OpenE2K
/
rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rollup merge of #55633 - nikic:memcpy-align, r=nagisa 

Support memcpy/memmove with differing src/dst alignment

If LLVM 7 is used, generate memcpy/memmove with differing src/dst alignment. I've added new FFI functions to construct these through the builder API, which is more convenient than dealing with differing intrinsic signatures depending on the LLVM version.

Fixes #49740.
This commit is contained in:
Mark Rousskov 2018-11-08 18:14:49 -07:00 committed by GitHub
parent 6153ce127a 463ad9098e
commit 78cab36139
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 98 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/librustc_codegen_llvm/abi.rs
View File

@ -225,9 +225,10 @@ impl ArgTypeExt<'ll, 'tcx> for ArgType<'tcx, Ty<'tcx>> {
// ...and then memcpy it to the intended destination.
base::call_memcpy(bx,
bx.pointercast(dst.llval, Type::i8p(cx)),
self.layout.align,
bx.pointercast(llscratch, Type::i8p(cx)),
scratch_align,
C_usize(cx, self.layout.size.bytes()),
self.layout.align.min(scratch_align),
MemFlags::empty());
bx.lifetime_end(llscratch, scratch_size);

27
src/librustc_codegen_llvm/base.rs
View File

@ -53,7 +53,7 @@ use mir::place::PlaceRef;
use attributes;
use builder::{Builder, MemFlags};
use callee;
use common::{C_bool, C_bytes_in_context, C_i32, C_usize};
use common::{C_bool, C_bytes_in_context, C_usize};
use rustc_mir::monomorphize::item::DefPathBasedNames;
use common::{C_struct_in_context, C_array, val_ty};
use consts;
@ -77,7 +77,6 @@ use rustc_data_structures::sync::Lrc;
use std::any::Any;
use std::cmp;
use std::ffi::CString;
use std::i32;
use std::ops::{Deref, DerefMut};
use std::sync::mpsc;
use std::time::{Instant, Duration};
@ -319,8 +318,8 @@ pub fn coerce_unsized_into(
}
if src_f.layout.ty == dst_f.layout.ty {
memcpy_ty(bx, dst_f.llval, src_f.llval, src_f.layout,
src_f.align.min(dst_f.align), MemFlags::empty());
memcpy_ty(bx, dst_f.llval, dst_f.align, src_f.llval, src_f.align,
src_f.layout, MemFlags::empty());
} else {
coerce_unsized_into(bx, src_f, dst_f);
}
@ -420,36 +419,34 @@ pub fn to_immediate_scalar(
pub fn call_memcpy(
bx: &Builder<'_, 'll, '_>,
dst: &'ll Value,
dst_align: Align,
src: &'ll Value,
src_align: Align,
n_bytes: &'ll Value,
align: Align,
flags: MemFlags,
) {
if flags.contains(MemFlags::NONTEMPORAL) {
// HACK(nox): This is inefficient but there is no nontemporal memcpy.
let val = bx.load(src, align);
let val = bx.load(src, src_align);
let ptr = bx.pointercast(dst, val_ty(val).ptr_to());
bx.store_with_flags(val, ptr, align, flags);
bx.store_with_flags(val, ptr, dst_align, flags);
return;
}
let cx = bx.cx;
let ptr_width = &cx.sess().target.target.target_pointer_width;
let key = format!("llvm.memcpy.p0i8.p0i8.i{}", ptr_width);
let memcpy = cx.get_intrinsic(&key);
let src_ptr = bx.pointercast(src, Type::i8p(cx));
let dst_ptr = bx.pointercast(dst, Type::i8p(cx));
let size = bx.intcast(n_bytes, cx.isize_ty, false);
let align = C_i32(cx, align.abi() as i32);
let volatile = C_bool(cx, flags.contains(MemFlags::VOLATILE));
bx.call(memcpy, &[dst_ptr, src_ptr, size, align, volatile], None);
let volatile = flags.contains(MemFlags::VOLATILE);
bx.memcpy(dst_ptr, dst_align.abi(), src_ptr, src_align.abi(), size, volatile);
}
pub fn memcpy_ty(
bx: &Builder<'_, 'll, 'tcx>,
dst: &'ll Value,
dst_align: Align,
src: &'ll Value,
src_align: Align,
layout: TyLayout<'tcx>,
align: Align,
flags: MemFlags,
) {
let size = layout.size.bytes();
@ -457,7 +454,7 @@ pub fn memcpy_ty(
return;
}
call_memcpy(bx, dst, src, C_usize(bx.cx, size), align, flags);
call_memcpy(bx, dst, dst_align, src, src_align, C_usize(bx.cx, size), flags);
}
pub fn call_memset(

18
src/librustc_codegen_llvm/builder.rs
View File

@ -781,6 +781,24 @@ impl Builder<'a, 'll, 'tcx> {
}
}
pub fn memcpy(&self, dst: &'ll Value, dst_align: u64,
src: &'ll Value, src_align: u64,
size: &'ll Value, is_volatile: bool) -> &'ll Value {
unsafe {
llvm::LLVMRustBuildMemCpy(self.llbuilder, dst, dst_align as c_uint,
src, src_align as c_uint, size, is_volatile)
}
}
pub fn memmove(&self, dst: &'ll Value, dst_align: u64,
src: &'ll Value, src_align: u64,
size: &'ll Value, is_volatile: bool) -> &'ll Value {
unsafe {
llvm::LLVMRustBuildMemMove(self.llbuilder, dst, dst_align as c_uint,
src, src_align as c_uint, size, is_volatile)
}
}
pub fn minnum(&self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
self.count_insn("minnum");
unsafe {

6
src/librustc_codegen_llvm/context.rs
View File

@ -530,12 +530,6 @@ fn declare_intrinsic(cx: &CodegenCx<'ll, '_>, key: &str) -> Option<&'ll Value> {
let t_v4f64 = Type::vector(t_f64, 4);
let t_v8f64 = Type::vector(t_f64, 8);
ifn!("llvm.memcpy.p0i8.p0i8.i16", fn(i8p, i8p, t_i16, t_i32, i1) -> void);
ifn!("llvm.memcpy.p0i8.p0i8.i32", fn(i8p, i8p, t_i32, t_i32, i1) -> void);
ifn!("llvm.memcpy.p0i8.p0i8.i64", fn(i8p, i8p, t_i64, t_i32, i1) -> void);
ifn!("llvm.memmove.p0i8.p0i8.i16", fn(i8p, i8p, t_i16, t_i32, i1) -> void);
ifn!("llvm.memmove.p0i8.p0i8.i32", fn(i8p, i8p, t_i32, t_i32, i1) -> void);
ifn!("llvm.memmove.p0i8.p0i8.i64", fn(i8p, i8p, t_i64, t_i32, i1) -> void);
ifn!("llvm.memset.p0i8.i16", fn(i8p, t_i8, t_i16, t_i32, i1) -> void);
ifn!("llvm.memset.p0i8.i32", fn(i8p, t_i8, t_i32, t_i32, i1) -> void);
ifn!("llvm.memset.p0i8.i64", fn(i8p, t_i8, t_i64, t_i32, i1) -> void);

28
src/librustc_codegen_llvm/intrinsic.rs
View File

@ -23,7 +23,7 @@ use glue;
use type_::Type;
use type_of::LayoutLlvmExt;
use rustc::ty::{self, Ty};
use rustc::ty::layout::{HasDataLayout, LayoutOf};
use rustc::ty::layout::LayoutOf;
use rustc::hir;
use syntax::ast;
use syntax::symbol::Symbol;
@ -690,28 +690,14 @@ fn copy_intrinsic(
let cx = bx.cx;
let (size, align) = cx.size_and_align_of(ty);
let size = C_usize(cx, size.bytes());
let align = C_i32(cx, align.abi() as i32);
let operation = if allow_overlap {
"memmove"
} else {
"memcpy"
};
let name = format!("llvm.{}.p0i8.p0i8.i{}", operation,
cx.data_layout().pointer_size.bits());
let align = align.abi();
let dst_ptr = bx.pointercast(dst, Type::i8p(cx));
let src_ptr = bx.pointercast(src, Type::i8p(cx));
let llfn = cx.get_intrinsic(&name);
bx.call(llfn,
&[dst_ptr,
src_ptr,
bx.mul(size, count),
align,
C_bool(cx, volatile)],
None)
if allow_overlap {
bx.memmove(dst_ptr, align, src_ptr, align, bx.mul(size, count), volatile)
} else {
bx.memcpy(dst_ptr, align, src_ptr, align, bx.mul(size, count), volatile)
}
}
fn memset_intrinsic(

16
src/librustc_codegen_llvm/llvm/ffi.rs
View File

@ -998,6 +998,22 @@ extern "C" {
Bundle: Option<&OperandBundleDef<'a>>,
Name: *const c_char)
-> &'a Value;
pub fn LLVMRustBuildMemCpy(B: &Builder<'a>,
Dst: &'a Value,
DstAlign: c_uint,
Src: &'a Value,
SrcAlign: c_uint,
Size: &'a Value,
IsVolatile: bool)
-> &'a Value;
pub fn LLVMRustBuildMemMove(B: &Builder<'a>,
Dst: &'a Value,
DstAlign: c_uint,
Src: &'a Value,
SrcAlign: c_uint,
Size: &'a Value,
IsVolatile: bool)
-> &'a Value;
pub fn LLVMBuildSelect(B: &Builder<'a>,
If: &'a Value,
Then: &'a Value,

3
src/librustc_codegen_llvm/mir/block.rs
View File

@ -784,7 +784,8 @@ impl FunctionCx<'a, 'll, 'tcx> {
// have scary latent bugs around.
let scratch = PlaceRef::alloca(bx, arg.layout, "arg");
base::memcpy_ty(bx, scratch.llval, llval, op.layout, align, MemFlags::empty());
base::memcpy_ty(bx, scratch.llval, scratch.align, llval, align,
op.layout, MemFlags::empty());
(scratch.llval, scratch.align, true)
} else {
(llval, align, true)

6
src/librustc_codegen_llvm/mir/operand.rs
View File

@ -282,8 +282,8 @@ impl OperandValue<'ll> {
}
match self {
OperandValue::Ref(r, None, source_align) => {
base::memcpy_ty(bx, dest.llval, r, dest.layout,
source_align.min(dest.align), flags)
base::memcpy_ty(bx, dest.llval, dest.align, r, source_align,
dest.layout, flags)
}
OperandValue::Ref(_, Some(_), _) => {
bug!("cannot directly store unsized values");
@ -324,7 +324,7 @@ impl OperandValue<'ll> {
// Allocate an appropriate region on the stack, and copy the value into it
let (llsize, _) = glue::size_and_align_of_dst(&bx, unsized_ty, Some(llextra));
let lldst = bx.array_alloca(Type::i8(bx.cx), llsize, "unsized_tmp", max_align);
base::call_memcpy(&bx, lldst, llptr, llsize, min_align, flags);
base::call_memcpy(&bx, lldst, max_align, llptr, min_align, llsize, flags);
// Store the allocated region and the extra to the indirect place.
let indirect_operand = OperandValue::Pair(lldst, llextra);

34
src/rustllvm/RustWrapper.cpp
View File

@ -1237,6 +1237,40 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMValueRef Fn,
unwrap(Fn), makeArrayRef(unwrap(Args), NumArgs), Bundles, Name));
}
extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B,
LLVMValueRef Dst, unsigned DstAlign,
LLVMValueRef Src, unsigned SrcAlign,
LLVMValueRef Size, bool IsVolatile) {
#if LLVM_VERSION_GE(7, 0)
return wrap(unwrap(B)->CreateMemCpy(
unwrap(Dst), DstAlign,
unwrap(Src), SrcAlign,
unwrap(Size), IsVolatile));
#else
unsigned Align = std::min(DstAlign, SrcAlign);
return wrap(unwrap(B)->CreateMemCpy(
unwrap(Dst), unwrap(Src),
unwrap(Size), Align, IsVolatile));
#endif
}
extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B,
LLVMValueRef Dst, unsigned DstAlign,
LLVMValueRef Src, unsigned SrcAlign,
LLVMValueRef Size, bool IsVolatile) {
#if LLVM_VERSION_GE(7, 0)
return wrap(unwrap(B)->CreateMemMove(
unwrap(Dst), DstAlign,
unwrap(Src), SrcAlign,
unwrap(Size), IsVolatile));
#else
unsigned Align = std::min(DstAlign, SrcAlign);
return wrap(unwrap(B)->CreateMemMove(
unwrap(Dst), unwrap(Src),
unwrap(Size), Align, IsVolatile));
#endif
}
extern "C" LLVMValueRef
LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef *Args,
unsigned NumArgs, LLVMBasicBlockRef Then,

4
src/test/codegen/packed.rs
View File

@ -65,7 +65,7 @@ pub struct BigPacked2 {
pub fn call_pkd1(f: fn() -> Array) -> BigPacked1 {
// CHECK: [[ALLOCA:%[_a-z0-9]+]] = alloca %Array
// CHECK: call void %{{.*}}(%Array* noalias nocapture sret dereferenceable(32) [[ALLOCA]])
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 %{{.*}}, i8* align 1 %{{.*}}, i{{[0-9]+}} 32, i1 false)
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 %{{.*}}, i8* align 4 %{{.*}}, i{{[0-9]+}} 32, i1 false)
// check that calls whose destination is a field of a packed struct
// go through an alloca rather than calling the function with an
// unaligned destination.
@ -77,7 +77,7 @@ pub fn call_pkd1(f: fn() -> Array) -> BigPacked1 {
pub fn call_pkd2(f: fn() -> Array) -> BigPacked2 {
// CHECK: [[ALLOCA:%[_a-z0-9]+]] = alloca %Array
// CHECK: call void %{{.*}}(%Array* noalias nocapture sret dereferenceable(32) [[ALLOCA]])
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 2 %{{.*}}, i8* align 2 %{{.*}}, i{{[0-9]+}} 32, i1 false)
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 2 %{{.*}}, i8* align 4 %{{.*}}, i{{[0-9]+}} 32, i1 false)
// check that calls whose destination is a field of a packed struct
// go through an alloca rather than calling the function with an
// unaligned destination.

4
src/test/codegen/stores.rs
View File

@ -31,7 +31,7 @@ pub fn small_array_alignment(x: &mut [i8; 4], y: [i8; 4]) {
// CHECK: store i32 %0, i32* [[TMP]]
// CHECK: [[Y8:%[0-9]+]] = bitcast [4 x i8]* %y to i8*
// CHECK: [[TMP8:%[0-9]+]] = bitcast i32* [[TMP]] to i8*
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 1 [[TMP8]], i{{[0-9]+}} 4, i1 false)
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 4 [[TMP8]], i{{[0-9]+}} 4, i1 false)
*x = y;
}
@ -45,6 +45,6 @@ pub fn small_struct_alignment(x: &mut Bytes, y: Bytes) {
// CHECK: store i32 %0, i32* [[TMP]]
// CHECK: [[Y8:%[0-9]+]] = bitcast %Bytes* %y to i8*
// CHECK: [[TMP8:%[0-9]+]] = bitcast i32* [[TMP]] to i8*
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 1 [[TMP8]], i{{[0-9]+}} 4, i1 false)
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 4 [[TMP8]], i{{[0-9]+}} 4, i1 false)
*x = y;
}