From 7a36141465d1f97936cfceca87ed428dbfafdd3f Mon Sep 17 00:00:00 2001 From: Eduard-Mihai Burtescu Date: Mon, 9 Oct 2017 19:56:41 +0300 Subject: [PATCH] rustc: unpack scalar pair newtype layout ABIs. --- src/librustc/ty/layout.rs | 41 +++++----- src/librustc_trans/base.rs | 25 ++++++ src/librustc_trans/mir/block.rs | 47 +++-------- src/librustc_trans/mir/constant.rs | 13 +++- src/librustc_trans/mir/lvalue.rs | 69 +++++++++------- src/librustc_trans/mir/operand.rs | 121 +++++++++++++++++------------ src/librustc_trans/mir/rvalue.rs | 13 ++-- src/librustc_trans/type_of.rs | 10 --- src/test/codegen/refs.rs | 2 +- 9 files changed, 185 insertions(+), 156 deletions(-) diff --git a/src/librustc/ty/layout.rs b/src/librustc/ty/layout.rs index 3bf711d3e23..fc5d4213949 100644 --- a/src/librustc/ty/layout.rs +++ b/src/librustc/ty/layout.rs @@ -1087,10 +1087,11 @@ impl<'a, 'tcx> CachedLayout { // We have exactly one non-ZST field. match (non_zst_fields.next(), non_zst_fields.next()) { (Some(field), None) => { - // Field size match and it has a scalar ABI. + // Field size matches and it has a scalar or scalar pair ABI. if size == field.size { match field.abi { - Abi::Scalar(_) => { + Abi::Scalar(_) | + Abi::ScalarPair(..) => { abi = field.abi.clone(); } _ => {} @@ -2228,17 +2229,7 @@ impl<'a, 'tcx> TyLayout<'tcx> { ty::TyAdt(def, substs) => { match self.variants { Variants::Single { index } => { - let mut field_ty = def.variants[index].fields[i].ty(tcx, substs); - - // Treat NonZero<*T> as containing &T. - // This is especially useful for fat pointers. - if Some(def.did) == tcx.lang_items().non_zero() { - if let ty::TyRawPtr(mt) = field_ty.sty { - field_ty = tcx.mk_ref(tcx.types.re_erased, mt); - } - } - - field_ty + def.variants[index].fields[i].ty(tcx, substs) } // Discriminant field for enums (where applicable). @@ -2294,21 +2285,22 @@ impl<'a, 'tcx> TyLayout<'tcx> { where C: LayoutOf, TyLayout = Result>> + HasTyCtxt<'tcx> { - if let Abi::Scalar(Scalar { value, ref valid_range }) = self.abi { + let scalar_component = |scalar: &Scalar, offset| { // FIXME(eddyb) support negative/wrap-around discriminant ranges. - return if valid_range.start < valid_range.end { + let Scalar { value, ref valid_range } = *scalar; + if valid_range.start < valid_range.end { let bits = value.size(cx).bits(); assert!(bits <= 128); let max_value = !0u128 >> (128 - bits); if valid_range.start > 0 { let niche = valid_range.start - 1; - Ok(Some((self.fields.offset(0), Scalar { + Ok(Some((offset, Scalar { value, valid_range: niche..=valid_range.end }, niche))) } else if valid_range.end < max_value { let niche = valid_range.end + 1; - Ok(Some((self.fields.offset(0), Scalar { + Ok(Some((offset, Scalar { value, valid_range: valid_range.start..=niche }, niche))) @@ -2317,7 +2309,20 @@ impl<'a, 'tcx> TyLayout<'tcx> { } } else { Ok(None) - }; + } + }; + + match self.abi { + Abi::Scalar(ref scalar) => { + return scalar_component(scalar, Size::from_bytes(0)); + } + Abi::ScalarPair(ref a, ref b) => { + if let Some(result) = scalar_component(a, Size::from_bytes(0))? { + return Ok(Some(result)); + } + return scalar_component(b, a.value.size(cx).abi_align(b.value.align(cx))); + } + _ => {} } // Perhaps one of the fields is non-zero, let's recurse and find out. diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs index ff70184b262..b7408681ed0 100644 --- a/src/librustc_trans/base.rs +++ b/src/librustc_trans/base.rs @@ -240,6 +240,31 @@ pub fn unsize_thin_ptr<'a, 'tcx>( let ptr_ty = bcx.ccx.layout_of(b).llvm_type(bcx.ccx).ptr_to(); (bcx.pointercast(src, ptr_ty), unsized_info(bcx.ccx, a, b, None)) } + (&ty::TyAdt(def_a, _), &ty::TyAdt(def_b, _)) => { + assert_eq!(def_a, def_b); + + let src_layout = bcx.ccx.layout_of(src_ty); + let dst_layout = bcx.ccx.layout_of(dst_ty); + let mut result = None; + for i in 0..src_layout.fields.count() { + let src_f = src_layout.field(bcx.ccx, i); + assert_eq!(src_layout.fields.offset(i).bytes(), 0); + assert_eq!(dst_layout.fields.offset(i).bytes(), 0); + if src_f.is_zst() { + continue; + } + assert_eq!(src_layout.size, src_f.size); + + let dst_f = dst_layout.field(bcx.ccx, i); + assert_ne!(src_f.ty, dst_f.ty); + assert_eq!(result, None); + result = Some(unsize_thin_ptr(bcx, src, src_f.ty, dst_f.ty)); + } + let (lldata, llextra) = result.unwrap(); + // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. + (bcx.bitcast(lldata, dst_layout.scalar_pair_element_llvm_type(bcx.ccx, 0)), + bcx.bitcast(llextra, dst_layout.scalar_pair_element_llvm_type(bcx.ccx, 1))) + } _ => bug!("unsize_thin_ptr: called on bad types"), } } diff --git a/src/librustc_trans/mir/block.rs b/src/librustc_trans/mir/block.rs index 6811861499d..67e0f35b46e 100644 --- a/src/librustc_trans/mir/block.rs +++ b/src/librustc_trans/mir/block.rs @@ -685,46 +685,19 @@ impl<'a, 'tcx> MirContext<'a, 'tcx> { let tuple = self.trans_operand(bcx, operand); // Handle both by-ref and immediate tuples. - match tuple.val { - Ref(llval, align) => { - let tuple_ptr = LvalueRef::new_sized(llval, tuple.layout, align); - for i in 0..tuple.layout.fields.count() { - let field_ptr = tuple_ptr.project_field(bcx, i); - self.trans_argument(bcx, field_ptr.load(bcx), llargs, &args[i]); - } - + if let Ref(llval, align) = tuple.val { + let tuple_ptr = LvalueRef::new_sized(llval, tuple.layout, align); + for i in 0..tuple.layout.fields.count() { + let field_ptr = tuple_ptr.project_field(bcx, i); + self.trans_argument(bcx, field_ptr.load(bcx), llargs, &args[i]); } - Immediate(llval) => { - for i in 0..tuple.layout.fields.count() { - let field = tuple.layout.field(bcx.ccx, i); - let elem = if field.is_zst() { - C_undef(field.llvm_type(bcx.ccx)) - } else { - // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. - bcx.bitcast(llval, field.immediate_llvm_type(bcx.ccx)) - }; - // If the tuple is immediate, the elements are as well - let op = OperandRef { - val: Immediate(elem), - layout: field, - }; - self.trans_argument(bcx, op, llargs, &args[i]); - } - } - Pair(a, b) => { - let elems = [a, b]; - assert_eq!(tuple.layout.fields.count(), 2); - for i in 0..2 { - // Pair is always made up of immediates - let op = OperandRef { - val: Immediate(elems[i]), - layout: tuple.layout.field(bcx.ccx, i), - }; - self.trans_argument(bcx, op, llargs, &args[i]); - } + } else { + // If the tuple is immediate, the elements are as well. + for i in 0..tuple.layout.fields.count() { + let op = tuple.extract_field(bcx, i); + self.trans_argument(bcx, op, llargs, &args[i]); } } - } fn get_personality_slot(&mut self, bcx: &Builder<'a, 'tcx>) -> LvalueRef<'tcx> { diff --git a/src/librustc_trans/mir/constant.rs b/src/librustc_trans/mir/constant.rs index f223227cd72..54907cb747c 100644 --- a/src/librustc_trans/mir/constant.rs +++ b/src/librustc_trans/mir/constant.rs @@ -127,8 +127,12 @@ impl<'a, 'tcx> Const<'tcx> { layout::Abi::ScalarPair(ref a, ref b) => { let offset = layout.fields.offset(i); if offset.bytes() == 0 { - assert_eq!(field.size, a.value.size(ccx)); - const_get_elt(self.llval, 0) + if field.size == layout.size { + self.llval + } else { + assert_eq!(field.size, a.value.size(ccx)); + const_get_elt(self.llval, 0) + } } else { assert_eq!(offset, a.value.size(ccx) .abi_align(b.value.align(ccx))); @@ -166,8 +170,9 @@ impl<'a, 'tcx> Const<'tcx> { let llvalty = val_ty(self.llval); let val = if llty == llvalty && layout.is_llvm_scalar_pair() { - let (a, b) = self.get_pair(ccx); - OperandValue::Pair(a, b) + OperandValue::Pair( + const_get_elt(self.llval, 0), + const_get_elt(self.llval, 1)) } else if llty == llvalty && layout.is_llvm_immediate() { // If the types match, we can use the value directly. OperandValue::Immediate(self.llval) diff --git a/src/librustc_trans/mir/lvalue.rs b/src/librustc_trans/mir/lvalue.rs index ff0b4482678..71d2fda09c9 100644 --- a/src/librustc_trans/mir/lvalue.rs +++ b/src/librustc_trans/mir/lvalue.rs @@ -135,6 +135,31 @@ impl<'a, 'tcx> LvalueRef<'tcx> { return OperandRef::new_zst(bcx.ccx, self.layout); } + let scalar_load_metadata = |load, scalar: &layout::Scalar| { + let (min, max) = (scalar.valid_range.start, scalar.valid_range.end); + let max_next = max.wrapping_add(1); + let bits = scalar.value.size(bcx.ccx).bits(); + assert!(bits <= 128); + let mask = !0u128 >> (128 - bits); + // For a (max) value of -1, max will be `-1 as usize`, which overflows. + // However, that is fine here (it would still represent the full range), + // i.e., if the range is everything. The lo==hi case would be + // rejected by the LLVM verifier (it would mean either an + // empty set, which is impossible, or the entire range of the + // type, which is pointless). + match scalar.value { + layout::Int(..) if max_next & mask != min & mask => { + // llvm::ConstantRange can deal with ranges that wrap around, + // so an overflow on (max + 1) is fine. + bcx.range_metadata(load, min..max_next); + } + layout::Pointer if 0 < min && min < max => { + bcx.nonnull_metadata(load); + } + _ => {} + } + }; + let val = if self.layout.is_llvm_immediate() { let mut const_llval = ptr::null_mut(); unsafe { @@ -149,39 +174,27 @@ impl<'a, 'tcx> LvalueRef<'tcx> { } else { let load = bcx.load(self.llval, self.alignment.non_abi()); if let layout::Abi::Scalar(ref scalar) = self.layout.abi { - let (min, max) = (scalar.valid_range.start, scalar.valid_range.end); - let max_next = max.wrapping_add(1); - let bits = scalar.value.size(bcx.ccx).bits(); - assert!(bits <= 128); - let mask = !0u128 >> (128 - bits); - // For a (max) value of -1, max will be `-1 as usize`, which overflows. - // However, that is fine here (it would still represent the full range), - // i.e., if the range is everything. The lo==hi case would be - // rejected by the LLVM verifier (it would mean either an - // empty set, which is impossible, or the entire range of the - // type, which is pointless). - match scalar.value { - layout::Int(..) if max_next & mask != min & mask => { - // llvm::ConstantRange can deal with ranges that wrap around, - // so an overflow on (max + 1) is fine. - bcx.range_metadata(load, min..max_next); - } - layout::Pointer if 0 < min && min < max => { - bcx.nonnull_metadata(load); - } - _ => {} - } + scalar_load_metadata(load, scalar); } load }; OperandValue::Immediate(base::to_immediate(bcx, llval, self.layout)) - } else if self.layout.is_llvm_scalar_pair() { - let load = |i| { - let x = self.project_field(bcx, i).load(bcx).immediate(); - // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. - bcx.bitcast(x, self.layout.scalar_pair_element_llvm_type(bcx.ccx, i)) + } else if let layout::Abi::ScalarPair(ref a, ref b) = self.layout.abi { + let load = |i, scalar: &layout::Scalar| { + let mut llptr = bcx.struct_gep(self.llval, i as u64); + // Make sure to always load i1 as i8. + if scalar.is_bool() { + llptr = bcx.pointercast(llptr, Type::i8p(bcx.ccx)); + } + let load = bcx.load(llptr, self.alignment.non_abi()); + scalar_load_metadata(load, scalar); + if scalar.is_bool() { + bcx.trunc(load, Type::i1(bcx.ccx)) + } else { + load + } }; - OperandValue::Pair(load(0), load(1)) + OperandValue::Pair(load(0, a), load(1, b)) } else { OperandValue::Ref(self.llval, self.alignment) }; diff --git a/src/librustc_trans/mir/operand.rs b/src/librustc_trans/mir/operand.rs index 7826d998df3..8c43bded1bf 100644 --- a/src/librustc_trans/mir/operand.rs +++ b/src/librustc_trans/mir/operand.rs @@ -15,10 +15,11 @@ use rustc::mir; use rustc_data_structures::indexed_vec::Idx; use base; -use common::{CrateContext, C_undef, C_usize}; +use common::{self, CrateContext, C_undef, C_usize}; use builder::Builder; use value::Value; use type_of::LayoutLlvmExt; +use type_::Type; use std::fmt; use std::ptr; @@ -84,7 +85,7 @@ impl<'a, 'tcx> OperandRef<'tcx> { layout: TyLayout<'tcx>) -> OperandRef<'tcx> { assert!(layout.is_zst()); OperandRef { - val: OperandValue::Immediate(C_undef(layout.llvm_type(ccx))), + val: OperandValue::Immediate(C_undef(layout.immediate_llvm_type(ccx))), layout } } @@ -148,6 +149,66 @@ impl<'a, 'tcx> OperandRef<'tcx> { }; OperandRef { val, layout } } + + pub fn extract_field(&self, bcx: &Builder<'a, 'tcx>, i: usize) -> OperandRef<'tcx> { + let field = self.layout.field(bcx.ccx, i); + let offset = self.layout.fields.offset(i); + + let mut val = match (self.val, &self.layout.abi) { + // If we're uninhabited, or the field is ZST, it has no data. + _ if self.layout.abi == layout::Abi::Uninhabited || field.is_zst() => { + return OperandRef { + val: OperandValue::Immediate(C_undef(field.immediate_llvm_type(bcx.ccx))), + layout: field + }; + } + + // Newtype of a scalar or scalar pair. + (OperandValue::Immediate(_), _) | + (OperandValue::Pair(..), _) if field.size == self.layout.size => { + assert_eq!(offset.bytes(), 0); + self.val + } + + // Extract a scalar component from a pair. + (OperandValue::Pair(a_llval, b_llval), &layout::Abi::ScalarPair(ref a, ref b)) => { + if offset.bytes() == 0 { + assert_eq!(field.size, a.value.size(bcx.ccx)); + OperandValue::Immediate(a_llval) + } else { + assert_eq!(offset, a.value.size(bcx.ccx) + .abi_align(b.value.align(bcx.ccx))); + assert_eq!(field.size, b.value.size(bcx.ccx)); + OperandValue::Immediate(b_llval) + } + } + + // `#[repr(simd)]` types are also immediate. + (OperandValue::Immediate(llval), &layout::Abi::Vector) => { + OperandValue::Immediate( + bcx.extract_element(llval, C_usize(bcx.ccx, i as u64))) + } + + _ => bug!("OperandRef::extract_field({:?}): not applicable", self) + }; + + // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. + match val { + OperandValue::Immediate(ref mut llval) => { + *llval = bcx.bitcast(*llval, field.immediate_llvm_type(bcx.ccx)); + } + OperandValue::Pair(ref mut a, ref mut b) => { + *a = bcx.bitcast(*a, field.scalar_pair_element_llvm_type(bcx.ccx, 0)); + *b = bcx.bitcast(*b, field.scalar_pair_element_llvm_type(bcx.ccx, 1)); + } + OperandValue::Ref(..) => bug!() + } + + OperandRef { + val, + layout: field + } + } } impl<'a, 'tcx> OperandValue { @@ -167,11 +228,12 @@ impl<'a, 'tcx> OperandValue { } OperandValue::Pair(a, b) => { for (i, &x) in [a, b].iter().enumerate() { - let field = dest.project_field(bcx, i); - // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. - let x = bcx.bitcast(x, field.layout.immediate_llvm_type(bcx.ccx)); - bcx.store(base::from_immediate(bcx, x), - field.llval, field.alignment.non_abi()); + let mut llptr = bcx.struct_gep(dest.llval, i as u64); + // Make sure to always store i1 as i8. + if common::val_ty(x) == Type::i1(bcx.ccx) { + llptr = bcx.pointercast(llptr, Type::i8p(bcx.ccx)); + } + bcx.store(base::from_immediate(bcx, x), llptr, dest.alignment.non_abi()); } } } @@ -202,52 +264,11 @@ impl<'a, 'tcx> MirContext<'a, 'tcx> { } } - // Moves out of pair fields are trivial. + // Moves out of scalar and scalar pair fields are trivial. if let &mir::Lvalue::Projection(ref proj) = lvalue { if let mir::ProjectionElem::Field(ref f, _) = proj.elem { if let Some(o) = self.maybe_trans_consume_direct(bcx, &proj.base) { - let layout = o.layout.field(bcx.ccx, f.index()); - let offset = o.layout.fields.offset(f.index()); - - // Handled in `trans_consume`. - assert!(!layout.is_zst()); - - // Offset has to match a scalar component. - let llval = match (o.val, &o.layout.abi) { - (OperandValue::Immediate(llval), - &layout::Abi::Scalar(ref scalar)) => { - assert_eq!(offset.bytes(), 0); - assert_eq!(layout.size, scalar.value.size(bcx.ccx)); - llval - } - (OperandValue::Pair(a_llval, b_llval), - &layout::Abi::ScalarPair(ref a, ref b)) => { - if offset.bytes() == 0 { - assert_eq!(layout.size, a.value.size(bcx.ccx)); - a_llval - } else { - assert_eq!(offset, a.value.size(bcx.ccx) - .abi_align(b.value.align(bcx.ccx))); - assert_eq!(layout.size, b.value.size(bcx.ccx)); - b_llval - } - } - - // `#[repr(simd)]` types are also immediate. - (OperandValue::Immediate(llval), - &layout::Abi::Vector) => { - bcx.extract_element(llval, C_usize(bcx.ccx, f.index() as u64)) - } - - _ => return None - }; - - // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. - let llval = bcx.bitcast(llval, layout.immediate_llvm_type(bcx.ccx)); - return Some(OperandRef { - val: OperandValue::Immediate(llval), - layout - }); + return Some(o.extract_field(bcx, f.index())); } } } diff --git a/src/librustc_trans/mir/rvalue.rs b/src/librustc_trans/mir/rvalue.rs index 33b1a7e3363..bf44434cafe 100644 --- a/src/librustc_trans/mir/rvalue.rs +++ b/src/librustc_trans/mir/rvalue.rs @@ -223,20 +223,17 @@ impl<'a, 'tcx> MirContext<'a, 'tcx> { operand.val } mir::CastKind::Unsize => { - // unsize targets other than to a fat pointer currently - // can't be operands. - assert!(common::type_is_fat_ptr(bcx.ccx, cast.ty)); - match operand.val { OperandValue::Pair(lldata, llextra) => { // unsize from a fat pointer - this is a // "trait-object-to-supertrait" coercion, for // example, // &'a fmt::Debug+Send => &'a fmt::Debug, - // So we need to pointercast the base to ensure - // the types match up. - let thin_ptr = cast.field(bcx.ccx, abi::FAT_PTR_ADDR); - let lldata = bcx.pointercast(lldata, thin_ptr.llvm_type(bcx.ccx)); + + // HACK(eddyb) have to bitcast pointers + // until LLVM removes pointee types. + let lldata = bcx.pointercast(lldata, + cast.scalar_pair_element_llvm_type(bcx.ccx, 0)); OperandValue::Pair(lldata, llextra) } OperandValue::Immediate(lldata) => { diff --git a/src/librustc_trans/type_of.rs b/src/librustc_trans/type_of.rs index d62e2ac1552..35da258cdb3 100644 --- a/src/librustc_trans/type_of.rs +++ b/src/librustc_trans/type_of.rs @@ -491,16 +491,6 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyLayout<'tcx> { } } } - - if let ty::TyAdt(def, _) = self.ty.sty { - if Some(def.did) == ccx.tcx().lang_items().non_zero() { - // FIXME(eddyb) Don't treat NonZero<*T> as - // as containing &T in ty::layout. - if let Some(ref mut pointee) = result { - pointee.safe = None; - } - } - } } } diff --git a/src/test/codegen/refs.rs b/src/test/codegen/refs.rs index 0c084131ea3..6c00ffa754b 100644 --- a/src/test/codegen/refs.rs +++ b/src/test/codegen/refs.rs @@ -24,7 +24,7 @@ pub fn helper(_: usize) { pub fn ref_dst(s: &[u8]) { // We used to generate an extra alloca and memcpy to ref the dst, so check that we copy // directly to the alloca for "x" -// CHECK: [[X0:%[0-9]+]] = bitcast { [0 x i8]*, [[USIZE]] }* %x to [0 x i8]** +// CHECK: [[X0:%[0-9]+]] = getelementptr {{.*}} { [0 x i8]*, [[USIZE]] }* %x, i32 0, i32 0 // CHECK: store [0 x i8]* %s.0, [0 x i8]** [[X0]] // CHECK: [[X1:%[0-9]+]] = getelementptr {{.*}} { [0 x i8]*, [[USIZE]] }* %x, i32 0, i32 1 // CHECK: store [[USIZE]] %s.1, [[USIZE]]* [[X1]]