Rollup merge of #62264 - RalfJung:inline-forcing, r=zackmdavis
Fix perf regression from Miri Machine trait changes Maybe this fixes the perf regression that https://github.com/rust-lang/rust/pull/62003 seemingly introduced? Cc @nnethercote
This commit is contained in:
commit
0383be8577
|
@ -11,7 +11,7 @@ use rustc::hir::def::DefKind;
|
|||
use rustc::hir::def_id::DefId;
|
||||
use rustc::mir::interpret::{ConstEvalErr, ErrorHandled, ScalarMaybeUndef};
|
||||
use rustc::mir;
|
||||
use rustc::ty::{self, TyCtxt, query::TyCtxtAt};
|
||||
use rustc::ty::{self, TyCtxt};
|
||||
use rustc::ty::layout::{self, LayoutOf, VariantIdx};
|
||||
use rustc::ty::subst::Subst;
|
||||
use rustc::traits::Reveal;
|
||||
|
@ -23,7 +23,7 @@ use crate::interpret::{self,
|
|||
PlaceTy, MPlaceTy, OpTy, ImmTy, Immediate, Scalar,
|
||||
RawConst, ConstValue,
|
||||
InterpResult, InterpErrorInfo, InterpError, GlobalId, InterpCx, StackPopCleanup,
|
||||
Allocation, AllocId, MemoryKind, Memory,
|
||||
Allocation, AllocId, MemoryKind,
|
||||
snapshot, RefTracking, intern_const_alloc_recursive,
|
||||
};
|
||||
|
||||
|
@ -409,18 +409,18 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
|
|||
}
|
||||
|
||||
fn find_foreign_static(
|
||||
_tcx: TyCtxt<'tcx>,
|
||||
_def_id: DefId,
|
||||
_tcx: TyCtxtAt<'tcx>,
|
||||
) -> InterpResult<'tcx, Cow<'tcx, Allocation<Self::PointerTag>>> {
|
||||
err!(ReadForeignStatic)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn tag_allocation<'b>(
|
||||
_memory_extra: &(),
|
||||
_id: AllocId,
|
||||
alloc: Cow<'b, Allocation>,
|
||||
_kind: Option<MemoryKind<!>>,
|
||||
_memory: &Memory<'mir, 'tcx, Self>,
|
||||
) -> (Cow<'b, Allocation<Self::PointerTag>>, Self::PointerTag) {
|
||||
// We do not use a tag so we can just cheaply forward the allocation
|
||||
(alloc, ())
|
||||
|
@ -428,8 +428,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
|
|||
|
||||
#[inline(always)]
|
||||
fn tag_static_base_pointer(
|
||||
_memory_extra: &(),
|
||||
_id: AllocId,
|
||||
_memory: &Memory<'mir, 'tcx, Self>,
|
||||
) -> Self::PointerTag {
|
||||
()
|
||||
}
|
||||
|
|
|
@ -222,6 +222,23 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
|
|||
&mut self.memory
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn force_ptr(
|
||||
&self,
|
||||
scalar: Scalar<M::PointerTag>,
|
||||
) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
|
||||
self.memory.force_ptr(scalar)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn force_bits(
|
||||
&self,
|
||||
scalar: Scalar<M::PointerTag>,
|
||||
size: Size
|
||||
) -> InterpResult<'tcx, u128> {
|
||||
self.memory.force_bits(scalar, size)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn tag_static_base_pointer(&self, ptr: Pointer) -> Pointer<M::PointerTag> {
|
||||
self.memory.tag_static_base_pointer(ptr)
|
||||
|
@ -253,6 +270,27 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
|
|||
self.frame().body
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn sign_extend(&self, value: u128, ty: TyLayout<'_>) -> u128 {
|
||||
assert!(ty.abi.is_signed());
|
||||
sign_extend(value, ty.size)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn truncate(&self, value: u128, ty: TyLayout<'_>) -> u128 {
|
||||
truncate(value, ty.size)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn type_is_sized(&self, ty: Ty<'tcx>) -> bool {
|
||||
ty.is_sized(self.tcx, self.param_env)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn type_is_freeze(&self, ty: Ty<'tcx>) -> bool {
|
||||
ty.is_freeze(*self.tcx, self.param_env, DUMMY_SP)
|
||||
}
|
||||
|
||||
pub(super) fn subst_and_normalize_erasing_regions<T: TypeFoldable<'tcx>>(
|
||||
&self,
|
||||
substs: T,
|
||||
|
@ -288,14 +326,6 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
|
|||
).ok_or_else(|| InterpError::TooGeneric.into())
|
||||
}
|
||||
|
||||
pub fn type_is_sized(&self, ty: Ty<'tcx>) -> bool {
|
||||
ty.is_sized(self.tcx, self.param_env)
|
||||
}
|
||||
|
||||
pub fn type_is_freeze(&self, ty: Ty<'tcx>) -> bool {
|
||||
ty.is_freeze(*self.tcx, self.param_env, DUMMY_SP)
|
||||
}
|
||||
|
||||
pub fn load_mir(
|
||||
&self,
|
||||
instance: ty::InstanceDef<'tcx>,
|
||||
|
@ -766,32 +796,4 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
|
|||
trace!("generate stacktrace: {:#?}, {:?}", frames, explicit_span);
|
||||
frames
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn sign_extend(&self, value: u128, ty: TyLayout<'_>) -> u128 {
|
||||
assert!(ty.abi.is_signed());
|
||||
sign_extend(value, ty.size)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn truncate(&self, value: u128, ty: TyLayout<'_>) -> u128 {
|
||||
truncate(value, ty.size)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn force_ptr(
|
||||
&self,
|
||||
scalar: Scalar<M::PointerTag>,
|
||||
) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
|
||||
self.memory.force_ptr(scalar)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn force_bits(
|
||||
&self,
|
||||
scalar: Scalar<M::PointerTag>,
|
||||
size: Size
|
||||
) -> InterpResult<'tcx, u128> {
|
||||
self.memory.force_bits(scalar, size)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -7,11 +7,11 @@ use std::hash::Hash;
|
|||
|
||||
use rustc::hir::def_id::DefId;
|
||||
use rustc::mir;
|
||||
use rustc::ty::{self, query::TyCtxtAt};
|
||||
use rustc::ty::{self, TyCtxt};
|
||||
|
||||
use super::{
|
||||
Allocation, AllocId, InterpResult, Scalar, AllocationExtra,
|
||||
InterpCx, PlaceTy, OpTy, ImmTy, MemoryKind, Pointer, Memory
|
||||
Allocation, AllocId, InterpResult, InterpError, Scalar, AllocationExtra,
|
||||
InterpCx, PlaceTy, OpTy, ImmTy, MemoryKind, Pointer, Memory,
|
||||
};
|
||||
|
||||
/// Whether this kind of memory is allowed to leak
|
||||
|
@ -151,8 +151,8 @@ pub trait Machine<'mir, 'tcx>: Sized {
|
|||
///
|
||||
/// This allocation will then be fed to `tag_allocation` to initialize the "extra" state.
|
||||
fn find_foreign_static(
|
||||
tcx: TyCtxt<'tcx>,
|
||||
def_id: DefId,
|
||||
tcx: TyCtxtAt<'tcx>,
|
||||
) -> InterpResult<'tcx, Cow<'tcx, Allocation>>;
|
||||
|
||||
/// Called for all binary operations on integer(-like) types when one operand is a pointer
|
||||
|
@ -189,10 +189,10 @@ pub trait Machine<'mir, 'tcx>: Sized {
|
|||
/// For static allocations, the tag returned must be the same as the one returned by
|
||||
/// `tag_static_base_pointer`.
|
||||
fn tag_allocation<'b>(
|
||||
memory_extra: &Self::MemoryExtra,
|
||||
id: AllocId,
|
||||
alloc: Cow<'b, Allocation>,
|
||||
kind: Option<MemoryKind<Self::MemoryKinds>>,
|
||||
memory: &Memory<'mir, 'tcx, Self>,
|
||||
) -> (Cow<'b, Allocation<Self::PointerTag, Self::AllocExtra>>, Self::PointerTag);
|
||||
|
||||
/// Return the "base" tag for the given static allocation: the one that is used for direct
|
||||
|
@ -201,8 +201,8 @@ pub trait Machine<'mir, 'tcx>: Sized {
|
|||
/// Be aware that requesting the `Allocation` for that `id` will lead to cycles
|
||||
/// for cyclic statics!
|
||||
fn tag_static_base_pointer(
|
||||
memory_extra: &Self::MemoryExtra,
|
||||
id: AllocId,
|
||||
memory: &Memory<'mir, 'tcx, Self>,
|
||||
) -> Self::PointerTag;
|
||||
|
||||
/// Executes a retagging operation
|
||||
|
@ -224,20 +224,22 @@ pub trait Machine<'mir, 'tcx>: Sized {
|
|||
extra: Self::FrameExtra,
|
||||
) -> InterpResult<'tcx>;
|
||||
|
||||
#[inline(always)]
|
||||
fn int_to_ptr(
|
||||
int: u64,
|
||||
_mem: &Memory<'mir, 'tcx, Self>,
|
||||
int: u64,
|
||||
) -> InterpResult<'tcx, Pointer<Self::PointerTag>> {
|
||||
if int == 0 {
|
||||
err!(InvalidNullPointerUsage)
|
||||
Err((if int == 0 {
|
||||
InterpError::InvalidNullPointerUsage
|
||||
} else {
|
||||
err!(ReadBytesAsPointer)
|
||||
}
|
||||
InterpError::ReadBytesAsPointer
|
||||
}).into())
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn ptr_to_int(
|
||||
_ptr: Pointer<Self::PointerTag>,
|
||||
_mem: &Memory<'mir, 'tcx, Self>,
|
||||
_ptr: Pointer<Self::PointerTag>,
|
||||
) -> InterpResult<'tcx, u64> {
|
||||
err!(ReadPointerAsBytes)
|
||||
}
|
||||
|
|
|
@ -143,7 +143,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
|
|||
|
||||
#[inline]
|
||||
pub fn tag_static_base_pointer(&self, ptr: Pointer) -> Pointer<M::PointerTag> {
|
||||
ptr.with_tag(M::tag_static_base_pointer(ptr.alloc_id, &self))
|
||||
ptr.with_tag(M::tag_static_base_pointer(&self.extra, ptr.alloc_id))
|
||||
}
|
||||
|
||||
pub fn create_fn_alloc(
|
||||
|
@ -189,7 +189,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
|
|||
kind: MemoryKind<M::MemoryKinds>,
|
||||
) -> Pointer<M::PointerTag> {
|
||||
let id = self.tcx.alloc_map.lock().reserve();
|
||||
let (alloc, tag) = M::tag_allocation(id, Cow::Owned(alloc), Some(kind), &self);
|
||||
let (alloc, tag) = M::tag_allocation(&self.extra, id, Cow::Owned(alloc), Some(kind));
|
||||
self.alloc_map.insert(id, (kind, alloc.into_owned()));
|
||||
Pointer::from(id).with_tag(tag)
|
||||
}
|
||||
|
@ -407,9 +407,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
|
|||
/// contains a reference to memory that was created during its evaluation (i.e., not to
|
||||
/// another static), those inner references only exist in "resolved" form.
|
||||
fn get_static_alloc(
|
||||
id: AllocId,
|
||||
memory_extra: &M::MemoryExtra,
|
||||
tcx: TyCtxtAt<'tcx>,
|
||||
memory: &Memory<'mir, 'tcx, M>,
|
||||
id: AllocId,
|
||||
) -> InterpResult<'tcx, Cow<'tcx, Allocation<M::PointerTag, M::AllocExtra>>> {
|
||||
let alloc = tcx.alloc_map.lock().get(id);
|
||||
let alloc = match alloc {
|
||||
|
@ -423,7 +423,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
|
|||
// We got a "lazy" static that has not been computed yet.
|
||||
if tcx.is_foreign_item(def_id) {
|
||||
trace!("static_alloc: foreign item {:?}", def_id);
|
||||
M::find_foreign_static(def_id, tcx)?
|
||||
M::find_foreign_static(tcx.tcx, def_id)?
|
||||
} else {
|
||||
trace!("static_alloc: Need to compute {:?}", def_id);
|
||||
let instance = Instance::mono(tcx.tcx, def_id);
|
||||
|
@ -453,10 +453,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
|
|||
// We got tcx memory. Let the machine figure out whether and how to
|
||||
// turn that into memory with the right pointer tag.
|
||||
Ok(M::tag_allocation(
|
||||
memory_extra,
|
||||
id, // always use the ID we got as input, not the "hidden" one.
|
||||
alloc,
|
||||
M::STATIC_KIND.map(MemoryKind::Machine),
|
||||
memory
|
||||
).0)
|
||||
}
|
||||
|
||||
|
@ -469,7 +469,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
|
|||
// `get_static_alloc` that we can actually use directly without inserting anything anywhere.
|
||||
// So the error type is `InterpResult<'tcx, &Allocation<M::PointerTag>>`.
|
||||
let a = self.alloc_map.get_or(id, || {
|
||||
let alloc = Self::get_static_alloc(id, self.tcx, &self).map_err(Err)?;
|
||||
let alloc = Self::get_static_alloc(&self.extra, self.tcx, id).map_err(Err)?;
|
||||
match alloc {
|
||||
Cow::Borrowed(alloc) => {
|
||||
// We got a ref, cheaply return that as an "error" so that the
|
||||
|
@ -498,11 +498,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
|
|||
id: AllocId,
|
||||
) -> InterpResult<'tcx, &mut Allocation<M::PointerTag, M::AllocExtra>> {
|
||||
let tcx = self.tcx;
|
||||
let alloc = Self::get_static_alloc(id, tcx, &self);
|
||||
let memory_extra = &self.extra;
|
||||
let a = self.alloc_map.get_mut_or(id, || {
|
||||
// Need to make a copy, even if `get_static_alloc` is able
|
||||
// to give us a cheap reference.
|
||||
let alloc = alloc?;
|
||||
let alloc = Self::get_static_alloc(memory_extra, tcx, id)?;
|
||||
if alloc.mutability == Mutability::Immutable {
|
||||
return err!(ModifiedConstantMemory);
|
||||
}
|
||||
|
@ -948,7 +948,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
|
|||
) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
|
||||
match scalar {
|
||||
Scalar::Ptr(ptr) => Ok(ptr),
|
||||
_ => M::int_to_ptr(scalar.to_usize(self)?, self)
|
||||
_ => M::int_to_ptr(&self, scalar.to_usize(self)?)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -959,7 +959,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
|
|||
) -> InterpResult<'tcx, u128> {
|
||||
match scalar.to_bits_or_ptr(size, self) {
|
||||
Ok(bits) => Ok(bits),
|
||||
Err(ptr) => Ok(M::ptr_to_int(ptr, self)? as u128)
|
||||
Err(ptr) => Ok(M::ptr_to_int(&self, ptr)? as u128)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue