Rollup merge of #62264 - RalfJung:inline-forcing, r=zackmdavis

Fix perf regression from Miri Machine trait changes

Maybe this fixes the perf regression that https://github.com/rust-lang/rust/pull/62003 seemingly introduced?

Cc @nnethercote

src/librustc_mir/const_eval.rs

@@ -11,7 +11,7 @@ use rustc::hir::def::DefKind;
 use rustc::hir::def_id::DefId;
 use rustc::mir::interpret::{ConstEvalErr, ErrorHandled, ScalarMaybeUndef};
 use rustc::mir;
-use rustc::ty::{self, TyCtxt, query::TyCtxtAt};
+use rustc::ty::{self, TyCtxt};
 use rustc::ty::layout::{self, LayoutOf, VariantIdx};
 use rustc::ty::subst::Subst;
 use rustc::traits::Reveal;
@@ -23,7 +23,7 @@ use crate::interpret::{self,
     PlaceTy, MPlaceTy, OpTy, ImmTy, Immediate, Scalar,
     RawConst, ConstValue,
     InterpResult, InterpErrorInfo, InterpError, GlobalId, InterpCx, StackPopCleanup,
-    Allocation, AllocId, MemoryKind, Memory,
+    Allocation, AllocId, MemoryKind,
     snapshot, RefTracking, intern_const_alloc_recursive,
 };
 
@@ -409,18 +409,18 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
     }
 
     fn find_foreign_static(
+        _tcx: TyCtxt<'tcx>,
         _def_id: DefId,
-        _tcx: TyCtxtAt<'tcx>,
     ) -> InterpResult<'tcx, Cow<'tcx, Allocation<Self::PointerTag>>> {
         err!(ReadForeignStatic)
     }
 
     #[inline(always)]
     fn tag_allocation<'b>(
+        _memory_extra: &(),
         _id: AllocId,
         alloc: Cow<'b, Allocation>,
         _kind: Option<MemoryKind<!>>,
-        _memory: &Memory<'mir, 'tcx, Self>,
     ) -> (Cow<'b, Allocation<Self::PointerTag>>, Self::PointerTag) {
         // We do not use a tag so we can just cheaply forward the allocation
         (alloc, ())
@@ -428,8 +428,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
 
     #[inline(always)]
     fn tag_static_base_pointer(
+        _memory_extra: &(),
         _id: AllocId,
-        _memory: &Memory<'mir, 'tcx, Self>,
     ) -> Self::PointerTag {
         ()
     }

src/librustc_mir/interpret/eval_context.rs

@@ -222,6 +222,23 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         &mut self.memory
     }
 
+    #[inline(always)]
+    pub fn force_ptr(
+        &self,
+        scalar: Scalar<M::PointerTag>,
+    ) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
+        self.memory.force_ptr(scalar)
+    }
+
+    #[inline(always)]
+    pub fn force_bits(
+        &self,
+        scalar: Scalar<M::PointerTag>,
+        size: Size
+    ) -> InterpResult<'tcx, u128> {
+        self.memory.force_bits(scalar, size)
+    }
+
     #[inline(always)]
     pub fn tag_static_base_pointer(&self, ptr: Pointer) -> Pointer<M::PointerTag> {
         self.memory.tag_static_base_pointer(ptr)
@@ -253,6 +270,27 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         self.frame().body
     }
 
+    #[inline(always)]
+    pub fn sign_extend(&self, value: u128, ty: TyLayout<'_>) -> u128 {
+        assert!(ty.abi.is_signed());
+        sign_extend(value, ty.size)
+    }
+
+    #[inline(always)]
+    pub fn truncate(&self, value: u128, ty: TyLayout<'_>) -> u128 {
+        truncate(value, ty.size)
+    }
+
+    #[inline]
+    pub fn type_is_sized(&self, ty: Ty<'tcx>) -> bool {
+        ty.is_sized(self.tcx, self.param_env)
+    }
+
+    #[inline]
+    pub fn type_is_freeze(&self, ty: Ty<'tcx>) -> bool {
+        ty.is_freeze(*self.tcx, self.param_env, DUMMY_SP)
+    }
+
     pub(super) fn subst_and_normalize_erasing_regions<T: TypeFoldable<'tcx>>(
         &self,
         substs: T,
@@ -288,14 +326,6 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         ).ok_or_else(|| InterpError::TooGeneric.into())
     }
 
-    pub fn type_is_sized(&self, ty: Ty<'tcx>) -> bool {
-        ty.is_sized(self.tcx, self.param_env)
-    }
-
-    pub fn type_is_freeze(&self, ty: Ty<'tcx>) -> bool {
-        ty.is_freeze(*self.tcx, self.param_env, DUMMY_SP)
-    }
-
     pub fn load_mir(
         &self,
         instance: ty::InstanceDef<'tcx>,
@@ -766,32 +796,4 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         trace!("generate stacktrace: {:#?}, {:?}", frames, explicit_span);
         frames
     }
-
-    #[inline(always)]
-    pub fn sign_extend(&self, value: u128, ty: TyLayout<'_>) -> u128 {
-        assert!(ty.abi.is_signed());
-        sign_extend(value, ty.size)
-    }
-
-    #[inline(always)]
-    pub fn truncate(&self, value: u128, ty: TyLayout<'_>) -> u128 {
-        truncate(value, ty.size)
-    }
-
-    #[inline(always)]
-    pub fn force_ptr(
-        &self,
-        scalar: Scalar<M::PointerTag>,
-    ) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
-        self.memory.force_ptr(scalar)
-    }
-
-    #[inline(always)]
-    pub fn force_bits(
-        &self,
-        scalar: Scalar<M::PointerTag>,
-        size: Size
-    ) -> InterpResult<'tcx, u128> {
-        self.memory.force_bits(scalar, size)
-    }
 }

src/librustc_mir/interpret/machine.rs

@@ -7,11 +7,11 @@ use std::hash::Hash;
 
 use rustc::hir::def_id::DefId;
 use rustc::mir;
-use rustc::ty::{self, query::TyCtxtAt};
+use rustc::ty::{self, TyCtxt};
 
 use super::{
-    Allocation, AllocId, InterpResult, Scalar, AllocationExtra,
+    Allocation, AllocId, InterpResult, InterpError, Scalar, AllocationExtra,
-    InterpCx, PlaceTy, OpTy, ImmTy, MemoryKind, Pointer, Memory
+    InterpCx, PlaceTy, OpTy, ImmTy, MemoryKind, Pointer, Memory,
 };
 
 /// Whether this kind of memory is allowed to leak
@@ -151,8 +151,8 @@ pub trait Machine<'mir, 'tcx>: Sized {
     ///
     /// This allocation will then be fed to `tag_allocation` to initialize the "extra" state.
     fn find_foreign_static(
+        tcx: TyCtxt<'tcx>,
         def_id: DefId,
-        tcx: TyCtxtAt<'tcx>,
     ) -> InterpResult<'tcx, Cow<'tcx, Allocation>>;
 
     /// Called for all binary operations on integer(-like) types when one operand is a pointer
@@ -189,10 +189,10 @@ pub trait Machine<'mir, 'tcx>: Sized {
     /// For static allocations, the tag returned must be the same as the one returned by
     /// `tag_static_base_pointer`.
     fn tag_allocation<'b>(
+        memory_extra: &Self::MemoryExtra,
         id: AllocId,
         alloc: Cow<'b, Allocation>,
         kind: Option<MemoryKind<Self::MemoryKinds>>,
-        memory: &Memory<'mir, 'tcx, Self>,
     ) -> (Cow<'b, Allocation<Self::PointerTag, Self::AllocExtra>>, Self::PointerTag);
 
     /// Return the "base" tag for the given static allocation: the one that is used for direct
@@ -201,8 +201,8 @@ pub trait Machine<'mir, 'tcx>: Sized {
     /// Be aware that requesting the `Allocation` for that `id` will lead to cycles
     /// for cyclic statics!
     fn tag_static_base_pointer(
+        memory_extra: &Self::MemoryExtra,
         id: AllocId,
-        memory: &Memory<'mir, 'tcx, Self>,
     ) -> Self::PointerTag;
 
     /// Executes a retagging operation
@@ -224,20 +224,22 @@ pub trait Machine<'mir, 'tcx>: Sized {
         extra: Self::FrameExtra,
     ) -> InterpResult<'tcx>;
 
+    #[inline(always)]
     fn int_to_ptr(
-        int: u64,
         _mem: &Memory<'mir, 'tcx, Self>,
+        int: u64,
     ) -> InterpResult<'tcx, Pointer<Self::PointerTag>> {
-        if int == 0 {
+        Err((if int == 0 {
-            err!(InvalidNullPointerUsage)
+            InterpError::InvalidNullPointerUsage
         } else {
-            err!(ReadBytesAsPointer)
+            InterpError::ReadBytesAsPointer
-        }
+        }).into())
     }
 
+    #[inline(always)]
     fn ptr_to_int(
-        _ptr: Pointer<Self::PointerTag>,
         _mem: &Memory<'mir, 'tcx, Self>,
+        _ptr: Pointer<Self::PointerTag>,
     ) -> InterpResult<'tcx, u64> {
         err!(ReadPointerAsBytes)
     }

src/librustc_mir/interpret/memory.rs

@@ -143,7 +143,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
 
     #[inline]
     pub fn tag_static_base_pointer(&self, ptr: Pointer) -> Pointer<M::PointerTag> {
-        ptr.with_tag(M::tag_static_base_pointer(ptr.alloc_id, &self))
+        ptr.with_tag(M::tag_static_base_pointer(&self.extra, ptr.alloc_id))
     }
 
     pub fn create_fn_alloc(
@@ -189,7 +189,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
         kind: MemoryKind<M::MemoryKinds>,
     ) -> Pointer<M::PointerTag> {
         let id = self.tcx.alloc_map.lock().reserve();
-        let (alloc, tag) = M::tag_allocation(id, Cow::Owned(alloc), Some(kind), &self);
+        let (alloc, tag) = M::tag_allocation(&self.extra, id, Cow::Owned(alloc), Some(kind));
         self.alloc_map.insert(id, (kind, alloc.into_owned()));
         Pointer::from(id).with_tag(tag)
     }
@@ -407,9 +407,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
     /// contains a reference to memory that was created during its evaluation (i.e., not to
     /// another static), those inner references only exist in "resolved" form.
     fn get_static_alloc(
-        id: AllocId,
+        memory_extra: &M::MemoryExtra,
         tcx: TyCtxtAt<'tcx>,
-        memory: &Memory<'mir, 'tcx, M>,
+        id: AllocId,
     ) -> InterpResult<'tcx, Cow<'tcx, Allocation<M::PointerTag, M::AllocExtra>>> {
         let alloc = tcx.alloc_map.lock().get(id);
         let alloc = match alloc {
@@ -423,7 +423,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
                 // We got a "lazy" static that has not been computed yet.
                 if tcx.is_foreign_item(def_id) {
                     trace!("static_alloc: foreign item {:?}", def_id);
-                    M::find_foreign_static(def_id, tcx)?
+                    M::find_foreign_static(tcx.tcx, def_id)?
                 } else {
                     trace!("static_alloc: Need to compute {:?}", def_id);
                     let instance = Instance::mono(tcx.tcx, def_id);
@@ -453,10 +453,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
         // We got tcx memory. Let the machine figure out whether and how to
         // turn that into memory with the right pointer tag.
         Ok(M::tag_allocation(
+            memory_extra,
             id, // always use the ID we got as input, not the "hidden" one.
             alloc,
             M::STATIC_KIND.map(MemoryKind::Machine),
-            memory
         ).0)
     }
 
@@ -469,7 +469,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
         // `get_static_alloc` that we can actually use directly without inserting anything anywhere.
         // So the error type is `InterpResult<'tcx, &Allocation<M::PointerTag>>`.
         let a = self.alloc_map.get_or(id, || {
-            let alloc = Self::get_static_alloc(id, self.tcx, &self).map_err(Err)?;
+            let alloc = Self::get_static_alloc(&self.extra, self.tcx, id).map_err(Err)?;
             match alloc {
                 Cow::Borrowed(alloc) => {
                     // We got a ref, cheaply return that as an "error" so that the
@@ -498,11 +498,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
         id: AllocId,
     ) -> InterpResult<'tcx, &mut Allocation<M::PointerTag, M::AllocExtra>> {
         let tcx = self.tcx;
-        let alloc = Self::get_static_alloc(id, tcx, &self);
+        let memory_extra = &self.extra;
         let a = self.alloc_map.get_mut_or(id, || {
             // Need to make a copy, even if `get_static_alloc` is able
             // to give us a cheap reference.
-            let alloc = alloc?;
+            let alloc = Self::get_static_alloc(memory_extra, tcx, id)?;
             if alloc.mutability == Mutability::Immutable {
                 return err!(ModifiedConstantMemory);
             }
@@ -948,7 +948,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
     ) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
         match scalar {
             Scalar::Ptr(ptr) => Ok(ptr),
-            _ => M::int_to_ptr(scalar.to_usize(self)?, self)
+            _ => M::int_to_ptr(&self, scalar.to_usize(self)?)
         }
     }
 
@@ -959,7 +959,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
     ) -> InterpResult<'tcx, u128> {
         match scalar.to_bits_or_ptr(size, self) {
             Ok(bits) => Ok(bits),
-            Err(ptr) => Ok(M::ptr_to_int(ptr, self)? as u128)
+            Err(ptr) => Ok(M::ptr_to_int(&self, ptr)? as u128)
         }
     }
 }