diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index 98c7ac3f2ef..16f3f5d5745 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -164,16 +164,26 @@ pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 { #[unstable(feature = "allocator_api", issue = "32838")] unsafe impl AllocRef for Global { #[inline] - fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result { + fn alloc(&mut self, layout: Layout) -> Result { unsafe { let size = layout.size(); if size == 0 { Ok(MemoryBlock { ptr: layout.dangling(), size: 0 }) } else { - let raw_ptr = match init { - AllocInit::Uninitialized => alloc(layout), - AllocInit::Zeroed => alloc_zeroed(layout), - }; + let raw_ptr = alloc(layout); + let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?; + Ok(MemoryBlock { ptr, size }) + } + } + } + + fn alloc_zeroed(&mut self, layout: Layout) -> Result { + unsafe { + let size = layout.size(); + if size == 0 { + Ok(MemoryBlock { ptr: layout.dangling(), size: 0 }) + } else { + let raw_ptr = alloc_zeroed(layout); let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?; Ok(MemoryBlock { ptr, size }) } @@ -193,8 +203,6 @@ unsafe impl AllocRef for Global { ptr: NonNull, layout: Layout, new_size: usize, - placement: ReallocPlacement, - init: AllocInit, ) -> Result { let size = layout.size(); debug_assert!( @@ -206,26 +214,49 @@ unsafe impl AllocRef for Global { return Ok(MemoryBlock { ptr, size }); } - match placement { - ReallocPlacement::InPlace => Err(AllocErr), - ReallocPlacement::MayMove if layout.size() == 0 => { - let new_layout = - unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; - self.alloc(new_layout, init) - } - ReallocPlacement::MayMove => { - // `realloc` probably checks for `new_size > size` or something similar. - let ptr = unsafe { - intrinsics::assume(new_size > size); - realloc(ptr.as_ptr(), layout, new_size) - }; - let memory = - MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }; - unsafe { - init.init_offset(memory, size); - } - Ok(memory) + if layout.size() == 0 { + let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + self.alloc(new_layout) + } else { + // `realloc` probably checks for `new_size > size` or something similar. + let ptr = unsafe { + intrinsics::assume(new_size > size); + realloc(ptr.as_ptr(), layout, new_size) + }; + Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }) + } + } + + unsafe fn grow_zeroed( + &mut self, + ptr: NonNull, + layout: Layout, + new_size: usize, + ) -> Result { + let size = layout.size(); + debug_assert!( + new_size >= size, + "`new_size` must be greater than or equal to `memory.size()`" + ); + + if size == new_size { + return Ok(MemoryBlock { ptr, size }); + } + + if layout.size() == 0 { + let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + self.alloc(new_layout) + } else { + // `realloc` probably checks for `new_size > size` or something similar. + let ptr = unsafe { + intrinsics::assume(new_size > size); + realloc(ptr.as_ptr(), layout, new_size) + }; + let memory = MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }; + unsafe { + memory.ptr.as_ptr().add(size).write_bytes(0, memory.size - size); } + Ok(memory) } } @@ -235,7 +266,6 @@ unsafe impl AllocRef for Global { ptr: NonNull, layout: Layout, new_size: usize, - placement: ReallocPlacement, ) -> Result { let size = layout.size(); debug_assert!( @@ -247,22 +277,18 @@ unsafe impl AllocRef for Global { return Ok(MemoryBlock { ptr, size }); } - match placement { - ReallocPlacement::InPlace => Err(AllocErr), - ReallocPlacement::MayMove if new_size == 0 => { - unsafe { - self.dealloc(ptr, layout); - } - Ok(MemoryBlock { ptr: layout.dangling(), size: 0 }) - } - ReallocPlacement::MayMove => { - // `realloc` probably checks for `new_size < size` or something similar. - let ptr = unsafe { - intrinsics::assume(new_size < size); - realloc(ptr.as_ptr(), layout, new_size) - }; - Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }) + if new_size == 0 { + unsafe { + self.dealloc(ptr, layout); } + Ok(MemoryBlock { ptr: layout.dangling(), size: 0 }) + } else { + // `realloc` probably checks for `new_size < size` or something similar. + let ptr = unsafe { + intrinsics::assume(new_size < size); + realloc(ptr.as_ptr(), layout, new_size) + }; + Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }) } } } @@ -274,7 +300,7 @@ unsafe impl AllocRef for Global { #[inline] unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 { let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; - match Global.alloc(layout, AllocInit::Uninitialized) { + match Global.alloc(layout) { Ok(memory) => memory.ptr.as_ptr(), Err(_) => handle_alloc_error(layout), } diff --git a/library/alloc/src/alloc/tests.rs b/library/alloc/src/alloc/tests.rs index 1c003983df9..2b4cb946bb4 100644 --- a/library/alloc/src/alloc/tests.rs +++ b/library/alloc/src/alloc/tests.rs @@ -8,9 +8,8 @@ use test::Bencher; fn allocate_zeroed() { unsafe { let layout = Layout::from_size_align(1024, 1).unwrap(); - let memory = Global - .alloc(layout.clone(), AllocInit::Zeroed) - .unwrap_or_else(|_| handle_alloc_error(layout)); + let memory = + Global.alloc_zeroed(layout.clone()).unwrap_or_else(|_| handle_alloc_error(layout)); let mut i = memory.ptr.cast::().as_ptr(); let end = i.add(layout.size()); diff --git a/library/alloc/src/boxed.rs b/library/alloc/src/boxed.rs index f225aa18853..cae98f8c76f 100644 --- a/library/alloc/src/boxed.rs +++ b/library/alloc/src/boxed.rs @@ -146,7 +146,7 @@ use core::pin::Pin; use core::ptr::{self, NonNull, Unique}; use core::task::{Context, Poll}; -use crate::alloc::{self, AllocInit, AllocRef, Global}; +use crate::alloc::{self, AllocRef, Global}; use crate::borrow::Cow; use crate::raw_vec::RawVec; use crate::str::from_boxed_utf8_unchecked; @@ -197,11 +197,8 @@ impl Box { #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit() -> Box> { let layout = alloc::Layout::new::>(); - let ptr = Global - .alloc(layout, AllocInit::Uninitialized) - .unwrap_or_else(|_| alloc::handle_alloc_error(layout)) - .ptr - .cast(); + let ptr = + Global.alloc(layout).unwrap_or_else(|_| alloc::handle_alloc_error(layout)).ptr.cast(); unsafe { Box::from_raw(ptr.as_ptr()) } } @@ -227,7 +224,7 @@ impl Box { pub fn new_zeroed() -> Box> { let layout = alloc::Layout::new::>(); let ptr = Global - .alloc(layout, AllocInit::Zeroed) + .alloc_zeroed(layout) .unwrap_or_else(|_| alloc::handle_alloc_error(layout)) .ptr .cast(); diff --git a/library/alloc/src/raw_vec.rs b/library/alloc/src/raw_vec.rs index ed81ce71ddf..99ac027bf0b 100644 --- a/library/alloc/src/raw_vec.rs +++ b/library/alloc/src/raw_vec.rs @@ -8,18 +8,20 @@ use core::ops::Drop; use core::ptr::{NonNull, Unique}; use core::slice; -use crate::alloc::{ - handle_alloc_error, - AllocInit::{self, *}, - AllocRef, Global, Layout, - ReallocPlacement::{self, *}, -}; +use crate::alloc::{handle_alloc_error, AllocRef, Global, Layout}; use crate::boxed::Box; use crate::collections::TryReserveError::{self, *}; #[cfg(test)] mod tests; +enum AllocInit { + /// The contents of the new memory are uninitialized. + Uninitialized, + /// The new memory is guaranteed to be zeroed. + Zeroed, +} + /// A low-level utility for more ergonomically allocating, reallocating, and deallocating /// a buffer of memory on the heap without having to worry about all the corner cases /// involved. This type is excellent for building your own data structures like Vec and VecDeque. @@ -156,14 +158,14 @@ impl RawVec { /// allocator for the returned `RawVec`. #[inline] pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { - Self::allocate_in(capacity, Uninitialized, alloc) + Self::allocate_in(capacity, AllocInit::Uninitialized, alloc) } /// Like `with_capacity_zeroed`, but parameterized over the choice /// of allocator for the returned `RawVec`. #[inline] pub fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { - Self::allocate_in(capacity, Zeroed, alloc) + Self::allocate_in(capacity, AllocInit::Zeroed, alloc) } fn allocate_in(capacity: usize, init: AllocInit, mut alloc: A) -> Self { @@ -180,7 +182,11 @@ impl RawVec { Ok(_) => {} Err(_) => capacity_overflow(), } - let memory = match alloc.alloc(layout, init) { + let result = match init { + AllocInit::Uninitialized => alloc.alloc(layout), + AllocInit::Zeroed => alloc.alloc_zeroed(layout), + }; + let memory = match result { Ok(memory) => memory, Err(_) => handle_alloc_error(layout), }; @@ -358,7 +364,7 @@ impl RawVec { /// /// Aborts on OOM. pub fn shrink_to_fit(&mut self, amount: usize) { - match self.shrink(amount, MayMove) { + match self.shrink(amount) { Err(CapacityOverflow) => capacity_overflow(), Err(AllocError { layout, .. }) => handle_alloc_error(layout), Ok(()) => { /* yay */ } @@ -450,22 +456,16 @@ impl RawVec { Ok(()) } - fn shrink( - &mut self, - amount: usize, - placement: ReallocPlacement, - ) -> Result<(), TryReserveError> { + fn shrink(&mut self, amount: usize) -> Result<(), TryReserveError> { assert!(amount <= self.capacity(), "Tried to shrink to a larger capacity"); let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) }; let new_size = amount * mem::size_of::(); let memory = unsafe { - self.alloc.shrink(ptr, layout, new_size, placement).map_err(|_| { - TryReserveError::AllocError { - layout: Layout::from_size_align_unchecked(new_size, layout.align()), - non_exhaustive: (), - } + self.alloc.shrink(ptr, layout, new_size).map_err(|_| TryReserveError::AllocError { + layout: Layout::from_size_align_unchecked(new_size, layout.align()), + non_exhaustive: (), })? }; self.set_memory(memory); @@ -492,9 +492,9 @@ where let memory = if let Some((ptr, old_layout)) = current_memory { debug_assert_eq!(old_layout.align(), new_layout.align()); - unsafe { alloc.grow(ptr, old_layout, new_layout.size(), MayMove, Uninitialized) } + unsafe { alloc.grow(ptr, old_layout, new_layout.size()) } } else { - alloc.alloc(new_layout, Uninitialized) + alloc.alloc(new_layout) } .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })?; diff --git a/library/alloc/src/raw_vec/tests.rs b/library/alloc/src/raw_vec/tests.rs index 5408faa079c..08a5cbee5a7 100644 --- a/library/alloc/src/raw_vec/tests.rs +++ b/library/alloc/src/raw_vec/tests.rs @@ -20,12 +20,12 @@ fn allocator_param() { fuel: usize, } unsafe impl AllocRef for BoundedAlloc { - fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result { + fn alloc(&mut self, layout: Layout) -> Result { let size = layout.size(); if size > self.fuel { return Err(AllocErr); } - match Global.alloc(layout, init) { + match Global.alloc(layout) { ok @ Ok(_) => { self.fuel -= size; ok diff --git a/library/alloc/src/rc.rs b/library/alloc/src/rc.rs index 96dfc2f4251..c3af42c0cbb 100644 --- a/library/alloc/src/rc.rs +++ b/library/alloc/src/rc.rs @@ -250,7 +250,7 @@ use core::pin::Pin; use core::ptr::{self, NonNull}; use core::slice::from_raw_parts_mut; -use crate::alloc::{box_free, handle_alloc_error, AllocInit, AllocRef, Global, Layout}; +use crate::alloc::{box_free, handle_alloc_error, AllocRef, Global, Layout}; use crate::borrow::{Cow, ToOwned}; use crate::string::String; use crate::vec::Vec; @@ -951,9 +951,7 @@ impl Rc { let layout = Layout::new::>().extend(value_layout).unwrap().0.pad_to_align(); // Allocate for the layout. - let mem = Global - .alloc(layout, AllocInit::Uninitialized) - .unwrap_or_else(|_| handle_alloc_error(layout)); + let mem = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout)); // Initialize the RcBox let inner = mem_to_rcbox(mem.ptr.as_ptr()); diff --git a/library/alloc/src/sync.rs b/library/alloc/src/sync.rs index 8a5f1ee5076..bd70639428a 100644 --- a/library/alloc/src/sync.rs +++ b/library/alloc/src/sync.rs @@ -23,7 +23,7 @@ use core::slice::from_raw_parts_mut; use core::sync::atomic; use core::sync::atomic::Ordering::{Acquire, Relaxed, Release, SeqCst}; -use crate::alloc::{box_free, handle_alloc_error, AllocInit, AllocRef, Global, Layout}; +use crate::alloc::{box_free, handle_alloc_error, AllocRef, Global, Layout}; use crate::borrow::{Cow, ToOwned}; use crate::boxed::Box; use crate::rc::is_dangling; @@ -906,9 +906,7 @@ impl Arc { // reference (see #54908). let layout = Layout::new::>().extend(value_layout).unwrap().0.pad_to_align(); - let mem = Global - .alloc(layout, AllocInit::Uninitialized) - .unwrap_or_else(|_| handle_alloc_error(layout)); + let mem = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout)); // Initialize the ArcInner let inner = mem_to_arcinner(mem.ptr.as_ptr()); diff --git a/library/alloc/tests/heap.rs b/library/alloc/tests/heap.rs index 62f062b83d7..a05340dc79a 100644 --- a/library/alloc/tests/heap.rs +++ b/library/alloc/tests/heap.rs @@ -1,4 +1,4 @@ -use std::alloc::{AllocInit, AllocRef, Global, Layout, System}; +use std::alloc::{AllocRef, Global, Layout, System}; /// Issue #45955 and #62251. #[test] @@ -20,13 +20,7 @@ fn check_overalign_requests(mut allocator: T) { unsafe { let pointers: Vec<_> = (0..iterations) .map(|_| { - allocator - .alloc( - Layout::from_size_align(size, align).unwrap(), - AllocInit::Uninitialized, - ) - .unwrap() - .ptr + allocator.alloc(Layout::from_size_align(size, align).unwrap()).unwrap().ptr }) .collect(); for &ptr in &pointers { diff --git a/library/core/src/alloc/mod.rs b/library/core/src/alloc/mod.rs index be4e051b1ca..f5632b578ae 100644 --- a/library/core/src/alloc/mod.rs +++ b/library/core/src/alloc/mod.rs @@ -29,66 +29,6 @@ impl fmt::Display for AllocErr { } } -/// A desired initial state for allocated memory. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[unstable(feature = "allocator_api", issue = "32838")] -pub enum AllocInit { - /// The contents of the new memory are uninitialized. - Uninitialized, - /// The new memory is guaranteed to be zeroed. - Zeroed, -} - -impl AllocInit { - /// Initialize the specified memory block. - /// - /// This behaves like calling [`AllocInit::init_offset(memory, 0)`][off]. - /// - /// [off]: AllocInit::init_offset - /// - /// # Safety - /// - /// * `memory.ptr` must be [valid] for writes of `memory.size` bytes. - /// - /// [valid]: ../../core/ptr/index.html#safety - #[inline] - #[unstable(feature = "allocator_api", issue = "32838")] - pub unsafe fn init(self, memory: MemoryBlock) { - // SAFETY: the safety contract for `init_offset` must be - // upheld by the caller. - unsafe { self.init_offset(memory, 0) } - } - - /// Initialize the memory block like specified by `init` at the specified `offset`. - /// - /// This is a no-op for [`AllocInit::Uninitialized`][] and writes zeroes for - /// [`AllocInit::Zeroed`][] at `ptr + offset` until `ptr + layout.size()`. - /// - /// # Safety - /// - /// * `memory.ptr` must be [valid] for writes of `memory.size` bytes. - /// * `offset` must be smaller than or equal to `memory.size` - /// - /// [valid]: ../../core/ptr/index.html#safety - #[inline] - #[unstable(feature = "allocator_api", issue = "32838")] - pub unsafe fn init_offset(self, memory: MemoryBlock, offset: usize) { - debug_assert!( - offset <= memory.size, - "`offset` must be smaller than or equal to `memory.size`" - ); - match self { - AllocInit::Uninitialized => (), - AllocInit::Zeroed => { - // SAFETY: the caller must guarantee that `offset` is smaller than or equal to `memory.size`, - // so the memory from `memory.ptr + offset` of length `memory.size - offset` - // is guaranteed to be contaned in `memory` and thus valid for writes. - unsafe { memory.ptr.as_ptr().add(offset).write_bytes(0, memory.size - offset) } - } - } - } -} - /// Represents a block of allocated memory returned by an allocator. #[derive(Debug, Copy, Clone)] #[unstable(feature = "allocator_api", issue = "32838")] @@ -97,24 +37,6 @@ pub struct MemoryBlock { pub size: usize, } -/// A placement constraint when growing or shrinking an existing allocation. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[unstable(feature = "allocator_api", issue = "32838")] -pub enum ReallocPlacement { - /// The allocator is allowed to move the allocation to a different memory address. - // FIXME(wg-allocators#46): Add a section to the module documentation "What is a legal - // allocator" and link it at "valid location". - /// - /// If the allocation _does_ move, it's the responsibility of the allocator - /// to also move the data from the previous location to the new location. - MayMove, - /// The address of the new memory must not change. - /// - /// If the allocation would have to be moved to a new location to fit, the - /// reallocation request will fail. - InPlace, -} - /// An implementation of `AllocRef` can allocate, grow, shrink, and deallocate arbitrary blocks of /// data described via [`Layout`][]. /// @@ -177,10 +99,8 @@ pub unsafe trait AllocRef { /// /// On success, returns a [`MemoryBlock`][] meeting the size and alignment guarantees of `layout`. /// - /// The returned block may have a larger size than specified by `layout.size()` and is - /// initialized as specified by [`init`], all the way up to the returned size of the block. - /// - /// [`init`]: AllocInit + /// The returned block may have a larger size than specified by `layout.size()`, and may or may + /// not have its contents initialized. /// /// # Errors /// @@ -195,7 +115,29 @@ pub unsafe trait AllocRef { /// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar. /// /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html - fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result; + fn alloc(&mut self, layout: Layout) -> Result; + + /// Behaves like `alloc`, but also ensures that the contents are set to zero before being returned. + /// + /// # Errors + /// + /// Returning `Err` indicates that either memory is exhausted or `layout` does not meet + /// allocator's size or alignment constraints. + /// + /// Implementations are encouraged to return `Err` on memory exhaustion rather than panicking or + /// aborting, but this is not a strict requirement. (Specifically: it is *legal* to implement + /// this trait atop an underlying native allocation library that aborts on memory exhaustion.) + /// + /// Clients wishing to abort computation in response to an allocation error are encouraged to + /// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + fn alloc_zeroed(&mut self, layout: Layout) -> Result { + let memory = self.alloc(layout)?; + // SAFETY: `alloc` returns a valid memory block + unsafe { memory.ptr.as_ptr().write_bytes(0, memory.size) } + Ok(memory) + } /// Deallocates the memory referenced by `ptr`. /// @@ -213,31 +155,11 @@ pub unsafe trait AllocRef { /// Returns a new [`MemoryBlock`][] containing a pointer and the actual size of the allocated /// memory. The pointer is suitable for holding data described by a new layout with `layout`’s /// alignment and a size given by `new_size`. To accomplish this, the allocator may extend the - /// allocation referenced by `ptr` to fit the new layout. If the [`placement`] is - /// [`InPlace`], the returned pointer is guaranteed to be the same as the passed `ptr`. - /// - /// If [`MayMove`] is used then ownership of the memory block referenced by `ptr` - /// is transferred to this allocator. The memory may or may not be freed, and should be - /// considered unusable (unless of course it is transferred back to the caller again via the - /// return value of this method). - /// + /// allocation referenced by `ptr` to fit the new layout. + ///~ /// If this method returns `Err`, then ownership of the memory block has not been transferred to /// this allocator, and the contents of the memory block are unaltered. /// - /// The memory block will contain the following contents after a successful call to `grow`: - /// * Bytes `0..layout.size()` are preserved from the original allocation. - /// * Bytes `layout.size()..old_size` will either be preserved or initialized according to - /// [`init`], depending on the allocator implementation. `old_size` refers to the size of - /// the `MemoryBlock` prior to the `grow` call, which may be larger than the size - /// that was originally requested when it was allocated. - /// * Bytes `old_size..new_size` are initialized according to [`init`]. `new_size` refers to - /// the size of the `MemoryBlock` returned by the `grow` call. - /// - /// [`InPlace`]: ReallocPlacement::InPlace - /// [`MayMove`]: ReallocPlacement::MayMove - /// [`placement`]: ReallocPlacement - /// [`init`]: AllocInit - /// /// # Safety /// /// * `ptr` must denote a block of memory [*currently allocated*] via this allocator, @@ -270,40 +192,105 @@ pub unsafe trait AllocRef { ptr: NonNull, layout: Layout, new_size: usize, - placement: ReallocPlacement, - init: AllocInit, ) -> Result { - match placement { - ReallocPlacement::InPlace => Err(AllocErr), - ReallocPlacement::MayMove => { - let size = layout.size(); - debug_assert!( - new_size >= size, - "`new_size` must be greater than or equal to `layout.size()`" - ); + let size = layout.size(); + debug_assert!( + new_size >= size, + "`new_size` must be greater than or equal to `layout.size()`" + ); - if new_size == size { - return Ok(MemoryBlock { ptr, size }); - } + if new_size == size { + return Ok(MemoryBlock { ptr, size }); + } - let new_layout = - // SAFETY: the caller must ensure that the `new_size` does not overflow. - // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout. - // The caller must ensure that `new_size` is greater than zero. - unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; - let new_memory = self.alloc(new_layout, init)?; + let new_layout = + // SAFETY: the caller must ensure that the `new_size` does not overflow. + // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout. + // The caller must ensure that `new_size` is greater than or equal to zero. If it's equal + // to zero, it's catched beforehand. + unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + let new_memory = self.alloc(new_layout)?; - // SAFETY: because `new_size` must be greater than or equal to `size`, both the old and new - // memory allocation are valid for reads and writes for `size` bytes. Also, because the old - // allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to - // `copy_nonoverlapping` is safe. - // The safety contract for `dealloc` must be upheld by the caller. - unsafe { - ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), size); - self.dealloc(ptr, layout); - Ok(new_memory) - } - } + // SAFETY: because `new_size` must be greater than or equal to `size`, both the old and new + // memory allocation are valid for reads and writes for `size` bytes. Also, because the old + // allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to + // `copy_nonoverlapping` is safe. + // The safety contract for `dealloc` must be upheld by the caller. + unsafe { + ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), size); + self.dealloc(ptr, layout); + Ok(new_memory) + } + } + + /// Behaves like `grow`, but also ensures that the new contents are set to zero before being + /// returned. + /// + /// The memory block will contain the following contents after a successful call to `grow`: + /// * Bytes `0..layout.size()` are preserved from the original allocation. + /// * Bytes `layout.size()..new_size` are zeroed. `new_size` refers to + /// the size of the `MemoryBlock` returned by the `grow` call. + /// + /// # Safety + /// + /// * `ptr` must denote a block of memory [*currently allocated*] via this allocator, + /// * `layout` must [*fit*] that block of memory (The `new_size` argument need not fit it.), + // We can't require that `new_size` is strictly greater than `memory.size` because of ZSTs. + // An alternative would be + // * `new_size must be strictly greater than `memory.size` or both are zero + /// * `new_size` must be greater than or equal to `layout.size()`, and + /// * `new_size`, when rounded up to the nearest multiple of `layout.align()`, must not overflow + /// (i.e., the rounded value must be less than or equal to `usize::MAX`). + /// + /// [*currently allocated*]: #currently-allocated-memory + /// [*fit*]: #memory-fitting + /// + /// # Errors + /// + /// Returns `Err` if the new layout does not meet the allocator's size and alignment + /// constraints of the allocator, or if growing otherwise fails. + /// + /// Implementations are encouraged to return `Err` on memory exhaustion rather than panicking or + /// aborting, but this is not a strict requirement. (Specifically: it is *legal* to implement + /// this trait atop an underlying native allocation library that aborts on memory exhaustion.) + /// + /// Clients wishing to abort computation in response to an allocation error are encouraged to + /// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + unsafe fn grow_zeroed( + &mut self, + ptr: NonNull, + layout: Layout, + new_size: usize, + ) -> Result { + let size = layout.size(); + debug_assert!( + new_size >= size, + "`new_size` must be greater than or equal to `layout.size()`" + ); + + if new_size == size { + return Ok(MemoryBlock { ptr, size }); + } + + let new_layout = + // SAFETY: the caller must ensure that the `new_size` does not overflow. + // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout. + // The caller must ensure that `new_size` is greater than or equal to zero. If it's equal + // to zero, it's catched beforehand. + unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + let new_memory = self.alloc_zeroed(new_layout)?; + + // SAFETY: because `new_size` must be greater than or equal to `size`, both the old and new + // memory allocation are valid for reads and writes for `size` bytes. Also, because the old + // allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to + // `copy_nonoverlapping` is safe. + // The safety contract for `dealloc` must be upheld by the caller. + unsafe { + ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), size); + self.dealloc(ptr, layout); + Ok(new_memory) } } @@ -312,8 +299,7 @@ pub unsafe trait AllocRef { /// Returns a new [`MemoryBlock`][] containing a pointer and the actual size of the allocated /// memory. The pointer is suitable for holding data described by a new layout with `layout`’s /// alignment and a size given by `new_size`. To accomplish this, the allocator may shrink the - /// allocation referenced by `ptr` to fit the new layout. If the [`placement`] is - /// [`InPlace`], the returned pointer is guaranteed to be the same as the passed `ptr`. + /// allocation referenced by `ptr` to fit the new layout. /// /// If this returns `Ok`, then ownership of the memory block referenced by `ptr` has been /// transferred to this allocator. The memory may or may not have been freed, and should be @@ -323,11 +309,6 @@ pub unsafe trait AllocRef { /// If this method returns `Err`, then ownership of the memory block has not been transferred to /// this allocator, and the contents of the memory block are unaltered. /// - /// The behavior of how the allocator tries to shrink the memory is specified by [`placement`]. - /// - /// [`InPlace`]: ReallocPlacement::InPlace - /// [`placement`]: ReallocPlacement - /// /// # Safety /// /// * `ptr` must denote a block of memory [*currently allocated*] via this allocator, @@ -358,39 +339,33 @@ pub unsafe trait AllocRef { ptr: NonNull, layout: Layout, new_size: usize, - placement: ReallocPlacement, ) -> Result { - match placement { - ReallocPlacement::InPlace => Err(AllocErr), - ReallocPlacement::MayMove => { - let size = layout.size(); - debug_assert!( - new_size <= size, - "`new_size` must be smaller than or equal to `layout.size()`" - ); + let size = layout.size(); + debug_assert!( + new_size <= size, + "`new_size` must be smaller than or equal to `layout.size()`" + ); - if new_size == size { - return Ok(MemoryBlock { ptr, size }); - } + if new_size == size { + return Ok(MemoryBlock { ptr, size }); + } - let new_layout = - // SAFETY: the caller must ensure that the `new_size` does not overflow. - // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout. - // The caller must ensure that `new_size` is greater than zero. - unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; - let new_memory = self.alloc(new_layout, AllocInit::Uninitialized)?; + let new_layout = + // SAFETY: the caller must ensure that the `new_size` does not overflow. + // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout. + // The caller must ensure that `new_size` is greater than zero. + unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + let new_memory = self.alloc(new_layout)?; - // SAFETY: because `new_size` must be lower than or equal to `size`, both the old and new - // memory allocation are valid for reads and writes for `new_size` bytes. Also, because the - // old allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to - // `copy_nonoverlapping` is safe. - // The safety contract for `dealloc` must be upheld by the caller. - unsafe { - ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), new_size); - self.dealloc(ptr, layout); - Ok(new_memory) - } - } + // SAFETY: because `new_size` must be lower than or equal to `size`, both the old and new + // memory allocation are valid for reads and writes for `new_size` bytes. Also, because the + // old allocation wasn't yet deallocated, it cannot overlap `new_memory`. Thus, the call to + // `copy_nonoverlapping` is safe. + // The safety contract for `dealloc` must be upheld by the caller. + unsafe { + ptr::copy_nonoverlapping(ptr.as_ptr(), new_memory.ptr.as_ptr(), new_size); + self.dealloc(ptr, layout); + Ok(new_memory) } } @@ -409,8 +384,13 @@ where A: AllocRef + ?Sized, { #[inline] - fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result { - (**self).alloc(layout, init) + fn alloc(&mut self, layout: Layout) -> Result { + (**self).alloc(layout) + } + + #[inline] + fn alloc_zeroed(&mut self, layout: Layout) -> Result { + (**self).alloc_zeroed(layout) } #[inline] @@ -425,11 +405,20 @@ where ptr: NonNull, layout: Layout, new_size: usize, - placement: ReallocPlacement, - init: AllocInit, ) -> Result { // SAFETY: the safety contract must be upheld by the caller - unsafe { (**self).grow(ptr, layout, new_size, placement, init) } + unsafe { (**self).grow(ptr, layout, new_size) } + } + + #[inline] + unsafe fn grow_zeroed( + &mut self, + ptr: NonNull, + layout: Layout, + new_size: usize, + ) -> Result { + // SAFETY: the safety contract must be upheld by the caller + unsafe { (**self).grow_zeroed(ptr, layout, new_size) } } #[inline] @@ -438,9 +427,8 @@ where ptr: NonNull, layout: Layout, new_size: usize, - placement: ReallocPlacement, ) -> Result { // SAFETY: the safety contract must be upheld by the caller - unsafe { (**self).shrink(ptr, layout, new_size, placement) } + unsafe { (**self).shrink(ptr, layout, new_size) } } } diff --git a/library/std/src/alloc.rs b/library/std/src/alloc.rs index ecfaaeace51..11c36d398f1 100644 --- a/library/std/src/alloc.rs +++ b/library/std/src/alloc.rs @@ -140,16 +140,27 @@ pub struct System; #[unstable(feature = "allocator_api", issue = "32838")] unsafe impl AllocRef for System { #[inline] - fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result { + fn alloc(&mut self, layout: Layout) -> Result { unsafe { let size = layout.size(); if size == 0 { Ok(MemoryBlock { ptr: layout.dangling(), size: 0 }) } else { - let raw_ptr = match init { - AllocInit::Uninitialized => GlobalAlloc::alloc(self, layout), - AllocInit::Zeroed => GlobalAlloc::alloc_zeroed(self, layout), - }; + let raw_ptr = GlobalAlloc::alloc(self, layout); + let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?; + Ok(MemoryBlock { ptr, size }) + } + } + } + + #[inline] + fn alloc_zeroed(&mut self, layout: Layout) -> Result { + unsafe { + let size = layout.size(); + if size == 0 { + Ok(MemoryBlock { ptr: layout.dangling(), size: 0 }) + } else { + let raw_ptr = GlobalAlloc::alloc_zeroed(self, layout); let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?; Ok(MemoryBlock { ptr, size }) } @@ -171,8 +182,6 @@ unsafe impl AllocRef for System { ptr: NonNull, layout: Layout, new_size: usize, - placement: ReallocPlacement, - init: AllocInit, ) -> Result { let size = layout.size(); debug_assert!( @@ -184,51 +193,95 @@ unsafe impl AllocRef for System { return Ok(MemoryBlock { ptr, size }); } - match placement { - ReallocPlacement::InPlace => Err(AllocErr), - ReallocPlacement::MayMove if layout.size() == 0 => { - let new_layout = - // SAFETY: The new size and layout alignement guarantees - // are transfered to the caller (they come from parameters). - // - // See the preconditions for `Layout::from_size_align` to - // see what must be checked. - unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; - self.alloc(new_layout, init) - } - ReallocPlacement::MayMove => { - // SAFETY: + if layout.size() == 0 { + let new_layout = + // SAFETY: The new size and layout alignement guarantees + // are transfered to the caller (they come from parameters). // - // The safety guarantees are explained in the documentation - // for the `GlobalAlloc` trait and its `dealloc` method. - // - // `realloc` probably checks for `new_size > size` or something - // similar. - // - // For the guarantees about `init_offset`, see its documentation: - // `ptr` is assumed valid (and checked for non-NUL) and - // `memory.size` is set to `new_size` so the offset being `size` - // is valid. - let memory = unsafe { - intrinsics::assume(new_size > size); - let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size); - let memory = - MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }; - init.init_offset(memory, size); - memory - }; + // See the preconditions for `Layout::from_size_align` to + // see what must be checked. + unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + self.alloc(new_layout) + } else { + // SAFETY: + // + // The safety guarantees are explained in the documentation + // for the `GlobalAlloc` trait and its `dealloc` method. + // + // `realloc` probably checks for `new_size > size` or something + // similar. + // + // For the guarantees about `init_offset`, see its documentation: + // `ptr` is assumed valid (and checked for non-NUL) and + // `memory.size` is set to `new_size` so the offset being `size` + // is valid. + unsafe { + intrinsics::assume(new_size > size); + let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size); + let memory = + MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }; Ok(memory) } } } + #[inline] + unsafe fn grow_zeroed( + &mut self, + ptr: NonNull, + layout: Layout, + new_size: usize, + ) -> Result { + let size = layout.size(); + debug_assert!( + new_size >= size, + "`new_size` must be greater than or equal to `memory.size()`" + ); + + if size == new_size { + return Ok(MemoryBlock { ptr, size }); + } + + if layout.size() == 0 { + let new_layout = + // SAFETY: The new size and layout alignement guarantees + // are transfered to the caller (they come from parameters). + // + // See the preconditions for `Layout::from_size_align` to + // see what must be checked. + unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + self.alloc_zeroed(new_layout) + } else { + // SAFETY: + // + // The safety guarantees are explained in the documentation + // for the `GlobalAlloc` trait and its `dealloc` method. + // + // `realloc` probably checks for `new_size > size` or something + // similar. + // + // For the guarantees about `init_offset`, see its documentation: + // `ptr` is assumed valid (and checked for non-NUL) and + // `memory.size` is set to `new_size` so the offset being `size` + // is valid. + let memory = unsafe { + intrinsics::assume(new_size > size); + let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size); + let memory = + MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size }; + memory.ptr.as_ptr().add(size).write_bytes(0, memory.size - size); + memory + }; + Ok(memory) + } + } + #[inline] unsafe fn shrink( &mut self, ptr: NonNull, layout: Layout, new_size: usize, - placement: ReallocPlacement, ) -> Result { let size = layout.size(); debug_assert!( @@ -240,32 +293,28 @@ unsafe impl AllocRef for System { return Ok(MemoryBlock { ptr, size }); } - match placement { - ReallocPlacement::InPlace => Err(AllocErr), - ReallocPlacement::MayMove if new_size == 0 => { - // SAFETY: see `GlobalAlloc::dealloc` for the guarantees that - // must be respected. `ptr` and `layout` are parameters and so - // those guarantees must be checked by the caller. - unsafe { self.dealloc(ptr, layout) }; - Ok(MemoryBlock { ptr: layout.dangling(), size: 0 }) - } - ReallocPlacement::MayMove => { - // SAFETY: - // - // See `GlobalAlloc::realloc` for more informations about the - // guarantees expected by this method. `ptr`, `layout` and - // `new_size` are parameters and the responsability for their - // correctness is left to the caller. - // - // `realloc` probably checks for `new_size < size` or something - // similar. - let memory = unsafe { - intrinsics::assume(new_size < size); - let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size); - MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size } - }; - Ok(memory) - } + if new_size == 0 { + // SAFETY: see `GlobalAlloc::dealloc` for the guarantees that + // must be respected. `ptr` and `layout` are parameters and so + // those guarantees must be checked by the caller. + unsafe { self.dealloc(ptr, layout) }; + Ok(MemoryBlock { ptr: layout.dangling(), size: 0 }) + } else { + // SAFETY: + // + // See `GlobalAlloc::realloc` for more informations about the + // guarantees expected by this method. `ptr`, `layout` and + // `new_size` are parameters and the responsability for their + // correctness is left to the caller. + // + // `realloc` probably checks for `new_size < size` or something + // similar. + let memory = unsafe { + intrinsics::assume(new_size < size); + let ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size); + MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size } + }; + Ok(memory) } } } diff --git a/src/test/ui/allocator/custom.rs b/src/test/ui/allocator/custom.rs index 184e4706a4c..f10d29f33fc 100644 --- a/src/test/ui/allocator/custom.rs +++ b/src/test/ui/allocator/custom.rs @@ -7,7 +7,7 @@ extern crate helper; -use std::alloc::{self, AllocInit, AllocRef, Global, Layout, System}; +use std::alloc::{self, AllocRef, Global, Layout, System}; use std::sync::atomic::{AtomicUsize, Ordering}; static HITS: AtomicUsize = AtomicUsize::new(0); @@ -37,7 +37,7 @@ fn main() { unsafe { let layout = Layout::from_size_align(4, 2).unwrap(); - let memory = Global.alloc(layout.clone(), AllocInit::Uninitialized).unwrap(); + let memory = Global.alloc(layout.clone()).unwrap(); helper::work_with(&memory.ptr); assert_eq!(HITS.load(Ordering::SeqCst), n + 1); Global.dealloc(memory.ptr, layout); @@ -49,7 +49,7 @@ fn main() { drop(s); assert_eq!(HITS.load(Ordering::SeqCst), n + 4); - let memory = System.alloc(layout.clone(), AllocInit::Uninitialized).unwrap(); + let memory = System.alloc(layout.clone()).unwrap(); assert_eq!(HITS.load(Ordering::SeqCst), n + 4); helper::work_with(&memory.ptr); System.dealloc(memory.ptr, layout); diff --git a/src/test/ui/allocator/xcrate-use.rs b/src/test/ui/allocator/xcrate-use.rs index 7de1ab7a553..c7d31f71074 100644 --- a/src/test/ui/allocator/xcrate-use.rs +++ b/src/test/ui/allocator/xcrate-use.rs @@ -9,7 +9,7 @@ extern crate custom; extern crate helper; -use std::alloc::{AllocInit, AllocRef, Global, Layout, System}; +use std::alloc::{AllocRef, Global, Layout, System}; use std::sync::atomic::{AtomicUsize, Ordering}; #[global_allocator] @@ -20,13 +20,13 @@ fn main() { let n = GLOBAL.0.load(Ordering::SeqCst); let layout = Layout::from_size_align(4, 2).unwrap(); - let memory = Global.alloc(layout.clone(), AllocInit::Uninitialized).unwrap(); + let memory = Global.alloc(layout.clone()).unwrap(); helper::work_with(&memory.ptr); assert_eq!(GLOBAL.0.load(Ordering::SeqCst), n + 1); Global.dealloc(memory.ptr, layout); assert_eq!(GLOBAL.0.load(Ordering::SeqCst), n + 2); - let memory = System.alloc(layout.clone(), AllocInit::Uninitialized).unwrap(); + let memory = System.alloc(layout.clone()).unwrap(); assert_eq!(GLOBAL.0.load(Ordering::SeqCst), n + 2); helper::work_with(&memory.ptr); System.dealloc(memory.ptr, layout); diff --git a/src/test/ui/realloc-16687.rs b/src/test/ui/realloc-16687.rs index 0687a9ce454..e9435bb476c 100644 --- a/src/test/ui/realloc-16687.rs +++ b/src/test/ui/realloc-16687.rs @@ -6,7 +6,7 @@ #![feature(allocator_api)] -use std::alloc::{handle_alloc_error, AllocInit, AllocRef, Global, Layout, ReallocPlacement}; +use std::alloc::{handle_alloc_error, AllocRef, Global, Layout}; use std::ptr::{self, NonNull}; fn main() { @@ -41,9 +41,7 @@ unsafe fn test_triangle() -> bool { println!("allocate({:?})", layout); } - let memory = Global - .alloc(layout, AllocInit::Uninitialized) - .unwrap_or_else(|_| handle_alloc_error(layout)); + let memory = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout)); if PRINT { println!("allocate({:?}) = {:?}", layout, memory.ptr); @@ -70,11 +68,9 @@ unsafe fn test_triangle() -> bool { NonNull::new_unchecked(ptr), old, new.size(), - ReallocPlacement::MayMove, - AllocInit::Uninitialized, ) } else { - Global.shrink(NonNull::new_unchecked(ptr), old, new.size(), ReallocPlacement::MayMove) + Global.shrink(NonNull::new_unchecked(ptr), old, new.size()) }; let memory = memory.unwrap_or_else(|_| { diff --git a/src/test/ui/regions/regions-mock-codegen.rs b/src/test/ui/regions/regions-mock-codegen.rs index 380310190be..7d433530033 100644 --- a/src/test/ui/regions/regions-mock-codegen.rs +++ b/src/test/ui/regions/regions-mock-codegen.rs @@ -4,7 +4,7 @@ // pretty-expanded FIXME #23616 #![feature(allocator_api)] -use std::alloc::{handle_alloc_error, AllocInit, AllocRef, Global, Layout}; +use std::alloc::{handle_alloc_error, AllocRef, Global, Layout}; use std::ptr::NonNull; struct arena(()); @@ -25,9 +25,7 @@ struct Ccx { fn alloc(_bcx: &arena) -> &Bcx<'_> { unsafe { let layout = Layout::new::(); - let memory = Global - .alloc(layout, AllocInit::Uninitialized) - .unwrap_or_else(|_| handle_alloc_error(layout)); + let memory = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout)); &*(memory.ptr.as_ptr() as *const _) } }