Auto merge of #72013 - nnethercote:make-RawVec-grow-mostly-non-generic, r=Amanieu
Make `RawVec::grow` mostly non-generic. `cargo-llvm-lines` shows that, in various benchmarks, `RawVec::grow` is instantiated 10s or 100s of times and accounts for 1-8% of lines of generated LLVM IR. This commit moves most of `RawVec::grow` into a separate function that isn't parameterized by `T`, which means it doesn't need to be instantiated many times. This reduces compile time significantly. r? @ghost
This commit is contained in:
commit
75e1463c52
|
@ -1354,7 +1354,9 @@ impl<T> VecDeque<T> {
|
||||||
/// ```
|
/// ```
|
||||||
#[stable(feature = "rust1", since = "1.0.0")]
|
#[stable(feature = "rust1", since = "1.0.0")]
|
||||||
pub fn push_front(&mut self, value: T) {
|
pub fn push_front(&mut self, value: T) {
|
||||||
self.grow_if_necessary();
|
if self.is_full() {
|
||||||
|
self.grow();
|
||||||
|
}
|
||||||
|
|
||||||
self.tail = self.wrap_sub(self.tail, 1);
|
self.tail = self.wrap_sub(self.tail, 1);
|
||||||
let tail = self.tail;
|
let tail = self.tail;
|
||||||
|
@ -1377,7 +1379,9 @@ impl<T> VecDeque<T> {
|
||||||
/// ```
|
/// ```
|
||||||
#[stable(feature = "rust1", since = "1.0.0")]
|
#[stable(feature = "rust1", since = "1.0.0")]
|
||||||
pub fn push_back(&mut self, value: T) {
|
pub fn push_back(&mut self, value: T) {
|
||||||
self.grow_if_necessary();
|
if self.is_full() {
|
||||||
|
self.grow();
|
||||||
|
}
|
||||||
|
|
||||||
let head = self.head;
|
let head = self.head;
|
||||||
self.head = self.wrap_add(self.head, 1);
|
self.head = self.wrap_add(self.head, 1);
|
||||||
|
@ -1485,7 +1489,9 @@ impl<T> VecDeque<T> {
|
||||||
#[stable(feature = "deque_extras_15", since = "1.5.0")]
|
#[stable(feature = "deque_extras_15", since = "1.5.0")]
|
||||||
pub fn insert(&mut self, index: usize, value: T) {
|
pub fn insert(&mut self, index: usize, value: T) {
|
||||||
assert!(index <= self.len(), "index out of bounds");
|
assert!(index <= self.len(), "index out of bounds");
|
||||||
self.grow_if_necessary();
|
if self.is_full() {
|
||||||
|
self.grow();
|
||||||
|
}
|
||||||
|
|
||||||
// Move the least number of elements in the ring buffer and insert
|
// Move the least number of elements in the ring buffer and insert
|
||||||
// the given object
|
// the given object
|
||||||
|
@ -2003,11 +2009,13 @@ impl<T> VecDeque<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// This may panic or abort
|
// This may panic or abort
|
||||||
#[inline]
|
#[inline(never)]
|
||||||
fn grow_if_necessary(&mut self) {
|
fn grow(&mut self) {
|
||||||
if self.is_full() {
|
if self.is_full() {
|
||||||
let old_cap = self.cap();
|
let old_cap = self.cap();
|
||||||
self.buf.double();
|
// Double the buffer size.
|
||||||
|
self.buf.reserve_exact(old_cap, old_cap);
|
||||||
|
assert!(self.cap() == old_cap * 2);
|
||||||
unsafe {
|
unsafe {
|
||||||
self.handle_capacity_increase(old_cap);
|
self.handle_capacity_increase(old_cap);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
#![unstable(feature = "raw_vec_internals", reason = "implementation detail", issue = "none")]
|
#![unstable(feature = "raw_vec_internals", reason = "implementation detail", issue = "none")]
|
||||||
#![doc(hidden)]
|
#![doc(hidden)]
|
||||||
|
|
||||||
use core::alloc::MemoryBlock;
|
use core::alloc::{LayoutErr, MemoryBlock};
|
||||||
use core::cmp;
|
use core::cmp;
|
||||||
use core::mem::{self, ManuallyDrop, MaybeUninit};
|
use core::mem::{self, ManuallyDrop, MaybeUninit};
|
||||||
use core::ops::Drop;
|
use core::ops::Drop;
|
||||||
|
@ -211,82 +211,6 @@ impl<T, A: AllocRef> RawVec<T, A> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Doubles the size of the type's backing allocation. This is common enough
|
|
||||||
/// to want to do that it's easiest to just have a dedicated method. Slightly
|
|
||||||
/// more efficient logic can be provided for this than the general case.
|
|
||||||
///
|
|
||||||
/// This function is ideal for when pushing elements one-at-a-time because
|
|
||||||
/// you don't need to incur the costs of the more general computations
|
|
||||||
/// reserve needs to do to guard against overflow. You do however need to
|
|
||||||
/// manually check if your `len == capacity`.
|
|
||||||
///
|
|
||||||
/// # Panics
|
|
||||||
///
|
|
||||||
/// * Panics if `T` is zero-sized on the assumption that you managed to exhaust
|
|
||||||
/// all `usize::MAX` slots in your imaginary buffer.
|
|
||||||
/// * Panics on 32-bit platforms if the requested capacity exceeds
|
|
||||||
/// `isize::MAX` bytes.
|
|
||||||
///
|
|
||||||
/// # Aborts
|
|
||||||
///
|
|
||||||
/// Aborts on OOM
|
|
||||||
///
|
|
||||||
/// # Examples
|
|
||||||
///
|
|
||||||
/// ```
|
|
||||||
/// # #![feature(raw_vec_internals)]
|
|
||||||
/// # extern crate alloc;
|
|
||||||
/// # use std::ptr;
|
|
||||||
/// # use alloc::raw_vec::RawVec;
|
|
||||||
/// struct MyVec<T> {
|
|
||||||
/// buf: RawVec<T>,
|
|
||||||
/// len: usize,
|
|
||||||
/// }
|
|
||||||
///
|
|
||||||
/// impl<T> MyVec<T> {
|
|
||||||
/// pub fn push(&mut self, elem: T) {
|
|
||||||
/// if self.len == self.buf.capacity() { self.buf.double(); }
|
|
||||||
/// // double would have aborted or panicked if the len exceeded
|
|
||||||
/// // `isize::MAX` so this is safe to do unchecked now.
|
|
||||||
/// unsafe {
|
|
||||||
/// ptr::write(self.buf.ptr().add(self.len), elem);
|
|
||||||
/// }
|
|
||||||
/// self.len += 1;
|
|
||||||
/// }
|
|
||||||
/// }
|
|
||||||
/// # fn main() {
|
|
||||||
/// # let mut vec = MyVec { buf: RawVec::new(), len: 0 };
|
|
||||||
/// # vec.push(1);
|
|
||||||
/// # }
|
|
||||||
/// ```
|
|
||||||
#[inline(never)]
|
|
||||||
#[cold]
|
|
||||||
pub fn double(&mut self) {
|
|
||||||
match self.grow(Double, MayMove, Uninitialized) {
|
|
||||||
Err(CapacityOverflow) => capacity_overflow(),
|
|
||||||
Err(AllocError { layout, .. }) => handle_alloc_error(layout),
|
|
||||||
Ok(()) => { /* yay */ }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Attempts to double the size of the type's backing allocation in place. This is common
|
|
||||||
/// enough to want to do that it's easiest to just have a dedicated method. Slightly
|
|
||||||
/// more efficient logic can be provided for this than the general case.
|
|
||||||
///
|
|
||||||
/// Returns `true` if the reallocation attempt has succeeded.
|
|
||||||
///
|
|
||||||
/// # Panics
|
|
||||||
///
|
|
||||||
/// * Panics if `T` is zero-sized on the assumption that you managed to exhaust
|
|
||||||
/// all `usize::MAX` slots in your imaginary buffer.
|
|
||||||
/// * Panics on 32-bit platforms if the requested capacity exceeds
|
|
||||||
/// `isize::MAX` bytes.
|
|
||||||
#[inline(never)]
|
|
||||||
#[cold]
|
|
||||||
pub fn double_in_place(&mut self) -> bool {
|
|
||||||
self.grow(Double, InPlace, Uninitialized).is_ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Ensures that the buffer contains at least enough space to hold
|
/// Ensures that the buffer contains at least enough space to hold
|
||||||
/// `used_capacity + needed_extra_capacity` elements. If it doesn't already have
|
/// `used_capacity + needed_extra_capacity` elements. If it doesn't already have
|
||||||
/// enough capacity, will reallocate enough space plus comfortable slack
|
/// enough capacity, will reallocate enough space plus comfortable slack
|
||||||
|
@ -354,7 +278,7 @@ impl<T, A: AllocRef> RawVec<T, A> {
|
||||||
needed_extra_capacity: usize,
|
needed_extra_capacity: usize,
|
||||||
) -> Result<(), TryReserveError> {
|
) -> Result<(), TryReserveError> {
|
||||||
if self.needs_to_grow(used_capacity, needed_extra_capacity) {
|
if self.needs_to_grow(used_capacity, needed_extra_capacity) {
|
||||||
self.grow(Amortized { used_capacity, needed_extra_capacity }, MayMove, Uninitialized)
|
self.grow_amortized(used_capacity, needed_extra_capacity, MayMove)
|
||||||
} else {
|
} else {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -381,8 +305,7 @@ impl<T, A: AllocRef> RawVec<T, A> {
|
||||||
// This is more readable than putting this in one line:
|
// This is more readable than putting this in one line:
|
||||||
// `!self.needs_to_grow(...) || self.grow(...).is_ok()`
|
// `!self.needs_to_grow(...) || self.grow(...).is_ok()`
|
||||||
if self.needs_to_grow(used_capacity, needed_extra_capacity) {
|
if self.needs_to_grow(used_capacity, needed_extra_capacity) {
|
||||||
self.grow(Amortized { used_capacity, needed_extra_capacity }, InPlace, Uninitialized)
|
self.grow_amortized(used_capacity, needed_extra_capacity, InPlace).is_ok()
|
||||||
.is_ok()
|
|
||||||
} else {
|
} else {
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
@ -423,7 +346,7 @@ impl<T, A: AllocRef> RawVec<T, A> {
|
||||||
needed_extra_capacity: usize,
|
needed_extra_capacity: usize,
|
||||||
) -> Result<(), TryReserveError> {
|
) -> Result<(), TryReserveError> {
|
||||||
if self.needs_to_grow(used_capacity, needed_extra_capacity) {
|
if self.needs_to_grow(used_capacity, needed_extra_capacity) {
|
||||||
self.grow(Exact { used_capacity, needed_extra_capacity }, MayMove, Uninitialized)
|
self.grow_exact(used_capacity, needed_extra_capacity)
|
||||||
} else {
|
} else {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -448,14 +371,6 @@ impl<T, A: AllocRef> RawVec<T, A> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Copy, Clone)]
|
|
||||||
enum Strategy {
|
|
||||||
Double,
|
|
||||||
Amortized { used_capacity: usize, needed_extra_capacity: usize },
|
|
||||||
Exact { used_capacity: usize, needed_extra_capacity: usize },
|
|
||||||
}
|
|
||||||
use Strategy::*;
|
|
||||||
|
|
||||||
impl<T, A: AllocRef> RawVec<T, A> {
|
impl<T, A: AllocRef> RawVec<T, A> {
|
||||||
/// Returns if the buffer needs to grow to fulfill the needed extra capacity.
|
/// Returns if the buffer needs to grow to fulfill the needed extra capacity.
|
||||||
/// Mainly used to make inlining reserve-calls possible without inlining `grow`.
|
/// Mainly used to make inlining reserve-calls possible without inlining `grow`.
|
||||||
|
@ -473,68 +388,59 @@ impl<T, A: AllocRef> RawVec<T, A> {
|
||||||
self.cap = Self::capacity_from_bytes(memory.size);
|
self.cap = Self::capacity_from_bytes(memory.size);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Single method to handle all possibilities of growing the buffer.
|
// This method is usually instantiated many times. So we want it to be as
|
||||||
fn grow(
|
// small as possible, to improve compile times. But we also want as much of
|
||||||
|
// its contents to be statically computable as possible, to make the
|
||||||
|
// generated code run faster. Therefore, this method is carefully written
|
||||||
|
// so that all of the code that depends on `T` is within it, while as much
|
||||||
|
// of the code that doesn't depend on `T` as possible is in functions that
|
||||||
|
// are non-generic over `T`.
|
||||||
|
fn grow_amortized(
|
||||||
&mut self,
|
&mut self,
|
||||||
strategy: Strategy,
|
used_capacity: usize,
|
||||||
|
needed_extra_capacity: usize,
|
||||||
placement: ReallocPlacement,
|
placement: ReallocPlacement,
|
||||||
init: AllocInit,
|
|
||||||
) -> Result<(), TryReserveError> {
|
) -> Result<(), TryReserveError> {
|
||||||
let elem_size = mem::size_of::<T>();
|
if mem::size_of::<T>() == 0 {
|
||||||
if elem_size == 0 {
|
|
||||||
// Since we return a capacity of `usize::MAX` when `elem_size` is
|
// Since we return a capacity of `usize::MAX` when `elem_size` is
|
||||||
// 0, getting to here necessarily means the `RawVec` is overfull.
|
// 0, getting to here necessarily means the `RawVec` is overfull.
|
||||||
return Err(CapacityOverflow);
|
return Err(CapacityOverflow);
|
||||||
}
|
}
|
||||||
let new_layout = match strategy {
|
|
||||||
Double => unsafe {
|
|
||||||
// Since we guarantee that we never allocate more than `isize::MAX` bytes,
|
|
||||||
// `elem_size * self.cap <= isize::MAX` as a precondition, so this can't overflow.
|
|
||||||
// Additionally the alignment will never be too large as to "not be satisfiable",
|
|
||||||
// so `Layout::from_size_align` will always return `Some`.
|
|
||||||
//
|
|
||||||
// TL;DR, we bypass runtime checks due to dynamic assertions in this module,
|
|
||||||
// allowing us to use `from_size_align_unchecked`.
|
|
||||||
let cap = if self.cap == 0 {
|
|
||||||
// Skip to 4 because tiny `Vec`'s are dumb; but not if that would cause overflow.
|
|
||||||
if elem_size > usize::MAX / 8 { 1 } else { 4 }
|
|
||||||
} else {
|
|
||||||
self.cap * 2
|
|
||||||
};
|
|
||||||
Layout::from_size_align_unchecked(cap * elem_size, mem::align_of::<T>())
|
|
||||||
},
|
|
||||||
Amortized { used_capacity, needed_extra_capacity } => {
|
|
||||||
// Nothing we can really do about these checks, sadly.
|
|
||||||
let required_cap =
|
|
||||||
used_capacity.checked_add(needed_extra_capacity).ok_or(CapacityOverflow)?;
|
|
||||||
// Cannot overflow, because `cap <= isize::MAX`, and type of `cap` is `usize`.
|
|
||||||
let double_cap = self.cap * 2;
|
|
||||||
// `double_cap` guarantees exponential growth.
|
|
||||||
let cap = cmp::max(double_cap, required_cap);
|
|
||||||
Layout::array::<T>(cap).map_err(|_| CapacityOverflow)?
|
|
||||||
}
|
|
||||||
Exact { used_capacity, needed_extra_capacity } => {
|
|
||||||
let cap =
|
|
||||||
used_capacity.checked_add(needed_extra_capacity).ok_or(CapacityOverflow)?;
|
|
||||||
Layout::array::<T>(cap).map_err(|_| CapacityOverflow)?
|
|
||||||
}
|
|
||||||
};
|
|
||||||
alloc_guard(new_layout.size())?;
|
|
||||||
|
|
||||||
let memory = if let Some((ptr, old_layout)) = self.current_memory() {
|
// Nothing we can really do about these checks, sadly.
|
||||||
debug_assert_eq!(old_layout.align(), new_layout.align());
|
let required_cap =
|
||||||
unsafe {
|
used_capacity.checked_add(needed_extra_capacity).ok_or(CapacityOverflow)?;
|
||||||
self.alloc
|
// Cannot overflow, because `cap <= isize::MAX`, and type of `cap` is `usize`.
|
||||||
.grow(ptr, old_layout, new_layout.size(), placement, init)
|
let double_cap = self.cap * 2;
|
||||||
.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })?
|
// `double_cap` guarantees exponential growth.
|
||||||
}
|
let cap = cmp::max(double_cap, required_cap);
|
||||||
} else {
|
let new_layout = Layout::array::<T>(cap);
|
||||||
match placement {
|
|
||||||
MayMove => self.alloc.alloc(new_layout, init),
|
// `finish_grow` is non-generic over `T`.
|
||||||
InPlace => Err(AllocErr),
|
let memory = finish_grow(new_layout, placement, self.current_memory(), &mut self.alloc)?;
|
||||||
}
|
self.set_memory(memory);
|
||||||
.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })?
|
Ok(())
|
||||||
};
|
}
|
||||||
|
|
||||||
|
// The constraints on this method are much the same as those on
|
||||||
|
// `grow_amortized`, but this method is usually instantiated less often so
|
||||||
|
// it's less critical.
|
||||||
|
fn grow_exact(
|
||||||
|
&mut self,
|
||||||
|
used_capacity: usize,
|
||||||
|
needed_extra_capacity: usize,
|
||||||
|
) -> Result<(), TryReserveError> {
|
||||||
|
if mem::size_of::<T>() == 0 {
|
||||||
|
// Since we return a capacity of `usize::MAX` when the type size is
|
||||||
|
// 0, getting to here necessarily means the `RawVec` is overfull.
|
||||||
|
return Err(CapacityOverflow);
|
||||||
|
}
|
||||||
|
|
||||||
|
let cap = used_capacity.checked_add(needed_extra_capacity).ok_or(CapacityOverflow)?;
|
||||||
|
let new_layout = Layout::array::<T>(cap);
|
||||||
|
|
||||||
|
// `finish_grow` is non-generic over `T`.
|
||||||
|
let memory = finish_grow(new_layout, MayMove, self.current_memory(), &mut self.alloc)?;
|
||||||
self.set_memory(memory);
|
self.set_memory(memory);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -562,6 +468,38 @@ impl<T, A: AllocRef> RawVec<T, A> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// This function is outside `RawVec` to minimize compile times. See the comment
|
||||||
|
// above `RawVec::grow_amortized` for details. (The `A` parameter isn't
|
||||||
|
// significant, because the number of different `A` types seen in practice is
|
||||||
|
// much smaller than the number of `T` types.)
|
||||||
|
fn finish_grow<A>(
|
||||||
|
new_layout: Result<Layout, LayoutErr>,
|
||||||
|
placement: ReallocPlacement,
|
||||||
|
current_memory: Option<(NonNull<u8>, Layout)>,
|
||||||
|
alloc: &mut A,
|
||||||
|
) -> Result<MemoryBlock, TryReserveError>
|
||||||
|
where
|
||||||
|
A: AllocRef,
|
||||||
|
{
|
||||||
|
// Check for the error here to minimize the size of `RawVec::grow_*`.
|
||||||
|
let new_layout = new_layout.map_err(|_| CapacityOverflow)?;
|
||||||
|
|
||||||
|
alloc_guard(new_layout.size())?;
|
||||||
|
|
||||||
|
let memory = if let Some((ptr, old_layout)) = current_memory {
|
||||||
|
debug_assert_eq!(old_layout.align(), new_layout.align());
|
||||||
|
unsafe { alloc.grow(ptr, old_layout, new_layout.size(), placement, Uninitialized) }
|
||||||
|
} else {
|
||||||
|
match placement {
|
||||||
|
MayMove => alloc.alloc(new_layout, Uninitialized),
|
||||||
|
InPlace => Err(AllocErr),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })?;
|
||||||
|
|
||||||
|
Ok(memory)
|
||||||
|
}
|
||||||
|
|
||||||
impl<T> RawVec<T, Global> {
|
impl<T> RawVec<T, Global> {
|
||||||
/// Converts the entire buffer into `Box<[MaybeUninit<T>]>` with the specified `len`.
|
/// Converts the entire buffer into `Box<[MaybeUninit<T>]>` with the specified `len`.
|
||||||
///
|
///
|
||||||
|
|
Loading…
Reference in New Issue