diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index 4fbcc4590f1..cb9daaea000 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -397,3 +397,26 @@ pub mod __alloc_error_handler { unsafe { oom_impl(layout) } } } + +/// Specialize clones into pre-allocated, uninitialized memory. +/// Used by `Box::clone` and `Rc`/`Arc::make_mut`. +pub(crate) trait WriteCloneIntoRaw: Sized { + unsafe fn write_clone_into_raw(&self, target: *mut Self); +} + +impl WriteCloneIntoRaw for T { + #[inline] + default unsafe fn write_clone_into_raw(&self, target: *mut Self) { + // Having allocated *first* may allow the optimizer to create + // the cloned value in-place, skipping the local and move. + unsafe { target.write(self.clone()) }; + } +} + +impl WriteCloneIntoRaw for T { + #[inline] + unsafe fn write_clone_into_raw(&self, target: *mut Self) { + // We can always copy in-place, without ever involving a local value. + unsafe { target.copy_from_nonoverlapping(self, 1) }; + } +} diff --git a/library/alloc/src/boxed.rs b/library/alloc/src/boxed.rs index 33b812ec59f..0aa52b35ced 100644 --- a/library/alloc/src/boxed.rs +++ b/library/alloc/src/boxed.rs @@ -151,7 +151,7 @@ use core::pin::Pin; use core::ptr::{self, Unique}; use core::task::{Context, Poll}; -use crate::alloc::{handle_alloc_error, AllocError, Allocator, Global, Layout}; +use crate::alloc::{handle_alloc_error, AllocError, Allocator, Global, Layout, WriteCloneIntoRaw}; use crate::borrow::Cow; use crate::raw_vec::RawVec; use crate::str::from_boxed_utf8_unchecked; @@ -1014,10 +1014,14 @@ impl Clone for Box { /// // But they are unique objects /// assert_ne!(&*x as *const i32, &*y as *const i32); /// ``` - #[rustfmt::skip] #[inline] fn clone(&self) -> Self { - Self::new_in((**self).clone(), self.1.clone()) + // Pre-allocate memory to allow writing the cloned value directly. + let mut boxed = Self::new_uninit_in(self.1.clone()); + unsafe { + (**self).write_clone_into_raw(boxed.as_mut_ptr()); + boxed.assume_init() + } } /// Copies `source`'s contents into `self` without creating a new allocation. diff --git a/library/alloc/src/rc.rs b/library/alloc/src/rc.rs index 8183a582d33..0973a6e362b 100644 --- a/library/alloc/src/rc.rs +++ b/library/alloc/src/rc.rs @@ -263,7 +263,9 @@ use core::pin::Pin; use core::ptr::{self, NonNull}; use core::slice::from_raw_parts_mut; -use crate::alloc::{box_free, handle_alloc_error, AllocError, Allocator, Global, Layout}; +use crate::alloc::{ + box_free, handle_alloc_error, AllocError, Allocator, Global, Layout, WriteCloneIntoRaw, +}; use crate::borrow::{Cow, ToOwned}; use crate::string::String; use crate::vec::Vec; @@ -1037,18 +1039,26 @@ impl Rc { #[stable(feature = "rc_unique", since = "1.4.0")] pub fn make_mut(this: &mut Self) -> &mut T { if Rc::strong_count(this) != 1 { - // Gotta clone the data, there are other Rcs - *this = Rc::new((**this).clone()) + // Gotta clone the data, there are other Rcs. + // Pre-allocate memory to allow writing the cloned value directly. + let mut rc = Self::new_uninit(); + unsafe { + let data = Rc::get_mut_unchecked(&mut rc); + (**this).write_clone_into_raw(data.as_mut_ptr()); + *this = rc.assume_init(); + } } else if Rc::weak_count(this) != 0 { // Can just steal the data, all that's left is Weaks + let mut rc = Self::new_uninit(); unsafe { - let mut swap = Rc::new(ptr::read(&this.ptr.as_ref().value)); - mem::swap(this, &mut swap); - swap.inner().dec_strong(); + let data = Rc::get_mut_unchecked(&mut rc); + data.as_mut_ptr().copy_from_nonoverlapping(&**this, 1); + + this.inner().dec_strong(); // Remove implicit strong-weak ref (no need to craft a fake // Weak here -- we know other Weaks can clean up for us) - swap.inner().dec_weak(); - forget(swap); + this.inner().dec_weak(); + ptr::write(this, rc.assume_init()); } } // This unsafety is ok because we're guaranteed that the pointer diff --git a/library/alloc/src/sync.rs b/library/alloc/src/sync.rs index 06ad6217271..05bfeccbda1 100644 --- a/library/alloc/src/sync.rs +++ b/library/alloc/src/sync.rs @@ -22,7 +22,9 @@ use core::slice::from_raw_parts_mut; use core::sync::atomic; use core::sync::atomic::Ordering::{Acquire, Relaxed, Release, SeqCst}; -use crate::alloc::{box_free, handle_alloc_error, AllocError, Allocator, Global, Layout}; +use crate::alloc::{ + box_free, handle_alloc_error, AllocError, Allocator, Global, Layout, WriteCloneIntoRaw, +}; use crate::borrow::{Cow, ToOwned}; use crate::boxed::Box; use crate::rc::is_dangling; @@ -1369,8 +1371,14 @@ impl Arc { // weak count, there's no chance the ArcInner itself could be // deallocated. if this.inner().strong.compare_exchange(1, 0, Acquire, Relaxed).is_err() { - // Another strong pointer exists; clone - *this = Arc::new((**this).clone()); + // Another strong pointer exists, so we must clone. + // Pre-allocate memory to allow writing the cloned value directly. + let mut arc = Self::new_uninit(); + unsafe { + let data = Arc::get_mut_unchecked(&mut arc); + (**this).write_clone_into_raw(data.as_mut_ptr()); + *this = arc.assume_init(); + } } else if this.inner().weak.load(Relaxed) != 1 { // Relaxed suffices in the above because this is fundamentally an // optimization: we are always racing with weak pointers being @@ -1386,17 +1394,14 @@ impl Arc { // Materialize our own implicit weak pointer, so that it can clean // up the ArcInner as needed. - let weak = Weak { ptr: this.ptr }; + let _weak = Weak { ptr: this.ptr }; - // mark the data itself as already deallocated + // Can just steal the data, all that's left is Weaks + let mut arc = Self::new_uninit(); unsafe { - // there is no data race in the implicit write caused by `read` - // here (due to zeroing) because data is no longer accessed by - // other threads (due to there being no more strong refs at this - // point). - let mut swap = Arc::new(ptr::read(&weak.ptr.as_ref().data)); - mem::swap(this, &mut swap); - mem::forget(swap); + let data = Arc::get_mut_unchecked(&mut arc); + data.as_mut_ptr().copy_from_nonoverlapping(&**this, 1); + ptr::write(this, arc.assume_init()); } } else { // We were the sole reference of either kind; bump back up the