From c1db77e9b99adc945ee3dbaef8d3d01f651b9b50 Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Wed, 23 Nov 2016 23:02:30 +0100 Subject: [PATCH 1/3] core: Convert utility macros for the slice iterator into a trait Use an extension trait for the slice iterator's pointer manipulations. --- src/libcore/slice.rs | 68 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/src/libcore/slice.rs b/src/libcore/slice.rs index 871b63145ca..fca4583f049 100644 --- a/src/libcore/slice.rs +++ b/src/libcore/slice.rs @@ -161,21 +161,35 @@ macro_rules! slice_offset { ($ptr:expr, $by:expr) => {{ let ptr = $ptr; if size_from_ptr(ptr) == 0 { - ::intrinsics::arith_offset(ptr as *mut i8, $by) as *mut _ + (ptr as *mut i8).wrapping_offset($by) as _ } else { ptr.offset($by) } }}; } -macro_rules! slice_ref { +// make a &T from a *const T +macro_rules! make_ref { + ($ptr:expr) => {{ + let ptr = $ptr; + if size_from_ptr(ptr) == 0 { + // Use a non-null pointer value + &*(1 as *mut _) + } else { + &*ptr + } + }}; +} + +// make a &mut T from a *mut T +macro_rules! make_ref_mut { ($ptr:expr) => {{ let ptr = $ptr; if size_from_ptr(ptr) == 0 { // Use a non-null pointer value &mut *(1 as *mut _) } else { - mem::transmute(ptr) + &mut *ptr } }}; } @@ -796,7 +810,7 @@ fn size_from_ptr(_: *const T) -> usize { // The shared definition of the `Iter` and `IterMut` iterators macro_rules! iterator { - (struct $name:ident -> $ptr:ty, $elem:ty) => { + (struct $name:ident -> $ptr:ty, $elem:ty, $mkref:ident) => { #[stable(feature = "rust1", since = "1.0.0")] impl<'a, T> Iterator for $name<'a, T> { type Item = $elem; @@ -812,9 +826,7 @@ macro_rules! iterator { if self.ptr == self.end { None } else { - let old = self.ptr; - self.ptr = slice_offset!(self.ptr, 1); - Some(slice_ref!(old)) + Some($mkref!(self.ptr.post_inc())) } } } @@ -857,8 +869,7 @@ macro_rules! iterator { if self.end == self.ptr { None } else { - self.end = slice_offset!(self.end, -1); - Some(slice_ref!(self.end)) + Some($mkref!(self.end.pre_dec())) } } } @@ -980,7 +991,7 @@ impl<'a, T> Iter<'a, T> { } } -iterator!{struct Iter -> *const T, &'a T} +iterator!{struct Iter -> *const T, &'a T, make_ref} #[stable(feature = "rust1", since = "1.0.0")] impl<'a, T> ExactSizeIterator for Iter<'a, T> {} @@ -1104,7 +1115,7 @@ impl<'a, T> IterMut<'a, T> { } } -iterator!{struct IterMut -> *mut T, &'a mut T} +iterator!{struct IterMut -> *mut T, &'a mut T, make_ref_mut} #[stable(feature = "rust1", since = "1.0.0")] impl<'a, T> ExactSizeIterator for IterMut<'a, T> {} @@ -1115,6 +1126,41 @@ impl<'a, T> FusedIterator for IterMut<'a, T> {} #[unstable(feature = "trusted_len", issue = "37572")] unsafe impl<'a, T> TrustedLen for IterMut<'a, T> {} + +// Extension methods for raw pointers, used by the iterators +trait PointerExt : Copy { + unsafe fn slice_offset(self, i: isize) -> Self; + + /// Increment self by 1, but return the old value + #[inline(always)] + unsafe fn post_inc(&mut self) -> Self { + let current = *self; + *self = self.slice_offset(1); + current + } + + /// Decrement self by 1, and return the new value + #[inline(always)] + unsafe fn pre_dec(&mut self) -> Self { + *self = self.slice_offset(-1); + *self + } +} + +impl PointerExt for *const T { + #[inline(always)] + unsafe fn slice_offset(self, i: isize) -> Self { + slice_offset!(self, i) + } +} + +impl PointerExt for *mut T { + #[inline(always)] + unsafe fn slice_offset(self, i: isize) -> Self { + slice_offset!(self, i) + } +} + /// An internal abstraction over the splitting iterators, so that /// splitn, splitn_mut etc can be implemented once. #[doc(hidden)] From 7611e424a7274ab1094ceebc0381e36e81b4db72 Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Wed, 23 Nov 2016 23:02:30 +0100 Subject: [PATCH 2/3] core: Add ptrdistance to slice module --- src/libcore/slice.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libcore/slice.rs b/src/libcore/slice.rs index fca4583f049..b2bd1219997 100644 --- a/src/libcore/slice.rs +++ b/src/libcore/slice.rs @@ -833,9 +833,7 @@ macro_rules! iterator { #[inline] fn size_hint(&self) -> (usize, Option) { - let diff = (self.end as usize).wrapping_sub(self.ptr as usize); - let size = mem::size_of::(); - let exact = diff / (if size == 0 {1} else {size}); + let exact = ptrdistance(self.ptr, self.end); (exact, Some(exact)) } @@ -1127,6 +1125,15 @@ impl<'a, T> FusedIterator for IterMut<'a, T> {} unsafe impl<'a, T> TrustedLen for IterMut<'a, T> {} +// Return the number of elements of `T` from `start` to `end`. +// Return the arithmetic difference if `T` is zero size. +#[inline(always)] +fn ptrdistance(start: *const T, end: *const T) -> usize { + let diff = (end as usize).wrapping_sub(start as usize); + let size = mem::size_of::(); + diff / (if size == 0 { 1 } else { size }) +} + // Extension methods for raw pointers, used by the iterators trait PointerExt : Copy { unsafe fn slice_offset(self, i: isize) -> Self; From a54ddfb676c59bcf0dd3dc8ef869a6e5ac4afc0b Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Wed, 23 Nov 2016 23:02:30 +0100 Subject: [PATCH 3/3] core: Unroll the loop in the slice iterator search methods Introduce a helper method .search_while() that generalizes internal iteration (Iterator's all, find, position, fold and so on). The compiler does not unroll loops with conditional exits; we can do this manually instead to improve the performance of for example Iterator::find and Iterator::position when used on the slice iterators. The unrolling is patterned on libstdc++'s implementation of std::find_if. --- src/libcore/slice.rs | 118 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/src/libcore/slice.rs b/src/libcore/slice.rs index b2bd1219997..263b9c22621 100644 --- a/src/libcore/slice.rs +++ b/src/libcore/slice.rs @@ -852,6 +852,64 @@ macro_rules! iterator { fn last(mut self) -> Option<$elem> { self.next_back() } + + fn all(&mut self, mut predicate: F) -> bool + where F: FnMut(Self::Item) -> bool, + { + self.search_while(true, move |elt| { + if predicate(elt) { + SearchWhile::Continue + } else { + SearchWhile::Done(false) + } + }) + } + + fn any(&mut self, mut predicate: F) -> bool + where F: FnMut(Self::Item) -> bool, + { + !self.all(move |elt| !predicate(elt)) + } + + fn find(&mut self, mut predicate: F) -> Option + where F: FnMut(&Self::Item) -> bool, + { + self.search_while(None, move |elt| { + if predicate(&elt) { + SearchWhile::Done(Some(elt)) + } else { + SearchWhile::Continue + } + }) + } + + fn position(&mut self, mut predicate: F) -> Option + where F: FnMut(Self::Item) -> bool, + { + let mut index = 0; + self.search_while(None, move |elt| { + if predicate(elt) { + SearchWhile::Done(Some(index)) + } else { + index += 1; + SearchWhile::Continue + } + }) + } + + fn rposition(&mut self, mut predicate: F) -> Option + where F: FnMut(Self::Item) -> bool, + { + let mut index = self.len(); + self.rsearch_while(None, move |elt| { + index -= 1; + if predicate(elt) { + SearchWhile::Done(Some(index)) + } else { + SearchWhile::Continue + } + }) + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -872,6 +930,48 @@ macro_rules! iterator { } } } + + // search_while is a generalization of the internal iteration methods. + impl<'a, T> $name<'a, T> { + // search through the iterator's element using the closure `g`. + // if no element was found, return `default`. + fn search_while(&mut self, default: Acc, mut g: G) -> Acc + where Self: Sized, + G: FnMut($elem) -> SearchWhile + { + // manual unrolling is needed when there are conditional exits from the loop + unsafe { + while ptrdistance(self.ptr, self.end) >= 4 { + search_while!(g($mkref!(self.ptr.post_inc()))); + search_while!(g($mkref!(self.ptr.post_inc()))); + search_while!(g($mkref!(self.ptr.post_inc()))); + search_while!(g($mkref!(self.ptr.post_inc()))); + } + while self.ptr != self.end { + search_while!(g($mkref!(self.ptr.post_inc()))); + } + } + default + } + + fn rsearch_while(&mut self, default: Acc, mut g: G) -> Acc + where Self: Sized, + G: FnMut($elem) -> SearchWhile + { + unsafe { + while ptrdistance(self.ptr, self.end) >= 4 { + search_while!(g($mkref!(self.end.pre_dec()))); + search_while!(g($mkref!(self.end.pre_dec()))); + search_while!(g($mkref!(self.end.pre_dec()))); + search_while!(g($mkref!(self.end.pre_dec()))); + } + while self.ptr != self.end { + search_while!(g($mkref!(self.end.pre_dec()))); + } + } + default + } + } } } @@ -903,6 +1003,24 @@ macro_rules! make_mut_slice { }} } +// An enum used for controlling the execution of `.search_while()`. +enum SearchWhile { + // Continue searching + Continue, + // Fold is complete and will return this value + Done(T), +} + +// helper macro for search while's control flow +macro_rules! search_while { + ($e:expr) => { + match $e { + SearchWhile::Continue => { } + SearchWhile::Done(done) => return done, + } + } +} + /// Immutable slice iterator /// /// This struct is created by the [`iter`] method on [slices].