From 06472075cf314a01442270fbaa8eee3f700d53dd Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Wed, 13 Jan 2021 19:51:07 -0500 Subject: [PATCH] Outline the drain to heap logic in TinyVec::push (#127) * Outline the drain to heap logic in TinyVec::push * Use #[cold] and explain the outlining in a comment --- src/tinyvec.rs | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/tinyvec.rs b/src/tinyvec.rs index aa8945d..86a7cc2 100644 --- a/src/tinyvec.rs +++ b/src/tinyvec.rs @@ -676,18 +676,34 @@ impl TinyVec { /// tv.push(4); /// assert_eq!(tv.as_slice(), &[1, 2, 3, 4]); /// ``` - #[inline(always)] + #[inline] pub fn push(&mut self, val: A::Item) { - let arr = match self { - TinyVec::Heap(v) => return v.push(val), - TinyVec::Inline(a) => a, - }; - - if let Some(x) = arr.try_push(val) { + // The code path for moving the inline contents to the heap produces a lot + // of instructions, but we have a strong guarantee that this is a cold + // path. LLVM doesn't know this, inlines it, and this tends to cause a + // cascade of other bad inlining decisions because the body of push looks + // huge even though nearly every call executes the same few instructions. + // + // Moving the logic out of line with #[cold] causes the hot code to be + // inlined together, and we take the extra cost of a function call only + // in rare cases. + #[cold] + fn drain_to_heap_and_push( + arr: &mut ArrayVec, val: A::Item, + ) -> TinyVec { /* Make the Vec twice the size to amortize the cost of draining */ let mut v = arr.drain_to_vec_and_reserve(arr.len()); - v.push(x); - *self = TinyVec::Heap(v); + v.push(val); + TinyVec::Heap(v) + } + + match self { + TinyVec::Heap(v) => v.push(val), + TinyVec::Inline(arr) => { + if let Some(x) = arr.try_push(val) { + *self = drain_to_heap_and_push(arr, x); + } + } } }