Use a union to reduce the size of SmallVec

.travis.yml

@@ -5,10 +5,9 @@ rust:
   - stable
 script: |
   cargo build --verbose &&
-  cargo build --all-features --verbose &&
   cargo test --verbose &&
-  cargo test --all-features --verbose &&
   ([ $TRAVIS_RUST_VERSION != nightly ] || cargo test --verbose --no-default-features) &&
+  ([ $TRAVIS_RUST_VERSION != nightly ] || cargo test --verbose --features union) &&
   ([ $TRAVIS_RUST_VERSION != nightly ] || cargo bench --verbose bench)
 notifications:
   webhooks: http://build.servo.org:54856/travis

Cargo.toml

@@ -12,6 +12,7 @@ documentation = "http://doc.servo.org/smallvec/"
 
 [features]
 std = []
+union = []
 default = ["std"]
 
 [lib]
@@ -19,6 +20,8 @@ name = "smallvec"
 path = "lib.rs"
 
 [dependencies]
+unreachable = "1.0.0"
+debug_unreachable = "0.1"
 serde = { version = "1", optional = true }
 
 [dev_dependencies]

lib.rs

@@ -17,6 +17,7 @@
 
 #![cfg_attr(not(feature = "std"), no_std)]
 #![cfg_attr(not(feature = "std"), feature(alloc))]
+#![cfg_attr(feature = "union", feature(untagged_unions))]
 #![deny(missing_docs)]
 
 
@@ -30,6 +31,13 @@ use alloc::Vec;
 #[cfg(feature = "serde")]
 extern crate serde;
 
+extern crate unreachable;
+use unreachable::UncheckedOptionExt;
+
+#[cfg(not(feature = "union"))]
+#[macro_use]
+extern crate debug_unreachable;
+
 #[cfg(not(feature = "std"))]
 mod std {
     pub use core::*;
@@ -41,6 +49,8 @@ use std::fmt;
 use std::hash::{Hash, Hasher};
 use std::iter::{IntoIterator, FromIterator, repeat};
 use std::mem;
+#[cfg(not(feature = "union"))]
+use std::mem::ManuallyDrop;
 use std::ops;
 use std::ptr;
 use std::slice;
@@ -53,8 +63,6 @@ use serde::de::{Deserialize, Deserializer, SeqAccess, Visitor};
 #[cfg(feature = "serde")]
 use std::marker::PhantomData;
 
-use SmallVecData::{Inline, Heap};
-
 /// Creates a [`SmallVec`] containing the arguments.
 ///
 /// `smallvec!` allows `SmallVec`s to be defined with the same syntax as array expressions.
@@ -240,40 +248,90 @@ impl<'a, T: 'a> Drop for Drain<'a,T> {
     }
 }
 
-enum SmallVecData<A: Array> {
-    Inline { array: A },
-    Heap { ptr: *mut A::Item, capacity: usize },
+#[cfg(feature = "union")]
+#[allow(unions_with_drop_fields)]
+union SmallVecData<A: Array> {
+    inline: A,
+    heap: (*mut A::Item, usize),
 }
 
+#[cfg(feature = "union")]
 impl<A: Array> SmallVecData<A> {
-    fn ptr_mut(&mut self) -> *mut A::Item {
+    #[inline]
+    unsafe fn inline(&self) -> &A {
+        &self.inline
+    }
+    #[inline]
+    unsafe fn inline_mut(&mut self) -> &mut A {
+        &mut self.inline
+    }
+    #[inline]
+    fn from_inline(inline: A) -> SmallVecData<A> {
+        SmallVecData { inline }
+    }
+    #[inline]
+    unsafe fn heap(&self) -> (*mut A::Item, usize) {
+        self.heap
+    }
+    #[inline]
+    unsafe fn heap_mut(&mut self) -> &mut (*mut A::Item, usize) {
+        &mut self.heap
+    }
+    #[inline]
+    fn from_heap(ptr: *mut A::Item, len: usize) -> SmallVecData<A> {
+        SmallVecData { heap: (ptr, len) }
+    }
+}
+
+#[cfg(not(feature = "union"))]
+enum SmallVecData<A: Array> {
+    Inline(ManuallyDrop<A>),
+    Heap((*mut A::Item, usize)),
+}
+
+#[cfg(not(feature = "union"))]
+impl<A: Array> SmallVecData<A> {
+    #[inline]
+    unsafe fn inline(&self) -> &A {
         match *self {
-            Inline { ref mut array } => array.ptr_mut(),
-            Heap { ptr, .. } => ptr,
+            SmallVecData::Inline(ref a) => a,
+            _ => debug_unreachable!(),
         }
     }
+    #[inline]
+    unsafe fn inline_mut(&mut self) -> &mut A {
+        match *self {
+            SmallVecData::Inline(ref mut a) => a,
+            _ => debug_unreachable!(),
+        }
+    }
+    #[inline]
+    fn from_inline(inline: A) -> SmallVecData<A> {
+        SmallVecData::Inline(ManuallyDrop::new(inline))
+    }
+    #[inline]
+    unsafe fn heap(&self) -> (*mut A::Item, usize) {
+        match *self {
+            SmallVecData::Heap(data) => data,
+            _ => debug_unreachable!(),
+        }
+    }
+    #[inline]
+    unsafe fn heap_mut(&mut self) -> &mut (*mut A::Item, usize) {
+        match *self {
+            SmallVecData::Heap(ref mut data) => data,
+            _ => debug_unreachable!(),
+        }
+    }
+    #[inline]
+    fn from_heap(ptr: *mut A::Item, len: usize) -> SmallVecData<A> {
+        SmallVecData::Heap((ptr, len))
+    }
 }
 
 unsafe impl<A: Array + Send> Send for SmallVecData<A> {}
 unsafe impl<A: Array + Sync> Sync for SmallVecData<A> {}
 
-impl<A: Array> Drop for SmallVecData<A> {
-    fn drop(&mut self) {
-        unsafe {
-            match *self {
-                ref mut inline @ Inline { .. } => {
-                    // Inhibit the array destructor.
-                    ptr::write(inline, Heap {
-                        ptr: ptr::null_mut(),
-                        capacity: 0,
-                    });
-                }
-                Heap { ptr, capacity } => deallocate(ptr, capacity),
-            }
-        }
-    }
-}
-
 /// A `Vec`-like container that can store a small number of elements inline.
 ///
 /// `SmallVec` acts like a vector, but can store a limited amount of data inline within the
@@ -301,7 +359,7 @@ impl<A: Array> Drop for SmallVecData<A> {
 /// assert!(v.spilled());
 /// ```
 pub struct SmallVec<A: Array> {
-    len: usize,
+    capacity: usize,
     data: SmallVecData<A>,
 }
 
@@ -311,8 +369,8 @@ impl<A: Array> SmallVec<A> {
     pub fn new() -> SmallVec<A> {
         unsafe {
             SmallVec {
-                len: 0,
-                data: Inline { array: mem::uninitialized() },
+                capacity: 0,
+                data: SmallVecData::from_inline(mem::uninitialized()),
             }
         }
     }
@@ -350,14 +408,25 @@ impl<A: Array> SmallVec<A> {
     /// ```
     #[inline]
     pub fn from_vec(mut vec: Vec<A::Item>) -> SmallVec<A> {
-        let (ptr, cap, len) = (vec.as_mut_ptr(), vec.capacity(), vec.len());
-        mem::forget(vec);
+        if vec.capacity() <= A::size() {
+            unsafe {
+                let mut data = SmallVecData::<A>::from_inline(mem::uninitialized());
+                let len = vec.len();
+                vec.set_len(0);
+                ptr::copy_nonoverlapping(vec.as_ptr(), data.inline_mut().ptr_mut(), len);
 
-        SmallVec {
-            len: len,
-            data: SmallVecData::Heap {
-                ptr: ptr,
-                capacity: cap
+                SmallVec {
+                    capacity: len,
+                    data,
+                }
+            }
+        } else {
+            let (ptr, cap, len) = (vec.as_mut_ptr(), vec.capacity(), vec.len());
+            mem::forget(vec);
+
+            SmallVec {
+                capacity: cap,
+                data: SmallVecData::from_heap(ptr, len),
             }
         }
     }
@@ -376,8 +445,8 @@ impl<A: Array> SmallVec<A> {
     #[inline]
     pub fn from_buf(buf: A) -> SmallVec<A> {
         SmallVec {
-            len: A::size(),
-            data: SmallVecData::Inline { array: buf },
+            capacity: A::size(),
+            data: SmallVecData::from_inline(buf),
         }
     }
 
@@ -387,7 +456,8 @@ impl<A: Array> SmallVec<A> {
     /// modifying its buffers, so it is up to the caller to ensure that the
     /// vector is actually the specified size.
     pub unsafe fn set_len(&mut self, new_len: usize) {
-        self.len = new_len
+        let (_, len_ptr, _) = self.triple_mut();
+        *len_ptr = new_len;
     }
 
     /// The maximum number of elements this vector can hold inline
@@ -399,41 +469,59 @@ impl<A: Array> SmallVec<A> {
     /// The number of elements stored in the vector
     #[inline]
     pub fn len(&self) -> usize {
-        self.len
+        self.triple().1
     }
 
     /// Returns `true` if the vector is empty
     #[inline]
     pub fn is_empty(&self) -> bool {
-        self.len == 0
+        self.len() == 0
     }
 
     /// The number of items the vector can hold without reallocating
     #[inline]
     pub fn capacity(&self) -> usize {
-        match self.data {
-            Inline { .. } => A::size(),
-            Heap { capacity, .. } => capacity,
+        self.triple().2
+    }
+
+    #[inline]
+    fn triple(&self) -> (*const A::Item, usize, usize) {
+        unsafe {
+            if self.spilled() {
+                let (ptr, len) = self.data.heap();
+                (ptr, len, self.capacity)
+            } else {
+                (self.data.inline().ptr(), self.capacity, A::size())
+            }
+        }
+    }
+
+    #[inline]
+    fn triple_mut(&mut self) -> (*mut A::Item, &mut usize, usize) {
+        unsafe {
+            if self.spilled() {
+                let &mut (ptr, ref mut len_ptr) = self.data.heap_mut();
+                (ptr, len_ptr, self.capacity)
+            } else {
+                (self.data.inline_mut().ptr_mut(), &mut self.capacity, A::size())
+            }
         }
     }
 
     /// Returns `true` if the data has spilled into a separate heap-allocated buffer.
     #[inline]
     pub fn spilled(&self) -> bool {
-        match self.data {
-            Inline { .. } => false,
-            Heap { .. } => true,
-        }
+        self.capacity > A::size()
     }
 
     /// Empty the vector and return an iterator over its former contents.
     pub fn drain(&mut self) -> Drain<A::Item> {
         unsafe {
+            let ptr = self.as_mut_ptr();
+
             let current_len = self.len();
             self.set_len(0);
 
-            let ptr = self.data.ptr_mut();
-
             let slice = slice::from_raw_parts_mut(ptr, current_len);
 
             Drain {
@@ -445,55 +533,57 @@ impl<A: Array> SmallVec<A> {
     /// Append an item to the vector.
     #[inline]
     pub fn push(&mut self, value: A::Item) {
-        let cap = self.capacity();
-        if self.len == cap {
-            self.grow(cmp::max(cap * 2, 1))
-        }
         unsafe {
-            let end = self.as_mut_ptr().offset(self.len as isize);
-            ptr::write(end, value);
-            let len = self.len;
-            self.set_len(len + 1)
+            let (_, &mut len, cap) = self.triple_mut();
+            if len == cap {
+                self.grow(cmp::max(cap * 2, 1))
+            }
+            let (ptr, len_ptr, _) = self.triple_mut();
+            *len_ptr = len + 1;
+            ptr::write(ptr.offset(len as isize), value);
         }
     }
 
     /// Remove an item from the end of the vector and return it, or None if empty.
     #[inline]
     pub fn pop(&mut self) -> Option<A::Item> {
-        if self.len == 0 {
-            return None
-        }
-        let last_index = self.len - 1;
-        if (last_index as isize) < 0 {
-            panic!("overflow")
-        }
         unsafe {
-            let end_ptr = self.as_ptr().offset(last_index as isize);
-            let value = ptr::read(end_ptr);
-            self.set_len(last_index);
-            Some(value)
+            let (ptr, len_ptr, _) = self.triple_mut();
+            if *len_ptr == 0 {
+                return None;
+            }
+            let last_index = *len_ptr - 1;
+            *len_ptr = last_index;
+            Some(ptr::read(ptr.offset(last_index as isize)))
         }
     }
 
-    /// Re-allocate to set the capacity to `new_cap`.
+    /// Re-allocate to set the capacity to `max(new_cap, inline_size())`.
     ///
     /// Panics if `new_cap` is less than the vector's length.
     pub fn grow(&mut self, new_cap: usize) {
-        assert!(new_cap >= self.len);
-        let mut vec: Vec<A::Item> = Vec::with_capacity(new_cap);
-        let new_alloc = vec.as_mut_ptr();
         unsafe {
-            mem::forget(vec);
-            ptr::copy_nonoverlapping(self.as_ptr(), new_alloc, self.len);
-
-            match self.data {
-                Inline { .. } => {}
-                Heap { ptr, capacity } => deallocate(ptr, capacity),
+            let (ptr, &mut len, cap) = self.triple_mut();
+            let spilled = self.spilled();
+            assert!(new_cap >= len);
+            if new_cap <= self.inline_size() {
+                if !spilled {
+                    return;
+                }
+                self.data = SmallVecData::from_inline(mem::uninitialized());
+                ptr::copy_nonoverlapping(ptr, self.data.inline_mut().ptr_mut(), len);
+                deallocate(ptr, cap);
+            } else if new_cap != cap {
+                let mut vec = Vec::with_capacity(new_cap);
+                let new_alloc = vec.as_mut_ptr();
+                mem::forget(vec);
+                ptr::copy_nonoverlapping(ptr, new_alloc, len);
+                self.data = SmallVecData::from_heap(new_alloc, len);
+                self.capacity = new_cap;
+                if spilled {
+                    deallocate(ptr, cap);
+                }
             }
-            ptr::write(&mut self.data, Heap {
-                ptr: new_alloc,
-                capacity: new_cap,
-            });
         }
     }
 
@@ -505,12 +595,15 @@ impl<A: Array> SmallVec<A> {
     /// instead. (This means that inserting `additional` new elements is not guaranteed to be
     /// possible after calling this function.)
     pub fn reserve(&mut self, additional: usize) {
-        let len = self.len();
-        if self.capacity() - len < additional {
-            match len.checked_add(additional).and_then(usize::checked_next_power_of_two) {
-                Some(cap) => self.grow(cap),
-                None => self.grow(usize::max_value()),
-            }
+        // prefer triple_mut() even if triple() would work
+        // so that the optimizer removes duplicated calls to it
+        // from callers like insert()
+        let (_, &mut len, cap) = self.triple_mut();
+        if cap - len < additional {
+            let new_cap = len.checked_add(additional).
+                and_then(usize::checked_next_power_of_two).
+                unwrap_or(usize::max_value());
+            self.grow(new_cap);
         }
     }
 
@@ -518,8 +611,8 @@ impl<A: Array> SmallVec<A> {
     ///
     /// Panics if the new capacity overflows `usize`.
     pub fn reserve_exact(&mut self, additional: usize) {
-        let len = self.len();
-        if self.capacity() - len < additional {
+        let (_, &mut len, cap) = self.triple_mut();
+        if cap - len < additional {
             match len.checked_add(additional) {
                 Some(cap) => self.grow(cap),
                 None => panic!("reserve_exact overflow"),
@@ -532,16 +625,17 @@ impl<A: Array> SmallVec<A> {
     /// When possible, this will move data from an external heap buffer to the vector's inline
     /// storage.
     pub fn shrink_to_fit(&mut self) {
-        let len = self.len;
+        if !self.spilled() {
+            return;
+        }
+        let len = self.len();
         if self.inline_size() >= len {
             unsafe {
-                let (ptr, capacity) = match self.data {
-                    Inline { .. } => return,
-                    Heap { ptr, capacity } => (ptr, capacity),
-                };
-                ptr::write(&mut self.data, Inline { array: mem::uninitialized() });
-                ptr::copy_nonoverlapping(ptr, self.as_mut_ptr(), len);
-                deallocate(ptr, capacity);
+                let (ptr, len) = self.data.heap();
+                self.data = SmallVecData::from_inline(mem::uninitialized());
+                ptr::copy_nonoverlapping(ptr, self.data.inline_mut().ptr_mut(), len);
+                deallocate(ptr, self.capacity);
+                self.capacity = len;
             }
         } else if self.capacity() > len {
             self.grow(len);
@@ -556,12 +650,12 @@ impl<A: Array> SmallVec<A> {
     /// This does not re-allocate.  If you want the vector's capacity to shrink, call
     /// `shrink_to_fit` after truncating.
     pub fn truncate(&mut self, len: usize) {
-        let end_ptr = self.as_ptr();
-        while len < self.len {
-            unsafe {
-                let last_index = self.len - 1;
-                self.set_len(last_index);
-                ptr::read(end_ptr.offset(last_index as isize));
+        unsafe {
+            let (ptr, len_ptr, _) = self.triple_mut();
+            while len < *len_ptr {
+                let last_index = *len_ptr - 1;
+                *len_ptr = last_index;
+                ptr::read(ptr.offset(last_index as isize));
             }
         }
     }
@@ -587,9 +681,9 @@ impl<A: Array> SmallVec<A> {
     /// Panics if `index` is out of bounds.
     #[inline]
     pub fn swap_remove(&mut self, index: usize) -> A::Item {
-        let len = self.len;
+        let len = self.len();
         self.swap(len - 1, index);
-        self.pop().unwrap()
+        unsafe { self.pop().unchecked_unwrap() }
     }
 
     /// Remove all elements from the vector.
@@ -603,15 +697,14 @@ impl<A: Array> SmallVec<A> {
     ///
     /// Panics if `index` is out of bounds.
     pub fn remove(&mut self, index: usize) -> A::Item {
-        let len = self.len();
-
-        assert!(index < len);
-
         unsafe {
-            let ptr = self.as_mut_ptr().offset(index as isize);
+            let (mut ptr, len_ptr, _) = self.triple_mut();
+            let len = *len_ptr;
+            assert!(index < len);
+            *len_ptr = len - 1;
+            ptr = ptr.offset(index as isize);
             let item = ptr::read(ptr);
             ptr::copy(ptr.offset(1), ptr, len - index - 1);
-            self.set_len(len - 1);
             item
         }
     }
@@ -622,14 +715,14 @@ impl<A: Array> SmallVec<A> {
     pub fn insert(&mut self, index: usize, element: A::Item) {
         self.reserve(1);
 
-        let len = self.len;
-        assert!(index <= len);
-
         unsafe {
-            let ptr = self.as_mut_ptr().offset(index as isize);
+            let (mut ptr, len_ptr, _) = self.triple_mut();
+            let len = *len_ptr;
+            assert!(index <= len);
+            *len_ptr = len + 1;
+            ptr = ptr.offset(index as isize);
             ptr::copy(ptr, ptr.offset(1), len - index);
             ptr::write(ptr, element);
-            self.set_len(len + 1);
         }
     }
 
@@ -647,21 +740,22 @@ impl<A: Array> SmallVec<A> {
         self.reserve(lower_size_bound);
 
         unsafe {
-            let old_len = self.len;
+            let old_len = self.len();
             assert!(index <= old_len);
             let ptr = self.as_mut_ptr().offset(index as isize);
             ptr::copy(ptr, ptr.offset(lower_size_bound as isize), old_len - index);
             for (off, element) in iter.enumerate() {
                 if off < lower_size_bound {
                     ptr::write(ptr.offset(off as isize), element);
-                    self.len = self.len + 1;
+                    let len = self.len() + 1;
+                    self.set_len(len);
                 } else {
                     // Iterator provided more elements than the hint.
                     assert!(index + off >= index);  // Protect against overflow.
                     self.insert(index + off, element);
                 }
             }
-            let num_added = self.len - old_len;
+            let num_added = self.len() - old_len;
             if num_added < lower_size_bound {
                 // Iterator provided fewer elements than the hint
                 ptr::copy(ptr.offset(lower_size_bound as isize), ptr.offset(num_added as isize), old_len - index);
@@ -672,13 +766,15 @@ impl<A: Array> SmallVec<A> {
     /// Convert a SmallVec to a Vec, without reallocating if the SmallVec has already spilled onto
     /// the heap.
     pub fn into_vec(self) -> Vec<A::Item> {
-        match self.data {
-            Inline { .. } => self.into_iter().collect(),
-            Heap { ptr, capacity } => unsafe {
-                let v = Vec::from_raw_parts(ptr, self.len, capacity);
+        if self.spilled() {
+            unsafe {
+                let (ptr, len) = self.data.heap();
+                let v = Vec::from_raw_parts(ptr, len, self.capacity);
                 mem::forget(self);
                 v
             }
+        } else {
+            self.into_iter().collect()
         }
     }
 
@@ -689,7 +785,7 @@ impl<A: Array> SmallVec<A> {
     /// elements.
     pub fn retain<F: FnMut(&mut A::Item) -> bool>(&mut self, mut f: F) {
         let mut del = 0;
-        let len = self.len;
+        let len = self.len();
         for i in 0..len {
             if !f(&mut self[i]) {
                 del += 1;
@@ -706,12 +802,12 @@ impl<A: Array> SmallVec<A> {
     }
 
     /// Removes consecutive duplicate elements using the given equality relation.
-    pub fn dedup_by<F>(&mut self, mut same_bucket: F) 
+    pub fn dedup_by<F>(&mut self, mut same_bucket: F)
         where F: FnMut(&mut A::Item, &mut A::Item) -> bool
     {
         // See the implementation of Vec::dedup_by in the
         // standard library for an explanation of this algorithm.
-        let len = self.len;
+        let len = self.len();
         if len <= 1 {
             return;
         }
@@ -737,9 +833,9 @@ impl<A: Array> SmallVec<A> {
     }
 
     /// Removes consecutive elements that map to the same key.
-    pub fn dedup_by_key<F, K>(&mut self, mut key: F) 
-        where F: FnMut(&mut A::Item) -> K, 
-              K: PartialEq<K> 
+    pub fn dedup_by_key<F, K>(&mut self, mut key: F)
+        where F: FnMut(&mut A::Item) -> K,
+              K: PartialEq<K>
     {
         self.dedup_by(|a, b| key(a) == key(b));
     }
@@ -762,7 +858,7 @@ impl<A: Array> SmallVec<A> where A::Item: Copy {
     pub fn insert_from_slice(&mut self, index: usize, slice: &[A::Item]) {
         self.reserve(slice.len());
 
-        let len = self.len;
+        let len = self.len();
         assert!(index <= len);
 
         unsafe {
@@ -804,7 +900,7 @@ impl<A: Array> SmallVec<A> where A::Item: Clone {
     /// Creates a `SmallVec` with `n` copies of `elem`.
     /// ```
     /// use smallvec::SmallVec;
-    /// 
+    ///
     /// let v = SmallVec::<[char; 128]>::from_elem('d', 2);
     /// assert_eq!(v, SmallVec::from_buf(['d', 'd']));
     /// ```
@@ -821,8 +917,8 @@ impl<A: Array> SmallVec<A> where A::Item: Clone {
                 }
 
                 SmallVec {
-                    data: Inline { array: arr },
-                    len: n,
+                    capacity: n,
+                    data: SmallVecData::from_inline(arr),
                 }
             }
         }
@@ -833,12 +929,9 @@ impl<A: Array> ops::Deref for SmallVec<A> {
     type Target = [A::Item];
     #[inline]
     fn deref(&self) -> &[A::Item] {
-        let ptr: *const _ = match self.data {
-            Inline { ref array } => array.ptr(),
-            Heap { ptr, .. } => ptr,
-        };
         unsafe {
-            slice::from_raw_parts(ptr, self.len)
+            let (ptr, len, _) = self.triple();
+            slice::from_raw_parts(ptr, len)
         }
     }
 }
@@ -846,9 +939,9 @@ impl<A: Array> ops::Deref for SmallVec<A> {
 impl<A: Array> ops::DerefMut for SmallVec<A> {
     #[inline]
     fn deref_mut(&mut self) -> &mut [A::Item] {
-        let ptr = self.data.ptr_mut();
         unsafe {
-            slice::from_raw_parts_mut(ptr, self.len)
+            let (ptr, &mut len, _) = self.triple_mut();
+            slice::from_raw_parts_mut(ptr, len)
         }
     }
 }
@@ -1021,27 +1114,20 @@ impl<A: Array> Extend<A::Item> for SmallVec<A> {
     fn extend<I: IntoIterator<Item=A::Item>>(&mut self, iterable: I) {
         let mut iter = iterable.into_iter();
         let (lower_size_bound, _) = iter.size_hint();
-
-        let target_len = self.len + lower_size_bound;
-
-        if target_len > self.capacity() {
-            self.grow(target_len);
-        }
+        self.reserve(lower_size_bound);
 
         unsafe {
-            let ptr = self.as_mut_ptr().offset(self.len() as isize);
-
             let len = self.len();
+            let ptr = self.as_mut_ptr().offset(len as isize);
             let mut count = 0;
-            for p in 0..lower_size_bound as isize {
+            while count < lower_size_bound {
                 if let Some(out) = iter.next() {
-                    ::std::ptr::write(ptr.offset(p), out);
+                    ptr::write(ptr.offset(count as isize), out);
                     count += 1;
                 } else {
                     break;
                 }
             }
-
             self.set_len(len + count);
         }
 
@@ -1066,12 +1152,15 @@ impl<A: Array> Default for SmallVec<A> {
 
 impl<A: Array> Drop for SmallVec<A> {
     fn drop(&mut self) {
-        // Note on panic safety: dropping an element may panic,
-        // but the inner SmallVecData destructor will still run
         unsafe {
-            let ptr = self.as_ptr();
-            for i in 0 .. self.len {
-                ptr::read(ptr.offset(i as isize));
+            if self.spilled() {
+                let (ptr, len) = self.data.heap();
+                Vec::from_raw_parts(ptr, len, self.capacity);
+            } else {
+                let ptr = self.as_ptr();
+                for i in 0..self.len() {
+                    ptr::read(ptr.offset(i as isize));
+                }
             }
         }
     }
@@ -1125,7 +1214,7 @@ unsafe impl<A: Array> Send for SmallVec<A> where A::Item: Send {}
 ///
 /// [1]: struct.SmallVec.html#method.into_iter
 pub struct IntoIter<A: Array> {
-    data: SmallVecData<A>,
+    data: SmallVec<A>,
     current: usize,
     end: usize,
 }
@@ -1148,7 +1237,7 @@ impl<A: Array> Iterator for IntoIter<A> {
             unsafe {
                 let current = self.current as isize;
                 self.current += 1;
-                Some(ptr::read(self.data.ptr_mut().offset(current)))
+                Some(ptr::read(self.data.as_ptr().offset(current)))
             }
         }
     }
@@ -1169,7 +1258,7 @@ impl<A: Array> DoubleEndedIterator for IntoIter<A> {
         else {
             unsafe {
                 self.end -= 1;
-                Some(ptr::read(self.data.ptr_mut().offset(self.end as isize)))
+                Some(ptr::read(self.data.as_ptr().offset(self.end as isize)))
             }
         }
     }
@@ -1181,13 +1270,12 @@ impl<A: Array> IntoIterator for SmallVec<A> {
     type IntoIter = IntoIter<A>;
     type Item = A::Item;
     fn into_iter(mut self) -> Self::IntoIter {
-        let len = self.len();
         unsafe {
-            // Only grab the `data` field, the `IntoIter` type handles dropping of the elements
-            let data = ptr::read(&mut self.data);
-            mem::forget(self);
+            // Set SmallVec len to zero as `IntoIter` drop handles dropping of the elements
+            let len = self.len();
+            self.set_len(0);
             IntoIter {
-                data: data,
+                data: self,
                 current: 0,
                 end: len,
             }