Merge branch 'memory'

fuzz/Cargo.toml

@@ -10,6 +10,8 @@ cargo-fuzz = true
 
 [dependencies.encoding_rs]
 path = ".."
+[dependencies.safe_encoding_rs_mem]
+path = "../../safe_encoding_rs_mem"
 [dependencies.libfuzzer-sys]
 git = "https://github.com/rust-fuzz/libfuzzer-sys.git"
 
@@ -27,3 +29,7 @@ path = "fuzzers/fuzz_encodings.rs"
 [[bin]]
 name = "fuzz_labels"
 path = "fuzzers/fuzz_labels.rs"
+
+[[bin]]
+name = "fuzz_mem"
+path = "fuzzers/fuzz_mem.rs"

fuzz/fuzzers/fuzz_mem.rs

@@ -0,0 +1,342 @@
+// Copyright 2015-2016 Mozilla Foundation. See the COPYRIGHT
+// file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![no_main]
+#[macro_use]
+extern crate libfuzzer_sys;
+extern crate encoding_rs;
+extern crate safe_encoding_rs_mem;
+
+fn check_utf8(data: &[u8]) {
+    if let Err(_) = ::std::str::from_utf8(data) {
+        panic!("Bogus UTF-8.");
+    }
+}
+
+fn check_utf16(data: &[u16]) {
+    let mut prev_was_high_surrogate = false;
+    for unit in data {
+        if *unit >= 0xD800 && *unit <= 0xDBFF {
+            assert!(!prev_was_high_surrogate);
+            prev_was_high_surrogate = true;
+        } else if *unit >= 0xDC00 && *unit <= 0xDFFF {
+            assert!(prev_was_high_surrogate);
+            prev_was_high_surrogate = false;
+        } else {
+            assert!(!prev_was_high_surrogate);
+            prev_was_high_surrogate = false;
+        }
+    }
+    assert!(!prev_was_high_surrogate);
+}
+
+fn as_u16_slice(data: &[u8]) -> &[u16] {
+    unsafe {
+        let ptr = data.as_ptr();
+        let len = data.len();
+        if len < 2 {
+            return ::std::slice::from_raw_parts(ptr as *const u16, 0);
+        }
+        let (adj_ptr, adj_len) = if ptr as usize & 1 == 0 {
+            (ptr, len / 2)
+        } else {
+            (ptr.offset(1), (len - 1) / 2)
+        };
+        ::std::slice::from_raw_parts(adj_ptr as *const u16, adj_len)
+    }
+}
+
+trait EigthOrSixteen: Clone {
+    fn zero() -> Self;
+}
+
+impl EigthOrSixteen for u8 {
+    fn zero() -> u8 {
+        0
+    }
+}
+
+impl EigthOrSixteen for u16 {
+    fn zero() -> u16 {
+        0
+    }
+}
+
+fn vec_with_len<T: EigthOrSixteen>(len: usize) -> Vec<T> {
+    let mut vec: Vec<T> = Vec::with_capacity(len);
+    vec.resize(len, T::zero());
+    vec
+}
+
+fn string_with_len(len: usize) -> String {
+    let mut s = String::with_capacity(len);
+    for _ in 0..len {
+        s.push('\u{0}');
+    }
+    s
+}
+
+fn fuzz_is_ascii(data: &[u8]) {
+    assert_eq!(encoding_rs::mem::is_ascii(data), safe_encoding_rs_mem::is_ascii(data));
+}
+
+fn fuzz_is_basic_latin(data: &[u16]) {
+    assert_eq!(encoding_rs::mem::is_basic_latin(data), safe_encoding_rs_mem::is_basic_latin(data));
+}
+
+fn fuzz_is_utf8_latin1(data: &[u8]) {
+    assert_eq!(encoding_rs::mem::is_utf8_latin1(data), safe_encoding_rs_mem::is_utf8_latin1(data));
+}
+
+fn fuzz_is_str_latin1(data: &[u8]) {
+    if let Ok(s) = std::str::from_utf8(data) {
+        assert_eq!(encoding_rs::mem::is_str_latin1(s), safe_encoding_rs_mem::is_str_latin1(s));
+    }
+}
+
+fn fuzz_is_utf16_latin1(data: &[u16]) {
+    assert_eq!(encoding_rs::mem::is_utf16_latin1(data), safe_encoding_rs_mem::is_utf16_latin1(data));
+}
+
+fn fuzz_convert_utf8_to_utf16(data: &[u8]) {
+    let needed = data.len() + 1;
+    let mut dst = vec_with_len::<u16>(needed);
+    let mut safe_dst = vec_with_len::<u16>(needed);
+    let len = encoding_rs::mem::convert_utf8_to_utf16(data, &mut dst[..]);
+    let safe_len = safe_encoding_rs_mem::convert_utf8_to_utf16(data, &mut safe_dst[..]);
+    dst.truncate(len);
+    safe_dst.truncate(safe_len);
+    assert_eq!(len, safe_len);
+    assert_eq!(dst, safe_dst);
+    check_utf16(&dst[..]);
+}
+
+fn fuzz_convert_str_to_utf16(data: &[u8]) {
+    if let Ok(s) = std::str::from_utf8(data) {
+        let needed = s.len();
+        let mut dst = vec_with_len::<u16>(needed);
+        let mut safe_dst = vec_with_len::<u16>(needed);
+        let len = encoding_rs::mem::convert_str_to_utf16(s, &mut dst[..]);
+        let safe_len = safe_encoding_rs_mem::convert_str_to_utf16(s, &mut safe_dst[..]);
+        dst.truncate(len);
+        safe_dst.truncate(safe_len);
+        assert_eq!(len, safe_len);
+        assert_eq!(dst, safe_dst);
+        check_utf16(&dst[..]);
+    }
+}
+
+fn fuzz_convert_utf16_to_utf8(data: &[u16]) {
+    let needed = data.len() * 3 + 1;
+    let mut dst = vec_with_len::<u8>(needed);
+    let mut safe_dst = vec_with_len::<u8>(needed);
+    let len = encoding_rs::mem::convert_utf16_to_utf8(data, &mut dst[..]);
+    let safe_len = safe_encoding_rs_mem::convert_utf16_to_utf8(data, &mut safe_dst[..]);
+    dst.truncate(len);
+    safe_dst.truncate(safe_len);
+    assert_eq!(len, safe_len);
+    assert_eq!(dst, safe_dst);
+    check_utf8(&dst[..]);
+}
+
+fn fuzz_convert_utf16_to_str(data: &[u16]) {
+    let needed = data.len() * 3 + 1;
+    let mut dst = string_with_len(needed);
+    let mut safe_dst = string_with_len(needed);
+    let len = encoding_rs::mem::convert_utf16_to_str(data, &mut dst[..]);
+    let safe_len = safe_encoding_rs_mem::convert_utf16_to_str(data, &mut safe_dst[..]);
+    check_utf8(dst.as_bytes());
+    check_utf8(safe_dst.as_bytes());
+    dst.truncate(len);
+    safe_dst.truncate(safe_len);
+    assert_eq!(len, safe_len);
+    assert_eq!(dst, safe_dst);
+}
+
+fn fuzz_convert_latin1_to_utf16(data: &[u8]) {
+    let needed = data.len();
+    let mut dst = vec_with_len::<u16>(needed);
+    let mut safe_dst = vec_with_len::<u16>(needed);
+    encoding_rs::mem::convert_latin1_to_utf16(data, &mut dst[..]);
+    safe_encoding_rs_mem::convert_latin1_to_utf16(data, &mut safe_dst[..]);
+    assert_eq!(dst, safe_dst);
+    check_utf16(&dst[..]);
+}
+
+fn fuzz_convert_latin1_to_utf8(data: &[u8]) {
+    let needed = data.len() * 2;
+    let mut dst = vec_with_len::<u8>(needed);
+    let mut safe_dst = vec_with_len::<u8>(needed);
+    let len = encoding_rs::mem::convert_latin1_to_utf8(data, &mut dst[..]);
+    let safe_len = safe_encoding_rs_mem::convert_latin1_to_utf8(data, &mut safe_dst[..]);
+    dst.truncate(len);
+    safe_dst.truncate(safe_len);
+    assert_eq!(len, safe_len);
+    assert_eq!(dst, safe_dst);
+    check_utf8(&dst[..]);
+}
+
+fn fuzz_convert_latin1_to_str(data: &[u8]) {
+    let needed = data.len() * 2;
+    let mut dst = string_with_len(needed);
+    let mut safe_dst = string_with_len(needed);
+    let len = encoding_rs::mem::convert_latin1_to_str(data, &mut dst[..]);
+    let safe_len = safe_encoding_rs_mem::convert_latin1_to_str(data, &mut safe_dst[..]);
+    check_utf8(dst.as_bytes());
+    check_utf8(safe_dst.as_bytes());
+    dst.truncate(len);
+    safe_dst.truncate(safe_len);
+    assert_eq!(len, safe_len);
+    assert_eq!(dst, safe_dst);
+}
+
+fn fuzz_convert_utf8_to_latin1_lossy(data: &[u8]) {
+    let needed = data.len();
+    let mut dst = vec_with_len::<u8>(needed);
+    let mut safe_dst = vec_with_len::<u8>(needed);
+    let len = encoding_rs::mem::convert_utf8_to_latin1_lossy(data, &mut dst[..]);
+    let safe_len = safe_encoding_rs_mem::convert_utf8_to_latin1_lossy(data, &mut safe_dst[..]);
+    if safe_encoding_rs_mem::is_utf8_latin1(data) {
+        dst.truncate(len);
+        safe_dst.truncate(safe_len);
+        assert_eq!(len, safe_len);
+        assert_eq!(dst, safe_dst);
+    }
+}
+
+fn fuzz_convert_utf16_to_latin1_lossy(data: &[u16]) {
+    let needed = data.len();
+    let mut dst = vec_with_len::<u8>(needed);
+    let mut safe_dst = vec_with_len::<u8>(needed);
+    encoding_rs::mem::convert_utf16_to_latin1_lossy(data, &mut dst[..]);
+    safe_encoding_rs_mem::convert_utf16_to_latin1_lossy(data, &mut safe_dst[..]);
+    if safe_encoding_rs_mem::is_utf16_latin1(data) {
+        assert_eq!(dst, safe_dst);
+    }
+}
+
+fn fuzz_utf16_valid_up_to(data: &[u16]) {
+    let up_to = encoding_rs::mem::utf16_valid_up_to(data);
+    let safe_up_to = safe_encoding_rs_mem::utf16_valid_up_to(data);
+    assert_eq!(up_to, safe_up_to);
+}
+
+fn fuzz_ensure_utf16_validity(data: &[u16]) {
+    let needed = data.len();
+    let mut dst = vec_with_len::<u16>(needed);
+    let mut safe_dst = vec_with_len::<u16>(needed);
+    dst.copy_from_slice(data);
+    safe_dst.copy_from_slice(data);
+    encoding_rs::mem::ensure_utf16_validity(&mut dst[..]);
+    safe_encoding_rs_mem::ensure_utf16_validity(&mut safe_dst[..]);
+    assert_eq!(dst, safe_dst);
+    check_utf16(&dst[..]);
+}
+
+fn fuzz_copy_ascii_to_ascii(data: &[u8]) {
+    let needed = data.len();
+    let mut dst = vec_with_len::<u8>(needed);
+    let mut safe_dst = vec_with_len::<u8>(needed);
+    let len = encoding_rs::mem::copy_ascii_to_ascii(data, &mut dst[..]);
+    let safe_len = safe_encoding_rs_mem::copy_ascii_to_ascii(data, &mut safe_dst[..]);
+    dst.truncate(len);
+    safe_dst.truncate(safe_len);
+    assert_eq!(len, safe_len);
+    assert_eq!(dst, safe_dst);
+}
+
+fn fuzz_copy_ascii_to_basic_latin(data: &[u8]) {
+    let needed = data.len();
+    let mut dst = vec_with_len::<u16>(needed);
+    let mut safe_dst = vec_with_len::<u16>(needed);
+    let len = encoding_rs::mem::copy_ascii_to_basic_latin(data, &mut dst[..]);
+    let safe_len = safe_encoding_rs_mem::copy_ascii_to_basic_latin(data, &mut safe_dst[..]);
+    dst.truncate(len);
+    safe_dst.truncate(safe_len);
+    assert_eq!(len, safe_len);
+    assert_eq!(dst, safe_dst);
+}
+
+fn fuzz_copy_basic_latin_to_ascii(data: &[u16]) {
+    let needed = data.len();
+    let mut dst = vec_with_len::<u8>(needed);
+    let mut safe_dst = vec_with_len::<u8>(needed);
+    let len = encoding_rs::mem::copy_basic_latin_to_ascii(data, &mut dst[..]);
+    let safe_len = safe_encoding_rs_mem::copy_basic_latin_to_ascii(data, &mut safe_dst[..]);
+    dst.truncate(len);
+    safe_dst.truncate(safe_len);
+    assert_eq!(len, safe_len);
+    assert_eq!(dst, safe_dst);
+}
+
+fn fuzz_is_utf8_bidi(data: &[u8]) {
+    assert_eq!(encoding_rs::mem::is_utf8_bidi(data), safe_encoding_rs_mem::is_utf8_bidi(data));
+}
+
+fn fuzz_is_str_bidi(data: &[u8]) {
+    if let Ok(s) = std::str::from_utf8(data) {
+        assert_eq!(encoding_rs::mem::is_str_bidi(s), safe_encoding_rs_mem::is_str_bidi(s));
+    }
+}
+
+fn fuzz_is_utf16_bidi(data: &[u16]) {
+    assert_eq!(encoding_rs::mem::is_utf16_bidi(data), safe_encoding_rs_mem::is_utf16_bidi(data));
+}
+
+// is_char_bidi() and is_utf16_code_unit_bidi() are tested exhaustively, so no need to fuzz them.as_u16_slice
+
+fn fuzz_check_utf8_for_latin1_and_bidi(data: &[u8]) {
+    assert_eq!(encoding_rs::mem::check_utf8_for_latin1_and_bidi(data), safe_encoding_rs_mem::check_utf8_for_latin1_and_bidi(data));
+}
+
+fn fuzz_check_str_for_latin1_and_bidi(data: &[u8]) {
+    if let Ok(s) = std::str::from_utf8(data) {
+        assert_eq!(encoding_rs::mem::check_str_for_latin1_and_bidi(s), safe_encoding_rs_mem::check_str_for_latin1_and_bidi(s));
+    }
+}
+
+fn fuzz_check_utf16_for_latin1_and_bidi(data: &[u16]) {
+    assert_eq!(encoding_rs::mem::check_utf16_for_latin1_and_bidi(data), safe_encoding_rs_mem::check_utf16_for_latin1_and_bidi(data));
+}
+
+fuzz_target!(
+    |data: &[u8]| {
+        if let Some(first) = data.first() {
+            match *first {
+                0 => fuzz_is_ascii(&data[1..]),
+                1 => fuzz_is_basic_latin(as_u16_slice(&data[1..])),
+                2 => fuzz_is_utf8_latin1(&data[1..]),
+                3 => fuzz_is_str_latin1(&data[1..]),
+                4 => fuzz_is_utf16_latin1(as_u16_slice(&data[1..])),
+                5 => fuzz_convert_utf8_to_utf16(&data[1..]),
+                6 => fuzz_convert_str_to_utf16(&data[1..]),
+                7 => fuzz_convert_utf16_to_utf8(as_u16_slice(&data[1..])),
+                8 => fuzz_convert_utf16_to_str(as_u16_slice(&data[1..])),
+                9 => fuzz_convert_latin1_to_utf16(&data[1..]),
+                10 => fuzz_convert_latin1_to_utf8(&data[1..]),
+                11 => fuzz_convert_latin1_to_str(&data[1..]),
+                12 => fuzz_convert_utf8_to_latin1_lossy(&data[1..]),
+                13 => fuzz_convert_utf16_to_latin1_lossy(as_u16_slice(&data[1..])),
+                14 => fuzz_utf16_valid_up_to(as_u16_slice(&data[1..])),
+                15 => fuzz_ensure_utf16_validity(as_u16_slice(&data[1..])),
+                16 => fuzz_copy_ascii_to_ascii(&data[1..]),
+                17 => fuzz_copy_ascii_to_basic_latin(&data[1..]),
+                18 => fuzz_copy_basic_latin_to_ascii(as_u16_slice(&data[1..])),
+                19 => fuzz_is_utf8_bidi(&data[1..]),
+                20 => fuzz_is_str_bidi(&data[1..]),
+                21 => fuzz_is_utf16_bidi(as_u16_slice(&data[1..])),
+                22 => fuzz_check_utf8_for_latin1_and_bidi(&data[1..]),
+                23 => fuzz_check_str_for_latin1_and_bidi(&data[1..]),
+                24 => fuzz_check_utf16_for_latin1_and_bidi(as_u16_slice(&data[1..])),
+                _ => return,
+            }
+        }
+        // Comment to make rustfmt not introduce a compilation error
+    }
+);

src/ascii.rs

@@ -24,6 +24,14 @@
 #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))]
 use simd_funcs::*;
 
+// `as` truncates, so works on 32-bit, too.
+#[allow(dead_code)]
+pub const ASCII_MASK: usize = 0x80808080_80808080u64 as usize;
+
+// `as` truncates, so works on 32-bit, too.
+#[allow(dead_code)]
+pub const BASIC_LATIN_MASK: usize = 0xFF80FF80_FF80FF80u64 as usize;
+
 #[allow(unused_macros)]
 macro_rules! ascii_naive {
     ($name:ident,
@@ -212,6 +220,62 @@ macro_rules! basic_latin_alu {
     });
 }
 
+#[allow(unused_macros)]
+macro_rules! latin1_alu {
+    ($name:ident,
+     $src_unit:ty,
+     $dst_unit:ty,
+     $stride_fn:ident) => (
+    #[cfg_attr(feature = "cargo-clippy", allow(never_loop))]
+    #[inline(always)]
+    pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) {
+        let mut offset = 0usize;
+        // This loop is only broken out of as a `goto` forward
+        loop {
+            let mut until_alignment = {
+                if ::std::mem::size_of::<$src_unit>() < ::std::mem::size_of::<$dst_unit>() {
+                    // unpack
+                    let src_until_alignment = (ALIGNMENT - ((src as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
+                    if (dst.offset(src_until_alignment as isize) as usize) & ALIGNMENT_MASK != 0 {
+                        break;
+                    }
+                    src_until_alignment
+                } else {
+                    // pack
+                    let dst_until_alignment = (ALIGNMENT - ((dst as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
+                    if (src.offset(dst_until_alignment as isize) as usize) & ALIGNMENT_MASK != 0 {
+                        break;
+                    }
+                    dst_until_alignment
+                }
+            };
+            if until_alignment + STRIDE_SIZE <= len {
+                while until_alignment != 0 {
+                    let code_unit = *(src.offset(offset as isize));
+                    *(dst.offset(offset as isize)) = code_unit as $dst_unit;
+                    offset += 1;
+                    until_alignment -= 1;
+                }
+                let len_minus_stride = len - STRIDE_SIZE;
+                loop {
+                    $stride_fn(src.offset(offset as isize) as *const usize,
+                               dst.offset(offset as isize) as *mut usize);
+                    offset += STRIDE_SIZE;
+                    if offset > len_minus_stride {
+                        break;
+                    }
+                }
+            }
+            break;
+        }
+        while offset < len {
+            let code_unit = *(src.offset(offset as isize));
+            *(dst.offset(offset as isize)) = code_unit as $dst_unit;
+            offset += 1;
+        }
+    });
+}
+
 #[allow(unused_macros)]
 macro_rules! ascii_simd_check_align {
     ($name:ident,
@@ -294,6 +358,89 @@ macro_rules! ascii_simd_check_align {
     });
 }
 
+#[allow(unused_macros)]
+macro_rules! latin1_simd_check_align {
+    ($name:ident,
+     $src_unit:ty,
+     $dst_unit:ty,
+     $stride_both_aligned:ident,
+     $stride_src_aligned:ident,
+     $stride_dst_aligned:ident,
+     $stride_neither_aligned:ident) => (
+    #[inline(always)]
+    pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) {
+        let mut offset = 0usize;
+        if STRIDE_SIZE <= len {
+            let len_minus_stride = len - STRIDE_SIZE;
+            // XXX Should we first process one stride unconditinoally as unaligned to
+            // avoid the cost of the branchiness below if the first stride fails anyway?
+            // XXX Should we just use unaligned SSE2 access unconditionally? It seems that
+            // on Haswell, it would make sense to just use unaligned and not bother
+            // checking. Need to benchmark older architectures before deciding.
+            let dst_masked = (dst as usize) & ALIGNMENT_MASK;
+            if ((src as usize) & ALIGNMENT_MASK) == 0 {
+                if dst_masked == 0 {
+                    loop {
+                        $stride_both_aligned(src.offset(offset as isize),
+                                             dst.offset(offset as isize));
+                        offset += STRIDE_SIZE;
+                        if offset > len_minus_stride {
+                            break;
+                        }
+                    }
+                } else {
+                    loop {
+                        $stride_src_aligned(src.offset(offset as isize),
+                                            dst.offset(offset as isize));
+                        offset += STRIDE_SIZE;
+                        if offset > len_minus_stride {
+                            break;
+                        }
+                    }
+                }
+            } else {
+                if dst_masked == 0 {
+                    loop {
+                        $stride_dst_aligned(src.offset(offset as isize),
+                                            dst.offset(offset as isize));
+                        offset += STRIDE_SIZE;
+                        if offset > len_minus_stride {
+                            break;
+                        }
+                    }
+                } else {
+                    loop {
+                        $stride_neither_aligned(src.offset(offset as isize),
+                                                dst.offset(offset as isize));
+                        offset += STRIDE_SIZE;
+                        if offset > len_minus_stride {
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        while offset < len {
+            let code_unit = *(src.offset(offset as isize));
+            // On x86_64, this loop autovectorizes but in the pack
+            // case there are instructions whose purpose is to make sure
+            // each u16 in the vector is truncated before packing. However,
+            // since we don't care about saturating behavior of SSE2 packing
+            // when the input isn't Latin1, those instructions are useless.
+            // Unfortunately, using the `assume` intrinsic to lie to the
+            // optimizer doesn't make LLVM omit the trunctation that we
+            // don't need. Possibly this loop could be manually optimized
+            // to do the sort of thing that LLVM does but without the
+            // ANDing the read vectors of u16 with a constant that discards
+            // the high half of each u16. As far as I can tell, the
+            // optimization assumes that doing a SIMD read past the end of
+            // the array is OK.
+            *(dst.offset(offset as isize)) = code_unit as $dst_unit;
+            offset += 1;
+        }
+    });
+}
+
 #[allow(unused_macros)]
 macro_rules! ascii_simd_unalign {
     ($name:ident,
@@ -328,6 +475,34 @@ macro_rules! ascii_simd_unalign {
     });
 }
 
+#[allow(unused_macros)]
+macro_rules! latin1_simd_unalign {
+    ($name:ident,
+     $src_unit:ty,
+     $dst_unit:ty,
+     $stride_neither_aligned:ident) => (
+    #[inline(always)]
+    pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) {
+        let mut offset = 0usize;
+        if STRIDE_SIZE <= len {
+            let len_minus_stride = len - STRIDE_SIZE;
+            loop {
+                $stride_neither_aligned(src.offset(offset as isize),
+                                        dst.offset(offset as isize));
+                offset += STRIDE_SIZE;
+                if offset > len_minus_stride {
+                    break;
+                }
+            }
+        }
+        while offset < len {
+            let code_unit = *(src.offset(offset as isize));
+            *(dst.offset(offset as isize)) = code_unit as $dst_unit;
+            offset += 1;
+        }
+    });
+}
+
 #[allow(unused_macros)]
 macro_rules! ascii_to_ascii_simd_stride {
     ($name:ident,
@@ -336,7 +511,7 @@ macro_rules! ascii_to_ascii_simd_stride {
     #[inline(always)]
     pub unsafe fn $name(src: *const u8, dst: *mut u8) -> bool {
         let simd = $load(src);
-        if !is_ascii(simd) {
+        if !simd_is_ascii(simd) {
             return false;
         }
         $store(dst, simd);
@@ -352,7 +527,7 @@ macro_rules! ascii_to_basic_latin_simd_stride {
     #[inline(always)]
     pub unsafe fn $name(src: *const u8, dst: *mut u16) -> bool {
         let simd = $load(src);
-        if !is_ascii(simd) {
+        if !simd_is_ascii(simd) {
             return false;
         }
         let (first, second) = simd_unpack(simd);
@@ -362,6 +537,20 @@ macro_rules! ascii_to_basic_latin_simd_stride {
     });
 }
 
+#[allow(unused_macros)]
+macro_rules! unpack_simd_stride {
+    ($name:ident,
+     $load:ident,
+     $store:ident) => (
+    #[inline(always)]
+    pub unsafe fn $name(src: *const u8, dst: *mut u16) {
+        let simd = $load(src);
+        let (first, second) = simd_unpack(simd);
+        $store(dst, first);
+        $store(dst.offset(8), second);
+    });
+}
+
 #[allow(unused_macros)]
 macro_rules! basic_latin_to_ascii_simd_stride {
     ($name:ident,
@@ -371,7 +560,7 @@ macro_rules! basic_latin_to_ascii_simd_stride {
     pub unsafe fn $name(src: *const u16, dst: *mut u8) -> bool {
         let first = $load(src);
         let second = $load(src.offset(8));
-        if is_basic_latin(first | second) {
+        if simd_is_basic_latin(first | second) {
             $store(dst, simd_pack(first, second));
             true
         } else {
@@ -380,23 +569,40 @@ macro_rules! basic_latin_to_ascii_simd_stride {
     });
 }
 
+#[allow(unused_macros)]
+macro_rules! pack_simd_stride {
+    ($name:ident,
+     $load:ident,
+     $store:ident) => (
+    #[inline(always)]
+    pub unsafe fn $name(src: *const u16, dst: *mut u8) {
+        let first = $load(src);
+        let second = $load(src.offset(8));
+        $store(dst, simd_pack(first, second));
+    });
+}
+
 cfg_if! {
     if #[cfg(all(feature = "simd-accel", target_endian = "little", target_arch = "aarch64"))] {
         // SIMD with the same instructions for aligned and unaligned loads and stores
 
         pub const STRIDE_SIZE: usize = 16;
 
-        const ALIGNMENT: usize = 8;
+//        pub const ALIGNMENT: usize = 8;
 
         ascii_to_ascii_simd_stride!(ascii_to_ascii_stride_neither_aligned, load16_unaligned, store16_unaligned);
 
         ascii_to_basic_latin_simd_stride!(ascii_to_basic_latin_stride_neither_aligned, load16_unaligned, store8_unaligned);
+        unpack_simd_stride!(unpack_stride_neither_aligned, load16_unaligned, store8_unaligned);
 
         basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_neither_aligned, load8_unaligned, store16_unaligned);
+        pack_simd_stride!(pack_stride_neither_aligned, load8_unaligned, store16_unaligned);
 
         ascii_simd_unalign!(ascii_to_ascii, u8, u8, ascii_to_ascii_stride_neither_aligned);
         ascii_simd_unalign!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_neither_aligned);
         ascii_simd_unalign!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_neither_aligned);
+        latin1_simd_unalign!(unpack_latin1, u8, u16, unpack_stride_neither_aligned);
+        latin1_simd_unalign!(pack_latin1, u16, u8, pack_stride_neither_aligned);
     } else if #[cfg(all(feature = "simd-accel", target_feature = "sse2"))] {
         // SIMD with different instructions for aligned and unaligned loads and stores.
         //
@@ -406,7 +612,7 @@ cfg_if! {
 
         pub const STRIDE_SIZE: usize = 16;
 
-        const ALIGNMENT_MASK: usize = 15;
+        pub const ALIGNMENT_MASK: usize = 15;
 
         ascii_to_ascii_simd_stride!(ascii_to_ascii_stride_both_aligned, load16_aligned, store16_aligned);
         ascii_to_ascii_simd_stride!(ascii_to_ascii_stride_src_aligned, load16_aligned, store16_unaligned);
@@ -418,31 +624,37 @@ cfg_if! {
         ascii_to_basic_latin_simd_stride!(ascii_to_basic_latin_stride_dst_aligned, load16_unaligned, store8_aligned);
         ascii_to_basic_latin_simd_stride!(ascii_to_basic_latin_stride_neither_aligned, load16_unaligned, store8_unaligned);
 
+        unpack_simd_stride!(unpack_stride_both_aligned, load16_aligned, store8_aligned);
+        unpack_simd_stride!(unpack_stride_src_aligned, load16_aligned, store8_unaligned);
+        unpack_simd_stride!(unpack_stride_dst_aligned, load16_unaligned, store8_aligned);
+        unpack_simd_stride!(unpack_stride_neither_aligned, load16_unaligned, store8_unaligned);
+
         basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_both_aligned, load8_aligned, store16_aligned);
         basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_src_aligned, load8_aligned, store16_unaligned);
         basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_dst_aligned, load8_unaligned, store16_aligned);
         basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_neither_aligned, load8_unaligned, store16_unaligned);
 
+        pack_simd_stride!(pack_stride_both_aligned, load8_aligned, store16_aligned);
+        pack_simd_stride!(pack_stride_src_aligned, load8_aligned, store16_unaligned);
+        pack_simd_stride!(pack_stride_dst_aligned, load8_unaligned, store16_aligned);
+        pack_simd_stride!(pack_stride_neither_aligned, load8_unaligned, store16_unaligned);
+
         ascii_simd_check_align!(ascii_to_ascii, u8, u8, ascii_to_ascii_stride_both_aligned, ascii_to_ascii_stride_src_aligned, ascii_to_ascii_stride_dst_aligned, ascii_to_ascii_stride_neither_aligned);
         ascii_simd_check_align!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_both_aligned, ascii_to_basic_latin_stride_src_aligned, ascii_to_basic_latin_stride_dst_aligned, ascii_to_basic_latin_stride_neither_aligned);
         ascii_simd_check_align!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_both_aligned, basic_latin_to_ascii_stride_src_aligned, basic_latin_to_ascii_stride_dst_aligned, basic_latin_to_ascii_stride_neither_aligned);
+        latin1_simd_check_align!(unpack_latin1, u8, u16, unpack_stride_both_aligned, unpack_stride_src_aligned, unpack_stride_dst_aligned, unpack_stride_neither_aligned);
+        latin1_simd_check_align!(pack_latin1, u16, u8, pack_stride_both_aligned, pack_stride_src_aligned, pack_stride_dst_aligned, pack_stride_neither_aligned);
     } else if #[cfg(all(target_endian = "little", target_pointer_width = "64"))] {
         // Aligned ALU word, little-endian, 64-bit
 
         pub const STRIDE_SIZE: usize = 16;
 
-        const ALIGNMENT: usize = 8;
+        pub const ALIGNMENT: usize = 8;
 
-        const ALIGNMENT_MASK: usize = 7;
+        pub const ALIGNMENT_MASK: usize = 7;
 
         #[inline(always)]
-        unsafe fn ascii_to_basic_latin_stride_little_64(src: *const usize, dst: *mut usize) -> bool {
-            let word = *src;
-            let second_word = *(src.offset(1));
-            // Check if the words contains non-ASCII
-            if (word & ASCII_MASK) | (second_word & ASCII_MASK) != 0 {
-                return false;
-            }
+        unsafe fn unpack_alu(word: usize, second_word: usize, dst: *mut usize) {
             let first = ((0x00000000_FF000000usize & word) << 24) |
                         ((0x00000000_00FF0000usize & word) << 16) |
                         ((0x00000000_0000FF00usize & word) << 8) |
@@ -463,18 +675,10 @@ cfg_if! {
             *(dst.offset(1)) = second;
             *(dst.offset(2)) = third;
             *(dst.offset(3)) = fourth;
-            true
         }
 
         #[inline(always)]
-        unsafe fn basic_latin_to_ascii_stride_little_64(src: *const usize, dst: *mut usize) -> bool {
-            let first = *src;
-            let second = *(src.offset(1));
-            let third = *(src.offset(2));
-            let fourth = *(src.offset(3));
-            if (first & BASIC_LATIN_MASK) | (second & BASIC_LATIN_MASK) | (third & BASIC_LATIN_MASK) | (fourth & BASIC_LATIN_MASK) != 0 {
-                return false;
-            }
+        unsafe fn pack_alu(first: usize, second: usize, third: usize, fourth: usize, dst: *mut usize) {
             let word = ((0x00FF0000_00000000usize & second) << 8) |
                        ((0x000000FF_00000000usize & second) << 16) |
                        ((0x00000000_00FF0000usize & second) << 24) |
@@ -493,28 +697,18 @@ cfg_if! {
                               (0x00000000_000000FFusize & third);
             *dst = word;
             *(dst.offset(1)) = second_word;
-            true
         }
-
-        basic_latin_alu!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_little_64);
-        basic_latin_alu!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_little_64);
     } else if #[cfg(all(target_endian = "little", target_pointer_width = "32"))] {
         // Aligned ALU word, little-endian, 32-bit
 
         pub const STRIDE_SIZE: usize = 8;
 
-        const ALIGNMENT: usize = 4;
+        pub const ALIGNMENT: usize = 4;
 
-        const ALIGNMENT_MASK: usize = 3;
+        pub const ALIGNMENT_MASK: usize = 3;
 
         #[inline(always)]
-        unsafe fn ascii_to_basic_latin_stride_little_32(src: *const usize, dst: *mut usize) -> bool {
-            let word = *src;
-            let second_word = *(src.offset(1));
-            // Check if the words contains non-ASCII
-            if (word & ASCII_MASK) | (second_word & ASCII_MASK) != 0 {
-                return false;
-            }
+        unsafe fn unpack_alu(word: usize, second_word: usize, dst: *mut usize) {
             let first = ((0x0000FF00usize & word) << 8) |
                         (0x000000FFusize & word);
             let second = ((0xFF000000usize & word) >> 8) |
@@ -527,18 +721,10 @@ cfg_if! {
             *(dst.offset(1)) = second;
             *(dst.offset(2)) = third;
             *(dst.offset(3)) = fourth;
-            return true;
         }
 
         #[inline(always)]
-        unsafe fn basic_latin_to_ascii_stride_little_32(src: *const usize, dst: *mut usize) -> bool {
-            let first = *src;
-            let second = *(src.offset(1));
-            let third = *(src.offset(2));
-            let fourth = *(src.offset(3));
-            if (first & BASIC_LATIN_MASK) | (second & BASIC_LATIN_MASK) | (third & BASIC_LATIN_MASK) | (fourth & BASIC_LATIN_MASK) != 0 {
-                return false;
-            }
+        unsafe fn pack_alu(first: usize, second: usize, third: usize, fourth: usize, dst: *mut usize) {
             let word = ((0x00FF0000usize & second) << 8) |
                        ((0x000000FFusize & second) << 16) |
                        ((0x00FF0000usize & first) >> 8) |
@@ -549,28 +735,18 @@ cfg_if! {
                               (0x000000FFusize & third);
             *dst = word;
             *(dst.offset(1)) = second_word;
-            return true;
         }
-
-        basic_latin_alu!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_little_32);
-        basic_latin_alu!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_little_32);
     } else if #[cfg(all(target_endian = "big", target_pointer_width = "64"))] {
         // Aligned ALU word, big-endian, 64-bit
 
         pub const STRIDE_SIZE: usize = 16;
 
-        const ALIGNMENT: usize = 8;
+        pub const ALIGNMENT: usize = 8;
 
-        const ALIGNMENT_MASK: usize = 7;
+        pub const ALIGNMENT_MASK: usize = 7;
 
         #[inline(always)]
-        unsafe fn ascii_to_basic_latin_stride_big_64(src: *const usize, dst: *mut usize) -> bool {
-            let word = *src;
-            let second_word = *(src.offset(1));
-            // Check if the words contains non-ASCII
-            if (word & ASCII_MASK) | (second_word & ASCII_MASK) != 0 {
-                return false;
-            }
+        unsafe fn unpack_alu(word: usize, second_word: usize, dst: *mut usize) {
             let first = ((0xFF000000_00000000usize & word) >> 8) |
                          ((0x00FF0000_00000000usize & word) >> 16) |
                          ((0x0000FF00_00000000usize & word) >> 24) |
@@ -591,18 +767,10 @@ cfg_if! {
             *(dst.offset(1)) = second;
             *(dst.offset(2)) = third;
             *(dst.offset(3)) = fourth;
-            return true;
         }
 
         #[inline(always)]
-        unsafe fn basic_latin_to_ascii_stride_big_64(src: *const usize, dst: *mut usize) -> bool {
-            let first = *src;
-            let second = *(src.offset(1));
-            let third = *(src.offset(2));
-            let fourth = *(src.offset(3));
-            if (first & BASIC_LATIN_MASK) | (second & BASIC_LATIN_MASK) | (third & BASIC_LATIN_MASK) | (fourth & BASIC_LATIN_MASK) != 0 {
-                return false;
-            }
+        unsafe fn pack_alu(first: usize, second: usize, third: usize, fourth: usize, dst: *mut usize) {
             let word = ((0x00FF0000_00000000usize & first) << 8) |
                        ((0x000000FF_00000000usize & first) << 16) |
                        ((0x00000000_00FF0000usize & first) << 24) |
@@ -621,28 +789,18 @@ cfg_if! {
                               (0x00000000_000000FFusize &  fourth);
             *dst = word;
             *(dst.offset(1)) = second_word;
-            return true;
         }
-
-        basic_latin_alu!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_big_64);
-        basic_latin_alu!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_big_64);
     } else if #[cfg(all(target_endian = "big", target_pointer_width = "32"))] {
         // Aligned ALU word, big-endian, 32-bit
 
         pub const STRIDE_SIZE: usize = 8;
 
-        const ALIGNMENT: usize = 4;
+        pub const ALIGNMENT: usize = 4;
 
-        const ALIGNMENT_MASK: usize = 3;
+        pub const ALIGNMENT_MASK: usize = 3;
 
         #[inline(always)]
-        unsafe fn ascii_to_basic_latin_stride_big_32(src: *const usize, dst: *mut usize) -> bool {
-            let word = *src;
-            let second_word = *(src.offset(1));
-            // Check if the words contains non-ASCII
-            if (word & ASCII_MASK) | (second_word & ASCII_MASK) != 0 {
-                return false;
-            }
+        unsafe fn unpack_alu(word: usize, second_word: usize, dst: *mut usize) {
             let first = ((0xFF000000usize & word) >> 8) |
                          ((0x00FF0000usize & word) >> 16);
             let second = ((0x0000FF00usize & word) << 8) |
@@ -655,18 +813,10 @@ cfg_if! {
             *(dst.offset(1)) = second;
             *(dst.offset(2)) = third;
             *(dst.offset(3)) = fourth;
-            return true;
         }
 
         #[inline(always)]
-        unsafe fn basic_latin_to_ascii_stride_big_32(src: *const usize, dst: *mut usize) -> bool {
-            let first = *src;
-            let second = *(src.offset(1));
-            let third = *(src.offset(2));
-            let fourth = *(src.offset(3));
-            if (first & BASIC_LATIN_MASK) | (second & BASIC_LATIN_MASK) | (third & BASIC_LATIN_MASK) | (fourth & BASIC_LATIN_MASK) != 0 {
-                return false;
-            }
+        unsafe fn pack_alu(first: usize, second: usize, third: usize, fourth: usize, dst: *mut usize) {
             let word = ((0x00FF0000usize & first) << 8) |
                        ((0x000000FFusize & first) << 16) |
                        ((0x00FF0000usize & second) >> 8) |
@@ -677,11 +827,7 @@ cfg_if! {
                               (0x000000FFusize & fourth);
             *dst = word;
             *(dst.offset(1)) = second_word;
-            return true;
         }
-
-        basic_latin_alu!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_big_32);
-        basic_latin_alu!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_big_32);
     } else {
         ascii_naive!(ascii_to_ascii, u8, u8);
         ascii_naive!(ascii_to_basic_latin, u8, u16);
@@ -716,7 +862,7 @@ cfg_if! {
                 let len_minus_stride = len - STRIDE_SIZE;
                 loop {
                     let simd = unsafe { load16_unaligned(src.offset(offset as isize)) };
-                    if !is_ascii(simd) {
+                    if !simd_is_ascii(simd) {
                         break;
                     }
                     offset += STRIDE_SIZE;
@@ -787,9 +933,51 @@ cfg_if! {
             None
         }
     } else {
-        // `as` truncates, so works on 32-bit, too.
-        const ASCII_MASK: usize = 0x80808080_80808080u64 as usize;
-        const BASIC_LATIN_MASK: usize = 0xFF80FF80_FF80FF80u64 as usize;
+        #[inline(always)]
+        unsafe fn unpack_latin1_stride_alu(src: *const usize, dst: *mut usize) {
+            let word = *src;
+            let second_word = *(src.offset(1));
+            unpack_alu(word, second_word, dst);
+        }
+
+        #[inline(always)]
+        unsafe fn pack_latin1_stride_alu(src: *const usize, dst: *mut usize) {
+            let first = *src;
+            let second = *(src.offset(1));
+            let third = *(src.offset(2));
+            let fourth = *(src.offset(3));
+            pack_alu(first, second, third, fourth, dst);
+        }
+
+        #[inline(always)]
+        unsafe fn ascii_to_basic_latin_stride_alu(src: *const usize, dst: *mut usize) -> bool {
+            let word = *src;
+            let second_word = *(src.offset(1));
+            // Check if the words contains non-ASCII
+            if (word & ASCII_MASK) | (second_word & ASCII_MASK) != 0 {
+                return false;
+            }
+            unpack_alu(word, second_word, dst);
+            true
+        }
+
+        #[inline(always)]
+        unsafe fn basic_latin_to_ascii_stride_alu(src: *const usize, dst: *mut usize) -> bool {
+            let first = *src;
+            let second = *(src.offset(1));
+            let third = *(src.offset(2));
+            let fourth = *(src.offset(3));
+            if (first & BASIC_LATIN_MASK) | (second & BASIC_LATIN_MASK) | (third & BASIC_LATIN_MASK) | (fourth & BASIC_LATIN_MASK) != 0 {
+                return false;
+            }
+            pack_alu(first, second, third, fourth, dst);
+            true
+        }
+
+        basic_latin_alu!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_alu);
+        basic_latin_alu!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_alu);
+        latin1_alu!(unpack_latin1, u8, u16, unpack_latin1_stride_alu);
+        latin1_alu!(pack_latin1, u16, u8, pack_latin1_stride_alu);
 
         #[inline(always)]
         unsafe fn ascii_to_ascii_stride(src: *const usize, dst: *mut usize) -> Option<usize> {

src/lib.rs

@@ -17,6 +17,10 @@
 //! streamability goals are browser-oriented, and that FFI-friendliness is a
 //! goal.
 //!
+//! Additionally, the `mem` module provides functions that are useful for
+//! applications that need to be able to deal with legacy in-memory
+//! representations of Unicode.
+//!
 //! # Availability
 //!
 //! The code is available under the
@@ -491,7 +495,7 @@
 //! </tbody>
 //! </table>
 
-#![cfg_attr(feature = "simd-accel", feature(cfg_target_feature, platform_intrinsics))]
+#![cfg_attr(feature = "simd-accel", feature(cfg_target_feature, platform_intrinsics, core_intrinsics))]
 
 #[macro_use]
 extern crate cfg_if;
@@ -539,6 +543,8 @@ mod handles;
 mod data;
 mod variant;
 
+pub mod mem;
+
 use variant::*;
 use utf_8::utf8_valid_up_to;
 use ascii::ascii_valid_up_to;
@@ -2030,20 +2036,20 @@ static ENCODINGS_IN_LABEL_SORT: [&'static Encoding; 219] = [&WINDOWS_1252_INIT,
 /// # Streaming vs. Non-Streaming
 ///
 /// When you have the entire input in a single buffer, you can use the
-/// methods [`decode()`][1], [`decode_with_bom_removal()`][2],
-/// [`decode_without_bom_handling()`][3],
-/// [`decode_without_bom_handling_and_without_replacement()`][4] and
-/// [`encode()`][5]. (These methods are available to Rust callers only and are
+/// methods [`decode()`][3], [`decode_with_bom_removal()`][3],
+/// [`decode_without_bom_handling()`][5],
+/// [`decode_without_bom_handling_and_without_replacement()`][6] and
+/// [`encode()`][7]. (These methods are available to Rust callers only and are
 /// not available in the C API.) Unlike the rest of the API available to Rust,
 /// these methods perform heap allocations. You should the `Decoder` and
 /// `Encoder` objects when your input is split into multiple buffers or when
 /// you want to control the allocation of the output buffers.
 ///
-/// [1]: #method.decode
-/// [2]: #method.decode_with_bom_removal
-/// [3]: #method.decode_without_bom_handling
-/// [4]: #method.decode_without_bom_handling_and_without_replacement
-/// [5]: #method.encode
+/// [3]: #method.decode
+/// [4]: #method.decode_with_bom_removal
+/// [5]: #method.decode_without_bom_handling
+/// [6]: #method.decode_without_bom_handling_and_without_replacement
+/// [7]: #method.encode
 ///
 /// # Instances
 ///
@@ -2222,6 +2228,7 @@ impl Encoding {
     /// unsafe fallback for labels that `for_label()` maps to `Some(REPLACEMENT)`.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn for_label_no_replacement(label: &[u8]) -> Option<&'static Encoding> {
         match Encoding::for_label(label) {
             None => None,
@@ -2246,6 +2253,7 @@ impl Encoding {
     /// or UTF-16BE BOM or `None` otherwise.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn for_bom(buffer: &[u8]) -> Option<(&'static Encoding, usize)> {
         if buffer.starts_with(b"\xEF\xBB\xBF") {
             Some((UTF_8, 3))
@@ -2264,6 +2272,7 @@ impl Encoding {
     /// `document.characterSet` property.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn name(&'static self) -> &'static str {
         self.name
     }
@@ -2272,6 +2281,7 @@ impl Encoding {
     /// `char`. (Only true if the output encoding is UTF-8.)
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn can_encode_everything(&'static self) -> bool {
         self.output_encoding() == UTF_8
     }
@@ -2280,12 +2290,14 @@ impl Encoding {
     /// U+0000...U+007F and vice versa.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn is_ascii_compatible(&'static self) -> bool {
         !(self == REPLACEMENT || self == UTF_16BE || self == UTF_16LE || self == ISO_2022_JP)
     }
 
     /// Checks whether the bytes 0x00...0x7F map mostly to the characters
     /// U+0000...U+007F and vice versa.
+    #[inline]
     fn is_potentially_borrowable(&'static self) -> bool {
         !(self == REPLACEMENT || self == UTF_16BE || self == UTF_16LE)
     }
@@ -2294,6 +2306,7 @@ impl Encoding {
     /// UTF-16BE, UTF-16LE and replacement and the encoding itself otherwise.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn output_encoding(&'static self) -> &'static Encoding {
         if self == REPLACEMENT || self == UTF_16BE || self == UTF_16LE {
             UTF_8
@@ -2336,6 +2349,7 @@ impl Encoding {
     /// `usize`.
     ///
     /// Available to Rust only.
+    #[inline]
     pub fn decode<'a>(&'static self, bytes: &'a [u8]) -> (Cow<'a, str>, &'static Encoding, bool) {
         let (encoding, without_bom) = match Encoding::for_bom(bytes) {
             Some((encoding, bom_length)) => (encoding, &bytes[bom_length..]),
@@ -2378,6 +2392,7 @@ impl Encoding {
     /// `usize`.
     ///
     /// Available to Rust only.
+    #[inline]
     pub fn decode_with_bom_removal<'a>(&'static self, bytes: &'a [u8]) -> (Cow<'a, str>, bool) {
         let without_bom = if self == UTF_8 && bytes.starts_with(b"\xEF\xBB\xBF") {
             &bytes[3..]
@@ -2689,6 +2704,7 @@ impl Encoding {
     /// for UTF-8, UTF-16LE or UTF-16BE instead of this encoding.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn new_decoder(&'static self) -> Decoder {
         Decoder::new(self, self.new_variant_decoder(), BomHandling::Sniff)
     }
@@ -2702,6 +2718,7 @@ impl Encoding {
     /// encoding.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn new_decoder_with_bom_removal(&'static self) -> Decoder {
         Decoder::new(self, self.new_variant_decoder(), BomHandling::Remove)
     }
@@ -2717,6 +2734,7 @@ impl Encoding {
     /// instead of this method to cause the BOM to be removed.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn new_decoder_without_bom_handling(&'static self) -> Decoder {
         Decoder::new(self, self.new_variant_decoder(), BomHandling::Off)
     }
@@ -2724,6 +2742,7 @@ impl Encoding {
     /// Instantiates a new encoder for the output encoding of this encoding.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn new_encoder(&'static self) -> Encoder {
         let enc = self.output_encoding();
         enc.variant.new_encoder(enc)
@@ -2767,6 +2786,7 @@ impl Encoding {
 }
 
 impl PartialEq for Encoding {
+    #[inline]
     fn eq(&self, other: &Encoding) -> bool {
         (self as *const Encoding) == (other as *const Encoding)
     }
@@ -2775,12 +2795,14 @@ impl PartialEq for Encoding {
 impl Eq for Encoding {}
 
 impl Hash for Encoding {
+    #[inline]
     fn hash<H: Hasher>(&self, state: &mut H) {
         (self as *const Encoding).hash(state);
     }
 }
 
 impl std::fmt::Debug for Encoding {
+    #[inline]
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "Encoding {{ {} }}", self.name)
     }
@@ -2788,6 +2810,7 @@ impl std::fmt::Debug for Encoding {
 
 #[cfg(feature = "serde")]
 impl Serialize for Encoding {
+    #[inline]
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
         where S: Serializer
     {
@@ -3054,6 +3077,7 @@ impl Decoder {
     /// of the decoder.
     ///
     /// Available via the C wrapper.
+    #[inline]
     pub fn encoding(&self) -> &'static Encoding {
         self.encoding
     }
@@ -3769,12 +3793,14 @@ impl Encoder {
     }
 
     /// The `Encoding` this `Encoder` is for.
+    #[inline]
     pub fn encoding(&self) -> &'static Encoding {
         self.encoding
     }
 
     /// Returns `true` if this is an ISO-2022-JP encoder that's not in the
     /// ASCII state and `false` otherwise.
+    #[inline]
     pub fn has_pending_state(&self) -> bool {
         self.variant.has_pending_state()
     }
@@ -4111,6 +4137,16 @@ fn in_range16(i: u16, start: u16, end: u16) -> bool {
     i.wrapping_sub(start) < (end - start)
 }
 
+#[inline(always)]
+fn in_range32(i: u32, start: u32, end: u32) -> bool {
+    i.wrapping_sub(start) < (end - start)
+}
+
+#[inline(always)]
+fn in_inclusive_range8(i: u8, start: u8, end: u8) -> bool {
+    i.wrapping_sub(start) <= (end - start)
+}
+
 #[inline(always)]
 fn in_inclusive_range16(i: u16, start: u16, end: u16) -> bool {
     i.wrapping_sub(start) <= (end - start)

src/simd_funcs.rs

@@ -21,6 +21,7 @@ pub unsafe fn load16_unaligned(ptr: *const u8) -> u8x16 {
     simd
 }
 
+#[allow(dead_code)]
 #[inline(always)]
 pub unsafe fn load16_aligned(ptr: *const u8) -> u8x16 {
     *(ptr as *const u8x16)
@@ -31,6 +32,7 @@ pub unsafe fn store16_unaligned(ptr: *mut u8, s: u8x16) {
     ::std::ptr::copy_nonoverlapping(&s as *const u8x16 as *const u8, ptr, 16);
 }
 
+#[allow(dead_code)]
 #[inline(always)]
 pub unsafe fn store16_aligned(ptr: *mut u8, s: u8x16) {
     *(ptr as *mut u8x16) = s;
@@ -43,6 +45,7 @@ pub unsafe fn load8_unaligned(ptr: *const u16) -> u16x8 {
     simd
 }
 
+#[allow(dead_code)]
 #[inline(always)]
 pub unsafe fn load8_aligned(ptr: *const u16) -> u16x8 {
     *(ptr as *const u16x8)
@@ -53,6 +56,7 @@ pub unsafe fn store8_unaligned(ptr: *mut u16, s: u16x8) {
     ::std::ptr::copy_nonoverlapping(&s as *const u16x8 as *const u8, ptr as *mut u8, 16);
 }
 
+#[allow(dead_code)]
 #[inline(always)]
 pub unsafe fn store8_aligned(ptr: *mut u16, s: u16x8) {
     *(ptr as *mut u16x8) = s;
@@ -89,7 +93,7 @@ cfg_if! {
 cfg_if! {
     if #[cfg(target_feature = "sse2")] {
         #[inline(always)]
-        pub fn is_ascii(s: u8x16) -> bool {
+        pub fn simd_is_ascii(s: u8x16) -> bool {
             unsafe {
                 let signed: i8x16 = ::std::mem::transmute_copy(&s);
                 x86_mm_movemask_epi8(signed) == 0
@@ -101,16 +105,42 @@ cfg_if! {
         }
 
         #[inline(always)]
-        pub fn is_ascii(s: u8x16) -> bool {
+        pub fn simd_is_ascii(s: u8x16) -> bool {
             unsafe {
                 aarch64_vmaxvq_u8(s) < 0x80
             }
         }
     } else {
         #[inline(always)]
-        pub fn is_ascii(s: u8x16) -> bool {
-            let highest_ascii = u8x16::splat(0x7F);
-            !s.gt(highest_ascii).any()
+        pub fn simd_is_ascii(s: u8x16) -> bool {
+            let above_ascii = u8x16::splat(0x80);
+            s.lt(above_ascii).all()
+        }
+    }
+}
+
+cfg_if! {
+    if #[cfg(target_feature = "sse2")] {
+        #[inline(always)]
+        pub fn simd_is_str_latin1(s: u8x16) -> bool {
+            if simd_is_ascii(s) {
+                return true;
+            }
+            let above_str_latin1 = u8x16::splat(0xC4);
+            s.lt(above_str_latin1).all()
+        }
+    } else if #[cfg(target_arch = "aarch64")]{
+        #[inline(always)]
+        pub fn simd_is_str_latin1(s: u8x16) -> bool {
+            unsafe {
+                aarch64_vmaxvq_u8(s) < 0xC4
+            }
+        }
+    } else {
+        #[inline(always)]
+        pub fn simd_is_str_latin1(s: u8x16) -> bool {
+            let above_str_latin1 = u8x16::splat(0xC4);
+            s.lt(above_str_latin1).all()
         }
     }
 }
@@ -122,20 +152,103 @@ cfg_if! {
         }
 
         #[inline(always)]
-        pub fn is_basic_latin(s: u16x8) -> bool {
+        pub fn simd_is_basic_latin(s: u16x8) -> bool {
             unsafe {
                 aarch64_vmaxvq_u16(s) < 0x80
             }
         }
+
+        #[inline(always)]
+        pub fn simd_is_latin1(s: u16x8) -> bool {
+            unsafe {
+                aarch64_vmaxvq_u16(s) < 0x100
+            }
+        }
     } else {
         #[inline(always)]
-        pub fn is_basic_latin(s: u16x8) -> bool {
-            let highest_ascii = u16x8::splat(0x7F);
-            !s.gt(highest_ascii).any()
+        pub fn simd_is_basic_latin(s: u16x8) -> bool {
+            let above_ascii = u16x8::splat(0x80);
+            s.lt(above_ascii).all()
+        }
+
+        #[inline(always)]
+        pub fn simd_is_latin1(s: u16x8) -> bool {
+            // For some reason, on SSE2 this formulation
+            // seems faster in this case while the above
+            // function is better the other way round...
+            let highest_latin1 = u16x8::splat(0xFF);
+            !s.gt(highest_latin1).any()
         }
     }
 }
 
+#[inline(always)]
+pub fn contains_surrogates(s: u16x8) -> bool {
+    let mask = u16x8::splat(0xF800);
+    let surrogate_bits = u16x8::splat(0xD800);
+    (s & mask).eq(surrogate_bits).any()
+}
+
+cfg_if! {
+    if #[cfg(target_arch = "aarch64")]{
+        macro_rules! aarch64_return_false_if_below_hebrew {
+            ($s:ident) => ({
+                unsafe {
+                    if aarch64_vmaxvq_u16($s) < 0x0590 {
+                        return false;
+                    }
+                }
+            })
+        }
+
+        macro_rules! non_aarch64_return_false_if_all {
+            ($s:ident) => ()
+        }
+    } else {
+        macro_rules! aarch64_return_false_if_below_hebrew {
+            ($s:ident) => ()
+        }
+
+        macro_rules! non_aarch64_return_false_if_all {
+            ($s:ident) => ({
+                if $s.all() {
+                    return false;
+                }
+            })
+        }
+    }
+}
+
+macro_rules! in_range16x8 {
+    ($s:ident, $start:expr, $end:expr) => ({
+        // SIMD sub is wrapping
+        ($s - u16x8::splat($start)).lt(u16x8::splat($end - $start))
+    })
+}
+
+#[inline(always)]
+pub fn is_u16x8_bidi(s: u16x8) -> bool {
+    // We try to first quickly refute the RTLness of the vector. If that
+    // fails, we do the real RTL check, so in that case we end up wasting
+    // the work for the up-front quick checks. Even the quick-check is
+    // two-fold in order to return `false` ASAP if everything is below
+    // Hebrew.
+
+    aarch64_return_false_if_below_hebrew!(s);
+
+    let below_hebrew = s.lt(u16x8::splat(0x0590));
+
+    non_aarch64_return_false_if_all!(below_hebrew);
+
+    if (below_hebrew | in_range16x8!(s, 0x0900, 0x200F) | in_range16x8!(s, 0x2068, 0xD802)).all() {
+        return false;
+    }
+
+    // Quick refutation failed. Let's do the full check.
+
+    (in_range16x8!(s, 0x0590, 0x0900) | in_range16x8!(s, 0xFB50, 0xFE00) | in_range16x8!(s, 0xFE70, 0xFF00) | in_range16x8!(s, 0xD802, 0xD804) | in_range16x8!(s, 0xD83A, 0xD83C) | s.eq(u16x8::splat(0x200F)) | s.eq(u16x8::splat(0x202B)) | s.eq(u16x8::splat(0x202E)) | s.eq(u16x8::splat(0x2067))).any()
+}
+
 #[inline(always)]
 pub fn simd_unpack(s: u8x16) -> (u16x8, u16x8) {
     unsafe {
@@ -206,7 +319,7 @@ mod tests {
     }
 
     #[test]
-    fn test_is_basic_latin_success() {
+    fn test_simd_is_basic_latin_success() {
         let ascii: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
                                0x72, 0x73, 0x74, 0x75, 0x76];
         let basic_latin: [u16; 16] = [0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70,
@@ -216,7 +329,7 @@ mod tests {
         let mut vec = Vec::with_capacity(16);
         vec.resize(16, 0u8);
         let ptr = vec.as_mut_ptr();
-        assert!(is_basic_latin(first | second));
+        assert!(simd_is_basic_latin(first | second));
         unsafe {
             store16_unaligned(ptr, simd_pack(first, second));
         }
@@ -224,46 +337,46 @@ mod tests {
     }
 
     #[test]
-    fn test_is_basic_latin_c0() {
+    fn test_simd_is_basic_latin_c0() {
         let input: [u16; 16] = [0x61, 0x62, 0x63, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
                                 0x72, 0x73, 0x74, 0x75, 0x76];
         let first = unsafe { load8_unaligned(input.as_ptr()) };
         let second = unsafe { load8_unaligned(input.as_ptr().offset(8)) };
-        assert!(!is_basic_latin(first | second));
+        assert!(!simd_is_basic_latin(first | second));
     }
 
     #[test]
-    fn test_is_basic_latin_0fff() {
+    fn test_simd_is_basic_latin_0fff() {
         let input: [u16; 16] = [0x61, 0x62, 0x63, 0x0FFF, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70,
                                 0x71, 0x72, 0x73, 0x74, 0x75, 0x76];
         let first = unsafe { load8_unaligned(input.as_ptr()) };
         let second = unsafe { load8_unaligned(input.as_ptr().offset(8)) };
-        assert!(!is_basic_latin(first | second));
+        assert!(!simd_is_basic_latin(first | second));
     }
 
     #[test]
-    fn test_is_basic_latin_ffff() {
+    fn test_simd_is_basic_latin_ffff() {
         let input: [u16; 16] = [0x61, 0x62, 0x63, 0xFFFF, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70,
                                 0x71, 0x72, 0x73, 0x74, 0x75, 0x76];
         let first = unsafe { load8_unaligned(input.as_ptr()) };
         let second = unsafe { load8_unaligned(input.as_ptr().offset(8)) };
-        assert!(!is_basic_latin(first | second));
+        assert!(!simd_is_basic_latin(first | second));
     }
 
     #[test]
-    fn test_is_ascii_success() {
+    fn test_simd_is_ascii_success() {
         let ascii: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
                                0x72, 0x73, 0x74, 0x75, 0x76];
         let simd = unsafe { load16_unaligned(ascii.as_ptr()) };
-        assert!(is_ascii(simd));
+        assert!(simd_is_ascii(simd));
     }
 
     #[test]
-    fn test_is_ascii_failure() {
+    fn test_simd_is_ascii_failure() {
         let input: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x81, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
                                0x72, 0x73, 0x74, 0x75, 0x76];
         let simd = unsafe { load16_unaligned(input.as_ptr()) };
-        assert!(!is_ascii(simd));
+        assert!(!simd_is_ascii(simd));
     }
 
     #[cfg(target_feature = "sse2")]

src/utf_8.rs

@@ -34,21 +34,21 @@ cfg_if! {
     }
 }
 
-const UTF8_NORMAL_TRAIL: u8 = 1 << 3;
+pub const UTF8_NORMAL_TRAIL: u8 = 1 << 3;
 
-const UTF8_THREE_BYTE_SPECIAL_LOWER_BOUND_TRAIL: u8 = 1 << 4;
+pub const UTF8_THREE_BYTE_SPECIAL_LOWER_BOUND_TRAIL: u8 = 1 << 4;
 
-const UTF8_THREE_BYTE_SPECIAL_UPPER_BOUND_TRAIL: u8 = 1 << 5;
+pub const UTF8_THREE_BYTE_SPECIAL_UPPER_BOUND_TRAIL: u8 = 1 << 5;
 
-const UTF8_FOUR_BYTE_SPECIAL_LOWER_BOUND_TRAIL: u8 = 1 << 6;
+pub const UTF8_FOUR_BYTE_SPECIAL_LOWER_BOUND_TRAIL: u8 = 1 << 6;
 
-const UTF8_FOUR_BYTE_SPECIAL_UPPER_BOUND_TRAIL: u8 = 1 << 7;
+pub const UTF8_FOUR_BYTE_SPECIAL_UPPER_BOUND_TRAIL: u8 = 1 << 7;
 
 // BEGIN GENERATED CODE. PLEASE DO NOT EDIT.
 // Instead, please regenerate using generate-encoding-data.py
 
 /// Bit is 1 if the trail is invalid.
-static UTF8_TRAIL_INVALID: [u8; 256] =
+pub static UTF8_TRAIL_INVALID: [u8; 256] =
     [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248,
      248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248,
      248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248,
@@ -433,16 +433,18 @@ pub struct Utf8Decoder {
 }
 
 impl Utf8Decoder {
+    pub fn new_inner() -> Utf8Decoder {
+        Utf8Decoder {
+            code_point: 0,
+            bytes_seen: 0,
+            bytes_needed: 0,
+            lower_boundary: 0x80u8,
+            upper_boundary: 0xBFu8,
+        }
+    }
+
     pub fn new() -> VariantDecoder {
-        VariantDecoder::Utf8(
-            Utf8Decoder {
-                code_point: 0,
-                bytes_seen: 0,
-                bytes_needed: 0,
-                lower_boundary: 0x80u8,
-                upper_boundary: 0xBFu8,
-            }
-        )
+        VariantDecoder::Utf8(Utf8Decoder::new_inner())
     }
 
     fn extra_from_state(&self) -> usize {