openharmony/third_party_rust_encoding_rs
1
0
Fork 0
mirror of https://github.com/openharmony/third_party_rust_encoding_rs.git synced 2026-07-21 02:05:23 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge NEON support.

Browse Source
This commit is contained in:
Henri Sivonen
2018-05-07 13:09:39 +03:00
parent f0bc93769d ce24104502
commit 1ee867e39b
5 changed files with 234 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+1 -1
View File
@@ -21,7 +21,7 @@ no-static-ideograph-encoder-tables = []
[dependencies]
cfg-if = "0.1.0"
simd = { version = "0.2.0", optional = true }
simd = { version = "0.2.2", optional = true }
serde = { version = "1.0", optional = true }
# rayon = { version = "0.8.0", optional = true }
src/ascii.rs
+171 -144
View File
@@ -22,7 +22,7 @@
// different approaches based on benchmarking on Raspberry Pi 3.
#[cfg(all(feature = "simd-accel",
any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))]
any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"), all(target_endian = "little", target_feature = "neon"))))]
use simd_funcs::*;
// `as` truncates, so works on 32-bit, too.
@@ -70,12 +70,12 @@ macro_rules! ascii_alu {
// to alignment.
// if ::std::mem::size_of::<$src_unit>() == ::std::mem::size_of::<$dst_unit>() {
// ascii_to_ascii
let src_alignment = (src as usize) & ALIGNMENT_MASK;
let dst_alignment = (dst as usize) & ALIGNMENT_MASK;
let src_alignment = (src as usize) & ALU_ALIGNMENT_MASK;
let dst_alignment = (dst as usize) & ALU_ALIGNMENT_MASK;
if src_alignment != dst_alignment {
break;
}
(ALIGNMENT - src_alignment) & ALIGNMENT_MASK
(ALU_ALIGNMENT - src_alignment) & ALU_ALIGNMENT_MASK
// } else if ::std::mem::size_of::<$src_unit>() < ::std::mem::size_of::<$dst_unit>() {
// ascii_to_basic_latin
// let src_until_alignment = (ALIGNMENT - ((src as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
@@ -92,7 +92,7 @@ macro_rules! ascii_alu {
// dst_until_alignment
// }
};
if until_alignment + STRIDE_SIZE <= len {
if until_alignment + ALU_STRIDE_SIZE <= len {
// Moving pointers to alignment seems to be a pessimization on
// x86_64 for operations that have UTF-16 as the internal
// Unicode representation. However, since it seems to be a win
@@ -110,14 +110,14 @@ macro_rules! ascii_alu {
offset += 1;
until_alignment -= 1;
}
let len_minus_stride = len - STRIDE_SIZE;
let len_minus_stride = len - ALU_STRIDE_SIZE;
loop {
if let Some(num_ascii) = $stride_fn(src.offset(offset as isize) as *const usize,
dst.offset(offset as isize) as *mut usize) {
offset += num_ascii;
return Some((*(src.offset(offset as isize)), offset));
}
offset += STRIDE_SIZE;
offset += ALU_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -163,21 +163,21 @@ macro_rules! basic_latin_alu {
// } else
if ::std::mem::size_of::<$src_unit>() < ::std::mem::size_of::<$dst_unit>() {
// ascii_to_basic_latin
let src_until_alignment = (ALIGNMENT - ((src as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
if (dst.offset(src_until_alignment as isize) as usize) & ALIGNMENT_MASK != 0 {
let src_until_alignment = (ALU_ALIGNMENT - ((src as usize) & ALU_ALIGNMENT_MASK)) & ALU_ALIGNMENT_MASK;
if (dst.offset(src_until_alignment as isize) as usize) & ALU_ALIGNMENT_MASK != 0 {
break;
}
src_until_alignment
} else {
// basic_latin_to_ascii
let dst_until_alignment = (ALIGNMENT - ((dst as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
if (src.offset(dst_until_alignment as isize) as usize) & ALIGNMENT_MASK != 0 {
let dst_until_alignment = (ALU_ALIGNMENT - ((dst as usize) & ALU_ALIGNMENT_MASK)) & ALU_ALIGNMENT_MASK;
if (src.offset(dst_until_alignment as isize) as usize) & ALU_ALIGNMENT_MASK != 0 {
break;
}
dst_until_alignment
}
};
if until_alignment + STRIDE_SIZE <= len {
if until_alignment + ALU_STRIDE_SIZE <= len {
// Moving pointers to alignment seems to be a pessimization on
// x86_64 for operations that have UTF-16 as the internal
// Unicode representation. However, since it seems to be a win
@@ -195,13 +195,13 @@ macro_rules! basic_latin_alu {
offset += 1;
until_alignment -= 1;
}
let len_minus_stride = len - STRIDE_SIZE;
let len_minus_stride = len - ALU_STRIDE_SIZE;
loop {
if !$stride_fn(src.offset(offset as isize) as *const usize,
dst.offset(offset as isize) as *mut usize) {
break;
}
offset += STRIDE_SIZE;
offset += ALU_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -236,32 +236,32 @@ macro_rules! latin1_alu {
let mut until_alignment = {
if ::std::mem::size_of::<$src_unit>() < ::std::mem::size_of::<$dst_unit>() {
// unpack
let src_until_alignment = (ALIGNMENT - ((src as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
if (dst.offset(src_until_alignment as isize) as usize) & ALIGNMENT_MASK != 0 {
let src_until_alignment = (ALU_ALIGNMENT - ((src as usize) & ALU_ALIGNMENT_MASK)) & ALU_ALIGNMENT_MASK;
if (dst.offset(src_until_alignment as isize) as usize) & ALU_ALIGNMENT_MASK != 0 {
break;
}
src_until_alignment
} else {
// pack
let dst_until_alignment = (ALIGNMENT - ((dst as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
if (src.offset(dst_until_alignment as isize) as usize) & ALIGNMENT_MASK != 0 {
let dst_until_alignment = (ALU_ALIGNMENT - ((dst as usize) & ALU_ALIGNMENT_MASK)) & ALU_ALIGNMENT_MASK;
if (src.offset(dst_until_alignment as isize) as usize) & ALU_ALIGNMENT_MASK != 0 {
break;
}
dst_until_alignment
}
};
if until_alignment + STRIDE_SIZE <= len {
if until_alignment + ALU_STRIDE_SIZE <= len {
while until_alignment != 0 {
let code_unit = *(src.offset(offset as isize));
*(dst.offset(offset as isize)) = code_unit as $dst_unit;
offset += 1;
until_alignment -= 1;
}
let len_minus_stride = len - STRIDE_SIZE;
let len_minus_stride = len - ALU_STRIDE_SIZE;
loop {
$stride_fn(src.offset(offset as isize) as *const usize,
dst.offset(offset as isize) as *mut usize);
offset += STRIDE_SIZE;
offset += ALU_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -289,22 +289,22 @@ macro_rules! ascii_simd_check_align {
#[inline(always)]
pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) -> Option<($src_unit, usize)> {
let mut offset = 0usize;
if STRIDE_SIZE <= len {
let len_minus_stride = len - STRIDE_SIZE;
if SIMD_STRIDE_SIZE <= len {
let len_minus_stride = len - SIMD_STRIDE_SIZE;
// XXX Should we first process one stride unconditinoally as unaligned to
// avoid the cost of the branchiness below if the first stride fails anyway?
// XXX Should we just use unaligned SSE2 access unconditionally? It seems that
// on Haswell, it would make sense to just use unaligned and not bother
// checking. Need to benchmark older architectures before deciding.
let dst_masked = (dst as usize) & ALIGNMENT_MASK;
if ((src as usize) & ALIGNMENT_MASK) == 0 {
let dst_masked = (dst as usize) & SIMD_ALIGNMENT_MASK;
if ((src as usize) & SIMD_ALIGNMENT_MASK) == 0 {
if dst_masked == 0 {
loop {
if !$stride_both_aligned(src.offset(offset as isize),
dst.offset(offset as isize)) {
break;
}
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -315,7 +315,7 @@ macro_rules! ascii_simd_check_align {
dst.offset(offset as isize)) {
break;
}
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -328,7 +328,7 @@ macro_rules! ascii_simd_check_align {
dst.offset(offset as isize)) {
break;
}
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -339,7 +339,7 @@ macro_rules! ascii_simd_check_align {
dst.offset(offset as isize)) {
break;
}
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -371,20 +371,20 @@ macro_rules! latin1_simd_check_align {
#[inline(always)]
pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) {
let mut offset = 0usize;
if STRIDE_SIZE <= len {
let len_minus_stride = len - STRIDE_SIZE;
if SIMD_STRIDE_SIZE <= len {
let len_minus_stride = len - SIMD_STRIDE_SIZE;
// XXX Should we first process one stride unconditinoally as unaligned to
// avoid the cost of the branchiness below if the first stride fails anyway?
// XXX Should we just use unaligned SSE2 access unconditionally? It seems that
// on Haswell, it would make sense to just use unaligned and not bother
// checking. Need to benchmark older architectures before deciding.
let dst_masked = (dst as usize) & ALIGNMENT_MASK;
if ((src as usize) & ALIGNMENT_MASK) == 0 {
let dst_masked = (dst as usize) & SIMD_ALIGNMENT_MASK;
if ((src as usize) & SIMD_ALIGNMENT_MASK) == 0 {
if dst_masked == 0 {
loop {
$stride_both_aligned(src.offset(offset as isize),
dst.offset(offset as isize));
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -393,7 +393,7 @@ macro_rules! latin1_simd_check_align {
loop {
$stride_src_aligned(src.offset(offset as isize),
dst.offset(offset as isize));
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -404,7 +404,7 @@ macro_rules! latin1_simd_check_align {
loop {
$stride_dst_aligned(src.offset(offset as isize),
dst.offset(offset as isize));
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -413,7 +413,7 @@ macro_rules! latin1_simd_check_align {
loop {
$stride_neither_aligned(src.offset(offset as isize),
dst.offset(offset as isize));
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -451,14 +451,14 @@ macro_rules! ascii_simd_unalign {
#[inline(always)]
pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) -> Option<($src_unit, usize)> {
let mut offset = 0usize;
if STRIDE_SIZE <= len {
let len_minus_stride = len - STRIDE_SIZE;
if SIMD_STRIDE_SIZE <= len {
let len_minus_stride = len - SIMD_STRIDE_SIZE;
loop {
if !$stride_neither_aligned(src.offset(offset as isize),
dst.offset(offset as isize)) {
break;
}
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -485,12 +485,12 @@ macro_rules! latin1_simd_unalign {
#[inline(always)]
pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) {
let mut offset = 0usize;
if STRIDE_SIZE <= len {
let len_minus_stride = len - STRIDE_SIZE;
if SIMD_STRIDE_SIZE <= len {
let len_minus_stride = len - SIMD_STRIDE_SIZE;
loop {
$stride_neither_aligned(src.offset(offset as isize),
dst.offset(offset as isize));
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -587,7 +587,9 @@ cfg_if! {
if #[cfg(all(feature = "simd-accel", target_endian = "little", target_arch = "aarch64"))] {
// SIMD with the same instructions for aligned and unaligned loads and stores
pub const STRIDE_SIZE: usize = 16;
pub const SIMD_STRIDE_SIZE: usize = 16;
pub const MAX_STRIDE_SIZE: usize = 16;
// pub const ALIGNMENT: usize = 8;
@@ -604,16 +606,18 @@ cfg_if! {
ascii_simd_unalign!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_neither_aligned);
latin1_simd_unalign!(unpack_latin1, u8, u16, unpack_stride_neither_aligned);
latin1_simd_unalign!(pack_latin1, u16, u8, pack_stride_neither_aligned);
} else if #[cfg(all(feature = "simd-accel", target_feature = "sse2"))] {
} else if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_feature = "neon"))))] {
// SIMD with different instructions for aligned and unaligned loads and stores.
//
// Newer microarchitectures are not supposed to have a performance difference between
// aligned and unaligned SSE2 loads and stores when the address is actually aligned,
// but the benchmark results I see don't agree.
pub const STRIDE_SIZE: usize = 16;
pub const SIMD_STRIDE_SIZE: usize = 16;
pub const ALIGNMENT_MASK: usize = 15;
pub const MAX_STRIDE_SIZE: usize = 16;
pub const SIMD_ALIGNMENT_MASK: usize = 15;
ascii_to_ascii_simd_stride!(ascii_to_ascii_stride_both_aligned, load16_aligned, store16_aligned);
ascii_to_ascii_simd_stride!(ascii_to_ascii_stride_src_aligned, load16_aligned, store16_unaligned);
@@ -648,11 +652,13 @@ cfg_if! {
} else if #[cfg(all(target_endian = "little", target_pointer_width = "64"))] {
// Aligned ALU word, little-endian, 64-bit
pub const STRIDE_SIZE: usize = 16;
pub const ALU_STRIDE_SIZE: usize = 16;
pub const ALIGNMENT: usize = 8;
pub const MAX_STRIDE_SIZE: usize = 16;
pub const ALIGNMENT_MASK: usize = 7;
pub const ALU_ALIGNMENT: usize = 8;
pub const ALU_ALIGNMENT_MASK: usize = 7;
#[inline(always)]
unsafe fn unpack_alu(word: usize, second_word: usize, dst: *mut usize) {
@@ -702,11 +708,13 @@ cfg_if! {
} else if #[cfg(all(target_endian = "little", target_pointer_width = "32"))] {
// Aligned ALU word, little-endian, 32-bit
pub const STRIDE_SIZE: usize = 8;
pub const ALU_STRIDE_SIZE: usize = 8;
pub const ALIGNMENT: usize = 4;
pub const MAX_STRIDE_SIZE: usize = 8;
pub const ALIGNMENT_MASK: usize = 3;
pub const ALU_ALIGNMENT: usize = 4;
pub const ALU_ALIGNMENT_MASK: usize = 3;
#[inline(always)]
unsafe fn unpack_alu(word: usize, second_word: usize, dst: *mut usize) {
@@ -740,11 +748,13 @@ cfg_if! {
} else if #[cfg(all(target_endian = "big", target_pointer_width = "64"))] {
// Aligned ALU word, big-endian, 64-bit
pub const STRIDE_SIZE: usize = 16;
pub const ALU_STRIDE_SIZE: usize = 16;
pub const ALIGNMENT: usize = 8;
pub const MAX_STRIDE_SIZE: usize = 16;
pub const ALIGNMENT_MASK: usize = 7;
pub const ALU_ALIGNMENT: usize = 8;
pub const ALU_ALIGNMENT_MASK: usize = 7;
#[inline(always)]
unsafe fn unpack_alu(word: usize, second_word: usize, dst: *mut usize) {
@@ -794,11 +804,13 @@ cfg_if! {
} else if #[cfg(all(target_endian = "big", target_pointer_width = "32"))] {
// Aligned ALU word, big-endian, 32-bit
pub const STRIDE_SIZE: usize = 8;
pub const ALU_STRIDE_SIZE: usize = 8;
pub const ALIGNMENT: usize = 4;
pub const MAX_STRIDE_SIZE: usize = 8;
pub const ALIGNMENT_MASK: usize = 3;
pub const ALU_ALIGNMENT: usize = 4;
pub const ALU_ALIGNMENT_MASK: usize = 3;
#[inline(always)]
unsafe fn unpack_alu(word: usize, second_word: usize, dst: *mut usize) {
@@ -859,14 +871,14 @@ cfg_if! {
let src = slice.as_ptr();
let len = slice.len();
let mut offset = 0usize;
if STRIDE_SIZE <= len {
let len_minus_stride = len - STRIDE_SIZE;
if SIMD_STRIDE_SIZE <= len {
let len_minus_stride = len - SIMD_STRIDE_SIZE;
loop {
let simd = unsafe { load16_unaligned(src.offset(offset as isize)) };
if !simd_is_ascii(simd) {
break;
}
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -887,14 +899,14 @@ cfg_if! {
let src = slice.as_ptr();
let len = slice.len();
let mut offset = 0usize;
if STRIDE_SIZE <= len {
let len_minus_stride = len - STRIDE_SIZE;
if SIMD_STRIDE_SIZE <= len {
let len_minus_stride = len - SIMD_STRIDE_SIZE;
// XXX Should we first process one stride unconditionally as unaligned to
// avoid the cost of the branchiness below if the first stride fails anyway?
// XXX Should we just use unaligned SSE2 access unconditionally? It seems that
// on Haswell, it would make sense to just use unaligned and not bother
// checking. Need to benchmark older architectures before deciding.
if ((src as usize) & ALIGNMENT_MASK) == 0 {
if ((src as usize) & SIMD_ALIGNMENT_MASK) == 0 {
loop {
let simd = unsafe { load16_aligned(src.offset(offset as isize)) };
let mask = mask_ascii(simd);
@@ -903,7 +915,7 @@ cfg_if! {
let non_ascii = unsafe { *src.offset(offset as isize) };
return Some((non_ascii, offset));
}
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -917,7 +929,7 @@ cfg_if! {
let non_ascii = unsafe { *src.offset(offset as isize) };
return Some((non_ascii, offset));
}
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -933,6 +945,93 @@ cfg_if! {
}
None
}
} else {
#[inline(always)]
fn find_non_ascii(word: usize, second_word: usize) -> Option<usize> {
let word_masked = word & ASCII_MASK;
let second_masked = second_word & ASCII_MASK;
if (word_masked | second_masked) == 0 {
return None;
}
if word_masked != 0 {
let zeros = count_zeros(word_masked);
// `zeros` now contains 7 (for the seven bits of non-ASCII)
// plus 8 times the number of ASCII in text order before the
// non-ASCII byte in the little-endian case or 8 times the number of ASCII in
// text order before the non-ASCII byte in the big-endian case.
let num_ascii = (zeros >> 3) as usize;
return Some(num_ascii);
}
let zeros = count_zeros(second_masked);
// `zeros` now contains 7 (for the seven bits of non-ASCII)
// plus 8 times the number of ASCII in text order before the
// non-ASCII byte in the little-endian case or 8 times the number of ASCII in
// text order before the non-ASCII byte in the big-endian case.
let num_ascii = (zeros >> 3) as usize;
Some(ALU_ALIGNMENT + num_ascii)
}
#[inline(always)]
unsafe fn validate_ascii_stride(src: *const usize) -> Option<usize> {
let word = *src;
let second_word = *(src.offset(1));
find_non_ascii(word, second_word)
}
#[inline(always)]
pub fn validate_ascii(slice: &[u8]) -> Option<(u8, usize)> {
let src = slice.as_ptr();
let len = slice.len();
let mut offset = 0usize;
let mut until_alignment = (ALU_ALIGNMENT - ((src as usize) & ALU_ALIGNMENT_MASK)) & ALU_ALIGNMENT_MASK;
if until_alignment + ALU_STRIDE_SIZE <= len {
while until_alignment != 0 {
let code_unit = slice[offset];
if code_unit > 127 {
return Some((code_unit, offset));
}
offset += 1;
until_alignment -= 1;
}
let len_minus_stride = len - ALU_STRIDE_SIZE;
loop {
let ptr = unsafe { src.offset(offset as isize) as *const usize };
if let Some(num_ascii) = unsafe { validate_ascii_stride(ptr) } {
offset += num_ascii;
return Some((unsafe { *(src.offset(offset as isize)) }, offset));
}
offset += ALU_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
}
}
while offset < len {
let code_unit = slice[offset];
if code_unit > 127 {
return Some((code_unit, offset));
}
offset += 1;
}
None
}
}
}
cfg_if! {
if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))] {
} else if #[cfg(all(feature = "simd-accel", target_endian = "little", target_feature = "neon"))] {
// Even with NEON enabled, we use the ALU path for ASCII validation, because testing
// on Exynos 5 indicated that using NEON isn't worthwhile where there are only
// vector reads without vector writes.
pub const ALU_STRIDE_SIZE: usize = 8;
pub const ALU_ALIGNMENT: usize = 4;
pub const ALU_ALIGNMENT_MASK: usize = 3;
} else {
#[inline(always)]
unsafe fn unpack_latin1_stride_alu(src: *const usize, dst: *mut usize) {
@@ -975,11 +1074,6 @@ cfg_if! {
true
}
basic_latin_alu!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_alu);
basic_latin_alu!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_alu);
latin1_alu!(unpack_latin1, u8, u16, unpack_latin1_stride_alu);
latin1_alu!(pack_latin1, u16, u8, pack_latin1_stride_alu);
#[inline(always)]
unsafe fn ascii_to_ascii_stride(src: *const usize, dst: *mut usize) -> Option<usize> {
let word = *src;
@@ -989,82 +1083,15 @@ cfg_if! {
find_non_ascii(word, second_word)
}
#[inline(always)]
unsafe fn validate_ascii_stride(src: *const usize) -> Option<usize> {
let word = *src;
let second_word = *(src.offset(1));
find_non_ascii(word, second_word)
}
#[inline(always)]
fn find_non_ascii(word: usize, second_word: usize) -> Option<usize> {
let word_masked = word & ASCII_MASK;
let second_masked = second_word & ASCII_MASK;
if (word_masked | second_masked) == 0 {
return None;
}
if word_masked != 0 {
let zeros = count_zeros(word_masked);
// `zeros` now contains 7 (for the seven bits of non-ASCII)
// plus 8 times the number of ASCII in text order before the
// non-ASCII byte in the little-endian case or 8 times the number of ASCII in
// text order before the non-ASCII byte in the big-endian case.
let num_ascii = (zeros >> 3) as usize;
return Some(num_ascii);
}
let zeros = count_zeros(second_masked);
// `zeros` now contains 7 (for the seven bits of non-ASCII)
// plus 8 times the number of ASCII in text order before the
// non-ASCII byte in the little-endian case or 8 times the number of ASCII in
// text order before the non-ASCII byte in the big-endian case.
let num_ascii = (zeros >> 3) as usize;
Some(ALIGNMENT + num_ascii)
}
#[cfg(not(all(feature = "simd-accel", target_endian = "little", target_arch = "aarch64")))]
basic_latin_alu!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_alu);
basic_latin_alu!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_alu);
latin1_alu!(unpack_latin1, u8, u16, unpack_latin1_stride_alu);
latin1_alu!(pack_latin1, u16, u8, pack_latin1_stride_alu);
ascii_alu!(ascii_to_ascii, u8, u8, ascii_to_ascii_stride);
#[inline(always)]
pub fn validate_ascii(slice: &[u8]) -> Option<(u8, usize)> {
let src = slice.as_ptr();
let len = slice.len();
let mut offset = 0usize;
let mut until_alignment = (ALIGNMENT - ((src as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
if until_alignment + STRIDE_SIZE <= len {
while until_alignment != 0 {
let code_unit = slice[offset];
if code_unit > 127 {
return Some((code_unit, offset));
}
offset += 1;
until_alignment -= 1;
}
let len_minus_stride = len - STRIDE_SIZE;
loop {
let ptr = unsafe { src.offset(offset as isize) as *const usize };
if let Some(num_ascii) = unsafe { validate_ascii_stride(ptr) } {
offset += num_ascii;
return Some((unsafe { *(src.offset(offset as isize)) }, offset));
}
offset += STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
}
}
while offset < len {
let code_unit = slice[offset];
if code_unit > 127 {
return Some((code_unit, offset));
}
offset += 1;
}
None
}
}
}
pub fn ascii_valid_up_to(bytes: &[u8]) -> usize {
match validate_ascii(bytes) {
None => bytes.len(),
src/lib.rs
+6 -6
View File
@@ -643,7 +643,7 @@
extern crate cfg_if;
#[cfg(all(feature = "simd-accel",
any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))]
any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"), all(target_endian = "little", target_feature = "neon"))))]
extern crate simd;
#[cfg(feature = "serde")]
@@ -661,7 +661,7 @@ extern crate serde_json;
mod macros;
#[cfg(all(feature = "simd-accel",
any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))]
any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"), all(target_endian = "little", target_feature = "neon"))))]
mod simd_funcs;
#[cfg(any(all(feature = "simd-accel", target_feature = "sse2"),
@@ -3482,11 +3482,11 @@ impl Decoder {
let (result, read, written, replaced) = self.decode_to_utf8(src, bytes, last);
let len = bytes.len();
let mut trail = written;
// Non-UTF-8 ASCII-compatible decoders may write up to `STRIDE_SIZE`
// Non-UTF-8 ASCII-compatible decoders may write up to `MAX_STRIDE_SIZE`
// bytes of trailing garbage. No need to optimize non-ASCII-compatible
// encodings to avoid overwriting here.
if self.encoding != UTF_8 {
let max = std::cmp::min(len, trail + ascii::STRIDE_SIZE);
let max = std::cmp::min(len, trail + ascii::MAX_STRIDE_SIZE);
while trail < max {
bytes[trail] = 0;
trail += 1;
@@ -3572,11 +3572,11 @@ impl Decoder {
let (result, read, written) = self.decode_to_utf8_without_replacement(src, bytes, last);
let len = bytes.len();
let mut trail = written;
// Non-UTF-8 ASCII-compatible decoders may write up to `STRIDE_SIZE`
// Non-UTF-8 ASCII-compatible decoders may write up to `MAX_STRIDE_SIZE`
// bytes of trailing garbage. No need to optimize non-ASCII-compatible
// encodings to avoid overwriting here.
if self.encoding != UTF_8 {
let max = std::cmp::min(len, trail + ascii::STRIDE_SIZE);
let max = std::cmp::min(len, trail + ascii::MAX_STRIDE_SIZE);
while trail < max {
bytes[trail] = 0;
trail += 1;
src/mem.rs
+52 -52
View File
@@ -75,16 +75,16 @@ macro_rules! by_unit_check_alu {
let mut accu = 0usize;
let unit_size = ::std::mem::size_of::<$unit>();
let len = buffer.len();
if len >= ALIGNMENT / unit_size {
if len >= ALU_ALIGNMENT / unit_size {
// The most common reason to return `false` is for the first code
// unit to fail the test, so check that first.
if buffer[0] >= $bound {
return false;
}
let src = buffer.as_ptr();
let mut until_alignment = ((ALIGNMENT - ((src as usize) & ALIGNMENT_MASK)) &
ALIGNMENT_MASK) / unit_size;
if until_alignment + ALIGNMENT / unit_size <= len {
let mut until_alignment = ((ALU_ALIGNMENT - ((src as usize) & ALU_ALIGNMENT_MASK)) &
ALU_ALIGNMENT_MASK) / unit_size;
if until_alignment + ALU_ALIGNMENT / unit_size <= len {
if until_alignment != 0 {
accu |= buffer[offset] as usize;
offset += 1;
@@ -98,18 +98,18 @@ macro_rules! by_unit_check_alu {
return false;
}
}
let len_minus_stride = len - ALIGNMENT / unit_size;
if offset + (4 * (ALIGNMENT / unit_size)) <= len {
let len_minus_unroll = len - (4 * (ALIGNMENT / unit_size));
let len_minus_stride = len - ALU_ALIGNMENT / unit_size;
if offset + (4 * (ALU_ALIGNMENT / unit_size)) <= len {
let len_minus_unroll = len - (4 * (ALU_ALIGNMENT / unit_size));
loop {
let unroll_accu = unsafe { *(src.offset(offset as isize) as *const usize) } |
unsafe { *(src.offset((offset + (ALIGNMENT / unit_size)) as isize) as *const usize) } |
unsafe { *(src.offset((offset + (2 * (ALIGNMENT / unit_size))) as isize) as *const usize) } |
unsafe { *(src.offset((offset + (3 * (ALIGNMENT / unit_size))) as isize) as *const usize) };
unsafe { *(src.offset((offset + (ALU_ALIGNMENT / unit_size)) as isize) as *const usize) } |
unsafe { *(src.offset((offset + (2 * (ALU_ALIGNMENT / unit_size))) as isize) as *const usize) } |
unsafe { *(src.offset((offset + (3 * (ALU_ALIGNMENT / unit_size))) as isize) as *const usize) };
if unroll_accu & $mask != 0 {
return false;
}
offset += 4 * (ALIGNMENT / unit_size);
offset += 4 * (ALU_ALIGNMENT / unit_size);
if offset > len_minus_unroll {
break;
}
@@ -117,7 +117,7 @@ macro_rules! by_unit_check_alu {
}
while offset <= len_minus_stride {
accu |= unsafe { *(src.offset(offset as isize) as *const usize) };
offset += ALIGNMENT / unit_size;
offset += ALU_ALIGNMENT / unit_size;
}
}
}
@@ -142,7 +142,7 @@ macro_rules! by_unit_check_simd {
let mut accu = 0usize;
let unit_size = ::std::mem::size_of::<$unit>();
let len = buffer.len();
if len >= STRIDE_SIZE / unit_size {
if len >= SIMD_STRIDE_SIZE / unit_size {
// The most common reason to return `false` is for the first code
// unit to fail the test, so check that first.
if buffer[0] >= $bound {
@@ -151,7 +151,7 @@ macro_rules! by_unit_check_simd {
let src = buffer.as_ptr();
let mut until_alignment = ((SIMD_ALIGNMENT - ((src as usize) & SIMD_ALIGNMENT_MASK)) &
SIMD_ALIGNMENT_MASK) / unit_size;
if until_alignment + STRIDE_SIZE / unit_size <= len {
if until_alignment + SIMD_STRIDE_SIZE / unit_size <= len {
if until_alignment != 0 {
accu |= buffer[offset] as usize;
offset += 1;
@@ -165,18 +165,18 @@ macro_rules! by_unit_check_simd {
return false;
}
}
let len_minus_stride = len - STRIDE_SIZE / unit_size;
if offset + (4 * (STRIDE_SIZE / unit_size)) <= len {
let len_minus_unroll = len - (4 * (STRIDE_SIZE / unit_size));
let len_minus_stride = len - SIMD_STRIDE_SIZE / unit_size;
if offset + (4 * (SIMD_STRIDE_SIZE / unit_size)) <= len {
let len_minus_unroll = len - (4 * (SIMD_STRIDE_SIZE / unit_size));
loop {
let unroll_accu = unsafe { *(src.offset(offset as isize) as *const $simd_ty) } |
unsafe { *(src.offset((offset + (STRIDE_SIZE / unit_size)) as isize) as *const $simd_ty) } |
unsafe { *(src.offset((offset + (2 * (STRIDE_SIZE / unit_size))) as isize) as *const $simd_ty) } |
unsafe { *(src.offset((offset + (3 * (STRIDE_SIZE / unit_size))) as isize) as *const $simd_ty) };
unsafe { *(src.offset((offset + (SIMD_STRIDE_SIZE / unit_size)) as isize) as *const $simd_ty) } |
unsafe { *(src.offset((offset + (2 * (SIMD_STRIDE_SIZE / unit_size))) as isize) as *const $simd_ty) } |
unsafe { *(src.offset((offset + (3 * (SIMD_STRIDE_SIZE / unit_size))) as isize) as *const $simd_ty) };
if !$func(unroll_accu) {
return false;
}
offset += 4 * (STRIDE_SIZE / unit_size);
offset += 4 * (SIMD_STRIDE_SIZE / unit_size);
if offset > len_minus_unroll {
break;
}
@@ -185,7 +185,7 @@ macro_rules! by_unit_check_simd {
let mut simd_accu = $splat;
while offset <= len_minus_stride {
simd_accu = simd_accu | unsafe { *(src.offset(offset as isize) as *const $simd_ty) };
offset += STRIDE_SIZE / unit_size;
offset += SIMD_STRIDE_SIZE / unit_size;
}
if !$func(simd_accu) {
return false;
@@ -200,7 +200,7 @@ macro_rules! by_unit_check_simd {
}
cfg_if!{
if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))] {
if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"), all(target_endian = "little", target_feature = "neon"))))] {
use simd_funcs::*;
use simd::u8x16;
use simd::u16x8;
@@ -227,13 +227,13 @@ cfg_if!{
let until_alignment = ((SIMD_ALIGNMENT - ((unsafe { src.offset(offset as isize) } as usize) & SIMD_ALIGNMENT_MASK)) &
SIMD_ALIGNMENT_MASK) / unit_size;
if until_alignment == 0 {
if offset + STRIDE_SIZE / unit_size > len {
if offset + SIMD_STRIDE_SIZE / unit_size > len {
break;
}
} else {
let offset_plus_until_alignment = offset + until_alignment;
let offset_plus_until_alignment_plus_one = offset_plus_until_alignment + 1;
if offset_plus_until_alignment_plus_one + STRIDE_SIZE / unit_size > len {
if offset_plus_until_alignment_plus_one + SIMD_STRIDE_SIZE / unit_size > len {
break;
}
let (up_to, last_valid_low) = utf16_valid_up_to_alu(&buffer[offset..offset_plus_until_alignment_plus_one]);
@@ -246,16 +246,16 @@ cfg_if!{
}
offset = offset_plus_until_alignment;
}
let len_minus_stride = len - STRIDE_SIZE / unit_size;
let len_minus_stride = len - SIMD_STRIDE_SIZE / unit_size;
'inner: loop {
let offset_plus_stride = offset + STRIDE_SIZE / unit_size;
let offset_plus_stride = offset + SIMD_STRIDE_SIZE / unit_size;
if contains_surrogates(unsafe { *(src.offset(offset as isize) as *const u16x8) }) {
if offset_plus_stride == len {
break 'outer;
}
let offset_plus_stride_plus_one = offset_plus_stride + 1;
let (up_to, last_valid_low) = utf16_valid_up_to_alu(&buffer[offset..offset_plus_stride_plus_one]);
if up_to < STRIDE_SIZE / unit_size {
if up_to < SIMD_STRIDE_SIZE / unit_size {
return offset + up_to;
}
if last_valid_low {
@@ -331,17 +331,17 @@ fn utf16_valid_up_to_alu(buffer: &[u16]) -> (usize, bool) {
}
cfg_if!{
if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))] {
if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"), all(target_endian = "little", target_feature = "neon"))))] {
#[inline(always)]
fn is_str_latin1_impl(buffer: &str) -> Option<usize> {
let mut offset = 0usize;
let bytes = buffer.as_bytes();
let len = bytes.len();
if len >= STRIDE_SIZE {
if len >= SIMD_STRIDE_SIZE {
let src = bytes.as_ptr();
let mut until_alignment = (SIMD_ALIGNMENT - ((src as usize) & SIMD_ALIGNMENT_MASK)) &
SIMD_ALIGNMENT_MASK;
if until_alignment + STRIDE_SIZE <= len {
if until_alignment + SIMD_STRIDE_SIZE <= len {
while until_alignment != 0 {
if bytes[offset] > 0xC3 {
return Some(offset);
@@ -349,7 +349,7 @@ cfg_if!{
offset += 1;
until_alignment -= 1;
}
let len_minus_stride = len - STRIDE_SIZE;
let len_minus_stride = len - SIMD_STRIDE_SIZE;
loop {
if !simd_is_str_latin1(unsafe { *(src.offset(offset as isize) as *const u8x16) }) {
// TODO: Ensure this compiles away when inlined into `is_str_latin1()`.
@@ -358,7 +358,7 @@ cfg_if!{
}
return Some(offset);
}
offset += STRIDE_SIZE;
offset += SIMD_STRIDE_SIZE;
if offset > len_minus_stride {
break;
}
@@ -420,16 +420,16 @@ fn is_utf8_latin1_impl(buffer: &[u8]) -> Option<usize> {
}
cfg_if!{
if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))] {
if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"), all(target_endian = "little", target_feature = "neon"))))] {
#[inline(always)]
fn is_utf16_bidi_impl(buffer: &[u16]) -> bool {
let mut offset = 0usize;
let len = buffer.len();
if len >= STRIDE_SIZE / 2 {
if len >= SIMD_STRIDE_SIZE / 2 {
let src = buffer.as_ptr();
let mut until_alignment = ((SIMD_ALIGNMENT - ((src as usize) & SIMD_ALIGNMENT_MASK)) &
SIMD_ALIGNMENT_MASK) / 2;
if until_alignment + (STRIDE_SIZE / 2) <= len {
if until_alignment + (SIMD_STRIDE_SIZE / 2) <= len {
while until_alignment != 0 {
if is_utf16_code_unit_bidi(buffer[offset]) {
return true;
@@ -437,12 +437,12 @@ cfg_if!{
offset += 1;
until_alignment -= 1;
}
let len_minus_stride = len - (STRIDE_SIZE / 2);
let len_minus_stride = len - (SIMD_STRIDE_SIZE / 2);
loop {
if is_u16x8_bidi(unsafe { *(src.offset(offset as isize) as *const u16x8) }) {
return true;
}
offset += STRIDE_SIZE / 2;
offset += SIMD_STRIDE_SIZE / 2;
if offset > len_minus_stride {
break;
}
@@ -470,16 +470,16 @@ cfg_if!{
}
cfg_if!{
if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))] {
if #[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"), all(target_endian = "little", target_feature = "neon"))))] {
#[inline(always)]
fn check_utf16_for_latin1_and_bidi_impl(buffer: &[u16]) -> Latin1Bidi {
let mut offset = 0usize;
let len = buffer.len();
if len >= STRIDE_SIZE / 2 {
if len >= SIMD_STRIDE_SIZE / 2 {
let src = buffer.as_ptr();
let mut until_alignment = ((SIMD_ALIGNMENT - ((src as usize) & SIMD_ALIGNMENT_MASK)) &
SIMD_ALIGNMENT_MASK) / 2;
if until_alignment + (STRIDE_SIZE / 2) <= len {
if until_alignment + (SIMD_STRIDE_SIZE / 2) <= len {
while until_alignment != 0 {
if buffer[offset] > 0xFF {
// This transition isn't optimal, since the aligment is recomputing
@@ -492,7 +492,7 @@ cfg_if!{
offset += 1;
until_alignment -= 1;
}
let len_minus_stride = len - (STRIDE_SIZE / 2);
let len_minus_stride = len - (SIMD_STRIDE_SIZE / 2);
loop {
let mut s = unsafe { *(src.offset(offset as isize) as *const u16x8) };
if !simd_is_latin1(s) {
@@ -500,7 +500,7 @@ cfg_if!{
if is_u16x8_bidi(s) {
return Latin1Bidi::Bidi;
}
offset += STRIDE_SIZE / 2;
offset += SIMD_STRIDE_SIZE / 2;
if offset > len_minus_stride {
for &u in &buffer[offset..] {
if is_utf16_code_unit_bidi(u) {
@@ -512,7 +512,7 @@ cfg_if!{
s = unsafe { *(src.offset(offset as isize) as *const u16x8) };
}
}
offset += STRIDE_SIZE / 2;
offset += SIMD_STRIDE_SIZE / 2;
if offset > len_minus_stride {
break;
}
@@ -545,11 +545,11 @@ cfg_if!{
fn check_utf16_for_latin1_and_bidi_impl(buffer: &[u16]) -> Latin1Bidi {
let mut offset = 0usize;
let len = buffer.len();
if len >= ALIGNMENT / 2 {
if len >= ALU_ALIGNMENT / 2 {
let src = buffer.as_ptr();
let mut until_alignment = ((ALIGNMENT - ((src as usize) & ALIGNMENT_MASK)) &
ALIGNMENT_MASK) / 2;
if until_alignment + ALIGNMENT / 2 <= len {
let mut until_alignment = ((ALU_ALIGNMENT - ((src as usize) & ALU_ALIGNMENT_MASK)) &
ALU_ALIGNMENT_MASK) / 2;
if until_alignment + ALU_ALIGNMENT / 2 <= len {
while until_alignment != 0 {
if buffer[offset] > 0xFF {
if is_utf16_bidi_impl(&buffer[offset..]) {
@@ -560,7 +560,7 @@ cfg_if!{
offset += 1;
until_alignment -= 1;
}
let len_minus_stride = len - ALIGNMENT / 2;
let len_minus_stride = len - ALU_ALIGNMENT / 2;
loop {
if unsafe { *(src.offset(offset as isize) as *const usize) } & LATIN1_MASK != 0 {
if is_utf16_bidi_impl(&buffer[offset..]) {
@@ -568,7 +568,7 @@ cfg_if!{
}
return Latin1Bidi::LeftToRight;
}
offset += ALIGNMENT / 2;
offset += ALU_ALIGNMENT / 2;
if offset > len_minus_stride {
break;
}
@@ -1575,7 +1575,7 @@ pub fn convert_utf16_to_str(src: &[u16], dst: &mut str) -> usize {
let written = convert_utf16_to_utf8(src, bytes);
let len = bytes.len();
let mut trail = written;
let max = ::std::cmp::min(len, trail + STRIDE_SIZE);
let max = ::std::cmp::min(len, trail + MAX_STRIDE_SIZE);
while trail < max {
bytes[trail] = 0;
trail += 1;
@@ -1683,7 +1683,7 @@ pub fn convert_latin1_to_str(src: &[u8], dst: &mut str) -> usize {
let written = convert_latin1_to_utf8(src, bytes);
let len = bytes.len();
let mut trail = written;
let max = ::std::cmp::min(len, trail + STRIDE_SIZE);
let max = ::std::cmp::min(len, trail + MAX_STRIDE_SIZE);
while trail < max {
bytes[trail] = 0;
trail += 1;
src/simd_funcs.rs
+4 -2
View File
@@ -113,8 +113,10 @@ cfg_if! {
} else {
#[inline(always)]
pub fn simd_is_ascii(s: u8x16) -> bool {
let above_ascii = u8x16::splat(0x80);
s.lt(above_ascii).all()
// This optimizes better on ARM than
// the lt formulation.
let highest_ascii = u8x16::splat(0x7F);
!s.gt(highest_ascii).any()
}
}
}