Ensure is_str_latin1_impl() reports an offset at code point boundary.

This commit is contained in:
Henri Sivonen
2018-01-17 15:20:06 +02:00
parent c5e088f3d2
commit 2d87c50cd5
+4 -1
View File
@@ -352,6 +352,10 @@ cfg_if!{
let len_minus_stride = len - STRIDE_SIZE;
loop {
if !simd_is_str_latin1(unsafe { *(src.offset(offset as isize) as *const u8x16) }) {
// TODO: Ensure this compiles away when inlined into `is_str_latin1()`.
while bytes[offset] & 0xC0 == 0x80 {
offset += 1;
}
return Some(offset);
}
offset += STRIDE_SIZE;
@@ -2473,7 +2477,6 @@ mod tests {
assert!(!is_utf8_bidi(b"\xD5\xBF\x61"));
assert!(!is_utf8_bidi(b"\xD6\x80\x61"));
assert!(!is_utf8_bidi(b"abc"));
assert!(!is_utf8_bidi(b"\xCB\xBB"));
assert!(is_utf8_bidi(b"\xD5\xBF\xC2"));
assert!(is_utf8_bidi(b"\xD6\x80\xC2"));
assert!(is_utf8_bidi(b"ab\xC2"));