mirror of
https://github.com/openharmony/third_party_rust_encoding_rs.git
synced 2026-07-21 02:05:23 -04:00
Add variants of conversions from UTF-16 and Latin1 to UTF-8 that can deal
with insufficient output space. The idea is to use these first with an allocation rounded up to jemalloc bucket size and only if there output doesn't fit allocating for the worst case.
This commit is contained in:
Henri Sivonen
+169
-54
@@ -1551,6 +1551,33 @@ pub fn convert_str_to_utf16(src: &str, dst: &mut [u16]) -> usize {
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts potentially-invalid UTF-16 to valid UTF-8 with errors replaced
|
||||
/// with the REPLACEMENT CHARACTER with potentially insufficient output
|
||||
/// space.
|
||||
///
|
||||
/// Returns the number of code units read and the number of bytes written.
|
||||
///
|
||||
/// Not all code units are read if there isn't enough output space.
|
||||
///
|
||||
/// Note that this method isn't designed for general streamability but for
|
||||
/// not allocating memory for the worst case up front. Specifically,
|
||||
/// if the input starts with or ends with an unpaired surrogate, those are
|
||||
/// replaced with the REPLACEMENT CHARACTER.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Note that this function may write garbage beyond the number of bytes
|
||||
/// indicated by the return value, so using a `&mut str` interpreted as
|
||||
/// `&mut [u8]` as the destination is not safe. If you want to convert into
|
||||
/// a `&mut str`, use `convert_utf16_to_str()` instead of this function.
|
||||
#[inline]
|
||||
pub fn convert_utf16_to_utf8_partial(src: &[u16], dst: &mut [u8]) -> (usize, usize) {
|
||||
let mut encoder = Utf8Encoder;
|
||||
let (result, read, written) = encoder.encode_from_utf16_raw(src, dst, true);
|
||||
debug_assert!(result == EncoderResult::OutputFull || read == src.len());
|
||||
(read, written)
|
||||
}
|
||||
|
||||
/// Converts potentially-invalid UTF-16 to valid UTF-8 with errors replaced
|
||||
/// with the REPLACEMENT CHARACTER.
|
||||
///
|
||||
@@ -1572,12 +1599,42 @@ pub fn convert_str_to_utf16(src: &str, dst: &mut [u16]) -> usize {
|
||||
#[inline]
|
||||
pub fn convert_utf16_to_utf8(src: &[u16], dst: &mut [u8]) -> usize {
|
||||
assert!(dst.len() >= src.len() * 3 + 1);
|
||||
let mut encoder = Utf8Encoder;
|
||||
let (result, _, written) = encoder.encode_from_utf16_raw(src, dst, true);
|
||||
debug_assert!(result == EncoderResult::InputEmpty);
|
||||
let (read, written) = convert_utf16_to_utf8_partial(src, dst);
|
||||
debug_assert_eq!(read, src.len());
|
||||
written
|
||||
}
|
||||
|
||||
/// Converts potentially-invalid UTF-16 to valid UTF-8 with errors replaced
|
||||
/// with the REPLACEMENT CHARACTER such that the validity of the output is
|
||||
/// signaled using the Rust type system with potentially insufficient output
|
||||
/// space.
|
||||
///
|
||||
/// Returns the number of code units read and the number of bytes written.
|
||||
///
|
||||
/// Not all code units are read if there isn't enough output space.
|
||||
///
|
||||
/// Note that this method isn't designed for general streamability but for
|
||||
/// not allocating memory for the worst case up front. Specifically,
|
||||
/// if the input starts with or ends with an unpaired surrogate, those are
|
||||
/// replaced with the REPLACEMENT CHARACTER.
|
||||
#[inline]
|
||||
pub fn convert_utf16_to_str_partial(src: &[u16], dst: &mut str) -> (usize, usize) {
|
||||
let bytes: &mut [u8] = unsafe { ::std::mem::transmute(dst) };
|
||||
let (read, written) = convert_utf16_to_utf8_partial(src, bytes);
|
||||
let len = bytes.len();
|
||||
let mut trail = written;
|
||||
let max = ::std::cmp::min(len, trail + MAX_STRIDE_SIZE);
|
||||
while trail < max {
|
||||
bytes[trail] = 0;
|
||||
trail += 1;
|
||||
}
|
||||
while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
|
||||
bytes[trail] = 0;
|
||||
trail += 1;
|
||||
}
|
||||
(read, written)
|
||||
}
|
||||
|
||||
/// Converts potentially-invalid UTF-16 to valid UTF-8 with errors replaced
|
||||
/// with the REPLACEMENT CHARACTER such that the validity of the output is
|
||||
/// signaled using the Rust type system.
|
||||
@@ -1592,19 +1649,9 @@ pub fn convert_utf16_to_utf8(src: &[u16], dst: &mut [u8]) -> usize {
|
||||
/// Panics if the destination buffer is shorter than stated above.
|
||||
#[inline]
|
||||
pub fn convert_utf16_to_str(src: &[u16], dst: &mut str) -> usize {
|
||||
let bytes: &mut [u8] = unsafe { ::std::mem::transmute(dst) };
|
||||
let written = convert_utf16_to_utf8(src, bytes);
|
||||
let len = bytes.len();
|
||||
let mut trail = written;
|
||||
let max = ::std::cmp::min(len, trail + MAX_STRIDE_SIZE);
|
||||
while trail < max {
|
||||
bytes[trail] = 0;
|
||||
trail += 1;
|
||||
}
|
||||
while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
|
||||
bytes[trail] = 0;
|
||||
trail += 1;
|
||||
}
|
||||
assert!(dst.len() >= src.len() * 3 + 1);
|
||||
let (read, written) = convert_utf16_to_str_partial(src, dst);
|
||||
debug_assert_eq!(read, src.len());
|
||||
written
|
||||
}
|
||||
|
||||
@@ -1633,6 +1680,59 @@ pub fn convert_latin1_to_utf16(src: &[u8], dst: &mut [u16]) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts bytes whose unsigned value is interpreted as Unicode code point
|
||||
/// (i.e. U+0000 to U+00FF, inclusive) to UTF-8 with potentially insufficient
|
||||
/// output space.
|
||||
///
|
||||
/// Returns the number of bytes read and the number of bytes written.
|
||||
///
|
||||
/// If the output isn't large enough, not all input is consumed.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// Note that this function may write garbage beyond the number of bytes
|
||||
/// indicated by the return value, so using a `&mut str` interpreted as
|
||||
/// `&mut [u8]` as the destination is not safe. If you want to convert into
|
||||
/// a `&mut str`, use `convert_utf16_to_str()` instead of this function.
|
||||
#[inline]
|
||||
pub fn convert_latin1_to_utf8_partial(src: &[u8], dst: &mut [u8]) -> (usize, usize) {
|
||||
let src_len = src.len();
|
||||
let src_ptr = src.as_ptr();
|
||||
let dst_ptr = dst.as_mut_ptr();
|
||||
let dst_len = dst.len();
|
||||
let mut total_read = 0usize;
|
||||
let mut total_written = 0usize;
|
||||
loop {
|
||||
// src can't advance more than dst
|
||||
let src_left = src_len - total_read;
|
||||
let dst_left = dst_len - total_written;
|
||||
let min_left = ::std::cmp::min(src_left, dst_left);
|
||||
if let Some((non_ascii, consumed)) = unsafe {
|
||||
ascii_to_ascii(
|
||||
src_ptr.offset(total_read as isize),
|
||||
dst_ptr.offset(total_written as isize),
|
||||
min_left,
|
||||
)
|
||||
} {
|
||||
total_read += consumed;
|
||||
total_written += consumed;
|
||||
if total_written.checked_add(2).unwrap() > dst_len {
|
||||
return (total_read, total_written);
|
||||
}
|
||||
|
||||
total_read += 1; // consume `non_ascii`
|
||||
|
||||
let code_point = non_ascii as u32;
|
||||
dst[total_written] = ((code_point >> 6) | 0xC0u32) as u8;
|
||||
total_written += 1;
|
||||
dst[total_written] = ((code_point as u32 & 0x3Fu32) | 0x80u32) as u8;
|
||||
total_written += 1;
|
||||
continue;
|
||||
}
|
||||
return (total_read + min_left, total_written + min_left);
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts bytes whose unsigned value is interpreted as Unicode code point
|
||||
/// (i.e. U+0000 to U+00FF, inclusive) to UTF-8.
|
||||
///
|
||||
@@ -1657,33 +1757,35 @@ pub fn convert_latin1_to_utf8(src: &[u8], dst: &mut [u8]) -> usize {
|
||||
dst.len() >= src.len() * 2,
|
||||
"Destination must not be shorter than the source times two."
|
||||
);
|
||||
let src_len = src.len();
|
||||
let src_ptr = src.as_ptr();
|
||||
let dst_ptr = dst.as_mut_ptr();
|
||||
let mut total_read = 0usize;
|
||||
let mut total_written = 0usize;
|
||||
loop {
|
||||
// src can't advance more than dst
|
||||
let src_left = src_len - total_read;
|
||||
if let Some((non_ascii, consumed)) = unsafe {
|
||||
ascii_to_ascii(
|
||||
src_ptr.offset(total_read as isize),
|
||||
dst_ptr.offset(total_written as isize),
|
||||
src_left,
|
||||
)
|
||||
} {
|
||||
total_read += consumed + 1;
|
||||
total_written += consumed;
|
||||
let (read, written) = convert_latin1_to_utf8_partial(src, dst);
|
||||
debug_assert_eq!(read, src.len());
|
||||
written
|
||||
}
|
||||
|
||||
let code_point = non_ascii as u32;
|
||||
dst[total_written] = ((code_point >> 6) | 0xC0u32) as u8;
|
||||
total_written += 1;
|
||||
dst[total_written] = ((code_point as u32 & 0x3Fu32) | 0x80u32) as u8;
|
||||
total_written += 1;
|
||||
continue;
|
||||
}
|
||||
return total_written + src_left;
|
||||
/// Converts bytes whose unsigned value is interpreted as Unicode code point
|
||||
/// (i.e. U+0000 to U+00FF, inclusive) to UTF-8 such that the validity of the
|
||||
/// output is signaled using the Rust type system with potentially insufficient
|
||||
/// output space.
|
||||
///
|
||||
/// Returns the number of bytes read and the number of bytes written.
|
||||
///
|
||||
/// If the output isn't large enough, not all input is consumed.
|
||||
#[inline]
|
||||
pub fn convert_latin1_to_str_partial(src: &[u8], dst: &mut str) -> (usize, usize) {
|
||||
let bytes: &mut [u8] = unsafe { ::std::mem::transmute(dst) };
|
||||
let (read, written) = convert_latin1_to_utf8_partial(src, bytes);
|
||||
let len = bytes.len();
|
||||
let mut trail = written;
|
||||
let max = ::std::cmp::min(len, trail + MAX_STRIDE_SIZE);
|
||||
while trail < max {
|
||||
bytes[trail] = 0;
|
||||
trail += 1;
|
||||
}
|
||||
while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
|
||||
bytes[trail] = 0;
|
||||
trail += 1;
|
||||
}
|
||||
(read, written)
|
||||
}
|
||||
|
||||
/// Converts bytes whose unsigned value is interpreted as Unicode code point
|
||||
@@ -1700,19 +1802,12 @@ pub fn convert_latin1_to_utf8(src: &[u8], dst: &mut [u8]) -> usize {
|
||||
/// Panics if the destination buffer is shorter than stated above.
|
||||
#[inline]
|
||||
pub fn convert_latin1_to_str(src: &[u8], dst: &mut str) -> usize {
|
||||
let bytes: &mut [u8] = unsafe { ::std::mem::transmute(dst) };
|
||||
let written = convert_latin1_to_utf8(src, bytes);
|
||||
let len = bytes.len();
|
||||
let mut trail = written;
|
||||
let max = ::std::cmp::min(len, trail + MAX_STRIDE_SIZE);
|
||||
while trail < max {
|
||||
bytes[trail] = 0;
|
||||
trail += 1;
|
||||
}
|
||||
while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
|
||||
bytes[trail] = 0;
|
||||
trail += 1;
|
||||
}
|
||||
assert!(
|
||||
dst.len() >= src.len() * 2,
|
||||
"Destination must not be shorter than the source times two."
|
||||
);
|
||||
let (read, written) = convert_latin1_to_str_partial(src, dst);
|
||||
debug_assert_eq!(read, src.len());
|
||||
written
|
||||
}
|
||||
|
||||
@@ -2114,6 +2209,18 @@ mod tests {
|
||||
assert_eq!(dst, reference);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_convert_utf16_to_utf8_partial() {
|
||||
let reference = "abcdefghijklmnopqrstu\u{1F4A9}v\u{2603}w\u{00B6}xyzz";
|
||||
let src: Vec<u16> = reference.encode_utf16().collect();
|
||||
let mut dst: Vec<u8> = Vec::with_capacity(src.len() * 3 + 1);
|
||||
dst.resize(src.len() * 3 + 1, 0);
|
||||
let (read, written) = convert_utf16_to_utf8_partial(&src[..], &mut dst[..24]);
|
||||
let len = written + convert_utf16_to_utf8(&src[read..], &mut dst[written..]);
|
||||
dst.truncate(len);
|
||||
assert_eq!(dst, reference.as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_convert_utf16_to_utf8() {
|
||||
let reference = "abcdefghijklmnopqrstu\u{1F4A9}v\u{2603}w\u{00B6}xyzz";
|
||||
@@ -2141,6 +2248,14 @@ mod tests {
|
||||
assert_eq!(dst, reference);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_convert_latin1_to_utf8_partial() {
|
||||
let mut dst = [0u8, 2];
|
||||
let (read, written) = convert_latin1_to_utf8_partial(b"a\xFF", &mut dst[..]);
|
||||
assert_eq!(read, 1);
|
||||
assert_eq!(written, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_convert_latin1_to_utf8() {
|
||||
let mut src: Vec<u8> = Vec::with_capacity(256);
|
||||
|
||||
Reference in New Issue
Block a user