openharmony/third_party_rust_encoding_rs
1
0
Fork 0
mirror of https://github.com/openharmony/third_party_rust_encoding_rs.git synced 2026-07-19 23:04:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

Make semantics of latin1_byte_compatible_up_to actually useful.

Browse Source
This commit is contained in:
Henri Sivonen
2019-09-16 14:25:25 +03:00
parent 13f2d9688b
commit c3eb642cdf
5 changed files with 83 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+8
View File
@@ -417,6 +417,14 @@ To regenerate the generated code:
## Release Notes
### 0.8.20
* Make `Decoder::latin1_byte_compatible_up_to` return `None` in more
cases to make the method actually useful. While this could be argued
to be a breaking change due to the bug fix changing semantics, it does
not break callers that had to handle the `None` case in a reasonable
way anyway.
### 0.8.19
* Removed a bunch of bound checks in `convert_str_to_utf16`.
generate-encoding-data.py
+25 -37
View File
@@ -1274,65 +1274,53 @@ write_variant_method("decode_to_utf8_raw", True, [("src", "&[u8]"),
variant_file.write('''
pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> usize {
if let Some(n) = match *self {
VariantDecoder::SingleByte(ref v) => Some(v.latin1_byte_compatible_up_to(buffer)),
pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> Option<usize> {
match *self {
VariantDecoder::SingleByte(ref v) => {
return Some(v.latin1_byte_compatible_up_to(buffer));
}
VariantDecoder::Utf8(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::Gb18030(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::Big5(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::EucJp(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::Iso2022Jp(ref v) => {
if v.in_neutral_state() {
Some(Encoding::iso_2022_jp_ascii_valid_up_to(buffer))
} else {
Some(0)
return Some(Encoding::iso_2022_jp_ascii_valid_up_to(buffer));
}
return None;
}
VariantDecoder::ShiftJis(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::EucKr(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::UserDefined(_) => None,
VariantDecoder::Replacement(_) | VariantDecoder::Utf16(_) => Some(0),
} {
n
} else {
Encoding::ascii_valid_up_to(buffer)
}
VariantDecoder::UserDefined(_) => {}
VariantDecoder::Replacement(_) | VariantDecoder::Utf16(_) => {
return None;
}
};
Some(Encoding::ascii_valid_up_to(buffer))
}
}
src/lib.rs
+19 -28
View File
@@ -4212,15 +4212,13 @@ impl Decoder {
/// Checks for compatibility with storing Unicode scalar values as unsigned
/// bytes taking into account the state of the decoder.
///
/// Returns `None` if the decoder is still waiting for (parts of) a potential
/// BOM.
/// Returns `None` if the decoder is not in a neutral state, including waiting
/// for the BOM or if the encoding is never Latin-byte-compatible.
///
/// Otherwise returns the index of the first byte whose unsigned value doesn't
/// directly correspond to the decoded Unicode scalar value, or the length
/// of the input if all bytes in the input decode directly to scalar values
/// corresponding to the unsigned byte values. (This is always returns zero
/// for UTF-16LE, UTF-16BE, and replacement. It's also zero when a multibyte
/// decoder is in the middle of a multibyte sequence.)
/// corresponding to the unsigned byte values.
///
/// Does not change the state of the decoder.
///
@@ -4230,7 +4228,9 @@ impl Decoder {
/// Available via the C wrapper.
pub fn latin1_byte_compatible_up_to(&self, bytes: &[u8]) -> Option<usize> {
match self.life_cycle {
DecoderLifeCycle::Converting => Some(self.variant.latin1_byte_compatible_up_to(bytes)),
DecoderLifeCycle::Converting => {
return self.variant.latin1_byte_compatible_up_to(bytes);
}
DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."),
_ => None,
}
@@ -5760,13 +5760,10 @@ mod tests {
.unwrap(),
1
);
assert_eq!(
REPLACEMENT
.new_decoder_without_bom_handling()
.latin1_byte_compatible_up_to(buffer)
.unwrap(),
0
);
assert!(REPLACEMENT
.new_decoder_without_bom_handling()
.latin1_byte_compatible_up_to(buffer)
.is_none());
assert_eq!(
SHIFT_JIS
.new_decoder_without_bom_handling()
@@ -5781,20 +5778,14 @@ mod tests {
.unwrap(),
1
);
assert_eq!(
UTF_16BE
.new_decoder_without_bom_handling()
.latin1_byte_compatible_up_to(buffer)
.unwrap(),
0
);
assert_eq!(
UTF_16LE
.new_decoder_without_bom_handling()
.latin1_byte_compatible_up_to(buffer)
.unwrap(),
0
);
assert!(UTF_16BE
.new_decoder_without_bom_handling()
.latin1_byte_compatible_up_to(buffer)
.is_none());
assert!(UTF_16LE
.new_decoder_without_bom_handling()
.latin1_byte_compatible_up_to(buffer)
.is_none());
assert_eq!(
ISO_2022_JP
.new_decoder_without_bom_handling()
@@ -6019,6 +6010,6 @@ mod tests {
let _ = decoder.decode_to_utf16(b"\xBB\xBF", &mut output, false);
assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), Some(1));
let _ = decoder.decode_to_utf16(b"\xEF", &mut output, false);
assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), Some(0));
assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), None);
}
}
src/mem.rs
+3 -3
View File
@@ -2468,17 +2468,17 @@ mod tests {
0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0x2603u16,
0xD83Du16, 0xDCA9u16, 0x00B6u16,
];
assert_eq!(utf16_valid_up_to(&valid[..]), 16);;
assert_eq!(utf16_valid_up_to(&valid[..]), 16);
let lone_high = vec![
0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
0x2603u16, 0xD83Du16, 0x00B6u16,
];
assert_eq!(utf16_valid_up_to(&lone_high[..]), 14);;
assert_eq!(utf16_valid_up_to(&lone_high[..]), 14);
let lone_low = vec![
0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
0x2603u16, 0xDCA9u16, 0x00B6u16,
];
assert_eq!(utf16_valid_up_to(&lone_low[..]), 14);;
assert_eq!(utf16_valid_up_to(&lone_low[..]), 14);
let lone_high_at_end = vec![
0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
0x2603u16, 0x00B6u16, 0xD83Du16,
src/variant.rs
+28 -38
View File
@@ -159,65 +159,53 @@ impl VariantDecoder {
}
}
pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> usize {
if let Some(n) = match *self {
VariantDecoder::SingleByte(ref v) => Some(v.latin1_byte_compatible_up_to(buffer)),
pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> Option<usize> {
match *self {
VariantDecoder::SingleByte(ref v) => {
return Some(v.latin1_byte_compatible_up_to(buffer));
}
VariantDecoder::Utf8(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::Gb18030(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::Big5(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::EucJp(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::Iso2022Jp(ref v) => {
if v.in_neutral_state() {
Some(Encoding::iso_2022_jp_ascii_valid_up_to(buffer))
} else {
Some(0)
return Some(Encoding::iso_2022_jp_ascii_valid_up_to(buffer));
}
return None;
}
VariantDecoder::ShiftJis(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::EucKr(ref v) => {
if v.in_neutral_state() {
None
} else {
Some(0)
if !v.in_neutral_state() {
return None;
}
}
VariantDecoder::UserDefined(_) => None,
VariantDecoder::Replacement(_) | VariantDecoder::Utf16(_) => Some(0),
} {
n
} else {
Encoding::ascii_valid_up_to(buffer)
}
VariantDecoder::UserDefined(_) => {}
VariantDecoder::Replacement(_) | VariantDecoder::Utf16(_) => {
return None;
}
};
Some(Encoding::ascii_valid_up_to(buffer))
}
}
@@ -397,7 +385,9 @@ impl VariantEncoding {
VariantEncoding::ShiftJis => ShiftJisEncoder::new(encoding),
VariantEncoding::EucKr => EucKrEncoder::new(encoding),
VariantEncoding::UserDefined => UserDefinedEncoder::new(encoding),
VariantEncoding::Utf16Be | VariantEncoding::Replacement | VariantEncoding::Utf16Le => unreachable!(),
VariantEncoding::Utf16Be | VariantEncoding::Replacement | VariantEncoding::Utf16Le => {
unreachable!()
}
}
}