diff --git a/README.md b/README.md index f9a9ecb..3f6a9ec 100644 --- a/README.md +++ b/README.md @@ -417,6 +417,14 @@ To regenerate the generated code: ## Release Notes +### 0.8.20 + +* Make `Decoder::latin1_byte_compatible_up_to` return `None` in more + cases to make the method actually useful. While this could be argued + to be a breaking change due to the bug fix changing semantics, it does + not break callers that had to handle the `None` case in a reasonable + way anyway. + ### 0.8.19 * Removed a bunch of bound checks in `convert_str_to_utf16`. diff --git a/generate-encoding-data.py b/generate-encoding-data.py index dd6556c..cecb88e 100644 --- a/generate-encoding-data.py +++ b/generate-encoding-data.py @@ -1274,65 +1274,53 @@ write_variant_method("decode_to_utf8_raw", True, [("src", "&[u8]"), variant_file.write(''' - pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> usize { - if let Some(n) = match *self { - VariantDecoder::SingleByte(ref v) => Some(v.latin1_byte_compatible_up_to(buffer)), + pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> Option { + match *self { + VariantDecoder::SingleByte(ref v) => { + return Some(v.latin1_byte_compatible_up_to(buffer)); + } VariantDecoder::Utf8(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::Gb18030(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::Big5(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::EucJp(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::Iso2022Jp(ref v) => { if v.in_neutral_state() { - Some(Encoding::iso_2022_jp_ascii_valid_up_to(buffer)) - } else { - Some(0) + return Some(Encoding::iso_2022_jp_ascii_valid_up_to(buffer)); } + return None; } VariantDecoder::ShiftJis(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::EucKr(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } - VariantDecoder::UserDefined(_) => None, - VariantDecoder::Replacement(_) | VariantDecoder::Utf16(_) => Some(0), - } { - n - } else { - Encoding::ascii_valid_up_to(buffer) - } + VariantDecoder::UserDefined(_) => {} + VariantDecoder::Replacement(_) | VariantDecoder::Utf16(_) => { + return None; + } + }; + Some(Encoding::ascii_valid_up_to(buffer)) } } diff --git a/src/lib.rs b/src/lib.rs index 17ac25d..eda66cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4212,15 +4212,13 @@ impl Decoder { /// Checks for compatibility with storing Unicode scalar values as unsigned /// bytes taking into account the state of the decoder. /// - /// Returns `None` if the decoder is still waiting for (parts of) a potential - /// BOM. + /// Returns `None` if the decoder is not in a neutral state, including waiting + /// for the BOM or if the encoding is never Latin-byte-compatible. /// /// Otherwise returns the index of the first byte whose unsigned value doesn't /// directly correspond to the decoded Unicode scalar value, or the length /// of the input if all bytes in the input decode directly to scalar values - /// corresponding to the unsigned byte values. (This is always returns zero - /// for UTF-16LE, UTF-16BE, and replacement. It's also zero when a multibyte - /// decoder is in the middle of a multibyte sequence.) + /// corresponding to the unsigned byte values. /// /// Does not change the state of the decoder. /// @@ -4230,7 +4228,9 @@ impl Decoder { /// Available via the C wrapper. pub fn latin1_byte_compatible_up_to(&self, bytes: &[u8]) -> Option { match self.life_cycle { - DecoderLifeCycle::Converting => Some(self.variant.latin1_byte_compatible_up_to(bytes)), + DecoderLifeCycle::Converting => { + return self.variant.latin1_byte_compatible_up_to(bytes); + } DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."), _ => None, } @@ -5760,13 +5760,10 @@ mod tests { .unwrap(), 1 ); - assert_eq!( - REPLACEMENT - .new_decoder_without_bom_handling() - .latin1_byte_compatible_up_to(buffer) - .unwrap(), - 0 - ); + assert!(REPLACEMENT + .new_decoder_without_bom_handling() + .latin1_byte_compatible_up_to(buffer) + .is_none()); assert_eq!( SHIFT_JIS .new_decoder_without_bom_handling() @@ -5781,20 +5778,14 @@ mod tests { .unwrap(), 1 ); - assert_eq!( - UTF_16BE - .new_decoder_without_bom_handling() - .latin1_byte_compatible_up_to(buffer) - .unwrap(), - 0 - ); - assert_eq!( - UTF_16LE - .new_decoder_without_bom_handling() - .latin1_byte_compatible_up_to(buffer) - .unwrap(), - 0 - ); + assert!(UTF_16BE + .new_decoder_without_bom_handling() + .latin1_byte_compatible_up_to(buffer) + .is_none()); + assert!(UTF_16LE + .new_decoder_without_bom_handling() + .latin1_byte_compatible_up_to(buffer) + .is_none()); assert_eq!( ISO_2022_JP .new_decoder_without_bom_handling() @@ -6019,6 +6010,6 @@ mod tests { let _ = decoder.decode_to_utf16(b"\xBB\xBF", &mut output, false); assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), Some(1)); let _ = decoder.decode_to_utf16(b"\xEF", &mut output, false); - assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), Some(0)); + assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), None); } } diff --git a/src/mem.rs b/src/mem.rs index ab8f10b..8c82704 100644 --- a/src/mem.rs +++ b/src/mem.rs @@ -2468,17 +2468,17 @@ mod tests { 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0x2603u16, 0xD83Du16, 0xDCA9u16, 0x00B6u16, ]; - assert_eq!(utf16_valid_up_to(&valid[..]), 16);; + assert_eq!(utf16_valid_up_to(&valid[..]), 16); let lone_high = vec![ 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0x2603u16, 0xD83Du16, 0x00B6u16, ]; - assert_eq!(utf16_valid_up_to(&lone_high[..]), 14);; + assert_eq!(utf16_valid_up_to(&lone_high[..]), 14); let lone_low = vec![ 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0x2603u16, 0xDCA9u16, 0x00B6u16, ]; - assert_eq!(utf16_valid_up_to(&lone_low[..]), 14);; + assert_eq!(utf16_valid_up_to(&lone_low[..]), 14); let lone_high_at_end = vec![ 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0x2603u16, 0x00B6u16, 0xD83Du16, diff --git a/src/variant.rs b/src/variant.rs index c70bd43..6c1dfeb 100644 --- a/src/variant.rs +++ b/src/variant.rs @@ -159,65 +159,53 @@ impl VariantDecoder { } } - pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> usize { - if let Some(n) = match *self { - VariantDecoder::SingleByte(ref v) => Some(v.latin1_byte_compatible_up_to(buffer)), + pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> Option { + match *self { + VariantDecoder::SingleByte(ref v) => { + return Some(v.latin1_byte_compatible_up_to(buffer)); + } VariantDecoder::Utf8(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::Gb18030(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::Big5(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::EucJp(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::Iso2022Jp(ref v) => { if v.in_neutral_state() { - Some(Encoding::iso_2022_jp_ascii_valid_up_to(buffer)) - } else { - Some(0) + return Some(Encoding::iso_2022_jp_ascii_valid_up_to(buffer)); } + return None; } VariantDecoder::ShiftJis(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } VariantDecoder::EucKr(ref v) => { - if v.in_neutral_state() { - None - } else { - Some(0) + if !v.in_neutral_state() { + return None; } } - VariantDecoder::UserDefined(_) => None, - VariantDecoder::Replacement(_) | VariantDecoder::Utf16(_) => Some(0), - } { - n - } else { - Encoding::ascii_valid_up_to(buffer) - } + VariantDecoder::UserDefined(_) => {} + VariantDecoder::Replacement(_) | VariantDecoder::Utf16(_) => { + return None; + } + }; + Some(Encoding::ascii_valid_up_to(buffer)) } } @@ -397,7 +385,9 @@ impl VariantEncoding { VariantEncoding::ShiftJis => ShiftJisEncoder::new(encoding), VariantEncoding::EucKr => EucKrEncoder::new(encoding), VariantEncoding::UserDefined => UserDefinedEncoder::new(encoding), - VariantEncoding::Utf16Be | VariantEncoding::Replacement | VariantEncoding::Utf16Le => unreachable!(), + VariantEncoding::Utf16Be | VariantEncoding::Replacement | VariantEncoding::Utf16Le => { + unreachable!() + } } }