diff --git a/scripts/unicode.py b/scripts/unicode.py index ccbc1c9..93e88a5 100644 --- a/scripts/unicode.py +++ b/scripts/unicode.py @@ -72,9 +72,9 @@ class UnicodeData(object): self.canon_comp = self._compute_canonical_comp() self.canon_fully_decomp, self.compat_fully_decomp = self._compute_fully_decomposed() - self.svar_decomp = {} - self.svar_fully_decomp = {} - self._load_standardized_variants() + self.cjk_compat_variants_decomp = {} + self.cjk_compat_variants_fully_decomp = {} + self._load_cjk_compat_ideograph_variants() def stats(name, table): count = sum(len(v) for v in table.values()) @@ -83,10 +83,10 @@ class UnicodeData(object): print("Decomposition table stats:") stats("Canonical decomp", self.canon_decomp) stats("Compatible decomp", self.compat_decomp) - stats("Standardized Variants", self.svar_decomp) + stats("CJK Compat Variants", self.cjk_compat_variants_decomp) stats("Canonical fully decomp", self.canon_fully_decomp) stats("Compatible fully decomp", self.compat_fully_decomp) - stats("Standardized Variants", self.svar_fully_decomp) + stats("CJK Compat Variants", self.cjk_compat_variants_fully_decomp) self.ss_leading, self.ss_trailing = self._compute_stream_safe_tables() @@ -122,38 +122,41 @@ class UnicodeData(object): if category == 'M' or 'M' in expanded_categories.get(category, []): self.general_category_mark.append(char_int) - def _load_standardized_variants(self): + def _load_cjk_compat_ideograph_variants(self): for line in self._fetch("StandardizedVariants.txt").splitlines(): strip_comments = line.split('#', 1)[0].strip() if not strip_comments: continue - pieces = strip_comments.split(';') - assert len(pieces) == 3 - - variation_sequence, description, differences = pieces[0], pieces[1].strip(), pieces[2] + variation_sequence, description, differences = strip_comments.split(';') + description = description.strip() # Don't use variations that only apply in particular shaping environments. if differences: continue # Look for entries where the description field is a codepoint name. - if description in self.name_to_char_int: - char_int = self.name_to_char_int[description] + if description not in self.name_to_char_int: + continue - assert not char_int in self.combining_classes, "Unexpected: standardized variant with a combining class" - assert not char_int in self.compat_decomp, "Unexpected: standardized variant and compatibility decomposition" - assert len(self.canon_decomp[char_int]) == 1, "Unexpected: standardized variant and non-singleton canonical decomposition" - # If we ever need to handle Hangul here, we'll need to handle it separately. - assert not (S_BASE <= char_int < S_BASE + S_COUNT) + # Only consider the CJK Compatibility Ideographs. + if not description.startswith('CJK COMPATIBILITY IDEOGRAPH-'): + continue - standardized_variant_parts = [int(c, 16) for c in variation_sequence.split()] - for c in standardized_variant_parts: - #assert not never_composes(c) TODO: Re-enable this once #67 lands. - assert not c in self.canon_decomp, "Unexpected: standardized variant is unnormalized (canon)" - assert not c in self.compat_decomp, "Unexpected: standardized variant is unnormalized (compat)" - self.svar_decomp[char_int] = standardized_variant_parts - self.svar_fully_decomp[char_int] = standardized_variant_parts + char_int = self.name_to_char_int[description] + + assert not char_int in self.combining_classes, "Unexpected: CJK compat variant with a combining class" + assert not char_int in self.compat_decomp, "Unexpected: CJK compat variant and compatibility decomposition" + assert len(self.canon_decomp[char_int]) == 1, "Unexpected: CJK compat variant and non-singleton canonical decomposition" + # If we ever need to handle Hangul here, we'll need to handle it separately. + assert not (S_BASE <= char_int < S_BASE + S_COUNT) + + cjk_compat_variant_parts = [int(c, 16) for c in variation_sequence.split()] + for c in cjk_compat_variant_parts: + assert not c in self.canon_decomp, "Unexpected: CJK compat variant is unnormalized (canon)" + assert not c in self.compat_decomp, "Unexpected: CJK compat variant is unnormalized (compat)" + self.cjk_compat_variants_decomp[char_int] = cjk_compat_variant_parts + self.cjk_compat_variants_fully_decomp[char_int] = cjk_compat_variant_parts def _load_norm_props(self): props = collections.defaultdict(list) @@ -364,8 +367,8 @@ def gen_composition_table(canon_comp, out): out.write(" }\n") out.write("}\n") -def gen_decomposition_tables(canon_decomp, compat_decomp, svar_decomp, out): - tables = [(canon_decomp, 'canonical'), (compat_decomp, 'compatibility'), (svar_decomp, 'svar')] +def gen_decomposition_tables(canon_decomp, compat_decomp, cjk_compat_variants_decomp, out): + tables = [(canon_decomp, 'canonical'), (compat_decomp, 'compatibility'), (cjk_compat_variants_decomp, 'cjk_compat_variants')] for table, name in tables: gen_mph_data(name + '_decomposed', table, "(u32, &'static [char])", lambda k: "(0x{:x}, &[{}])".format(k, @@ -535,7 +538,7 @@ if __name__ == '__main__': gen_composition_table(data.canon_comp, out) out.write("\n") - gen_decomposition_tables(data.canon_fully_decomp, data.compat_fully_decomp, data.svar_fully_decomp, out) + gen_decomposition_tables(data.canon_fully_decomp, data.compat_fully_decomp, data.cjk_compat_variants_fully_decomp, out) gen_combining_mark(data.general_category_mark, out) out.write("\n") diff --git a/src/lib.rs b/src/lib.rs index 1994f80..cb623ba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -86,7 +86,7 @@ mod test; /// Methods for composing and decomposing characters. pub mod char { pub use crate::normalize::{ - compose, decompose_canonical, decompose_compatible, decompose_svar, + compose, decompose_canonical, decompose_cjk_compat_variants, decompose_compatible, }; pub use crate::lookups::{canonical_combining_class, is_combining_mark}; @@ -112,17 +112,17 @@ pub trait UnicodeNormalization> { /// (compatibility decomposition followed by canonical composition). fn nfkc(self) -> Recompositions; - /// A transformation which replaces codepoints with normal forms using - /// Standardized Variation Sequences. This is not part of the canonical - /// or compatibility decomposition algorithms, but performing it before - /// those algorithms produces normalized output which better preserves - /// the intent of the original text. + /// A transformation which replaces CJK Compatibility Ideograph codepoints + /// with normal forms using Standardized Variation Sequences. This is not + /// part of the canonical or compatibility decomposition algorithms, but + /// performing it before those algorithms produces normalized output which + /// better preserves the intent of the original text. /// /// Note that many systems today ignore variation selectors, so these /// may not immediately help text display as intended, but they at /// least preserve the information in a standardized form, giving /// implementations the option to recognize them. - fn svar(self) -> Replacements; + fn cjk_compat_variants(self) -> Replacements; /// An Iterator over the string with Conjoining Grapheme Joiner characters /// inserted according to the Stream-Safe Text Process (UAX15-D4) @@ -151,8 +151,8 @@ impl<'a> UnicodeNormalization> for &'a str { } #[inline] - fn svar(self) -> Replacements> { - replace::new_svar(self.chars()) + fn cjk_compat_variants(self) -> Replacements> { + replace::new_cjk_compat_variants(self.chars()) } #[inline] @@ -183,8 +183,8 @@ impl> UnicodeNormalization for I { } #[inline] - fn svar(self) -> Replacements { - replace::new_svar(self) + fn cjk_compat_variants(self) -> Replacements { + replace::new_cjk_compat_variants(self) } #[inline] diff --git a/src/lookups.rs b/src/lookups.rs index fdf8333..0111d20 100644 --- a/src/lookups.rs +++ b/src/lookups.rs @@ -64,11 +64,11 @@ pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]> ) } -pub(crate) fn svar_fully_decomposed(c: char) -> Option<&'static [char]> { +pub(crate) fn cjk_compat_variants_fully_decomposed(c: char) -> Option<&'static [char]> { mph_lookup( c.into(), - SVAR_DECOMPOSED_SALT, - SVAR_DECOMPOSED_KV, + CJK_COMPAT_VARIANTS_DECOMPOSED_SALT, + CJK_COMPAT_VARIANTS_DECOMPOSED_KV, pair_lookup_fk, pair_lookup_fv_opt, None, diff --git a/src/normalize.rs b/src/normalize.rs index 6e584bc..b144bd7 100644 --- a/src/normalize.rs +++ b/src/normalize.rs @@ -10,8 +10,8 @@ //! Functions for computing canonical and compatible decompositions for Unicode characters. use crate::lookups::{ - canonical_fully_decomposed, compatibility_fully_decomposed, composition_table, - svar_fully_decomposed, + canonical_fully_decomposed, cjk_compat_variants_fully_decomposed, + compatibility_fully_decomposed, composition_table, }; use core::{char, ops::FnMut}; @@ -47,7 +47,7 @@ pub fn decompose_compatible(c: char, emit_char: F) { /// [Unicode 6.3 Release Summary](https://www.unicode.org/versions/Unicode6.3.0/#Summary) /// for more information. #[inline] -pub fn decompose_svar(c: char, mut emit_char: F) +pub fn decompose_cjk_compat_variants(c: char, mut emit_char: F) where F: FnMut(char), { @@ -59,7 +59,7 @@ where // Don't perform decomposition for Hangul - if let Some(decomposed) = svar_fully_decomposed(c) { + if let Some(decomposed) = cjk_compat_variants_fully_decomposed(c) { for &d in decomposed { emit_char(d); } diff --git a/src/replace.rs b/src/replace.rs index 1661db3..6e5cb1a 100644 --- a/src/replace.rs +++ b/src/replace.rs @@ -20,7 +20,7 @@ pub struct Replacements { } #[inline] -pub fn new_svar>(iter: I) -> Replacements { +pub fn new_cjk_compat_variants>(iter: I) -> Replacements { Replacements { iter, buffer: None } } @@ -37,7 +37,7 @@ impl> Iterator for Replacements { Some(ch) => { // At this time, the longest replacement sequence has length 2. let mut buffer = TinyVec::<[char; 2]>::new(); - super::char::decompose_svar(ch, |d| buffer.push(d)); + super::char::decompose_cjk_compat_variants(ch, |d| buffer.push(d)); self.buffer = buffer.get(1).copied(); Some(buffer[0]) } diff --git a/src/tables.rs b/src/tables.rs index 14c7b8d..81ebf23 100644 --- a/src/tables.rs +++ b/src/tables.rs @@ -15161,7 +15161,7 @@ pub(crate) const COMPATIBILITY_DECOMPOSED_KV: &[(u32, &'static [char])] = &[ (0x2106, &['\u{0063}', '\u{002F}', '\u{0075}']), ]; -pub(crate) const SVAR_DECOMPOSED_SALT: &[u16] = &[ +pub(crate) const CJK_COMPAT_VARIANTS_DECOMPOSED_SALT: &[u16] = &[ 0x5, 0x0, 0x0, @@ -16165,7 +16165,7 @@ pub(crate) const SVAR_DECOMPOSED_SALT: &[u16] = &[ 0x5, 0x1, ]; -pub(crate) const SVAR_DECOMPOSED_KV: &[(u32, &'static [char])] = &[ +pub(crate) const CJK_COMPAT_VARIANTS_DECOMPOSED_KV: &[(u32, &'static [char])] = &[ (0xfa08, &['\u{884C}', '\u{FE00}']), (0x2f825, &['\u{52C7}', '\u{FE01}']), (0x2f838, &['\u{20B63}', '\u{FE00}']), diff --git a/tests/cjk_compat_variants.rs b/tests/cjk_compat_variants.rs new file mode 100644 index 0000000..1bf7a2d --- /dev/null +++ b/tests/cjk_compat_variants.rs @@ -0,0 +1,80 @@ +//! Test the standard variation sequence replacements. + +use unicode_normalization::UnicodeNormalization; + +#[test] +fn test_cjk_compat_variants() { + // These codepoints have singleton decompositions in the canonical + // decomposition, and can use standardized variations. + let s = "\u{2f999}\u{2f8a6}"; + + // These codepoints have canonical decompositions. + let mut nfd_iter = s.chars().nfd(); + assert_eq!(nfd_iter.next(), Some('\u{831d}')); + assert_eq!(nfd_iter.next(), Some('\u{6148}')); + assert_eq!(nfd_iter.next(), None); + + let mut nfkd_iter = s.chars().nfkd(); + assert_eq!(nfkd_iter.next(), Some('\u{831d}')); + assert_eq!(nfkd_iter.next(), Some('\u{6148}')); + assert_eq!(nfkd_iter.next(), None); + + let mut nfc_iter = s.chars().nfc(); + assert_eq!(nfc_iter.next(), Some('\u{831d}')); + assert_eq!(nfc_iter.next(), Some('\u{6148}')); + assert_eq!(nfc_iter.next(), None); + + let mut nfkc_iter = s.chars().nfkc(); + assert_eq!(nfkc_iter.next(), Some('\u{831d}')); + assert_eq!(nfkc_iter.next(), Some('\u{6148}')); + assert_eq!(nfkc_iter.next(), None); + + // However they also have standardized variants. + let mut var_iter = s.chars().cjk_compat_variants(); + assert_eq!(var_iter.next(), Some('\u{831d}')); + assert_eq!(var_iter.next(), Some('\u{fe00}')); + assert_eq!(var_iter.next(), Some('\u{6148}')); + assert_eq!(var_iter.next(), Some('\u{fe00}')); + assert_eq!(var_iter.next(), None); + + // The standardized variants are normalization-stable. + let mut var_nfc_iter = s.chars().cjk_compat_variants().nfc(); + assert_eq!(var_nfc_iter.next(), Some('\u{831d}')); + assert_eq!(var_nfc_iter.next(), Some('\u{fe00}')); + assert_eq!(var_nfc_iter.next(), Some('\u{6148}')); + assert_eq!(var_nfc_iter.next(), Some('\u{fe00}')); + assert_eq!(var_nfc_iter.next(), None); + + let mut var_nfd_iter = s.chars().cjk_compat_variants().nfd(); + assert_eq!(var_nfd_iter.next(), Some('\u{831d}')); + assert_eq!(var_nfd_iter.next(), Some('\u{fe00}')); + assert_eq!(var_nfd_iter.next(), Some('\u{6148}')); + assert_eq!(var_nfd_iter.next(), Some('\u{fe00}')); + assert_eq!(var_nfd_iter.next(), None); + + let mut var_nfkc_iter = s.chars().cjk_compat_variants().nfkc(); + assert_eq!(var_nfkc_iter.next(), Some('\u{831d}')); + assert_eq!(var_nfkc_iter.next(), Some('\u{fe00}')); + assert_eq!(var_nfkc_iter.next(), Some('\u{6148}')); + assert_eq!(var_nfkc_iter.next(), Some('\u{fe00}')); + assert_eq!(var_nfkc_iter.next(), None); + + let mut var_nfkd_iter = s.chars().cjk_compat_variants().nfkd(); + assert_eq!(var_nfkd_iter.next(), Some('\u{831d}')); + assert_eq!(var_nfkd_iter.next(), Some('\u{fe00}')); + assert_eq!(var_nfkd_iter.next(), Some('\u{6148}')); + assert_eq!(var_nfkd_iter.next(), Some('\u{fe00}')); + assert_eq!(var_nfkd_iter.next(), None); +} + +/// `cjk_compat_variants` shouldn't decompose Hangul. +#[test] +fn test_cjk_compat_variants_with_hangul() { + assert_eq!( + "중국어 (홍콩)" + .chars() + .cjk_compat_variants() + .collect::(), + "중국어 (홍콩)" + ); +} diff --git a/tests/svar.rs b/tests/svar.rs deleted file mode 100644 index cc8b923..0000000 --- a/tests/svar.rs +++ /dev/null @@ -1,77 +0,0 @@ -//! Test the standard variation sequence replacements. - -use unicode_normalization::UnicodeNormalization; - -#[test] -fn test_standardized_variations_for_cjk_singleton_decompositions() { - // These codepoints have singleton decompositions in the canonical - // decomposition, and can use standardized variations. - let s = "\u{2f999}\u{2f8a6}"; - - // These codepoints have canonical decompositions. - let mut nfd_iter = s.chars().nfd(); - assert_eq!(nfd_iter.next(), Some('\u{831d}')); - assert_eq!(nfd_iter.next(), Some('\u{6148}')); - assert_eq!(nfd_iter.next(), None); - - let mut nfkd_iter = s.chars().nfkd(); - assert_eq!(nfkd_iter.next(), Some('\u{831d}')); - assert_eq!(nfkd_iter.next(), Some('\u{6148}')); - assert_eq!(nfkd_iter.next(), None); - - let mut nfc_iter = s.chars().nfc(); - assert_eq!(nfc_iter.next(), Some('\u{831d}')); - assert_eq!(nfc_iter.next(), Some('\u{6148}')); - assert_eq!(nfc_iter.next(), None); - - let mut nfkc_iter = s.chars().nfkc(); - assert_eq!(nfkc_iter.next(), Some('\u{831d}')); - assert_eq!(nfkc_iter.next(), Some('\u{6148}')); - assert_eq!(nfkc_iter.next(), None); - - // However they also have standardized variants. - let mut svar_iter = s.chars().svar(); - assert_eq!(svar_iter.next(), Some('\u{831d}')); - assert_eq!(svar_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_iter.next(), Some('\u{6148}')); - assert_eq!(svar_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_iter.next(), None); - - // The standardized variants are normalization-stable. - let mut svar_nfc_iter = s.chars().svar().nfc(); - assert_eq!(svar_nfc_iter.next(), Some('\u{831d}')); - assert_eq!(svar_nfc_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_nfc_iter.next(), Some('\u{6148}')); - assert_eq!(svar_nfc_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_nfc_iter.next(), None); - - let mut svar_nfd_iter = s.chars().svar().nfd(); - assert_eq!(svar_nfd_iter.next(), Some('\u{831d}')); - assert_eq!(svar_nfd_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_nfd_iter.next(), Some('\u{6148}')); - assert_eq!(svar_nfd_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_nfd_iter.next(), None); - - let mut svar_nfkc_iter = s.chars().svar().nfkc(); - assert_eq!(svar_nfkc_iter.next(), Some('\u{831d}')); - assert_eq!(svar_nfkc_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_nfkc_iter.next(), Some('\u{6148}')); - assert_eq!(svar_nfkc_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_nfkc_iter.next(), None); - - let mut svar_nfkd_iter = s.chars().svar().nfkd(); - assert_eq!(svar_nfkd_iter.next(), Some('\u{831d}')); - assert_eq!(svar_nfkd_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_nfkd_iter.next(), Some('\u{6148}')); - assert_eq!(svar_nfkd_iter.next(), Some('\u{fe00}')); - assert_eq!(svar_nfkd_iter.next(), None); -} - -/// `svar` shouldn't decompose Hangul. -#[test] -fn test_svar_hangul() { - assert_eq!( - "중국어 (홍콩)".chars().svar().collect::(), - "중국어 (홍콩)" - ); -}