Switch to a more explicit API.

Switch to a dedicated `svar()` iterator function, which just does
standardized variation sequences, rather than framing this functionality
as an open-ended "extended" version of the standard normalization
algorithms. This makes for a more factored API, gives users more control
over exactly what transformations are done, and has less impact on users
that don't need this new functionality.

scripts/unicode.py

@@ -72,8 +72,8 @@ class UnicodeData(object):
         self.canon_comp = self._compute_canonical_comp()
         self.canon_fully_decomp, self.compat_fully_decomp = self._compute_fully_decomposed()
 
-        self.ext_decomp = {}
-        self.ext_fully_decomp = {}
+        self.svar_decomp = {}
+        self.svar_fully_decomp = {}
         self._load_standardized_variants()
 
         def stats(name, table):
@@ -82,11 +82,11 @@ class UnicodeData(object):
 
         print("Decomposition table stats:")
         stats("Canonical decomp", self.canon_decomp)
-        stats("Canonical decomp with extensions", self.ext_decomp)
         stats("Compatible decomp", self.compat_decomp)
+        stats("Standardized Variants", self.svar_decomp)
         stats("Canonical fully decomp", self.canon_fully_decomp)
-        stats("Canonical fully decomp with extensions", self.ext_fully_decomp)
         stats("Compatible fully decomp", self.compat_fully_decomp)
+        stats("Standardized Variants", self.svar_fully_decomp)
 
         self.ss_leading, self.ss_trailing = self._compute_stream_safe_tables()
 
@@ -152,8 +152,8 @@ class UnicodeData(object):
                     #assert not never_composes(c) TODO: Re-enable this once #67 lands.
                     assert not c in self.canon_decomp, "Unexpected: standardized variant is unnormalized (canon)"
                     assert not c in self.compat_decomp, "Unexpected: standardized variant is unnormalized (compat)"
-                self.ext_decomp[char_int] = standardized_variant_parts
-                self.ext_fully_decomp[char_int] = standardized_variant_parts
+                self.svar_decomp[char_int] = standardized_variant_parts
+                self.svar_fully_decomp[char_int] = standardized_variant_parts
 
     def _load_norm_props(self):
         props = collections.defaultdict(list)
@@ -364,8 +364,8 @@ def gen_composition_table(canon_comp, out):
     out.write("    }\n")
     out.write("}\n")
 
-def gen_decomposition_tables(canon_decomp, ext_decomp, compat_decomp, out):
-    tables = [(canon_decomp, 'canonical'), (ext_decomp, 'ext'), (compat_decomp, 'compatibility')]
+def gen_decomposition_tables(canon_decomp, compat_decomp, svar_decomp, out):
+    tables = [(canon_decomp, 'canonical'), (compat_decomp, 'compatibility'), (svar_decomp, 'svar')]
     for table, name in tables:
         gen_mph_data(name + '_decomposed', table, "(u32, &'static [char])",
             lambda k: "(0x{:x}, &[{}])".format(k,
@@ -535,7 +535,7 @@ if __name__ == '__main__':
         gen_composition_table(data.canon_comp, out)
         out.write("\n")
 
-        gen_decomposition_tables(data.canon_fully_decomp, data.ext_fully_decomp, data.compat_fully_decomp, out)
+        gen_decomposition_tables(data.canon_fully_decomp, data.compat_fully_decomp, data.svar_fully_decomp, out)
 
         gen_combining_mark(data.general_category_mark, out)
         out.write("\n")

src/decompose.rs

@@ -16,8 +16,6 @@ use tinyvec::TinyVec;
 enum DecompositionType {
     Canonical,
     Compatible,
-    CanonicalExt,
-    CompatibleExt,
 }
 
 /// External iterator for a string decomposition's characters.
@@ -58,26 +56,6 @@ pub fn new_compatible<I: Iterator<Item = char>>(iter: I) -> Decompositions<I> {
     }
 }
 
-#[inline]
-pub fn new_canonical_ext<I: Iterator<Item = char>>(iter: I) -> Decompositions<I> {
-    Decompositions {
-        kind: self::DecompositionType::CanonicalExt,
-        iter: iter.fuse(),
-        buffer: TinyVec::new(),
-        ready: 0..0,
-    }
-}
-
-#[inline]
-pub fn new_compatible_ext<I: Iterator<Item = char>>(iter: I) -> Decompositions<I> {
-    Decompositions {
-        kind: self::DecompositionType::CompatibleExt,
-        iter: iter.fuse(),
-        buffer: TinyVec::new(),
-        ready: 0..0,
-    }
-}
-
 impl<I> Decompositions<I> {
     #[inline]
     fn push_back(&mut self, ch: char) {
@@ -135,12 +113,6 @@ impl<I: Iterator<Item = char>> Iterator for Decompositions<I> {
                 (Some(ch), &DecompositionType::Compatible) => {
                     super::char::decompose_compatible(ch, |d| self.push_back(d));
                 }
-                (Some(ch), &DecompositionType::CanonicalExt) => {
-                    super::char::decompose_canonical_ext(ch, |d| self.push_back(d));
-                }
-                (Some(ch), &DecompositionType::CompatibleExt) => {
-                    super::char::decompose_compatible_ext(ch, |d| self.push_back(d));
-                }
                 (None, _) => {
                     if self.buffer.is_empty() {
                         return None;

src/lib.rs

@@ -59,6 +59,7 @@ pub use crate::quick_check::{
     IsNormalized,
 };
 pub use crate::recompose::Recompositions;
+pub use crate::replace::Replacements;
 pub use crate::stream_safe::StreamSafe;
 pub use crate::tables::UNICODE_VERSION;
 use core::str::Chars;
@@ -71,6 +72,7 @@ mod normalize;
 mod perfect_hash;
 mod quick_check;
 mod recompose;
+mod replace;
 mod stream_safe;
 
 #[rustfmt::skip]
@@ -84,8 +86,7 @@ mod test;
 /// Methods for composing and decomposing characters.
 pub mod char {
     pub use crate::normalize::{
-        compose, decompose_canonical, decompose_canonical_ext, decompose_compatible,
-        decompose_compatible_ext,
+        compose, decompose_canonical, decompose_compatible, decompose_svar,
     };
 
     pub use crate::lookups::{canonical_combining_class, is_combining_mark};
@@ -111,41 +112,16 @@ pub trait UnicodeNormalization<I: Iterator<Item = char>> {
     /// (compatibility decomposition followed by canonical composition).
     fn nfkc(self) -> Recompositions<I>;
 
-    /// Similar to `nfd`, but with extensions which differ from the standard
-    /// decomposition algorithm and which don't have a stability guarantee,
-    /// but which still produce valid NFD and provide better results:
-    ///  - Standardized Variation Seqeuences are used to avoid losing
-    ///    information when normalizing "CJK Compatibility Ideographs"
-    ///    codepoints. Note that many systemes today ignore variation
-    ///    selectors, but the information is at least preserved in a
-    ///    standardized form.
+    /// A non-standard transformation which replaces select codepoints with
+    /// normal forms using Standardized Variation Sequences. These are
+    /// different than the standard decompositions, but they better preserve
+    /// the intent of the original text.
     ///
-    /// Additional extensions may be added in future versions.
-    ///
-    /// If you need to match the standard `toNFD` algorithm exactly, or you
-    /// need a stability guarantee, use `nfd` instead.
-    fn nfd_ext(self) -> Decompositions<I>;
-
-    /// Similar to `nfkd`, and the result is valid NFKD, but with the same
-    /// extensions as `nfd`.
-    ///
-    /// If you need to match the standard `toNFKD` algorithm exactly, or you
-    /// need a stability guarantee, use `nfd` instead.
-    fn nfkd_ext(self) -> Decompositions<I>;
-
-    /// Similar to `nfc`, and the result is valid NFC, but with the same
-    /// extensions as `nfd`.
-    ///
-    /// If you need to match the standard `toNFC` algorithm exactly, or you
-    /// need a stability guarantee, use `nfd` instead.
-    fn nfc_ext(self) -> Recompositions<I>;
-
-    /// Similar to `nfkc`, and the result is valid NFKC, but with the same
-    /// extensions as `nfd`.
-    ///
-    /// If you need to match the standard `toNFKC` algorithm exactly, or you
-    /// need a stability guarantee, use `nfd` instead.
-    fn nfkc_ext(self) -> Recompositions<I>;
+    /// Note that many systems today ignore variation selectors, so these
+    /// may not immediately help text display as intended, but they at
+    /// least preserve the information in a standardized form, giving
+    /// implementations the option to recognize them.
+    fn svar(self) -> Replacements<I>;
 
     /// An Iterator over the string with Conjoining Grapheme Joiner characters
     /// inserted according to the Stream-Safe Text Process (UAX15-D4)
@@ -174,23 +150,8 @@ impl<'a> UnicodeNormalization<Chars<'a>> for &'a str {
     }
 
     #[inline]
-    fn nfd_ext(self) -> Decompositions<Chars<'a>> {
-        decompose::new_canonical_ext(self.chars())
-    }
-
-    #[inline]
-    fn nfkd_ext(self) -> Decompositions<Chars<'a>> {
-        decompose::new_compatible_ext(self.chars())
-    }
-
-    #[inline]
-    fn nfc_ext(self) -> Recompositions<Chars<'a>> {
-        recompose::new_canonical_ext(self.chars())
-    }
-
-    #[inline]
-    fn nfkc_ext(self) -> Recompositions<Chars<'a>> {
-        recompose::new_compatible_ext(self.chars())
+    fn svar(self) -> Replacements<Chars<'a>> {
+        replace::new_svar(self.chars())
     }
 
     #[inline]
@@ -221,23 +182,8 @@ impl<I: Iterator<Item = char>> UnicodeNormalization<I> for I {
     }
 
     #[inline]
-    fn nfd_ext(self) -> Decompositions<I> {
-        decompose::new_canonical_ext(self)
-    }
-
-    #[inline]
-    fn nfkd_ext(self) -> Decompositions<I> {
-        decompose::new_compatible_ext(self)
-    }
-
-    #[inline]
-    fn nfc_ext(self) -> Recompositions<I> {
-        recompose::new_canonical_ext(self)
-    }
-
-    #[inline]
-    fn nfkc_ext(self) -> Recompositions<I> {
-        recompose::new_compatible_ext(self)
+    fn svar(self) -> Replacements<I> {
+        replace::new_svar(self)
     }
 
     #[inline]

src/lookups.rs

@@ -53,22 +53,22 @@ pub(crate) fn canonical_fully_decomposed(c: char) -> Option<&'static [char]> {
     )
 }
 
-pub(crate) fn ext_fully_decomposed(c: char) -> Option<&'static [char]> {
+pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]> {
     mph_lookup(
         c.into(),
-        EXT_DECOMPOSED_SALT,
-        EXT_DECOMPOSED_KV,
+        COMPATIBILITY_DECOMPOSED_SALT,
+        COMPATIBILITY_DECOMPOSED_KV,
         pair_lookup_fk,
         pair_lookup_fv_opt,
         None,
     )
 }
 
-pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]> {
+pub(crate) fn svar_fully_decomposed(c: char) -> Option<&'static [char]> {
     mph_lookup(
         c.into(),
-        COMPATIBILITY_DECOMPOSED_SALT,
-        COMPATIBILITY_DECOMPOSED_KV,
+        SVAR_DECOMPOSED_SALT,
+        SVAR_DECOMPOSED_KV,
         pair_lookup_fk,
         pair_lookup_fv_opt,
         None,

src/normalize.rs

@@ -11,7 +11,7 @@
 //! Functions for computing canonical and compatible decompositions for Unicode characters.
 use crate::lookups::{
     canonical_fully_decomposed, compatibility_fully_decomposed, composition_table,
-    ext_fully_decomposed,
+    svar_fully_decomposed,
 };
 
 use core::{char, ops::FnMut};
@@ -37,35 +37,21 @@ pub fn decompose_compatible<F: FnMut(char)>(c: char, emit_char: F) {
     decompose(c, decompose_char, emit_char)
 }
 
-/// Compute "extended" canonical Unicode decomposition for character.
+/// Compute standard-variation decomposition for character.
 ///
-/// This is `decompose_canonical` plus extensions, which currently consist of:
-///  - [Standardized Variation Sequences] are used instead of the standard canonical
-///    decompositions for CJK codepoints with singleton canonical decompositions, to
-///    avoid losing information. See the
-///    [Unicode Variation Sequence FAQ](http://unicode.org/faq/vs.html) and the
-///    "Other Enhancements" section of the
-///    [Unicode 6.3 Release Summary](https://www.unicode.org/versions/Unicode6.3.0/#Summary)
-///    for more information.
+/// [Standardized Variation Sequences] are used instead of the standard canonical
+/// decompositions, notably for CJK codepoints with singleton canonical decompositions,
+/// to avoid losing information. See the
+/// [Unicode Variation Sequence FAQ](http://unicode.org/faq/vs.html) and the
+/// "Other Enhancements" section of the
+/// [Unicode 6.3 Release Summary](https://www.unicode.org/versions/Unicode6.3.0/#Summary)
+/// for more information.
 #[inline]
-pub fn decompose_canonical_ext<F>(c: char, emit_char: F)
+pub fn decompose_svar<F>(c: char, emit_char: F)
 where
     F: FnMut(char),
 {
-    let decompose_char = |c| ext_fully_decomposed(c).or_else(|| canonical_fully_decomposed(c));
-    decompose(c, decompose_char, emit_char)
-}
-
-/// Compute "extended" compatible Unicode decomposition for character.
-///
-/// This is `decompose_compatible` plus the same extensions as `decompose_canonical_ext`.
-#[inline]
-pub fn decompose_compatible_ext<F: FnMut(char)>(c: char, emit_char: F) {
-    let decompose_char = |c| {
-        ext_fully_decomposed(c)
-            .or_else(|| compatibility_fully_decomposed(c).or_else(|| canonical_fully_decomposed(c)))
-    };
-    decompose(c, decompose_char, emit_char)
+    decompose(c, svar_fully_decomposed, emit_char)
 }
 
 #[inline]

src/recompose.rs

@@ -51,28 +51,6 @@ pub fn new_compatible<I: Iterator<Item = char>>(iter: I) -> Recompositions<I> {
     }
 }
 
-#[inline]
-pub fn new_canonical_ext<I: Iterator<Item = char>>(iter: I) -> Recompositions<I> {
-    Recompositions {
-        iter: super::decompose::new_canonical_ext(iter),
-        state: self::RecompositionState::Composing,
-        buffer: TinyVec::new(),
-        composee: None,
-        last_ccc: None,
-    }
-}
-
-#[inline]
-pub fn new_compatible_ext<I: Iterator<Item = char>>(iter: I) -> Recompositions<I> {
-    Recompositions {
-        iter: super::decompose::new_compatible_ext(iter),
-        state: self::RecompositionState::Composing,
-        buffer: TinyVec::new(),
-        composee: None,
-        last_ccc: None,
-    }
-}
-
 impl<I: Iterator<Item = char>> Iterator for Recompositions<I> {
     type Item = char;
 

src/replace.rs

@@ -0,0 +1,61 @@
+// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+use core::fmt::{self, Write};
+use tinyvec::TinyVec;
+
+/// External iterator for replacements for a string's characters.
+#[derive(Clone)]
+pub struct Replacements<I> {
+    iter: I,
+    // At this time, the longest replacement sequence has length 2, so we just
+    // need buffer space for 1 codepoint.
+    buffer: Option<char>,
+}
+
+#[inline]
+pub fn new_svar<I: Iterator<Item = char>>(iter: I) -> Replacements<I> {
+    Replacements { iter, buffer: None }
+}
+
+impl<I: Iterator<Item = char>> Iterator for Replacements<I> {
+    type Item = char;
+
+    #[inline]
+    fn next(&mut self) -> Option<char> {
+        if let Some(c) = self.buffer.take() {
+            return Some(c);
+        }
+
+        match self.iter.next() {
+            Some(ch) => {
+                // At this time, the longest replacement sequence has length 2.
+                let mut buffer = TinyVec::<[char; 2]>::new();
+                super::char::decompose_svar(ch, |d| buffer.push(d));
+                self.buffer = buffer.get(1).copied();
+                Some(buffer[0])
+            }
+            None => None,
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let (lower, _) = self.iter.size_hint();
+        (lower, None)
+    }
+}
+
+impl<I: Iterator<Item = char> + Clone> fmt::Display for Replacements<I> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        for c in self.clone() {
+            f.write_char(c)?;
+        }
+        Ok(())
+    }
+}

tests/ext.rs

@@ -1,119 +0,0 @@
-//! Test the extended versions of `nfd`, `nfc`, `nfkc`, and `nfkd`.
-
-use unicode_normalization::UnicodeNormalization;
-
-#[test]
-fn test_standardized_variations_for_cjk_singleton_decompositions() {
-    // These codepoints have singleton decompositions in the canonical
-    // decomposition, and can use standardized variations in the extended
-    // decomposition.
-    let s = "\u{2f999}\u{2f8a6}";
-
-    let mut nfd_iter = s.chars().nfd();
-    assert_eq!(nfd_iter.next(), Some('\u{831d}'));
-    assert_eq!(nfd_iter.next(), Some('\u{6148}'));
-    assert_eq!(nfd_iter.next(), None);
-
-    let mut nfd_ext_iter = s.chars().nfd_ext();
-    assert_eq!(nfd_ext_iter.next(), Some('\u{831d}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{fe00}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{6148}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{fe00}'));
-    assert_eq!(nfd_ext_iter.next(), None);
-
-    let mut nfkd_iter = s.chars().nfkd();
-    assert_eq!(nfkd_iter.next(), Some('\u{831d}'));
-    assert_eq!(nfkd_iter.next(), Some('\u{6148}'));
-    assert_eq!(nfkd_iter.next(), None);
-
-    let mut nfkd_ext_iter = s.chars().nfkd_ext();
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{831d}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{fe00}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{6148}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{fe00}'));
-    assert_eq!(nfkd_ext_iter.next(), None);
-
-    let mut nfc_iter = s.chars().nfc();
-    assert_eq!(nfc_iter.next(), Some('\u{831d}'));
-    assert_eq!(nfc_iter.next(), Some('\u{6148}'));
-    assert_eq!(nfc_iter.next(), None);
-
-    let mut nfc_ext_iter = s.chars().nfc_ext();
-    assert_eq!(nfc_ext_iter.next(), Some('\u{831d}'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{fe00}'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{6148}'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{fe00}'));
-    assert_eq!(nfc_ext_iter.next(), None);
-
-    let mut nfkc_iter = s.chars().nfkc();
-    assert_eq!(nfkc_iter.next(), Some('\u{831d}'));
-    assert_eq!(nfkc_iter.next(), Some('\u{6148}'));
-    assert_eq!(nfkc_iter.next(), None);
-
-    let mut nfkc_ext_iter = s.chars().nfkc_ext();
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{831d}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{fe00}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{6148}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{fe00}'));
-    assert_eq!(nfkc_ext_iter.next(), None);
-}
-
-/// Test that the ext iterators incude the usual NFC/NFD/NFKC/NKFD normalizations.
-#[test]
-fn test_underlying_nfd_nfc_nfkd_nfkc() {
-    let s = "hi\u{212b}\u{4d}\u{3a9}\u{2156}\u{31}\u{2044}\u{33}";
-
-    let mut nfd_ext_iter = s.chars().nfd_ext();
-    assert_eq!(nfd_ext_iter.next(), Some('h'));
-    assert_eq!(nfd_ext_iter.next(), Some('i'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{41}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{30a}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{4d}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{3a9}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{2156}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{31}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{2044}'));
-    assert_eq!(nfd_ext_iter.next(), Some('\u{33}'));
-    assert_eq!(nfd_ext_iter.next(), None);
-
-    let mut nfkd_ext_iter = s.chars().nfkd_ext();
-    assert_eq!(nfkd_ext_iter.next(), Some('h'));
-    assert_eq!(nfkd_ext_iter.next(), Some('i'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{41}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{30a}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{4d}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{3a9}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{32}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{2044}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{35}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{31}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{2044}'));
-    assert_eq!(nfkd_ext_iter.next(), Some('\u{33}'));
-    assert_eq!(nfkd_ext_iter.next(), None);
-
-    let mut nfc_ext_iter = s.chars().nfc_ext();
-    assert_eq!(nfc_ext_iter.next(), Some('h'));
-    assert_eq!(nfc_ext_iter.next(), Some('i'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{c5}'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{4d}'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{3a9}'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{2156}'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{31}'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{2044}'));
-    assert_eq!(nfc_ext_iter.next(), Some('\u{33}'));
-    assert_eq!(nfc_ext_iter.next(), None);
-
-    let mut nfkc_ext_iter = s.chars().nfkc_ext();
-    assert_eq!(nfkc_ext_iter.next(), Some('h'));
-    assert_eq!(nfkc_ext_iter.next(), Some('i'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{c5}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{4d}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{3a9}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{32}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{2044}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{35}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{31}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{2044}'));
-    assert_eq!(nfkc_ext_iter.next(), Some('\u{33}'));
-    assert_eq!(nfkc_ext_iter.next(), None);
-}

tests/svar.rs

@@ -0,0 +1,68 @@
+//! Test the standard variation sequence replacements.
+
+use unicode_normalization::UnicodeNormalization;
+
+#[test]
+fn test_standardized_variations_for_cjk_singleton_decompositions() {
+    // These codepoints have singleton decompositions in the canonical
+    // decomposition, and can use standardized variations.
+    let s = "\u{2f999}\u{2f8a6}";
+
+    // These codepoints have canonical decompositions.
+    let mut nfd_iter = s.chars().nfd();
+    assert_eq!(nfd_iter.next(), Some('\u{831d}'));
+    assert_eq!(nfd_iter.next(), Some('\u{6148}'));
+    assert_eq!(nfd_iter.next(), None);
+
+    let mut nfkd_iter = s.chars().nfkd();
+    assert_eq!(nfkd_iter.next(), Some('\u{831d}'));
+    assert_eq!(nfkd_iter.next(), Some('\u{6148}'));
+    assert_eq!(nfkd_iter.next(), None);
+
+    let mut nfc_iter = s.chars().nfc();
+    assert_eq!(nfc_iter.next(), Some('\u{831d}'));
+    assert_eq!(nfc_iter.next(), Some('\u{6148}'));
+    assert_eq!(nfc_iter.next(), None);
+
+    let mut nfkc_iter = s.chars().nfkc();
+    assert_eq!(nfkc_iter.next(), Some('\u{831d}'));
+    assert_eq!(nfkc_iter.next(), Some('\u{6148}'));
+    assert_eq!(nfkc_iter.next(), None);
+
+    // However they also have standardized variants.
+    let mut svar_iter = s.chars().svar();
+    assert_eq!(svar_iter.next(), Some('\u{831d}'));
+    assert_eq!(svar_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_iter.next(), Some('\u{6148}'));
+    assert_eq!(svar_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_iter.next(), None);
+
+    // The standardized variants are normalization-stable.
+    let mut svar_nfc_iter = s.chars().svar().nfc();
+    assert_eq!(svar_nfc_iter.next(), Some('\u{831d}'));
+    assert_eq!(svar_nfc_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_nfc_iter.next(), Some('\u{6148}'));
+    assert_eq!(svar_nfc_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_nfc_iter.next(), None);
+
+    let mut svar_nfd_iter = s.chars().svar().nfd();
+    assert_eq!(svar_nfd_iter.next(), Some('\u{831d}'));
+    assert_eq!(svar_nfd_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_nfd_iter.next(), Some('\u{6148}'));
+    assert_eq!(svar_nfd_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_nfd_iter.next(), None);
+
+    let mut svar_nfkc_iter = s.chars().svar().nfkc();
+    assert_eq!(svar_nfkc_iter.next(), Some('\u{831d}'));
+    assert_eq!(svar_nfkc_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_nfkc_iter.next(), Some('\u{6148}'));
+    assert_eq!(svar_nfkc_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_nfkc_iter.next(), None);
+
+    let mut svar_nfkd_iter = s.chars().svar().nfkd();
+    assert_eq!(svar_nfkd_iter.next(), Some('\u{831d}'));
+    assert_eq!(svar_nfkd_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_nfkd_iter.next(), Some('\u{6148}'));
+    assert_eq!(svar_nfkd_iter.next(), Some('\u{fe00}'));
+    assert_eq!(svar_nfkd_iter.next(), None);
+}