diff --git a/src/stream_safe.rs b/src/stream_safe.rs index 7ee71c9..3a442ac 100644 --- a/src/stream_safe.rs +++ b/src/stream_safe.rs @@ -32,25 +32,26 @@ impl> Iterator for StreamSafe { #[inline] fn next(&mut self) -> Option { - if let Some(ch) = self.buffer.take() { - return Some(ch); - } - let next_ch = match self.iter.next() { + let next_ch = match self.buffer.take().or_else(|| self.iter.next()) { None => return None, Some(c) => c, }; let d = classify_nonstarters(next_ch); if self.nonstarter_count + d.leading_nonstarters > MAX_NONSTARTERS { + // Since we're emitting a CGJ, the suffix of the emitted string in NFKD has no trailing + // nonstarters, so we can reset the counter to zero. Put `next_ch` back into the + // iterator (via `self.buffer`), and we'll reclassify it next iteration. + self.nonstarter_count = 0; self.buffer = Some(next_ch); - self.nonstarter_count = d.decomposition_len; return Some(COMBINING_GRAPHEME_JOINER); } - // No starters in the decomposition, so keep accumulating + // Is the character all nonstarters in NFKD? If so, increment our counter of contiguous + // nonstarters in NKFD. if d.leading_nonstarters == d.decomposition_len { self.nonstarter_count += d.decomposition_len; } - // Otherwise, restart the nonstarter counter. + // Otherwise, reset the counter to the decomposition's number of trailing nonstarters. else { self.nonstarter_count = d.trailing_nonstarters; } @@ -104,7 +105,7 @@ pub(crate) fn classify_nonstarters(c: char) -> Decomposition { #[cfg(test)] mod tests { - use super::{classify_nonstarters, StreamSafe}; + use super::{COMBINING_GRAPHEME_JOINER, classify_nonstarters, StreamSafe}; use crate::lookups::canonical_combining_class; use crate::normalize::decompose_compatible; @@ -131,6 +132,17 @@ mod tests { assert_eq!(stream_safe(woah_nelly), its_cool); } + #[test] + fn test_all_nonstarters() { + let s = "\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}"; + + let first_block = "\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}"; + let second_block = "\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}"; + let expected = format!("{}{}{}", first_block, COMBINING_GRAPHEME_JOINER, second_block); + + assert_eq!(stream_safe(s), expected); + } + #[test] fn test_classify_nonstarters() { // Highest character in the `compat_fully_decomp` table is 2FA1D