diff --git a/benches/bench.rs b/benches/bench.rs index 0e0a5df..153012b 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -6,52 +6,83 @@ extern crate test; use test::Bencher; use unicode_normalization::UnicodeNormalization; +const ASCII: &'static str = "all types of normalized"; +const NFC: &'static str = "Introducci\u{00f3}n a Unicode.pdf"; +const NFD: &'static str = "Introduccio\u{0301}n a Unicode.pdf"; + #[bench] fn bench_is_nfc_ascii(b: &mut Bencher) { - b.iter(|| unicode_normalization::is_nfc("all types of normalized")); + b.iter(|| unicode_normalization::is_nfc(ASCII)); } #[bench] fn bench_is_nfc_normalized(b: &mut Bencher) { - b.iter(|| unicode_normalization::is_nfc("Introducci\u{00f3}n a Unicode.pdf")); + b.iter(|| unicode_normalization::is_nfc(NFC)); } #[bench] fn bench_is_nfc_not_normalized(b: &mut Bencher) { - b.iter(|| unicode_normalization::is_nfc("Introduccio\u{0301}n a Unicode.pdf")); + b.iter(|| unicode_normalization::is_nfc(NFD)); } #[bench] fn bench_is_nfd_ascii(b: &mut Bencher) { - b.iter(|| unicode_normalization::is_nfd("an easy string to check")); + b.iter(|| unicode_normalization::is_nfd(ASCII)); } #[bench] fn bench_is_nfd_normalized(b: &mut Bencher) { - b.iter(|| unicode_normalization::is_nfd("Introduccio\u{0301}n a Unicode.pdf")); + b.iter(|| unicode_normalization::is_nfd(NFD)); } #[bench] fn bench_is_nfd_not_normalized(b: &mut Bencher) { - b.iter(|| unicode_normalization::is_nfd("Introducci\u{00f3}n a Unicode.pdf")); + b.iter(|| unicode_normalization::is_nfd(NFC)); +} + +#[bench] +fn bench_is_nfc_stream_safe_ascii(b: &mut Bencher) { + b.iter(|| unicode_normalization::is_nfc_stream_safe(ASCII)); +} + +#[bench] +fn bench_is_nfc_stream_safe_normalized(b: &mut Bencher) { + b.iter(|| unicode_normalization::is_nfc_stream_safe(NFC)); +} + +#[bench] +fn bench_is_nfc_stream_safe_not_normalized(b: &mut Bencher) { + b.iter(|| unicode_normalization::is_nfc_stream_safe(NFD)); +} + +#[bench] +fn bench_is_nfd_stream_safe_ascii(b: &mut Bencher) { + b.iter(|| unicode_normalization::is_nfd_stream_safe(ASCII)); +} + +#[bench] +fn bench_is_nfd_stream_safe_normalized(b: &mut Bencher) { + b.iter(|| unicode_normalization::is_nfd_stream_safe(NFD)); +} + +#[bench] +fn bench_is_nfd_stream_safe_not_normalized(b: &mut Bencher) { + b.iter(|| unicode_normalization::is_nfd_stream_safe(NFC)); } #[bench] fn bench_nfc_ascii(b: &mut Bencher) { - let s = "normalize me please"; - b.iter(|| s.nfc().count()); + b.iter(|| ASCII.nfc().count()); } #[bench] fn bench_nfd_ascii(b: &mut Bencher) { - let s = "decompose me entirely"; - b.iter(|| s.nfd().count()); + b.iter(|| ASCII.nfd().count()); } #[bench] fn bench_streamsafe_ascii(b: &mut Bencher) { - let s = "quite nonthreatening"; - b.iter(|| s.stream_safe().count()); + b.iter(|| ASCII.stream_safe().count()); } #[bench] diff --git a/src/lib.rs b/src/lib.rs index c26c388..30b7a3c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,8 +47,12 @@ pub use quick_check::{ IsNormalized, is_nfc, is_nfc_quick, + is_nfc_stream_safe, + is_nfc_stream_safe_quick, is_nfd, is_nfd_quick, + is_nfd_stream_safe, + is_nfd_stream_safe_quick, }; pub use recompose::Recompositions; pub use stream_safe::StreamSafe; diff --git a/src/quick_check.rs b/src/quick_check.rs index 4316cf5..63dab1c 100644 --- a/src/quick_check.rs +++ b/src/quick_check.rs @@ -1,9 +1,11 @@ use UnicodeNormalization; +use stream_safe; use tables; /// The QuickCheck algorithm can quickly determine if a text is or isn't /// normalized without any allocations in many cases, but it has to be able to /// return `Maybe` when a full decomposition and recomposition is necessary. +#[derive(Debug, Eq, PartialEq)] pub enum IsNormalized { /// The text is definitely normalized. Yes, @@ -15,17 +17,20 @@ pub enum IsNormalized { // https://unicode.org/reports/tr15/#Detecting_Normalization_Forms #[inline] -fn quick_check(s: I, is_allowed: F) -> IsNormalized +fn quick_check(s: I, is_allowed: F, stream_safe: bool) -> IsNormalized where I: Iterator, F: Fn(char) -> IsNormalized { let mut last_cc = 0u8; + let mut nonstarter_count = 0; let mut result = IsNormalized::Yes; for ch in s { // For ASCII we know it's always allowed and a starter if ch <= '\x7f' { last_cc = 0; + nonstarter_count = 0; continue; } + // Otherwise, lookup the combining class and QC property let cc = tables::canonical_combining_class(ch); if last_cc > cc && cc != 0 { @@ -38,6 +43,20 @@ fn quick_check(s: I, is_allowed: F) -> IsNormalized result = IsNormalized::Maybe; }, } + if stream_safe { + let decomp = stream_safe::classify_nonstarters(ch); + + // If we're above `MAX_NONSTARTERS`, we're definitely *not* + // stream-safe normalized. + if nonstarter_count + decomp.leading_nonstarters > stream_safe::MAX_NONSTARTERS { + return IsNormalized::No; + } + if decomp.leading_nonstarters == decomp.decomposition_len { + nonstarter_count += decomp.decomposition_len; + } else { + nonstarter_count = decomp.trailing_nonstarters; + } + } last_cc = cc; } result @@ -48,16 +67,29 @@ fn quick_check(s: I, is_allowed: F) -> IsNormalized /// like `s.chars().nfc().eq(s.chars())` should suffice. #[inline] pub fn is_nfc_quick>(s: I) -> IsNormalized { - quick_check(s, tables::qc_nfc) + quick_check(s, tables::qc_nfc, false) } /// Quickly check if a string is in NFD. #[inline] pub fn is_nfd_quick>(s: I) -> IsNormalized { - quick_check(s, tables::qc_nfd) + quick_check(s, tables::qc_nfd, false) +} + +/// Quickly check if a string is Stream-Safe NFC. +#[inline] +pub fn is_nfc_stream_safe_quick>(s: I) -> IsNormalized { + quick_check(s, tables::qc_nfc, true) +} + +/// Quickly check if a string is Stream-Safe NFD. +#[inline] +pub fn is_nfd_stream_safe_quick>(s: I) -> IsNormalized { + quick_check(s, tables::qc_nfd, true) } /// Authoritatively check if a string is in NFC. +#[inline] pub fn is_nfc(s: &str) -> bool { match is_nfc_quick(s.chars()) { IsNormalized::Yes => true, @@ -67,6 +99,7 @@ pub fn is_nfc(s: &str) -> bool { } /// Authoritatively check if a string is in NFD. +#[inline] pub fn is_nfd(s: &str) -> bool { match is_nfd_quick(s.chars()) { IsNormalized::Yes => true, @@ -74,3 +107,50 @@ pub fn is_nfd(s: &str) -> bool { IsNormalized::Maybe => s.chars().eq(s.chars().nfd()), } } + +/// Authoritatively check if a string is Stream-Safe NFC. +#[inline] +pub fn is_nfc_stream_safe(s: &str) -> bool { + match is_nfc_stream_safe_quick(s.chars()) { + IsNormalized::Yes => true, + IsNormalized::No => false, + IsNormalized::Maybe => s.chars().eq(s.chars().stream_safe().nfc()), + } +} + +/// Authoritatively check if a string is Stream-Safe NFD. +#[inline] +pub fn is_nfd_stream_safe(s: &str) -> bool { + match is_nfd_stream_safe_quick(s.chars()) { + IsNormalized::Yes => true, + IsNormalized::No => false, + IsNormalized::Maybe => s.chars().eq(s.chars().stream_safe().nfd()), + } +} + +#[cfg(test)] +mod tests { + use super::{ + IsNormalized, + is_nfc_stream_safe_quick, + is_nfd_stream_safe_quick, + }; + + #[test] + fn test_stream_safe_nfd() { + let okay = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone"; + assert_eq!(is_nfd_stream_safe_quick(okay.chars()), IsNormalized::Yes); + + let too_much = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone"; + assert_eq!(is_nfd_stream_safe_quick(too_much.chars()), IsNormalized::No); + } + + #[test] + fn test_stream_safe_nfc() { + let okay = "ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y"; + assert_eq!(is_nfc_stream_safe_quick(okay.chars()), IsNormalized::Maybe); + + let too_much = "not ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y"; + assert_eq!(is_nfc_stream_safe_quick(too_much.chars()), IsNormalized::No); + } +} diff --git a/src/stream_safe.rs b/src/stream_safe.rs index ee88c83..8bc48f6 100644 --- a/src/stream_safe.rs +++ b/src/stream_safe.rs @@ -4,7 +4,7 @@ use normalize::{ }; use tables; -const MAX_NONSTARTERS: usize = 30; +pub(crate) const MAX_NONSTARTERS: usize = 30; const COMBINING_GRAPHEME_JOINER: char = '\u{034F}'; /// UAX15-D4: This iterator keeps track of how many non-starters there have been @@ -54,14 +54,14 @@ impl> Iterator for StreamSafe { } #[derive(Debug)] -struct Decomposition { - leading_nonstarters: usize, - trailing_nonstarters: usize, - decomposition_len: usize, +pub(crate) struct Decomposition { + pub(crate) leading_nonstarters: usize, + pub(crate) trailing_nonstarters: usize, + pub(crate) decomposition_len: usize, } #[inline] -fn classify_nonstarters(c: char) -> Decomposition { +pub(crate) fn classify_nonstarters(c: char) -> Decomposition { // As usual, fast path for ASCII (which is always a starter) if c <= '\x7f' { return Decomposition {