diff --git a/src/lib.rs b/src/lib.rs index ddef92a..7a4b4e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -65,6 +65,7 @@ pub use stream_safe::StreamSafe; use std::str::Chars; mod decompose; +mod lookups; mod normalize; mod perfect_hash; mod recompose; @@ -81,7 +82,7 @@ mod normalization_tests; pub mod char { pub use normalize::{decompose_canonical, decompose_compatible, compose}; - pub use perfect_hash::{canonical_combining_class, is_combining_mark}; + pub use lookups::{canonical_combining_class, is_combining_mark}; } diff --git a/src/lookups.rs b/src/lookups.rs new file mode 100644 index 0000000..edaa0a0 --- /dev/null +++ b/src/lookups.rs @@ -0,0 +1,89 @@ +// Copyright 2019 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Lookups of unicode properties using minimal perfect hashing. + +use perfect_hash::mph_lookup; +use tables::*; + +/// Look up the canonical combining class for a codepoint. +/// +/// The value returned is as defined in the Unicode Character Database. +pub fn canonical_combining_class(c: char) -> u8 { + mph_lookup(c.into(), CANONICAL_COMBINING_CLASS_SALT, CANONICAL_COMBINING_CLASS_KV, + u8_lookup_fk, u8_lookup_fv, 0) +} + +pub(crate) fn composition_table(c1: char, c2: char) -> Option { + if c1 < '\u{10000}' && c2 < '\u{10000}' { + mph_lookup((c1 as u32) << 16 | (c2 as u32), + COMPOSITION_TABLE_SALT, COMPOSITION_TABLE_KV, + pair_lookup_fk, pair_lookup_fv_opt, None) + } else { + composition_table_astral(c1, c2) + } +} + +pub(crate) fn canonical_fully_decomposed(c: char) -> Option<&'static [char]> { + mph_lookup(c.into(), CANONICAL_DECOMPOSED_SALT, CANONICAL_DECOMPOSED_KV, + pair_lookup_fk, pair_lookup_fv_opt, None) +} + +pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]> { + mph_lookup(c.into(), COMPATIBILITY_DECOMPOSED_SALT, COMPATIBILITY_DECOMPOSED_KV, + pair_lookup_fk, pair_lookup_fv_opt, None) +} + +/// Return whether the given character is a combining mark (`General_Category=Mark`) +pub fn is_combining_mark(c: char) -> bool { + mph_lookup(c.into(), COMBINING_MARK_SALT, COMBINING_MARK_KV, + bool_lookup_fk, bool_lookup_fv, false) +} + +pub fn stream_safe_trailing_nonstarters(c: char) -> usize { + mph_lookup(c.into(), TRAILING_NONSTARTERS_SALT, TRAILING_NONSTARTERS_KV, + u8_lookup_fk, u8_lookup_fv, 0) as usize +} + +/// Extract the key in a 24 bit key and 8 bit value packed in a u32. +#[inline] +fn u8_lookup_fk(kv: u32) -> u32 { + kv >> 8 +} + +/// Extract the value in a 24 bit key and 8 bit value packed in a u32. +#[inline] +fn u8_lookup_fv(kv: u32) -> u8 { + (kv & 0xff) as u8 +} + +/// Extract the key for a boolean lookup. +#[inline] +fn bool_lookup_fk(kv: u32) -> u32 { + kv +} + +/// Extract the value for a boolean lookup. +#[inline] +fn bool_lookup_fv(_kv: u32) -> bool { + true +} + +/// Extract the key in a pair. +#[inline] +fn pair_lookup_fk(kv: (u32, T)) -> u32 { + kv.0 +} + +/// Extract the value in a pair, returning an option. +#[inline] +fn pair_lookup_fv_opt(kv: (u32, T)) -> Option { + Some(kv.1) +} diff --git a/src/normalize.rs b/src/normalize.rs index c2a1e4d..8208634 100644 --- a/src/normalize.rs +++ b/src/normalize.rs @@ -11,7 +11,7 @@ //! Functions for computing canonical and compatible decompositions for Unicode characters. use std::char; use std::ops::FnMut; -use perfect_hash::{canonical_fully_decomposed, composition_table, compatibility_fully_decomposed}; +use lookups::{canonical_fully_decomposed, composition_table, compatibility_fully_decomposed}; /// Compute canonical Unicode decomposition for character. /// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/) diff --git a/src/perfect_hash.rs b/src/perfect_hash.rs index c9157fe..0a81714 100644 --- a/src/perfect_hash.rs +++ b/src/perfect_hash.rs @@ -10,8 +10,6 @@ //! Support for lookups based on minimal perfect hashing. -use tables::*; - // This function is based on multiplication being fast and is "good enough". Also // it can share some work between the unsalted and salted versions. #[inline] @@ -29,7 +27,7 @@ fn my_hash(key: u32, salt: u32, n: usize) -> usize { /// The hash function doesn't have to be very good, just good enough that the /// resulting map is unique. #[inline] -fn mph_lookup(x: u32, salt: &[u16], kv: &[KV], fk: FK, fv: FV, +pub(crate) fn mph_lookup(x: u32, salt: &[u16], kv: &[KV], fk: FK, fv: FV, default: V) -> V where KV: Copy, FK: Fn(KV) -> u32, FV: Fn(KV) -> V { @@ -41,78 +39,3 @@ fn mph_lookup(x: u32, salt: &[u16], kv: &[KV], fk: FK, fv: FV, default } } - -/// Extract the key in a 24 bit key and 8 bit value packed in a u32. -#[inline] -fn u8_lookup_fk(kv: u32) -> u32 { - kv >> 8 -} - -/// Extract the value in a 24 bit key and 8 bit value packed in a u32. -#[inline] -fn u8_lookup_fv(kv: u32) -> u8 { - (kv & 0xff) as u8 -} - -/// Extract the key for a boolean lookup. -#[inline] -fn bool_lookup_fk(kv: u32) -> u32 { - kv -} - -/// Extract the value for a boolean lookup. -#[inline] -fn bool_lookup_fv(_kv: u32) -> bool { - true -} - -/// Extract the key in a pair. -#[inline] -fn pair_lookup_fk(kv: (u32, T)) -> u32 { - kv.0 -} - -/// Extract the value in a pair, returning an option. -#[inline] -fn pair_lookup_fv_opt(kv: (u32, T)) -> Option { - Some(kv.1) -} - -/// Look up the canonical combining class for a codepoint. -/// -/// The value returned is as defined in the Unicode Character Database. -pub fn canonical_combining_class(c: char) -> u8 { - mph_lookup(c.into(), CANONICAL_COMBINING_CLASS_SALT, CANONICAL_COMBINING_CLASS_KV, - u8_lookup_fk, u8_lookup_fv, 0) -} - -pub(crate) fn composition_table(c1: char, c2: char) -> Option { - if c1 < '\u{10000}' && c2 < '\u{10000}' { - mph_lookup((c1 as u32) << 16 | (c2 as u32), - COMPOSITION_TABLE_SALT, COMPOSITION_TABLE_KV, - pair_lookup_fk, pair_lookup_fv_opt, None) - } else { - composition_table_astral(c1, c2) - } -} - -pub(crate) fn canonical_fully_decomposed(c: char) -> Option<&'static [char]> { - mph_lookup(c.into(), CANONICAL_DECOMPOSED_SALT, CANONICAL_DECOMPOSED_KV, - pair_lookup_fk, pair_lookup_fv_opt, None) -} - -pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]> { - mph_lookup(c.into(), COMPATIBILITY_DECOMPOSED_SALT, COMPATIBILITY_DECOMPOSED_KV, - pair_lookup_fk, pair_lookup_fv_opt, None) -} - -/// Return whether the given character is a combining mark (`General_Category=Mark`) -pub fn is_combining_mark(c: char) -> bool { - mph_lookup(c.into(), COMBINING_MARK_SALT, COMBINING_MARK_KV, - bool_lookup_fk, bool_lookup_fv, false) -} - -pub fn stream_safe_trailing_nonstarters(c: char) -> usize { - mph_lookup(c.into(), TRAILING_NONSTARTERS_SALT, TRAILING_NONSTARTERS_KV, - u8_lookup_fk, u8_lookup_fv, 0) as usize -} diff --git a/src/quick_check.rs b/src/quick_check.rs index c12768a..49b1460 100644 --- a/src/quick_check.rs +++ b/src/quick_check.rs @@ -1,5 +1,5 @@ use UnicodeNormalization; -use perfect_hash::canonical_combining_class; +use lookups::canonical_combining_class; use stream_safe; use tables; diff --git a/src/stream_safe.rs b/src/stream_safe.rs index cbb0304..38bb42c 100644 --- a/src/stream_safe.rs +++ b/src/stream_safe.rs @@ -2,7 +2,7 @@ use normalize::{ hangul_decomposition_length, is_hangul_syllable, }; -use perfect_hash::{ +use lookups::{ canonical_combining_class, canonical_fully_decomposed, compatibility_fully_decomposed, stream_safe_trailing_nonstarters, }; @@ -113,7 +113,7 @@ mod tests { use std::char; use normalization_tests::NORMALIZATION_TESTS; use normalize::decompose_compatible; - use perfect_hash::canonical_combining_class; + use lookups::canonical_combining_class; fn stream_safe(s: &str) -> String { StreamSafe::new(s.chars()).collect()