openharmony/third_party_rust_unicode-normalization
1
0
Fork 0
mirror of https://github.com/openharmony/third_party_rust_unicode-normalization.git synced 2026-07-19 15:03:34 -04:00
Code Issues Packages Projects Releases Wiki Activity

More lookups moved to perfect hashing

Browse Source
This commit is contained in:
Raph Levien
2019-04-09 16:46:28 -07:00
parent 274b267525
commit b68dbec11f
3 changed files with 6425 additions and 3215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/unicode.py
+23 -17
View File
@@ -287,7 +287,6 @@ class UnicodeData(object):
hexify = lambda c: '{:04X}'.format(c)
def gen_combining_class(combining_classes, out):
out.write("#[inline]\n")
(salt, keys) = minimal_perfect_hash(combining_classes)
out.write("pub fn canonical_combining_class(c: char) -> u8 {\n")
out.write(" mph_lookup(c.into(), &[\n")
@@ -299,7 +298,7 @@ def gen_combining_class(combining_classes, out):
kv = int(combining_classes[k]) | (k << 8)
out.write(" 0x{:x},\n".format(kv))
out.write(" ],\n")
out.write(" u8_lookup_fk, u8_lookup_fv, 0) as u8\n")
out.write(" u8_lookup_fk, u8_lookup_fv, 0)\n")
out.write("}\n")
def gen_composition_table(canon_comp, out):
@@ -376,16 +375,20 @@ def gen_nfkd_qc(prop_tables, out):
def gen_combining_mark(general_category_mark, out):
out.write("#[inline]\n")
(salt, keys) = minimal_perfect_hash(general_category_mark)
out.write("pub fn is_combining_mark(c: char) -> bool {\n")
out.write(" match c {\n")
for char in sorted(general_category_mark):
out.write(" '\\u{%s}' => true,\n" % hexify(char))
out.write(" _ => false,\n")
out.write(" }\n")
out.write(" mph_lookup(c.into(), &[\n")
for s in salt:
out.write(" 0x{:x},\n".format(s))
out.write(" ],\n")
out.write(" &[\n")
for k in keys:
out.write(" 0x{:x},\n".format(k))
out.write(" ],\n")
out.write(" bool_lookup_fk, bool_lookup_fv, false)\n")
out.write("}\n")
def gen_stream_safe(leading, trailing, out):
out.write("#[inline]\n")
out.write("pub fn stream_safe_leading_nonstarters(c: char) -> usize {\n")
@@ -399,15 +402,18 @@ def gen_stream_safe(leading, trailing, out):
out.write("}\n")
out.write("\n")
out.write("#[inline]\n")
(salt, keys) = minimal_perfect_hash(trailing)
out.write("pub fn stream_safe_trailing_nonstarters(c: char) -> usize {\n")
out.write(" match c {\n")
for char, num_trailing in sorted(trailing.items()):
out.write(" '\\u{%s}' => %d,\n" % (hexify(char), num_trailing))
out.write(" _ => 0,\n")
out.write(" }\n")
out.write(" mph_lookup(c.into(), &[\n")
for s in salt:
out.write(" 0x{:x},\n".format(s))
out.write(" ],\n")
out.write(" &[\n")
for k in keys:
kv = int(trailing[k]) | (k << 8)
out.write(" 0x{:x},\n".format(kv))
out.write(" ],\n")
out.write(" u8_lookup_fk, u8_lookup_fv, 0) as usize\n")
out.write("}\n")
def gen_tests(tests, out):