openharmony/third_party_rust_unicode-normalization
1
0
Fork 0
mirror of https://github.com/openharmony/third_party_rust_unicode-normalization.git synced 2026-07-18 21:14:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add new normalization algorithms using Standardized Variants

Browse Source 
The standard normalization algorithm decomposes CJK compatibility ideographs
into nominally equivalent codepoints, but which traditionally look different,
and is one of the main reasons normalization is considered destructive in
practice.

[Unicode 6.3] introduced a solution for this, by providing
[standardized variation sequences] for these codepoints. For example, while
U+2F8A6 "CJK COMPATIBILITY-IDEOGRAPH-2F8A6" canonically decomposes to U+6148
with a different appearance, in Unicode 6.3 and later the standardized variation
sequences in the StandardizedVariants.txt file include the following:

> 6148 FE00; CJK COMPATIBILITY IDEOGRAPH-2F8A6;

which says that "CJK COMPATIBILITY IDEOGRAPH-2F8A6" corresponds to
U+6148 U+FE00, where U+FE00 is "VARIATION SELECTOR-1".

U+6148 and U+FE00 are both normalized codepoints, so we can transform text
containing U+2F8A6 into normal form without losing information about the
distinct appearance. At this time, many popular implementations ignore these
variation selectors, however this technique at least preserves the information
in a standardized way, so implementations could use it if they chose.

This PR adds "ext" versions of the `nfd`, `nfc`, `nfkd`, and `nkfd`
iterators, which perform the standard algorithms extended with this technique.
They don't match the standard decompositions, and don't guarantee stability,
but they do produce appropriately normalized output.

I used the generic term "ext" to reflect that other extensions could
theoretically be added in the future. The standard decomposition tables are
limited by their stability requirements, but these "ext" versions could be
free to adopt new useful rules.

I'm not an expert in any of these topics, so please correct me if I'm mistaken
in any of this. Also, I'm open to ideas about how to best present this
functionality in the API.

[Unicode 6.3]: https://www.unicode.org/versions/Unicode6.3.0/#Summary
[standardized variation sequences]: http://unicode.org/faq/vs.html
This commit is contained in:
Dan Gohman
2020-12-07 08:16:58 -08:00
parent 8dfab5ee50
commit 107879735b
8 changed files with 2354 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/unicode.py
+53 -9
View File
@@ -14,9 +14,10 @@
# - DerivedNormalizationProps.txt
# - NormalizationTest.txt
# - UnicodeData.txt
# - StandardizedVariants.txt
#
# Since this should not require frequent updates, we just store this
# out-of-line and check the unicode.rs file into git.
# out-of-line and check the tables.rs and normalization_tests.rs files into git.
import collections
import urllib.request
@@ -57,6 +58,11 @@ expanded_categories = {
'Cc': ['C'], 'Cf': ['C'], 'Cs': ['C'], 'Co': ['C'], 'Cn': ['C'],
}
# Constants from Unicode 9.0.0 Section 3.12 Conjoining Jamo Behavior
# http://www.unicode.org/versions/Unicode9.0.0/ch03.pdf#M9.32468.Heading.310.Combining.Jamo.Behavior
S_BASE, L_COUNT, V_COUNT, T_COUNT = 0xAC00, 19, 21, 28
S_COUNT = L_COUNT * V_COUNT * T_COUNT
class UnicodeData(object):
def __init__(self):
self._load_unicode_data()
@@ -66,14 +72,20 @@ class UnicodeData(object):
self.canon_comp = self._compute_canonical_comp()
self.canon_fully_decomp, self.compat_fully_decomp = self._compute_fully_decomposed()
self.ext_decomp = {}
self.ext_fully_decomp = {}
self._load_standardized_variants()
def stats(name, table):
count = sum(len(v) for v in table.values())
print("%s: %d chars => %d decomposed chars" % (name, len(table), count))
print("Decomposition table stats:")
stats("Canonical decomp", self.canon_decomp)
stats("Canonical decomp with extensions", self.ext_decomp)
stats("Compatible decomp", self.compat_decomp)
stats("Canonical fully decomp", self.canon_fully_decomp)
stats("Canonical fully decomp with extensions", self.ext_fully_decomp)
stats("Compatible fully decomp", self.compat_fully_decomp)
self.ss_leading, self.ss_trailing = self._compute_stream_safe_tables()
@@ -83,6 +95,7 @@ class UnicodeData(object):
return resp.read().decode('utf-8')
def _load_unicode_data(self):
self.name_to_char_int = {}
self.combining_classes = {}
self.compat_decomp = {}
self.canon_decomp = {}
@@ -95,6 +108,9 @@ class UnicodeData(object):
char, category, cc, decomp = pieces[0], pieces[2], pieces[3], pieces[5]
char_int = int(char, 16)
name = pieces[1].strip()
self.name_to_char_int[name] = char_int
if cc != '0':
self.combining_classes[char_int] = cc
@@ -106,6 +122,39 @@ class UnicodeData(object):
if category == 'M' or 'M' in expanded_categories.get(category, []):
self.general_category_mark.append(char_int)
def _load_standardized_variants(self):
for line in self._fetch("StandardizedVariants.txt").splitlines():
strip_comments = line.split('#', 1)[0].strip()
if not strip_comments:
continue
pieces = strip_comments.split(';')
assert len(pieces) == 3
variation_sequence, description, differences = pieces[0], pieces[1].strip(), pieces[2]
# Don't use variations that only apply in particular shaping environments.
if differences:
continue
# Look for entries where the description field is a codepoint name.
if description in self.name_to_char_int:
char_int = self.name_to_char_int[description]
assert not char_int in self.combining_classes, "Unexpected: standardized variant with a combining class"
assert not char_int in self.compat_decomp, "Unexpected: standardized variant and compatibility decomposition"
assert len(self.canon_decomp[char_int]) == 1, "Unexpected: standardized variant and non-singleton canonical decomposition"
# If we ever need to handle Hangul here, we'll need to handle it separately.
assert not (S_BASE <= char_int < S_BASE + S_COUNT)
standardized_variant_parts = [int(c, 16) for c in variation_sequence.split()]
for c in standardized_variant_parts:
#assert not never_composes(c) TODO: Re-enable this once #67 lands.
assert not c in self.canon_decomp, "Unexpected: standardized variant is unnormalized (canon)"
assert not c in self.compat_decomp, "Unexpected: standardized variant is unnormalized (compat)"
self.ext_decomp[char_int] = standardized_variant_parts
self.ext_fully_decomp[char_int] = standardized_variant_parts
def _load_norm_props(self):
props = collections.defaultdict(list)
@@ -178,11 +227,6 @@ class UnicodeData(object):
The upshot is that decomposition code is very simple and easy to inline
at mild code size cost.
"""
# Constants from Unicode 9.0.0 Section 3.12 Conjoining Jamo Behavior
# http://www.unicode.org/versions/Unicode9.0.0/ch03.pdf#M9.32468.Heading.310.Combining.Jamo.Behavior
S_BASE, L_COUNT, V_COUNT, T_COUNT = 0xAC00, 19, 21, 28
S_COUNT = L_COUNT * V_COUNT * T_COUNT
def _decompose(char_int, compatible):
# 7-bit ASCII never decomposes
if char_int <= 0x7f:
@@ -320,8 +364,8 @@ def gen_composition_table(canon_comp, out):
out.write(" }\n")
out.write("}\n")
def gen_decomposition_tables(canon_decomp, compat_decomp, out):
tables = [(canon_decomp, 'canonical'), (compat_decomp, 'compatibility')]
def gen_decomposition_tables(canon_decomp, ext_decomp, compat_decomp, out):
tables = [(canon_decomp, 'canonical'), (ext_decomp, 'ext'), (compat_decomp, 'compatibility')]
for table, name in tables:
gen_mph_data(name + '_decomposed', table, "(u32, &'static [char])",
lambda k: "(0x{:x}, &[{}])".format(k,
@@ -491,7 +535,7 @@ if __name__ == '__main__':
gen_composition_table(data.canon_comp, out)
out.write("\n")
gen_decomposition_tables(data.canon_fully_decomp, data.compat_fully_decomp, out)
gen_decomposition_tables(data.canon_fully_decomp, data.ext_fully_decomp, data.compat_fully_decomp, out)
gen_combining_mark(data.general_category_mark, out)
out.write("\n")