Fix is_public_assigned to include Hangul Syllable and other ranges.

Hangul Syllables and several other ranges are defined in UnicodeData.txt
as just their first and last values:

```
AC00;<Hangul Syllable, First>;Lo;0;L;;;;;N;;;;;
D7A3;<Hangul Syllable, Last>;Lo;0;L;;;;;N;;;;;
```

Teach the unicode.py script how to recognize these, so that it correctly
classifies them as assigned ranges, for the `is_public_assigned`
predicate.
This commit is contained in:
Dan Gohman
2021-05-28 14:15:26 -07:00
parent 74f416f8ea
commit 33e73008da
3 changed files with 70 additions and 23 deletions
+13 -2
View File
@@ -102,12 +102,13 @@ class UnicodeData(object):
assigned_start = 0;
prev_char_int = -1;
prev_name = "";
for line in self._fetch("UnicodeData.txt").splitlines():
# See ftp://ftp.unicode.org/Public/3.0-Update/UnicodeData-3.0.0.html
pieces = line.split(';')
assert len(pieces) == 15
char, category, cc, decomp = pieces[0], pieces[2], pieces[3], pieces[5]
char, name, category, cc, decomp = pieces[0], pieces[1], pieces[2], pieces[3], pieces[5]
char_int = int(char, 16)
name = pieces[1].strip()
@@ -126,10 +127,11 @@ class UnicodeData(object):
assert category != 'Cn', "Unexpected: Unassigned codepoint in UnicodeData.txt"
if category not in ['Co', 'Cs']:
if char_int != prev_char_int + 1:
if char_int != prev_char_int + 1 and not is_first_and_last(prev_name, name):
self.general_category_public_assigned.append((assigned_start, prev_char_int))
assigned_start = char_int
prev_char_int = char_int
prev_name = name;
self.general_category_public_assigned.append((assigned_start, prev_char_int))
@@ -343,6 +345,15 @@ class UnicodeData(object):
hexify = lambda c: '{:04X}'.format(c)
# Test whether `first` and `last` are corresponding "<..., First>" and
# "<..., Last>" markers.
def is_first_and_last(first, last):
if not first.startswith('<') or not first.endswith(', First>'):
return False
if not last.startswith('<') or not last.endswith(', Last>'):
return False
return first[1:-8] == last[1:-7]
def gen_mph_data(name, d, kv_type, kv_callback):
(salt, keys) = minimal_perfect_hash(d)
out.write("pub(crate) const %s_SALT: &[u16] = &[\n" % name.upper())