Implement fast-gb-hanzi-encode.

Cargo.toml

@@ -21,6 +21,7 @@ less-slow-big5-hanzi-encode = []
 less-slow-gb-hanzi-encode = []
 fast-hangul-encode = []
 fast-kanji-encode = []
+fast-gb-hanzi-encode = []
 
 [dependencies]
 cfg-if = "0.1.0"

generate-encoding-data.py

@@ -846,8 +846,8 @@ static_u16_table("KSX1001_OTHER_UNSORTED_OFFSETS", offsets[:-1])
 hangul_bytes = [None] * (0xD7A4 - 0xAC00)
 for row in xrange(73):
   for column in xrange(190):
-    i = column + (row * 190)
-    code_point = index[i]
+    pointer = column + (row * 190)
+    code_point = index[pointer]
     if code_point and code_point >= 0xAC00 and code_point <= 0xD7A3:
       hangul_lead = 0x81 + row
       hangul_trail = 0x41 + column
@@ -1057,6 +1057,19 @@ level1_hanzi_pairs.sort(key=lambda x: x[0])
 static_u16_table_from_indexable("GB2312_LEVEL1_HANZI_CODE_POINTS", level1_hanzi_pairs, 0, "gb-hanzi-encode")
 static_u8_pair_table_from_indexable("GB2312_LEVEL1_HANZI_BYTES", level1_hanzi_pairs, 1, "gb-hanzi-encode")
 
+# Fast Hanzi encoder table
+hanzi_bytes = [None] * (0x9FA7 - 0x4E00)
+for row in xrange(126):
+  for column in xrange(190):
+    pointer = column + (row * 190)
+    code_point = index[pointer]
+    if code_point and code_point >= 0x4E00 and code_point <= 0x9FA6:
+      hanzi_lead = 0x81 + row
+      hanzi_trail = column + (0x40 if column < 0x3F else 0x41)
+      hanzi_bytes[code_point - 0x4E00] = (hanzi_lead, hanzi_trail)
+
+static_u8_pair_table("GBK_HANZI_BYTES", hanzi_bytes, "fast-gb-hanzi-encode")
+
 data_file.write('''#[inline(always)]
 fn map_with_ranges(haystack: &[u16], other: &[u16], needle: u16) -> u16 {
     debug_assert_eq!(haystack.len(), other.len());
@@ -1112,6 +1125,7 @@ pub fn gbk_top_ideograph_decode(pointer: u16) -> u16 {
     )
 }
 
+#[cfg(not(feature = "fast-gb-hanzi-encode"))]
 #[inline(always)]
 pub fn gbk_top_ideograph_encode(bmp: u16) -> u16 {
     map_with_ranges(
@@ -1130,6 +1144,7 @@ pub fn gbk_left_ideograph_decode(pointer: u16) -> u16 {
     )
 }
 
+#[cfg(not(feature = "fast-gb-hanzi-encode"))]
 #[inline(always)]
 pub fn gbk_left_ideograph_encode(bmp: u16) -> u16 {
     map_with_ranges(
@@ -1220,7 +1235,17 @@ pub fn gb2312_other_encode(bmp: u16) -> Option<u16> {
     )
 }
 
-#[cfg(not(feature = "less-slow-gb-hanzi-encode"))]
+#[cfg(feature = "fast-gb-hanzi-encode")]
+#[inline(always)]
+pub fn gbk_hanzi_encode(bmp_minus_start: u16) -> (u8, u8) {
+    let pair = &GBK_HANZI_BYTES[bmp_minus_start as usize];
+    (pair[0], pair[1])
+}
+
+#[cfg(not(any(
+    feature = "less-slow-gb-hanzi-encode",
+    feature = "fast-gb-hanzi-encode")
+))]
 #[inline(always)]
 pub fn gb2312_level1_hanzi_encode(bmp: u16) -> Option<(u8, u8)> {
     position(&GB2312_HANZI[..(94 * (0xD8 - 0xB0) - 5)], bmp).map(|hanzi_pointer| {
@@ -1230,7 +1255,10 @@ pub fn gb2312_level1_hanzi_encode(bmp: u16) -> Option<(u8, u8)> {
     })
 }
 
-#[cfg(feature = "less-slow-gb-hanzi-encode")]
+#[cfg(all(
+    feature = "less-slow-gb-hanzi-encode",
+    not(feature = "fast-gb-hanzi-encode")
+))]
 #[inline(always)]
 pub fn gb2312_level1_hanzi_encode(bmp: u16) -> Option<(u8, u8)> {
     match GB2312_LEVEL1_HANZI_CODE_POINTS.binary_search(&bmp) {
@@ -1242,6 +1270,7 @@ pub fn gb2312_level1_hanzi_encode(bmp: u16) -> Option<(u8, u8)> {
     }
 }
 
+#[cfg(not(feature = "fast-gb-hanzi-encode"))]
 #[inline(always)]
 pub fn gb2312_level2_hanzi_encode(bmp: u16) -> Option<usize> {
     // TODO: optimize

src/gb18030.rs

@@ -395,6 +395,40 @@ fn gbk_encode_non_unified(bmp: u16) -> Option<(usize, usize)> {
     None
 }
 
+#[cfg(not(feature = "fast-gb-hanzi-encode"))]
+#[inline(always)]
+fn encode_hanzi(bmp: u16, _: u16) -> (u8, u8) {
+    if let Some((lead, trail)) = gb2312_level1_hanzi_encode(bmp) {
+        (lead, trail)
+    } else if let Some(hanzi_pointer) = gb2312_level2_hanzi_encode(bmp) {
+        let hanzi_lead = (hanzi_pointer / 94) + (0xD8);
+        let hanzi_trail = (hanzi_pointer % 94) + 0xA1;
+        (hanzi_lead as u8, hanzi_trail as u8)
+    } else {
+        let (lead, gbk_trail) = if bmp < 0x72DC {
+            // Above GB2312
+            let pointer = gbk_top_ideograph_encode(bmp) as usize;
+            let lead = (pointer / 190) + 0x81;
+            let gbk_trail = pointer % 190;
+            (lead, gbk_trail)
+        } else {
+            // To the left of GB2312
+            let gbk_left_ideograph_pointer = gbk_left_ideograph_encode(bmp) as usize;
+            let lead = (gbk_left_ideograph_pointer / (190 - 94)) + (0x81 + 0x29);
+            let gbk_trail = gbk_left_ideograph_pointer % (190 - 94);
+            (lead, gbk_trail)
+        };
+        let offset = if gbk_trail < 0x3F { 0x40 } else { 0x41 };
+        (lead as u8, (gbk_trail + offset) as u8)
+    }
+}
+
+#[cfg(feature = "fast-gb-hanzi-encode")]
+#[inline(always)]
+fn encode_hanzi(_: u16, bmp_minus_unified_start: u16) -> (u8, u8) {
+    gbk_hanzi_encode(bmp_minus_unified_start)
+}
+
 pub struct Gb18030Encoder {
     extended: bool,
 }
@@ -451,33 +485,8 @@ impl Gb18030Encoder {
                 // CJK Unified Ideographs
                 // Can't fail now, since all are
                 // mapped.
-                // XXX Can we do something smarter
-                // than linear search for GB2312
-                // Level 2 Hanzi, which are almost
-                // Unicode-ordered?
-                if let Some((lead, trail)) = gb2312_level1_hanzi_encode(bmp) {
-                    handle.write_two(lead, trail)
-                } else if let Some(hanzi_pointer) = gb2312_level2_hanzi_encode(bmp) {
-                    let hanzi_lead = (hanzi_pointer / 94) + (0xD8);
-                    let hanzi_trail = (hanzi_pointer % 94) + 0xA1;
-                    handle.write_two(hanzi_lead as u8, hanzi_trail as u8)
-                } else {
-                    let (lead, gbk_trail) = if bmp < 0x72DC {
-                        // Above GB2312
-                        let pointer = gbk_top_ideograph_encode(bmp) as usize;
-                        let lead = (pointer / 190) + 0x81;
-                        let gbk_trail = pointer % 190;
-                        (lead, gbk_trail)
-                    } else {
-                        // To the left of GB2312
-                        let gbk_left_ideograph_pointer = gbk_left_ideograph_encode(bmp) as usize;
-                        let lead = (gbk_left_ideograph_pointer / (190 - 94)) + (0x81 + 0x29);
-                        let gbk_trail = gbk_left_ideograph_pointer % (190 - 94);
-                        (lead, gbk_trail)
-                    };
-                    let offset = if gbk_trail < 0x3F { 0x40 } else { 0x41 };
-                    handle.write_two(lead as u8, (gbk_trail + offset) as u8)
-                }
+                let (lead, trail) = encode_hanzi(bmp, bmp_minus_unified_start);
+                handle.write_two(lead, trail)
             } else if bmp == 0xE5E5 {
                 // It's not optimal to check for the unmappable
                 // and for euro at this stage, but getting