store smaller slices in unicode data tables

Rust's default slices are convenient, but for tables like:

  const f: &'static [(char, &'static [char])]

they take up far too much space.  An element of the above array consumes
24 bytes on 64-bit platforms, and unicode-normalization contains about
6000 such array elements.

A better approach is to manually store a smaller slice type:

  struct Slice {
    offset: u16,
    length: u16,
  }

  const f: &'static [(char, Slice)]

and store the actual character data in a separate array on the side.
The `Slice` structures then point in to this separate array, but at a
much smaller space cost: elements of the modified `f` take up only 8
bytes on 64-bit platforms, which implies a space savings of ~96K on
64-bit platforms.  On some systems, this strategy also eliminates the
necessity of run-time relocations, which can be a further, significant
savings in binary size and runtime cost.

This change is strictly local to the library; it does not affect the
public API.

scripts/unicode.py

@@ -18,7 +18,7 @@
 # Since this should not require frequent updates, we just store this
 # out-of-line and check the unicode.rs file into git.
 
-import fileinput, re, os, sys
+import fileinput, re, os, sys, collections
 
 preamble = '''// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
 // file at the top-level directory of this distribution and at
@@ -213,6 +213,43 @@ def emit_table(f, name, t_data, t_type = "&'static [(char, char)]", is_pub=True,
     format_table_content(f, [pfun(d) for d in t_data], 8)
     f.write("\n    ];\n\n")
 
+def emit_strtab_table(f, name, keys, vfun, is_pub=True,
+                      tab_entry_type='char', slice_element_sfun=escape_char):
+    pub_string = ""
+    if is_pub:
+        pub_string = "pub "
+    f.write("    %s const %s: &'static [(char, Slice)] = &[\n"
+            % (pub_string, name))
+
+    strtab = collections.OrderedDict()
+    strtab_offset = 0
+
+    # TODO: a more sophisticated algorithm here would not only check for the
+    # existence of v in the strtab, but also v in contiguous substrings of
+    # strtab, if that's possible.
+    for k in keys:
+        v = tuple(vfun(k))
+        if v in strtab:
+            item_slice = strtab[v]
+        else:
+            value_len = len(v)
+            item_slice = (strtab_offset, value_len)
+            strtab[v] = item_slice
+            strtab_offset += value_len
+
+        f.write("%s(%s, Slice { offset: %d, length: %d }),\n"
+                % (" "*8, escape_char(k), item_slice[0], item_slice[1]))
+
+    f.write("\n    ];\n\n")
+
+    f.write("    %s const %s_STRTAB: &'static [%s] = &[\n"
+            % (pub_string, name, tab_entry_type))
+
+    for (v, _) in strtab.iteritems():
+        f.write("%s%s,\n" % (" "*8, ', '.join(slice_element_sfun(c) for c in v)))
+
+    f.write("\n    ];\n\n")
+
 def emit_norm_module(f, canon, compat, combine, norm_props, general_category_mark):
     canon_keys = canon.keys()
     canon_keys.sort()
@@ -234,35 +271,38 @@ def emit_norm_module(f, canon, compat, combine, norm_props, general_category_mar
     canon_comp_keys.sort()
 
     f.write("pub mod normalization {\n")
+    f.write("""
+pub struct Slice {
+  pub offset: u16,
+  pub length: u16,
+}
+""")
 
     def mkdata_fun(table):
         def f(char):
-            data = "(%s,&[" % escape_char(char)
-            chars = [escape_char(d) for d in table[char]]
-            data += ','.join(chars)
-            data += "])"
-            return data
+            return table[char]
         return f
 
+    # TODO: should the strtab of these two tables be of type &'static str, for
+    # smaller data?
     f.write("    // Canonical decompositions\n")
-    emit_table(f, "canonical_table", canon_keys, "&'static [(char, &'static [char])]",
-        pfun=mkdata_fun(canon))
+    emit_strtab_table(f, "canonical_table", canon_keys,
+                      vfun=mkdata_fun(canon))
 
     f.write("    // Compatibility decompositions\n")
-    emit_table(f, "compatibility_table", compat_keys, "&'static [(char, &'static [char])]",
-        pfun=mkdata_fun(compat))
+    emit_strtab_table(f, "compatibility_table", compat_keys,
+                      vfun=mkdata_fun(compat))
 
-    def comp_pfun(char):
-        data = "(%s,&[" % escape_char(char)
-        canon_comp[char].sort(lambda x, y: x[0] - y[0])
-        data += ','.join("(%s,%s)" % (escape_char(pair[0]), escape_char(pair[1]))
-                         for pair in canon_comp[char])
-        data += "])"
-        return data
+    def comp_vfun(char):
+        return sorted(canon_comp[char], lambda x, y: x[0] - y[0])
 
     f.write("    // Canonical compositions\n")
-    emit_table(f, "composition_table", canon_comp_keys,
-        "&'static [(char, &'static [(char, char)])]", pfun=comp_pfun)
+    # "&'static [(char, &'static [(char, char)])]", pfun=comp_pfun)
+    emit_strtab_table(f, "composition_table", canon_comp_keys,
+                      vfun=comp_vfun,
+                      tab_entry_type="(char, char)",
+                      slice_element_sfun=lambda pair: "(%s,%s)" % (escape_char(pair[0]),
+                                                                   escape_char(pair[1])))
 
     f.write("""
     fn bsearch_range_value_table(c: char, r: &'static [(char, char, u8)]) -> u8 {