Bug 1533481 - Part 4: Prepare update to ICU 64. r=jwalden!

Summary:

- ICU no longer supports removing resource files manually, instead the new data
  filter builder needs to be used.
- This includes test data, which also must be preserved in ICU 64.
- umutex.h contains an implicit call to `new` in ICU64 which needs to be allowed
  in check_vanilla_allocations.py.
- Drive-by: Make ICU with multiple jobs for faster rebuilds.

Reviewers: jwalden

Reviewed By: jwalden

Bug #: 1533481

Differential Revision: https://phabricator.services.mozilla.com/D25264

--HG--
extra : rebase_source : 9fde775ee9b189c84a29f69f11ebd955f0030b38

config/check_vanilla_allocations.py

@@ -150,6 +150,14 @@ def main():
         if "ProfilingStack" in filename:
             continue
 
+        # Ignore implicit call to operator new in std::condition_variable_any.
+        #
+        # From intl/icu/source/common/umutex.h:
+        # On Linux, the default constructor of std::condition_variable_any
+        # produces an in-line reference to global operator new(), [...].
+        if filename == 'umutex.o':
+            continue
+
         fn = m.group(2)
         if filename == 'jsutil.o':
             jsutil_cpp.add(fn)

intl/icu/data_filter.json

@@ -0,0 +1,44 @@
+{
+    "featureFilters": {
+        "brkitr_dictionaries": {
+            "filterType": "exclude"
+        },
+        "brkitr_rules": {
+            "filterType": "exclude"
+        },
+        "brkitr_tree": {
+            "filterType": "exclude"
+        },
+        "conversion_mappings": {
+            "whitelist": [
+                "ibm-37_P100-1995",
+                "ibm-1047_P100-1995"
+            ]
+        },
+        "lang_tree": {
+            "filterType": "exclude"
+        },
+        "rbnf_tree": {
+            "filterType": "exclude"
+        },
+        "region_tree": {
+            "filterType": "exclude"
+        },
+        "translit": {
+            "filterType": "exclude"
+        },
+        "unit_tree": {
+            "filterType": "exclude"
+        }
+    },
+    "resourceFilters": [
+        {
+            "categories": [
+                "zone_tree"
+            ],
+            "rules": [
+                "-/zoneStrings/*/ec"
+            ]
+        }
+    ]
+}

intl/icu_sources_data.py

@@ -13,6 +13,7 @@
 from __future__ import print_function
 
 import glob
+import multiprocessing
 import os
 import sets
 import shutil
@@ -217,6 +218,13 @@ def update_data_file(topsrcdir):
                      '-DUCONFIG_NO_BREAK_ITERATION ' +
                      '-DU_CHARSET_IS_UTF8')
     })
+
+    # Exclude data that we currently don't need.
+    #
+    # The file format for ICU's data build tool is described at
+    # <https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md>.
+    env["ICU_DATA_FILTER_FILE"] = mozpath.join(topsrcdir, 'intl/icu/data_filter.json')
+
     print('Running ICU configure...')
     if not try_run(
             'icu-configure',
@@ -235,7 +243,12 @@ def update_data_file(topsrcdir):
             env=env):
         return False
     print('Running ICU make...')
-    if not try_run('icu-make', ['make'], cwd=objdir):
+    if not try_run(
+            'icu-make',
+            ['make',
+             '--jobs=%d' % multiprocessing.cpu_count(),
+             '--output-sync'],
+            cwd=objdir):
         return False
     print('Copying ICU data file...')
     tree_data_path = mozpath.join(topsrcdir,

intl/update-icu.sh

@@ -44,43 +44,11 @@ git -C ${tmpclonedir} log -1 > ${icu_dir}/GIT-INFO
 # Clean up after ourselves.
 rm -rf ${tmpclonedir}
 
-# remove layoutex, tests, and samples, but leave makefiles in place
+# Remove layoutex, tests, and samples, but leave makefiles and test data in place.
 find ${icu_dir}/source/layoutex -name '*Makefile.in' -prune -or -type f -print | xargs rm
-find ${icu_dir}/source/test -name '*Makefile.in' -prune -or -type f -print | xargs rm
+find ${icu_dir}/source/test -name '*Makefile.in' -prune -or -name 'testdata' -prune -or -type f -print | xargs rm
 find ${icu_dir}/source/samples -name '*Makefile.in' -prune -or -type f -print | xargs rm
 
-# remove data that we currently don't need
-rm -rf ${icu_dir}/source/data/brkitr/*
-rm ${icu_dir}/source/data/lang/*.mk
-rm ${icu_dir}/source/data/lang/*.txt
-rm ${icu_dir}/source/data/mappings/*.mk
-find ${icu_dir}/source/data/mappings \
-    -name ibm-37_P100-1995.ucm -prune -or \
-    -name ibm-1047_P100-1995.ucm -prune -or \
-    -name '*.ucm' -print | xargs rm
-rm ${icu_dir}/source/data/rbnf/*
-rm ${icu_dir}/source/data/region/*.mk
-rm ${icu_dir}/source/data/region/*.txt
-rm ${icu_dir}/source/data/translit/*
-rm ${icu_dir}/source/data/unit/*.mk
-rm ${icu_dir}/source/data/unit/*.txt
-
-# Remove all exemplar cities 'ec'. (bug 1225401 and bug 1345336)
-find ${icu_dir}/source/data/zone \
-    -name root.txt -prune -or \
-    -name tzdbNames.txt -prune -or \
-    -name '*.txt' -print | xargs sed -i '/^\s*ec{\".*\"}$/ { d }'
-# Remove empty time zone entries after exemplar cities removal.
-find ${icu_dir}/source/data/zone \
-    -name root.txt -prune -or \
-    -name tzdbNames.txt -prune -or \
-    -name '*.txt' -print | xargs sed -i '/^\s*\"[A-Z][a-zA-Z:_-]*\"{/{N; s/^\s*\"[A-Z][a-zA-Z:_-]*\"{\n\s*}// }; /^$/d'
-# And finally remove any empty 'zoneStrings' entries.
-find ${icu_dir}/source/data/zone \
-    -name root.txt -prune -or \
-    -name tzdbNames.txt -prune -or \
-    -name '*.txt' -print | xargs sed -i '/^\s*zoneStrings{/{N; s/^\s*zoneStrings{\n\s*}// }; /^$/d'
-
 for patch in \
  bug-915735 \
  suppress-warnings.diff \