darling-JavaScriptCore/Scripts/generateIntlCanonicalizeLanguage.py

#!/usr/bin/env python

# Copyright (C) 2018 Andy VanWagoner (andy@vanwagoner.family)
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# This tool processes the IANA file language-subtag-registry.txt to create
# information required to canonicalize language tags according to ECMA 402 and
# RFC 5646 Section 4.5.
# https://www.iana.org/assignments/language-subtag-registry
# https://tc39.github.io/ecma402/#sec-canonicalizelanguagetag
# https://tools.ietf.org/html/rfc5646#section-4.5

import sys
import optparse
import os

header = """// DO NO EDIT! - This file was generated by """ + __file__ + """
"""


footer = """
"""


def openOrExit(path, mode):
    try:
        if sys.version_info.major >= 3:
            return open(path, mode, encoding="UTF-8")
        else:
            return open(path, mode)
    except IOError as e:
        print("I/O error opening {0}, ({1}): {2}".format(path, e.errno, e.strerror))
        exit(1)


class SubtagRegistry:
    def __init__(self):
        self.languageMap = {}
        self.extlangMap = {}
        self.regionMap = {}
        self.redundantMap = {}
        self.grandfatheredMap = {}

    def parse(self, file):
        record = {}
        for line in file:
            line = line.strip()
            if line.startswith("#"):
                continue

            if line == "%%":
                self.process(record)
                record = {}
                continue

            if ":" in line:
                key, value = line.split(":", 1)
                record[key.strip()] = value.strip()
            else:
                # Description often continues on the next line
                record[key.strip()] = " " + line
        self.process(record)

    def process(self, record):
        if "File-Date" in record:
            self.fileDate = record["File-Date"]

        if not ("Type" in record):
            return

        type = record["Type"]
        preferred = record.get("Preferred-Value")
        if type == "language" and preferred:
            self.languageMap[record["Subtag"]] = preferred
        elif type == "extlang":
            self.extlangMap[record["Subtag"]] = record["Prefix"]
        elif type == "region" and preferred:
            self.regionMap[record["Subtag"]] = preferred
        elif type == "redundant" and preferred:
            lang = self.extlangMap.get(preferred)
            if "{}-{}".format(lang, preferred) != record["Tag"]:
                self.redundantMap[record["Tag"]] = preferred
        elif type == "variant" and preferred:
            key = "{}-{}".format(record['Prefix'], record['Subtag'])
            if preferred == "alalc97":
                preferred = "ja-Latn-alalc97"
            self.redundantMap[key] = preferred
        elif type == "grandfathered":
            key = record["Tag"].lower()
            value = record.get("Preferred-Value", key)
            self.grandfatheredMap[key] = value

    def dump(self, file):
        if self.fileDate:
            file.write("// language-subtag-registry file date: {}\n".format(self.fileDate))
        file.write("\n#pragma once\n")
        file.write("\n#if ENABLE(INTL)\n")
        file.write("\nnamespace JSC {\n")
        self.dumpLookup(file, "intlPreferredLanguageTag", self.languageMap)
        self.dumpLookup(file, "intlPreferredExtlangTag", self.extlangMap)
        self.dumpLookup(file, "intlPreferredRegionTag", self.regionMap)
        self.dumpLookup(file, "intlRedundantLanguageTag", self.redundantMap)
        self.dumpLookup(file, "intlGrandfatheredLanguageTag", self.grandfatheredMap)
        file.write("\n} // namespace JSC\n")
        file.write("\n#endif // ENABLE(INTL)\n")

    def dumpLookup(self, file, name, map):
        file.write("\nstatic String {}(const String& tag)\n{{\n".format(name))
        file.write("    // {} possible replacements\n".format(len(map)))
        # We could pick the lookup implementation per map if desired
        # Anecdotal perf: if > switch > hash (slowest)
        # Code complexity: switch > if > hash (least complex)
        # Algo complexity: if = O(N) > switch > O(log N) > hash = O(1) (least complex)
        self.dumpIfLookup(file, name, map)
        file.write("}\n")

    def dumpHashLookup(self, file, name, map):
        file.write("    static NeverDestroyed<HashMap<String, String>> cache;\n")
        file.write("    HashMap<String, String>& map = cache.get();\n")
        file.write("    if (UNLIKELY(map.isEmpty())) {\n")
        entries = ["        map.add(\"{}\"_s, \"{}\"_s);\n".format(k, v) for k, v in map.items()]
        entries.sort()
        file.write("".join(entries))
        file.write("    }\n")
        file.write("    return map.get(tag);\n")

    def dumpIfLookup(self, file, name, map):
        entries = ["    if (tag == \"{}\")\n        return \"{}\"_s;".format(k, v) for k, v in map.items()]
        entries.sort()
        file.write("\n".join(entries))
        file.write("\n    return String();\n")

    def dumpSwitchLookup(self, file, name, map):
        tree = {}
        for k, v in map.items():
            node = tree
            for char in k:
                if not (char in node):
                    node[char] = {}
                node = node[char]
            node["value"] = v
        self.dumpSwitchLookupTree(file, tree, 0)
        file.write("\n    return String();\n")

    def dumpSwitchLookupTree(self, file, tree, level):
        indent = "".ljust((level + 1) * 4)
        if "value" in tree:
            file.write(indent + "if (tag.length() == {})\n".format(level))
            file.write(indent + "    return \"{}\"_s;\n".format(tree["value"]))
            del tree["value"]
        keys = tree.keys()
        keys.sort()
        if len(keys) == 0:
            return
        file.write(indent + "switch (tag[{}]) {{\n".format(level))
        for key in keys:
            file.write(indent + "case {}:\n".format(ord(key)))
            self.dumpSwitchLookupTree(file, tree[key], level + 1)
            file.write(indent + "    break;\n")
        file.write(indent + "default: break;\n")
        file.write(indent + "}\n")


if __name__ == "__main__":
    parser = optparse.OptionParser(usage="usage: %prog <language-subtag-registry.txt> <IntlCanonicalizeLanguage.h>")
    (options, args) = parser.parse_args()

    if len(args) != 2:
        parser.error("<language-subtag-registry.txt> <IntlCanonicalizeLanguage.h>")

    registryPath = args[0]
    intlCanonHPath = args[1]

    registryFile = openOrExit(registryPath, "r")
    intlCanonHFile = openOrExit(intlCanonHPath, "w")

    intlCanonHFile.write(header)

    registry = SubtagRegistry()
    registry.parse(registryFile)
    registry.dump(intlCanonHFile)

    intlCanonHFile.write(footer)

    exit(0)