diff --git a/serve/server.py b/serve/server.py index 76b5a2c79..ad0e10a90 100644 --- a/serve/server.py +++ b/serve/server.py @@ -65,6 +65,7 @@ from serve.frontend import (THINK_END, ChatTemplate, Event, OutputParser, anthro tool_choice_of, unmark_think_literals) from serve.mcp import McpCancelled, hub_from_config # noqa: E402 from serve import runconfig # noqa: E402 +from serve.webfiles import subtree_file as web_subtree_file # noqa: E402 from strata_control import engine_component # noqa: E402 from strata_control import tls # noqa: E402 @@ -6387,6 +6388,21 @@ def make_handler(svc: Service): self.end_headers() self.wfile.write(body) return + sub = web_subtree_file(path) + if path.startswith(("/web/locales/", "/web/vendor/")): + # the page's translations (serve/web/locales/.json) and the vendored KaTeX (serve/web/vendor): + # only whitelisted names and types, never a path out of those folders + if sub is None: + self._json(404, {"error": {"message": "not found"}}) + return + body = sub[0].read_bytes() + self.send_response(200) + self.send_header("Content-Type", sub[1]) + self.send_header("Cache-Control", "no-cache") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return if path.startswith("/web/"): # the web app's own files (serve/web): styles, script, icon sprite - same origin, no CDN name = path[len("/web/"):] diff --git a/serve/test_web_i18n.js b/serve/test_web_i18n.js new file mode 100644 index 000000000..a712f6e7a --- /dev/null +++ b/serve/test_web_i18n.js @@ -0,0 +1,92 @@ +// The web page's translation mechanism (serve/web/i18n.js), run in Node with a small stub of the DOM and fetch. +// node serve/test_web_i18n.js (serve/test_web_i18n.py runs it too and skips when node is missing) +"use strict"; +const assert = require("node:assert/strict"); +const path = require("node:path"); + +// a page element that holds data-i18n marks and keeps its attributes +class El { + constructor(attrs, text = "") { this.attrs = {...attrs}; this.textContent = text; this.dataset = {}; } + getAttribute(k) { return k in this.attrs ? this.attrs[k] : null; } + setAttribute(k, v) { this.attrs[k] = String(v); } +} +const marks = [ + new El({"data-i18n": "a.text"}, "Rewind to here"), + new El({"data-i18n-title": "a.title", title: "Light / dark"}), + new El({"data-i18n-label": "a.label", "aria-label": "Send"}), + new El({"data-i18n-placeholder": "a.hint", placeholder: "Ask anything…"}), +]; +globalThis.document = { + documentElement: {lang: "en"}, + querySelectorAll: (sel) => marks.filter((e) => sel.slice(1, -1) in e.attrs), +}; +const catalogs = { + de: {"a.text": "Hierher zurückspulen", "a.title": "Hell / Dunkel", "chat.hidden": {one: "{n} Nachricht ausgeblendet", other: "{n} Nachrichten ausgeblendet"}}, + ru: {"chat.hidden": {one: "{n} сообщение скрыто", few: "{n} сообщения скрыто", many: "{n} сообщений скрыто", other: "{n} сообщения скрыто"}}, + ja: {"chat.hidden": {other: "{n} 件のメッセージを非表示"}}, +}; +let fetched = []; +globalThis.fetch = async (url) => { + fetched.push(url); + const m = /web\/locales\/([\w-]+)\.json$/.exec(url); + if (m && catalogs[m[1]]) return {ok: true, json: async () => catalogs[m[1]]}; + if (m && m[1] === "zh") return {ok: true, json: async () => ["not", "an", "object"]}; + if (m && m[1] === "es") throw new Error("offline"); + return {ok: false, json: async () => { throw new Error("404"); }}; +}; +const i18n = require(path.join(__dirname, "web", "i18n.js")); +const {tr} = i18n; +const plural = {one: "{n} message hidden", other: "{n} messages hidden"}; + +(async () => { + // English: the text in the code, with its {placeholders}; a key with no text shows the key rather than nothing + assert.equal(tr("x.y", "Hello {name}", {name: "Ada"}), "Hello Ada"); + assert.equal(tr("x.y", "Hello {name}"), "Hello {name}", "an unknown placeholder stays visible"); + assert.equal(tr("chat.hidden", plural, {n: 1}), "1 message hidden"); + assert.equal(tr("chat.hidden", plural, {n: 3}), "3 messages hidden"); + assert.equal(tr("x.y", undefined), "x.y"); + assert.equal(globalThis.tr, tr, "tr is global for app.js"); + i18n.apply(); + assert.equal(marks[0].textContent, "Rewind to here", "English leaves the marked text alone"); + + // a catalog replaces the text; a key it does not have falls back to the English text in the code + assert.equal(await i18n.setLanguage("de"), "de"); + assert.equal(document.documentElement.lang, "de"); + assert.equal(marks[0].textContent, "Hierher zurückspulen"); + assert.equal(marks[1].attrs.title, "Hell / Dunkel"); + assert.equal(marks[2].attrs["aria-label"], "Send", "no German text for this key: English stays"); + assert.equal(tr("chat.hidden", plural, {n: 1}), "1 Nachricht ausgeblendet"); + assert.equal(tr("chat.hidden", plural, {n: 2}), "2 Nachrichten ausgeblendet"); + assert.equal(tr("only.english", "Only English"), "Only English"); + + // plural forms of the language: Russian has few and many, Japanese only other + await i18n.setLanguage("ru"); + assert.deepEqual([1, 2, 5, 21, 22].map((n) => tr("chat.hidden", plural, {n})), + ["1 сообщение скрыто", "2 сообщения скрыто", "5 сообщений скрыто", "21 сообщение скрыто", "22 сообщения скрыто"]); + await i18n.setLanguage("ja"); + assert.equal(tr("chat.hidden", plural, {n: 1}), "1 件のメッセージを非表示"); + + // switching back restores the English text that was in the page + await i18n.setLanguage("de"); + await i18n.setLanguage("en"); + assert.equal(marks[0].textContent, "Rewind to here"); + assert.equal(marks[1].attrs.title, "Light / dark"); + assert.equal(marks[3].attrs.placeholder, "Ask anything…"); + assert.equal(document.documentElement.lang, "en"); + + // a language that is not offered, a catalog that is not an object, one that cannot be fetched, a missing file: + // all stay English, and never throw + fetched = []; + assert.equal(await i18n.setLanguage("xx"), "en"); + assert.deepEqual(fetched, [], "an unknown code is not even requested"); + assert.equal(await i18n.setLanguage("zh"), "en"); + assert.equal(await i18n.setLanguage("es"), "en"); + assert.equal(await i18n.setLanguage("fr"), "en", "French is not in LANGUAGES yet"); + assert.equal(await i18n.setLanguage("ko"), "en", "the file does not exist in this test"); + assert.equal(tr("chat.hidden", plural, {n: 3}), "3 messages hidden"); + + // the list: English first, every entry [code, own name] + assert.equal(i18n.LANGUAGES[0][0], "en"); + for (const [code, name] of i18n.LANGUAGES) assert.ok(/^[a-z]{2,3}(-[A-Z]{2})?$/.test(code) && name, code); + console.log("ok web i18n"); +})().catch((e) => { console.error(e); process.exitCode = 1; }); diff --git a/serve/test_web_i18n.py b/serve/test_web_i18n.py new file mode 100644 index 000000000..528533739 --- /dev/null +++ b/serve/test_web_i18n.py @@ -0,0 +1,108 @@ +"""The web page's translations: the mechanism (node, serve/test_web_i18n.js), the files the server hands out for it +(serve/web/locales, serve/web/vendor), and the catalogs against the text in the code (tools/web_i18n.py).""" +from __future__ import annotations + +import shutil +import subprocess +import sys +import unittest +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) + +import web_i18n # noqa: E402 +from serve.frontend import ChatTemplate # noqa: E402 +from serve.server import ByteTokenizer, MockEngine, Service, serve # noqa: E402 +from serve.webfiles import subtree_file # noqa: E402 + + +def run_node(script: str, *args: str) -> None: + result = subprocess.run([shutil.which("node"), str(Path(__file__).with_name(script)), *args], + capture_output=True, text=True, timeout=60) + if result.returncode: + raise AssertionError((result.stdout + result.stderr).strip()) + + +@unittest.skipUnless(shutil.which("node"), "node is not installed") +class Mechanism(unittest.TestCase): + def test_tr_apply_and_set_language(self): + run_node("test_web_i18n.js") + + +class WhatTheServerHandsOut(unittest.TestCase): + """/web/locales and /web/vendor: whitelisted names and types, nothing outside those folders.""" + + def test_names_and_types(self): + web = Path(__file__).parent / "web" + locales = web / "locales" + if (locales / "de.json").is_file(): + self.assertEqual(subtree_file("/web/locales/de.json")[1], "application/json; charset=utf-8") + for bad in ("/web/locales/de.js", "/web/locales/DE.json", "/web/locales/../app.js", "/web/locales/x/y.json", + "/web/locales/%2e%2e%2fapp.json", "/web/locales/de.json%00.js", "/web/locales/missing.json", + "/web/locales/C:\\x.json", "/web/vendor/katex/../../app.js", "/web/vendor/katex/dist/katex.min.js.map", + "/web/vendor/other/dist/katex.min.js", "/web/vendor/katex/dist/fonts/x.ttf", "/web/vendor/katex/dist/"): + with self.subTest(bad=bad): + self.assertIsNone(subtree_file(bad)) + + def test_a_link_out_of_the_folder_is_refused(self): + import tempfile + with tempfile.TemporaryDirectory() as d: + web = Path(d) / "web" + (web / "vendor" / "katex").mkdir(parents=True) + (web / "locales").mkdir() + (web / "secret.js").write_text("secret") + (web / "vendor" / "katex" / "ok.js").write_text("ok") + self.assertEqual(subtree_file("/web/vendor/katex/ok.js", web)[1], "text/javascript; charset=utf-8") + try: + (web / "vendor" / "katex" / "link.js").symlink_to(web / "secret.js") + except (OSError, NotImplementedError): + self.skipTest("symlinks need a privilege here") + self.assertIsNone(subtree_file("/web/vendor/katex/link.js", web)) + + +class Served(unittest.TestCase): + @classmethod + def setUpClass(cls): + tok = ByteTokenizer() + cls.svc = Service(MockEngine(tok, "ok"), tok, ChatTemplate(ROOT / "serve" / "chat_template.jinja")) + cls.httpd = serve(cls.svc, port=0) + cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}" + + @classmethod + def tearDownClass(cls): + cls.httpd.shutdown() + cls.httpd.server_close() + + def get(self, path): + try: + with urllib.request.urlopen(self.base + path, timeout=10) as r: + return r.status, r.headers.get("Content-Type", ""), r.read() + except urllib.error.HTTPError as e: + return e.code, e.headers.get("Content-Type", ""), e.read() + + def test_i18n_script_and_refusals(self): + code, ctype, body = self.get("/web/i18n.js") + self.assertEqual(code, 200) + self.assertIn(b"StrataI18n", body) + for path in ("/web/locales/../server.py", "/web/locales/nope.json", "/web/vendor/katex/../../app.js", + "/web/vendor/katex/dist/missing.js", "/web/locales/de.json%00"): + with self.subTest(path=path): + self.assertEqual(self.get(path)[0], 404) + + +@unittest.skipUnless((Path(__file__).parent / "web" / "locales" / "en.json").is_file(), "no catalogs yet") +class Catalogs(unittest.TestCase): + """Every language in serve/web/i18n.js has a catalog with exactly the keys and {placeholders} of the code.""" + + def test_the_text_in_the_code_is_plain_literals(self): + self.assertTrue(web_i18n.english()) + + def test_catalogs_are_complete(self): + self.assertEqual(web_i18n.check(), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/serve/web/app.js b/serve/web/app.js index ac84f85dc..b3448d92b 100644 --- a/serve/web/app.js +++ b/serve/web/app.js @@ -1096,14 +1096,35 @@ $("drawer-close").onclick = () => openDrawer(false); $("scrim").onclick = () => openDrawer(false); document.addEventListener("keydown", (e) => { if (e.key === "Escape" && $("drawer").dataset.open === "true") openDrawer(false); }); +// ------------------------------------------------------------------ language (serve/web/i18n.js) +// The user's own choice, kept in this browser: English until they pick another. /?lang=de shows one for this visit only. +const urlLang = new URLSearchParams(location.search).get("lang"); +$("lang").innerHTML = StrataI18n.LANGUAGES.map(([code, name]) => ``).join(""); +function showLanguage() { + $("lang").value = StrataI18n.lang; + renderChat(); + setBusy(!!busy); + if (lastMetrics) render(lastMetrics); +} +$("lang").onchange = async () => { + store.set("lang", $("lang").value); + await StrataI18n.setLanguage($("lang").value); + showLanguage(); +}; + // ------------------------------------------------------------------ start -setBusy(false); -renderChat(); -const startQuestion = new URLSearchParams(location.search).get("q"); // /?q=... starts a chat (a shortcut) -if (startQuestion) history.replaceState(null, "", location.pathname + location.hash); -loadHealth().then(loadMcp).then(() => { if (startQuestion) { $("input").value = startQuestion; send(); } }); -showTab(location.hash.slice(1) || "chat"); -poll(); +async function start() { + await StrataI18n.setLanguage(urlLang || store.get("lang", "en")); + $("lang").value = StrataI18n.lang; + setBusy(false); + renderChat(); + const startQuestion = new URLSearchParams(location.search).get("q"); // /?q=... starts a chat (a shortcut) + if (startQuestion) history.replaceState(null, "", location.pathname + location.hash); + loadHealth().then(loadMcp).then(() => { if (startQuestion) { $("input").value = startQuestion; send(); } }); + showTab(location.hash.slice(1) || "chat"); + poll(); +} +start(); // ------------------------------------------------------------------ images (POST /v1/images/generations) $("img-form").onsubmit = async (ev) => { diff --git a/serve/web/i18n.js b/serve/web/i18n.js new file mode 100644 index 000000000..81defdbb1 --- /dev/null +++ b/serve/web/i18n.js @@ -0,0 +1,77 @@ +// serve/web/i18n.js - the web page's translations. +// +// The English text sits in the code, next to the key: tr("chat.rewind", "Rewind to here") and +// tr("chat.hidden", {one: "{n} message hidden", other: "{n} messages hidden"}, {n: 3}). Static page text is marked +// data-i18n="key" (also data-i18n-title, data-i18n-label, data-i18n-placeholder) and holds its English text in the HTML. +// So the page reads fine with no catalog at all. Every other language is a catalog, serve/web/locales/.json +// (a flat {"key": "text"} object; a plural is {"one": "...", "other": "..."}, the CLDR names of the language). +// serve/test_web_i18n.py keeps the catalogs complete: same keys as the English text in the code, same {placeholders}. +// +// The language is the user's choice (About > Settings), kept in this browser. Nothing is chosen for them from the +// browser's language: a page that is translated in part would mix two languages. +"use strict"; +(function (root) { + // [code, the language's own name]. A new language is a locales/.json file plus one line here. + const LANGUAGES = [["en", "English"], ["de", "Deutsch"], ["es", "Español"], ["ja", "日本語"], ["ko", "한국어"], + ["pt", "Português"], ["ru", "Русский"], ["zh", "中文"]]; + let lang = "en", catalog = null, rules = null; + + const fill = (s, vars) => String(s).replace(/\{(\w+)\}/g, (m, k) => (vars && k in vars ? String(vars[k]) : m)); + function pick(entry, vars) { + if (entry && typeof entry === "object") { + const n = vars && vars.n != null ? Number(vars.n) : 1; + let cat = "other"; + try { cat = (rules || new Intl.PluralRules(lang === "en" ? "en" : lang)).select(n); } catch (e) { /* other */ } + return entry[cat] != null ? entry[cat] : entry.other != null ? entry.other : entry.one; + } + return entry; + } + // tr(key, english, vars): the catalog's text for the chosen language, else the English text in the code + function tr(key, english, vars) { + const own = catalog && Object.prototype.hasOwnProperty.call(catalog, key) ? catalog[key] : undefined; + const text = pick(own != null ? own : english, vars); + return fill(text == null ? key : text, vars); + } + + // marked page text: the English text is read once from the HTML and kept, so switching back restores it + const ATTRS = [["data-i18n", null], ["data-i18n-title", "title"], ["data-i18n-label", "aria-label"], + ["data-i18n-placeholder", "placeholder"]]; + function apply(scope) { + const doc = scope || root.document; + if (!doc || !doc.querySelectorAll) return; + for (const [mark, attr] of ATTRS) { + for (const el of doc.querySelectorAll(`[${mark}]`)) { + const key = el.getAttribute(mark); + const slot = "en_" + mark.replace(/[^a-z]/g, ""); + if (el.dataset[slot] === undefined) el.dataset[slot] = attr ? (el.getAttribute(attr) || "") : el.textContent; + const text = tr(key, el.dataset[slot]); + if (attr) el.setAttribute(attr, text); else el.textContent = text; + } + } + if (root.document && root.document.documentElement) root.document.documentElement.lang = lang; + } + + const known = (code) => LANGUAGES.some((l) => l[0] === code); + // load a catalog; English, or a catalog that cannot be read, leaves the English text in the code + async function setLanguage(code) { + code = known(code) ? code : "en"; + let next = null; + if (code !== "en") { + try { + const r = await root.fetch(`web/locales/${code}.json`); + const j = r.ok ? await r.json() : null; + if (j && typeof j === "object" && !Array.isArray(j)) next = j; + } catch (e) { /* the English text stays */ } + if (!next) code = "en"; + } + lang = code; catalog = next; rules = null; + try { rules = new Intl.PluralRules(lang); } catch (e) { /* other */ } + apply(); + return lang; + } + + const api = {LANGUAGES, tr, apply, setLanguage, get lang() { return lang; }, fill, pick}; + root.StrataI18n = api; + root.tr = tr; + if (typeof module !== "undefined" && module.exports) module.exports = api; +})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/serve/web/index.html b/serve/web/index.html index 0c7847f22..0fdb4e6ed 100644 --- a/serve/web/index.html +++ b/serve/web/index.html @@ -192,6 +192,11 @@ +
+ + + Chat, its settings and the buttons above are translated. The Monitor, Images and About views are still in English. +
Dark theme

Chats, settings and the key are kept in this browser only. @@ -262,6 +267,7 @@

+ diff --git a/serve/webfiles.py b/serve/webfiles.py new file mode 100644 index 000000000..d79b107d4 --- /dev/null +++ b/serve/webfiles.py @@ -0,0 +1,35 @@ +"""Which files of serve/web the server hands out below /web/: the page's translations and the vendored KaTeX. + +The flat files (app.js, app.css, ...) are served by serve/server.py itself. The two folders here hold more than one +level, so each has its own whitelist: a name pattern and the types it may have. Nothing outside these folders is ever +read, a name with a backslash, a colon or a percent sign is refused, and the resolved path must stay inside the folder. +""" +from __future__ import annotations + +import re +from pathlib import Path + +WEB = Path(__file__).resolve().parent / "web" + +# /web/locales/.json e.g. de.json, pt-BR.json +LOCALE = re.compile(r"^/web/locales/(?P[a-z]{2,3}(?:-[A-Z]{2})?\.json)$") +# /web/vendor/katex/dist/... : the script, the style sheet and the fonts only +VENDOR_TYPES = {".js": "text/javascript; charset=utf-8", ".css": "text/css; charset=utf-8", ".woff2": "font/woff2"} +VENDOR_PREFIX = "/web/vendor/katex/" + + +def subtree_file(path: str, web: Path = WEB): + """(file, content type) for a /web/locales or /web/vendor path, or None when it is not served.""" + if any(c in path for c in ("\\", ":", "%", "\0")) or ".." in path: + return None + m = LOCALE.match(path) + if m: + f = web / "locales" / m.group("name") + return (f, "application/json; charset=utf-8") if f.is_file() else None + if path.startswith(VENDOR_PREFIX): + vendor = (web / "vendor" / "katex").resolve() + f = (vendor / path[len(VENDOR_PREFIX):]).resolve() + ctype = VENDOR_TYPES.get(f.suffix) + if ctype and f.is_relative_to(vendor) and f.is_file(): + return f, ctype + return None diff --git a/tools/web_i18n.py b/tools/web_i18n.py new file mode 100644 index 000000000..747f11019 --- /dev/null +++ b/tools/web_i18n.py @@ -0,0 +1,203 @@ +"""The web page's translations: read the English text out of serve/web, write locales/en.json, check the catalogs. + + python tools/web_i18n.py # check every catalog against the text in the code (exit 1 on a problem) + python tools/web_i18n.py --write # also rewrite serve/web/locales/en.json from the code + +The English text lives in the code (see serve/web/i18n.js): + tr("key", "English text", vars) in serve/web/*.js + tr("key", {one: "{n} chat", other: "{n} chats"}, vars) a plural + English text in serve/web/index.html (also data-i18n-title, -label, -placeholder) +Both must be plain string literals (no template strings, no joined strings), so that this reader and a translator see +the same text. serve/test_web_i18n.py runs the check. +""" +from __future__ import annotations + +import html.parser +import json +import re +import sys +from pathlib import Path + +WEB = Path(__file__).resolve().parents[1] / "serve" / "web" +LOCALES = WEB / "locales" +PLACEHOLDER = re.compile(r"\{(\w+)\}") +CALL = re.compile(r"(? dict: + found = {} + for m in CALL.finditer(src): + key = m.group(1) + i = m.end() + try: + value, _ = _plural(src, i) if src[i] == "{" else _string(src, i) + except Problem as e: + line = src.count("\n", 0, m.start()) + 1 + raise Problem(f"{name}:{line}: tr({key!r}): {e}") + found[key] = value + return found + + +class _Html(html.parser.HTMLParser): + ATTRS = {"data-i18n-title": "title", "data-i18n-label": "aria-label", "data-i18n-placeholder": "placeholder"} + + def __init__(self): + super().__init__(convert_charrefs=True) + self.found: dict = {} + self.open = [] # [(tag, key, [text])] + + def handle_starttag(self, tag, attrs): + a = dict(attrs) + for mark, attr in self.ATTRS.items(): + if mark in a: + self.found[a[mark]] = a.get(attr) or "" + if "data-i18n" in a: + self.open.append((tag, a["data-i18n"], [])) + elif self.open: + self.open.append((tag, None, self.open[-1][2])) + + def handle_data(self, data): + if self.open: + self.open[-1][2].append(data) + + def handle_endtag(self, tag): + if self.open and self.open[-1][0] == tag: + _, key, text = self.open.pop() + if key: + self.found[key] = " ".join("".join(text).split()) + + +def html_strings(src: str) -> dict: + p = _Html() + p.feed(src) + return p.found + + +def english() -> dict: + """Every key in serve/web with its English text; the same key twice must have the same text.""" + out: dict = {} + sources = {p.name: js_strings(p.read_text(encoding="utf-8"), p.name) for p in sorted(WEB.glob("*.js"))} + sources["index.html"] = html_strings((WEB / "index.html").read_text(encoding="utf-8")) + for name, found in sources.items(): + for key, value in found.items(): + if key in out and out[key] != value: + raise Problem(f"{key!r} has two different English texts (one is in {name})") + out[key] = value + return dict(sorted(out.items())) + + +def languages() -> list: + src = (WEB / "i18n.js").read_text(encoding="utf-8") + block = re.search(r"const LANGUAGES = (\[.*?\]);", src, re.S).group(1) + return [m for m in re.findall(r'\["([\w-]+)",', block)] + + +def placeholders(value) -> set: + forms = value.values() if isinstance(value, dict) else [value] + return {name for form in forms for name in PLACEHOLDER.findall(form)} + + +def check_catalog(code: str, catalog: dict, en: dict) -> list: + problems = [] + for key in sorted(set(en) - set(catalog)): + problems.append(f"{code}: missing {key!r}") + for key in sorted(set(catalog) - set(en)): + problems.append(f"{code}: {key!r} is not in the page (renamed or removed?)") + for key, value in catalog.items(): + if key not in en: + continue + forms = value if isinstance(value, dict) else {"": value} + if isinstance(value, dict): + bad = [c for c in value if c not in PLURAL_CATEGORIES] + if bad or "other" not in value: + problems.append(f"{code}: {key!r}: plural forms must be CLDR names with an 'other' form, got {sorted(value)}") + if not isinstance(en[key], dict): + problems.append(f"{code}: {key!r} is a plural here but not in the code") + elif isinstance(en[key], dict): + problems.append(f"{code}: {key!r} is a plain text here but a plural in the code") + for form, text in forms.items(): + if not isinstance(text, str) or not text.strip(): + problems.append(f"{code}: {key!r} {form} is empty or not a string") + if placeholders(value) != placeholders(en[key]) and not ( + isinstance(value, dict) and isinstance(en[key], dict) + and all(placeholders(f) <= placeholders(en[key]) for f in value.values())): + problems.append(f"{code}: {key!r} uses {sorted(placeholders(value))}, the code uses {sorted(placeholders(en[key]))}") + return problems + + +def check(write: bool = False) -> list: + en = english() + if write: + LOCALES.mkdir(exist_ok=True) + (LOCALES / "en.json").write_text(json.dumps(en, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + problems = [] + on_disk = LOCALES / "en.json" + if not on_disk.is_file() or json.loads(on_disk.read_text(encoding="utf-8")) != en: + problems.append("en.json is not the text in the code (python tools/web_i18n.py --write)") + for code in languages(): + if code == "en": + continue + f = LOCALES / f"{code}.json" + if not f.is_file(): + problems.append(f"{code}: serve/web/locales/{code}.json is missing") + continue + problems += check_catalog(code, json.loads(f.read_text(encoding="utf-8")), en) + for f in sorted(LOCALES.glob("*.json")): + if f.stem not in languages(): + problems.append(f"{f.name}: not in LANGUAGES of serve/web/i18n.js") + return problems + + +if __name__ == "__main__": + try: + problems = check(write="--write" in sys.argv) + except Problem as e: + print(f"web_i18n: {e}") + sys.exit(1) + for p in problems: + print("web_i18n:", p) + sys.exit(1 if problems else 0)