box64/docs/gen/gen.py

# SPDX-License-Identifier: MIT
# Usage: python gen.py

import json
import os
import re
from collections import defaultdict

script_dir = os.path.dirname(os.path.abspath(__file__))
usage_file = os.path.join(script_dir, 'usage.json')

# Language definitions — add a key here and a corresponding usage_<lang>.json
# (if translations are needed) to add a new language.
LANG = {
    "en": {
        "suffix": "",
        "preamble": """<!--- This file is generated by gen.py, do not edit it directly. -->
Usage
----

There are many environment variables to control Box64's behaviour, which will be listed below by category.

There are 2 types of Box64 builds: the Wine WOW64 build (WowBox64) and the regular Linux build. Note that only some of the environment variables are available in WowBox64.

### Configuration files

In addition to environment variables, if you're using the regular Linux build, Box64 also looks for 2 places for rcfile by default: the system-wide `/etc/box64.box64rc` and user-specific `~/.box64rc`.
While in WowBox64, the configuration file is checked at `%USERPROFILE%/.box64rc` only.
Settings priority follows this order (from highest to lowest): `~/.box64rc` > `/etc/box64.box64rc` > environment variables.

Example configuration:

```
[factorio]
BOX64_DYNAREC_SAFEFLAGS=0
BOX64_DYNAREC_BIGBLOCK=2
BOX64_DYNAREC_FORWARD=1024
BOX64_DYNAREC_CALLRET=1
```

This configuration will apply the specified settings application-wide to any executable named `factorio`.

Shared options can also be defined in a special `[*]` section. Those settings apply globally, and any more specific section only overrides the options it defines itself.

### Advanced usage for Linux build

1. **Wildcard Matching**

   Asterisks (`*`) can be used for basic pattern matching in application names. For instance, `[*setup*]` will match any program containing "setup" in its name. Note that this implements simple wildcard matching rather than full regex support.
2. **Shared Settings**

   The special `[*]` section defines shared rcfile options for all applications. Unlike wildcard sections such as `[*setup*]`, this is a dedicated global section.
3. **Custom Configuration File**

   The `BOX64_RCFILE` environment variable can specify an alternative configuration file instead of the default `/etc/box64.box64rc`.
4. **Per-File Settings**

   Sections starting with `/` apply to specific files. For example:
   ```
   [/d3d9.dll]
   BOX64_DYNAREC_SAFEFLAGS=0
   ```
   These settings will only affect the `d3d9.dll` file. This syntax also works for **emulated** Linux libraries, e.g., `[/libstdc++.so.6]`.

----

""",
        "wine": "Available in WowBox64.",
        "default": "[Default]",
        "top_cat": "Performance",
        "categories": {},
    },
    "cn": {
        "suffix": "_CN",
        "preamble": """<!--- 此文件由 gen.py 自动生成，请勿直接编辑。 -->
使用说明
----

Box64 有许多环境变量可用于控制其行为，下面将按类别列出。

Box64 有两种构建类型：Wine WOW64 构建（WowBox64）和常规 Linux 构建。请注意，只有部分环境变量在 WowBox64 中可用。

### 配置文件

除了环境变量之外，如果你使用的是常规 Linux 构建，Box64 默认还会在两个位置查找 rcfile：系统范围的 `/etc/box64.box64rc` 和用户目录下的 `~/.box64rc`。
而在 WowBox64 中，配置文件仅在 `%USERPROFILE%/.box64rc` 处检查。
设置的优先级如下（从高到低）：`~/.box64rc` > `/etc/box64.box64rc` > 环境变量。

配置示例：

```
[factorio]
BOX64_DYNAREC_SAFEFLAGS=0
BOX64_DYNAREC_BIGBLOCK=2
BOX64_DYNAREC_FORWARD=1024
BOX64_DYNAREC_CALLRET=1
```

此配置将使所有名为 `factorio` 的可执行文件应用指定的设置。

共享选项也可以定义在特殊的 `[*]` 配置段中。这些设置会全局应用，而更具体的配置段只会覆盖它自己显式定义的选项。

### Linux 构建的高级用法

1. **通配符匹配**

   可以使用星号（`*`）对应用程序名称进行基本的模式匹配。例如，`[*setup*]` 将匹配名称中包含 "setup" 的任何程序。请注意，这里实现的是简单的通配符匹配，而非完整的正则表达式支持。
2. **共享设置**

   特殊的 `[*]` 配置段可为所有应用程序定义共享的 rcfile 选项。它不同于 `[*setup*]` 这类通配符配置段，是一个专用的全局配置段。
3. **自定义配置文件**

   `BOX64_RCFILE` 环境变量可以指定一个替代的配置文件，而不是默认的 `/etc/box64.box64rc`。
4. **按文件设置**

   以 `/` 开头的配置项适用于特定文件。例如：
   ```
   [/d3d9.dll]
   BOX64_DYNAREC_SAFEFLAGS=0
   ```
   这些设置将仅影响 `d3d9.dll` 文件。此语法同样适用于**模拟的** Linux 库，例如 `[/libstdc++.so.6]`。

----

""",
        "wine": "在 WowBox64 中可用。",
        "default": "[默认值]",
        "top_cat": "性能",
        "categories": {
            "Performance": "性能",
            "Arguments": "参数",
            "Compatibility": "兼容性",
            "Debugging": "调试",
            "Environment": "环境变量",
            "Libraries": "库",
            "Fragile or Legacy": "不稳定或遗留选项",
        },
    },
}


def write_md(data, lang, t_data=None):
    L = LANG[lang]
    out_path = os.path.join(script_dir, f'../USAGE{L["suffix"]}.md')

    with open(out_path, 'w') as md_file:
        md_file.write(L['preamble'])

        categories = defaultdict(list)
        for entry in data:
            entry = dict(entry)
            entry['category'] = L['categories'].get(entry['category'], entry['category'])
            categories[entry["category"]].append(entry)

        sorted_categories = sorted(categories.items(), key=lambda x: (x[0] != L['top_cat'], x[0]))
        for category, entries in sorted_categories:
            md_file.write(f"## {category}\n\n")
            for entry in entries:
                if t_data:
                    t_entry = t_data.get(entry['name'], {})
                    desc = t_entry.get('description', entry['description'])
                else:
                    desc = entry['description']
                wine_note = f" {L['wine']}" if entry['wine'] else ""
                md_file.write(f"### {entry['name']}\n\n{desc}{wine_note}\n\n")
                for option in entry['options']:
                    if t_data:
                        t_opts = t_data.get(entry['name'], {}).get('options', {})
                        opt_desc = t_opts.get(option['key'], option['description'])
                    else:
                        opt_desc = option['description']
                    default_mark = f" {L['default']}" if option['default'] else ""
                    md_file.write(f" * {option['key']}: {opt_desc}{default_mark}\n")
                md_file.write("\n")


with open(usage_file, 'r') as file:
    data = json.load(file)

for lang in LANG:
    t_file = os.path.join(script_dir, f'usage_{lang}.json')
    t_data = None
    if os.path.exists(t_file):
        with open(t_file, 'r') as file:
            t_data = json.load(file)
    write_md(data, lang, t_data)

with open(os.path.join(script_dir, '../box64.pod'), 'w') as pod_file:
    pod_file.write(
        """=head1 NAME

box64 - Linux Userspace x86_64 Emulator with a twist

=head1 SYNOPSIS

B<box64> [B<--help>] [B<--version>] I<executable>

=head1 DESCRIPTION

B<Box64> lets you run x86_64 Linux programs (such as games) on non-x86_64 Linux
systems, like ARM (host system needs to be 64-bit little-endian). Since B<Box64>
uses the native versions of some "system" libraries, like libc, libm, SDL, and
OpenGL, it's easy to integrate and use with most applications, and performance
can be surprisingly high in many cases. B<Box64> integrates with DynaRec (dynamic
recompiler) for the ARM64 platform, providing a speed boost between 5 to 10
times faster than using only the interpreter.

=head1 OPTIONS

=over 8

=item B<-h,--help>

Print box64 help and quit.

=item B<-v,--version>

Print box64 version and quit.

=back

=head1 BRIEF USAGE

There are many environment variables to control B<Box64>'s behaviour. In
addition to environment variables, B<Box64> also looks for 2 places for rcfile:
F</etc/box64.box64rc> and F<~/.box64rc>, in the format of .ini files.
Settings priority: F<~/.box64rc> > F</etc/box64.box64rc> > environment variables.
Example:

    [factorio]
    BOX64_DYNAREC_SAFEFLAGS=0
    BOX64_DYNAREC_BIGBLOCK=2
    BOX64_DYNAREC_FORWARD=1024
    BOX64_DYNAREC_CALLRET=1

The special C<[*]> section defines shared rcfile options for all applications.
More specific sections override only the options they define themselves.

=head1 ENVIRONMENT VARIABLES

=over 8

"""
    )

    for entry in data:
        pod_file.write(f"\n=item B<{entry['name']}> =I<{ '|'.join(option['key'] for option in entry['options']) }>\n\n{entry['description']}{' Available in WowBox64.' if entry['wine'] else ''}\n\n")
        for option in entry['options']:
            pod_file.write(f" * {option['key']} : {option['description']} {'[Default]' if option['default'] else ''}\n")
        pod_file.write("\n")

    pod_file.write("""
=back

=cut
""")

####################
# Validation
####################

PADDING = 30
INCLUDE_BLANK = False

def get_usage_entry(usage, define):
    entry = list(e for e in usage if e["name"] == define)
    if len(entry) == 0:
        print(f"{define:<{PADDING}}: missing usage.json entry")
    elif len(entry) > 1:
        print(f"{define:<{PADDING}}: multiple usage entries found", len(entry))
    else:
        return entry[0]
    return None

# check `default` uniqueness
for entry in data:
    i = list(d["default"] for d in entry["options"]).count(True)
    if i > 1:
        print(f"{entry['name']:<{PADDING}}: multiple default values usage.json")


# regex to match env.h C code
regex = {
    "INTEGER": re.compile(
        r"^\s*INTEGER\((?P<define>\w+), (?P<name>\w+), (?P<default>\w+), (?P<min>\w+), (?P<max>\w+), (?P<wine>\w+)(?:, \w+)?\)"
    ),
    "INTEGER64": re.compile(
        r"^\s*INTEGER64\((?P<define>\w+), (?P<name>\w+), (?P<default>\w+), (?P<wine>\w+)(?:, \w+)?\)"
    ),
    "BOOLEAN": re.compile(
        r"^\s*BOOLEAN\((?P<define>\w+), (?P<name>\w+), (?P<default>\w+), (?P<wine>\w+)(?:, \w+)?\)"
    ),
    "ADDRESS": re.compile(r"^\s*ADDRESS\((?P<define>\w+), (?P<name>\w+), (?P<wine>\w+)(?:, \w+)?\)"),
    "STRING": re.compile(r"^\s*STRING\((?P<define>\w+), (?P<name>\w+), (?P<wine>\w+)(?:, \w+)?\)"),
}

env_file = os.path.join(script_dir, "../../src/include/env.h")
with open(env_file, "r") as file:
    matches = {}
    for line in file.readlines():
        for t, r in regex.items():
            m = r.search(line)
            if m:
                # filter out comments and other non useful code
                if m.group("define") == "NAME":
                    continue

                if t not in matches:
                    matches[t] = {}
                if (m.group("define") in matches[t]):
                    # multiple definitions, no default
                    matches[t][m.group("define")]['no_default'] = True
                    break
                matches[t][m.group("define")] = m.groupdict()
                matches[t][m.group("define")]['no_default'] = False

for define, m in matches["INTEGER"].items():
    name = m["name"]
    default = None if m["no_default"] or not m["default"].isdigit() else int(m["default"])
    min = int(m["min"])
    max = int(m["max"])

    # Check default in valid range
    if default and (default < min or default > max):
        print(f"{define:<{PADDING}}: default lays outside of min/max range")

    # Check consistency with usage.json
    if e := get_usage_entry(data, define):
        # guess min/max values if possible
        min2 = max2 = None
        # blank means that the entry has an 'XXXX' entry which usually indicated that arbitrary values are valid
        blank = False
        default2 = None

        if int(m["wine"]) != int(e["wine"]):
            print(f"{define:<{PADDING}}: wine mismatch: env.h={int(m['wine'])}, usage.json={int(e['wine'])}")

        for o in e["options"]:
            if o["key"] == "XXXX":
                blank = True
                continue

            val = int(o["key"])  # not supposed to fail

            min2 = min2 if min2 is not None and min2 < val else val
            max2 = max2 if max2 is not None and max2 > val else val

            if o["default"]:
                default2 = val

        if min2 and min2 != min:
            if not blank or (blank and INCLUDE_BLANK):
                print(
                    f"{define:<{PADDING}}: min value mismatch: env.h={min}, usage.json={min2}{(' (possible false positive)' if blank else '')}"
                )
        if max2 and max2 != max:
            if not blank or (blank and INCLUDE_BLANK):
                print(
                    f"{define:<{PADDING}}: max value mismatch: env.h={max}, usage.json={max2}{(' (possible false positive)' if blank else '')}"
                )
        if default2 != default:
            print(
                f"{define:<{PADDING}}: default value mismatch: env.h={default}, usage.json={default2}"
            )

for define, m in matches['INTEGER64'].items():
    # similar to INTEGER but without min/max
    name = m["name"]
    default = None if m["no_default"] or not m["default"].isdigit() else int(m["default"])

    # Check consistency with usage.json
    if e := get_usage_entry(data, define):
        default2 = None

        if int(m["wine"]) != int(e["wine"]):
            print(f"{define:<{PADDING}}: wine mismatch: env.h={int(m['wine'])}, usage.json={int(e['wine'])}")

        for o in e["options"]:
            if o["key"] == "XXXX":
                continue

            val = int(o["key"])

            if o["default"]:
                default2 = val

        if default2 != default:
            print(
                f"{define:<{PADDING}}: default value mismatch: env.h={default}, usage.json={default2}"
            )

for define, m in matches["BOOLEAN"].items():
    name = m["name"]
    default = None if m["no_default"] or m["default"] not in ["0", "1"] else bool(m["default"])

    # Check consistency with usage.json
    if e := get_usage_entry(data, define):
        default2 = None

        if int(m["wine"]) != int(e["wine"]):
            print(f"{define:<{PADDING}}: wine mismatch: env.h={int(m['wine'])}, usage.json={int(e['wine'])}")

        for o in e["options"]:
            try:
                val = bool(o["key"])
            except ValueError:
                print(f"{define:<{PADDING}}: failed to parse boolean {o['key']}")

            if o["default"]:
                default2 = val

        if default2 != default:
            print(
                f"{define:<{PADDING}}: default value mismatch: env.h={default}, usage.json={default2}"
            )

# ADDRESS and STRING are not that interesting
for define, m in matches["ADDRESS"].items():
    if e := get_usage_entry(data, define):
        if int(m["wine"]) != int(e["wine"]):
            print(f"{define:<{PADDING}}: wine mismatch: env.h={int(m['wine'])}, usage.json={int(e['wine'])}")

for define, m in matches["STRING"].items():
    # skip BOX64_ENV[1-5] entries, they mismatch but this is fine
    if define.startswith("BOX64_ENV"):
        continue
    if e := get_usage_entry(data, define):
        if int(m["wine"]) != int(e["wine"]):
            print(f"{define:<{PADDING}}: wine mismatch: env.h={int(m['wine'])}, usage.json={int(e['wine'])}")

# check that everything from usage.json is in env.h
for e in data:
    define = e["name"]

    # skip BOX64_ENV[1-5] entries, they mismatch but this is fine
    if define.startswith("BOX64_ENV"):
        continue

    found = False
    for t in matches:
        for d in matches[t]:
            if d == define:
                found = True

    if not found:
        print(f"{define:<{PADDING}}: missing env.h entry")