Check native mods on the newest Nightly, together, before publish (#308)

Players' native builds broke on mods made for an older Horizon (targets moved
by the crate split, a support module refused by a glob) and the whole build
failed. scripts/native_compat.py applies mods as the launcher builds them:

- check (pull requests): each new native-mod version on the newest Nightly
  with the 20 most downloaded native mods, horizon-mixins apply then cargo
  check, and its support modules' (file, name) against every registry mod's.
- health (daily, native-compat.yml): every native mod's newest version on the
  newest Nightly; the ones horizon-mixins refuses go to the native-health
  release, and publish marks them `broken_on` in index.json so the launcher
  warns before an install.

The PR job runs main's script with a read-only token; the submission's build
scripts run under that token only.

Co-authored-by: Horizon agent <[email protected]>
Co-authored-by: Claude Opus 5.5 <[email protected]>

.github/workflows/native-compat.yml

@@ -0,0 +1,85 @@
+# Native mods on the newest Nightly, as players' launchers build them (scripts/native_compat.py):
+# - on a pull request, each new native-mod version, applied together with the most downloaded native mods
+#   and compiled (`cargo check`); its support modules' names against every registry mod's;
+# - every day after the Nightly, every native mod's newest version, applied: the ones horizon-mixins
+#   refuses go to the `native-health` release, which publish adds to the index (`broken_on`) so the
+#   launcher warns before an install.
+# The check compiles the submission, whose crates may have build scripts: it runs with a token that can
+# only read, and main's script, never the pull request's.
+name: native-compat
+on:
+  pull_request:
+    branches: [main]
+    paths: ["mods/*.toml"]
+  schedule:
+    - cron: "40 18 * * *"
+  workflow_dispatch:
+    inputs:
+      horizon:
+        description: "Horizon commit, or `latest` (the newest Nightly)"
+        default: latest
+permissions:
+  contents: read
+concurrency:
+  group: native-compat-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+env:
+  # The Rust the launcher builds native mods with (Horizon's launcher/src/rust_toolchain.rs).
+  RUST: "1.99.1"
+jobs:
+  check:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+      - uses: actions/checkout@v4
+        with:
+          path: head
+          persist-credentials: false
+      - name: Does it change a native mod?
+        id: native
+        run: |
+          if [ -n "$(python3 base/scripts/native_compat.py new base head)" ]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+      - if: steps.native.outputs.changed == 'true'
+        run: rustup toolchain install $RUST --profile minimal && rustup default $RUST
+      - if: steps.native.outputs.changed == 'true'
+        run: sudo apt-get update && sudo apt-get install -y libasound2-dev libudev-dev pkg-config
+      - if: steps.native.outputs.changed == 'true'
+        name: On the newest Nightly, with the popular native mods
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          set -o pipefail
+          python3 base/scripts/native_compat.py check base head | tee -a "$GITHUB_STEP_SUMMARY"
+  health:
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    # Each mod horizon-mixins refuses costs one more apply of the whole set.
+    timeout-minutes: 300
+    permissions:
+      contents: write
+    env:
+      GH_TOKEN: ${{ github.token }}
+      GH_REPO: ${{ github.repository }}
+    steps:
+      - uses: actions/checkout@v4
+      - run: rustup toolchain install $RUST --profile minimal && rustup default $RUST
+      - name: Every native mod's newest version on the newest Nightly
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          HORIZON: ${{ inputs.horizon || 'latest' }}
+        run: |
+          set -o pipefail
+          python3 scripts/native_compat.py health --horizon "$HORIZON" --out native-health.json | tee -a "$GITHUB_STEP_SUMMARY"
+      - name: Keep it for publish
+        run: |
+          gh release view native-health >/dev/null 2>&1 \
+            || gh release create native-health --prerelease --title "Native mods' health" \
+                 --notes "native-health.json: the native mods that don't apply on the newest Nightly (native-compat.yml). publish adds them to the index as broken_on."
+          gh release upload native-health native-health.json --clobber

.github/workflows/publish.yml

@@ -73,6 +73,21 @@ jobs:
               | if $n > 0 then .downloads = $n else . end)' site/index.json > index.counted.json
           mv index.counted.json site/index.json
           jq -r '.projects[] | "\(.id) \(.downloads // 0)"' site/index.json
+      - name: Native mods that don't apply on the newest Nightly
+        run: |
+          # native-compat.yml's daily health: the version each names gets `broken_on`, the Horizon
+          # commit horizon-mixins refused it on and its code, so the launcher warns before an install.
+          if gh release download native-health --pattern native-health.json --dir health 2>/dev/null; then
+            jq --slurpfile h health/native-health.json '
+              $h[0] as $health
+              | .projects |= map(. as $p | .versions |= map(
+                  ($health.broken[$p.id] // null) as $b
+                  | if $b != null and $b.version == .version
+                    then .broken_on = [{horizon: $health.horizon, code: $b.code}] else . end))' \
+              site/index.json > index.health.json
+            mv index.health.json site/index.json
+            jq -r '.projects[] | select(.versions[0].broken_on) | "\(.id) \(.versions[0].version): \(.versions[0].broken_on[0].code)"' site/index.json
+          fi
       - name: The site
         run: |
           cp -R icons site/icons

README.md

@@ -11,9 +11,10 @@ with no file, network or clock access, and can only do what their page lists.
 Native source mods (`.hnmod`, kind `native-mod`) are built into an isolated game by
 a launcher with native-mod support. They run with the player’s account permissions.
 Their bundles pin the exact Horizon and Horizon Mixins revisions used for the build.
-Other revisions have not been validated. Registry checks inspect the source bundle without
-executing native code; reviewers must also examine the source and pinned-build
-validation. Publishing these requires the native-mod changes to `cargo-horizon-mod`
+Other revisions have not been validated. Registry checks inspect the source bundle and compile it
+on the newest Nightly with the most downloaded native mods (in a job whose token
+can only read) without running it; reviewers must also examine the source and
+pinned-build validation. Publishing these requires the native-mod changes to `cargo-horizon-mod`
 and the launcher to have shipped first.
 
 **Who is responsible.** Except the projects marked Official (made by the Horizon project), everything here is made and
@@ -65,6 +66,7 @@ why.
 | `.github/workflows/check.yml` | A pull request's checks |
 | `.github/workflows/publish.yml` | On merge: releases, then `index.json` on Pages |
 | `.github/workflows/cleanup.yml` | Weekly: staging files nobody needs any more |
+| `.github/workflows/native-compat.yml` | Native mods on the newest Nightly, as the launcher builds them: a pull request's new version applied and compiled with the most downloaded native mods, its module names against every mod's; daily, every mod's newest version, the ones that don't apply marked `broken_on` in the index (`scripts/native_compat.py`) |
 | `.github/workflows/native.yml` | Horizon's native mods: each changed one's tests, `.hnmod` and game check, all of them together, and weekly on the newest Nightly |
 | `native/<name>/` | Horizon's own native mods (Zoom, FPS Counter, Keystrokes, Minimap, Recipes, Coordinates HUD): the source of their `.hnmod` files |
 | `native/tools/` | Shared by them: apply on Horizon, check, build, package, prove (`native/tools/native.py`) |

scripts/native_compat.py

@@ -0,0 +1,306 @@
+#!/usr/bin/env python3
+"""Whether native mods build on the newest Nightly, together, as a player's launcher builds them.
+
+    python3 scripts/native_compat.py check BASE HEAD                 # a pull request's new native versions
+    python3 scripts/native_compat.py health --out health.json        # every native mod's newest version
+
+`check` takes each native-mod version HEAD lists and BASE doesn't, and fails when:
+  - one of its support modules has the name, in the same file, of another registry mod's (the launcher
+    can't build both: E_MODULE_CONFLICT);
+  - horizon-mixins doesn't apply it on the newest Nightly with the most downloaded native mods
+    (`--popular`), as the launcher moves mods on the hook points to the Horizon it builds;
+  - the game doesn't compile (`cargo check`) with it and those mods, when it does without it.
+Popular mods that don't apply on that Nightly are left out, as the launcher leaves them out, and listed.
+
+`health` applies every native mod's newest version on the newest Nightly and writes the ones horizon-mixins
+refuses, with its diagnostic: the publish job adds them to the index (`broken_on`), and the launcher warns
+before an install.
+
+horizon-mixins only reads the bundles; `cargo check` runs a mod crate's build script, if it has one, so
+`check` runs in a job whose token can only read (native-compat.yml). Horizon's source comes from GitHub's archive.
+"""
+import argparse
+import hashlib
+import io
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tomllib
+import zipfile
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "native" / "tools"))
+import native  # noqa: E402  (native/tools/native.py: source, latest_nightly, get, log)
+
+INDEX = "https://horizon-rust.github.io/horizon-mods/index.json"
+REGISTRY = "horizon-rust/horizon-mods"
+MIXINS_GIT = "https://github.com/horizon-rust/horizon-mixins"
+WORK = native.TARGET / "compat"
+
+
+def log(*args):
+    native.log(*args)
+
+
+class Bundle:
+    """An unpacked `.hnmod`: its mixin bundle in WORK/mods/<set>/<id>/."""
+
+    def __init__(self, data, origin):
+        archive = zipfile.ZipFile(io.BytesIO(data))
+        meta = json.loads(archive.read("horizon-native.json"))
+        self.id, self.version, self.origin = meta["id"], meta["version"], origin
+        self.mixins = meta.get("mixins")
+        prefix = f"mixins/{self.id}/"
+        self.files = {n[len(prefix):]: archive.read(n) for n in archive.namelist()
+                      if n.startswith(prefix) and not n.endswith("/")}
+        manifest = tomllib.loads(self.files["horizon-mixin.toml"].decode())
+        self.modules = {(m["file"], m["name"]) for m in manifest.get("module", [])}
+        self.needs = [d for d, v in manifest.get("dependencies", {}).items()
+                      if not (isinstance(v, dict) and v.get("optional"))]
+
+    def unpack(self, into, revision):
+        out = into / self.id
+        for name, data in self.files.items():
+            path = out / name
+            if ".." in Path(name).parts:
+                sys.exit(f"{self.id}: {name} leaves its folder")
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(data)
+        # As the launcher does for a mod built on another Horizon than it pins.
+        toml = out / "horizon-mixin.toml"
+        text = re.sub(r'horizon_revision\s*=\s*"[0-9a-f]{40}"', f'horizon_revision = "{revision}"', toml.read_text())
+        toml.write_text(text)
+
+
+def download(url, sha256):
+    cache = WORK / "files" / sha256
+    if not cache.is_file():
+        data = native.get(url)
+        if hashlib.sha256(data).hexdigest() != sha256:
+            sys.exit(f"{url}: its SHA-256 isn't the index's")
+        cache.parent.mkdir(parents=True, exist_ok=True)
+        cache.write_bytes(data)
+    return cache.read_bytes()
+
+
+def published():
+    """The newest version of every native mod in the published index, most downloaded first."""
+    index = json.loads(native.get(INDEX))
+    revoked = {r.get("sha256") for r in index.get("revoked", [])}
+    found = []
+    for project in sorted(index["projects"], key=lambda p: -p.get("downloads", 0)):
+        if project.get("kind") != "native-mod":
+            continue
+        version = next((v for v in project["versions"] if v["file"]["sha256"] not in revoked), None)
+        if version:
+            found.append(Bundle(download(version["file"]["url"], version["file"]["sha256"]), "registry"))
+    return found
+
+
+def mixins_tool(bundles):
+    """horizon-mixins as the launcher picks it: the newest of its own (Horizon main's MIXINS_REV) and the
+    mods' pins, by commit date."""
+    text = native.get(f"https://raw.githubusercontent.com/{native.REPO}/main/launcher/src/native_mods.rs").decode()
+    revs = {re.search(r'pub const MIXINS_REV: &str = "([0-9a-f]{40})"', text).group(1)}
+    revs |= {b.mixins for b in bundles if b.mixins}
+    def date(rev):
+        commit = json.loads(native.get(f"https://api.github.com/repos/horizon-rust/horizon-mixins/commits/{rev}"))
+        return commit["commit"]["committer"]["date"]
+    rev = max(revs, key=date)
+    root = native.TARGET / "horizon-mixins" / rev
+    binary = root / "bin" / "horizon-mixins"
+    if not binary.is_file():
+        log(f"Building horizon-mixins {rev[:7]}")
+        subprocess.run(["cargo", "install", "--locked", "--quiet", "--git", MIXINS_GIT, "--rev", rev,
+                        "--root", str(root), "horizon-mixins-cli"], check=True)
+    return binary
+
+
+def culprit(diagnostic, bundles):
+    """The bundle a horizon-mixins diagnostic names: by mod id, its manifest's path, its module's name or
+    destination; between two mods' modules, the later one (the launcher's rule)."""
+    by_id = {b.id: b for b in bundles}
+    if diagnostic.get("mod_id") in by_id:
+        return by_id[diagnostic["mod_id"]]
+    file = diagnostic.get("file", "")
+    for b in bundles:
+        if f"/{b.id}/horizon-mixin.toml" in file:
+            return b
+    named = re.search(r"support module '([^']+)'", diagnostic.get("message", ""))
+    declaring = [b for b in bundles if any((named and n == named.group(1)) or f == file for f, n in b.modules)]
+    return declaring[-1] if declaring else None
+
+
+def apply(tool, tree, bundles, revision, name):
+    """horizon-mixins on `bundles` together, leaving out each one it refuses (and the ones needing it) until
+    the rest applies: (stage folder or None, {id: diagnostic})."""
+    left, refused = list(bundles), {}
+    while left:
+        mods = WORK / "mods" / name
+        shutil.rmtree(mods, ignore_errors=True)
+        mods.mkdir(parents=True)
+        for b in left:
+            b.unpack(mods, revision)
+        out = WORK / "stage" / name
+        shutil.rmtree(out, ignore_errors=True)
+        run = subprocess.run([str(tool), "apply", "--workspace", str(tree), "--mods", str(mods), "--out", str(out)],
+                             capture_output=True, text=True)
+        if run.returncode == 0:
+            return out, refused
+        text = run.stderr + run.stdout
+        try:
+            diagnostic = json.loads(text[text.index("{"):text.rindex("}") + 1])
+        except ValueError:
+            sys.exit(f"horizon-mixins failed without a diagnostic:\n{text[-2000:]}")
+        bad = culprit(diagnostic, left)
+        if bad is None:
+            sys.exit(f"horizon-mixins refused the set, naming no mod: {json.dumps(diagnostic)}")
+        log(f"  {bad.id} {bad.version}: {diagnostic.get('code')}: {diagnostic.get('message')}")
+        refused[bad.id] = diagnostic
+        gone = {bad.id}
+        while True:
+            more = [b for b in left if b.id not in gone and any(n in gone for n in b.needs)]
+            if not more:
+                break
+            for b in more:
+                refused[b.id] = {"code": "E_DEPENDENCY", "message": f"needs {', '.join(b.needs)}"}
+                gone.add(b.id)
+        left = [b for b in left if b.id not in gone]
+    return None, refused
+
+
+def compiles(stage):
+    """`cargo check` of the game in `stage`, as the launcher compiles it."""
+    env = dict(os.environ, CARGO_TARGET_DIR=str(WORK / "cargo"))
+    run = subprocess.run(["cargo", "check", "--locked", "-p", "horizon-client", "--bin", "horizon-client"],
+                         cwd=stage, env=env, capture_output=True, text=True)
+    if run.returncode != 0:
+        errors = [line for line in run.stderr.splitlines() if line.startswith("error")]
+        log("\n".join(errors[:10]) or run.stderr[-3000:])
+    return run.returncode == 0
+
+
+def new_entries(base, head):
+    """(id, version, file, sha256) of each native-mod version HEAD lists and BASE doesn't."""
+    def versions(root):
+        out = {}
+        for path in sorted((Path(root) / "mods").glob("*.toml")):
+            entry = tomllib.loads(path.read_text())
+            if entry.get("kind") == "native-mod":
+                for v in entry.get("versions", []):
+                    out[(entry["id"], v["version"])] = (v["file"], v["sha256"])
+        return out
+    before = versions(base)
+    return [(i, v, f, s) for (i, v), (f, s) in versions(head).items() if (i, v) not in before]
+
+
+def new_versions(base, head):
+    """The native-mod versions HEAD lists that BASE doesn't: their file from its release once published, else
+    from the `staging` prerelease the submission form uploads it to (scripts/fetch.sh)."""
+    found = []
+    for id, version, file, sha in new_entries(base, head):
+        releases = f"https://github.com/{REGISTRY}/releases/download"
+        try:
+            data = download(f"{releases}/{id}-{version}/{file}", sha)
+        except Exception:
+            data = download(f"{releases}/staging/{sha}.hnmod", sha)
+        found.append(Bundle(data, "submitted"))
+    return found
+
+
+def check(args):
+    submitted = new_versions(args.base, args.head)
+    if not submitted:
+        print("No new native mod version.")
+        return 0
+    registry = published()
+    revision = native.latest_nightly() if args.horizon == "latest" else args.horizon
+    tree = native.source(revision)
+    failed = []
+    for mod in submitted:
+        others = [b for b in registry if b.id != mod.id]
+        print(f"## {mod.id} {mod.version} on Horizon {revision[:7]} (the newest Nightly)")
+        clashes = sorted({(b.id, f, n) for b in others for f, n in mod.modules & b.modules})
+        for other, file, name in clashes:
+            print(f"- support module `{name}` in `{file}` is also {other}'s: the launcher can't build both."
+                  f" Name it after the mod (`{mod.id}_…`).")
+        popular = others[:args.popular]
+        tool = mixins_tool([mod, *popular])
+        stage, refused = apply(tool, tree, [mod, *popular], revision, f"{mod.id}-together")
+        for other, diagnostic in refused.items():
+            if other != mod.id:
+                print(f"- {other} (popular) doesn't apply on this Nightly either: {diagnostic.get('code')}."
+                      " The launcher leaves it out.")
+        if mod.id in refused:
+            d = refused[mod.id]
+            print(f"- **doesn't apply**: `{d.get('code')}`: {d.get('message')}"
+                  + (f" (injection `{d['injection_id']}`)" if d.get("injection_id") else ""))
+            failed.append(mod.id)
+            continue
+        if clashes:
+            failed.append(mod.id)
+        if args.no_compile:
+            continue
+        if compiles(stage):
+            print(f"- applies and compiles with {len(popular) - len(refused)} popular mods.")
+            continue
+        alone, _ = apply(tool, tree, [mod], revision, f"{mod.id}-alone")
+        without, _ = apply(tool, tree, [b for b in popular if b.id not in refused], revision, f"{mod.id}-without")
+        if not compiles(alone):
+            print("- **doesn't compile** on this Nightly, even alone (the errors are in the log).")
+            failed.append(mod.id)
+        elif without is None or compiles(without):
+            print("- **doesn't compile together with the popular mods**, while each side compiles alone.")
+            failed.append(mod.id)
+        else:
+            print("- compiles alone; the popular mods already don't compile together on this Nightly.")
+    return 1 if failed else 0
+
+
+def health(args):
+    registry = published()
+    revision = native.latest_nightly() if args.horizon == "latest" else args.horizon
+    tree = native.source(revision)
+    tool = mixins_tool([])
+    _, refused = apply(tool, tree, registry, revision, "health")
+    versions = {b.id: b.version for b in registry}
+    report = {
+        "horizon": revision,
+        "mixins": tool.parent.parent.name,
+        "broken": {i: {"version": versions[i], "code": d.get("code"), "message": d.get("message")}
+                   for i, d in sorted(refused.items())},
+    }
+    Path(args.out).write_text(json.dumps(report, indent=2) + "\n")
+    print(f"{len(refused)} of {len(registry)} native mods don't apply on Horizon {revision[:7]}: "
+          + ", ".join(sorted(refused)))
+    return 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = parser.add_subparsers(dest="action", required=True)
+    c = sub.add_parser("check")
+    c.add_argument("base")
+    c.add_argument("head")
+    c.add_argument("--popular", type=int, default=20)
+    c.add_argument("--no-compile", action="store_true")
+    n = sub.add_parser("new", help="list the new native-mod versions, one per line")
+    n.add_argument("base")
+    n.add_argument("head")
+    h = sub.add_parser("health")
+    h.add_argument("--out", required=True)
+    for p in (c, h):
+        p.add_argument("--horizon", default="latest", help="a Horizon commit, or `latest` (the newest Nightly)")
+    args = parser.parse_args()
+    if args.action == "new":
+        for id, version, _, _ in new_entries(args.base, args.head):
+            print(id, version)
+        return
+    sys.exit(check(args) if args.action == "check" else health(args))
+
+
+if __name__ == "__main__":
+    main()