mirror of
https://github.com/horizon-rust/horizon.git
synced 2026-11-07 11:06:11 -10:00
CI: run every job that can on our runners
The bots move to a shared horizon-light label (8 light runners on playhorizon, from 2), and triage command, mojang republish and nightly build's answer join them. checks' verify and launcher, mod-registry, telemetry-server and the pack bench run on the desktop's isolated horizon-checks runners, forks included: main's pushes and the pull requests whose number ends in 0-2 by default (GitHub's pool keeps the others, so both work at once), every one with SELF_HOSTED_CI=all, none with light (desktop paused) or off (everything back to GitHub). Jobs with a secret or a token that can push, publish or start workflows stay on GitHub: tests/test_ci_runners.py pins it. playhorizon's job hook now also drops a checkout's .git past 4 GB: each light runner's held 6822 packs, 50 GB. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Ludovic MarieandClaude Opus 5.5 committed
1 parent
b33bc70c4f
commit
bfe0be195c
16 files changed
+246
-114
No files matched your search
@@ -9,9 +9,9 @@ permissions:
|
||||
actions: write
|
||||
jobs:
|
||||
cancel:
|
||||
# One of playhorizon's small runners, out of GitHub's shared pool: no checkout, no code of the pull request
|
||||
# One of our small runners (horizon-light), out of GitHub's pool: no checkout, no code of the pull request
|
||||
# (docs/ci-runners.md). SELF_HOSTED_CI=off: GitHub's runners.
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "playhorizon-light"]') || 'ubuntu-latest' }}
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "horizon-light"]') || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- name: Cancel the pull request's unfinished runs
|
||||
env:
|
||||
|
||||
@@ -27,13 +27,18 @@ jobs:
|
||||
# One job, not one per check: each job takes a runner from the organisation's shared pool, and with every pull request
|
||||
# and every merge to main starting this workflow, three jobs a run left the pool's queue tens of minutes deep.
|
||||
#
|
||||
# Where: GitHub's runners. With the repository variable SELF_HOSTED_CI=all (for when that pool is full: their 4 CPUs
|
||||
# are about one GitHub runner, docs/ci-runners.md), main's pushes and this repository's own pull requests run on
|
||||
# playhorizon's checks runners (label playhorizon-ci); a fork's pull request never does (and the runners refuse it
|
||||
# themselves). There the build folder persists (CARGO_TARGET_DIR, outside the checkout), so the apt packages,
|
||||
# Python and the Rust cache steps are GitHub's only.
|
||||
# Where (docs/ci-runners.md): our checks runners (label horizon-checks: the desktop's, an isolated user, so a fork's
|
||||
# pull request runs there too) take main's pushes and the pull requests whose number ends in 0, 1 or 2; GitHub's
|
||||
# runners take the others, so both pools work at once. The repository variable SELF_HOSTED_CI moves them: `all`
|
||||
# every job to ours, `light` (the desktop paused) or `off` every job to GitHub's. On ours the build folder persists
|
||||
# (CARGO_TARGET_DIR, outside the checkout), so the apt packages, Python and the Rust cache steps are GitHub's only.
|
||||
verify:
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI == 'all' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "linux", "playhorizon-ci"]') || 'ubuntu-latest' }}
|
||||
runs-on: >-
|
||||
${{ vars.SELF_HOSTED_CI != 'off' && vars.SELF_HOSTED_CI != 'light' && (vars.SELF_HOSTED_CI == 'all' ||
|
||||
github.event_name != 'pull_request' || endsWith(format('{0}', github.event.pull_request.number), '0') ||
|
||||
endsWith(format('{0}', github.event.pull_request.number), '1') ||
|
||||
endsWith(format('{0}', github.event.pull_request.number), '2')) &&
|
||||
fromJSON('["self-hosted", "linux", "horizon-checks"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -104,7 +109,12 @@ jobs:
|
||||
# changes it or a crate it builds is checked here, and so is every push to main: two pull requests can each pass
|
||||
# alone and break it together (#2059 and the registry ratings did, 2026-10-26).
|
||||
launcher:
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI == 'all' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON('["self-hosted", "linux", "playhorizon-ci"]') || 'ubuntu-latest' }}
|
||||
runs-on: >-
|
||||
${{ vars.SELF_HOSTED_CI != 'off' && vars.SELF_HOSTED_CI != 'light' && (vars.SELF_HOSTED_CI == 'all' ||
|
||||
github.event_name != 'pull_request' || endsWith(format('{0}', github.event.pull_request.number), '0') ||
|
||||
endsWith(format('{0}', github.event.pull_request.number), '1') ||
|
||||
endsWith(format('{0}', github.event.pull_request.number), '2')) &&
|
||||
fromJSON('["self-hosted", "linux", "horizon-checks"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -145,8 +155,8 @@ jobs:
|
||||
# a harness, an oracle, a probe or the runner, or carries the run-parity label (run-parity.yml, when it is added).
|
||||
parity-scope:
|
||||
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
|
||||
# A git diff of this repository's own branch: one of playhorizon's small runners (docs/ci-runners.md).
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "playhorizon-light"]') || 'ubuntu-latest' }}
|
||||
# A git diff of this repository's own branch: one of our small runners (horizon-light, docs/ci-runners.md).
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "horizon-light"]') || 'ubuntu-latest' }}
|
||||
outputs:
|
||||
run: ${{ steps.scope.outputs.run }}
|
||||
steps:
|
||||
|
||||
@@ -29,8 +29,8 @@ jobs:
|
||||
# the skipped runs of ordinary comments must not cancel a waiting one.
|
||||
concurrency:
|
||||
group: claim-${{ github.event.issue.number }}
|
||||
# One of playhorizon's small runners, out of GitHub's shared pool: main's script only (docs/ci-runners.md).
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "playhorizon-light"]') || 'ubuntu-latest' }}
|
||||
# One of our small runners (horizon-light), out of GitHub's pool: main's script only (docs/ci-runners.md).
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "horizon-light"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -56,8 +56,8 @@ jobs:
|
||||
run: python3 tools/ci/claim_command.py command
|
||||
release:
|
||||
if: github.event_name != 'issue_comment' && github.repository == 'horizon-rust/horizon'
|
||||
# One of playhorizon's small runners, out of GitHub's shared pool: main's script only (docs/ci-runners.md).
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "playhorizon-light"]') || 'ubuntu-latest' }}
|
||||
# One of our small runners (horizon-light), out of GitHub's pool: main's script only (docs/ci-runners.md).
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "horizon-light"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -17,7 +17,8 @@ permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
# Our checks runners (label horizon-checks, docs/ci-runners.md); SELF_HOSTED_CI=light or off: GitHub's.
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && vars.SELF_HOSTED_CI != 'light' && fromJSON('["self-hosted", "linux", "horizon-checks"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
defaults:
|
||||
run:
|
||||
@@ -25,7 +26,9 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: rustup toolchain install stable --profile minimal --component rustfmt --component clippy
|
||||
# Ours keep their build folder between jobs.
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
if: runner.environment == 'github-hosted'
|
||||
with:
|
||||
workspaces: crates/horizon-mod-registry
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
@@ -21,7 +21,8 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
# One of our small runners (label horizon-light, docs/ci-runners.md): main's script, Mojang's public manifest.
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "horizon-light"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -253,7 +253,8 @@ jobs:
|
||||
# Tells the thread that asked for this build how it ended. A conflict `pick` already explained there.
|
||||
needs: [pick, build, sign, publish]
|
||||
if: always() && inputs.issue != '' && needs.pick.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
# One comment, no checkout and no secret: one of our small runners (label horizon-light, docs/ci-runners.md).
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "horizon-light"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
issues: write
|
||||
|
||||
@@ -19,9 +19,9 @@ jobs:
|
||||
concurrency:
|
||||
group: overlap-note-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
# One of playhorizon's small runners, out of GitHub's shared pool: it runs main's script only (above), so a fork's
|
||||
# One of our small runners (horizon-light), out of GitHub's pool: it runs main's script only (above), so a fork's
|
||||
# pull request runs none of its code there (docs/ci-runners.md). SELF_HOSTED_CI=off: GitHub's runners.
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "playhorizon-light"]') || 'ubuntu-latest' }}
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "horizon-light"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -22,17 +22,24 @@ env:
|
||||
|
||||
jobs:
|
||||
bench:
|
||||
runs-on: ubuntu-latest
|
||||
# Our checks runners (label horizon-checks, docs/ci-runners.md): the branch and the base on the same machine, as
|
||||
# the comparison needs. SELF_HOSTED_CI=light or off: GitHub's.
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && vars.SELF_HOSTED_CI != 'light' && fromJSON('["self-hosted", "linux", "horizon-checks"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Install Rust and the ALSA and udev headers
|
||||
env:
|
||||
HOSTED: ${{ runner.environment == 'github-hosted' }}
|
||||
run: |
|
||||
sudo apt-get update && sudo apt-get install -y --no-install-recommends libasound2-dev libudev-dev
|
||||
if [ "$HOSTED" = true ]; then
|
||||
sudo apt-get update && sudo apt-get install -y --no-install-recommends libasound2-dev libudev-dev
|
||||
fi
|
||||
rustup toolchain install stable --profile minimal
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
if: runner.environment == 'github-hosted'
|
||||
with:
|
||||
shared-key: pack-bench
|
||||
- name: Bench the branch
|
||||
@@ -46,11 +53,15 @@ jobs:
|
||||
SCALE: ${{ inputs.scale || '1' }}
|
||||
run: |
|
||||
git fetch https://github.com/horizon-rust/horizon.git "$BASE"
|
||||
# Our runners keep the folder beside the checkout between jobs.
|
||||
git worktree remove --force ../base 2> /dev/null || rm -rf ../base
|
||||
git worktree prune
|
||||
git worktree add ../base FETCH_HEAD
|
||||
cp crates/horizon-client-render/examples/pack_load_bench.rs ../base/crates/horizon-client-render/examples/
|
||||
# A bench row that times a function the base lacks keeps it from building there: then
|
||||
# the branch's own rows compare the two ways.
|
||||
(cd ../base && CARGO_TARGET_DIR="$GITHUB_WORKSPACE/target" cargo run --profile fast -p horizon-client-render \
|
||||
target=${CARGO_TARGET_DIR:-$GITHUB_WORKSPACE/target}
|
||||
(cd ../base && CARGO_TARGET_DIR="$target" cargo run --profile fast -p horizon-client-render \
|
||||
--example pack_load_bench -- --scale "$SCALE") > base.md \
|
||||
|| echo "The bench does not build on the base." > base.md
|
||||
- name: Summary
|
||||
|
||||
@@ -16,9 +16,9 @@ jobs:
|
||||
label:
|
||||
# Title-only edits change nothing: skip them before a runner starts.
|
||||
if: github.event.action != 'edited' || github.event.changes.body != null
|
||||
# One of playhorizon's small runners, out of GitHub's shared pool: it runs main's script only (above), so a fork's
|
||||
# One of our small runners (horizon-light), out of GitHub's pool: it runs main's script only (above), so a fork's
|
||||
# pull request runs none of its code there (docs/ci-runners.md). SELF_HOSTED_CI=off: GitHub's runners.
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "playhorizon-light"]') || 'ubuntu-latest' }}
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "horizon-light"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -17,7 +17,8 @@ permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
# Our checks runners (label horizon-checks, docs/ci-runners.md); SELF_HOSTED_CI=light or off: GitHub's.
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && vars.SELF_HOSTED_CI != 'light' && fromJSON('["self-hosted", "linux", "horizon-checks"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
defaults:
|
||||
run:
|
||||
@@ -25,7 +26,9 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: rustup toolchain install stable --profile minimal --component rustfmt --component clippy
|
||||
# Ours keep their build folder between jobs.
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
if: runner.environment == 'github-hosted'
|
||||
with:
|
||||
workspaces: crates/horizon-telemetry-server
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
@@ -21,7 +21,8 @@ jobs:
|
||||
startsWith(github.event.comment.body, '/reopen') ||
|
||||
startsWith(github.event.comment.body, '/label') ||
|
||||
startsWith(github.event.comment.body, '/unlabel'))
|
||||
runs-on: ubuntu-latest
|
||||
# One of our small runners (label horizon-light, docs/ci-runners.md): main's script only, as above.
|
||||
runs-on: ${{ vars.SELF_HOSTED_CI != 'off' && fromJSON('["self-hosted", "linux", "horizon-light"]') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ To play, use [the launcher](../launcher/README.md); the project site is [playhor
|
||||
|
||||
- [Rust rewrite status](rust-parity.md): completion criteria, how to reproduce the validation, and per-area scope.
|
||||
- [Parity workflow](parity-workflow.md): `tools/parity.py`, the gate, other machines, merging.
|
||||
- [CI runners](ci-runners.md): which jobs run on GitHub's runners and which on ours (playhorizon, the Mac mini), and why a fork's pull request never runs on ours.
|
||||
- [CI runners](ci-runners.md): which jobs run on GitHub's runners and which on ours (the desktop, playhorizon, the Mac mini), the `SELF_HOSTED_CI` switch, and where a fork's pull request may run.
|
||||
- Area notes: [block behaviours and ticking](parity/blocks.md), [section meshing](parity/meshing.md).
|
||||
- [Port ledger](port-ledger.md): `tools/port_ledger.py`, every vanilla method next to the Rust functions that port it, both ways, with the gaps where a port isn't wired like vanilla.
|
||||
- [Version ports](version-ports.md): how a new Minecraft release is detected, diffed against the pinned one and ported, and what stays manual.
|
||||
|
||||
+71
-50
@@ -1,66 +1,71 @@
|
||||
# CI runners
|
||||
|
||||
GitHub gives the organisation 20 hosted jobs at a time, shared by every pull request, every push to main and the
|
||||
bots. On 2026-10-31 that pool ran ~12 jobs on average with 400+ runs queued and every job waited 20–25 minutes for a
|
||||
runner, while its work took 0.3–4 minutes. A quarter of that pool's time went to bots that run for seconds; those
|
||||
run on our own machines now, and the heavy checks can follow when the pool is full.
|
||||
bots. On 2026-11-03 that pool had 283–351 jobs waiting (median wait 43–55 minutes) for jobs that work 1–18
|
||||
minutes, and the bots' two small runners had 327–371 waiting. Since then (the maintainer's decision: "use everything
|
||||
we have to the max") every job that can runs on our own machines, and GitHub's pool keeps only what can't move.
|
||||
|
||||
| Runner | Machine | Labels | Takes |
|
||||
| --- | --- | --- | --- |
|
||||
| `playhorizon-ci-1`, `playhorizon-ci-2` | playhorizon (partner's server, 8 vCPU) | `playhorizon-ci` | Only with `SELF_HOSTED_CI=all`: `checks`' `verify` and `launcher` for main's pushes and this repository's own pull requests |
|
||||
| `playhorizon-light-1`, `playhorizon-light-2` | playhorizon | `playhorizon-light` | The bots that run main's own scripts: `pr labels`, `overlap note`, `cancel-on-close`, `claim command`, and `checks`' `parity-scope` |
|
||||
| `playhorizon-horizon` | playhorizon | `horizon-game`, `playhorizon` | Parity ([release-flow.md](release-flow.md#the-parity-job)) |
|
||||
| `ludodesktop-checks-1`…`-3` | ludodesktop (the maintainer's desktop, 24 threads), isolated user `gha` | `horizon-checks` | `checks`' `verify` and `launcher` (main's pushes, and the pull requests whose number ends in 0, 1 or 2, forks included), `mod-registry`, `telemetry-server`, `pack load bench` |
|
||||
| `playhorizon-light-1`…`-8` | playhorizon (partner's server, 8 vCPU) | `horizon-light`, `playhorizon-light` | The bots that run main's own scripts: `pr labels`, `overlap note`, `cancel-on-close`, `claim command`, `triage command`, `mojang republish`, `nightly build`'s `answer`, and `checks`' `parity-scope` |
|
||||
| `playhorizon-ci-1`, `playhorizon-ci-2` | playhorizon | `playhorizon-ci` | Nothing routed since 2026-11-03 (they took `checks` of this repository's own pull requests with the old `SELF_HOSTED_CI=all`) |
|
||||
| `playhorizon-horizon`, `ludodesktop-horizon`, `ludodesktop-parity-2` | playhorizon, ludodesktop | `horizon-game` | Parity ([release-flow.md](release-flow.md#the-parity-job)) |
|
||||
| `ludovics-mac-mini*` | Mac mini | `horizon-game`, `horizon-parity-main`, `horizon-release`, `horizon-nightly` | Parity, parity-main, macOS builds, releases |
|
||||
|
||||
Everything else stays on GitHub's runners: `checks` by default, a fork's pull request always, the Windows and Linux
|
||||
builds, and every job that holds a secret (signing, publishing: the partner keeps hypervisor access to playhorizon).
|
||||
The desktop's runners are declared in the maintainer's NixOS configuration (his private fleet repository,
|
||||
`nixos/ludodesktop/horizon-runner.nix`), not installed from here; playhorizon's by `tools/ci/checks-runners.sh`.
|
||||
|
||||
## What stays on GitHub, and why
|
||||
|
||||
- **Every job that holds a secret, or whose token can push, tag, publish or start workflows** (`contents: write`,
|
||||
`actions: write`): `release`'s and `launcher`'s `decide` (they choose the commit a release builds and start the
|
||||
build), `nightly build`'s `pick`, `sign` and `publish`, `launcher`'s `sign` and `publish`, `nightly command` (it
|
||||
writes the set of pull requests the Nightly holds), `native hook stamp` (it pushes a branch). playhorizon is a
|
||||
partner's server that the partner can reach, and a checks runner keeps what a fork's job left behind: neither may
|
||||
decide or sign what players download. `tests/test_ci_runners.py` pins it.
|
||||
- **Release builds**: Linux on `ubuntu-22.04` (players' binaries must link against its older glibc; our Linux
|
||||
machines have newer ones), Windows on `windows-latest` (no Windows machine). macOS builds already run on the Mac
|
||||
mini.
|
||||
- **Seven of ten pull requests' `verify` and `launcher`** (number ending in 3–9): three runners can't take
|
||||
~90 `checks` runs an hour alone (each brings `verify`, 10–18 minutes of work on GitHub, and `launcher`, 1–11),
|
||||
so both pools work at once. `SELF_HOSTED_CI=all` sends them all to ours.
|
||||
|
||||
## Which jobs, and when
|
||||
|
||||
The repository variable `SELF_HOSTED_CI` decides:
|
||||
The repository variable `SELF_HOSTED_CI` decides, from the next job queued (queued jobs keep the runner they asked
|
||||
for):
|
||||
|
||||
| `SELF_HOSTED_CI` | Bots (`playhorizon-light`) | `checks` of main and same-repository pull requests (`playhorizon-ci`) |
|
||||
| `SELF_HOSTED_CI` | Bots (`horizon-light`) | `checks`, `mod-registry`, `telemetry-server`, `pack load bench` (`horizon-checks`) |
|
||||
| --- | --- | --- |
|
||||
| unset (default) | playhorizon | GitHub |
|
||||
| `all` | playhorizon | playhorizon |
|
||||
| `off` | GitHub | GitHub |
|
||||
| unset (default) | ours | ours for main's pushes and the pull requests ending in 0–2; GitHub's for the others (`verify`, `launcher`); ours for the rest |
|
||||
| `all` | ours | ours, every pull request |
|
||||
| `light` | ours | GitHub's: the desktop is paused |
|
||||
| `off` | GitHub's | GitHub's: everything back, in one switch |
|
||||
|
||||
`gh variable set SELF_HOSTED_CI --body all --repo horizon-rust/horizon` (or `off`; `gh variable delete` for the
|
||||
default) takes effect from the next run; queued jobs keep the runner they asked for.
|
||||
```sh
|
||||
gh variable set SELF_HOSTED_CI --body light --repo horizon-rust/horizon # or all, off
|
||||
gh variable delete SELF_HOSTED_CI --repo horizon-rust/horizon # the default
|
||||
```
|
||||
|
||||
Why the checks stay on GitHub by default: the checks runners have CPUs 4–7, shared with the parity runner, about
|
||||
one GitHub runner's worth. With every same-repository pull request on them (#5415, 2026-10-31 20:25–20:45 UTC), 19
|
||||
jobs waited there, the oldest 18 minutes, while GitHub's pool, rid of the bots, ran 9–12 of its 20 jobs with nothing
|
||||
waiting. Use `all` when GitHub's pool is full again (hosted jobs waiting tens of minutes) and playhorizon is quiet.
|
||||
Set `off` when playhorizon is down or moves: the bots would wait for it.
|
||||
Set `light` when the desktop is paused (the maintainer uses it), `off` when playhorizon is down or moves: the bots
|
||||
would wait for it. Back to the default when they return.
|
||||
|
||||
## How long
|
||||
## Forks
|
||||
|
||||
Measured 2026-10-31 on #5415, with playhorizon's parity runner busy on the same CPUs (load ~15–20):
|
||||
A fork's pull request is code nobody here wrote. It may run on the checks runners only:
|
||||
|
||||
| Job | GitHub's runners: wait + work | playhorizon: wait + work |
|
||||
| --- | --- | --- |
|
||||
| `verify` (Rust changed) | 20–25 min + ~8 min (clippy 2.4, tests 4.2, Python 1.3) | ~0 + 10.5 min (clippy 3.7, tests 5.0, Python 1.7) |
|
||||
| `launcher` (launcher changed) | 20–25 min + 2.1 min | ~0 + 2.5 min |
|
||||
| a bot (`pr labels`, `overlap note`...) | 20–26 min + 0.3–0.9 min | ~0 + seconds |
|
||||
|
||||
The work is ~30% slower here (4 CPUs shared with parity, against a GitHub runner's 4 of its own), up to 16 minutes
|
||||
for `verify` when the machine is loaded (load ~30). That pays only while GitHub's pool makes jobs wait longer than
|
||||
that. A runner's first build of a workspace is cold: ~14 min for `verify`, ~11 for `launcher`.
|
||||
|
||||
## Forks never run here
|
||||
|
||||
A fork's pull request is code nobody here wrote. Three locks keep it off our runners:
|
||||
|
||||
1. The workflows pick the runner by event: `checks` sends a `pull_request` to playhorizon only when
|
||||
`github.event.pull_request.head.repo.full_name == github.repository` (and `SELF_HOSTED_CI=all`), else to
|
||||
`ubuntu-latest`. The bots run on
|
||||
`pull_request_target`, `issue_comment` and `schedule`, which always run main's workflow and main's scripts, never
|
||||
the pull request's (each workflow's header says how). `tests/test_ci_runners.py` pins both.
|
||||
2. A fork's pull request runs its *own* copy of the workflows, so it could name these labels anyway. Before every
|
||||
job, the runners' `job-started.sh` (`ACTIONS_RUNNER_HOOK_JOB_STARTED`) fails any pull request event whose head
|
||||
repository isn't this one, before a line of it is on disk.
|
||||
3. The repository asks a maintainer to approve the runs of every outside contributor's pull request (Settings →
|
||||
1. The checks runners are an isolated system user on the desktop (`gha`: no sudo, no logins, no SSH keys, `/home`
|
||||
invisible, the rest of the system read-only, no GPU or device), and their jobs' tokens write nothing and see no
|
||||
secret (`tests/test_ci_runners.py`).
|
||||
2. The light runners take main's own scripts only: the bots run on `pull_request_target`, `issue_comment`,
|
||||
`schedule` and main's pushes, which always run main's workflow and main's scripts, never the pull request's
|
||||
(each workflow's header says how); `checks`' `parity-scope` only for this repository's own branches. The test
|
||||
pins both.
|
||||
3. A fork's pull request runs its *own* copy of the workflows, so it could name the light labels anyway. Before
|
||||
every job, playhorizon's runners' `job-started.sh` (`ACTIONS_RUNNER_HOOK_JOB_STARTED`) fails any pull request
|
||||
event whose head repository isn't this one, before a line of it is on disk.
|
||||
4. The repository asks a maintainer to approve the runs of every outside contributor's pull request (Settings →
|
||||
Actions → "Require approval for all external contributors").
|
||||
|
||||
## On playhorizon
|
||||
@@ -75,14 +80,30 @@ ssh ludovic@playhorizon 'bash ~/horizon-install/checks-runners.sh status'
|
||||
```
|
||||
|
||||
- Each runner is a systemd user service (`horizon-runner-ci-<n>`, `horizon-runner-light-<n>`) in
|
||||
`~/.actions-runners/horizon-{ci,light}-<n>`, with its own work folder and `HOME`.
|
||||
`~/.actions-runners/horizon-{ci,light}-<n>`, with its own work folder and `HOME`. `LIGHT_RUNNERS` (default 8) and
|
||||
`CI_RUNNERS` (default 2) say how many.
|
||||
- All of them run nice and pinned to CPUs 4–7, inside `horizon-ci.slice` (the CPU quota and memory cap of all CI on
|
||||
the machine, `linux-runner.sh`). The checks runners have their own `horizon-ci-checks.slice` (memory ≤ 16 GB).
|
||||
CPUs 0–3 stay for `tools/remote.sh` runs, benchmarks and the servers.
|
||||
- `job-started.sh`, before every job: refuses a fork's pull request; empties a cargo build folder over 40 GB
|
||||
(`MAX_TARGET_GB`); drops a checkout's `.git` over 4 GB (`MAX_GIT_GB`), which the checkout then clones again. Every
|
||||
job's fetch adds a pack and nothing repacks them: on 2026-11-03 each light runner's checkout held 6822 packs, 50 GB.
|
||||
- The checks runners share one cargo home (`~/.actions-runners/ci-shared/cargo`: registry and git sources). Each has
|
||||
its own rustup home and its own build folder (`CARGO_TARGET_DIR`, outside the checkout, so the checkout's clean
|
||||
keeps it): a run rebuilds only what changed. `CARGO_BUILD_JOBS=4`. `job-started.sh` empties a build folder over
|
||||
40 GB (`MAX_TARGET_GB`) before the job.
|
||||
- No sccache: each build folder already holds the dependencies, and the host's shared cache is the remote runs'.
|
||||
its own rustup home and its own build folder (`CARGO_TARGET_DIR`, outside the checkout). `CARGO_BUILD_JOBS=4`.
|
||||
- Python is the host's 3.13 (`ci-shared/bin/python3`); the apt packages the jobs need are installed by the script, so
|
||||
the jobs skip their `apt-get`, `setup-python` and Rust cache steps there (`runner.environment == 'github-hosted'`).
|
||||
|
||||
## On the desktop
|
||||
|
||||
`gha.slice` holds every runner there (parity and checks): CPU and I/O weight 20, 22 of the 24 threads (12 while the
|
||||
CPU is at 92 °C or more), memory 24 GB soft / 26 GB hard. Each checks runner has `CARGO_BUILD_JOBS=6`, its own
|
||||
build folder (`/var/lib/gha/checks-<n>/target`, kept between jobs, emptied past 30 GB before the runner starts), its
|
||||
own rustup and cargo homes, Python 3.13, gh (no login), the libraries the game and GPUI link against, and its own
|
||||
Xvfb with Mesa's software Vulkan. They run all the time; the maintainer pauses them when he uses the desktop:
|
||||
|
||||
```sh
|
||||
ssh desktop 'sudo -n touch /var/lib/gha-control/pause && sudo -n systemctl stop gha-runners.target' # pause
|
||||
ssh desktop 'sudo -n rm -f /var/lib/gha-control/pause && sudo -n systemctl start gha-runners.target' # resume
|
||||
```
|
||||
|
||||
With the desktop paused, set `SELF_HOSTED_CI=light`, or `checks` waits for it.
|
||||
@@ -13,7 +13,7 @@ reopened; labels never start it):
|
||||
|
||||
| Job | Runs on | What | Time |
|
||||
| --- | --- | --- | --- |
|
||||
| `verify` | GitHub `ubuntu-latest`; with `SELF_HOSTED_CI=all`, playhorizon's checks runners for main's pushes and this repository's own pull requests, never a fork's ([CI runners](ci-runners.md)) | Public-tree check, Python tests, rustfmt, clippy with warnings denied, Rust tests, one after another in one job (one runner from the organisation's shared pool per run). Dependencies and the workspace build come from a cache saved by main's runs (GitHub) or the runner's own build folder (playhorizon) | ~8 min |
|
||||
| `verify` | Our checks runners (label `horizon-checks`, the desktop's) for main's pushes and the pull requests whose number ends in 0–2, forks included; GitHub `ubuntu-latest` for the others. `SELF_HOSTED_CI` moves them: `all` all to ours, `light` or `off` all to GitHub ([CI runners](ci-runners.md)) | Public-tree check, Python tests, rustfmt, clippy with warnings denied, Rust tests, one after another in one job. Dependencies and the workspace build come from a cache saved by main's runs (GitHub) or the runner's own build folder (ours) | ~8 min |
|
||||
| `parity / parity` | A runner with the label `horizon-game`: the Mac mini's and playhorizon's (below) | Only for a change to a harness, an oracle, a probe or the runner (or the `run-parity` label): the vanilla harnesses the change can affect (`tools/parity.py --changed <base>`). Other pull requests skip it: [parity-main](#parity-on-main) checks them on main | seconds when nothing is affected; minutes per affected harness |
|
||||
|
||||
Pushes to main run `verify` only; the nightly audit covers main's harnesses.
|
||||
|
||||
+94
-26
@@ -1,12 +1,22 @@
|
||||
"""playhorizon's checks runners (docs/ci-runners.md) never run a fork's pull request."""
|
||||
"""Which jobs may run on our own runners (docs/ci-runners.md).
|
||||
|
||||
- horizon-light (playhorizon's and other small runners, which refuse a fork's pull request): main's own scripts only.
|
||||
- horizon-checks (isolated runners that take any pull request, forks included): no secret and no token that writes.
|
||||
- Neither takes a job that holds a secret or whose token can push, publish or start workflows: playhorizon is a
|
||||
partner's server, and a checks runner keeps what a fork's job left behind.
|
||||
"""
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
WORKFLOWS = ROOT / ".github/workflows"
|
||||
LABELS = ("playhorizon-ci", "playhorizon-light")
|
||||
LIGHT = ("horizon-light", "playhorizon-light", "playhorizon-ci")
|
||||
CHECKS = ("horizon-checks",)
|
||||
FORK_GUARD = "github.event.pull_request.head.repo.full_name == github.repository"
|
||||
TRUSTED_WRITES = re.compile(r"^\s*(contents|actions|packages|id-token|deployments|attestations): write", re.M)
|
||||
# actions: write can start workflows too; cancel-on-close needs it to cancel runs and runs no checkout at all.
|
||||
CANCELS_RUNS = {"cancel-on-close.yml cancel"}
|
||||
|
||||
|
||||
def jobs(text):
|
||||
@@ -22,31 +32,89 @@ def triggers(text):
|
||||
return set(re.findall(r"^ ([a-z_]+):", on.group(1), re.M)) if on else set()
|
||||
|
||||
|
||||
class ChecksRunnerTests(unittest.TestCase):
|
||||
def test_self_hosted_jobs_never_take_a_fork_pull_request(self):
|
||||
routed = 0
|
||||
for path in sorted(WORKFLOWS.glob("*.yml")):
|
||||
text = path.read_text()
|
||||
on = triggers(text)
|
||||
for name, block in jobs(text):
|
||||
runs_on = next((l for l in block.splitlines() if l.strip().startswith("runs-on:")), "")
|
||||
if not any(label in runs_on for label in LABELS):
|
||||
continue
|
||||
routed += 1
|
||||
where = f"{path.name} {name}"
|
||||
# The switch back to GitHub's runners.
|
||||
self.assertTrue("vars.SELF_HOSTED_CI != 'off'" in runs_on or "vars.SELF_HOSTED_CI == 'all'" in runs_on,
|
||||
where)
|
||||
self.assertIn("'ubuntu-latest'", runs_on, where)
|
||||
if "pull_request" in on:
|
||||
job_if = next((l for l in block.splitlines() if l.strip().startswith("if:")), "")
|
||||
self.assertTrue(FORK_GUARD in runs_on or FORK_GUARD in job_if, f"{where}: a fork's pull request")
|
||||
if "pull_request_target" in on:
|
||||
# main's code only: every checkout is main's.
|
||||
self.assertEqual(block.count("uses: actions/checkout"), block.count("ref: main"), where)
|
||||
self.assertGreaterEqual(routed, 3)
|
||||
def runs_on(block):
|
||||
"""The job's runs-on value, a folded (`>-`) one included."""
|
||||
lines = block.splitlines()
|
||||
for i, line in enumerate(lines):
|
||||
if line.strip().startswith("runs-on:"):
|
||||
value = line.split("runs-on:", 1)[1].strip()
|
||||
if value in (">-", ">", "|"):
|
||||
indent = len(line) - len(line.lstrip()) + 2
|
||||
# The folded lines right under it.
|
||||
folded = []
|
||||
for l in lines[i + 1:]:
|
||||
if not l.startswith(" " * indent):
|
||||
break
|
||||
folded.append(l.strip())
|
||||
value = " ".join(folded)
|
||||
return value
|
||||
return ""
|
||||
|
||||
def test_the_runners_refuse_forks_themselves(self):
|
||||
|
||||
def routed():
|
||||
"""(where, triggers, workflow text, job block, runs-on, pools) for every job that can run on our runners."""
|
||||
for path in sorted(WORKFLOWS.glob("*.yml")):
|
||||
text = path.read_text()
|
||||
on = triggers(text)
|
||||
for name, block in jobs(text):
|
||||
value = runs_on(block)
|
||||
pools = {pool for pool, labels in (("light", LIGHT), ("checks", CHECKS))
|
||||
if any(f'"{label}"' in value for label in labels)}
|
||||
if pools:
|
||||
yield f"{path.name} {name}", on, text, block, value, pools
|
||||
|
||||
|
||||
class OwnRunnerTests(unittest.TestCase):
|
||||
def test_every_job_on_our_runners_can_go_back_to_github(self):
|
||||
count = 0
|
||||
for where, _, _, _, value, pools in routed():
|
||||
count += 1
|
||||
self.assertIn("vars.SELF_HOSTED_CI != 'off'", value, where)
|
||||
self.assertTrue(value.endswith("|| 'ubuntu-latest' }}"), where)
|
||||
if "checks" in pools:
|
||||
# The desktop paused: every checks job back to GitHub, the bots stay.
|
||||
self.assertIn("vars.SELF_HOSTED_CI != 'light'", value, where)
|
||||
self.assertGreaterEqual(count, 10)
|
||||
|
||||
def test_light_runners_take_main_code_only(self):
|
||||
for where, on, _, block, _, pools in routed():
|
||||
if "light" not in pools:
|
||||
continue
|
||||
if "pull_request" in on:
|
||||
job_if = next((l for l in block.splitlines() if l.strip().startswith("if:")), "")
|
||||
self.assertIn(FORK_GUARD, job_if, f"{where}: a fork's pull request")
|
||||
if "pull_request_target" in on:
|
||||
# main's code only: every checkout is main's.
|
||||
self.assertEqual(block.count("uses: actions/checkout"), block.count("ref: main"), where)
|
||||
|
||||
def test_no_secret_or_writing_token_on_our_runners(self):
|
||||
for where, _, text, block, _, pools in routed():
|
||||
self.assertNotIn("secrets.", block, where)
|
||||
self.assertNotIn("environment:", block, where)
|
||||
head = text.split("\njobs:\n", 1)[0]
|
||||
for part in (block, head):
|
||||
writes = {m.group(1) for m in TRUSTED_WRITES.finditer(part)}
|
||||
if where in CANCELS_RUNS:
|
||||
writes -= {"actions"}
|
||||
self.assertNotIn("actions/checkout", block, where)
|
||||
self.assertEqual(writes, set(), where)
|
||||
if "checks" in pools:
|
||||
# A checks runner keeps what a fork's job left: its jobs' tokens write nothing at all.
|
||||
self.assertNotRegex(block, r":\s*write", where)
|
||||
self.assertNotRegex(head, r":\s*write", where)
|
||||
|
||||
def test_checks_split_between_both_pools(self):
|
||||
ci = (WORKFLOWS / "ci.yml").read_text()
|
||||
verify = dict(jobs(ci))["verify"]
|
||||
value = runs_on(verify)
|
||||
# Main's pushes on ours; a pull request by the last digit of its number unless SELF_HOSTED_CI=all.
|
||||
self.assertIn("github.event_name != 'pull_request'", value)
|
||||
self.assertIn("vars.SELF_HOSTED_CI == 'all'", value)
|
||||
self.assertIn("endsWith(format('{0}', github.event.pull_request.number), '0')", value)
|
||||
self.assertNotIn(FORK_GUARD, value)
|
||||
self.assertEqual(value, runs_on(dict(jobs(ci))["launcher"]))
|
||||
|
||||
def test_the_light_runners_refuse_forks_themselves(self):
|
||||
script = (ROOT / "tools/ci/checks-runners.sh").read_text()
|
||||
self.assertIn("ACTIONS_RUNNER_HOOK_JOB_STARTED=$shared/job-started.sh", script)
|
||||
self.assertIn('.pull_request.head.repo.full_name // "unknown"', script)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# The checks runners on playhorizon (the partner's Linux server, docs/ci-runners.md): GitHub's 20 hosted jobs at a
|
||||
# time are shared by every pull request, so the same-repository ones and main's pushes run here instead.
|
||||
# The bots' runners and two checks runners on playhorizon (the partner's Linux server, docs/ci-runners.md): GitHub's
|
||||
# 20 hosted jobs at a time are shared by every pull request and every bot, so the bots run here instead.
|
||||
#
|
||||
# bash tools/ci/checks-runners.sh install < token # register (token on stdin), install, start; again to update
|
||||
# bash tools/ci/checks-runners.sh status
|
||||
@@ -11,15 +11,19 @@
|
||||
# ssh ludovic@playhorizon 'bash ~/horizon-install/checks-runners.sh install'
|
||||
#
|
||||
# Two kinds of runner, each a systemd user service in ~/.actions-runners/<name>:
|
||||
# - horizon-ci-<n> (label playhorizon-ci, CI_RUNNERS of them, default 2): ci.yml's verify and launcher jobs. Each has
|
||||
# - horizon-ci-<n> (label playhorizon-ci, CI_RUNNERS of them, default 2): this repository's own verify and launcher
|
||||
# jobs (no workflow routes them here since 2026-11-03: the desktop's horizon-checks runners take them). Each has
|
||||
# its own work folder, rustup home and cargo target folder (outside the checkout, so the checkout's clean leaves it),
|
||||
# and they share one cargo home (registry and git sources).
|
||||
# - horizon-light-<n> (label playhorizon-light, LIGHT_RUNNERS, default 2): the bots that run main's own scripts on
|
||||
# pull_request_target and comment events (pr labels, overlap note, cancel-on-close, claim command). No Rust.
|
||||
# - horizon-light-<n> (labels playhorizon-light and horizon-light, LIGHT_RUNNERS, default 8): the bots that run main's
|
||||
# own scripts on pull_request_target, comment, schedule and main's push events (pr labels, overlap note,
|
||||
# cancel-on-close, claim command...). No Rust. Their jobs last seconds, so they are many; horizon-light is the label
|
||||
# the workflows ask for.
|
||||
# All of them run nice, pinned to CPUs 4-7 inside horizon-ci.slice (linux-runner.sh: CPU quota and memory cap for
|
||||
# all CI on the machine), so CPUs 0-3 stay for tools/remote.sh runs, benchmarks and the servers.
|
||||
# Before every job, ci-shared/job-started.sh fails a job from a fork's pull request (whatever its workflow file says)
|
||||
# and empties a cargo target folder grown past MAX_TARGET_GB.
|
||||
# Before every job, ci-shared/job-started.sh fails a job from a fork's pull request (whatever its workflow file says),
|
||||
# empties a cargo target folder grown past MAX_TARGET_GB and drops a checkout's .git grown past MAX_GIT_GB (every
|
||||
# fetch adds a pack and nothing repacks them: 6822 packs, 50 GB, in each light runner's checkout on 2026-11-03).
|
||||
set -euo pipefail
|
||||
repo=horizon-rust/horizon
|
||||
base=$HOME/.actions-runners
|
||||
@@ -27,13 +31,13 @@ shared=$base/ci-shared
|
||||
units=${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user
|
||||
version=2.338.0
|
||||
sha=af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32
|
||||
ci=${CI_RUNNERS:-2} light=${LIGHT_RUNNERS:-2} cpus=${RUNNER_CPU_LIST:-4-7} jobs=${CI_JOBS:-4}
|
||||
ci=${CI_RUNNERS:-2} light=${LIGHT_RUNNERS:-8} cpus=${RUNNER_CPU_LIST:-4-7} jobs=${CI_JOBS:-4}
|
||||
packages="build-essential pkg-config git curl jq gh libasound2-dev libudev-dev libxkbcommon-x11-dev libwayland-dev
|
||||
libx11-xcb-dev libfontconfig-dev libvulkan-dev"
|
||||
|
||||
runners() {
|
||||
for i in $(seq 1 "$ci"); do echo "horizon-ci-$i playhorizon-ci"; done
|
||||
for i in $(seq 1 "$light"); do echo "horizon-light-$i playhorizon-light"; done
|
||||
for i in $(seq 1 "$light"); do echo "horizon-light-$i playhorizon-light,horizon-light"; done
|
||||
}
|
||||
|
||||
shared_files() {
|
||||
@@ -64,6 +68,15 @@ if [ -n "${CARGO_TARGET_DIR:-}" ] && [ -d "$CARGO_TARGET_DIR" ]; then
|
||||
rm -rf "${CARGO_TARGET_DIR:?}"
|
||||
fi
|
||||
fi
|
||||
# The checkout's history: actions/checkout fetches into the same .git every job and never repacks. Without .git it
|
||||
# clones again (and replaces the rest of the folder).
|
||||
if [ -n "${GITHUB_WORKSPACE:-}" ] && [ -d "$GITHUB_WORKSPACE/.git" ]; then
|
||||
size=$(du -s --block-size=1G "$GITHUB_WORKSPACE/.git" | cut -f1)
|
||||
if [ "$size" -gt "${MAX_GIT_GB:-4}" ]; then
|
||||
echo "Checkout's .git at ${size} GB (over ${MAX_GIT_GB:-4}): removed, this job clones again"
|
||||
rm -rf "${GITHUB_WORKSPACE:?}/.git"
|
||||
fi
|
||||
fi
|
||||
EOF
|
||||
chmod +x "$shared/job-started.sh"
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user