From 4f2f4078d1dcfb012f949f1bcf9654ad300f926d Mon Sep 17 00:00:00 2001 From: Horizon agent Date: Sat, 7 Nov 2026 11:53:10 +0100 Subject: [PATCH] ci: see runner jobs after a runner self-update; keep the Mac's disk free GitHub's runner moved its binaries to bin./ after a self-update, so `pgrep -f "$runner/bin/Runner.Worker"` never matched: runner-gc took busy runners for idle and deleted the probe builds and old prepared resources of two running parity jobs (2026-11-07). Match bin*/ in runner-gc, checks-runners.sh and linux-runner.sh. runner-gc also counts shared-local/parity-target (19-24 GB per runner), recreates it empty after removing it (a dangling .local link failed every harness), and drops harness folders over 2 GB untouched for 6 hours. Co-Authored-By: Claude Opus 5.5 --- tools/ci/checks-runners.sh | 4 ++-- tools/ci/linux-runner.sh | 2 +- tools/ci/runner-gc.sh | 15 +++++++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/ci/checks-runners.sh b/tools/ci/checks-runners.sh index 43bd691dbe..04dd6bb5bf 100644 --- a/tools/ci/checks-runners.sh +++ b/tools/ci/checks-runners.sh @@ -184,7 +184,7 @@ EOF local unit=horizon-runner-${name#horizon-}.service systemctl --user enable "$unit" > /dev/null 2>&1 # The runner reads .env when it starts: restart only an idle one (a restart cancels its job). - if pgrep -f "$base/$name/bin/Runner.Worker" > /dev/null; then echo "$name in a job: restart $unit after it" + if pgrep -f "$base/$name/bin[^/]*/Runner.Worker" > /dev/null; then echo "$name in a job: restart $unit after it" else systemctl --user restart "$unit"; fi done < <(runners) status @@ -196,7 +196,7 @@ status() { for dir in "$base"/horizon-ci-* "$base"/horizon-light-*; do [ -d "$dir" ] || continue printf '%-16s %s, target %s\n' "$(basename "$dir")" \ - "$(pgrep -f "$dir/bin/Runner.Worker" > /dev/null && echo busy || echo idle)" \ + "$(pgrep -f "$dir/bin[^/]*/Runner.Worker" > /dev/null && echo busy || echo idle)" \ "$(du -sh "$dir/target" 2>/dev/null | cut -f1 || echo -)" done } diff --git a/tools/ci/linux-runner.sh b/tools/ci/linux-runner.sh index 1911ff94a0..bdbdb5df9e 100755 --- a/tools/ci/linux-runner.sh +++ b/tools/ci/linux-runner.sh @@ -213,7 +213,7 @@ EOF systemctl --user enable horizon-runner-pause.service > /dev/null # The runner reads .env when it starts: stop it, unless it is in a job (that would cancel the job); the watcher # starts it again when it may run. - if pgrep -f "$dir/bin/Runner.Worker" > /dev/null; then echo "In a job: stop horizon-runner.service after it" + if pgrep -f "$dir/bin[^/]*/Runner.Worker" > /dev/null; then echo "In a job: stop horizon-runner.service after it" else systemctl --user stop horizon-runner.service; rm -f "$dir/paused"; fi systemctl --user restart horizon-runner-pause.service status diff --git a/tools/ci/runner-gc.sh b/tools/ci/runner-gc.sh index 7ccf69ff05..590a83f0fb 100755 --- a/tools/ci/runner-gc.sh +++ b/tools/ci/runner-gc.sh @@ -18,7 +18,7 @@ root=${RUNNERS:-$HOME/.actions-runners} max_kb=$(( ${MAX_TARGET_GB:-12} * 1024 * 1024 )) keep=${KEEP_PREPARED:-2} stale_min=$(( ${STALE_DAYS:-3} * 24 * 60 )) -busy() { pgrep -f "$1/bin/Runner.Worker" > /dev/null; } +busy() { pgrep -f "$1/bin[^/]*/Runner.Worker" > /dev/null; } any_busy=0 for runner in "$root"/horizon "$root"/horizon-[0-9]*; do @@ -31,17 +31,28 @@ for runner in "$root"/horizon "$root"/horizon-[0-9]*; do for dir in "$checkout"/.local/*/; do dir=${dir%/} [ -L "$dir" ] || [ "$(basename "$dir")" = parity-runner ] && continue - if [ -z "$(find "$dir" -mmin -"$stale_min" -print -quit 2>/dev/null)" ]; then rm -rf "$dir"; stale=$((stale + 1)); fi + if [ -z "$(find "$dir" -mmin -"$stale_min" -print -quit 2>/dev/null)" ]; then rm -rf "$dir"; stale=$((stale + 1)); continue; fi + # A few harness folders reach 5+ GB (stored_effect_notifications, attribute_map_queries; the Mac fell to 1.3 GB + # free on 2026-11-07): drop folders over 2 GB once untouched for 6 hours, not 3 days. + if [ -z "$(find "$dir" -mmin -360 -print -quit 2>/dev/null)" ] && [ "$(du -sk "$dir" | cut -f1)" -gt 2097152 ]; then + rm -rf "$dir"; stale=$((stale + 1)) + fi done [ "$stale" = 0 ] || echo "$(basename "$runner"): removed $stale harness folders untouched for ${STALE_DAYS:-3} days" targets=() [ -f "$checkout/target/CACHEDIR.TAG" ] && targets+=("$checkout/target") while IFS= read -r tag; do targets+=("$(dirname "$tag")"); done < <( find "$checkout/.local" -mindepth 2 -maxdepth 2 -name CACHEDIR.TAG -type f 2>/dev/null) + # The probes' build folder lives in shared-local (a symlink from .local), which the scan above doesn't follow: + # it grew to 19-24 GB per runner (2026-11-06, the Mac down to 6 GB free). + [ -d "$runner/_work/horizon/shared-local/parity-target" ] && targets+=("$runner/_work/horizon/shared-local/parity-target") [ ${#targets[@]} -gt 0 ] || continue size=$(du -sk "${targets[@]}" | awk '{s += $1} END {print s}') if [ "$size" -gt "$max_kb" ]; then rm -rf "${targets[@]}" + # .local/parity-target is a symlink into shared-local: cargo can't create a directory through a dangling + # link ("Not a directory"), which failed every harness of parity-main on 2026-11-06. Recreate it empty. + mkdir -p "$runner/_work/horizon/shared-local/parity-target" 2>/dev/null || true echo "$(basename "$runner"): removed ${#targets[@]} build folders ($(( size / 1024 / 1024 )) GB)" fi done