ci: see runner jobs after a runner self-update; keep the Mac's disk free

GitHub's runner moved its binaries to bin.<version>/ after a self-update, so
`pgrep -f "$runner/bin/Runner.Worker"` never matched: runner-gc took busy
runners for idle and deleted the probe builds and old prepared resources of
two running parity jobs (2026-11-07). Match bin*/ in runner-gc,
checks-runners.sh and linux-runner.sh.

runner-gc also counts shared-local/parity-target (19-24 GB per runner),
recreates it empty after removing it (a dangling .local link failed every
harness), and drops harness folders over 2 GB untouched for 6 hours.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Horizon agentandClaude Opus 5.5 committed 2026-11-07 11:53:10 +01:00
1 parent fd06d1c265
commit 4f2f4078d1
3 files changed
+16 -5

No files matched your search

+2 -2
View File
@@ -184,7 +184,7 @@ EOF
local unit=horizon-runner-${name#horizon-}.service
systemctl --user enable "$unit" > /dev/null 2>&1
# The runner reads .env when it starts: restart only an idle one (a restart cancels its job).
if pgrep -f "$base/$name/bin/Runner.Worker" > /dev/null; then echo "$name in a job: restart $unit after it"
if pgrep -f "$base/$name/bin[^/]*/Runner.Worker" > /dev/null; then echo "$name in a job: restart $unit after it"
else systemctl --user restart "$unit"; fi
done < <(runners)
status
@@ -196,7 +196,7 @@ status() {
for dir in "$base"/horizon-ci-* "$base"/horizon-light-*; do
[ -d "$dir" ] || continue
printf '%-16s %s, target %s\n' "$(basename "$dir")" \
"$(pgrep -f "$dir/bin/Runner.Worker" > /dev/null && echo busy || echo idle)" \
"$(pgrep -f "$dir/bin[^/]*/Runner.Worker" > /dev/null && echo busy || echo idle)" \
"$(du -sh "$dir/target" 2>/dev/null | cut -f1 || echo -)"
done
}
+1 -1
View File
@@ -213,7 +213,7 @@ EOF
systemctl --user enable horizon-runner-pause.service > /dev/null
# The runner reads .env when it starts: stop it, unless it is in a job (that would cancel the job); the watcher
# starts it again when it may run.
if pgrep -f "$dir/bin/Runner.Worker" > /dev/null; then echo "In a job: stop horizon-runner.service after it"
if pgrep -f "$dir/bin[^/]*/Runner.Worker" > /dev/null; then echo "In a job: stop horizon-runner.service after it"
else systemctl --user stop horizon-runner.service; rm -f "$dir/paused"; fi
systemctl --user restart horizon-runner-pause.service
status
+13 -2
View File
@@ -18,7 +18,7 @@ root=${RUNNERS:-$HOME/.actions-runners}
max_kb=$(( ${MAX_TARGET_GB:-12} * 1024 * 1024 ))
keep=${KEEP_PREPARED:-2}
stale_min=$(( ${STALE_DAYS:-3} * 24 * 60 ))
busy() { pgrep -f "$1/bin/Runner.Worker" > /dev/null; }
busy() { pgrep -f "$1/bin[^/]*/Runner.Worker" > /dev/null; }
any_busy=0
for runner in "$root"/horizon "$root"/horizon-[0-9]*; do
@@ -31,17 +31,28 @@ for runner in "$root"/horizon "$root"/horizon-[0-9]*; do
for dir in "$checkout"/.local/*/; do
dir=${dir%/}
[ -L "$dir" ] || [ "$(basename "$dir")" = parity-runner ] && continue
if [ -z "$(find "$dir" -mmin -"$stale_min" -print -quit 2>/dev/null)" ]; then rm -rf "$dir"; stale=$((stale + 1)); fi
if [ -z "$(find "$dir" -mmin -"$stale_min" -print -quit 2>/dev/null)" ]; then rm -rf "$dir"; stale=$((stale + 1)); continue; fi
# A few harness folders reach 5+ GB (stored_effect_notifications, attribute_map_queries; the Mac fell to 1.3 GB
# free on 2026-11-07): drop folders over 2 GB once untouched for 6 hours, not 3 days.
if [ -z "$(find "$dir" -mmin -360 -print -quit 2>/dev/null)" ] && [ "$(du -sk "$dir" | cut -f1)" -gt 2097152 ]; then
rm -rf "$dir"; stale=$((stale + 1))
fi
done
[ "$stale" = 0 ] || echo "$(basename "$runner"): removed $stale harness folders untouched for ${STALE_DAYS:-3} days"
targets=()
[ -f "$checkout/target/CACHEDIR.TAG" ] && targets+=("$checkout/target")
while IFS= read -r tag; do targets+=("$(dirname "$tag")"); done < <(
find "$checkout/.local" -mindepth 2 -maxdepth 2 -name CACHEDIR.TAG -type f 2>/dev/null)
# The probes' build folder lives in shared-local (a symlink from .local), which the scan above doesn't follow:
# it grew to 19-24 GB per runner (2026-11-06, the Mac down to 6 GB free).
[ -d "$runner/_work/horizon/shared-local/parity-target" ] && targets+=("$runner/_work/horizon/shared-local/parity-target")
[ ${#targets[@]} -gt 0 ] || continue
size=$(du -sk "${targets[@]}" | awk '{s += $1} END {print s}')
if [ "$size" -gt "$max_kb" ]; then
rm -rf "${targets[@]}"
# .local/parity-target is a symlink into shared-local: cargo can't create a directory through a dangling
# link ("Not a directory"), which failed every harness of parity-main on 2026-11-06. Recreate it empty.
mkdir -p "$runner/_work/horizon/shared-local/parity-target" 2>/dev/null || true
echo "$(basename "$runner"): removed ${#targets[@]} build folders ($(( size / 1024 / 1024 )) GB)"
fi
done