mirror of
https://github.com/horizon-rust/horizon.git
synced 2026-11-07 02:56:17 -10:00
ci: see runner jobs after a runner self-update; keep the Mac's disk free
GitHub's runner moved its binaries to bin.<version>/ after a self-update, so `pgrep -f "$runner/bin/Runner.Worker"` never matched: runner-gc took busy runners for idle and deleted the probe builds and old prepared resources of two running parity jobs (2026-11-07). Match bin*/ in runner-gc, checks-runners.sh and linux-runner.sh. runner-gc also counts shared-local/parity-target (19-24 GB per runner), recreates it empty after removing it (a dangling .local link failed every harness), and drops harness folders over 2 GB untouched for 6 hours. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
fd06d1c265
commit
4f2f4078d1
3 files changed
+16
-5
No files matched your search
@@ -184,7 +184,7 @@ EOF
|
||||
local unit=horizon-runner-${name#horizon-}.service
|
||||
systemctl --user enable "$unit" > /dev/null 2>&1
|
||||
# The runner reads .env when it starts: restart only an idle one (a restart cancels its job).
|
||||
if pgrep -f "$base/$name/bin/Runner.Worker" > /dev/null; then echo "$name in a job: restart $unit after it"
|
||||
if pgrep -f "$base/$name/bin[^/]*/Runner.Worker" > /dev/null; then echo "$name in a job: restart $unit after it"
|
||||
else systemctl --user restart "$unit"; fi
|
||||
done < <(runners)
|
||||
status
|
||||
@@ -196,7 +196,7 @@ status() {
|
||||
for dir in "$base"/horizon-ci-* "$base"/horizon-light-*; do
|
||||
[ -d "$dir" ] || continue
|
||||
printf '%-16s %s, target %s\n' "$(basename "$dir")" \
|
||||
"$(pgrep -f "$dir/bin/Runner.Worker" > /dev/null && echo busy || echo idle)" \
|
||||
"$(pgrep -f "$dir/bin[^/]*/Runner.Worker" > /dev/null && echo busy || echo idle)" \
|
||||
"$(du -sh "$dir/target" 2>/dev/null | cut -f1 || echo -)"
|
||||
done
|
||||
}
|
||||
|
||||
@@ -213,7 +213,7 @@ EOF
|
||||
systemctl --user enable horizon-runner-pause.service > /dev/null
|
||||
# The runner reads .env when it starts: stop it, unless it is in a job (that would cancel the job); the watcher
|
||||
# starts it again when it may run.
|
||||
if pgrep -f "$dir/bin/Runner.Worker" > /dev/null; then echo "In a job: stop horizon-runner.service after it"
|
||||
if pgrep -f "$dir/bin[^/]*/Runner.Worker" > /dev/null; then echo "In a job: stop horizon-runner.service after it"
|
||||
else systemctl --user stop horizon-runner.service; rm -f "$dir/paused"; fi
|
||||
systemctl --user restart horizon-runner-pause.service
|
||||
status
|
||||
|
||||
+13
-2
@@ -18,7 +18,7 @@ root=${RUNNERS:-$HOME/.actions-runners}
|
||||
max_kb=$(( ${MAX_TARGET_GB:-12} * 1024 * 1024 ))
|
||||
keep=${KEEP_PREPARED:-2}
|
||||
stale_min=$(( ${STALE_DAYS:-3} * 24 * 60 ))
|
||||
busy() { pgrep -f "$1/bin/Runner.Worker" > /dev/null; }
|
||||
busy() { pgrep -f "$1/bin[^/]*/Runner.Worker" > /dev/null; }
|
||||
|
||||
any_busy=0
|
||||
for runner in "$root"/horizon "$root"/horizon-[0-9]*; do
|
||||
@@ -31,17 +31,28 @@ for runner in "$root"/horizon "$root"/horizon-[0-9]*; do
|
||||
for dir in "$checkout"/.local/*/; do
|
||||
dir=${dir%/}
|
||||
[ -L "$dir" ] || [ "$(basename "$dir")" = parity-runner ] && continue
|
||||
if [ -z "$(find "$dir" -mmin -"$stale_min" -print -quit 2>/dev/null)" ]; then rm -rf "$dir"; stale=$((stale + 1)); fi
|
||||
if [ -z "$(find "$dir" -mmin -"$stale_min" -print -quit 2>/dev/null)" ]; then rm -rf "$dir"; stale=$((stale + 1)); continue; fi
|
||||
# A few harness folders reach 5+ GB (stored_effect_notifications, attribute_map_queries; the Mac fell to 1.3 GB
|
||||
# free on 2026-11-07): drop folders over 2 GB once untouched for 6 hours, not 3 days.
|
||||
if [ -z "$(find "$dir" -mmin -360 -print -quit 2>/dev/null)" ] && [ "$(du -sk "$dir" | cut -f1)" -gt 2097152 ]; then
|
||||
rm -rf "$dir"; stale=$((stale + 1))
|
||||
fi
|
||||
done
|
||||
[ "$stale" = 0 ] || echo "$(basename "$runner"): removed $stale harness folders untouched for ${STALE_DAYS:-3} days"
|
||||
targets=()
|
||||
[ -f "$checkout/target/CACHEDIR.TAG" ] && targets+=("$checkout/target")
|
||||
while IFS= read -r tag; do targets+=("$(dirname "$tag")"); done < <(
|
||||
find "$checkout/.local" -mindepth 2 -maxdepth 2 -name CACHEDIR.TAG -type f 2>/dev/null)
|
||||
# The probes' build folder lives in shared-local (a symlink from .local), which the scan above doesn't follow:
|
||||
# it grew to 19-24 GB per runner (2026-11-06, the Mac down to 6 GB free).
|
||||
[ -d "$runner/_work/horizon/shared-local/parity-target" ] && targets+=("$runner/_work/horizon/shared-local/parity-target")
|
||||
[ ${#targets[@]} -gt 0 ] || continue
|
||||
size=$(du -sk "${targets[@]}" | awk '{s += $1} END {print s}')
|
||||
if [ "$size" -gt "$max_kb" ]; then
|
||||
rm -rf "${targets[@]}"
|
||||
# .local/parity-target is a symlink into shared-local: cargo can't create a directory through a dangling
|
||||
# link ("Not a directory"), which failed every harness of parity-main on 2026-11-06. Recreate it empty.
|
||||
mkdir -p "$runner/_work/horizon/shared-local/parity-target" 2>/dev/null || true
|
||||
echo "$(basename "$runner"): removed ${#targets[@]} build folders ($(( size / 1024 / 1024 )) GB)"
|
||||
fi
|
||||
done
|
||||
|
||||
Reference in new issue
Block a user