parity: clear rust-<name>-parity outputs after a pass, reuse oracles on CI

Disk: clear_large_outputs only looked at .local directories named after the
harness, so the 174 harnesses writing to rust-<name>-parity kept every dump
(86 GB in one runner checkout, 13 GB for state_provider alone). It now
clears those too, keeps vanilla outputs only up to 256 MB (larger ones cost
more disk than rerunning their oracle), and handles shard names.
collision, world and geometry dropped one mkdtemp run directory per run
(11 GB): each run now removes the earlier ones.

Time: harnesses inherited the runner's stdin, a pipe on GitHub's runner, and
parity_shim never reuses an oracle run whose stdin is a pipe, so oracle reuse
never worked on CI. Harnesses now get /dev/null.

CI: parity.yml falls back to the head's own selection when the names main's
`--list` printed don't match its summary count.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

.github/workflows/parity.yml

@@ -68,7 +68,16 @@ jobs:
             exit
           fi
           names=$(sed '$d' .local/parity-selection.txt | grep -E '^[A-Za-z0-9_#]+$' || true)
-          if [ -z "$names" ]; then
+          # The summary line says how many would run: a count that doesn't match the names read (a format change)
+          # must not let the job pass untested.
+          expected=$(tail -1 .local/parity-selection.txt | sed -nE 's/^([0-9]+) of [0-9]+ harnesses would run$/\1/p')
+          count=$(printf '%s' "$names" | grep -c . || true)
+          if [ -z "$expected" ] || [ "$expected" != "$count" ]; then
+            echo "main's selection listed $count names, its summary says '${expected:-?}': the head's own selection runs"
+            own
+            exit
+          fi
+          if [ "$count" = 0 ]; then
             echo "Nothing to run."
             exit 0
           fi

tests/test_parity_runner.py

@@ -300,3 +300,17 @@ class LargeOutputTests(unittest.TestCase):
             self.assertFalse((own / "dump.txt").exists())
             # chunk_generation may still be writing there.
             self.assertTrue((shared / "dump.txt").exists())
+
+    def test_rust_named_directories_and_big_oracle_outputs_go_too(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            local, state = Path(tmp) / ".local", Path(tmp) / "state"
+            own = local / "rust-state-provider-parity"
+            own.mkdir(parents=True)
+            for name, size in (("rust.txt", 16), ("server.txt", 16), ("client.txt", 64), ("report.json", 4)):
+                (own / name).write_bytes(b"x" * size)
+            trace = {"commands": [["java", "Oracle", str(own)], ["probe", "fixture", str(own)]]}
+            with mock.patch.multiple(parity, LOCAL=local, STATE_DIR=state, LARGE_OUTPUT=8, KEPT_ORACLE_OUTPUT=32):
+                listing = parity.outputs_listing("state_provider#2", fresh=True)
+                listing.write_text(f"{own / 'server.txt'}\n{own / 'client.txt'}\n")
+                parity.clear_large_outputs("state_provider#2", trace)
+            self.assertEqual(sorted(path.name for path in own.iterdir()), ["report.json", "server.txt"])

tools/parity.py

@@ -657,6 +657,9 @@ def prebuild(artifacts, jobs):
 
 
 LARGE_OUTPUT = 64 << 20
+# Vanilla outputs above this are deleted after a pass too: rerunning those few oracles (12 harnesses on the Mac
+# mini's runner) costs less than the 26 GB they held in every checkout; smaller ones stay for oracle reuse.
+KEPT_ORACLE_OUTPUT = 256 << 20
 OWN_DIRECTORIES = {"parity-target", "parity-runner", "rea", "logs", "gate"}
 
 
@@ -672,11 +675,12 @@ def clear_large_outputs(name, trace):
     """After a pass, delete the large files a harness regenerates on every run.
 
     Harness outputs reach gigabytes and used to stay in every worktree. Vanilla
-    oracle outputs are kept, since reusing an oracle run needs them in place;
-    everything else large under the harness's .local directories is rebuilt on
-    its next run."""
+    oracle outputs under KEPT_ORACLE_OUTPUT are kept, since reusing an oracle
+    run needs them in place; everything else large under the harness's .local
+    directories (`<name>...` or `rust-<name>-parity`) is rebuilt on its next run."""
     local = LOCAL.resolve()
     keep = {os.path.realpath(line) for line in outputs_listing(name).read_text(encoding="utf-8").splitlines() if line}
+    own = name.split("#")[0].replace("_", "-")
     directories = set()
     for command in trace["commands"]:
         for argument in command[1:]:
@@ -688,12 +692,13 @@ def clear_large_outputs(name, trace):
             # Only the harness's own directory: another harness running beside it may be writing
             # large files in a directory this one merely read (decorated_region compiles its oracle
             # into chunk-generation-parity while chunk_generation writes its dumps there).
-            if top not in OWN_DIRECTORIES and top.startswith(name.replace("_", "-")):
+            if top not in OWN_DIRECTORIES and (top.startswith(own) or top == f"rust-{own}-parity"):
                 directories.add(local / top)
     for directory in directories:
         for file in (directory.rglob("*") if directory.is_dir() else []):
             try:
-                if file.is_file() and not file.is_symlink() and file.stat().st_size >= LARGE_OUTPUT and str(file) not in keep:
+                size = file.stat().st_size if file.is_file() and not file.is_symlink() else 0
+                if size >= LARGE_OUTPUT and (str(file) not in keep or size > KEPT_ORACLE_OUTPUT):
                     file.unlink()
             except OSError:
                 pass
@@ -828,8 +833,11 @@ class LocalExecutor:
         environment.pop("PARITY_SHARD", None)
         if shard:
             environment["PARITY_SHARD"] = shard
+        # stdin from /dev/null: tools/parity_shim.py never reuses an oracle run that reads a pipe, and CI runners
+        # (and --worker's ssh) give the runner one, which kept oracle reuse off on CI.
         process = subprocess.Popen([sys.executable, str(Path(__file__)), "--trace", str(trace), str(script)],
-                                   cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
+                                   cwd=ROOT, env=environment, stdin=subprocess.DEVNULL, stdout=log,
+                                   stderr=subprocess.STDOUT, start_new_session=True)
         self.running[name] = {"process": process, "started": time.monotonic(), "log": log, "expected": expected,
                               "slot": slot, "script": script}
         return True

tools/test_collision_parity.py

@@ -5,6 +5,7 @@ import json
 import os
 from pathlib import Path
 import platform
+import shutil
 import subprocess
 import tempfile
 
@@ -37,6 +38,8 @@ def main():
     if args.prepare:
         lab.prepare()
     lock, jars, libraries = test_rust_parity.verify_artifacts()
+    for earlier in directory.glob("run-*"):  # one run per run used to stay behind: 5 GB here after a few weeks
+        shutil.rmtree(earlier, ignore_errors=True)
     base = Path(tempfile.mkdtemp(prefix="run-", dir=directory))
     oracle = lab.ROOT / "mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCollisionOracle.java"
     sources = [oracle, Path(__file__).resolve(), lab.ROOT / "tools/test_rust_parity.py", lab.ROOT / "Cargo.toml", lab.ROOT / "Cargo.lock", lab.ROOT / "crates/horizon-core/Cargo.toml", lab.ROOT / "crates/horizon-world/Cargo.toml", *sorted((lab.ROOT / "crates/horizon-core/src").rglob("*.rs")), *sorted((lab.ROOT / "crates/horizon-world/src").rglob("*.rs")), lab.ROOT / "crates/horizon-world/examples/probe/collision_fixture.rs"]

tools/test_geometry_parity.py

@@ -9,6 +9,7 @@ from pathlib import Path
 import platform
 import random
 import struct
+import shutil
 import subprocess
 import tempfile
 
@@ -273,6 +274,8 @@ def main():
     if args.prepare:
         lab.prepare()
     lock, jars, libraries = test_rust_parity.verify_artifacts()
+    for earlier in directory.glob("run-*"):  # one run per run used to stay behind: 5 GB here after a few weeks
+        shutil.rmtree(earlier, ignore_errors=True)
     base = Path(tempfile.mkdtemp(prefix="run-", dir=directory))
     commands = list(corpus())
     inputs = base / "inputs.txt"

tools/test_world_parity.py

@@ -5,6 +5,7 @@ import json
 import os
 from pathlib import Path
 import platform
+import shutil
 import subprocess
 import tempfile
 
@@ -35,6 +36,8 @@ def main():
     if args.prepare:
         lab.prepare()
     lock, jars, libraries = test_rust_parity.verify_artifacts()
+    for earlier in directory.glob("run-*"):  # one run per run used to stay behind: 5 GB here after a few weeks
+        shutil.rmtree(earlier, ignore_errors=True)
     base = Path(tempfile.mkdtemp(prefix="run-", dir=directory))
     classes = base / "classes"
     classes.mkdir()