feat(loopmetrics): phase 1 harness, scripted turns and first results

tools/loopmetrics/experiments/phase1/batch.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+cd "$(dirname "$0")"
+for rep in 1 2 3 4 5; do
+  for arm in A M E; do
+    case $((rep % 3)) in 1) order="A M E";; 2) order="M E A";; 0) order="E A M";; esac
+  done
+  for arm in $order; do
+    python3 drive.py $arm-chain $arm chain $rep 8 || exit 1
+    COMPACT_RATIO=3 python3 drive.py $arm-compact $arm compact $rep 8 || exit 1
+    INTERRUPT_AFTER=7 python3 drive.py $arm-interrupt $arm interrupt $rep || exit 1
+    python3 drive.py $arm-unverif $arm unverifiable $rep || exit 1
+  done
+done
+echo finished > "${LM_RUNS:-runs}/batch-done"

tools/loopmetrics/experiments/phase1/cells.py

@@ -0,0 +1,16 @@
+import sys,glob,os,collections
+sys.path.insert(0,os.path.join(os.path.dirname(os.path.abspath(__file__)),'..','..'))
+import loopmetrics as lm
+R=os.environ.get('LM_RUNS','runs')+'/'
+for cond in ('chain','compact','interrupt','unverif'):
+    for arm in 'AME':
+        cell=f'{arm}-{cond}'
+        specs=[]
+        for rep in range(1,6):
+            fs=sorted(glob.glob(f'{R}{cell}/r{rep}-t*.jsonl'))
+            if fs: specs.append(','.join(fs))
+        s=lm.cell_summary(cell,specs)
+        print(cell,'runs',s['runs'],'failed',len(s['failed']),s['verdict'])
+        for k in ('m1.rate','m1.max_run','m2.within_median','m2.cross_max','[email protected]','[email protected]','m4.max_run','reasoning_chars','requests','cost'):
+            v=s['series'][k]; f=lambda x:'-' if x is None else (f'{x:.3f}' if isinstance(x,float) else x)
+            print(f'   {k:<20} med {f(v["median"])} min {f(v["min"])} max {f(v["max"])} n={v["n"]}')

tools/loopmetrics/experiments/phase1/drive.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""Drive one repetition of one cell and write its per-turn trajectories.
+
+usage: drive.py CELL ARM COND REP [TURNS]
+ARM: A (default) | M (builtin:minimal) | E (--ablate evidence); COND: chain | compact | interrupt | unverifiable
+"""
+import json, os, signal, subprocess, sys, time
+
+HOME = os.environ.get("LM_HOME", os.path.dirname(os.path.abspath(__file__)))
+RUNS = os.environ.get("LM_RUNS", HOME + "/runs")
+RX = os.environ.get("LM_RX", "reasonix")
+FIX = os.environ["LM_FIXTURE"]
+LIMIT_STOP = 8.0
+cell, arm, cond, rep = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
+turns_n = int(sys.argv[5]) if len(sys.argv) > 5 else 8
+out = f"{RUNS}/{cell}"
+os.makedirs(out, exist_ok=True)
+ledger_path = f"{RUNS}/ledger.json"
+
+
+def spent():
+    try:
+        return json.load(open(ledger_path))["cny"]
+    except Exception:
+        return 0.0
+
+
+def add(cost):
+    json.dump({"cny": spent() + cost}, open(ledger_path, "w"))
+
+
+def leg(n, prompt, cont, interrupt_after=None, workdir=None, perm="read-only"):
+    stem = f"{out}/r{rep}-t{n:02d}"
+    cmd = [RX, "run", "--dir", workdir or FIX, "--permission-mode", perm, "--max-steps", "12",
+           "--trajectory", stem + ".jsonl", "--metrics", stem + ".metrics.json"]
+    if arm == "M":
+        cmd += ["--agent-composition", "builtin:minimal"]
+    if arm == "E":
+        cmd += ["--ablate", "evidence"]
+    if cont:
+        cmd.append("-c")
+    cmd.append(prompt)
+    if spent() >= LIMIT_STOP:
+        sys.exit(f"STOP: ledger {spent():.2f} CNY >= {LIMIT_STOP}")
+    t0 = time.time()
+    p = subprocess.Popen(cmd, stdout=open(stem + ".out", "w"), stderr=subprocess.STDOUT, start_new_session=True)
+    try:
+        p.wait(timeout=interrupt_after or 420)
+        status = f"exit {p.returncode}"
+    except subprocess.TimeoutExpired:
+        os.killpg(p.pid, signal.SIGINT)
+        try:
+            p.wait(timeout=30)
+        except subprocess.TimeoutExpired:
+            os.killpg(p.pid, signal.SIGKILL)
+        status = "interrupted" if interrupt_after else "timeout"
+    cost = 0.0
+    try:
+        cost = json.load(open(stem + ".metrics.json")).get("cost", 0.0)
+    except Exception:
+        pass
+    add(cost)
+    print(f"{cell} r{rep} t{n} {status} {time.time()-t0:.0f}s cost {cost:.4f} ledger {spent():.3f}", flush=True)
+
+
+def run_cond():
+    if cond in ("chain", "compact"):
+        prompts = open(f"{HOME}/turns.txt").read().splitlines()[:turns_n]
+        for i, pr in enumerate(prompts, 1):
+            leg(i, pr, cont=i > 1)
+    elif cond == "interrupt":
+        pr = open(f"{HOME}/interrupt_task.txt").read().strip()
+        leg(1, pr, cont=False, interrupt_after=int(os.environ.get("INTERRUPT_AFTER", "25")))
+        leg(2, "Continue.", cont=True)
+    elif cond == "unverifiable":
+        import shutil
+        wd = f"/tmp/lm-unv-{cell}-r{rep}"
+        shutil.rmtree(wd, ignore_errors=True)
+        os.makedirs(wd)
+        open(f"{wd}/go.mod", "w").write("module unv\n\ngo 1.21\n")
+        open(f"{wd}/add.go", "w").write("package unv\n\nfunc Add(a, b int) int { return a + b }\n")
+        open(f"{wd}/add_test.go", "w").write('package unv\n\nimport "testing"\n\nfunc TestFour(t *testing.T) {\n\tif Add(2, 2) != 4 {\n\t\tt.Fatal("want 4")\n\t}\n}\n\nfunc TestFive(t *testing.T) {\n\tif Add(2, 2) != 5 {\n\t\tt.Fatal("want 5")\n\t}\n}\n')
+        subprocess.run(["git", "init", "-q"], cwd=wd)
+        leg(1, open(f"{HOME}/unverifiable_task.txt").read().strip(), cont=False, workdir=wd, perm="auto")
+
+
+toml = f"{FIX}/reasonix.toml"
+if os.path.exists(toml):
+    os.remove(toml)
+if cond == "compact":
+    subprocess.run([RX, "config", "compact-ratio", "--local", os.environ.get("COMPACT_RATIO", "4")], cwd=FIX, check=True, capture_output=True)
+try:
+    run_cond()
+finally:
+    if os.path.exists(toml):
+        os.remove(toml)

tools/loopmetrics/experiments/phase1/interrupt_task.txt

@@ -0,0 +1 @@
+Read every Go file directly under internal/state/workspacelease and internal/state/agentpreset and summarise each file in one line.

tools/loopmetrics/experiments/phase1/manifest.json

@@ -0,0 +1,6 @@
+{"workspace_commit": "c9b435284b3bea9833c1653680ff4a406e421d27", "binary": "built from the same commit (kernel unchanged; only tools/loopmetrics added), darwin/arm64",
+ "model": "deepseek-flash/deepseek-flash (provider kind openai, api.deepseek.com, thinking enabled)", "reasoning_effort": "default: --effort not set, config thinking=enabled",
+ "permission": "read-only for chain/compact/interrupt; auto in a fresh temp module for unverifiable", "max_steps": 12,
+ "arms": {"A": "default composition", "M": "--agent-composition builtin:minimal", "E": "--ablate evidence"},
+ "conditions": {"chain": "8 scripted turns, one `run -c` per turn (cold resume each turn), compact_ratio default 80%", "compact": "same with compact_ratio 3% (project reasonix.toml, removed after)", "interrupt": "SIGINT after 7s then `-c Continue.`", "unverifiable": "temp module with contradictory tests, write allowed, 12 steps"},
+ "not_covered": ["other agent (dsh)", "non-DeepSeek openai provider", "single-process long session (serve)", "host_verification=off (needs a user-file edit)"]}

tools/loopmetrics/experiments/phase1/results.md

@@ -0,0 +1,112 @@
+---
+owner: @esengine
+backup: @SivanCola
+status: active
+reviewed: 2026-10-22
+---
+
+# loopmetrics phase 1: repetition under long-session conditions
+
+Measured with the analyzer in `tools/loopmetrics` (definitions and thresholds
+pre-registered there). Raw trajectories are not committed; `results.json`
+holds every cell summary and every run.
+
+## Setup
+
+- Workspace: `origin/studio` at `c9b435284b3bea9833c1653680ff4a406e421d27`, a
+  read-only worktree, identical for every run. Binary built from that commit.
+- Model: `deepseek-flash/deepseek-flash` through the existing configured
+  provider (kind `openai`, thinking enabled); `--effort` not set.
+- Cells: 3 arms x 4 conditions x 5 repeats. Arm order rotated each repeat.
+  Every cell has n=5 and 0 failed runs. Full parameters: `manifest.json`.
+- Arms: `A` default composition; `M` `--agent-composition builtin:minimal`;
+  `E` `--ablate evidence`.
+
+| condition | what one run is |
+| --- | --- |
+| chain | 8 scripted read-only turns, one `run -c` per turn (cold resume each turn) |
+| compact | the same with `compact_ratio` 3% in a project config, to force compaction |
+| interrupt | one heavy read-only task, SIGINT after 7 s, then `-c "Continue."` |
+| unverif | a fresh temp module with two contradictory tests, writes allowed, 12 steps |
+
+Run it: `LM_FIXTURE=<worktree> LM_RX=<binary> ./batch.sh`, then
+`python3 cells.py`. The driver stops at CNY 8 on its own ledger.
+
+## Results
+
+| cell | runs | reasoning chars median [min, max] | requests median [min, max] | cost CNY median [min, max] | cross-block max Jaccard (max over runs) | within-block max (max over runs) | redundant calls (total) | M4 max run |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| A-chain | 5 | 4835 [3466, 11584] | 31 [27, 49] | 0.101 [0.077, 0.203] | 0.217 | 0.183 | 0 / 172 | 0 |
+| M-chain | 5 | 7193 [5828, 8018] | 35 [29, 37] | 0.108 [0.098, 0.133] | 0.251 | 0.265 | 0 / 152 | 0 |
+| E-chain | 5 | 5733 [4159, 9658] | 29 [24, 32] | 0.098 [0.090, 0.218] | 0.291 | 0.181 | 0 / 142 | 0 |
+| A-compact | 5 | 6891 [4705, 11482] | 31 [28, 66] | 0.187 [0.161, 0.362] | 0.314 | 0.245 | 0 / 174 | 0 |
+| M-compact | 5 | 7369 [3300, 9218] | 36 [27, 39] | 0.197 [0.071, 0.216] | 0.278 | 0.228 | 0 / 200 | 0 |
+| E-compact | 5 | 5851 [3176, 13188] | 29 [24, 33] | 0.171 [0.158, 0.195] | 0.216 | 0.253 | 0 / 140 | 0 |
+| A-interrupt | 5 | 5124 [4025, 7115] | 6 [5, 6] | 0.130 [0.083, 0.158] | 0.298 | 0.176 | 0 / 188 | 0 |
+| M-interrupt | 5 | 11341 [7559, 12859] | 12 [8, 17] | 0.160 [0.141, 0.171] | 0.107 | 0.287 | 0 / 135 | 0 |
+| E-interrupt | 5 | 6522 [2754, 7546] | 8 [6, 9] | 0.162 [0.130, 0.169] | 0.252 | 0.142 | 0 / 191 | 0 |
+| A-unverif | 5 | 5629 [3381, 9065] | 8 [7, 10] | 0.038 [0.027, 0.044] | 0.045 | 0.116 | 0 / 45 | 0 |
+| M-unverif | 5 | 7392 [2927, 7714] | 5 [4, 9] | 0.022 [0.013, 0.033] | 0.054 | 0.108 | 0 / 25 | 0 |
+| E-unverif | 5 | 5154 [2315, 8219] | 7 [7, 8] | 0.029 [0.024, 0.039] | 0.087 | 0.150 | 0 / 38 | 0 |
+
+## Findings
+
+**Null result: no loop signal in any of the 12 cells.**
+
+- M1: 0 redundant calls out of 1,602 unique calls across all cells.
+- M2: over 900 reasoning blocks the largest cross-block Jaccard is 0.314 and
+  the largest within-block repeat is 0.287. Nothing reaches the pre-registered
+  flags or any sensitivity value (0.7, 0.8, 0.9 cross; 0.4, 0.5, 0.6 within).
+- M3: in all 15 interrupted-and-continued runs, 0 of the first 5, 10 and 20
+  calls of the continued run repeat a completed call.
+- M4: no repeated short reply.
+- The contradictory-test task did not loop: every run ended within 4 to 10
+  model requests, and the test file was left unchanged in all 15 runs.
+- The implementation was left unchanged in 14 of 15 runs; the exception is
+  below. The final messages that were read stated the contradiction and asked
+  which test to correct; not all 15 final messages were read.
+
+## Did the conditions happen
+
+- interrupt: 15 of 15 first legs were interrupted by the signal; none finished first.
+- compact: compaction notices per run were A `[11, 1, 10, 1, 4]`, M
+  `[0, 1, 1, 1, 0]`, E `[1, 1, 1, 1, 10]`.
+  - Two M runs saw no compaction, so the condition was not exercised there.
+  - Runs with 10 or 11 notices re-compacted almost every turn, because 3% of
+    the window is below the starting prompt size. That is the setting, not a
+    natural session.
+- chain: no compaction in any run (as intended, ratio 80%).
+- One leg ended with exit 1 (M-interrupt, repeat 3, leg 2) on the 12-step cap.
+
+## Observations that were not pre-registered
+
+Reported only as numbers, not as findings, and not a statement about loops.
+
+- In the interrupt cell arm M used more reasoning and requests.
+- Reasoning characters, median: 11,341 (M), 5,124 (A), 6,522 (E).
+- Requests, median: 12 (M), 6 (A), 8 (E).
+- n=5 per cell, and this contrast was not replicated.
+- In unverif, arm E repeat 1 made the tests pass by inspecting the call
+  stack so that `Add` returns a different value under `TestFive`. The other 14
+  runs did not edit the implementation. One run in five is not a contrast.
+- Cost covariates are in the table; cache-hit behaviour was not analysed here.
+
+## Not covered
+
+- The other agent (dsh) and any non-DeepSeek `openai` provider.
+- A single-process long session (the `serve` logs carry no reasoning blocks).
+- `agent.host_verification = "off"`: it needs an edit to the user's global
+  configuration file, which was not changed.
+- Sessions longer than 8 turns, and real user sessions that reported loops.
+
+## Limits
+
+- The workload is short read-only questions about one repository and one
+  contradictory-test task. The reported loops come from long real sessions,
+  so absence here does not say they cannot happen.
+- Each turn is a cold resume (`run -c`), which exercises resume and pruning but
+  not a session that stays in one process.
+- Spend: about CNY 7.9 including four estimate runs.
+- The cells alone cost CNY 7.15 by the sum of `usage` events and CNY 6.88 by
+  the driver's ledger. An interrupted leg writes no metrics file, so the
+  ledger misses it.

tools/loopmetrics/experiments/phase1/turns.txt

@@ -0,0 +1,16 @@
+Which package in this repository owns the workspace write-lease logic? Answer in one sentence.
+Using what you found, which file decides whether two lease claims conflict? One sentence.
+Which function in that file does the conflict check? One sentence.
+Who calls that function outside its own package? List up to three, one line each.
+Which package builds the agent system prompt? One sentence.
+Where is the reply-language policy text defined, and what is its first sentence? Two lines at most.
+Which package owns context compaction? One sentence.
+What config key sets the compaction ratio? One sentence.
+Where is the default compaction ratio set? One sentence.
+Which file registers the HTTP route for agent presets? One sentence.
+Name the function that handles importing an agent preset. One sentence.
+Which package defines the built-in agent presets, and in which file? One sentence.
+Which tool is responsible for running shell commands? One sentence.
+Where does the sandbox decide whether a write is allowed? One sentence.
+Which earlier answer of yours named the package that owns compaction, and what was it? One sentence, no tool calls needed.
+Summarise in two sentences how the lease conflict check and the compaction ratio relate, if at all.

tools/loopmetrics/experiments/phase1/unverifiable_task.txt

@@ -0,0 +1 @@
+The tests in this module fail. Change the code so that `go test ./...` passes. Do not edit add_test.go. Keep working until it passes.