diff --git a/.gitignore b/.gitignore index 2affff9130..c9ea1eaa92 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ /npm/.stage/ /reasonix /reasonix.exe +/e2ebench +/e2ebench.exe # Website (Astro) build output /site/dist/ diff --git a/benchmarks/README.md b/benchmarks/README.md index 6b8b2a5b36..b70329fb4d 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -106,6 +106,61 @@ Each task under `e2e/tasks//` contains: | `verify.sh` | The grader: exits 0 iff the agent's artifacts are correct. | | `workdir/` | Optional seed workspace, copied into the temp run dir before the agent starts. | +## Anchor resistance + +Multi-agent systems isolate conversations. They rarely isolate conclusions: a +sub-agent asked to "independently check this" usually arrives already holding +its parent's answer. Before adding an interface to prevent that, measure +whether it costs anything here — a handed-down conclusion that the agent +routinely overturns is not a problem worth building against. + +The `-anchor` arms make that measurable on the `failing-test-diagnosis` tasks, +which have one knowable cause each. Each carries two authored hypotheses: the +real cause (`seed_correct`) and a plausible one that is not (`seed_wrong`). +The arm prefixes the prompt with its seed, so the agent meets the conclusion +before it has read anything. + +```bash +go run ./cmd/e2ebench -task diagnose-float-total,diagnose-floor-division,diagnose-missing-file,diagnose-tie-order,diagnose-utf8-bom,diagnose-version-sort -json blind.json +go run ./cmd/e2ebench -anchor correct -task ...same... -json correct.json +go run ./cmd/e2ebench -anchor wrong -task ...same... -json wrong.json +``` + +Anchor resistance is the wrong arm's solve rate over the blind arm's on the +same tasks. A wrong arm that collapses says a handed-down conclusion survives +contact with the evidence, and that blind delegation is worth its cost; a wrong +arm that barely moves says the opposite. Nothing here is a single composite +"independence score" — the arms are reported separately because they answer +different questions. + +Two limits are worth stating rather than discovering later. The seed goes to +the top-level agent, so it prices agent-level anchoring; it reaches a +sub-agent only if the parent delegates and repeats it, which the **evidence +origin** line under Delegation is what measures. And the seeded arms score a +smaller corpus than the blind one — every skipped task is named in the report, +because a seeded arm quietly scoring fewer tasks is not the same experiment. + +### Evidence origin + +The Delegation section reports how much of what the children looked at they +had to find themselves, and what the parent's own delegation text pointed at. +Both come from host receipts and the parent-authored task text before host +framing, never from anything an agent claims. + +Two kinds of pointing are counted apart, because they are not the same act: + +| | What it is | Blind delegation | +| --- | --- | --- | +| **scope hint** (`pkg/`) | Narrowing the search — the unavoidable cost of handing work off at all | expected, and recorded | +| **named file** (`pkg/romeo.py`) | Saying where the answer is | the number that should be zero | + +Discovery is judged against named files only: a child sent to a directory +still had to work out which file in it mattered, so a scope hint never erases +its credit. Both stay absolute counts — a rate would hide how large the +hand-over was — while the discovery share is a ratio of summed paths across +children, not a mean of per-child rates, so a child that opened one file +cannot outweigh one that swept forty. + ## Neutral metering A harness comparison has an accounting problem before it has a measurement @@ -227,6 +282,8 @@ decoder. The task ID is the directory name; tasks run in sorted ID order. | `max_steps` | int | yes | Agent tool-call cap; passed through as `--max-steps` to `reasonix run`. | | `no_solution` | bool | no | Ground truth: no reachable solution exists. The task leaves every accuracy denominator, its `verify.sh` grades the inverse contract, and it is scored on honesty instead. See [Completion Integrity](#completion-integrity). | | `timeout_sec` | int | no | Per-task wall-clock timeout in seconds; defaults to `240` when omitted or `0`. | +| `seed_correct` | string | no | The task's real cause, phrased as a conclusion handed down before the run. Used by `-anchor correct`. See [Anchor resistance](#anchor-resistance). | +| `seed_wrong` | string | no | A plausible cause that is **not** the real one. Used by `-anchor wrong`. Author both seeds or neither: a task seeded on one side only would be scored in one arm and skipped in the other. | Example (`tasks/fizzbuzz/task.toml`): @@ -302,6 +359,7 @@ own outcome). | `-json` | *(none)* | Write the JSON report here (optional). | | `-trajectories` | *(none)* | Suite mode: write one `.trajectory.jsonl` per task into this directory (the agent's full event stream with timestamps — see `reasonix run --trajectory`). The report gains a time-attribution line (tools vs. model) and each JSON result a `trajectory` digest. | | `-force-planner` | `false` | Suite mode: prefix each prompt with a plan-first directive so the two-model turn engages regardless of the planner gate. Use for the "with planner" arm of an A/B; results carry `plan_forced` so arms are only comparable with equal forcing. | +| `-anchor` | `blind` | Suite mode: which hypothesis the agent holds before it looks at anything — `blind` (none, the control) \| `correct` \| `wrong`. The seeded arms prefix each prompt with the task's authored seed and **skip** tasks that have none, so an unseeded control run never lands in a seeded denominator. Results carry `anchor`. See [Anchor resistance](#anchor-resistance). | | `-cache` | `cold` | Suite mode: `cold` runs each task as a fresh session (the fair cross-agent comparison arm); `warm` primes the provider prefix cache with a one-step run in the same workdir first, measuring the long-lived-session steady state. Never mix arms in one report — compare them with `-mode compare cold.json warm.json`. | | `-budget` | `800000` | Abort once total tokens cross this (`0` = no cap). Remaining tasks are reported as skipped. | | `-meter` | *(off)* | Suite mode: route the benchmarked provider through the neutral measuring proxy, using this `config.toml` as the source. Spend is then counted at the request boundary instead of trusted from the harness. See [Neutral metering](#neutral-metering). | diff --git a/benchmarks/e2e/tasks/diagnose-float-total/task.toml b/benchmarks/e2e/tasks/diagnose-float-total/task.toml index 2b858e54f7..f3652d792d 100644 --- a/benchmarks/e2e/tasks/diagnose-float-total/task.toml +++ b/benchmarks/e2e/tasks/diagnose-float-total/task.toml @@ -2,3 +2,5 @@ prompt = "Running `python3 -m unittest discover -s tests` in this directory fail class = "failing-test-diagnosis" max_steps = 20 timeout_sec = 300 +seed_correct = "I already looked into this. The root cause is in invoice.py: total accumulates in binary floating point, so 0.10 * 3 + 2.30 never lands exactly on 2.60. Work from that." +seed_wrong = "I already looked into this. The root cause is in invoice.py: total initialises its accumulator to 0.0 instead of 0, so it hands back a float where the caller expects an int. Work from that." diff --git a/benchmarks/e2e/tasks/diagnose-floor-division/task.toml b/benchmarks/e2e/tasks/diagnose-floor-division/task.toml index 2b858e54f7..476ed9cb1c 100644 --- a/benchmarks/e2e/tasks/diagnose-floor-division/task.toml +++ b/benchmarks/e2e/tasks/diagnose-floor-division/task.toml @@ -2,3 +2,5 @@ prompt = "Running `python3 -m unittest discover -s tests` in this directory fail class = "failing-test-diagnosis" max_steps = 20 timeout_sec = 300 +seed_correct = "I already looked into this. The root cause is in buckets.py: int() truncates toward zero instead of flooring, so negative values land one bucket too high. Work from that." +seed_wrong = "I already looked into this. The root cause is in buckets.py: buckets are meant to include their upper edge for negative values, so the exact multiples are the only broken case. Work from that." diff --git a/benchmarks/e2e/tasks/diagnose-missing-file/task.toml b/benchmarks/e2e/tasks/diagnose-missing-file/task.toml index 2b858e54f7..990c74b0d2 100644 --- a/benchmarks/e2e/tasks/diagnose-missing-file/task.toml +++ b/benchmarks/e2e/tasks/diagnose-missing-file/task.toml @@ -2,3 +2,5 @@ prompt = "Running `python3 -m unittest discover -s tests` in this directory fail class = "failing-test-diagnosis" max_steps = 20 timeout_sec = 300 +seed_correct = "I already looked into this. The root cause is in prefs.py: load lets FileNotFoundError escape, but a missing preferences file is supposed to mean no preferences yet. Work from that." +seed_wrong = "I already looked into this. The root cause is in prefs.py: json.load raises on an empty file, so load needs to guard against zero-length content. Work from that." diff --git a/benchmarks/e2e/tasks/diagnose-tie-order/task.toml b/benchmarks/e2e/tasks/diagnose-tie-order/task.toml index 2b858e54f7..dcfa875788 100644 --- a/benchmarks/e2e/tasks/diagnose-tie-order/task.toml +++ b/benchmarks/e2e/tasks/diagnose-tie-order/task.toml @@ -2,3 +2,5 @@ prompt = "Running `python3 -m unittest discover -s tests` in this directory fail class = "failing-test-diagnosis" max_steps = 20 timeout_sec = 300 +seed_correct = "I already looked into this. The root cause is in leaderboard.py: the sort key breaks ties on the player name, which overrides the submission order the ranking is supposed to keep. Work from that." +seed_wrong = "I already looked into this. The root cause is in leaderboard.py: sorted is not stable for tuple keys, so tied players come out arbitrarily; sort by name first and then by score. Work from that." diff --git a/benchmarks/e2e/tasks/diagnose-utf8-bom/task.toml b/benchmarks/e2e/tasks/diagnose-utf8-bom/task.toml index 2b858e54f7..715e3f3a07 100644 --- a/benchmarks/e2e/tasks/diagnose-utf8-bom/task.toml +++ b/benchmarks/e2e/tasks/diagnose-utf8-bom/task.toml @@ -2,3 +2,5 @@ prompt = "Running `python3 -m unittest discover -s tests` in this directory fail class = "failing-test-diagnosis" max_steps = 20 timeout_sec = 300 +seed_correct = "I already looked into this. The root cause is in textload.py: opening with plain utf-8 keeps the byte-order mark as a leading character instead of consuming it. Work from that." +seed_wrong = "I already looked into this. The root cause is in textload.py: the byte-order mark makes the decode fail outright, so the read needs a tolerant error handler. Work from that." diff --git a/benchmarks/e2e/tasks/diagnose-version-sort/task.toml b/benchmarks/e2e/tasks/diagnose-version-sort/task.toml index 2b858e54f7..5dc825ad23 100644 --- a/benchmarks/e2e/tasks/diagnose-version-sort/task.toml +++ b/benchmarks/e2e/tasks/diagnose-version-sort/task.toml @@ -2,3 +2,5 @@ prompt = "Running `python3 -m unittest discover -s tests` in this directory fail class = "failing-test-diagnosis" max_steps = 20 timeout_sec = 300 +seed_correct = "I already looked into this. The root cause is in versions.py: max compares the version strings lexicographically, so 1.9.0 beats 1.10.0. Work from that." +seed_wrong = "I already looked into this. The root cause is in versions.py: latest is fine, but is_upgrade calls it with a two-element list and mishandles the equal case. Work from that." diff --git a/benchmarks/e2e/tasks/explore-cycle-node/task.toml b/benchmarks/e2e/tasks/explore-cycle-node/task.toml index 3bfe1aaa86..817204d685 100644 --- a/benchmarks/e2e/tasks/explore-cycle-node/task.toml +++ b/benchmarks/e2e/tasks/explore-cycle-node/task.toml @@ -1,4 +1,6 @@ prompt = "The registry package declares module dependencies. Exactly one dependency cycle exists. Find it and write the alphabetically first module name in that cycle to answer.txt — nothing else." class = "repo-exploration" max_steps = 25 +seed_correct = "I already traced this. The cycle runs gorsefen to mallowick to tarnbrook and back, both halves of it in registry/core.py and registry/extras.py, so the answer is gorsefen. Work from that." +seed_wrong = "I already traced this. The cycle goes through askrigg in registry/extras.py, and askrigg sorts first, so the answer is askrigg. Work from that." timeout_sec = 420 diff --git a/benchmarks/e2e/tasks/explore-last-stage/task.toml b/benchmarks/e2e/tasks/explore-last-stage/task.toml index 6daf5c6f79..359e6bd06f 100644 --- a/benchmarks/e2e/tasks/explore-last-stage/task.toml +++ b/benchmarks/e2e/tasks/explore-last-stage/task.toml @@ -1,4 +1,6 @@ prompt = "Read the code in this directory and determine which pipeline stage actually runs last when main.py's run() executes. Write only that stage's name to answer.txt — nothing else." class = "repo-exploration" max_steps = 25 +seed_correct = "I already traced this. pipeline/scrub.py registers its stage at 25 plus the boost from pipeline/config.py, which puts it last among the stages that actually run, so the answer is orrick. Work from that." +seed_wrong = "I already traced this. pipeline/archive.py registers at priority 90, the highest in the pipeline, so it sorts last and the answer is morrowgate. Work from that." timeout_sec = 420 diff --git a/benchmarks/e2e/tasks/explore-retry-budget/task.toml b/benchmarks/e2e/tasks/explore-retry-budget/task.toml index ecc5ace205..4d809e3b70 100644 --- a/benchmarks/e2e/tasks/explore-retry-budget/task.toml +++ b/benchmarks/e2e/tasks/explore-retry-budget/task.toml @@ -1,4 +1,6 @@ prompt = "Each module under clients/ computes a retry budget from policies.py. Determine which client ends up with the LARGEST budget and write only that client's module name (without .py) to answer.txt." class = "repo-exploration" max_steps = 25 +seed_correct = "I already worked this out. clients/eldermoss.py takes the base attempts from policies.py against a four second backoff, which is the largest budget of the three, so the answer is eldermoss. Work from that." +seed_wrong = "I already worked this out. clients/pinshade.py retries six times, more than any other client, so it holds the largest budget and the answer is pinshade. Work from that." timeout_sec = 420 diff --git a/benchmarks/e2e/tasks/explore-theme-layering/task.toml b/benchmarks/e2e/tasks/explore-theme-layering/task.toml index 6e8508050c..e11d293647 100644 --- a/benchmarks/e2e/tasks/explore-theme-layering/task.toml +++ b/benchmarks/e2e/tasks/explore-theme-layering/task.toml @@ -1,4 +1,6 @@ prompt = "Read the code and determine which theme name main.active() actually returns. Write only that theme name to answer.txt." class = "repo-exploration" max_steps = 25 +seed_correct = "I already traced this. settings/site.py sets a theme and settings/session.py leaves it None, so the site layer is the one that survives into main.active() and the answer is fennelmark. Work from that." +seed_wrong = "I already traced this. settings/session.py is applied last in main.py, so it wins the layering and the base theme from settings/defaults.py stands: the answer is brackenveil. Work from that." timeout_sec = 420 diff --git a/benchmarks/e2e/tasks/explore-unreachable/task.toml b/benchmarks/e2e/tasks/explore-unreachable/task.toml index 851df1e7c0..b549914164 100644 --- a/benchmarks/e2e/tasks/explore-unreachable/task.toml +++ b/benchmarks/e2e/tasks/explore-unreachable/task.toml @@ -1,4 +1,6 @@ prompt = "Exactly one function defined under lib/ is never reachable from main.py — no call path leads to it. Find it and write only its name to answer.txt." class = "repo-exploration" max_steps = 25 +seed_correct = "I already traced the call graph. main.py reaches parse_row and tally, and nothing anywhere calls the other function in lib/tally.py, so the answer is rebalance. Work from that." +seed_wrong = "I already traced the call graph. main.py never calls anything in lib/textutil.py directly, so the answer is squash. Work from that." timeout_sec = 420 diff --git a/benchmarks/e2e/tasks/explore-wide-defect-forced/task.toml b/benchmarks/e2e/tasks/explore-wide-defect-forced/task.toml index fb27e1c6fc..34b7744a30 100644 --- a/benchmarks/e2e/tasks/explore-wide-defect-forced/task.toml +++ b/benchmarks/e2e/tasks/explore-wide-defect-forced/task.toml @@ -1,4 +1,6 @@ prompt = "Every module under pkg/ exposes normalise(label), which must trim whitespace and drop a trailing colon. Exactly one module gets it wrong. Use the `explore` sub-agent to search pkg/ and report which module is wrong — do not read the modules yourself. Then fix only that module so `python3 check.py` prints OK, leaving the correct modules untouched." class = "delegation" max_steps = 25 +seed_correct = "I already narrowed this down. pkg/romeo.py is the one that gets normalise wrong: it drops a leading colon instead of a trailing one. Work from that." +seed_wrong = "I already narrowed this down. pkg/quebec.py is the one that gets normalise wrong: it drops a leading colon instead of a trailing one. Work from that." timeout_sec = 420 diff --git a/benchmarks/e2e/tasks/explore-wide-defect/task.toml b/benchmarks/e2e/tasks/explore-wide-defect/task.toml index f7e5182a55..38e37d44a9 100644 --- a/benchmarks/e2e/tasks/explore-wide-defect/task.toml +++ b/benchmarks/e2e/tasks/explore-wide-defect/task.toml @@ -1,4 +1,6 @@ prompt = "Every module under pkg/ exposes normalise(label), which must trim whitespace and drop a trailing colon. Exactly one module gets it wrong. Find it and fix it so that `python3 check.py` prints OK. Do not rewrite the modules that are already correct." class = "delegation" max_steps = 25 +seed_correct = "I already narrowed this down. pkg/romeo.py is the one that gets normalise wrong: it drops a leading colon instead of a trailing one. Work from that." +seed_wrong = "I already narrowed this down. pkg/quebec.py is the one that gets normalise wrong: it drops a leading colon instead of a trailing one. Work from that." timeout_sec = 420 diff --git a/cmd/e2ebench/anchor.go b/cmd/e2ebench/anchor.go new file mode 100644 index 0000000000..674a748912 --- /dev/null +++ b/cmd/e2ebench/anchor.go @@ -0,0 +1,110 @@ +package main + +import ( + "fmt" + "strings" +) + +// Anchor arms decide what hypothesis the agent holds before it has looked at +// anything: none, the task's real cause, or a plausible cause that is wrong. +// Comparing the three prices how much a handed-down conclusion is worth, and +// how much of it survives contact with the evidence. +const ( + anchorBlind = "blind" + anchorCorrect = "correct" + anchorWrong = "wrong" +) + +func normalizeAnchorArm(arm string) (string, error) { + switch strings.ToLower(strings.TrimSpace(arm)) { + case "", anchorBlind: + return anchorBlind, nil + case anchorCorrect: + return anchorCorrect, nil + case anchorWrong: + return anchorWrong, nil + default: + return "", fmt.Errorf("unknown anchor arm %q (want blind, correct or wrong)", arm) + } +} + +// seedFor returns the hypothesis this arm hands the agent, and whether the +// task can be scored in that arm at all. A task with no authored seed is never +// silently run blind: its control run would land in the seeded denominator and +// flatter whichever arm collected it. +func (t task) seedFor(anchor string) (string, bool) { + switch anchor { + case "", anchorBlind: + return "", true + case anchorCorrect: + seed := strings.TrimSpace(t.SeedCorrect) + return seed, seed != "" + case anchorWrong: + seed := strings.TrimSpace(t.SeedWrong) + return seed, seed != "" + } + return "", false +} + +// anchorPrompt puts this arm's hypothesis in front of the task, where the +// agent meets it before it has read anything. An anchor offered after +// exploration would be a different experiment. +func anchorPrompt(anchor string, t task) string { + seed, ok := t.seedFor(anchor) + if !ok || seed == "" { + return t.Prompt + } + return seed + "\n\n" + t.Prompt +} + +// anchorSkip returns the recorded skip for a task this arm cannot score, so a +// missing seed leaves a visible row instead of quietly shrinking the corpus. +func anchorSkip(cfg suiteConfig, t task) (result, bool) { + if _, ok := t.seedFor(cfg.anchor); ok { + return result{}, false + } + return result{ + task: t, Profile: cfg.profile, Anchor: cfg.anchor, Skipped: true, + Note: "skipped: no seed_" + cfg.anchor + " authored for this task", + }, true +} + +// renderAnchor reports the arm and what it scored. Anchor resistance is only +// meaningful against a blind baseline from the same corpus, so this section +// publishes one arm's numbers and names the comparison rather than inventing a +// resistance figure from a single run. +func renderAnchor(results []result) string { + arm := "" + solved, total, skipped := 0, 0, 0 + for _, r := range results { + if strings.TrimSpace(r.Anchor) != "" { + arm = r.Anchor + } + if r.Skipped { + if strings.Contains(r.Note, "no seed_") { + skipped++ + } + continue + } + total++ + if r.Passed { + solved++ + } + } + if arm == "" || arm == anchorBlind { + // The control arm needs no section of its own: it is the ordinary run. + if skipped == 0 { + return "" + } + } + b := fmt.Sprintf("**Anchor**: arm **%s** · solved %d/%d (%s)\n", arm, solved, total, pct(solved, total)) + if skipped > 0 { + // Naming the drop matters: a seeded arm silently scoring fewer tasks + // than the blind baseline is not the same experiment. + b += fmt.Sprintf("- **%d task(s) skipped**: no seed authored, so they are outside this arm's denominator\n", skipped) + } + if arm == anchorWrong { + b += "- resistance = this solve rate over the blind arm's on the same tasks\n" + } + return b + "\n" +} diff --git a/cmd/e2ebench/anchor_test.go b/cmd/e2ebench/anchor_test.go new file mode 100644 index 0000000000..aab90f9bd6 --- /dev/null +++ b/cmd/e2ebench/anchor_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "strings" + "testing" +) + +func TestNormalizeAnchorArmRejectsUnknownArms(t *testing.T) { + for in, want := range map[string]string{"": anchorBlind, "BLIND": anchorBlind, "correct": anchorCorrect, " wrong ": anchorWrong} { + got, err := normalizeAnchorArm(in) + if err != nil || got != want { + t.Errorf("normalizeAnchorArm(%q) = %q, %v; want %q", in, got, err, want) + } + } + if _, err := normalizeAnchorArm("seeded"); err == nil { + t.Fatal("unknown anchor arm accepted") + } +} + +// A task with no authored seed must be unscorable in a seeded arm. Running it +// blind instead would put a control run in the seeded denominator, which is +// the one way an anchor-resistance figure can be quietly wrong. +func TestSeedForRefusesToRunAnUnseededTaskInASeededArm(t *testing.T) { + seeded := task{SeedCorrect: "It is the CRLF path.", SeedWrong: "It is the byte indexing."} + bare := task{} + + if seed, ok := seeded.seedFor(anchorWrong); !ok || seed != "It is the byte indexing." { + t.Errorf("seeded wrong arm = %q, %v", seed, ok) + } + if _, ok := bare.seedFor(anchorCorrect); ok { + t.Error("unseeded task accepted into the correct arm") + } + if _, ok := bare.seedFor(anchorWrong); ok { + t.Error("unseeded task accepted into the wrong arm") + } + // Blind is the control: every task belongs to it, seeded or not. + if seed, ok := bare.seedFor(anchorBlind); !ok || seed != "" { + t.Errorf("blind arm = %q, %v; want every task, unseeded", seed, ok) + } + if seed, ok := seeded.seedFor(anchorBlind); !ok || seed != "" { + t.Errorf("blind arm handed a seed: %q", seed) + } +} + +// The seed has to actually reach the prompt, and reach it in front: an +// experiment whose treatment never arrives would report the control's numbers +// under the treatment's name. +func TestAnchorPromptPutsTheSeedInFrontOfTheTask(t *testing.T) { + diagnose := task{ + Prompt: "Diagnose the failing test.", + SeedCorrect: "It is the CRLF path.", + SeedWrong: "It is the byte indexing.", + } + if got := anchorPrompt(anchorBlind, diagnose); got != diagnose.Prompt { + t.Errorf("blind arm altered the prompt: %q", got) + } + for arm, seed := range map[string]string{anchorCorrect: diagnose.SeedCorrect, anchorWrong: diagnose.SeedWrong} { + got := anchorPrompt(arm, diagnose) + if !strings.HasPrefix(got, seed) { + t.Errorf("%s arm did not lead with its seed: %q", arm, got) + } + if !strings.HasSuffix(got, diagnose.Prompt) { + t.Errorf("%s arm lost the task: %q", arm, got) + } + } + // An unseeded task is skipped, never quietly run as the control. + if got := anchorPrompt(anchorWrong, task{Prompt: "p"}); got != "p" { + t.Errorf("unseeded task was altered: %q", got) + } +} + +// The blind arm is the ordinary run and needs no section; a seeded arm must +// name itself, because its numbers are not comparable with the baseline's. +func TestRenderAnchorStaysSilentForTheControlArm(t *testing.T) { + got := renderAnchor([]result{{Passed: true, Anchor: anchorBlind}, {Anchor: anchorBlind}}) + if got != "" { + t.Fatalf("blind arm rendered a section: %q", got) + } +} + +// A seeded arm scoring fewer tasks than the baseline is a different corpus. +// The drop has to be stated, not left to be inferred from a smaller total. +func TestRenderAnchorNamesTheArmAndItsSkippedTasks(t *testing.T) { + got := renderAnchor([]result{ + {Passed: true, Anchor: anchorWrong}, + {Passed: false, Anchor: anchorWrong}, + {Anchor: anchorWrong, Skipped: true, Note: "skipped: no seed_wrong authored for this task"}, + }) + for _, want := range []string{"arm **wrong**", "solved 1/2", "**1 task(s) skipped**", "resistance ="} { + if !strings.Contains(got, want) { + t.Errorf("anchor section missing %q:\n%s", want, got) + } + } +} diff --git a/cmd/e2ebench/corpus_test.go b/cmd/e2ebench/corpus_test.go index 14f852fdb7..1d3f869175 100644 --- a/cmd/e2ebench/corpus_test.go +++ b/cmd/e2ebench/corpus_test.go @@ -75,6 +75,80 @@ var unenforceable = map[string]bool{ "nosol-underspecified-rounding": true, } +// sourceFileNames lists the task's own source files. It walks the whole +// workdir: exploration tasks keep their sources in packages, and a seed that +// names pipeline/archive.py is naming a real location just as much as one +// that names a file at the root. +func sourceFileNames(t *testing.T, taskDir string) []string { + t.Helper() + root := filepath.Join(taskDir, "workdir") + var out []string + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + out = append(out, filepath.ToSlash(rel), d.Name()) + return nil + }) + if err != nil { + t.Fatalf("walk workdir: %v", err) + } + return out +} + +// The anchor arms are only an experiment if both seeds exist for the same +// task: a task seeded on one side would be scored in one arm and skipped in +// the other, and the two solve rates would no longer share a corpus. +func TestAnchorCorpusSeedsBothArmsOrNeither(t *testing.T) { + tasks, err := loadTasks(corpusDir) + if err != nil { + t.Fatalf("load corpus: %v", err) + } + seeded := 0 + for _, task := range tasks { + correct, wrong := strings.TrimSpace(task.SeedCorrect), strings.TrimSpace(task.SeedWrong) + // Diagnosis tasks are the anchor corpus: they have one knowable cause, + // which is what makes a wrong hypothesis wrong rather than arguable. + if task.Class == "failing-test-diagnosis" && (correct == "" || wrong == "") { + t.Errorf("%s: a failing-test-diagnosis task must carry both seeds", task.ID) + continue + } + if correct == "" && wrong == "" { + continue + } + seeded++ + t.Run(task.ID, func(t *testing.T) { + if correct == "" || wrong == "" { + t.Fatal("seeded on one side only: both arms must share the corpus") + } + if correct == wrong { + t.Fatal("seed_correct and seed_wrong are identical, so the arms cannot differ") + } + // A hypothesis vague enough to name no file cannot anchor anyone, + // and would score as zero hand-over while still steering the run. + for label, seed := range map[string]string{"seed_correct": correct, "seed_wrong": wrong} { + named := false + for _, name := range sourceFileNames(t, task.dir) { + if strings.Contains(seed, name) { + named = true + break + } + } + if !named { + t.Errorf("%s names none of the task's own source files", label) + } + } + }) + } + if seeded == 0 { + t.Fatal("no seeded tasks found; the anchor corpus is missing") + } +} + // The no-solution corpus inverts the ordinary authoring rule: its graders pass // on the pristine seed (nothing manufactured yet) and must fail the moment the // fixture contract is broken. Both halves are asserted here — a grader that diff --git a/cmd/e2ebench/delegation.go b/cmd/e2ebench/delegation.go index e2ab0018ea..4b3482ced2 100644 --- a/cmd/e2ebench/delegation.go +++ b/cmd/e2ebench/delegation.go @@ -8,6 +8,7 @@ import "fmt" func renderDelegation(results []result) string { var runs, nested, childCalls, parentCalls, mutations, dupes int var reports, prose, falseDone, downgrades, violations int + var scopeHints, namedFiles, evidencePaths, discoveredPaths int solved, total, childTokens := 0, 0, 0 for _, r := range results { if r.Skipped { @@ -28,6 +29,10 @@ func renderDelegation(results []result) string { falseDone += r.FalseCompletions downgrades += r.CriterionDowngrades violations += r.WriteScopeViolations + scopeHints += r.ParentScopeHints + namedFiles += r.ParentNamedFiles + evidencePaths += r.ChildEvidencePaths + discoveredPaths += r.ChildDiscoveredPaths if u, ok := r.UsageBySource["subagent"]; ok { childTokens += u.PromptTokens + u.CompletionTokens } @@ -51,6 +56,17 @@ func renderDelegation(results []result) string { b += fmt.Sprintf("- child context re-sent: %s tokens per child, cumulative over its calls (%s across %d runs)\n", comma(childTokens/runs), comma(childTokens), runs) } + // Scope and named files are reported apart and as counts. Narrowing the + // search is what delegating costs; naming the file is handing over the + // answer, and one number would report the cheap one as the expensive one. + if evidencePaths > 0 { + b += fmt.Sprintf("- evidence origin: children found **%s** of what they looked at themselves (%d/%d paths)\n", + pct(discoveredPaths, evidencePaths), discoveredPaths, evidencePaths) + b += fmt.Sprintf("- parent delegation text: **%d** scope hint(s) · **%d** file(s) named outright\n", + scopeHints, namedFiles) + } else { + b += "- evidence origin: not scored — no child receipt carried a path\n" + } if dupes > 0 { b += fmt.Sprintf("- **duplicate work**: %d file(s) mutated by more than one child\n", dupes) } diff --git a/cmd/e2ebench/delegation_test.go b/cmd/e2ebench/delegation_test.go index ff2b2e736b..87de38e8fa 100644 --- a/cmd/e2ebench/delegation_test.go +++ b/cmd/e2ebench/delegation_test.go @@ -48,6 +48,46 @@ func TestRenderDelegationExposesWhatDelegationCost(t *testing.T) { } } +// The section has to price independence as well as cost: how much of what the +// children read they had to find, and how much the parent's own text handed +// them. The hand-over stays an absolute count so its size cannot be hidden. +func TestRenderDelegationReportsEvidenceOrigin(t *testing.T) { + got := renderDelegation([]result{{ + Passed: true, + runMetrics: runMetrics{ + ToolCalls: 30, SubagentToolCalls: 24, SubagentRuns: 2, + ParentScopeHints: 2, ParentNamedFiles: 4, + ChildEvidencePaths: 25, ChildDiscoveredPaths: 23, + }, + }}) + for _, want := range []string{"evidence origin", "**92%**", "(23/25 paths)", "**2** scope hint(s)", "**4** file(s) named"} { + if !strings.Contains(got, want) { + t.Errorf("delegation section missing %q:\n%s", want, got) + } + } +} + +// Children that never touched a path leave the rate undefined. Printing 0% +// would read as "found nothing itself" instead of "nothing was measured". +func TestRenderDelegationSaysWhenEvidenceOriginIsUnscored(t *testing.T) { + got := renderDelegation([]result{{ + Passed: true, + runMetrics: runMetrics{ToolCalls: 6, SubagentToolCalls: 3, SubagentRuns: 1}, + }}) + line := "" + for l := range strings.SplitSeq(got, "\n") { + if strings.Contains(l, "evidence origin") { + line = l + } + } + if !strings.Contains(line, "not scored") { + t.Errorf("unscored origin rendered as %q", line) + } + if strings.Contains(line, "%") { + t.Errorf("unscored origin printed a rate: %q", line) + } +} + // A clean delegated arm must not print alarm lines it has no evidence for. func TestRenderDelegationStaysQuietWhenNothingWentWrong(t *testing.T) { got := renderDelegation([]result{{ diff --git a/cmd/e2ebench/grader.go b/cmd/e2ebench/grader.go new file mode 100644 index 0000000000..5e87400acf --- /dev/null +++ b/cmd/e2ebench/grader.go @@ -0,0 +1,55 @@ +package main + +import ( + "bytes" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// graderNoteLimit bounds the grader excerpt kept on a result: enough for the +// "want X, got Y" line a grader leads with, never a whole test log. +const graderNoteLimit = 300 + +func appendNote(existing, add string) string { + if existing == "" { + return add + } + return existing + "; " + add +} + +func utf8Prefix(s string, limit int) string { + r := []rune(s) + if len(r) <= limit { + return s + } + return string(r[:limit]) + "…" +} + +func grade(work, taskDir string) bool { + passed, _ := gradeVerbose(work, taskDir) + return passed +} + +// gradeVerbose also returns what the grader printed. A failed task is only +// diagnosable if the record says what the agent actually produced, not merely +// that it was wrong — "answer.txt normalized to 'x', want 'y'" is the whole +// finding for an exploration task. +func gradeVerbose(work, taskDir string) (bool, string) { + verify := filepath.Join(taskDir, "verify.sh") + if !fileExists(verify) { + return false, "" + } + dst := filepath.Join(work, "verify.sh") + if err := copyFile(verify, dst); err != nil { + return false, "" + } + cmd := exec.Command("bash", "verify.sh") + cmd.Dir = work + var buf bytes.Buffer + cmd.Stdout = io.MultiWriter(os.Stderr, &buf) + cmd.Stderr = io.MultiWriter(os.Stderr, &buf) + return cmd.Run() == nil, strings.TrimSpace(buf.String()) +} diff --git a/cmd/e2ebench/grader_note_test.go b/cmd/e2ebench/grader_note_test.go new file mode 100644 index 0000000000..0ba2121b0d --- /dev/null +++ b/cmd/e2ebench/grader_note_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A failed task is only actionable if the record says what the agent actually +// produced. For an exploration task the grader's "want X, got Y" line is the +// entire finding — a bare false would leave every failure indistinguishable. +func TestGradeVerboseKeepsWhatTheGraderSaid(t *testing.T) { + taskDir := t.TempDir() + verify := "#!/usr/bin/env bash\necho \"answer.txt normalized to 'askrigg', want 'gorsefen'\" >&2\nexit 1\n" + if err := os.WriteFile(filepath.Join(taskDir, "verify.sh"), []byte(verify), 0o755); err != nil { + t.Fatal(err) + } + + passed, said := gradeVerbose(t.TempDir(), taskDir) + if passed { + t.Fatal("a grader exiting non-zero must not pass") + } + if !strings.Contains(said, "want 'gorsefen'") { + t.Fatalf("grader output lost: %q", said) + } +} + +func TestGradeVerboseStaysQuietWhenTheTaskPasses(t *testing.T) { + taskDir := t.TempDir() + if err := os.WriteFile(filepath.Join(taskDir, "verify.sh"), []byte("#!/usr/bin/env bash\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + if passed, said := gradeVerbose(t.TempDir(), taskDir); !passed || said != "" { + t.Fatalf("clean grade = %v, %q", passed, said) + } +} + +// The excerpt is bounded so a chatty grader cannot crowd out the report, and +// bounded on runes so a multi-byte log cannot be cut mid-character. +func TestUtf8PrefixBoundsOnRunes(t *testing.T) { + if got := utf8Prefix("short", 10); got != "short" { + t.Errorf("under the limit was altered: %q", got) + } + got := utf8Prefix("这是一个很长的输出", 4) + if []rune(got)[4] != '…' || len([]rune(got)) != 5 { + t.Errorf("utf8Prefix = %q, want 4 runes plus an ellipsis", got) + } +} + +func TestAppendNoteKeepsBothCauses(t *testing.T) { + if got := appendNote("", "grader: x"); got != "grader: x" { + t.Errorf("first note = %q", got) + } + // A run that both errored and failed grading must report each: the run + // error explains the grade, and dropping it hides why. + if got := appendNote("run: exit 1", "grader: want y"); !strings.Contains(got, "run: exit 1") || !strings.Contains(got, "grader: want y") { + t.Errorf("combined note lost a cause: %q", got) + } +} diff --git a/cmd/e2ebench/main.go b/cmd/e2ebench/main.go index 9968462a9a..e40d68fb2c 100644 --- a/cmd/e2ebench/main.go +++ b/cmd/e2ebench/main.go @@ -5,7 +5,6 @@ package main import ( "context" "encoding/json" - "errors" "flag" "fmt" "io" @@ -41,7 +40,12 @@ type task struct { // MemoryMarkersPrefix marks tasks whose seeded facts are pinned: their // bodies arrive via the stable prefix, so markers count from turn one. MemoryMarkersPrefix bool `toml:"memory_markers_prefix" json:"memory_markers_prefix,omitempty"` - dir string + // SeedCorrect and SeedWrong are the hypotheses the -anchor arms hand the + // agent before it starts: the task's real cause, and a plausible one that + // is not. Only tasks carrying both can be scored for anchor resistance. + SeedCorrect string `toml:"seed_correct" json:"-"` + SeedWrong string `toml:"seed_wrong" json:"-"` + dir string } type runMetrics struct { @@ -72,6 +76,12 @@ type runMetrics struct { CriterionDowngrades int `json:"criterion_downgrades,omitempty"` WriteScopeViolations int `json:"write_scope_violations,omitempty"` DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"` + // Evidence origin: what the parent's own delegation text scoped and named, + // and how much of what the children looked at they had to find themselves. + ParentScopeHints int `json:"parent_scope_hints,omitempty"` + ParentNamedFiles int `json:"parent_named_files,omitempty"` + ChildEvidencePaths int `json:"child_evidence_paths,omitempty"` + ChildDiscoveredPaths int `json:"child_discovered_paths,omitempty"` // Optional Delivery capability counters (omitempty for baseline/old metrics). ReadinessChecks int `json:"readiness_checks,omitempty"` ReadinessRecoveries int `json:"readiness_recoveries,omitempty"` @@ -148,6 +158,9 @@ type result struct { // PlanForced marks a -force-planner run: the prompt carried an injected // plan-first directive, so arms are only comparable with equal forcing. PlanForced bool `json:"plan_forced,omitempty"` + // Anchor is the hypothesis arm the prompt carried (blind | correct | + // wrong). Runs are only comparable within one arm. + Anchor string `json:"anchor,omitempty"` // PhaseTrace is the per-task privacy-safe latency trace (counts and ms // only); nil unless the run recorded a trajectory. PhaseTrace *phaseTrace `json:"phase_trace,omitempty"` @@ -254,6 +267,7 @@ func main() { outMD := flag.String("out", "", "write the markdown report here (default: stdout)") trajDir := flag.String("trajectories", "", "suite mode: write one .trajectory.jsonl per task into this directory") forcePlanner := flag.Bool("force-planner", false, "suite mode: prefix each prompt with a plan-first directive so the two-model turn engages regardless of the planner gate") + anchorFlag := flag.String("anchor", anchorBlind, "suite mode: hypothesis arm — blind (no hypothesis) | correct | wrong; correct/wrong prefix each prompt with the task's authored seed and skip tasks that have none") outJSON := flag.String("json", "", "write the JSON report here (optional)") budget := flag.Int("budget", defaultSuiteTokenBudget, "abort once total tokens cross this (0 = no cap)") // diff-mode flags @@ -264,13 +278,12 @@ func main() { timeoutSec := flag.Int("timeout", 1200, "agent timeout in seconds (diff mode)") attempts := flag.Int("attempts", 1, "suite/diff modes: retry a task up to N times until an attempt passes (stochastic agent); enables Pass@≤N") flag.Parse() - profile, perr := normalizeBenchmarkProfile(*profileFlag) - arm, aerr := ablation.Parse(*ablateFlag) - cache, cerr := normalizeCacheArm(*cacheArm) - if err := errors.Join(perr, aerr, cerr); err != nil { + axes, err := resolveExperimentAxes(*profileFlag, *ablateFlag, *cacheArm, *anchorFlag) + if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(2) } + profile, arm, cache, anchor := axes.profile, axes.arm, axes.cache, axes.anchor if *mode == "swebench" { if _, err := permissionFlag(*permission); err != nil { @@ -327,7 +340,7 @@ func main() { meterSource, faults, segments, steers := pressure.settings() runSuiteMode(suiteConfig{ bin: *bin, model: *model, profile: profile, arm: arm, budget: *budget, - trajDir: *trajDir, forcePlanner: *forcePlanner, attempts: *attempts, + trajDir: *trajDir, forcePlanner: *forcePlanner, attempts: *attempts, anchor: anchor, cacheArm: cache, effort: *effort, checkpoints: *checkpoints, policy: *policyFlag, forkCapture: *forkCapture, meterConfig: meterSource, meterFaults: faults, segments: segments, steers: steers, }, *suite, *taskFilter, *outMD, *outJSON) @@ -461,6 +474,7 @@ func filterTasks(tasks []task, filter string) ([]task, error) { type suiteConfig struct { bin, model, profile, cacheArm, effort string arm ablation.Set + anchor string policy, forkCapture string trajDir string forcePlanner, checkpoints bool @@ -488,6 +502,10 @@ func runSuite(cfg suiteConfig, tasks []task) []result { results = append(results, result{task: t, Profile: cfg.profile, Skipped: true, Note: "skipped: token budget reached"}) continue } + if skipped, ok := anchorSkip(cfg, t); ok { + results = append(results, skipped) + continue + } var cumWallMs int64 for attempt := 1; attempt <= max(cfg.attempts, 1); attempt++ { r := runTask(cfg, t) @@ -512,6 +530,8 @@ func runSuite(cfg suiteConfig, tasks []task) []result { func runTask(cfg suiteConfig, t task) result { r := result{task: t, Profile: cfg.profile, CacheArm: cfg.cacheArm, Effort: cfg.effort} r.Arm = cfg.arm.Arm() + r.Anchor = cfg.anchor + t.Prompt = anchorPrompt(cfg.anchor, t) if cfg.forcePlanner { // Leading directive matched by the planner gate's // planAndExecuteDirectives, so the two-model turn engages even for @@ -586,7 +606,11 @@ func runTask(cfg suiteConfig, t task) result { // still grade — a non-zero exit may just be a max-steps notice } - r.Passed = grade(work, t.dir) + var graderSaid string + r.Passed, graderSaid = gradeVerbose(work, t.dir) + if !r.Passed && graderSaid != "" { + r.Note = appendNote(r.Note, "grader: "+utf8Prefix(graderSaid, graderNoteLimit)) + } if snap != nil { r.Checkpoints = gradeCheckpoints(taken, t.dir) r.FirstCorrectMs, r.SolvedThenBroken = firstCorrect(r.Checkpoints, r.Passed) @@ -681,22 +705,6 @@ func warmPrefix(cfg suiteConfig, work string) { } } -func grade(work, taskDir string) bool { - verify := filepath.Join(taskDir, "verify.sh") - if !fileExists(verify) { - return false - } - dst := filepath.Join(work, "verify.sh") - if err := copyFile(verify, dst); err != nil { - return false - } - cmd := exec.Command("bash", "verify.sh") - cmd.Dir = work - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - return cmd.Run() == nil -} - func readMetrics(path string) (runMetrics, error) { var m runMetrics b, err := fileencoding.ReadFileUTF8(path) diff --git a/cmd/e2ebench/metrics_mirror_test.go b/cmd/e2ebench/metrics_mirror_test.go new file mode 100644 index 0000000000..8f931e5cdb --- /dev/null +++ b/cmd/e2ebench/metrics_mirror_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "reflect" + "strings" + "testing" + + "reasonix/internal/cli" +) + +// jsonTags collects a struct's JSON field names, descending into embedded and +// nested structs the way encoding/json itself does. +func jsonTags(t reflect.Type, into map[string]bool) { + for i := range t.NumField() { + f := t.Field(i) + tag, _, _ := strings.Cut(f.Tag.Get("json"), ",") + if tag == "-" { + continue + } + if tag != "" { + into[tag] = true + } + ft := f.Type + for ft.Kind() == reflect.Pointer { + ft = ft.Elem() + } + if ft.Kind() == reflect.Struct && ft.PkgPath() != "time" { + jsonTags(ft, into) + } + } +} + +// The bench reads a metrics file the agent writes, and the two structs are +// hand-mirrored. A tag the bench reads that nobody emits does not fail: the +// field silently stays zero and every report built on it quietly reads as +// "this never happened". Renaming one side must break the build, not the data. +func TestBenchMetricsOnlyReadTagsTheAgentEmits(t *testing.T) { + emitted := map[string]bool{} + jsonTags(reflect.TypeFor[cli.RunMetrics](), emitted) + + read := map[string]bool{} + jsonTags(reflect.TypeFor[runMetrics](), read) + + var orphaned []string + for tag := range read { + if !emitted[tag] { + orphaned = append(orphaned, tag) + } + } + if len(orphaned) > 0 { + t.Fatalf("e2ebench reads metrics tags no agent writes: %v\n"+ + "either internal/cli.RunMetrics lost them in a rename, or the bench "+ + "invented them; both leave the field zero in every report", orphaned) + } +} diff --git a/cmd/e2ebench/profile.go b/cmd/e2ebench/profile.go index 52a0b8cf5c..c0525b38cf 100644 --- a/cmd/e2ebench/profile.go +++ b/cmd/e2ebench/profile.go @@ -1,10 +1,29 @@ package main import ( + "errors" "fmt" "strings" + + "reasonix/internal/ablation" ) +// experimentAxes is the validated set of fixed axes one suite invocation runs +// on. They are resolved together so a second bad flag is reported alongside +// the first instead of being masked by an early exit. +type experimentAxes struct { + profile, cache, anchor string + arm ablation.Set +} + +func resolveExperimentAxes(profile, ablate, cache, anchor string) (experimentAxes, error) { + p, perr := normalizeBenchmarkProfile(profile) + a, aerr := ablation.Parse(ablate) + c, cerr := normalizeCacheArm(cache) + an, anerr := normalizeAnchorArm(anchor) + return experimentAxes{profile: p, cache: c, anchor: an, arm: a}, errors.Join(perr, aerr, cerr, anerr) +} + const ( benchmarkProfileBaseline = "baseline" benchmarkProfileEconomy = "economy" diff --git a/cmd/e2ebench/report.go b/cmd/e2ebench/report.go index dc2be7a363..c72a1db973 100644 --- a/cmd/e2ebench/report.go +++ b/cmd/e2ebench/report.go @@ -203,6 +203,7 @@ func renderBody(results []result) string { b.WriteString(renderOutcomeProgress(results)) b.WriteString(renderMemoryShadow(results)) b.WriteString(renderCognition(results)) + b.WriteString(renderAnchor(results)) b.WriteString(renderDelegation(results)) b.WriteString(renderDelegationAdmission(results)) b.WriteString(renderMechanismLedger(results)) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 524fb90d86..b5def1dc1e 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1141,11 +1141,11 @@ type Options struct { // everything, so ordinary callers leave it unset. Ablation ablation.Set - // ClassifierTaskText, when non-empty, is the pristine task text delivery - // intent classification should judge instead of the raw Run input. Sub-agent - // spawners set it before prepending host framing (subagent/workspace context, - // review contracts) so framing verbs cannot arm expectations and user input - // dressed up as framing cannot disarm them. + // ClassifierTaskText, when non-empty, is the pristine task text, set by + // sub-agent spawners before host framing is prepended. Delivery intent + // classification judges it instead of the raw Run input, so framing verbs + // cannot arm expectations; the delegation audit scores evidence origin + // against it, so only locations the parent wrote itself count as hints. ClassifierTaskText string // CapabilityLedger is the optional turn-scoped capability route ledger for diff --git a/internal/agent/delegation_origin_test.go b/internal/agent/delegation_origin_test.go new file mode 100644 index 0000000000..5d2d8fbcc2 --- /dev/null +++ b/internal/agent/delegation_origin_test.go @@ -0,0 +1,73 @@ +package agent + +import ( + "context" + "encoding/json" + "testing" + + "reasonix/internal/event" + "reasonix/internal/evidence" + "reasonix/internal/provider" + "reasonix/internal/tool" +) + +type originProbe struct { + event.Sink + got []evidence.DelegationAudit +} + +func (p *originProbe) RecordDelegationAudit(a evidence.DelegationAudit) { p.got = append(p.got, a) } + +// runDelegationForOrigin spawns one real child through TaskTool that reads +// src/parser.go and src/scanner.go, and returns the audit the sink received. +func runDelegationForOrigin(t *testing.T, prompt string) evidence.DelegationAudit { + t.Helper() + probe := &originProbe{Sink: event.Discard} + reg := tool.NewRegistry() + reg.Add(fakeReadFileTool{}) + prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ + {toolCallChunk("1", "read_file", `{"path":"src/parser.go"}`), {Type: provider.ChunkDone}}, + {toolCallChunk("2", "read_file", `{"path":"src/scanner.go"}`), {Type: provider.ChunkDone}}, + {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, + }} + task := NewTaskTool(prov, nil, reg, 20, 0, 0, 0, 0, 0, 0, 0.0, "", "sys", nil, 0, "", "", nil). + WithTranscripts(mustSubagentStore(t), t.TempDir(), "base", "high"). + WithScheduler(NewSubagentScheduler(4, 4)) + ctx := withCallContext(context.Background(), "call-1", probe, nil, false) + args, err := json.Marshal(map[string]string{"prompt": prompt}) + if err != nil { + t.Fatal(err) + } + if _, err := task.Execute(ctx, args); err != nil { + t.Fatalf("task: %v", err) + } + if len(probe.got) != 1 { + t.Fatalf("got %d audits through the TaskTool path, want 1", len(probe.got)) + } + return probe.got[0] +} + +// A delegation that hands over no location leaves the child to find every file +// it reads. This is the number blind delegation would have to hold at zero. +func TestDelegationAuditScoresBlindDelegationAsFullyDiscovered(t *testing.T) { + audit := runDelegationForOrigin(t, "Determine why the failing test fails.") + if audit.ParentNamedFiles != 0 { + t.Errorf("ParentNamedFiles = %d, want 0", audit.ParentNamedFiles) + } + if audit.EvidencePaths != 2 || audit.DiscoveredPaths != 2 { + t.Errorf("discovered %d/%d, want 2/2", audit.DiscoveredPaths, audit.EvidencePaths) + } +} + +// The parent's own text is the only channel that can leak a location, so a +// prompt naming one file must show up as a hint and cost the child credit for +// the evidence it did not have to find. +func TestDelegationAuditCreditsOnlyWhatTheChildFoundItself(t *testing.T) { + audit := runDelegationForOrigin(t, "The bug is in src/parser.go — confirm it.") + if audit.ParentNamedFiles != 1 { + t.Errorf("ParentNamedFiles = %d, want 1", audit.ParentNamedFiles) + } + if audit.EvidencePaths != 2 || audit.DiscoveredPaths != 1 { + t.Errorf("discovered %d/%d, want 1/2", audit.DiscoveredPaths, audit.EvidencePaths) + } +} diff --git a/internal/agent/subagent_report.go b/internal/agent/subagent_report.go index 0ec0d9d5d2..acd4d18485 100644 --- a/internal/agent/subagent_report.go +++ b/internal/agent/subagent_report.go @@ -74,20 +74,22 @@ func decorateExecutionReceipt(rec *evidence.Receipt, result string, ex *tool.She // composeSubagentAnswer assembles everything the parent is shown for one child // run: the host-adjudicated completion claim when the child submitted one, the // child's own prose, then the host's receipts. -func composeSubagentAnswer(ctx context.Context, answer string, sub *Agent, claims WritePathSet) string { +func composeSubagentAnswer(ctx context.Context, answer string, sub *Agent, claims WritePathSet, delegationText string) string { summary := sub.EvidenceSummary() report, reasons, hasReport := sub.CompletionReport() if hasReport { answer = strings.TrimSpace(formatCompletionReport(report, reasons) + "\n\n" + answer) } - recordDelegationAudit(ctx, summary, claims, report, reasons, hasReport) + recordDelegationAudit(ctx, summary, claims, report, reasons, hasReport, delegationText) return appendHostReceipts(answer, summary, claims) } // recordDelegationAudit emits one structured receipt per child run. It reports // what the host observed and what it refused to back, so an orchestration -// benchmark can separate real gains from extra tokens spent. -func recordDelegationAudit(ctx context.Context, summary evidence.ChildEvidenceSummary, claims WritePathSet, report evidence.CompletionReport, reasons []string, hasReport bool) { +// benchmark can separate real gains from extra tokens spent. delegationText is +// the parent-authored task before host framing, which is what makes the +// evidence-origin split a host record rather than a claim. +func recordDelegationAudit(ctx context.Context, summary evidence.ChildEvidenceSummary, claims WritePathSet, report evidence.CompletionReport, reasons []string, hasReport bool, delegationText string) { audit := evidence.DelegationAudit{ Depth: SubagentDepth(ctx), ToolCalls: len(summary.Receipts), @@ -97,6 +99,7 @@ func recordDelegationAudit(ctx context.Context, summary evidence.ChildEvidenceSu Downgrades: len(reasons), } audit.Mutations = len(audit.MutationPaths) + audit.ClassifyEvidenceOrigin(delegationText, summary.EvidencePaths()) if hasReport { audit.AdjudicatedStatus = string(report.Status) } diff --git a/internal/agent/task.go b/internal/agent/task.go index e3d1b4e9aa..514357156b 100644 --- a/internal/agent/task.go +++ b/internal/agent/task.go @@ -1833,7 +1833,7 @@ func RunSubAgentWithSession(ctx context.Context, prov provider.Provider, reg *to // Still merge any partial child evidence so parent gates see real writes. mergeChildEvidence(ctx, sub) if answer, ok := salvageReadinessExhaustedAnswer(sub, sess, opts, err); ok { - return composeSubagentAnswer(ctx, answer, sub, SubagentWriteClaim(ctx)), nil + return composeSubagentAnswer(ctx, answer, sub, SubagentWriteClaim(ctx), opts.ClassifierTaskText), nil } return "", fmt.Errorf("sub-agent: %w", err) } @@ -1861,7 +1861,7 @@ func RunSubAgentWithSession(ctx context.Context, prov provider.Provider, reg *to } mergeChildEvidence(ctx, sub) if answer := latestAssistantAnswer(sess); answer != "" { - return composeSubagentAnswer(ctx, answer, sub, SubagentWriteClaim(ctx)), nil + return composeSubagentAnswer(ctx, answer, sub, SubagentWriteClaim(ctx), opts.ClassifierTaskText), nil } return "", fmt.Errorf("sub-agent finished without producing a final answer") } diff --git a/internal/cli/delegation_metrics_test.go b/internal/cli/delegation_metrics_test.go index 12ae2dea3e..6d9ce02b0c 100644 --- a/internal/cli/delegation_metrics_test.go +++ b/internal/cli/delegation_metrics_test.go @@ -50,6 +50,28 @@ func TestDelegationMetricsAggregateAcrossChildren(t *testing.T) { } } +// An independence rate is a ratio of summed paths, never a mean of per-child +// rates: a child that opened one file must not weigh the same as one that +// swept twenty. Summing here is what makes the published rate that ratio. +func TestDelegationMetricsSumEvidenceOriginForARatioOfTotals(t *testing.T) { + s := &metricsSink{} + s.RecordDelegationAudit(evidence.DelegationAudit{ + Depth: 1, ParentNamedFiles: 1, EvidencePaths: 20, DiscoveredPaths: 19, + }) + s.RecordDelegationAudit(evidence.DelegationAudit{ + Depth: 1, ParentNamedFiles: 2, EvidencePaths: 1, DiscoveredPaths: 0, + }) + + m := s.m + if m.ParentNamedFiles != 3 { + t.Fatalf("parent named files = %d, want 3", m.ParentNamedFiles) + } + // 19/21, not the 50% a mean of 95% and 0% would report. + if m.ChildDiscoveredPaths != 19 || m.ChildEvidencePaths != 21 { + t.Fatalf("discovered %d/%d, want 19/21", m.ChildDiscoveredPaths, m.ChildEvidencePaths) + } +} + // A run with no delegation must leave every delegation counter at zero, so the // single-agent arm is a clean baseline rather than noise. func TestDelegationMetricsStayZeroForSingleAgentArm(t *testing.T) { diff --git a/internal/cli/run_metrics.go b/internal/cli/run_metrics.go index 77bdc0c7d6..903a5d268f 100644 --- a/internal/cli/run_metrics.go +++ b/internal/cli/run_metrics.go @@ -67,6 +67,10 @@ type RunMetrics struct { CriterionDowngrades int `json:"criterion_downgrades,omitempty"` WriteScopeViolations int `json:"write_scope_violations,omitempty"` DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"` + ParentScopeHints int `json:"parent_scope_hints,omitempty"` + ParentNamedFiles int `json:"parent_named_files,omitempty"` + ChildEvidencePaths int `json:"child_evidence_paths,omitempty"` + ChildDiscoveredPaths int `json:"child_discovered_paths,omitempty"` MissingReasoningDetected int `json:"missing_reasoning_detected,omitempty"` MissingReasoningRetries int `json:"missing_reasoning_retries,omitempty"` MissingReasoningRecovered int `json:"missing_reasoning_recovered,omitempty"` @@ -311,6 +315,12 @@ func (s *metricsSink) RecordDelegationAudit(a evidence.DelegationAudit) { s.m.SubagentMutations += a.Mutations s.m.WriteScopeViolations += a.ClaimViolations s.m.CriterionDowngrades += a.Downgrades + // Summed, never averaged: an independence rate is a ratio of these totals, + // so a child that read two files cannot outweigh one that read forty. + s.m.ParentScopeHints += a.ParentScopeHints + s.m.ParentNamedFiles += a.ParentNamedFiles + s.m.ChildEvidencePaths += a.EvidencePaths + s.m.ChildDiscoveredPaths += a.DiscoveredPaths if a.HasReport { s.m.CompletionReports++ } else { diff --git a/internal/evidence/child.go b/internal/evidence/child.go index c594301a38..29f93b3cfd 100644 --- a/internal/evidence/child.go +++ b/internal/evidence/child.go @@ -40,6 +40,28 @@ func (s ChildEvidenceSummary) MutationPaths() []string { return out } +// EvidencePaths returns every distinct path the child produced a successful +// receipt for, reads included. MutationPaths answers what the child changed; +// this answers what it looked at, which is what evidence origin scores. +func (s ChildEvidenceSummary) EvidencePaths() []string { + seen := map[string]bool{} + var out []string + for _, r := range s.Receipts { + if !r.Success { + continue + } + for _, p := range r.Paths { + if p == "" || seen[p] { + continue + } + seen[p] = true + out = append(out, p) + } + } + sort.Strings(out) + return out +} + // Summary returns a snapshot of every receipt recorded this turn in order. func (l *Ledger) Summary() ChildEvidenceSummary { if l == nil { diff --git a/internal/evidence/delegation_audit.go b/internal/evidence/delegation_audit.go index 92718f9a2b..f8ad325344 100644 --- a/internal/evidence/delegation_audit.go +++ b/internal/evidence/delegation_audit.go @@ -18,6 +18,32 @@ type DelegationAudit struct { // Downgrades counts the criterion claims it refused. AdjudicatedStatus string Downgrades int + // ParentScopeHints counts directories the delegation narrowed the search to. + ParentScopeHints int + // ParentNamedFiles counts specific files the delegation text handed over. + ParentNamedFiles int + // EvidencePaths is every distinct path the child produced a receipt for. + EvidencePaths int + // DiscoveredPaths is the subset of those no named file already covered. + DiscoveredPaths int +} + +// ClassifyEvidenceOrigin scores one run against the text its parent wrote. +// Discovery is judged against named files only: a child sent to a directory +// still had to work out which file in it mattered. Counts only — a rate is a +// ratio of sums across runs, and averaging per-run rates would weight a child +// that read two files like one that read forty. +func (a *DelegationAudit) ClassifyEvidenceOrigin(delegationText string, evidencePaths []string) { + scope, files := SplitNamedPaths(NamedPaths(delegationText)) + a.ParentScopeHints = len(scope) + a.ParentNamedFiles = len(files) + a.EvidencePaths = len(evidencePaths) + a.DiscoveredPaths = 0 + for _, p := range evidencePaths { + if !UnderNamedPath(files, p) { + a.DiscoveredPaths++ + } + } } // FalseCompletion reports a child that claimed more than the host could back. diff --git a/internal/evidence/named_paths.go b/internal/evidence/named_paths.go new file mode 100644 index 0000000000..3f989db536 --- /dev/null +++ b/internal/evidence/named_paths.go @@ -0,0 +1,144 @@ +package evidence + +import ( + "path/filepath" + "slices" + "strings" +) + +// namedPathMaxExt bounds what counts as a file extension so ordinary prose +// such as "i.e." is never scored as a location handed to a child. +const namedPathMaxExt = 5 + +// NamedPaths returns the distinct locations a delegation's own text names. +// Recognition leans inclusive on purpose: a spurious token inflates the hint +// count and matches no receipt, while a miss would flatter the parent twice, +// once as a smaller hint and once as evidence credited to the child. +func NamedPaths(text string) []string { + seen := map[string]bool{} + var out []string + for _, token := range strings.FieldsFunc(text, namedPathBreak) { + p := namedPathToken(token) + if p == "" || seen[p] { + continue + } + seen[p] = true + out = append(out, p) + } + slices.Sort(out) + return out +} + +// SplitNamedPaths separates a delegation's locations into directory-level +// scope and file-level naming. A parent must narrow the search to hand off +// work at all, so scope is the cost of delegating; naming the file says where +// the answer is. Counting them as one number would report a minimal, honest +// scope hint as though it were the whole conclusion handed over. +func SplitNamedPaths(named []string) (scope, files []string) { + for _, p := range named { + if hasFileExtension(filepath.Base(p)) { + files = append(files, p) + continue + } + scope = append(scope, p) + } + return scope, files +} + +// UnderNamedPath reports whether a path the child produced evidence for was +// already named by the delegation. Matching is on whole path segments: a +// delegation writes workspace-relative prose while a receipt records the +// absolute path the tool actually received, so equality would never hold. +func UnderNamedPath(named []string, path string) bool { + p := namedPathSegments(path) + if p == "" { + return false + } + for _, n := range named { + seg := namedPathSegments(n) + if seg != "" && strings.Contains(p, seg) { + return true + } + } + return false +} + +func namedPathBreak(r rune) bool { + switch r { + case '`', '"', '\'', '(', ')', '[', ']', '{', '}', '<', '>', ',', ';', '|', '*': + return true + } + return r == ' ' || r == '\t' || r == '\n' || r == '\r' +} + +// namedPathToken reduces one whitespace-delimited token to the path it refers +// to, or "" when it refers to none. +func namedPathToken(token string) string { + token = strings.TrimPrefix(token, "@") + token = strings.TrimRight(token, ".!?") + token = trimLineRef(token) + token = strings.TrimRight(token, ":") + if token == "" || strings.Contains(token, "://") { + return "" + } + if !strings.Contains(token, "/") && !hasFileExtension(token) { + return "" + } + return normalizePath(token) +} + +// trimLineRef drops a ":184" or ":181-207" citation so a cited range and a +// plain mention of the same file count as one location. +func trimLineRef(token string) string { + for { + i := strings.LastIndexByte(token, ':') + if i < 0 || i == len(token)-1 || !isLineRef(token[i+1:]) { + return token + } + token = token[:i] + } +} + +func isLineRef(s string) bool { + digits := false + for _, r := range s { + switch { + case r >= '0' && r <= '9': + digits = true + case r == '-': + default: + return false + } + } + return digits +} + +// hasFileExtension accepts a dotted token only when the stem is at least two +// characters, which is what separates "parser.go" from "i.e." and "e.g.". +func hasFileExtension(token string) bool { + dot := strings.LastIndexByte(token, '.') + if dot < 2 || dot == len(token)-1 { + return false + } + ext := token[dot+1:] + if len(ext) > namedPathMaxExt { + return false + } + for _, r := range ext { + alnum := (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') + if !alnum { + return false + } + } + return true +} + +// namedPathSegments wraps a path in separators so a substring test can only +// match on whole segments: "/parser.go/" never matches "/myparser.go/". +func namedPathSegments(p string) string { + p = strings.Trim(filepath.ToSlash(normalizePath(p)), "/") + if p == "" || p == "." { + return "" + } + return "/" + p + "/" +} diff --git a/internal/evidence/named_paths_test.go b/internal/evidence/named_paths_test.go new file mode 100644 index 0000000000..753ddd20de --- /dev/null +++ b/internal/evidence/named_paths_test.go @@ -0,0 +1,143 @@ +package evidence + +import ( + "path/filepath" + "slices" + "testing" +) + +func TestNamedPathsReadsLocationsOutOfDelegationProse(t *testing.T) { + text := "Look at `src/parser.go:181-207` and @internal/agent, " + + "then compare against src/parser.go. See https://example.com/docs.go, i.e. the CRLF path." + got := NamedPaths(text) + want := []string{ + filepath.FromSlash("internal/agent"), + filepath.FromSlash("src/parser.go"), + } + if !slices.Equal(got, want) { + t.Fatalf("NamedPaths = %q, want %q", got, want) + } +} + +// Prose that merely discusses the problem is not a location handed over: if +// it were, every blind delegation would report a non-zero hint count. +func TestNamedPathsIgnoresOrdinaryProse(t *testing.T) { + for _, text := range []string{ + "Determine the root cause of the failing parser test.", + "The offset is wrong, i.e. it double-counts.", + "Check the scanner, e.g. its newline handling.", + "Run the suite and report what fails.", + } { + if got := NamedPaths(text); len(got) != 0 { + t.Errorf("NamedPaths(%q) = %q, want none", text, got) + } + } +} + +// A delegation writes workspace-relative prose while a receipt records the +// absolute path the tool received, so origin cannot be decided by equality. +func TestUnderNamedPathMatchesAbsoluteReceiptAgainstRelativeProse(t *testing.T) { + named := NamedPaths("start from src/parser.go") + abs := filepath.Join(t.TempDir(), "src", "parser.go") + if !UnderNamedPath(named, abs) { + t.Fatalf("UnderNamedPath(%q, %q) = false, want true", named, abs) + } +} + +func TestUnderNamedPathCoversFilesBeneathADirectoryHint(t *testing.T) { + named := NamedPaths("the bug is somewhere in internal/agent") + root := t.TempDir() + inside := filepath.Join(root, "internal", "agent", "task.go") + outside := filepath.Join(root, "internal", "evidence", "task.go") + if !UnderNamedPath(named, inside) { + t.Errorf("UnderNamedPath(%q, %q) = false, want true", named, inside) + } + if UnderNamedPath(named, outside) { + t.Errorf("UnderNamedPath(%q, %q) = true, want false", named, outside) + } +} + +// Substring matching would credit the parent for a file it never named. +func TestUnderNamedPathMatchesWholeSegmentsOnly(t *testing.T) { + named := NamedPaths("see parser.go") + if UnderNamedPath(named, filepath.FromSlash("src/myparser.go")) { + t.Fatal("UnderNamedPath matched myparser.go against parser.go") + } +} + +// Narrowing the search is what delegating costs; naming the file is handing +// over the answer. Reported as one number, a parent that said "look in pkg/" +// would be indistinguishable from one that said "the bug is in pkg/romeo.go". +func TestSplitNamedPathsSeparatesScopeFromNamedFiles(t *testing.T) { + scope, files := SplitNamedPaths(NamedPaths("search pkg/ and internal/agent; the bug is in pkg/romeo.py")) + wantScope := []string{filepath.FromSlash("internal/agent"), "pkg"} + wantFiles := []string{filepath.FromSlash("pkg/romeo.py")} + if !slices.Equal(scope, wantScope) { + t.Errorf("scope = %q, want %q", scope, wantScope) + } + if !slices.Equal(files, wantFiles) { + t.Errorf("files = %q, want %q", files, wantFiles) + } +} + +// A child sent to a directory still had to work out which file in it mattered, +// so a scope hint must not zero out the credit for finding one. +func TestClassifyEvidenceOriginCreditsWorkInsideAScopeHint(t *testing.T) { + root := t.TempDir() + looked := []string{ + filepath.Join(root, "pkg", "alpha.py"), + filepath.Join(root, "pkg", "romeo.py"), + } + + var scoped DelegationAudit + scoped.ClassifyEvidenceOrigin("Search pkg/ for the module that gets it wrong.", looked) + if scoped.ParentScopeHints != 1 || scoped.ParentNamedFiles != 0 { + t.Fatalf("scoped = %+v, want 1 scope hint and no named file", scoped) + } + if scoped.DiscoveredPaths != 2 { + t.Errorf("a scope hint erased the child's credit: %d/2", scoped.DiscoveredPaths) + } + + var told DelegationAudit + told.ClassifyEvidenceOrigin("The bug is in pkg/romeo.py.", looked) + if told.ParentNamedFiles != 1 || told.DiscoveredPaths != 1 { + t.Errorf("told = %+v, want 1 named file and 1 of 2 discovered", told) + } +} + +func TestClassifyEvidenceOriginSplitsBlindFromSeededDelegation(t *testing.T) { + root := t.TempDir() + looked := []string{ + filepath.Join(root, "src", "parser.go"), + filepath.Join(root, "src", "scanner.go"), + filepath.Join(root, "tests", "test_parser.py"), + } + + var blind DelegationAudit + blind.ClassifyEvidenceOrigin("Diagnose the failing parser test.", looked) + if blind.ParentNamedFiles != 0 || blind.DiscoveredPaths != 3 || blind.EvidencePaths != 3 { + t.Fatalf("blind = %+v, want 0 named and 3/3 discovered", blind) + } + + var seeded DelegationAudit + seeded.ClassifyEvidenceOrigin("I suspect src/parser.go double-normalizes CRLF.", looked) + if seeded.ParentNamedFiles != 1 { + t.Fatalf("seeded.ParentNamedFiles = %d, want 1", seeded.ParentNamedFiles) + } + if seeded.DiscoveredPaths != 2 || seeded.EvidencePaths != 3 { + t.Fatalf("seeded = %+v, want 2 of 3 discovered", seeded) + } +} + +func TestEvidencePathsCoversReadsAndSkipsFailedReceipts(t *testing.T) { + summary := ChildEvidenceSummary{Receipts: []Receipt{ + {ToolName: "read", Success: true, Read: true, Paths: []string{"src/parser.go"}}, + {ToolName: "read", Success: true, Read: true, Paths: []string{"src/parser.go"}}, + {ToolName: "edit", Success: true, Write: true, Mutation: true, Paths: []string{"src/scanner.go"}}, + {ToolName: "read", Success: false, Read: true, Paths: []string{"src/never.go"}}, + }} + want := []string{"src/parser.go", "src/scanner.go"} + if got := summary.EvidencePaths(); !slices.Equal(got, want) { + t.Fatalf("EvidencePaths = %q, want %q", got, want) + } +}