mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 09:49:34 -10:00
perf(turn-bench): split pass/fail into oracle tiers so the pass rate can't be misread as correctness
The turn benchmark counted any exit-0 task as passed, including the 24 read-only tasks (nav/longproc/longctx/parallel) that carry no verification oracle — so the JSON tasksPassed / "N/M passed" headline could read as a correctness verdict it never was. Split pass/fail into three oracle tiers: - correctness (edit grep + fix go test): the only pass rate that can move with model quality (tasksVerified / tasksPassed / correctnessPassRate) - build-only (refactor go build): a non-positive oracle, reported as buildPassRate and excluded from correctnessPassRate; the manifest declares buildOnlyClasses: ["refactor"] - latency-only (no oracle): counted in latencyOnlyTasks, never in any pass rate NewTurnExecRunner no longer sets Passed=true for tasks with no verificationCommand, and the summary headline names each tier explicitly so an exit-0 read-only run cannot inflate a correctness number. Bumps TurnSchemaVersion to 2. Class breakdown and tier contract documented in internal/perfbench/MANIFEST.md and reports/README.md. Also install the trace first-token/visible-event forwarding handlers in agent.Run whenever options.Trace is set, not only when a UI OnText / OnReasoning callback is set, so a headless traced run (zero exec --trace) still captures FirstTokenAt. forwardedVisibleText stays tied to the user callback so stall-retry semantics are unchanged. Adds a regression assertion to the existing traced-run test.
This commit is contained in:
Vasanthdev2004 committed
1 parent
40806aa4b1
commit
2bfb340668
8 files changed
+414
-72
No files matched your search
+28
-8
@@ -284,18 +284,38 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// conversation-state duplication.
|
||||
forwardedVisibleText := false
|
||||
forwardingOpts := zeroruntime.CollectOptions{OnUsage: options.OnUsage}
|
||||
if options.OnText != nil {
|
||||
// Install text/reasoning forwarding handlers whenever EITHER a user
|
||||
// callback OR a trace recorder is set. A headless traced run (e.g. `zero
|
||||
// exec --trace`) sets Trace but no OnText/OnReasoning; without these
|
||||
// handlers the stream is still collected, but FirstTokenAt would never
|
||||
// stamp and the trace would lose its TTFT signal. The trace recorder's
|
||||
// stamp methods are nil-safe, but we guard on `trace != nil` so a run with
|
||||
// a user callback but no recorder still works. forwardedVisibleText stays
|
||||
// tied to the USER callback only, preserving the stall-retry semantics
|
||||
// (a trace-only handler forwards nothing the user would see duplicated).
|
||||
rec := options.Trace
|
||||
onText := options.OnText
|
||||
if onText != nil || rec != nil {
|
||||
forwardingOpts.OnText = func(s string) {
|
||||
forwardedVisibleText = true
|
||||
options.Trace.StampFirstVisibleEvent()
|
||||
options.Trace.StampFirstToken()
|
||||
options.OnText(s)
|
||||
if rec != nil {
|
||||
rec.StampFirstVisibleEvent()
|
||||
rec.StampFirstToken()
|
||||
}
|
||||
if onText != nil {
|
||||
forwardedVisibleText = true
|
||||
onText(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.OnReasoning != nil {
|
||||
onReasoning := options.OnReasoning
|
||||
if onReasoning != nil || rec != nil {
|
||||
forwardingOpts.OnReasoning = func(s string) {
|
||||
options.Trace.StampFirstToken()
|
||||
options.OnReasoning(s)
|
||||
if rec != nil {
|
||||
rec.StampFirstToken()
|
||||
}
|
||||
if onReasoning != nil {
|
||||
onReasoning(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.OnToolCallStart != nil {
|
||||
|
||||
@@ -3488,6 +3488,13 @@ func TestRunTracingWrapperStampsUsage(t *testing.T) {
|
||||
if tr.StartedAt.IsZero() {
|
||||
t.Fatal("tracing wrapper did not Start the recorder")
|
||||
}
|
||||
// FirstTokenAt must stamp even though no OnText/OnReasoning user callback is
|
||||
// set: a headless traced run (e.g. `zero exec --trace`) sets Trace but no UI
|
||||
// callbacks, so the loop installs trace-only forwarding handlers to capture
|
||||
// TTFT. Without them FirstTokenAt stays zero and the trace loses its signal.
|
||||
if tr.FirstTokenAt.IsZero() {
|
||||
t.Fatal("FirstTokenAt not stamped for a traced run with no OnText/OnReasoning callbacks")
|
||||
}
|
||||
if got := tr.Counter(trace.CounterInputTokens); got != 100 {
|
||||
t.Fatalf("input token counter = %d, want 100", got)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# Turn benchmark manifest
|
||||
|
||||
The baseline manifest (`manifests/baseline.json`) is the per-turn benchmark's
|
||||
program keystone: it defines the tasks the harness runs, the workspace each
|
||||
starts in, and — critically — what "pass" means for each task. Because `make
|
||||
baseline` is re-run on every perf change, the manifest's pass/fail contract
|
||||
matters for months, so the contract is written down here.
|
||||
|
||||
## Task count
|
||||
|
||||
48 tasks across seven classes:
|
||||
|
||||
| class | count | oracle | tier |
|
||||
|-----------|-------|-----------------------------|-------------|
|
||||
| nav | 10 | none | latency |
|
||||
| edit | 10 | substring grep | correctness |
|
||||
| fix | 8 | scoped `go test -run <name>` | correctness |
|
||||
| refactor | 6 | `go build ./...` | build |
|
||||
| longproc | 4 | none | latency |
|
||||
| longctx | 4 | none | latency |
|
||||
| parallel | 6 | none | latency |
|
||||
| **total** | **48**| | |
|
||||
|
||||
## Oracle tiers
|
||||
|
||||
Pass/fail is reported per tier so the report cannot be misread as a blanket
|
||||
correctness verdict. The tier is decided per task from oracle presence and the
|
||||
manifest's `buildOnlyClasses` list:
|
||||
|
||||
- **Correctness** (18 tasks: 10 edit + 8 fix) — a positive oracle (substring
|
||||
grep that the requested change landed, or a scoped `go test -run` that the
|
||||
bug fix passes). This is the only pass rate that can move with model quality:
|
||||
`tasksVerified` / `tasksPassed` / `correctnessPassRate`.
|
||||
- **Build-only** (6 tasks: refactor) — `go build ./...` proves the edit
|
||||
compiles, not that the refactor achieved its goal (a no-op refactor passes).
|
||||
The manifest declares `buildOnlyClasses: ["refactor"]`. Reported as
|
||||
`buildCheckedTasks` / `buildPassedTasks` / `buildPassRate`, never in
|
||||
`correctnessPassRate`.
|
||||
- **Latency-only** (24 tasks: nav, longproc, longctx, parallel) — no
|
||||
`verificationCommand`. An exit 0 only proves the turn ran, not that the answer
|
||||
was right. They contribute to latency and span attribution only and are
|
||||
counted in `latencyOnlyTasks`, never in any pass rate.
|
||||
|
||||
A task's tier is driven by oracle **presence** first: a task with no
|
||||
`verificationCommand` is latency-only even if its class is listed in
|
||||
`buildOnlyClasses`, so a missing oracle can never silently pass on exit 0.
|
||||
|
||||
## Known limitations (deferred)
|
||||
|
||||
These are accepted for the Phase 0 baseline and tracked in a follow-up issue;
|
||||
they do not block the baseline because the tier split keeps the report honest:
|
||||
|
||||
- The edit grep oracles are substring checks. A rename oracle asserts the new
|
||||
name is present AND the old name is gone (compound `bash -c`), but an
|
||||
add-field oracle (`grep -R Label .`) only proves the string appears, not that
|
||||
it landed on the right struct. Strengthening these to `go vet`/`go build` +
|
||||
structural assertions is follow-up work.
|
||||
- The refactor build oracle is non-positive (a no-op refactor passes). A
|
||||
structural verifier that proves the refactor happened is follow-up work;
|
||||
until then refactor is `buildPassRate`, not `correctnessPassRate`.
|
||||
- The 24 read-only tasks have no oracle. Adding deterministic oracles for nav
|
||||
(diff against an expected answer) and documenting longproc/longctx/parallel
|
||||
as permanently latency-only is follow-up work.
|
||||
|
||||
## Fixtures
|
||||
|
||||
Each task's `workspaceFixture` points at a small self-contained workspace under
|
||||
`testdata/` so the suite runs offline and repeatably. Mutating tasks (edit,
|
||||
fix, refactor) run against a per-invocation **copy** of their fixture, so the
|
||||
checked-in fixtures stay clean and one task's edits can't bleed into the next
|
||||
iteration or a later task.
|
||||
@@ -1,7 +1,8 @@
|
||||
{
|
||||
"id": "zero-baseline-turn",
|
||||
"name": "Zero per-turn baseline (Phase 0)",
|
||||
"description": "Latency-only baseline: measures where a turn spends wall time, not task correctness. Read-only tasks (nav, longproc, longctx, parallel) carry no verification oracle by design — a zero-exit does not prove the answer was right, only that the turn ran. Edit/fix/refactor oracles are intentionally lightweight (substring grep or `go build`) so the suite stays a latency probe; do not read pass/fail as a correctness verdict.",
|
||||
"description": "Latency-first baseline: measures where a turn spends wall time, with pass/fail reported per oracle tier so it cannot be misread as a blanket correctness verdict. Read-only classes (nav, longproc, longctx, parallel) carry no verification oracle — a zero-exit only proves the turn ran, so they count as latency-only and never in any pass rate. Correctness classes (edit, fix) carry a positive oracle (substring grep or a scoped `go test`) and are the only tasks in correctnessPassRate. Build-only classes (refactor) carry `go build ./...`, which proves the edit compiles but not that the refactor achieved its goal, so they are reported as buildPassRate and excluded from correctnessPassRate.",
|
||||
"buildOnlyClasses": ["refactor"],
|
||||
"tasks": [
|
||||
{"id": "nav-01", "name": "list repo files", "class": "nav", "prompt": "List the files in this repository and report their count. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-02", "name": "find a function", "class": "nav", "prompt": "Find the function named 'greet' and report the file and line it is defined on. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
|
||||
@@ -39,9 +39,26 @@ spent.
|
||||
**Coverage** is the fraction of wall covered by the union of all span
|
||||
intervals (capped at 1.0) — the honest "≥95% of wall accounted for" metric. A
|
||||
run with `coverage < 0.95` has uninstrumented gaps, not an inflated attribution.
|
||||
Note that the suite is **latency-only** (see the manifest `description`): the
|
||||
top-three ranking is about latency, and pass/fail oracles are intentionally
|
||||
lightweight; do not read them as a correctness verdict.
|
||||
|
||||
### Pass/fail is reported per oracle tier
|
||||
|
||||
Pass/fail is split into three tiers so it cannot be misread as a blanket
|
||||
correctness verdict (see `MANIFEST.md` for the class breakdown):
|
||||
|
||||
- **Correctness** (`tasksVerified` / `tasksPassed` / `correctnessPassRate`):
|
||||
tasks with a positive oracle — edit's substring grep, fix's scoped `go test`.
|
||||
This is the only pass rate that can move with model quality.
|
||||
- **Build-only** (`buildCheckedTasks` / `buildPassedTasks` / `buildPassRate`):
|
||||
refactor's `go build ./...`, which proves the edit compiles but not that the
|
||||
refactor achieved its goal. Reported separately, never in `correctnessPassRate`.
|
||||
- **Latency-only** (`latencyOnlyTasks`): the read-only classes (nav, longproc,
|
||||
longctx, parallel) carry no oracle — an exit 0 only proves the turn ran. They
|
||||
contribute to latency and span attribution and are excluded from every pass
|
||||
rate.
|
||||
|
||||
`tasksAttempted` is still the total across all three tiers. The tier class lists
|
||||
are echoed in the report so a consumer can see exactly which classes each rate
|
||||
is computed over.
|
||||
|
||||
## What to commit
|
||||
|
||||
|
||||
@@ -29,6 +29,14 @@ type TaskSet struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Tasks []BenchTask `json:"tasks"`
|
||||
// BuildOnlyClasses lists task classes whose verificationCommand is a
|
||||
// non-positive build check (e.g. refactor's `go build ./...`): it proves the
|
||||
// edit compiles, not that the refactor achieved its goal. The turn benchmark
|
||||
// reports these separately from correctness oracles so a build-pass cannot be
|
||||
// misread as a correctness pass. Classes with a verificationCommand that are
|
||||
// NOT listed here are treated as correctness classes; classes whose tasks
|
||||
// carry no verificationCommand are latency-only regardless of this list.
|
||||
BuildOnlyClasses []string `json:"buildOnlyClasses,omitempty"`
|
||||
}
|
||||
|
||||
// BenchTask is one benchmark task. WorkspaceFixture is the relative path of the
|
||||
|
||||
@@ -20,7 +20,14 @@ import (
|
||||
|
||||
// TurnSchemaVersion is the schema version of a published turn-benchmark result.
|
||||
// Bump when the TurnBenchResult shape changes so consumers can detect drift.
|
||||
const TurnSchemaVersion = 1
|
||||
//
|
||||
// v2 splits pass/fail into three oracle tiers so the report cannot be misread as
|
||||
// a blanket correctness verdict: tasksVerified/tasksPassed/correctnessPassRate
|
||||
// cover only the positive-oracle classes (edit/fix); buildCheckedTasks/
|
||||
// buildPassedTasks/buildPassRate cover the non-positive build-check classes
|
||||
// (refactor); latencyOnlyTasks covers the no-oracle classes (nav/longproc/
|
||||
// longctx/parallel). tasksAttempted is still every task run.
|
||||
const TurnSchemaVersion = 2
|
||||
|
||||
// TurnRunner runs one benchmark task and reports its outcome plus the captured
|
||||
// per-turn trace. A non-nil Err means the run failed to execute (process crash);
|
||||
@@ -80,31 +87,64 @@ type LatencySource struct {
|
||||
}
|
||||
|
||||
// ClassSummary is the per-class (task group) roll-up.
|
||||
//
|
||||
// Passed is the count that passed THIS class's oracle: a correctness pass for
|
||||
// edit/fix, a build pass for refactor, and always 0 for the no-oracle classes
|
||||
// (nav/longproc/longctx/parallel). Verified is how many tasks in the class
|
||||
// carry an oracle (correctness or build); LatencyOnly is how many carry none.
|
||||
// A latency-only class therefore reports Passed=0, Verified=0, LatencyOnly=Tasks.
|
||||
type ClassSummary struct {
|
||||
Tasks int `json:"tasks"`
|
||||
Passed int `json:"passed"`
|
||||
WallMs NumericStats `json:"wallMs"`
|
||||
SpanTotals map[string]float64 `json:"spanTotals"`
|
||||
Tasks int `json:"tasks"`
|
||||
Verified int `json:"verified"`
|
||||
Passed int `json:"passed"`
|
||||
LatencyOnly int `json:"latencyOnly"`
|
||||
WallMs NumericStats `json:"wallMs"`
|
||||
SpanTotals map[string]float64 `json:"spanTotals"`
|
||||
}
|
||||
|
||||
// TurnBenchResult is the publishable turn-benchmark record.
|
||||
//
|
||||
// Pass/fail is split into three oracle tiers so it cannot be misread as a
|
||||
// blanket correctness verdict:
|
||||
// - Correctness (tasksVerified / tasksPassed / correctnessPassRate): tasks
|
||||
// with a positive oracle (edit/fix). This is the only pass rate that can
|
||||
// move with model quality.
|
||||
// - Build-only (buildCheckedTasks / buildPassedTasks / buildPassRate): tasks
|
||||
// whose oracle is a non-positive build check (refactor `go build`). A pass
|
||||
// means it compiles, not that the refactor is correct.
|
||||
// - Latency-only (latencyOnlyTasks): tasks with no oracle (nav/longproc/
|
||||
// longctx/parallel). They ran for latency and span attribution only and are
|
||||
// excluded from every pass rate.
|
||||
//
|
||||
// tasksAttempted is still the total number of tasks run across all three tiers.
|
||||
// The tier class lists are echoed so a consumer can see exactly which classes
|
||||
// each rate is computed over.
|
||||
type TurnBenchResult struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Suite string `json:"suite"`
|
||||
Model string `json:"model"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
SelfCorrect bool `json:"selfCorrect"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Date string `json:"date"`
|
||||
TasksAttempted int `json:"tasksAttempted"`
|
||||
TasksPassed int `json:"tasksPassed"`
|
||||
Iterations int `json:"iterations"`
|
||||
PerSpan map[string]SpanStats `json:"perSpan"`
|
||||
TopLatency []LatencySource `json:"topLatency"`
|
||||
PerClass map[string]ClassSummary `json:"perClass"`
|
||||
Totals TurnBenchTotals `json:"totals"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Suite string `json:"suite"`
|
||||
Model string `json:"model"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
SelfCorrect bool `json:"selfCorrect"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Date string `json:"date"`
|
||||
TasksAttempted int `json:"tasksAttempted"`
|
||||
TasksVerified int `json:"tasksVerified"`
|
||||
TasksPassed int `json:"tasksPassed"`
|
||||
LatencyOnlyTasks int `json:"latencyOnlyTasks"`
|
||||
BuildCheckedTasks int `json:"buildCheckedTasks"`
|
||||
BuildPassedTasks int `json:"buildPassedTasks"`
|
||||
CorrectnessPassRate float64 `json:"correctnessPassRate"`
|
||||
BuildPassRate float64 `json:"buildPassRate,omitempty"`
|
||||
CorrectnessClasses []string `json:"correctnessClasses,omitempty"`
|
||||
BuildOnlyClasses []string `json:"buildOnlyClasses,omitempty"`
|
||||
LatencyOnlyClasses []string `json:"latencyOnlyClasses,omitempty"`
|
||||
Iterations int `json:"iterations"`
|
||||
PerSpan map[string]SpanStats `json:"perSpan"`
|
||||
TopLatency []LatencySource `json:"topLatency"`
|
||||
PerClass map[string]ClassSummary `json:"perClass"`
|
||||
Totals TurnBenchTotals `json:"totals"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// TurnBenchTotals aggregates token and count totals across the whole run.
|
||||
@@ -148,9 +188,25 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe
|
||||
classWalls := map[string][]float64{}
|
||||
classSpanTotals := map[string]map[string]float64{}
|
||||
classTasks := map[string]int{}
|
||||
classVerified := map[string]int{}
|
||||
classPassed := map[string]int{}
|
||||
classLatencyOnly := map[string]int{}
|
||||
correctnessClasses := map[string]bool{}
|
||||
buildOnlyClasses := map[string]bool{}
|
||||
latencyOnlyClasses := map[string]bool{}
|
||||
var totals TurnBenchTotals
|
||||
|
||||
// A class is build-only when the manifest declares it in BuildOnlyClasses.
|
||||
// A task's tier is then decided per-task: no verificationCommand => latency-
|
||||
// only; otherwise build-only if its class is declared, else correctness. The
|
||||
// latency-only tier is always driven by oracle presence, never by the
|
||||
// declared list, so a declared build-only class with a missing oracle still
|
||||
// counts as latency-only rather than silently passing on exit 0.
|
||||
buildOnly := map[string]bool{}
|
||||
for _, c := range set.BuildOnlyClasses {
|
||||
buildOnly[strings.TrimSpace(c)] = true
|
||||
}
|
||||
|
||||
result := TurnBenchResult{
|
||||
SchemaVersion: TurnSchemaVersion,
|
||||
Suite: strings.TrimSpace(set.ID),
|
||||
@@ -223,9 +279,33 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe
|
||||
}
|
||||
}
|
||||
result.TasksAttempted++
|
||||
if passedForTask {
|
||||
result.TasksPassed++
|
||||
classPassed[class]++
|
||||
|
||||
// Classify the task into an oracle tier and update only that tier's
|
||||
// counters. A latency-only task (no verificationCommand) is never counted
|
||||
// in any pass rate even when the runner reports Passed — an exit-0
|
||||
// read-only run proves the turn ran, not that the answer was right.
|
||||
hasOracle := len(task.VerificationCommand) > 0
|
||||
switch {
|
||||
case !hasOracle:
|
||||
result.LatencyOnlyTasks++
|
||||
classLatencyOnly[class]++
|
||||
latencyOnlyClasses[class] = true
|
||||
case buildOnly[class]:
|
||||
result.BuildCheckedTasks++
|
||||
classVerified[class]++
|
||||
buildOnlyClasses[class] = true
|
||||
if passedForTask {
|
||||
result.BuildPassedTasks++
|
||||
classPassed[class]++
|
||||
}
|
||||
default:
|
||||
result.TasksVerified++
|
||||
classVerified[class]++
|
||||
correctnessClasses[class] = true
|
||||
if passedForTask {
|
||||
result.TasksPassed++
|
||||
classPassed[class]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,16 +320,44 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe
|
||||
wallStats = SummarizeSamples(walls)
|
||||
}
|
||||
result.PerClass[class] = ClassSummary{
|
||||
Tasks: classTasks[class],
|
||||
Passed: classPassed[class],
|
||||
WallMs: wallStats,
|
||||
SpanTotals: classSpanTotals[class],
|
||||
Tasks: classTasks[class],
|
||||
Verified: classVerified[class],
|
||||
Passed: classPassed[class],
|
||||
LatencyOnly: classLatencyOnly[class],
|
||||
WallMs: wallStats,
|
||||
SpanTotals: classSpanTotals[class],
|
||||
}
|
||||
}
|
||||
result.CorrectnessClasses = sortedSetKeys(correctnessClasses)
|
||||
result.BuildOnlyClasses = sortedSetKeys(buildOnlyClasses)
|
||||
result.LatencyOnlyClasses = sortedSetKeys(latencyOnlyClasses)
|
||||
result.CorrectnessPassRate = passRate(result.TasksPassed, result.TasksVerified)
|
||||
result.BuildPassRate = passRate(result.BuildPassedTasks, result.BuildCheckedTasks)
|
||||
result.Totals = totals
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// passRate is passed/total rounded to the benchmark's metric precision, or 0
|
||||
// when the denominator is zero (no tasks in that tier were run).
|
||||
func passRate(passed, total int) float64 {
|
||||
if total <= 0 {
|
||||
return 0
|
||||
}
|
||||
return RoundMetric(float64(passed) / float64(total))
|
||||
}
|
||||
|
||||
func sortedSetKeys(set map[string]bool) []string {
|
||||
if len(set) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func summarizeSpan(samples []float64) SpanStats {
|
||||
if len(samples) == 0 {
|
||||
return SpanStats{}
|
||||
@@ -315,7 +423,15 @@ func FormatTurnBenchSummary(result TurnBenchResult) string {
|
||||
lines := []string{
|
||||
"Zero turn benchmark: " + displayOrUnknown(result.Suite),
|
||||
"model: " + displayOrUnknown(result.Model),
|
||||
fmt.Sprintf("tasks: %d/%d passed across %d iteration(s)", result.TasksPassed, result.TasksAttempted, result.Iterations),
|
||||
// The headline separates the three oracle tiers so an exit-0 read-only
|
||||
// task can never inflate a "pass rate" that reads as correctness:
|
||||
// correctness (positive oracle, edit/fix), build (non-positive `go build`,
|
||||
// refactor), and latency-only (no oracle, nav/longproc/longctx/parallel).
|
||||
fmt.Sprintf("tasks: %d total | correctness %d/%d (%.0f%%) | build %d/%d (%.0f%%) | latency-only %d | %d iter",
|
||||
result.TasksAttempted,
|
||||
result.TasksPassed, result.TasksVerified, result.CorrectnessPassRate*100,
|
||||
result.BuildPassedTasks, result.BuildCheckedTasks, result.BuildPassRate*100,
|
||||
result.LatencyOnlyTasks, result.Iterations),
|
||||
}
|
||||
if result.Mode != "" {
|
||||
lines = append(lines, "mode: "+result.Mode)
|
||||
@@ -333,11 +449,30 @@ func FormatTurnBenchSummary(result TurnBenchResult) string {
|
||||
for _, class := range sortedClasses(result.PerClass) {
|
||||
summary := result.PerClass[class]
|
||||
median := FormatMetric(summary.WallMs.Median, "ms")
|
||||
lines = append(lines, fmt.Sprintf(" [%s] %d/%d passed, wall median %s", class, summary.Passed, summary.Tasks, median))
|
||||
tier := classTier(result, class)
|
||||
lines = append(lines, fmt.Sprintf(" [%s/%s] %d/%d passed, %d latency-only, wall median %s",
|
||||
class, tier, summary.Passed, summary.Verified, summary.LatencyOnly, median))
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// classTier returns the oracle tier label a class was classified into, so the
|
||||
// per-class roll-up states explicitly which oracle (if any) its "passed" count
|
||||
// is measured against.
|
||||
func classTier(result TurnBenchResult, class string) string {
|
||||
for _, c := range result.BuildOnlyClasses {
|
||||
if c == class {
|
||||
return "build"
|
||||
}
|
||||
}
|
||||
for _, c := range result.LatencyOnlyClasses {
|
||||
if c == class {
|
||||
return "latency"
|
||||
}
|
||||
}
|
||||
return "correctness"
|
||||
}
|
||||
|
||||
func sortedClasses(perClass map[string]ClassSummary) []string {
|
||||
classes := make([]string, 0, len(perClass))
|
||||
for c := range perClass {
|
||||
@@ -435,8 +570,13 @@ func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner {
|
||||
outcome.VerifyErr = strings.TrimSpace(vOutcome.Detail)
|
||||
return outcome
|
||||
}
|
||||
// A positive oracle (grep/test/build) passed: this is the only path
|
||||
// that sets Passed. A task with no verificationCommand is latency-only
|
||||
// (read-only nav/longproc/longctx/parallel): its exit 0 proves the turn
|
||||
// ran, not that the answer was right, so it never reports Passed and the
|
||||
// harness counts it in latencyOnlyTasks rather than any pass rate.
|
||||
outcome.Passed = true
|
||||
}
|
||||
outcome.Passed = true
|
||||
return outcome
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,15 +54,18 @@ func TestRunTurnBenchAggregation(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "fake-suite",
|
||||
Tasks: []BenchTask{
|
||||
{ID: "t1", Class: "nav", Prompt: "p1"},
|
||||
{ID: "t2", Class: "nav", Prompt: "p2"},
|
||||
{ID: "t3", Class: "edit", Prompt: "p3"},
|
||||
{ID: "t1", Class: "nav", Prompt: "p1"}, // latency-only
|
||||
{ID: "t2", Class: "nav", Prompt: "p2"}, // latency-only
|
||||
{ID: "t3", Class: "edit", Prompt: "p3", VerificationCommand: []string{"true"}}, // correctness
|
||||
{ID: "t4", Class: "refactor", Prompt: "p4", VerificationCommand: []string{"true"}}, // build-only
|
||||
},
|
||||
BuildOnlyClasses: []string{"refactor"},
|
||||
}
|
||||
canned := map[string]*trace.TurnTrace{
|
||||
"t1": cannedTrace(100, 10, 1000),
|
||||
"t2": cannedTrace(300, 10, 1000),
|
||||
"t3": cannedTrace(200, 50, 2000),
|
||||
"t4": cannedTrace(150, 20, 1500),
|
||||
}
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
@@ -74,8 +77,33 @@ func TestRunTurnBenchAggregation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
if result.TasksAttempted != 3 || result.TasksPassed != 3 {
|
||||
t.Fatalf("attempted=%d passed=%d, want 3/3", result.TasksAttempted, result.TasksPassed)
|
||||
// 4 tasks attempted; the tier split is 2 latency-only, 1 correctness, 1 build.
|
||||
if result.TasksAttempted != 4 {
|
||||
t.Fatalf("attempted=%d, want 4", result.TasksAttempted)
|
||||
}
|
||||
if result.TasksVerified != 1 || result.TasksPassed != 1 {
|
||||
t.Fatalf("correctness verified=%d passed=%d, want 1/1", result.TasksVerified, result.TasksPassed)
|
||||
}
|
||||
if result.LatencyOnlyTasks != 2 {
|
||||
t.Fatalf("latencyOnly=%d, want 2", result.LatencyOnlyTasks)
|
||||
}
|
||||
if result.BuildCheckedTasks != 1 || result.BuildPassedTasks != 1 {
|
||||
t.Fatalf("build checked=%d passed=%d, want 1/1", result.BuildCheckedTasks, result.BuildPassedTasks)
|
||||
}
|
||||
if result.CorrectnessPassRate != 1.0 {
|
||||
t.Fatalf("correctnessPassRate=%v, want 1.0", result.CorrectnessPassRate)
|
||||
}
|
||||
if result.BuildPassRate != 1.0 {
|
||||
t.Fatalf("buildPassRate=%v, want 1.0", result.BuildPassRate)
|
||||
}
|
||||
if len(result.CorrectnessClasses) != 1 || result.CorrectnessClasses[0] != "edit" {
|
||||
t.Fatalf("correctnessClasses=%v, want [edit]", result.CorrectnessClasses)
|
||||
}
|
||||
if len(result.BuildOnlyClasses) != 1 || result.BuildOnlyClasses[0] != "refactor" {
|
||||
t.Fatalf("buildOnlyClasses=%v, want [refactor]", result.BuildOnlyClasses)
|
||||
}
|
||||
if len(result.LatencyOnlyClasses) != 1 || result.LatencyOnlyClasses[0] != "nav" {
|
||||
t.Fatalf("latencyOnlyClasses=%v, want [nav]", result.LatencyOnlyClasses)
|
||||
}
|
||||
if result.SchemaVersion != TurnSchemaVersion {
|
||||
t.Fatalf("schemaVersion = %d, want %d", result.SchemaVersion, TurnSchemaVersion)
|
||||
@@ -84,21 +112,21 @@ func TestRunTurnBenchAggregation(t *testing.T) {
|
||||
t.Fatalf("date = %q", result.Date)
|
||||
}
|
||||
|
||||
// Per-span: generation appears in all three (100+300+200=600ms), tool in all
|
||||
// three (10+10+50=70ms). Count must equal the number of tasks * iterations.
|
||||
// Per-span: generation appears in all four (100+300+200+150=750ms), tool in
|
||||
// all four (10+10+50+20=90ms). Count must equal the number of tasks * iterations.
|
||||
gen := result.PerSpan[trace.SpanGeneration]
|
||||
if gen.Count != 3 {
|
||||
t.Fatalf("generation count = %d, want 3", gen.Count)
|
||||
if gen.Count != 4 {
|
||||
t.Fatalf("generation count = %d, want 4", gen.Count)
|
||||
}
|
||||
if gen.TotalMs != 600 {
|
||||
t.Fatalf("generation totalMs = %v, want 600", gen.TotalMs)
|
||||
if gen.TotalMs != 750 {
|
||||
t.Fatalf("generation totalMs = %v, want 750", gen.TotalMs)
|
||||
}
|
||||
tool := result.PerSpan[trace.SpanToolExecution]
|
||||
if tool.TotalMs != 70 {
|
||||
t.Fatalf("tool totalMs = %v, want 70", tool.TotalMs)
|
||||
if tool.TotalMs != 90 {
|
||||
t.Fatalf("tool totalMs = %v, want 90", tool.TotalMs)
|
||||
}
|
||||
|
||||
// Top latency: generation (600) ranks above tool (70). Exactly two spans
|
||||
// Top latency: generation (750) ranks above tool (90). Exactly two spans
|
||||
// here, so both appear and generation is first.
|
||||
if len(result.TopLatency) != 2 || result.TopLatency[0].Span != trace.SpanGeneration {
|
||||
t.Fatalf("topLatency = %+v", result.TopLatency)
|
||||
@@ -107,29 +135,70 @@ func TestRunTurnBenchAggregation(t *testing.T) {
|
||||
t.Fatalf("top latency not ranked by share: %+v", result.TopLatency)
|
||||
}
|
||||
|
||||
// Totals: 3 model requests, 3 tool calls, input tokens 1000+1000+2000=4000.
|
||||
if result.Totals.ModelRequests != 3 {
|
||||
t.Fatalf("modelRequests = %d, want 3", result.Totals.ModelRequests)
|
||||
// Totals: 4 model requests, 4 tool calls, input tokens 1000+1000+2000+1500=5500.
|
||||
if result.Totals.ModelRequests != 4 {
|
||||
t.Fatalf("modelRequests = %d, want 4", result.Totals.ModelRequests)
|
||||
}
|
||||
if result.Totals.ToolCalls != 3 {
|
||||
t.Fatalf("toolCalls = %d, want 3", result.Totals.ToolCalls)
|
||||
if result.Totals.ToolCalls != 4 {
|
||||
t.Fatalf("toolCalls = %d, want 4", result.Totals.ToolCalls)
|
||||
}
|
||||
if result.Totals.InputTokens != 4000 {
|
||||
t.Fatalf("inputTokens = %d, want 4000", result.Totals.InputTokens)
|
||||
if result.Totals.InputTokens != 5500 {
|
||||
t.Fatalf("inputTokens = %d, want 5500", result.Totals.InputTokens)
|
||||
}
|
||||
if result.Totals.OutputTokens != 2000 {
|
||||
t.Fatalf("outputTokens = %d, want 2000", result.Totals.OutputTokens)
|
||||
if result.Totals.OutputTokens != 2750 {
|
||||
t.Fatalf("outputTokens = %d, want 2750", result.Totals.OutputTokens)
|
||||
}
|
||||
|
||||
// Per-class: nav has two tasks, edit has one; both fully passed.
|
||||
// Per-class tier roll-up: nav is latency-only (0 verified, 2 latency-only),
|
||||
// edit is correctness (1/1 verified passed), refactor is build (1/1 passed).
|
||||
nav := result.PerClass["nav"]
|
||||
if nav.Tasks != 2 || nav.Passed != 2 {
|
||||
if nav.Tasks != 2 || nav.Verified != 0 || nav.Passed != 0 || nav.LatencyOnly != 2 {
|
||||
t.Fatalf("nav class = %+v", nav)
|
||||
}
|
||||
edit := result.PerClass["edit"]
|
||||
if edit.Tasks != 1 || edit.Passed != 1 {
|
||||
if edit.Tasks != 1 || edit.Verified != 1 || edit.Passed != 1 || edit.LatencyOnly != 0 {
|
||||
t.Fatalf("edit class = %+v", edit)
|
||||
}
|
||||
refactor := result.PerClass["refactor"]
|
||||
if refactor.Tasks != 1 || refactor.Verified != 1 || refactor.Passed != 1 || refactor.LatencyOnly != 0 {
|
||||
t.Fatalf("refactor class = %+v", refactor)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunTurnBenchLatencyOnlyNeverPassed asserts the honesty gate: a task with
|
||||
// no verificationCommand reports Passed=true from the (stub) runner, yet the
|
||||
// harness counts it ONLY in latencyOnlyTasks — never in tasksPassed or any pass
|
||||
// rate — so an exit-0 read-only run cannot inflate a correctness number.
|
||||
func TestRunTurnBenchLatencyOnlyNeverPassed(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "lo-suite",
|
||||
Tasks: []BenchTask{
|
||||
{ID: "n1", Class: "nav", Prompt: "p"}, // no oracle — runner still says Passed=true
|
||||
},
|
||||
}
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Runner: fakeTurnRunner(map[string]*trace.TurnTrace{"n1": cannedTrace(50, 5, 100)}),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
}
|
||||
result, err := RunTurnBench(context.Background(), set, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
if result.TasksPassed != 0 || result.TasksVerified != 0 {
|
||||
t.Fatalf("latency-only leaked into pass: passed=%d verified=%d, want 0/0",
|
||||
result.TasksPassed, result.TasksVerified)
|
||||
}
|
||||
if result.CorrectnessPassRate != 0 || result.BuildPassRate != 0 {
|
||||
t.Fatalf("pass rates should be 0 with no oracle tasks: c=%v b=%v",
|
||||
result.CorrectnessPassRate, result.BuildPassRate)
|
||||
}
|
||||
if result.LatencyOnlyTasks != 1 {
|
||||
t.Fatalf("latencyOnlyTasks=%d, want 1", result.LatencyOnlyTasks)
|
||||
}
|
||||
if result.PerClass["nav"].Passed != 0 || result.PerClass["nav"].LatencyOnly != 1 {
|
||||
t.Fatalf("nav class = %+v, want passed=0 latencyOnly=1", result.PerClass["nav"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTurnBenchIterationsAggregates(t *testing.T) {
|
||||
@@ -196,7 +265,9 @@ func TestTopLatencySourcesRanksByTotalAndCapsTopN(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestWriteTurnBenchJSONRoundTrip(t *testing.T) {
|
||||
set := TaskSet{ID: "json-suite", Tasks: []BenchTask{{ID: "t1", Class: "nav", Prompt: "p1"}}}
|
||||
set := TaskSet{ID: "json-suite", Tasks: []BenchTask{
|
||||
{ID: "t1", Class: "edit", Prompt: "p1", VerificationCommand: []string{"true"}},
|
||||
}}
|
||||
canned := map[string]*trace.TurnTrace{"t1": cannedTrace(150, 20, 800)}
|
||||
result, err := RunTurnBench(context.Background(), set, TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
@@ -214,8 +285,15 @@ func TestWriteTurnBenchJSONRoundTrip(t *testing.T) {
|
||||
if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v\n%s", err, buf.String())
|
||||
}
|
||||
if decoded.SchemaVersion != TurnSchemaVersion || decoded.TasksPassed != 1 {
|
||||
t.Fatalf("decoded = %+v", decoded)
|
||||
if decoded.SchemaVersion != TurnSchemaVersion {
|
||||
t.Fatalf("schemaVersion = %d, want %d", decoded.SchemaVersion, TurnSchemaVersion)
|
||||
}
|
||||
if decoded.TasksVerified != 1 || decoded.TasksPassed != 1 || decoded.LatencyOnlyTasks != 0 {
|
||||
t.Fatalf("decoded tier counts = verified=%d passed=%d latency=%d, want 1/1/0",
|
||||
decoded.TasksVerified, decoded.TasksPassed, decoded.LatencyOnlyTasks)
|
||||
}
|
||||
if decoded.CorrectnessPassRate != 1.0 {
|
||||
t.Fatalf("decoded correctnessPassRate = %v, want 1.0", decoded.CorrectnessPassRate)
|
||||
}
|
||||
if decoded.PerSpan[trace.SpanGeneration].TotalMs != 150 {
|
||||
t.Fatalf("decoded generation totalMs = %v, want 150", decoded.PerSpan[trace.SpanGeneration].TotalMs)
|
||||
|
||||
Reference in new issue
Block a user