chore(perfbench): never commit generated baseline reports

.gitignore

@@ -55,3 +55,6 @@ docs/superpowers/
 zero-termux-arm64
 zero-linux-sandbox
 zero-seccomp
+
+# Generated benchmark reports are evidence for one configuration, never tree state
+internal/perfbench/reports/*.json

cmd/zero-perf-bench/main_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"strings"
 	"testing"
+
+	"github.com/Gitlawb/zero/internal/perfbench"
 )
 
 func TestParseArgsUsesCliAndEnvOverrides(t *testing.T) {
@@ -92,3 +94,18 @@ func TestHelpText(t *testing.T) {
 func emptyEnv(string) string {
 	return ""
 }
+
+func TestTurnExitCodeFailsWhenEveryTaskErrored(t *testing.T) {
+	var stderr bytes.Buffer
+	allErrored := perfbench.TurnBenchResult{TasksAttempted: 3, TasksErrored: 3}
+	if code := turnExitCode(allErrored, &stderr); code != 1 {
+		t.Fatalf("exit=%d, want 1 when every task errored", code)
+	}
+	if !strings.Contains(stderr.String(), "no accepted benchmark sample") {
+		t.Fatalf("stderr missing explanation: %q", stderr.String())
+	}
+	partial := perfbench.TurnBenchResult{TasksAttempted: 3, TasksErrored: 2}
+	if code := turnExitCode(partial, &stderr); code != 0 {
+		t.Fatalf("exit=%d, want 0 for partial errors (summary surfaces them)", code)
+	}
+}

cmd/zero-perf-bench/turn.go

@@ -86,9 +86,23 @@ func runTurnCommand(args []string, getenv func(string) string, stdout io.Writer,
 			_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
 			return 1
 		}
-		return 0
+		return turnExitCode(result, stderr)
 	}
 	_, _ = fmt.Fprintln(stdout, perfbench.FormatTurnBenchSummary(result))
+	return turnExitCode(result, stderr)
+}
+
+// turnExitCode fails the command when every attempted task errored (no
+// iteration produced an accepted benchmark sample): such a report measures
+// nothing, and exiting 0 would let a broken configuration (missing binary, bad
+// path, failing harness step) pass as a clean baseline. Partial errors keep
+// exit 0 — the summary surfaces them loudly and the surviving samples are
+// still valid measurements.
+func turnExitCode(result perfbench.TurnBenchResult, stderr io.Writer) int {
+	if result.TasksAttempted > 0 && result.TasksErrored == result.TasksAttempted {
+		_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: every task errored with no accepted benchmark sample (see warnings); the report contains no valid measurements")
+		return 1
+	}
 	return 0
 }
 

internal/perfbench/turn_bench.go

@@ -40,7 +40,19 @@ import (
 // prevents a v2→v2 cross-version comparison from misreading the jump as a model
 // improvement. New counts: correctness 34 (edit 10 + fix 8 + nav 10 + refactor
 // 6), build 0, latency 14 (longproc 4 + longctx 4 + parallel 6).
-const TurnSchemaVersion = 3
+//
+// v4 adds tasksErrored: the number of tasks whose every iteration produced no
+// accepted benchmark sample. That covers pre-run failures (missing binary,
+// spawn error, process crash) and harness errors after a successful agent run
+// (e.g. oracle stamping failed) alike — in every case the iteration yields no
+// usable measurement. Errored tasks were previously visible only in warnings,
+// so a run where nothing was actually measured still printed a clean-looking
+// 0% pass summary and exited 0. tasksErrored makes that state first-class:
+// the summary surfaces it, and the turn command exits nonzero when every
+// attempted task errored. Accounting: an errored oracle task stays in its
+// tier denominator as a failure; an errored latency-only task counts in
+// latencyOnlyTasks and, as always, in no pass rate.
+const TurnSchemaVersion = 4
 
 // TurnRunner runs one benchmark task and reports its outcome plus the captured
 // per-turn trace. A non-nil Err means the run failed to execute (process crash);
@@ -144,6 +156,7 @@ type TurnBenchResult struct {
 	Commit              string                  `json:"commit,omitempty"`
 	Date                string                  `json:"date"`
 	TasksAttempted      int                     `json:"tasksAttempted"`
+	TasksErrored        int                     `json:"tasksErrored"`
 	TasksVerified       int                     `json:"tasksVerified"`
 	TasksPassed         int                     `json:"tasksPassed"`
 	LatencyOnlyTasks    int                     `json:"latencyOnlyTasks"`
@@ -249,6 +262,7 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe
 		// runner is cold-start, so a flaky pass on one iteration and a fail on
 		// another is a real regression signal, not noise to average away.
 		passedForTask := true
+		erroredIterations := 0
 		for iter := 0; iter < iterations; iter++ {
 			outcome := cfg.Runner(ctx, task, rc)
 			if outcome.TraceIssue != "" {
@@ -266,6 +280,7 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe
 					Message: fmt.Sprintf("task %s: %v", task.ID, outcome.Err),
 				})
 				passedForTask = false
+				erroredIterations++
 				continue
 			}
 			if !outcome.Passed {
@@ -306,6 +321,14 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe
 		// counters. A latency-only task (no verificationCommand) is never counted
 		// in any pass rate even when the runner reports Passed — an exit-0
 		// read-only run proves the turn ran, not that the answer was right.
+		if erroredIterations == iterations {
+			// Every iteration produced no accepted sample (spawn failure or a
+			// harness error after the run). The task still counts in its tier
+			// below — an oracle task as a failure, a latency-only task in no
+			// pass rate as always — but the errored state is first-class so a
+			// broken run can never print a clean-looking summary.
+			result.TasksErrored++
+		}
 		hasOracle := len(task.VerificationCommand) > 0
 		switch {
 		case !hasOracle:
@@ -456,6 +479,20 @@ func FormatTurnBenchSummary(result TurnBenchResult) string {
 			result.BuildPassedTasks, result.BuildCheckedTasks, result.BuildPassRate*100,
 			result.LatencyOnlyTasks, result.Iterations),
 	}
+	if result.TasksErrored > 0 {
+		lines = append(lines, fmt.Sprintf("ERRORED: %d task(s) produced no accepted benchmark sample (spawn/crash or harness error) — errored oracle tasks count as failures in their tier's pass rate; latency-only tasks stay out of pass rates as always", result.TasksErrored))
+		shown := 0
+		for _, warning := range result.Warnings {
+			if warning.Metric != "run" {
+				continue
+			}
+			lines = append(lines, "  "+warning.Message)
+			shown++
+			if shown == 3 {
+				break
+			}
+		}
+	}
 	if result.Mode != "" {
 		lines = append(lines, "mode: "+result.Mode)
 	}
@@ -747,7 +784,15 @@ func ResolveBinary(explicit string) (string, error) {
 		if _, err := os.Stat(v); err != nil {
 			return "", fmt.Errorf("trace binary not found: %w", err)
 		}
-		return v, nil
+		// Pin the explicit path to an absolute one: the turn runner sets each
+		// child's cmd.Dir to a per-task fixture copy, so a relative path
+		// (./zero, the Makefile default) that stats fine here would fail to
+		// spawn from inside every fixture dir, silently erroring all tasks.
+		absolute, err := filepath.Abs(v)
+		if err != nil {
+			return "", fmt.Errorf("resolve trace binary path: %w", err)
+		}
+		return absolute, nil
 	}
 	if path, err := exec.LookPath("zero"); err == nil {
 		return path, nil

internal/perfbench/turn_bench_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
@@ -530,14 +531,15 @@ func TestRunTurnBenchBuildOnlyMechanism(t *testing.T) {
 	}
 }
 
-// TestTurnSchemaVersion3 pins the schema bump. v3 records the tier
-// reclassification (refactor structural-positive and nav answer-oracles moved
-// into correctnessPassRate), so a v2->v3 cross-version comparison cannot
-// silently misread the jump as a model improvement — exactly the misread the
-// tier system exists to prevent.
-func TestTurnSchemaVersion3(t *testing.T) {
-	if TurnSchemaVersion != 3 {
-		t.Fatalf("TurnSchemaVersion = %d, want 3", TurnSchemaVersion)
+// TestTurnSchemaVersion pins the schema bump so a shape change is a conscious
+// decision. v3 recorded the tier reclassification (refactor structural-positive
+// and nav answer-oracles moved into correctnessPassRate). v4 adds tasksErrored:
+// tasks whose every iteration died before the agent produced a run are now
+// first-class in the report instead of visible only in warnings, so a
+// spawn-broken run cannot print a clean-looking summary or exit 0.
+func TestTurnSchemaVersion(t *testing.T) {
+	if TurnSchemaVersion != 4 {
+		t.Fatalf("TurnSchemaVersion = %d, want 4", TurnSchemaVersion)
 	}
 }
 
@@ -1067,3 +1069,95 @@ printf '%s\n' '{"type":"run_end","exitCode":0}'
 `)
 	assertVerifyFailed(t, "nav-08 fmt-only-no-testing", outcome)
 }
+
+func TestResolveBinaryAbsolutizesExplicitPath(t *testing.T) {
+	dir := t.TempDir()
+	name := "zero-probe.exe"
+	if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chdir(dir); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = os.Chdir(cwd) })
+
+	resolved, err := ResolveBinary("./" + name)
+	if err != nil {
+		t.Fatalf("ResolveBinary: %v", err)
+	}
+	if !filepath.IsAbs(resolved) {
+		t.Fatalf("ResolveBinary returned relative path %q; the turn runner sets cmd.Dir per task, so a relative binary fails to spawn from every fixture copy", resolved)
+	}
+}
+
+func TestRunTurnBenchCountsErroredTasks(t *testing.T) {
+	set := TaskSet{
+		ID: "errored-suite",
+		Tasks: []BenchTask{
+			{ID: "t1", Class: "nav", Prompt: "p1", VerificationCommand: []string{"true"}},
+			{ID: "t2", Class: "longproc", Prompt: "p2"},
+		},
+	}
+	cfg := TurnBenchConfig{
+		Model:      "fake-model",
+		Iterations: 1,
+		Runner: func(context.Context, BenchTask, RunContext) TurnTaskOutcome {
+			return TurnTaskOutcome{Err: errors.New("fork/exec ./zero: file does not exist")}
+		},
+		Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
+	}
+	result, err := RunTurnBench(context.Background(), set, cfg)
+	if err != nil {
+		t.Fatalf("RunTurnBench: %v", err)
+	}
+	if result.TasksErrored != 2 {
+		t.Fatalf("TasksErrored=%d, want 2 (every iteration of both tasks errored)", result.TasksErrored)
+	}
+	if len(result.Warnings) != 2 {
+		t.Fatalf("warnings=%d, want 2 run warnings", len(result.Warnings))
+	}
+	summary := FormatTurnBenchSummary(result)
+	if !strings.Contains(summary, "ERRORED: 2 task(s)") {
+		t.Fatalf("summary does not surface the errored tasks:\n%s", summary)
+	}
+	if !strings.Contains(summary, "fork/exec ./zero") {
+		t.Fatalf("summary does not echo the underlying run error:\n%s", summary)
+	}
+}
+
+func TestRunTurnBenchPartialErrorStillCounts(t *testing.T) {
+	set := TaskSet{
+		ID: "partial-suite",
+		Tasks: []BenchTask{
+			{ID: "ok", Class: "nav", Prompt: "p", VerificationCommand: []string{"true"}},
+			{ID: "dead", Class: "nav", Prompt: "p", VerificationCommand: []string{"true"}},
+		},
+	}
+	canned := map[string]*trace.TurnTrace{"ok": cannedTrace(100, 10, 1000)}
+	inner := fakeTurnRunner(canned)
+	cfg := TurnBenchConfig{
+		Model:      "fake-model",
+		Iterations: 1,
+		Runner: func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome {
+			if task.ID == "dead" {
+				return TurnTaskOutcome{Err: errors.New("spawn failed")}
+			}
+			return inner(ctx, task, rc)
+		},
+		Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
+	}
+	result, err := RunTurnBench(context.Background(), set, cfg)
+	if err != nil {
+		t.Fatalf("RunTurnBench: %v", err)
+	}
+	if result.TasksErrored != 1 {
+		t.Fatalf("TasksErrored=%d, want 1", result.TasksErrored)
+	}
+	if result.TasksAttempted != 2 {
+		t.Fatalf("TasksAttempted=%d, want 2", result.TasksAttempted)
+	}
+}