fix(context-retrieval): stop scoring runs that never produced a transcript

A run that failed before its first model turn was appended as a zero-valued
contextMetrics, so results.jsonl gained an empty row, summarize counted it in
Scored and its stage tables, and batch.json reported it as completed with an
empty task id. Stamp such a run with its task, arm and NoEvidence stage, mark it
failed, and keep it out of every statistic while counting it in the manifest as
failed_runs.

benchmarks/context-retrieval/audit_evidence_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// runBlockedBatch runs one dry index batch where every arm's runOne fails
+// before any model turn: the task's own directory is a regular file, so the
+// fixture cannot be built. In production the same early return happens when the
+// fixture build, the session load, or the answer-on-disk guard fails.
+func runBlockedBatch(t *testing.T) string {
+	t.Helper()
+	tk := indexTasks()[0]
+	root := t.TempDir()
+	if err := os.WriteFile(filepath.Join(root, tk.ID), []byte("not a directory\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if code := runExperiment("run-index", experimentIndex, root, true, []contextTask{tk}); code != 0 {
+		t.Fatalf("runExperiment exit = %d, want 0 (every run failed, but the batch finished)", code)
+	}
+	return root
+}
+
+func readResultLines(t *testing.T, root string) ([]contextMetrics, []string) {
+	t.Helper()
+	raw, err := os.ReadFile(filepath.Join(root, "results.jsonl"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var runs []contextMetrics
+	var lines []string
+	for text := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") {
+		var m contextMetrics
+		if err := json.Unmarshal([]byte(text), &m); err != nil {
+			t.Fatalf("results.jsonl: %v", err)
+		}
+		runs = append(runs, m)
+		lines = append(lines, text)
+	}
+	return runs, lines
+}
+
+// A run that failed before the model turn produced no measurement. Recording it
+// as a zero-valued contextMetrics puts it in the funnel's Scored denominator,
+// in "Where it broke", and in the dose curve as a real zero.
+func TestBatchDoesNotRecordAFailedRunAsAResult(t *testing.T) {
+	root := runBlockedBatch(t)
+	runs, lines := readResultLines(t, root)
+	if len(runs) == 0 {
+		t.Fatal("no runs recorded at all")
+	}
+	for i, m := range runs {
+		if m.Task == "" || m.Arm == "" || m.FailureStage == "" {
+			t.Fatalf("results.jsonl line %d records a run that never ran as a result "+
+				"(task=%q arm=%q failure_stage=%q):\n  %s", i+1, m.Task, m.Arm, m.FailureStage, lines[i])
+		}
+	}
+}
+
+// The manifest is the artefact that says which runs counted, so a failed run
+// must not be added to CompletedRuns or to the task id list.
+func TestBatchManifestDoesNotReportFailedRunsAsCompleted(t *testing.T) {
+	root := runBlockedBatch(t)
+	var manifest batchManifest
+	data, err := os.ReadFile(filepath.Join(root, "batch.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := json.Unmarshal(data, &manifest); err != nil {
+		t.Fatal(err)
+	}
+	for _, id := range manifest.TaskIDs {
+		if id == "" {
+			t.Fatalf("manifest lists an empty task id: task_ids=%q (completed=%d planned=%d)",
+				manifest.TaskIDs, manifest.CompletedRuns, manifest.PlannedRuns)
+		}
+	}
+	if manifest.CompletedRuns == manifest.PlannedRuns {
+		t.Fatalf("manifest reports every planned run completed although none produced evidence: "+
+			"completed=%d planned=%d task_ids=%q", manifest.CompletedRuns, manifest.PlannedRuns, manifest.TaskIDs)
+	}
+}

benchmarks/context-retrieval/batch.go

@@ -41,6 +41,10 @@ type batchManifest struct {
 	Arms          []string `json:"arms"`
 	PlannedRuns   int      `json:"planned_runs"`
 	CompletedRuns int      `json:"completed_runs"`
+	// FailedRuns counts the planned runs that ended before a transcript existed.
+	// They are rows in results.jsonl, never part of CompletedRuns, and the field
+	// is absent when there were none so an intact batch is unchanged.
+	FailedRuns int `json:"failed_runs,omitempty"`
 	// Usage totals are absent when nothing reported any: an unmeasured cost must
 	// not read as a measured zero. UsageReportedRuns stays, because zero there
 	// is a fact about the batch.
@@ -98,9 +102,15 @@ func sortedKeys(set map[string]bool) []string {
 // last, so its digests are of what is already on disk.
 func writeBatchManifest(root string, info batchInfo, all []contextMetrics) error {
 	ids, arms := map[string]bool{}, map[string]bool{}
-	var prompt, completion, cacheHit, cacheMiss, requests, reported int
+	var prompt, completion, cacheHit, cacheMiss, requests, reported, failed int
 	estimated := false
 	for _, m := range all {
+		// A failed run has no task, arm or evidence to contribute: it is counted
+		// as a failure below and is not part of what the batch measured.
+		if m.Failed {
+			failed++
+			continue
+		}
 		ids[m.Task] = true
 		arms[m.Arm] = true
 		if m.UsageReportedRequests > 0 {
@@ -125,7 +135,7 @@ func writeBatchManifest(root string, info batchInfo, all []contextMetrics) error
 		GitCommit:  commit, GitDirty: dirty,
 		Model: model, Window: fixtureWindow, Generations: fixtureGenerations,
 		TaskIDs: sortedKeys(ids), Arms: sortedKeys(arms),
-		PlannedRuns: info.Planned, CompletedRuns: len(all),
+		PlannedRuns: info.Planned, CompletedRuns: len(all) - failed, FailedRuns: failed,
 		UsageReportedRuns: reported, UsageReportedRequests: requests,
 		PromptTokens: prompt, CompletionTokens: completion,
 		CacheHitTokens: cacheHit, CacheMissTokens: cacheMiss, UsageEstimated: estimated,

benchmarks/context-retrieval/contextbench.go

@@ -18,7 +18,8 @@ import (
 // canonical position, and never the model's prose. "I will search the history"
 // is not a search; a recall call with a query is.
 
-// failure stages, in funnel order. Exactly one is reported per run.
+// failure stages, in funnel order. Exactly one is reported per run, except for
+// NoEvidence: that stage belongs to a run with no measurement at all.
 const (
 	stageReadMissed   = "NoSearch"           // never asked, and never read the target either
 	stageSearchMiss   = "SearchMiss"         // searched, target never came back
@@ -26,6 +27,7 @@ const (
 	stageAnswerWrong  = "ReadButAnswerWrong" // opened it and still answered wrong
 	stageRecovered    = "Recovered"
 	stageContaminated = "Contaminated"
+	stageNoEvidence   = "NoEvidence"
 
 	// Routing is judged by model round, not by the linear order of tool events:
 	// two calls in one round are one parallel fan-out, and calling that
@@ -116,6 +118,12 @@ type contextMetrics struct {
 	FailureStage  string `json:"failure_stage"`
 	TimeoutReason string `json:"timeout_reason,omitempty"`
 
+	// Failed marks a run that ended before a transcript existed: it has no score
+	// and no place in the statistics. The row stays in results.jsonl with its
+	// task, arm and stage, so a failure is counted rather than dropped.
+	Failed bool   `json:"failed,omitempty"`
+	Reason string `json:"reason,omitempty"`
+
 	// UnexpectedWorkTools are investigation tools used instead of recall.
 	// Reaching for the workspace is behaviour, not noise: how a model routes
 	// its uncertainty when memory is missing is part of what is measured.
@@ -462,6 +470,12 @@ type funnel struct {
 func summarize(arm string, runs []contextMetrics) funnel {
 	f := funnel{Arm: arm, Runs: len(runs), Stages: map[string]int{}, Routes: map[string]int{}}
 	for _, r := range runs {
+		// A run that never produced a transcript is not a scored run and has no
+		// stage to place: counting it would put an empty name in the funnel, the
+		// stage table and the routes.
+		if r.Failed {
+			continue
+		}
 		if r.Contaminated {
 			f.Contaminated++
 			f.Stages[r.FailureStage]++
@@ -631,7 +645,7 @@ func reportStopping(all []contextMetrics) string {
 	var extraRounds, extraTokens []float64
 	sufficient, answeredNext := 0, 0
 	for _, m := range all {
-		if m.Contaminated {
+		if m.Failed || m.Contaminated {
 			continue
 		}
 		classes[m.StoppingClass]++

benchmarks/context-retrieval/dose.go

@@ -121,7 +121,7 @@ var doseMetrics = []struct {
 func reportDoseResponse(all []contextMetrics) string {
 	valid := map[string]map[string]contextMetrics{}
 	for _, m := range all {
-		if m.Contaminated || m.FailureStage == "RunError" {
+		if m.Failed || m.Contaminated || m.FailureStage == "RunError" {
 			continue
 		}
 		scale := strings.TrimPrefix(m.Arm, "index-")

benchmarks/context-retrieval/failed_batch_test.go

@@ -0,0 +1,306 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// blockTask makes one task fail before its first model turn: its own directory
+// is a regular file, so buildFixture cannot create the session under it. It is
+// the same early exit as a failed fixture build, session load, or
+// answer-on-disk guard, and it needs no provider.
+func blockTask(t *testing.T, root, id string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(root, id), []byte("not a directory\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// mixedTaskPair is one task whose runs all fail and one whose runs all succeed.
+// Both are drawn from the corpus, so the batch is the real pipeline.
+func mixedTaskPair() (blocked, ok contextTask) {
+	blocked, ok = indexTasks()[0], indexTasks()[1]
+	for _, t := range indexTasks() {
+		if t.Role == roleEfficiency && len(armsFor(t)) == 2 {
+			ok = t
+			break
+		}
+	}
+	return blocked, ok
+}
+
+// runBlockedExperiment runs one dry index batch over the given tasks, with the
+// named task failing before any model turn, and returns the batch directory, the
+// runs, the manifest, and everything the run printed.
+func runBlockedExperiment(t *testing.T, tasks []contextTask, blocked ...string) (string, []contextMetrics, batchManifest, string) {
+	t.Helper()
+	root := t.TempDir()
+	for _, id := range blocked {
+		blockTask(t, root, id)
+	}
+	out := captureStdout(t, func() {
+		if code := runExperiment("run-index", experimentIndex, root, true, tasks); code != 0 {
+			t.Errorf("runExperiment exit = %d, want 0 (the batch finished)", code)
+		}
+	})
+	runs, _ := readResultLines(t, root)
+	manifest, _ := readManifest(t, root)
+	return root, runs, manifest, out
+}
+
+// captureStdout reads everything the function prints. os.Pipe is unbuffered, so
+// the reader must run beside the call.
+func captureStdout(t *testing.T, f func()) string {
+	t.Helper()
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	old := os.Stdout
+	os.Stdout = w
+	done := make(chan string, 1)
+	go func() {
+		var b strings.Builder
+		buf := make([]byte, 4096)
+		for {
+			n, err := r.Read(buf)
+			b.Write(buf[:n])
+			if err != nil {
+				break
+			}
+		}
+		done <- b.String()
+	}()
+	f()
+	os.Stdout = old
+	w.Close()
+	text := <-done
+	r.Close()
+	return text
+}
+
+// funnelScored reads the "runs=N" column of the named arm's funnel line out of
+// the printed tables, so the assertion is on the number a reader sees.
+func funnelScored(t *testing.T, out, arm string) int {
+	t.Helper()
+	for line := range strings.SplitSeq(out, "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 2 || fields[0] != arm || !strings.HasPrefix(fields[1], "runs=") {
+			continue
+		}
+		n, err := strconv.Atoi(strings.TrimPrefix(fields[1], "runs="))
+		if err != nil {
+			t.Fatalf("funnel line for %s: %v\n%s", arm, err, line)
+		}
+		return n
+	}
+	t.Fatalf("no funnel line for arm %s:\n%s", arm, out)
+	return 0
+}
+
+// The batch report states both counts on one line, so a reader of the scrollback
+// sees the scored set and the failures without counting funnel rows.
+func reportCounts(t *testing.T, out string) (scored, failed int) {
+	t.Helper()
+	for line := range strings.SplitSeq(out, "\n") {
+		if !strings.Contains(line, " scored=") || !strings.Contains(line, " failed=") {
+			continue
+		}
+		for f := range strings.FieldsSeq(line) {
+			switch {
+			case strings.HasPrefix(f, "scored="):
+				scored, _ = strconv.Atoi(strings.TrimPrefix(f, "scored="))
+			case strings.HasPrefix(f, "failed="):
+				failed, _ = strconv.Atoi(strings.TrimPrefix(f, "failed="))
+			}
+		}
+		return scored, failed
+	}
+	t.Fatalf("no line reports scored= and failed=:\n%s", out)
+	return 0, 0
+}
+
+// A batch that mixes a task that never ran with one that did must read as its
+// successful runs only: the failed task leaves Scored, the funnel, and the
+// completed count, and is reported as a failure with its identity instead.
+// Every assertion here is a number, not the absence of an empty row.
+func TestMixedBatchCountsOnlyTheRunThatProducedEvidence(t *testing.T) {
+	blocked, ok := mixedTaskPair()
+	_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked, ok}, blocked.ID)
+
+	wantScored, wantFailed := len(armsFor(ok)), len(armsFor(blocked))
+	if manifest.CompletedRuns != wantScored || manifest.FailedRuns != wantFailed {
+		t.Fatalf("manifest completed=%d failed=%d, want %d and %d (planned=%d)",
+			manifest.CompletedRuns, manifest.FailedRuns, wantScored, wantFailed, manifest.PlannedRuns)
+	}
+	if len(runs) != manifest.CompletedRuns+manifest.FailedRuns {
+		t.Fatalf("results.jsonl has %d rows but completed=%d failed=%d",
+			len(runs), manifest.CompletedRuns, manifest.FailedRuns)
+	}
+	for _, id := range manifest.TaskIDs {
+		if id == blocked.ID {
+			t.Fatalf("task_ids lists %s, which produced no evidence: %v", blocked.ID, manifest.TaskIDs)
+		}
+	}
+	scored, failures := 0, 0
+	for _, m := range runs {
+		if m.Failed {
+			failures++
+			if m.Task != blocked.ID || m.FailureStage == "" || m.Reason == "" {
+				t.Fatalf("failed row lost its identity or stage: %+v", m)
+			}
+			continue
+		}
+		scored++
+		if m.Task != ok.ID {
+			t.Fatalf("a run scored from the wrong task: %+v", m)
+		}
+	}
+	if scored != wantScored || failures != wantFailed {
+		t.Fatalf("rows scored=%d failed=%d, want %d and %d", scored, failures, wantScored, wantFailed)
+	}
+	// Each arm's funnel holds exactly the successful task's runs, and the two
+	// scored counts agree with each other and with the manifest.
+	total := 0
+	for _, arm := range armsFor(ok) {
+		total += funnelScored(t, out, arm.name)
+	}
+	if total != manifest.CompletedRuns {
+		t.Fatalf("funnels scored %d runs, manifest completed_runs = %d:\n%s", total, manifest.CompletedRuns, out)
+	}
+	if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != manifest.FailedRuns {
+		t.Fatalf("report says scored=%d failed=%d, manifest says %d and %d",
+			gotScored, gotFailed, manifest.CompletedRuns, manifest.FailedRuns)
+	}
+	if strings.Contains(out, "## Where it broke") && strings.Contains(out, "=-\n") {
+		t.Fatalf("the stage table kept an empty stage:\n%s", out)
+	}
+}
+
+// A batch where every run fails must not read as an empty success: zero scored
+// runs, a nonzero failed count, and both stated.
+func TestAllFailedBatchReportsZeroScoredAndNonzeroFailed(t *testing.T) {
+	blocked, _ := mixedTaskPair()
+	_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked}, blocked.ID)
+
+	if manifest.CompletedRuns != 0 {
+		t.Fatalf("completed_runs = %d, want 0 (nothing produced evidence)", manifest.CompletedRuns)
+	}
+	if manifest.FailedRuns != manifest.PlannedRuns || manifest.FailedRuns == 0 {
+		t.Fatalf("failed_runs = %d, planned_runs = %d: every planned run failed",
+			manifest.FailedRuns, manifest.PlannedRuns)
+	}
+	if len(manifest.TaskIDs) != 0 {
+		t.Fatalf("task_ids = %v, want none", manifest.TaskIDs)
+	}
+	for i, m := range runs {
+		if !m.Failed || m.FailureStage == "" || m.Task != blocked.ID {
+			t.Fatalf("results.jsonl line %d is not a marked failure: %+v", i+1, m)
+		}
+	}
+	total := 0
+	for _, arm := range armsFor(blocked) {
+		total += funnelScored(t, out, arm.name)
+	}
+	if total != 0 {
+		t.Fatalf("funnels scored %d runs although nothing ran:\n%s", total, out)
+	}
+	if gotScored, gotFailed := reportCounts(t, out); gotScored != 0 || gotFailed != manifest.FailedRuns {
+		t.Fatalf("report says scored=%d failed=%d, want 0 and %d", gotScored, gotFailed, manifest.FailedRuns)
+	}
+}
+
+// The control: with nothing blocked, a batch is what it always was. Every run
+// is completed with a stage, two identical batches write byte-identical results
+// and the same manifest header, and the new failure field stays out of it.
+func TestAllSuccessfulBatchKeepsItsArtifacts(t *testing.T) {
+	tasks := indexTasks()
+	first, runs, manifest, out := runBlockedExperiment(t, tasks)
+	second, again, repeat, _ := runBlockedExperiment(t, tasks)
+	if manifest.CompletedRuns != len(runs) || manifest.FailedRuns != 0 {
+		t.Fatalf("completed=%d failed=%d rows=%d, want every row completed",
+			manifest.CompletedRuns, manifest.FailedRuns, len(runs))
+	}
+	if manifest.CompletedRuns != manifest.PlannedRuns {
+		t.Fatalf("completed=%d planned=%d, want all planned", manifest.CompletedRuns, manifest.PlannedRuns)
+	}
+	for i, m := range runs {
+		if m.Failed || m.Task == "" || m.Arm == "" || m.FailureStage == "" {
+			t.Fatalf("results.jsonl line %d is not a completed run: %+v", i+1, m)
+		}
+	}
+	if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != 0 {
+		t.Fatalf("report says scored=%d failed=%d, want %d and 0", gotScored, gotFailed, manifest.CompletedRuns)
+	}
+	if !reflect.DeepEqual(runs, again) {
+		t.Fatal("two identical successful batches scored differently")
+	}
+	// Byte-for-byte on the runs. Trajectories carry per-message wall-clock
+	// stamps, so they are compared field by field instead.
+	if got, want := readFile(t, second, "results.jsonl"), readFile(t, first, "results.jsonl"); got != want {
+		t.Fatalf("an unchanged batch wrote different results.jsonl:\n%s\n---\n%s", want, got)
+	}
+	gotTraj, wantTraj := withoutStamps(t, readFile(t, second, "trajectories.jsonl")),
+		withoutStamps(t, readFile(t, first, "trajectories.jsonl"))
+	if gotTraj != wantTraj {
+		t.Fatalf("an unchanged batch wrote different trajectories:\n%s\n---\n%s", wantTraj, gotTraj)
+	}
+	repeat.StartedAt, repeat.FinishedAt = manifest.StartedAt, manifest.FinishedAt
+	repeat.TrajectoriesSHA256 = manifest.TrajectoriesSHA256
+	if !reflect.DeepEqual(repeat, manifest) {
+		t.Fatalf("an unchanged batch wrote a different manifest:\n%+v\n---\n%+v", manifest, repeat)
+	}
+	if repeat.FailedRuns != 0 {
+		t.Fatalf("a batch with no failure reports failed_runs=%d", repeat.FailedRuns)
+	}
+}
+
+// withoutStamps drops the createdAt stamp every message carries, which is the
+// one field a dry batch cannot reproduce between two runs.
+func withoutStamps(t *testing.T, text string) string {
+	t.Helper()
+	var b strings.Builder
+	for line := range strings.SplitSeq(strings.TrimSpace(text), "\n") {
+		var row map[string]any
+		if err := json.Unmarshal([]byte(line), &row); err != nil {
+			t.Fatal(err)
+		}
+		stripStamp(row)
+		out, err := json.Marshal(row)
+		if err != nil {
+			t.Fatal(err)
+		}
+		b.Write(out)
+		b.WriteByte('\n')
+	}
+	return b.String()
+}
+
+func stripStamp(v any) {
+	switch typed := v.(type) {
+	case map[string]any:
+		delete(typed, "createdAt")
+		delete(typed, "workDurationMs")
+		for _, child := range typed {
+			stripStamp(child)
+		}
+	case []any:
+		for _, child := range typed {
+			stripStamp(child)
+		}
+	}
+}
+
+func readFile(t *testing.T, root, name string) string {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join(root, name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(data)
+}

benchmarks/context-retrieval/reformulation.go

@@ -130,7 +130,7 @@ type queryAudit struct {
 func auditQueries(runs []contextMetrics) queryAudit {
 	a := queryAudit{Reformulations: map[string]int{}, MissedProbeTerms: map[string]int{}}
 	for _, r := range runs {
-		if r.Contaminated {
+		if r.Failed || r.Contaminated {
 			continue
 		}
 		a.Runs++

benchmarks/context-retrieval/run.go

@@ -85,6 +85,7 @@ func runAdversarial(root string) int {
 			adversarialPrompts[t.Experiment])
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s: %v\n", id, err)
+			m = failedMetrics(t, "adversarial", err)
 		}
 		verdict := "isolated"
 		if m.EscapeFoundAnswer {
@@ -173,6 +174,16 @@ func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root
 	return m, runErr
 }
 
+// failedMetrics is what a run that never reached a transcript leaves behind. It
+// carries the run's identity and the stage that ended it, so the failure stays a
+// countable fact; Failed keeps it out of every statistic.
+func failedMetrics(t contextTask, armName string, err error) contextMetrics {
+	return contextMetrics{
+		Task: t.ID, Arm: armName, Failed: true,
+		FailureStage: stageNoEvidence, Reason: err.Error(),
+	}
+}
+
 // finalAnswer is the last assistant text the turn produced.
 func finalAnswer(msgs []provider.Message) string {
 	for _, m := range slices.Backward(msgs) {
@@ -250,6 +261,7 @@ func runBoundaries(mode, root string) int {
 			m, err := runOne(p, t, armFor(side.scale, false), name, root, side.has, false)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, name, err)
+				m = failedMetrics(t, name, err)
 			}
 			all = append(all, m)
 			if side.has {
@@ -349,6 +361,7 @@ func runExperiment(mode, experiment, root string, dry bool, tasks []contextTask)
 			m, err := runOne(p, t, armFor(arm.scale, arm.searchOff), arm.name, root, cueVisible, dry)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, arm.name, err)
+				m = failedMetrics(t, arm.name, err)
 			}
 			byArm[arm.name] = append(byArm[arm.name], m)
 			all = append(all, m)
@@ -430,11 +443,17 @@ func writeResults(root string, info batchInfo, all []contextMetrics) int {
 		return 2
 	}
 	defer f.Close()
+	scored, failed := 0, 0
 	for _, m := range all {
 		if err := writeJSONLine(f, m); err != nil {
 			fmt.Fprintln(os.Stderr, err)
 			return 2
 		}
+		if m.Failed {
+			failed++
+		} else {
+			scored++
+		}
 	}
 	if err := writeTrajectories(root, all); err != nil {
 		fmt.Fprintln(os.Stderr, "write trajectories:", err)
@@ -444,7 +463,10 @@ func writeResults(root string, info batchInfo, all []contextMetrics) int {
 		fmt.Fprintln(os.Stderr, "write batch manifest:", err)
 		return 2
 	}
-	fmt.Printf("\n%d runs written to %s (with trajectories.jsonl and batch.json)\n", len(all), path)
+	// Both counts are named: a reader who sees only the batch size cannot tell
+	// measurements from attempts.
+	fmt.Printf("\n%d runs written to %s: scored=%d failed=%d (with trajectories.jsonl and batch.json)\n",
+		len(all), path, scored, failed)
 	return 0
 }