feat(bench): persist what a context-retrieval batch was beside its results

A batch printed its own provenance to the terminal and kept the per-run records
on disk, so the two could not be read together afterwards. The fingerprint
(model, endpoint, window, generations, commit, start time) existed only in
scrollback, and every aggregate the report is read for — funnels, dose response,
pooled deltas — was printed rather than stored. Reconstructing a batch after the
fact, or attributing a figure in the README to the runs that produced it, meant
trusting a terminal transcript; the 24-run and 36-run totals in the last batch
are exactly that failure.

Each paying mode now writes batch.json beside results.jsonl: mode, start and
finish, the checkout's commit and dirty state, model, window, generations, the
task ids and arms the batch covered, planned against completed runs, the summed
usage of the runs that reported any, and the sha256 of both artifacts so a
manifest and the records it describes cannot drift apart.

Two things stay honest. Usage totals are absent when nothing reported any,
because an unmeasured cost must not read as a measured zero, while
usage_reported_runs is written even when it is zero, because that is a fact. And
a checkout git cannot describe is recorded as an unknown commit with no dirty
flag at all, rather than as a clean tree.

benchmarks/context-retrieval/batch.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+// batchInfo is what the mode itself knows before any run happens. Everything
+// else in the manifest is derived from the runs that were written, so a figure
+// in it cannot drift from the artifact beside it.
+type batchInfo struct {
+	Mode    string
+	Planned int
+	Started time.Time
+	Dry     bool
+	// Repo is the checkout the batch ran from, captured before the harness
+	// isolates its workspace: git cannot describe a directory that is gone.
+	Repo string
+}
+
+// batchManifest is written beside results.jsonl so a later reader does not have
+// to reconstruct what the batch was from a terminal scrollback.
+type batchManifest struct {
+	SchemaVersion int      `json:"schema_version"`
+	Mode          string   `json:"mode"`
+	StartedAt     string   `json:"started_at"`
+	FinishedAt    string   `json:"finished_at"`
+	GitCommit     string   `json:"git_commit"`
+	GitDirty      *bool    `json:"git_dirty,omitempty"`
+	Model         string   `json:"model"`
+	Window        int      `json:"window"`
+	Generations   int      `json:"generations"`
+	TaskIDs       []string `json:"task_ids"`
+	Arms          []string `json:"arms"`
+	PlannedRuns   int      `json:"planned_runs"`
+	CompletedRuns int      `json:"completed_runs"`
+	// Usage totals are absent when nothing reported any: an unmeasured cost must
+	// not read as a measured zero. UsageReportedRuns stays, because zero there
+	// is a fact about the batch.
+	UsageReportedRuns     int    `json:"usage_reported_runs"`
+	UsageReportedRequests int    `json:"usage_reported_requests,omitempty"`
+	PromptTokens          int    `json:"prompt_tokens,omitempty"`
+	CompletionTokens      int    `json:"completion_tokens,omitempty"`
+	CacheHitTokens        int    `json:"cache_hit_tokens,omitempty"`
+	CacheMissTokens       int    `json:"cache_miss_tokens,omitempty"`
+	UsageEstimated        bool   `json:"usage_estimated,omitempty"`
+	ResultsSHA256         string `json:"results_sha256"`
+	TrajectoriesSHA256    string `json:"trajectories_sha256"`
+}
+
+// gitCommitAndDirty reports the checkout the batch ran from. Unknown is written
+// rather than omitted, so an absent field is never mistaken for a clean tree.
+func gitCommitAndDirty(repo string) (commit string, dirty *bool) {
+	commit = "unknown"
+	if repo == "" {
+		return commit, nil
+	}
+	in := func(args ...string) ([]byte, error) {
+		cmd := exec.Command("git", args...)
+		cmd.Dir = repo
+		return cmd.Output()
+	}
+	if out, err := in("rev-parse", "--short", "HEAD"); err == nil {
+		commit = strings.TrimSpace(string(out))
+	}
+	if out, err := in("status", "--porcelain"); err == nil {
+		state := strings.TrimSpace(string(out)) != ""
+		dirty = &state
+	}
+	return commit, dirty
+}
+
+func fileSHA256(path string) (string, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%x", sha256.Sum256(data)), nil
+}
+
+func sortedKeys(set map[string]bool) []string {
+	out := make([]string, 0, len(set))
+	for k := range set {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// writeBatchManifest records the batch beside the runs it describes. It runs
+// last, so its digests are of what is already on disk.
+func writeBatchManifest(root string, info batchInfo, all []contextMetrics) error {
+	ids, arms := map[string]bool{}, map[string]bool{}
+	var prompt, completion, cacheHit, cacheMiss, requests, reported int
+	estimated := false
+	for _, m := range all {
+		ids[m.Task] = true
+		arms[m.Arm] = true
+		if m.UsageReportedRequests > 0 {
+			reported++
+			requests += m.UsageReportedRequests
+			prompt += m.PromptTokens
+			completion += m.CompletionTokens
+			cacheHit += m.CacheHitTokens
+			cacheMiss += m.CacheMissTokens
+			estimated = estimated || m.UsageEstimated
+		}
+	}
+	commit, dirty := gitCommitAndDirty(info.Repo)
+	model := runModel
+	if info.Dry {
+		model = "scripted"
+	}
+	manifest := batchManifest{
+		SchemaVersion: 1, Mode: info.Mode,
+		StartedAt:  info.Started.UTC().Format(time.RFC3339),
+		FinishedAt: time.Now().UTC().Format(time.RFC3339),
+		GitCommit:  commit, GitDirty: dirty,
+		Model: model, Window: fixtureWindow, Generations: fixtureGenerations,
+		TaskIDs: sortedKeys(ids), Arms: sortedKeys(arms),
+		PlannedRuns: info.Planned, CompletedRuns: len(all),
+		UsageReportedRuns: reported, UsageReportedRequests: requests,
+		PromptTokens: prompt, CompletionTokens: completion,
+		CacheHitTokens: cacheHit, CacheMissTokens: cacheMiss, UsageEstimated: estimated,
+	}
+	var err error
+	if manifest.ResultsSHA256, err = fileSHA256(filepath.Join(root, "results.jsonl")); err != nil {
+		return err
+	}
+	if manifest.TrajectoriesSHA256, err = fileSHA256(filepath.Join(root, "trajectories.jsonl")); err != nil {
+		return err
+	}
+	data, err := json.MarshalIndent(manifest, "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(root, "batch.json"), append(data, '\n'), 0o644)
+}

benchmarks/context-retrieval/batch_test.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func seedArtifacts(t *testing.T, root string) {
+	t.Helper()
+	for _, name := range []string{"results.jsonl", "trajectories.jsonl"} {
+		if err := os.WriteFile(filepath.Join(root, name), []byte("{}\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func readManifest(t *testing.T, root string) (batchManifest, string) {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join(root, "batch.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got batchManifest
+	if err := json.Unmarshal(data, &got); err != nil {
+		t.Fatal(err)
+	}
+	return got, string(data)
+}
+
+func TestBatchManifestRecordsWhatTheBatchWas(t *testing.T) {
+	root := t.TempDir()
+	seedArtifacts(t, root)
+	all := []contextMetrics{
+		{Task: "b", Arm: "index-off"},
+		{Task: "a", Arm: "index-off"},
+		{Task: "a", Arm: "index-half", UsageReportedRequests: 2, PromptTokens: 100,
+			CompletionTokens: 10, CacheHitTokens: 60, CacheMissTokens: 40},
+	}
+	if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 4, Started: time.Now().Add(-time.Minute)}, all); err != nil {
+		t.Fatal(err)
+	}
+	got, _ := readManifest(t, root)
+	if got.SchemaVersion != 1 || got.Mode != "run-index" || got.PlannedRuns != 4 || got.CompletedRuns != 3 {
+		t.Fatalf("manifest header = %+v", got)
+	}
+	if strings.Join(got.TaskIDs, ",") != "a,b" || strings.Join(got.Arms, ",") != "index-half,index-off" {
+		t.Fatalf("ids/arms = %v / %v, want sorted and unique", got.TaskIDs, got.Arms)
+	}
+	if got.UsageReportedRuns != 1 || got.UsageReportedRequests != 2 || got.PromptTokens != 100 ||
+		got.CompletionTokens != 10 || got.CacheHitTokens != 60 || got.CacheMissTokens != 40 {
+		t.Fatalf("usage totals = %+v", got)
+	}
+	// No repo means git could not describe the checkout: unknown commit, and no
+	// dirty flag at all, because "clean" is a claim this batch cannot make.
+	if got.GitCommit != "unknown" || got.GitDirty != nil {
+		t.Fatalf("git state = %q / %v, want unknown and no dirty flag", got.GitCommit, got.GitDirty)
+	}
+	want := fmt.Sprintf("%x", sha256.Sum256([]byte("{}\n")))
+	if got.ResultsSHA256 != want || got.TrajectoriesSHA256 != want {
+		t.Fatalf("digests = %q / %q, want %q", got.ResultsSHA256, got.TrajectoriesSHA256, want)
+	}
+}
+
+// A dry batch pays nothing, so its cost fields must be absent rather than zero:
+// an unmeasured spend must not read as a measured one.
+func TestBatchManifestLeavesUnreportedUsageAbsent(t *testing.T) {
+	root := t.TempDir()
+	seedArtifacts(t, root)
+	all := []contextMetrics{{Task: "a", Arm: "index-off"}, {Task: "a", Arm: "index-half"}}
+	if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 2, Started: time.Now(), Dry: true}, all); err != nil {
+		t.Fatal(err)
+	}
+	got, raw := readManifest(t, root)
+	if got.UsageReportedRuns != 0 {
+		t.Fatalf("usage_reported_runs = %d, want 0 reported", got.UsageReportedRuns)
+	}
+	if got.Model != "scripted" {
+		t.Fatalf("dry batch model = %q, want scripted", got.Model)
+	}
+	for _, absent := range []string{"usage_reported_requests", "prompt_tokens", "completion_tokens", "cache_hit_tokens"} {
+		if strings.Contains(raw, absent) {
+			t.Fatalf("a dry batch wrote %q:\n%s", absent, raw)
+		}
+	}
+}

benchmarks/context-retrieval/main.go

@@ -54,9 +54,9 @@ func main() {
 	case "preflight":
 		os.Exit(runPreflight(tasks, root))
 	case "run-search":
-		os.Exit(runExperiment(experimentSearch, root, *dry, tasks))
+		os.Exit(runExperiment(*mode, experimentSearch, root, *dry, tasks))
 	case "run-boundary":
-		os.Exit(runBoundaries(root))
+		os.Exit(runBoundaries(*mode, root))
 	case "snippet":
 		os.Exit(runSnippetAudit(root, 100))
 	case "adversarial":
@@ -69,7 +69,7 @@ func main() {
 			// stays measurable on its own.
 			batch = efficiencyTasks()
 		}
-		os.Exit(runExperiment(experimentIndex, root, *dry, batch))
+		os.Exit(runExperiment(*mode, experimentIndex, root, *dry, batch))
 	default:
 		fmt.Fprintf(os.Stderr, "unknown mode %q\n", *mode)
 		os.Exit(2)

benchmarks/context-retrieval/run.go

@@ -216,7 +216,10 @@ func isolateWorkspace() (func(), error) {
 
 // The two arms get separate nonces: reusing one would leave the first arm's
 // answer in a host-readable history for the second to find.
-func runBoundaries(root string) int {
+func runBoundaries(mode, root string) int {
+	started := time.Now()
+	planned := 0
+	repo, _ := os.Getwd()
 	root, err := filepath.Abs(root)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
@@ -243,6 +246,7 @@ func runBoundaries(root string) int {
 			has   bool
 		}{{cue, true}, {noCue, false}} {
 			name := "index-" + side.scale
+			planned++
 			m, err := runOne(p, t, armFor(side.scale, false), name, root, side.has, false)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, name, err)
@@ -260,7 +264,7 @@ func runBoundaries(root string) int {
 	}
 	reportDeltas(cueSide, noCueSide)
 	fmt.Print(reportStopping(all))
-	return writeResults(root, all)
+	return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Repo: repo}, all)
 }
 
 // reportDeltas is the paired reading: each task against itself, then pooled.
@@ -302,7 +306,10 @@ func deltaBool(a, b bool) string {
 // runExperiment runs one experiment's tasks across its arms. A dry run drives
 // the same pipeline with a scripted provider: it proves fixture, run, scoring
 // and report hold together before any of it is paid for.
-func runExperiment(experiment, root string, dry bool, tasks []contextTask) int {
+func runExperiment(mode, experiment, root string, dry bool, tasks []contextTask) int {
+	started := time.Now()
+	planned := 0
+	repo, _ := os.Getwd()
 	root, err := filepath.Abs(root)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
@@ -338,6 +345,7 @@ func runExperiment(experiment, root string, dry bool, tasks []contextTask) int {
 		// test: a bad minute would land entirely on one arm.
 		for _, arm := range shuffledArms(t) {
 			cueVisible := t.Experiment == experimentIndex && cueExpectedVisible(t, arm.scale)
+			planned++
 			m, err := runOne(p, t, armFor(arm.scale, arm.searchOff), arm.name, root, cueVisible, dry)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, arm.name, err)
@@ -359,7 +367,7 @@ func runExperiment(experiment, root string, dry bool, tasks []contextTask) int {
 	if experiment == experimentIndex {
 		reportBoundaries(all)
 	}
-	return writeResults(root, all)
+	return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Dry: dry, Repo: repo}, all)
 }
 
 func cueExpectedVisible(t contextTask, scale string) bool {
@@ -414,7 +422,7 @@ func writeTrajectories(root string, all []contextMetrics) error {
 	return nil
 }
 
-func writeResults(root string, all []contextMetrics) int {
+func writeResults(root string, info batchInfo, all []contextMetrics) int {
 	path := filepath.Join(root, "results.jsonl")
 	f, err := os.Create(path)
 	if err != nil {
@@ -432,7 +440,11 @@ func writeResults(root string, all []contextMetrics) int {
 		fmt.Fprintln(os.Stderr, "write trajectories:", err)
 		return 2
 	}
-	fmt.Printf("\n%d runs written to %s (with trajectories.jsonl)\n", len(all), path)
+	if err := writeBatchManifest(root, info, all); err != nil {
+		fmt.Fprintln(os.Stderr, "write batch manifest:", err)
+		return 2
+	}
+	fmt.Printf("\n%d runs written to %s (with trajectories.jsonl and batch.json)\n", len(all), path)
 	return 0
 }