feat(bench): account for what each context-retrieval run costs

The harness measures context cost and could not report its own. It handed the
agent event.Discard, so the usage every billed completion already reports went
nowhere, and a batch's spend could only be read from a vendor console: this run's
index batch cost 604,544 prompt tokens (527,232 cached) plus 12,600 completion
tokens over 24 runs, and nothing in the tree would have said so.

The bench now installs a sink and totals what the agent reports — the same record
the product's own round path, compaction, command routing and failure paths bill
to. One path, not a second count of the same fact.

Two things stay honest about what the number means. A dry run prints an empty
suffix rather than "prompt-tok=0", because an unmeasured cost must not read as a
measured one. And the count is called usage_reported_requests: a request that
failed before reporting usage leaves no event, so it is not a count of attempts.

benchmarks/context-retrieval/contextbench.go

@@ -89,6 +89,15 @@ type contextMetrics struct {
 	// resident prefix; this is what is paid dynamically instead.
 	RecallReturnedTokens int `json:"recall_returned_tokens"`
 
+	// Provider usage as reported on the stream, or empty for a dry run. The count
+	// is completions that reported usage, so it is not a count of attempts.
+	UsageReportedRequests int  `json:"usage_reported_requests,omitempty"`
+	PromptTokens          int  `json:"prompt_tokens,omitempty"`
+	CompletionTokens      int  `json:"completion_tokens,omitempty"`
+	CacheHitTokens        int  `json:"cache_hit_tokens,omitempty"`
+	CacheMissTokens       int  `json:"cache_miss_tokens,omitempty"`
+	UsageEstimated        bool `json:"usage_estimated,omitempty"`
+
 	Searches []searchAttempt `json:"searches,omitempty"`
 	// PostHitSearches are queries issued after the target had already been
 	// returned: not a query problem, a result-to-read problem.

benchmarks/context-retrieval/run.go

@@ -14,7 +14,6 @@ import (
 
 	"reasonix/internal/assembly/boot"
 	"reasonix/internal/contract/ablation"
-	"reasonix/internal/contract/event"
 	"reasonix/internal/contract/provider"
 	"reasonix/internal/contract/tool"
 	_ "reasonix/internal/model/openai"
@@ -132,10 +131,11 @@ func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root
 	}
 	boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
 
+	var usage usageSink
 	a := agent.New(p, reg, sess, agent.Options{
 		ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
 		SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
-	}, event.Discard)
+	}, &usage)
 	a.LoadProjectionSidecar(f.Path)
 	// From here the transcript lives only in memory. On disk it is the answer.
 	sealFixture(f.Path)
@@ -161,6 +161,10 @@ func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root
 	appended := sess.Snapshot()[before:]
 	m := scoreRun(appended, inst, f.Target, armName, cueVisible)
 	m.trajectory = appended
+	if u, requests := usage.snapshot(); requests > 0 {
+		m.UsageReportedRequests, m.PromptTokens, m.CompletionTokens = requests, u.PromptTokens, u.CompletionTokens
+		m.CacheHitTokens, m.CacheMissTokens, m.UsageEstimated = u.CacheHitTokens, u.CacheMissTokens, u.Estimated
+	}
 	m.scoreAnswer(finalAnswer(appended), inst)
 	if runErr != nil {
 		m.FailureStage = "RunError"
@@ -249,9 +253,9 @@ func runBoundaries(root string) int {
 			} else {
 				noCueSide = append(noCueSide, m)
 			}
-			fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d escape=%d\n",
+			fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d%s escape=%d\n",
 				t.ID, name, side.has, m.FailureStage, m.SearchCalls, m.ReadCalls,
-				m.CueDirectRead, m.RecallReturnedTokens, m.EscapeCalls)
+				m.CueDirectRead, m.RecallReturnedTokens, promptTokenSuffix(m), m.EscapeCalls)
 		}
 	}
 	reportDeltas(cueSide, noCueSide)
@@ -340,9 +344,9 @@ func runExperiment(experiment, root string, dry bool, tasks []contextTask) int {
 			}
 			byArm[arm.name] = append(byArm[arm.name], m)
 			all = append(all, m)
-			fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d %v\n",
+			fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d%s %v\n",
 				t.ID, arm.name, m.FailureStage, m.SearchCalls, m.TargetSearchHits, m.ReadCalls,
-				m.RecallReturnedTokens, m.UnexpectedWorkTools)
+				m.RecallReturnedTokens, promptTokenSuffix(m), m.UnexpectedWorkTools)
 		}
 	}
 	reportFunnels(byArm)

benchmarks/context-retrieval/usage.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"sync"
+
+	"reasonix/internal/contract/event"
+	"reasonix/internal/contract/provider"
+)
+
+// usageSink reads what a run cost from the events the agent already emits. Every
+// billed completion reports event.Usage — the round path, compaction, command
+// routing and failure diagnosis all do — so the bench reads the same record the
+// product reads instead of wrapping a provider and counting a second time.
+type usageSink struct {
+	mu       sync.Mutex
+	requests int
+	usage    provider.Usage
+}
+
+func (s *usageSink) Emit(e event.Event) {
+	if e.Kind != event.Usage || e.Usage == nil {
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	// One event is one completion that reported usage; RequestCount covers a
+	// completion that merged attempts. A request that failed before reporting
+	// leaves no event, so this is not a count of attempts.
+	if e.Usage.RequestCount > 0 {
+		s.requests += e.Usage.RequestCount
+	} else {
+		s.requests++
+	}
+	u := *e.Usage
+	s.usage.PromptTokens += u.PromptTokens
+	s.usage.CompletionTokens += u.CompletionTokens
+	s.usage.TotalTokens += u.TotalTokens
+	s.usage.CacheHitTokens += u.CacheHitTokens
+	s.usage.CacheMissTokens += u.CacheMissTokens
+	s.usage.ReasoningTokens += u.ReasoningTokens
+	s.usage.Estimated = s.usage.Estimated || u.Estimated
+}
+
+func (s *usageSink) snapshot() (provider.Usage, int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.usage, s.requests
+}
+
+// promptTokenSuffix renders what a run cost only when something reported it. A
+// dry run reports nothing, and a trailing "prompt-tok=0" would read as a
+// measured zero rather than as no measurement.
+func promptTokenSuffix(m contextMetrics) string {
+	if m.UsageReportedRequests == 0 {
+		return ""
+	}
+	return fmt.Sprintf(" prompt-tok=%d", m.PromptTokens)
+}

benchmarks/context-retrieval/usage_test.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"testing"
+
+	"reasonix/internal/contract/event"
+	"reasonix/internal/contract/provider"
+)
+
+// The cost a run reports must be what the agent reported, summed across every
+// completion it billed — including one that merged attempts and one that could
+// only estimate. Any other event kind is not a cost.
+func TestUsageSinkAccumulatesWhatTheAgentReports(t *testing.T) {
+	var sink usageSink
+	sink.Emit(event.Event{}) // a kind that is not Usage
+	sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
+		PromptTokens: 100, CompletionTokens: 10, TotalTokens: 110, CacheHitTokens: 60, CacheMissTokens: 40,
+	}})
+	sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
+		PromptTokens: 7, CompletionTokens: 3, CacheHitTokens: 2, CacheMissTokens: 5, RequestCount: 2, Estimated: true,
+	}})
+	sink.Emit(event.Event{Kind: event.Usage}) // a Usage event with no payload is not a cost
+
+	usage, requests := sink.snapshot()
+	if requests != 3 {
+		t.Fatalf("requests = %d, want 3 (1 + the merged attempt's 2; the empty event adds none)", requests)
+	}
+	if usage.PromptTokens != 107 || usage.CompletionTokens != 13 || usage.TotalTokens != 110 {
+		t.Fatalf("accumulated usage = %+v", usage)
+	}
+	if usage.CacheHitTokens != 62 || usage.CacheMissTokens != 45 {
+		t.Fatalf("cache split = %+v", usage)
+	}
+	if !usage.Estimated {
+		t.Fatal("Estimated was dropped when one completion reported an estimate")
+	}
+}
+
+// A dry run must not claim a cost it never paid, in the JSON or in the line a
+// person reads: an empty suffix, not "prompt-tok=0".
+func TestUsageSinkStaysEmptyWithoutUsage(t *testing.T) {
+	var sink usageSink
+	sink.Emit(event.Event{})
+	usage, requests := sink.snapshot()
+	if requests != 0 || usage.PromptTokens != 0 || usage.CompletionTokens != 0 {
+		t.Fatalf("nothing reported usage, got %+v requests=%d", usage, requests)
+	}
+	if suffix := promptTokenSuffix(contextMetrics{}); suffix != "" {
+		t.Fatalf("a run with no reported usage still printed %q", suffix)
+	}
+	if suffix := promptTokenSuffix(contextMetrics{UsageReportedRequests: 1, PromptTokens: 12}); suffix != " prompt-tok=12" {
+		t.Fatalf("a reported cost printed %q", suffix)
+	}
+}