test(bench): preflight the request the model is handed, not the projection

The fixtures were checked against the model-visible projection. That is what
the agent believes; a provider.Request is what the model is told, and
rendering, sanitization and tool-schema construction sit in between. So each of
the 36 arm states now resumes the way a restart does — transcript from disk,
projection from its sidecar, the shipped tool surface at that arm — runs one
round against a capture provider, and is judged on the request it produced.

The capture answers with a clean terminal response. An empty one would buy an
empty-final retry and a second request, and the round under inspection is the
first. The turn's request is the one carrying tools: a fold on the way in asks
the same provider for a digest, and taking request[0] blindly inspected the
summarizer's prompt instead.

The scan enumerates the provider-visible surface rather than marshalling a
Message whole — content, reasoning_content, tool-call names and arguments,
responses_items, and every tool name, description and parameter schema — so a
leak is reported where it is, and local-only fields the boundary is supposed to
strip cannot produce a false hit. Those are asserted absent in their own right:
raw_content, LocalOnly, decision receipts, tool execution and memory citations
crossing the boundary is a regression whether or not a marker rides along.

It found what the projection check could not. i05's answer marker "session"
was readable in tools[0].parameters and tools[2].parameters — in all four arms,
because a tool schema is not part of any projection. r01's "partial" was one
ordinary word away from the same fate. Both are nonces now.

Search arms are held to the ablation's definition: both keep positions, only
the control has query, and the search-off arm's request may not contain the
wording that offers a search — checked over the whole surface, not just the
schema, so a description or an index line cannot reintroduce it.

Getting a resumed fixture to keep its projection turned up one more thing worth
recording: projectionCoversTail cannot revalidate a projection that covers the
entire transcript, since it has neither a matching transcript version nor a
live tail to sit in front of. Real sessions always have one, because work
continues after a fold. The fixtures now do too.

Verified by making each fail: stop scanning tool schemas and i05's leak goes
quiet; leave the search tool unswapped and the search-off arm is named twice,
once for the parameter and once for the wording.

benchmarks/* join cmd/ and desktop/ as hosts in the layering rule. Each is a
main that assembles a real agent, and a harness reproducing the shipped tool
surface has to reach the frontend that defines it. Nothing is opened to the
kernel by this: the rule that matters is that nothing below a frontend may
import one, and a benchmark is above every frontend.

No provider is called anywhere in this commit. This is the gate that says the
API spend would measure what it claims to.

.gitignore

@@ -7,6 +7,7 @@
 /reasonix.exe
 /e2ebench
 /e2ebench.exe
+/context-retrieval
 
 # Website (Astro) build output
 /site/dist/

benchmarks/context-retrieval/capture.go

@@ -0,0 +1,170 @@
+// capture.go — what the agent actually hands a provider.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"reasonix/internal/ablation"
+	"reasonix/internal/agent"
+	"reasonix/internal/boot"
+	"reasonix/internal/event"
+	"reasonix/internal/provider"
+	"reasonix/internal/tool"
+	_ "reasonix/internal/tool/builtin"
+)
+
+// A projection is what the agent believes; a provider.Request is what the model
+// is told. Rendering, sanitization and tool-schema construction sit between
+// them, so the arm's contract is checked on the request.
+
+type captureProvider struct {
+	requests []provider.Request
+}
+
+func (p *captureProvider) Name() string { return "contextbench-capture" }
+
+// Stream records the request and answers with a clean terminal response. An
+// empty answer would buy an empty-final retry and a second request, which is
+// not the one round this is here to inspect.
+func (p *captureProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
+	p.requests = append(p.requests, cloneRequest(req))
+	ch := make(chan provider.Chunk, 2)
+	ch <- provider.Chunk{Type: provider.ChunkText, Text: "contextbench preflight complete"}
+	ch <- provider.Chunk{Type: provider.ChunkDone}
+	close(ch)
+	return ch, nil
+}
+
+func cloneRequest(req provider.Request) provider.Request {
+	out := req
+	out.Messages = append([]provider.Message(nil), req.Messages...)
+	out.Tools = append([]provider.ToolSchema(nil), req.Tools...)
+	return out
+}
+
+// captureRequest rebuilds the session the way a resume does — transcript from
+// disk, projection from its sidecar, the shipped tool surface at this arm —
+// and returns the request the first round hands the provider.
+func captureRequest(f builtFixture, arm ablation.Set, prompt string) (provider.Request, error) {
+	sess, err := agent.LoadSession(f.Path)
+	if err != nil {
+		return provider.Request{}, fmt.Errorf("load session: %w", err)
+	}
+	reg := tool.NewRegistry()
+	for _, t := range tool.Builtins() {
+		reg.Add(t)
+	}
+	boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
+
+	cap := &captureProvider{}
+	a := agent.New(cap, reg, sess, agent.Options{
+		ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
+		SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
+	}, event.Discard)
+	a.LoadProjectionSidecar(f.Path)
+
+	if err := a.Run(context.Background(), prompt); err != nil {
+		return provider.Request{}, fmt.Errorf("run: %w", err)
+	}
+	// The turn's own request is the one carrying tools. A fold on the way in
+	// asks the same provider for a digest, and that request has none — taking
+	// the first request blindly would inspect the summarizer's prompt.
+	for _, req := range cap.requests {
+		if len(req.Tools) > 0 {
+			return req, nil
+		}
+	}
+	return provider.Request{}, fmt.Errorf("no request carried tools (%d captured)", len(cap.requests))
+}
+
+// surfaceField is one place a model could read a string, named so a leak
+// report says where rather than only that.
+type surfaceField struct {
+	Path string
+	Text string
+}
+
+// providerSurface enumerates every provider-visible string in a request.
+// Enumerated rather than marshalled whole: a Message also carries local-only
+// fields, and scanning those would fail on bytes no model ever sees.
+func providerSurface(req provider.Request) []surfaceField {
+	var out []surfaceField
+	add := func(path, text string) {
+		if strings.TrimSpace(text) != "" {
+			out = append(out, surfaceField{Path: path, Text: text})
+		}
+	}
+	for i, m := range req.Messages {
+		add(fmt.Sprintf("messages[%d].content", i), m.Content)
+		add(fmt.Sprintf("messages[%d].reasoning_content", i), m.ReasoningContent)
+		add(fmt.Sprintf("messages[%d].name", i), m.Name)
+		for j, tc := range m.ToolCalls {
+			add(fmt.Sprintf("messages[%d].tool_calls[%d].name", i, j), tc.Name)
+			add(fmt.Sprintf("messages[%d].tool_calls[%d].arguments", i, j), tc.Arguments)
+		}
+		for j, item := range m.ResponsesItems {
+			add(fmt.Sprintf("messages[%d].responses_items[%d]", i, j), string(item))
+		}
+	}
+	for i, t := range req.Tools {
+		add(fmt.Sprintf("tools[%d].name", i), t.Name)
+		add(fmt.Sprintf("tools[%d].description", i), t.Description)
+		if len(t.Parameters) > 0 {
+			b, err := json.Marshal(t.Parameters)
+			if err == nil {
+				add(fmt.Sprintf("tools[%d].parameters", i), string(b))
+			}
+		}
+	}
+	return out
+}
+
+// findInSurface reports every place a marker is readable.
+func findInSurface(req provider.Request, marker string) []string {
+	var hits []string
+	for _, f := range providerSurface(req) {
+		if strings.Contains(f.Text, marker) {
+			hits = append(hits, f.Path)
+		}
+	}
+	return hits
+}
+
+// recallToolSchema returns the recall tool as the provider sees it.
+func recallToolSchema(req provider.Request) (provider.ToolSchema, bool) {
+	for _, t := range req.Tools {
+		if t.Name == "recall" {
+			return t, true
+		}
+	}
+	return provider.ToolSchema{}, false
+}
+
+// localOnlyLeaks reports host-side fields that should never survive the
+// provider boundary. A hit here is a boundary regression, marker or no marker.
+func localOnlyLeaks(req provider.Request) []string {
+	var out []string
+	for i, m := range req.Messages {
+		switch {
+		case m.RawContent != "":
+			out = append(out, fmt.Sprintf("messages[%d].raw_content", i))
+		case m.LocalOnly:
+			out = append(out, fmt.Sprintf("messages[%d].local_only", i))
+		case m.DecisionReceipt != nil || len(m.DecisionReceipts) > 0:
+			out = append(out, fmt.Sprintf("messages[%d].decision_receipt", i))
+		case m.ToolExecution != nil:
+			out = append(out, fmt.Sprintf("messages[%d].tool_execution", i))
+		case len(m.MemoryCitations) > 0:
+			out = append(out, fmt.Sprintf("messages[%d].memory_citations", i))
+		}
+	}
+	return out
+}
+
+// searchWording is what a search-off arm must never be told. The arm exists to
+// reproduce the runtime before search; an arm offered a capability it does not
+// have is a different experiment.
+var searchWording = []string{"recall with a query", "search the whole folded region", "query still finds"}

benchmarks/context-retrieval/corpus.go

@@ -176,12 +176,12 @@ func indexTasks() []contextTask {
 		{
 			ID: "i05-cache-probe", Experiment: experimentIndex, TargetKind: "tool_input",
 			Prompt:        "cache probe 当时记录的 prefix salt 和 scope 是什么？",
-			AnswerMarkers: []string{"comet-42", "session"},
+			AnswerMarkers: []string{"comet-42", "per-lineage"},
 			ProbeQuery:    "cache probe prefix salt scope",
 			CueMarker:     "cache-probe", CueTier: tierDefault,
 			PlantAfterGen: 0,
 			Plant: plantCall("cb1", "read_file", `{"path":"scratch/cache-probe.txt"}`,
-				"prefix salt = comet-42\nscope = session"),
+				"prefix salt = comet-42\nscope = per-lineage"),
 		},
 		{
 			ID: "i06-dispatch-handoff", Experiment: experimentIndex, TargetKind: "tool_input",

benchmarks/context-retrieval/fixture.go

@@ -13,8 +13,7 @@ import (
 )
 
 // A fixture is a real session that has really been compacted: no bench-only
-// state shape, so the loader, the projection restore and the arm all have to
-// work on it or the experiment measures something the runtime never makes.
+// state shape, so the loader and the projection restore have to work on it.
 
 const (
 	fixtureWindow      = 60000
@@ -97,6 +96,10 @@ func buildFixture(t contextTask, arm ablation.Set, sessionPath string) (builtFix
 	if at < 0 {
 		return builtFixture{}, fmt.Errorf("%s: PlantAfterGen %d is past the last generation", t.ID, t.PlantAfterGen)
 	}
+	// projectionCoversTail cannot revalidate a projection covering the whole
+	// transcript: it needs a live tail, which a real session always has because
+	// work continues after a fold. Neutral text — the tail is provider-visible.
+	sess.Add(provider.Message{Role: provider.RoleAssistant, Content: "Backlog swept; ready for the next question."})
 	if err := sess.Save(sessionPath); err != nil {
 		return builtFixture{}, fmt.Errorf("%s: save session: %w", t.ID, err)
 	}

benchmarks/context-retrieval/preflight.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"slices"
 	"strings"
 
 	"reasonix/internal/ablation"
+	"reasonix/internal/provider"
 	"reasonix/internal/retrieval"
 )
 
@@ -109,10 +111,81 @@ func checkTask(t contextTask, root string) ([]finding, error) {
 		} else if strings.Contains(visibleIndexOnly(f), t.ID) {
 			out = append(out, finding{t.ID, arm.name, "a search task must not be addressed by the fold index"})
 		}
+
+		// The projection is what the agent believes. This is what the model is
+		// told, and it is the one that has to hold.
+		req, err := captureRequest(f, armFor(arm.scale, arm.searchOff), t.Prompt)
+		if err != nil {
+			return nil, fmt.Errorf("%s [%s]: capture: %w", t.ID, arm.name, err)
+		}
+		out = append(out, checkCapturedRequest(t, arm.name, arm.scale, arm.searchOff, req)...)
 	}
 	return out, nil
 }
 
+// checkCapturedRequest holds one arm's real provider request to its contract:
+// the answer unreadable, the cue exactly where its tier says, the recall tool
+// offering only what this arm can do, and no host-side field surviving the
+// boundary.
+func checkCapturedRequest(t contextTask, armName, scale string, searchOff bool, req provider.Request) []finding {
+	var out []finding
+	for _, marker := range t.AnswerMarkers {
+		if len(marker) < 6 {
+			continue // a bare number is not a nonce; the set as a whole still is
+		}
+		for _, at := range findInSurface(req, marker) {
+			out = append(out, finding{t.ID, armName, fmt.Sprintf("answer marker %q leaked at %s", marker, at)})
+		}
+	}
+	if t.Experiment == experimentIndex {
+		wantVisible, wantHidden := tierScales(t.CueTier)
+		where := findInSurface(req, t.CueMarker)
+		switch {
+		case contains(wantVisible, scale) && len(where) == 0:
+			out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is not provider-visible; the tier expects it", t.CueMarker)})
+		case contains(wantHidden, scale) && len(where) > 0:
+			out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is provider-visible at %s: %s", t.CueMarker, where[0], scale)})
+		}
+	}
+
+	schema, ok := recallToolSchema(req)
+	if !ok {
+		out = append(out, finding{t.ID, armName, "recall is not in the provider's tool list"})
+		return append(out, boundaryFindings(t, armName, req)...)
+	}
+	params := ""
+	if b, err := json.Marshal(schema.Parameters); err == nil {
+		params = string(b)
+	}
+	// Both arms keep the read half: the ablation removes search, not recall.
+	if !strings.Contains(params, `"positions"`) {
+		out = append(out, finding{t.ID, armName, "recall lost its positions parameter"})
+	}
+	hasQuery := strings.Contains(params, `"query"`)
+	switch {
+	case searchOff && hasQuery:
+		out = append(out, finding{t.ID, armName, "recall still offers a query parameter"})
+	case !searchOff && !hasQuery:
+		out = append(out, finding{t.ID, armName, "recall has no query parameter"})
+	}
+	if searchOff {
+		for _, phrase := range searchWording {
+			for _, at := range findInSurface(req, phrase) {
+				out = append(out, finding{t.ID, armName, fmt.Sprintf("search is offered to an arm without it: %q at %s", phrase, at)})
+			}
+		}
+	}
+	return append(out, boundaryFindings(t, armName, req)...)
+}
+
+func boundaryFindings(t contextTask, armName string, req provider.Request) []finding {
+	var out []finding
+	for _, at := range localOnlyLeaks(req) {
+		out = append(out, finding{t.ID, armName, "host-only field crossed the provider boundary at " + at})
+	}
+	return out
+}
+
 // checkCueVisibility holds an index task to its tier: the cue is addressed at
 // the scales the matrix names and at no others. A policy change that moves it
 // fails here rather than turning two arms into one experiment.

internal/boot/agent_preset.go

@@ -143,3 +143,10 @@ func applyUnifiedProviderToolSurface(reg *tool.Registry, goalTurnsUnreachable bo
 	}
 	reg.SetProviderVisibleTools(allow)
 }
+
+// ApplyUnifiedProviderToolSurface restricts a registry to what a provider is
+// shown, at this arm. Exported so a harness reproduces the real surface
+// instead of a second copy of this rule.
+func ApplyUnifiedProviderToolSurface(reg *tool.Registry, goalTurnsUnreachable bool, arm ablation.Set) {
+	applyUnifiedProviderToolSurface(reg, goalTurnsUnreachable, arm)
+}

tools/repolint/layers.go

@@ -81,7 +81,14 @@ func violates(pkg, dep string) string {
 	return ""
 }
 
-func host(pkg string) bool { return under(pkg, "cmd") || under(pkg, "desktop") }
+// A host assembles the whole system to run it. benchmarks/* are hosts in that
+// sense — each is a main that builds a real agent — and a harness reproducing
+// the shipped surface has to reach the frontend that defines it. This opens
+// nothing to the kernel: the rule that matters is that nothing below a
+// frontend may import one, and benchmarks are above every frontend.
+func host(pkg string) bool {
+	return under(pkg, "cmd") || under(pkg, "desktop") || under(pkg, "benchmarks")
+}
 
 func under(pkg, root string) bool { return pkg == root || strings.HasPrefix(pkg, root+"/") }