fix(agent): keep policy reads the recorded failure, not the text prefix (#8155)

KeepErrors only matched tool results starting with "error:" or "blocked:",
the shape the host formats for an execution failure. A tool that ran fine
but reported a failure — a go test FAIL, a compiler error, any non-zero
exit — never matched, so its evidence folded into the digest and the
specific assertion was gone. Measured against a full-history control on
benchmarks/compaction, evidence buried mid-result survived 0 of 6 runs
while the control kept it every time.

provider.Message.ToolExecution already carries State/ExitCode/Verification
and is already stripped before provider requests, so isErrorMessage reads
it instead of guessing. Because the failure then stays identifiable without
its text prefix, a protected failure can also be shortened: snipFailureResult
keeps the failure-carrying lines with two lines of context and drops the
passing noise, holding the projection cost to +3.5% rather than the +100%
that keeping the whole result costs. A text-only failure has no such anchor
and is still carried whole.

The rewrite happens on three paths — the planning pass, the re-write that
follows archiving, and the fold's kept partition — now unified through
maintenanceReplacement/keptForProjection so one cannot silently overwrite
another.

benchmarks/compaction gains the probe that measures this, a -snip arm,
cross-generation planting for revisions that land across a fold, and
alignment with boot's default KeepPolicy. Buried evidence now survives
18 of 18 runs.
This commit is contained in:
YHH authored and GitHub committed 2026-08-10 14:26:36 +08:00
1 parent 96b5d32a76
commit d8d75e462c
14 files changed
+648 -29

No files matched your search

+32 -11
View File
@@ -42,6 +42,7 @@ func main() {
window := flag.Int("window", 128_000, "context window in tokens")
control := flag.Bool("control", true, "fidelity: also score probes against full history")
arm := flag.String("arm", "full", "full | incremental: re-derive each digest from canonical, or fold the previous projection")
snip := flag.Bool("snip", false, "run the stale-tool-result snip pass before each fold, as the 0.6 threshold does in a real session")
out := flag.String("out", "", "write the JSON report here")
flag.Parse()
@@ -49,14 +50,14 @@ func main() {
res []genResult
err error
)
incremental := *arm == "incremental"
a := arms{incremental: *arm == "incremental", snip: *snip}
switch {
case *arm != "full" && *arm != "incremental":
err = fmt.Errorf("unknown arm %q", *arm)
case *mode == "cost":
res, err = runCost(*gens, *window, incremental)
res, err = runCost(*gens, *window, a)
case *mode == "fidelity":
res, err = runFidelity(*gens, *window, *control, incremental)
res, err = runFidelity(*gens, *window, *control, a)
default:
err = fmt.Errorf("unknown mode %q", *mode)
}
@@ -83,6 +84,8 @@ type genResult struct {
SummarizerInput int `json:"summarizer_input_tokens"`
LargestCall int `json:"largest_call_tokens"`
Mode string `json:"mode,omitempty"`
SnippedResults int `json:"snipped_results,omitempty"`
SnippedChars int `json:"snipped_chars,omitempty"`
Seconds float64 `json:"seconds"`
Error string `json:"error,omitempty"`
Survived map[string]int `json:"survived,omitempty"` // probe class -> 1 kept, 0 lost
@@ -97,9 +100,10 @@ type harness struct {
agentA *agent.Agent
path string
calls *callRecorder
snip bool
}
func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorder, incremental bool) *harness {
func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorder, arm arms) *harness {
sess := newSession()
path := filepath.Join(t.dir, "session.jsonl")
a := agent.New(p, tool.NewRegistry(), sess, agent.Options{
@@ -107,13 +111,23 @@ func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorde
ArchiveDir: filepath.Join(t.dir, "archive"),
SessionPath: path,
RecentKeep: 4,
Ablation: foldArm(incremental),
// boot's default when cfg.Agent.Keep is unset; without it the bench
// would measure a configuration no real session runs.
KeepPolicy: agent.KeepErrors,
Ablation: foldArm(arm.incremental),
}, rec.sink())
return &harness{sess: sess, agentA: a, path: path, calls: rec}
return &harness{sess: sess, agentA: a, path: path, calls: rec, snip: arm.snip}
}
// foldArm switches full re-derivation off, which is what makes a fold read the
// previous projection instead of the canonical transcript.
// arms selects the maintenance behaviour under test. Snipping is off by default
// so a run stays comparable with baselines recorded before it existed.
type arms struct {
incremental bool
snip bool
}
func foldArm(incremental bool) ablation.Set {
if incremental {
return ablation.New(ablation.FullFold)
@@ -128,6 +142,13 @@ func (h *harness) runGeneration(ctx context.Context, gen int, probes []probe) ge
h.calls.reset()
start := time.Now()
if h.snip {
st, serr := h.agentA.SnipStaleToolResults()
if serr != nil {
r.Error = serr.Error()
}
r.SnippedResults, r.SnippedChars = st.Results, st.SavedChars
}
err := h.agentA.CompactNow(ctx, "")
r.Seconds = time.Since(start).Seconds()
if err != nil {
@@ -145,7 +166,7 @@ func (h *harness) runGeneration(ctx context.Context, gen int, probes []probe) ge
return r
}
func runCost(gens, window int, incremental bool) ([]genResult, error) {
func runCost(gens, window int, a arms) ([]genResult, error) {
dir, cleanup, err := tempDir()
if err != nil {
return nil, err
@@ -154,7 +175,7 @@ func runCost(gens, window int, incremental bool) ([]genResult, error) {
rec := &callRecorder{}
p := &scriptedProvider{rec: rec, reply: syntheticDigest, window: window}
h := newHarness(dir, p, window, rec, incremental)
h := newHarness(dir, p, window, rec, a)
var out []genResult
for gen := range gens {
@@ -163,7 +184,7 @@ func runCost(gens, window int, incremental bool) ([]genResult, error) {
return out, nil
}
func runFidelity(gens, window int, control, incremental bool) ([]genResult, error) {
func runFidelity(gens, window int, control bool, a arms) ([]genResult, error) {
key := os.Getenv("DEEPSEEK_API_KEY")
if key == "" {
return nil, fmt.Errorf("fidelity mode needs DEEPSEEK_API_KEY")
@@ -179,7 +200,7 @@ func runFidelity(gens, window int, control, incremental bool) ([]genResult, erro
defer cleanup()
rec := &callRecorder{}
h := newHarness(dir, &recordingProvider{inner: p, rec: rec}, window, rec, incremental)
h := newHarness(dir, &recordingProvider{inner: p, rec: rec}, window, rec, a)
probes := probeSuite()
ctx := context.Background()
@@ -193,7 +214,7 @@ func runFidelity(gens, window int, control, incremental bool) ([]genResult, erro
return nil, verr
}
for _, probe := range probes {
if probe.plantAt > gen {
if probe.settledAt() > gen {
continue
}
answer, aerr := ask(ctx, p, visible, probe.question)
+1 -1
View File
@@ -12,7 +12,7 @@ func TestRepeatedFoldsStayBoundedAndKeepSucceeding(t *testing.T) {
reserve = 8192 // summaryOutputReserve: the digest the call must still return
maxCalls = 7 // maxSummarySpans + the merge pass
)
res, err := runCost(gens, window, false)
res, err := runCost(gens, window, arms{})
if err != nil {
t.Fatalf("runCost: %v", err)
}
+121 -4
View File
@@ -8,6 +8,7 @@ import (
"reasonix/internal/agent"
"reasonix/internal/provider"
"reasonix/internal/tool"
)
// probeAnswerContract rides with every probe question. A context full of tool
@@ -33,9 +34,13 @@ func invalidAnswer(s string) bool {
// The question is asked against the compacted context, so a wrong answer means
// compaction lost the fact — not that the model is weak.
type probe struct {
class string
plantAt int
plant func(*agent.Session)
class string
plantAt int
plant func(*agent.Session)
// later plants more of the same fact in later generations, so a decision
// that changes across folds is tested the way it actually happens: each
// revision lands on the far side of a compaction, not next to the last one.
later map[int]func(*agent.Session)
question string
want []string // answer must contain one of these, lowercased
reject []string // ...and none of these: the pre-correction answer
@@ -52,6 +57,24 @@ func toolRound(id, name, args, result string) func(*agent.Session) {
}
}
// failedToolRound is a tool round the host recorded as failed, the way a real
// non-zero bash run arrives. Without the execution record the keep policy sees
// only text, which is exactly the gap the buried-evidence probe measures.
func failedToolRound(id, name, args, result string, code int) func(*agent.Session) {
return func(s *agent.Session) {
exit := code
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: name, Arguments: args}}})
s.Add(provider.Message{
Role: provider.RoleTool, ToolCallID: id, Name: name, Content: result,
ToolExecution: &provider.ToolExecution{
Kind: "shell",
State: tool.ShellStateFailed,
ExitCode: &exit,
},
})
}
}
func seq(fns ...func(*agent.Session)) func(*agent.Session) {
return func(s *agent.Session) {
for _, fn := range fns {
@@ -64,6 +87,50 @@ func seq(fns ...func(*agent.Session)) func(*agent.Session) {
// freshness and correction probes are the ones summaries classically get wrong:
// both have a plausible stale answer that reads as correct.
func probeSuite() []probe {
suite := append(factProbes(), revisionProbes()...)
return append(suite, snipProbes()...)
}
// buriedTestLog is a `go test -v` log whose only failure detail sits in the
// middle. Geometric snipping keeps the first 80 and last 12 lines, so this is
// precisely the shape it drops while reporting a plausible-looking remainder.
// It names a package no other probe touches: a run covering config/ would be a
// truthful "yes" for verification-freshness and score that probe as lost.
func buriedTestLog() string {
var b strings.Builder
pass := func(i int) {
fmt.Fprintf(&b, "=== RUN TestEmitterCase%03d\n--- PASS: TestEmitterCase%03d (0.00s)\n", i, i)
}
for i := range 120 {
pass(i)
}
b.WriteString("=== RUN TestEmitterQuoting\n")
b.WriteString(" emitter_test.go:412: assertion failed: expected beta-7d21, got gamma-4a88\n")
b.WriteString("--- FAIL: TestEmitterQuoting (0.01s)\n")
for i := 120; i < 200; i++ {
pass(i)
}
b.WriteString("FAIL\nexit status 1\nFAIL\treasonix/emitter\t0.234s\n")
return b.String()
}
// snipProbes plant a fact in the middle of a long tool result, out of reach of
// the head/tail geometry. They score the maintenance pass rather than the fold:
// every other probe in the suite is short enough that snipping cannot touch it.
func snipProbes() []probe {
return []probe{
{
class: "buried-evidence",
plantAt: 1,
plant: failedToolRound("bt1", "bash", `{"cmd":"go test ./emitter -v"}`, buriedTestLog(), 1),
question: "In the failed test, what value did the assertion expect? Reply with just the value.",
want: []string{"beta-7d21"},
reject: []string{"gamma-4a88"},
},
}
}
func factProbes() []probe {
return []probe{
{
class: "user-constraint",
@@ -165,6 +232,45 @@ func probeSuite() []probe {
}
}
// revisionProbes state a fact and then change it in a later generation. The
// revision lands on the far side of a fold, so carrying it forward means
// superseding the digest's own earlier claim rather than a neighbouring turn.
func revisionProbes() []probe {
return []probe{
{
class: "late-constraint",
plantAt: 3,
plant: userTurn("New hard rule from here on: never edit anything under internal/store/."),
question: "Are you allowed to edit files under internal/store/? Answer yes or no.",
want: []string{"no"},
reject: []string{"yes"},
},
{
class: "reversal-chain",
plantAt: 0,
plant: userTurn("Use PostgreSQL for the datastore."),
later: map[int]func(*agent.Session){
1: userTurn("Change of plan: drop PostgreSQL, use SQLite instead."),
2: userTurn("Final call: back to PostgreSQL after all."),
},
question: "Which datastore is the current decision? Reply with one word.",
want: []string{"postgresql", "postgres"},
reject: []string{"sqlite"},
},
{
class: "distractor-file",
plantAt: 1,
plant: userTurn("Draft note: the fix might belong in config/legacy_parser.go."),
later: map[int]func(*agent.Session){
3: userTurn("Confirmed: the fix belongs in config/emitter.go; the legacy parser is not involved."),
},
question: "Which file does the fix belong in? Reply with just the filename, no path.",
want: []string{"emitter"},
reject: []string{"legacy"},
},
}
}
// score reports whether an answer keeps the planted fact. A rejected token
// anywhere in the answer counts as lost even when the wanted token also
// appears, so "yes, but it was not re-run" does not pass as "no".
@@ -194,4 +300,15 @@ func matchesToken(lowered, want string) bool {
}), want)
}
func (p probe) String() string { return fmt.Sprintf("%s@gen%d", p.class, p.plantAt) }
// settledAt is the first generation whose answer is the one want/reject scores.
// A probe revised across folds has a different correct answer while the chain is
// still running, so asking before it settles would score a right answer as lost.
func (p probe) settledAt() int {
at := p.plantAt
for gen := range p.later {
at = max(at, gen)
}
return at
}
func (p probe) String() string { return fmt.Sprintf("%s@gen%d", p.class, p.settledAt()) }
+3
View File
@@ -46,6 +46,9 @@ func growSession(sess *agent.Session, gen int, probes []probe) {
if p.plantAt == gen {
p.plant(sess)
}
if revise, ok := p.later[gen]; ok {
revise(sess)
}
}
for i := range unitsPerGeneration {
workUnit(sess, gen, i)
+66
View File
@@ -0,0 +1,66 @@
package main
import (
"strings"
"testing"
)
// The buried-evidence probe only measures something if its planted fact sits
// outside the head/tail window the snip geometry keeps. Guard the probe design
// against a later edit that shortens the log back into reach.
func TestBuriedFactSitsOutsideSnipGeometry(t *testing.T) {
const (
head = 80 // defaultReadOnlySnip.head
tail = 12 // defaultReadOnlySnip.tail
)
lines := strings.Split(buriedTestLog(), "\n")
idx := -1
for i, line := range lines {
if strings.Contains(line, "beta-7d21") {
idx = i
break
}
}
if idx < 0 {
t.Fatal("buried fact absent from the planted log")
}
if idx < head || idx >= len(lines)-tail {
t.Errorf("fact at line %d of %d is inside the kept head(%d)/tail(%d); snipping would preserve it and the probe would measure nothing", idx, len(lines), head, tail)
}
}
// A tool result the host recorded as failed outranks the snip geometry: the
// assertion detail sits where head/tail cannot reach it, so it survives only
// because the keep policy reads the execution record rather than the text.
func TestSnipKeepsRecordedFailure(t *testing.T) {
dir, cleanup, err := tempDir()
if err != nil {
t.Fatal(err)
}
defer cleanup()
rec := &callRecorder{}
const window = 64_000
h := newHarness(dir, &scriptedProvider{rec: rec, reply: syntheticDigest, window: window}, window, rec, arms{snip: true})
for gen := range 3 {
growSession(h.sess, gen, probeSuite())
}
before := renderAll(h.sess.Snapshot())
if !strings.Contains(before, "beta-7d21") {
t.Fatal("planted fact missing before the snip pass")
}
st, err := h.agentA.SnipStaleToolResults()
if err != nil {
t.Fatal(err)
}
if st.Results == 0 {
t.Fatal("snip pass rewrote nothing")
}
visible, err := visibleContext(h.path, h.sess)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(renderAll(visible), "beta-7d21") {
t.Errorf("recorded failure was snipped away (%d results, %d chars); keep policy did not protect it", st.Results, st.SavedChars)
}
}
+12 -6
View File
@@ -275,9 +275,14 @@ Long tasks eventually fill the model's context window. Reasonix manages this wit
limits. `model_overrides.<model>.max_output_tokens` can specialize mixed
gateways; Anthropic still supplies a mandatory `max_tokens` fallback.
- Tool-result snip/prune never removes messages, so assistant `tool_calls` and
tool results stay paired. `KeepErrors` preserves error/blocked tool outputs,
and the recent tail is not rewritten. Snipped results can later be upgraded to
pruned placeholders; already-pruned results are left alone.
tool results stay paired. `KeepErrors` preserves failed tool outputs,
recognised from the recorded execution (state, exit code, verification) or,
when there is none, an `error:`/`blocked:` text prefix. A failure with a
recorded execution keeps its failure-carrying lines rather than the whole
result, since the record still identifies it as a failure after a rewrite; a
text-only failure has no such anchor and is kept whole. The recent tail is not
rewritten. Snipped results can later be upgraded to pruned placeholders;
already-pruned results are left alone.
- Automatic maintenance is planned once before a sampling request from the
current visible projection plus its append-only canonical tail. It never
rewrites the canonical transcript. A failed or non-convergent view fingerprint
@@ -290,7 +295,7 @@ Long tasks eventually fill the model's context window. Reasonix manages this wit
- When summary compaction runs, the fold region (everything between the pinned
prefix and the recent tail) is split three ways: the first few **small user
turns** are hoisted verbatim ahead of the digest, messages the keep policy
protects stay verbatim, and **everything else** — assistant/tool work, later
protects are kept, and **everything else** — assistant/tool work, later
user turns, and any prior digest — is summarized into a single digest, using
the executor's own provider, no tools. The split is a partition: a message in
the region is either kept verbatim or reaches the summarizer, never neither.
@@ -345,8 +350,9 @@ Long tasks eventually fill the model's context window. Reasonix manages this wit
**What survives a fold.** Verbatim, at every compaction: the system prompt, the
first user turn when it is small enough to be a brief, the first few small user
turns of the fold region, the messages the keep policy protects, and the recent
tail. Everything else is **best-effort** — it reaches the summarizer and survives
turns of the fold region, and the recent tail. The messages the keep policy
protects also survive, though a failure with a recorded execution keeps only its
failure-carrying lines. Everything else is **best-effort** — it reaches the summarizer and survives
only as well as the digest captured it. That includes small user turns beyond the
hoisted window, so a durable constraint is safest restated in a recent turn
rather than assumed to hold from turn 4 of a long session.
+4 -3
View File
@@ -164,7 +164,7 @@ tool result 的 snip/prune 不删除消息,确保 assistant `tool_calls` 与 t
原生工具清理不会替代 Reasonix 的摘要折叠,也不会修改 canonical history。
摘要折叠时,固定前缀与近期 tail 之间的区间被划分为三部分:开头若干条小体量用户回合原样提升到
digest 之前,keep policy 保护的消息原样保留,其余全部——assistant/tool 工作、后续用户回合、以及
digest 之前,keep policy 保护的消息予以保留,其余全部——assistant/tool 工作、后续用户回合、以及
已有 digest——折叠进同一条 digest。三者构成一个划分:区间内的消息要么原样保留,要么进入摘要输入,
不存在两者皆非的情况。
@@ -174,8 +174,9 @@ digest 之前,keep policy 保护的消息原样保留,其余全部——assi
被拆成连续几部分分别摘要,再由最后一次调用合并,并对份数设上限,使单次 compaction 的调用数有界;
某一部分被迫丢弃的内容,会写在摘要器读到的文本里。
因此逐字保留的是:system prompt、体量足够小的首个用户回合、折叠区间开头若干条小体量用户回合、
keep policy 保护的消息,以及近期 tail。其余均为尽力而为——只在 digest 抓住它的前提下留存,
因此逐字保留的是:system prompt、体量足够小的首个用户回合、折叠区间开头若干条小体量用户回合,
以及近期 tail。keep policy 保护的消息同样留存,但带有执行记录的失败只保留承载失败的若干行。
其余均为尽力而为——只在 digest 抓住它的前提下留存,
其中包括超出提升窗口的小体量用户回合,所以长期有效的约束更适合在近期回合中重述。
两个性质限制了这种损失:每次折叠都从 canonical transcript 重新生成 digest,而不是在上一条 digest
+20
View File
@@ -18,6 +18,7 @@ import (
"reasonix/internal/ablation"
"reasonix/internal/event"
"reasonix/internal/provider"
"reasonix/internal/tool"
)
// Compaction is a low-frequency cache-reset point: the prompt grows append-only
@@ -366,10 +367,29 @@ func isErrorMessage(m provider.Message) bool {
if m.Role != provider.RoleTool {
return false
}
if failedExecution(m.ToolExecution) {
return true
}
s := strings.TrimSpace(strings.ToLower(m.Content))
return strings.HasPrefix(s, "error:") || strings.HasPrefix(s, "blocked:")
}
// failedExecution reads the failure the host already recorded, rather than
// guessing from the text. A `go test` run that reports FAIL exits non-zero
// while its output starts with "=== RUN", which no prefix match can see.
func failedExecution(ex *provider.ToolExecution) bool {
if ex == nil {
return false
}
if ex.State == tool.ShellStateFailed || ex.State == tool.ShellStateTimedOut {
return true
}
if ex.ExitCode != nil && *ex.ExitCode != 0 {
return true
}
return ex.Verification == tool.ShellVerificationFailed
}
func isUserMarked(m provider.Message) bool {
if m.Role != provider.RoleUser {
return false
+6 -2
View File
@@ -545,7 +545,7 @@ func (a *Agent) partitionFoldForProjection(region []provider.Message) (early, ca
}
fold = append(fold, m)
case policyKeep[i]:
kept = append(kept, m)
kept = append(kept, a.keptForProjection(m))
default:
fold = append(fold, m)
}
@@ -702,7 +702,11 @@ func (a *Agent) installPruneProjection(view []provider.Message, st PruneStats) e
continue
}
m := currentVisible[i]
m.Content = rewriteToolResult(m, st.Mode, archive, a.snipStrategyFor(m.Name))
replacement, ok := a.maintenanceReplacement(m, st.Mode, archive)
if !ok {
continue
}
m.Content = replacement
view[i] = m
}
dst = estimateMessagesTokens(view)
+103
View File
@@ -0,0 +1,103 @@
package agent
import (
"fmt"
"strings"
"reasonix/internal/provider"
)
// Keeping a recorded failure whole is what makes a projection grow without
// bound: a failing `go test` log is mostly passing cases, and the keep policy
// protects all of it forever. These shorten one to the lines that carry the
// failure, which is the part a re-run would otherwise be needed to recover.
const (
failureContextLines = 2 // lines kept either side of a failure line
failureMaxKeptLines = 60 // ceiling, so a log that fails throughout still shrinks
failureMinLines = 24 // below this the result is already small enough to keep whole
)
// elisionMarker labels the gap a previous pass left behind. Its presence means
// the content is already shortened: shortening again would swallow the marker
// and restate the count, so a projection folded twice would differ byte for
// byte from one folded once, for no gain.
const elisionMarker = " lines omitted …"
// failureMarkers mark a line as carrying the failure. Deliberately generous:
// keeping a spare line costs a few tokens, dropping the assertion costs a re-run.
var failureMarkers = []string{
"fail", "error", "panic:", "fatal", "assert", "expected", "want:", "got:",
"exit status", "undefined:", "cannot ", "no such", "timeout", "timed out",
}
// keptForProjection shortens a protected failure before it rides into the
// projection. The keep policy exists so the failure is not lost, not so its
// passing noise is carried through every later fold.
func (a *Agent) keptForProjection(m provider.Message) provider.Message {
if !failedExecution(m.ToolExecution) {
return m
}
m.Content = snipFailureResult(m.Content)
return m
}
func isFailureLine(s string) bool {
lowered := strings.ToLower(s)
for _, marker := range failureMarkers {
if strings.Contains(lowered, marker) {
return true
}
}
return false
}
// snipFailureResult keeps the failure-carrying lines of content and elides the
// rest. It returns content unchanged when there is nothing worth dropping, so
// an unchanged return means "leave this message alone".
func snipFailureResult(content string) string {
if strings.Contains(content, elisionMarker) {
return content
}
lines := strings.Split(content, "\n")
if len(lines) < failureMinLines {
return content
}
keep := make([]bool, len(lines))
kept := 0
for i, line := range lines {
if kept >= failureMaxKeptLines {
break
}
if !isFailureLine(line) {
continue
}
for j := max(0, i-failureContextLines); j <= min(len(lines)-1, i+failureContextLines); j++ {
if !keep[j] {
keep[j], kept = true, kept+1
}
}
}
if kept == 0 || kept >= len(lines) {
return content
}
var b strings.Builder
omitted := 0
flush := func() {
if omitted > 0 {
fmt.Fprintf(&b, "… %d%s\n", omitted, elisionMarker)
omitted = 0
}
}
for i, line := range lines {
if !keep[i] {
omitted++
continue
}
flush()
b.WriteString(line)
b.WriteByte('\n')
}
flush()
return strings.TrimRight(b.String(), "\n")
}
+86
View File
@@ -0,0 +1,86 @@
package agent
import (
"fmt"
"strings"
"testing"
)
func padded(prefix string, n int) []string {
out := make([]string, n)
for i := range n {
out[i] = fmt.Sprintf("%s%03d", prefix, i)
}
return out
}
// The failure markers are a heuristic, so the toolchains a coding agent
// actually runs are the coverage that matters: each case buries its detail in
// the middle of otherwise unremarkable output.
func TestSnipFailureAcrossToolchains(t *testing.T) {
for _, tc := range []struct {
name string
middle []string
want []string
}{
{
name: "pytest",
middle: []string{
"=================================== FAILURES ===================================",
"_____________________________ test_quoted_values ______________________________",
"> assert config.dumps(cfg) == 'beta-7d21'",
"E AssertionError: assert 'gamma-4a88' == 'beta-7d21'",
"tests/test_format.py:412: AssertionError",
},
want: []string{"beta-7d21", "gamma-4a88", "test_format.py:412"},
},
{
name: "cargo",
middle: []string{
"error[E0308]: mismatched types",
" --> src/config.rs:412:9",
"412 | beta_7d21",
" | ^^^^^^^^^ expected `String`, found `&str`",
},
want: []string{"E0308", "config.rs:412", "expected"},
},
{
name: "tsc",
middle: []string{
"src/config.ts(412,9): error TS2322: Type 'string' is not assignable to type 'number'.",
},
want: []string{"TS2322", "config.ts(412,9)"},
},
{
name: "jest",
middle: []string{
" ● Config › round-trips quoted values",
" expect(received).toBe(expected) // Object.is equality",
" Expected: \"beta-7d21\"",
" Received: \"gamma-4a88\"",
" at Object.<anonymous> (src/config.test.ts:412:24)",
},
want: []string{"beta-7d21", "gamma-4a88", "config.test.ts:412"},
},
} {
t.Run(tc.name, func(t *testing.T) {
lines := padded("collecting item ", 120)
lines = append(lines, tc.middle...)
lines = append(lines, padded("ok case ", 120)...)
content := strings.Join(lines, "\n")
got := snipFailureResult(content)
if got == content {
t.Fatalf("nothing elided from %d lines of mostly-quiet output", len(lines))
}
for _, want := range tc.want {
if !strings.Contains(got, want) {
t.Errorf("detail %q did not survive:\n%s", want, got)
}
}
if len(got) >= len(content) {
t.Errorf("result grew: %d -> %d bytes", len(content), len(got))
}
})
}
}
+105
View File
@@ -0,0 +1,105 @@
package agent
import (
"fmt"
"strings"
"testing"
)
func passingLog(n int) []string {
out := make([]string, 0, n*2)
for i := range n {
out = append(out,
fmt.Sprintf("=== RUN TestConfigCase%03d", i),
fmt.Sprintf("--- PASS: TestConfigCase%03d (0.00s)", i))
}
return out
}
func TestSnipFailureKeepsTheAssertionAndDropsThePasses(t *testing.T) {
lines := passingLog(60)
lines = append(lines,
"=== RUN TestQuotedValues",
" format_test.go:412: assertion failed: expected beta-7d21, got gamma-4a88",
"--- FAIL: TestQuotedValues (0.01s)")
lines = append(lines, passingLog(60)...)
content := strings.Join(lines, "\n")
got := snipFailureResult(content)
if got == content {
t.Fatal("nothing was elided from a mostly-passing log")
}
for _, want := range []string{"beta-7d21", "gamma-4a88", "format_test.go:412"} {
if !strings.Contains(got, want) {
t.Errorf("failure detail %q did not survive", want)
}
}
if !strings.Contains(got, "lines omitted") {
t.Error("no elision marker: the reader cannot tell content was dropped")
}
if len(got) >= len(content) {
t.Errorf("result grew: %d -> %d bytes", len(content), len(got))
}
if strings.Count(got, "PASS") > 2*failureContextLines+2 {
t.Errorf("kept too many passing lines:\n%s", got)
}
}
// Small results are left alone: there is nothing to win and the elision marker
// would cost more than it saves.
func TestSnipFailureLeavesShortResultsWhole(t *testing.T) {
content := "error: build failed\n./main.go:12:5: undefined: foo\nexit status 1"
if got := snipFailureResult(content); got != content {
t.Errorf("short result rewritten:\n%s", got)
}
}
// A long result with no recognisable failure line is returned untouched rather
// than emptied — the caller decides what to do with it.
func TestSnipFailureKeepsUnrecognisedOutputWhole(t *testing.T) {
content := strings.Join(passingLog(40), "\n")
if got := snipFailureResult(content); got != content {
t.Errorf("output with no failure markers was rewritten:\n%.200s", got)
}
}
// keptForProjection runs on every fold, so shortening must reach a fixed point
// on the first pass. A second pass that swallowed the elision marker and
// restated the count would make a twice-folded projection differ byte for byte
// from a once-folded one, breaking the prompt cache for nothing.
func TestSnipFailureIsIdempotent(t *testing.T) {
allFailures := make([]string, 200)
for i := range allFailures {
allFailures[i] = fmt.Sprintf("--- FAIL: TestCase%03d (0.00s)", i)
}
mixed := passingLog(60)
mixed = append(mixed, " format_test.go:412: assertion failed: expected beta-7d21, got gamma-4a88")
mixed = append(mixed, passingLog(60)...)
for name, lines := range map[string][]string{"all failures": allFailures, "one failure": mixed} {
t.Run(name, func(t *testing.T) {
once := snipFailureResult(strings.Join(lines, "\n"))
twice := snipFailureResult(once)
if once != twice {
t.Errorf("not a fixed point:\n--- once ---\n%s\n--- twice ---\n%s", once, twice)
}
})
}
}
// A log that is failures throughout must still shrink, or the ceiling does not
// bound anything.
func TestSnipFailureBoundsAnAllFailureLog(t *testing.T) {
lines := make([]string, 0, 400)
for i := range 400 {
lines = append(lines, fmt.Sprintf("--- FAIL: TestCase%03d (0.00s)", i))
}
content := strings.Join(lines, "\n")
got := snipFailureResult(content)
if got == content {
t.Fatal("an all-failure log was not bounded")
}
if n := strings.Count(got, "FAIL"); n > failureMaxKeptLines {
t.Errorf("kept %d failure lines, over the %d ceiling", n, failureMaxKeptLines)
}
}
+70
View File
@@ -0,0 +1,70 @@
package agent
import (
"testing"
"reasonix/internal/provider"
"reasonix/internal/tool"
)
func exitCode(n int) *int { return &n }
// A failing `go test` is the case prefix matching cannot see: the output opens
// with "=== RUN" and the failure only shows in the recorded execution.
func TestKeepErrorsSeesStructuredFailure(t *testing.T) {
failing := "=== RUN TestQuotedValues\n format_test.go:412: expected beta-7d21, got gamma-4a88\n--- FAIL: TestQuotedValues (0.01s)\nFAIL\n"
for _, tc := range []struct {
name string
ex *provider.ToolExecution
want bool
}{
{"state failed", &provider.ToolExecution{State: tool.ShellStateFailed}, true},
{"timed out", &provider.ToolExecution{State: tool.ShellStateTimedOut}, true},
{"non-zero exit", &provider.ToolExecution{State: tool.ShellStateCompleted, ExitCode: exitCode(1)}, true},
{"verification failed", &provider.ToolExecution{Verification: tool.ShellVerificationFailed}, true},
{"clean run", &provider.ToolExecution{State: tool.ShellStateCompleted, ExitCode: exitCode(0)}, false},
{"no execution record", nil, false},
} {
t.Run(tc.name, func(t *testing.T) {
m := provider.Message{Role: provider.RoleTool, Content: failing, ToolExecution: tc.ex}
if got := isErrorMessage(m); got != tc.want {
t.Errorf("isErrorMessage = %v, want %v", got, tc.want)
}
})
}
}
// The text fallback still classifies host-formatted failures, and a passing run
// must not be kept just because it mentions a failure in passing.
func TestKeepErrorsTextFallbackUnchanged(t *testing.T) {
cases := map[string]struct {
content string
want bool
}{
"error prefix": {"error: file not found", true},
"blocked prefix": {"blocked: permission denied", true},
"mentions failure": {"all tests pass; the earlier FAIL is fixed", false},
"ordinary output": {"ok reasonix/config\t0.2s", false},
"uppercase prefix": {"Error: broken", true},
"prefix mid-string": {"note: error: not at the start", false},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
m := provider.Message{Role: provider.RoleTool, Content: tc.content}
if got := isErrorMessage(m); got != tc.want {
t.Errorf("isErrorMessage(%q) = %v, want %v", tc.content, got, tc.want)
}
})
}
}
// Only tool results are keep-policy candidates; a failing execution record on
// another role must not promote it.
func TestKeepErrorsIgnoresNonToolRoles(t *testing.T) {
ex := &provider.ToolExecution{State: tool.ShellStateFailed, ExitCode: exitCode(2)}
for _, role := range []provider.Role{provider.RoleUser, provider.RoleAssistant, provider.RoleSystem} {
if isErrorMessage(provider.Message{Role: role, Content: "error: x", ToolExecution: ex}) {
t.Errorf("role %v classified as an error tool result", role)
}
}
}
+19 -2
View File
@@ -223,6 +223,23 @@ func (a *Agent) persistCompactionStateLocked() error {
return SaveCompactionState(a.sessionPath, a.compactionState)
}
// maintenanceReplacement is the rewrite for one stale tool result. Both the
// planning pass and the re-write that follows archiving go through it, so a
// result cannot be shortened one way while planning and another way on install.
// ok is false when the policy protects the message outright.
func (a *Agent) maintenanceReplacement(m provider.Message, mode toolResultMaintenanceMode, archive string) (string, bool) {
if a.keepPolicy&KeepErrors != 0 && isErrorMessage(m) {
// A recorded failure stays recognisable once its text is rewritten, so
// its passing noise can go. A text-only failure has no such anchor:
// eliding it would hide that it was ever an error.
if !failedExecution(m.ToolExecution) {
return "", false
}
return snipFailureResult(m.Content), true
}
return rewriteToolResult(m, mode, archive, a.snipStrategyFor(m.Name)), true
}
// applyToolResultMaintenanceView returns a copy of msgs with stale tool results
// snipped or pruned. The canonical transcript is never modified.
func (a *Agent) applyToolResultMaintenanceView(msgs []provider.Message, mode toolResultMaintenanceMode) ([]provider.Message, PruneStats) {
@@ -249,10 +266,10 @@ func (a *Agent) applyToolResultMaintenanceView(msgs []provider.Message, mode too
if !shouldMaintainToolResult(m, mode) {
continue
}
if a.keepPolicy&KeepErrors != 0 && isErrorMessage(m) {
replacement, ok := a.maintenanceReplacement(m, mode, "projection-view")
if !ok {
continue
}
replacement := rewriteToolResult(m, mode, "projection-view", a.snipStrategyFor(m.Name))
if replacement == m.Content {
continue
}