mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-08 08:04:50 -10:00
fix(agent): keep policy reads the recorded failure, not the text prefix (#8155)
KeepErrors only matched tool results starting with "error:" or "blocked:", the shape the host formats for an execution failure. A tool that ran fine but reported a failure — a go test FAIL, a compiler error, any non-zero exit — never matched, so its evidence folded into the digest and the specific assertion was gone. Measured against a full-history control on benchmarks/compaction, evidence buried mid-result survived 0 of 6 runs while the control kept it every time. provider.Message.ToolExecution already carries State/ExitCode/Verification and is already stripped before provider requests, so isErrorMessage reads it instead of guessing. Because the failure then stays identifiable without its text prefix, a protected failure can also be shortened: snipFailureResult keeps the failure-carrying lines with two lines of context and drops the passing noise, holding the projection cost to +3.5% rather than the +100% that keeping the whole result costs. A text-only failure has no such anchor and is still carried whole. The rewrite happens on three paths — the planning pass, the re-write that follows archiving, and the fold's kept partition — now unified through maintenanceReplacement/keptForProjection so one cannot silently overwrite another. benchmarks/compaction gains the probe that measures this, a -snip arm, cross-generation planting for revisions that land across a fold, and alignment with boot's default KeepPolicy. Buried evidence now survives 18 of 18 runs.
This commit is contained in:
14 files changed
+648
-29
No files matched your search
@@ -42,6 +42,7 @@ func main() {
|
||||
window := flag.Int("window", 128_000, "context window in tokens")
|
||||
control := flag.Bool("control", true, "fidelity: also score probes against full history")
|
||||
arm := flag.String("arm", "full", "full | incremental: re-derive each digest from canonical, or fold the previous projection")
|
||||
snip := flag.Bool("snip", false, "run the stale-tool-result snip pass before each fold, as the 0.6 threshold does in a real session")
|
||||
out := flag.String("out", "", "write the JSON report here")
|
||||
flag.Parse()
|
||||
|
||||
@@ -49,14 +50,14 @@ func main() {
|
||||
res []genResult
|
||||
err error
|
||||
)
|
||||
incremental := *arm == "incremental"
|
||||
a := arms{incremental: *arm == "incremental", snip: *snip}
|
||||
switch {
|
||||
case *arm != "full" && *arm != "incremental":
|
||||
err = fmt.Errorf("unknown arm %q", *arm)
|
||||
case *mode == "cost":
|
||||
res, err = runCost(*gens, *window, incremental)
|
||||
res, err = runCost(*gens, *window, a)
|
||||
case *mode == "fidelity":
|
||||
res, err = runFidelity(*gens, *window, *control, incremental)
|
||||
res, err = runFidelity(*gens, *window, *control, a)
|
||||
default:
|
||||
err = fmt.Errorf("unknown mode %q", *mode)
|
||||
}
|
||||
@@ -83,6 +84,8 @@ type genResult struct {
|
||||
SummarizerInput int `json:"summarizer_input_tokens"`
|
||||
LargestCall int `json:"largest_call_tokens"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
SnippedResults int `json:"snipped_results,omitempty"`
|
||||
SnippedChars int `json:"snipped_chars,omitempty"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Survived map[string]int `json:"survived,omitempty"` // probe class -> 1 kept, 0 lost
|
||||
@@ -97,9 +100,10 @@ type harness struct {
|
||||
agentA *agent.Agent
|
||||
path string
|
||||
calls *callRecorder
|
||||
snip bool
|
||||
}
|
||||
|
||||
func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorder, incremental bool) *harness {
|
||||
func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorder, arm arms) *harness {
|
||||
sess := newSession()
|
||||
path := filepath.Join(t.dir, "session.jsonl")
|
||||
a := agent.New(p, tool.NewRegistry(), sess, agent.Options{
|
||||
@@ -107,13 +111,23 @@ func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorde
|
||||
ArchiveDir: filepath.Join(t.dir, "archive"),
|
||||
SessionPath: path,
|
||||
RecentKeep: 4,
|
||||
Ablation: foldArm(incremental),
|
||||
// boot's default when cfg.Agent.Keep is unset; without it the bench
|
||||
// would measure a configuration no real session runs.
|
||||
KeepPolicy: agent.KeepErrors,
|
||||
Ablation: foldArm(arm.incremental),
|
||||
}, rec.sink())
|
||||
return &harness{sess: sess, agentA: a, path: path, calls: rec}
|
||||
return &harness{sess: sess, agentA: a, path: path, calls: rec, snip: arm.snip}
|
||||
}
|
||||
|
||||
// foldArm switches full re-derivation off, which is what makes a fold read the
|
||||
// previous projection instead of the canonical transcript.
|
||||
// arms selects the maintenance behaviour under test. Snipping is off by default
|
||||
// so a run stays comparable with baselines recorded before it existed.
|
||||
type arms struct {
|
||||
incremental bool
|
||||
snip bool
|
||||
}
|
||||
|
||||
func foldArm(incremental bool) ablation.Set {
|
||||
if incremental {
|
||||
return ablation.New(ablation.FullFold)
|
||||
@@ -128,6 +142,13 @@ func (h *harness) runGeneration(ctx context.Context, gen int, probes []probe) ge
|
||||
|
||||
h.calls.reset()
|
||||
start := time.Now()
|
||||
if h.snip {
|
||||
st, serr := h.agentA.SnipStaleToolResults()
|
||||
if serr != nil {
|
||||
r.Error = serr.Error()
|
||||
}
|
||||
r.SnippedResults, r.SnippedChars = st.Results, st.SavedChars
|
||||
}
|
||||
err := h.agentA.CompactNow(ctx, "")
|
||||
r.Seconds = time.Since(start).Seconds()
|
||||
if err != nil {
|
||||
@@ -145,7 +166,7 @@ func (h *harness) runGeneration(ctx context.Context, gen int, probes []probe) ge
|
||||
return r
|
||||
}
|
||||
|
||||
func runCost(gens, window int, incremental bool) ([]genResult, error) {
|
||||
func runCost(gens, window int, a arms) ([]genResult, error) {
|
||||
dir, cleanup, err := tempDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -154,7 +175,7 @@ func runCost(gens, window int, incremental bool) ([]genResult, error) {
|
||||
|
||||
rec := &callRecorder{}
|
||||
p := &scriptedProvider{rec: rec, reply: syntheticDigest, window: window}
|
||||
h := newHarness(dir, p, window, rec, incremental)
|
||||
h := newHarness(dir, p, window, rec, a)
|
||||
|
||||
var out []genResult
|
||||
for gen := range gens {
|
||||
@@ -163,7 +184,7 @@ func runCost(gens, window int, incremental bool) ([]genResult, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func runFidelity(gens, window int, control, incremental bool) ([]genResult, error) {
|
||||
func runFidelity(gens, window int, control bool, a arms) ([]genResult, error) {
|
||||
key := os.Getenv("DEEPSEEK_API_KEY")
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("fidelity mode needs DEEPSEEK_API_KEY")
|
||||
@@ -179,7 +200,7 @@ func runFidelity(gens, window int, control, incremental bool) ([]genResult, erro
|
||||
defer cleanup()
|
||||
|
||||
rec := &callRecorder{}
|
||||
h := newHarness(dir, &recordingProvider{inner: p, rec: rec}, window, rec, incremental)
|
||||
h := newHarness(dir, &recordingProvider{inner: p, rec: rec}, window, rec, a)
|
||||
probes := probeSuite()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -193,7 +214,7 @@ func runFidelity(gens, window int, control, incremental bool) ([]genResult, erro
|
||||
return nil, verr
|
||||
}
|
||||
for _, probe := range probes {
|
||||
if probe.plantAt > gen {
|
||||
if probe.settledAt() > gen {
|
||||
continue
|
||||
}
|
||||
answer, aerr := ask(ctx, p, visible, probe.question)
|
||||
|
||||
@@ -12,7 +12,7 @@ func TestRepeatedFoldsStayBoundedAndKeepSucceeding(t *testing.T) {
|
||||
reserve = 8192 // summaryOutputReserve: the digest the call must still return
|
||||
maxCalls = 7 // maxSummarySpans + the merge pass
|
||||
)
|
||||
res, err := runCost(gens, window, false)
|
||||
res, err := runCost(gens, window, arms{})
|
||||
if err != nil {
|
||||
t.Fatalf("runCost: %v", err)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"reasonix/internal/agent"
|
||||
"reasonix/internal/provider"
|
||||
"reasonix/internal/tool"
|
||||
)
|
||||
|
||||
// probeAnswerContract rides with every probe question. A context full of tool
|
||||
@@ -33,9 +34,13 @@ func invalidAnswer(s string) bool {
|
||||
// The question is asked against the compacted context, so a wrong answer means
|
||||
// compaction lost the fact — not that the model is weak.
|
||||
type probe struct {
|
||||
class string
|
||||
plantAt int
|
||||
plant func(*agent.Session)
|
||||
class string
|
||||
plantAt int
|
||||
plant func(*agent.Session)
|
||||
// later plants more of the same fact in later generations, so a decision
|
||||
// that changes across folds is tested the way it actually happens: each
|
||||
// revision lands on the far side of a compaction, not next to the last one.
|
||||
later map[int]func(*agent.Session)
|
||||
question string
|
||||
want []string // answer must contain one of these, lowercased
|
||||
reject []string // ...and none of these: the pre-correction answer
|
||||
@@ -52,6 +57,24 @@ func toolRound(id, name, args, result string) func(*agent.Session) {
|
||||
}
|
||||
}
|
||||
|
||||
// failedToolRound is a tool round the host recorded as failed, the way a real
|
||||
// non-zero bash run arrives. Without the execution record the keep policy sees
|
||||
// only text, which is exactly the gap the buried-evidence probe measures.
|
||||
func failedToolRound(id, name, args, result string, code int) func(*agent.Session) {
|
||||
return func(s *agent.Session) {
|
||||
exit := code
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: name, Arguments: args}}})
|
||||
s.Add(provider.Message{
|
||||
Role: provider.RoleTool, ToolCallID: id, Name: name, Content: result,
|
||||
ToolExecution: &provider.ToolExecution{
|
||||
Kind: "shell",
|
||||
State: tool.ShellStateFailed,
|
||||
ExitCode: &exit,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func seq(fns ...func(*agent.Session)) func(*agent.Session) {
|
||||
return func(s *agent.Session) {
|
||||
for _, fn := range fns {
|
||||
@@ -64,6 +87,50 @@ func seq(fns ...func(*agent.Session)) func(*agent.Session) {
|
||||
// freshness and correction probes are the ones summaries classically get wrong:
|
||||
// both have a plausible stale answer that reads as correct.
|
||||
func probeSuite() []probe {
|
||||
suite := append(factProbes(), revisionProbes()...)
|
||||
return append(suite, snipProbes()...)
|
||||
}
|
||||
|
||||
// buriedTestLog is a `go test -v` log whose only failure detail sits in the
|
||||
// middle. Geometric snipping keeps the first 80 and last 12 lines, so this is
|
||||
// precisely the shape it drops while reporting a plausible-looking remainder.
|
||||
// It names a package no other probe touches: a run covering config/ would be a
|
||||
// truthful "yes" for verification-freshness and score that probe as lost.
|
||||
func buriedTestLog() string {
|
||||
var b strings.Builder
|
||||
pass := func(i int) {
|
||||
fmt.Fprintf(&b, "=== RUN TestEmitterCase%03d\n--- PASS: TestEmitterCase%03d (0.00s)\n", i, i)
|
||||
}
|
||||
for i := range 120 {
|
||||
pass(i)
|
||||
}
|
||||
b.WriteString("=== RUN TestEmitterQuoting\n")
|
||||
b.WriteString(" emitter_test.go:412: assertion failed: expected beta-7d21, got gamma-4a88\n")
|
||||
b.WriteString("--- FAIL: TestEmitterQuoting (0.01s)\n")
|
||||
for i := 120; i < 200; i++ {
|
||||
pass(i)
|
||||
}
|
||||
b.WriteString("FAIL\nexit status 1\nFAIL\treasonix/emitter\t0.234s\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// snipProbes plant a fact in the middle of a long tool result, out of reach of
|
||||
// the head/tail geometry. They score the maintenance pass rather than the fold:
|
||||
// every other probe in the suite is short enough that snipping cannot touch it.
|
||||
func snipProbes() []probe {
|
||||
return []probe{
|
||||
{
|
||||
class: "buried-evidence",
|
||||
plantAt: 1,
|
||||
plant: failedToolRound("bt1", "bash", `{"cmd":"go test ./emitter -v"}`, buriedTestLog(), 1),
|
||||
question: "In the failed test, what value did the assertion expect? Reply with just the value.",
|
||||
want: []string{"beta-7d21"},
|
||||
reject: []string{"gamma-4a88"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func factProbes() []probe {
|
||||
return []probe{
|
||||
{
|
||||
class: "user-constraint",
|
||||
@@ -165,6 +232,45 @@ func probeSuite() []probe {
|
||||
}
|
||||
}
|
||||
|
||||
// revisionProbes state a fact and then change it in a later generation. The
|
||||
// revision lands on the far side of a fold, so carrying it forward means
|
||||
// superseding the digest's own earlier claim rather than a neighbouring turn.
|
||||
func revisionProbes() []probe {
|
||||
return []probe{
|
||||
{
|
||||
class: "late-constraint",
|
||||
plantAt: 3,
|
||||
plant: userTurn("New hard rule from here on: never edit anything under internal/store/."),
|
||||
question: "Are you allowed to edit files under internal/store/? Answer yes or no.",
|
||||
want: []string{"no"},
|
||||
reject: []string{"yes"},
|
||||
},
|
||||
{
|
||||
class: "reversal-chain",
|
||||
plantAt: 0,
|
||||
plant: userTurn("Use PostgreSQL for the datastore."),
|
||||
later: map[int]func(*agent.Session){
|
||||
1: userTurn("Change of plan: drop PostgreSQL, use SQLite instead."),
|
||||
2: userTurn("Final call: back to PostgreSQL after all."),
|
||||
},
|
||||
question: "Which datastore is the current decision? Reply with one word.",
|
||||
want: []string{"postgresql", "postgres"},
|
||||
reject: []string{"sqlite"},
|
||||
},
|
||||
{
|
||||
class: "distractor-file",
|
||||
plantAt: 1,
|
||||
plant: userTurn("Draft note: the fix might belong in config/legacy_parser.go."),
|
||||
later: map[int]func(*agent.Session){
|
||||
3: userTurn("Confirmed: the fix belongs in config/emitter.go; the legacy parser is not involved."),
|
||||
},
|
||||
question: "Which file does the fix belong in? Reply with just the filename, no path.",
|
||||
want: []string{"emitter"},
|
||||
reject: []string{"legacy"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// score reports whether an answer keeps the planted fact. A rejected token
|
||||
// anywhere in the answer counts as lost even when the wanted token also
|
||||
// appears, so "yes, but it was not re-run" does not pass as "no".
|
||||
@@ -194,4 +300,15 @@ func matchesToken(lowered, want string) bool {
|
||||
}), want)
|
||||
}
|
||||
|
||||
func (p probe) String() string { return fmt.Sprintf("%s@gen%d", p.class, p.plantAt) }
|
||||
// settledAt is the first generation whose answer is the one want/reject scores.
|
||||
// A probe revised across folds has a different correct answer while the chain is
|
||||
// still running, so asking before it settles would score a right answer as lost.
|
||||
func (p probe) settledAt() int {
|
||||
at := p.plantAt
|
||||
for gen := range p.later {
|
||||
at = max(at, gen)
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
func (p probe) String() string { return fmt.Sprintf("%s@gen%d", p.class, p.settledAt()) }
|
||||
@@ -46,6 +46,9 @@ func growSession(sess *agent.Session, gen int, probes []probe) {
|
||||
if p.plantAt == gen {
|
||||
p.plant(sess)
|
||||
}
|
||||
if revise, ok := p.later[gen]; ok {
|
||||
revise(sess)
|
||||
}
|
||||
}
|
||||
for i := range unitsPerGeneration {
|
||||
workUnit(sess, gen, i)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The buried-evidence probe only measures something if its planted fact sits
|
||||
// outside the head/tail window the snip geometry keeps. Guard the probe design
|
||||
// against a later edit that shortens the log back into reach.
|
||||
func TestBuriedFactSitsOutsideSnipGeometry(t *testing.T) {
|
||||
const (
|
||||
head = 80 // defaultReadOnlySnip.head
|
||||
tail = 12 // defaultReadOnlySnip.tail
|
||||
)
|
||||
lines := strings.Split(buriedTestLog(), "\n")
|
||||
idx := -1
|
||||
for i, line := range lines {
|
||||
if strings.Contains(line, "beta-7d21") {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
t.Fatal("buried fact absent from the planted log")
|
||||
}
|
||||
if idx < head || idx >= len(lines)-tail {
|
||||
t.Errorf("fact at line %d of %d is inside the kept head(%d)/tail(%d); snipping would preserve it and the probe would measure nothing", idx, len(lines), head, tail)
|
||||
}
|
||||
}
|
||||
|
||||
// A tool result the host recorded as failed outranks the snip geometry: the
|
||||
// assertion detail sits where head/tail cannot reach it, so it survives only
|
||||
// because the keep policy reads the execution record rather than the text.
|
||||
func TestSnipKeepsRecordedFailure(t *testing.T) {
|
||||
dir, cleanup, err := tempDir()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
rec := &callRecorder{}
|
||||
const window = 64_000
|
||||
h := newHarness(dir, &scriptedProvider{rec: rec, reply: syntheticDigest, window: window}, window, rec, arms{snip: true})
|
||||
for gen := range 3 {
|
||||
growSession(h.sess, gen, probeSuite())
|
||||
}
|
||||
before := renderAll(h.sess.Snapshot())
|
||||
if !strings.Contains(before, "beta-7d21") {
|
||||
t.Fatal("planted fact missing before the snip pass")
|
||||
}
|
||||
st, err := h.agentA.SnipStaleToolResults()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.Results == 0 {
|
||||
t.Fatal("snip pass rewrote nothing")
|
||||
}
|
||||
visible, err := visibleContext(h.path, h.sess)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(renderAll(visible), "beta-7d21") {
|
||||
t.Errorf("recorded failure was snipped away (%d results, %d chars); keep policy did not protect it", st.Results, st.SavedChars)
|
||||
}
|
||||
}
|
||||
+12
-6
@@ -275,9 +275,14 @@ Long tasks eventually fill the model's context window. Reasonix manages this wit
|
||||
limits. `model_overrides.<model>.max_output_tokens` can specialize mixed
|
||||
gateways; Anthropic still supplies a mandatory `max_tokens` fallback.
|
||||
- Tool-result snip/prune never removes messages, so assistant `tool_calls` and
|
||||
tool results stay paired. `KeepErrors` preserves error/blocked tool outputs,
|
||||
and the recent tail is not rewritten. Snipped results can later be upgraded to
|
||||
pruned placeholders; already-pruned results are left alone.
|
||||
tool results stay paired. `KeepErrors` preserves failed tool outputs,
|
||||
recognised from the recorded execution (state, exit code, verification) or,
|
||||
when there is none, an `error:`/`blocked:` text prefix. A failure with a
|
||||
recorded execution keeps its failure-carrying lines rather than the whole
|
||||
result, since the record still identifies it as a failure after a rewrite; a
|
||||
text-only failure has no such anchor and is kept whole. The recent tail is not
|
||||
rewritten. Snipped results can later be upgraded to pruned placeholders;
|
||||
already-pruned results are left alone.
|
||||
- Automatic maintenance is planned once before a sampling request from the
|
||||
current visible projection plus its append-only canonical tail. It never
|
||||
rewrites the canonical transcript. A failed or non-convergent view fingerprint
|
||||
@@ -290,7 +295,7 @@ Long tasks eventually fill the model's context window. Reasonix manages this wit
|
||||
- When summary compaction runs, the fold region (everything between the pinned
|
||||
prefix and the recent tail) is split three ways: the first few **small user
|
||||
turns** are hoisted verbatim ahead of the digest, messages the keep policy
|
||||
protects stay verbatim, and **everything else** — assistant/tool work, later
|
||||
protects are kept, and **everything else** — assistant/tool work, later
|
||||
user turns, and any prior digest — is summarized into a single digest, using
|
||||
the executor's own provider, no tools. The split is a partition: a message in
|
||||
the region is either kept verbatim or reaches the summarizer, never neither.
|
||||
@@ -345,8 +350,9 @@ Long tasks eventually fill the model's context window. Reasonix manages this wit
|
||||
|
||||
**What survives a fold.** Verbatim, at every compaction: the system prompt, the
|
||||
first user turn when it is small enough to be a brief, the first few small user
|
||||
turns of the fold region, the messages the keep policy protects, and the recent
|
||||
tail. Everything else is **best-effort** — it reaches the summarizer and survives
|
||||
turns of the fold region, and the recent tail. The messages the keep policy
|
||||
protects also survive, though a failure with a recorded execution keeps only its
|
||||
failure-carrying lines. Everything else is **best-effort** — it reaches the summarizer and survives
|
||||
only as well as the digest captured it. That includes small user turns beyond the
|
||||
hoisted window, so a durable constraint is safest restated in a recent turn
|
||||
rather than assumed to hold from turn 4 of a long session.
|
||||
|
||||
+4
-3
@@ -164,7 +164,7 @@ tool result 的 snip/prune 不删除消息,确保 assistant `tool_calls` 与 t
|
||||
原生工具清理不会替代 Reasonix 的摘要折叠,也不会修改 canonical history。
|
||||
|
||||
摘要折叠时,固定前缀与近期 tail 之间的区间被划分为三部分:开头若干条小体量用户回合原样提升到
|
||||
digest 之前,keep policy 保护的消息原样保留,其余全部——assistant/tool 工作、后续用户回合、以及
|
||||
digest 之前,keep policy 保护的消息予以保留,其余全部——assistant/tool 工作、后续用户回合、以及
|
||||
已有 digest——折叠进同一条 digest。三者构成一个划分:区间内的消息要么原样保留,要么进入摘要输入,
|
||||
不存在两者皆非的情况。
|
||||
|
||||
@@ -174,8 +174,9 @@ digest 之前,keep policy 保护的消息原样保留,其余全部——assi
|
||||
被拆成连续几部分分别摘要,再由最后一次调用合并,并对份数设上限,使单次 compaction 的调用数有界;
|
||||
某一部分被迫丢弃的内容,会写在摘要器读到的文本里。
|
||||
|
||||
因此逐字保留的是:system prompt、体量足够小的首个用户回合、折叠区间开头若干条小体量用户回合、
|
||||
keep policy 保护的消息,以及近期 tail。其余均为尽力而为——只在 digest 抓住它的前提下留存,
|
||||
因此逐字保留的是:system prompt、体量足够小的首个用户回合、折叠区间开头若干条小体量用户回合,
|
||||
以及近期 tail。keep policy 保护的消息同样留存,但带有执行记录的失败只保留承载失败的若干行。
|
||||
其余均为尽力而为——只在 digest 抓住它的前提下留存,
|
||||
其中包括超出提升窗口的小体量用户回合,所以长期有效的约束更适合在近期回合中重述。
|
||||
|
||||
两个性质限制了这种损失:每次折叠都从 canonical transcript 重新生成 digest,而不是在上一条 digest
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"reasonix/internal/ablation"
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/provider"
|
||||
"reasonix/internal/tool"
|
||||
)
|
||||
|
||||
// Compaction is a low-frequency cache-reset point: the prompt grows append-only
|
||||
@@ -366,10 +367,29 @@ func isErrorMessage(m provider.Message) bool {
|
||||
if m.Role != provider.RoleTool {
|
||||
return false
|
||||
}
|
||||
if failedExecution(m.ToolExecution) {
|
||||
return true
|
||||
}
|
||||
s := strings.TrimSpace(strings.ToLower(m.Content))
|
||||
return strings.HasPrefix(s, "error:") || strings.HasPrefix(s, "blocked:")
|
||||
}
|
||||
|
||||
// failedExecution reads the failure the host already recorded, rather than
|
||||
// guessing from the text. A `go test` run that reports FAIL exits non-zero
|
||||
// while its output starts with "=== RUN", which no prefix match can see.
|
||||
func failedExecution(ex *provider.ToolExecution) bool {
|
||||
if ex == nil {
|
||||
return false
|
||||
}
|
||||
if ex.State == tool.ShellStateFailed || ex.State == tool.ShellStateTimedOut {
|
||||
return true
|
||||
}
|
||||
if ex.ExitCode != nil && *ex.ExitCode != 0 {
|
||||
return true
|
||||
}
|
||||
return ex.Verification == tool.ShellVerificationFailed
|
||||
}
|
||||
|
||||
func isUserMarked(m provider.Message) bool {
|
||||
if m.Role != provider.RoleUser {
|
||||
return false
|
||||
|
||||
@@ -545,7 +545,7 @@ func (a *Agent) partitionFoldForProjection(region []provider.Message) (early, ca
|
||||
}
|
||||
fold = append(fold, m)
|
||||
case policyKeep[i]:
|
||||
kept = append(kept, m)
|
||||
kept = append(kept, a.keptForProjection(m))
|
||||
default:
|
||||
fold = append(fold, m)
|
||||
}
|
||||
@@ -702,7 +702,11 @@ func (a *Agent) installPruneProjection(view []provider.Message, st PruneStats) e
|
||||
continue
|
||||
}
|
||||
m := currentVisible[i]
|
||||
m.Content = rewriteToolResult(m, st.Mode, archive, a.snipStrategyFor(m.Name))
|
||||
replacement, ok := a.maintenanceReplacement(m, st.Mode, archive)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
m.Content = replacement
|
||||
view[i] = m
|
||||
}
|
||||
dst = estimateMessagesTokens(view)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
// Keeping a recorded failure whole is what makes a projection grow without
|
||||
// bound: a failing `go test` log is mostly passing cases, and the keep policy
|
||||
// protects all of it forever. These shorten one to the lines that carry the
|
||||
// failure, which is the part a re-run would otherwise be needed to recover.
|
||||
const (
|
||||
failureContextLines = 2 // lines kept either side of a failure line
|
||||
failureMaxKeptLines = 60 // ceiling, so a log that fails throughout still shrinks
|
||||
failureMinLines = 24 // below this the result is already small enough to keep whole
|
||||
)
|
||||
|
||||
// elisionMarker labels the gap a previous pass left behind. Its presence means
|
||||
// the content is already shortened: shortening again would swallow the marker
|
||||
// and restate the count, so a projection folded twice would differ byte for
|
||||
// byte from one folded once, for no gain.
|
||||
const elisionMarker = " lines omitted …"
|
||||
|
||||
// failureMarkers mark a line as carrying the failure. Deliberately generous:
|
||||
// keeping a spare line costs a few tokens, dropping the assertion costs a re-run.
|
||||
var failureMarkers = []string{
|
||||
"fail", "error", "panic:", "fatal", "assert", "expected", "want:", "got:",
|
||||
"exit status", "undefined:", "cannot ", "no such", "timeout", "timed out",
|
||||
}
|
||||
|
||||
// keptForProjection shortens a protected failure before it rides into the
|
||||
// projection. The keep policy exists so the failure is not lost, not so its
|
||||
// passing noise is carried through every later fold.
|
||||
func (a *Agent) keptForProjection(m provider.Message) provider.Message {
|
||||
if !failedExecution(m.ToolExecution) {
|
||||
return m
|
||||
}
|
||||
m.Content = snipFailureResult(m.Content)
|
||||
return m
|
||||
}
|
||||
|
||||
func isFailureLine(s string) bool {
|
||||
lowered := strings.ToLower(s)
|
||||
for _, marker := range failureMarkers {
|
||||
if strings.Contains(lowered, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// snipFailureResult keeps the failure-carrying lines of content and elides the
|
||||
// rest. It returns content unchanged when there is nothing worth dropping, so
|
||||
// an unchanged return means "leave this message alone".
|
||||
func snipFailureResult(content string) string {
|
||||
if strings.Contains(content, elisionMarker) {
|
||||
return content
|
||||
}
|
||||
lines := strings.Split(content, "\n")
|
||||
if len(lines) < failureMinLines {
|
||||
return content
|
||||
}
|
||||
keep := make([]bool, len(lines))
|
||||
kept := 0
|
||||
for i, line := range lines {
|
||||
if kept >= failureMaxKeptLines {
|
||||
break
|
||||
}
|
||||
if !isFailureLine(line) {
|
||||
continue
|
||||
}
|
||||
for j := max(0, i-failureContextLines); j <= min(len(lines)-1, i+failureContextLines); j++ {
|
||||
if !keep[j] {
|
||||
keep[j], kept = true, kept+1
|
||||
}
|
||||
}
|
||||
}
|
||||
if kept == 0 || kept >= len(lines) {
|
||||
return content
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
omitted := 0
|
||||
flush := func() {
|
||||
if omitted > 0 {
|
||||
fmt.Fprintf(&b, "… %d%s\n", omitted, elisionMarker)
|
||||
omitted = 0
|
||||
}
|
||||
}
|
||||
for i, line := range lines {
|
||||
if !keep[i] {
|
||||
omitted++
|
||||
continue
|
||||
}
|
||||
flush()
|
||||
b.WriteString(line)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
flush()
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func padded(prefix string, n int) []string {
|
||||
out := make([]string, n)
|
||||
for i := range n {
|
||||
out[i] = fmt.Sprintf("%s%03d", prefix, i)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The failure markers are a heuristic, so the toolchains a coding agent
|
||||
// actually runs are the coverage that matters: each case buries its detail in
|
||||
// the middle of otherwise unremarkable output.
|
||||
func TestSnipFailureAcrossToolchains(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
middle []string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "pytest",
|
||||
middle: []string{
|
||||
"=================================== FAILURES ===================================",
|
||||
"_____________________________ test_quoted_values ______________________________",
|
||||
"> assert config.dumps(cfg) == 'beta-7d21'",
|
||||
"E AssertionError: assert 'gamma-4a88' == 'beta-7d21'",
|
||||
"tests/test_format.py:412: AssertionError",
|
||||
},
|
||||
want: []string{"beta-7d21", "gamma-4a88", "test_format.py:412"},
|
||||
},
|
||||
{
|
||||
name: "cargo",
|
||||
middle: []string{
|
||||
"error[E0308]: mismatched types",
|
||||
" --> src/config.rs:412:9",
|
||||
"412 | beta_7d21",
|
||||
" | ^^^^^^^^^ expected `String`, found `&str`",
|
||||
},
|
||||
want: []string{"E0308", "config.rs:412", "expected"},
|
||||
},
|
||||
{
|
||||
name: "tsc",
|
||||
middle: []string{
|
||||
"src/config.ts(412,9): error TS2322: Type 'string' is not assignable to type 'number'.",
|
||||
},
|
||||
want: []string{"TS2322", "config.ts(412,9)"},
|
||||
},
|
||||
{
|
||||
name: "jest",
|
||||
middle: []string{
|
||||
" ● Config › round-trips quoted values",
|
||||
" expect(received).toBe(expected) // Object.is equality",
|
||||
" Expected: \"beta-7d21\"",
|
||||
" Received: \"gamma-4a88\"",
|
||||
" at Object.<anonymous> (src/config.test.ts:412:24)",
|
||||
},
|
||||
want: []string{"beta-7d21", "gamma-4a88", "config.test.ts:412"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
lines := padded("collecting item ", 120)
|
||||
lines = append(lines, tc.middle...)
|
||||
lines = append(lines, padded("ok case ", 120)...)
|
||||
content := strings.Join(lines, "\n")
|
||||
|
||||
got := snipFailureResult(content)
|
||||
if got == content {
|
||||
t.Fatalf("nothing elided from %d lines of mostly-quiet output", len(lines))
|
||||
}
|
||||
for _, want := range tc.want {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("detail %q did not survive:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if len(got) >= len(content) {
|
||||
t.Errorf("result grew: %d -> %d bytes", len(content), len(got))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func passingLog(n int) []string {
|
||||
out := make([]string, 0, n*2)
|
||||
for i := range n {
|
||||
out = append(out,
|
||||
fmt.Sprintf("=== RUN TestConfigCase%03d", i),
|
||||
fmt.Sprintf("--- PASS: TestConfigCase%03d (0.00s)", i))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSnipFailureKeepsTheAssertionAndDropsThePasses(t *testing.T) {
|
||||
lines := passingLog(60)
|
||||
lines = append(lines,
|
||||
"=== RUN TestQuotedValues",
|
||||
" format_test.go:412: assertion failed: expected beta-7d21, got gamma-4a88",
|
||||
"--- FAIL: TestQuotedValues (0.01s)")
|
||||
lines = append(lines, passingLog(60)...)
|
||||
content := strings.Join(lines, "\n")
|
||||
|
||||
got := snipFailureResult(content)
|
||||
if got == content {
|
||||
t.Fatal("nothing was elided from a mostly-passing log")
|
||||
}
|
||||
for _, want := range []string{"beta-7d21", "gamma-4a88", "format_test.go:412"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("failure detail %q did not survive", want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(got, "lines omitted") {
|
||||
t.Error("no elision marker: the reader cannot tell content was dropped")
|
||||
}
|
||||
if len(got) >= len(content) {
|
||||
t.Errorf("result grew: %d -> %d bytes", len(content), len(got))
|
||||
}
|
||||
if strings.Count(got, "PASS") > 2*failureContextLines+2 {
|
||||
t.Errorf("kept too many passing lines:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Small results are left alone: there is nothing to win and the elision marker
|
||||
// would cost more than it saves.
|
||||
func TestSnipFailureLeavesShortResultsWhole(t *testing.T) {
|
||||
content := "error: build failed\n./main.go:12:5: undefined: foo\nexit status 1"
|
||||
if got := snipFailureResult(content); got != content {
|
||||
t.Errorf("short result rewritten:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A long result with no recognisable failure line is returned untouched rather
|
||||
// than emptied — the caller decides what to do with it.
|
||||
func TestSnipFailureKeepsUnrecognisedOutputWhole(t *testing.T) {
|
||||
content := strings.Join(passingLog(40), "\n")
|
||||
if got := snipFailureResult(content); got != content {
|
||||
t.Errorf("output with no failure markers was rewritten:\n%.200s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// keptForProjection runs on every fold, so shortening must reach a fixed point
|
||||
// on the first pass. A second pass that swallowed the elision marker and
|
||||
// restated the count would make a twice-folded projection differ byte for byte
|
||||
// from a once-folded one, breaking the prompt cache for nothing.
|
||||
func TestSnipFailureIsIdempotent(t *testing.T) {
|
||||
allFailures := make([]string, 200)
|
||||
for i := range allFailures {
|
||||
allFailures[i] = fmt.Sprintf("--- FAIL: TestCase%03d (0.00s)", i)
|
||||
}
|
||||
mixed := passingLog(60)
|
||||
mixed = append(mixed, " format_test.go:412: assertion failed: expected beta-7d21, got gamma-4a88")
|
||||
mixed = append(mixed, passingLog(60)...)
|
||||
|
||||
for name, lines := range map[string][]string{"all failures": allFailures, "one failure": mixed} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
once := snipFailureResult(strings.Join(lines, "\n"))
|
||||
twice := snipFailureResult(once)
|
||||
if once != twice {
|
||||
t.Errorf("not a fixed point:\n--- once ---\n%s\n--- twice ---\n%s", once, twice)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A log that is failures throughout must still shrink, or the ceiling does not
|
||||
// bound anything.
|
||||
func TestSnipFailureBoundsAnAllFailureLog(t *testing.T) {
|
||||
lines := make([]string, 0, 400)
|
||||
for i := range 400 {
|
||||
lines = append(lines, fmt.Sprintf("--- FAIL: TestCase%03d (0.00s)", i))
|
||||
}
|
||||
content := strings.Join(lines, "\n")
|
||||
got := snipFailureResult(content)
|
||||
if got == content {
|
||||
t.Fatal("an all-failure log was not bounded")
|
||||
}
|
||||
if n := strings.Count(got, "FAIL"); n > failureMaxKeptLines {
|
||||
t.Errorf("kept %d failure lines, over the %d ceiling", n, failureMaxKeptLines)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/provider"
|
||||
"reasonix/internal/tool"
|
||||
)
|
||||
|
||||
func exitCode(n int) *int { return &n }
|
||||
|
||||
// A failing `go test` is the case prefix matching cannot see: the output opens
|
||||
// with "=== RUN" and the failure only shows in the recorded execution.
|
||||
func TestKeepErrorsSeesStructuredFailure(t *testing.T) {
|
||||
failing := "=== RUN TestQuotedValues\n format_test.go:412: expected beta-7d21, got gamma-4a88\n--- FAIL: TestQuotedValues (0.01s)\nFAIL\n"
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ex *provider.ToolExecution
|
||||
want bool
|
||||
}{
|
||||
{"state failed", &provider.ToolExecution{State: tool.ShellStateFailed}, true},
|
||||
{"timed out", &provider.ToolExecution{State: tool.ShellStateTimedOut}, true},
|
||||
{"non-zero exit", &provider.ToolExecution{State: tool.ShellStateCompleted, ExitCode: exitCode(1)}, true},
|
||||
{"verification failed", &provider.ToolExecution{Verification: tool.ShellVerificationFailed}, true},
|
||||
{"clean run", &provider.ToolExecution{State: tool.ShellStateCompleted, ExitCode: exitCode(0)}, false},
|
||||
{"no execution record", nil, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
m := provider.Message{Role: provider.RoleTool, Content: failing, ToolExecution: tc.ex}
|
||||
if got := isErrorMessage(m); got != tc.want {
|
||||
t.Errorf("isErrorMessage = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The text fallback still classifies host-formatted failures, and a passing run
|
||||
// must not be kept just because it mentions a failure in passing.
|
||||
func TestKeepErrorsTextFallbackUnchanged(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
content string
|
||||
want bool
|
||||
}{
|
||||
"error prefix": {"error: file not found", true},
|
||||
"blocked prefix": {"blocked: permission denied", true},
|
||||
"mentions failure": {"all tests pass; the earlier FAIL is fixed", false},
|
||||
"ordinary output": {"ok reasonix/config\t0.2s", false},
|
||||
"uppercase prefix": {"Error: broken", true},
|
||||
"prefix mid-string": {"note: error: not at the start", false},
|
||||
}
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
m := provider.Message{Role: provider.RoleTool, Content: tc.content}
|
||||
if got := isErrorMessage(m); got != tc.want {
|
||||
t.Errorf("isErrorMessage(%q) = %v, want %v", tc.content, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Only tool results are keep-policy candidates; a failing execution record on
|
||||
// another role must not promote it.
|
||||
func TestKeepErrorsIgnoresNonToolRoles(t *testing.T) {
|
||||
ex := &provider.ToolExecution{State: tool.ShellStateFailed, ExitCode: exitCode(2)}
|
||||
for _, role := range []provider.Role{provider.RoleUser, provider.RoleAssistant, provider.RoleSystem} {
|
||||
if isErrorMessage(provider.Message{Role: role, Content: "error: x", ToolExecution: ex}) {
|
||||
t.Errorf("role %v classified as an error tool result", role)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -223,6 +223,23 @@ func (a *Agent) persistCompactionStateLocked() error {
|
||||
return SaveCompactionState(a.sessionPath, a.compactionState)
|
||||
}
|
||||
|
||||
// maintenanceReplacement is the rewrite for one stale tool result. Both the
|
||||
// planning pass and the re-write that follows archiving go through it, so a
|
||||
// result cannot be shortened one way while planning and another way on install.
|
||||
// ok is false when the policy protects the message outright.
|
||||
func (a *Agent) maintenanceReplacement(m provider.Message, mode toolResultMaintenanceMode, archive string) (string, bool) {
|
||||
if a.keepPolicy&KeepErrors != 0 && isErrorMessage(m) {
|
||||
// A recorded failure stays recognisable once its text is rewritten, so
|
||||
// its passing noise can go. A text-only failure has no such anchor:
|
||||
// eliding it would hide that it was ever an error.
|
||||
if !failedExecution(m.ToolExecution) {
|
||||
return "", false
|
||||
}
|
||||
return snipFailureResult(m.Content), true
|
||||
}
|
||||
return rewriteToolResult(m, mode, archive, a.snipStrategyFor(m.Name)), true
|
||||
}
|
||||
|
||||
// applyToolResultMaintenanceView returns a copy of msgs with stale tool results
|
||||
// snipped or pruned. The canonical transcript is never modified.
|
||||
func (a *Agent) applyToolResultMaintenanceView(msgs []provider.Message, mode toolResultMaintenanceMode) ([]provider.Message, PruneStats) {
|
||||
@@ -249,10 +266,10 @@ func (a *Agent) applyToolResultMaintenanceView(msgs []provider.Message, mode too
|
||||
if !shouldMaintainToolResult(m, mode) {
|
||||
continue
|
||||
}
|
||||
if a.keepPolicy&KeepErrors != 0 && isErrorMessage(m) {
|
||||
replacement, ok := a.maintenanceReplacement(m, mode, "projection-view")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
replacement := rewriteToolResult(m, mode, "projection-view", a.snipStrategyFor(m.Name))
|
||||
if replacement == m.Content {
|
||||
continue
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user