mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 10:45:59 -10:00
feat(compaction): report user turns the retention budget could not hold (#8424)
Retention landed with no way to see it fail. A turn past the budget still folds into the digest, and the projection reads exactly the same either way — so the one case where compaction hands a user's own words to the summarizer was the one case nobody could observe. That also left [[keep]] useless in practice: an escape hatch only helps someone told it exists at the moment it is needed. keepUserTurns now returns a userTurnRetention count that rides the existing telemetry to CompactionTelemetry (user_kept / user_dropped), and a committed checkpoint that dropped a turn emits a warning naming [[keep]]. The notice fires after commit, never on a rejected candidate: a checkpoint that was not installed folded nothing. The retention code moves to compact_user_turns.go — compact.go was 705 lines against the 800 ceiling, and this is one responsibility. partitionFoldForProjection drops its early/carried return values, which have been nil since content-driven summary replaced the hoist, rather than growing to five results. Coverage follows the two gaps this series left open: - A boot-level effect test drives the real Build assembly through a fold and asserts the constraint reaches the provider request, with a summarizer whose digest deliberately records nothing. Verified by mutation: stubbing keepUserTurns to a no-op turns it red. - Sub-agents inherit retention through subagentOptions, the single construction point they share; a test pins that inheritance and the child-scoped budget, since a parent's instructions are the child's constraints and nothing else in the child transcript records them. partitionCoversRegion additionally asserts the conservation law: every user turn in a region is either kept or counted as dropped, so a turn cannot go missing from the accounting the way it previously could from the projection.
This commit is contained in:
12 files changed
+384
-51
No files matched your search
+4
-1
@@ -358,7 +358,10 @@ digest, so a constraint keeps its protection across repeated compaction.
|
||||
|
||||
A turn past those bounds folds like any other content. Prefix it with `[[keep]]`
|
||||
(keep policy `user_marked`, on by default) to hold it verbatim regardless of
|
||||
size.
|
||||
size. That drop is never silent: compaction telemetry carries `user_kept` and
|
||||
`user_dropped` counts, and a committed checkpoint that had to fold one of your
|
||||
turns emits a warning naming `[[keep]]` — the projection reads as complete
|
||||
either way, so the count is the only thing that distinguishes them.
|
||||
|
||||
Two properties bound that loss. Each fold re-derives its digest from the
|
||||
canonical transcript rather than from the previous digest, so digests do not
|
||||
|
||||
@@ -162,6 +162,9 @@ transcript,仅在唯一自动阈值被跨越时安装 provider 可见的短 **
|
||||
工作区重新推导。预算是必须的:无上限地保留会把候选撑过验收天花板,使压缩直接
|
||||
失败而非降级。该保护不以最近一次 digest 为界,因此能跨多次压缩存活;超出预算的
|
||||
轮次可用 `[[keep]]` 前缀(keep 策略 `user_marked`,默认开启)强制原样保留。
|
||||
丢弃不是静默的:压缩 telemetry 带 `user_kept` / `user_dropped` 计数,且已提交的
|
||||
checkpoint 若折叠了用户轮次会发出提示 `[[keep]]` 的警告——两种情况下 projection
|
||||
读起来都是完整的,计数是唯一能区分它们的东西。
|
||||
- 用户可用 `reasonix config compact-ratio [--local] [VALUE]` 查看或修改阈值。
|
||||
项目配置优先于桌面与新 CLI 会话共用的用户全局配置。UI 始终展示**实际生效**值。
|
||||
- `max_output_tokens` 是独立的**本轮**输出上限。
|
||||
|
||||
@@ -305,7 +305,7 @@ func (a *Agent) fixedPinnableUserTurn(m provider.Message) bool {
|
||||
return fixedTokenEstimate(m) <= budget
|
||||
}
|
||||
|
||||
func (a *Agent) keepIndexes(region []provider.Message) []bool {
|
||||
func (a *Agent) keepIndexes(region []provider.Message) ([]bool, userTurnRetention) {
|
||||
keep := make([]bool, len(region))
|
||||
policyStart := 0
|
||||
for i, m := range region {
|
||||
@@ -320,7 +320,7 @@ func (a *Agent) keepIndexes(region []provider.Message) []bool {
|
||||
keep[i] = true
|
||||
}
|
||||
}
|
||||
a.keepUserTurns(region, keep)
|
||||
retention := a.keepUserTurns(region, keep)
|
||||
for i, m := range region {
|
||||
if !keep[i] {
|
||||
continue
|
||||
@@ -334,39 +334,7 @@ func (a *Agent) keepIndexes(region []provider.Message) []bool {
|
||||
keepToolCallGroup(region, keep, i)
|
||||
}
|
||||
}
|
||||
return keep
|
||||
}
|
||||
|
||||
// keepUserTurns protects the user's own words from summarizer judgement: a
|
||||
// constraint stated mid-session is unrecoverable once a digest drops it, while
|
||||
// the work it governs stays re-derivable from the workspace. Unlike the keep
|
||||
// policy it ignores policyStart — a bounded budget, not a fold horizon, is what
|
||||
// stops it growing.
|
||||
func (a *Agent) keepUserTurns(region []provider.Message, keep []bool) {
|
||||
budget := a.keptUserTurnsBudget()
|
||||
// Oldest-first: the recent tail already covers the newest turns verbatim,
|
||||
// and an old turn has survived more folds than a new one.
|
||||
for i, m := range region {
|
||||
if keep[i] || m.Role != provider.RoleUser || m.LocalOnly || isCompactionSummary(m) {
|
||||
continue
|
||||
}
|
||||
cost := fixedTokenEstimate(m)
|
||||
if cost > maxKeptUserTurnTokens || cost > budget {
|
||||
continue
|
||||
}
|
||||
keep[i] = true
|
||||
budget -= cost
|
||||
}
|
||||
}
|
||||
|
||||
// keptUserTurnsBudget caps what user turns may spend of the checkpoint. Hoisting
|
||||
// them unbounded is what made an earlier revision pad candidates past the
|
||||
// acceptance ceiling, which fails compaction outright rather than degrading it.
|
||||
func (a *Agent) keptUserTurnsBudget() int {
|
||||
if a.contextWindow <= 0 {
|
||||
return keptUserTurnsBudgetTokens
|
||||
}
|
||||
return min(keptUserTurnsBudgetTokens, int(float64(a.contextWindow)*keptUserTurnsWindowFrac))
|
||||
return keep, retention
|
||||
}
|
||||
|
||||
// fixedTokenEstimate is what every verbatim-retention decision measures with.
|
||||
|
||||
@@ -12,9 +12,15 @@ import (
|
||||
// every provider-visible message in the region lands in exactly one group.
|
||||
func partitionCoversRegion(t *testing.T, a *Agent, region []provider.Message) (kept, fold []provider.Message) {
|
||||
t.Helper()
|
||||
early, carried, kept, fold := a.partitionFoldForProjection(region)
|
||||
if len(early) != 0 || len(carried) != 0 {
|
||||
t.Fatalf("early=%d carried=%d, want both empty under content-driven summary", len(early), len(carried))
|
||||
kept, fold, retention := a.partitionFoldForProjection(region)
|
||||
userTurns := 0
|
||||
for _, m := range region {
|
||||
if m.Role == provider.RoleUser && !m.LocalOnly && !isCompactionSummary(m) {
|
||||
userTurns++
|
||||
}
|
||||
}
|
||||
if got := retention.Kept + retention.Dropped; got != userTurns {
|
||||
t.Errorf("retention accounts for %d user turns, region has %d", got, userTurns)
|
||||
}
|
||||
seen := map[string]int{}
|
||||
for _, group := range [][]provider.Message{kept, fold} {
|
||||
|
||||
@@ -401,7 +401,7 @@ func (a *Agent) compactToProjection(ctx context.Context, trigger, instructions s
|
||||
if !ok {
|
||||
return CompactionNoop, nil
|
||||
}
|
||||
_, _, kept, fold := a.partitionFoldForProjection(msgs[head:start])
|
||||
kept, fold, retention := a.partitionFoldForProjection(msgs[head:start])
|
||||
if len(fold) == 0 || (!force && !foldEconomics(fold)) {
|
||||
return CompactionNoop, nil
|
||||
}
|
||||
@@ -449,6 +449,7 @@ func (a *Agent) compactToProjection(ctx context.Context, trigger, instructions s
|
||||
projTokens := a.estimatedPromptTokens(projMsgs)
|
||||
fixedPrefixTokens = a.estimatedPromptTokens(msgs[:head])
|
||||
tele.ProjectionTokens = projTokens
|
||||
tele.UserTurnsKept, tele.UserTurnsDropped = retention.Kept, retention.Dropped
|
||||
a.emitCompactionTelemetry(tele)
|
||||
if err := a.acceptCheckpointCandidate(trigger, force, sourceTokens, projTokens, fixedPrefixTokens); err != nil {
|
||||
a.emitCompactionAborted(trigger)
|
||||
@@ -469,6 +470,8 @@ func (a *Agent) compactToProjection(ctx context.Context, trigger, instructions s
|
||||
a.sink.Emit(event.Event{Kind: event.CompactionDone, Compaction: event.Compaction{
|
||||
Trigger: trigger, Messages: len(fold), Summary: summary,
|
||||
}})
|
||||
// Only once the checkpoint is committed: a rejected candidate folded nothing.
|
||||
a.noticeDroppedUserTurns(retention)
|
||||
return CompactionInstalled, nil
|
||||
}
|
||||
|
||||
@@ -551,8 +554,8 @@ func (a *Agent) planFoldRegion(msgs []provider.Message, force bool) (head, start
|
||||
return head, start, start > head
|
||||
}
|
||||
|
||||
func (a *Agent) partitionFoldForProjection(region []provider.Message) (early, carried, kept, fold []provider.Message) {
|
||||
policyKeep := a.keepIndexes(region)
|
||||
func (a *Agent) partitionFoldForProjection(region []provider.Message) (kept, fold []provider.Message, retention userTurnRetention) {
|
||||
policyKeep, retention := a.keepIndexes(region)
|
||||
for i, m := range region {
|
||||
switch {
|
||||
case m.LocalOnly: // display-only output never reaches a provider
|
||||
@@ -565,7 +568,7 @@ func (a *Agent) partitionFoldForProjection(region []provider.Message) (early, ca
|
||||
fold = append(fold, m)
|
||||
}
|
||||
}
|
||||
return nil, nil, kept, fold
|
||||
return kept, fold, retention
|
||||
}
|
||||
|
||||
// runCompactionSummary uses the single local summarizer path for every provider.
|
||||
|
||||
@@ -71,7 +71,7 @@ func foldRegionOf(a *Agent) []provider.Message {
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
_, _, _, fold := a.partitionFoldForProjection(msgs[head:start])
|
||||
_, fold, _ := a.partitionFoldForProjection(msgs[head:start])
|
||||
return fold
|
||||
}
|
||||
|
||||
|
||||
@@ -168,7 +168,7 @@ func TestKeepIndexesKeepsSiblingToolResultsForKeptError(t *testing.T) {
|
||||
{Role: provider.RoleTool, ToolCallID: "ok", Name: "read_file", Content: "package main"},
|
||||
}
|
||||
|
||||
keep := (&Agent{keepPolicy: KeepErrors}).keepIndexes(region)
|
||||
keep, _ := (&Agent{keepPolicy: KeepErrors}).keepIndexes(region)
|
||||
for i, kept := range keep {
|
||||
if !kept {
|
||||
t.Fatalf("keep[%d] = false, want all sibling tool-call messages kept: %v", i, keep)
|
||||
@@ -186,7 +186,7 @@ func TestKeepIndexesScopesPolicyAfterLatestSummary(t *testing.T) {
|
||||
{Role: provider.RoleTool, ToolCallID: "new", Name: "bash", Content: "error: new failure"},
|
||||
}
|
||||
|
||||
keep := (&Agent{keepPolicy: KeepErrors}).keepIndexes(region)
|
||||
keep, _ := (&Agent{keepPolicy: KeepErrors}).keepIndexes(region)
|
||||
want := []bool{false, false, false, true, true}
|
||||
for i := range want {
|
||||
if keep[i] != want[i] {
|
||||
@@ -515,9 +515,12 @@ func TestInterruptedDisplayStaysOutOfCompactionPromptAndProjection(t *testing.T)
|
||||
InterruptedTurn: &provider.InterruptedTurnRecovery{Pending: true},
|
||||
}
|
||||
a := &Agent{}
|
||||
early, carried, kept, fold := a.partitionFoldForProjection([]provider.Message{local})
|
||||
if len(early) != 0 || len(carried) != 0 || len(kept) != 0 || len(fold) != 0 {
|
||||
t.Fatalf("compaction partition early=%+v carried=%+v kept=%+v fold=%+v, want display-only output in none of them", early, carried, kept, fold)
|
||||
kept, fold, retention := a.partitionFoldForProjection([]provider.Message{local})
|
||||
if len(kept) != 0 || len(fold) != 0 {
|
||||
t.Fatalf("compaction partition kept=%+v fold=%+v, want display-only output in neither", kept, fold)
|
||||
}
|
||||
if retention.Kept != 0 || retention.Dropped != 0 {
|
||||
t.Fatalf("retention = %+v, want display-only output counted as neither kept nor dropped", retention)
|
||||
}
|
||||
if transcript := renderTranscript([]provider.Message{local}); transcript != "" {
|
||||
t.Fatalf("local interrupted output leaked into compaction prompt: %q", transcript)
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
// userTurnRetention is what one fold could and could not hold verbatim. A
|
||||
// dropped turn is the loss compaction cannot undo — the workspace still holds
|
||||
// the code a constraint governs, nothing holds the constraint — so it is
|
||||
// counted and surfaced rather than absorbed.
|
||||
type userTurnRetention struct {
|
||||
Kept int
|
||||
Dropped int
|
||||
DroppedTokens int
|
||||
}
|
||||
|
||||
// keepUserTurns protects the user's own words from summarizer judgement: a
|
||||
// constraint stated mid-session is unrecoverable once a digest drops it, while
|
||||
// the work it governs stays re-derivable from the workspace. Unlike the keep
|
||||
// policy it ignores policyStart — a bounded budget, not a fold horizon, is what
|
||||
// stops it growing.
|
||||
func (a *Agent) keepUserTurns(region []provider.Message, keep []bool) userTurnRetention {
|
||||
budget := a.keptUserTurnsBudget()
|
||||
var ret userTurnRetention
|
||||
// Oldest-first: the recent tail already covers the newest turns verbatim,
|
||||
// and an old turn has survived more folds than a new one.
|
||||
for i, m := range region {
|
||||
if m.Role != provider.RoleUser || m.LocalOnly || isCompactionSummary(m) {
|
||||
continue
|
||||
}
|
||||
if keep[i] {
|
||||
// Already held by the keep policy — a [[keep]] marker, which is the
|
||||
// documented way past the size budget below.
|
||||
ret.Kept++
|
||||
continue
|
||||
}
|
||||
cost := fixedTokenEstimate(m)
|
||||
if cost > maxKeptUserTurnTokens || cost > budget {
|
||||
ret.Dropped++
|
||||
ret.DroppedTokens += cost
|
||||
continue
|
||||
}
|
||||
keep[i] = true
|
||||
budget -= cost
|
||||
ret.Kept++
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// keptUserTurnsBudget caps what user turns may spend of the checkpoint. Hoisting
|
||||
// them unbounded is what made an earlier revision pad candidates past the
|
||||
// acceptance ceiling, which fails compaction outright rather than degrading it.
|
||||
func (a *Agent) keptUserTurnsBudget() int {
|
||||
if a.contextWindow <= 0 {
|
||||
return keptUserTurnsBudgetTokens
|
||||
}
|
||||
return min(keptUserTurnsBudgetTokens, int(float64(a.contextWindow)*keptUserTurnsWindowFrac))
|
||||
}
|
||||
|
||||
// noticeDroppedUserTurns reports the turns the budget could not hold. Without
|
||||
// it the drop is invisible: the projection reads as complete, and the escape
|
||||
// hatch is only useful to someone told it exists at the moment it is needed.
|
||||
func (a *Agent) noticeDroppedUserTurns(ret userTurnRetention) {
|
||||
if ret.Dropped == 0 {
|
||||
return
|
||||
}
|
||||
a.sink.Emit(event.Event{Kind: event.Notice, Level: event.LevelWarn,
|
||||
Text: fmt.Sprintf("%s of yours %s too large to keep whole and now survive only through the summary. Prefix a turn with [[keep]] to hold it verbatim.",
|
||||
pluralTurns(ret.Dropped), wereOrWas(ret.Dropped)),
|
||||
Detail: fmt.Sprintf("compaction dropped %d user turn(s) (~%d tokens) past the retention budget of %d",
|
||||
ret.Dropped, ret.DroppedTokens, a.keptUserTurnsBudget())})
|
||||
}
|
||||
|
||||
func pluralTurns(n int) string {
|
||||
if n == 1 {
|
||||
return "1 earlier message"
|
||||
}
|
||||
return fmt.Sprintf("%d earlier messages", n)
|
||||
}
|
||||
|
||||
func wereOrWas(n int) string {
|
||||
if n == 1 {
|
||||
return "was"
|
||||
}
|
||||
return "were"
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/provider"
|
||||
"reasonix/internal/tool"
|
||||
)
|
||||
|
||||
// retentionSession puts one user turn of the given size in the fold region,
|
||||
// behind enough assistant work that the recent tail cannot reach it.
|
||||
func retentionSession(midTurn string) *Session {
|
||||
big := strings.Repeat("work output line with detail. ", 250)
|
||||
return &Session{Messages: []provider.Message{
|
||||
{Role: provider.RoleSystem, Content: "sys"},
|
||||
{Role: provider.RoleUser, Content: "first task"},
|
||||
{Role: provider.RoleAssistant, Content: big},
|
||||
{Role: provider.RoleTool, ToolCallID: "1", Name: "read_file", Content: big},
|
||||
{Role: provider.RoleUser, Content: midTurn},
|
||||
{Role: provider.RoleAssistant, Content: big},
|
||||
{Role: provider.RoleTool, ToolCallID: "2", Name: "read_file", Content: big},
|
||||
{Role: provider.RoleUser, Content: "next"},
|
||||
{Role: provider.RoleAssistant, Content: "ok"},
|
||||
}}
|
||||
}
|
||||
|
||||
func compactWithSink(t *testing.T, sess *Session) []event.Event {
|
||||
t.Helper()
|
||||
var got []event.Event
|
||||
sink := event.FuncSink(func(e event.Event) { got = append(got, e) })
|
||||
a := New(&fakeProvider{reply: "digest"}, tool.NewRegistry(), sess,
|
||||
Options{ContextWindow: 8_000, CompactRatio: 0.85, RecentKeep: 2}, sink)
|
||||
if err := a.compact(context.Background(), "manual", "", true); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func noticeMentioning(events []event.Event, substr string) (event.Event, bool) {
|
||||
for _, e := range events {
|
||||
if e.Kind == event.Notice && strings.Contains(e.Text+e.Detail, substr) {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return event.Event{}, false
|
||||
}
|
||||
|
||||
// A turn past the budget is the one case where compaction still hands a user's
|
||||
// own words to the summarizer. That has to be visible: the projection reads as
|
||||
// complete either way, so silence here is indistinguishable from success.
|
||||
func TestCompactionReportsDroppedUserTurns(t *testing.T) {
|
||||
oversize := strings.Repeat("constraint detail. ", 500) // ~2375 tokens, past the per-turn ceiling
|
||||
events := compactWithSink(t, retentionSession(oversize))
|
||||
|
||||
notice, ok := noticeMentioning(events, "[[keep]]")
|
||||
if !ok {
|
||||
t.Fatalf("a dropped user turn was not reported; events=%+v", noticeTexts(events))
|
||||
}
|
||||
if notice.Level != event.LevelWarn {
|
||||
t.Errorf("dropped-turn notice level = %v, want warn", notice.Level)
|
||||
}
|
||||
tele, ok := noticeMentioning(events, "user_dropped=")
|
||||
if !ok {
|
||||
t.Fatal("compaction telemetry carries no user-turn retention counts")
|
||||
}
|
||||
if !strings.Contains(tele.Detail, "user_dropped=1") {
|
||||
t.Errorf("telemetry detail = %q, want user_dropped=1", tele.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// The notice must stay rare enough to mean something: a fold that kept every
|
||||
// user turn has nothing to warn about.
|
||||
func TestCompactionSilentWhenEveryUserTurnKept(t *testing.T) {
|
||||
events := compactWithSink(t, retentionSession("by the way, always use pnpm not npm"))
|
||||
|
||||
if _, ok := noticeMentioning(events, "[[keep]]"); ok {
|
||||
t.Errorf("warned about dropped turns when none were dropped; events=%+v", noticeTexts(events))
|
||||
}
|
||||
tele, ok := noticeMentioning(events, "user_kept=")
|
||||
if !ok {
|
||||
t.Fatal("compaction telemetry carries no user-turn retention counts")
|
||||
}
|
||||
if !strings.Contains(tele.Detail, "user_dropped=0") {
|
||||
t.Errorf("telemetry detail = %q, want user_dropped=0", tele.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// A sub-agent's "user turns" are the parent's instructions, and nothing else in
|
||||
// the child transcript records them — so the protection has to travel down the
|
||||
// one construction point sub-agents share. This pins the inheritance rather than
|
||||
// the mechanism, which compact_partition_test.go already covers.
|
||||
func TestSubagentInheritsUserTurnRetention(t *testing.T) {
|
||||
parent := &TaskTool{keepPolicy: KeepErrors | KeepUserMarked, recentKeep: 2, compactRatio: 0.85}
|
||||
opts := parent.subagentOptions(context.Background(), 8, nil, 32_000, 1, "", nil)
|
||||
if opts.KeepPolicy != KeepErrors|KeepUserMarked {
|
||||
t.Fatalf("child KeepPolicy = %v, want the parent's", opts.KeepPolicy)
|
||||
}
|
||||
if opts.ContextWindow != 32_000 {
|
||||
t.Fatalf("child ContextWindow = %d, want the resolved sub-session window", opts.ContextWindow)
|
||||
}
|
||||
|
||||
child := New(&fakeProvider{reply: "ok"}, tool.NewRegistry(), &Session{}, opts, event.Discard)
|
||||
if got, want := child.keptUserTurnsBudget(), int(32_000*keptUserTurnsWindowFrac); got != want {
|
||||
t.Fatalf("child retention budget = %d, want %d scaled to its own window", got, want)
|
||||
}
|
||||
kept, _, retention := child.partitionFoldForProjection([]provider.Message{
|
||||
{Role: provider.RoleUser, Content: "parent instruction: do not touch the public API"},
|
||||
{Role: provider.RoleAssistant, Content: "child work"},
|
||||
})
|
||||
if retention.Kept != 1 || len(kept) != 1 {
|
||||
t.Fatalf("kept=%d retention=%+v, want the parent's instruction held verbatim", len(kept), retention)
|
||||
}
|
||||
}
|
||||
|
||||
func noticeTexts(events []event.Event) []string {
|
||||
var out []string
|
||||
for _, e := range events {
|
||||
if e.Kind == event.Notice {
|
||||
out = append(out, e.Text)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -123,9 +123,10 @@ func (a *Agent) recordContextMaintenanceOutcome(inputHash, trigger, action, stat
|
||||
}
|
||||
|
||||
func (a *Agent) emitCompactionTelemetry(t CompactionTelemetry) {
|
||||
detail := fmt.Sprintf("trigger=%s mode=%s cache=%s src=%d fold=%d spans=%d proj=%d in=%d out=%d hit=%d miss=%d write=%d reqs=%d",
|
||||
detail := fmt.Sprintf("trigger=%s mode=%s cache=%s src=%d fold=%d spans=%d proj=%d in=%d out=%d hit=%d miss=%d write=%d reqs=%d user_kept=%d user_dropped=%d",
|
||||
t.Trigger, t.Mode, t.CacheState, t.SourceTokens, t.FoldTokens, t.Spans, t.ProjectionTokens,
|
||||
t.InputTokens, t.OutputTokens, t.CacheHitTokens, t.CacheMissTokens, t.CacheWriteTokens, t.RequestCount)
|
||||
t.InputTokens, t.OutputTokens, t.CacheHitTokens, t.CacheMissTokens, t.CacheWriteTokens, t.RequestCount,
|
||||
t.UserTurnsKept, t.UserTurnsDropped)
|
||||
if t.ProviderRequestID != "" {
|
||||
detail += " provider_request_id=" + t.ProviderRequestID
|
||||
}
|
||||
|
||||
@@ -142,6 +142,8 @@ type CompactionTelemetry struct {
|
||||
FoldTokens int `json:"fold_tokens"` // summarizer input after any shortening
|
||||
Spans int `json:"spans"` // summarizer calls the fold needed; 1 unless it was split
|
||||
ProjectionTokens int `json:"projection_tokens"`
|
||||
UserTurnsKept int `json:"user_turns_kept"`
|
||||
UserTurnsDropped int `json:"user_turns_dropped"` // past the retention budget, now summary-only
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
CacheHitTokens int `json:"cache_hit_tokens"`
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package boot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
// compactionEffectProvider answers ordinary turns with enough text to drive the
|
||||
// window past the compaction trigger, and summarizer turns with a digest that
|
||||
// deliberately records nothing — so a constraint can only reach a later request
|
||||
// by surviving verbatim.
|
||||
type compactionEffectProvider struct {
|
||||
mu sync.Mutex
|
||||
reqs []provider.Request
|
||||
bulk string
|
||||
}
|
||||
|
||||
func (p *compactionEffectProvider) Name() string { return "boot-compaction-effect" }
|
||||
|
||||
func (p *compactionEffectProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.mu.Lock()
|
||||
p.reqs = append(p.reqs, req)
|
||||
p.mu.Unlock()
|
||||
text := p.bulk
|
||||
if len(req.Messages) > 0 && strings.Contains(req.Messages[0].Content, "compacting the earlier part") {
|
||||
text = "## Standing facts\n- none recorded"
|
||||
}
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: text}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (p *compactionEffectProvider) requests() []provider.Request {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return append([]provider.Request(nil), p.reqs...)
|
||||
}
|
||||
|
||||
// TestEffectConstraintSurvivesCompactionThroughRealBuild is the final-boundary
|
||||
// proof: a constraint stated on an early turn must still appear verbatim in a
|
||||
// provider request issued after compaction folded that turn's neighbourhood.
|
||||
// The summarizer here records nothing, so a digest cannot carry it — component
|
||||
// correctness in the partition is not the same as the model still being told.
|
||||
func TestEffectConstraintSurvivesCompactionThroughRealBuild(t *testing.T) {
|
||||
isolateConfigHome(t)
|
||||
dir := robustTempDir(t)
|
||||
t.Chdir(dir)
|
||||
|
||||
rec := &compactionEffectProvider{bulk: strings.Repeat("work output line with detail. ", 400)}
|
||||
provider.Register("boot-compaction-effect", func(provider.Config) (provider.Provider, error) {
|
||||
return rec, nil
|
||||
})
|
||||
// 32000 stays under the 64K threshold where the recent tail takes its 32K
|
||||
// floor, which would otherwise swallow the whole transcript and never fold.
|
||||
writeFile(t, dir, "reasonix.toml", `
|
||||
default_model = "test-model"
|
||||
|
||||
[agent]
|
||||
system_prompt = "BASE"
|
||||
compact_ratio = 0.5
|
||||
recent_keep = 2
|
||||
|
||||
[[providers]]
|
||||
name = "test-model"
|
||||
kind = "boot-compaction-effect"
|
||||
model = "x"
|
||||
context_window = 32000
|
||||
`)
|
||||
|
||||
ctrl, err := Build(context.Background(), Options{Sink: event.Discard})
|
||||
if err != nil {
|
||||
t.Fatalf("Build: %v", err)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
|
||||
// The constraint is the *second* user turn on purpose: the first one is
|
||||
// pinned into the stable prefix, so only a later turn exercises the fold-
|
||||
// region retention this test is about.
|
||||
const constraint = "standing constraint: never change the public API"
|
||||
for _, prompt := range []string{"start the task", constraint,
|
||||
"keep going", "keep going", "keep going", "keep going", "keep going"} {
|
||||
if err := ctrl.Run(context.Background(), prompt); err != nil {
|
||||
t.Fatalf("Run(%q): %v", prompt, err)
|
||||
}
|
||||
}
|
||||
|
||||
reqs := rec.requests()
|
||||
compacted := -1
|
||||
for i, req := range reqs {
|
||||
for _, m := range req.Messages {
|
||||
if strings.Contains(m.Content, "<compaction-summary>") {
|
||||
compacted = i
|
||||
}
|
||||
}
|
||||
}
|
||||
if compacted < 0 {
|
||||
t.Fatalf("no request carried a digest; the fixture never compacted (%d requests)", len(reqs))
|
||||
}
|
||||
|
||||
var found bool
|
||||
for _, m := range reqs[compacted].Messages {
|
||||
if strings.Contains(m.Content, constraint) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("constraint absent from the post-compaction request; the model was never told it.\nmessages=%s",
|
||||
messageDigest(reqs[compacted].Messages))
|
||||
}
|
||||
}
|
||||
|
||||
func messageDigest(msgs []provider.Message) string {
|
||||
var b strings.Builder
|
||||
for _, m := range msgs {
|
||||
content := m.Content
|
||||
if len(content) > 120 {
|
||||
content = content[:120] + "..."
|
||||
}
|
||||
b.WriteString("\n " + string(m.Role) + ": " + content)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
Reference in new issue
Block a user