Merge pull request #12322 from esengine/fix/summarizer-not-a-digest

fix(compaction): a summarizer reply that is not a briefing is a failed summary
This commit is contained in:
YHH authored and GitHub committed 2026-10-07 08:03:17 -07:00
commit 693d0fb9b7
32 files changed
+353 -47

No files matched your search

+1 -1
View File
@@ -36,7 +36,7 @@ func (p *digestProvider) Name() string { return "fixture-digest" }
func (p *digestProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
p.calls++
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Earlier work continued across several files; nothing outstanding was recorded."}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Earlier work\nEarlier work continued across several files; nothing outstanding was recorded."}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
@@ -5,6 +5,7 @@ export const FOLD_WHY: Record<string, string> = {
summary_timeout: "生成摘要的请求停滞了:连续 6 分钟没有任何输出",
summary_ceiling: "生成摘要的请求运行满 30 分钟仍未完成,已停止",
summary_truncated: "摘要在输出上限处被截断,没有采用",
summary_not_digest: "摘要模型没有按要求的标题给出摘要,没有采用",
summary_input_too_large: "待折叠的内容缩减后仍超过单次摘要请求的容量",
hook_refused: "扩展拒绝了这次折叠",
persist_failed: "折叠结果没能保存",
@@ -460,6 +460,7 @@ export const EN_SETTINGS: Record<string, string> = {
"生成摘要的请求运行满 30 分钟仍未完成,已停止":
"The request that writes the summary was still running after 30 minutes and was stopped",
"摘要在输出上限处被截断,没有采用": "The summary hit the output limit and was discarded",
"摘要模型没有按要求的标题给出摘要,没有采用": "The summarizer did not return a briefing under the required headings, so it was not used",
"待折叠的内容缩减后仍超过单次摘要请求的容量": "The content to fold still exceeds one summary request after shortening",
"摘要生成期间对话发生了变化,已放弃这次折叠": "The conversation changed while the summary was written, so this fold was dropped",
"摘要没有记下这段内容里的任何一处改动,已放弃这次折叠": "The summary recorded none of this span's changes, so this fold was dropped",
@@ -0,0 +1,126 @@
package boot
import (
"context"
"strings"
"sync"
"testing"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
)
const pseudoToolCall = "<||DSML|| calls>\n<||DSML|| invoke name=\"read_file\">\n<||DSML|| parameter name=\"path\">internal/a.go</||DSML|| parameter>\n</||DSML|| invoke>\n</||DSML|| calls>"
// notDigestProvider answers its first summary request the way a model carrying
// on the agent's work does, and later ones with a briefing.
type notDigestProvider struct {
mu sync.Mutex
reqs []provider.Request
summaries int
bulk string
}
func (p *notDigestProvider) Name() string { return "boot-not-digest" }
func (p *notDigestProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
p.mu.Lock()
p.reqs = append(p.reqs, req)
text := p.bulk
if isSummaryRequest(req) {
p.summaries++
text = "## Goal\n- keep working"
if p.summaries == 1 {
text = pseudoToolCall
}
}
p.mu.Unlock()
ch := make(chan provider.Chunk, 3)
ch <- provider.Chunk{Type: provider.ChunkText, Text: text}
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{PromptTokens: 100, CompletionTokens: 10, TotalTokens: 110}}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
}
// A summarizer answer that is not a briefing is a failed summary: it is typed,
// it is billed to the compaction, and the answer never reaches a later request
// as the digest. The request that asks for the briefing ends on that ask.
func TestEffectSummaryThatIsNotABriefingNeverBecomesTheDigest(t *testing.T) {
isolateConfigHome(t)
dir := robustTempDir(t)
t.Chdir(dir)
kind := uniqueKind("boot-not-digest")
rec := &notDigestProvider{bulk: strings.Repeat("work output line with detail. ", 400)}
provider.Register(kind, func(provider.Config) (provider.Provider, error) { return rec, nil })
writeFile(t, dir, "reasonix.toml", `
default_model = "test-model"
[agent]
system_prompt = "BASE"
compact_ratio = 0.5
recent_keep = 2
[[providers]]
name = "test-model"
kind = "`+kind+`"
model = "x"
context_window = 32000
`)
approveWorkspace(t, dir)
var mu sync.Mutex
var codes []string
var billed int
sink := event.FuncSink(func(e event.Event) {
mu.Lock()
defer mu.Unlock()
switch {
case e.Kind == event.ContextMaintenanceEvent && e.Maintenance != nil && e.Maintenance.Status == "failed":
codes = append(codes, e.Maintenance.Code)
case e.Kind == event.Usage && e.UsageSource == event.UsageSourceCompaction:
billed++
}
})
ctrl, err := Build(context.Background(), Options{Sink: sink})
if err != nil {
t.Fatalf("Build: %v", err)
}
defer ctrl.Close()
for i := range 16 {
if err := ctrl.Run(context.Background(), "keep going"); err != nil {
t.Fatalf("turn %d: %v", i, err)
}
}
mu.Lock()
gotCodes, gotBilled := append([]string(nil), codes...), billed
mu.Unlock()
if len(gotCodes) == 0 || gotCodes[0] != "summary_not_digest" {
t.Fatalf("failure codes %v, want the first to be summary_not_digest", gotCodes)
}
if gotBilled < 1 {
t.Fatalf("the rejected summary was not billed to the compaction (%d usage events)", gotBilled)
}
installed := false
for _, req := range rec.reqs {
if isSummaryRequest(req) {
last := req.Messages[len(req.Messages)-1]
if last.Role != provider.RoleUser || !strings.Contains(last.Content, "Reply now with the briefing") {
t.Fatalf("a summary request does not end on the ask: %q", last.Content[max(0, len(last.Content)-160):])
}
continue
}
for _, m := range req.Messages {
if strings.Contains(m.Content, "DSML") {
t.Fatalf("the pseudo tool call reached a later request:\n%s", messageDigest(req.Messages))
}
if strings.Contains(m.Content, "<compaction-summary>") && strings.Contains(m.Content, "keep working") {
installed = true
}
}
}
if !installed {
t.Fatalf("the retry after the rejection never installed a briefing (%d requests, %d summaries)", len(rec.reqs), rec.summaries)
}
}
@@ -31,7 +31,7 @@ func (p *promptSizeProvider) Stream(_ context.Context, req provider.Request) (<-
ch := make(chan provider.Chunk, 3)
if len(req.Messages) > 0 && strings.Contains(req.Messages[0].Content, "compacting the earlier part") {
p.digests++
ch <- provider.Chunk{Type: provider.ChunkText, Text: "SUMMARY of the earlier work."}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nSUMMARY of the earlier work."}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
+2 -1
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"os"
"path/filepath"
"reasonix/internal/runtime/agent/testutil"
"reflect"
"strings"
"sync"
@@ -40,7 +41,7 @@ func (p *effectRecordingProvider) Stream(ctx context.Context, req provider.Reque
p.rawInputs = append(p.rawInputs, agent.RawUserInput(ctx, ""))
p.mu.Unlock()
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: "ok"}
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, "ok")}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
+1
View File
@@ -31,6 +31,7 @@ var English = Messages{
"summary_timeout": "The request that writes the summary stalled: no output for 6 minutes",
"summary_ceiling": "The request that writes the summary was still running after 30 minutes and was stopped",
"summary_truncated": "The summary was cut off at the output limit and was not used",
"summary_not_digest": "The summarizer did not return a briefing under the required headings, so it was not used",
"summary_input_too_large": "The content to fold still exceeds one summary request after shortening",
"hook_refused": "An extension refused this fold",
"persist_failed": "The result of the fold could not be saved",
+1
View File
@@ -32,6 +32,7 @@ var Chinese = Messages{
"summary_timeout": "生成摘要的请求停滞了:连续 6 分钟没有任何输出",
"summary_ceiling": "生成摘要的请求运行满 30 分钟仍未完成,已停止",
"summary_truncated": "摘要在输出上限处被截断,没有采用",
"summary_not_digest": "摘要模型没有按要求的标题给出摘要,没有采用",
"summary_input_too_large": "待折叠的内容缩减后仍超过单次摘要请求的容量",
"hook_refused": "扩展拒绝了这次折叠",
"persist_failed": "折叠结果没能保存",
+1
View File
@@ -32,6 +32,7 @@ var ChineseTraditional = Messages{
"summary_timeout": "生成摘要的請求停滯了:連續 6 分鐘沒有任何輸出",
"summary_ceiling": "生成摘要的請求執行滿 30 分鐘仍未完成,已停止",
"summary_truncated": "摘要在輸出上限處被截斷,沒有採用",
"summary_not_digest": "摘要模型沒有按要求的標題給出摘要,沒有採用",
"summary_input_too_large": "待折疊的內容縮減後仍超過單次摘要請求的容量",
"hook_refused": "擴充功能拒絕了這次折疊",
"persist_failed": "折疊結果沒能儲存",
-23
View File
@@ -190,29 +190,6 @@ type ToolSchema struct {
Parameters json.RawMessage `json:"parameters"`
}
// Request is a single completion request.
type Request struct {
Messages []Message
Tools []ToolSchema
Temperature *float64 // nil = omit; non-nil = send the value, including 0
MaxTokens int
// ResponseFormat, when non-nil, asks the endpoint for structured JSON
// output (Responses: text.format.type=json_object). Nil omits the field
// entirely — the common path must stay byte-stable for prompt caching.
ResponseFormat *ResponseFormat `json:"ResponseFormat,omitempty"`
EffortOverride string `json:"EffortOverride,omitempty"` // per-call reasoning-depth override; adapters apply it only when the endpoint's effort vocabulary accepts it
// Mode is the session's model mode id. An adapter sends only a mode its
// config declared for this endpoint, so an unknown one reaches no wire.
Mode string `json:"Mode,omitempty"`
}
// ResponseFormat asks a provider to constrain its output shape.
type ResponseFormat struct {
// Type is the structured format: "json_object" is the only shape the
// Responses endpoints currently define (MiMo/DashScope/OpenAI).
Type string `json:"type"`
}
// Auto ladder for max_output_tokens=0. Bounds completion only; never compact_ratio.
const (
DefaultOrdinaryOutputTokens = 16 * 1024 // non-reasoning
+27
View File
@@ -0,0 +1,27 @@
package provider
// Request is a single completion request.
type Request struct {
Messages []Message
Tools []ToolSchema
Temperature *float64 // nil = omit; non-nil = send the value, including 0
MaxTokens int
// ResponseFormat, when non-nil, asks the endpoint for structured JSON
// output (Responses: text.format.type=json_object). Nil omits the field
// entirely — the common path must stay byte-stable for prompt caching.
ResponseFormat *ResponseFormat `json:"ResponseFormat,omitempty"`
EffortOverride string `json:"EffortOverride,omitempty"` // per-call reasoning-depth override; adapters apply it only when the endpoint's effort vocabulary accepts it
// Mode is the session's model mode id. An adapter sends only a mode its
// config declared for this endpoint, so an unknown one reaches no wire.
Mode string `json:"Mode,omitempty"`
// Summary marks the request that asks for a compaction briefing, so a
// double or an adapter can tell it from a turn without reading its prose.
Summary bool `json:"-"`
}
// ResponseFormat asks a provider to constrain its output shape.
type ResponseFormat struct {
// Type is the structured format: "json_object" is the only shape the
// Responses endpoints currently define (MiMo/DashScope/OpenAI).
Type string `json:"type"`
}
+2 -1
View File
@@ -628,10 +628,11 @@ func (a *contextWindow) summarize(ctx context.Context, region []provider.Message
req := provider.Request{
Messages: []provider.Message{
{Role: provider.RoleSystem, Content: sys},
{Role: provider.RoleUser, Content: renderTranscript(region)},
{Role: provider.RoleUser, Content: renderTranscript(region) + summaryClosingInstruction},
},
MaxTokens: maxOut,
EffortOverride: summaryEffort,
Summary: true,
Temperature: provider.OptionalTemperature(a.temperature),
}
if budget, clipped, budgetErr := a.effectiveOutputBudget(req); budgetErr != nil {
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"reasonix/internal/runtime/agent/testutil"
"reasonix/internal/state/sessionstore"
"slices"
"strings"
@@ -32,7 +33,7 @@ type scriptedSummarizer struct {
func (p *scriptedSummarizer) Name() string { return "scripted" }
func (p *scriptedSummarizer) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
func (p *scriptedSummarizer) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
p.mu.Lock()
at := p.calls
p.calls++
@@ -43,7 +44,7 @@ func (p *scriptedSummarizer) Stream(_ context.Context, _ provider.Request) (<-ch
r := p.replies[at]
ch := make(chan provider.Chunk, 4)
if r.text != "" {
ch <- provider.Chunk{Type: provider.ChunkText, Text: r.text}
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, r.text)}
}
if r.usage != nil {
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: r.usage}
@@ -3,6 +3,7 @@ package agent
import (
"context"
"fmt"
"reasonix/internal/runtime/agent/testutil"
"reasonix/internal/state/sessionstore"
"strings"
"testing"
@@ -23,7 +24,7 @@ func (p *countingProvider) Name() string { return "counting" }
func (p *countingProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
p.got = append(p.got, req)
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: fmt.Sprintf("%s %d", p.reply, len(p.got))}
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, fmt.Sprintf("%s %d", p.reply, len(p.got)))}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
@@ -2,6 +2,7 @@ package agent
import (
"context"
"reasonix/internal/runtime/agent/testutil"
"reasonix/internal/state/sessionstore"
"strings"
"testing"
@@ -84,7 +85,7 @@ func (p *recordingProvider) Name() string { return "recording" }
func (p *recordingProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
p.got = append(p.got, req)
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: p.reply}
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, p.reply)}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
@@ -61,7 +61,7 @@ func (m *loopMock) handler(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
if isSummarizeRequest(body) {
writeSSE(w, m.t,
streamChunk(deltaText("- goal: keep going\n- pending: continue the task")),
streamChunk(deltaText("## Goal\n- goal: keep going\n- pending: continue the task")),
finishChunk("stop"),
usageChunk(80, 30, 0, 80))
return
@@ -693,6 +693,9 @@ func (a *contextWindow) partitionFoldWith(region []provider.Message, policy Keep
// runCompactionSummary uses the single local summarizer path for every provider.
func (a *contextWindow) runCompactionSummary(ctx context.Context, fold []provider.Message, instructions string) (summary, mode string, usage *provider.Usage, providerReqID string, err error) {
summary, usage, err = a.summarizeOnce(ctx, fold, instructions)
if err == nil && !hasDigestHeading(summary) {
err = errSummaryNotDigest
}
if err != nil {
return "", CompactionModeSummarized, usage, "", err
}
@@ -134,7 +134,7 @@ func (p *deadlineProbe) Stream(ctx context.Context, _ provider.Request) (<-chan
p.remaining = time.Until(d)
}
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: "digest"}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Digest\ndigest"}
close(ch)
return ch, nil
}
@@ -27,7 +27,7 @@ type oneTurnMock struct {
func (m *oneTurnMock) handler(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
if isSummarizeRequest(body) {
writeSSE(w, m.t, streamChunk(deltaText("- digest")), finishChunk("stop"), usageChunk(80, 30, 0, 80))
writeSSE(w, m.t, streamChunk(deltaText("## Digest\n- digest")), finishChunk("stop"), usageChunk(80, 30, 0, 80))
return
}
promptTok := charsOf(decodeMessages(body)) / 4
+12 -1
View File
@@ -2,6 +2,7 @@ package agent
import (
"context"
"reasonix/internal/runtime/agent/testutil"
"reasonix/internal/state/sessionstore"
"slices"
"strings"
@@ -34,10 +35,20 @@ type fakeProvider struct {
got []provider.Message
streamErr error // when set, Stream emits a ChunkError instead of the reply
hang bool // when true, Stream returns a channel that never sends or closes
verbatim bool // when true, a summary request is answered with reply exactly as written
}
func (f *fakeProvider) Name() string { return "fake" }
// answer speaks the summary contract unless the test asks for the reply as
// written.
func (f *fakeProvider) answer(req provider.Request) string {
if f.verbatim {
return f.reply
}
return testutil.SummaryReply(req, f.reply)
}
func (f *fakeProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
f.got = req.Messages
if f.hang {
@@ -49,7 +60,7 @@ func (f *fakeProvider) Stream(_ context.Context, req provider.Request) (<-chan p
close(ch)
return ch, nil
}
ch <- provider.Chunk{Type: provider.ChunkText, Text: f.reply}
ch <- provider.Chunk{Type: provider.ChunkText, Text: f.answer(req)}
if f.promptTokens > 0 {
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{PromptTokens: f.promptTokens, TotalTokens: f.promptTokens}}
}
+4 -1
View File
@@ -22,6 +22,7 @@ const (
FailSummaryTimeout CompactionNoopReason = "summary_timeout"
FailSummaryCeiling CompactionNoopReason = "summary_ceiling"
FailSummaryTruncated CompactionNoopReason = "summary_truncated"
FailSummaryNotDigest CompactionNoopReason = "summary_not_digest"
FailSummaryInputTooLarge CompactionNoopReason = "summary_input_too_large"
FailContextChanged CompactionNoopReason = "context_changed"
FailHookRefused CompactionNoopReason = "hook_refused"
@@ -50,7 +51,7 @@ func compactionHookRefusal(err error) error {
func classifySummaryError(parent context.Context, err error) error {
switch {
case err == nil, errors.Is(err, context.Canceled), parent.Err() != nil,
errors.Is(err, errSummaryOutputTruncated), errors.Is(err, errSummaryInputTooLarge),
errors.Is(err, errSummaryOutputTruncated), errors.Is(err, errSummaryInputTooLarge), errors.Is(err, errSummaryNotDigest),
errors.Is(err, errSummaryTimeout), errors.Is(err, errSummaryCeiling):
return err
case errors.Is(err, context.DeadlineExceeded):
@@ -73,6 +74,8 @@ func compactionFailureCode(err error) CompactionNoopReason {
return rejected.code
case errors.Is(err, errSummaryOutputTruncated):
return FailSummaryTruncated
case errors.Is(err, errSummaryNotDigest):
return FailSummaryNotDigest
case errors.Is(err, errCompressStaleContext):
return FailContextChanged
case errors.Is(err, errSummaryInputTooLarge):
+3 -3
View File
@@ -268,13 +268,13 @@ func (p *singleflightCompressProvider) Stream(_ context.Context, _ provider.Requ
close(p.started)
go func() {
<-p.release
ch <- provider.Chunk{Type: provider.ChunkText, Text: "summary"}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nsummary"}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
}()
return ch, nil
}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "duplicate summary"}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nduplicate summary"}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
@@ -287,7 +287,7 @@ func (p *staleCompressProvider) Stream(_ context.Context, _ provider.Request) (<
close(p.started)
go func() {
<-p.release
ch <- provider.Chunk{Type: provider.ChunkText, Text: "summary"}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nsummary"}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
}()
+1 -1
View File
@@ -26,7 +26,7 @@ func (a *contextWindow) contextMaintenanceInputHash(visible []provider.Message)
// the input or the install, so retrying the same input cannot change it.
func transientSummaryFailure(code string) bool {
switch CompactionNoopReason(code) {
case FailSummaryTruncated, FailSummaryTimeout, FailSummaryCeiling, FailSummaryFailed:
case FailSummaryTruncated, FailSummaryTimeout, FailSummaryCeiling, FailSummaryFailed, FailSummaryNotDigest:
return true
}
return false
+2 -2
View File
@@ -1113,7 +1113,7 @@ func TestToolAfterFailurePolicy(t *testing.T) {
func newCompactionAgent(t *testing.T, d *dispatch.Dispatcher) (*mockProvider, *Agent) {
t.Helper()
mp := &mockProvider{name: "p", chunks: []provider.Chunk{
{Type: provider.ChunkText, Text: "SUMMARY TEXT"}, {Type: provider.ChunkDone},
{Type: provider.ChunkText, Text: "## Summary\nSUMMARY TEXT"}, {Type: provider.ChunkDone},
}}
sess := sessionstore.NewSession("sys")
big := strings.Repeat("a", 8000)
@@ -1243,7 +1243,7 @@ func TestCompactionCompleteReplace(t *testing.T) {
if err := json.Unmarshal(payload, &in); err != nil {
return protocol.InterceptResult{}, err
}
if in.Summary != "SUMMARY TEXT" {
if in.Summary != "## Summary\nSUMMARY TEXT" {
t.Errorf("complete payload summary = %q, want the produced summary", in.Summary)
}
return replaceWith(t, dispatch.CompactionCompletePayload{Summary: "EXTENSION SUMMARY"}), nil
+1 -1
View File
@@ -30,7 +30,7 @@ func (p *sharedWindowTestProvider) Stream(_ context.Context, req provider.Reques
p.last = req
p.calls++
ch := make(chan provider.Chunk, 3)
ch <- provider.Chunk{Type: provider.ChunkText, Text: "summary"}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nsummary"}
if p.finish != "" {
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: p.finish}}
}
+26
View File
@@ -0,0 +1,26 @@
package agent
import (
"errors"
"strings"
)
// summaryClosingInstruction ends the summarizer's user turn. The transcript
// before it ends on the last tool output, which reads as a conversation
// waiting for the agent's next move.
const summaryClosingInstruction = "\n\n---\nThe transcript above is complete. Do not continue the work and do not call tools. Reply now with the briefing, under the required headings."
var errSummaryNotDigest = errors.New("summarizer did not return a briefing under the required headings")
// hasDigestHeading reports whether text has an ATX heading line (one to six
// hashes), the shape the summary prompt asks for in level 2. A request that
// offers no tools can still be answered with the agent's next move; that
// answer has no such line.
func hasDigestHeading(text string) bool {
for line := range strings.SplitSeq(text, "\n") {
if hashes := len(line) - len(strings.TrimLeft(line, "#")); hashes >= 1 && hashes <= 6 && strings.HasPrefix(line[hashes:], " ") {
return true
}
}
return false
}
@@ -0,0 +1,99 @@
package agent
import (
"context"
"strings"
"testing"
"reasonix/internal/base/testenv"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
"reasonix/internal/contract/tool"
"reasonix/internal/state/sessionstore"
)
func readOnlyWorkSession() *sessionstore.Session {
sess := &sessionstore.Session{Messages: []provider.Message{
{Role: provider.RoleSystem, Content: "sys"},
{Role: provider.RoleUser, Content: "task"},
}}
for round := range 6 {
id := string(rune('a' + round))
sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: "read_file", Arguments: `{"path":"f.go"}`}}})
sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: strings.Repeat("package f // line\n", 200)})
}
sess.Add(provider.Message{Role: provider.RoleUser, Content: "continue"})
return sess
}
// The summarizer is offered no tools, so an answer that is not a briefing under
// the required headings is the model carrying on the agent's work. A read-only
// fold has no changed file for the coverage guard to compare, which is why such
// an answer used to be installed as the digest.
func TestSummaryThatIsNotABriefingIsNeverInstalled(t *testing.T) {
for name, reply := range map[string]string{
"pseudo tool call": "<||DSML|| calls>\n<||DSML|| invoke name=\"read_file\">\n</||DSML|| invoke>",
"prose": "I'll add the doc comment to the next package.",
} {
t.Run(name, func(t *testing.T) {
sess := readOnlyWorkSession()
a := New(&fakeProvider{reply: reply, verbatim: true}, tool.NewRegistry(), sess,
Options{ContextWindow: 20000, CompactRatio: 0.8, RecentKeep: 2, ArchiveDir: testenv.TempDir(t)}, event.Discard)
outcome, _, err := a.window().compactToProjection(context.Background(), CompactionTriggerManual, "", compactionScope{ignoreThreshold: true, ignoreEconomics: true}, false)
if outcome == CompactionInstalled {
t.Fatalf("a reply that is not a briefing was installed as the digest: %v", visibleContext(a))
}
if got := CompactionFailureCode(err); got != FailSummaryNotDigest {
t.Fatalf("failure code %q (err %v), want %q", got, err, FailSummaryNotDigest)
}
if !transientSummaryFailure(string(FailSummaryNotDigest)) {
t.Fatal("a model answering off-task is variance, so a later attempt must be allowed")
}
})
}
}
// Under hard pressure the fold has to free space, so the same reply degrades to
// the host's own statement that no summary exists instead of failing the turn.
func TestSummaryThatIsNotABriefingDegradesWhenTheFoldMustFree(t *testing.T) {
sess := readOnlyWorkSession()
a := New(&fakeProvider{reply: "<invoke name=\"read_file\"></invoke>", verbatim: true}, tool.NewRegistry(), sess,
Options{ContextWindow: 20000, CompactRatio: 0.8, RecentKeep: 2, ArchiveDir: testenv.TempDir(t)}, event.Discard)
outcome, _, err := a.window().compactToProjection(context.Background(), CompactionTriggerOverflow, "", compactionScope{ignoreThreshold: true, ignoreEconomics: true}, true)
if err != nil || outcome != CompactionInstalled {
t.Fatalf("outcome %v, err %v", outcome, err)
}
for _, m := range visibleContext(a) {
if strings.Contains(m.Content, "invoke name") {
t.Fatalf("the off-task reply reached the projection: %q", m.Content)
}
}
}
// The transcript is one user message that ends on the last tool output, which
// reads as a conversation waiting for its next move. The request has to end on
// what it wants.
func TestSummaryRequestEndsWithItsInstruction(t *testing.T) {
sess := readOnlyWorkSession()
prov := &fakeProvider{reply: "## Goal\nread the files"}
a := New(prov, tool.NewRegistry(), sess,
Options{ContextWindow: 20000, CompactRatio: 0.8, RecentKeep: 2, ArchiveDir: testenv.TempDir(t)}, event.Discard)
if outcome, reason, err := a.window().compactToProjection(context.Background(), CompactionTriggerManual, "", compactionScope{ignoreThreshold: true, ignoreEconomics: true}, false); err != nil || outcome != CompactionInstalled {
t.Fatalf("outcome %v, reason %q, err %v", outcome, reason, err)
}
last := prov.got[len(prov.got)-1]
if last.Role != provider.RoleUser || !strings.HasSuffix(last.Content, summaryClosingInstruction) {
t.Fatalf("summary request ends with %q, want the closing instruction", last.Content[max(0, len(last.Content)-200):])
}
}
func TestAnyAtxHeadingIsTheDigestShape(t *testing.T) {
for text, want := range map[string]bool{
"# Goal\nx": true, "## Goal\nx": true, "###### x": true, "preamble\n### Files\n- a": true,
"####### seven": false, "#nospace": false, "plain prose": false, "": false,
} {
if got := hasDigestHeading(text); got != want {
t.Errorf("hasDigestHeading(%q) = %v, want %v", text, got, want)
}
}
}
@@ -95,7 +95,7 @@ func (p *MockProvider) Stream(ctx context.Context, req provider.Request) (<-chan
chunks = append(chunks, provider.Chunk{Type: provider.ChunkReasoning, Text: t.Reasoning})
}
if t.Text != "" {
chunks = append(chunks, provider.Chunk{Type: provider.ChunkText, Text: t.Text})
chunks = append(chunks, provider.Chunk{Type: provider.ChunkText, Text: SummaryReply(req, t.Text)})
}
for i := range t.ToolCalls {
tc := t.ToolCalls[i]
@@ -0,0 +1,23 @@
package testutil
import (
"strings"
"reasonix/internal/contract/provider"
)
// SummaryReply is what a double that follows the summary contract answers: the
// summarizer prompt requires headings, and the host refuses a reply with
// none. A plain reply to a summary request gets one, so a test that only cares
// about the digest's text keeps reading it.
func SummaryReply(req provider.Request, text string) string {
if !req.Summary {
return text
}
for line := range strings.SplitSeq(text, "\n") {
if hashes := len(line) - len(strings.TrimLeft(line, "#")); hashes >= 1 && hashes <= 6 && strings.HasPrefix(line[hashes:], " ") {
return text
}
}
return "## Summary\n" + text
}
+1 -1
View File
@@ -24,7 +24,7 @@ func (p *summaryCounter) Name() string { return "summary-counter" }
func (p *summaryCounter) Stream(context.Context, provider.Request) (<-chan provider.Chunk, error) {
p.calls.Add(1)
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: "SUMMARY: earlier work, condensed."}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nSUMMARY: earlier work, condensed."}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
+2 -1
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path/filepath"
"reasonix/internal/runtime/agent/testutil"
"reasonix/internal/state/sessionstore"
"reflect"
"slices"
@@ -63,7 +64,7 @@ func (p *scriptedProvider) Stream(ctx context.Context, req provider.Request) (<-
ch <- provider.Chunk{Type: provider.ChunkReasoning, Text: resp.reasoning}
}
if resp.text != "" {
ch <- provider.Chunk{Type: provider.ChunkText, Text: resp.text}
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, resp.text)}
}
if resp.usage != nil {
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: resp.usage}
+1 -1
View File
@@ -184,7 +184,7 @@ func runTwoTurns(t *testing.T) (*Controller, *agent.Agent, *[]event.Event) {
prov := &scriptedTurns{turns: [][]provider.Chunk{
textTurn("first answer"),
textTurn("second answer"),
textTurn("edited answer"),
textTurn("## Summary\nedited answer"),
}}
ag := agent.New(prov, tool.NewRegistry(), sessionstore.NewSession("sys"), agent.Options{}, event.Discard)
var events []event.Event