mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 10:45:59 -10:00
Merge pull request #12322 from esengine/fix/summarizer-not-a-digest
fix(compaction): a summarizer reply that is not a briefing is a failed summary
This commit is contained in:
32 files changed
+353
-47
No files matched your search
@@ -36,7 +36,7 @@ func (p *digestProvider) Name() string { return "fixture-digest" }
|
||||
func (p *digestProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.calls++
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Earlier work continued across several files; nothing outstanding was recorded."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Earlier work\nEarlier work continued across several files; nothing outstanding was recorded."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
|
||||
@@ -5,6 +5,7 @@ export const FOLD_WHY: Record<string, string> = {
|
||||
summary_timeout: "生成摘要的请求停滞了:连续 6 分钟没有任何输出",
|
||||
summary_ceiling: "生成摘要的请求运行满 30 分钟仍未完成,已停止",
|
||||
summary_truncated: "摘要在输出上限处被截断,没有采用",
|
||||
summary_not_digest: "摘要模型没有按要求的标题给出摘要,没有采用",
|
||||
summary_input_too_large: "待折叠的内容缩减后仍超过单次摘要请求的容量",
|
||||
hook_refused: "扩展拒绝了这次折叠",
|
||||
persist_failed: "折叠结果没能保存",
|
||||
|
||||
@@ -460,6 +460,7 @@ export const EN_SETTINGS: Record<string, string> = {
|
||||
"生成摘要的请求运行满 30 分钟仍未完成,已停止":
|
||||
"The request that writes the summary was still running after 30 minutes and was stopped",
|
||||
"摘要在输出上限处被截断,没有采用": "The summary hit the output limit and was discarded",
|
||||
"摘要模型没有按要求的标题给出摘要,没有采用": "The summarizer did not return a briefing under the required headings, so it was not used",
|
||||
"待折叠的内容缩减后仍超过单次摘要请求的容量": "The content to fold still exceeds one summary request after shortening",
|
||||
"摘要生成期间对话发生了变化,已放弃这次折叠": "The conversation changed while the summary was written, so this fold was dropped",
|
||||
"摘要没有记下这段内容里的任何一处改动,已放弃这次折叠": "The summary recorded none of this span's changes, so this fold was dropped",
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package boot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
const pseudoToolCall = "<||DSML|| calls>\n<||DSML|| invoke name=\"read_file\">\n<||DSML|| parameter name=\"path\">internal/a.go</||DSML|| parameter>\n</||DSML|| invoke>\n</||DSML|| calls>"
|
||||
|
||||
// notDigestProvider answers its first summary request the way a model carrying
|
||||
// on the agent's work does, and later ones with a briefing.
|
||||
type notDigestProvider struct {
|
||||
mu sync.Mutex
|
||||
reqs []provider.Request
|
||||
summaries int
|
||||
bulk string
|
||||
}
|
||||
|
||||
func (p *notDigestProvider) Name() string { return "boot-not-digest" }
|
||||
|
||||
func (p *notDigestProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.mu.Lock()
|
||||
p.reqs = append(p.reqs, req)
|
||||
text := p.bulk
|
||||
if isSummaryRequest(req) {
|
||||
p.summaries++
|
||||
text = "## Goal\n- keep working"
|
||||
if p.summaries == 1 {
|
||||
text = pseudoToolCall
|
||||
}
|
||||
}
|
||||
p.mu.Unlock()
|
||||
ch := make(chan provider.Chunk, 3)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: text}
|
||||
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{PromptTokens: 100, CompletionTokens: 10, TotalTokens: 110}}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// A summarizer answer that is not a briefing is a failed summary: it is typed,
|
||||
// it is billed to the compaction, and the answer never reaches a later request
|
||||
// as the digest. The request that asks for the briefing ends on that ask.
|
||||
func TestEffectSummaryThatIsNotABriefingNeverBecomesTheDigest(t *testing.T) {
|
||||
isolateConfigHome(t)
|
||||
dir := robustTempDir(t)
|
||||
t.Chdir(dir)
|
||||
kind := uniqueKind("boot-not-digest")
|
||||
rec := ¬DigestProvider{bulk: strings.Repeat("work output line with detail. ", 400)}
|
||||
provider.Register(kind, func(provider.Config) (provider.Provider, error) { return rec, nil })
|
||||
writeFile(t, dir, "reasonix.toml", `
|
||||
default_model = "test-model"
|
||||
|
||||
[agent]
|
||||
system_prompt = "BASE"
|
||||
compact_ratio = 0.5
|
||||
recent_keep = 2
|
||||
|
||||
[[providers]]
|
||||
name = "test-model"
|
||||
kind = "`+kind+`"
|
||||
model = "x"
|
||||
context_window = 32000
|
||||
`)
|
||||
approveWorkspace(t, dir)
|
||||
|
||||
var mu sync.Mutex
|
||||
var codes []string
|
||||
var billed int
|
||||
sink := event.FuncSink(func(e event.Event) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch {
|
||||
case e.Kind == event.ContextMaintenanceEvent && e.Maintenance != nil && e.Maintenance.Status == "failed":
|
||||
codes = append(codes, e.Maintenance.Code)
|
||||
case e.Kind == event.Usage && e.UsageSource == event.UsageSourceCompaction:
|
||||
billed++
|
||||
}
|
||||
})
|
||||
ctrl, err := Build(context.Background(), Options{Sink: sink})
|
||||
if err != nil {
|
||||
t.Fatalf("Build: %v", err)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
for i := range 16 {
|
||||
if err := ctrl.Run(context.Background(), "keep going"); err != nil {
|
||||
t.Fatalf("turn %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
gotCodes, gotBilled := append([]string(nil), codes...), billed
|
||||
mu.Unlock()
|
||||
if len(gotCodes) == 0 || gotCodes[0] != "summary_not_digest" {
|
||||
t.Fatalf("failure codes %v, want the first to be summary_not_digest", gotCodes)
|
||||
}
|
||||
if gotBilled < 1 {
|
||||
t.Fatalf("the rejected summary was not billed to the compaction (%d usage events)", gotBilled)
|
||||
}
|
||||
installed := false
|
||||
for _, req := range rec.reqs {
|
||||
if isSummaryRequest(req) {
|
||||
last := req.Messages[len(req.Messages)-1]
|
||||
if last.Role != provider.RoleUser || !strings.Contains(last.Content, "Reply now with the briefing") {
|
||||
t.Fatalf("a summary request does not end on the ask: %q", last.Content[max(0, len(last.Content)-160):])
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, m := range req.Messages {
|
||||
if strings.Contains(m.Content, "DSML") {
|
||||
t.Fatalf("the pseudo tool call reached a later request:\n%s", messageDigest(req.Messages))
|
||||
}
|
||||
if strings.Contains(m.Content, "<compaction-summary>") && strings.Contains(m.Content, "keep working") {
|
||||
installed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !installed {
|
||||
t.Fatalf("the retry after the rejection never installed a briefing (%d requests, %d summaries)", len(rec.reqs), rec.summaries)
|
||||
}
|
||||
}
|
||||
@@ -31,7 +31,7 @@ func (p *promptSizeProvider) Stream(_ context.Context, req provider.Request) (<-
|
||||
ch := make(chan provider.Chunk, 3)
|
||||
if len(req.Messages) > 0 && strings.Contains(req.Messages[0].Content, "compacting the earlier part") {
|
||||
p.digests++
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "SUMMARY of the earlier work."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nSUMMARY of the earlier work."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reasonix/internal/runtime/agent/testutil"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -40,7 +41,7 @@ func (p *effectRecordingProvider) Stream(ctx context.Context, req provider.Reque
|
||||
p.rawInputs = append(p.rawInputs, agent.RawUserInput(ctx, ""))
|
||||
p.mu.Unlock()
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "ok"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, "ok")}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
|
||||
@@ -31,6 +31,7 @@ var English = Messages{
|
||||
"summary_timeout": "The request that writes the summary stalled: no output for 6 minutes",
|
||||
"summary_ceiling": "The request that writes the summary was still running after 30 minutes and was stopped",
|
||||
"summary_truncated": "The summary was cut off at the output limit and was not used",
|
||||
"summary_not_digest": "The summarizer did not return a briefing under the required headings, so it was not used",
|
||||
"summary_input_too_large": "The content to fold still exceeds one summary request after shortening",
|
||||
"hook_refused": "An extension refused this fold",
|
||||
"persist_failed": "The result of the fold could not be saved",
|
||||
|
||||
@@ -32,6 +32,7 @@ var Chinese = Messages{
|
||||
"summary_timeout": "生成摘要的请求停滞了:连续 6 分钟没有任何输出",
|
||||
"summary_ceiling": "生成摘要的请求运行满 30 分钟仍未完成,已停止",
|
||||
"summary_truncated": "摘要在输出上限处被截断,没有采用",
|
||||
"summary_not_digest": "摘要模型没有按要求的标题给出摘要,没有采用",
|
||||
"summary_input_too_large": "待折叠的内容缩减后仍超过单次摘要请求的容量",
|
||||
"hook_refused": "扩展拒绝了这次折叠",
|
||||
"persist_failed": "折叠结果没能保存",
|
||||
|
||||
@@ -32,6 +32,7 @@ var ChineseTraditional = Messages{
|
||||
"summary_timeout": "生成摘要的請求停滯了:連續 6 分鐘沒有任何輸出",
|
||||
"summary_ceiling": "生成摘要的請求執行滿 30 分鐘仍未完成,已停止",
|
||||
"summary_truncated": "摘要在輸出上限處被截斷,沒有採用",
|
||||
"summary_not_digest": "摘要模型沒有按要求的標題給出摘要,沒有採用",
|
||||
"summary_input_too_large": "待折疊的內容縮減後仍超過單次摘要請求的容量",
|
||||
"hook_refused": "擴充功能拒絕了這次折疊",
|
||||
"persist_failed": "折疊結果沒能儲存",
|
||||
|
||||
@@ -190,29 +190,6 @@ type ToolSchema struct {
|
||||
Parameters json.RawMessage `json:"parameters"`
|
||||
}
|
||||
|
||||
// Request is a single completion request.
|
||||
type Request struct {
|
||||
Messages []Message
|
||||
Tools []ToolSchema
|
||||
Temperature *float64 // nil = omit; non-nil = send the value, including 0
|
||||
MaxTokens int
|
||||
// ResponseFormat, when non-nil, asks the endpoint for structured JSON
|
||||
// output (Responses: text.format.type=json_object). Nil omits the field
|
||||
// entirely — the common path must stay byte-stable for prompt caching.
|
||||
ResponseFormat *ResponseFormat `json:"ResponseFormat,omitempty"`
|
||||
EffortOverride string `json:"EffortOverride,omitempty"` // per-call reasoning-depth override; adapters apply it only when the endpoint's effort vocabulary accepts it
|
||||
// Mode is the session's model mode id. An adapter sends only a mode its
|
||||
// config declared for this endpoint, so an unknown one reaches no wire.
|
||||
Mode string `json:"Mode,omitempty"`
|
||||
}
|
||||
|
||||
// ResponseFormat asks a provider to constrain its output shape.
|
||||
type ResponseFormat struct {
|
||||
// Type is the structured format: "json_object" is the only shape the
|
||||
// Responses endpoints currently define (MiMo/DashScope/OpenAI).
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// Auto ladder for max_output_tokens=0. Bounds completion only; never compact_ratio.
|
||||
const (
|
||||
DefaultOrdinaryOutputTokens = 16 * 1024 // non-reasoning
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package provider
|
||||
|
||||
// Request is a single completion request.
|
||||
type Request struct {
|
||||
Messages []Message
|
||||
Tools []ToolSchema
|
||||
Temperature *float64 // nil = omit; non-nil = send the value, including 0
|
||||
MaxTokens int
|
||||
// ResponseFormat, when non-nil, asks the endpoint for structured JSON
|
||||
// output (Responses: text.format.type=json_object). Nil omits the field
|
||||
// entirely — the common path must stay byte-stable for prompt caching.
|
||||
ResponseFormat *ResponseFormat `json:"ResponseFormat,omitempty"`
|
||||
EffortOverride string `json:"EffortOverride,omitempty"` // per-call reasoning-depth override; adapters apply it only when the endpoint's effort vocabulary accepts it
|
||||
// Mode is the session's model mode id. An adapter sends only a mode its
|
||||
// config declared for this endpoint, so an unknown one reaches no wire.
|
||||
Mode string `json:"Mode,omitempty"`
|
||||
// Summary marks the request that asks for a compaction briefing, so a
|
||||
// double or an adapter can tell it from a turn without reading its prose.
|
||||
Summary bool `json:"-"`
|
||||
}
|
||||
|
||||
// ResponseFormat asks a provider to constrain its output shape.
|
||||
type ResponseFormat struct {
|
||||
// Type is the structured format: "json_object" is the only shape the
|
||||
// Responses endpoints currently define (MiMo/DashScope/OpenAI).
|
||||
Type string `json:"type"`
|
||||
}
|
||||
@@ -628,10 +628,11 @@ func (a *contextWindow) summarize(ctx context.Context, region []provider.Message
|
||||
req := provider.Request{
|
||||
Messages: []provider.Message{
|
||||
{Role: provider.RoleSystem, Content: sys},
|
||||
{Role: provider.RoleUser, Content: renderTranscript(region)},
|
||||
{Role: provider.RoleUser, Content: renderTranscript(region) + summaryClosingInstruction},
|
||||
},
|
||||
MaxTokens: maxOut,
|
||||
EffortOverride: summaryEffort,
|
||||
Summary: true,
|
||||
Temperature: provider.OptionalTemperature(a.temperature),
|
||||
}
|
||||
if budget, clipped, budgetErr := a.effectiveOutputBudget(req); budgetErr != nil {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reasonix/internal/runtime/agent/testutil"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -32,7 +33,7 @@ type scriptedSummarizer struct {
|
||||
|
||||
func (p *scriptedSummarizer) Name() string { return "scripted" }
|
||||
|
||||
func (p *scriptedSummarizer) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
|
||||
func (p *scriptedSummarizer) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.mu.Lock()
|
||||
at := p.calls
|
||||
p.calls++
|
||||
@@ -43,7 +44,7 @@ func (p *scriptedSummarizer) Stream(_ context.Context, _ provider.Request) (<-ch
|
||||
r := p.replies[at]
|
||||
ch := make(chan provider.Chunk, 4)
|
||||
if r.text != "" {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: r.text}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, r.text)}
|
||||
}
|
||||
if r.usage != nil {
|
||||
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: r.usage}
|
||||
|
||||
@@ -3,6 +3,7 @@ package agent
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reasonix/internal/runtime/agent/testutil"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -23,7 +24,7 @@ func (p *countingProvider) Name() string { return "counting" }
|
||||
func (p *countingProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.got = append(p.got, req)
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: fmt.Sprintf("%s %d", p.reply, len(p.got))}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, fmt.Sprintf("%s %d", p.reply, len(p.got)))}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
|
||||
@@ -2,6 +2,7 @@ package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reasonix/internal/runtime/agent/testutil"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -84,7 +85,7 @@ func (p *recordingProvider) Name() string { return "recording" }
|
||||
func (p *recordingProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.got = append(p.got, req)
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: p.reply}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, p.reply)}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
|
||||
@@ -61,7 +61,7 @@ func (m *loopMock) handler(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if isSummarizeRequest(body) {
|
||||
writeSSE(w, m.t,
|
||||
streamChunk(deltaText("- goal: keep going\n- pending: continue the task")),
|
||||
streamChunk(deltaText("## Goal\n- goal: keep going\n- pending: continue the task")),
|
||||
finishChunk("stop"),
|
||||
usageChunk(80, 30, 0, 80))
|
||||
return
|
||||
|
||||
@@ -693,6 +693,9 @@ func (a *contextWindow) partitionFoldWith(region []provider.Message, policy Keep
|
||||
// runCompactionSummary uses the single local summarizer path for every provider.
|
||||
func (a *contextWindow) runCompactionSummary(ctx context.Context, fold []provider.Message, instructions string) (summary, mode string, usage *provider.Usage, providerReqID string, err error) {
|
||||
summary, usage, err = a.summarizeOnce(ctx, fold, instructions)
|
||||
if err == nil && !hasDigestHeading(summary) {
|
||||
err = errSummaryNotDigest
|
||||
}
|
||||
if err != nil {
|
||||
return "", CompactionModeSummarized, usage, "", err
|
||||
}
|
||||
|
||||
@@ -134,7 +134,7 @@ func (p *deadlineProbe) Stream(ctx context.Context, _ provider.Request) (<-chan
|
||||
p.remaining = time.Until(d)
|
||||
}
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "digest"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Digest\ndigest"}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ type oneTurnMock struct {
|
||||
func (m *oneTurnMock) handler(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if isSummarizeRequest(body) {
|
||||
writeSSE(w, m.t, streamChunk(deltaText("- digest")), finishChunk("stop"), usageChunk(80, 30, 0, 80))
|
||||
writeSSE(w, m.t, streamChunk(deltaText("## Digest\n- digest")), finishChunk("stop"), usageChunk(80, 30, 0, 80))
|
||||
return
|
||||
}
|
||||
promptTok := charsOf(decodeMessages(body)) / 4
|
||||
|
||||
@@ -2,6 +2,7 @@ package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reasonix/internal/runtime/agent/testutil"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -34,10 +35,20 @@ type fakeProvider struct {
|
||||
got []provider.Message
|
||||
streamErr error // when set, Stream emits a ChunkError instead of the reply
|
||||
hang bool // when true, Stream returns a channel that never sends or closes
|
||||
verbatim bool // when true, a summary request is answered with reply exactly as written
|
||||
}
|
||||
|
||||
func (f *fakeProvider) Name() string { return "fake" }
|
||||
|
||||
// answer speaks the summary contract unless the test asks for the reply as
|
||||
// written.
|
||||
func (f *fakeProvider) answer(req provider.Request) string {
|
||||
if f.verbatim {
|
||||
return f.reply
|
||||
}
|
||||
return testutil.SummaryReply(req, f.reply)
|
||||
}
|
||||
|
||||
func (f *fakeProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
f.got = req.Messages
|
||||
if f.hang {
|
||||
@@ -49,7 +60,7 @@ func (f *fakeProvider) Stream(_ context.Context, req provider.Request) (<-chan p
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: f.reply}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: f.answer(req)}
|
||||
if f.promptTokens > 0 {
|
||||
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{PromptTokens: f.promptTokens, TotalTokens: f.promptTokens}}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ const (
|
||||
FailSummaryTimeout CompactionNoopReason = "summary_timeout"
|
||||
FailSummaryCeiling CompactionNoopReason = "summary_ceiling"
|
||||
FailSummaryTruncated CompactionNoopReason = "summary_truncated"
|
||||
FailSummaryNotDigest CompactionNoopReason = "summary_not_digest"
|
||||
FailSummaryInputTooLarge CompactionNoopReason = "summary_input_too_large"
|
||||
FailContextChanged CompactionNoopReason = "context_changed"
|
||||
FailHookRefused CompactionNoopReason = "hook_refused"
|
||||
@@ -50,7 +51,7 @@ func compactionHookRefusal(err error) error {
|
||||
func classifySummaryError(parent context.Context, err error) error {
|
||||
switch {
|
||||
case err == nil, errors.Is(err, context.Canceled), parent.Err() != nil,
|
||||
errors.Is(err, errSummaryOutputTruncated), errors.Is(err, errSummaryInputTooLarge),
|
||||
errors.Is(err, errSummaryOutputTruncated), errors.Is(err, errSummaryInputTooLarge), errors.Is(err, errSummaryNotDigest),
|
||||
errors.Is(err, errSummaryTimeout), errors.Is(err, errSummaryCeiling):
|
||||
return err
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
@@ -73,6 +74,8 @@ func compactionFailureCode(err error) CompactionNoopReason {
|
||||
return rejected.code
|
||||
case errors.Is(err, errSummaryOutputTruncated):
|
||||
return FailSummaryTruncated
|
||||
case errors.Is(err, errSummaryNotDigest):
|
||||
return FailSummaryNotDigest
|
||||
case errors.Is(err, errCompressStaleContext):
|
||||
return FailContextChanged
|
||||
case errors.Is(err, errSummaryInputTooLarge):
|
||||
|
||||
@@ -268,13 +268,13 @@ func (p *singleflightCompressProvider) Stream(_ context.Context, _ provider.Requ
|
||||
close(p.started)
|
||||
go func() {
|
||||
<-p.release
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "summary"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nsummary"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "duplicate summary"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nduplicate summary"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
@@ -287,7 +287,7 @@ func (p *staleCompressProvider) Stream(_ context.Context, _ provider.Request) (<
|
||||
close(p.started)
|
||||
go func() {
|
||||
<-p.release
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "summary"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nsummary"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
}()
|
||||
|
||||
@@ -26,7 +26,7 @@ func (a *contextWindow) contextMaintenanceInputHash(visible []provider.Message)
|
||||
// the input or the install, so retrying the same input cannot change it.
|
||||
func transientSummaryFailure(code string) bool {
|
||||
switch CompactionNoopReason(code) {
|
||||
case FailSummaryTruncated, FailSummaryTimeout, FailSummaryCeiling, FailSummaryFailed:
|
||||
case FailSummaryTruncated, FailSummaryTimeout, FailSummaryCeiling, FailSummaryFailed, FailSummaryNotDigest:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -1113,7 +1113,7 @@ func TestToolAfterFailurePolicy(t *testing.T) {
|
||||
func newCompactionAgent(t *testing.T, d *dispatch.Dispatcher) (*mockProvider, *Agent) {
|
||||
t.Helper()
|
||||
mp := &mockProvider{name: "p", chunks: []provider.Chunk{
|
||||
{Type: provider.ChunkText, Text: "SUMMARY TEXT"}, {Type: provider.ChunkDone},
|
||||
{Type: provider.ChunkText, Text: "## Summary\nSUMMARY TEXT"}, {Type: provider.ChunkDone},
|
||||
}}
|
||||
sess := sessionstore.NewSession("sys")
|
||||
big := strings.Repeat("a", 8000)
|
||||
@@ -1243,7 +1243,7 @@ func TestCompactionCompleteReplace(t *testing.T) {
|
||||
if err := json.Unmarshal(payload, &in); err != nil {
|
||||
return protocol.InterceptResult{}, err
|
||||
}
|
||||
if in.Summary != "SUMMARY TEXT" {
|
||||
if in.Summary != "## Summary\nSUMMARY TEXT" {
|
||||
t.Errorf("complete payload summary = %q, want the produced summary", in.Summary)
|
||||
}
|
||||
return replaceWith(t, dispatch.CompactionCompletePayload{Summary: "EXTENSION SUMMARY"}), nil
|
||||
|
||||
@@ -30,7 +30,7 @@ func (p *sharedWindowTestProvider) Stream(_ context.Context, req provider.Reques
|
||||
p.last = req
|
||||
p.calls++
|
||||
ch := make(chan provider.Chunk, 3)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "summary"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nsummary"}
|
||||
if p.finish != "" {
|
||||
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: p.finish}}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// summaryClosingInstruction ends the summarizer's user turn. The transcript
|
||||
// before it ends on the last tool output, which reads as a conversation
|
||||
// waiting for the agent's next move.
|
||||
const summaryClosingInstruction = "\n\n---\nThe transcript above is complete. Do not continue the work and do not call tools. Reply now with the briefing, under the required headings."
|
||||
|
||||
var errSummaryNotDigest = errors.New("summarizer did not return a briefing under the required headings")
|
||||
|
||||
// hasDigestHeading reports whether text has an ATX heading line (one to six
|
||||
// hashes), the shape the summary prompt asks for in level 2. A request that
|
||||
// offers no tools can still be answered with the agent's next move; that
|
||||
// answer has no such line.
|
||||
func hasDigestHeading(text string) bool {
|
||||
for line := range strings.SplitSeq(text, "\n") {
|
||||
if hashes := len(line) - len(strings.TrimLeft(line, "#")); hashes >= 1 && hashes <= 6 && strings.HasPrefix(line[hashes:], " ") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/base/testenv"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
)
|
||||
|
||||
func readOnlyWorkSession() *sessionstore.Session {
|
||||
sess := &sessionstore.Session{Messages: []provider.Message{
|
||||
{Role: provider.RoleSystem, Content: "sys"},
|
||||
{Role: provider.RoleUser, Content: "task"},
|
||||
}}
|
||||
for round := range 6 {
|
||||
id := string(rune('a' + round))
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: "read_file", Arguments: `{"path":"f.go"}`}}})
|
||||
sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: strings.Repeat("package f // line\n", 200)})
|
||||
}
|
||||
sess.Add(provider.Message{Role: provider.RoleUser, Content: "continue"})
|
||||
return sess
|
||||
}
|
||||
|
||||
// The summarizer is offered no tools, so an answer that is not a briefing under
|
||||
// the required headings is the model carrying on the agent's work. A read-only
|
||||
// fold has no changed file for the coverage guard to compare, which is why such
|
||||
// an answer used to be installed as the digest.
|
||||
func TestSummaryThatIsNotABriefingIsNeverInstalled(t *testing.T) {
|
||||
for name, reply := range map[string]string{
|
||||
"pseudo tool call": "<||DSML|| calls>\n<||DSML|| invoke name=\"read_file\">\n</||DSML|| invoke>",
|
||||
"prose": "I'll add the doc comment to the next package.",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
sess := readOnlyWorkSession()
|
||||
a := New(&fakeProvider{reply: reply, verbatim: true}, tool.NewRegistry(), sess,
|
||||
Options{ContextWindow: 20000, CompactRatio: 0.8, RecentKeep: 2, ArchiveDir: testenv.TempDir(t)}, event.Discard)
|
||||
outcome, _, err := a.window().compactToProjection(context.Background(), CompactionTriggerManual, "", compactionScope{ignoreThreshold: true, ignoreEconomics: true}, false)
|
||||
if outcome == CompactionInstalled {
|
||||
t.Fatalf("a reply that is not a briefing was installed as the digest: %v", visibleContext(a))
|
||||
}
|
||||
if got := CompactionFailureCode(err); got != FailSummaryNotDigest {
|
||||
t.Fatalf("failure code %q (err %v), want %q", got, err, FailSummaryNotDigest)
|
||||
}
|
||||
if !transientSummaryFailure(string(FailSummaryNotDigest)) {
|
||||
t.Fatal("a model answering off-task is variance, so a later attempt must be allowed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Under hard pressure the fold has to free space, so the same reply degrades to
|
||||
// the host's own statement that no summary exists instead of failing the turn.
|
||||
func TestSummaryThatIsNotABriefingDegradesWhenTheFoldMustFree(t *testing.T) {
|
||||
sess := readOnlyWorkSession()
|
||||
a := New(&fakeProvider{reply: "<invoke name=\"read_file\"></invoke>", verbatim: true}, tool.NewRegistry(), sess,
|
||||
Options{ContextWindow: 20000, CompactRatio: 0.8, RecentKeep: 2, ArchiveDir: testenv.TempDir(t)}, event.Discard)
|
||||
outcome, _, err := a.window().compactToProjection(context.Background(), CompactionTriggerOverflow, "", compactionScope{ignoreThreshold: true, ignoreEconomics: true}, true)
|
||||
if err != nil || outcome != CompactionInstalled {
|
||||
t.Fatalf("outcome %v, err %v", outcome, err)
|
||||
}
|
||||
for _, m := range visibleContext(a) {
|
||||
if strings.Contains(m.Content, "invoke name") {
|
||||
t.Fatalf("the off-task reply reached the projection: %q", m.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The transcript is one user message that ends on the last tool output, which
|
||||
// reads as a conversation waiting for its next move. The request has to end on
|
||||
// what it wants.
|
||||
func TestSummaryRequestEndsWithItsInstruction(t *testing.T) {
|
||||
sess := readOnlyWorkSession()
|
||||
prov := &fakeProvider{reply: "## Goal\nread the files"}
|
||||
a := New(prov, tool.NewRegistry(), sess,
|
||||
Options{ContextWindow: 20000, CompactRatio: 0.8, RecentKeep: 2, ArchiveDir: testenv.TempDir(t)}, event.Discard)
|
||||
if outcome, reason, err := a.window().compactToProjection(context.Background(), CompactionTriggerManual, "", compactionScope{ignoreThreshold: true, ignoreEconomics: true}, false); err != nil || outcome != CompactionInstalled {
|
||||
t.Fatalf("outcome %v, reason %q, err %v", outcome, reason, err)
|
||||
}
|
||||
last := prov.got[len(prov.got)-1]
|
||||
if last.Role != provider.RoleUser || !strings.HasSuffix(last.Content, summaryClosingInstruction) {
|
||||
t.Fatalf("summary request ends with %q, want the closing instruction", last.Content[max(0, len(last.Content)-200):])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnyAtxHeadingIsTheDigestShape(t *testing.T) {
|
||||
for text, want := range map[string]bool{
|
||||
"# Goal\nx": true, "## Goal\nx": true, "###### x": true, "preamble\n### Files\n- a": true,
|
||||
"####### seven": false, "#nospace": false, "plain prose": false, "": false,
|
||||
} {
|
||||
if got := hasDigestHeading(text); got != want {
|
||||
t.Errorf("hasDigestHeading(%q) = %v, want %v", text, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -95,7 +95,7 @@ func (p *MockProvider) Stream(ctx context.Context, req provider.Request) (<-chan
|
||||
chunks = append(chunks, provider.Chunk{Type: provider.ChunkReasoning, Text: t.Reasoning})
|
||||
}
|
||||
if t.Text != "" {
|
||||
chunks = append(chunks, provider.Chunk{Type: provider.ChunkText, Text: t.Text})
|
||||
chunks = append(chunks, provider.Chunk{Type: provider.ChunkText, Text: SummaryReply(req, t.Text)})
|
||||
}
|
||||
for i := range t.ToolCalls {
|
||||
tc := t.ToolCalls[i]
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// SummaryReply is what a double that follows the summary contract answers: the
|
||||
// summarizer prompt requires headings, and the host refuses a reply with
|
||||
// none. A plain reply to a summary request gets one, so a test that only cares
|
||||
// about the digest's text keeps reading it.
|
||||
func SummaryReply(req provider.Request, text string) string {
|
||||
if !req.Summary {
|
||||
return text
|
||||
}
|
||||
for line := range strings.SplitSeq(text, "\n") {
|
||||
if hashes := len(line) - len(strings.TrimLeft(line, "#")); hashes >= 1 && hashes <= 6 && strings.HasPrefix(line[hashes:], " ") {
|
||||
return text
|
||||
}
|
||||
}
|
||||
return "## Summary\n" + text
|
||||
}
|
||||
@@ -24,7 +24,7 @@ func (p *summaryCounter) Name() string { return "summary-counter" }
|
||||
func (p *summaryCounter) Stream(context.Context, provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.calls.Add(1)
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "SUMMARY: earlier work, condensed."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Summary\nSUMMARY: earlier work, condensed."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reasonix/internal/runtime/agent/testutil"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"reflect"
|
||||
"slices"
|
||||
@@ -63,7 +64,7 @@ func (p *scriptedProvider) Stream(ctx context.Context, req provider.Request) (<-
|
||||
ch <- provider.Chunk{Type: provider.ChunkReasoning, Text: resp.reasoning}
|
||||
}
|
||||
if resp.text != "" {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: resp.text}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: testutil.SummaryReply(req, resp.text)}
|
||||
}
|
||||
if resp.usage != nil {
|
||||
ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: resp.usage}
|
||||
|
||||
@@ -184,7 +184,7 @@ func runTwoTurns(t *testing.T) (*Controller, *agent.Agent, *[]event.Event) {
|
||||
prov := &scriptedTurns{turns: [][]provider.Chunk{
|
||||
textTurn("first answer"),
|
||||
textTurn("second answer"),
|
||||
textTurn("edited answer"),
|
||||
textTurn("## Summary\nedited answer"),
|
||||
}}
|
||||
ag := agent.New(prov, tool.NewRegistry(), sessionstore.NewSession("sys"), agent.Options{}, event.Discard)
|
||||
var events []event.Event
|
||||
|
||||
Reference in new issue
Block a user