From 00c8d9142ef990e4352da87789a895dc2b6875ec Mon Sep 17 00:00:00 2001 From: SivanCola <32437197+SivanCola@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:09:21 +0800 Subject: [PATCH] fix(agent): count triage usage in the root token budget Problem: an explicitly bounded root omitted billed command classifications and external-content screening, allowing extra normal tool rounds. Root cause: the shared triage terminal emitted usage without charging the existing context-bound root counter. Fix: observe terminal usage through the accepted private root helper before the optional display guard. Preserve verdicts, screening notices, event conditions, missing usage, normal telemetry and cancellation authority. Verification: all 239 root packages pass in disjoint commands; full Agent race and real boot HTTP/SSE matrices pass. The compiled preceding owner fails ten events while five controls pass. Static gates, four platform cross-compiles and the literal 15-event Windows contract pass locally. --- .github/workflows/ci.yml | 30 +++ docs/ROOT_TRIAGE_TOKEN_BUDGET.md | 89 +++++++++ .../boot/effect_root_triage_budget_test.go | 171 ++++++++++++++++++ internal/runtime/agent/command_class.go | 1 + .../runtime/agent/root_triage_budget_test.go | 119 ++++++++++++ .../14025-root-triage-token-usage.md | 2 + 6 files changed, 412 insertions(+) create mode 100644 docs/ROOT_TRIAGE_TOKEN_BUDGET.md create mode 100644 internal/assembly/boot/effect_root_triage_budget_test.go create mode 100644 internal/runtime/agent/root_triage_budget_test.go create mode 100644 release-notes/studio/unreleased/14025-root-triage-token-usage.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0eb3871fe6..86d448096d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -421,6 +421,36 @@ jobs: retention-days: 7 + - name: test (Windows root triage budget) + if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request' + timeout-minutes: 5 + shell: bash + run: | + set -o pipefail + go test -count=1 -timeout=3m -json -run '^Test(RootTriage|EffectRootTriage)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-triage-budget.jsonl" + jq -se ' + [.[] | select(.Test == null and .Action == "pass") | .Package] as $packages | + [.[] | select(.Test != null and .Action == "pass") | .Test] as $tests | + all(.[]; .Action != "skip" and .Action != "fail") and + (["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and + (["TestRootTriageTokenBudget", "TestRootTriageTokenBudget/classifier", "TestRootTriageTokenBudget/screen", + "TestRootTriageTokenBudget/billed_failure", "TestRootTriageTokenBudget/missing", "TestRootTriageTokenBudget/disabled", + "TestRootTriageTokenBudget/no_sink", "TestRootTriageTokenBudget/counts_without_total", + "TestRootTriageOutsideRunDoesNotCreateScope", "TestEffectRootTriageScreenBudget", + "TestEffectRootTriageScreenBudget/reported", "TestEffectRootTriageScreenBudget/flagged", + "TestEffectRootTriageScreenBudget/disabled", "TestEffectRootTriageScreenBudget/missing", + "TestEffectRootTriageScreenBudget/screen_disabled"] - $tests | length) == 0 + ' "$RUNNER_TEMP/root-triage-budget.jsonl" + + - name: upload Windows root triage budget inventory + if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: root-triage-budget-windows-${{ github.run_attempt }} + path: ${{ runner.temp }}/root-triage-budget.jsonl + if-no-files-found: ignore + retention-days: 7 + - name: test (Windows root auxiliary budget) if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request' timeout-minutes: 5 diff --git a/docs/ROOT_TRIAGE_TOKEN_BUDGET.md b/docs/ROOT_TRIAGE_TOKEN_BUDGET.md new file mode 100644 index 0000000000..d9729d3e93 --- /dev/null +++ b/docs/ROOT_TRIAGE_TOKEN_BUDGET.md @@ -0,0 +1,89 @@ +--- +owner: @SivanCola +backup: @esengine +status: active +reviewed: 2026-10-26 +--- + +# Root triage token accounting / 根任务 triage token 计账 + +Command classification and external-content screening share one triage terminal. +Its reported prompt/completion counts now charge the active context's existing +Agent-root token counter, including usage received before a stream failure. + +命令分类和外部内容筛查共用一个 triage 终态。其已报告的 prompt/completion 计数 +现在计入活跃 context 已有的 Agent 根 token counter，包括流失败前收到的用量。 + +## Ownership and compatibility / 所有权与兼容 + +The terminal uses the existing private root accounting helper. It reads the +request's scope, not mutable current-task state, and does not create a scope for +out-of-run classifications. Nested and budget-only projected contexts retain +the existing root lifetime and target-job cancellation authority. + +终态复用既有私有根计账 helper，读取请求 scope，不读取可变当前任务状态；Run +之外的分类不创建 scope。嵌套和仅投射预算的 context 沿用既有根寿命与目标 job +取消权限。 + +Root observation is independent of the optional sink and positive-total +emission condition. Positive prompt/completion counts still count when the +provider omits its total; event source, model, pricing and emission stay the same. +Missing usage stays unknown, with no invented token or HTTP counts. + +根观察不依赖可选 usage sink 或既有正 total 上报条件。供应商省略 total 时， +正 prompt/completion 计数仍扣计。既有事件 source、model、pricing 与上报条件 +保持不变，不推造缺失 token 或 HTTP 次数；缺失用量仍未知。 + +One classification verdict still governs the same cached read-only decision. +The screen remains advisory: CLEAN, INJECTION and failed screening do not block, +drop or rewrite the fetched result. An INJECTION verdict keeps its existing +host notice. The accounting change adds no permission or safety exception. + +分类结论继续控制同一缓存的只读判断。筛查仍为提示性质：CLEAN、INJECTION 或 +筛查失败都不阻断、丢弃或改写已获取结果。INJECTION 保留既有 host 提示。 +计账改动不新增权限或安全例外。 + +## Limit boundary / 阈值边界 + +An explicit token threshold may end the normal loop earlier, after a tool +round with existing finalization. Zero stays unbounded. No reservation, timer, +provider cancellation or process termination is added. Normal-round telemetry +and monetary completeness/currency rules stay unchanged. + +显式 token 阈值可能更早结束正常循环；检查仍在工具轮后执行，并保留既有收尾。 +本机制不做请求前预留、timer、供应商取消或进程终止。零值仍不限。辅助根计账 +不改变正常轮次局部遥测、金额完整性或币种规则。 + +The classification and screening prompts, model selection, output/time bounds, +verdict cache and default screening setting stay the same. No API, config, +persisted or provider-wire field is added. Serialized main prefixes and tool +schemas are unchanged. + +分类/筛查 prompt、模型选择、输出/时间边界、结论缓存与默认筛查开关保持原样。 +不新增 API、配置、持久化或 provider-wire 字段；主序列化前缀和工具 schema 不变。 + +## Evidence boundary / 证据边界 + +Real Agent regressions cover classifier/screen usage, billed failures, missing +counts, disabled limits, absent sinks and omitted totals. Boot HTTP/SSE uses an +actual file-reading tool with declared external provenance through screening. +This is a constructed fixture, not live web-fetch or a production incident. + +真实 Agent 回归覆盖分类/筛查用量、已计费失败、缺失计数、禁用阈值、无 sink 与 +省略 total。真实 boot HTTP/SSE 夹具使用声明外部来源的文件读取工具，执行实际 +筛查管线。这是构造的外部结果夹具，不是 live web-fetch 传输或生产事故复现。 + +The flagged control keeps both the actual file result and existing notice in +the next provider request. Disabled and missing-usage controls retain their +round counts. Serialized prefix equality in the fixture is a cache guard, +not a live-provider hit-rate measurement. + +flagged 对照保留实际文件结果及下一供应商请求中的既有提示。禁用与缺失用量 +对照保持轮次数。夹具中的序列化前缀相等是缓存守卫，不是供应商命中率实测。 + +This unit does not complete all auxiliary-provider admission, shared Coordinator +budgets, root currency aggregation, durable accounting or the full isolation +and recovery plan. See [root auxiliary tokens](ROOT_AUXILIARY_TOKEN_BUDGET.md). + +本单元不完成全部辅助 provider 准入、共享 Coordinator 预算、根币种汇总、持久 +计账或完整隔离恢复方案。作者分析、独立审查与原生平台证据分别记录。 diff --git a/internal/assembly/boot/effect_root_triage_budget_test.go b/internal/assembly/boot/effect_root_triage_budget_test.go new file mode 100644 index 0000000000..8e8e1fa3d5 --- /dev/null +++ b/internal/assembly/boot/effect_root_triage_budget_test.go @@ -0,0 +1,171 @@ +package boot + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "reasonix/internal/contract/event" + "reasonix/internal/contract/tool" + "reasonix/internal/runtime/agent" + "reasonix/internal/session/control" + "reasonix/internal/state/sessionstore" +) + +type triageFixtureRead struct { + path string + reads atomic.Int32 +} + +func (*triageFixtureRead) Name() string { return "triage_fixture" } +func (*triageFixtureRead) Description() string { return "Read the screening fixture." } +func (*triageFixtureRead) ReadOnly() bool { return true } +func (*triageFixtureRead) Schema() json.RawMessage { return json.RawMessage(`{"type":"object"}`) } +func (*triageFixtureRead) Provenance(json.RawMessage) tool.Provenance { + return tool.Provenance{Kind: tool.ProvenanceWeb, Source: "screening-fixture"} +} +func (p *triageFixtureRead) Execute(context.Context, json.RawMessage) (string, error) { + b, err := os.ReadFile(p.path) + if err == nil { + p.reads.Add(1) + } + return string(b), err +} + +func TestEffectRootTriageScreenBudget(t *testing.T) { + for _, tc := range []struct { + name string + limit int + missing bool + flag bool + screenOff bool + wantReads int32 + wantTriage int32 + }{ + {name: "reported", limit: 1500, wantReads: 1, wantTriage: 1}, + {name: "flagged", limit: 1500, flag: true, wantReads: 1, wantTriage: 1}, + {name: "disabled", wantReads: 4, wantTriage: 4}, + {name: "missing", limit: 1500, missing: true, wantReads: 2, wantTriage: 2}, + {name: "screen_disabled", limit: 1500, screenOff: true, wantReads: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + home, root := statedBootHome(t), t.TempDir() + writeFile(t, root, "fixture.txt", "screening fixture content") + var normal, triage, notices atomic.Int32 + var sawNotice atomic.Bool + var mainPrefix atomic.Pointer[string] + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Model string `json:"model"` + Messages []json.RawMessage `json:"messages"` + Tools json.RawMessage `json:"tools"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "text/event-stream") + if req.Model == "triage" { + triage.Add(1) + if len(req.Messages) != 2 || len(req.Tools) > 0 && string(req.Tools) != "null" { + t.Errorf("triage request messages/tools=%d/%s", len(req.Messages), req.Tools) + } + if !strings.Contains(string(req.Messages[1]), "screening fixture content") { + t.Error("screen did not receive the actual fixture result") + } + answer := "CLEAN" + if tc.flag { + answer = "INJECTION" + } + _, _ = fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":%q},\"finish_reason\":\"stop\"}]}\n\n", answer) + } else { + prefix := string(req.Messages[0]) + string(req.Tools) + if previous := mainPrefix.Swap(&prefix); previous != nil && prefix != *previous { + t.Error("main serialized prefix changed during the run") + } + for _, m := range req.Messages { + if strings.Contains(string(m), "screening model judged") { + sawNotice.Store(true) + } + } + if n := normal.Add(1); n <= 4 { + _, _ = fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"fixture-%d\",\"type\":\"function\",\"function\":{\"name\":\"triage_fixture\",\"arguments\":\"{}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\n", n) + } else { + _, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"Finished.\"},\"finish_reason\":\"stop\"}]}\n\n") + } + } + if req.Model != "triage" || !tc.missing { + _, _ = io.WriteString(w, "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":1000,\"completion_tokens\":100,\"total_tokens\":1100}}\n\n") + } + _, _ = io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(srv.Close) + writeFile(t, root, "reasonix.toml", fmt.Sprintf(`default_model = "worker" +[agent] +system_prompt = "BASE" +triage_model = "triage" +screen_external_content = %t +task_token_budget = %d +[codegraph] +enabled = false +[[providers]] +name = "worker" +kind = "openai" +model = "worker" +context_window = 200000 +base_url = %q +api_key_env = "ROOT_TRIAGE_KEY" +[[providers]] +name = "triage" +kind = "openai" +model = "triage" +base_url = %q +api_key_env = "ROOT_TRIAGE_KEY" +`, !tc.screenOff, tc.limit, srv.URL, srv.URL)) + t.Setenv("ROOT_TRIAGE_KEY", "test") + approveWorkspaceAt(t, home, root) + sink := event.FuncSink(func(e event.Event) { + if e.Kind == event.Notice && e.Code == event.NoticeCodeSuspectedInjection { + notices.Add(1) + } + if e.Kind == event.Usage && e.UsageSource == event.UsageSourceInjectionScreen && (e.ModelRef != "triage" || e.Usage.PromptTokens+e.Usage.CompletionTokens != 1100) { + t.Errorf("triage billing changed: model=%s usage=%+v", e.ModelRef, e.Usage) + } + }) + ctrl, err := Build(t.Context(), Options{Home: home, WorkspaceRoot: root, Sink: sink}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(ctrl.Close) + ctrl.SetSessionPath(sessionstore.NewSessionPath(ctrl.SessionDir(), ctrl.Label())) + ctrl.SetToolApprovalMode(control.ToolApprovalYolo) + probe := &triageFixtureRead{path: filepath.Join(root, "fixture.txt")} + reg := tool.NewRegistry() + reg.Add(probe) + ctrl.Executor().SetTools(reg) + err = ctrl.Run(t.Context(), "read screening fixture") + if tc.limit == 0 { + if err != nil { + t.Fatal(err) + } + } else if pause, ok := agent.InspectRunPause(err); !ok || pause.Key != "token" { + t.Fatalf("pause=%+v err=%v", pause, err) + } + if probe.reads.Load() != tc.wantReads || triage.Load() != tc.wantTriage || normal.Load() != tc.wantReads+1 { + t.Fatalf("read/triage/normal=%d/%d/%d want%d/%d/%d", probe.reads.Load(), triage.Load(), normal.Load(), tc.wantReads, tc.wantTriage, tc.wantReads+1) + } + if tc.flag != (notices.Load() == 1) || tc.flag != sawNotice.Load() { + t.Fatalf("flag=%v notices=%d modelNotice=%v", tc.flag, notices.Load(), sawNotice.Load()) + } + }) + } +} diff --git a/internal/runtime/agent/command_class.go b/internal/runtime/agent/command_class.go index 33d1534910..024662ebf6 100644 --- a/internal/runtime/agent/command_class.go +++ b/internal/runtime/agent/command_class.go @@ -162,6 +162,7 @@ func (a *Agent) askTriage(ctx context.Context, systemPrompt, input, source strin var usage *provider.Usage defer func() { + observeRootTokenUsage(ctx, usage) if usage != nil && usage.TotalTokens > 0 && a.svc.sink != nil { // Reported as what it is, on the model that answered it. Billing a // classification to compaction at the turn's own price hid both how diff --git a/internal/runtime/agent/root_triage_budget_test.go b/internal/runtime/agent/root_triage_budget_test.go new file mode 100644 index 0000000000..ab9824b970 --- /dev/null +++ b/internal/runtime/agent/root_triage_budget_test.go @@ -0,0 +1,119 @@ +package agent + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "reasonix/internal/contract/event" + "reasonix/internal/contract/tool" + "reasonix/internal/state/sessionstore" +) + +type rootTriageTool struct { + readProbe + triage *Agent + calls int + source string +} + +func (p *rootTriageTool) Execute(ctx context.Context, _ json.RawMessage) (string, error) { + p.calls++ + _, _ = p.triage.askTriage(ctx, "classify fixture", "fixture content", p.source) + return "classification attempted", nil +} + +func TestRootTriageTokenBudget(t *testing.T) { + for _, tc := range []struct { + name string + source string + failure bool + missing bool + noEmission bool + promptOnly bool + disabled bool + wantCalls int + }{ + {name: "classifier", source: event.UsageSourceClassifier, wantCalls: 1}, + {name: "screen", source: event.UsageSourceInjectionScreen, wantCalls: 1}, + {name: "billed_failure", source: event.UsageSourceClassifier, failure: true, wantCalls: 1}, + {name: "missing", source: event.UsageSourceClassifier, missing: true, wantCalls: 2}, + {name: "disabled", source: event.UsageSourceClassifier, disabled: true, wantCalls: 4}, + {name: "no_sink", source: event.UsageSourceClassifier, noEmission: true, wantCalls: 1}, + {name: "counts_without_total", source: event.UsageSourceClassifier, promptOnly: true, wantCalls: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + u := billed(1000, 100) + if tc.missing { + u = nil + } + if tc.promptOnly { + u.TotalTokens = 0 + } + var fail error + if tc.failure { + fail = errors.New("billed classification failed") + } + replies := make([]scriptedReply, 4) + for i := range replies { + replies[i] = scriptedReply{text: "READONLY", usage: u, fail: fail} + } + sink := &recordSink{} + triage := New(&mockProvider{}, tool.NewRegistry(), sessionstore.NewSession("triage"), Options{TriageProvider: &scriptedSummarizer{replies: replies}}, sink) + if tc.noEmission { + triage.svc.sink = nil + } + probe := &rootTriageTool{triage: triage, source: tc.source} + reg := tool.NewRegistry() + reg.Add(probe) + p := &spendingProvider{max: 4} + root := New(p, reg, sessionstore.NewSession("root"), Options{TaskBudget: TaskBudget{Tokens: 1500}}, sink) + ctx := t.Context() + if tc.disabled { + ctx = WithTaskBudget(ctx, TaskBudget{}) + } + err := root.Run(ctx, "classify fixture") + if tc.disabled { + if err != nil { + t.Fatal(err) + } + } else if pause, ok := InspectRunPause(err); !ok || pause.Key != "token" { + t.Fatalf("pause=%+v err=%v", pause, err) + } + if probe.calls != tc.wantCalls || p.rounds.Load() != int32(tc.wantCalls+1) { + t.Fatalf("triage/normal=%d/%d want%d/%d", probe.calls, p.rounds.Load(), tc.wantCalls, tc.wantCalls+1) + } + var auxiliary int + for _, e := range sink.kinds(event.Usage) { + if e.Kind == event.Usage && e.UsageSource == tc.source { + auxiliary += e.Usage.PromptTokens + e.Usage.CompletionTokens + } + } + want := int(p.rounds.Load()) * 1100 + charged := 0 + if !tc.missing { + charged = probe.calls * 1100 + } + if root.task.budget.promptTokens+root.task.budget.outputTokens != want || root.task.budget.rootTokens.used.Load() != uint64(want+charged) { + t.Fatalf("local/root=%d/%d want%d/%d", root.task.budget.promptTokens+root.task.budget.outputTokens, root.task.budget.rootTokens.used.Load(), want, want+charged) + } + if !tc.missing && !tc.noEmission && !tc.promptOnly && auxiliary != probe.calls*1100 { + t.Fatal("triage fixture failed to bill exact usage") + } + if (tc.missing || tc.noEmission || tc.promptOnly) && auxiliary != 0 { + t.Fatal("existing usage emission conditions changed") + } + }) + } +} + +func TestRootTriageOutsideRunDoesNotCreateScope(t *testing.T) { + a := New(&mockProvider{}, tool.NewRegistry(), sessionstore.NewSession("triage"), Options{TriageProvider: &scriptedSummarizer{replies: []scriptedReply{{text: "READONLY", usage: billed(1000, 100)}}}}, event.Discard) + if text, ok := a.askTriage(t.Context(), "classify", "fixture", event.UsageSourceClassifier); !ok || text != "READONLY" { + t.Fatalf("reply=%q ok=%v", text, ok) + } + if a.task.budget.rootTokens != nil { + t.Fatal("out-of-run classification created an unrelated root") + } +} diff --git a/release-notes/studio/unreleased/14025-root-triage-token-usage.md b/release-notes/studio/unreleased/14025-root-triage-token-usage.md new file mode 100644 index 0000000000..462acc5a27 --- /dev/null +++ b/release-notes/studio/unreleased/14025-root-triage-token-usage.md @@ -0,0 +1,2 @@ +- 修复：活跃根任务的 token 软预算现在计入命令分类和外部内容筛查用量；分类结论、筛查提示、缺失用量与零阈值行为保留。关联 #14025。 +- Fixed: active root token soft budgets now include command-classification and external-screening usage, preserving verdicts, advisory notices, missing usage and zero limits. Refs #14025.