Show the cache hit rate after the cold start, and the output ratio

The session's first executor request pays for the prefix every later one
reuses, so the session-wide hit rate mixes "did the cache work once warm"
with "how much a cold start could reuse". usagereport already records the
cold start, but nothing a live session reads projected it.

The live figures come from the serve ledger, which restarts on /new,
/resume and /fork; usageTally does not, so its ColdStart can belong to an
earlier conversation in the same process. The broadcaster therefore takes
the cold start from the ledger's own population: the first request the
ledger files under the executor (pricing.LedgerSource, the rule Add
already used), split by the same CacheSplit the ledger applies, and reset
with it. SessionUsage carries it as coldStart.

- /status (TUI): "warm cache" row, ledger hit rate with the cold start
  taken out of hit, miss and unknown; "out/in" row, output ÷ input.
- Desktop session overview behind the cache meter: the same two figures,
  computed in state/overview beside cacheRate.
- wire-parity: ColdStartUsage mirrored in port/session.ts.
This commit is contained in:
yhh committed 2026-11-07 09:06:37 +00:00
1 parent 2e2566c84d
commit 07d2d4bdd0
20 files changed
+258 -15

No files matched your search

@@ -79,6 +79,11 @@ export const SESSION: Messages = {
"session.hit_hit_miss": { zh: "命中 ÷ (命中 + 未命中)", en: "hit ÷ (hit + miss)" },
"session.hit_hit_miss_leaves_out": { zh: "命中 ÷ (命中 + 未命中);不含服务商未区分的 {n} tokens", en: "hit ÷ (hit + miss); leaves out {n} tokens the provider did not split" },
"session.each_sub_agents_write_scope": { zh: "子代理同时改文件时各自能写的范围", en: "What each sub-agent may write when several change files at once" },
"session.leaves_out_the_cold_start": { zh: "不含首个主代理请求(冷启动)", en: "Leaves out the first main-agent request (the cold start)" },
"session.leaves_out_the_cold_start_n": { zh: "不含首个主代理请求(冷启动,{n} tokens)", en: "Leaves out the first main-agent request (the cold start, {n} tokens)" },
"session.output_input": { zh: "输出 ÷ 输入", en: "output ÷ input" },
"session.output_ratio": { zh: "输出比", en: "Output ratio" },
"session.warm_cache_hit": { zh: "扣除冷启动后命中", en: "Hit after cold start" },
"session.in_a_out_b": { zh: "输入 {a} · 输出 {b}", en: "In {a} · out {b}" },
"session.keep_it_short_and_recognizable": { zh: "保持简短且易于识别", en: "Keep it short and recognizable" },
"session.live_of_them_are_running": { zh: "其中 {live} 个对话正在运行,停止后才能移除", en: "{live} of them are running; stop them before removing" },
+10
View File
@@ -399,4 +399,14 @@ export interface SessionUsage {
cacheUnknown?: number;
workMs: number;
sources: SourceUsage[];
// The first executor request in these totals, which it is also in.
coldStart?: ColdStartUsage;
}
// The session's opening executor request, split the way the ledger split it.
export interface ColdStartUsage {
promptTokens: number;
cacheHit: number;
cacheMiss: number;
cacheUnknown?: number;
}
@@ -14,6 +14,20 @@ export function cacheRate(hit: number, miss: number, unknown = 0): number | null
return Math.min(1, hit / known);
}
/** The session rate with its cold start taken out of hit, miss and unknown alike:
* the first executor request pays for the prefix every later one reuses. Null
* until a request after it split its prompt. */
export function warmCacheRate(u: Pick<SessionUsage, "cacheHit" | "cacheMiss" | "cacheUnknown" | "coldStart">): number | null {
const c = u.coldStart;
if (!c) return null;
return cacheRate(u.cacheHit - c.cacheHit, u.cacheMiss - c.cacheMiss, (u.cacheUnknown ?? 0) - (c.cacheUnknown ?? 0));
}
/** output ÷ input tokens over the same requests. Null with no input. */
export function outputRatio(u: Pick<SessionUsage, "promptTokens" | "completionTokens">): number | null {
return u.promptTokens > 0 ? u.completionTokens / u.promptTokens : null;
}
export type CostSay =
| { kind: "exact"; amount: number; currency: string }
| { kind: "atLeast"; amount: number; currency: string }
@@ -93,6 +93,34 @@ describe("the session overview behind the cache meter", () => {
expect(button().textContent).toContain("99%");
});
it("states the hit rate with the cold start left out of hit, miss and unknown alike", () => {
rail({ usage: { ...usage, coldStart: { promptTokens: 1000, cacheHit: 800, cacheMiss: 200 } } });
const row = within(card()).getByText("扣除冷启动后命中").closest("div")!;
expect(row.textContent).toContain("75%");
expect(row.textContent).toContain("不含首个主代理请求(冷启动,1k tokens)");
expect(within(card()).getByText("缓存命中").closest("div")!.textContent).toContain("76%");
expect(button().textContent).toContain("76%");
});
it("owes no warm rate before a cold start is known, or before a request follows it", () => {
rail({ usage: { ...usage, coldStart: undefined } });
expect(within(card()).getByText("扣除冷启动后命中").closest("div")!.textContent).toContain("—");
cleanup();
const only = { promptTokens: 3000, cacheHit: 2300, cacheMiss: 700 };
rail({ usage: { ...usage, ...only, cacheUnknown: 0, completionTokens: 10, requests: 1, coldStart: only } });
expect(within(card()).getByText("扣除冷启动后命中").closest("div")!.textContent).toContain("—");
});
it("states output over input for the same requests", () => {
rail();
const row = within(card()).getByText("输出比").closest("div")!;
expect(row.textContent).toContain("5.6%");
expect(row.textContent).toContain("输出 ÷ 输入");
cleanup();
rail({ usage: { ...usage, promptTokens: 0, completionTokens: 0 } });
expect(within(card()).getByText("输出比").closest("div")!.textContent).toContain("—");
});
it("lists each source with its own cache and cost, and an unpriced one as unknown", () => {
rail();
const lines = [...card().querySelectorAll(".studio-overview-row")];
@@ -2,11 +2,12 @@ import { t } from "../i18n";
import { money } from "../i18n/format";
import type { SessionUsage } from "../port/session";
import type { CostQuote } from "../port/wire";
import { cacheRate, costSay, sessionTokens, sourceTokens, type CostSay } from "../state/overview";
import { cacheRate, costSay, outputRatio, sessionTokens, sourceTokens, warmCacheRate, type CostSay } from "../state/overview";
import { MASK } from "./wallet";
const tokens = (n: number): string => (n >= 1_000_000 ? `${(n / 1_000_000).toFixed(1)}M` : n >= 1_000 ? `${Math.round(n / 100) / 10}k` : String(n));
const percent = (r: number | null): string => (r === null ? "—" : `${Math.floor(r * 100)}%`);
const fine = (r: number | null): string => (r === null ? "—" : `${(r * 100).toFixed(1)}%`);
function clock(ms: number): string {
if (ms <= 0) return "—";
@@ -78,6 +79,11 @@ export function SessionOverviewCard({ usage, quote, hideAmounts, onContext, clas
<dt>{t("common.cache_hit")}</dt><dd>{percent(cacheRate(usage.cacheHit, usage.cacheMiss, unknown))}</dd>
<small>{unknown > 0 ? t("session.hit_hit_miss_leaves_out", { n: tokens(unknown) }) : t("session.hit_hit_miss")}</small>
</div>
<div>
<dt>{t("session.warm_cache_hit")}</dt><dd>{percent(warmCacheRate(usage))}</dd>
<small>{usage.coldStart ? t("session.leaves_out_the_cold_start_n", { n: tokens(usage.coldStart.promptTokens) }) : t("session.leaves_out_the_cold_start")}</small>
</div>
<div><dt>{t("session.output_ratio")}</dt><dd>{fine(outputRatio(usage))}</dd><small>{t("session.output_input")}</small></div>
</dl>
{sources.length > 0 && (
<section aria-label={t("common.by_source")}>
+1
View File
@@ -175,6 +175,7 @@ type Messages struct {
QueueMessages
ResumePickerMessages
GoalMessages
UsageMessages
SettingMessages
ApprovalNeededFmt string // notification text for a pending approval, tool only
ApprovalNeededWithSubjectFmt string // notification text for a pending approval with subject
+1
View File
@@ -201,6 +201,7 @@ var English = Messages{
QueueMessages: queueEN,
ResumePickerMessages: resumePickerEN,
GoalMessages: goalEN,
UsageMessages: usageEN,
SettingMessages: settingEN,
ApprovalNeededFmt: "approval needed: %s",
ApprovalNeededWithSubjectFmt: "approval needed: %s %s",
+1
View File
@@ -201,6 +201,7 @@ var Chinese = Messages{
QueueMessages: queueZH,
ResumePickerMessages: resumePickerZH,
GoalMessages: goalZH,
UsageMessages: usageZH,
SettingMessages: settingZH,
ApprovalNeededFmt: "需要审批:%s",
ApprovalNeededWithSubjectFmt: "需要审批:%s %s",
+1
View File
@@ -202,6 +202,7 @@ var ChineseTraditional = Messages{
QueueMessages: queueZHTW,
ResumePickerMessages: resumePickerZHTW,
GoalMessages: goalZHTW,
UsageMessages: usageZHTW,
SettingMessages: settingZHTW,
ApprovalNeededFmt: "需要核准:%s",
ApprovalNeededWithSubjectFmt: "需要核准:%s %s",
+7
View File
@@ -0,0 +1,7 @@
package i18n
var usageEN = UsageMessages{ChatStatusCacheWarmFmt: "%s after the cold start", ChatStatusOutputRatioFmt: "%s (out %s / in %s)"}
var usageZH = UsageMessages{ChatStatusCacheWarmFmt: "扣除冷启动后 %s", ChatStatusOutputRatioFmt: "%s(输出 %s / 输入 %s)"}
var usageZHTW = UsageMessages{ChatStatusCacheWarmFmt: "扣除冷啟動後 %s", ChatStatusOutputRatioFmt: "%s(輸出 %s / 輸入 %s)"}
+7
View File
@@ -0,0 +1,7 @@
package i18n
// UsageMessages is how /status words the session's ledger figures; Messages embeds it.
type UsageMessages struct {
ChatStatusCacheWarmFmt string // ledger hit rate with the cold start left out
ChatStatusOutputRatioFmt string // output ÷ input percent, output tokens, input tokens
}
+17 -11
View File
@@ -57,6 +57,18 @@ func NewLedger() *Ledger {
return &Ledger{Version: LedgerVersion, Entries: map[string]LedgerEntry{}}
}
// LedgerSource is the usage source a ledger files q under. A quote that names
// none is the executor's: the field postdates it.
func LedgerSource(q CostQuote) string {
if q.UsageSource != "" {
return q.UsageSource
}
return LedgerSourceExecutor
}
// LedgerSourceExecutor is the main agent's source, the one a session opens on.
const LedgerSourceExecutor = "executor"
// Add merges a quote into the ledger under its natural key.
func (l *Ledger) Add(q CostQuote, tokens UsageTokens, occurred time.Time) {
if l == nil {
@@ -72,10 +84,7 @@ func (l *Ledger) Add(q CostQuote, tokens UsageTokens, occurred time.Time) {
occurred = time.Now().UTC()
}
q = NormalizeQuote(q)
source := q.UsageSource
if source == "" {
source = "executor"
}
source := LedgerSource(q)
key := LedgerKey(q.ModelRef, source, q.PricingFingerprint, q.RateDate)
ent, ok := l.Entries[key]
if !ok {
@@ -151,13 +160,10 @@ func (l *Ledger) Add(q CostQuote, tokens UsageTokens, occurred time.Time) {
ent.PromptTokens += tokens.PromptTokens
ent.CompletionTokens += tokens.CompletionTokens
ent.TotalTokens += tokens.PromptTokens + tokens.CompletionTokens
if tokens.CacheHitTokens+tokens.CacheMissTokens > 0 {
ent.CacheHitTokens += tokens.CacheHitTokens
ent.CacheMissTokens += tokens.CacheMissTokens
} else {
ent.CacheMissTokens += tokens.PromptTokens
ent.CacheUnknownTokens += tokens.PromptTokens
}
hit, miss, unknown := tokens.CacheSplit()
ent.CacheHitTokens += hit
ent.CacheMissTokens += miss
ent.CacheUnknownTokens += unknown
ent.RequestCount++
if occurred.After(ent.OccurredAt) {
ent.OccurredAt = occurred
+10
View File
@@ -146,6 +146,16 @@ type UsageTokens struct {
Estimated bool
}
// CacheSplit is how a ledger counts one request's prompt: a request the
// endpoint did not split bills its whole prompt as a miss, and reports it as
// unknown too so a hit rate can leave it out.
func (t UsageTokens) CacheSplit() (hit, miss, unknown int) {
if t.CacheHitTokens+t.CacheMissTokens > 0 {
return t.CacheHitTokens, t.CacheMissTokens, 0
}
return 0, t.PromptTokens, t.PromptTokens
}
// QuoteInput drives a single CostQuote computation.
type QuoteInput struct {
Usage UsageTokens
+8 -2
View File
@@ -22,6 +22,7 @@ type Broadcaster struct {
ledger *pricing.Ledger
displayCurrency string
workMs int64
coldStart *ColdStartUsage
// The transport's, not the session's: numbering outlives /new and /resume, so
// resuming across one is told to rebuild rather than handed another
// conversation's frames under numbers it already has.
@@ -85,6 +86,7 @@ func (b *Broadcaster) ResetSession() {
b.mu.Lock()
b.ledger = pricing.NewLedger()
b.workMs = 0
b.coldStart = nil
b.replay.reset()
b.initialNoticesReplayed = false
b.mu.Unlock()
@@ -150,12 +152,16 @@ func (b *Broadcaster) Emit(e event.Event) {
if b.ledger == nil {
b.ledger = pricing.NewLedger()
}
b.ledger.Add(*e.CostQuote, pricing.UsageTokens{
tokens := pricing.UsageTokens{
PromptTokens: e.Usage.PromptTokens, CompletionTokens: e.Usage.CompletionTokens,
CacheHitTokens: e.Usage.CacheHitTokens, CacheMissTokens: e.Usage.CacheMissTokens,
CacheWriteTokens: e.Usage.CacheWriteTokens, CacheWriteBilledTokens: e.Usage.CacheWriteBilledTokens,
Estimated: e.Usage.Estimated,
}, time.Now().UTC())
}
b.ledger.Add(*e.CostQuote, tokens, time.Now().UTC())
if b.coldStart == nil && pricing.LedgerSource(*e.CostQuote) == pricing.LedgerSourceExecutor {
b.coldStart = coldStartOf(tokens)
}
}
for s := range b.subs {
s.push(Frame{Seq: w.Seq, Data: data}, drop)
+22
View File
@@ -35,6 +35,24 @@ type SessionUsage struct {
CacheUnknown int `json:"cacheUnknown,omitempty"`
WorkMs int64 `json:"workMs"`
Sources []SourceUsage `json:"sources"`
// ColdStart is the first executor request in these totals, which it is also
// in; subtracting it gives the rate once the prefix it paid for was warm.
ColdStart *ColdStartUsage `json:"coldStart,omitempty"`
}
// ColdStartUsage is the session's opening executor request, split the way the
// ledger split it. It pays for the prefix every later request reuses, so it is
// the one request a warm hit rate leaves out.
type ColdStartUsage struct {
PromptTokens int `json:"promptTokens"`
CacheHit int `json:"cacheHit"`
CacheMiss int `json:"cacheMiss"`
CacheUnknown int `json:"cacheUnknown,omitempty"`
}
func coldStartOf(tokens pricing.UsageTokens) *ColdStartUsage {
hit, miss, unknown := tokens.CacheSplit()
return &ColdStartUsage{PromptTokens: tokens.PromptTokens, CacheHit: hit, CacheMiss: miss, CacheUnknown: unknown}
}
// SessionUsage totals the ledger by source. WorkMs is the answered work time of
@@ -47,6 +65,10 @@ func (b *Broadcaster) SessionUsage() SessionUsage {
b.mu.Lock()
defer b.mu.Unlock()
out.WorkMs = b.workMs
if b.coldStart != nil {
cold := *b.coldStart
out.ColdStart = &cold
}
grouped := b.ledger.EntriesBySource()
names := make([]string, 0, len(grouped))
for name := range grouped {
@@ -63,3 +63,45 @@ func TestSessionUsageStartsAgainWhenTheSessionDoes(t *testing.T) {
t.Fatalf("after a reset: %+v", got)
}
}
// The cold start is the first request the ledger files under the executor:
// requests from other sources before it carry their own prefixes and are not it.
func TestSessionUsageColdStartIsTheLedgersFirstExecutorRequest(t *testing.T) {
bc := NewBroadcaster()
if got := bc.SessionUsage(); got.ColdStart != nil {
t.Fatalf("no request yet, cold start = %+v", got.ColdStart)
}
bc.Emit(usageEvent("title", 300, 10, 0, 300))
bc.Emit(usageEvent("", 4000, 100, 1000, 3000))
bc.Emit(usageEvent("executor", 5000, 50, 4900, 100))
got := bc.SessionUsage()
want := ColdStartUsage{PromptTokens: 4000, CacheHit: 1000, CacheMiss: 3000}
if got.ColdStart == nil || *got.ColdStart != want {
t.Fatalf("cold start = %+v, want %+v", got.ColdStart, want)
}
if got.CacheHit-got.ColdStart.CacheHit != 4900 {
t.Fatalf("the cold start must be part of the totals it is subtracted from: %+v", got)
}
}
func TestSessionUsageColdStartKeepsAnUnsplitPromptUnknown(t *testing.T) {
bc := NewBroadcaster()
bc.Emit(usageEvent("executor", 2000, 10, 0, 0))
got := bc.SessionUsage().ColdStart
if got == nil || got.CacheMiss != 2000 || got.CacheUnknown != 2000 || got.CacheHit != 0 {
t.Fatalf("cold start = %+v, want the prompt billed as a miss and held as unknown", got)
}
}
func TestSessionUsageColdStartStartsAgainWhenTheSessionDoes(t *testing.T) {
bc := NewBroadcaster()
bc.Emit(usageEvent("executor", 100, 10, 0, 100))
bc.ResetSession()
if got := bc.SessionUsage().ColdStart; got != nil {
t.Fatalf("after a reset: %+v", got)
}
bc.Emit(usageEvent("executor", 900, 10, 800, 100))
if got := bc.SessionUsage().ColdStart; got == nil || got.PromptTokens != 900 {
t.Fatalf("the next session's cold start = %+v", got)
}
}
+19 -1
View File
@@ -240,7 +240,25 @@ type Status struct {
CacheHitTokens int
CacheMissTokens int
} `json:"lastUsage"`
SessionCostQuote *CostQuote `json:"sessionCostQuote"`
SessionCostQuote *CostQuote `json:"sessionCostQuote"`
SessionUsage *SessionUsage `json:"sessionUsage"`
}
// SessionUsage is the slice of the kernel's ledger totals /status reports.
type SessionUsage struct {
PromptTokens int `json:"promptTokens"`
CompletionTokens int `json:"completionTokens"`
CacheHit int `json:"cacheHit"`
CacheMiss int `json:"cacheMiss"`
CacheUnknown int `json:"cacheUnknown"`
ColdStart *ColdStartUsage `json:"coldStart"`
}
// ColdStartUsage is the session's opening executor request, already in the totals.
type ColdStartUsage struct {
CacheHit int `json:"cacheHit"`
CacheMiss int `json:"cacheMiss"`
CacheUnknown int `json:"cacheUnknown"`
}
// CostQuote is the session's spend as the kernel priced it.
+34
View File
@@ -150,6 +150,40 @@ func TestStatusReportsTheFieldsTheFooterCompresses(t *testing.T) {
}
}
// Both rows read the ledger /status sends: the warm rate leaves the cold start
// out of hit and of the split prompt, and the ratio is output over input.
func TestStatusReportsTheWarmCacheRateAndTheOutputRatio(t *testing.T) {
m, _ := queueModel(t, map[string]any{
"GET /status": map[string]any{
"label": "deepseek-chat", "cacheHit": 0, "cacheMiss": 0,
"sessionUsage": map[string]any{
"promptTokens": 20000, "completionTokens": 500, "cacheHit": 13000, "cacheMiss": 7000, "cacheUnknown": 2000,
"coldStart": map[string]any{"promptTokens": 5000, "cacheHit": 1000, "cacheMiss": 4000},
},
},
})
queueSend(m, "/status")
got := queueLastNotice(m)
for _, want := range []string{"warm cache 92.31% after the cold start", "out/in 2.5% (out 500 / in 20.0K)"} {
if !strings.Contains(got, want) {
t.Fatalf("status lacks %q:\n%s", want, got)
}
}
}
func TestStatusOwesNoWarmRateUntilARequestFollowsTheColdStart(t *testing.T) {
m, _ := queueModel(t, map[string]any{
"GET /status": map[string]any{
"label": "deepseek-chat",
"sessionUsage": map[string]any{"promptTokens": 5000, "cacheHit": 1000, "cacheMiss": 4000, "coldStart": map[string]any{"promptTokens": 5000, "cacheHit": 1000, "cacheMiss": 4000}},
},
})
queueSend(m, "/status")
if got := queueLastNotice(m); strings.Contains(got, "warm cache") {
t.Fatalf("a session of one request has no warm rate:\n%s", got)
}
}
// The kernel owns where the file goes; this screen only reports it.
func TestExportAsksTheKernelAndReportsItsPath(t *testing.T) {
m, k := queueModel(t, map[string]any{
+23
View File
@@ -34,6 +34,13 @@ func (m *model) statusDetails() string {
if body, _, ok := cacheStatus(s); ok {
lines = append(lines, " cache "+body)
}
if rate, ok := warmCacheRate(s.SessionUsage); ok {
lines = append(lines, " warm cache "+fmt.Sprintf(i18n.M.ChatStatusCacheWarmFmt, fmt.Sprintf("%.2f%%", rate)))
}
if u := s.SessionUsage; u != nil && u.PromptTokens > 0 {
ratio := fmt.Sprintf("%.1f%%", pct(u.CompletionTokens, u.PromptTokens))
lines = append(lines, " out/in "+fmt.Sprintf(i18n.M.ChatStatusOutputRatioFmt, ratio, shortTokens(u.CompletionTokens), shortTokens(u.PromptTokens)))
}
if git := m.gitText(); git != "" {
lines = append(lines, " git "+git)
}
@@ -49,6 +56,22 @@ func (m *model) statusDetails() string {
return strings.Join(append(lines, " config "+activeConfigTag()), "\n")
}
// warmCacheRate is the ledger's hit rate with its cold start taken out of both
// sides: hit over the prompt the endpoint split, the same rate the desktop's
// session overview states. Not owed until a request after the cold start split.
func warmCacheRate(u *SessionUsage) (float64, bool) {
if u == nil || u.ColdStart == nil {
return 0, false
}
c := u.ColdStart
hit := u.CacheHit - c.CacheHit
known := hit + (u.CacheMiss - c.CacheMiss) - (u.CacheUnknown - c.CacheUnknown)
if known <= 0 {
return 0, false
}
return pct(min(hit, known), known), true
}
// modeText is the footer's mode badge without its colour.
func (m *model) modeText() string {
s := m.status
+1
View File
@@ -122,6 +122,7 @@ var mirroredWireTypesHead = []wireMirror{
{"internal/frontend/serve/subagents.go", "subagentRunView", tsSubagentFile, "SubagentRun"},
{"internal/frontend/serve/subagents.go", "subagentTranscriptView", tsSubagentFile, "SubagentTranscript"},
{"internal/frontend/serve/session_usage.go", "SourceUsage", tsSessionFile, "SourceUsage"},
{"internal/frontend/serve/session_usage.go", "ColdStartUsage", tsSessionFile, "ColdStartUsage"},
// The agent's tabs. The target id is what the window draws a view by, so a
// tab the page cannot read one for is a tab it cannot show.
{"internal/platform/browser/session.go", "TabInfo", tsSessionFile, "BrowserTab"},