feat(completion): the claim becomes checkable — reconcile it against the receipts

update_goal could say "complete" but never say what it ran, what it skipped, or
what it is worried about. The first is checkable and the other two are not, and
that asymmetry is the design: `verified` commands are matched against this
session's real receipts, while `unverified` and `risks` are declarations the
host has no way to infer and therefore keeps verbatim.

A claim can only add. Every host-found gap survives reconciliation untouched, so
accepting the model's account costs nothing in strictness — and a verification
that never ran, exited non-zero, or last ran before the newest mutation becomes
an unbacked_claim gap, leading the report because it is the worst kind.

The claim rides the ledger: update_goal receipts already carry their arguments,
so the report reads the latest successful call and needs no new plumbing. A
rejected call claims nothing.

e2ebench prices it as unbacked claims — asserted verifications the ledger
denied. Behavior is unchanged: the goal FSM still gates exactly as before, and
the report still only observes.
This commit is contained in:
esengine committed 2026-08-09 03:49:25 -07:00
1 parent c215d821fd
commit 4caf5d7f2f
17 files changed
+489 -19

No files matched your search

+6
View File
@@ -183,12 +183,15 @@ func renderCompletionReport(results []result) string {
verdicts := map[string]int{}
kinds := map[string]int{}
recorded, done, overclaim, failed, caught := 0, 0, 0, 0, 0
claimed, unbacked := 0, 0
for _, r := range results {
t := r.Trajectory
if t == nil || t.CompletionVerdict == "" {
continue
}
recorded++
claimed += t.ClaimsVerified
unbacked += t.ClaimsUnbacked
verdicts[t.CompletionVerdict]++
for _, kind := range t.CompletionGapKinds {
kinds[kind]++
@@ -220,6 +223,9 @@ func renderCompletionReport(results []result) string {
if failed > 0 {
line += fmt.Sprintf(" · **caught** %s (%d/%d failed runs declared a gap)", pct(caught, failed), caught, failed)
}
if claimed > 0 {
line += fmt.Sprintf(" · **unbacked claims** %s (%d/%d asserted verifications the ledger denied)", pct(unbacked, claimed), unbacked, claimed)
}
if census := gapCensus(kinds); census != "" {
line += " · gaps " + census
}
+2
View File
@@ -174,6 +174,7 @@ func TestRenderCompletionReportPricesOverclaim(t *testing.T) {
warned.Trajectory = &trajectorySummary{
CompletionVerdict: "partial", CompletionGaps: 2,
CompletionGapKinds: []string{"unreviewed_change", "stale_verification"},
ClaimsVerified: 4, ClaimsUnbacked: 1,
}
got := renderCompletionReport([]result{honest, overclaimed, warned})
for _, want := range []string{
@@ -181,6 +182,7 @@ func TestRenderCompletionReportPricesOverclaim(t *testing.T) {
"**overclaim** 50% (1/2 done runs the grader failed)",
"**caught** 50% (1/2 failed runs declared a gap)",
"gaps stale_verification ×1 · unreviewed_change ×1",
"**unbacked claims** 25% (1/4 asserted verifications the ledger denied)",
} {
if !strings.Contains(got, want) {
t.Fatalf("completion line missing %q:\n%s", want, got)
+9 -3
View File
@@ -110,6 +110,8 @@ type trajectorySummary struct {
CompletionVerdict string `json:"completion_verdict,omitempty"`
CompletionGaps int `json:"completion_gaps,omitempty"`
CompletionGapKinds []string `json:"completion_gap_kinds,omitempty"`
ClaimsVerified int `json:"claims_verified,omitempty"`
ClaimsUnbacked int `json:"claims_unbacked,omitempty"`
// Outcome shadow: the runtime outcome scorer's per-round series condensed,
// or a verification-receipt backfill for recordings that predate it.
@@ -150,9 +152,11 @@ type trajectoryRecord struct {
Complete bool `json:"complete"`
} `json:"contract_shadow"`
CompletionReport *struct {
Verdict string `json:"verdict"`
Gaps int `json:"gaps"`
GapKinds []string `json:"gap_kinds"`
Verdict string `json:"verdict"`
Gaps int `json:"gaps"`
GapKinds []string `json:"gap_kinds"`
ClaimsVerified int `json:"claims_verified"`
ClaimsUnbacked int `json:"claims_unbacked"`
} `json:"completion_report"`
OutcomeProgress *struct {
Exploration int `json:"exploration"`
@@ -371,6 +375,8 @@ func (t *trajScan) record(rec trajectoryRecord) {
t.s.CompletionVerdict = cr.Verdict
t.s.CompletionGaps = cr.Gaps
t.s.CompletionGapKinds = cr.GapKinds
t.s.ClaimsVerified = cr.ClaimsVerified
t.s.ClaimsUnbacked = cr.ClaimsUnbacked
}
if op := rec.OutcomeProgress; op != nil {
t.outcomePoints = append(t.outcomePoints, outcomePoint{
+13
View File
@@ -8,6 +8,7 @@ Reasonix 的 Goal 模式(`/goal`)将目标推进(Goal)、验收(Delive
|------|----------|------|
| 结构化完成协议 | `update_goal` 工具 | 每轮目标 turn 结束时模型通过工具报告 continue/complete/blocked(含 reason 与 next_action),取代旧的 `[goal:*]` footer 文本标记 |
| 完成校验 | 默认 | `complete` 声明必须通过 Delivery readiness(todos、验证、review、签收、能力门禁)才会真正完成;不满足时用缺失项开启下一轮 |
| 完成自述与对账 | `update_goal` 的 `completion` | `complete` 可附带自述:`verified` 命令逐条与本会话真实 receipt 对账,没跑过 / 跑失败 / 早于最后一次改动都记为 unbacked claim;`unverified` 与 `risks` 是宿主推断不出的声明,只增不减,永远不阻塞完成 |
| 独立评审 | 无报告时 | 模型未调用 `update_goal` 时,宿主调用一次独立 bounded evaluator 判定;评审不可用/出错/不确定时安全暂停,绝不默认继续 |
| 执行预算 | 默认 | **轮次与无进展熔断**:简单 10 轮、写入型 20 轮、研究型 40 轮;连续 4 轮无宿主可验证进展则暂停。累计 token 只做观测展示,**没有 token 硬上限**,也没有 provider 请求前预算准入 |
| 暂停/恢复 | `/goal pause` / `/goal resume` | 暂停保留 Goal、todo、Delivery checkpoint 与运行历史;轮次型暂停恢复时追加一档同类别**轮数**(`budget_extensions` 统计轮次追加次数) |
@@ -28,6 +29,18 @@ Reasonix 的 Goal 模式(`/goal`)将目标推进(Goal)、验收(Delive
- `complete`(仅在请求完成、输出格式与约束满足、验证已尝试或声明不可用时)— 宿主会用 Delivery readiness 校验该声明;
- `blocked`(仅当下一步需要用户独有信息、不可逆或对外可见操作、或范围变化时)— 立即停止。
`complete` 还可以附一份自述 `completion`:
```json
{"status":"complete","completion":{
"verified":["go test ./..."],
"unverified":["desktop UI 未实际操作验证"],
"risks":["迁移不可逆"]
}}
```
宿主对这份自述只做一件事——**对账**。`verified` 里的每条命令都会去本会话的真实 receipt 里找:没跑过、跑失败、或者最后一次运行早于最后一次改动,都会被记成一条 unbacked claim。`unverified` 与 `risks` 宿主无从推断,因此原样保留;它们**只增不减**,一份自述永远不能抹掉宿主自己发现的缺口,也永远不会因为诚实申报而阻塞完成。
`update_goal` 只在活动 Goal turn 中可用;普通聊天调用会收到结构化错误且不改变任何状态。同值重复调用幂等,`continue` 可升级为 `complete`/`blocked`,终态后冲突调用被拒绝;目标被替换或清除后,迟到的报告/用量一律按 scope+epoch 拒绝。
### 预算与暂停
+1 -1
View File
@@ -23,7 +23,7 @@ This document records the provider-visible contract for Reasonix compile-time bu
| `notebook_edit` | false | Edit one cell of a Jupyter notebook (.ipynb). Target a cell by 0-based cell_number (or cell_id). edit_mode: "replace" (default) swaps the cell's source; "insert" adds a new cell after cell_number (use -1 to prepend at the top), taking cell_type and new_source; "delete" removes the cell. cell_type is "code" or "markdown" (required for insert). Editing a code cell clears its outputs. Prefer this over edit_file for notebooks - it keeps the JSON valid. |
| `read_file` | true | Read a text file with optional line offset/limit. Output prefixes each line with its 1-based number so subsequent edit_file calls can target exact lines. Use `offset` and `limit` to page through large files; the tool reports total length and pagination hints in a trailer. |
| `todo_write` | true | Record and update a structured task list for the current work. Send the COMPLETE list every call - it replaces the previous one. Use it to plan multi-step work and show progress: keep exactly one item in_progress at a time, and flip an item to completed the moment it's done (don't batch completions). Skip it for trivial single-step tasks. |
| `update_goal` | true | Report this turn's disposition for the active goal: `continue` (work is ongoing - give a concrete next_action), `complete` (fully done and verified), or `blocked` (only the user can unblock). The host validates the claim against Delivery acceptance criteria and budget and decides whether to continue automatically. Outside an active goal turn the call fails closed without changing any state. |
| `update_goal` | true | Report this turn's disposition for the active goal: `continue` (work is ongoing - give a concrete next_action), `complete` (fully done and verified), or `blocked` (only the user can unblock). An optional `completion` account may accompany `complete`: `verified` commands are reconciled against the session's real receipts, while `unverified` and `risks` are declarations the host cannot infer. The host validates the claim against Delivery acceptance criteria and budget and decides whether to continue automatically. Outside an active goal turn the call fails closed without changing any state. |
| `wait` | true | Block until background jobs finish, then return each job's status and final output/answer. Use to collect the result of a task(run_in_background) or bash(run_in_background) before continuing. Omit job_ids to wait for every running job. |
| `web_fetch` | true | Fetch a URL over HTTPS/HTTP and return its text content. HTML pages are reduced to readable text; JSON / plain text / markdown bodies come back verbatim. Use to read documentation pages, API responses, or source files hosted somewhere the local filesystem can't reach. |
| `write_file` | false | Write content to a file at the given path (overwriting existing content). Creates parent directories as needed. |
+10 -4
View File
@@ -10,10 +10,16 @@ import (
// summary: counts, enums, and gap kinds, never a path or a command.
func completionReportAudit(rep completion.Report) event.CompletionReportAudit {
audit := event.CompletionReportAudit{
Verdict: rep.Verdict.String(),
Risk: riskName(rep.Risk),
Gaps: len(rep.Gaps),
GapKinds: rep.GapKinds(),
Verdict: rep.Verdict.String(),
Risk: riskName(rep.Risk),
Gaps: len(rep.Gaps),
GapKinds: rep.GapKinds(),
ClaimsVerified: len(rep.Claimed.Verified),
}
for _, gap := range rep.Gaps {
if gap.Kind == completion.GapUnbackedClaim {
audit.ClaimsUnbacked++
}
}
for _, criterion := range rep.Criteria {
if !criterion.Required {
+16
View File
@@ -73,6 +73,22 @@ func TestCompletionShadowCountsFailedVerification(t *testing.T) {
}
}
func TestCompletionShadowCountsUnbackedClaims(t *testing.T) {
_, report := shadowReport("fix the add bug in calc.py",
evidence.Receipt{ToolName: "edit_file", Mutation: true, Write: true, Success: true, Paths: []string{"calc.py"}},
evidence.Receipt{ToolName: "read_file", Read: true, Success: true, Paths: []string{"calc.py"}, OutputBytes: 64},
evidence.Receipt{ToolName: "update_goal", Success: true, Args: []byte(
`{"status":"complete","completion":{"verified":["pytest","go test ./..."]}}`)},
)
if report.ClaimsVerified != 2 || report.ClaimsUnbacked != 2 {
t.Fatalf("claims = %d verified / %d unbacked, want 2/2 — neither command ever ran",
report.ClaimsVerified, report.ClaimsUnbacked)
}
if !slices.Contains(report.GapKinds, "unbacked_claim") {
t.Fatalf("gap kinds = %v, want the fabricated verifications flagged", report.GapKinds)
}
}
func TestCompletionShadowStaysQuietOnAConversationTurn(t *testing.T) {
_, report := shadowReport("what does this function do?",
evidence.Receipt{ToolName: "read_file", Read: true, Success: true, Paths: []string{"calc.py"}, OutputBytes: 64},
+3 -3
View File
@@ -1,7 +1,7 @@
{
"SystemHash": "c4a54b2f61c5e15f",
"ToolsHash": "9674df33a22e9ee4",
"PrefixHash": "75fb4282ce36487d",
"ToolsHash": "ab2f66c27b039773",
"PrefixHash": "9e132ad357f9874b",
"LogRewriteVersion": 0,
"ToolSchemaTokens": 13603
"ToolSchemaTokens": 13801
}
+28 -1
View File
@@ -1620,9 +1620,36 @@
},
{
"name": "update_goal",
"description": "Report this turn's disposition for the active goal. Call it at the end of every goal turn instead of using prose markers: `continue` (work is ongoing — give a concrete next_action), `complete` (the request is fully done, output format and constraints satisfied, and verification was attempted or reported unavailable), or `blocked` (only the user can unblock: missing user-only information, an irreversible/externally visible operation, or changed scope). The host validates your claim against Delivery acceptance criteria and decides whether to continue automatically. Fields: `status` (required, one of continue|complete|blocked), `reason` (required for continue and blocked, optional for complete), `next_action` (optional concrete next step; recommended for continue).",
"description": "Report this turn's disposition for the active goal. Call it at the end of every goal turn instead of using prose markers: `continue` (work is ongoing — give a concrete next_action), `complete` (the request is fully done, output format and constraints satisfied, and verification was attempted or reported unavailable), or `blocked` (only the user can unblock: missing user-only information, an irreversible/externally visible operation, or changed scope). The host validates your claim against Delivery acceptance criteria and decides whether to continue automatically. Fields: `status` (required, one of continue|complete|blocked), `reason` (required for continue and blocked, optional for complete), `next_action` (optional concrete next step; recommended for continue), `completion` (recommended with complete: `verified` is checked against real receipts, while `unverified` and `risks` are yours to declare and never count against you).",
"parameters": {
"properties": {
"completion": {
"description": "Your own account of the finished work.",
"properties": {
"risks": {
"description": "Known risks to carry forward.",
"items": {
"type": "string"
},
"type": "array"
},
"unverified": {
"description": "What you did NOT verify. The host cannot infer what you skipped, so stating it is the only way it is known — and it never blocks completion.",
"items": {
"type": "string"
},
"type": "array"
},
"verified": {
"description": "Commands you ran as proof, as they actually ran. One that never ran, failed, or predates your latest change is recorded as an unbacked claim.",
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
},
"next_action": {
"description": "Optional concrete next step. Recommended for continue so the host can guide the next turn.",
"type": "string"
+28 -1
View File
@@ -1634,10 +1634,37 @@
},
{
"Name": "update_goal",
"Description": "Report this turn's disposition for the active goal. Call it at the end of every goal turn instead of using prose markers: `continue` (work is ongoing — give a concrete next_action), `complete` (the request is fully done, output format and constraints satisfied, and verification was attempted or reported unavailable), or `blocked` (only the user can unblock: missing user-only information, an irreversible/externally visible operation, or changed scope). The host validates your claim against Delivery acceptance criteria and decides whether to continue automatically. Fields: `status` (required, one of continue|complete|blocked), `reason` (required for continue and blocked, optional for complete), `next_action` (optional concrete next step; recommended for continue).",
"Description": "Report this turn's disposition for the active goal. Call it at the end of every goal turn instead of using prose markers: `continue` (work is ongoing — give a concrete next_action), `complete` (the request is fully done, output format and constraints satisfied, and verification was attempted or reported unavailable), or `blocked` (only the user can unblock: missing user-only information, an irreversible/externally visible operation, or changed scope). The host validates your claim against Delivery acceptance criteria and decides whether to continue automatically. Fields: `status` (required, one of continue|complete|blocked), `reason` (required for continue and blocked, optional for complete), `next_action` (optional concrete next step; recommended for continue), `completion` (recommended with complete: `verified` is checked against real receipts, while `unverified` and `risks` are yours to declare and never count against you).",
"ReadOnly": true,
"Schema": {
"properties": {
"completion": {
"description": "Your own account of the finished work.",
"properties": {
"risks": {
"description": "Known risks to carry forward.",
"items": {
"type": "string"
},
"type": "array"
},
"unverified": {
"description": "What you did NOT verify. The host cannot infer what you skipped, so stating it is the only way it is known — and it never blocks completion.",
"items": {
"type": "string"
},
"type": "array"
},
"verified": {
"description": "Commands you ran as proof, as they actually ran. One that never ran, failed, or predates your latest change is recorded as an unbacked claim.",
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
},
"next_action": {
"description": "Optional concrete next step. Recommended for continue so the host can guide the next turn.",
"type": "string"
+114
View File
@@ -0,0 +1,114 @@
package completion
import (
"encoding/json"
"strings"
"reasonix/internal/evidence"
)
// Claim is the model's own account of the work, as passed to update_goal. It
// is the only model-authored part of a report, and it can only ever add to
// what the host found: Verified is checked against the ledger, while
// Unverified and Risks are declarations the host cannot verify but has no
// reason to suppress.
type Claim struct {
Verified []string
Unverified []string
Risks []string
}
// Empty reports whether the turn made no claim at all.
func (c Claim) Empty() bool {
return len(c.Verified) == 0 && len(c.Unverified) == 0 && len(c.Risks) == 0
}
// claimOf extracts the latest successful update_goal completion claim. A
// failed call claims nothing: outside an active goal turn the tool fails
// closed, and a rejected claim must not reach the report.
func claimOf(receipts []evidence.Receipt) Claim {
var out Claim
for _, r := range receipts {
if !r.Success || r.ToolName != "update_goal" || len(r.Args) == 0 {
continue
}
var payload struct {
Completion struct {
Verified []string `json:"verified"`
Unverified []string `json:"unverified"`
Risks []string `json:"risks"`
} `json:"completion"`
}
if json.Unmarshal(r.Args, &payload) != nil {
continue
}
out = Claim{
Verified: trimAll(payload.Completion.Verified),
Unverified: trimAll(payload.Completion.Unverified),
Risks: trimAll(payload.Completion.Risks),
}
}
return out
}
func trimAll(in []string) []string {
var out []string
for _, s := range in {
if s = strings.TrimSpace(s); s != "" {
out = append(out, s)
}
}
return out
}
// reconcile folds the claim into a host-built report. Claimed verifications
// are matched against real receipts, and every mismatch becomes a gap; the
// model's own declarations are carried through untouched. Nothing here can
// clear a gap the host found — a claim only ever adds.
func reconcile(rep Report, claim Claim, receipts []evidence.Receipt) Report {
rep.Claimed = claim
rep.Risks = claim.Risks
var gaps []Gap
for _, command := range claim.Verified {
if why := unbackedClaim(command, receipts); why != "" {
gaps = append(gaps, Gap{GapUnbackedClaim, why})
}
}
for _, note := range claim.Unverified {
gaps = append(gaps, Gap{GapDeclaredUnverified, note})
}
rep.Gaps = append(gaps, rep.Gaps...)
return rep
}
// unbackedClaim returns why a claimed verification is not backed by the
// ledger, or "" when a successful run of it survives the latest mutation.
func unbackedClaim(command string, receipts []evidence.Receipt) string {
lastMutation := -1
for i, r := range receipts {
if r.Success && (r.Mutation || r.Write) {
lastMutation = i
}
}
matched, index, success := false, -1, false
for i, r := range receipts {
ran := strings.TrimSpace(r.Command)
if ran == "" {
continue
}
if ran != command && !evidence.CommandMatches(command, ran) {
continue
}
matched, index, success = true, i, r.Success
}
switch {
case !matched:
return command + " — claimed as verification, but no run of it was recorded"
case !success:
return command + " — claimed as verification, but its last run failed"
case index < lastMutation:
return command + " — claimed as verification, but it last ran before the latest change"
default:
return ""
}
}
+147
View File
@@ -0,0 +1,147 @@
package completion
import (
"slices"
"strings"
"testing"
"reasonix/internal/evidence"
)
func claimed(status string, completion string) evidence.Receipt {
args := `{"status":"` + status + `","completion":` + completion + `}`
return evidence.Receipt{ToolName: "update_goal", Success: true, Args: []byte(args)}
}
func TestClaimedVerificationThatNeverRanIsUnbacked(t *testing.T) {
rep := Build(nil, ledgerOf(
wrote("parser.go"),
read("parser.go"),
claimed("complete", `{"verified":["go test ./..."]}`),
))
if got := gapKinds(rep); !slices.Contains(got, "unbacked_claim") {
t.Fatalf("gap kinds = %v, want the fabricated verification caught", got)
}
if !strings.Contains(rep.Gaps[0].Detail, "no run of it was recorded") {
t.Fatalf("gap detail = %q, want the reason named", rep.Gaps[0].Detail)
}
if rep.Gaps[0].Kind != GapUnbackedClaim {
t.Fatalf("an unbacked claim must lead the gap list, got %v", rep.Gaps[0].Kind)
}
}
func TestClaimedVerificationThatFailedIsUnbacked(t *testing.T) {
rep := Build(nil, ledgerOf(
wrote("parser.go"),
read("parser.go"),
ran("go test ./...", false),
claimed("complete", `{"verified":["go test ./..."]}`),
))
if !strings.Contains(rep.Gaps[0].Detail, "its last run failed") {
t.Fatalf("gap = %+v, want the failed run named", rep.Gaps[0])
}
}
func TestClaimedVerificationBeforeTheLatestChangeIsUnbacked(t *testing.T) {
rep := Build(nil, ledgerOf(
ran("go test ./...", true),
wrote("parser.go"),
read("parser.go"),
claimed("complete", `{"verified":["go test ./..."]}`),
))
if !strings.Contains(rep.Gaps[0].Detail, "before the latest change") {
t.Fatalf("gap = %+v, want the stale claim named", rep.Gaps[0])
}
}
func TestBackedClaimAddsNoGap(t *testing.T) {
rep := Build(nil, ledgerOf(
wrote("parser.go"),
read("parser.go"),
ran("go test ./...", true),
claimed("complete", `{"verified":["go test ./..."]}`),
))
if len(rep.Gaps) != 0 {
t.Fatalf("gaps = %+v, want none: the claim matches a fresh successful receipt", rep.Gaps)
}
if rep.Verdict != VerdictDone {
t.Fatalf("verdict = %v, want done", rep.Verdict)
}
if len(rep.Claimed.Verified) != 1 {
t.Fatalf("claim not recorded: %+v", rep.Claimed)
}
}
// A cited command need not be byte-identical to the one that ran; the shared
// segment matcher decides, exactly as complete_step does.
func TestClaimMatchesTheCommandAsItActuallyRan(t *testing.T) {
rep := Build(nil, ledgerOf(
wrote("parser.go"),
read("parser.go"),
ran("cd /repo && go test ./...", true),
claimed("complete", `{"verified":["go test ./..."]}`),
))
if len(rep.Gaps) != 0 {
t.Fatalf("gaps = %+v, want none: a cd prefix is not a different command", rep.Gaps)
}
}
func TestDeclaredUnverifiedAndRisksSurvive(t *testing.T) {
rep := Build(nil, ledgerOf(
wrote("parser.go"),
read("parser.go"),
ran("go test ./...", true),
claimed("complete", `{"unverified":["desktop UI never exercised"],"risks":["the migration is one-way"]}`),
))
if got := gapKinds(rep); !slices.Equal(got, []string{"declared_unverified"}) {
t.Fatalf("gap kinds = %v, want the self-declared gap kept", got)
}
if rep.Verdict != VerdictPartial {
t.Fatalf("verdict = %v, want partial: declaring a gap is honest, not clean", rep.Verdict)
}
if !slices.Equal(rep.Risks, []string{"the migration is one-way"}) {
t.Fatalf("risks = %v", rep.Risks)
}
}
// The invariant that makes the claim safe to accept at all.
func TestClaimCannotClearAHostFoundGap(t *testing.T) {
receipts := []evidence.Receipt{wrote("parser.go"), ran("go test ./...", true)}
bare := Build(nil, ledgerOf(receipts...))
withClaim := Build(nil, ledgerOf(append(receipts,
claimed("complete", `{"verified":["go test ./..."],"unverified":[],"risks":[]}`))...))
if len(withClaim.Gaps) < len(bare.Gaps) {
t.Fatalf("a claim removed a host-found gap: %d -> %d", len(bare.Gaps), len(withClaim.Gaps))
}
if !slices.Contains(gapKinds(withClaim), "unreviewed_change") {
t.Fatalf("gap kinds = %v, want the host's own finding intact", gapKinds(withClaim))
}
}
func TestFailedUpdateGoalClaimsNothing(t *testing.T) {
rejected := evidence.Receipt{
ToolName: "update_goal", Success: false,
Args: []byte(`{"status":"complete","completion":{"verified":["go test ./..."]}}`),
}
rep := Build(nil, ledgerOf(wrote("parser.go"), read("parser.go"), rejected))
if !rep.Claimed.Empty() {
t.Fatalf("a rejected update_goal must claim nothing, got %+v", rep.Claimed)
}
if got := gapKinds(rep); slices.Contains(got, "unbacked_claim") {
t.Fatalf("gap kinds = %v, want no claim gap from a failed call", got)
}
}
func TestLatestClaimWins(t *testing.T) {
rep := Build(nil, ledgerOf(
wrote("parser.go"),
read("parser.go"),
claimed("continue", `{"unverified":["nothing run yet"]}`),
ran("go test ./...", true),
claimed("complete", `{"verified":["go test ./..."]}`),
))
if len(rep.Gaps) != 0 {
t.Fatalf("gaps = %+v, want the superseded claim dropped", rep.Gaps)
}
}
+16 -3
View File
@@ -62,16 +62,24 @@ type Verification struct {
type GapKind uint8
const (
GapUnprovenCriterion GapKind = iota
// GapUnbackedClaim is first because it is the worst: the turn asserted a
// verification the ledger does not support.
GapUnbackedClaim GapKind = iota
GapUnprovenCriterion
GapMissingCheck
GapFailedVerification
GapStaleVerification
GapUnverifiedChange
GapUnreviewedChange
GapDeclaredUnverified
)
func (k GapKind) String() string {
switch k {
case GapUnbackedClaim:
return "unbacked_claim"
case GapDeclaredUnverified:
return "declared_unverified"
case GapUnprovenCriterion:
return "unproven_criterion"
case GapMissingCheck:
@@ -106,6 +114,10 @@ type Report struct {
Changes []Change
Verifications []Verification
Gaps []Gap
// Claimed is what the turn said about itself; Risks is its declared risk
// list. Both are model-authored and never clear a host-found gap.
Claimed Claim
Risks []string
}
// Build derives the report from a contract and the turn's receipts. Both may
@@ -123,6 +135,7 @@ func Build(c *taskcontract.Contract, ledger *evidence.Ledger) Report {
rep.Risk = c.Risk
}
rep.Gaps = gapsOf(rep, c)
rep = reconcile(rep, claimOf(receipts), receipts)
rep.Verdict = verdictOf(rep, c)
return rep
}
@@ -272,7 +285,7 @@ func checkLabel(check taskcontract.Check) string {
func verdictOf(rep Report, c *taskcontract.Contract) Verdict {
declared := c != nil && (len(c.Requirements) > 0 || len(c.Checks) > 0)
switch {
case !declared && rep.Mutations == 0 && len(rep.Verifications) == 0:
case !declared && rep.Mutations == 0 && len(rep.Verifications) == 0 && rep.Claimed.Empty():
return VerdictUnknown
case c != nil && !c.Complete():
return VerdictIncomplete
@@ -305,7 +318,7 @@ func (r Report) Summary() string {
func (r Report) GapKinds() []string {
seen := map[GapKind]bool{}
var out []string
for _, kind := range []GapKind{GapUnprovenCriterion, GapMissingCheck, GapFailedVerification, GapStaleVerification, GapUnverifiedChange, GapUnreviewedChange} {
for _, kind := range []GapKind{GapUnbackedClaim, GapUnprovenCriterion, GapMissingCheck, GapFailedVerification, GapStaleVerification, GapUnverifiedChange, GapUnreviewedChange, GapDeclaredUnverified} {
for _, gap := range r.Gaps {
if gap.Kind == kind && !seen[kind] {
seen[kind] = true
+4
View File
@@ -636,6 +636,10 @@ type CompletionReportAudit struct {
VerificationsStale int
Gaps int
GapKinds []string
// ClaimsVerified counts the turn's own asserted verifications;
// ClaimsUnbacked is how many of them the ledger did not support.
ClaimsVerified int
ClaimsUnbacked int
}
// CompletionReportAuditSink is an optional sink capability; implementations
+37 -3
View File
@@ -23,7 +23,7 @@ type updateGoal struct{}
func (updateGoal) Name() string { return "update_goal" }
func (updateGoal) Description() string {
return "Report this turn's disposition for the active goal. Call it at the end of every goal turn instead of using prose markers: `continue` (work is ongoing — give a concrete next_action), `complete` (the request is fully done, output format and constraints satisfied, and verification was attempted or reported unavailable), or `blocked` (only the user can unblock: missing user-only information, an irreversible/externally visible operation, or changed scope). The host validates your claim against Delivery acceptance criteria and decides whether to continue automatically. Fields: `status` (required, one of continue|complete|blocked), `reason` (required for continue and blocked, optional for complete), `next_action` (optional concrete next step; recommended for continue)."
return "Report this turn's disposition for the active goal. Call it at the end of every goal turn instead of using prose markers: `continue` (work is ongoing — give a concrete next_action), `complete` (the request is fully done, output format and constraints satisfied, and verification was attempted or reported unavailable), or `blocked` (only the user can unblock: missing user-only information, an irreversible/externally visible operation, or changed scope). The host validates your claim against Delivery acceptance criteria and decides whether to continue automatically. Fields: `status` (required, one of continue|complete|blocked), `reason` (required for continue and blocked, optional for complete), `next_action` (optional concrete next step; recommended for continue), `completion` (recommended with complete: `verified` is checked against real receipts, while `unverified` and `risks` are yours to declare and never count against you)."
}
func (updateGoal) Schema() json.RawMessage {
@@ -32,7 +32,16 @@ func (updateGoal) Schema() json.RawMessage {
"properties":{
"status":{"type":"string","enum":["continue","complete","blocked"],"description":"continue = keep working autonomously; complete = the goal is fully done and verified; blocked = only the user can unblock."},
"reason":{"type":"string","description":"Short explanation. REQUIRED for continue and blocked; optional for complete."},
"next_action":{"type":"string","description":"Optional concrete next step. Recommended for continue so the host can guide the next turn."}
"next_action":{"type":"string","description":"Optional concrete next step. Recommended for continue so the host can guide the next turn."},
"completion":{
"type":"object",
"description":"Your own account of the finished work.",
"properties":{
"verified":{"type":"array","items":{"type":"string"},"description":"Commands you ran as proof, as they actually ran. One that never ran, failed, or predates your latest change is recorded as an unbacked claim."},
"unverified":{"type":"array","items":{"type":"string"},"description":"What you did NOT verify. The host cannot infer what you skipped, so stating it is the only way it is known — and it never blocks completion."},
"risks":{"type":"array","items":{"type":"string"},"description":"Known risks to carry forward."}
}
}
},
"required":["status"]
}`)
@@ -57,6 +66,11 @@ func (updateGoal) Execute(ctx context.Context, args json.RawMessage) (string, er
Status string `json:"status"`
Reason string `json:"reason"`
NextAction string `json:"next_action"`
Completion struct {
Verified []string `json:"verified"`
Unverified []string `json:"unverified"`
Risks []string `json:"risks"`
} `json:"completion"`
}
if err := json.Unmarshal(args, &p); err != nil {
return "", fmt.Errorf("invalid update_goal args: %w", err)
@@ -70,13 +84,33 @@ func (updateGoal) Execute(ctx context.Context, args json.RawMessage) (string, er
if (p.Status == "continue" || p.Status == "blocked") && strings.TrimSpace(p.Reason) == "" {
return "", fmt.Errorf("update_goal: reason is required for %s — no goal state was changed", p.Status)
}
for i, command := range p.Completion.Verified {
if strings.TrimSpace(command) == "" {
return "", fmt.Errorf("update_goal: completion.verified[%d] is empty — cite the command as it actually ran, or leave the list out", i)
}
}
recorder, ok := tool.GoalTurnRecorderFromContext(ctx)
if !ok {
return "", fmt.Errorf("update_goal is only available while an active goal turn is running — no goal state was changed")
}
return recorder.RecordGoalReport(tool.GoalReport{
result, err := recorder.RecordGoalReport(tool.GoalReport{
Status: p.Status,
Reason: strings.TrimSpace(p.Reason),
NextAction: strings.TrimSpace(p.NextAction),
})
if err != nil || p.Status != "complete" {
return result, err
}
return result + completionNote(len(p.Completion.Verified), len(p.Completion.Unverified), len(p.Completion.Risks)), nil
}
// completionNote tells the model what its own account will contribute to the
// host's completion report. The claim is reconciled against real receipts
// after the turn; it is recorded, never trusted.
func completionNote(verified, unverified, risks int) string {
if verified+unverified+risks == 0 {
return " No completion account was given: the report will carry only what the host could observe."
}
return fmt.Sprintf(" Completion account recorded (%d verified, %d unverified, %d risks); the verified commands are reconciled against this session's receipts.",
verified, unverified, risks)
}
+51
View File
@@ -90,6 +90,57 @@ func TestUpdateGoalRecordsReport(t *testing.T) {
}
}
func TestUpdateGoalAcknowledgesTheCompletionAccount(t *testing.T) {
toolFn, _, ctx := goalTool(t)
got, err := toolFn.Execute(ctx, json.RawMessage(
`{"status":"complete","completion":{"verified":["go test ./..."],"unverified":["desktop UI"],"risks":["one-way migration"]}}`))
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if !strings.Contains(got, "1 verified, 1 unverified, 1 risks") {
t.Fatalf("result = %q, want the recorded account echoed back", got)
}
if !strings.Contains(got, "reconciled against this session's receipts") {
t.Fatalf("result = %q, want the claim's fate stated", got)
}
}
func TestUpdateGoalCompleteWithoutAnAccountSaysSo(t *testing.T) {
toolFn, _, ctx := goalTool(t)
got, err := toolFn.Execute(ctx, json.RawMessage(`{"status":"complete"}`))
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if !strings.Contains(got, "No completion account was given") {
t.Fatalf("result = %q, want the silence named", got)
}
}
func TestUpdateGoalRejectsAnEmptyVerifiedEntry(t *testing.T) {
toolFn, rec, ctx := goalTool(t)
_, err := toolFn.Execute(ctx, json.RawMessage(`{"status":"complete","completion":{"verified":[" "]}}`))
if err == nil || !strings.Contains(err.Error(), "completion.verified[0] is empty") {
t.Fatalf("Execute() error = %v, want the empty citation rejected", err)
}
if len(rec.reports) != 0 {
t.Fatalf("rejected call still recorded a report: %+v", rec.reports)
}
}
// A continue turn may still declare what it has not verified; only the
// completion echo is reserved for complete.
func TestUpdateGoalContinueKeepsItsPlainResult(t *testing.T) {
toolFn, _, ctx := goalTool(t)
got, err := toolFn.Execute(ctx, json.RawMessage(
`{"status":"continue","reason":"still fixing","completion":{"unverified":["nothing run yet"]}}`))
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if strings.Contains(got, "Completion account recorded") {
t.Fatalf("result = %q, want no completion echo on continue", got)
}
}
func TestUpdateGoalRecorderErrorsPropagate(t *testing.T) {
toolFn, rec, ctx := goalTool(t)
rec.err = errors.New("conflicting reports this turn")
+4
View File
@@ -108,6 +108,8 @@ type CompletionReport struct {
VerificationsStale int `json:"verifications_stale,omitempty"`
Gaps int `json:"gaps,omitempty"`
GapKinds []string `json:"gap_kinds,omitempty"`
ClaimsVerified int `json:"claims_verified,omitempty"`
ClaimsUnbacked int `json:"claims_unbacked,omitempty"`
}
// ReadinessAudit mirrors evidence.ReadinessAudit with stable snake_case keys.
@@ -229,6 +231,8 @@ func (r *Recorder) RecordCompletionReport(a event.CompletionReportAudit) {
VerificationsStale: a.VerificationsStale,
Gaps: a.Gaps,
GapKinds: a.GapKinds,
ClaimsVerified: a.ClaimsVerified,
ClaimsUnbacked: a.ClaimsUnbacked,
}})
event.RecordCompletionReport(r.inner, a)
}