From 0bb0f44785db28d2201e20703d0b210d17a9812c Mon Sep 17 00:00:00 2001 From: esengine <359807859@qq.com> Date: Mon, 2 Nov 2026 07:16:52 -0800 Subject: [PATCH] refactor(delegation): sub-agents return prose, not a structured completion report complete_subtask, the host adjudication of its criteria and the 'lowered by the host' note are gone, with the handoff audit channel and its metrics, trajectory field and benchmark columns. A delegated child ends the way any turn does and the parent reads its last reply. The host's own receipts appended to a child's reply and the write-scope check stay: they need a record of what the child changed, which is replaced with the evidence ledger later. --- cmd/e2ebench/delegation.go | 11 +- cmd/e2ebench/delegation_test.go | 11 +- cmd/e2ebench/main.go | 14 +- .../assembly/boot/audit_forwarding_test.go | 1 - .../boot/effect_repo_map_example_test.go | 2 +- .../boot/effect_tool_declaration_test.go | 2 +- internal/contract/event/audit_forwarder.go | 4 - internal/contract/event/coalesce.go | 7 - internal/contract/event/sink_capabilities.go | 1 - .../contract/event/sink_capabilities_test.go | 3 - internal/contract/event/subagent_handoff.go | 69 ----- internal/contract/event/sync.go | 8 - .../contract/hostaudit/delegation_audit.go | 13 - .../frontend/cli/delegation_metrics_test.go | 23 +- internal/frontend/cli/run_metrics.go | 99 +------ .../cli/subagent_handoff_metrics_test.go | 45 --- internal/runtime/agent/agent_options.go | 10 +- internal/runtime/agent/child_registry.go | 1 - internal/runtime/agent/child_run.go | 162 +---------- internal/runtime/agent/complete_subtask.go | 135 --------- .../runtime/agent/complete_subtask_test.go | 179 ------------ .../runtime/agent/readonly_handoff_test.go | 112 -------- .../delegation/complete_subtask_test.go | 45 --- .../delegation/readonly_handoff_test.go | 27 -- .../runtime/delegation/readonly_subagent.go | 4 +- internal/runtime/delegation/task.go | 3 +- internal/runtime/delegation/task_test.go | 4 +- .../evidence/completion_adjudication_test.go | 97 ------- internal/safety/evidence/completion_report.go | 270 ------------------ .../evidence/completion_report_parse_test.go | 53 ---- internal/safety/evidence/evidence.go | 2 - internal/safety/evidence/quote_short_test.go | 33 --- .../state/trajectory/audit_routing_test.go | 51 ---- internal/state/trajectory/recorder.go | 72 ----- tools/repolint/baseline.json | 4 +- 35 files changed, 33 insertions(+), 1544 deletions(-) delete mode 100644 internal/contract/event/subagent_handoff.go delete mode 100644 internal/frontend/cli/subagent_handoff_metrics_test.go delete mode 100644 internal/runtime/agent/complete_subtask.go delete mode 100644 internal/runtime/agent/complete_subtask_test.go delete mode 100644 internal/runtime/agent/readonly_handoff_test.go delete mode 100644 internal/runtime/delegation/complete_subtask_test.go delete mode 100644 internal/runtime/delegation/readonly_handoff_test.go delete mode 100644 internal/safety/evidence/completion_adjudication_test.go delete mode 100644 internal/safety/evidence/completion_report.go delete mode 100644 internal/safety/evidence/completion_report_parse_test.go delete mode 100644 internal/safety/evidence/quote_short_test.go delete mode 100644 internal/state/trajectory/audit_routing_test.go diff --git a/cmd/e2ebench/delegation.go b/cmd/e2ebench/delegation.go index 4b3482ced2..c7c3228d8c 100644 --- a/cmd/e2ebench/delegation.go +++ b/cmd/e2ebench/delegation.go @@ -7,7 +7,7 @@ import "fmt" // arm costs more: did the extra agents produce verified work, or just tokens. func renderDelegation(results []result) string { var runs, nested, childCalls, parentCalls, mutations, dupes int - var reports, prose, falseDone, downgrades, violations int + var violations int var scopeHints, namedFiles, evidencePaths, discoveredPaths int solved, total, childTokens := 0, 0, 0 for _, r := range results { @@ -24,10 +24,6 @@ func renderDelegation(results []result) string { parentCalls += r.ToolCalls - r.SubagentToolCalls mutations += r.SubagentMutations dupes += r.DuplicateWorkPaths - reports += r.CompletionReports - prose += r.CompletionsProsedOnly - falseDone += r.FalseCompletions - downgrades += r.CriterionDowngrades violations += r.WriteScopeViolations scopeHints += r.ParentScopeHints namedFiles += r.ParentNamedFiles @@ -70,11 +66,6 @@ func renderDelegation(results []result) string { if dupes > 0 { b += fmt.Sprintf("- **duplicate work**: %d file(s) mutated by more than one child\n", dupes) } - closed := reports + prose - b += fmt.Sprintf("- closed with a checkable claim: **%d/%d** (%s)\n", reports, closed, pct(reports, closed)) - if falseDone > 0 { - b += fmt.Sprintf("- **false completions**: %d run(s), %d criterion claim(s) the host refused\n", falseDone, downgrades) - } if violations > 0 { b += fmt.Sprintf("- **write-scope violations**: %d\n", violations) } diff --git a/cmd/e2ebench/delegation_test.go b/cmd/e2ebench/delegation_test.go index 87de38e8fa..a85ae70ddc 100644 --- a/cmd/e2ebench/delegation_test.go +++ b/cmd/e2ebench/delegation_test.go @@ -27,9 +27,7 @@ func TestRenderDelegationExposesWhatDelegationCost(t *testing.T) { runMetrics: runMetrics{ ToolCalls: 30, SubagentToolCalls: 24, SubagentRuns: 3, SubagentNestedRuns: 1, SubagentMutations: 5, - DuplicateWorkPaths: 2, - CompletionReports: 2, CompletionsProsedOnly: 1, - FalseCompletions: 1, CriterionDowngrades: 3, + DuplicateWorkPaths: 2, WriteScopeViolations: 1, }, }}) @@ -38,8 +36,6 @@ func TestRenderDelegationExposesWhatDelegationCost(t *testing.T) { "parent **6** tool calls", "children **24**", "duplicate work**: 2", - "checkable claim: **2/3**", - "false completions**: 1 run(s), 3 criterion", "write-scope violations**: 1", } { if !strings.Contains(got, want) { @@ -92,14 +88,11 @@ func TestRenderDelegationSaysWhenEvidenceOriginIsUnscored(t *testing.T) { func TestRenderDelegationStaysQuietWhenNothingWentWrong(t *testing.T) { got := renderDelegation([]result{{ Passed: true, - runMetrics: runMetrics{ToolCalls: 12, SubagentToolCalls: 8, SubagentRuns: 2, CompletionReports: 2}, + runMetrics: runMetrics{ToolCalls: 12, SubagentToolCalls: 8, SubagentRuns: 2}, }}) for _, unwanted := range []string{"duplicate work", "false completions", "write-scope violations"} { if strings.Contains(got, unwanted) { t.Errorf("clean run reported %q:\n%s", unwanted, got) } } - if !strings.Contains(got, "checkable claim: **2/2**") { - t.Errorf("clean run lost its completion coverage:\n%s", got) - } } diff --git a/cmd/e2ebench/main.go b/cmd/e2ebench/main.go index eb5558513a..f9ad155e43 100644 --- a/cmd/e2ebench/main.go +++ b/cmd/e2ebench/main.go @@ -66,15 +66,11 @@ type runMetrics struct { // Delegation counters mirror internal/frontend/cli.RunMetrics. They are what makes a // single-agent arm comparable against a delegated one for the same model. - SubagentRuns int `json:"subagent_runs,omitempty"` - SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"` - SubagentMutations int `json:"subagent_mutations,omitempty"` - CompletionReports int `json:"completion_reports,omitempty"` - CompletionsProsedOnly int `json:"completions_prose_only,omitempty"` - FalseCompletions int `json:"false_completions,omitempty"` - CriterionDowngrades int `json:"criterion_downgrades,omitempty"` - WriteScopeViolations int `json:"write_scope_violations,omitempty"` - DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"` + SubagentRuns int `json:"subagent_runs,omitempty"` + SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"` + SubagentMutations int `json:"subagent_mutations,omitempty"` + WriteScopeViolations int `json:"write_scope_violations,omitempty"` + DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"` // Evidence origin: what the parent's own delegation text scoped and named, // and how much of what the children looked at they had to find themselves. ParentScopeHints int `json:"parent_scope_hints,omitempty"` diff --git a/internal/assembly/boot/audit_forwarding_test.go b/internal/assembly/boot/audit_forwarding_test.go index 30430928c7..4743e72224 100644 --- a/internal/assembly/boot/audit_forwarding_test.go +++ b/internal/assembly/boot/audit_forwarding_test.go @@ -23,7 +23,6 @@ func TestAuditCapabilitiesAreForwardedByEveryWrapper(t *testing.T) { "DelegationAuditSink": reflect.TypeFor[event.DelegationAuditSink](), "MemoryRecallSink": reflect.TypeFor[event.MemoryRecallSink](), "TurnCompletionSink": reflect.TypeFor[event.TurnCompletionSink](), - "SubagentHandoffSink": reflect.TypeFor[event.SubagentHandoffSink](), } wrappers := []event.Sink{ event.Sync(event.Discard), diff --git a/internal/assembly/boot/effect_repo_map_example_test.go b/internal/assembly/boot/effect_repo_map_example_test.go index b06366d14c..156941aac7 100644 --- a/internal/assembly/boot/effect_repo_map_example_test.go +++ b/internal/assembly/boot/effect_repo_map_example_test.go @@ -193,7 +193,7 @@ model = "x" if !strings.HasPrefix(systemMessage(req.Messages), profileBody) { t.Fatal("installed profile body did not reach the child") } - if !slices.Equal(toolSchemaNames(req.Tools), []string{"complete_subtask", "glob", "grep", "ls", "read_file"}) { + if !slices.Equal(toolSchemaNames(req.Tools), []string{"glob", "grep", "ls", "read_file"}) { t.Fatalf("child tool ceiling = %v", toolSchemaNames(req.Tools)) } var user string diff --git a/internal/assembly/boot/effect_tool_declaration_test.go b/internal/assembly/boot/effect_tool_declaration_test.go index 9336a6457c..34be9f717a 100644 --- a/internal/assembly/boot/effect_tool_declaration_test.go +++ b/internal/assembly/boot/effect_tool_declaration_test.go @@ -15,7 +15,7 @@ func TestEffectConditionalDeclarationsDoNotReachTheProviderPrefix(t *testing.T) for _, schema := range req.Tools { sent[schema.Name] = true } - for _, n := range []string{"advise", "best_of_n", "apply_isolated", "discard_isolated", "complete_subtask", "system_one", "submit_plan", "conclude_no_changes", "review_report"} { + for _, n := range []string{"advise", "best_of_n", "apply_isolated", "discard_isolated", "system_one", "submit_plan", "conclude_no_changes", "review_report"} { if sent[n] { t.Errorf("%s is declared conditional but reached the default request tools", n) } diff --git a/internal/contract/event/audit_forwarder.go b/internal/contract/event/audit_forwarder.go index e471323081..2c4db64d62 100644 --- a/internal/contract/event/audit_forwarder.go +++ b/internal/contract/event/audit_forwarder.go @@ -31,10 +31,6 @@ func (f AuditForwarder) RecordRunBudget(s RunBudgetSample) { RecordRunBudget(f.Inner, s) } -func (f AuditForwarder) RecordSubagentHandoff(a SubagentHandoffAudit) { - RecordSubagentHandoff(f.Inner, a) -} - // DelegationAuditSink receives one receipt per completed sub-agent run. type DelegationAuditSink interface { RecordDelegationAudit(a hostaudit.DelegationAudit) diff --git a/internal/contract/event/coalesce.go b/internal/contract/event/coalesce.go index 471fe77356..e92944edd7 100644 --- a/internal/contract/event/coalesce.go +++ b/internal/contract/event/coalesce.go @@ -185,13 +185,6 @@ func (c *coalescer) RecordDelegationAudit(a hostaudit.DelegationAudit) { RecordDelegationAudit(c.inner, a) } -func (c *coalescer) RecordSubagentHandoff(a SubagentHandoffAudit) { - c.mu.Lock() - c.enqueueFlushLocked() - c.drainAndUnlock() - RecordSubagentHandoff(c.inner, a) -} - func (c *coalescer) RecordTurnCompletion() { c.mu.Lock() c.enqueueFlushLocked() diff --git a/internal/contract/event/sink_capabilities.go b/internal/contract/event/sink_capabilities.go index 9ae70654c8..95f1e8a027 100644 --- a/internal/contract/event/sink_capabilities.go +++ b/internal/contract/event/sink_capabilities.go @@ -11,7 +11,6 @@ var capabilityContracts = map[string]reflect.Type{ "DelegationAudit": reflect.TypeFor[DelegationAuditSink](), "MemoryRecall": reflect.TypeFor[MemoryRecallSink](), "RunBudget": reflect.TypeFor[RunBudgetSink](), - "SubagentHandoff": reflect.TypeFor[SubagentHandoffSink](), "TurnCompletion": reflect.TypeFor[TurnCompletionSink](), "UsageQuote": reflect.TypeFor[UsageQuoter](), "WorkspaceMutation": reflect.TypeFor[WorkspaceMutationSink](), diff --git a/internal/contract/event/sink_capabilities_test.go b/internal/contract/event/sink_capabilities_test.go index fd0165d022..226b0362a5 100644 --- a/internal/contract/event/sink_capabilities_test.go +++ b/internal/contract/event/sink_capabilities_test.go @@ -37,9 +37,6 @@ func (s *capturingSink) RecordWorkspaceMutation(WorkspaceMutation) { func (s *capturingSink) RecordRunBudget(RunBudgetSample) { s.recorded = append(s.recorded, "RunBudget") } -func (s *capturingSink) RecordSubagentHandoff(SubagentHandoffAudit) { - s.recorded = append(s.recorded, "SubagentHandoff") -} // AuditForwarder is why a new capability does not have to be repeated at every // layer, so it must itself be complete. It reads the registry rather than a diff --git a/internal/contract/event/subagent_handoff.go b/internal/contract/event/subagent_handoff.go deleted file mode 100644 index f8f09690d6..0000000000 --- a/internal/contract/event/subagent_handoff.go +++ /dev/null @@ -1,69 +0,0 @@ -// subagent_handoff.go — how a delegated child closed, counted. -package event - -import "reasonix/internal/base/nilutil" - -// SubagentHandoffAudit is one child run's closing protocol, counted. It exists -// to answer whether a typed handoff is submitted naturally, before anything is -// built to insist on one — so it observes and never decides. Content-free: -// counts, enums and rounds, never a prompt, a path or a summary. -type SubagentHandoffAudit struct { - // Entrance names the delegation path (read_only_task, fleet, …) and Depth - // how deep the child sits. An aggregate hides an entrance that is failing. - Entrance string - Depth int - ReadOnly bool - // Expected is set where the host actually appended the contract, never - // inferred from a registry: the denominator has to be the host's own - // instruction, not a guess about one. - Expected bool - // Exit says how the run ended. A provider timeout that produced no report - // is not a compliance failure and must leave the denominator. - Exit string // completed | error | cancelled | no_answer - - // Attempts counts complete_subtask calls, Accepted the ones the tool took, - // Malformed the ones it refused. Missing with attempts is a schema - // problem; missing without them is a protocol one. - Attempts int - Accepted int - Malformed int - - // ReportRound and FinalRound say whether the report actually closed the - // run: the contract asks for it as the final call, and "submitted it - // eventually" is a different behaviour from "closed with it". - ReportRound int - FinalRound int - ToolCallsAfterReport int - - // Closed and NeedsWork are the verdicts the host issued, counted when each - // was issued. Accepted only says a call was well-formed; these say whether - // the host considered the sub-task done. - Closed int - NeedsWork int - - // What the host made of the claim, read from the adjudicated report rather - // than re-derived here. - ClaimedStatus string - AdjudicatedStatus string - LoweredClaims int - Criteria int - Evidence int - Unresolved int -} - -// SubagentHandoffSink is an optional sink capability; implementations must keep -// it content-free, like every other audit channel. -type SubagentHandoffSink interface { - RecordSubagentHandoff(SubagentHandoffAudit) -} - -// RecordSubagentHandoff forwards one child's closing protocol only to sinks -// that explicitly opt in. Ordinary UI sinks receive nothing. -func RecordSubagentHandoff(s Sink, a SubagentHandoffAudit) { - if nilutil.IsNil(s) { - return - } - if hs, ok := s.(SubagentHandoffSink); ok { - hs.RecordSubagentHandoff(a) - } -} diff --git a/internal/contract/event/sync.go b/internal/contract/event/sync.go index 602c959326..86c6011e69 100644 --- a/internal/contract/event/sync.go +++ b/internal/contract/event/sync.go @@ -44,14 +44,6 @@ func (s *syncSink) RecordDelegationAudit(a hostaudit.DelegationAudit) { RecordDelegationAudit(s.inner, a) } -func (s *syncSink) RecordSubagentHandoff(a SubagentHandoffAudit) { - s.mu.Lock() - defer s.mu.Unlock() - if hs, ok := s.inner.(SubagentHandoffSink); ok { - hs.RecordSubagentHandoff(a) - } -} - func (s *syncSink) RecordTurnCompletion() { s.mu.Lock() defer s.mu.Unlock() diff --git a/internal/contract/hostaudit/delegation_audit.go b/internal/contract/hostaudit/delegation_audit.go index ecdce05056..5e3087f754 100644 --- a/internal/contract/hostaudit/delegation_audit.go +++ b/internal/contract/hostaudit/delegation_audit.go @@ -12,12 +12,6 @@ type DelegationAudit struct { MutationPaths []string // ClaimViolations counts writes the host saw outside the declared claim. ClaimViolations int - // HasReport is false when the child ended in prose instead of a typed claim. - HasReport bool - // AdjudicatedStatus is the status the host was willing to back, and - // Downgrades counts the criterion claims it refused. - AdjudicatedStatus string - Downgrades int // ParentScopeHints counts directories the delegation narrowed the search to. ParentScopeHints int // ParentNamedFiles counts specific files the delegation text handed over. @@ -27,10 +21,3 @@ type DelegationAudit struct { // DiscoveredPaths is the subset of those no named file already covered. DiscoveredPaths int } - -// FalseCompletion reports a child that claimed more than the host could back. -// It is the count of refused criteria, not a comparison against a claimed -// status the host never independently recorded. -func (a DelegationAudit) FalseCompletion() bool { - return a.HasReport && a.Downgrades > 0 -} diff --git a/internal/frontend/cli/delegation_metrics_test.go b/internal/frontend/cli/delegation_metrics_test.go index 51212deb63..158579bcbe 100644 --- a/internal/frontend/cli/delegation_metrics_test.go +++ b/internal/frontend/cli/delegation_metrics_test.go @@ -3,8 +3,6 @@ package cli import ( "reasonix/internal/contract/hostaudit" "testing" - - "reasonix/internal/safety/evidence" ) // The instrument must be able to say, from one run, whether delegation @@ -13,17 +11,12 @@ func TestDelegationMetricsAggregateAcrossChildren(t *testing.T) { s := &metricsSink{} s.RecordDelegationAudit(hostaudit.DelegationAudit{ Depth: 1, ToolCalls: 6, Mutations: 2, - MutationPaths: []string{"api/handler.go", "api/handler_test.go"}, - HasReport: true, - AdjudicatedStatus: string(evidence.CompletionComplete), + MutationPaths: []string{"api/handler.go", "api/handler_test.go"}, }) s.RecordDelegationAudit(hostaudit.DelegationAudit{ Depth: 2, ToolCalls: 4, Mutations: 1, - MutationPaths: []string{"api/handler.go"}, - ClaimViolations: 1, - HasReport: true, - AdjudicatedStatus: string(evidence.CompletionPartial), - Downgrades: 2, + MutationPaths: []string{"api/handler.go"}, + ClaimViolations: 1, }) s.RecordDelegationAudit(hostaudit.DelegationAudit{Depth: 1, ToolCalls: 3}) @@ -34,14 +27,6 @@ func TestDelegationMetricsAggregateAcrossChildren(t *testing.T) { if m.SubagentMutations != 3 { t.Fatalf("mutations = %d, want 3", m.SubagentMutations) } - if m.CompletionReports != 2 || m.CompletionsProsedOnly != 1 { - t.Fatalf("reports = %d prose-only = %d, want 2/1", m.CompletionReports, m.CompletionsProsedOnly) - } - // One child claimed criteria the host refused: that is the false-completion - // signal an orchestration benchmark exists to surface. - if m.FalseCompletions != 1 || m.CriterionDowngrades != 2 { - t.Fatalf("false completions = %d downgrades = %d, want 1/2", m.FalseCompletions, m.CriterionDowngrades) - } if m.WriteScopeViolations != 1 { t.Fatalf("write scope violations = %d, want 1", m.WriteScopeViolations) } @@ -77,7 +62,7 @@ func TestDelegationMetricsSumEvidenceOriginForARatioOfTotals(t *testing.T) { // single-agent arm is a clean baseline rather than noise. func TestDelegationMetricsStayZeroForSingleAgentArm(t *testing.T) { s := &metricsSink{} - if m := s.m; m.SubagentRuns != 0 || m.CompletionReports != 0 || m.DuplicateWorkPaths != 0 { + if m := s.m; m.SubagentRuns != 0 || m.DuplicateWorkPaths != 0 { t.Fatalf("single-agent baseline is not zero: %+v", m) } } diff --git a/internal/frontend/cli/run_metrics.go b/internal/frontend/cli/run_metrics.go index 48e1641519..76c70f512a 100644 --- a/internal/frontend/cli/run_metrics.go +++ b/internal/frontend/cli/run_metrics.go @@ -75,45 +75,19 @@ type RunMetrics struct { // Delegation counters let one model be compared across orchestration arms // without scraping prose. Child tool calls are already split out as // SubagentToolCalls below; parent calls are ToolCalls minus that. - SubagentRuns int `json:"subagent_runs,omitempty"` - SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"` - SubagentMutations int `json:"subagent_mutations,omitempty"` - CompletionReports int `json:"completion_reports,omitempty"` - CompletionsProsedOnly int `json:"completions_prose_only,omitempty"` - FalseCompletions int `json:"false_completions,omitempty"` - CriterionDowngrades int `json:"criterion_downgrades,omitempty"` - WriteScopeViolations int `json:"write_scope_violations,omitempty"` - DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"` - ParentScopeHints int `json:"parent_scope_hints,omitempty"` - ParentNamedFiles int `json:"parent_named_files,omitempty"` - ChildEvidencePaths int `json:"child_evidence_paths,omitempty"` - ChildDiscoveredPaths int `json:"child_discovered_paths,omitempty"` - // Subagent handoff: whether a delegated child closed the way it was asked - // to. Judged is the denominator — expected to report and not killed by the - // provider — because a run that never got to close is not a refusal. - SubagentHandoffs int `json:"subagent_handoffs"` - SubagentHandoffExpected int `json:"subagent_handoff_expected"` - SubagentHandoffJudged int `json:"subagent_handoff_judged"` - SubagentHandoffAttempted int `json:"subagent_handoff_attempted"` - SubagentHandoffAccepted int `json:"subagent_handoff_accepted"` - SubagentHandoffMalformed int `json:"subagent_handoff_malformed"` - SubagentHandoffClosed int `json:"subagent_handoff_closed"` - SubagentHandoffNeedsWork int `json:"subagent_handoff_needs_work"` - SubagentHandoffNotTried int `json:"subagent_handoff_never_attempted"` - SubagentHandoffClosedWith int `json:"subagent_handoff_closed_with_report"` - SubagentHandoffToolsAfter int `json:"subagent_handoff_tool_calls_after_report"` - SubagentHandoffLowered int `json:"subagent_handoff_lowered_claims"` - SubagentHandoffReadOnly int `json:"subagent_handoff_read_only"` - SubagentHandoffExits map[string]int `json:"subagent_handoff_exits,omitempty"` + SubagentRuns int `json:"subagent_runs,omitempty"` + SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"` + SubagentMutations int `json:"subagent_mutations,omitempty"` + WriteScopeViolations int `json:"write_scope_violations,omitempty"` + DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"` + ParentScopeHints int `json:"parent_scope_hints,omitempty"` + ParentNamedFiles int `json:"parent_named_files,omitempty"` + ChildEvidencePaths int `json:"child_evidence_paths,omitempty"` + ChildDiscoveredPaths int `json:"child_discovered_paths,omitempty"` // Repairs that happened inside a delegated run rather than the parent's own // loop, so a child that continued only because the host pulled it back is // not read as one that continued on its own. SubagentRecoveries int `json:"subagent_recoveries"` - // Goal resumes that carried a frozen verification contract, and how many of - // those found the current declaration naming different criteria. Neither - // says which declaration should govern. - GoalResumesWithContract int `json:"goal_resumes_with_verification_contract"` - GoalVerificationDrifts int `json:"goal_verification_contract_drifts"` // Fanout times what the delegation counters only count: what the run waited // on its fan-outs, against what the same work costs one at a time. Fanout *FanoutMetrics `json:"fanout,omitempty"` @@ -459,21 +433,12 @@ func (s *metricsSink) RecordDelegationAudit(a hostaudit.DelegationAudit) { } s.m.SubagentMutations += a.Mutations s.m.WriteScopeViolations += a.ClaimViolations - s.m.CriterionDowngrades += a.Downgrades // Summed, never averaged: an independence rate is a ratio of these totals, // so a child that read two files cannot outweigh one that read forty. s.m.ParentScopeHints += a.ParentScopeHints s.m.ParentNamedFiles += a.ParentNamedFiles s.m.ChildEvidencePaths += a.EvidencePaths s.m.ChildDiscoveredPaths += a.DiscoveredPaths - if a.HasReport { - s.m.CompletionReports++ - } else { - s.m.CompletionsProsedOnly++ - } - if a.FalseCompletion() { - s.m.FalseCompletions++ - } if s.childMutations == nil { s.childMutations = map[string]int{} } @@ -485,52 +450,6 @@ func (s *metricsSink) RecordDelegationAudit(a hostaudit.DelegationAudit) { } } -func (s *metricsSink) RecordSubagentHandoff(a event.SubagentHandoffAudit) { - if s == nil { - return - } - defer event.RecordSubagentHandoff(s.inner, a) - s.mu.Lock() - defer s.mu.Unlock() - s.m.SubagentHandoffs++ - if a.Exit != "" { - if s.m.SubagentHandoffExits == nil { - s.m.SubagentHandoffExits = map[string]int{} - } - s.m.SubagentHandoffExits[a.Exit]++ - } - if a.Expected { - s.m.SubagentHandoffExpected++ - } - // A run the provider killed never reached the point of closing, so it - // leaves the compliance denominator instead of counting as a refusal. - if !a.Expected || a.Exit != "completed" { - return - } - s.m.SubagentHandoffJudged++ - if a.ReadOnly { - s.m.SubagentHandoffReadOnly++ - } - if a.Attempts == 0 { - s.m.SubagentHandoffNotTried++ - } else { - s.m.SubagentHandoffAttempted++ - } - if a.Accepted > 0 { - s.m.SubagentHandoffAccepted++ - } - if a.Malformed > 0 { - s.m.SubagentHandoffMalformed++ - } - if a.ReportRound > 0 && a.ToolCallsAfterReport == 0 { - s.m.SubagentHandoffClosedWith++ - } - s.m.SubagentHandoffClosed += a.Closed - s.m.SubagentHandoffNeedsWork += a.NeedsWork - s.m.SubagentHandoffToolsAfter += a.ToolCallsAfterReport - s.m.SubagentHandoffLowered += a.LoweredClaims -} - // MergeCapabilityAuditCounters copies capability counters into RunMetrics. func (m *RunMetrics) MergeCapabilityAuditCounters( routes, routedCandidates, routedRequire, routedPrefer, routedSuggest, declines int, diff --git a/internal/frontend/cli/subagent_handoff_metrics_test.go b/internal/frontend/cli/subagent_handoff_metrics_test.go deleted file mode 100644 index ea2e54f435..0000000000 --- a/internal/frontend/cli/subagent_handoff_metrics_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package cli - -import ( - "testing" - - "reasonix/internal/contract/event" -) - -// The compliance denominator is the point of this test: a child the provider -// killed never reached the point of closing, so it must leave the denominator -// rather than count as a refusal. Routing is asserted too — the audit is sent -// through a wrapper, not called on the sink directly, because a method that -// exists and is never reached is the bug this channel already had once. -func TestMetricsSinkCountsSubagentHandoffCompliance(t *testing.T) { - s := &metricsSink{inner: event.Discard} - chain := event.Sync(s) - - event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{ - Expected: true, Exit: "completed", ReadOnly: true, - }) - event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{ - Expected: true, Exit: "cancelled", - }) - event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{ - Expected: true, Exit: "completed", - Attempts: 1, Accepted: 1, ReportRound: 2, FinalRound: 2, LoweredClaims: 3, - }) - - m := s.m - if m.SubagentHandoffs != 3 || m.SubagentHandoffExpected != 3 || m.SubagentHandoffJudged != 2 { - t.Fatalf("totals = %d/%d/%d, want 3 observed, 3 expected, 2 judged", - m.SubagentHandoffs, m.SubagentHandoffExpected, m.SubagentHandoffJudged) - } - if m.SubagentHandoffNotTried != 1 || m.SubagentHandoffAttempted != 1 || m.SubagentHandoffAccepted != 1 { - t.Fatalf("never-attempted/attempted/accepted = %d/%d/%d, want 1/1/1 — the cancelled run must not be either", - m.SubagentHandoffNotTried, m.SubagentHandoffAttempted, m.SubagentHandoffAccepted) - } - if m.SubagentHandoffClosedWith != 1 || m.SubagentHandoffLowered != 3 || m.SubagentHandoffReadOnly != 1 { - t.Fatalf("closed/lowered/read-only = %d/%d/%d, want 1/3/1", - m.SubagentHandoffClosedWith, m.SubagentHandoffLowered, m.SubagentHandoffReadOnly) - } - if m.SubagentHandoffExits["completed"] != 2 || m.SubagentHandoffExits["cancelled"] != 1 { - t.Fatalf("exits = %v, want 2 completed and 1 cancelled", m.SubagentHandoffExits) - } -} diff --git a/internal/runtime/agent/agent_options.go b/internal/runtime/agent/agent_options.go index 16a851616f..dae1726955 100644 --- a/internal/runtime/agent/agent_options.go +++ b/internal/runtime/agent/agent_options.go @@ -8,7 +8,6 @@ import ( "reasonix/internal/ext/extension/dispatch" "reasonix/internal/runtime/capability" "reasonix/internal/runtime/writeclaim" - "reasonix/internal/safety/evidence" "reasonix/internal/safety/sandbox" "reasonix/internal/state/checkpoint" "reasonix/internal/state/instruction" @@ -182,9 +181,7 @@ type Options struct { // classification judges it instead of the raw Run input, so framing verbs // cannot arm expectations; the delegation audit scores evidence origin // against it, so only locations the parent wrote itself count as hints. - ClassifierTaskText string - ExpectCompletionReport bool // set where the contract is appended, never inferred - HandoffEntrance string // delegation path, for telemetry only + ClassifierTaskText string // CapabilityLedger is the optional turn-scoped capability route ledger for // Delivery require/prefer gates. Nil disables capability gates. @@ -192,11 +189,6 @@ type Options struct { // CapabilityAudit is the optional non-persisted metrics sink for routing. CapabilityAudit *capability.Audit - // RequireReviewReportKind, when non-empty, makes RunSubAgentWithSession fail - // unless the subagent recorded a successful review_report of this kind — - // review/security subagents must return typed, host-verifiable reports. - RequireReviewReportKind evidence.ReviewKind - // ReasoningLanguage controls visible reasoning language preference as transient // user-turn context. Empty/auto injects nothing. ReasoningLanguage string diff --git a/internal/runtime/agent/child_registry.go b/internal/runtime/agent/child_registry.go index fcb7a89ac2..4056236207 100644 --- a/internal/runtime/agent/child_registry.go +++ b/internal/runtime/agent/child_registry.go @@ -98,7 +98,6 @@ func SubagentToolRegistryForDepthWithRuntime(parent *tool.Registry, names []stri sub := FilterRegistry(parent, names, exclude...) bindLocal := builtin.ForkFileViews(sub) stripDirectMCPTools(sub) - AttachCompleteSubtaskTool(sub) attachSubagentCapabilityProxy(parent, sub, names, runtime, bindLocal) if bash, ok := sub.Get("bash"); ok { sub.Add(foregroundOnlyBash{inner: bash}) diff --git a/internal/runtime/agent/child_run.go b/internal/runtime/agent/child_run.go index 6f25c7b222..76048a6643 100644 --- a/internal/runtime/agent/child_run.go +++ b/internal/runtime/agent/child_run.go @@ -2,7 +2,6 @@ package agent import ( "context" - "errors" "fmt" "reasonix/internal/contract/hostaudit" "reasonix/internal/runtime/writeclaim" @@ -65,14 +64,8 @@ func RunSubAgentWithSession(ctx context.Context, prov provider.Provider, reg *to if planWorkflow && !strings.Contains(prompt, planmode.Marker) { prompt = planmode.Marker + "\n\n" + prompt } - if kind := opts.RequireReviewReportKind; kind != "" { - prompt = prompt + "\n\n" + reviewReportTaskContract(kind) - } ctx = WithoutRunTimeSink(ctx) sub := newDelegatedAgent(ctx, prov, reg, sess, opts, sink) - // One defer: this returns from a salvage, a review failure, a provider - // error and a clean answer, and per-path calls would miss one. - defer func() { observeSubagentHandoff(sink, sub, sess, opts, answer, err) }() sub.SetPlanMode(planWorkflow) if err := sub.Run(ctx, prompt); err != nil { // Still merge any partial child evidence so parent gates see real writes. @@ -230,15 +223,10 @@ func (a *Agent) EvidenceSummary() evidence.ChildEvidenceSummary { } // composeSubagentAnswer assembles everything the parent is shown for one child -// run: the host-adjudicated completion claim when the child submitted one, the -// child's own prose, then the host's receipts. +// run: the child's own prose, then the host's receipts. func composeSubagentAnswer(ctx context.Context, answer string, sub *Agent, claims writeclaim.WritePathSet, delegationText string) string { summary := sub.EvidenceSummary() - report, reasons, hasReport := sub.CompletionReport() - if hasReport { - answer = strings.TrimSpace(formatCompletionReport(report, reasons) + "\n\n" + answer) - } - recordDelegationAudit(ctx, summary, claims, report, reasons, hasReport, delegationText) + recordDelegationAudit(ctx, summary, claims, delegationText) return AppendHostReceipts(answer, summary, claims) } @@ -259,52 +247,6 @@ func newDelegatedAgent(ctx context.Context, prov provider.Provider, reg *tool.Re return sub } -// observeSubagentHandoff records one child run's closing protocol. Called from -// a single defer so every exit path is counted the same: a salvage, a review -// failure and a clean answer are all runs that either submitted a report or -// did not. -func observeSubagentHandoff(sink event.Sink, sub *Agent, sess *sessionstore.Session, opts Options, answer string, runErr error) { - if sub == nil || sess == nil { - return - } - audit := event.SubagentHandoffAudit{ - Entrance: opts.HandoffEntrance, - Depth: opts.SubagentDepth, - ReadOnly: opts.ReadOnlyExecution, - Expected: opts.ExpectCompletionReport, - Exit: handoffExit(answer, runErr), - } - countReportCalls(&audit, sess.Snapshot()) - // Claimed and adjudicated are both kept: a child that submits reports - // readily and has them lowered every time is a different problem from one - // that does not submit, and a single status cannot say which. - if sub.task.ledger != nil { - verdicts := sub.task.ledger.ClosureVerdicts() - audit.Closed, audit.NeedsWork = verdicts.Closed, verdicts.NeedsWork - if claimed, ok := sub.task.ledger.LatestCompletionReport(); ok { - adjudicated, reasons := sub.task.ledger.AdjudicateCompletion(claimed) - audit.ClaimedStatus = string(claimed.Status) - audit.AdjudicatedStatus = string(adjudicated.Status) - audit.LoweredClaims = len(reasons) - audit.Criteria = len(adjudicated.Criteria) - audit.Unresolved = len(adjudicated.Unresolved) - for _, c := range adjudicated.Criteria { - audit.Evidence += len(c.Evidence) - } - } - } - event.RecordSubagentHandoff(sink, audit) -} - -// reviewReportTaskContract is appended to the task prompt of a review subagent -// whose run must end with a typed report. The skill body describes how to -// review; this states the non-negotiable submission protocol. -func reviewReportTaskContract(kind evidence.ReviewKind) string { - return fmt.Sprintf(` -Before your final answer you MUST call the review_report tool exactly once with kind=%q, your verdict (pass | warn | block), reviewed_paths listing only files you actually read this run, and your findings. The host discards a review run that ends without a successful review_report call — your prose summary alone does not count. -`, string(kind)) -} - const subagentStartContext = ` Before acting, check the available skills and tools. If a relevant skill is available, invoke it before continuing. Delegate to another sub-agent only when the task genuinely benefits from isolated context and the delegation tool is available. ` @@ -325,52 +267,20 @@ type mutationObserverSetter interface { SetMutationObserver(*checkpoint.MutationObserver) } -// formatCompletionReport renders the child's claim after the host has lowered -// whatever its receipts could not back. Downgrades are shown, never silently -// applied: a parent that cannot see the adjudication cannot trust the status. -func formatCompletionReport(report evidence.CompletionReport, reasons []string) string { - var b strings.Builder - b.WriteString("status: ") - b.WriteString(string(report.Status)) - if len(reasons) > 0 { - b.WriteString(" (lowered by the host: unbacked criterion claims)") - } - b.WriteString("\nsummary: ") - b.WriteString(report.Summary) - for _, c := range report.Criteria { - b.WriteString("\n " + c.ID + " " + string(c.Status)) - if proof := criterionProof(c); proof != "" { - b.WriteString(" — " + proof) - } - } - for _, reason := range reasons { - b.WriteString("\n host lowered " + reason) - } - for _, u := range report.Unresolved { - b.WriteString("\nunresolved: " + u) - } - return b.String() -} - // recordDelegationAudit emits one structured receipt per child run. It reports // what the host observed and what it refused to back, so an orchestration // benchmark can separate real gains from extra tokens spent. delegationText is // the parent-authored task before host framing, which is what makes the // evidence-origin split a host record rather than a claim. -func recordDelegationAudit(ctx context.Context, summary evidence.ChildEvidenceSummary, claims writeclaim.WritePathSet, report evidence.CompletionReport, reasons []string, hasReport bool, delegationText string) { +func recordDelegationAudit(ctx context.Context, summary evidence.ChildEvidenceSummary, claims writeclaim.WritePathSet, delegationText string) { audit := hostaudit.DelegationAudit{ Depth: SubagentDepth(ctx), ToolCalls: len(summary.Receipts), MutationPaths: summary.MutationPaths(), ClaimViolations: len(claimViolations(summary, claims)), - HasReport: hasReport, - Downgrades: len(reasons), } audit.Mutations = len(audit.MutationPaths) evidence.ClassifyEvidenceOrigin(&audit, delegationText, summary.EvidencePaths()) - if hasReport { - audit.AdjudicatedStatus = string(report.Status) - } _, sink, _, _ := CallContext(ctx) event.RecordDelegationAudit(sink, audit) } @@ -390,57 +300,6 @@ func AppendHostReceipts(answer string, summary evidence.ChildEvidenceSummary, cl return answer + "\n\n" + block } -// handoffExit classifies how the run ended. A provider failure that produced no -// report is not a compliance failure, and counting it as one would make the -// denominator answer a different question. -func handoffExit(answer string, runErr error) string { - switch { - case errors.Is(runErr, context.Canceled): - return "cancelled" - case runErr != nil: - return "error" - case strings.TrimSpace(answer) == "": - return "no_answer" - default: - return "completed" - } -} - -// countReportCalls walks the child's transcript for complete_subtask calls and -// what the tool made of them. Attempted-and-refused is a schema problem; -// never-attempted is a protocol one, and only the counts can tell them apart. -func countReportCalls(audit *event.SubagentHandoffAudit, msgs []provider.Message) { - calls := map[string]bool{} - round := 0 - for _, m := range msgs { - if len(m.ToolCalls) > 0 { - round++ - } - for _, tc := range m.ToolCalls { - if tc.Name != completeSubtaskToolName { - if audit.ReportRound > 0 { - audit.ToolCallsAfterReport++ - } - continue - } - calls[tc.ID] = true - audit.Attempts++ - if audit.ReportRound == 0 { - audit.ReportRound = round - } - } - if m.Role != provider.RoleTool || !calls[m.ToolCallID] { - continue - } - if isErrorMessage(m) { - audit.Malformed++ - continue - } - audit.Accepted++ - } - audit.FinalRound = round -} - // claimViolations returns the mutations the host observed outside the write // claim the child declared. Tool-level confinement already refuses these, so a // non-empty result means a write reached the workspace through a surface the @@ -458,21 +317,6 @@ func claimViolations(summary evidence.ChildEvidenceSummary, claims writeclaim.Wr return out } -func criterionProof(c evidence.AcceptanceCriterion) string { - var parts []string - for _, e := range c.Evidence { - switch { - case strings.TrimSpace(e.Command) != "": - parts = append(parts, e.Command) - case len(e.Paths) > 0: - parts = append(parts, strings.Join(e.Paths, " ")) - case strings.TrimSpace(e.Summary) != "": - parts = append(parts, e.Kind+": "+e.Summary) - } - } - return joinBoundedReceipts(parts) -} - // formatHostReceipts renders only what the host is willing to attest to: files // the child really changed, and commands whose outcome the host observed. // Ordinary reads and greps are excluded on purpose — they are not claims a diff --git a/internal/runtime/agent/complete_subtask.go b/internal/runtime/agent/complete_subtask.go deleted file mode 100644 index 4fce5a45aa..0000000000 --- a/internal/runtime/agent/complete_subtask.go +++ /dev/null @@ -1,135 +0,0 @@ -package agent - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - "reasonix/internal/contract/tool" - "reasonix/internal/safety/evidence" -) - -// CompleteSubtaskTool is visible only inside sub-agent registries. It ends a -// delegated run with a structured, host-checkable claim instead of prose. It is -// never registered on the parent agent's tool surface. -type CompleteSubtaskTool struct{} - -func NewCompleteSubtaskTool() *CompleteSubtaskTool { return &CompleteSubtaskTool{} } - -// completeSubtaskToolName is the one spelling the contract, the registry and -// the telemetry all mean. -const completeSubtaskToolName = "complete_subtask" - -func init() { tool.RegisterConditional(&CompleteSubtaskTool{}) } - -func (*CompleteSubtaskTool) Name() string { return completeSubtaskToolName } - -func (*CompleteSubtaskTool) Description() string { - return "Close out this delegated sub-task with a structured result the parent can verify. Call once, last. status is complete, partial, blocked, or failed; summary states what is now true; acceptance_criteria lists each condition with the evidence for it; unresolved lists what you did not finish. The host checks every cited command and path against what it actually observed you do, and lowers any claim it cannot back." -} - -// ReadOnly is true: submitting a report changes no workspace state. It is the -// claim itself, not a mutation. -func (*CompleteSubtaskTool) ReadOnly() bool { return true } - -func (*CompleteSubtaskTool) Schema() json.RawMessage { - // Fixed schema — sub-agent registries only, so it never enters the parent - // prefix. - return json.RawMessage(`{ -"type":"object", -"properties":{ - "status":{"type":"string","description":"complete | partial | blocked | failed. Claim the truth: the host lowers a status its receipts cannot back."}, - "summary":{"type":"string","description":"What is now true as a result of this sub-task."}, - "acceptance_criteria":{ - "type":"array", - "description":"Each condition this sub-task had to meet, with proof.", - "items":{ - "type":"object", - "properties":{ - "id":{"type":"string","description":"Short stable id, e.g. AC1."}, - "status":{"type":"string","description":"satisfied | unsatisfied"}, - "evidence":{ - "type":"array", - "items":{ - "type":"object", - "properties":{ - "kind":{"type":"string","enum":["verification","review","diff","files","manual"],"description":"verification = a command was run (command REQUIRED); review = a review completed; diff = a code change (paths REQUIRED); files = files created/edited/inspected (paths REQUIRED); manual = a manual check, which the host cannot back on its own."}, - "summary":{"type":"string","description":"The evidence itself."}, - "command":{"type":"string","description":"REQUIRED for verification: the command as it actually ran."}, - "paths":{"type":"array","items":{"type":"string"},"description":"REQUIRED for diff/files: the files this evidence refers to."} - }, - "required":["kind","summary"] - } - } - }, - "required":["id","status"] - } - }, - "unresolved":{"type":"array","items":{"type":"string"},"description":"What remains undone, unverified, or blocked. Put anything you assumed rather than checked here."} -}, -"required":["status","summary"] -}`) -} - -func (*CompleteSubtaskTool) Execute(ctx context.Context, args json.RawMessage) (string, error) { - report, err := evidence.ParseCompletionReport(args) - if err != nil { - return "", err - } - led, ok := evidence.FromContext(ctx) - if !ok { - return "", fmt.Errorf("complete_subtask requires the host evidence ledger; submit it from inside a sub-agent run") - } - // The submission always succeeds; what varies is the verdict. A rejection - // the parent never learns about serves it worse than a lowered claim it can - // see, so the call stands and the closure decision is said out loud. - adjudicated, reasons := led.AdjudicateCompletion(report) - closed := len(reasons) == 0 - led.NoteClosureVerdict(closed) - if closed { - return fmt.Sprintf("complete_subtask closed: status=%s criteria=%d unresolved=%d — the sub-task is done and nothing further is asked of you", - adjudicated.Status, len(adjudicated.Criteria), len(adjudicated.Unresolved)), nil - } - // The host knows which claims it could not back and why; reporting only the - // count leaves the sub-agent guessing which criterion and what would fix it. - return fmt.Sprintf("complete_subtask needs_work: status=%s criteria=%d unresolved=%d — these claims are not backed by anything the host observed:\n- %s\nDo that work, then submit again.", - adjudicated.Status, len(adjudicated.Criteria), len(adjudicated.Unresolved), strings.Join(reasons, "\n- ")), nil -} - -// AttachCompleteSubtaskTool adds complete_subtask to a sub-agent registry. -// Parent registries never receive it. -func AttachCompleteSubtaskTool(reg *tool.Registry) { - if reg == nil { - return - } - reg.Add(NewCompleteSubtaskTool()) -} - -// CompletionReport returns this agent's adjudicated completion claim, if it -// submitted one. Adjudication re-runs here so the returned status reflects the -// full run, including receipts recorded after the tool call itself. -func (a *Agent) CompletionReport() (evidence.CompletionReport, []string, bool) { - if a == nil || a.task.ledger == nil { - return evidence.CompletionReport{}, nil, false - } - report, ok := a.task.ledger.LatestCompletionReport() - if !ok { - return evidence.CompletionReport{}, nil, false - } - adjudicated, reasons := a.task.ledger.AdjudicateCompletion(report) - return adjudicated, reasons, true -} - -// CompleteSubtaskContract is appended to a sub-agent's task prompt when the -// host expects a typed completion claim. The profile body says how to work; -// this states the non-negotiable closing protocol. -const CompleteSubtaskContract = ` -Call complete_subtask when you believe this sub-task is done. State the acceptance -criteria you were held to and attach, for each, the command you ran or the paths you -changed or inspected. The host checks every citation against what it observed you -actually do, and answers with a verdict: needs_work means a claim is not backed by -anything it saw — do that work and submit again; closed means the sub-task is done and -nothing further is asked of you. Cite real work only and put anything you assumed -rather than verified in unresolved. -` diff --git a/internal/runtime/agent/complete_subtask_test.go b/internal/runtime/agent/complete_subtask_test.go deleted file mode 100644 index cb916170a0..0000000000 --- a/internal/runtime/agent/complete_subtask_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package agent - -import ( - "context" - "encoding/json" - "strings" - "testing" - - "reasonix/internal/safety/evidence" -) - -func submitCompleteSubtask(t *testing.T, led *evidence.Ledger, args string) string { - t.Helper() - ctx := evidence.WithLedger(context.Background(), led) - out, err := NewCompleteSubtaskTool().Execute(ctx, json.RawMessage(args)) - if err != nil { - t.Fatalf("complete_subtask: %v", err) - } - return out -} - -// The child may claim anything; the status the parent sees is the host's. -func TestCompleteSubtaskHostLowersUnbackedClaim(t *testing.T) { - led := evidence.NewLedger() - led.Record(evidence.Receipt{ToolName: "bash", Command: "go test ./parser", Success: true, OutputBytes: 12}) - - args := `{ - "status":"complete", - "summary":"fixed the parser", - "acceptance_criteria":[ - {"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"unit tests","command":"go test ./parser"}]}, - {"id":"AC2","status":"satisfied","evidence":[{"kind":"verification","summary":"integration suite","command":"go test ./integration"}]} - ]}` - if out := submitCompleteSubtask(t, led, args); !strings.Contains(out, "status=partial") { - t.Fatalf("tool result = %q, want the host-lowered status", out) - } - - // The agent host, not the tool, records the call; replay that receipt so the - // ledger lookup the report renderer uses is covered too. - led.Record(evidence.ReceiptFromToolCall("complete_subtask", json.RawMessage(args), true, evidence.ToolFacts{ReadOnly: true})) - report, ok := led.LatestCompletionReport() - if !ok { - t.Fatal("a recorded complete_subtask call must be recoverable from the ledger") - } - adjudicated, reasons := led.AdjudicateCompletion(report) - if adjudicated.Status != evidence.CompletionPartial { - t.Fatalf("status = %q, want partial", adjudicated.Status) - } - if adjudicated.Criteria[0].Status != evidence.CriterionSatisfied { - t.Fatal("AC1 was backed by a real command receipt and must survive") - } - if adjudicated.Criteria[1].Status != evidence.CriterionUnsatisfied { - t.Fatal("AC2 cited a command that never ran and must be lowered") - } - if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "AC2:") { - t.Fatalf("reasons = %v, want one naming AC2", reasons) - } -} - -func TestCompleteSubtaskKeepsFullyBackedClaim(t *testing.T) { - led := evidence.NewLedger() - led.Record(evidence.Receipt{ToolName: "bash", Command: "go test ./parser", Success: true, OutputBytes: 12}) - led.Record(evidence.Receipt{ToolName: "write_file", Success: true, Mutation: true, Write: true, Paths: []string{"parser.go"}}) - - out := submitCompleteSubtask(t, led, `{ - "status":"complete", - "summary":"fixed the parser", - "acceptance_criteria":[ - {"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"tests","command":"go test ./parser"}]}, - {"id":"AC2","status":"satisfied","evidence":[{"kind":"diff","summary":"the fix","paths":["parser.go"]}]} - ]}`) - if !strings.Contains(out, "status=complete") || strings.Contains(out, "lowered") { - t.Fatalf("tool result = %q, want an untouched complete status", out) - } -} - -// A satisfied criterion resting only on the model's word is not host-backed. -func TestCompleteSubtaskLowersManualOnlyAndEvidenceFreeClaims(t *testing.T) { - led := evidence.NewLedger() - report, err := evidence.ParseCompletionReport(json.RawMessage(`{ - "status":"complete","summary":"done", - "acceptance_criteria":[ - {"id":"AC1","status":"satisfied","evidence":[{"kind":"manual","summary":"I checked it"}]}, - {"id":"AC2","status":"satisfied"} - ]}`)) - if err != nil { - t.Fatal(err) - } - adjudicated, reasons := led.AdjudicateCompletion(report) - if adjudicated.Status != evidence.CompletionPartial || len(reasons) != 2 { - t.Fatalf("status = %q reasons = %v, want partial with both lowered", adjudicated.Status, reasons) - } -} - -func TestParseCompletionReportRejectsMalformedClaims(t *testing.T) { - for name, args := range map[string]string{ - "bad status": `{"status":"done","summary":"x"}`, - "missing summary": `{"status":"complete"}`, - "verification no cmd": `{"status":"complete","summary":"x","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"tests"}]}]}`, - "diff without paths": `{"status":"complete","summary":"x","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"diff","summary":"change"}]}]}`, - "unknown evidence": `{"status":"complete","summary":"x","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"vibes","summary":"trust me"}]}]}`, - "criterion without id": `{"status":"complete","summary":"x","acceptance_criteria":[{"status":"satisfied"}]}`, - } { - if _, err := evidence.ParseCompletionReport(json.RawMessage(args)); err == nil { - t.Errorf("%s: accepted a malformed report", name) - } - } -} - -// A verdict is not an error. A malformed submission fails the call; a claim the -// host cannot back is a well-formed call the host answers with needs_work, and -// keeping those apart is what lets "submit again" and "you are done" be -// different answers instead of two readings of the word accepted. -func TestCompleteSubtaskVerdictSeparatesClosureFromMoreWork(t *testing.T) { - const criterion = `{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"tests","command":"go test ./parser"}]}` - - backed := evidence.NewLedger() - backed.Record(evidence.Receipt{ToolName: "bash", Command: "go test ./parser", Success: true, OutputBytes: 12}) - out := submitCompleteSubtask(t, backed, `{"status":"complete","summary":"fixed it","acceptance_criteria":[`+criterion+`]}`) - if !strings.Contains(out, "complete_subtask closed:") { - t.Fatalf("backed report = %q, want a closed verdict", out) - } - if v := backed.ClosureVerdicts(); v.Closed != 1 || v.NeedsWork != 0 { - t.Fatalf("verdicts = %+v, want one closure", v) - } - - // The same claim with nothing behind it: the call still succeeds. - unbacked := evidence.NewLedger() - out = submitCompleteSubtask(t, unbacked, `{"status":"complete","summary":"fixed it","acceptance_criteria":[`+criterion+`]}`) - if !strings.Contains(out, "complete_subtask needs_work:") { - t.Fatalf("unbacked report = %q, want a needs_work verdict", out) - } - if v := unbacked.ClosureVerdicts(); v.NeedsWork != 1 || v.Closed != 0 { - t.Fatalf("verdicts = %+v, want one needs_work", v) - } - - // A malformed submission is the other layer and still fails the call. - ctx := evidence.WithLedger(context.Background(), evidence.NewLedger()) - if _, err := NewCompleteSubtaskTool().Execute(ctx, json.RawMessage(`{"status":"nonsense"}`)); err == nil { - t.Fatal("a malformed report must fail the call, not return a verdict") - } -} - -// A rejection the sub-agent cannot act on costs a whole round: it resubmits -// with a different evidence shape and hopes. The host knows which criterion it -// could not back and what it looked for, so the verdict says both. -func TestCompleteSubtaskRejectionNamesWhatTheHostLookedFor(t *testing.T) { - led := evidence.NewLedger() - ctx := evidence.WithLedger(context.Background(), led) - - args := []byte(`{"status":"complete","summary":"done", - "acceptance_criteria":[ - {"id":"c1","status":"satisfied","evidence":[ - {"kind":"files","summary":"read the files","paths":["internal/runtime/agent/agent.go","internal/runtime/agent/run_loop.go"]}]}, - {"id":"c2","status":"satisfied","evidence":[ - {"kind":"verification","summary":"ran the tests","command":"go test ./internal/runtime/agent/"}]}, - {"id":"c3","status":"satisfied","evidence":[ - {"kind":"review","summary":"first reviewer"}, - {"kind":"review","summary":"second reviewer"}]}]}`) - - out, err := (&CompleteSubtaskTool{}).Execute(ctx, args) - if err != nil { - t.Fatalf("Execute: %v", err) - } - if !strings.Contains(out, "needs_work") { - t.Fatalf("verdict = %q, want needs_work for claims with no receipts", out) - } - if n := strings.Count(out, "no completed review"); n > 1 { - t.Errorf("one missing receipt is reported %d times, which reads as %d problems:\n%s", n, n, out) - } - for _, want := range []string{ - "c1:", "no read or write recorded for", "internal/runtime/agent/agent.go", - "c2:", "no successful run of", "go test ./internal/runtime/agent/", - } { - if !strings.Contains(out, want) { - t.Errorf("verdict does not say %q; the sub-agent cannot tell what to fix:\n%s", want, out) - } - } -} diff --git a/internal/runtime/agent/readonly_handoff_test.go b/internal/runtime/agent/readonly_handoff_test.go deleted file mode 100644 index 94f939ba53..0000000000 --- a/internal/runtime/agent/readonly_handoff_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package agent - -import ( - "context" - "errors" - "fmt" - "reasonix/internal/contract/tool" - "strings" - "testing" - - "reasonix/internal/contract/event" - "reasonix/internal/contract/provider" -) - -// A writer child submits a typed claim the host adjudicates. A read-only child -// used to end in prose, so what it found reached the parent as an unverifiable -// sentence while its receipts sat elsewhere in the host ledger. - -// Both kinds of child are told the same closing protocol, and the wording has -// to fit a child that only looked at things. It also has to name both verdicts: -// a contract that asks for one call cannot be honoured against a host that -// answers a lowered claim by asking for the work again. -func TestCompletionContractCoversInspectionNotOnlyChange(t *testing.T) { - if !strings.Contains(CompleteSubtaskContract, "complete_subtask") { - t.Error("the contract no longer names the call it requires") - } - for _, verdict := range []string{"needs_work", "closed"} { - if !strings.Contains(CompleteSubtaskContract, verdict) { - t.Errorf("the contract does not say what %q means, so the child cannot act on it", verdict) - } - } - if !strings.Contains(CompleteSubtaskContract, "inspected") { - t.Errorf("the contract asks only about changes, so a read-only child has nothing to cite:\n%s", - CompleteSubtaskContract) - } -} - -// The shadow answers "did the child close the way it was asked to", so its -// denominator must be the instruction the host actually gave. -func TestHandoffShadowCountsAttemptsApartFromAcceptance(t *testing.T) { - audit := event.SubagentHandoffAudit{} - countReportCalls(&audit, []provider.Message{ - {Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{ - {ID: "a", Name: "read_file", Arguments: `{"path":"x"}`}, - }}, - {Role: provider.RoleTool, ToolCallID: "a", Name: "read_file", Content: "ok"}, - {Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{ - {ID: "r1", Name: completeSubtaskToolName, Arguments: `{}`}, - }}, - {Role: provider.RoleTool, ToolCallID: "r1", Name: completeSubtaskToolName, Content: "error: invalid status"}, - {Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{ - {ID: "r2", Name: completeSubtaskToolName, Arguments: `{"status":"complete"}`}, - }}, - {Role: provider.RoleTool, ToolCallID: "r2", Name: completeSubtaskToolName, Content: "recorded"}, - {Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{ - {ID: "b", Name: "read_file", Arguments: `{"path":"y"}`}, - }}, - {Role: provider.RoleTool, ToolCallID: "b", Name: "read_file", Content: "ok"}, - }) - - // A refused call is a schema problem; never calling is a protocol one, and - // one count cannot say which happened. - if audit.Attempts != 2 || audit.Accepted != 1 || audit.Malformed != 1 { - t.Errorf("attempts=%d accepted=%d malformed=%d, want 2/1/1", - audit.Attempts, audit.Accepted, audit.Malformed) - } - // The contract asks for the report as the final call. Work after it is a - // different behaviour from submitting one at all. - if audit.ToolCallsAfterReport != 1 { - t.Errorf("tool calls after the report = %d, want 1", audit.ToolCallsAfterReport) - } - if audit.ReportRound != 2 || audit.FinalRound != 4 { - t.Errorf("rounds = report %d final %d, want 2 and 4", audit.ReportRound, audit.FinalRound) - } -} - -// A run that died on the provider never had the chance to comply. Counting it -// as a missing report would make the adoption rate answer a different question. -func TestHandoffShadowSeparatesFailureFromNonCompliance(t *testing.T) { - for _, tc := range []struct { - name string - answer string - err error - want string - }{ - {"clean", "here is what I found", nil, "completed"}, - {"provider failed", "", errors.New("stream error"), "error"}, - // A wrapped sentinel, not a lookalike string: identity is what the - // classifier reads, so a message that merely says so must not pass. - {"user cancelled", "", fmt.Errorf("sub-agent: %w", context.Canceled), "cancelled"}, - {"message only says cancelled", "", errors.New("context canceled"), "error"}, - {"ran but said nothing", " ", nil, "no_answer"}, - } { - if got := handoffExit(tc.answer, tc.err); got != tc.want { - t.Errorf("%s: exit = %q, want %q", tc.name, got, tc.want) - } - } -} - -func TestCompleteSubtaskSurvivesTheStrictReadOnlyFilter(t *testing.T) { - reg := tool.NewRegistry() - reg.Add(subagentRegistryTool{name: "read_file", readOnly: true}) - AttachCompleteSubtaskTool(reg) - - // The child's registry is filtered once more at construction; a protocol - // tool that did not survive would leave the contract asking for a call the - // child cannot make. - strict := strictReadOnlyExecutionRegistry(reg) - if _, ok := strict.Get("complete_subtask"); !ok { - t.Fatalf("complete_subtask did not survive strict read-only construction: %v", strict.Names()) - } -} diff --git a/internal/runtime/delegation/complete_subtask_test.go b/internal/runtime/delegation/complete_subtask_test.go deleted file mode 100644 index 18d3a02bfc..0000000000 --- a/internal/runtime/delegation/complete_subtask_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package delegation - -import ( - "context" - "reasonix/internal/runtime/agent" - "reasonix/internal/state/sessionstore" - "strings" - "testing" - - "reasonix/internal/contract/event" - "reasonix/internal/contract/provider" - "reasonix/internal/contract/tool" -) - -// End to end: the parent's view leads with the adjudicated status, not prose. -func TestSubAgentAnswerLeadsWithAdjudicatedStatus(t *testing.T) { - reg := tool.NewRegistry() - reg.Add(fakeWriteFileTool{}) - agent.AttachCompleteSubtaskTool(reg) - prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ - {toolCallChunk("1", "write_file", `{"path":"parser.go"}`), {Type: provider.ChunkDone}}, - {toolCallChunk("2", "complete_subtask", `{"status":"complete","summary":"fixed the parser","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"diff","summary":"the fix","paths":["parser.go"]}]},{"id":"AC2","status":"satisfied","evidence":[{"kind":"verification","summary":"suite","command":"go test ./..."}]}],"unresolved":["integration suite not executed"]}`), {Type: provider.ChunkDone}}, - {{Type: provider.ChunkText, Text: "all good"}, {Type: provider.ChunkDone}}, - }} - - answer, err := agent.RunSubAgentWithSession(context.Background(), prov, reg, sessionstore.NewSession("sys"), - "fix the parser", agent.Options{CheckTargetAccess: allowTestFileAccess}, event.Discard) - if err != nil { - t.Fatalf("RunSubAgentWithSession: %v", err) - } - if !strings.HasPrefix(answer, "status: partial") { - t.Fatalf("answer must lead with the host-adjudicated status:\n%s", answer) - } - for _, want := range []string{ - "AC1 satisfied", - "AC2 unsatisfied", - "host lowered AC2", - "unresolved: integration suite not executed", - agent.HostReceiptsHeader, - } { - if !strings.Contains(answer, want) { - t.Fatalf("answer missing %q:\n%s", want, answer) - } - } -} diff --git a/internal/runtime/delegation/readonly_handoff_test.go b/internal/runtime/delegation/readonly_handoff_test.go deleted file mode 100644 index d61c7f48da..0000000000 --- a/internal/runtime/delegation/readonly_handoff_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package delegation - -import ( - "reasonix/internal/runtime/agent" - "testing" - - "reasonix/internal/contract/tool" -) - -// The protocol tool is not a capability. Attaching it before the "can this -// child investigate anything?" check would let a child with no research tools -// pass as having one, and it would then report on work it could not do. -func TestReportingToolDoesNotCountAsAResearchCapability(t *testing.T) { - parent := tool.NewRegistry() - parent.Add(subagentRegistryTool{name: "write_file"}) // not read-only - - sub := agent.ReadOnlySubagentToolRegistry(parent, nil) - if sub.Len() != 0 { - t.Fatalf("a parent with no read-only tools produced %v", sub.Names()) - } - // And once it is attached, the registry is no longer empty — which is - // exactly why the emptiness check has to come first. - agent.AttachCompleteSubtaskTool(sub) - if sub.Len() == 0 { - t.Fatal("complete_subtask was not attached at all") - } -} diff --git a/internal/runtime/delegation/readonly_subagent.go b/internal/runtime/delegation/readonly_subagent.go index 0ba81f372a..2a0661fa54 100644 --- a/internal/runtime/delegation/readonly_subagent.go +++ b/internal/runtime/delegation/readonly_subagent.go @@ -19,7 +19,6 @@ func (t *TaskTool) readOnlySubRegistry(spec *ProfileExecSpec, toolNames []string if sub.Len() == 0 && !spec.Grant.AllowNoTools { return nil, fmt.Errorf("no read-only tools available for this sub-agent") } - agent.AttachCompleteSubtaskTool(sub) return sub, nil } @@ -32,7 +31,6 @@ func (t *TaskTool) prepareSubSession(ctx context.Context, prompt string, opts ag // The pristine task, before framing: delivery classification judges the // task and not the wrapper. opts.ClassifierTaskText = prompt - opts.ExpectCompletionReport, opts.HandoffEntrance = true, entrance - prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt) + "\n\n" + agent.CompleteSubtaskContract + prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt) return agent.WithUserImages(ctx, agent.SubagentImageCandidates(ctx)), prompt, opts } diff --git a/internal/runtime/delegation/task.go b/internal/runtime/delegation/task.go index e28de51288..3e34e49c6d 100644 --- a/internal/runtime/delegation/task.go +++ b/internal/runtime/delegation/task.go @@ -824,8 +824,7 @@ func (t *TaskTool) runSubSession(ctx context.Context, prompt string, subReg *too // Capture the pristine task before host framing is prepended: delivery // intent classification must judge the task, not the wrapper. opts.ClassifierTaskText = prompt - prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt) + "\n\n" + agent.CompleteSubtaskContract - opts.ExpectCompletionReport, opts.HandoffEntrance = true, entrance + prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt) // Child provider owns the final vision decision. Text-only providers // retain the attachment metadata but omit image parts during serialization. ctx = agent.WithUserImages(ctx, agent.SubagentImageCandidates(ctx)) diff --git a/internal/runtime/delegation/task_test.go b/internal/runtime/delegation/task_test.go index b1cc7b42ec..8c97238e3f 100644 --- a/internal/runtime/delegation/task_test.go +++ b/internal/runtime/delegation/task_test.go @@ -54,7 +54,7 @@ func TestTaskToolReturnsSubAgentFinalAnswer(t *testing.T) { if sys := sub.lastReq.Messages[0]; sys.Role != provider.RoleSystem || sys.Content != "test-sys-prompt" { t.Errorf("first message = %+v, want system 'test-sys-prompt'", sys) } - if got := lastUser(sub.lastReq); !strings.Contains(got, ``) || !strings.Contains(got, "find callers of Foo") || !strings.Contains(got, agent.CompleteSubtaskContract) { + if got := lastUser(sub.lastReq); !strings.Contains(got, ``) || !strings.Contains(got, "find callers of Foo") { t.Errorf("sub-agent user = %q, want SubagentStart context plus prompt", got) } } @@ -89,7 +89,7 @@ func TestTaskToolInjectsWorkspaceContextIntoSubagentPrompt(t *testing.T) { if !strings.Contains(got, ``) || !strings.Contains(got, "Current workspace: "+strconv.Quote(workspace)) || !strings.Contains(got, `prefer "." or relative paths`) || - !strings.Contains(got, "inspect project") || !strings.Contains(got, agent.CompleteSubtaskContract) { + !strings.Contains(got, "inspect project") { t.Fatalf("sub-agent user = %q, want workspace context plus prompt", got) } } diff --git a/internal/safety/evidence/completion_adjudication_test.go b/internal/safety/evidence/completion_adjudication_test.go deleted file mode 100644 index 1f6ee15836..0000000000 --- a/internal/safety/evidence/completion_adjudication_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package evidence - -import ( - "strings" - "testing" -) - -func report(status CompletionStatus, criteria ...AcceptanceCriterion) CompletionReport { - return CompletionReport{Status: status, Summary: "done", Criteria: criteria} -} - -func satisfied(id string, evidence ...CompletionEvidence) AcceptanceCriterion { - return AcceptanceCriterion{ID: id, Status: CriterionSatisfied, Evidence: evidence} -} - -func TestAdjudicateCompletionKeepsWhatTheHostsOwnReceiptsBack(t *testing.T) { - l := recorded( - Receipt{ToolName: "bash", Success: true, Command: "go test ./..."}, - Receipt{ToolName: "edit_file", Success: true, Write: true, Mutation: true, Paths: []string{"a.go"}}, - Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"b.go"}}, - ) - in := report(CompletionComplete, - satisfied("tests", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test ./..."}), - satisfied("diff", CompletionEvidence{Kind: CompletionEvidenceDiff, Paths: []string{"a.go"}}), - satisfied("files", CompletionEvidence{Kind: CompletionEvidenceFiles, Paths: []string{"b.go"}}), - ) - got, reasons := l.AdjudicateCompletion(in) - if got.Status != CompletionComplete || len(reasons) != 0 { - t.Fatalf("a fully backed report was lowered: %q %v", got.Status, reasons) - } - for _, c := range got.Criteria { - if c.Status != CriterionSatisfied { - t.Fatalf("criterion %s was downgraded", c.ID) - } - } -} - -func TestAdjudicateCompletionDowngradesEveryClaimNoReceiptBacksAndCannotStayComplete(t *testing.T) { - l := recorded( - Receipt{ToolName: "bash", Success: false, Command: "go test ./..."}, - Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"a.go"}}, - ) - in := report(CompletionComplete, - satisfied("failed-run", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test ./..."}), - satisfied("read-is-not-a-write", CompletionEvidence{Kind: CompletionEvidenceDiff, Paths: []string{"a.go"}}), - satisfied("manual-only", CompletionEvidence{Kind: CompletionEvidenceManual, Summary: "trust me"}), - satisfied("no-evidence"), - AcceptanceCriterion{ID: "already-unsatisfied", Status: CriterionUnsatisfied}, - ) - got, reasons := l.AdjudicateCompletion(in) - if got.Status != CompletionPartial { - t.Fatalf("status = %q, want the report lowered to partial", got.Status) - } - if len(reasons) != 4 { - t.Fatalf("reasons = %v, want one per downgraded claim", reasons) - } - for _, want := range []string{"failed-run: no successful run", "read-is-not-a-write: no write recorded", "manual-only:", "no-evidence: claimed satisfied with no evidence"} { - found := false - for _, r := range reasons { - found = found || strings.Contains(r, want) - } - if !found { - t.Errorf("no reason mentions %q: %v", want, reasons) - } - } - for _, c := range got.Criteria { - if c.Status != CriterionUnsatisfied { - t.Errorf("criterion %s is %s after adjudication", c.ID, c.Status) - } - } -} - -func TestAdjudicateCompletionNeverRaisesAStatusAndAcceptsANilLedger(t *testing.T) { - l := recorded(Receipt{ToolName: "bash", Success: true, Command: "make check"}) - for _, status := range []CompletionStatus{CompletionPartial, CompletionBlocked, CompletionFailed} { - in := report(status, satisfied("ok", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "make check"})) - if got, _ := l.AdjudicateCompletion(in); got.Status != status { - t.Errorf("%s became %s: the host may lower a claim, never raise it", status, got.Status) - } - } - var nilLedger *Ledger - in := report(CompletionComplete, satisfied("x")) - if got, reasons := nilLedger.AdjudicateCompletion(in); got.Status != CompletionComplete || reasons != nil { - t.Fatalf("a nil ledger adjudicates nothing: %q %v", got.Status, reasons) - } -} - -func TestAdjudicateCompletionSaysACitedMissIsOneProblemNotTwo(t *testing.T) { - l := recorded() - got, reasons := l.AdjudicateCompletion(report(CompletionComplete, satisfied("twice", - CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test"}, - CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test"}, - ))) - if len(reasons) != 1 || strings.Count(reasons[0], "no successful run") != 1 || got.Status != CompletionPartial { - t.Fatalf("reasons = %v status = %q", reasons, got.Status) - } -} diff --git a/internal/safety/evidence/completion_report.go b/internal/safety/evidence/completion_report.go deleted file mode 100644 index 747d75a8df..0000000000 --- a/internal/safety/evidence/completion_report.go +++ /dev/null @@ -1,270 +0,0 @@ -package evidence - -import ( - "encoding/json" - "fmt" - "slices" - "strconv" - "strings" - "unicode/utf8" -) - -// CompletionStatus is a sub-agent's claim about its own run. The host may lower -// it to what receipts support; nothing may raise it. -type CompletionStatus string - -const ( - CompletionComplete CompletionStatus = "complete" - CompletionPartial CompletionStatus = "partial" - CompletionBlocked CompletionStatus = "blocked" - CompletionFailed CompletionStatus = "failed" -) - -// CriterionStatus is the child's claim about one acceptance criterion. -type CriterionStatus string - -const ( - CriterionSatisfied CriterionStatus = "satisfied" - CriterionUnsatisfied CriterionStatus = "unsatisfied" -) - -// Completion evidence kinds mirror complete_step so a sub-agent and the root -// agent prove work in the same vocabulary. -const ( - CompletionEvidenceVerification = "verification" - CompletionEvidenceReview = "review" - CompletionEvidenceDiff = "diff" - CompletionEvidenceFiles = "files" - CompletionEvidenceManual = "manual" -) - -// CompletionEvidence is one proof attached to an acceptance criterion. -type CompletionEvidence struct { - Kind string `json:"kind"` - Summary string `json:"summary"` - Command string `json:"command,omitempty"` - Paths []string `json:"paths,omitempty"` -} - -// AcceptanceCriterion is one checkable condition the sub-task had to meet. -type AcceptanceCriterion struct { - ID string `json:"id"` - Status CriterionStatus `json:"status"` - Evidence []CompletionEvidence `json:"evidence,omitempty"` -} - -// CompletionReport is the structured payload submitted via complete_subtask. -type CompletionReport struct { - Status CompletionStatus `json:"status"` - Summary string `json:"summary"` - Criteria []AcceptanceCriterion `json:"acceptance_criteria,omitempty"` - Unresolved []string `json:"unresolved,omitempty"` -} - -// ParseCompletionReport validates and normalizes a complete_subtask argument -// object. It checks shape only: whether the claims are true is the host's job. -func ParseCompletionReport(raw json.RawMessage) (CompletionReport, error) { - var r CompletionReport - if err := json.Unmarshal(raw, &r); err != nil { - return CompletionReport{}, fmt.Errorf("invalid complete_subtask JSON: %w", err) - } - r.Status = CompletionStatus(strings.ToLower(strings.TrimSpace(string(r.Status)))) - switch r.Status { - case CompletionComplete, CompletionPartial, CompletionBlocked, CompletionFailed: - default: - return CompletionReport{}, fmt.Errorf("complete_subtask.status must be complete, partial, blocked, or failed") - } - r.Summary = strings.TrimSpace(r.Summary) - if r.Summary == "" { - return CompletionReport{}, fmt.Errorf("complete_subtask.summary is required") - } - for i := range r.Criteria { - c := &r.Criteria[i] - c.ID = strings.TrimSpace(c.ID) - if c.ID == "" { - return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].id is required", i) - } - c.Status = CriterionStatus(strings.ToLower(strings.TrimSpace(string(c.Status)))) - switch c.Status { - case CriterionSatisfied, CriterionUnsatisfied: - default: - return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].status must be satisfied or unsatisfied", i) - } - for j := range c.Evidence { - e := &c.Evidence[j] - e.Kind = strings.ToLower(strings.TrimSpace(e.Kind)) - switch e.Kind { - case CompletionEvidenceVerification: - if strings.TrimSpace(e.Command) == "" { - return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].evidence[%d]: verification requires command", i, j) - } - case CompletionEvidenceDiff, CompletionEvidenceFiles: - if len(e.Paths) == 0 { - return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].evidence[%d]: %s requires paths", i, j, e.Kind) - } - case CompletionEvidenceReview, CompletionEvidenceManual: - default: - return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].evidence[%d].kind is not a known evidence kind", i, j) - } - } - } - return r, nil -} - -// LatestCompletionReport returns the most recent successful complete_subtask -// payload recorded this run. -func (l *Ledger) LatestCompletionReport() (CompletionReport, bool) { - if l == nil { - return CompletionReport{}, false - } - l.mu.Lock() - defer l.mu.Unlock() - for _, r := range slices.Backward(l.receipts) { - if r.ToolName != "complete_subtask" || !r.Success { - continue - } - report, err := ParseCompletionReport(r.Args) - if err != nil { - continue - } - return report, true - } - return CompletionReport{}, false -} - -// ClosureTally counts the verdicts this run's completion reports received, at -// the moment each was issued. Recomputing them later answers a different -// question: the ledger has grown since, so a claim the host could not back -// then may be backed now. -type ClosureTally struct { - Closed int - NeedsWork int -} - -// NoteClosureVerdict records what the host answered one report with. -func (l *Ledger) NoteClosureVerdict(closed bool) { - if l == nil { - return - } - l.mu.Lock() - defer l.mu.Unlock() - if closed { - l.closure.Closed++ - return - } - l.closure.NeedsWork++ -} - -// ClosureVerdicts returns the verdicts issued so far. -func (l *Ledger) ClosureVerdicts() ClosureTally { - if l == nil { - return ClosureTally{} - } - l.mu.Lock() - defer l.mu.Unlock() - return l.closure -} - -// AdjudicateCompletion lowers a report to what the host's own receipts support. -// A criterion claimed satisfied whose evidence no receipt backs is downgraded, -// and a report holding any downgraded criterion cannot stay "complete". The -// returned reasons name every downgrade so the parent sees why. -func (l *Ledger) AdjudicateCompletion(r CompletionReport) (CompletionReport, []string) { - if l == nil { - return r, nil - } - var reasons []string - downgraded := false - for i := range r.Criteria { - c := &r.Criteria[i] - if c.Status != CriterionSatisfied { - continue - } - if backed, why := l.criterionIsBacked(*c); !backed { - c.Status = CriterionUnsatisfied - downgraded = true - reasons = append(reasons, c.ID+": "+why) - } - } - if downgraded && r.Status == CompletionComplete { - r.Status = CompletionPartial - } - return r, reasons -} - -// criterionIsBacked reports whether at least one attached proof is one the host -// itself observed. A manual claim is never host-backed on its own. -func (l *Ledger) criterionIsBacked(c AcceptanceCriterion) (bool, string) { - if len(c.Evidence) == 0 { - return false, "claimed satisfied with no evidence" - } - var missing []string - for _, e := range c.Evidence { - switch e.Kind { - case CompletionEvidenceVerification: - if l.HasSuccessfulCommand(e.Command) { - return true, "" - } - missing = append(missing, "no successful run of "+quoteShort(e.Command)) - case CompletionEvidenceDiff: - if l.HasSuccessfulWrite(e.Paths) { - return true, "" - } - missing = append(missing, "no write recorded for "+joinShort(e.Paths)) - case CompletionEvidenceFiles: - if l.HasSuccessfulReadOrWrite(e.Paths) { - return true, "" - } - missing = append(missing, "no read or write recorded for "+joinShort(e.Paths)) - case CompletionEvidenceReview: - if l.HasCompletedReview() { - return true, "" - } - missing = append(missing, "no completed review") - } - } - if len(missing) == 0 { - return false, "no host receipt backs the cited evidence" - } - // Two evidence entries can miss for the same reason; saying it twice reads - // as two problems. Order is kept: it is the order they were cited in. - seen := make(map[string]bool, len(missing)) - uniq := missing[:0] - for _, m := range missing { - if seen[m] { - continue - } - seen[m] = true - uniq = append(uniq, m) - } - return false, strings.Join(uniq, "; ") -} - -// quoteShort renders one cited command for a rejection message. -func quoteShort(s string) string { - const max = 80 - s = strings.TrimSpace(s) - if s == "" { - return "an unnamed command" - } - if len(s) > max { - cut := max - for cut > 0 && !utf8.RuneStart(s[cut]) { - cut-- - } - s = s[:cut] + "…" - } - return strconv.Quote(s) -} - -// joinShort names the cited paths, capped so one rejection cannot become the -// whole reply. -func joinShort(paths []string) string { - if len(paths) == 0 { - return "no path" - } - if len(paths) > 3 { - return strings.Join(paths[:3], ", ") + fmt.Sprintf(" (+%d more)", len(paths)-3) - } - return strings.Join(paths, ", ") -} diff --git a/internal/safety/evidence/completion_report_parse_test.go b/internal/safety/evidence/completion_report_parse_test.go deleted file mode 100644 index 6a051adf0a..0000000000 --- a/internal/safety/evidence/completion_report_parse_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package evidence - -import ( - "encoding/json" - "strings" - "testing" -) - -func TestParseCompletionReportNormalisesAValidReport(t *testing.T) { - raw := json.RawMessage(`{"status":" Complete ","summary":" fixed the loop ","acceptance_criteria":[ - {"id":" c1 ","status":"SATISFIED","evidence":[ - {"kind":" Verification ","summary":"ran","command":"go test ./..."}, - {"kind":"diff","summary":"d","paths":["a.go"]}, - {"kind":"files","summary":"f","paths":["b.go"]}, - {"kind":"review","summary":"r"}, - {"kind":"manual","summary":"m"}]}], - "unresolved":["integration run"]}`) - got, err := ParseCompletionReport(raw) - if err != nil { - t.Fatal(err) - } - if got.Status != CompletionComplete || got.Summary != "fixed the loop" { - t.Fatalf("status/summary = %q / %q", got.Status, got.Summary) - } - c := got.Criteria[0] - if c.ID != "c1" || c.Status != CriterionSatisfied || c.Evidence[0].Kind != CompletionEvidenceVerification { - t.Fatalf("criterion = %+v", c) - } - if len(got.Unresolved) != 1 { - t.Fatalf("unresolved = %v", got.Unresolved) - } -} - -func TestParseCompletionReportRefusesWhatIsMalformedAndSaysWhere(t *testing.T) { - for name, tc := range map[string]struct{ raw, want string }{ - "not json": {`{`, "invalid complete_subtask JSON"}, - "unknown status": {`{"status":"done","summary":"x"}`, "status must be"}, - "missing status": {`{"summary":"x"}`, "status must be"}, - "blank summary": {`{"status":"complete","summary":" "}`, "summary is required"}, - "criterion without id": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":" ","status":"satisfied"}]}`, "acceptance_criteria[0].id"}, - "criterion with odd status": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"maybe"}]}`, "acceptance_criteria[0].status"}, - "verification with no cmd": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"verification","summary":"s"}]}]}`, "verification requires command"}, - "diff with no paths": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"diff","summary":"s"}]}]}`, "diff requires paths"}, - "files with no paths": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"files","summary":"s"}]}]}`, "files requires paths"}, - "unknown evidence kind": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"vibes","summary":"s"}]}]}`, "not a known evidence kind"}, - "second criterion is broken": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied"},{"id":"","status":"satisfied"}]}`, "acceptance_criteria[1].id"}, - } { - _, err := ParseCompletionReport(json.RawMessage(tc.raw)) - if err == nil || !strings.Contains(err.Error(), tc.want) { - t.Errorf("%s: err = %v, want it to mention %q", name, err, tc.want) - } - } -} diff --git a/internal/safety/evidence/evidence.go b/internal/safety/evidence/evidence.go index 824d99f130..670df03dac 100644 --- a/internal/safety/evidence/evidence.go +++ b/internal/safety/evidence/evidence.go @@ -92,7 +92,6 @@ type Ledger struct { mu sync.Mutex receipts []Receipt backgroundLeases []BackgroundLease - closure ClosureTally } func NewLedger() *Ledger { return &Ledger{} } @@ -106,7 +105,6 @@ func (l *Ledger) Reset() { defer l.mu.Unlock() l.receipts = nil l.backgroundLeases = nil - l.closure = ClosureTally{} } // Record appends a receipt. Failed receipts are retained for auditability but diff --git a/internal/safety/evidence/quote_short_test.go b/internal/safety/evidence/quote_short_test.go deleted file mode 100644 index 4811432621..0000000000 --- a/internal/safety/evidence/quote_short_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package evidence - -import ( - "strconv" - "strings" - "testing" -) - -func TestQuoteShortCutsOnRuneBoundary(t *testing.T) { - // 80 bytes into this command falls inside a three-byte character, and - // strconv.Quote would render the split bytes as \x escapes. - cmd := `echo "` + strings.Repeat("测试", 20) + `"` - got := quoteShort(cmd) - if strings.Contains(got, `\x`) { - t.Fatalf("quoteShort split a character: %s", got) - } - unquoted, err := strconv.Unquote(got) - if err != nil { - t.Fatalf("quoteShort result does not unquote: %v", err) - } - if !strings.HasSuffix(unquoted, "…") || !strings.HasPrefix(cmd, strings.TrimSuffix(unquoted, "…")) { - t.Fatalf("quoteShort = %s, want a prefix of the command ending in …", got) - } - if len(strings.TrimSuffix(unquoted, "…")) > 80 { - t.Fatalf("quoteShort kept more than 80 bytes: %s", got) - } -} - -func TestQuoteShortKeepsShortASCII(t *testing.T) { - if got := quoteShort(" go test ./... "); got != `"go test ./..."` { - t.Fatalf("quoteShort = %s", got) - } -} diff --git a/internal/state/trajectory/audit_routing_test.go b/internal/state/trajectory/audit_routing_test.go deleted file mode 100644 index 7473a7fe18..0000000000 --- a/internal/state/trajectory/audit_routing_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package trajectory - -import ( - "path/filepath" - "testing" - "time" - - "reasonix/internal/base/testenv" - "reasonix/internal/contract/event" -) - -// The bug this guards: SubagentHandoffAudit had a producer, a struct, an -// interface and a green unit test, and still reached nothing — the type -// assertion inside event.RecordSubagentHandoff matched no sink in the chain. -// Conformance proves a method exists; only a stack proves a value arrives. -func TestSubagentHandoffSurvivesTheSessionSinkStack(t *testing.T) { - path := filepath.Join(testenv.TempDir(t), "run.trajectory.jsonl") - r, err := New(event.Discard, path, nil) - if err != nil { - t.Fatalf("New: %v", err) - } - // boot's assembly order: Coalesce -> Sync -> CostQuote -> recorder. - chain := event.Coalesce(event.Sync(event.NewCostQuoteSink(r, nil)), time.Millisecond) - - event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{ - Entrance: "read_only_task", Depth: 1, ReadOnly: true, Expected: true, - Exit: "completed", Attempts: 2, Accepted: 1, Malformed: 1, - ReportRound: 3, FinalRound: 4, ToolCallsAfterReport: 2, - ClaimedStatus: "complete", AdjudicatedStatus: "partial", - LoweredClaims: 1, Criteria: 3, Evidence: 4, Unresolved: 1, - }) - if err := r.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - - recs := readRecords(t, path) - if len(recs) != 1 || recs[0].SubagentHandoff == nil { - t.Fatalf("handoff never reached the recorder: %+v", recs) - } - got := *recs[0].SubagentHandoff - want := SubagentHandoff{ - Entrance: "read_only_task", Depth: 1, ReadOnly: true, Expected: true, - Exit: "completed", Attempts: 2, Accepted: 1, Malformed: 1, - ReportRound: 3, FinalRound: 4, ToolCallsAfterReport: 2, - ClaimedStatus: "complete", AdjudicatedStatus: "partial", - LoweredClaims: 1, Criteria: 3, Evidence: 4, Unresolved: 1, - } - if got != want { - t.Fatalf("handoff record = %+v, want %+v", got, want) - } -} diff --git a/internal/state/trajectory/recorder.go b/internal/state/trajectory/recorder.go index 9d17c4add1..ac13b9564c 100644 --- a/internal/state/trajectory/recorder.go +++ b/internal/state/trajectory/recorder.go @@ -32,12 +32,10 @@ type Record struct { RecoveryChild string `json:"protocol_recovery_child,omitempty"` TurnCompletion bool `json:"turn_completion,omitempty"` ContractShadow *ContractShadowAudit `json:"contract_shadow,omitempty"` - CompletionReport *CompletionReport `json:"completion_report,omitempty"` EvidenceBundle *EvidenceBundle `json:"evidence_bundle,omitempty"` OutcomeProgress *OutcomeProgress `json:"outcome_progress,omitempty"` MemoryRecall *MemoryRecall `json:"memory_recall,omitempty"` ProjectCheck *ProjectCheckProbe `json:"project_check_probe,omitempty"` - SubagentHandoff *SubagentHandoff `json:"subagent_handoff,omitempty"` // Deltas counts the streamed increments merged into this record; absent // means one. TS stays the first increment's — that is the observation // time-to-first-token readers key on — and EndTS carries the last's. @@ -127,31 +125,6 @@ type ContractShadowAudit struct { ReadyToFinalize bool `json:"ready_to_finalize,omitempty"` } -// SubagentHandoff mirrors event.SubagentHandoffAudit: how one delegated child -// closed. Every field travels, because which cut explains a refusal — entrance, -// depth, read-only, how long the run was — is what the aggregate cannot say. -type SubagentHandoff struct { - Entrance string `json:"entrance,omitempty"` - Depth int `json:"depth,omitempty"` - ReadOnly bool `json:"read_only,omitempty"` - Expected bool `json:"expected,omitempty"` - Exit string `json:"exit,omitempty"` - Attempts int `json:"attempts,omitempty"` - Accepted int `json:"accepted,omitempty"` - Malformed int `json:"malformed,omitempty"` - Closed int `json:"closed,omitempty"` - NeedsWork int `json:"needs_work,omitempty"` - ReportRound int `json:"report_round,omitempty"` - FinalRound int `json:"final_round,omitempty"` - ToolCallsAfterReport int `json:"tool_calls_after_report,omitempty"` - ClaimedStatus string `json:"claimed_status,omitempty"` - AdjudicatedStatus string `json:"adjudicated_status,omitempty"` - LoweredClaims int `json:"lowered_claims,omitempty"` - Criteria int `json:"criteria,omitempty"` - Evidence int `json:"evidence,omitempty"` - Unresolved int `json:"unresolved,omitempty"` -} - // ProjectCheckProbe mirrors event.ProjectCheckProbe: the shadow comparison // between the readiness gate's project-check derivation and the ledger's // obligations. Identities are recorded, not counted — a class is only @@ -173,26 +146,6 @@ type ProjectCheckDiff struct { Class string `json:"class"` } -// CompletionReport mirrors event.CompletionReportAudit with stable keys. -type CompletionReport struct { - Verdict string `json:"verdict"` - Risk string `json:"risk,omitempty"` - Criteria int `json:"criteria,omitempty"` - CriteriaSatisfied int `json:"criteria_satisfied,omitempty"` - Changes int `json:"changes,omitempty"` - ChangesUnreviewed int `json:"changes_unreviewed,omitempty"` - Verifications int `json:"verifications,omitempty"` - VerificationsFailed int `json:"verifications_failed,omitempty"` - VerificationsStale int `json:"verifications_stale,omitempty"` - VerificationsInconclusive int `json:"verifications_inconclusive,omitempty"` - Gaps int `json:"gaps,omitempty"` - GapKinds []string `json:"gap_kinds,omitempty"` - ClaimsVerified int `json:"claims_verified,omitempty"` - ClaimsUnbacked int `json:"claims_unbacked,omitempty"` - OriginRecorded int `json:"origin_recorded,omitempty"` - OriginMatched int `json:"origin_matched,omitempty"` -} - // EvidenceBundle mirrors event.EvidenceBundleAudit with stable keys. type EvidenceBundle struct { Sealed bool `json:"sealed"` @@ -357,31 +310,6 @@ func (r *Recorder) RecordDelegationAudit(a hostaudit.DelegationAudit) { event.RecordDelegationAudit(r.inner, a) } -func (r *Recorder) RecordSubagentHandoff(a event.SubagentHandoffAudit) { - r.append(Record{SubagentHandoff: &SubagentHandoff{ - Entrance: a.Entrance, - Depth: a.Depth, - ReadOnly: a.ReadOnly, - Expected: a.Expected, - Exit: a.Exit, - Attempts: a.Attempts, - Accepted: a.Accepted, - Malformed: a.Malformed, - Closed: a.Closed, - NeedsWork: a.NeedsWork, - ReportRound: a.ReportRound, - FinalRound: a.FinalRound, - ToolCallsAfterReport: a.ToolCallsAfterReport, - ClaimedStatus: a.ClaimedStatus, - AdjudicatedStatus: a.AdjudicatedStatus, - LoweredClaims: a.LoweredClaims, - Criteria: a.Criteria, - Evidence: a.Evidence, - Unresolved: a.Unresolved, - }}) - event.RecordSubagentHandoff(r.inner, a) -} - func (r *Recorder) RecordMemoryRecall(a event.MemoryRecallAudit) { rec := &MemoryRecall{ CharBudget: a.CharBudget, UsedChars: a.UsedChars, Omitted: a.Omitted, diff --git a/tools/repolint/baseline.json b/tools/repolint/baseline.json index 6bbe77ae16..f3fff748c9 100644 --- a/tools/repolint/baseline.json +++ b/tools/repolint/baseline.json @@ -13,7 +13,7 @@ "doc-prose": 2128, "error-text": 9, "essay": 2576, - "file-size": 18416, + "file-size": 18415, "flat-view": 0, "frontend-parity": 13, "function-size": 2485, @@ -1080,7 +1080,7 @@ "internal/runtime/delegation/task.go": { "complexity": 8, "essay": 9, - "file-size": 160, + "file-size": 159, "function-size": 55, "orphan": 1 },