mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 02:36:01 -10:00
refactor(delegation): sub-agents return prose, not a structured completion report
complete_subtask, the host adjudication of its criteria and the 'lowered by the host' note are gone, with the handoff audit channel and its metrics, trajectory field and benchmark columns. A delegated child ends the way any turn does and the parent reads its last reply. The host's own receipts appended to a child's reply and the write-scope check stay: they need a record of what the child changed, which is replaced with the evidence ledger later.
This commit is contained in:
1 parent
5766a8ebeb
commit
0bb0f44785
35 files changed
+33
-1544
No files matched your search
@@ -7,7 +7,7 @@ import "fmt"
|
||||
// arm costs more: did the extra agents produce verified work, or just tokens.
|
||||
func renderDelegation(results []result) string {
|
||||
var runs, nested, childCalls, parentCalls, mutations, dupes int
|
||||
var reports, prose, falseDone, downgrades, violations int
|
||||
var violations int
|
||||
var scopeHints, namedFiles, evidencePaths, discoveredPaths int
|
||||
solved, total, childTokens := 0, 0, 0
|
||||
for _, r := range results {
|
||||
@@ -24,10 +24,6 @@ func renderDelegation(results []result) string {
|
||||
parentCalls += r.ToolCalls - r.SubagentToolCalls
|
||||
mutations += r.SubagentMutations
|
||||
dupes += r.DuplicateWorkPaths
|
||||
reports += r.CompletionReports
|
||||
prose += r.CompletionsProsedOnly
|
||||
falseDone += r.FalseCompletions
|
||||
downgrades += r.CriterionDowngrades
|
||||
violations += r.WriteScopeViolations
|
||||
scopeHints += r.ParentScopeHints
|
||||
namedFiles += r.ParentNamedFiles
|
||||
@@ -70,11 +66,6 @@ func renderDelegation(results []result) string {
|
||||
if dupes > 0 {
|
||||
b += fmt.Sprintf("- **duplicate work**: %d file(s) mutated by more than one child\n", dupes)
|
||||
}
|
||||
closed := reports + prose
|
||||
b += fmt.Sprintf("- closed with a checkable claim: **%d/%d** (%s)\n", reports, closed, pct(reports, closed))
|
||||
if falseDone > 0 {
|
||||
b += fmt.Sprintf("- **false completions**: %d run(s), %d criterion claim(s) the host refused\n", falseDone, downgrades)
|
||||
}
|
||||
if violations > 0 {
|
||||
b += fmt.Sprintf("- **write-scope violations**: %d\n", violations)
|
||||
}
|
||||
|
||||
@@ -27,9 +27,7 @@ func TestRenderDelegationExposesWhatDelegationCost(t *testing.T) {
|
||||
runMetrics: runMetrics{
|
||||
ToolCalls: 30, SubagentToolCalls: 24,
|
||||
SubagentRuns: 3, SubagentNestedRuns: 1, SubagentMutations: 5,
|
||||
DuplicateWorkPaths: 2,
|
||||
CompletionReports: 2, CompletionsProsedOnly: 1,
|
||||
FalseCompletions: 1, CriterionDowngrades: 3,
|
||||
DuplicateWorkPaths: 2,
|
||||
WriteScopeViolations: 1,
|
||||
},
|
||||
}})
|
||||
@@ -38,8 +36,6 @@ func TestRenderDelegationExposesWhatDelegationCost(t *testing.T) {
|
||||
"parent **6** tool calls",
|
||||
"children **24**",
|
||||
"duplicate work**: 2",
|
||||
"checkable claim: **2/3**",
|
||||
"false completions**: 1 run(s), 3 criterion",
|
||||
"write-scope violations**: 1",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
@@ -92,14 +88,11 @@ func TestRenderDelegationSaysWhenEvidenceOriginIsUnscored(t *testing.T) {
|
||||
func TestRenderDelegationStaysQuietWhenNothingWentWrong(t *testing.T) {
|
||||
got := renderDelegation([]result{{
|
||||
Passed: true,
|
||||
runMetrics: runMetrics{ToolCalls: 12, SubagentToolCalls: 8, SubagentRuns: 2, CompletionReports: 2},
|
||||
runMetrics: runMetrics{ToolCalls: 12, SubagentToolCalls: 8, SubagentRuns: 2},
|
||||
}})
|
||||
for _, unwanted := range []string{"duplicate work", "false completions", "write-scope violations"} {
|
||||
if strings.Contains(got, unwanted) {
|
||||
t.Errorf("clean run reported %q:\n%s", unwanted, got)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(got, "checkable claim: **2/2**") {
|
||||
t.Errorf("clean run lost its completion coverage:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -66,15 +66,11 @@ type runMetrics struct {
|
||||
|
||||
// Delegation counters mirror internal/frontend/cli.RunMetrics. They are what makes a
|
||||
// single-agent arm comparable against a delegated one for the same model.
|
||||
SubagentRuns int `json:"subagent_runs,omitempty"`
|
||||
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
|
||||
SubagentMutations int `json:"subagent_mutations,omitempty"`
|
||||
CompletionReports int `json:"completion_reports,omitempty"`
|
||||
CompletionsProsedOnly int `json:"completions_prose_only,omitempty"`
|
||||
FalseCompletions int `json:"false_completions,omitempty"`
|
||||
CriterionDowngrades int `json:"criterion_downgrades,omitempty"`
|
||||
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
|
||||
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
|
||||
SubagentRuns int `json:"subagent_runs,omitempty"`
|
||||
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
|
||||
SubagentMutations int `json:"subagent_mutations,omitempty"`
|
||||
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
|
||||
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
|
||||
// Evidence origin: what the parent's own delegation text scoped and named,
|
||||
// and how much of what the children looked at they had to find themselves.
|
||||
ParentScopeHints int `json:"parent_scope_hints,omitempty"`
|
||||
|
||||
@@ -23,7 +23,6 @@ func TestAuditCapabilitiesAreForwardedByEveryWrapper(t *testing.T) {
|
||||
"DelegationAuditSink": reflect.TypeFor[event.DelegationAuditSink](),
|
||||
"MemoryRecallSink": reflect.TypeFor[event.MemoryRecallSink](),
|
||||
"TurnCompletionSink": reflect.TypeFor[event.TurnCompletionSink](),
|
||||
"SubagentHandoffSink": reflect.TypeFor[event.SubagentHandoffSink](),
|
||||
}
|
||||
wrappers := []event.Sink{
|
||||
event.Sync(event.Discard),
|
||||
|
||||
@@ -193,7 +193,7 @@ model = "x"
|
||||
if !strings.HasPrefix(systemMessage(req.Messages), profileBody) {
|
||||
t.Fatal("installed profile body did not reach the child")
|
||||
}
|
||||
if !slices.Equal(toolSchemaNames(req.Tools), []string{"complete_subtask", "glob", "grep", "ls", "read_file"}) {
|
||||
if !slices.Equal(toolSchemaNames(req.Tools), []string{"glob", "grep", "ls", "read_file"}) {
|
||||
t.Fatalf("child tool ceiling = %v", toolSchemaNames(req.Tools))
|
||||
}
|
||||
var user string
|
||||
|
||||
@@ -15,7 +15,7 @@ func TestEffectConditionalDeclarationsDoNotReachTheProviderPrefix(t *testing.T)
|
||||
for _, schema := range req.Tools {
|
||||
sent[schema.Name] = true
|
||||
}
|
||||
for _, n := range []string{"advise", "best_of_n", "apply_isolated", "discard_isolated", "complete_subtask", "system_one", "submit_plan", "conclude_no_changes", "review_report"} {
|
||||
for _, n := range []string{"advise", "best_of_n", "apply_isolated", "discard_isolated", "system_one", "submit_plan", "conclude_no_changes", "review_report"} {
|
||||
if sent[n] {
|
||||
t.Errorf("%s is declared conditional but reached the default request tools", n)
|
||||
}
|
||||
|
||||
@@ -31,10 +31,6 @@ func (f AuditForwarder) RecordRunBudget(s RunBudgetSample) {
|
||||
RecordRunBudget(f.Inner, s)
|
||||
}
|
||||
|
||||
func (f AuditForwarder) RecordSubagentHandoff(a SubagentHandoffAudit) {
|
||||
RecordSubagentHandoff(f.Inner, a)
|
||||
}
|
||||
|
||||
// DelegationAuditSink receives one receipt per completed sub-agent run.
|
||||
type DelegationAuditSink interface {
|
||||
RecordDelegationAudit(a hostaudit.DelegationAudit)
|
||||
|
||||
@@ -185,13 +185,6 @@ func (c *coalescer) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
||||
RecordDelegationAudit(c.inner, a)
|
||||
}
|
||||
|
||||
func (c *coalescer) RecordSubagentHandoff(a SubagentHandoffAudit) {
|
||||
c.mu.Lock()
|
||||
c.enqueueFlushLocked()
|
||||
c.drainAndUnlock()
|
||||
RecordSubagentHandoff(c.inner, a)
|
||||
}
|
||||
|
||||
func (c *coalescer) RecordTurnCompletion() {
|
||||
c.mu.Lock()
|
||||
c.enqueueFlushLocked()
|
||||
|
||||
@@ -11,7 +11,6 @@ var capabilityContracts = map[string]reflect.Type{
|
||||
"DelegationAudit": reflect.TypeFor[DelegationAuditSink](),
|
||||
"MemoryRecall": reflect.TypeFor[MemoryRecallSink](),
|
||||
"RunBudget": reflect.TypeFor[RunBudgetSink](),
|
||||
"SubagentHandoff": reflect.TypeFor[SubagentHandoffSink](),
|
||||
"TurnCompletion": reflect.TypeFor[TurnCompletionSink](),
|
||||
"UsageQuote": reflect.TypeFor[UsageQuoter](),
|
||||
"WorkspaceMutation": reflect.TypeFor[WorkspaceMutationSink](),
|
||||
|
||||
@@ -37,9 +37,6 @@ func (s *capturingSink) RecordWorkspaceMutation(WorkspaceMutation) {
|
||||
func (s *capturingSink) RecordRunBudget(RunBudgetSample) {
|
||||
s.recorded = append(s.recorded, "RunBudget")
|
||||
}
|
||||
func (s *capturingSink) RecordSubagentHandoff(SubagentHandoffAudit) {
|
||||
s.recorded = append(s.recorded, "SubagentHandoff")
|
||||
}
|
||||
|
||||
// AuditForwarder is why a new capability does not have to be repeated at every
|
||||
// layer, so it must itself be complete. It reads the registry rather than a
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
// subagent_handoff.go — how a delegated child closed, counted.
|
||||
package event
|
||||
|
||||
import "reasonix/internal/base/nilutil"
|
||||
|
||||
// SubagentHandoffAudit is one child run's closing protocol, counted. It exists
|
||||
// to answer whether a typed handoff is submitted naturally, before anything is
|
||||
// built to insist on one — so it observes and never decides. Content-free:
|
||||
// counts, enums and rounds, never a prompt, a path or a summary.
|
||||
type SubagentHandoffAudit struct {
|
||||
// Entrance names the delegation path (read_only_task, fleet, …) and Depth
|
||||
// how deep the child sits. An aggregate hides an entrance that is failing.
|
||||
Entrance string
|
||||
Depth int
|
||||
ReadOnly bool
|
||||
// Expected is set where the host actually appended the contract, never
|
||||
// inferred from a registry: the denominator has to be the host's own
|
||||
// instruction, not a guess about one.
|
||||
Expected bool
|
||||
// Exit says how the run ended. A provider timeout that produced no report
|
||||
// is not a compliance failure and must leave the denominator.
|
||||
Exit string // completed | error | cancelled | no_answer
|
||||
|
||||
// Attempts counts complete_subtask calls, Accepted the ones the tool took,
|
||||
// Malformed the ones it refused. Missing with attempts is a schema
|
||||
// problem; missing without them is a protocol one.
|
||||
Attempts int
|
||||
Accepted int
|
||||
Malformed int
|
||||
|
||||
// ReportRound and FinalRound say whether the report actually closed the
|
||||
// run: the contract asks for it as the final call, and "submitted it
|
||||
// eventually" is a different behaviour from "closed with it".
|
||||
ReportRound int
|
||||
FinalRound int
|
||||
ToolCallsAfterReport int
|
||||
|
||||
// Closed and NeedsWork are the verdicts the host issued, counted when each
|
||||
// was issued. Accepted only says a call was well-formed; these say whether
|
||||
// the host considered the sub-task done.
|
||||
Closed int
|
||||
NeedsWork int
|
||||
|
||||
// What the host made of the claim, read from the adjudicated report rather
|
||||
// than re-derived here.
|
||||
ClaimedStatus string
|
||||
AdjudicatedStatus string
|
||||
LoweredClaims int
|
||||
Criteria int
|
||||
Evidence int
|
||||
Unresolved int
|
||||
}
|
||||
|
||||
// SubagentHandoffSink is an optional sink capability; implementations must keep
|
||||
// it content-free, like every other audit channel.
|
||||
type SubagentHandoffSink interface {
|
||||
RecordSubagentHandoff(SubagentHandoffAudit)
|
||||
}
|
||||
|
||||
// RecordSubagentHandoff forwards one child's closing protocol only to sinks
|
||||
// that explicitly opt in. Ordinary UI sinks receive nothing.
|
||||
func RecordSubagentHandoff(s Sink, a SubagentHandoffAudit) {
|
||||
if nilutil.IsNil(s) {
|
||||
return
|
||||
}
|
||||
if hs, ok := s.(SubagentHandoffSink); ok {
|
||||
hs.RecordSubagentHandoff(a)
|
||||
}
|
||||
}
|
||||
@@ -44,14 +44,6 @@ func (s *syncSink) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
||||
RecordDelegationAudit(s.inner, a)
|
||||
}
|
||||
|
||||
func (s *syncSink) RecordSubagentHandoff(a SubagentHandoffAudit) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if hs, ok := s.inner.(SubagentHandoffSink); ok {
|
||||
hs.RecordSubagentHandoff(a)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *syncSink) RecordTurnCompletion() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -12,12 +12,6 @@ type DelegationAudit struct {
|
||||
MutationPaths []string
|
||||
// ClaimViolations counts writes the host saw outside the declared claim.
|
||||
ClaimViolations int
|
||||
// HasReport is false when the child ended in prose instead of a typed claim.
|
||||
HasReport bool
|
||||
// AdjudicatedStatus is the status the host was willing to back, and
|
||||
// Downgrades counts the criterion claims it refused.
|
||||
AdjudicatedStatus string
|
||||
Downgrades int
|
||||
// ParentScopeHints counts directories the delegation narrowed the search to.
|
||||
ParentScopeHints int
|
||||
// ParentNamedFiles counts specific files the delegation text handed over.
|
||||
@@ -27,10 +21,3 @@ type DelegationAudit struct {
|
||||
// DiscoveredPaths is the subset of those no named file already covered.
|
||||
DiscoveredPaths int
|
||||
}
|
||||
|
||||
// FalseCompletion reports a child that claimed more than the host could back.
|
||||
// It is the count of refused criteria, not a comparison against a claimed
|
||||
// status the host never independently recorded.
|
||||
func (a DelegationAudit) FalseCompletion() bool {
|
||||
return a.HasReport && a.Downgrades > 0
|
||||
}
|
||||
@@ -3,8 +3,6 @@ package cli
|
||||
import (
|
||||
"reasonix/internal/contract/hostaudit"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/safety/evidence"
|
||||
)
|
||||
|
||||
// The instrument must be able to say, from one run, whether delegation
|
||||
@@ -13,17 +11,12 @@ func TestDelegationMetricsAggregateAcrossChildren(t *testing.T) {
|
||||
s := &metricsSink{}
|
||||
s.RecordDelegationAudit(hostaudit.DelegationAudit{
|
||||
Depth: 1, ToolCalls: 6, Mutations: 2,
|
||||
MutationPaths: []string{"api/handler.go", "api/handler_test.go"},
|
||||
HasReport: true,
|
||||
AdjudicatedStatus: string(evidence.CompletionComplete),
|
||||
MutationPaths: []string{"api/handler.go", "api/handler_test.go"},
|
||||
})
|
||||
s.RecordDelegationAudit(hostaudit.DelegationAudit{
|
||||
Depth: 2, ToolCalls: 4, Mutations: 1,
|
||||
MutationPaths: []string{"api/handler.go"},
|
||||
ClaimViolations: 1,
|
||||
HasReport: true,
|
||||
AdjudicatedStatus: string(evidence.CompletionPartial),
|
||||
Downgrades: 2,
|
||||
MutationPaths: []string{"api/handler.go"},
|
||||
ClaimViolations: 1,
|
||||
})
|
||||
s.RecordDelegationAudit(hostaudit.DelegationAudit{Depth: 1, ToolCalls: 3})
|
||||
|
||||
@@ -34,14 +27,6 @@ func TestDelegationMetricsAggregateAcrossChildren(t *testing.T) {
|
||||
if m.SubagentMutations != 3 {
|
||||
t.Fatalf("mutations = %d, want 3", m.SubagentMutations)
|
||||
}
|
||||
if m.CompletionReports != 2 || m.CompletionsProsedOnly != 1 {
|
||||
t.Fatalf("reports = %d prose-only = %d, want 2/1", m.CompletionReports, m.CompletionsProsedOnly)
|
||||
}
|
||||
// One child claimed criteria the host refused: that is the false-completion
|
||||
// signal an orchestration benchmark exists to surface.
|
||||
if m.FalseCompletions != 1 || m.CriterionDowngrades != 2 {
|
||||
t.Fatalf("false completions = %d downgrades = %d, want 1/2", m.FalseCompletions, m.CriterionDowngrades)
|
||||
}
|
||||
if m.WriteScopeViolations != 1 {
|
||||
t.Fatalf("write scope violations = %d, want 1", m.WriteScopeViolations)
|
||||
}
|
||||
@@ -77,7 +62,7 @@ func TestDelegationMetricsSumEvidenceOriginForARatioOfTotals(t *testing.T) {
|
||||
// single-agent arm is a clean baseline rather than noise.
|
||||
func TestDelegationMetricsStayZeroForSingleAgentArm(t *testing.T) {
|
||||
s := &metricsSink{}
|
||||
if m := s.m; m.SubagentRuns != 0 || m.CompletionReports != 0 || m.DuplicateWorkPaths != 0 {
|
||||
if m := s.m; m.SubagentRuns != 0 || m.DuplicateWorkPaths != 0 {
|
||||
t.Fatalf("single-agent baseline is not zero: %+v", m)
|
||||
}
|
||||
}
|
||||
@@ -75,45 +75,19 @@ type RunMetrics struct {
|
||||
// Delegation counters let one model be compared across orchestration arms
|
||||
// without scraping prose. Child tool calls are already split out as
|
||||
// SubagentToolCalls below; parent calls are ToolCalls minus that.
|
||||
SubagentRuns int `json:"subagent_runs,omitempty"`
|
||||
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
|
||||
SubagentMutations int `json:"subagent_mutations,omitempty"`
|
||||
CompletionReports int `json:"completion_reports,omitempty"`
|
||||
CompletionsProsedOnly int `json:"completions_prose_only,omitempty"`
|
||||
FalseCompletions int `json:"false_completions,omitempty"`
|
||||
CriterionDowngrades int `json:"criterion_downgrades,omitempty"`
|
||||
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
|
||||
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
|
||||
ParentScopeHints int `json:"parent_scope_hints,omitempty"`
|
||||
ParentNamedFiles int `json:"parent_named_files,omitempty"`
|
||||
ChildEvidencePaths int `json:"child_evidence_paths,omitempty"`
|
||||
ChildDiscoveredPaths int `json:"child_discovered_paths,omitempty"`
|
||||
// Subagent handoff: whether a delegated child closed the way it was asked
|
||||
// to. Judged is the denominator — expected to report and not killed by the
|
||||
// provider — because a run that never got to close is not a refusal.
|
||||
SubagentHandoffs int `json:"subagent_handoffs"`
|
||||
SubagentHandoffExpected int `json:"subagent_handoff_expected"`
|
||||
SubagentHandoffJudged int `json:"subagent_handoff_judged"`
|
||||
SubagentHandoffAttempted int `json:"subagent_handoff_attempted"`
|
||||
SubagentHandoffAccepted int `json:"subagent_handoff_accepted"`
|
||||
SubagentHandoffMalformed int `json:"subagent_handoff_malformed"`
|
||||
SubagentHandoffClosed int `json:"subagent_handoff_closed"`
|
||||
SubagentHandoffNeedsWork int `json:"subagent_handoff_needs_work"`
|
||||
SubagentHandoffNotTried int `json:"subagent_handoff_never_attempted"`
|
||||
SubagentHandoffClosedWith int `json:"subagent_handoff_closed_with_report"`
|
||||
SubagentHandoffToolsAfter int `json:"subagent_handoff_tool_calls_after_report"`
|
||||
SubagentHandoffLowered int `json:"subagent_handoff_lowered_claims"`
|
||||
SubagentHandoffReadOnly int `json:"subagent_handoff_read_only"`
|
||||
SubagentHandoffExits map[string]int `json:"subagent_handoff_exits,omitempty"`
|
||||
SubagentRuns int `json:"subagent_runs,omitempty"`
|
||||
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
|
||||
SubagentMutations int `json:"subagent_mutations,omitempty"`
|
||||
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
|
||||
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
|
||||
ParentScopeHints int `json:"parent_scope_hints,omitempty"`
|
||||
ParentNamedFiles int `json:"parent_named_files,omitempty"`
|
||||
ChildEvidencePaths int `json:"child_evidence_paths,omitempty"`
|
||||
ChildDiscoveredPaths int `json:"child_discovered_paths,omitempty"`
|
||||
// Repairs that happened inside a delegated run rather than the parent's own
|
||||
// loop, so a child that continued only because the host pulled it back is
|
||||
// not read as one that continued on its own.
|
||||
SubagentRecoveries int `json:"subagent_recoveries"`
|
||||
// Goal resumes that carried a frozen verification contract, and how many of
|
||||
// those found the current declaration naming different criteria. Neither
|
||||
// says which declaration should govern.
|
||||
GoalResumesWithContract int `json:"goal_resumes_with_verification_contract"`
|
||||
GoalVerificationDrifts int `json:"goal_verification_contract_drifts"`
|
||||
// Fanout times what the delegation counters only count: what the run waited
|
||||
// on its fan-outs, against what the same work costs one at a time.
|
||||
Fanout *FanoutMetrics `json:"fanout,omitempty"`
|
||||
@@ -459,21 +433,12 @@ func (s *metricsSink) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
||||
}
|
||||
s.m.SubagentMutations += a.Mutations
|
||||
s.m.WriteScopeViolations += a.ClaimViolations
|
||||
s.m.CriterionDowngrades += a.Downgrades
|
||||
// Summed, never averaged: an independence rate is a ratio of these totals,
|
||||
// so a child that read two files cannot outweigh one that read forty.
|
||||
s.m.ParentScopeHints += a.ParentScopeHints
|
||||
s.m.ParentNamedFiles += a.ParentNamedFiles
|
||||
s.m.ChildEvidencePaths += a.EvidencePaths
|
||||
s.m.ChildDiscoveredPaths += a.DiscoveredPaths
|
||||
if a.HasReport {
|
||||
s.m.CompletionReports++
|
||||
} else {
|
||||
s.m.CompletionsProsedOnly++
|
||||
}
|
||||
if a.FalseCompletion() {
|
||||
s.m.FalseCompletions++
|
||||
}
|
||||
if s.childMutations == nil {
|
||||
s.childMutations = map[string]int{}
|
||||
}
|
||||
@@ -485,52 +450,6 @@ func (s *metricsSink) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *metricsSink) RecordSubagentHandoff(a event.SubagentHandoffAudit) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
defer event.RecordSubagentHandoff(s.inner, a)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.m.SubagentHandoffs++
|
||||
if a.Exit != "" {
|
||||
if s.m.SubagentHandoffExits == nil {
|
||||
s.m.SubagentHandoffExits = map[string]int{}
|
||||
}
|
||||
s.m.SubagentHandoffExits[a.Exit]++
|
||||
}
|
||||
if a.Expected {
|
||||
s.m.SubagentHandoffExpected++
|
||||
}
|
||||
// A run the provider killed never reached the point of closing, so it
|
||||
// leaves the compliance denominator instead of counting as a refusal.
|
||||
if !a.Expected || a.Exit != "completed" {
|
||||
return
|
||||
}
|
||||
s.m.SubagentHandoffJudged++
|
||||
if a.ReadOnly {
|
||||
s.m.SubagentHandoffReadOnly++
|
||||
}
|
||||
if a.Attempts == 0 {
|
||||
s.m.SubagentHandoffNotTried++
|
||||
} else {
|
||||
s.m.SubagentHandoffAttempted++
|
||||
}
|
||||
if a.Accepted > 0 {
|
||||
s.m.SubagentHandoffAccepted++
|
||||
}
|
||||
if a.Malformed > 0 {
|
||||
s.m.SubagentHandoffMalformed++
|
||||
}
|
||||
if a.ReportRound > 0 && a.ToolCallsAfterReport == 0 {
|
||||
s.m.SubagentHandoffClosedWith++
|
||||
}
|
||||
s.m.SubagentHandoffClosed += a.Closed
|
||||
s.m.SubagentHandoffNeedsWork += a.NeedsWork
|
||||
s.m.SubagentHandoffToolsAfter += a.ToolCallsAfterReport
|
||||
s.m.SubagentHandoffLowered += a.LoweredClaims
|
||||
}
|
||||
|
||||
// MergeCapabilityAuditCounters copies capability counters into RunMetrics.
|
||||
func (m *RunMetrics) MergeCapabilityAuditCounters(
|
||||
routes, routedCandidates, routedRequire, routedPrefer, routedSuggest, declines int,
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
)
|
||||
|
||||
// The compliance denominator is the point of this test: a child the provider
|
||||
// killed never reached the point of closing, so it must leave the denominator
|
||||
// rather than count as a refusal. Routing is asserted too — the audit is sent
|
||||
// through a wrapper, not called on the sink directly, because a method that
|
||||
// exists and is never reached is the bug this channel already had once.
|
||||
func TestMetricsSinkCountsSubagentHandoffCompliance(t *testing.T) {
|
||||
s := &metricsSink{inner: event.Discard}
|
||||
chain := event.Sync(s)
|
||||
|
||||
event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{
|
||||
Expected: true, Exit: "completed", ReadOnly: true,
|
||||
})
|
||||
event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{
|
||||
Expected: true, Exit: "cancelled",
|
||||
})
|
||||
event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{
|
||||
Expected: true, Exit: "completed",
|
||||
Attempts: 1, Accepted: 1, ReportRound: 2, FinalRound: 2, LoweredClaims: 3,
|
||||
})
|
||||
|
||||
m := s.m
|
||||
if m.SubagentHandoffs != 3 || m.SubagentHandoffExpected != 3 || m.SubagentHandoffJudged != 2 {
|
||||
t.Fatalf("totals = %d/%d/%d, want 3 observed, 3 expected, 2 judged",
|
||||
m.SubagentHandoffs, m.SubagentHandoffExpected, m.SubagentHandoffJudged)
|
||||
}
|
||||
if m.SubagentHandoffNotTried != 1 || m.SubagentHandoffAttempted != 1 || m.SubagentHandoffAccepted != 1 {
|
||||
t.Fatalf("never-attempted/attempted/accepted = %d/%d/%d, want 1/1/1 — the cancelled run must not be either",
|
||||
m.SubagentHandoffNotTried, m.SubagentHandoffAttempted, m.SubagentHandoffAccepted)
|
||||
}
|
||||
if m.SubagentHandoffClosedWith != 1 || m.SubagentHandoffLowered != 3 || m.SubagentHandoffReadOnly != 1 {
|
||||
t.Fatalf("closed/lowered/read-only = %d/%d/%d, want 1/3/1",
|
||||
m.SubagentHandoffClosedWith, m.SubagentHandoffLowered, m.SubagentHandoffReadOnly)
|
||||
}
|
||||
if m.SubagentHandoffExits["completed"] != 2 || m.SubagentHandoffExits["cancelled"] != 1 {
|
||||
t.Fatalf("exits = %v, want 2 completed and 1 cancelled", m.SubagentHandoffExits)
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"reasonix/internal/ext/extension/dispatch"
|
||||
"reasonix/internal/runtime/capability"
|
||||
"reasonix/internal/runtime/writeclaim"
|
||||
"reasonix/internal/safety/evidence"
|
||||
"reasonix/internal/safety/sandbox"
|
||||
"reasonix/internal/state/checkpoint"
|
||||
"reasonix/internal/state/instruction"
|
||||
@@ -182,9 +181,7 @@ type Options struct {
|
||||
// classification judges it instead of the raw Run input, so framing verbs
|
||||
// cannot arm expectations; the delegation audit scores evidence origin
|
||||
// against it, so only locations the parent wrote itself count as hints.
|
||||
ClassifierTaskText string
|
||||
ExpectCompletionReport bool // set where the contract is appended, never inferred
|
||||
HandoffEntrance string // delegation path, for telemetry only
|
||||
ClassifierTaskText string
|
||||
|
||||
// CapabilityLedger is the optional turn-scoped capability route ledger for
|
||||
// Delivery require/prefer gates. Nil disables capability gates.
|
||||
@@ -192,11 +189,6 @@ type Options struct {
|
||||
// CapabilityAudit is the optional non-persisted metrics sink for routing.
|
||||
CapabilityAudit *capability.Audit
|
||||
|
||||
// RequireReviewReportKind, when non-empty, makes RunSubAgentWithSession fail
|
||||
// unless the subagent recorded a successful review_report of this kind —
|
||||
// review/security subagents must return typed, host-verifiable reports.
|
||||
RequireReviewReportKind evidence.ReviewKind
|
||||
|
||||
// ReasoningLanguage controls visible reasoning language preference as transient
|
||||
// user-turn context. Empty/auto injects nothing.
|
||||
ReasoningLanguage string
|
||||
|
||||
@@ -98,7 +98,6 @@ func SubagentToolRegistryForDepthWithRuntime(parent *tool.Registry, names []stri
|
||||
sub := FilterRegistry(parent, names, exclude...)
|
||||
bindLocal := builtin.ForkFileViews(sub)
|
||||
stripDirectMCPTools(sub)
|
||||
AttachCompleteSubtaskTool(sub)
|
||||
attachSubagentCapabilityProxy(parent, sub, names, runtime, bindLocal)
|
||||
if bash, ok := sub.Get("bash"); ok {
|
||||
sub.Add(foregroundOnlyBash{inner: bash})
|
||||
|
||||
@@ -2,7 +2,6 @@ package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reasonix/internal/contract/hostaudit"
|
||||
"reasonix/internal/runtime/writeclaim"
|
||||
@@ -65,14 +64,8 @@ func RunSubAgentWithSession(ctx context.Context, prov provider.Provider, reg *to
|
||||
if planWorkflow && !strings.Contains(prompt, planmode.Marker) {
|
||||
prompt = planmode.Marker + "\n\n" + prompt
|
||||
}
|
||||
if kind := opts.RequireReviewReportKind; kind != "" {
|
||||
prompt = prompt + "\n\n" + reviewReportTaskContract(kind)
|
||||
}
|
||||
ctx = WithoutRunTimeSink(ctx)
|
||||
sub := newDelegatedAgent(ctx, prov, reg, sess, opts, sink)
|
||||
// One defer: this returns from a salvage, a review failure, a provider
|
||||
// error and a clean answer, and per-path calls would miss one.
|
||||
defer func() { observeSubagentHandoff(sink, sub, sess, opts, answer, err) }()
|
||||
sub.SetPlanMode(planWorkflow)
|
||||
if err := sub.Run(ctx, prompt); err != nil {
|
||||
// Still merge any partial child evidence so parent gates see real writes.
|
||||
@@ -230,15 +223,10 @@ func (a *Agent) EvidenceSummary() evidence.ChildEvidenceSummary {
|
||||
}
|
||||
|
||||
// composeSubagentAnswer assembles everything the parent is shown for one child
|
||||
// run: the host-adjudicated completion claim when the child submitted one, the
|
||||
// child's own prose, then the host's receipts.
|
||||
// run: the child's own prose, then the host's receipts.
|
||||
func composeSubagentAnswer(ctx context.Context, answer string, sub *Agent, claims writeclaim.WritePathSet, delegationText string) string {
|
||||
summary := sub.EvidenceSummary()
|
||||
report, reasons, hasReport := sub.CompletionReport()
|
||||
if hasReport {
|
||||
answer = strings.TrimSpace(formatCompletionReport(report, reasons) + "\n\n" + answer)
|
||||
}
|
||||
recordDelegationAudit(ctx, summary, claims, report, reasons, hasReport, delegationText)
|
||||
recordDelegationAudit(ctx, summary, claims, delegationText)
|
||||
return AppendHostReceipts(answer, summary, claims)
|
||||
}
|
||||
|
||||
@@ -259,52 +247,6 @@ func newDelegatedAgent(ctx context.Context, prov provider.Provider, reg *tool.Re
|
||||
return sub
|
||||
}
|
||||
|
||||
// observeSubagentHandoff records one child run's closing protocol. Called from
|
||||
// a single defer so every exit path is counted the same: a salvage, a review
|
||||
// failure and a clean answer are all runs that either submitted a report or
|
||||
// did not.
|
||||
func observeSubagentHandoff(sink event.Sink, sub *Agent, sess *sessionstore.Session, opts Options, answer string, runErr error) {
|
||||
if sub == nil || sess == nil {
|
||||
return
|
||||
}
|
||||
audit := event.SubagentHandoffAudit{
|
||||
Entrance: opts.HandoffEntrance,
|
||||
Depth: opts.SubagentDepth,
|
||||
ReadOnly: opts.ReadOnlyExecution,
|
||||
Expected: opts.ExpectCompletionReport,
|
||||
Exit: handoffExit(answer, runErr),
|
||||
}
|
||||
countReportCalls(&audit, sess.Snapshot())
|
||||
// Claimed and adjudicated are both kept: a child that submits reports
|
||||
// readily and has them lowered every time is a different problem from one
|
||||
// that does not submit, and a single status cannot say which.
|
||||
if sub.task.ledger != nil {
|
||||
verdicts := sub.task.ledger.ClosureVerdicts()
|
||||
audit.Closed, audit.NeedsWork = verdicts.Closed, verdicts.NeedsWork
|
||||
if claimed, ok := sub.task.ledger.LatestCompletionReport(); ok {
|
||||
adjudicated, reasons := sub.task.ledger.AdjudicateCompletion(claimed)
|
||||
audit.ClaimedStatus = string(claimed.Status)
|
||||
audit.AdjudicatedStatus = string(adjudicated.Status)
|
||||
audit.LoweredClaims = len(reasons)
|
||||
audit.Criteria = len(adjudicated.Criteria)
|
||||
audit.Unresolved = len(adjudicated.Unresolved)
|
||||
for _, c := range adjudicated.Criteria {
|
||||
audit.Evidence += len(c.Evidence)
|
||||
}
|
||||
}
|
||||
}
|
||||
event.RecordSubagentHandoff(sink, audit)
|
||||
}
|
||||
|
||||
// reviewReportTaskContract is appended to the task prompt of a review subagent
|
||||
// whose run must end with a typed report. The skill body describes how to
|
||||
// review; this states the non-negotiable submission protocol.
|
||||
func reviewReportTaskContract(kind evidence.ReviewKind) string {
|
||||
return fmt.Sprintf(`<review-report-contract event="SubagentReviewReport">
|
||||
Before your final answer you MUST call the review_report tool exactly once with kind=%q, your verdict (pass | warn | block), reviewed_paths listing only files you actually read this run, and your findings. The host discards a review run that ends without a successful review_report call — your prose summary alone does not count.
|
||||
</review-report-contract>`, string(kind))
|
||||
}
|
||||
|
||||
const subagentStartContext = `<subagent-context event="SubagentStart">
|
||||
Before acting, check the available skills and tools. If a relevant skill is available, invoke it before continuing. Delegate to another sub-agent only when the task genuinely benefits from isolated context and the delegation tool is available.
|
||||
</subagent-context>`
|
||||
@@ -325,52 +267,20 @@ type mutationObserverSetter interface {
|
||||
SetMutationObserver(*checkpoint.MutationObserver)
|
||||
}
|
||||
|
||||
// formatCompletionReport renders the child's claim after the host has lowered
|
||||
// whatever its receipts could not back. Downgrades are shown, never silently
|
||||
// applied: a parent that cannot see the adjudication cannot trust the status.
|
||||
func formatCompletionReport(report evidence.CompletionReport, reasons []string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("status: ")
|
||||
b.WriteString(string(report.Status))
|
||||
if len(reasons) > 0 {
|
||||
b.WriteString(" (lowered by the host: unbacked criterion claims)")
|
||||
}
|
||||
b.WriteString("\nsummary: ")
|
||||
b.WriteString(report.Summary)
|
||||
for _, c := range report.Criteria {
|
||||
b.WriteString("\n " + c.ID + " " + string(c.Status))
|
||||
if proof := criterionProof(c); proof != "" {
|
||||
b.WriteString(" — " + proof)
|
||||
}
|
||||
}
|
||||
for _, reason := range reasons {
|
||||
b.WriteString("\n host lowered " + reason)
|
||||
}
|
||||
for _, u := range report.Unresolved {
|
||||
b.WriteString("\nunresolved: " + u)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// recordDelegationAudit emits one structured receipt per child run. It reports
|
||||
// what the host observed and what it refused to back, so an orchestration
|
||||
// benchmark can separate real gains from extra tokens spent. delegationText is
|
||||
// the parent-authored task before host framing, which is what makes the
|
||||
// evidence-origin split a host record rather than a claim.
|
||||
func recordDelegationAudit(ctx context.Context, summary evidence.ChildEvidenceSummary, claims writeclaim.WritePathSet, report evidence.CompletionReport, reasons []string, hasReport bool, delegationText string) {
|
||||
func recordDelegationAudit(ctx context.Context, summary evidence.ChildEvidenceSummary, claims writeclaim.WritePathSet, delegationText string) {
|
||||
audit := hostaudit.DelegationAudit{
|
||||
Depth: SubagentDepth(ctx),
|
||||
ToolCalls: len(summary.Receipts),
|
||||
MutationPaths: summary.MutationPaths(),
|
||||
ClaimViolations: len(claimViolations(summary, claims)),
|
||||
HasReport: hasReport,
|
||||
Downgrades: len(reasons),
|
||||
}
|
||||
audit.Mutations = len(audit.MutationPaths)
|
||||
evidence.ClassifyEvidenceOrigin(&audit, delegationText, summary.EvidencePaths())
|
||||
if hasReport {
|
||||
audit.AdjudicatedStatus = string(report.Status)
|
||||
}
|
||||
_, sink, _, _ := CallContext(ctx)
|
||||
event.RecordDelegationAudit(sink, audit)
|
||||
}
|
||||
@@ -390,57 +300,6 @@ func AppendHostReceipts(answer string, summary evidence.ChildEvidenceSummary, cl
|
||||
return answer + "\n\n" + block
|
||||
}
|
||||
|
||||
// handoffExit classifies how the run ended. A provider failure that produced no
|
||||
// report is not a compliance failure, and counting it as one would make the
|
||||
// denominator answer a different question.
|
||||
func handoffExit(answer string, runErr error) string {
|
||||
switch {
|
||||
case errors.Is(runErr, context.Canceled):
|
||||
return "cancelled"
|
||||
case runErr != nil:
|
||||
return "error"
|
||||
case strings.TrimSpace(answer) == "":
|
||||
return "no_answer"
|
||||
default:
|
||||
return "completed"
|
||||
}
|
||||
}
|
||||
|
||||
// countReportCalls walks the child's transcript for complete_subtask calls and
|
||||
// what the tool made of them. Attempted-and-refused is a schema problem;
|
||||
// never-attempted is a protocol one, and only the counts can tell them apart.
|
||||
func countReportCalls(audit *event.SubagentHandoffAudit, msgs []provider.Message) {
|
||||
calls := map[string]bool{}
|
||||
round := 0
|
||||
for _, m := range msgs {
|
||||
if len(m.ToolCalls) > 0 {
|
||||
round++
|
||||
}
|
||||
for _, tc := range m.ToolCalls {
|
||||
if tc.Name != completeSubtaskToolName {
|
||||
if audit.ReportRound > 0 {
|
||||
audit.ToolCallsAfterReport++
|
||||
}
|
||||
continue
|
||||
}
|
||||
calls[tc.ID] = true
|
||||
audit.Attempts++
|
||||
if audit.ReportRound == 0 {
|
||||
audit.ReportRound = round
|
||||
}
|
||||
}
|
||||
if m.Role != provider.RoleTool || !calls[m.ToolCallID] {
|
||||
continue
|
||||
}
|
||||
if isErrorMessage(m) {
|
||||
audit.Malformed++
|
||||
continue
|
||||
}
|
||||
audit.Accepted++
|
||||
}
|
||||
audit.FinalRound = round
|
||||
}
|
||||
|
||||
// claimViolations returns the mutations the host observed outside the write
|
||||
// claim the child declared. Tool-level confinement already refuses these, so a
|
||||
// non-empty result means a write reached the workspace through a surface the
|
||||
@@ -458,21 +317,6 @@ func claimViolations(summary evidence.ChildEvidenceSummary, claims writeclaim.Wr
|
||||
return out
|
||||
}
|
||||
|
||||
func criterionProof(c evidence.AcceptanceCriterion) string {
|
||||
var parts []string
|
||||
for _, e := range c.Evidence {
|
||||
switch {
|
||||
case strings.TrimSpace(e.Command) != "":
|
||||
parts = append(parts, e.Command)
|
||||
case len(e.Paths) > 0:
|
||||
parts = append(parts, strings.Join(e.Paths, " "))
|
||||
case strings.TrimSpace(e.Summary) != "":
|
||||
parts = append(parts, e.Kind+": "+e.Summary)
|
||||
}
|
||||
}
|
||||
return joinBoundedReceipts(parts)
|
||||
}
|
||||
|
||||
// formatHostReceipts renders only what the host is willing to attest to: files
|
||||
// the child really changed, and commands whose outcome the host observed.
|
||||
// Ordinary reads and greps are excluded on purpose — they are not claims a
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/safety/evidence"
|
||||
)
|
||||
|
||||
// CompleteSubtaskTool is visible only inside sub-agent registries. It ends a
|
||||
// delegated run with a structured, host-checkable claim instead of prose. It is
|
||||
// never registered on the parent agent's tool surface.
|
||||
type CompleteSubtaskTool struct{}
|
||||
|
||||
func NewCompleteSubtaskTool() *CompleteSubtaskTool { return &CompleteSubtaskTool{} }
|
||||
|
||||
// completeSubtaskToolName is the one spelling the contract, the registry and
|
||||
// the telemetry all mean.
|
||||
const completeSubtaskToolName = "complete_subtask"
|
||||
|
||||
func init() { tool.RegisterConditional(&CompleteSubtaskTool{}) }
|
||||
|
||||
func (*CompleteSubtaskTool) Name() string { return completeSubtaskToolName }
|
||||
|
||||
func (*CompleteSubtaskTool) Description() string {
|
||||
return "Close out this delegated sub-task with a structured result the parent can verify. Call once, last. status is complete, partial, blocked, or failed; summary states what is now true; acceptance_criteria lists each condition with the evidence for it; unresolved lists what you did not finish. The host checks every cited command and path against what it actually observed you do, and lowers any claim it cannot back."
|
||||
}
|
||||
|
||||
// ReadOnly is true: submitting a report changes no workspace state. It is the
|
||||
// claim itself, not a mutation.
|
||||
func (*CompleteSubtaskTool) ReadOnly() bool { return true }
|
||||
|
||||
func (*CompleteSubtaskTool) Schema() json.RawMessage {
|
||||
// Fixed schema — sub-agent registries only, so it never enters the parent
|
||||
// prefix.
|
||||
return json.RawMessage(`{
|
||||
"type":"object",
|
||||
"properties":{
|
||||
"status":{"type":"string","description":"complete | partial | blocked | failed. Claim the truth: the host lowers a status its receipts cannot back."},
|
||||
"summary":{"type":"string","description":"What is now true as a result of this sub-task."},
|
||||
"acceptance_criteria":{
|
||||
"type":"array",
|
||||
"description":"Each condition this sub-task had to meet, with proof.",
|
||||
"items":{
|
||||
"type":"object",
|
||||
"properties":{
|
||||
"id":{"type":"string","description":"Short stable id, e.g. AC1."},
|
||||
"status":{"type":"string","description":"satisfied | unsatisfied"},
|
||||
"evidence":{
|
||||
"type":"array",
|
||||
"items":{
|
||||
"type":"object",
|
||||
"properties":{
|
||||
"kind":{"type":"string","enum":["verification","review","diff","files","manual"],"description":"verification = a command was run (command REQUIRED); review = a review completed; diff = a code change (paths REQUIRED); files = files created/edited/inspected (paths REQUIRED); manual = a manual check, which the host cannot back on its own."},
|
||||
"summary":{"type":"string","description":"The evidence itself."},
|
||||
"command":{"type":"string","description":"REQUIRED for verification: the command as it actually ran."},
|
||||
"paths":{"type":"array","items":{"type":"string"},"description":"REQUIRED for diff/files: the files this evidence refers to."}
|
||||
},
|
||||
"required":["kind","summary"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required":["id","status"]
|
||||
}
|
||||
},
|
||||
"unresolved":{"type":"array","items":{"type":"string"},"description":"What remains undone, unverified, or blocked. Put anything you assumed rather than checked here."}
|
||||
},
|
||||
"required":["status","summary"]
|
||||
}`)
|
||||
}
|
||||
|
||||
func (*CompleteSubtaskTool) Execute(ctx context.Context, args json.RawMessage) (string, error) {
|
||||
report, err := evidence.ParseCompletionReport(args)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
led, ok := evidence.FromContext(ctx)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("complete_subtask requires the host evidence ledger; submit it from inside a sub-agent run")
|
||||
}
|
||||
// The submission always succeeds; what varies is the verdict. A rejection
|
||||
// the parent never learns about serves it worse than a lowered claim it can
|
||||
// see, so the call stands and the closure decision is said out loud.
|
||||
adjudicated, reasons := led.AdjudicateCompletion(report)
|
||||
closed := len(reasons) == 0
|
||||
led.NoteClosureVerdict(closed)
|
||||
if closed {
|
||||
return fmt.Sprintf("complete_subtask closed: status=%s criteria=%d unresolved=%d — the sub-task is done and nothing further is asked of you",
|
||||
adjudicated.Status, len(adjudicated.Criteria), len(adjudicated.Unresolved)), nil
|
||||
}
|
||||
// The host knows which claims it could not back and why; reporting only the
|
||||
// count leaves the sub-agent guessing which criterion and what would fix it.
|
||||
return fmt.Sprintf("complete_subtask needs_work: status=%s criteria=%d unresolved=%d — these claims are not backed by anything the host observed:\n- %s\nDo that work, then submit again.",
|
||||
adjudicated.Status, len(adjudicated.Criteria), len(adjudicated.Unresolved), strings.Join(reasons, "\n- ")), nil
|
||||
}
|
||||
|
||||
// AttachCompleteSubtaskTool adds complete_subtask to a sub-agent registry.
|
||||
// Parent registries never receive it.
|
||||
func AttachCompleteSubtaskTool(reg *tool.Registry) {
|
||||
if reg == nil {
|
||||
return
|
||||
}
|
||||
reg.Add(NewCompleteSubtaskTool())
|
||||
}
|
||||
|
||||
// CompletionReport returns this agent's adjudicated completion claim, if it
|
||||
// submitted one. Adjudication re-runs here so the returned status reflects the
|
||||
// full run, including receipts recorded after the tool call itself.
|
||||
func (a *Agent) CompletionReport() (evidence.CompletionReport, []string, bool) {
|
||||
if a == nil || a.task.ledger == nil {
|
||||
return evidence.CompletionReport{}, nil, false
|
||||
}
|
||||
report, ok := a.task.ledger.LatestCompletionReport()
|
||||
if !ok {
|
||||
return evidence.CompletionReport{}, nil, false
|
||||
}
|
||||
adjudicated, reasons := a.task.ledger.AdjudicateCompletion(report)
|
||||
return adjudicated, reasons, true
|
||||
}
|
||||
|
||||
// CompleteSubtaskContract is appended to a sub-agent's task prompt when the
|
||||
// host expects a typed completion claim. The profile body says how to work;
|
||||
// this states the non-negotiable closing protocol.
|
||||
const CompleteSubtaskContract = `<completion-contract>
|
||||
Call complete_subtask when you believe this sub-task is done. State the acceptance
|
||||
criteria you were held to and attach, for each, the command you ran or the paths you
|
||||
changed or inspected. The host checks every citation against what it observed you
|
||||
actually do, and answers with a verdict: needs_work means a claim is not backed by
|
||||
anything it saw — do that work and submit again; closed means the sub-task is done and
|
||||
nothing further is asked of you. Cite real work only and put anything you assumed
|
||||
rather than verified in unresolved.
|
||||
</completion-contract>`
|
||||
@@ -1,179 +0,0 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/safety/evidence"
|
||||
)
|
||||
|
||||
func submitCompleteSubtask(t *testing.T, led *evidence.Ledger, args string) string {
|
||||
t.Helper()
|
||||
ctx := evidence.WithLedger(context.Background(), led)
|
||||
out, err := NewCompleteSubtaskTool().Execute(ctx, json.RawMessage(args))
|
||||
if err != nil {
|
||||
t.Fatalf("complete_subtask: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The child may claim anything; the status the parent sees is the host's.
|
||||
func TestCompleteSubtaskHostLowersUnbackedClaim(t *testing.T) {
|
||||
led := evidence.NewLedger()
|
||||
led.Record(evidence.Receipt{ToolName: "bash", Command: "go test ./parser", Success: true, OutputBytes: 12})
|
||||
|
||||
args := `{
|
||||
"status":"complete",
|
||||
"summary":"fixed the parser",
|
||||
"acceptance_criteria":[
|
||||
{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"unit tests","command":"go test ./parser"}]},
|
||||
{"id":"AC2","status":"satisfied","evidence":[{"kind":"verification","summary":"integration suite","command":"go test ./integration"}]}
|
||||
]}`
|
||||
if out := submitCompleteSubtask(t, led, args); !strings.Contains(out, "status=partial") {
|
||||
t.Fatalf("tool result = %q, want the host-lowered status", out)
|
||||
}
|
||||
|
||||
// The agent host, not the tool, records the call; replay that receipt so the
|
||||
// ledger lookup the report renderer uses is covered too.
|
||||
led.Record(evidence.ReceiptFromToolCall("complete_subtask", json.RawMessage(args), true, evidence.ToolFacts{ReadOnly: true}))
|
||||
report, ok := led.LatestCompletionReport()
|
||||
if !ok {
|
||||
t.Fatal("a recorded complete_subtask call must be recoverable from the ledger")
|
||||
}
|
||||
adjudicated, reasons := led.AdjudicateCompletion(report)
|
||||
if adjudicated.Status != evidence.CompletionPartial {
|
||||
t.Fatalf("status = %q, want partial", adjudicated.Status)
|
||||
}
|
||||
if adjudicated.Criteria[0].Status != evidence.CriterionSatisfied {
|
||||
t.Fatal("AC1 was backed by a real command receipt and must survive")
|
||||
}
|
||||
if adjudicated.Criteria[1].Status != evidence.CriterionUnsatisfied {
|
||||
t.Fatal("AC2 cited a command that never ran and must be lowered")
|
||||
}
|
||||
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "AC2:") {
|
||||
t.Fatalf("reasons = %v, want one naming AC2", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteSubtaskKeepsFullyBackedClaim(t *testing.T) {
|
||||
led := evidence.NewLedger()
|
||||
led.Record(evidence.Receipt{ToolName: "bash", Command: "go test ./parser", Success: true, OutputBytes: 12})
|
||||
led.Record(evidence.Receipt{ToolName: "write_file", Success: true, Mutation: true, Write: true, Paths: []string{"parser.go"}})
|
||||
|
||||
out := submitCompleteSubtask(t, led, `{
|
||||
"status":"complete",
|
||||
"summary":"fixed the parser",
|
||||
"acceptance_criteria":[
|
||||
{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"tests","command":"go test ./parser"}]},
|
||||
{"id":"AC2","status":"satisfied","evidence":[{"kind":"diff","summary":"the fix","paths":["parser.go"]}]}
|
||||
]}`)
|
||||
if !strings.Contains(out, "status=complete") || strings.Contains(out, "lowered") {
|
||||
t.Fatalf("tool result = %q, want an untouched complete status", out)
|
||||
}
|
||||
}
|
||||
|
||||
// A satisfied criterion resting only on the model's word is not host-backed.
|
||||
func TestCompleteSubtaskLowersManualOnlyAndEvidenceFreeClaims(t *testing.T) {
|
||||
led := evidence.NewLedger()
|
||||
report, err := evidence.ParseCompletionReport(json.RawMessage(`{
|
||||
"status":"complete","summary":"done",
|
||||
"acceptance_criteria":[
|
||||
{"id":"AC1","status":"satisfied","evidence":[{"kind":"manual","summary":"I checked it"}]},
|
||||
{"id":"AC2","status":"satisfied"}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adjudicated, reasons := led.AdjudicateCompletion(report)
|
||||
if adjudicated.Status != evidence.CompletionPartial || len(reasons) != 2 {
|
||||
t.Fatalf("status = %q reasons = %v, want partial with both lowered", adjudicated.Status, reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCompletionReportRejectsMalformedClaims(t *testing.T) {
|
||||
for name, args := range map[string]string{
|
||||
"bad status": `{"status":"done","summary":"x"}`,
|
||||
"missing summary": `{"status":"complete"}`,
|
||||
"verification no cmd": `{"status":"complete","summary":"x","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"tests"}]}]}`,
|
||||
"diff without paths": `{"status":"complete","summary":"x","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"diff","summary":"change"}]}]}`,
|
||||
"unknown evidence": `{"status":"complete","summary":"x","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"vibes","summary":"trust me"}]}]}`,
|
||||
"criterion without id": `{"status":"complete","summary":"x","acceptance_criteria":[{"status":"satisfied"}]}`,
|
||||
} {
|
||||
if _, err := evidence.ParseCompletionReport(json.RawMessage(args)); err == nil {
|
||||
t.Errorf("%s: accepted a malformed report", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A verdict is not an error. A malformed submission fails the call; a claim the
|
||||
// host cannot back is a well-formed call the host answers with needs_work, and
|
||||
// keeping those apart is what lets "submit again" and "you are done" be
|
||||
// different answers instead of two readings of the word accepted.
|
||||
func TestCompleteSubtaskVerdictSeparatesClosureFromMoreWork(t *testing.T) {
|
||||
const criterion = `{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"tests","command":"go test ./parser"}]}`
|
||||
|
||||
backed := evidence.NewLedger()
|
||||
backed.Record(evidence.Receipt{ToolName: "bash", Command: "go test ./parser", Success: true, OutputBytes: 12})
|
||||
out := submitCompleteSubtask(t, backed, `{"status":"complete","summary":"fixed it","acceptance_criteria":[`+criterion+`]}`)
|
||||
if !strings.Contains(out, "complete_subtask closed:") {
|
||||
t.Fatalf("backed report = %q, want a closed verdict", out)
|
||||
}
|
||||
if v := backed.ClosureVerdicts(); v.Closed != 1 || v.NeedsWork != 0 {
|
||||
t.Fatalf("verdicts = %+v, want one closure", v)
|
||||
}
|
||||
|
||||
// The same claim with nothing behind it: the call still succeeds.
|
||||
unbacked := evidence.NewLedger()
|
||||
out = submitCompleteSubtask(t, unbacked, `{"status":"complete","summary":"fixed it","acceptance_criteria":[`+criterion+`]}`)
|
||||
if !strings.Contains(out, "complete_subtask needs_work:") {
|
||||
t.Fatalf("unbacked report = %q, want a needs_work verdict", out)
|
||||
}
|
||||
if v := unbacked.ClosureVerdicts(); v.NeedsWork != 1 || v.Closed != 0 {
|
||||
t.Fatalf("verdicts = %+v, want one needs_work", v)
|
||||
}
|
||||
|
||||
// A malformed submission is the other layer and still fails the call.
|
||||
ctx := evidence.WithLedger(context.Background(), evidence.NewLedger())
|
||||
if _, err := NewCompleteSubtaskTool().Execute(ctx, json.RawMessage(`{"status":"nonsense"}`)); err == nil {
|
||||
t.Fatal("a malformed report must fail the call, not return a verdict")
|
||||
}
|
||||
}
|
||||
|
||||
// A rejection the sub-agent cannot act on costs a whole round: it resubmits
|
||||
// with a different evidence shape and hopes. The host knows which criterion it
|
||||
// could not back and what it looked for, so the verdict says both.
|
||||
func TestCompleteSubtaskRejectionNamesWhatTheHostLookedFor(t *testing.T) {
|
||||
led := evidence.NewLedger()
|
||||
ctx := evidence.WithLedger(context.Background(), led)
|
||||
|
||||
args := []byte(`{"status":"complete","summary":"done",
|
||||
"acceptance_criteria":[
|
||||
{"id":"c1","status":"satisfied","evidence":[
|
||||
{"kind":"files","summary":"read the files","paths":["internal/runtime/agent/agent.go","internal/runtime/agent/run_loop.go"]}]},
|
||||
{"id":"c2","status":"satisfied","evidence":[
|
||||
{"kind":"verification","summary":"ran the tests","command":"go test ./internal/runtime/agent/"}]},
|
||||
{"id":"c3","status":"satisfied","evidence":[
|
||||
{"kind":"review","summary":"first reviewer"},
|
||||
{"kind":"review","summary":"second reviewer"}]}]}`)
|
||||
|
||||
out, err := (&CompleteSubtaskTool{}).Execute(ctx, args)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "needs_work") {
|
||||
t.Fatalf("verdict = %q, want needs_work for claims with no receipts", out)
|
||||
}
|
||||
if n := strings.Count(out, "no completed review"); n > 1 {
|
||||
t.Errorf("one missing receipt is reported %d times, which reads as %d problems:\n%s", n, n, out)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"c1:", "no read or write recorded for", "internal/runtime/agent/agent.go",
|
||||
"c2:", "no successful run of", "go test ./internal/runtime/agent/",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("verdict does not say %q; the sub-agent cannot tell what to fix:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reasonix/internal/contract/tool"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// A writer child submits a typed claim the host adjudicates. A read-only child
|
||||
// used to end in prose, so what it found reached the parent as an unverifiable
|
||||
// sentence while its receipts sat elsewhere in the host ledger.
|
||||
|
||||
// Both kinds of child are told the same closing protocol, and the wording has
|
||||
// to fit a child that only looked at things. It also has to name both verdicts:
|
||||
// a contract that asks for one call cannot be honoured against a host that
|
||||
// answers a lowered claim by asking for the work again.
|
||||
func TestCompletionContractCoversInspectionNotOnlyChange(t *testing.T) {
|
||||
if !strings.Contains(CompleteSubtaskContract, "complete_subtask") {
|
||||
t.Error("the contract no longer names the call it requires")
|
||||
}
|
||||
for _, verdict := range []string{"needs_work", "closed"} {
|
||||
if !strings.Contains(CompleteSubtaskContract, verdict) {
|
||||
t.Errorf("the contract does not say what %q means, so the child cannot act on it", verdict)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(CompleteSubtaskContract, "inspected") {
|
||||
t.Errorf("the contract asks only about changes, so a read-only child has nothing to cite:\n%s",
|
||||
CompleteSubtaskContract)
|
||||
}
|
||||
}
|
||||
|
||||
// The shadow answers "did the child close the way it was asked to", so its
|
||||
// denominator must be the instruction the host actually gave.
|
||||
func TestHandoffShadowCountsAttemptsApartFromAcceptance(t *testing.T) {
|
||||
audit := event.SubagentHandoffAudit{}
|
||||
countReportCalls(&audit, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "a", Name: "read_file", Arguments: `{"path":"x"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "a", Name: "read_file", Content: "ok"},
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "r1", Name: completeSubtaskToolName, Arguments: `{}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "r1", Name: completeSubtaskToolName, Content: "error: invalid status"},
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "r2", Name: completeSubtaskToolName, Arguments: `{"status":"complete"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "r2", Name: completeSubtaskToolName, Content: "recorded"},
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "b", Name: "read_file", Arguments: `{"path":"y"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "b", Name: "read_file", Content: "ok"},
|
||||
})
|
||||
|
||||
// A refused call is a schema problem; never calling is a protocol one, and
|
||||
// one count cannot say which happened.
|
||||
if audit.Attempts != 2 || audit.Accepted != 1 || audit.Malformed != 1 {
|
||||
t.Errorf("attempts=%d accepted=%d malformed=%d, want 2/1/1",
|
||||
audit.Attempts, audit.Accepted, audit.Malformed)
|
||||
}
|
||||
// The contract asks for the report as the final call. Work after it is a
|
||||
// different behaviour from submitting one at all.
|
||||
if audit.ToolCallsAfterReport != 1 {
|
||||
t.Errorf("tool calls after the report = %d, want 1", audit.ToolCallsAfterReport)
|
||||
}
|
||||
if audit.ReportRound != 2 || audit.FinalRound != 4 {
|
||||
t.Errorf("rounds = report %d final %d, want 2 and 4", audit.ReportRound, audit.FinalRound)
|
||||
}
|
||||
}
|
||||
|
||||
// A run that died on the provider never had the chance to comply. Counting it
|
||||
// as a missing report would make the adoption rate answer a different question.
|
||||
func TestHandoffShadowSeparatesFailureFromNonCompliance(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
answer string
|
||||
err error
|
||||
want string
|
||||
}{
|
||||
{"clean", "here is what I found", nil, "completed"},
|
||||
{"provider failed", "", errors.New("stream error"), "error"},
|
||||
// A wrapped sentinel, not a lookalike string: identity is what the
|
||||
// classifier reads, so a message that merely says so must not pass.
|
||||
{"user cancelled", "", fmt.Errorf("sub-agent: %w", context.Canceled), "cancelled"},
|
||||
{"message only says cancelled", "", errors.New("context canceled"), "error"},
|
||||
{"ran but said nothing", " ", nil, "no_answer"},
|
||||
} {
|
||||
if got := handoffExit(tc.answer, tc.err); got != tc.want {
|
||||
t.Errorf("%s: exit = %q, want %q", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteSubtaskSurvivesTheStrictReadOnlyFilter(t *testing.T) {
|
||||
reg := tool.NewRegistry()
|
||||
reg.Add(subagentRegistryTool{name: "read_file", readOnly: true})
|
||||
AttachCompleteSubtaskTool(reg)
|
||||
|
||||
// The child's registry is filtered once more at construction; a protocol
|
||||
// tool that did not survive would leave the contract asking for a call the
|
||||
// child cannot make.
|
||||
strict := strictReadOnlyExecutionRegistry(reg)
|
||||
if _, ok := strict.Get("complete_subtask"); !ok {
|
||||
t.Fatalf("complete_subtask did not survive strict read-only construction: %v", strict.Names())
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package delegation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reasonix/internal/runtime/agent"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
)
|
||||
|
||||
// End to end: the parent's view leads with the adjudicated status, not prose.
|
||||
func TestSubAgentAnswerLeadsWithAdjudicatedStatus(t *testing.T) {
|
||||
reg := tool.NewRegistry()
|
||||
reg.Add(fakeWriteFileTool{})
|
||||
agent.AttachCompleteSubtaskTool(reg)
|
||||
prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{
|
||||
{toolCallChunk("1", "write_file", `{"path":"parser.go"}`), {Type: provider.ChunkDone}},
|
||||
{toolCallChunk("2", "complete_subtask", `{"status":"complete","summary":"fixed the parser","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"diff","summary":"the fix","paths":["parser.go"]}]},{"id":"AC2","status":"satisfied","evidence":[{"kind":"verification","summary":"suite","command":"go test ./..."}]}],"unresolved":["integration suite not executed"]}`), {Type: provider.ChunkDone}},
|
||||
{{Type: provider.ChunkText, Text: "all good"}, {Type: provider.ChunkDone}},
|
||||
}}
|
||||
|
||||
answer, err := agent.RunSubAgentWithSession(context.Background(), prov, reg, sessionstore.NewSession("sys"),
|
||||
"fix the parser", agent.Options{CheckTargetAccess: allowTestFileAccess}, event.Discard)
|
||||
if err != nil {
|
||||
t.Fatalf("RunSubAgentWithSession: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(answer, "status: partial") {
|
||||
t.Fatalf("answer must lead with the host-adjudicated status:\n%s", answer)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"AC1 satisfied",
|
||||
"AC2 unsatisfied",
|
||||
"host lowered AC2",
|
||||
"unresolved: integration suite not executed",
|
||||
agent.HostReceiptsHeader,
|
||||
} {
|
||||
if !strings.Contains(answer, want) {
|
||||
t.Fatalf("answer missing %q:\n%s", want, answer)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package delegation
|
||||
|
||||
import (
|
||||
"reasonix/internal/runtime/agent"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/tool"
|
||||
)
|
||||
|
||||
// The protocol tool is not a capability. Attaching it before the "can this
|
||||
// child investigate anything?" check would let a child with no research tools
|
||||
// pass as having one, and it would then report on work it could not do.
|
||||
func TestReportingToolDoesNotCountAsAResearchCapability(t *testing.T) {
|
||||
parent := tool.NewRegistry()
|
||||
parent.Add(subagentRegistryTool{name: "write_file"}) // not read-only
|
||||
|
||||
sub := agent.ReadOnlySubagentToolRegistry(parent, nil)
|
||||
if sub.Len() != 0 {
|
||||
t.Fatalf("a parent with no read-only tools produced %v", sub.Names())
|
||||
}
|
||||
// And once it is attached, the registry is no longer empty — which is
|
||||
// exactly why the emptiness check has to come first.
|
||||
agent.AttachCompleteSubtaskTool(sub)
|
||||
if sub.Len() == 0 {
|
||||
t.Fatal("complete_subtask was not attached at all")
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,6 @@ func (t *TaskTool) readOnlySubRegistry(spec *ProfileExecSpec, toolNames []string
|
||||
if sub.Len() == 0 && !spec.Grant.AllowNoTools {
|
||||
return nil, fmt.Errorf("no read-only tools available for this sub-agent")
|
||||
}
|
||||
agent.AttachCompleteSubtaskTool(sub)
|
||||
return sub, nil
|
||||
}
|
||||
|
||||
@@ -32,7 +31,6 @@ func (t *TaskTool) prepareSubSession(ctx context.Context, prompt string, opts ag
|
||||
// The pristine task, before framing: delivery classification judges the
|
||||
// task and not the wrapper.
|
||||
opts.ClassifierTaskText = prompt
|
||||
opts.ExpectCompletionReport, opts.HandoffEntrance = true, entrance
|
||||
prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt) + "\n\n" + agent.CompleteSubtaskContract
|
||||
prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt)
|
||||
return agent.WithUserImages(ctx, agent.SubagentImageCandidates(ctx)), prompt, opts
|
||||
}
|
||||
@@ -824,8 +824,7 @@ func (t *TaskTool) runSubSession(ctx context.Context, prompt string, subReg *too
|
||||
// Capture the pristine task before host framing is prepended: delivery
|
||||
// intent classification must judge the task, not the wrapper.
|
||||
opts.ClassifierTaskText = prompt
|
||||
prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt) + "\n\n" + agent.CompleteSubtaskContract
|
||||
opts.ExpectCompletionReport, opts.HandoffEntrance = true, entrance
|
||||
prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt)
|
||||
// Child provider owns the final vision decision. Text-only providers
|
||||
// retain the attachment metadata but omit image parts during serialization.
|
||||
ctx = agent.WithUserImages(ctx, agent.SubagentImageCandidates(ctx))
|
||||
|
||||
@@ -54,7 +54,7 @@ func TestTaskToolReturnsSubAgentFinalAnswer(t *testing.T) {
|
||||
if sys := sub.lastReq.Messages[0]; sys.Role != provider.RoleSystem || sys.Content != "test-sys-prompt" {
|
||||
t.Errorf("first message = %+v, want system 'test-sys-prompt'", sys)
|
||||
}
|
||||
if got := lastUser(sub.lastReq); !strings.Contains(got, `<subagent-context event="SubagentStart">`) || !strings.Contains(got, "find callers of Foo") || !strings.Contains(got, agent.CompleteSubtaskContract) {
|
||||
if got := lastUser(sub.lastReq); !strings.Contains(got, `<subagent-context event="SubagentStart">`) || !strings.Contains(got, "find callers of Foo") {
|
||||
t.Errorf("sub-agent user = %q, want SubagentStart context plus prompt", got)
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func TestTaskToolInjectsWorkspaceContextIntoSubagentPrompt(t *testing.T) {
|
||||
if !strings.Contains(got, `<workspace-context event="SubagentWorkspace">`) ||
|
||||
!strings.Contains(got, "Current workspace: "+strconv.Quote(workspace)) ||
|
||||
!strings.Contains(got, `prefer "." or relative paths`) ||
|
||||
!strings.Contains(got, "inspect project") || !strings.Contains(got, agent.CompleteSubtaskContract) {
|
||||
!strings.Contains(got, "inspect project") {
|
||||
t.Fatalf("sub-agent user = %q, want workspace context plus prompt", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func report(status CompletionStatus, criteria ...AcceptanceCriterion) CompletionReport {
|
||||
return CompletionReport{Status: status, Summary: "done", Criteria: criteria}
|
||||
}
|
||||
|
||||
func satisfied(id string, evidence ...CompletionEvidence) AcceptanceCriterion {
|
||||
return AcceptanceCriterion{ID: id, Status: CriterionSatisfied, Evidence: evidence}
|
||||
}
|
||||
|
||||
func TestAdjudicateCompletionKeepsWhatTheHostsOwnReceiptsBack(t *testing.T) {
|
||||
l := recorded(
|
||||
Receipt{ToolName: "bash", Success: true, Command: "go test ./..."},
|
||||
Receipt{ToolName: "edit_file", Success: true, Write: true, Mutation: true, Paths: []string{"a.go"}},
|
||||
Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"b.go"}},
|
||||
)
|
||||
in := report(CompletionComplete,
|
||||
satisfied("tests", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test ./..."}),
|
||||
satisfied("diff", CompletionEvidence{Kind: CompletionEvidenceDiff, Paths: []string{"a.go"}}),
|
||||
satisfied("files", CompletionEvidence{Kind: CompletionEvidenceFiles, Paths: []string{"b.go"}}),
|
||||
)
|
||||
got, reasons := l.AdjudicateCompletion(in)
|
||||
if got.Status != CompletionComplete || len(reasons) != 0 {
|
||||
t.Fatalf("a fully backed report was lowered: %q %v", got.Status, reasons)
|
||||
}
|
||||
for _, c := range got.Criteria {
|
||||
if c.Status != CriterionSatisfied {
|
||||
t.Fatalf("criterion %s was downgraded", c.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdjudicateCompletionDowngradesEveryClaimNoReceiptBacksAndCannotStayComplete(t *testing.T) {
|
||||
l := recorded(
|
||||
Receipt{ToolName: "bash", Success: false, Command: "go test ./..."},
|
||||
Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"a.go"}},
|
||||
)
|
||||
in := report(CompletionComplete,
|
||||
satisfied("failed-run", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test ./..."}),
|
||||
satisfied("read-is-not-a-write", CompletionEvidence{Kind: CompletionEvidenceDiff, Paths: []string{"a.go"}}),
|
||||
satisfied("manual-only", CompletionEvidence{Kind: CompletionEvidenceManual, Summary: "trust me"}),
|
||||
satisfied("no-evidence"),
|
||||
AcceptanceCriterion{ID: "already-unsatisfied", Status: CriterionUnsatisfied},
|
||||
)
|
||||
got, reasons := l.AdjudicateCompletion(in)
|
||||
if got.Status != CompletionPartial {
|
||||
t.Fatalf("status = %q, want the report lowered to partial", got.Status)
|
||||
}
|
||||
if len(reasons) != 4 {
|
||||
t.Fatalf("reasons = %v, want one per downgraded claim", reasons)
|
||||
}
|
||||
for _, want := range []string{"failed-run: no successful run", "read-is-not-a-write: no write recorded", "manual-only:", "no-evidence: claimed satisfied with no evidence"} {
|
||||
found := false
|
||||
for _, r := range reasons {
|
||||
found = found || strings.Contains(r, want)
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no reason mentions %q: %v", want, reasons)
|
||||
}
|
||||
}
|
||||
for _, c := range got.Criteria {
|
||||
if c.Status != CriterionUnsatisfied {
|
||||
t.Errorf("criterion %s is %s after adjudication", c.ID, c.Status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdjudicateCompletionNeverRaisesAStatusAndAcceptsANilLedger(t *testing.T) {
|
||||
l := recorded(Receipt{ToolName: "bash", Success: true, Command: "make check"})
|
||||
for _, status := range []CompletionStatus{CompletionPartial, CompletionBlocked, CompletionFailed} {
|
||||
in := report(status, satisfied("ok", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "make check"}))
|
||||
if got, _ := l.AdjudicateCompletion(in); got.Status != status {
|
||||
t.Errorf("%s became %s: the host may lower a claim, never raise it", status, got.Status)
|
||||
}
|
||||
}
|
||||
var nilLedger *Ledger
|
||||
in := report(CompletionComplete, satisfied("x"))
|
||||
if got, reasons := nilLedger.AdjudicateCompletion(in); got.Status != CompletionComplete || reasons != nil {
|
||||
t.Fatalf("a nil ledger adjudicates nothing: %q %v", got.Status, reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdjudicateCompletionSaysACitedMissIsOneProblemNotTwo(t *testing.T) {
|
||||
l := recorded()
|
||||
got, reasons := l.AdjudicateCompletion(report(CompletionComplete, satisfied("twice",
|
||||
CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test"},
|
||||
CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test"},
|
||||
)))
|
||||
if len(reasons) != 1 || strings.Count(reasons[0], "no successful run") != 1 || got.Status != CompletionPartial {
|
||||
t.Fatalf("reasons = %v status = %q", reasons, got.Status)
|
||||
}
|
||||
}
|
||||
@@ -1,270 +0,0 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// CompletionStatus is a sub-agent's claim about its own run. The host may lower
|
||||
// it to what receipts support; nothing may raise it.
|
||||
type CompletionStatus string
|
||||
|
||||
const (
|
||||
CompletionComplete CompletionStatus = "complete"
|
||||
CompletionPartial CompletionStatus = "partial"
|
||||
CompletionBlocked CompletionStatus = "blocked"
|
||||
CompletionFailed CompletionStatus = "failed"
|
||||
)
|
||||
|
||||
// CriterionStatus is the child's claim about one acceptance criterion.
|
||||
type CriterionStatus string
|
||||
|
||||
const (
|
||||
CriterionSatisfied CriterionStatus = "satisfied"
|
||||
CriterionUnsatisfied CriterionStatus = "unsatisfied"
|
||||
)
|
||||
|
||||
// Completion evidence kinds mirror complete_step so a sub-agent and the root
|
||||
// agent prove work in the same vocabulary.
|
||||
const (
|
||||
CompletionEvidenceVerification = "verification"
|
||||
CompletionEvidenceReview = "review"
|
||||
CompletionEvidenceDiff = "diff"
|
||||
CompletionEvidenceFiles = "files"
|
||||
CompletionEvidenceManual = "manual"
|
||||
)
|
||||
|
||||
// CompletionEvidence is one proof attached to an acceptance criterion.
|
||||
type CompletionEvidence struct {
|
||||
Kind string `json:"kind"`
|
||||
Summary string `json:"summary"`
|
||||
Command string `json:"command,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// AcceptanceCriterion is one checkable condition the sub-task had to meet.
|
||||
type AcceptanceCriterion struct {
|
||||
ID string `json:"id"`
|
||||
Status CriterionStatus `json:"status"`
|
||||
Evidence []CompletionEvidence `json:"evidence,omitempty"`
|
||||
}
|
||||
|
||||
// CompletionReport is the structured payload submitted via complete_subtask.
|
||||
type CompletionReport struct {
|
||||
Status CompletionStatus `json:"status"`
|
||||
Summary string `json:"summary"`
|
||||
Criteria []AcceptanceCriterion `json:"acceptance_criteria,omitempty"`
|
||||
Unresolved []string `json:"unresolved,omitempty"`
|
||||
}
|
||||
|
||||
// ParseCompletionReport validates and normalizes a complete_subtask argument
|
||||
// object. It checks shape only: whether the claims are true is the host's job.
|
||||
func ParseCompletionReport(raw json.RawMessage) (CompletionReport, error) {
|
||||
var r CompletionReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return CompletionReport{}, fmt.Errorf("invalid complete_subtask JSON: %w", err)
|
||||
}
|
||||
r.Status = CompletionStatus(strings.ToLower(strings.TrimSpace(string(r.Status))))
|
||||
switch r.Status {
|
||||
case CompletionComplete, CompletionPartial, CompletionBlocked, CompletionFailed:
|
||||
default:
|
||||
return CompletionReport{}, fmt.Errorf("complete_subtask.status must be complete, partial, blocked, or failed")
|
||||
}
|
||||
r.Summary = strings.TrimSpace(r.Summary)
|
||||
if r.Summary == "" {
|
||||
return CompletionReport{}, fmt.Errorf("complete_subtask.summary is required")
|
||||
}
|
||||
for i := range r.Criteria {
|
||||
c := &r.Criteria[i]
|
||||
c.ID = strings.TrimSpace(c.ID)
|
||||
if c.ID == "" {
|
||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].id is required", i)
|
||||
}
|
||||
c.Status = CriterionStatus(strings.ToLower(strings.TrimSpace(string(c.Status))))
|
||||
switch c.Status {
|
||||
case CriterionSatisfied, CriterionUnsatisfied:
|
||||
default:
|
||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].status must be satisfied or unsatisfied", i)
|
||||
}
|
||||
for j := range c.Evidence {
|
||||
e := &c.Evidence[j]
|
||||
e.Kind = strings.ToLower(strings.TrimSpace(e.Kind))
|
||||
switch e.Kind {
|
||||
case CompletionEvidenceVerification:
|
||||
if strings.TrimSpace(e.Command) == "" {
|
||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].evidence[%d]: verification requires command", i, j)
|
||||
}
|
||||
case CompletionEvidenceDiff, CompletionEvidenceFiles:
|
||||
if len(e.Paths) == 0 {
|
||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].evidence[%d]: %s requires paths", i, j, e.Kind)
|
||||
}
|
||||
case CompletionEvidenceReview, CompletionEvidenceManual:
|
||||
default:
|
||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].evidence[%d].kind is not a known evidence kind", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// LatestCompletionReport returns the most recent successful complete_subtask
|
||||
// payload recorded this run.
|
||||
func (l *Ledger) LatestCompletionReport() (CompletionReport, bool) {
|
||||
if l == nil {
|
||||
return CompletionReport{}, false
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
for _, r := range slices.Backward(l.receipts) {
|
||||
if r.ToolName != "complete_subtask" || !r.Success {
|
||||
continue
|
||||
}
|
||||
report, err := ParseCompletionReport(r.Args)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
return report, true
|
||||
}
|
||||
return CompletionReport{}, false
|
||||
}
|
||||
|
||||
// ClosureTally counts the verdicts this run's completion reports received, at
|
||||
// the moment each was issued. Recomputing them later answers a different
|
||||
// question: the ledger has grown since, so a claim the host could not back
|
||||
// then may be backed now.
|
||||
type ClosureTally struct {
|
||||
Closed int
|
||||
NeedsWork int
|
||||
}
|
||||
|
||||
// NoteClosureVerdict records what the host answered one report with.
|
||||
func (l *Ledger) NoteClosureVerdict(closed bool) {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if closed {
|
||||
l.closure.Closed++
|
||||
return
|
||||
}
|
||||
l.closure.NeedsWork++
|
||||
}
|
||||
|
||||
// ClosureVerdicts returns the verdicts issued so far.
|
||||
func (l *Ledger) ClosureVerdicts() ClosureTally {
|
||||
if l == nil {
|
||||
return ClosureTally{}
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.closure
|
||||
}
|
||||
|
||||
// AdjudicateCompletion lowers a report to what the host's own receipts support.
|
||||
// A criterion claimed satisfied whose evidence no receipt backs is downgraded,
|
||||
// and a report holding any downgraded criterion cannot stay "complete". The
|
||||
// returned reasons name every downgrade so the parent sees why.
|
||||
func (l *Ledger) AdjudicateCompletion(r CompletionReport) (CompletionReport, []string) {
|
||||
if l == nil {
|
||||
return r, nil
|
||||
}
|
||||
var reasons []string
|
||||
downgraded := false
|
||||
for i := range r.Criteria {
|
||||
c := &r.Criteria[i]
|
||||
if c.Status != CriterionSatisfied {
|
||||
continue
|
||||
}
|
||||
if backed, why := l.criterionIsBacked(*c); !backed {
|
||||
c.Status = CriterionUnsatisfied
|
||||
downgraded = true
|
||||
reasons = append(reasons, c.ID+": "+why)
|
||||
}
|
||||
}
|
||||
if downgraded && r.Status == CompletionComplete {
|
||||
r.Status = CompletionPartial
|
||||
}
|
||||
return r, reasons
|
||||
}
|
||||
|
||||
// criterionIsBacked reports whether at least one attached proof is one the host
|
||||
// itself observed. A manual claim is never host-backed on its own.
|
||||
func (l *Ledger) criterionIsBacked(c AcceptanceCriterion) (bool, string) {
|
||||
if len(c.Evidence) == 0 {
|
||||
return false, "claimed satisfied with no evidence"
|
||||
}
|
||||
var missing []string
|
||||
for _, e := range c.Evidence {
|
||||
switch e.Kind {
|
||||
case CompletionEvidenceVerification:
|
||||
if l.HasSuccessfulCommand(e.Command) {
|
||||
return true, ""
|
||||
}
|
||||
missing = append(missing, "no successful run of "+quoteShort(e.Command))
|
||||
case CompletionEvidenceDiff:
|
||||
if l.HasSuccessfulWrite(e.Paths) {
|
||||
return true, ""
|
||||
}
|
||||
missing = append(missing, "no write recorded for "+joinShort(e.Paths))
|
||||
case CompletionEvidenceFiles:
|
||||
if l.HasSuccessfulReadOrWrite(e.Paths) {
|
||||
return true, ""
|
||||
}
|
||||
missing = append(missing, "no read or write recorded for "+joinShort(e.Paths))
|
||||
case CompletionEvidenceReview:
|
||||
if l.HasCompletedReview() {
|
||||
return true, ""
|
||||
}
|
||||
missing = append(missing, "no completed review")
|
||||
}
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return false, "no host receipt backs the cited evidence"
|
||||
}
|
||||
// Two evidence entries can miss for the same reason; saying it twice reads
|
||||
// as two problems. Order is kept: it is the order they were cited in.
|
||||
seen := make(map[string]bool, len(missing))
|
||||
uniq := missing[:0]
|
||||
for _, m := range missing {
|
||||
if seen[m] {
|
||||
continue
|
||||
}
|
||||
seen[m] = true
|
||||
uniq = append(uniq, m)
|
||||
}
|
||||
return false, strings.Join(uniq, "; ")
|
||||
}
|
||||
|
||||
// quoteShort renders one cited command for a rejection message.
|
||||
func quoteShort(s string) string {
|
||||
const max = 80
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return "an unnamed command"
|
||||
}
|
||||
if len(s) > max {
|
||||
cut := max
|
||||
for cut > 0 && !utf8.RuneStart(s[cut]) {
|
||||
cut--
|
||||
}
|
||||
s = s[:cut] + "…"
|
||||
}
|
||||
return strconv.Quote(s)
|
||||
}
|
||||
|
||||
// joinShort names the cited paths, capped so one rejection cannot become the
|
||||
// whole reply.
|
||||
func joinShort(paths []string) string {
|
||||
if len(paths) == 0 {
|
||||
return "no path"
|
||||
}
|
||||
if len(paths) > 3 {
|
||||
return strings.Join(paths[:3], ", ") + fmt.Sprintf(" (+%d more)", len(paths)-3)
|
||||
}
|
||||
return strings.Join(paths, ", ")
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseCompletionReportNormalisesAValidReport(t *testing.T) {
|
||||
raw := json.RawMessage(`{"status":" Complete ","summary":" fixed the loop ","acceptance_criteria":[
|
||||
{"id":" c1 ","status":"SATISFIED","evidence":[
|
||||
{"kind":" Verification ","summary":"ran","command":"go test ./..."},
|
||||
{"kind":"diff","summary":"d","paths":["a.go"]},
|
||||
{"kind":"files","summary":"f","paths":["b.go"]},
|
||||
{"kind":"review","summary":"r"},
|
||||
{"kind":"manual","summary":"m"}]}],
|
||||
"unresolved":["integration run"]}`)
|
||||
got, err := ParseCompletionReport(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Status != CompletionComplete || got.Summary != "fixed the loop" {
|
||||
t.Fatalf("status/summary = %q / %q", got.Status, got.Summary)
|
||||
}
|
||||
c := got.Criteria[0]
|
||||
if c.ID != "c1" || c.Status != CriterionSatisfied || c.Evidence[0].Kind != CompletionEvidenceVerification {
|
||||
t.Fatalf("criterion = %+v", c)
|
||||
}
|
||||
if len(got.Unresolved) != 1 {
|
||||
t.Fatalf("unresolved = %v", got.Unresolved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCompletionReportRefusesWhatIsMalformedAndSaysWhere(t *testing.T) {
|
||||
for name, tc := range map[string]struct{ raw, want string }{
|
||||
"not json": {`{`, "invalid complete_subtask JSON"},
|
||||
"unknown status": {`{"status":"done","summary":"x"}`, "status must be"},
|
||||
"missing status": {`{"summary":"x"}`, "status must be"},
|
||||
"blank summary": {`{"status":"complete","summary":" "}`, "summary is required"},
|
||||
"criterion without id": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":" ","status":"satisfied"}]}`, "acceptance_criteria[0].id"},
|
||||
"criterion with odd status": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"maybe"}]}`, "acceptance_criteria[0].status"},
|
||||
"verification with no cmd": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"verification","summary":"s"}]}]}`, "verification requires command"},
|
||||
"diff with no paths": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"diff","summary":"s"}]}]}`, "diff requires paths"},
|
||||
"files with no paths": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"files","summary":"s"}]}]}`, "files requires paths"},
|
||||
"unknown evidence kind": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"vibes","summary":"s"}]}]}`, "not a known evidence kind"},
|
||||
"second criterion is broken": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied"},{"id":"","status":"satisfied"}]}`, "acceptance_criteria[1].id"},
|
||||
} {
|
||||
_, err := ParseCompletionReport(json.RawMessage(tc.raw))
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Errorf("%s: err = %v, want it to mention %q", name, err, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -92,7 +92,6 @@ type Ledger struct {
|
||||
mu sync.Mutex
|
||||
receipts []Receipt
|
||||
backgroundLeases []BackgroundLease
|
||||
closure ClosureTally
|
||||
}
|
||||
|
||||
func NewLedger() *Ledger { return &Ledger{} }
|
||||
@@ -106,7 +105,6 @@ func (l *Ledger) Reset() {
|
||||
defer l.mu.Unlock()
|
||||
l.receipts = nil
|
||||
l.backgroundLeases = nil
|
||||
l.closure = ClosureTally{}
|
||||
}
|
||||
|
||||
// Record appends a receipt. Failed receipts are retained for auditability but
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQuoteShortCutsOnRuneBoundary(t *testing.T) {
|
||||
// 80 bytes into this command falls inside a three-byte character, and
|
||||
// strconv.Quote would render the split bytes as \x escapes.
|
||||
cmd := `echo "` + strings.Repeat("测试", 20) + `"`
|
||||
got := quoteShort(cmd)
|
||||
if strings.Contains(got, `\x`) {
|
||||
t.Fatalf("quoteShort split a character: %s", got)
|
||||
}
|
||||
unquoted, err := strconv.Unquote(got)
|
||||
if err != nil {
|
||||
t.Fatalf("quoteShort result does not unquote: %v", err)
|
||||
}
|
||||
if !strings.HasSuffix(unquoted, "…") || !strings.HasPrefix(cmd, strings.TrimSuffix(unquoted, "…")) {
|
||||
t.Fatalf("quoteShort = %s, want a prefix of the command ending in …", got)
|
||||
}
|
||||
if len(strings.TrimSuffix(unquoted, "…")) > 80 {
|
||||
t.Fatalf("quoteShort kept more than 80 bytes: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuoteShortKeepsShortASCII(t *testing.T) {
|
||||
if got := quoteShort(" go test ./... "); got != `"go test ./..."` {
|
||||
t.Fatalf("quoteShort = %s", got)
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package trajectory
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/base/testenv"
|
||||
"reasonix/internal/contract/event"
|
||||
)
|
||||
|
||||
// The bug this guards: SubagentHandoffAudit had a producer, a struct, an
|
||||
// interface and a green unit test, and still reached nothing — the type
|
||||
// assertion inside event.RecordSubagentHandoff matched no sink in the chain.
|
||||
// Conformance proves a method exists; only a stack proves a value arrives.
|
||||
func TestSubagentHandoffSurvivesTheSessionSinkStack(t *testing.T) {
|
||||
path := filepath.Join(testenv.TempDir(t), "run.trajectory.jsonl")
|
||||
r, err := New(event.Discard, path, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
// boot's assembly order: Coalesce -> Sync -> CostQuote -> recorder.
|
||||
chain := event.Coalesce(event.Sync(event.NewCostQuoteSink(r, nil)), time.Millisecond)
|
||||
|
||||
event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{
|
||||
Entrance: "read_only_task", Depth: 1, ReadOnly: true, Expected: true,
|
||||
Exit: "completed", Attempts: 2, Accepted: 1, Malformed: 1,
|
||||
ReportRound: 3, FinalRound: 4, ToolCallsAfterReport: 2,
|
||||
ClaimedStatus: "complete", AdjudicatedStatus: "partial",
|
||||
LoweredClaims: 1, Criteria: 3, Evidence: 4, Unresolved: 1,
|
||||
})
|
||||
if err := r.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
|
||||
recs := readRecords(t, path)
|
||||
if len(recs) != 1 || recs[0].SubagentHandoff == nil {
|
||||
t.Fatalf("handoff never reached the recorder: %+v", recs)
|
||||
}
|
||||
got := *recs[0].SubagentHandoff
|
||||
want := SubagentHandoff{
|
||||
Entrance: "read_only_task", Depth: 1, ReadOnly: true, Expected: true,
|
||||
Exit: "completed", Attempts: 2, Accepted: 1, Malformed: 1,
|
||||
ReportRound: 3, FinalRound: 4, ToolCallsAfterReport: 2,
|
||||
ClaimedStatus: "complete", AdjudicatedStatus: "partial",
|
||||
LoweredClaims: 1, Criteria: 3, Evidence: 4, Unresolved: 1,
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("handoff record = %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -32,12 +32,10 @@ type Record struct {
|
||||
RecoveryChild string `json:"protocol_recovery_child,omitempty"`
|
||||
TurnCompletion bool `json:"turn_completion,omitempty"`
|
||||
ContractShadow *ContractShadowAudit `json:"contract_shadow,omitempty"`
|
||||
CompletionReport *CompletionReport `json:"completion_report,omitempty"`
|
||||
EvidenceBundle *EvidenceBundle `json:"evidence_bundle,omitempty"`
|
||||
OutcomeProgress *OutcomeProgress `json:"outcome_progress,omitempty"`
|
||||
MemoryRecall *MemoryRecall `json:"memory_recall,omitempty"`
|
||||
ProjectCheck *ProjectCheckProbe `json:"project_check_probe,omitempty"`
|
||||
SubagentHandoff *SubagentHandoff `json:"subagent_handoff,omitempty"`
|
||||
// Deltas counts the streamed increments merged into this record; absent
|
||||
// means one. TS stays the first increment's — that is the observation
|
||||
// time-to-first-token readers key on — and EndTS carries the last's.
|
||||
@@ -127,31 +125,6 @@ type ContractShadowAudit struct {
|
||||
ReadyToFinalize bool `json:"ready_to_finalize,omitempty"`
|
||||
}
|
||||
|
||||
// SubagentHandoff mirrors event.SubagentHandoffAudit: how one delegated child
|
||||
// closed. Every field travels, because which cut explains a refusal — entrance,
|
||||
// depth, read-only, how long the run was — is what the aggregate cannot say.
|
||||
type SubagentHandoff struct {
|
||||
Entrance string `json:"entrance,omitempty"`
|
||||
Depth int `json:"depth,omitempty"`
|
||||
ReadOnly bool `json:"read_only,omitempty"`
|
||||
Expected bool `json:"expected,omitempty"`
|
||||
Exit string `json:"exit,omitempty"`
|
||||
Attempts int `json:"attempts,omitempty"`
|
||||
Accepted int `json:"accepted,omitempty"`
|
||||
Malformed int `json:"malformed,omitempty"`
|
||||
Closed int `json:"closed,omitempty"`
|
||||
NeedsWork int `json:"needs_work,omitempty"`
|
||||
ReportRound int `json:"report_round,omitempty"`
|
||||
FinalRound int `json:"final_round,omitempty"`
|
||||
ToolCallsAfterReport int `json:"tool_calls_after_report,omitempty"`
|
||||
ClaimedStatus string `json:"claimed_status,omitempty"`
|
||||
AdjudicatedStatus string `json:"adjudicated_status,omitempty"`
|
||||
LoweredClaims int `json:"lowered_claims,omitempty"`
|
||||
Criteria int `json:"criteria,omitempty"`
|
||||
Evidence int `json:"evidence,omitempty"`
|
||||
Unresolved int `json:"unresolved,omitempty"`
|
||||
}
|
||||
|
||||
// ProjectCheckProbe mirrors event.ProjectCheckProbe: the shadow comparison
|
||||
// between the readiness gate's project-check derivation and the ledger's
|
||||
// obligations. Identities are recorded, not counted — a class is only
|
||||
@@ -173,26 +146,6 @@ type ProjectCheckDiff struct {
|
||||
Class string `json:"class"`
|
||||
}
|
||||
|
||||
// CompletionReport mirrors event.CompletionReportAudit with stable keys.
|
||||
type CompletionReport struct {
|
||||
Verdict string `json:"verdict"`
|
||||
Risk string `json:"risk,omitempty"`
|
||||
Criteria int `json:"criteria,omitempty"`
|
||||
CriteriaSatisfied int `json:"criteria_satisfied,omitempty"`
|
||||
Changes int `json:"changes,omitempty"`
|
||||
ChangesUnreviewed int `json:"changes_unreviewed,omitempty"`
|
||||
Verifications int `json:"verifications,omitempty"`
|
||||
VerificationsFailed int `json:"verifications_failed,omitempty"`
|
||||
VerificationsStale int `json:"verifications_stale,omitempty"`
|
||||
VerificationsInconclusive int `json:"verifications_inconclusive,omitempty"`
|
||||
Gaps int `json:"gaps,omitempty"`
|
||||
GapKinds []string `json:"gap_kinds,omitempty"`
|
||||
ClaimsVerified int `json:"claims_verified,omitempty"`
|
||||
ClaimsUnbacked int `json:"claims_unbacked,omitempty"`
|
||||
OriginRecorded int `json:"origin_recorded,omitempty"`
|
||||
OriginMatched int `json:"origin_matched,omitempty"`
|
||||
}
|
||||
|
||||
// EvidenceBundle mirrors event.EvidenceBundleAudit with stable keys.
|
||||
type EvidenceBundle struct {
|
||||
Sealed bool `json:"sealed"`
|
||||
@@ -357,31 +310,6 @@ func (r *Recorder) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
||||
event.RecordDelegationAudit(r.inner, a)
|
||||
}
|
||||
|
||||
func (r *Recorder) RecordSubagentHandoff(a event.SubagentHandoffAudit) {
|
||||
r.append(Record{SubagentHandoff: &SubagentHandoff{
|
||||
Entrance: a.Entrance,
|
||||
Depth: a.Depth,
|
||||
ReadOnly: a.ReadOnly,
|
||||
Expected: a.Expected,
|
||||
Exit: a.Exit,
|
||||
Attempts: a.Attempts,
|
||||
Accepted: a.Accepted,
|
||||
Malformed: a.Malformed,
|
||||
Closed: a.Closed,
|
||||
NeedsWork: a.NeedsWork,
|
||||
ReportRound: a.ReportRound,
|
||||
FinalRound: a.FinalRound,
|
||||
ToolCallsAfterReport: a.ToolCallsAfterReport,
|
||||
ClaimedStatus: a.ClaimedStatus,
|
||||
AdjudicatedStatus: a.AdjudicatedStatus,
|
||||
LoweredClaims: a.LoweredClaims,
|
||||
Criteria: a.Criteria,
|
||||
Evidence: a.Evidence,
|
||||
Unresolved: a.Unresolved,
|
||||
}})
|
||||
event.RecordSubagentHandoff(r.inner, a)
|
||||
}
|
||||
|
||||
func (r *Recorder) RecordMemoryRecall(a event.MemoryRecallAudit) {
|
||||
rec := &MemoryRecall{
|
||||
CharBudget: a.CharBudget, UsedChars: a.UsedChars, Omitted: a.Omitted,
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"doc-prose": 2128,
|
||||
"error-text": 9,
|
||||
"essay": 2576,
|
||||
"file-size": 18416,
|
||||
"file-size": 18415,
|
||||
"flat-view": 0,
|
||||
"frontend-parity": 13,
|
||||
"function-size": 2485,
|
||||
@@ -1080,7 +1080,7 @@
|
||||
"internal/runtime/delegation/task.go": {
|
||||
"complexity": 8,
|
||||
"essay": 9,
|
||||
"file-size": 160,
|
||||
"file-size": 159,
|
||||
"function-size": 55,
|
||||
"orphan": 1
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user