mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 10:45:59 -10:00
refactor(delegation): sub-agents return prose, not a structured completion report
complete_subtask, the host adjudication of its criteria and the 'lowered by the host' note are gone, with the handoff audit channel and its metrics, trajectory field and benchmark columns. A delegated child ends the way any turn does and the parent reads its last reply. The host's own receipts appended to a child's reply and the write-scope check stay: they need a record of what the child changed, which is replaced with the evidence ledger later.
This commit is contained in:
1 parent
5766a8ebeb
commit
0bb0f44785
35 files changed
+33
-1544
No files matched your search
@@ -7,7 +7,7 @@ import "fmt"
|
|||||||
// arm costs more: did the extra agents produce verified work, or just tokens.
|
// arm costs more: did the extra agents produce verified work, or just tokens.
|
||||||
func renderDelegation(results []result) string {
|
func renderDelegation(results []result) string {
|
||||||
var runs, nested, childCalls, parentCalls, mutations, dupes int
|
var runs, nested, childCalls, parentCalls, mutations, dupes int
|
||||||
var reports, prose, falseDone, downgrades, violations int
|
var violations int
|
||||||
var scopeHints, namedFiles, evidencePaths, discoveredPaths int
|
var scopeHints, namedFiles, evidencePaths, discoveredPaths int
|
||||||
solved, total, childTokens := 0, 0, 0
|
solved, total, childTokens := 0, 0, 0
|
||||||
for _, r := range results {
|
for _, r := range results {
|
||||||
@@ -24,10 +24,6 @@ func renderDelegation(results []result) string {
|
|||||||
parentCalls += r.ToolCalls - r.SubagentToolCalls
|
parentCalls += r.ToolCalls - r.SubagentToolCalls
|
||||||
mutations += r.SubagentMutations
|
mutations += r.SubagentMutations
|
||||||
dupes += r.DuplicateWorkPaths
|
dupes += r.DuplicateWorkPaths
|
||||||
reports += r.CompletionReports
|
|
||||||
prose += r.CompletionsProsedOnly
|
|
||||||
falseDone += r.FalseCompletions
|
|
||||||
downgrades += r.CriterionDowngrades
|
|
||||||
violations += r.WriteScopeViolations
|
violations += r.WriteScopeViolations
|
||||||
scopeHints += r.ParentScopeHints
|
scopeHints += r.ParentScopeHints
|
||||||
namedFiles += r.ParentNamedFiles
|
namedFiles += r.ParentNamedFiles
|
||||||
@@ -70,11 +66,6 @@ func renderDelegation(results []result) string {
|
|||||||
if dupes > 0 {
|
if dupes > 0 {
|
||||||
b += fmt.Sprintf("- **duplicate work**: %d file(s) mutated by more than one child\n", dupes)
|
b += fmt.Sprintf("- **duplicate work**: %d file(s) mutated by more than one child\n", dupes)
|
||||||
}
|
}
|
||||||
closed := reports + prose
|
|
||||||
b += fmt.Sprintf("- closed with a checkable claim: **%d/%d** (%s)\n", reports, closed, pct(reports, closed))
|
|
||||||
if falseDone > 0 {
|
|
||||||
b += fmt.Sprintf("- **false completions**: %d run(s), %d criterion claim(s) the host refused\n", falseDone, downgrades)
|
|
||||||
}
|
|
||||||
if violations > 0 {
|
if violations > 0 {
|
||||||
b += fmt.Sprintf("- **write-scope violations**: %d\n", violations)
|
b += fmt.Sprintf("- **write-scope violations**: %d\n", violations)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,9 +27,7 @@ func TestRenderDelegationExposesWhatDelegationCost(t *testing.T) {
|
|||||||
runMetrics: runMetrics{
|
runMetrics: runMetrics{
|
||||||
ToolCalls: 30, SubagentToolCalls: 24,
|
ToolCalls: 30, SubagentToolCalls: 24,
|
||||||
SubagentRuns: 3, SubagentNestedRuns: 1, SubagentMutations: 5,
|
SubagentRuns: 3, SubagentNestedRuns: 1, SubagentMutations: 5,
|
||||||
DuplicateWorkPaths: 2,
|
DuplicateWorkPaths: 2,
|
||||||
CompletionReports: 2, CompletionsProsedOnly: 1,
|
|
||||||
FalseCompletions: 1, CriterionDowngrades: 3,
|
|
||||||
WriteScopeViolations: 1,
|
WriteScopeViolations: 1,
|
||||||
},
|
},
|
||||||
}})
|
}})
|
||||||
@@ -38,8 +36,6 @@ func TestRenderDelegationExposesWhatDelegationCost(t *testing.T) {
|
|||||||
"parent **6** tool calls",
|
"parent **6** tool calls",
|
||||||
"children **24**",
|
"children **24**",
|
||||||
"duplicate work**: 2",
|
"duplicate work**: 2",
|
||||||
"checkable claim: **2/3**",
|
|
||||||
"false completions**: 1 run(s), 3 criterion",
|
|
||||||
"write-scope violations**: 1",
|
"write-scope violations**: 1",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(got, want) {
|
if !strings.Contains(got, want) {
|
||||||
@@ -92,14 +88,11 @@ func TestRenderDelegationSaysWhenEvidenceOriginIsUnscored(t *testing.T) {
|
|||||||
func TestRenderDelegationStaysQuietWhenNothingWentWrong(t *testing.T) {
|
func TestRenderDelegationStaysQuietWhenNothingWentWrong(t *testing.T) {
|
||||||
got := renderDelegation([]result{{
|
got := renderDelegation([]result{{
|
||||||
Passed: true,
|
Passed: true,
|
||||||
runMetrics: runMetrics{ToolCalls: 12, SubagentToolCalls: 8, SubagentRuns: 2, CompletionReports: 2},
|
runMetrics: runMetrics{ToolCalls: 12, SubagentToolCalls: 8, SubagentRuns: 2},
|
||||||
}})
|
}})
|
||||||
for _, unwanted := range []string{"duplicate work", "false completions", "write-scope violations"} {
|
for _, unwanted := range []string{"duplicate work", "false completions", "write-scope violations"} {
|
||||||
if strings.Contains(got, unwanted) {
|
if strings.Contains(got, unwanted) {
|
||||||
t.Errorf("clean run reported %q:\n%s", unwanted, got)
|
t.Errorf("clean run reported %q:\n%s", unwanted, got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !strings.Contains(got, "checkable claim: **2/2**") {
|
|
||||||
t.Errorf("clean run lost its completion coverage:\n%s", got)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -66,15 +66,11 @@ type runMetrics struct {
|
|||||||
|
|
||||||
// Delegation counters mirror internal/frontend/cli.RunMetrics. They are what makes a
|
// Delegation counters mirror internal/frontend/cli.RunMetrics. They are what makes a
|
||||||
// single-agent arm comparable against a delegated one for the same model.
|
// single-agent arm comparable against a delegated one for the same model.
|
||||||
SubagentRuns int `json:"subagent_runs,omitempty"`
|
SubagentRuns int `json:"subagent_runs,omitempty"`
|
||||||
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
|
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
|
||||||
SubagentMutations int `json:"subagent_mutations,omitempty"`
|
SubagentMutations int `json:"subagent_mutations,omitempty"`
|
||||||
CompletionReports int `json:"completion_reports,omitempty"`
|
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
|
||||||
CompletionsProsedOnly int `json:"completions_prose_only,omitempty"`
|
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
|
||||||
FalseCompletions int `json:"false_completions,omitempty"`
|
|
||||||
CriterionDowngrades int `json:"criterion_downgrades,omitempty"`
|
|
||||||
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
|
|
||||||
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
|
|
||||||
// Evidence origin: what the parent's own delegation text scoped and named,
|
// Evidence origin: what the parent's own delegation text scoped and named,
|
||||||
// and how much of what the children looked at they had to find themselves.
|
// and how much of what the children looked at they had to find themselves.
|
||||||
ParentScopeHints int `json:"parent_scope_hints,omitempty"`
|
ParentScopeHints int `json:"parent_scope_hints,omitempty"`
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ func TestAuditCapabilitiesAreForwardedByEveryWrapper(t *testing.T) {
|
|||||||
"DelegationAuditSink": reflect.TypeFor[event.DelegationAuditSink](),
|
"DelegationAuditSink": reflect.TypeFor[event.DelegationAuditSink](),
|
||||||
"MemoryRecallSink": reflect.TypeFor[event.MemoryRecallSink](),
|
"MemoryRecallSink": reflect.TypeFor[event.MemoryRecallSink](),
|
||||||
"TurnCompletionSink": reflect.TypeFor[event.TurnCompletionSink](),
|
"TurnCompletionSink": reflect.TypeFor[event.TurnCompletionSink](),
|
||||||
"SubagentHandoffSink": reflect.TypeFor[event.SubagentHandoffSink](),
|
|
||||||
}
|
}
|
||||||
wrappers := []event.Sink{
|
wrappers := []event.Sink{
|
||||||
event.Sync(event.Discard),
|
event.Sync(event.Discard),
|
||||||
|
|||||||
@@ -193,7 +193,7 @@ model = "x"
|
|||||||
if !strings.HasPrefix(systemMessage(req.Messages), profileBody) {
|
if !strings.HasPrefix(systemMessage(req.Messages), profileBody) {
|
||||||
t.Fatal("installed profile body did not reach the child")
|
t.Fatal("installed profile body did not reach the child")
|
||||||
}
|
}
|
||||||
if !slices.Equal(toolSchemaNames(req.Tools), []string{"complete_subtask", "glob", "grep", "ls", "read_file"}) {
|
if !slices.Equal(toolSchemaNames(req.Tools), []string{"glob", "grep", "ls", "read_file"}) {
|
||||||
t.Fatalf("child tool ceiling = %v", toolSchemaNames(req.Tools))
|
t.Fatalf("child tool ceiling = %v", toolSchemaNames(req.Tools))
|
||||||
}
|
}
|
||||||
var user string
|
var user string
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ func TestEffectConditionalDeclarationsDoNotReachTheProviderPrefix(t *testing.T)
|
|||||||
for _, schema := range req.Tools {
|
for _, schema := range req.Tools {
|
||||||
sent[schema.Name] = true
|
sent[schema.Name] = true
|
||||||
}
|
}
|
||||||
for _, n := range []string{"advise", "best_of_n", "apply_isolated", "discard_isolated", "complete_subtask", "system_one", "submit_plan", "conclude_no_changes", "review_report"} {
|
for _, n := range []string{"advise", "best_of_n", "apply_isolated", "discard_isolated", "system_one", "submit_plan", "conclude_no_changes", "review_report"} {
|
||||||
if sent[n] {
|
if sent[n] {
|
||||||
t.Errorf("%s is declared conditional but reached the default request tools", n)
|
t.Errorf("%s is declared conditional but reached the default request tools", n)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,10 +31,6 @@ func (f AuditForwarder) RecordRunBudget(s RunBudgetSample) {
|
|||||||
RecordRunBudget(f.Inner, s)
|
RecordRunBudget(f.Inner, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f AuditForwarder) RecordSubagentHandoff(a SubagentHandoffAudit) {
|
|
||||||
RecordSubagentHandoff(f.Inner, a)
|
|
||||||
}
|
|
||||||
|
|
||||||
// DelegationAuditSink receives one receipt per completed sub-agent run.
|
// DelegationAuditSink receives one receipt per completed sub-agent run.
|
||||||
type DelegationAuditSink interface {
|
type DelegationAuditSink interface {
|
||||||
RecordDelegationAudit(a hostaudit.DelegationAudit)
|
RecordDelegationAudit(a hostaudit.DelegationAudit)
|
||||||
|
|||||||
@@ -185,13 +185,6 @@ func (c *coalescer) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
|||||||
RecordDelegationAudit(c.inner, a)
|
RecordDelegationAudit(c.inner, a)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *coalescer) RecordSubagentHandoff(a SubagentHandoffAudit) {
|
|
||||||
c.mu.Lock()
|
|
||||||
c.enqueueFlushLocked()
|
|
||||||
c.drainAndUnlock()
|
|
||||||
RecordSubagentHandoff(c.inner, a)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *coalescer) RecordTurnCompletion() {
|
func (c *coalescer) RecordTurnCompletion() {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
c.enqueueFlushLocked()
|
c.enqueueFlushLocked()
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ var capabilityContracts = map[string]reflect.Type{
|
|||||||
"DelegationAudit": reflect.TypeFor[DelegationAuditSink](),
|
"DelegationAudit": reflect.TypeFor[DelegationAuditSink](),
|
||||||
"MemoryRecall": reflect.TypeFor[MemoryRecallSink](),
|
"MemoryRecall": reflect.TypeFor[MemoryRecallSink](),
|
||||||
"RunBudget": reflect.TypeFor[RunBudgetSink](),
|
"RunBudget": reflect.TypeFor[RunBudgetSink](),
|
||||||
"SubagentHandoff": reflect.TypeFor[SubagentHandoffSink](),
|
|
||||||
"TurnCompletion": reflect.TypeFor[TurnCompletionSink](),
|
"TurnCompletion": reflect.TypeFor[TurnCompletionSink](),
|
||||||
"UsageQuote": reflect.TypeFor[UsageQuoter](),
|
"UsageQuote": reflect.TypeFor[UsageQuoter](),
|
||||||
"WorkspaceMutation": reflect.TypeFor[WorkspaceMutationSink](),
|
"WorkspaceMutation": reflect.TypeFor[WorkspaceMutationSink](),
|
||||||
|
|||||||
@@ -37,9 +37,6 @@ func (s *capturingSink) RecordWorkspaceMutation(WorkspaceMutation) {
|
|||||||
func (s *capturingSink) RecordRunBudget(RunBudgetSample) {
|
func (s *capturingSink) RecordRunBudget(RunBudgetSample) {
|
||||||
s.recorded = append(s.recorded, "RunBudget")
|
s.recorded = append(s.recorded, "RunBudget")
|
||||||
}
|
}
|
||||||
func (s *capturingSink) RecordSubagentHandoff(SubagentHandoffAudit) {
|
|
||||||
s.recorded = append(s.recorded, "SubagentHandoff")
|
|
||||||
}
|
|
||||||
|
|
||||||
// AuditForwarder is why a new capability does not have to be repeated at every
|
// AuditForwarder is why a new capability does not have to be repeated at every
|
||||||
// layer, so it must itself be complete. It reads the registry rather than a
|
// layer, so it must itself be complete. It reads the registry rather than a
|
||||||
|
|||||||
@@ -1,69 +0,0 @@
|
|||||||
// subagent_handoff.go — how a delegated child closed, counted.
|
|
||||||
package event
|
|
||||||
|
|
||||||
import "reasonix/internal/base/nilutil"
|
|
||||||
|
|
||||||
// SubagentHandoffAudit is one child run's closing protocol, counted. It exists
|
|
||||||
// to answer whether a typed handoff is submitted naturally, before anything is
|
|
||||||
// built to insist on one — so it observes and never decides. Content-free:
|
|
||||||
// counts, enums and rounds, never a prompt, a path or a summary.
|
|
||||||
type SubagentHandoffAudit struct {
|
|
||||||
// Entrance names the delegation path (read_only_task, fleet, …) and Depth
|
|
||||||
// how deep the child sits. An aggregate hides an entrance that is failing.
|
|
||||||
Entrance string
|
|
||||||
Depth int
|
|
||||||
ReadOnly bool
|
|
||||||
// Expected is set where the host actually appended the contract, never
|
|
||||||
// inferred from a registry: the denominator has to be the host's own
|
|
||||||
// instruction, not a guess about one.
|
|
||||||
Expected bool
|
|
||||||
// Exit says how the run ended. A provider timeout that produced no report
|
|
||||||
// is not a compliance failure and must leave the denominator.
|
|
||||||
Exit string // completed | error | cancelled | no_answer
|
|
||||||
|
|
||||||
// Attempts counts complete_subtask calls, Accepted the ones the tool took,
|
|
||||||
// Malformed the ones it refused. Missing with attempts is a schema
|
|
||||||
// problem; missing without them is a protocol one.
|
|
||||||
Attempts int
|
|
||||||
Accepted int
|
|
||||||
Malformed int
|
|
||||||
|
|
||||||
// ReportRound and FinalRound say whether the report actually closed the
|
|
||||||
// run: the contract asks for it as the final call, and "submitted it
|
|
||||||
// eventually" is a different behaviour from "closed with it".
|
|
||||||
ReportRound int
|
|
||||||
FinalRound int
|
|
||||||
ToolCallsAfterReport int
|
|
||||||
|
|
||||||
// Closed and NeedsWork are the verdicts the host issued, counted when each
|
|
||||||
// was issued. Accepted only says a call was well-formed; these say whether
|
|
||||||
// the host considered the sub-task done.
|
|
||||||
Closed int
|
|
||||||
NeedsWork int
|
|
||||||
|
|
||||||
// What the host made of the claim, read from the adjudicated report rather
|
|
||||||
// than re-derived here.
|
|
||||||
ClaimedStatus string
|
|
||||||
AdjudicatedStatus string
|
|
||||||
LoweredClaims int
|
|
||||||
Criteria int
|
|
||||||
Evidence int
|
|
||||||
Unresolved int
|
|
||||||
}
|
|
||||||
|
|
||||||
// SubagentHandoffSink is an optional sink capability; implementations must keep
|
|
||||||
// it content-free, like every other audit channel.
|
|
||||||
type SubagentHandoffSink interface {
|
|
||||||
RecordSubagentHandoff(SubagentHandoffAudit)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RecordSubagentHandoff forwards one child's closing protocol only to sinks
|
|
||||||
// that explicitly opt in. Ordinary UI sinks receive nothing.
|
|
||||||
func RecordSubagentHandoff(s Sink, a SubagentHandoffAudit) {
|
|
||||||
if nilutil.IsNil(s) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if hs, ok := s.(SubagentHandoffSink); ok {
|
|
||||||
hs.RecordSubagentHandoff(a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -44,14 +44,6 @@ func (s *syncSink) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
|||||||
RecordDelegationAudit(s.inner, a)
|
RecordDelegationAudit(s.inner, a)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *syncSink) RecordSubagentHandoff(a SubagentHandoffAudit) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if hs, ok := s.inner.(SubagentHandoffSink); ok {
|
|
||||||
hs.RecordSubagentHandoff(a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *syncSink) RecordTurnCompletion() {
|
func (s *syncSink) RecordTurnCompletion() {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|||||||
@@ -12,12 +12,6 @@ type DelegationAudit struct {
|
|||||||
MutationPaths []string
|
MutationPaths []string
|
||||||
// ClaimViolations counts writes the host saw outside the declared claim.
|
// ClaimViolations counts writes the host saw outside the declared claim.
|
||||||
ClaimViolations int
|
ClaimViolations int
|
||||||
// HasReport is false when the child ended in prose instead of a typed claim.
|
|
||||||
HasReport bool
|
|
||||||
// AdjudicatedStatus is the status the host was willing to back, and
|
|
||||||
// Downgrades counts the criterion claims it refused.
|
|
||||||
AdjudicatedStatus string
|
|
||||||
Downgrades int
|
|
||||||
// ParentScopeHints counts directories the delegation narrowed the search to.
|
// ParentScopeHints counts directories the delegation narrowed the search to.
|
||||||
ParentScopeHints int
|
ParentScopeHints int
|
||||||
// ParentNamedFiles counts specific files the delegation text handed over.
|
// ParentNamedFiles counts specific files the delegation text handed over.
|
||||||
@@ -27,10 +21,3 @@ type DelegationAudit struct {
|
|||||||
// DiscoveredPaths is the subset of those no named file already covered.
|
// DiscoveredPaths is the subset of those no named file already covered.
|
||||||
DiscoveredPaths int
|
DiscoveredPaths int
|
||||||
}
|
}
|
||||||
|
|
||||||
// FalseCompletion reports a child that claimed more than the host could back.
|
|
||||||
// It is the count of refused criteria, not a comparison against a claimed
|
|
||||||
// status the host never independently recorded.
|
|
||||||
func (a DelegationAudit) FalseCompletion() bool {
|
|
||||||
return a.HasReport && a.Downgrades > 0
|
|
||||||
}
|
|
||||||
@@ -3,8 +3,6 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"reasonix/internal/contract/hostaudit"
|
"reasonix/internal/contract/hostaudit"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"reasonix/internal/safety/evidence"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The instrument must be able to say, from one run, whether delegation
|
// The instrument must be able to say, from one run, whether delegation
|
||||||
@@ -13,17 +11,12 @@ func TestDelegationMetricsAggregateAcrossChildren(t *testing.T) {
|
|||||||
s := &metricsSink{}
|
s := &metricsSink{}
|
||||||
s.RecordDelegationAudit(hostaudit.DelegationAudit{
|
s.RecordDelegationAudit(hostaudit.DelegationAudit{
|
||||||
Depth: 1, ToolCalls: 6, Mutations: 2,
|
Depth: 1, ToolCalls: 6, Mutations: 2,
|
||||||
MutationPaths: []string{"api/handler.go", "api/handler_test.go"},
|
MutationPaths: []string{"api/handler.go", "api/handler_test.go"},
|
||||||
HasReport: true,
|
|
||||||
AdjudicatedStatus: string(evidence.CompletionComplete),
|
|
||||||
})
|
})
|
||||||
s.RecordDelegationAudit(hostaudit.DelegationAudit{
|
s.RecordDelegationAudit(hostaudit.DelegationAudit{
|
||||||
Depth: 2, ToolCalls: 4, Mutations: 1,
|
Depth: 2, ToolCalls: 4, Mutations: 1,
|
||||||
MutationPaths: []string{"api/handler.go"},
|
MutationPaths: []string{"api/handler.go"},
|
||||||
ClaimViolations: 1,
|
ClaimViolations: 1,
|
||||||
HasReport: true,
|
|
||||||
AdjudicatedStatus: string(evidence.CompletionPartial),
|
|
||||||
Downgrades: 2,
|
|
||||||
})
|
})
|
||||||
s.RecordDelegationAudit(hostaudit.DelegationAudit{Depth: 1, ToolCalls: 3})
|
s.RecordDelegationAudit(hostaudit.DelegationAudit{Depth: 1, ToolCalls: 3})
|
||||||
|
|
||||||
@@ -34,14 +27,6 @@ func TestDelegationMetricsAggregateAcrossChildren(t *testing.T) {
|
|||||||
if m.SubagentMutations != 3 {
|
if m.SubagentMutations != 3 {
|
||||||
t.Fatalf("mutations = %d, want 3", m.SubagentMutations)
|
t.Fatalf("mutations = %d, want 3", m.SubagentMutations)
|
||||||
}
|
}
|
||||||
if m.CompletionReports != 2 || m.CompletionsProsedOnly != 1 {
|
|
||||||
t.Fatalf("reports = %d prose-only = %d, want 2/1", m.CompletionReports, m.CompletionsProsedOnly)
|
|
||||||
}
|
|
||||||
// One child claimed criteria the host refused: that is the false-completion
|
|
||||||
// signal an orchestration benchmark exists to surface.
|
|
||||||
if m.FalseCompletions != 1 || m.CriterionDowngrades != 2 {
|
|
||||||
t.Fatalf("false completions = %d downgrades = %d, want 1/2", m.FalseCompletions, m.CriterionDowngrades)
|
|
||||||
}
|
|
||||||
if m.WriteScopeViolations != 1 {
|
if m.WriteScopeViolations != 1 {
|
||||||
t.Fatalf("write scope violations = %d, want 1", m.WriteScopeViolations)
|
t.Fatalf("write scope violations = %d, want 1", m.WriteScopeViolations)
|
||||||
}
|
}
|
||||||
@@ -77,7 +62,7 @@ func TestDelegationMetricsSumEvidenceOriginForARatioOfTotals(t *testing.T) {
|
|||||||
// single-agent arm is a clean baseline rather than noise.
|
// single-agent arm is a clean baseline rather than noise.
|
||||||
func TestDelegationMetricsStayZeroForSingleAgentArm(t *testing.T) {
|
func TestDelegationMetricsStayZeroForSingleAgentArm(t *testing.T) {
|
||||||
s := &metricsSink{}
|
s := &metricsSink{}
|
||||||
if m := s.m; m.SubagentRuns != 0 || m.CompletionReports != 0 || m.DuplicateWorkPaths != 0 {
|
if m := s.m; m.SubagentRuns != 0 || m.DuplicateWorkPaths != 0 {
|
||||||
t.Fatalf("single-agent baseline is not zero: %+v", m)
|
t.Fatalf("single-agent baseline is not zero: %+v", m)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -75,45 +75,19 @@ type RunMetrics struct {
|
|||||||
// Delegation counters let one model be compared across orchestration arms
|
// Delegation counters let one model be compared across orchestration arms
|
||||||
// without scraping prose. Child tool calls are already split out as
|
// without scraping prose. Child tool calls are already split out as
|
||||||
// SubagentToolCalls below; parent calls are ToolCalls minus that.
|
// SubagentToolCalls below; parent calls are ToolCalls minus that.
|
||||||
SubagentRuns int `json:"subagent_runs,omitempty"`
|
SubagentRuns int `json:"subagent_runs,omitempty"`
|
||||||
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
|
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
|
||||||
SubagentMutations int `json:"subagent_mutations,omitempty"`
|
SubagentMutations int `json:"subagent_mutations,omitempty"`
|
||||||
CompletionReports int `json:"completion_reports,omitempty"`
|
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
|
||||||
CompletionsProsedOnly int `json:"completions_prose_only,omitempty"`
|
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
|
||||||
FalseCompletions int `json:"false_completions,omitempty"`
|
ParentScopeHints int `json:"parent_scope_hints,omitempty"`
|
||||||
CriterionDowngrades int `json:"criterion_downgrades,omitempty"`
|
ParentNamedFiles int `json:"parent_named_files,omitempty"`
|
||||||
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
|
ChildEvidencePaths int `json:"child_evidence_paths,omitempty"`
|
||||||
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
|
ChildDiscoveredPaths int `json:"child_discovered_paths,omitempty"`
|
||||||
ParentScopeHints int `json:"parent_scope_hints,omitempty"`
|
|
||||||
ParentNamedFiles int `json:"parent_named_files,omitempty"`
|
|
||||||
ChildEvidencePaths int `json:"child_evidence_paths,omitempty"`
|
|
||||||
ChildDiscoveredPaths int `json:"child_discovered_paths,omitempty"`
|
|
||||||
// Subagent handoff: whether a delegated child closed the way it was asked
|
|
||||||
// to. Judged is the denominator — expected to report and not killed by the
|
|
||||||
// provider — because a run that never got to close is not a refusal.
|
|
||||||
SubagentHandoffs int `json:"subagent_handoffs"`
|
|
||||||
SubagentHandoffExpected int `json:"subagent_handoff_expected"`
|
|
||||||
SubagentHandoffJudged int `json:"subagent_handoff_judged"`
|
|
||||||
SubagentHandoffAttempted int `json:"subagent_handoff_attempted"`
|
|
||||||
SubagentHandoffAccepted int `json:"subagent_handoff_accepted"`
|
|
||||||
SubagentHandoffMalformed int `json:"subagent_handoff_malformed"`
|
|
||||||
SubagentHandoffClosed int `json:"subagent_handoff_closed"`
|
|
||||||
SubagentHandoffNeedsWork int `json:"subagent_handoff_needs_work"`
|
|
||||||
SubagentHandoffNotTried int `json:"subagent_handoff_never_attempted"`
|
|
||||||
SubagentHandoffClosedWith int `json:"subagent_handoff_closed_with_report"`
|
|
||||||
SubagentHandoffToolsAfter int `json:"subagent_handoff_tool_calls_after_report"`
|
|
||||||
SubagentHandoffLowered int `json:"subagent_handoff_lowered_claims"`
|
|
||||||
SubagentHandoffReadOnly int `json:"subagent_handoff_read_only"`
|
|
||||||
SubagentHandoffExits map[string]int `json:"subagent_handoff_exits,omitempty"`
|
|
||||||
// Repairs that happened inside a delegated run rather than the parent's own
|
// Repairs that happened inside a delegated run rather than the parent's own
|
||||||
// loop, so a child that continued only because the host pulled it back is
|
// loop, so a child that continued only because the host pulled it back is
|
||||||
// not read as one that continued on its own.
|
// not read as one that continued on its own.
|
||||||
SubagentRecoveries int `json:"subagent_recoveries"`
|
SubagentRecoveries int `json:"subagent_recoveries"`
|
||||||
// Goal resumes that carried a frozen verification contract, and how many of
|
|
||||||
// those found the current declaration naming different criteria. Neither
|
|
||||||
// says which declaration should govern.
|
|
||||||
GoalResumesWithContract int `json:"goal_resumes_with_verification_contract"`
|
|
||||||
GoalVerificationDrifts int `json:"goal_verification_contract_drifts"`
|
|
||||||
// Fanout times what the delegation counters only count: what the run waited
|
// Fanout times what the delegation counters only count: what the run waited
|
||||||
// on its fan-outs, against what the same work costs one at a time.
|
// on its fan-outs, against what the same work costs one at a time.
|
||||||
Fanout *FanoutMetrics `json:"fanout,omitempty"`
|
Fanout *FanoutMetrics `json:"fanout,omitempty"`
|
||||||
@@ -459,21 +433,12 @@ func (s *metricsSink) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
|||||||
}
|
}
|
||||||
s.m.SubagentMutations += a.Mutations
|
s.m.SubagentMutations += a.Mutations
|
||||||
s.m.WriteScopeViolations += a.ClaimViolations
|
s.m.WriteScopeViolations += a.ClaimViolations
|
||||||
s.m.CriterionDowngrades += a.Downgrades
|
|
||||||
// Summed, never averaged: an independence rate is a ratio of these totals,
|
// Summed, never averaged: an independence rate is a ratio of these totals,
|
||||||
// so a child that read two files cannot outweigh one that read forty.
|
// so a child that read two files cannot outweigh one that read forty.
|
||||||
s.m.ParentScopeHints += a.ParentScopeHints
|
s.m.ParentScopeHints += a.ParentScopeHints
|
||||||
s.m.ParentNamedFiles += a.ParentNamedFiles
|
s.m.ParentNamedFiles += a.ParentNamedFiles
|
||||||
s.m.ChildEvidencePaths += a.EvidencePaths
|
s.m.ChildEvidencePaths += a.EvidencePaths
|
||||||
s.m.ChildDiscoveredPaths += a.DiscoveredPaths
|
s.m.ChildDiscoveredPaths += a.DiscoveredPaths
|
||||||
if a.HasReport {
|
|
||||||
s.m.CompletionReports++
|
|
||||||
} else {
|
|
||||||
s.m.CompletionsProsedOnly++
|
|
||||||
}
|
|
||||||
if a.FalseCompletion() {
|
|
||||||
s.m.FalseCompletions++
|
|
||||||
}
|
|
||||||
if s.childMutations == nil {
|
if s.childMutations == nil {
|
||||||
s.childMutations = map[string]int{}
|
s.childMutations = map[string]int{}
|
||||||
}
|
}
|
||||||
@@ -485,52 +450,6 @@ func (s *metricsSink) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *metricsSink) RecordSubagentHandoff(a event.SubagentHandoffAudit) {
|
|
||||||
if s == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer event.RecordSubagentHandoff(s.inner, a)
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
s.m.SubagentHandoffs++
|
|
||||||
if a.Exit != "" {
|
|
||||||
if s.m.SubagentHandoffExits == nil {
|
|
||||||
s.m.SubagentHandoffExits = map[string]int{}
|
|
||||||
}
|
|
||||||
s.m.SubagentHandoffExits[a.Exit]++
|
|
||||||
}
|
|
||||||
if a.Expected {
|
|
||||||
s.m.SubagentHandoffExpected++
|
|
||||||
}
|
|
||||||
// A run the provider killed never reached the point of closing, so it
|
|
||||||
// leaves the compliance denominator instead of counting as a refusal.
|
|
||||||
if !a.Expected || a.Exit != "completed" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.m.SubagentHandoffJudged++
|
|
||||||
if a.ReadOnly {
|
|
||||||
s.m.SubagentHandoffReadOnly++
|
|
||||||
}
|
|
||||||
if a.Attempts == 0 {
|
|
||||||
s.m.SubagentHandoffNotTried++
|
|
||||||
} else {
|
|
||||||
s.m.SubagentHandoffAttempted++
|
|
||||||
}
|
|
||||||
if a.Accepted > 0 {
|
|
||||||
s.m.SubagentHandoffAccepted++
|
|
||||||
}
|
|
||||||
if a.Malformed > 0 {
|
|
||||||
s.m.SubagentHandoffMalformed++
|
|
||||||
}
|
|
||||||
if a.ReportRound > 0 && a.ToolCallsAfterReport == 0 {
|
|
||||||
s.m.SubagentHandoffClosedWith++
|
|
||||||
}
|
|
||||||
s.m.SubagentHandoffClosed += a.Closed
|
|
||||||
s.m.SubagentHandoffNeedsWork += a.NeedsWork
|
|
||||||
s.m.SubagentHandoffToolsAfter += a.ToolCallsAfterReport
|
|
||||||
s.m.SubagentHandoffLowered += a.LoweredClaims
|
|
||||||
}
|
|
||||||
|
|
||||||
// MergeCapabilityAuditCounters copies capability counters into RunMetrics.
|
// MergeCapabilityAuditCounters copies capability counters into RunMetrics.
|
||||||
func (m *RunMetrics) MergeCapabilityAuditCounters(
|
func (m *RunMetrics) MergeCapabilityAuditCounters(
|
||||||
routes, routedCandidates, routedRequire, routedPrefer, routedSuggest, declines int,
|
routes, routedCandidates, routedRequire, routedPrefer, routedSuggest, declines int,
|
||||||
|
|||||||
@@ -1,45 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"reasonix/internal/contract/event"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The compliance denominator is the point of this test: a child the provider
|
|
||||||
// killed never reached the point of closing, so it must leave the denominator
|
|
||||||
// rather than count as a refusal. Routing is asserted too — the audit is sent
|
|
||||||
// through a wrapper, not called on the sink directly, because a method that
|
|
||||||
// exists and is never reached is the bug this channel already had once.
|
|
||||||
func TestMetricsSinkCountsSubagentHandoffCompliance(t *testing.T) {
|
|
||||||
s := &metricsSink{inner: event.Discard}
|
|
||||||
chain := event.Sync(s)
|
|
||||||
|
|
||||||
event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{
|
|
||||||
Expected: true, Exit: "completed", ReadOnly: true,
|
|
||||||
})
|
|
||||||
event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{
|
|
||||||
Expected: true, Exit: "cancelled",
|
|
||||||
})
|
|
||||||
event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{
|
|
||||||
Expected: true, Exit: "completed",
|
|
||||||
Attempts: 1, Accepted: 1, ReportRound: 2, FinalRound: 2, LoweredClaims: 3,
|
|
||||||
})
|
|
||||||
|
|
||||||
m := s.m
|
|
||||||
if m.SubagentHandoffs != 3 || m.SubagentHandoffExpected != 3 || m.SubagentHandoffJudged != 2 {
|
|
||||||
t.Fatalf("totals = %d/%d/%d, want 3 observed, 3 expected, 2 judged",
|
|
||||||
m.SubagentHandoffs, m.SubagentHandoffExpected, m.SubagentHandoffJudged)
|
|
||||||
}
|
|
||||||
if m.SubagentHandoffNotTried != 1 || m.SubagentHandoffAttempted != 1 || m.SubagentHandoffAccepted != 1 {
|
|
||||||
t.Fatalf("never-attempted/attempted/accepted = %d/%d/%d, want 1/1/1 — the cancelled run must not be either",
|
|
||||||
m.SubagentHandoffNotTried, m.SubagentHandoffAttempted, m.SubagentHandoffAccepted)
|
|
||||||
}
|
|
||||||
if m.SubagentHandoffClosedWith != 1 || m.SubagentHandoffLowered != 3 || m.SubagentHandoffReadOnly != 1 {
|
|
||||||
t.Fatalf("closed/lowered/read-only = %d/%d/%d, want 1/3/1",
|
|
||||||
m.SubagentHandoffClosedWith, m.SubagentHandoffLowered, m.SubagentHandoffReadOnly)
|
|
||||||
}
|
|
||||||
if m.SubagentHandoffExits["completed"] != 2 || m.SubagentHandoffExits["cancelled"] != 1 {
|
|
||||||
t.Fatalf("exits = %v, want 2 completed and 1 cancelled", m.SubagentHandoffExits)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"reasonix/internal/ext/extension/dispatch"
|
"reasonix/internal/ext/extension/dispatch"
|
||||||
"reasonix/internal/runtime/capability"
|
"reasonix/internal/runtime/capability"
|
||||||
"reasonix/internal/runtime/writeclaim"
|
"reasonix/internal/runtime/writeclaim"
|
||||||
"reasonix/internal/safety/evidence"
|
|
||||||
"reasonix/internal/safety/sandbox"
|
"reasonix/internal/safety/sandbox"
|
||||||
"reasonix/internal/state/checkpoint"
|
"reasonix/internal/state/checkpoint"
|
||||||
"reasonix/internal/state/instruction"
|
"reasonix/internal/state/instruction"
|
||||||
@@ -182,9 +181,7 @@ type Options struct {
|
|||||||
// classification judges it instead of the raw Run input, so framing verbs
|
// classification judges it instead of the raw Run input, so framing verbs
|
||||||
// cannot arm expectations; the delegation audit scores evidence origin
|
// cannot arm expectations; the delegation audit scores evidence origin
|
||||||
// against it, so only locations the parent wrote itself count as hints.
|
// against it, so only locations the parent wrote itself count as hints.
|
||||||
ClassifierTaskText string
|
ClassifierTaskText string
|
||||||
ExpectCompletionReport bool // set where the contract is appended, never inferred
|
|
||||||
HandoffEntrance string // delegation path, for telemetry only
|
|
||||||
|
|
||||||
// CapabilityLedger is the optional turn-scoped capability route ledger for
|
// CapabilityLedger is the optional turn-scoped capability route ledger for
|
||||||
// Delivery require/prefer gates. Nil disables capability gates.
|
// Delivery require/prefer gates. Nil disables capability gates.
|
||||||
@@ -192,11 +189,6 @@ type Options struct {
|
|||||||
// CapabilityAudit is the optional non-persisted metrics sink for routing.
|
// CapabilityAudit is the optional non-persisted metrics sink for routing.
|
||||||
CapabilityAudit *capability.Audit
|
CapabilityAudit *capability.Audit
|
||||||
|
|
||||||
// RequireReviewReportKind, when non-empty, makes RunSubAgentWithSession fail
|
|
||||||
// unless the subagent recorded a successful review_report of this kind —
|
|
||||||
// review/security subagents must return typed, host-verifiable reports.
|
|
||||||
RequireReviewReportKind evidence.ReviewKind
|
|
||||||
|
|
||||||
// ReasoningLanguage controls visible reasoning language preference as transient
|
// ReasoningLanguage controls visible reasoning language preference as transient
|
||||||
// user-turn context. Empty/auto injects nothing.
|
// user-turn context. Empty/auto injects nothing.
|
||||||
ReasoningLanguage string
|
ReasoningLanguage string
|
||||||
|
|||||||
@@ -98,7 +98,6 @@ func SubagentToolRegistryForDepthWithRuntime(parent *tool.Registry, names []stri
|
|||||||
sub := FilterRegistry(parent, names, exclude...)
|
sub := FilterRegistry(parent, names, exclude...)
|
||||||
bindLocal := builtin.ForkFileViews(sub)
|
bindLocal := builtin.ForkFileViews(sub)
|
||||||
stripDirectMCPTools(sub)
|
stripDirectMCPTools(sub)
|
||||||
AttachCompleteSubtaskTool(sub)
|
|
||||||
attachSubagentCapabilityProxy(parent, sub, names, runtime, bindLocal)
|
attachSubagentCapabilityProxy(parent, sub, names, runtime, bindLocal)
|
||||||
if bash, ok := sub.Get("bash"); ok {
|
if bash, ok := sub.Get("bash"); ok {
|
||||||
sub.Add(foregroundOnlyBash{inner: bash})
|
sub.Add(foregroundOnlyBash{inner: bash})
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package agent
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"reasonix/internal/contract/hostaudit"
|
"reasonix/internal/contract/hostaudit"
|
||||||
"reasonix/internal/runtime/writeclaim"
|
"reasonix/internal/runtime/writeclaim"
|
||||||
@@ -65,14 +64,8 @@ func RunSubAgentWithSession(ctx context.Context, prov provider.Provider, reg *to
|
|||||||
if planWorkflow && !strings.Contains(prompt, planmode.Marker) {
|
if planWorkflow && !strings.Contains(prompt, planmode.Marker) {
|
||||||
prompt = planmode.Marker + "\n\n" + prompt
|
prompt = planmode.Marker + "\n\n" + prompt
|
||||||
}
|
}
|
||||||
if kind := opts.RequireReviewReportKind; kind != "" {
|
|
||||||
prompt = prompt + "\n\n" + reviewReportTaskContract(kind)
|
|
||||||
}
|
|
||||||
ctx = WithoutRunTimeSink(ctx)
|
ctx = WithoutRunTimeSink(ctx)
|
||||||
sub := newDelegatedAgent(ctx, prov, reg, sess, opts, sink)
|
sub := newDelegatedAgent(ctx, prov, reg, sess, opts, sink)
|
||||||
// One defer: this returns from a salvage, a review failure, a provider
|
|
||||||
// error and a clean answer, and per-path calls would miss one.
|
|
||||||
defer func() { observeSubagentHandoff(sink, sub, sess, opts, answer, err) }()
|
|
||||||
sub.SetPlanMode(planWorkflow)
|
sub.SetPlanMode(planWorkflow)
|
||||||
if err := sub.Run(ctx, prompt); err != nil {
|
if err := sub.Run(ctx, prompt); err != nil {
|
||||||
// Still merge any partial child evidence so parent gates see real writes.
|
// Still merge any partial child evidence so parent gates see real writes.
|
||||||
@@ -230,15 +223,10 @@ func (a *Agent) EvidenceSummary() evidence.ChildEvidenceSummary {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// composeSubagentAnswer assembles everything the parent is shown for one child
|
// composeSubagentAnswer assembles everything the parent is shown for one child
|
||||||
// run: the host-adjudicated completion claim when the child submitted one, the
|
// run: the child's own prose, then the host's receipts.
|
||||||
// child's own prose, then the host's receipts.
|
|
||||||
func composeSubagentAnswer(ctx context.Context, answer string, sub *Agent, claims writeclaim.WritePathSet, delegationText string) string {
|
func composeSubagentAnswer(ctx context.Context, answer string, sub *Agent, claims writeclaim.WritePathSet, delegationText string) string {
|
||||||
summary := sub.EvidenceSummary()
|
summary := sub.EvidenceSummary()
|
||||||
report, reasons, hasReport := sub.CompletionReport()
|
recordDelegationAudit(ctx, summary, claims, delegationText)
|
||||||
if hasReport {
|
|
||||||
answer = strings.TrimSpace(formatCompletionReport(report, reasons) + "\n\n" + answer)
|
|
||||||
}
|
|
||||||
recordDelegationAudit(ctx, summary, claims, report, reasons, hasReport, delegationText)
|
|
||||||
return AppendHostReceipts(answer, summary, claims)
|
return AppendHostReceipts(answer, summary, claims)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -259,52 +247,6 @@ func newDelegatedAgent(ctx context.Context, prov provider.Provider, reg *tool.Re
|
|||||||
return sub
|
return sub
|
||||||
}
|
}
|
||||||
|
|
||||||
// observeSubagentHandoff records one child run's closing protocol. Called from
|
|
||||||
// a single defer so every exit path is counted the same: a salvage, a review
|
|
||||||
// failure and a clean answer are all runs that either submitted a report or
|
|
||||||
// did not.
|
|
||||||
func observeSubagentHandoff(sink event.Sink, sub *Agent, sess *sessionstore.Session, opts Options, answer string, runErr error) {
|
|
||||||
if sub == nil || sess == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
audit := event.SubagentHandoffAudit{
|
|
||||||
Entrance: opts.HandoffEntrance,
|
|
||||||
Depth: opts.SubagentDepth,
|
|
||||||
ReadOnly: opts.ReadOnlyExecution,
|
|
||||||
Expected: opts.ExpectCompletionReport,
|
|
||||||
Exit: handoffExit(answer, runErr),
|
|
||||||
}
|
|
||||||
countReportCalls(&audit, sess.Snapshot())
|
|
||||||
// Claimed and adjudicated are both kept: a child that submits reports
|
|
||||||
// readily and has them lowered every time is a different problem from one
|
|
||||||
// that does not submit, and a single status cannot say which.
|
|
||||||
if sub.task.ledger != nil {
|
|
||||||
verdicts := sub.task.ledger.ClosureVerdicts()
|
|
||||||
audit.Closed, audit.NeedsWork = verdicts.Closed, verdicts.NeedsWork
|
|
||||||
if claimed, ok := sub.task.ledger.LatestCompletionReport(); ok {
|
|
||||||
adjudicated, reasons := sub.task.ledger.AdjudicateCompletion(claimed)
|
|
||||||
audit.ClaimedStatus = string(claimed.Status)
|
|
||||||
audit.AdjudicatedStatus = string(adjudicated.Status)
|
|
||||||
audit.LoweredClaims = len(reasons)
|
|
||||||
audit.Criteria = len(adjudicated.Criteria)
|
|
||||||
audit.Unresolved = len(adjudicated.Unresolved)
|
|
||||||
for _, c := range adjudicated.Criteria {
|
|
||||||
audit.Evidence += len(c.Evidence)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
event.RecordSubagentHandoff(sink, audit)
|
|
||||||
}
|
|
||||||
|
|
||||||
// reviewReportTaskContract is appended to the task prompt of a review subagent
|
|
||||||
// whose run must end with a typed report. The skill body describes how to
|
|
||||||
// review; this states the non-negotiable submission protocol.
|
|
||||||
func reviewReportTaskContract(kind evidence.ReviewKind) string {
|
|
||||||
return fmt.Sprintf(`<review-report-contract event="SubagentReviewReport">
|
|
||||||
Before your final answer you MUST call the review_report tool exactly once with kind=%q, your verdict (pass | warn | block), reviewed_paths listing only files you actually read this run, and your findings. The host discards a review run that ends without a successful review_report call — your prose summary alone does not count.
|
|
||||||
</review-report-contract>`, string(kind))
|
|
||||||
}
|
|
||||||
|
|
||||||
const subagentStartContext = `<subagent-context event="SubagentStart">
|
const subagentStartContext = `<subagent-context event="SubagentStart">
|
||||||
Before acting, check the available skills and tools. If a relevant skill is available, invoke it before continuing. Delegate to another sub-agent only when the task genuinely benefits from isolated context and the delegation tool is available.
|
Before acting, check the available skills and tools. If a relevant skill is available, invoke it before continuing. Delegate to another sub-agent only when the task genuinely benefits from isolated context and the delegation tool is available.
|
||||||
</subagent-context>`
|
</subagent-context>`
|
||||||
@@ -325,52 +267,20 @@ type mutationObserverSetter interface {
|
|||||||
SetMutationObserver(*checkpoint.MutationObserver)
|
SetMutationObserver(*checkpoint.MutationObserver)
|
||||||
}
|
}
|
||||||
|
|
||||||
// formatCompletionReport renders the child's claim after the host has lowered
|
|
||||||
// whatever its receipts could not back. Downgrades are shown, never silently
|
|
||||||
// applied: a parent that cannot see the adjudication cannot trust the status.
|
|
||||||
func formatCompletionReport(report evidence.CompletionReport, reasons []string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
b.WriteString("status: ")
|
|
||||||
b.WriteString(string(report.Status))
|
|
||||||
if len(reasons) > 0 {
|
|
||||||
b.WriteString(" (lowered by the host: unbacked criterion claims)")
|
|
||||||
}
|
|
||||||
b.WriteString("\nsummary: ")
|
|
||||||
b.WriteString(report.Summary)
|
|
||||||
for _, c := range report.Criteria {
|
|
||||||
b.WriteString("\n " + c.ID + " " + string(c.Status))
|
|
||||||
if proof := criterionProof(c); proof != "" {
|
|
||||||
b.WriteString(" — " + proof)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, reason := range reasons {
|
|
||||||
b.WriteString("\n host lowered " + reason)
|
|
||||||
}
|
|
||||||
for _, u := range report.Unresolved {
|
|
||||||
b.WriteString("\nunresolved: " + u)
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordDelegationAudit emits one structured receipt per child run. It reports
|
// recordDelegationAudit emits one structured receipt per child run. It reports
|
||||||
// what the host observed and what it refused to back, so an orchestration
|
// what the host observed and what it refused to back, so an orchestration
|
||||||
// benchmark can separate real gains from extra tokens spent. delegationText is
|
// benchmark can separate real gains from extra tokens spent. delegationText is
|
||||||
// the parent-authored task before host framing, which is what makes the
|
// the parent-authored task before host framing, which is what makes the
|
||||||
// evidence-origin split a host record rather than a claim.
|
// evidence-origin split a host record rather than a claim.
|
||||||
func recordDelegationAudit(ctx context.Context, summary evidence.ChildEvidenceSummary, claims writeclaim.WritePathSet, report evidence.CompletionReport, reasons []string, hasReport bool, delegationText string) {
|
func recordDelegationAudit(ctx context.Context, summary evidence.ChildEvidenceSummary, claims writeclaim.WritePathSet, delegationText string) {
|
||||||
audit := hostaudit.DelegationAudit{
|
audit := hostaudit.DelegationAudit{
|
||||||
Depth: SubagentDepth(ctx),
|
Depth: SubagentDepth(ctx),
|
||||||
ToolCalls: len(summary.Receipts),
|
ToolCalls: len(summary.Receipts),
|
||||||
MutationPaths: summary.MutationPaths(),
|
MutationPaths: summary.MutationPaths(),
|
||||||
ClaimViolations: len(claimViolations(summary, claims)),
|
ClaimViolations: len(claimViolations(summary, claims)),
|
||||||
HasReport: hasReport,
|
|
||||||
Downgrades: len(reasons),
|
|
||||||
}
|
}
|
||||||
audit.Mutations = len(audit.MutationPaths)
|
audit.Mutations = len(audit.MutationPaths)
|
||||||
evidence.ClassifyEvidenceOrigin(&audit, delegationText, summary.EvidencePaths())
|
evidence.ClassifyEvidenceOrigin(&audit, delegationText, summary.EvidencePaths())
|
||||||
if hasReport {
|
|
||||||
audit.AdjudicatedStatus = string(report.Status)
|
|
||||||
}
|
|
||||||
_, sink, _, _ := CallContext(ctx)
|
_, sink, _, _ := CallContext(ctx)
|
||||||
event.RecordDelegationAudit(sink, audit)
|
event.RecordDelegationAudit(sink, audit)
|
||||||
}
|
}
|
||||||
@@ -390,57 +300,6 @@ func AppendHostReceipts(answer string, summary evidence.ChildEvidenceSummary, cl
|
|||||||
return answer + "\n\n" + block
|
return answer + "\n\n" + block
|
||||||
}
|
}
|
||||||
|
|
||||||
// handoffExit classifies how the run ended. A provider failure that produced no
|
|
||||||
// report is not a compliance failure, and counting it as one would make the
|
|
||||||
// denominator answer a different question.
|
|
||||||
func handoffExit(answer string, runErr error) string {
|
|
||||||
switch {
|
|
||||||
case errors.Is(runErr, context.Canceled):
|
|
||||||
return "cancelled"
|
|
||||||
case runErr != nil:
|
|
||||||
return "error"
|
|
||||||
case strings.TrimSpace(answer) == "":
|
|
||||||
return "no_answer"
|
|
||||||
default:
|
|
||||||
return "completed"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// countReportCalls walks the child's transcript for complete_subtask calls and
|
|
||||||
// what the tool made of them. Attempted-and-refused is a schema problem;
|
|
||||||
// never-attempted is a protocol one, and only the counts can tell them apart.
|
|
||||||
func countReportCalls(audit *event.SubagentHandoffAudit, msgs []provider.Message) {
|
|
||||||
calls := map[string]bool{}
|
|
||||||
round := 0
|
|
||||||
for _, m := range msgs {
|
|
||||||
if len(m.ToolCalls) > 0 {
|
|
||||||
round++
|
|
||||||
}
|
|
||||||
for _, tc := range m.ToolCalls {
|
|
||||||
if tc.Name != completeSubtaskToolName {
|
|
||||||
if audit.ReportRound > 0 {
|
|
||||||
audit.ToolCallsAfterReport++
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
calls[tc.ID] = true
|
|
||||||
audit.Attempts++
|
|
||||||
if audit.ReportRound == 0 {
|
|
||||||
audit.ReportRound = round
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if m.Role != provider.RoleTool || !calls[m.ToolCallID] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if isErrorMessage(m) {
|
|
||||||
audit.Malformed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
audit.Accepted++
|
|
||||||
}
|
|
||||||
audit.FinalRound = round
|
|
||||||
}
|
|
||||||
|
|
||||||
// claimViolations returns the mutations the host observed outside the write
|
// claimViolations returns the mutations the host observed outside the write
|
||||||
// claim the child declared. Tool-level confinement already refuses these, so a
|
// claim the child declared. Tool-level confinement already refuses these, so a
|
||||||
// non-empty result means a write reached the workspace through a surface the
|
// non-empty result means a write reached the workspace through a surface the
|
||||||
@@ -458,21 +317,6 @@ func claimViolations(summary evidence.ChildEvidenceSummary, claims writeclaim.Wr
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func criterionProof(c evidence.AcceptanceCriterion) string {
|
|
||||||
var parts []string
|
|
||||||
for _, e := range c.Evidence {
|
|
||||||
switch {
|
|
||||||
case strings.TrimSpace(e.Command) != "":
|
|
||||||
parts = append(parts, e.Command)
|
|
||||||
case len(e.Paths) > 0:
|
|
||||||
parts = append(parts, strings.Join(e.Paths, " "))
|
|
||||||
case strings.TrimSpace(e.Summary) != "":
|
|
||||||
parts = append(parts, e.Kind+": "+e.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return joinBoundedReceipts(parts)
|
|
||||||
}
|
|
||||||
|
|
||||||
// formatHostReceipts renders only what the host is willing to attest to: files
|
// formatHostReceipts renders only what the host is willing to attest to: files
|
||||||
// the child really changed, and commands whose outcome the host observed.
|
// the child really changed, and commands whose outcome the host observed.
|
||||||
// Ordinary reads and greps are excluded on purpose — they are not claims a
|
// Ordinary reads and greps are excluded on purpose — they are not claims a
|
||||||
|
|||||||
@@ -1,135 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"reasonix/internal/contract/tool"
|
|
||||||
"reasonix/internal/safety/evidence"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CompleteSubtaskTool is visible only inside sub-agent registries. It ends a
|
|
||||||
// delegated run with a structured, host-checkable claim instead of prose. It is
|
|
||||||
// never registered on the parent agent's tool surface.
|
|
||||||
type CompleteSubtaskTool struct{}
|
|
||||||
|
|
||||||
func NewCompleteSubtaskTool() *CompleteSubtaskTool { return &CompleteSubtaskTool{} }
|
|
||||||
|
|
||||||
// completeSubtaskToolName is the one spelling the contract, the registry and
|
|
||||||
// the telemetry all mean.
|
|
||||||
const completeSubtaskToolName = "complete_subtask"
|
|
||||||
|
|
||||||
func init() { tool.RegisterConditional(&CompleteSubtaskTool{}) }
|
|
||||||
|
|
||||||
func (*CompleteSubtaskTool) Name() string { return completeSubtaskToolName }
|
|
||||||
|
|
||||||
func (*CompleteSubtaskTool) Description() string {
|
|
||||||
return "Close out this delegated sub-task with a structured result the parent can verify. Call once, last. status is complete, partial, blocked, or failed; summary states what is now true; acceptance_criteria lists each condition with the evidence for it; unresolved lists what you did not finish. The host checks every cited command and path against what it actually observed you do, and lowers any claim it cannot back."
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadOnly is true: submitting a report changes no workspace state. It is the
|
|
||||||
// claim itself, not a mutation.
|
|
||||||
func (*CompleteSubtaskTool) ReadOnly() bool { return true }
|
|
||||||
|
|
||||||
func (*CompleteSubtaskTool) Schema() json.RawMessage {
|
|
||||||
// Fixed schema — sub-agent registries only, so it never enters the parent
|
|
||||||
// prefix.
|
|
||||||
return json.RawMessage(`{
|
|
||||||
"type":"object",
|
|
||||||
"properties":{
|
|
||||||
"status":{"type":"string","description":"complete | partial | blocked | failed. Claim the truth: the host lowers a status its receipts cannot back."},
|
|
||||||
"summary":{"type":"string","description":"What is now true as a result of this sub-task."},
|
|
||||||
"acceptance_criteria":{
|
|
||||||
"type":"array",
|
|
||||||
"description":"Each condition this sub-task had to meet, with proof.",
|
|
||||||
"items":{
|
|
||||||
"type":"object",
|
|
||||||
"properties":{
|
|
||||||
"id":{"type":"string","description":"Short stable id, e.g. AC1."},
|
|
||||||
"status":{"type":"string","description":"satisfied | unsatisfied"},
|
|
||||||
"evidence":{
|
|
||||||
"type":"array",
|
|
||||||
"items":{
|
|
||||||
"type":"object",
|
|
||||||
"properties":{
|
|
||||||
"kind":{"type":"string","enum":["verification","review","diff","files","manual"],"description":"verification = a command was run (command REQUIRED); review = a review completed; diff = a code change (paths REQUIRED); files = files created/edited/inspected (paths REQUIRED); manual = a manual check, which the host cannot back on its own."},
|
|
||||||
"summary":{"type":"string","description":"The evidence itself."},
|
|
||||||
"command":{"type":"string","description":"REQUIRED for verification: the command as it actually ran."},
|
|
||||||
"paths":{"type":"array","items":{"type":"string"},"description":"REQUIRED for diff/files: the files this evidence refers to."}
|
|
||||||
},
|
|
||||||
"required":["kind","summary"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"required":["id","status"]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"unresolved":{"type":"array","items":{"type":"string"},"description":"What remains undone, unverified, or blocked. Put anything you assumed rather than checked here."}
|
|
||||||
},
|
|
||||||
"required":["status","summary"]
|
|
||||||
}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*CompleteSubtaskTool) Execute(ctx context.Context, args json.RawMessage) (string, error) {
|
|
||||||
report, err := evidence.ParseCompletionReport(args)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
led, ok := evidence.FromContext(ctx)
|
|
||||||
if !ok {
|
|
||||||
return "", fmt.Errorf("complete_subtask requires the host evidence ledger; submit it from inside a sub-agent run")
|
|
||||||
}
|
|
||||||
// The submission always succeeds; what varies is the verdict. A rejection
|
|
||||||
// the parent never learns about serves it worse than a lowered claim it can
|
|
||||||
// see, so the call stands and the closure decision is said out loud.
|
|
||||||
adjudicated, reasons := led.AdjudicateCompletion(report)
|
|
||||||
closed := len(reasons) == 0
|
|
||||||
led.NoteClosureVerdict(closed)
|
|
||||||
if closed {
|
|
||||||
return fmt.Sprintf("complete_subtask closed: status=%s criteria=%d unresolved=%d — the sub-task is done and nothing further is asked of you",
|
|
||||||
adjudicated.Status, len(adjudicated.Criteria), len(adjudicated.Unresolved)), nil
|
|
||||||
}
|
|
||||||
// The host knows which claims it could not back and why; reporting only the
|
|
||||||
// count leaves the sub-agent guessing which criterion and what would fix it.
|
|
||||||
return fmt.Sprintf("complete_subtask needs_work: status=%s criteria=%d unresolved=%d — these claims are not backed by anything the host observed:\n- %s\nDo that work, then submit again.",
|
|
||||||
adjudicated.Status, len(adjudicated.Criteria), len(adjudicated.Unresolved), strings.Join(reasons, "\n- ")), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// AttachCompleteSubtaskTool adds complete_subtask to a sub-agent registry.
|
|
||||||
// Parent registries never receive it.
|
|
||||||
func AttachCompleteSubtaskTool(reg *tool.Registry) {
|
|
||||||
if reg == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
reg.Add(NewCompleteSubtaskTool())
|
|
||||||
}
|
|
||||||
|
|
||||||
// CompletionReport returns this agent's adjudicated completion claim, if it
|
|
||||||
// submitted one. Adjudication re-runs here so the returned status reflects the
|
|
||||||
// full run, including receipts recorded after the tool call itself.
|
|
||||||
func (a *Agent) CompletionReport() (evidence.CompletionReport, []string, bool) {
|
|
||||||
if a == nil || a.task.ledger == nil {
|
|
||||||
return evidence.CompletionReport{}, nil, false
|
|
||||||
}
|
|
||||||
report, ok := a.task.ledger.LatestCompletionReport()
|
|
||||||
if !ok {
|
|
||||||
return evidence.CompletionReport{}, nil, false
|
|
||||||
}
|
|
||||||
adjudicated, reasons := a.task.ledger.AdjudicateCompletion(report)
|
|
||||||
return adjudicated, reasons, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// CompleteSubtaskContract is appended to a sub-agent's task prompt when the
|
|
||||||
// host expects a typed completion claim. The profile body says how to work;
|
|
||||||
// this states the non-negotiable closing protocol.
|
|
||||||
const CompleteSubtaskContract = `<completion-contract>
|
|
||||||
Call complete_subtask when you believe this sub-task is done. State the acceptance
|
|
||||||
criteria you were held to and attach, for each, the command you ran or the paths you
|
|
||||||
changed or inspected. The host checks every citation against what it observed you
|
|
||||||
actually do, and answers with a verdict: needs_work means a claim is not backed by
|
|
||||||
anything it saw — do that work and submit again; closed means the sub-task is done and
|
|
||||||
nothing further is asked of you. Cite real work only and put anything you assumed
|
|
||||||
rather than verified in unresolved.
|
|
||||||
</completion-contract>`
|
|
||||||
@@ -1,179 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"reasonix/internal/safety/evidence"
|
|
||||||
)
|
|
||||||
|
|
||||||
func submitCompleteSubtask(t *testing.T, led *evidence.Ledger, args string) string {
|
|
||||||
t.Helper()
|
|
||||||
ctx := evidence.WithLedger(context.Background(), led)
|
|
||||||
out, err := NewCompleteSubtaskTool().Execute(ctx, json.RawMessage(args))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("complete_subtask: %v", err)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// The child may claim anything; the status the parent sees is the host's.
|
|
||||||
func TestCompleteSubtaskHostLowersUnbackedClaim(t *testing.T) {
|
|
||||||
led := evidence.NewLedger()
|
|
||||||
led.Record(evidence.Receipt{ToolName: "bash", Command: "go test ./parser", Success: true, OutputBytes: 12})
|
|
||||||
|
|
||||||
args := `{
|
|
||||||
"status":"complete",
|
|
||||||
"summary":"fixed the parser",
|
|
||||||
"acceptance_criteria":[
|
|
||||||
{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"unit tests","command":"go test ./parser"}]},
|
|
||||||
{"id":"AC2","status":"satisfied","evidence":[{"kind":"verification","summary":"integration suite","command":"go test ./integration"}]}
|
|
||||||
]}`
|
|
||||||
if out := submitCompleteSubtask(t, led, args); !strings.Contains(out, "status=partial") {
|
|
||||||
t.Fatalf("tool result = %q, want the host-lowered status", out)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The agent host, not the tool, records the call; replay that receipt so the
|
|
||||||
// ledger lookup the report renderer uses is covered too.
|
|
||||||
led.Record(evidence.ReceiptFromToolCall("complete_subtask", json.RawMessage(args), true, evidence.ToolFacts{ReadOnly: true}))
|
|
||||||
report, ok := led.LatestCompletionReport()
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("a recorded complete_subtask call must be recoverable from the ledger")
|
|
||||||
}
|
|
||||||
adjudicated, reasons := led.AdjudicateCompletion(report)
|
|
||||||
if adjudicated.Status != evidence.CompletionPartial {
|
|
||||||
t.Fatalf("status = %q, want partial", adjudicated.Status)
|
|
||||||
}
|
|
||||||
if adjudicated.Criteria[0].Status != evidence.CriterionSatisfied {
|
|
||||||
t.Fatal("AC1 was backed by a real command receipt and must survive")
|
|
||||||
}
|
|
||||||
if adjudicated.Criteria[1].Status != evidence.CriterionUnsatisfied {
|
|
||||||
t.Fatal("AC2 cited a command that never ran and must be lowered")
|
|
||||||
}
|
|
||||||
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "AC2:") {
|
|
||||||
t.Fatalf("reasons = %v, want one naming AC2", reasons)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompleteSubtaskKeepsFullyBackedClaim(t *testing.T) {
|
|
||||||
led := evidence.NewLedger()
|
|
||||||
led.Record(evidence.Receipt{ToolName: "bash", Command: "go test ./parser", Success: true, OutputBytes: 12})
|
|
||||||
led.Record(evidence.Receipt{ToolName: "write_file", Success: true, Mutation: true, Write: true, Paths: []string{"parser.go"}})
|
|
||||||
|
|
||||||
out := submitCompleteSubtask(t, led, `{
|
|
||||||
"status":"complete",
|
|
||||||
"summary":"fixed the parser",
|
|
||||||
"acceptance_criteria":[
|
|
||||||
{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"tests","command":"go test ./parser"}]},
|
|
||||||
{"id":"AC2","status":"satisfied","evidence":[{"kind":"diff","summary":"the fix","paths":["parser.go"]}]}
|
|
||||||
]}`)
|
|
||||||
if !strings.Contains(out, "status=complete") || strings.Contains(out, "lowered") {
|
|
||||||
t.Fatalf("tool result = %q, want an untouched complete status", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A satisfied criterion resting only on the model's word is not host-backed.
|
|
||||||
func TestCompleteSubtaskLowersManualOnlyAndEvidenceFreeClaims(t *testing.T) {
|
|
||||||
led := evidence.NewLedger()
|
|
||||||
report, err := evidence.ParseCompletionReport(json.RawMessage(`{
|
|
||||||
"status":"complete","summary":"done",
|
|
||||||
"acceptance_criteria":[
|
|
||||||
{"id":"AC1","status":"satisfied","evidence":[{"kind":"manual","summary":"I checked it"}]},
|
|
||||||
{"id":"AC2","status":"satisfied"}
|
|
||||||
]}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
adjudicated, reasons := led.AdjudicateCompletion(report)
|
|
||||||
if adjudicated.Status != evidence.CompletionPartial || len(reasons) != 2 {
|
|
||||||
t.Fatalf("status = %q reasons = %v, want partial with both lowered", adjudicated.Status, reasons)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseCompletionReportRejectsMalformedClaims(t *testing.T) {
|
|
||||||
for name, args := range map[string]string{
|
|
||||||
"bad status": `{"status":"done","summary":"x"}`,
|
|
||||||
"missing summary": `{"status":"complete"}`,
|
|
||||||
"verification no cmd": `{"status":"complete","summary":"x","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"tests"}]}]}`,
|
|
||||||
"diff without paths": `{"status":"complete","summary":"x","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"diff","summary":"change"}]}]}`,
|
|
||||||
"unknown evidence": `{"status":"complete","summary":"x","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"vibes","summary":"trust me"}]}]}`,
|
|
||||||
"criterion without id": `{"status":"complete","summary":"x","acceptance_criteria":[{"status":"satisfied"}]}`,
|
|
||||||
} {
|
|
||||||
if _, err := evidence.ParseCompletionReport(json.RawMessage(args)); err == nil {
|
|
||||||
t.Errorf("%s: accepted a malformed report", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A verdict is not an error. A malformed submission fails the call; a claim the
|
|
||||||
// host cannot back is a well-formed call the host answers with needs_work, and
|
|
||||||
// keeping those apart is what lets "submit again" and "you are done" be
|
|
||||||
// different answers instead of two readings of the word accepted.
|
|
||||||
func TestCompleteSubtaskVerdictSeparatesClosureFromMoreWork(t *testing.T) {
|
|
||||||
const criterion = `{"id":"AC1","status":"satisfied","evidence":[{"kind":"verification","summary":"tests","command":"go test ./parser"}]}`
|
|
||||||
|
|
||||||
backed := evidence.NewLedger()
|
|
||||||
backed.Record(evidence.Receipt{ToolName: "bash", Command: "go test ./parser", Success: true, OutputBytes: 12})
|
|
||||||
out := submitCompleteSubtask(t, backed, `{"status":"complete","summary":"fixed it","acceptance_criteria":[`+criterion+`]}`)
|
|
||||||
if !strings.Contains(out, "complete_subtask closed:") {
|
|
||||||
t.Fatalf("backed report = %q, want a closed verdict", out)
|
|
||||||
}
|
|
||||||
if v := backed.ClosureVerdicts(); v.Closed != 1 || v.NeedsWork != 0 {
|
|
||||||
t.Fatalf("verdicts = %+v, want one closure", v)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The same claim with nothing behind it: the call still succeeds.
|
|
||||||
unbacked := evidence.NewLedger()
|
|
||||||
out = submitCompleteSubtask(t, unbacked, `{"status":"complete","summary":"fixed it","acceptance_criteria":[`+criterion+`]}`)
|
|
||||||
if !strings.Contains(out, "complete_subtask needs_work:") {
|
|
||||||
t.Fatalf("unbacked report = %q, want a needs_work verdict", out)
|
|
||||||
}
|
|
||||||
if v := unbacked.ClosureVerdicts(); v.NeedsWork != 1 || v.Closed != 0 {
|
|
||||||
t.Fatalf("verdicts = %+v, want one needs_work", v)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A malformed submission is the other layer and still fails the call.
|
|
||||||
ctx := evidence.WithLedger(context.Background(), evidence.NewLedger())
|
|
||||||
if _, err := NewCompleteSubtaskTool().Execute(ctx, json.RawMessage(`{"status":"nonsense"}`)); err == nil {
|
|
||||||
t.Fatal("a malformed report must fail the call, not return a verdict")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A rejection the sub-agent cannot act on costs a whole round: it resubmits
|
|
||||||
// with a different evidence shape and hopes. The host knows which criterion it
|
|
||||||
// could not back and what it looked for, so the verdict says both.
|
|
||||||
func TestCompleteSubtaskRejectionNamesWhatTheHostLookedFor(t *testing.T) {
|
|
||||||
led := evidence.NewLedger()
|
|
||||||
ctx := evidence.WithLedger(context.Background(), led)
|
|
||||||
|
|
||||||
args := []byte(`{"status":"complete","summary":"done",
|
|
||||||
"acceptance_criteria":[
|
|
||||||
{"id":"c1","status":"satisfied","evidence":[
|
|
||||||
{"kind":"files","summary":"read the files","paths":["internal/runtime/agent/agent.go","internal/runtime/agent/run_loop.go"]}]},
|
|
||||||
{"id":"c2","status":"satisfied","evidence":[
|
|
||||||
{"kind":"verification","summary":"ran the tests","command":"go test ./internal/runtime/agent/"}]},
|
|
||||||
{"id":"c3","status":"satisfied","evidence":[
|
|
||||||
{"kind":"review","summary":"first reviewer"},
|
|
||||||
{"kind":"review","summary":"second reviewer"}]}]}`)
|
|
||||||
|
|
||||||
out, err := (&CompleteSubtaskTool{}).Execute(ctx, args)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Execute: %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(out, "needs_work") {
|
|
||||||
t.Fatalf("verdict = %q, want needs_work for claims with no receipts", out)
|
|
||||||
}
|
|
||||||
if n := strings.Count(out, "no completed review"); n > 1 {
|
|
||||||
t.Errorf("one missing receipt is reported %d times, which reads as %d problems:\n%s", n, n, out)
|
|
||||||
}
|
|
||||||
for _, want := range []string{
|
|
||||||
"c1:", "no read or write recorded for", "internal/runtime/agent/agent.go",
|
|
||||||
"c2:", "no successful run of", "go test ./internal/runtime/agent/",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(out, want) {
|
|
||||||
t.Errorf("verdict does not say %q; the sub-agent cannot tell what to fix:\n%s", want, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,112 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"reasonix/internal/contract/tool"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"reasonix/internal/contract/event"
|
|
||||||
"reasonix/internal/contract/provider"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A writer child submits a typed claim the host adjudicates. A read-only child
|
|
||||||
// used to end in prose, so what it found reached the parent as an unverifiable
|
|
||||||
// sentence while its receipts sat elsewhere in the host ledger.
|
|
||||||
|
|
||||||
// Both kinds of child are told the same closing protocol, and the wording has
|
|
||||||
// to fit a child that only looked at things. It also has to name both verdicts:
|
|
||||||
// a contract that asks for one call cannot be honoured against a host that
|
|
||||||
// answers a lowered claim by asking for the work again.
|
|
||||||
func TestCompletionContractCoversInspectionNotOnlyChange(t *testing.T) {
|
|
||||||
if !strings.Contains(CompleteSubtaskContract, "complete_subtask") {
|
|
||||||
t.Error("the contract no longer names the call it requires")
|
|
||||||
}
|
|
||||||
for _, verdict := range []string{"needs_work", "closed"} {
|
|
||||||
if !strings.Contains(CompleteSubtaskContract, verdict) {
|
|
||||||
t.Errorf("the contract does not say what %q means, so the child cannot act on it", verdict)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(CompleteSubtaskContract, "inspected") {
|
|
||||||
t.Errorf("the contract asks only about changes, so a read-only child has nothing to cite:\n%s",
|
|
||||||
CompleteSubtaskContract)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The shadow answers "did the child close the way it was asked to", so its
|
|
||||||
// denominator must be the instruction the host actually gave.
|
|
||||||
func TestHandoffShadowCountsAttemptsApartFromAcceptance(t *testing.T) {
|
|
||||||
audit := event.SubagentHandoffAudit{}
|
|
||||||
countReportCalls(&audit, []provider.Message{
|
|
||||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
|
||||||
{ID: "a", Name: "read_file", Arguments: `{"path":"x"}`},
|
|
||||||
}},
|
|
||||||
{Role: provider.RoleTool, ToolCallID: "a", Name: "read_file", Content: "ok"},
|
|
||||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
|
||||||
{ID: "r1", Name: completeSubtaskToolName, Arguments: `{}`},
|
|
||||||
}},
|
|
||||||
{Role: provider.RoleTool, ToolCallID: "r1", Name: completeSubtaskToolName, Content: "error: invalid status"},
|
|
||||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
|
||||||
{ID: "r2", Name: completeSubtaskToolName, Arguments: `{"status":"complete"}`},
|
|
||||||
}},
|
|
||||||
{Role: provider.RoleTool, ToolCallID: "r2", Name: completeSubtaskToolName, Content: "recorded"},
|
|
||||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
|
||||||
{ID: "b", Name: "read_file", Arguments: `{"path":"y"}`},
|
|
||||||
}},
|
|
||||||
{Role: provider.RoleTool, ToolCallID: "b", Name: "read_file", Content: "ok"},
|
|
||||||
})
|
|
||||||
|
|
||||||
// A refused call is a schema problem; never calling is a protocol one, and
|
|
||||||
// one count cannot say which happened.
|
|
||||||
if audit.Attempts != 2 || audit.Accepted != 1 || audit.Malformed != 1 {
|
|
||||||
t.Errorf("attempts=%d accepted=%d malformed=%d, want 2/1/1",
|
|
||||||
audit.Attempts, audit.Accepted, audit.Malformed)
|
|
||||||
}
|
|
||||||
// The contract asks for the report as the final call. Work after it is a
|
|
||||||
// different behaviour from submitting one at all.
|
|
||||||
if audit.ToolCallsAfterReport != 1 {
|
|
||||||
t.Errorf("tool calls after the report = %d, want 1", audit.ToolCallsAfterReport)
|
|
||||||
}
|
|
||||||
if audit.ReportRound != 2 || audit.FinalRound != 4 {
|
|
||||||
t.Errorf("rounds = report %d final %d, want 2 and 4", audit.ReportRound, audit.FinalRound)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A run that died on the provider never had the chance to comply. Counting it
|
|
||||||
// as a missing report would make the adoption rate answer a different question.
|
|
||||||
func TestHandoffShadowSeparatesFailureFromNonCompliance(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
answer string
|
|
||||||
err error
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"clean", "here is what I found", nil, "completed"},
|
|
||||||
{"provider failed", "", errors.New("stream error"), "error"},
|
|
||||||
// A wrapped sentinel, not a lookalike string: identity is what the
|
|
||||||
// classifier reads, so a message that merely says so must not pass.
|
|
||||||
{"user cancelled", "", fmt.Errorf("sub-agent: %w", context.Canceled), "cancelled"},
|
|
||||||
{"message only says cancelled", "", errors.New("context canceled"), "error"},
|
|
||||||
{"ran but said nothing", " ", nil, "no_answer"},
|
|
||||||
} {
|
|
||||||
if got := handoffExit(tc.answer, tc.err); got != tc.want {
|
|
||||||
t.Errorf("%s: exit = %q, want %q", tc.name, got, tc.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompleteSubtaskSurvivesTheStrictReadOnlyFilter(t *testing.T) {
|
|
||||||
reg := tool.NewRegistry()
|
|
||||||
reg.Add(subagentRegistryTool{name: "read_file", readOnly: true})
|
|
||||||
AttachCompleteSubtaskTool(reg)
|
|
||||||
|
|
||||||
// The child's registry is filtered once more at construction; a protocol
|
|
||||||
// tool that did not survive would leave the contract asking for a call the
|
|
||||||
// child cannot make.
|
|
||||||
strict := strictReadOnlyExecutionRegistry(reg)
|
|
||||||
if _, ok := strict.Get("complete_subtask"); !ok {
|
|
||||||
t.Fatalf("complete_subtask did not survive strict read-only construction: %v", strict.Names())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
package delegation
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"reasonix/internal/runtime/agent"
|
|
||||||
"reasonix/internal/state/sessionstore"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"reasonix/internal/contract/event"
|
|
||||||
"reasonix/internal/contract/provider"
|
|
||||||
"reasonix/internal/contract/tool"
|
|
||||||
)
|
|
||||||
|
|
||||||
// End to end: the parent's view leads with the adjudicated status, not prose.
|
|
||||||
func TestSubAgentAnswerLeadsWithAdjudicatedStatus(t *testing.T) {
|
|
||||||
reg := tool.NewRegistry()
|
|
||||||
reg.Add(fakeWriteFileTool{})
|
|
||||||
agent.AttachCompleteSubtaskTool(reg)
|
|
||||||
prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{
|
|
||||||
{toolCallChunk("1", "write_file", `{"path":"parser.go"}`), {Type: provider.ChunkDone}},
|
|
||||||
{toolCallChunk("2", "complete_subtask", `{"status":"complete","summary":"fixed the parser","acceptance_criteria":[{"id":"AC1","status":"satisfied","evidence":[{"kind":"diff","summary":"the fix","paths":["parser.go"]}]},{"id":"AC2","status":"satisfied","evidence":[{"kind":"verification","summary":"suite","command":"go test ./..."}]}],"unresolved":["integration suite not executed"]}`), {Type: provider.ChunkDone}},
|
|
||||||
{{Type: provider.ChunkText, Text: "all good"}, {Type: provider.ChunkDone}},
|
|
||||||
}}
|
|
||||||
|
|
||||||
answer, err := agent.RunSubAgentWithSession(context.Background(), prov, reg, sessionstore.NewSession("sys"),
|
|
||||||
"fix the parser", agent.Options{CheckTargetAccess: allowTestFileAccess}, event.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("RunSubAgentWithSession: %v", err)
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(answer, "status: partial") {
|
|
||||||
t.Fatalf("answer must lead with the host-adjudicated status:\n%s", answer)
|
|
||||||
}
|
|
||||||
for _, want := range []string{
|
|
||||||
"AC1 satisfied",
|
|
||||||
"AC2 unsatisfied",
|
|
||||||
"host lowered AC2",
|
|
||||||
"unresolved: integration suite not executed",
|
|
||||||
agent.HostReceiptsHeader,
|
|
||||||
} {
|
|
||||||
if !strings.Contains(answer, want) {
|
|
||||||
t.Fatalf("answer missing %q:\n%s", want, answer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
package delegation
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reasonix/internal/runtime/agent"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"reasonix/internal/contract/tool"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The protocol tool is not a capability. Attaching it before the "can this
|
|
||||||
// child investigate anything?" check would let a child with no research tools
|
|
||||||
// pass as having one, and it would then report on work it could not do.
|
|
||||||
func TestReportingToolDoesNotCountAsAResearchCapability(t *testing.T) {
|
|
||||||
parent := tool.NewRegistry()
|
|
||||||
parent.Add(subagentRegistryTool{name: "write_file"}) // not read-only
|
|
||||||
|
|
||||||
sub := agent.ReadOnlySubagentToolRegistry(parent, nil)
|
|
||||||
if sub.Len() != 0 {
|
|
||||||
t.Fatalf("a parent with no read-only tools produced %v", sub.Names())
|
|
||||||
}
|
|
||||||
// And once it is attached, the registry is no longer empty — which is
|
|
||||||
// exactly why the emptiness check has to come first.
|
|
||||||
agent.AttachCompleteSubtaskTool(sub)
|
|
||||||
if sub.Len() == 0 {
|
|
||||||
t.Fatal("complete_subtask was not attached at all")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -19,7 +19,6 @@ func (t *TaskTool) readOnlySubRegistry(spec *ProfileExecSpec, toolNames []string
|
|||||||
if sub.Len() == 0 && !spec.Grant.AllowNoTools {
|
if sub.Len() == 0 && !spec.Grant.AllowNoTools {
|
||||||
return nil, fmt.Errorf("no read-only tools available for this sub-agent")
|
return nil, fmt.Errorf("no read-only tools available for this sub-agent")
|
||||||
}
|
}
|
||||||
agent.AttachCompleteSubtaskTool(sub)
|
|
||||||
return sub, nil
|
return sub, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -32,7 +31,6 @@ func (t *TaskTool) prepareSubSession(ctx context.Context, prompt string, opts ag
|
|||||||
// The pristine task, before framing: delivery classification judges the
|
// The pristine task, before framing: delivery classification judges the
|
||||||
// task and not the wrapper.
|
// task and not the wrapper.
|
||||||
opts.ClassifierTaskText = prompt
|
opts.ClassifierTaskText = prompt
|
||||||
opts.ExpectCompletionReport, opts.HandoffEntrance = true, entrance
|
prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt)
|
||||||
prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt) + "\n\n" + agent.CompleteSubtaskContract
|
|
||||||
return agent.WithUserImages(ctx, agent.SubagentImageCandidates(ctx)), prompt, opts
|
return agent.WithUserImages(ctx, agent.SubagentImageCandidates(ctx)), prompt, opts
|
||||||
}
|
}
|
||||||
@@ -824,8 +824,7 @@ func (t *TaskTool) runSubSession(ctx context.Context, prompt string, subReg *too
|
|||||||
// Capture the pristine task before host framing is prepended: delivery
|
// Capture the pristine task before host framing is prepended: delivery
|
||||||
// intent classification must judge the task, not the wrapper.
|
// intent classification must judge the task, not the wrapper.
|
||||||
opts.ClassifierTaskText = prompt
|
opts.ClassifierTaskText = prompt
|
||||||
prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt) + "\n\n" + agent.CompleteSubtaskContract
|
prompt = agent.SubagentImageNote(ctx) + t.withWorkspaceContext(upstreamNote(ctx)+prompt)
|
||||||
opts.ExpectCompletionReport, opts.HandoffEntrance = true, entrance
|
|
||||||
// Child provider owns the final vision decision. Text-only providers
|
// Child provider owns the final vision decision. Text-only providers
|
||||||
// retain the attachment metadata but omit image parts during serialization.
|
// retain the attachment metadata but omit image parts during serialization.
|
||||||
ctx = agent.WithUserImages(ctx, agent.SubagentImageCandidates(ctx))
|
ctx = agent.WithUserImages(ctx, agent.SubagentImageCandidates(ctx))
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func TestTaskToolReturnsSubAgentFinalAnswer(t *testing.T) {
|
|||||||
if sys := sub.lastReq.Messages[0]; sys.Role != provider.RoleSystem || sys.Content != "test-sys-prompt" {
|
if sys := sub.lastReq.Messages[0]; sys.Role != provider.RoleSystem || sys.Content != "test-sys-prompt" {
|
||||||
t.Errorf("first message = %+v, want system 'test-sys-prompt'", sys)
|
t.Errorf("first message = %+v, want system 'test-sys-prompt'", sys)
|
||||||
}
|
}
|
||||||
if got := lastUser(sub.lastReq); !strings.Contains(got, `<subagent-context event="SubagentStart">`) || !strings.Contains(got, "find callers of Foo") || !strings.Contains(got, agent.CompleteSubtaskContract) {
|
if got := lastUser(sub.lastReq); !strings.Contains(got, `<subagent-context event="SubagentStart">`) || !strings.Contains(got, "find callers of Foo") {
|
||||||
t.Errorf("sub-agent user = %q, want SubagentStart context plus prompt", got)
|
t.Errorf("sub-agent user = %q, want SubagentStart context plus prompt", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -89,7 +89,7 @@ func TestTaskToolInjectsWorkspaceContextIntoSubagentPrompt(t *testing.T) {
|
|||||||
if !strings.Contains(got, `<workspace-context event="SubagentWorkspace">`) ||
|
if !strings.Contains(got, `<workspace-context event="SubagentWorkspace">`) ||
|
||||||
!strings.Contains(got, "Current workspace: "+strconv.Quote(workspace)) ||
|
!strings.Contains(got, "Current workspace: "+strconv.Quote(workspace)) ||
|
||||||
!strings.Contains(got, `prefer "." or relative paths`) ||
|
!strings.Contains(got, `prefer "." or relative paths`) ||
|
||||||
!strings.Contains(got, "inspect project") || !strings.Contains(got, agent.CompleteSubtaskContract) {
|
!strings.Contains(got, "inspect project") {
|
||||||
t.Fatalf("sub-agent user = %q, want workspace context plus prompt", got)
|
t.Fatalf("sub-agent user = %q, want workspace context plus prompt", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,97 +0,0 @@
|
|||||||
package evidence
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func report(status CompletionStatus, criteria ...AcceptanceCriterion) CompletionReport {
|
|
||||||
return CompletionReport{Status: status, Summary: "done", Criteria: criteria}
|
|
||||||
}
|
|
||||||
|
|
||||||
func satisfied(id string, evidence ...CompletionEvidence) AcceptanceCriterion {
|
|
||||||
return AcceptanceCriterion{ID: id, Status: CriterionSatisfied, Evidence: evidence}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdjudicateCompletionKeepsWhatTheHostsOwnReceiptsBack(t *testing.T) {
|
|
||||||
l := recorded(
|
|
||||||
Receipt{ToolName: "bash", Success: true, Command: "go test ./..."},
|
|
||||||
Receipt{ToolName: "edit_file", Success: true, Write: true, Mutation: true, Paths: []string{"a.go"}},
|
|
||||||
Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"b.go"}},
|
|
||||||
)
|
|
||||||
in := report(CompletionComplete,
|
|
||||||
satisfied("tests", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test ./..."}),
|
|
||||||
satisfied("diff", CompletionEvidence{Kind: CompletionEvidenceDiff, Paths: []string{"a.go"}}),
|
|
||||||
satisfied("files", CompletionEvidence{Kind: CompletionEvidenceFiles, Paths: []string{"b.go"}}),
|
|
||||||
)
|
|
||||||
got, reasons := l.AdjudicateCompletion(in)
|
|
||||||
if got.Status != CompletionComplete || len(reasons) != 0 {
|
|
||||||
t.Fatalf("a fully backed report was lowered: %q %v", got.Status, reasons)
|
|
||||||
}
|
|
||||||
for _, c := range got.Criteria {
|
|
||||||
if c.Status != CriterionSatisfied {
|
|
||||||
t.Fatalf("criterion %s was downgraded", c.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdjudicateCompletionDowngradesEveryClaimNoReceiptBacksAndCannotStayComplete(t *testing.T) {
|
|
||||||
l := recorded(
|
|
||||||
Receipt{ToolName: "bash", Success: false, Command: "go test ./..."},
|
|
||||||
Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"a.go"}},
|
|
||||||
)
|
|
||||||
in := report(CompletionComplete,
|
|
||||||
satisfied("failed-run", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test ./..."}),
|
|
||||||
satisfied("read-is-not-a-write", CompletionEvidence{Kind: CompletionEvidenceDiff, Paths: []string{"a.go"}}),
|
|
||||||
satisfied("manual-only", CompletionEvidence{Kind: CompletionEvidenceManual, Summary: "trust me"}),
|
|
||||||
satisfied("no-evidence"),
|
|
||||||
AcceptanceCriterion{ID: "already-unsatisfied", Status: CriterionUnsatisfied},
|
|
||||||
)
|
|
||||||
got, reasons := l.AdjudicateCompletion(in)
|
|
||||||
if got.Status != CompletionPartial {
|
|
||||||
t.Fatalf("status = %q, want the report lowered to partial", got.Status)
|
|
||||||
}
|
|
||||||
if len(reasons) != 4 {
|
|
||||||
t.Fatalf("reasons = %v, want one per downgraded claim", reasons)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"failed-run: no successful run", "read-is-not-a-write: no write recorded", "manual-only:", "no-evidence: claimed satisfied with no evidence"} {
|
|
||||||
found := false
|
|
||||||
for _, r := range reasons {
|
|
||||||
found = found || strings.Contains(r, want)
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Errorf("no reason mentions %q: %v", want, reasons)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, c := range got.Criteria {
|
|
||||||
if c.Status != CriterionUnsatisfied {
|
|
||||||
t.Errorf("criterion %s is %s after adjudication", c.ID, c.Status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdjudicateCompletionNeverRaisesAStatusAndAcceptsANilLedger(t *testing.T) {
|
|
||||||
l := recorded(Receipt{ToolName: "bash", Success: true, Command: "make check"})
|
|
||||||
for _, status := range []CompletionStatus{CompletionPartial, CompletionBlocked, CompletionFailed} {
|
|
||||||
in := report(status, satisfied("ok", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "make check"}))
|
|
||||||
if got, _ := l.AdjudicateCompletion(in); got.Status != status {
|
|
||||||
t.Errorf("%s became %s: the host may lower a claim, never raise it", status, got.Status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var nilLedger *Ledger
|
|
||||||
in := report(CompletionComplete, satisfied("x"))
|
|
||||||
if got, reasons := nilLedger.AdjudicateCompletion(in); got.Status != CompletionComplete || reasons != nil {
|
|
||||||
t.Fatalf("a nil ledger adjudicates nothing: %q %v", got.Status, reasons)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdjudicateCompletionSaysACitedMissIsOneProblemNotTwo(t *testing.T) {
|
|
||||||
l := recorded()
|
|
||||||
got, reasons := l.AdjudicateCompletion(report(CompletionComplete, satisfied("twice",
|
|
||||||
CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test"},
|
|
||||||
CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test"},
|
|
||||||
)))
|
|
||||||
if len(reasons) != 1 || strings.Count(reasons[0], "no successful run") != 1 || got.Status != CompletionPartial {
|
|
||||||
t.Fatalf("reasons = %v status = %q", reasons, got.Status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,270 +0,0 @@
|
|||||||
package evidence
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"unicode/utf8"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CompletionStatus is a sub-agent's claim about its own run. The host may lower
|
|
||||||
// it to what receipts support; nothing may raise it.
|
|
||||||
type CompletionStatus string
|
|
||||||
|
|
||||||
const (
|
|
||||||
CompletionComplete CompletionStatus = "complete"
|
|
||||||
CompletionPartial CompletionStatus = "partial"
|
|
||||||
CompletionBlocked CompletionStatus = "blocked"
|
|
||||||
CompletionFailed CompletionStatus = "failed"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CriterionStatus is the child's claim about one acceptance criterion.
|
|
||||||
type CriterionStatus string
|
|
||||||
|
|
||||||
const (
|
|
||||||
CriterionSatisfied CriterionStatus = "satisfied"
|
|
||||||
CriterionUnsatisfied CriterionStatus = "unsatisfied"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Completion evidence kinds mirror complete_step so a sub-agent and the root
|
|
||||||
// agent prove work in the same vocabulary.
|
|
||||||
const (
|
|
||||||
CompletionEvidenceVerification = "verification"
|
|
||||||
CompletionEvidenceReview = "review"
|
|
||||||
CompletionEvidenceDiff = "diff"
|
|
||||||
CompletionEvidenceFiles = "files"
|
|
||||||
CompletionEvidenceManual = "manual"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CompletionEvidence is one proof attached to an acceptance criterion.
|
|
||||||
type CompletionEvidence struct {
|
|
||||||
Kind string `json:"kind"`
|
|
||||||
Summary string `json:"summary"`
|
|
||||||
Command string `json:"command,omitempty"`
|
|
||||||
Paths []string `json:"paths,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// AcceptanceCriterion is one checkable condition the sub-task had to meet.
|
|
||||||
type AcceptanceCriterion struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Status CriterionStatus `json:"status"`
|
|
||||||
Evidence []CompletionEvidence `json:"evidence,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// CompletionReport is the structured payload submitted via complete_subtask.
|
|
||||||
type CompletionReport struct {
|
|
||||||
Status CompletionStatus `json:"status"`
|
|
||||||
Summary string `json:"summary"`
|
|
||||||
Criteria []AcceptanceCriterion `json:"acceptance_criteria,omitempty"`
|
|
||||||
Unresolved []string `json:"unresolved,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParseCompletionReport validates and normalizes a complete_subtask argument
|
|
||||||
// object. It checks shape only: whether the claims are true is the host's job.
|
|
||||||
func ParseCompletionReport(raw json.RawMessage) (CompletionReport, error) {
|
|
||||||
var r CompletionReport
|
|
||||||
if err := json.Unmarshal(raw, &r); err != nil {
|
|
||||||
return CompletionReport{}, fmt.Errorf("invalid complete_subtask JSON: %w", err)
|
|
||||||
}
|
|
||||||
r.Status = CompletionStatus(strings.ToLower(strings.TrimSpace(string(r.Status))))
|
|
||||||
switch r.Status {
|
|
||||||
case CompletionComplete, CompletionPartial, CompletionBlocked, CompletionFailed:
|
|
||||||
default:
|
|
||||||
return CompletionReport{}, fmt.Errorf("complete_subtask.status must be complete, partial, blocked, or failed")
|
|
||||||
}
|
|
||||||
r.Summary = strings.TrimSpace(r.Summary)
|
|
||||||
if r.Summary == "" {
|
|
||||||
return CompletionReport{}, fmt.Errorf("complete_subtask.summary is required")
|
|
||||||
}
|
|
||||||
for i := range r.Criteria {
|
|
||||||
c := &r.Criteria[i]
|
|
||||||
c.ID = strings.TrimSpace(c.ID)
|
|
||||||
if c.ID == "" {
|
|
||||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].id is required", i)
|
|
||||||
}
|
|
||||||
c.Status = CriterionStatus(strings.ToLower(strings.TrimSpace(string(c.Status))))
|
|
||||||
switch c.Status {
|
|
||||||
case CriterionSatisfied, CriterionUnsatisfied:
|
|
||||||
default:
|
|
||||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].status must be satisfied or unsatisfied", i)
|
|
||||||
}
|
|
||||||
for j := range c.Evidence {
|
|
||||||
e := &c.Evidence[j]
|
|
||||||
e.Kind = strings.ToLower(strings.TrimSpace(e.Kind))
|
|
||||||
switch e.Kind {
|
|
||||||
case CompletionEvidenceVerification:
|
|
||||||
if strings.TrimSpace(e.Command) == "" {
|
|
||||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].evidence[%d]: verification requires command", i, j)
|
|
||||||
}
|
|
||||||
case CompletionEvidenceDiff, CompletionEvidenceFiles:
|
|
||||||
if len(e.Paths) == 0 {
|
|
||||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].evidence[%d]: %s requires paths", i, j, e.Kind)
|
|
||||||
}
|
|
||||||
case CompletionEvidenceReview, CompletionEvidenceManual:
|
|
||||||
default:
|
|
||||||
return CompletionReport{}, fmt.Errorf("acceptance_criteria[%d].evidence[%d].kind is not a known evidence kind", i, j)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return r, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// LatestCompletionReport returns the most recent successful complete_subtask
|
|
||||||
// payload recorded this run.
|
|
||||||
func (l *Ledger) LatestCompletionReport() (CompletionReport, bool) {
|
|
||||||
if l == nil {
|
|
||||||
return CompletionReport{}, false
|
|
||||||
}
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
for _, r := range slices.Backward(l.receipts) {
|
|
||||||
if r.ToolName != "complete_subtask" || !r.Success {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
report, err := ParseCompletionReport(r.Args)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return report, true
|
|
||||||
}
|
|
||||||
return CompletionReport{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// ClosureTally counts the verdicts this run's completion reports received, at
|
|
||||||
// the moment each was issued. Recomputing them later answers a different
|
|
||||||
// question: the ledger has grown since, so a claim the host could not back
|
|
||||||
// then may be backed now.
|
|
||||||
type ClosureTally struct {
|
|
||||||
Closed int
|
|
||||||
NeedsWork int
|
|
||||||
}
|
|
||||||
|
|
||||||
// NoteClosureVerdict records what the host answered one report with.
|
|
||||||
func (l *Ledger) NoteClosureVerdict(closed bool) {
|
|
||||||
if l == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
if closed {
|
|
||||||
l.closure.Closed++
|
|
||||||
return
|
|
||||||
}
|
|
||||||
l.closure.NeedsWork++
|
|
||||||
}
|
|
||||||
|
|
||||||
// ClosureVerdicts returns the verdicts issued so far.
|
|
||||||
func (l *Ledger) ClosureVerdicts() ClosureTally {
|
|
||||||
if l == nil {
|
|
||||||
return ClosureTally{}
|
|
||||||
}
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
return l.closure
|
|
||||||
}
|
|
||||||
|
|
||||||
// AdjudicateCompletion lowers a report to what the host's own receipts support.
|
|
||||||
// A criterion claimed satisfied whose evidence no receipt backs is downgraded,
|
|
||||||
// and a report holding any downgraded criterion cannot stay "complete". The
|
|
||||||
// returned reasons name every downgrade so the parent sees why.
|
|
||||||
func (l *Ledger) AdjudicateCompletion(r CompletionReport) (CompletionReport, []string) {
|
|
||||||
if l == nil {
|
|
||||||
return r, nil
|
|
||||||
}
|
|
||||||
var reasons []string
|
|
||||||
downgraded := false
|
|
||||||
for i := range r.Criteria {
|
|
||||||
c := &r.Criteria[i]
|
|
||||||
if c.Status != CriterionSatisfied {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if backed, why := l.criterionIsBacked(*c); !backed {
|
|
||||||
c.Status = CriterionUnsatisfied
|
|
||||||
downgraded = true
|
|
||||||
reasons = append(reasons, c.ID+": "+why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if downgraded && r.Status == CompletionComplete {
|
|
||||||
r.Status = CompletionPartial
|
|
||||||
}
|
|
||||||
return r, reasons
|
|
||||||
}
|
|
||||||
|
|
||||||
// criterionIsBacked reports whether at least one attached proof is one the host
|
|
||||||
// itself observed. A manual claim is never host-backed on its own.
|
|
||||||
func (l *Ledger) criterionIsBacked(c AcceptanceCriterion) (bool, string) {
|
|
||||||
if len(c.Evidence) == 0 {
|
|
||||||
return false, "claimed satisfied with no evidence"
|
|
||||||
}
|
|
||||||
var missing []string
|
|
||||||
for _, e := range c.Evidence {
|
|
||||||
switch e.Kind {
|
|
||||||
case CompletionEvidenceVerification:
|
|
||||||
if l.HasSuccessfulCommand(e.Command) {
|
|
||||||
return true, ""
|
|
||||||
}
|
|
||||||
missing = append(missing, "no successful run of "+quoteShort(e.Command))
|
|
||||||
case CompletionEvidenceDiff:
|
|
||||||
if l.HasSuccessfulWrite(e.Paths) {
|
|
||||||
return true, ""
|
|
||||||
}
|
|
||||||
missing = append(missing, "no write recorded for "+joinShort(e.Paths))
|
|
||||||
case CompletionEvidenceFiles:
|
|
||||||
if l.HasSuccessfulReadOrWrite(e.Paths) {
|
|
||||||
return true, ""
|
|
||||||
}
|
|
||||||
missing = append(missing, "no read or write recorded for "+joinShort(e.Paths))
|
|
||||||
case CompletionEvidenceReview:
|
|
||||||
if l.HasCompletedReview() {
|
|
||||||
return true, ""
|
|
||||||
}
|
|
||||||
missing = append(missing, "no completed review")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(missing) == 0 {
|
|
||||||
return false, "no host receipt backs the cited evidence"
|
|
||||||
}
|
|
||||||
// Two evidence entries can miss for the same reason; saying it twice reads
|
|
||||||
// as two problems. Order is kept: it is the order they were cited in.
|
|
||||||
seen := make(map[string]bool, len(missing))
|
|
||||||
uniq := missing[:0]
|
|
||||||
for _, m := range missing {
|
|
||||||
if seen[m] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[m] = true
|
|
||||||
uniq = append(uniq, m)
|
|
||||||
}
|
|
||||||
return false, strings.Join(uniq, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// quoteShort renders one cited command for a rejection message.
|
|
||||||
func quoteShort(s string) string {
|
|
||||||
const max = 80
|
|
||||||
s = strings.TrimSpace(s)
|
|
||||||
if s == "" {
|
|
||||||
return "an unnamed command"
|
|
||||||
}
|
|
||||||
if len(s) > max {
|
|
||||||
cut := max
|
|
||||||
for cut > 0 && !utf8.RuneStart(s[cut]) {
|
|
||||||
cut--
|
|
||||||
}
|
|
||||||
s = s[:cut] + "…"
|
|
||||||
}
|
|
||||||
return strconv.Quote(s)
|
|
||||||
}
|
|
||||||
|
|
||||||
// joinShort names the cited paths, capped so one rejection cannot become the
|
|
||||||
// whole reply.
|
|
||||||
func joinShort(paths []string) string {
|
|
||||||
if len(paths) == 0 {
|
|
||||||
return "no path"
|
|
||||||
}
|
|
||||||
if len(paths) > 3 {
|
|
||||||
return strings.Join(paths[:3], ", ") + fmt.Sprintf(" (+%d more)", len(paths)-3)
|
|
||||||
}
|
|
||||||
return strings.Join(paths, ", ")
|
|
||||||
}
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
package evidence
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseCompletionReportNormalisesAValidReport(t *testing.T) {
|
|
||||||
raw := json.RawMessage(`{"status":" Complete ","summary":" fixed the loop ","acceptance_criteria":[
|
|
||||||
{"id":" c1 ","status":"SATISFIED","evidence":[
|
|
||||||
{"kind":" Verification ","summary":"ran","command":"go test ./..."},
|
|
||||||
{"kind":"diff","summary":"d","paths":["a.go"]},
|
|
||||||
{"kind":"files","summary":"f","paths":["b.go"]},
|
|
||||||
{"kind":"review","summary":"r"},
|
|
||||||
{"kind":"manual","summary":"m"}]}],
|
|
||||||
"unresolved":["integration run"]}`)
|
|
||||||
got, err := ParseCompletionReport(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got.Status != CompletionComplete || got.Summary != "fixed the loop" {
|
|
||||||
t.Fatalf("status/summary = %q / %q", got.Status, got.Summary)
|
|
||||||
}
|
|
||||||
c := got.Criteria[0]
|
|
||||||
if c.ID != "c1" || c.Status != CriterionSatisfied || c.Evidence[0].Kind != CompletionEvidenceVerification {
|
|
||||||
t.Fatalf("criterion = %+v", c)
|
|
||||||
}
|
|
||||||
if len(got.Unresolved) != 1 {
|
|
||||||
t.Fatalf("unresolved = %v", got.Unresolved)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseCompletionReportRefusesWhatIsMalformedAndSaysWhere(t *testing.T) {
|
|
||||||
for name, tc := range map[string]struct{ raw, want string }{
|
|
||||||
"not json": {`{`, "invalid complete_subtask JSON"},
|
|
||||||
"unknown status": {`{"status":"done","summary":"x"}`, "status must be"},
|
|
||||||
"missing status": {`{"summary":"x"}`, "status must be"},
|
|
||||||
"blank summary": {`{"status":"complete","summary":" "}`, "summary is required"},
|
|
||||||
"criterion without id": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":" ","status":"satisfied"}]}`, "acceptance_criteria[0].id"},
|
|
||||||
"criterion with odd status": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"maybe"}]}`, "acceptance_criteria[0].status"},
|
|
||||||
"verification with no cmd": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"verification","summary":"s"}]}]}`, "verification requires command"},
|
|
||||||
"diff with no paths": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"diff","summary":"s"}]}]}`, "diff requires paths"},
|
|
||||||
"files with no paths": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"files","summary":"s"}]}]}`, "files requires paths"},
|
|
||||||
"unknown evidence kind": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied","evidence":[{"kind":"vibes","summary":"s"}]}]}`, "not a known evidence kind"},
|
|
||||||
"second criterion is broken": {`{"status":"partial","summary":"x","acceptance_criteria":[{"id":"a","status":"satisfied"},{"id":"","status":"satisfied"}]}`, "acceptance_criteria[1].id"},
|
|
||||||
} {
|
|
||||||
_, err := ParseCompletionReport(json.RawMessage(tc.raw))
|
|
||||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
|
||||||
t.Errorf("%s: err = %v, want it to mention %q", name, err, tc.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -92,7 +92,6 @@ type Ledger struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
receipts []Receipt
|
receipts []Receipt
|
||||||
backgroundLeases []BackgroundLease
|
backgroundLeases []BackgroundLease
|
||||||
closure ClosureTally
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewLedger() *Ledger { return &Ledger{} }
|
func NewLedger() *Ledger { return &Ledger{} }
|
||||||
@@ -106,7 +105,6 @@ func (l *Ledger) Reset() {
|
|||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
l.receipts = nil
|
l.receipts = nil
|
||||||
l.backgroundLeases = nil
|
l.backgroundLeases = nil
|
||||||
l.closure = ClosureTally{}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record appends a receipt. Failed receipts are retained for auditability but
|
// Record appends a receipt. Failed receipts are retained for auditability but
|
||||||
|
|||||||
@@ -1,33 +0,0 @@
|
|||||||
package evidence
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestQuoteShortCutsOnRuneBoundary(t *testing.T) {
|
|
||||||
// 80 bytes into this command falls inside a three-byte character, and
|
|
||||||
// strconv.Quote would render the split bytes as \x escapes.
|
|
||||||
cmd := `echo "` + strings.Repeat("测试", 20) + `"`
|
|
||||||
got := quoteShort(cmd)
|
|
||||||
if strings.Contains(got, `\x`) {
|
|
||||||
t.Fatalf("quoteShort split a character: %s", got)
|
|
||||||
}
|
|
||||||
unquoted, err := strconv.Unquote(got)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("quoteShort result does not unquote: %v", err)
|
|
||||||
}
|
|
||||||
if !strings.HasSuffix(unquoted, "…") || !strings.HasPrefix(cmd, strings.TrimSuffix(unquoted, "…")) {
|
|
||||||
t.Fatalf("quoteShort = %s, want a prefix of the command ending in …", got)
|
|
||||||
}
|
|
||||||
if len(strings.TrimSuffix(unquoted, "…")) > 80 {
|
|
||||||
t.Fatalf("quoteShort kept more than 80 bytes: %s", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestQuoteShortKeepsShortASCII(t *testing.T) {
|
|
||||||
if got := quoteShort(" go test ./... "); got != `"go test ./..."` {
|
|
||||||
t.Fatalf("quoteShort = %s", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
package trajectory
|
|
||||||
|
|
||||||
import (
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"reasonix/internal/base/testenv"
|
|
||||||
"reasonix/internal/contract/event"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The bug this guards: SubagentHandoffAudit had a producer, a struct, an
|
|
||||||
// interface and a green unit test, and still reached nothing — the type
|
|
||||||
// assertion inside event.RecordSubagentHandoff matched no sink in the chain.
|
|
||||||
// Conformance proves a method exists; only a stack proves a value arrives.
|
|
||||||
func TestSubagentHandoffSurvivesTheSessionSinkStack(t *testing.T) {
|
|
||||||
path := filepath.Join(testenv.TempDir(t), "run.trajectory.jsonl")
|
|
||||||
r, err := New(event.Discard, path, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New: %v", err)
|
|
||||||
}
|
|
||||||
// boot's assembly order: Coalesce -> Sync -> CostQuote -> recorder.
|
|
||||||
chain := event.Coalesce(event.Sync(event.NewCostQuoteSink(r, nil)), time.Millisecond)
|
|
||||||
|
|
||||||
event.RecordSubagentHandoff(chain, event.SubagentHandoffAudit{
|
|
||||||
Entrance: "read_only_task", Depth: 1, ReadOnly: true, Expected: true,
|
|
||||||
Exit: "completed", Attempts: 2, Accepted: 1, Malformed: 1,
|
|
||||||
ReportRound: 3, FinalRound: 4, ToolCallsAfterReport: 2,
|
|
||||||
ClaimedStatus: "complete", AdjudicatedStatus: "partial",
|
|
||||||
LoweredClaims: 1, Criteria: 3, Evidence: 4, Unresolved: 1,
|
|
||||||
})
|
|
||||||
if err := r.Close(); err != nil {
|
|
||||||
t.Fatalf("Close: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
recs := readRecords(t, path)
|
|
||||||
if len(recs) != 1 || recs[0].SubagentHandoff == nil {
|
|
||||||
t.Fatalf("handoff never reached the recorder: %+v", recs)
|
|
||||||
}
|
|
||||||
got := *recs[0].SubagentHandoff
|
|
||||||
want := SubagentHandoff{
|
|
||||||
Entrance: "read_only_task", Depth: 1, ReadOnly: true, Expected: true,
|
|
||||||
Exit: "completed", Attempts: 2, Accepted: 1, Malformed: 1,
|
|
||||||
ReportRound: 3, FinalRound: 4, ToolCallsAfterReport: 2,
|
|
||||||
ClaimedStatus: "complete", AdjudicatedStatus: "partial",
|
|
||||||
LoweredClaims: 1, Criteria: 3, Evidence: 4, Unresolved: 1,
|
|
||||||
}
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("handoff record = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -32,12 +32,10 @@ type Record struct {
|
|||||||
RecoveryChild string `json:"protocol_recovery_child,omitempty"`
|
RecoveryChild string `json:"protocol_recovery_child,omitempty"`
|
||||||
TurnCompletion bool `json:"turn_completion,omitempty"`
|
TurnCompletion bool `json:"turn_completion,omitempty"`
|
||||||
ContractShadow *ContractShadowAudit `json:"contract_shadow,omitempty"`
|
ContractShadow *ContractShadowAudit `json:"contract_shadow,omitempty"`
|
||||||
CompletionReport *CompletionReport `json:"completion_report,omitempty"`
|
|
||||||
EvidenceBundle *EvidenceBundle `json:"evidence_bundle,omitempty"`
|
EvidenceBundle *EvidenceBundle `json:"evidence_bundle,omitempty"`
|
||||||
OutcomeProgress *OutcomeProgress `json:"outcome_progress,omitempty"`
|
OutcomeProgress *OutcomeProgress `json:"outcome_progress,omitempty"`
|
||||||
MemoryRecall *MemoryRecall `json:"memory_recall,omitempty"`
|
MemoryRecall *MemoryRecall `json:"memory_recall,omitempty"`
|
||||||
ProjectCheck *ProjectCheckProbe `json:"project_check_probe,omitempty"`
|
ProjectCheck *ProjectCheckProbe `json:"project_check_probe,omitempty"`
|
||||||
SubagentHandoff *SubagentHandoff `json:"subagent_handoff,omitempty"`
|
|
||||||
// Deltas counts the streamed increments merged into this record; absent
|
// Deltas counts the streamed increments merged into this record; absent
|
||||||
// means one. TS stays the first increment's — that is the observation
|
// means one. TS stays the first increment's — that is the observation
|
||||||
// time-to-first-token readers key on — and EndTS carries the last's.
|
// time-to-first-token readers key on — and EndTS carries the last's.
|
||||||
@@ -127,31 +125,6 @@ type ContractShadowAudit struct {
|
|||||||
ReadyToFinalize bool `json:"ready_to_finalize,omitempty"`
|
ReadyToFinalize bool `json:"ready_to_finalize,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SubagentHandoff mirrors event.SubagentHandoffAudit: how one delegated child
|
|
||||||
// closed. Every field travels, because which cut explains a refusal — entrance,
|
|
||||||
// depth, read-only, how long the run was — is what the aggregate cannot say.
|
|
||||||
type SubagentHandoff struct {
|
|
||||||
Entrance string `json:"entrance,omitempty"`
|
|
||||||
Depth int `json:"depth,omitempty"`
|
|
||||||
ReadOnly bool `json:"read_only,omitempty"`
|
|
||||||
Expected bool `json:"expected,omitempty"`
|
|
||||||
Exit string `json:"exit,omitempty"`
|
|
||||||
Attempts int `json:"attempts,omitempty"`
|
|
||||||
Accepted int `json:"accepted,omitempty"`
|
|
||||||
Malformed int `json:"malformed,omitempty"`
|
|
||||||
Closed int `json:"closed,omitempty"`
|
|
||||||
NeedsWork int `json:"needs_work,omitempty"`
|
|
||||||
ReportRound int `json:"report_round,omitempty"`
|
|
||||||
FinalRound int `json:"final_round,omitempty"`
|
|
||||||
ToolCallsAfterReport int `json:"tool_calls_after_report,omitempty"`
|
|
||||||
ClaimedStatus string `json:"claimed_status,omitempty"`
|
|
||||||
AdjudicatedStatus string `json:"adjudicated_status,omitempty"`
|
|
||||||
LoweredClaims int `json:"lowered_claims,omitempty"`
|
|
||||||
Criteria int `json:"criteria,omitempty"`
|
|
||||||
Evidence int `json:"evidence,omitempty"`
|
|
||||||
Unresolved int `json:"unresolved,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ProjectCheckProbe mirrors event.ProjectCheckProbe: the shadow comparison
|
// ProjectCheckProbe mirrors event.ProjectCheckProbe: the shadow comparison
|
||||||
// between the readiness gate's project-check derivation and the ledger's
|
// between the readiness gate's project-check derivation and the ledger's
|
||||||
// obligations. Identities are recorded, not counted — a class is only
|
// obligations. Identities are recorded, not counted — a class is only
|
||||||
@@ -173,26 +146,6 @@ type ProjectCheckDiff struct {
|
|||||||
Class string `json:"class"`
|
Class string `json:"class"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CompletionReport mirrors event.CompletionReportAudit with stable keys.
|
|
||||||
type CompletionReport struct {
|
|
||||||
Verdict string `json:"verdict"`
|
|
||||||
Risk string `json:"risk,omitempty"`
|
|
||||||
Criteria int `json:"criteria,omitempty"`
|
|
||||||
CriteriaSatisfied int `json:"criteria_satisfied,omitempty"`
|
|
||||||
Changes int `json:"changes,omitempty"`
|
|
||||||
ChangesUnreviewed int `json:"changes_unreviewed,omitempty"`
|
|
||||||
Verifications int `json:"verifications,omitempty"`
|
|
||||||
VerificationsFailed int `json:"verifications_failed,omitempty"`
|
|
||||||
VerificationsStale int `json:"verifications_stale,omitempty"`
|
|
||||||
VerificationsInconclusive int `json:"verifications_inconclusive,omitempty"`
|
|
||||||
Gaps int `json:"gaps,omitempty"`
|
|
||||||
GapKinds []string `json:"gap_kinds,omitempty"`
|
|
||||||
ClaimsVerified int `json:"claims_verified,omitempty"`
|
|
||||||
ClaimsUnbacked int `json:"claims_unbacked,omitempty"`
|
|
||||||
OriginRecorded int `json:"origin_recorded,omitempty"`
|
|
||||||
OriginMatched int `json:"origin_matched,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// EvidenceBundle mirrors event.EvidenceBundleAudit with stable keys.
|
// EvidenceBundle mirrors event.EvidenceBundleAudit with stable keys.
|
||||||
type EvidenceBundle struct {
|
type EvidenceBundle struct {
|
||||||
Sealed bool `json:"sealed"`
|
Sealed bool `json:"sealed"`
|
||||||
@@ -357,31 +310,6 @@ func (r *Recorder) RecordDelegationAudit(a hostaudit.DelegationAudit) {
|
|||||||
event.RecordDelegationAudit(r.inner, a)
|
event.RecordDelegationAudit(r.inner, a)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Recorder) RecordSubagentHandoff(a event.SubagentHandoffAudit) {
|
|
||||||
r.append(Record{SubagentHandoff: &SubagentHandoff{
|
|
||||||
Entrance: a.Entrance,
|
|
||||||
Depth: a.Depth,
|
|
||||||
ReadOnly: a.ReadOnly,
|
|
||||||
Expected: a.Expected,
|
|
||||||
Exit: a.Exit,
|
|
||||||
Attempts: a.Attempts,
|
|
||||||
Accepted: a.Accepted,
|
|
||||||
Malformed: a.Malformed,
|
|
||||||
Closed: a.Closed,
|
|
||||||
NeedsWork: a.NeedsWork,
|
|
||||||
ReportRound: a.ReportRound,
|
|
||||||
FinalRound: a.FinalRound,
|
|
||||||
ToolCallsAfterReport: a.ToolCallsAfterReport,
|
|
||||||
ClaimedStatus: a.ClaimedStatus,
|
|
||||||
AdjudicatedStatus: a.AdjudicatedStatus,
|
|
||||||
LoweredClaims: a.LoweredClaims,
|
|
||||||
Criteria: a.Criteria,
|
|
||||||
Evidence: a.Evidence,
|
|
||||||
Unresolved: a.Unresolved,
|
|
||||||
}})
|
|
||||||
event.RecordSubagentHandoff(r.inner, a)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Recorder) RecordMemoryRecall(a event.MemoryRecallAudit) {
|
func (r *Recorder) RecordMemoryRecall(a event.MemoryRecallAudit) {
|
||||||
rec := &MemoryRecall{
|
rec := &MemoryRecall{
|
||||||
CharBudget: a.CharBudget, UsedChars: a.UsedChars, Omitted: a.Omitted,
|
CharBudget: a.CharBudget, UsedChars: a.UsedChars, Omitted: a.Omitted,
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
"doc-prose": 2128,
|
"doc-prose": 2128,
|
||||||
"error-text": 9,
|
"error-text": 9,
|
||||||
"essay": 2576,
|
"essay": 2576,
|
||||||
"file-size": 18416,
|
"file-size": 18415,
|
||||||
"flat-view": 0,
|
"flat-view": 0,
|
||||||
"frontend-parity": 13,
|
"frontend-parity": 13,
|
||||||
"function-size": 2485,
|
"function-size": 2485,
|
||||||
@@ -1080,7 +1080,7 @@
|
|||||||
"internal/runtime/delegation/task.go": {
|
"internal/runtime/delegation/task.go": {
|
||||||
"complexity": 8,
|
"complexity": 8,
|
||||||
"essay": 9,
|
"essay": 9,
|
||||||
"file-size": 160,
|
"file-size": 159,
|
||||||
"function-size": 55,
|
"function-size": 55,
|
||||||
"orphan": 1
|
"orphan": 1
|
||||||
},
|
},
|
||||||
|
|||||||
Reference in new issue
Block a user