Merge pull request #11017 from esengine/feat/plan-criterion-verifier

feat(contract): verify plan criteria by the commands their step names

docs/design/TRUSTED_EXECUTION.md

@@ -521,9 +521,9 @@ Honesty metrics carry about ±10pp noise at one sample per task. A stage exit MU
 
 | Target | Today | Gap |
 | --- | --- | --- |
-| Canonical contract | `internal/runtime/contract` holds revisions in THS (`contract/1`), derived from the baseline check identities and captured test criteria; host policy accepts only tightening, the checkpoint names the revision, and a goal resumed in another process keeps it. The gate still reads `internal/runtime/taskcontract`, rebuilt each turn | templates, plan criteria, admission, the gate reading the revision |
+| Canonical contract | `internal/runtime/contract` holds revisions in THS (`contract/1`), derived from the baseline check identities, captured test criteria and the approved plan's acceptance criteria (each verified by its step's verification commands; a step naming none gives `verifier.none`). Host policy accepts only tightening, the checkpoint names the revision, and a goal resumed in another process keeps it. The gate still reads `internal/runtime/taskcontract`, rebuilt each turn | a user-approved plan revision that drops a criterion is refused like any relaxation; must be accepted as the User's before the gate reads the revision |
 | I2 claim inertness | a todo is recorded as the model's own breakdown: not required, and marking it `completed` satisfies nothing; a turn that only marked todos gets no report verdict rather than `done`. Unfinished todos stay the readiness gate's | none for todos |
-| I2 claim inertness | `complete_step` checks that the cited command ran; the criterion binding is the model's | P2 moves binding into the frozen verifier |
+| I2 claim inertness | `complete_step` still binds a check to a plan criterion on the model's word for the report; the frozen contract verifies plan criteria by their step's commands and never reads that binding | drop the binding when the gate reads the revision |
 | Atomic tasks | `taskcontract.Atomic` treats any mutation as proof of the ask | acceptable at L1 only |
 | Durable evidence | the root agent seals each turn's contract, report and receipts (arguments by digest) as a `shadow_bundle/1` record in THS; receipts carry no blob digests and no snapshot yet | P1: snapshot, verdicts, divergence |
 | Frozen test criteria | `evidence.TestCriterion` keeps host-owned bytes and digest identity | reuse as the `test` verifier subject |

internal/runtime/agent/task_contract.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/hex"
+	"slices"
+	"strings"
 
 	"reasonix/internal/runtime/contract"
 	"reasonix/internal/runtime/verdict"
@@ -29,7 +31,7 @@ func (a *Agent) settleContract(ctx context.Context, seal *EvidenceSeal) contract
 	if err != nil {
 		return contractState{Failure: trustedstate.FailureCode(err)}
 	}
-	derived := contract.Derive(contract.Sources{Checks: a.task.checkpoint.BaselineChecks, Tests: a.baselineTestIdentities()})
+	derived := contract.Derive(contract.Sources{Checks: a.task.checkpoint.BaselineChecks, Tests: a.baselineTestIdentities(), Plan: a.planCriteria()})
 	next, decision := contract.Accept(cur, record, derived, a.contractID())
 	if decision == contract.Accepted || decision == contract.Tightened {
 		rec, err := contract.Seal(ctx, seal.Store, seal.Stream, next)
@@ -109,10 +111,37 @@ func (a *Agent) frozenResults(criteria []contract.Criterion) []verdict.Frozen {
 		case contract.VerifierTest:
 			f.Unverifiable = !known[c.Verifier.Identity]
 			f.Satisfied = !f.Unverifiable && !owedTests["baseline_test@"+c.Verifier.Identity]
+		case contract.VerifierCommands:
+			f.Satisfied = !changed || !slices.ContainsFunc(c.Verifier.Identities, func(id string) bool {
+				return !ledger.HasSuccessfulCommandAfter(id, at)
+			})
 		default:
-			f.Unverifiable = true
+			f.Unverifiable, f.NoVerifier = true, true
 		}
 		out = append(out, f)
 	}
 	return out
 }
+
+// planCriteria are the approved plan's acceptance criteria, each verified by
+// the verification commands its step names. The user approved those commands
+// with the plan, so they are the plan's own verifiers rather than the model's.
+func (a *Agent) planCriteria() []contract.PlanCriterion {
+	plan := a.PlanContract()
+	if plan == nil {
+		return nil
+	}
+	var out []contract.PlanCriterion
+	for _, step := range plan.Steps {
+		var commands []string
+		for _, v := range step.Verification {
+			if id := evidence.VerificationIdentity(strings.TrimSpace(v.Command)); id != "" {
+				commands = append(commands, id)
+			}
+		}
+		for _, c := range step.Acceptance {
+			out = append(out, contract.PlanCriterion{ID: c.ID, Required: !c.Optional, Commands: commands})
+		}
+	}
+	return out
+}

internal/runtime/agent/task_contract_test.go

@@ -9,6 +9,7 @@ import (
 	"reasonix/internal/contract/provider"
 	"reasonix/internal/contract/tool"
 	"reasonix/internal/runtime/contract"
+	"reasonix/internal/runtime/plancontract"
 	"reasonix/internal/safety/evidence"
 	"reasonix/internal/state/instruction"
 	"reasonix/internal/state/sessionstore"
@@ -46,10 +47,11 @@ func contractAgent(t *testing.T, prov *scriptedProvider, declared string, store
 	reg.Add(fakeTool{name: "write_file", readOnly: false, writesPaths: true})
 	reg.Add(fakeTool{name: "bash", readOnly: false})
 	sink := &contractAuditSink{}
-	a := New(prov, reg, sessionstore.NewSession(""), Options{
-		ProjectChecks: []instruction.VerifyCheck{{Command: declared, SourcePath: "REASONIX.md", Line: 3}},
-		EvidenceSeal:  &EvidenceSeal{Store: store, Stream: "ws"},
-	}, sink)
+	opts := Options{EvidenceSeal: &EvidenceSeal{Store: store, Stream: "ws"}}
+	if declared != "" {
+		opts.ProjectChecks = []instruction.VerifyCheck{{Command: declared, SourcePath: "REASONIX.md", Line: 3}}
+	}
+	a := New(prov, reg, sessionstore.NewSession(""), opts, sink)
 	return a, sink
 }
 
@@ -74,7 +76,7 @@ func TestContractOwesTheCheckTheTaskBeganUnder(t *testing.T) {
 		t.Fatal(err)
 	}
 	want := contract.Derive(contract.Sources{Checks: []string{evidence.VerificationIdentity(began)}})
-	if !slices.Equal(c.Criteria, want) {
+	if !slices.EqualFunc(c.Criteria, want, contract.Criterion.Equal) {
 		t.Fatalf("criteria = %+v, want %+v", c.Criteria, want)
 	}
 }
@@ -159,3 +161,74 @@ func TestContractLoadFailureIsNotReplaced(t *testing.T) {
 		t.Fatalf("audit = %+v, want a failure and no revision accepted in its place", got)
 	}
 }
+
+func verifiedPlan() plancontract.Plan {
+	return plancontract.Plan{
+		Objective: "fix the parser",
+		Steps: []plancontract.Step{
+			{
+				ID: "p1", Title: "fix it",
+				Acceptance: []plancontract.Criterion{
+					{Text: "quoted fields parse"},
+					{Text: "timing is logged", Optional: true},
+				},
+				Verification: []plancontract.Verification{{Command: "go test ./parser/"}},
+			},
+			{ID: "p2", Title: "document it", Acceptance: []plancontract.Criterion{{Text: "the README says so"}}},
+		},
+	}.Normalize()
+}
+
+// An approved plan's criterion is verified by the commands its step names:
+// they came with the plan the user approved, so the host runs them against the
+// ledger instead of taking a citation on the model's word.
+func TestPlanCriterionIsVerifiedByItsStepsCommands(t *testing.T) {
+	plan := verifiedPlan()
+	quoted, readme := plan.Steps[0].Acceptance[0].ID, plan.Steps[1].Acceptance[0].ID
+	for _, tc := range []struct {
+		ran  string
+		want string
+	}{{"go test ./parser/", "satisfied"}, {"go vet ./...", "owed"}} {
+		store := trustedstate.Open(t.TempDir(), nil)
+		a, sink := contractAgent(t, writeThenRun(tc.ran), "", store)
+		a.SetPlanContract(&plan)
+		_ = a.Run(deliveryGoalContext("goal-1", "fix"), "fix")
+		record := sink.last(t).Record
+		if v := sealedVerdict(t, store, record, "contract@plan@"+quoted); v != tc.want {
+			t.Fatalf("after %q the plan criterion is %q, want %q", tc.ran, v, tc.want)
+		}
+		if v := sealedVerdict(t, store, record, "contract@plan@"+readme); v != "unverifiable" {
+			t.Fatalf("a criterion whose step names no command is %q, want unverifiable", v)
+		}
+		if hasObligation(t, store, record, "criterion@"+quoted) {
+			t.Fatal("the plan criterion is counted twice: once frozen, once from the replayed contract")
+		}
+	}
+}
+
+func hasObligation(t *testing.T, store *trustedstate.Store, record, id string) bool {
+	t.Helper()
+	rec, err := store.Record(trustedstate.Digest(record))
+	if err != nil {
+		t.Fatal(err)
+	}
+	payload, err := store.Object(rec.Payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var b struct {
+		Verdict struct {
+			Obligations []struct {
+				ID string `json:"id"`
+			} `json:"obligations"`
+		} `json:"verdict"`
+	}
+	if err := json.Unmarshal(payload, &b); err != nil {
+		t.Fatal(err)
+	}
+	return slices.ContainsFunc(b.Verdict.Obligations, func(o struct {
+		ID string `json:"id"`
+	}) bool {
+		return o.ID == id
+	})
+}

internal/runtime/contract/contract.go

@@ -10,15 +10,19 @@ import (
 
 // Verifier kinds.
 const (
-	VerifierCommand = "command"
-	VerifierTest    = "test"
+	VerifierCommand  = "command"
+	VerifierTest     = "test"
+	VerifierCommands = "commands"
+	VerifierNone     = "none"
 )
 
 // Verifier is what satisfies a criterion: a verification command recognised
-// by its canonical identity, or a captured test criterion by its identity.
+// by its canonical identity, a captured test criterion by its identity, every
+// command of a plan step by theirs, or nothing the host can run.
 type Verifier struct {
-	Kind     string `json:"kind"`
-	Identity string `json:"identity"`
+	Kind       string   `json:"kind"`
+	Identity   string   `json:"identity,omitempty"`
+	Identities []string `json:"identities,omitempty"`
 }
 
 // Criterion is one accepted acceptance criterion.
@@ -45,12 +49,24 @@ type Sources struct {
 	Checks []string
 	// Tests are captured test criterion identities.
 	Tests []string
+	// Plan are the approved plan's acceptance criteria, each verified by the
+	// verification commands of the step it belongs to.
+	Plan []PlanCriterion
+}
+
+// PlanCriterion is one plan acceptance criterion and the canonical identities
+// of its step's verification commands; none means the host cannot check it.
+type PlanCriterion struct {
+	ID       string
+	Required bool
+	Commands []string
 }
 
 // Sources a criterion came from.
 const (
 	SourceProjectCheck = "project_check"
 	SourceBaselineTest = "baseline_test"
+	SourcePlan         = "plan"
 )
 
 // PolicyTemplate names the host policy that accepts a derived revision.
@@ -74,6 +90,17 @@ func Derive(s Sources) []Criterion {
 	}
 	add(SourceProjectCheck, VerifierCommand, s.Checks)
 	add(SourceBaselineTest, VerifierTest, s.Tests)
+	for _, p := range s.Plan {
+		id := strings.TrimSpace(p.ID)
+		if id == "" || slices.ContainsFunc(out, func(x Criterion) bool { return x.ID == "plan@"+id }) {
+			continue
+		}
+		v := Verifier{Kind: VerifierNone}
+		if ids := canonicalIDs(p.Commands); len(ids) > 0 {
+			v = Verifier{Kind: VerifierCommands, Identities: ids}
+		}
+		out = append(out, Criterion{ID: "plan@" + id, Source: SourcePlan, Required: p.Required, Verifier: v})
+	}
 	slices.SortFunc(out, func(a, b Criterion) int { return strings.Compare(a.ID, b.ID) })
 	return out
 }
@@ -101,7 +128,7 @@ func Accept(current *Contract, parentDigest string, derived []Criterion, id stri
 		return Contract{ID: id, Revision: 1, Criteria: derived, AcceptedBy: PolicyTemplate}, Accepted
 	}
 	for _, c := range current.Criteria {
-		if !slices.Contains(derived, c) {
+		if !slices.ContainsFunc(derived, func(d Criterion) bool { return d.Equal(c) }) {
 			return *current, RelaxationRefused
 		}
 	}
@@ -125,3 +152,21 @@ func (c Contract) Encode() []byte {
 
 // Digest identifies a revision's exact content.
 func (c Contract) Digest() string { return string(trustedstate.DigestOf(c.Encode())) }
+
+func canonicalIDs(in []string) []string {
+	var out []string
+	for _, id := range in {
+		if id = strings.TrimSpace(id); id != "" && !slices.Contains(out, id) {
+			out = append(out, id)
+		}
+	}
+	slices.Sort(out)
+	return out
+}
+
+// Equal reports whether two criteria demand the same thing of the same verifier.
+func (c Criterion) Equal(o Criterion) bool {
+	return c.ID == o.ID && c.Source == o.Source && c.Required == o.Required &&
+		c.Verifier.Kind == o.Verifier.Kind && c.Verifier.Identity == o.Verifier.Identity &&
+		slices.Equal(c.Verifier.Identities, o.Verifier.Identities)
+}

internal/runtime/contract/contract_test.go

@@ -12,7 +12,7 @@ import (
 func TestDeriveIsCanonical(t *testing.T) {
 	a := Derive(Sources{Checks: []string{"go test ./...", "  ", "go vet ./..."}, Tests: []string{"baseline_test@x@1"}})
 	b := Derive(Sources{Checks: []string{"go vet ./...", "go test ./...", "go test ./..."}, Tests: []string{"baseline_test@x@1"}})
-	if !slices.Equal(a, b) || len(a) != 3 {
+	if !slices.EqualFunc(a, b, Criterion.Equal) || len(a) != 3 {
 		t.Fatalf("derivations differ or kept blanks/duplicates:\n%v\n%v", a, b)
 	}
 	if (Contract{Criteria: a}).Digest() != (Contract{Criteria: b}).Digest() {
@@ -71,3 +71,22 @@ func TestLoadRefusesARecordOfAnotherKind(t *testing.T) {
 		t.Fatal("a malformed record name loaded")
 	}
 }
+
+func TestPlanCriteriaCarryTheirStepsCommands(t *testing.T) {
+	got := Derive(Sources{Plan: []PlanCriterion{
+		{ID: "c1", Required: true, Commands: []string{"go vet ./...", "go test ./...", "go test ./..."}},
+		{ID: "c2", Required: false},
+	}})
+	want := []Criterion{
+		{ID: "plan@c1", Source: SourcePlan, Required: true, Verifier: Verifier{Kind: VerifierCommands, Identities: []string{"go test ./...", "go vet ./..."}}},
+		{ID: "plan@c2", Source: SourcePlan, Verifier: Verifier{Kind: VerifierNone}},
+	}
+	if !slices.EqualFunc(got, want, Criterion.Equal) {
+		t.Fatalf("Derive = %+v, want %+v", got, want)
+	}
+	first, _ := Accept(nil, "", got, "t")
+	changed := Derive(Sources{Plan: []PlanCriterion{{ID: "c1", Required: true, Commands: []string{"go test ./..."}}, {ID: "c2"}}})
+	if _, dec := Accept(&first, "r", changed, "t"); dec != RelaxationRefused {
+		t.Fatalf("dropping a verifier command was %s, want refused", dec)
+	}
+}

internal/runtime/verdict/verdict.go

@@ -6,6 +6,8 @@
 package verdict
 
 import (
+	"strings"
+
 	"reasonix/internal/runtime/taskcontract"
 	"reasonix/internal/safety/evidence"
 )
@@ -52,6 +54,8 @@ type Frozen struct {
 	Identity     string
 	Satisfied    bool
 	Unverifiable bool
+	// NoVerifier marks a criterion the accepted contract gives nothing to run.
+	NoVerifier bool
 }
 
 // Obligation is one derived obligation and its verdict.
@@ -87,8 +91,17 @@ type Input struct {
 // Evaluate derives every obligation's verdict and the outcome.
 func Evaluate(in Input) Result {
 	var obs []Obligation
+	frozenPlan := map[string]bool{}
+	for _, f := range in.Frozen {
+		if id, ok := strings.CutPrefix(f.ID, "plan@"); ok {
+			frozenPlan[id] = true
+		}
+	}
 	if c := in.Contract; c != nil {
 		for _, req := range c.Requirements {
+			if frozenPlan[req.ID] {
+				continue
+			}
 			obs = append(obs, criterion(req, in.Receipts))
 		}
 		for _, check := range c.Checks {
@@ -173,6 +186,8 @@ func outcomeOf(obs []Obligation, blocked bool) Outcome {
 func frozenObligation(f Frozen) Obligation {
 	o := Obligation{ID: "contract@" + f.ID, Source: "contract:" + f.Source, Required: true}
 	switch {
+	case f.NoVerifier:
+		o.Verdict, o.Cause = Unverifiable, CauseNoVerifier
 	case f.Unverifiable:
 		o.Verdict, o.Cause = Unverifiable, CauseSubjectMissing
 	case f.Satisfied:

internal/session/control/goal_resume_contract_test.go

@@ -69,7 +69,7 @@ func TestGoalResumeKeepsTheAcceptedContractRevision(t *testing.T) {
 		ID: "command@" + evidence.VerificationIdentity(checkA), Source: contract.SourceProjectCheck, Required: true,
 		Verifier: contract.Verifier{Kind: contract.VerifierCommand, Identity: evidence.VerificationIdentity(checkA)},
 	}
-	if c.Revision != 1 || !slices.Equal(c.Criteria, []contract.Criterion{want}) {
+	if c.Revision != 1 || !slices.EqualFunc(c.Criteria, []contract.Criterion{want}, contract.Criterion.Equal) {
 		t.Fatalf("revision = %+v, want revision 1 holding only the check the goal began under", c)
 	}
 }