test(evidence): cover completion adjudication and path evidence

internal/safety/evidence/completion_adjudication_test.go

@@ -0,0 +1,97 @@
+package evidence
+
+import (
+	"strings"
+	"testing"
+)
+
+func report(status CompletionStatus, criteria ...AcceptanceCriterion) CompletionReport {
+	return CompletionReport{Status: status, Summary: "done", Criteria: criteria}
+}
+
+func satisfied(id string, evidence ...CompletionEvidence) AcceptanceCriterion {
+	return AcceptanceCriterion{ID: id, Status: CriterionSatisfied, Evidence: evidence}
+}
+
+func TestAdjudicateCompletionKeepsWhatTheHostsOwnReceiptsBack(t *testing.T) {
+	l := recorded(
+		Receipt{ToolName: "bash", Success: true, Command: "go test ./..."},
+		Receipt{ToolName: "edit_file", Success: true, Write: true, Mutation: true, Paths: []string{"a.go"}},
+		Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"b.go"}},
+	)
+	in := report(CompletionComplete,
+		satisfied("tests", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test ./..."}),
+		satisfied("diff", CompletionEvidence{Kind: CompletionEvidenceDiff, Paths: []string{"a.go"}}),
+		satisfied("files", CompletionEvidence{Kind: CompletionEvidenceFiles, Paths: []string{"b.go"}}),
+	)
+	got, reasons := l.AdjudicateCompletion(in)
+	if got.Status != CompletionComplete || len(reasons) != 0 {
+		t.Fatalf("a fully backed report was lowered: %q %v", got.Status, reasons)
+	}
+	for _, c := range got.Criteria {
+		if c.Status != CriterionSatisfied {
+			t.Fatalf("criterion %s was downgraded", c.ID)
+		}
+	}
+}
+
+func TestAdjudicateCompletionDowngradesEveryClaimNoReceiptBacksAndCannotStayComplete(t *testing.T) {
+	l := recorded(
+		Receipt{ToolName: "bash", Success: false, Command: "go test ./..."},
+		Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"a.go"}},
+	)
+	in := report(CompletionComplete,
+		satisfied("failed-run", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test ./..."}),
+		satisfied("read-is-not-a-write", CompletionEvidence{Kind: CompletionEvidenceDiff, Paths: []string{"a.go"}}),
+		satisfied("manual-only", CompletionEvidence{Kind: CompletionEvidenceManual, Summary: "trust me"}),
+		satisfied("no-evidence"),
+		AcceptanceCriterion{ID: "already-unsatisfied", Status: CriterionUnsatisfied},
+	)
+	got, reasons := l.AdjudicateCompletion(in)
+	if got.Status != CompletionPartial {
+		t.Fatalf("status = %q, want the report lowered to partial", got.Status)
+	}
+	if len(reasons) != 4 {
+		t.Fatalf("reasons = %v, want one per downgraded claim", reasons)
+	}
+	for _, want := range []string{"failed-run: no successful run", "read-is-not-a-write: no write recorded", "manual-only:", "no-evidence: claimed satisfied with no evidence"} {
+		found := false
+		for _, r := range reasons {
+			found = found || strings.Contains(r, want)
+		}
+		if !found {
+			t.Errorf("no reason mentions %q: %v", want, reasons)
+		}
+	}
+	for _, c := range got.Criteria {
+		if c.Status != CriterionUnsatisfied {
+			t.Errorf("criterion %s is %s after adjudication", c.ID, c.Status)
+		}
+	}
+}
+
+func TestAdjudicateCompletionNeverRaisesAStatusAndAcceptsANilLedger(t *testing.T) {
+	l := recorded(Receipt{ToolName: "bash", Success: true, Command: "make check"})
+	for _, status := range []CompletionStatus{CompletionPartial, CompletionBlocked, CompletionFailed} {
+		in := report(status, satisfied("ok", CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "make check"}))
+		if got, _ := l.AdjudicateCompletion(in); got.Status != status {
+			t.Errorf("%s became %s: the host may lower a claim, never raise it", status, got.Status)
+		}
+	}
+	var nilLedger *Ledger
+	in := report(CompletionComplete, satisfied("x"))
+	if got, reasons := nilLedger.AdjudicateCompletion(in); got.Status != CompletionComplete || reasons != nil {
+		t.Fatalf("a nil ledger adjudicates nothing: %q %v", got.Status, reasons)
+	}
+}
+
+func TestAdjudicateCompletionSaysACitedMissIsOneProblemNotTwo(t *testing.T) {
+	l := recorded()
+	got, reasons := l.AdjudicateCompletion(report(CompletionComplete, satisfied("twice",
+		CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test"},
+		CompletionEvidence{Kind: CompletionEvidenceVerification, Command: "go test"},
+	)))
+	if len(reasons) != 1 || strings.Count(reasons[0], "no successful run") != 1 || got.Status != CompletionPartial {
+		t.Fatalf("reasons = %v status = %q", reasons, got.Status)
+	}
+}

internal/safety/evidence/path_evidence_test.go

@@ -0,0 +1,72 @@
+package evidence
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"slices"
+	"testing"
+)
+
+func TestToolCallPathsReadsOnlyDeclaredPathFieldsTrimmedAndDistinct(t *testing.T) {
+	args := json.RawMessage(`{"path":" a.go ","file_path":"a.go","notebook_path":"n.ipynb","source_path":"s","destination_path":"d","paths":["p1"," p2","p1",""],"file_paths":["f1"],"command":"rm -rf x","content":"not/a/path.go"}`)
+	got := ToolCallPaths(args)
+	want := []string{"a.go", "n.ipynb", "s", "d", "p1", "p2", "f1"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("ToolCallPaths = %v, want %v", got, want)
+	}
+	for name, raw := range map[string]string{"not json": `{`, "a list": `["a.go"]`, "no paths": `{"command":"ls"}`, "non-string path": `{"path":5}`} {
+		if got := ToolCallPaths(json.RawMessage(raw)); len(got) != 0 {
+			t.Errorf("%s: ToolCallPaths = %v, want none (bash and unknown targets are not parsed)", name, got)
+		}
+	}
+}
+
+func TestTouchedPathsAreRecentFirstDistinctBoundedAndSayWhetherTheyWereWritten(t *testing.T) {
+	l := recorded(
+		Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"old.go"}},
+		Receipt{ToolName: "edit_file", Success: true, Write: true, Paths: []string{"a.go", "b.go"}},
+		Receipt{ToolName: "edit_file", Success: false, Write: true, Paths: []string{"failed.go"}},
+		Receipt{ToolName: "bash", Success: true, Paths: []string{"not-a-read-or-write.go"}},
+		Receipt{ToolName: "read_file", Success: true, Read: true, Paths: []string{"a.go", "c.go"}},
+	)
+	if got, want := l.TouchedPaths(10, false), []string{"a.go", "c.go", "b.go", "old.go"}; !slices.Equal(got, want) {
+		t.Fatalf("touched = %v, want %v", got, want)
+	}
+	if got, want := l.TouchedPaths(10, true), []string{"a.go", "b.go"}; !slices.Equal(got, want) {
+		t.Fatalf("written = %v, want %v", got, want)
+	}
+	if got := l.TouchedPaths(2, false); len(got) != 2 {
+		t.Fatalf("limit ignored: %v", got)
+	}
+	var nilLedger *Ledger
+	if nilLedger.TouchedPaths(5, false) != nil || l.TouchedPaths(0, false) != nil {
+		t.Fatal("a nil ledger and a zero limit touch nothing")
+	}
+}
+
+func TestCitationFormsNamesTheSameFileRelativeAndAbsoluteButNeverOneOutsideTheRoot(t *testing.T) {
+	root := t.TempDir()
+	abs := filepath.Join(root, "pkg", "a.go")
+	rel := filepath.Join("pkg", "a.go")
+	for name, tc := range map[string]struct {
+		cited string
+		want  []string
+	}{
+		"relative inside":    {rel, []string{normalizePath(rel), normalizePath(abs)}},
+		"absolute inside":    {abs, []string{normalizePath(abs), normalizePath(rel)}},
+		"relative escaping":  {filepath.Join("..", "x.go"), []string{normalizePath(filepath.Join("..", "x.go"))}},
+		"absolute outside":   {filepath.Join(filepath.Dir(root), "other", "x.go"), []string{normalizePath(filepath.Join(filepath.Dir(root), "other", "x.go"))}},
+		"the root itself":    {root, []string{normalizePath(root)}},
+		"backslash relative": {"pkg\\a.go", []string{normalizePath(rel), normalizePath(abs)}},
+	} {
+		if got := CitationForms(root, tc.cited); !slices.Equal(got, tc.want) {
+			t.Errorf("%s: CitationForms(%q) = %v, want %v", name, tc.cited, got, tc.want)
+		}
+	}
+	if got := CitationForms("", rel); !slices.Equal(got, []string{normalizePath(rel)}) {
+		t.Errorf("no root: %v", got)
+	}
+	if got := CitationForms(root, "   "); got != nil {
+		t.Errorf("a blank citation names no file: %v", got)
+	}
+}