mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-12 08:41:32 -10:00
Retry shell commands outside the sandbox when sandboxed output only reflects the sandbox namespace, and keep network retries sandboxed when a network grant is enough. Show user-facing permission reasons for escalated shell commands instead of exposing internal sandbox policy text. Tested: env GOCACHE=/tmp/zero-go-cache go test ./internal/agent ./internal/tui -run 'TestRunUsesEscalatedJustificationForPermissionPrompt|TestRunUsesUserFacingEscalatedFallbackForPermissionPrompt|TestPermissionPromptMapsEscalatedSandboxReason|TestRunRetriesShellUnsandboxedAfterSandboxNamespaceLimitedOutput' Tested: env GOCACHE=/tmp/zero-go-cache go test ./internal/tools Tested: env HOME=/tmp/zero-test-home-clean XDG_CONFIG_HOME=/tmp/zero-test-config-clean GOCACHE=/tmp/zero-go-cache go test ./...
3248 lines
119 KiB
Go
3248 lines
119 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/Gitlawb/zero/internal/sandbox"
|
|
"github.com/Gitlawb/zero/internal/specmode"
|
|
"github.com/Gitlawb/zero/internal/tools"
|
|
"github.com/Gitlawb/zero/internal/zeroruntime"
|
|
)
|
|
|
|
type mockProvider struct {
|
|
turns [][]zeroruntime.StreamEvent
|
|
requests []zeroruntime.CompletionRequest
|
|
}
|
|
|
|
func (provider *mockProvider) StreamCompletion(ctx context.Context, request zeroruntime.CompletionRequest) (<-chan zeroruntime.StreamEvent, error) {
|
|
provider.requests = append(provider.requests, request)
|
|
|
|
events := []zeroruntime.StreamEvent{{Type: zeroruntime.StreamEventDone}}
|
|
if len(provider.turns) >= len(provider.requests) {
|
|
events = provider.turns[len(provider.requests)-1]
|
|
}
|
|
|
|
ch := make(chan zeroruntime.StreamEvent, len(events))
|
|
for _, event := range events {
|
|
ch <- event
|
|
}
|
|
close(ch)
|
|
return ch, nil
|
|
}
|
|
|
|
type sandboxDeniedRetryTool struct {
|
|
calls []map[string]any
|
|
}
|
|
|
|
func (tool *sandboxDeniedRetryTool) Name() string { return "bash" }
|
|
func (tool *sandboxDeniedRetryTool) Description() string { return "test shell tool" }
|
|
func (tool *sandboxDeniedRetryTool) Parameters() tools.Schema {
|
|
return tools.Schema{
|
|
Type: "object",
|
|
Properties: map[string]tools.PropertySchema{
|
|
"command": {Type: "string"},
|
|
"sandbox_permissions": {Type: "string"},
|
|
"justification": {Type: "string"},
|
|
},
|
|
Required: []string{"command"},
|
|
AdditionalProperties: false,
|
|
}
|
|
}
|
|
func (tool *sandboxDeniedRetryTool) Safety() tools.Safety {
|
|
return tools.Safety{SideEffect: tools.SideEffectShell, Permission: tools.PermissionPrompt, Reason: "runs shell commands"}
|
|
}
|
|
func (tool *sandboxDeniedRetryTool) Run(_ context.Context, args map[string]any) tools.Result {
|
|
tool.calls = append(tool.calls, cloneArgs(args))
|
|
if args["sandbox_permissions"] == string(tools.SandboxPermissionsRequireEscalated) {
|
|
return tools.Result{Status: tools.StatusOK, Output: "installed"}
|
|
}
|
|
return tools.Result{
|
|
Status: tools.StatusError,
|
|
Output: "touch: cannot touch '/home/user/.npm/cache': Read-only file system",
|
|
Meta: map[string]string{
|
|
"exit_code": "1",
|
|
tools.SandboxLikelyDeniedMeta: "true",
|
|
tools.SandboxDenialReasonMeta: "sandbox blocked command execution",
|
|
tools.SandboxDenialKeywordMeta: "read-only file system",
|
|
},
|
|
}
|
|
}
|
|
|
|
type sandboxDeniedExecCommandRetryTool struct {
|
|
sandboxDeniedRetryTool
|
|
}
|
|
|
|
func (tool *sandboxDeniedExecCommandRetryTool) Name() string { return "exec_command" }
|
|
|
|
type sandboxNetworkDeniedRetryTool struct {
|
|
calls []map[string]any
|
|
}
|
|
|
|
func (tool *sandboxNetworkDeniedRetryTool) Name() string { return "bash" }
|
|
func (tool *sandboxNetworkDeniedRetryTool) Description() string { return "test shell tool" }
|
|
func (tool *sandboxNetworkDeniedRetryTool) Parameters() tools.Schema {
|
|
return tools.Schema{
|
|
Type: "object",
|
|
Properties: map[string]tools.PropertySchema{
|
|
"command": {Type: "string"},
|
|
},
|
|
Required: []string{"command"},
|
|
AdditionalProperties: false,
|
|
}
|
|
}
|
|
func (tool *sandboxNetworkDeniedRetryTool) Safety() tools.Safety {
|
|
return tools.Safety{SideEffect: tools.SideEffectShell, Permission: tools.PermissionPrompt, Reason: "runs shell commands"}
|
|
}
|
|
func (tool *sandboxNetworkDeniedRetryTool) Run(ctx context.Context, args map[string]any) tools.Result {
|
|
return tool.RunWithOptions(ctx, args, tools.RunOptions{})
|
|
}
|
|
func (tool *sandboxNetworkDeniedRetryTool) RunWithOptions(ctx context.Context, args map[string]any, options tools.RunOptions) tools.Result {
|
|
tool.calls = append(tool.calls, cloneArgs(args))
|
|
if shellNetworkAllowed(ctx, options.Sandbox, args) {
|
|
return tools.Result{Status: tools.StatusOK, Output: "server started"}
|
|
}
|
|
return tools.Result{
|
|
Status: tools.StatusOK,
|
|
Output: "Cannot open a network socket.",
|
|
Meta: map[string]string{
|
|
"exit_code": "0",
|
|
tools.SandboxLikelyDeniedMeta: "true",
|
|
tools.SandboxDenialKindMeta: tools.SandboxDenialKindNetwork,
|
|
tools.SandboxDenialReasonMeta: "sandbox blocked command execution",
|
|
tools.SandboxDenialKeywordMeta: "cannot open a network socket",
|
|
},
|
|
}
|
|
}
|
|
|
|
func shellNetworkAllowed(ctx context.Context, engine *sandbox.Engine, args map[string]any) bool {
|
|
if engine == nil {
|
|
return false
|
|
}
|
|
decision := engine.Evaluate(ctx, sandbox.Request{
|
|
ToolName: "bash",
|
|
SideEffect: sandbox.SideEffectShell,
|
|
Permission: sandbox.PermissionPrompt,
|
|
PermissionGranted: true,
|
|
PermissionMode: sandbox.PermissionModeAsk,
|
|
Args: map[string]any{"command": "curl https://example.com"},
|
|
})
|
|
return decision.Action == sandbox.ActionAllow
|
|
}
|
|
|
|
func agentNativeBackendStub() sandbox.Backend {
|
|
return sandbox.Backend{
|
|
Name: sandbox.BackendLinuxBwrap,
|
|
Available: true,
|
|
Executable: "/nonexistent/zero-linux-sandbox-stub",
|
|
CommandWrapping: true,
|
|
NativeIsolation: true,
|
|
}
|
|
}
|
|
|
|
type sandboxNamespaceLimitedRetryTool struct {
|
|
calls []map[string]any
|
|
}
|
|
|
|
func (tool *sandboxNamespaceLimitedRetryTool) Name() string { return "bash" }
|
|
func (tool *sandboxNamespaceLimitedRetryTool) Description() string { return "test shell tool" }
|
|
func (tool *sandboxNamespaceLimitedRetryTool) Parameters() tools.Schema {
|
|
return tools.Schema{
|
|
Type: "object",
|
|
Properties: map[string]tools.PropertySchema{
|
|
"command": {Type: "string"},
|
|
"sandbox_permissions": {Type: "string"},
|
|
},
|
|
Required: []string{"command"},
|
|
AdditionalProperties: false,
|
|
}
|
|
}
|
|
func (tool *sandboxNamespaceLimitedRetryTool) Safety() tools.Safety {
|
|
return tools.Safety{SideEffect: tools.SideEffectShell, Permission: tools.PermissionPrompt, Reason: "runs shell commands"}
|
|
}
|
|
func (tool *sandboxNamespaceLimitedRetryTool) Run(_ context.Context, args map[string]any) tools.Result {
|
|
tool.calls = append(tool.calls, cloneArgs(args))
|
|
if args["sandbox_permissions"] == string(tools.SandboxPermissionsRequireEscalated) {
|
|
return tools.Result{Status: tools.StatusOK, Output: "stdout:\nUSER PID COMMAND\nanaxy 42 firefox\nanaxy 43 Discord"}
|
|
}
|
|
return tools.Result{
|
|
Status: tools.StatusOK,
|
|
Output: "stdout:\nUSER PID COMMAND\nanaxy 1 bwrap --new-session --die-with-parent --unshare-user --unshare-pid --unshare-net --proc /proc -- /bin/sh -c ps aux\nanaxy 2 ps aux",
|
|
Meta: map[string]string{
|
|
"sandbox_backend": string(sandbox.BackendLinuxBwrap),
|
|
"sandbox_target_backend": string(sandbox.BackendLinuxBwrap),
|
|
"sandbox_wrapped": "true",
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestRunRetriesShellUnsandboxedAfterSandboxNamespaceLimitedOutput(t *testing.T) {
|
|
root := t.TempDir()
|
|
retryTool := &sandboxNamespaceLimitedRetryTool{}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(retryTool)
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"ps aux"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
var events []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "check running applications", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: agentNativeBackendStub(),
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllowPrefix, Reason: "inspect host processes"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
events = append(events, event)
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("final answer = %q", result.FinalAnswer)
|
|
}
|
|
if len(retryTool.calls) != 2 {
|
|
t.Fatalf("tool calls = %#v, want sandboxed attempt plus unsandboxed retry", retryTool.calls)
|
|
}
|
|
if retryTool.calls[1]["sandbox_permissions"] != string(tools.SandboxPermissionsRequireEscalated) {
|
|
t.Fatalf("retry args = %#v, want require_escalated", retryTool.calls[1])
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("permission requests = %#v, want one unsandboxed retry approval", requests)
|
|
}
|
|
if !strings.Contains(requests[0].Reason, "sandbox PID namespace") {
|
|
t.Fatalf("permission reason = %q, want sandbox namespace reason", requests[0].Reason)
|
|
}
|
|
if !equalStringSlices(requests[0].CommandPrefix, []string{"ps", "aux"}) {
|
|
t.Fatalf("request command prefix = %#v, want ps aux", requests[0].CommandPrefix)
|
|
}
|
|
if len(events) != 1 || events[0].Action != PermissionActionAllow || events[0].DecisionAction != PermissionDecisionAllowPrefix {
|
|
t.Fatalf("permission events = %#v, want approved unsandboxed retry", events)
|
|
}
|
|
if len(provider.requests) < 2 {
|
|
t.Fatalf("provider requests = %d, want retry result sent back to model", len(provider.requests))
|
|
}
|
|
last := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if !strings.Contains(last.Content, "firefox") || strings.Contains(last.Content, "bwrap --new-session") {
|
|
t.Fatalf("tool result sent to model = %q, want unsandboxed retry output only", last.Content)
|
|
}
|
|
}
|
|
|
|
func TestRunUsesEscalatedJustificationForPermissionPrompt(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(&sandboxDeniedRetryTool{})
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"ps aux","sandbox_permissions":"require_escalated","justification":"Need host process visibility."}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requestReasons []string
|
|
|
|
_, err := Run(context.Background(), "check host processes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: agentNativeBackendStub(),
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requestReasons = append(requestReasons, request.Reason)
|
|
return PermissionDecision{Action: PermissionDecisionDeny, Reason: "not needed"}, nil
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(requestReasons) != 1 {
|
|
t.Fatalf("permission reasons = %#v, want one request", requestReasons)
|
|
}
|
|
if requestReasons[0] != "Need host process visibility." {
|
|
t.Fatalf("permission reason = %q, want model justification", requestReasons[0])
|
|
}
|
|
}
|
|
|
|
func TestRunUsesUserFacingEscalatedFallbackForPermissionPrompt(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(&sandboxDeniedExecCommandRetryTool{})
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "exec_command"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"ps aux","sandbox_permissions":"require_escalated"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requestReasons []string
|
|
|
|
_, err := Run(context.Background(), "check host processes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: agentNativeBackendStub(),
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requestReasons = append(requestReasons, request.Reason)
|
|
return PermissionDecision{Action: PermissionDecisionDeny, Reason: "not needed"}, nil
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(requestReasons) != 1 {
|
|
t.Fatalf("permission reasons = %#v, want one request", requestReasons)
|
|
}
|
|
if requestReasons[0] != "This command needs to run outside the sandbox." {
|
|
t.Fatalf("permission reason = %q, want user-facing fallback", requestReasons[0])
|
|
}
|
|
if strings.Contains(requestReasons[0], sandbox.ReasonEscalatedSandboxRequired) {
|
|
t.Fatalf("permission reason leaked internal sandbox copy: %q", requestReasons[0])
|
|
}
|
|
}
|
|
|
|
func TestRunRetriesNetworkDeniedShellWithNetworkGrant(t *testing.T) {
|
|
root := t.TempDir()
|
|
retryTool := &sandboxNetworkDeniedRetryTool{}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(retryTool)
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"node server.js"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
|
|
result, err := Run(context.Background(), "start server", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: agentNativeBackendStub(),
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllow, Reason: "approve network"}, nil
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("final answer = %q", result.FinalAnswer)
|
|
}
|
|
if len(retryTool.calls) != 2 {
|
|
t.Fatalf("tool calls = %#v, want denied attempt plus network-granted retry", retryTool.calls)
|
|
}
|
|
if _, escalated := retryTool.calls[1]["sandbox_permissions"]; escalated {
|
|
t.Fatalf("network retry must stay sandboxed, got retry args %#v", retryTool.calls[1])
|
|
}
|
|
if len(requests) != 1 || requests[0].Reason != sandbox.ReasonNetworkBlocked {
|
|
t.Fatalf("permission requests = %#v, want one network approval", requests)
|
|
}
|
|
if len(provider.requests) < 2 {
|
|
t.Fatalf("provider requests = %d, want retry result sent back to model", len(provider.requests))
|
|
}
|
|
last := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if !strings.Contains(last.Content, "server started") {
|
|
t.Fatalf("tool result sent to model = %q, want network retry output", last.Content)
|
|
}
|
|
}
|
|
|
|
func TestIsRetriableToolError(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
result ToolResult
|
|
want bool
|
|
}{
|
|
{"success", ToolResult{Status: tools.StatusOK}, false},
|
|
{"bad arguments", ToolResult{Status: tools.StatusError, Output: "Error: Failed to parse arguments for x: bad json"}, true},
|
|
{"execution failure", ToolResult{Status: tools.StatusError, Output: "Error: read foo.txt: no such file"}, true},
|
|
{"disabled tool", ToolResult{Status: tools.StatusError, Output: `Error: Tool "x" is not enabled for this run.`}, false},
|
|
{"permission denied (meta)", ToolResult{Status: tools.StatusError, Output: "Error: Permission denied for x: needs approval", Meta: map[string]string{"permission_action": "deny"}}, false},
|
|
{"permission required", ToolResult{Status: tools.StatusError, Output: "Error: Permission required for x: approve first"}, false},
|
|
{"sandbox block", ToolResult{Status: tools.StatusError, Output: "Error: Sandbox block: outside_workspace"}, false},
|
|
{"sandbox approval", ToolResult{Status: tools.StatusError, Output: "Error: Sandbox approval required for x: network"}, false},
|
|
// Structured denial categories are authoritative regardless of message text.
|
|
{"denial: filtered", ToolResult{Status: tools.StatusError, Output: "anything", DenialReason: DenialFiltered}, false},
|
|
{"denial: permission", ToolResult{Status: tools.StatusError, Output: "anything", DenialReason: DenialPermissionDenied}, false},
|
|
{"denial: approval canceled", ToolResult{Status: tools.StatusError, Output: "anything", DenialReason: DenialApprovalCanceled}, false},
|
|
{"denial: sandbox", ToolResult{Status: tools.StatusError, Output: "anything", DenialReason: DenialSandboxBlock}, false},
|
|
}
|
|
for _, c := range cases {
|
|
if got := isRetriableToolError(c.result); got != c.want {
|
|
t.Errorf("%s: isRetriableToolError = %v, want %v", c.name, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunRetriesShellUnsandboxedAfterSandboxDeniedExit(t *testing.T) {
|
|
root := t.TempDir()
|
|
retryTool := &sandboxDeniedRetryTool{}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(retryTool)
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"npm install --save http-server"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
var events []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "install package", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: agentNativeBackendStub(),
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllow, Reason: "retry outside sandbox"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
events = append(events, event)
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("final answer = %q", result.FinalAnswer)
|
|
}
|
|
if len(retryTool.calls) != 2 {
|
|
t.Fatalf("tool calls = %#v, want sandboxed attempt plus unsandboxed retry", retryTool.calls)
|
|
}
|
|
if retryTool.calls[1]["sandbox_permissions"] != string(tools.SandboxPermissionsRequireEscalated) {
|
|
t.Fatalf("retry args = %#v, want require_escalated", retryTool.calls[1])
|
|
}
|
|
if len(requests) != 2 {
|
|
t.Fatalf("permission requests = %#v, want network approval plus retry approval", requests)
|
|
}
|
|
if requests[0].Reason != sandbox.ReasonNetworkBlocked {
|
|
t.Fatalf("first request reason = %q, want network approval", requests[0].Reason)
|
|
}
|
|
if !strings.Contains(requests[1].Reason, "sandbox blocked command execution") || !strings.Contains(requests[1].Reason, "read-only file system") {
|
|
t.Fatalf("retry request reason = %q", requests[1].Reason)
|
|
}
|
|
if len(events) != 1 || events[0].Action != PermissionActionAllow || events[0].DecisionAction != PermissionDecisionAllow {
|
|
t.Fatalf("permission events = %#v, want approved unsandboxed retry", events)
|
|
}
|
|
if len(provider.requests) < 2 {
|
|
t.Fatalf("provider requests = %d, want retry result sent back to model", len(provider.requests))
|
|
}
|
|
last := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if !strings.Contains(last.Content, "installed") {
|
|
t.Fatalf("tool result sent to model = %q, want retry output", last.Content)
|
|
}
|
|
}
|
|
|
|
func TestRunDoesNotRetryUnsandboxedWhenDeniedReadsActive(t *testing.T) {
|
|
root := t.TempDir()
|
|
retryTool := &sandboxDeniedRetryTool{}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(retryTool)
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"touch cache-file"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
var events []PermissionEvent
|
|
policy := sandbox.DefaultPolicy()
|
|
policy.DenyRead = []string{filepath.Join(root, "secret.txt")}
|
|
|
|
result, err := Run(context.Background(), "touch cache", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: policy,
|
|
Backend: agentNativeBackendStub(),
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllow, Reason: "unexpected"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
events = append(events, event)
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("final answer = %q", result.FinalAnswer)
|
|
}
|
|
if len(retryTool.calls) != 1 {
|
|
t.Fatalf("tool calls = %#v, want only the original sandboxed attempt", retryTool.calls)
|
|
}
|
|
if len(requests) != 0 {
|
|
t.Fatalf("permission requests = %#v, want no unsandboxed retry request", requests)
|
|
}
|
|
for _, event := range events {
|
|
if strings.Contains(event.Reason, "sandbox blocked command execution") {
|
|
t.Fatalf("permission events = %#v, want no unsandboxed retry event", events)
|
|
}
|
|
}
|
|
if len(provider.requests) < 2 {
|
|
t.Fatalf("provider requests = %d, want tool result sent back to model", len(provider.requests))
|
|
}
|
|
last := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if !strings.Contains(last.Content, "Read-only file system") {
|
|
t.Fatalf("tool result sent to model = %q, want original sandbox denial", last.Content)
|
|
}
|
|
}
|
|
|
|
func TestRunReturnsProviderText(t *testing.T) {
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "hello"},
|
|
{Type: zeroruntime.StreamEventText, Content: " zero"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "say hi", provider, Options{
|
|
Registry: tools.NewRegistry(),
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "hello zero" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
if len(provider.requests) != 1 {
|
|
t.Fatalf("expected one provider turn, got %d", len(provider.requests))
|
|
}
|
|
assertMessage(t, provider.requests[0].Messages[0], zeroruntime.MessageRoleSystem, "")
|
|
assertMessage(t, provider.requests[0].Messages[1], zeroruntime.MessageRoleUser, "say hi")
|
|
}
|
|
|
|
func TestRunDoesNotPersistReasoningAsAssistantText(t *testing.T) {
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventReasoning, Content: "private reasoning"},
|
|
{Type: zeroruntime.StreamEventText, Content: "public answer"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
var reasoning []string
|
|
|
|
result, err := Run(context.Background(), "say hi", provider, Options{
|
|
Registry: tools.NewRegistry(),
|
|
OnReasoning: func(delta string) { reasoning = append(reasoning, delta) },
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "public answer" {
|
|
t.Fatalf("final answer = %q, want public answer", result.FinalAnswer)
|
|
}
|
|
if len(result.Messages) == 0 {
|
|
t.Fatal("expected persisted messages")
|
|
}
|
|
last := result.Messages[len(result.Messages)-1]
|
|
if last.Role != zeroruntime.MessageRoleAssistant || last.Content != "public answer" {
|
|
t.Fatalf("assistant message = %#v, want answer-only assistant content", last)
|
|
}
|
|
if len(reasoning) != 1 || reasoning[0] != "private reasoning" {
|
|
t.Fatalf("reasoning callbacks = %#v", reasoning)
|
|
}
|
|
}
|
|
|
|
func TestRunReportsTruncationFinishReason(t *testing.T) {
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "partial answer"},
|
|
{Type: zeroruntime.StreamEventDone, FinishReason: zeroruntime.FinishReasonLength},
|
|
}},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "write a long thing", provider, Options{
|
|
Registry: tools.NewRegistry(),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "partial answer" {
|
|
t.Fatalf("final answer = %q", result.FinalAnswer)
|
|
}
|
|
if result.FinishReason != zeroruntime.FinishReasonLength {
|
|
t.Fatalf("FinishReason = %q, want %q", result.FinishReason, zeroruntime.FinishReasonLength)
|
|
}
|
|
if !result.Truncated() {
|
|
t.Fatal("Truncated() = false, want true for a length-capped response")
|
|
}
|
|
if result.TruncationNotice() == "" {
|
|
t.Fatal("TruncationNotice() empty for a truncated response")
|
|
}
|
|
}
|
|
|
|
func TestRunNormalCompletionIsNotTruncated(t *testing.T) {
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "hi", provider, Options{Registry: tools.NewRegistry()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Truncated() || result.TruncationNotice() != "" {
|
|
t.Fatalf("normal completion reported as truncated: reason=%q", result.FinishReason)
|
|
}
|
|
}
|
|
|
|
func TestResultTruncationNotice(t *testing.T) {
|
|
cases := map[string]struct {
|
|
reason string
|
|
wantNotice bool
|
|
}{
|
|
"length": {zeroruntime.FinishReasonLength, true},
|
|
"content_filter": {zeroruntime.FinishReasonContentFilter, true},
|
|
"unknown": {"weird_reason", true},
|
|
"normal": {"", false},
|
|
}
|
|
for name, tc := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
notice := Result{FinishReason: tc.reason}.TruncationNotice()
|
|
if (notice != "") != tc.wantNotice {
|
|
t.Fatalf("notice = %q, wantNotice = %v", notice, tc.wantNotice)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunEmitsTextDeltas(t *testing.T) {
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "hello"},
|
|
{Type: zeroruntime.StreamEventText, Content: " zero"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
var deltas []string
|
|
_, err := Run(context.Background(), "say hi", provider, Options{
|
|
Registry: tools.NewRegistry(),
|
|
OnText: func(delta string) { deltas = append(deltas, delta) },
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Join(deltas, "|") != "hello| zero" {
|
|
t.Fatalf("expected text deltas, got %#v", deltas)
|
|
}
|
|
}
|
|
|
|
func TestRunEmitsUsageEvents(t *testing.T) {
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventUsage, Usage: zeroruntime.Usage{PromptTokens: 12, CompletionTokens: 5, CachedInputTokens: 2}},
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
var usages []zeroruntime.Usage
|
|
_, err := Run(context.Background(), "track usage", provider, Options{
|
|
Registry: tools.NewRegistry(),
|
|
OnUsage: func(usage zeroruntime.Usage) { usages = append(usages, usage) },
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(usages) != 1 {
|
|
t.Fatalf("expected one usage event, got %#v", usages)
|
|
}
|
|
if usages[0].PromptTokens != 12 || usages[0].CompletionTokens != 5 || usages[0].CachedInputTokens != 2 {
|
|
t.Fatalf("unexpected usage event: %#v", usages[0])
|
|
}
|
|
}
|
|
|
|
func TestRunAdvertisesRuntimeToolDefinitions(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
_, err := Run(context.Background(), "what tools exist?", provider, Options{
|
|
Registry: registry,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(provider.requests) != 1 {
|
|
t.Fatalf("expected one request, got %d", len(provider.requests))
|
|
}
|
|
if len(provider.requests[0].Tools) != 1 {
|
|
t.Fatalf("expected one advertised tool, got %#v", provider.requests[0].Tools)
|
|
}
|
|
|
|
toolDefinition := provider.requests[0].Tools[0]
|
|
if toolDefinition.Name != "read_file" {
|
|
t.Fatalf("expected read_file definition, got %#v", toolDefinition)
|
|
}
|
|
parameters := toolDefinition.Parameters
|
|
if parameters["type"] != "object" {
|
|
t.Fatalf("expected object schema, got %#v", parameters)
|
|
}
|
|
if parameters["additionalProperties"] != false {
|
|
t.Fatalf("expected additionalProperties=false, got %#v", parameters["additionalProperties"])
|
|
}
|
|
properties, ok := parameters["properties"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected properties map, got %#v", parameters["properties"])
|
|
}
|
|
pathProperty, ok := properties["path"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected path property map, got %#v", properties["path"])
|
|
}
|
|
if pathProperty["type"] != "string" || pathProperty["description"] == "" {
|
|
t.Fatalf("unexpected path property schema: %#v", pathProperty)
|
|
}
|
|
required, ok := parameters["required"].([]string)
|
|
if !ok || len(required) != 1 || required[0] != "path" {
|
|
t.Fatalf("unexpected required fields: %#v", parameters["required"])
|
|
}
|
|
}
|
|
|
|
func TestRunAdvertisesWebFetchInAutoMode(t *testing.T) {
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWebFetchTool())
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
_, err := Run(context.Background(), "what tools exist?", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAuto,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(provider.requests) != 1 {
|
|
t.Fatalf("expected one request, got %d", len(provider.requests))
|
|
}
|
|
if len(provider.requests[0].Tools) != 1 || provider.requests[0].Tools[0].Name != "web_fetch" {
|
|
t.Fatalf("expected web_fetch to be advertised in auto mode, got %#v", provider.requests[0].Tools)
|
|
}
|
|
}
|
|
|
|
func TestRunRejectsLocalWebFetchBeforePermissionRequest(t *testing.T) {
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWebFetchTool())
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "web_fetch"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"url":"http://localhost:8000/index.html"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
|
|
result, err := Run(context.Background(), "fetch local page", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionDeny, Reason: "unexpected permission request"}, nil
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("final answer = %q", result.FinalAnswer)
|
|
}
|
|
if len(requests) != 0 {
|
|
t.Fatalf("expected no permission request, got %#v", requests)
|
|
}
|
|
if len(provider.requests) < 2 {
|
|
t.Fatalf("expected tool result to be sent back to provider, got %d requests", len(provider.requests))
|
|
}
|
|
lastMessage := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if lastMessage.ToolCallID != "call-1" {
|
|
t.Fatalf("expected tool_call_id call-1, got %q", lastMessage.ToolCallID)
|
|
}
|
|
if strings.Contains(lastMessage.Content, "Permission required") {
|
|
t.Fatalf("local web_fetch must not request permission first: %q", lastMessage.Content)
|
|
}
|
|
if !strings.Contains(lastMessage.Content, "bash with curl") {
|
|
t.Fatalf("expected curl guidance, got %q", lastMessage.Content)
|
|
}
|
|
}
|
|
|
|
func TestRunAdvertisesAllowedWebSearchInAutoMode(t *testing.T) {
|
|
t.Setenv("ZERO_WEBSEARCH_BASE_URL", "https://search.example/api")
|
|
registry := tools.NewRegistry()
|
|
for _, tool := range tools.CoreNetworkTools() {
|
|
if tool.Name() == "web_search" {
|
|
registry.Register(tool)
|
|
}
|
|
}
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
_, err := Run(context.Background(), "what tools exist?", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAuto,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(provider.requests) != 1 {
|
|
t.Fatalf("expected one request, got %d", len(provider.requests))
|
|
}
|
|
if len(provider.requests[0].Tools) != 1 || provider.requests[0].Tools[0].Name != "web_search" {
|
|
t.Fatalf("expected web_search to be advertised in auto mode, got %#v", provider.requests[0].Tools)
|
|
}
|
|
}
|
|
|
|
func TestRunFiltersAdvertisedTools(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
registry.Register(tools.NewGrepTool(root))
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
_, err := Run(context.Background(), "what tools exist?", provider, Options{
|
|
Registry: registry,
|
|
EnabledTools: []string{"read_file", "grep"},
|
|
DisabledTools: []string{"grep"},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(provider.requests) != 1 {
|
|
t.Fatalf("expected one request, got %d", len(provider.requests))
|
|
}
|
|
if len(provider.requests[0].Tools) != 1 || provider.requests[0].Tools[0].Name != "read_file" {
|
|
t.Fatalf("expected only read_file to be advertised, got %#v", provider.requests[0].Tools)
|
|
}
|
|
}
|
|
|
|
func TestRunRejectsFilteredToolCalls(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call_1", ToolName: "read_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call_1", ArgumentsFragment: `{"path":"README.md"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call_1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "read", provider, Options{
|
|
Registry: registry,
|
|
DisabledTools: []string{"read_file"},
|
|
MaxTurns: 2,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("expected final answer after filtered tool result, got %q", result.FinalAnswer)
|
|
}
|
|
lastMessage := result.Messages[len(result.Messages)-2]
|
|
if !strings.Contains(lastMessage.Content, "not enabled") {
|
|
t.Fatalf("expected filtered tool error message, got %#v", result.Messages)
|
|
}
|
|
}
|
|
|
|
func TestRunRejectsToolCallsOutsideEnabledList(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call_1", ToolName: "read_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call_1", ArgumentsFragment: `{"path":"README.md"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call_1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "read", provider, Options{
|
|
Registry: registry,
|
|
EnabledTools: []string{"grep"},
|
|
MaxTurns: 2,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("expected final answer after filtered tool result, got %q", result.FinalAnswer)
|
|
}
|
|
lastMessage := result.Messages[len(result.Messages)-2]
|
|
if !strings.Contains(lastMessage.Content, "not enabled") {
|
|
t.Fatalf("expected filtered tool error message, got %#v", result.Messages)
|
|
}
|
|
}
|
|
|
|
func TestRunExecutesToolCallThroughRegistry(t *testing.T) {
|
|
root := t.TempDir()
|
|
writeAgentTestFile(t, filepath.Join(root, "notes.txt"), "alpha\nbeta\n")
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "read_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":"notes.txt"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "read done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
var toolResults []ToolResult
|
|
result, err := Run(context.Background(), "read notes", provider, Options{
|
|
Registry: registry,
|
|
OnToolResult: func(result ToolResult) { toolResults = append(toolResults, result) },
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "read done" {
|
|
t.Fatalf("expected final answer from second turn, got %q", result.FinalAnswer)
|
|
}
|
|
if len(provider.requests) != 2 {
|
|
t.Fatalf("expected provider to be called twice, got %d", len(provider.requests))
|
|
}
|
|
lastMessage := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
assertMessage(t, lastMessage, zeroruntime.MessageRoleTool, "alpha")
|
|
if lastMessage.ToolCallID != "call-1" {
|
|
t.Fatalf("expected tool_call_id call-1, got %q", lastMessage.ToolCallID)
|
|
}
|
|
if len(toolResults) != 1 || toolResults[0].Status != tools.StatusOK {
|
|
t.Fatalf("expected one ok tool result, got %#v", toolResults)
|
|
}
|
|
}
|
|
|
|
func TestRunSanitizesMalformedToolCallArgumentsBeforeRetry(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "read_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":"README.md"}{"path":"AGENTS.md"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "recovered"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "read files", provider, Options{
|
|
Registry: registry,
|
|
MaxTurns: 2,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "recovered" {
|
|
t.Fatalf("expected recovery turn final answer, got %q", result.FinalAnswer)
|
|
}
|
|
if len(provider.requests) != 2 {
|
|
t.Fatalf("expected retry request after malformed tool args, got %d requests", len(provider.requests))
|
|
}
|
|
|
|
var assistantCall *zeroruntime.ToolCall
|
|
var toolParseError string
|
|
for _, message := range provider.requests[1].Messages {
|
|
if message.Role == zeroruntime.MessageRoleAssistant && len(message.ToolCalls) > 0 {
|
|
assistantCall = &message.ToolCalls[0]
|
|
}
|
|
if message.Role == zeroruntime.MessageRoleTool && strings.Contains(message.Content, "Failed to parse arguments for read_file") {
|
|
toolParseError = message.Content
|
|
}
|
|
}
|
|
if assistantCall == nil {
|
|
t.Fatalf("expected retry request to include assistant tool call history, got %#v", provider.requests[1].Messages)
|
|
}
|
|
if !json.Valid([]byte(assistantCall.Arguments)) {
|
|
t.Fatalf("assistant tool-call arguments must be valid JSON for provider replay, got %q", assistantCall.Arguments)
|
|
}
|
|
if assistantCall.Arguments != "{}" {
|
|
t.Fatalf("malformed arguments should be sanitized to an empty JSON object, got %q", assistantCall.Arguments)
|
|
}
|
|
if toolParseError == "" {
|
|
t.Fatalf("expected model-visible tool result to keep the parse error, messages: %#v", provider.requests[1].Messages)
|
|
}
|
|
}
|
|
|
|
func TestRunDefersSelfCorrectFeedbackUntilAfterToolBatch(t *testing.T) {
|
|
// A single assistant turn with two tool calls where the first mutates and
|
|
// self-correct fails must keep both tool_results contiguous, with the feedback
|
|
// appended only after the whole batch — otherwise a user message interleaves
|
|
// between tool_results and breaks strict provider replay.
|
|
root := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(root, "existing.txt"), []byte("data"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "write_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":"notes.txt","content":"hello"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-2", ToolName: "read_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: `{"path":"existing.txt"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-2"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
corrector := NewSelfCorrector(root, erroringChecker{err: errors.New("lsp boom")}, nil, SelfCorrectConfig{
|
|
Enabled: true,
|
|
IncludeLSP: true,
|
|
Autonomy: "high",
|
|
})
|
|
|
|
if _, err := Run(context.Background(), "go", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
SelfCorrect: corrector,
|
|
OnPermissionRequest: func(_ context.Context, _ PermissionRequest) (PermissionDecision, error) {
|
|
return PermissionDecision{Action: PermissionDecisionAllow, Reason: "test"}, nil
|
|
},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(provider.requests) < 2 {
|
|
t.Fatalf("expected a second completion request, got %d", len(provider.requests))
|
|
}
|
|
|
|
msgs := provider.requests[1].Messages
|
|
var toolIdx []int
|
|
feedbackIdx := -1
|
|
for i, m := range msgs {
|
|
switch m.Role {
|
|
case zeroruntime.MessageRoleTool:
|
|
toolIdx = append(toolIdx, i)
|
|
case zeroruntime.MessageRoleUser:
|
|
if strings.Contains(m.Content, "Verification failed after your edit") {
|
|
feedbackIdx = i
|
|
}
|
|
}
|
|
}
|
|
if len(toolIdx) != 2 {
|
|
t.Fatalf("expected 2 tool_result messages, got %d: %#v", len(toolIdx), msgs)
|
|
}
|
|
if feedbackIdx == -1 {
|
|
t.Fatalf("expected a self-correct feedback message, messages: %#v", msgs)
|
|
}
|
|
if toolIdx[1] != toolIdx[0]+1 {
|
|
t.Fatalf("tool_results must be contiguous, got indices %v: %#v", toolIdx, msgs)
|
|
}
|
|
if feedbackIdx < toolIdx[1] {
|
|
t.Fatalf("self-correct feedback (idx %d) must come after both tool_results %v", feedbackIdx, toolIdx)
|
|
}
|
|
}
|
|
|
|
func TestRunBatchesSelfCorrectOncePerTurn(t *testing.T) {
|
|
// Two mutating tool calls in one assistant turn must trigger a single
|
|
// AfterEdit over the union of changed files — not one per call — so the per-run
|
|
// attempt budget isn't consumed twice and an intermediate edit isn't verified
|
|
// after a later call in the same turn supersedes it.
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "write_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":"a.txt","content":"hello"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-2", ToolName: "write_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: `{"path":"b.txt","content":"hello"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-2"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
corrector := NewSelfCorrector(root, erroringChecker{err: errors.New("boom")}, nil, SelfCorrectConfig{
|
|
Enabled: true,
|
|
IncludeLSP: true,
|
|
Autonomy: "high",
|
|
})
|
|
|
|
if _, err := Run(context.Background(), "go", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
SelfCorrect: corrector,
|
|
OnPermissionRequest: func(_ context.Context, _ PermissionRequest) (PermissionDecision, error) {
|
|
return PermissionDecision{Action: PermissionDecisionAllow, Reason: "test"}, nil
|
|
},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// One AfterEdit for the whole turn -> exactly one corrective attempt consumed
|
|
// (the old per-call path would have consumed two).
|
|
if corrector.attempts != 1 {
|
|
t.Fatalf("AfterEdit should run once per turn (attempts=1), got %d", corrector.attempts)
|
|
}
|
|
feedbackCount := 0
|
|
for _, m := range provider.requests[1].Messages {
|
|
if m.Role == zeroruntime.MessageRoleUser && strings.Contains(m.Content, "Verification failed after your edit") {
|
|
feedbackCount++
|
|
}
|
|
}
|
|
if feedbackCount != 1 {
|
|
t.Fatalf("expected exactly one self-correct feedback message, got %d", feedbackCount)
|
|
}
|
|
}
|
|
|
|
func TestRunDeniesPromptToolWithoutUnsafePermission(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("write denied")
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write notes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "write denied" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, "notes.txt")); !os.IsNotExist(err) {
|
|
t.Fatalf("expected denied write to leave file missing, stat error: %v", err)
|
|
}
|
|
lastMessage := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if !strings.Contains(lastMessage.Content, "Permission required for write_file") {
|
|
t.Fatalf("expected permission denial tool result, got %q", lastMessage.Content)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.ToolCallID != "call-1" || event.ToolName != "write_file" || event.Action != PermissionActionPrompt {
|
|
t.Fatalf("unexpected permission event: %#v", event)
|
|
}
|
|
if event.Permission != string(tools.PermissionPrompt) || event.PermissionMode != PermissionModeAsk || event.SideEffect != string(tools.SideEffectWrite) {
|
|
t.Fatalf("unexpected permission metadata: %#v", event)
|
|
}
|
|
if !strings.Contains(event.Reason, "Creates or overwrites files") {
|
|
t.Fatalf("expected tool safety reason in permission event, got %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestRunRequestsPromptToolPermissionBeforeExecution(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("write approved")
|
|
var requests []PermissionRequest
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write notes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllow, Reason: "approved for test"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "write approved" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
content, err := os.ReadFile(filepath.Join(root, "notes.txt"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(content) != "hello" {
|
|
t.Fatalf("expected approved write content, got %q", content)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected one permission request, got %#v", requests)
|
|
}
|
|
request := requests[0]
|
|
if request.ToolCallID != "call-1" || request.ToolName != "write_file" || request.Action != PermissionActionPrompt {
|
|
t.Fatalf("unexpected permission request: %#v", request)
|
|
}
|
|
if request.PermissionMode != PermissionModeAsk || request.SideEffect != string(tools.SideEffectWrite) || request.Autonomy != "medium" {
|
|
t.Fatalf("unexpected request metadata: %#v", request)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one final permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.Action != PermissionActionAllow || !event.PermissionGranted {
|
|
t.Fatalf("expected final allow event after approval, got %#v", event)
|
|
}
|
|
if event.DecisionReason != "approved for test" {
|
|
t.Fatalf("expected decision reason in final event, got %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestRunAllowsWorkspaceWriteWithoutPromptWhenSandboxPolicyPermits(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("write done")
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write notes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
}),
|
|
OnPermissionRequest: func(context.Context, PermissionRequest) (PermissionDecision, error) {
|
|
t.Fatal("workspace write should not request permission")
|
|
return PermissionDecision{}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "write done" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
content, err := os.ReadFile(filepath.Join(root, "notes.txt"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(content) != "hello" {
|
|
t.Fatalf("expected workspace write content, got %q", content)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one auto permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.Action != PermissionActionAllow || event.PermissionGranted || event.GrantMatched {
|
|
t.Fatalf("expected workspace allow without user grant, got %#v", event)
|
|
}
|
|
if event.Reason != "workspace write is allowed" {
|
|
t.Fatalf("expected workspace-write reason, got %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestRunDeniesPromptToolWhenPermissionRequestDenied(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("write denied")
|
|
var requests []PermissionRequest
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write notes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionDeny, Reason: "not this command"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "write denied" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, "notes.txt")); !os.IsNotExist(err) {
|
|
t.Fatalf("expected denied write to leave file missing, stat error: %v", err)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected one permission request, got %#v", requests)
|
|
}
|
|
lastMessage := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if !strings.Contains(lastMessage.Content, "Permission denied for write_file") || !strings.Contains(lastMessage.Content, "not this command") {
|
|
t.Fatalf("expected denied permission tool result, got %q", lastMessage.Content)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one final permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.Action != PermissionActionDeny || event.PermissionGranted {
|
|
t.Fatalf("expected final deny event, got %#v", event)
|
|
}
|
|
if event.DecisionReason != "not this command" {
|
|
t.Fatalf("expected denial reason in final event, got %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestRunAbortsWhenPermissionRequestCanceled(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("write should not continue")
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write notes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
OnPermissionRequest: func(context.Context, PermissionRequest) (PermissionDecision, error) {
|
|
return PermissionDecision{Action: PermissionDecisionCancel, Reason: "redirect needed"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if !errors.Is(err, errPermissionApprovalCanceled) {
|
|
t.Fatalf("expected permission approval cancel error, got %v", err)
|
|
}
|
|
if result.FinalAnswer != "" {
|
|
t.Fatalf("canceled run must not continue to final answer, got %q", result.FinalAnswer)
|
|
}
|
|
if len(provider.requests) != 1 {
|
|
t.Fatalf("canceled permission prompt should stop after first turn, got %d provider requests", len(provider.requests))
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, "notes.txt")); !os.IsNotExist(err) {
|
|
t.Fatalf("expected canceled write to leave file missing, stat error: %v", err)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one cancel permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.Action != PermissionActionCancel || event.DecisionAction != PermissionDecisionCancel || event.PermissionGranted {
|
|
t.Fatalf("expected final cancel event, got %#v", event)
|
|
}
|
|
if event.DecisionReason != "redirect needed" {
|
|
t.Fatalf("expected cancel reason in final event, got %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestAvailablePermissionDecisionsSplitDenyAndCancel(t *testing.T) {
|
|
options := Options{Sandbox: sandbox.NewEngine(sandbox.EngineOptions{WorkspaceRoot: t.TempDir(), Policy: sandbox.DefaultPolicy()})}
|
|
cases := []struct {
|
|
name string
|
|
tool string
|
|
want []PermissionDecisionAction
|
|
}{
|
|
{
|
|
name: "bash keeps recoverable deny and explicit cancel",
|
|
tool: "bash",
|
|
want: []PermissionDecisionAction{
|
|
PermissionDecisionAllow,
|
|
PermissionDecisionAllowForSession,
|
|
PermissionDecisionDeny,
|
|
PermissionDecisionCancel,
|
|
},
|
|
},
|
|
{
|
|
name: "apply patch uses cancel without recoverable deny",
|
|
tool: "apply_patch",
|
|
want: []PermissionDecisionAction{
|
|
PermissionDecisionAllow,
|
|
PermissionDecisionAllowForSession,
|
|
PermissionDecisionCancel,
|
|
},
|
|
},
|
|
{
|
|
name: "file write keeps recoverable deny",
|
|
tool: "write_file",
|
|
want: []PermissionDecisionAction{
|
|
PermissionDecisionAllow,
|
|
PermissionDecisionAllowForSession,
|
|
PermissionDecisionDeny,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := availablePermissionDecisions(PermissionEvent{
|
|
ToolName: tc.tool,
|
|
Action: PermissionActionPrompt,
|
|
}, nil, options)
|
|
if !equalPermissionDecisions(got, tc.want) {
|
|
t.Fatalf("decisions = %#v, want %#v", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunPersistsAlwaysAllowPermissionDecision(t *testing.T) {
|
|
root := t.TempDir()
|
|
store, err := sandbox.NewGrantStore(sandbox.StoreOptions{FilePath: filepath.Join(t.TempDir(), "sandbox-grants.json")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("write approved")
|
|
var permissionEvents []PermissionEvent
|
|
policy := sandbox.DefaultPolicy()
|
|
policy.EnforceWorkspace = false
|
|
|
|
result, err := Run(context.Background(), "write notes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: policy,
|
|
Store: store,
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
if request.Risk.Level == "" {
|
|
t.Fatalf("expected request risk to be populated: %#v", request)
|
|
}
|
|
return PermissionDecision{Action: PermissionDecisionAlwaysAllow, Reason: "trust write_file for this workspace"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "write approved" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
// The grant is scoped to exactly the file the call wrote, anchored to the
|
|
// workspace — not a blanket tool-wide allow.
|
|
notesPath := filepath.Join(root, "notes.txt")
|
|
lookup, err := store.Lookup("write_file", notesPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !lookup.Matched || lookup.Grant.Decision != sandbox.GrantAllow {
|
|
t.Fatalf("expected persistent allow grant, got %#v", lookup)
|
|
}
|
|
if lookup.Grant.ScopeKind != sandbox.ScopeFile || lookup.Grant.Scope != notesPath {
|
|
t.Fatalf("expected file-scoped grant for %q, got %#v", notesPath, lookup.Grant)
|
|
}
|
|
// A different file in the same workspace is NOT covered by that grant.
|
|
other, err := store.Lookup("write_file", filepath.Join(root, "other.txt"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if other.Matched {
|
|
t.Fatalf("a sibling file must not be covered by a file-scoped grant: %#v", other)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one final permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.Action != PermissionActionAllow || !event.PermissionGranted || event.Grant == nil {
|
|
t.Fatalf("expected allow event with persisted grant, got %#v", event)
|
|
}
|
|
if event.Grant.Decision != sandbox.GrantAllow || event.Grant.ToolName != "write_file" {
|
|
t.Fatalf("unexpected persisted grant in event: %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestRunSessionAllowSkipsMatchingPromptWithoutPersistentGrant(t *testing.T) {
|
|
root := t.TempDir()
|
|
store, err := sandbox.NewGrantStore(sandbox.StoreOptions{FilePath: filepath.Join(t.TempDir(), "sandbox-grants.json")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "write_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":"notes.txt","content":"first","overwrite":true}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-2", ToolName: "write_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: `{"path":"notes.txt","content":"second","overwrite":true}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-2"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
var permissionEvents []PermissionEvent
|
|
policy := sandbox.DefaultPolicy()
|
|
policy.EnforceWorkspace = false
|
|
|
|
result, err := Run(context.Background(), "write notes twice", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: policy,
|
|
Store: store,
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllowForSession, Reason: "trust this file for the session"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
content, err := os.ReadFile(filepath.Join(root, "notes.txt"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(content) != "second" {
|
|
t.Fatalf("expected second write content, got %q", content)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected one permission request, got %#v", requests)
|
|
}
|
|
if len(permissionEvents) != 2 {
|
|
t.Fatalf("expected two permission events, got %#v", permissionEvents)
|
|
}
|
|
if permissionEvents[0].DecisionAction != PermissionDecisionAllowForSession || permissionEvents[0].Grant == nil || !permissionEvents[0].Grant.Session {
|
|
t.Fatalf("expected first event to carry session grant, got %#v", permissionEvents[0])
|
|
}
|
|
if !permissionEvents[1].GrantMatched || permissionEvents[1].Grant == nil || !permissionEvents[1].Grant.Session {
|
|
t.Fatalf("expected second event to be session-grant matched, got %#v", permissionEvents[1])
|
|
}
|
|
lookup, err := store.Lookup("write_file", filepath.Join(root, "notes.txt"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if lookup.Matched {
|
|
t.Fatalf("session approval must not persist a grant, got %#v", lookup)
|
|
}
|
|
}
|
|
|
|
func TestRunCommandPrefixApprovalSkipsLaterMatchingBashPrompt(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewBashTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"echo prefix-ok"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-2", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: `{"command":"echo prefix-ok again"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-2"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "run twice", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: sandbox.Backend{Name: sandbox.BackendUnavailable, Message: "native sandbox unavailable"},
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
if !containsPermissionDecision(request.AvailableDecisions, PermissionDecisionAllowPrefix) {
|
|
t.Fatalf("bash prompt missing prefix approval decision: %#v", request.AvailableDecisions)
|
|
}
|
|
if !equalStringSlices(request.CommandPrefix, []string{"echo", "prefix-ok"}) {
|
|
t.Fatalf("request command prefix = %#v", request.CommandPrefix)
|
|
}
|
|
return PermissionDecision{Action: PermissionDecisionAllowPrefix, Reason: "trust this command prefix"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected only the first bash call to prompt, got %#v", requests)
|
|
}
|
|
if len(permissionEvents) != 2 {
|
|
t.Fatalf("expected two bash permission events, got %#v", permissionEvents)
|
|
}
|
|
for index, event := range permissionEvents {
|
|
if event.DecisionAction != PermissionDecisionAllowPrefix || !event.PermissionGranted {
|
|
t.Fatalf("event %d = %#v, want prefix allow", index, event)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunCommandPrefixApprovalBypassesSandboxForMatchingShellCalls(t *testing.T) {
|
|
root := t.TempDir()
|
|
retryTool := &sandboxDeniedRetryTool{}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(retryTool)
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"echo prefix-ok"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-2", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: `{"command":"echo prefix-ok again"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-2"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
|
|
result, err := Run(context.Background(), "run twice", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: sandbox.Backend{Name: sandbox.BackendUnavailable, Message: "native sandbox unavailable"},
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllowPrefix, Reason: "trust this command prefix"}, nil
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected only first command to request prefix approval, got %#v", requests)
|
|
}
|
|
if len(retryTool.calls) != 2 {
|
|
t.Fatalf("tool calls = %#v, want two first-attempt executions", retryTool.calls)
|
|
}
|
|
for index, call := range retryTool.calls {
|
|
if call["sandbox_permissions"] != string(tools.SandboxPermissionsRequireEscalated) {
|
|
t.Fatalf("call %d args = %#v, want prefix-approved require_escalated execution", index, call)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunCommandPrefixApprovalCoversSegmentedShellWithSafeTail(t *testing.T) {
|
|
root := t.TempDir()
|
|
retryTool := &sandboxDeniedRetryTool{}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(retryTool)
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"ps aux"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-2", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: `{"command":"ps aux | head -5"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-2"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
|
|
result, err := Run(context.Background(), "inspect processes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: sandbox.Backend{Name: sandbox.BackendUnavailable, Message: "native sandbox unavailable"},
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
if !equalStringSlices(request.CommandPrefix, []string{"ps", "aux"}) {
|
|
t.Fatalf("request command prefix = %#v, want ps aux", request.CommandPrefix)
|
|
}
|
|
return PermissionDecision{Action: PermissionDecisionAllowPrefix, Reason: "trust this command prefix"}, nil
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected only first command to request approval, got %#v", requests)
|
|
}
|
|
if len(retryTool.calls) != 2 {
|
|
t.Fatalf("tool calls = %#v, want two first-attempt executions", retryTool.calls)
|
|
}
|
|
for index, call := range retryTool.calls {
|
|
if call["sandbox_permissions"] != string(tools.SandboxPermissionsRequireEscalated) {
|
|
t.Fatalf("call %d args = %#v, want require_escalated execution", index, call)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunPersistentCommandPrefixApprovalSkipsFutureSessionPrompt(t *testing.T) {
|
|
root := t.TempDir()
|
|
store, err := sandbox.NewGrantStore(sandbox.StoreOptions{FilePath: filepath.Join(t.TempDir(), "sandbox-grants.json")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewBashTool(root))
|
|
policy := sandbox.DefaultPolicy()
|
|
policy.Network = sandbox.NetworkAllow
|
|
|
|
firstProvider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"echo persist-ok"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "first done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var firstRequests []PermissionRequest
|
|
var firstEvents []PermissionEvent
|
|
if _, err := Run(context.Background(), "run once", firstProvider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: policy,
|
|
Store: store,
|
|
Backend: sandbox.Backend{Name: sandbox.BackendUnavailable, Message: "native sandbox unavailable"},
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
firstRequests = append(firstRequests, request)
|
|
if !containsPermissionDecision(request.AvailableDecisions, PermissionDecisionAlwaysAllowPrefix) {
|
|
t.Fatalf("bash prompt missing persistent prefix approval decision: %#v", request.AvailableDecisions)
|
|
}
|
|
return PermissionDecision{Action: PermissionDecisionAlwaysAllowPrefix, Reason: "trust this command prefix"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
firstEvents = append(firstEvents, event)
|
|
},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(firstRequests) != 1 {
|
|
t.Fatalf("expected first run to prompt once, got %#v", firstRequests)
|
|
}
|
|
if len(firstEvents) != 1 || firstEvents[0].DecisionAction != PermissionDecisionAlwaysAllowPrefix {
|
|
t.Fatalf("expected persistent prefix event, got %#v", firstEvents)
|
|
}
|
|
prefixes, err := store.ListCommandPrefixes()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(prefixes) != 1 || !equalStringSlices(prefixes[0].Prefix, []string{"echo", "persist-ok"}) {
|
|
t.Fatalf("persisted prefixes = %#v, want echo persist-ok", prefixes)
|
|
}
|
|
|
|
secondProvider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-2", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: `{"command":"echo persist-ok again"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-2"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "second done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var secondEvents []PermissionEvent
|
|
if _, err := Run(context.Background(), "run again", secondProvider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: policy,
|
|
Store: store,
|
|
Backend: sandbox.Backend{Name: sandbox.BackendUnavailable, Message: "native sandbox unavailable"},
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
t.Fatalf("persistent command prefix should skip prompt, got %#v", request)
|
|
return PermissionDecision{}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
secondEvents = append(secondEvents, event)
|
|
},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(secondEvents) != 1 || secondEvents[0].DecisionAction != PermissionDecisionAlwaysAllowPrefix || !secondEvents[0].PermissionGranted {
|
|
t.Fatalf("expected persistent prefix match event, got %#v", secondEvents)
|
|
}
|
|
}
|
|
|
|
func TestRunPersistentCommandPrefixStillPromptsForNetwork(t *testing.T) {
|
|
root := t.TempDir()
|
|
store, err := sandbox.NewGrantStore(sandbox.StoreOptions{FilePath: filepath.Join(t.TempDir(), "sandbox-grants.json")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.GrantCommandPrefix(sandbox.CommandPrefixInput{ToolName: "bash", Prefix: []string{"curl"}}); err != nil {
|
|
t.Fatalf("seed command prefix: %v", err)
|
|
}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewBashTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"curl https://example.com"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var events []PermissionEvent
|
|
var requests []PermissionRequest
|
|
if _, err := Run(context.Background(), "curl", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Store: store,
|
|
Backend: sandbox.Backend{Name: sandbox.BackendUnavailable, Message: "native sandbox unavailable"},
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionDeny, Reason: "network not approved"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
events = append(events, event)
|
|
},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(requests) != 1 || requests[0].Reason != sandbox.ReasonNetworkBlocked {
|
|
t.Fatalf("expected one network permission request despite prefix match, got %#v", requests)
|
|
}
|
|
if len(events) != 1 || events[0].Action != PermissionActionDeny || events[0].DecisionReason != "network not approved" {
|
|
t.Fatalf("expected denied network permission event despite prefix match, got %#v", events)
|
|
}
|
|
}
|
|
|
|
func TestRunApprovedNetworkBashPromptAppliesTurnNetworkGrant(t *testing.T) {
|
|
root := t.TempDir()
|
|
command := "PATH=.:$PATH curl https://example.com"
|
|
if runtime.GOOS == "windows" {
|
|
command = "set PATH=.;%PATH% && curl https://example.com"
|
|
fakeCurl := filepath.Join(root, "curl.cmd")
|
|
if err := os.WriteFile(fakeCurl, []byte("@echo fake curl %*\r\n"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
} else {
|
|
fakeCurl := filepath.Join(root, "curl")
|
|
if err := os.WriteFile(fakeCurl, []byte("#!/bin/sh\necho fake curl \"$@\"\n"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewBashTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":` + quoteJSONString(command) + `}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
var events []PermissionEvent
|
|
result, err := Run(context.Background(), "curl", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: sandbox.Backend{Name: sandbox.BackendUnavailable, Message: "native sandbox unavailable"},
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllow, Reason: "approve network once"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
events = append(events, event)
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("final answer = %q", result.FinalAnswer)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected one permission request, got %#v", requests)
|
|
}
|
|
request := requests[0]
|
|
if request.Reason != sandbox.ReasonNetworkBlocked || !sandbox.HasRiskCategory(request.Risk, "network") {
|
|
t.Fatalf("expected network permission request, got %#v", request)
|
|
}
|
|
for _, decision := range request.AvailableDecisions {
|
|
if decision == PermissionDecisionAllowPrefix || decision == PermissionDecisionAlwaysAllowPrefix {
|
|
t.Fatalf("network prompt must not offer command-prefix approvals: %#v", request.AvailableDecisions)
|
|
}
|
|
}
|
|
if len(events) != 1 || events[0].Action != PermissionActionAllow || events[0].DecisionAction != PermissionDecisionAllow {
|
|
t.Fatalf("expected approved permission event, got %#v", events)
|
|
}
|
|
if len(provider.requests) < 2 {
|
|
t.Fatalf("expected tool result to be sent back to provider, got %d requests", len(provider.requests))
|
|
}
|
|
lastMessage := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if !strings.Contains(lastMessage.Content, "fake curl https://example.com") {
|
|
t.Fatalf("expected approved network command output after degraded execution, got %q", lastMessage.Content)
|
|
}
|
|
}
|
|
|
|
func TestRunDoesNotOfferPrefixApprovalForUnsafeBashCommand(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewBashTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"echo hi && npm install","prefix_rule":["echo"]}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
_, err := Run(context.Background(), "run unsafe", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: sandbox.Backend{Name: sandbox.BackendUnavailable, Message: "native sandbox unavailable"},
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
if len(request.CommandPrefix) != 0 {
|
|
t.Fatalf("unsafe command should not have a command prefix: %#v", request.CommandPrefix)
|
|
}
|
|
if containsPermissionDecision(request.AvailableDecisions, PermissionDecisionAllowPrefix) {
|
|
t.Fatalf("unsafe command should not offer prefix approval: %#v", request.AvailableDecisions)
|
|
}
|
|
return PermissionDecision{Action: PermissionDecisionDeny, Reason: "deny unsafe"}, nil
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestRunPromptsForDestructiveShellInsteadOfSandboxDeny(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewBashTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"echo rm -rf /"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
var events []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "run dangerous command", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Backend: sandbox.Backend{Name: sandbox.BackendUnavailable, Message: "native sandbox unavailable"},
|
|
}),
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllow, Reason: "approve once"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
events = append(events, event)
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("final answer = %q", result.FinalAnswer)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected one permission request, got %#v", requests)
|
|
}
|
|
request := requests[0]
|
|
if request.Action != PermissionActionPrompt || request.Reason != "destructive shell command requires approval" || !sandbox.HasRiskCategory(request.Risk, "destructive") {
|
|
t.Fatalf("expected destructive shell prompt, got %#v", request)
|
|
}
|
|
if request.Block != nil {
|
|
t.Fatalf("destructive shell prompt should not be a sandbox block: %#v", request.Block)
|
|
}
|
|
if len(events) != 1 || events[0].Action != PermissionActionAllow || events[0].DecisionAction != PermissionDecisionAllow {
|
|
t.Fatalf("expected approved permission event, got %#v", events)
|
|
}
|
|
if len(provider.requests) < 2 {
|
|
t.Fatalf("expected tool result to be sent back to provider, got %d requests", len(provider.requests))
|
|
}
|
|
lastMessage := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if !strings.Contains(lastMessage.Content, "rm -rf /") {
|
|
t.Fatalf("expected approved shell command output after degraded execution, got %q", lastMessage.Content)
|
|
}
|
|
}
|
|
|
|
func TestRunAlwaysAllowWithoutSandboxStillAllowsCall(t *testing.T) {
|
|
// Choosing "always allow" with NO sandbox engine configured must still allow
|
|
// THIS call (there is just nowhere to persist a grant for future calls). The
|
|
// prior code denied it because persistPermissionGrant errors when Sandbox==nil.
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("write approved")
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write notes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: nil, // no sandbox engine configured
|
|
OnPermissionRequest: func(_ context.Context, _ PermissionRequest) (PermissionDecision, error) {
|
|
return PermissionDecision{Action: PermissionDecisionAlwaysAllow, Reason: "trust it"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "write approved" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
// The tool actually ran: the file exists. Under the bug it would be denied
|
|
// and never written.
|
|
if _, statErr := os.Stat(filepath.Join(root, "notes.txt")); statErr != nil {
|
|
t.Fatalf("write_file should have run under always-allow with no sandbox: %v", statErr)
|
|
}
|
|
if len(permissionEvents) != 1 || permissionEvents[0].Action != PermissionActionAllow || !permissionEvents[0].PermissionGranted {
|
|
t.Fatalf("expected one allow event with permission granted, got %#v", permissionEvents)
|
|
}
|
|
}
|
|
|
|
func containsPermissionDecision(decisions []PermissionDecisionAction, want PermissionDecisionAction) bool {
|
|
for _, decision := range decisions {
|
|
if decision == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// cancelMidStreamProvider cancels the run while the provider stream is open and
|
|
// never sends a terminal event, so CollectStream returns via ctx.Done().
|
|
type cancelMidStreamProvider struct{ cancel context.CancelFunc }
|
|
|
|
func (p cancelMidStreamProvider) StreamCompletion(_ context.Context, _ zeroruntime.CompletionRequest) (<-chan zeroruntime.StreamEvent, error) {
|
|
p.cancel()
|
|
return make(chan zeroruntime.StreamEvent), nil
|
|
}
|
|
|
|
func TestRunCancellationPreservesContextCanceledIdentity(t *testing.T) {
|
|
// On cancellation the collected error is the stringified ctx error; the loop
|
|
// must return ctx.Err() itself so errors.Is(err, context.Canceled) holds for
|
|
// callers that branch on it.
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
_, err := Run(ctx, "hi", cancelMidStreamProvider{cancel: cancel}, Options{Registry: tools.NewRegistry()})
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("expected errors.Is(err, context.Canceled), got %v (%T)", err, err)
|
|
}
|
|
}
|
|
|
|
func TestRunGrantsPromptToolInUnsafeMode(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("write done")
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write notes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeUnsafe,
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "write done" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
content, err := os.ReadFile(filepath.Join(root, "notes.txt"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(content) != "hello" {
|
|
t.Fatalf("expected written content, got %q", content)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.Action != PermissionActionAllow || !event.PermissionGranted {
|
|
t.Fatalf("expected unsafe approval permission event, got %#v", event)
|
|
}
|
|
if event.ToolName != "write_file" || event.PermissionMode != PermissionModeUnsafe {
|
|
t.Fatalf("unexpected unsafe approval metadata: %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestRunEmitsPermissionEventForPersistentSandboxGrant(t *testing.T) {
|
|
root := t.TempDir()
|
|
store, err := sandbox.NewGrantStore(sandbox.StoreOptions{FilePath: filepath.Join(t.TempDir(), "sandbox-grants.json")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
grant, err := store.Grant(sandbox.GrantInput{
|
|
ToolName: "write_file",
|
|
Decision: sandbox.GrantAllow,
|
|
Reason: "trusted workspace edits",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("write done")
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write notes", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
Store: store,
|
|
}),
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "write done" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
content, err := os.ReadFile(filepath.Join(root, "notes.txt"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(content) != "hello" {
|
|
t.Fatalf("expected written content, got %q", content)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.Action != PermissionActionAllow || event.PermissionGranted {
|
|
t.Fatalf("expected grant-backed allow without unsafe permission, got %#v", event)
|
|
}
|
|
if !event.GrantMatched || event.Grant == nil || event.Grant.ToolName != grant.ToolName || event.Grant.Decision != sandbox.GrantAllow {
|
|
t.Fatalf("expected persistent grant details, got %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestRunPromptsAndAllowsOutsideWorkspaceWrite(t *testing.T) {
|
|
root := t.TempDir()
|
|
outside := filepath.Join(tempDirOutsideDefaultTemp(t), "escape.txt")
|
|
engine := sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
})
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewScopedWriteFileTool(root, engine.Scope()))
|
|
provider := providerCallingWritePathThenAnswer(outside, "write done")
|
|
var requests []PermissionRequest
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write outside", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: engine,
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllow, Reason: "approve outside write"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "write done" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
content, err := os.ReadFile(outside)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(content) != "hello" {
|
|
t.Fatalf("expected outside write content, got %q", content)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected one permission request, got %#v", requests)
|
|
}
|
|
request := requests[0]
|
|
if request.Action != PermissionActionPrompt || request.Block == nil || request.Block.Code != sandbox.BlockOutsideWorkspace || !request.Block.Recoverable {
|
|
t.Fatalf("expected recoverable outside-workspace prompt, got %#v", request)
|
|
}
|
|
if !containsPermissionDecision(request.AvailableDecisions, PermissionDecisionAllowForSession) {
|
|
t.Fatalf("expected session allow decision for outside-workspace prompt, got %#v", request.AvailableDecisions)
|
|
}
|
|
if containsPermissionDecision(request.AvailableDecisions, PermissionDecisionAlwaysAllow) {
|
|
t.Fatalf("outside-workspace prompt must not offer unsupported persistent allow, got %#v", request.AvailableDecisions)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.Action != PermissionActionAllow || !event.PermissionGranted || event.DecisionAction != PermissionDecisionAllow {
|
|
t.Fatalf("expected approved permission event, got %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestRunSessionAllowsLaterOutsideWorkspaceWrite(t *testing.T) {
|
|
root := t.TempDir()
|
|
outside := filepath.Join(tempDirOutsideDefaultTemp(t), "escape.txt")
|
|
engine := sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
})
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewScopedWriteFileTool(root, engine.Scope()))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "write_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":` + quoteJSONString(outside) + `,"content":"first"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-2", ToolName: "write_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: `{"path":` + quoteJSONString(outside) + `,"content":"second","overwrite":true}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-2"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
var requests []PermissionRequest
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write outside twice", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeAsk,
|
|
Autonomy: "medium",
|
|
Sandbox: engine,
|
|
OnPermissionRequest: func(_ context.Context, request PermissionRequest) (PermissionDecision, error) {
|
|
requests = append(requests, request)
|
|
return PermissionDecision{Action: PermissionDecisionAllowForSession, Reason: "trust outside file this session"}, nil
|
|
},
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
content, err := os.ReadFile(outside)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(content) != "second" {
|
|
t.Fatalf("expected second outside write content, got %q", content)
|
|
}
|
|
if len(requests) != 1 {
|
|
t.Fatalf("expected one permission request, got %#v", requests)
|
|
}
|
|
if len(permissionEvents) != 2 {
|
|
t.Fatalf("expected two permission events, got %#v", permissionEvents)
|
|
}
|
|
if permissionEvents[0].DecisionAction != PermissionDecisionAllowForSession || !permissionEvents[0].PermissionGranted {
|
|
t.Fatalf("expected first event to approve session grant, got %#v", permissionEvents[0])
|
|
}
|
|
if permissionEvents[1].Action != PermissionActionAllow || permissionEvents[1].PermissionGranted {
|
|
t.Fatalf("expected second event to run from session filesystem grant, got %#v", permissionEvents[1])
|
|
}
|
|
}
|
|
|
|
func TestRunAppliesSandboxEvenInUnsafeMode(t *testing.T) {
|
|
root := t.TempDir()
|
|
outside := filepath.Join(tempDirOutsideDefaultTemp(t), "escape.txt")
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWritePathThenAnswer(outside, "sandbox handled")
|
|
var permissionEvents []PermissionEvent
|
|
|
|
result, err := Run(context.Background(), "write outside", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeUnsafe,
|
|
Autonomy: "high",
|
|
Sandbox: sandbox.NewEngine(sandbox.EngineOptions{
|
|
WorkspaceRoot: root,
|
|
Policy: sandbox.DefaultPolicy(),
|
|
}),
|
|
OnPermission: func(event PermissionEvent) {
|
|
permissionEvents = append(permissionEvents, event)
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "sandbox handled" {
|
|
t.Fatalf("expected final answer, got %q", result.FinalAnswer)
|
|
}
|
|
if _, err := os.Stat(outside); !os.IsNotExist(err) {
|
|
t.Fatalf("expected sandbox to prevent outside write, stat error: %v", err)
|
|
}
|
|
lastMessage := provider.requests[1].Messages[len(provider.requests[1].Messages)-1]
|
|
if !strings.Contains(lastMessage.Content, "Sandbox block") || !strings.Contains(lastMessage.Content, "outside_workspace") {
|
|
t.Fatalf("expected sandbox block tool result, got %q", lastMessage.Content)
|
|
}
|
|
if len(permissionEvents) != 1 {
|
|
t.Fatalf("expected one permission event, got %#v", permissionEvents)
|
|
}
|
|
event := permissionEvents[0]
|
|
if event.Action != PermissionActionDeny {
|
|
t.Fatalf("expected denied permission event, got %#v", event)
|
|
}
|
|
if event.Block == nil || event.Block.Code != sandbox.BlockOutsideWorkspace {
|
|
t.Fatalf("expected outside_workspace block in permission event, got %#v", event)
|
|
}
|
|
if event.Risk.Level != sandbox.RiskCritical {
|
|
t.Fatalf("expected critical risk in permission event, got %#v", event)
|
|
}
|
|
}
|
|
|
|
func TestRunStopsAfterMaxTurns(t *testing.T) {
|
|
root := t.TempDir()
|
|
writeAgentTestFile(t, filepath.Join(root, "notes.txt"), "alpha")
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "read_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":"notes.txt"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "loop", provider, Options{
|
|
Registry: registry,
|
|
MaxTurns: 1,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "Agent reached maximum number of turns without a final answer." {
|
|
t.Fatalf("expected max-turns answer, got %q", result.FinalAnswer)
|
|
}
|
|
if result.Turns != 1 {
|
|
t.Fatalf("expected one turn, got %d", result.Turns)
|
|
}
|
|
}
|
|
|
|
func TestRunRequestsFinalAnswerAfterMaxTurns(t *testing.T) {
|
|
root := t.TempDir()
|
|
writeAgentTestFile(t, filepath.Join(root, "notes.txt"), "alpha")
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "read_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":"notes.txt"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "I read notes.txt and found alpha."},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "loop", provider, Options{
|
|
Registry: registry,
|
|
MaxTurns: 1,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "I read notes.txt and found alpha." {
|
|
t.Fatalf("expected final answer from finalization turn, got %q", result.FinalAnswer)
|
|
}
|
|
if len(provider.requests) != 2 {
|
|
t.Fatalf("expected final no-tools request after max turns, got %d requests", len(provider.requests))
|
|
}
|
|
finalRequest := provider.requests[1]
|
|
if len(finalRequest.Tools) != 0 {
|
|
t.Fatalf("finalization request must not advertise tools, got %#v", finalRequest.Tools)
|
|
}
|
|
lastMessage := finalRequest.Messages[len(finalRequest.Messages)-1]
|
|
if lastMessage.Role != zeroruntime.MessageRoleUser || !strings.Contains(lastMessage.Content, "tool-turn limit") {
|
|
t.Fatalf("expected max-turns finalization prompt, got %#v", lastMessage)
|
|
}
|
|
if result.Turns != 1 {
|
|
t.Fatalf("expected tool turns to remain 1, got %d", result.Turns)
|
|
}
|
|
}
|
|
|
|
func providerCallingWriteFileThenAnswer(answer string) *mockProvider {
|
|
return providerCallingWritePathThenAnswer("notes.txt", answer)
|
|
}
|
|
|
|
func providerCallingWritePathThenAnswer(path string, answer string) *mockProvider {
|
|
return &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "write_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":` + quoteJSONString(path) + `,"content":"hello"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: answer},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func quoteJSONString(value string) string {
|
|
encoded, _ := json.Marshal(value)
|
|
return string(encoded)
|
|
}
|
|
|
|
func equalPermissionDecisions(a, b []PermissionDecisionAction) bool {
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
for i := range a {
|
|
if a[i] != b[i] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func TestRunAppendsConfirmationPolicyToSystemPrompt(t *testing.T) {
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "ok"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
if _, err := Run(context.Background(), "do work", provider, Options{
|
|
Registry: tools.NewRegistry(),
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if len(provider.requests) == 0 {
|
|
t.Fatal("expected at least one provider request")
|
|
}
|
|
system := provider.requests[0].Messages[0]
|
|
if system.Role != zeroruntime.MessageRoleSystem {
|
|
t.Fatalf("expected first message to be system, got %s", system.Role)
|
|
}
|
|
// The overhauled core prompt: identity + the mandatory testing gate.
|
|
for _, marker := range []string{"You are Zero", "Testing gate"} {
|
|
if !strings.Contains(system.Content, marker) {
|
|
t.Fatalf("system prompt missing core marker %q: %q", marker, system.Content)
|
|
}
|
|
}
|
|
// Key markers from CONFIRMATION_POLICY.md must be present so the model self-polices.
|
|
for _, marker := range []string{"Confirmation Modes", "BLOCKED", "ALWAYS CONFIRM"} {
|
|
if !strings.Contains(system.Content, marker) {
|
|
t.Fatalf("system prompt missing confirmation policy marker %q", marker)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSystemPromptEmbedsConfirmationPolicy(t *testing.T) {
|
|
prompt := buildSystemPrompt(Options{})
|
|
if !strings.HasPrefix(prompt, "You are Zero") {
|
|
t.Fatalf("system prompt should start with the core instructions, got %q", prompt)
|
|
}
|
|
if !strings.Contains(prompt, "Confirmation Modes") {
|
|
t.Fatalf("embedded confirmation policy missing from system prompt")
|
|
}
|
|
// No workspace context without a cwd (keeps headless/test runs deterministic).
|
|
if strings.Contains(prompt, "<environment>") {
|
|
t.Fatalf("system prompt should omit the environment block when cwd is unset")
|
|
}
|
|
}
|
|
|
|
func TestGitBranchForPromptResolvesRelativeWorktreeGitdir(t *testing.T) {
|
|
root := t.TempDir()
|
|
// The real gitdir for the worktree, where HEAD lives.
|
|
gitdir := filepath.Join(root, "realgit", "worktrees", "wt")
|
|
if err := os.MkdirAll(gitdir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(gitdir, "HEAD"), []byte("ref: refs/heads/feature-x\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The worktree checkout: a .git FILE pointing at the gitdir via a RELATIVE path.
|
|
worktree := filepath.Join(root, "checkout")
|
|
if err := os.MkdirAll(worktree, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(worktree, ".git"), []byte("gitdir: ../realgit/worktrees/wt\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := gitBranchForPrompt(worktree); got != "feature-x" {
|
|
t.Fatalf("gitBranchForPrompt = %q, want feature-x (relative worktree gitdir resolved against cwd)", got)
|
|
}
|
|
}
|
|
|
|
func TestBuildSystemPromptInjectsWorkspaceContext(t *testing.T) {
|
|
cwd := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(cwd, "AGENTS.md"), []byte("Always run `make lint` before committing."), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prompt := buildSystemPrompt(Options{Cwd: cwd})
|
|
if !strings.Contains(prompt, "<environment>") || !strings.Contains(prompt, "Working directory: "+cwd) {
|
|
t.Fatalf("expected environment block with cwd, got %q", prompt)
|
|
}
|
|
if !strings.Contains(prompt, "Project guidelines (AGENTS.md)") || !strings.Contains(prompt, "make lint") {
|
|
t.Fatalf("expected AGENTS.md project guidelines injected, got %q", prompt)
|
|
}
|
|
}
|
|
|
|
func TestBuildSystemPromptInjectsHostShellContext(t *testing.T) {
|
|
prompt := buildSystemPrompt(Options{Cwd: t.TempDir()})
|
|
if !strings.Contains(prompt, "Operating system: "+runtime.GOOS) {
|
|
t.Fatalf("expected operating system in environment block, got %q", prompt)
|
|
}
|
|
if runtime.GOOS == "windows" {
|
|
if !strings.Contains(prompt, "Windows cmd.exe syntax") || !strings.Contains(prompt, "cwd argument") {
|
|
t.Fatalf("expected Windows cmd.exe shell guidance in prompt, got %q", prompt)
|
|
}
|
|
} else if !strings.Contains(prompt, "/bin/sh syntax") {
|
|
t.Fatalf("expected POSIX shell guidance in prompt, got %q", prompt)
|
|
}
|
|
}
|
|
|
|
func TestBuildSystemPromptInjectsRepoMap(t *testing.T) {
|
|
cwd := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(cwd, "go.mod"), []byte("module example.com/app\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(cwd, "internal", "service"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(cwd, "internal", "service", "service.go"), []byte("package service\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(cwd, "node_modules", "leftpad"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(cwd, "node_modules", "leftpad", "index.js"), []byte("module.exports = 1\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
prompt := buildSystemPrompt(Options{Cwd: cwd})
|
|
for _, want := range []string{
|
|
"## Repo map",
|
|
"Important files: go.mod",
|
|
"Languages:",
|
|
"internal/service/service.go",
|
|
} {
|
|
if !strings.Contains(prompt, want) {
|
|
t.Fatalf("expected repo map marker %q in prompt, got %q", want, prompt)
|
|
}
|
|
}
|
|
if strings.Contains(prompt, "node_modules/leftpad") {
|
|
t.Fatalf("repo map should omit ignored dependency dirs, got %q", prompt)
|
|
}
|
|
}
|
|
|
|
func TestBuildSystemPromptAllowsSpecModeOverride(t *testing.T) {
|
|
prompt := buildSystemPrompt(Options{SystemPrompt: specmode.DraftSystemPrompt})
|
|
if !strings.HasPrefix(prompt, "Specification drafting is active.") {
|
|
t.Fatalf("expected spec prompt override, got %q", prompt)
|
|
}
|
|
if !strings.Contains(prompt, "Confirmation Modes") {
|
|
t.Fatalf("expected confirmation policy to remain appended")
|
|
}
|
|
}
|
|
|
|
func TestSpecDraftAdvertisesOnlySafeDraftTools(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
for _, tool := range tools.CoreTools(root) {
|
|
registry.Register(tool)
|
|
}
|
|
specmode.RegisterDraftTools(registry, root, nil)
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
_, err := Run(context.Background(), "draft", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeSpecDraft,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
names := map[string]bool{}
|
|
for _, definition := range provider.requests[0].Tools {
|
|
names[definition.Name] = true
|
|
}
|
|
for _, want := range []string{"read_file", "list_directory", "glob", "grep", "skill", "ask_user", specmode.SubmitToolName} {
|
|
if !names[want] {
|
|
t.Fatalf("spec draft tools missing %q from %#v", want, names)
|
|
}
|
|
}
|
|
for _, denied := range []string{"write_file", "edit_file", "apply_patch", "bash", "update_plan", "web_fetch"} {
|
|
if names[denied] {
|
|
t.Fatalf("spec draft advertised denied tool %q in %#v", denied, names)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSpecDraftDeniesHiddenToolCalls(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewWriteFileTool(root))
|
|
provider := providerCallingWriteFileThenAnswer("done")
|
|
|
|
result, err := Run(context.Background(), "draft", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeSpecDraft,
|
|
MaxTurns: 2,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("expected final answer after denial, got %q", result.FinalAnswer)
|
|
}
|
|
var denied string
|
|
for _, message := range result.Messages {
|
|
if message.Role == zeroruntime.MessageRoleTool {
|
|
denied = message.Content
|
|
break
|
|
}
|
|
}
|
|
if !strings.Contains(denied, "not available in spec-draft mode") {
|
|
t.Fatalf("expected spec-draft denial, got %q", denied)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, "notes.txt")); !os.IsNotExist(err) {
|
|
t.Fatalf("write_file should not have written notes.txt, stat err=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestSpecDraftDeniesBashToolCalls(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewBashTool(root))
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "bash"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"command":"printf ran > ran.txt"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "draft", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeSpecDraft,
|
|
MaxTurns: 2,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "done" {
|
|
t.Fatalf("expected final answer after denial, got %q", result.FinalAnswer)
|
|
}
|
|
var denied string
|
|
for _, message := range result.Messages {
|
|
if message.Role == zeroruntime.MessageRoleTool {
|
|
denied = message.Content
|
|
break
|
|
}
|
|
}
|
|
if !strings.Contains(denied, "not available in spec-draft mode") {
|
|
t.Fatalf("expected spec-draft bash denial, got %q", denied)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, "ran.txt")); !os.IsNotExist(err) {
|
|
t.Fatalf("bash should not have written ran.txt, stat err=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestRunStopsWhenSubmitSpecReturnsReviewControl(t *testing.T) {
|
|
root := t.TempDir()
|
|
registry := tools.NewRegistry()
|
|
specmode.RegisterDraftTools(registry, root, nil)
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "exit-1", ToolName: specmode.SubmitToolName},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "exit-1", ArgumentsFragment: `{"title":"Implementation Plan","plan":"# Goal\n\nAdd implementation plan."}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "exit-1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "draft", provider, Options{
|
|
Registry: registry,
|
|
PermissionMode: PermissionModeSpecDraft,
|
|
MaxTurns: 3,
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.StopReason != StopReasonSpecReviewRequired {
|
|
t.Fatalf("StopReason = %q, want %q", result.StopReason, StopReasonSpecReviewRequired)
|
|
}
|
|
if len(provider.requests) != 1 {
|
|
t.Fatalf("expected run to stop after submit_spec, got %d requests", len(provider.requests))
|
|
}
|
|
if !strings.Contains(result.FinalAnswer, ".zero/specs/") {
|
|
t.Fatalf("final answer should mention saved spec, got %q", result.FinalAnswer)
|
|
}
|
|
}
|
|
|
|
func assertMessage(t *testing.T, message zeroruntime.Message, role zeroruntime.MessageRole, contentContains string) {
|
|
t.Helper()
|
|
|
|
if message.Role != role {
|
|
t.Fatalf("expected role %s, got %s", role, message.Role)
|
|
}
|
|
if contentContains != "" && !strings.Contains(message.Content, contentContains) {
|
|
t.Fatalf("expected message content to contain %q, got %q", contentContains, message.Content)
|
|
}
|
|
}
|
|
|
|
func writeAgentTestFile(t *testing.T, path string, content string) {
|
|
t.Helper()
|
|
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestRunRetriesOnDroppedToolCall(t *testing.T) {
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "Let me write the files."},
|
|
{Type: zeroruntime.StreamEventToolCallDropped},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "All done."},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "build it", provider, Options{Registry: tools.NewRegistry()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(provider.requests) != 2 {
|
|
t.Fatalf("expected the loop to retry (2 turns), got %d", len(provider.requests))
|
|
}
|
|
if result.FinalAnswer != "All done." {
|
|
t.Fatalf("expected final answer from retry turn, got %q", result.FinalAnswer)
|
|
}
|
|
// The retry turn must carry synthetic feedback to the model.
|
|
var fedback bool
|
|
for _, m := range provider.requests[1].Messages {
|
|
if m.Role == zeroruntime.MessageRoleUser && strings.Contains(strings.ToLower(m.Content), "tool name") {
|
|
fedback = true
|
|
}
|
|
}
|
|
if !fedback {
|
|
t.Fatalf("expected a synthetic tool-error message on the retry turn, messages: %+v", provider.requests[1].Messages)
|
|
}
|
|
}
|
|
|
|
func TestRunSurfacesDroppedToolCallAlongsideValidCall(t *testing.T) {
|
|
root := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(root, "notes.txt"), []byte("hi"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
registry := tools.NewRegistry()
|
|
registry.Register(tools.NewReadFileTool(root))
|
|
|
|
provider := &mockProvider{
|
|
turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
// One valid tool call AND a dropped (nameless) call in the same turn.
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "read_file"},
|
|
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{"path":"notes.txt"}`},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
|
{Type: zeroruntime.StreamEventToolCallDropped},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "All done."},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := Run(context.Background(), "do it", provider, Options{Registry: registry})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.FinalAnswer != "All done." {
|
|
t.Fatalf("expected final answer from retry turn, got %q", result.FinalAnswer)
|
|
}
|
|
if len(provider.requests) != 2 {
|
|
t.Fatalf("expected the loop to continue (2 turns), got %d", len(provider.requests))
|
|
}
|
|
// The valid tool call must still have executed (a tool result for call-1).
|
|
var sawToolResult bool
|
|
for _, m := range provider.requests[1].Messages {
|
|
if m.Role == zeroruntime.MessageRoleTool && m.ToolCallID == "call-1" {
|
|
sawToolResult = true
|
|
}
|
|
}
|
|
if !sawToolResult {
|
|
t.Fatalf("expected the valid tool call to execute, messages: %+v", provider.requests[1].Messages)
|
|
}
|
|
// The dropped call must ALSO be surfaced via the malformed-call notice.
|
|
var sawDroppedNotice bool
|
|
for _, m := range provider.requests[1].Messages {
|
|
if m.Role == zeroruntime.MessageRoleUser && strings.Contains(strings.ToLower(m.Content), "malformed") {
|
|
sawDroppedNotice = true
|
|
}
|
|
}
|
|
if !sawDroppedNotice {
|
|
t.Fatalf("expected a malformed-call notice for the dropped call, messages: %+v", provider.requests[1].Messages)
|
|
}
|
|
}
|
|
|
|
// TestRunAppendsAbortedPlaceholderForUnexecutedToolCallsOnGuardStop verifies
|
|
// that when a turn carries multiple tool calls and the repeated-failure guard
|
|
// halts the run on a call that is NOT the last, every advertised tool_use still
|
|
// gets a matching tool_result: the executed call gets its real result and the
|
|
// remaining (unexecuted) calls get aborted-placeholder results, so the recorded
|
|
// messages stay structurally valid for a strict provider replay.
|
|
func TestRunAppendsAbortedPlaceholderForUnexecutedToolCallsOnGuardStop(t *testing.T) {
|
|
registry := tools.NewRegistry()
|
|
registry.Register(alwaysFailingTool{})
|
|
|
|
// Three prior single-call failures prime the streak to one below the stop
|
|
// cap, so the FIRST call of the next multi-call turn is the 4th failure and
|
|
// trips outcome.Stop.
|
|
primingTurns := repeatedFlakyTurns(toolFailureStopAt - 1)
|
|
|
|
// The halting turn carries TWO tool calls: the first (flaky-stop) trips the
|
|
// guard before the second (flaky-2) is executed.
|
|
haltingTurn := []zeroruntime.StreamEvent{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "flaky-stop", ToolName: "flaky"},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "flaky-stop"},
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "flaky-2", ToolName: "flaky"},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "flaky-2"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
}
|
|
|
|
provider := &mockProvider{turns: append(primingTurns, haltingTurn)}
|
|
|
|
result, err := Run(context.Background(), "go", provider, Options{Registry: registry, MaxTurns: 12})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(result.FinalAnswer, "flaky") || !strings.Contains(result.FinalAnswer, "failed") {
|
|
t.Fatalf("expected repeated-failure stop answer, got %q", result.FinalAnswer)
|
|
}
|
|
|
|
// Both tool calls must have a matching tool_result message.
|
|
toolResultIDs := map[string]string{}
|
|
for _, message := range result.Messages {
|
|
if message.Role == zeroruntime.MessageRoleTool {
|
|
toolResultIDs[message.ToolCallID] = message.Content
|
|
}
|
|
}
|
|
if _, ok := toolResultIDs["flaky-stop"]; !ok {
|
|
t.Fatalf("expected a tool result for the executed call flaky-stop, messages: %+v", result.Messages)
|
|
}
|
|
placeholder, ok := toolResultIDs["flaky-2"]
|
|
if !ok {
|
|
t.Fatalf("expected an aborted-placeholder tool result for the unexecuted call flaky-2, messages: %+v", result.Messages)
|
|
}
|
|
if !strings.Contains(strings.ToLower(placeholder), "aborted") {
|
|
t.Fatalf("expected the placeholder result to mark the call as aborted, got %q", placeholder)
|
|
}
|
|
|
|
// Every tool_use in the final assistant message must have a matching result.
|
|
for _, message := range result.Messages {
|
|
if message.Role != zeroruntime.MessageRoleAssistant {
|
|
continue
|
|
}
|
|
for _, call := range message.ToolCalls {
|
|
if _, ok := toolResultIDs[call.ID]; !ok {
|
|
t.Fatalf("tool_use %q (%s) has no matching tool_result", call.ID, call.Name)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
type secretEmittingTool struct{ output string }
|
|
|
|
func (t secretEmittingTool) Name() string { return "leak" }
|
|
func (t secretEmittingTool) Description() string { return "emits text for testing" }
|
|
func (t secretEmittingTool) Parameters() tools.Schema {
|
|
return tools.Schema{Type: "object", AdditionalProperties: false}
|
|
}
|
|
func (t secretEmittingTool) Safety() tools.Safety {
|
|
return tools.Safety{SideEffect: tools.SideEffectRead, Permission: tools.PermissionAllow}
|
|
}
|
|
func (t secretEmittingTool) Run(_ context.Context, _ map[string]any) tools.Result {
|
|
return tools.Result{Status: tools.StatusOK, Output: t.output}
|
|
}
|
|
|
|
func TestRunScrubsSecretsFromToolOutput(t *testing.T) {
|
|
secret := "sk-proj-ABCDEFGHIJKLMNOP1234567890"
|
|
registry := tools.NewRegistry()
|
|
registry.Register(secretEmittingTool{output: "the token is " + secret + " ok"})
|
|
|
|
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "c1", ToolName: "leak"},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "c1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{
|
|
{Type: zeroruntime.StreamEventText, Content: "done"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
}}
|
|
|
|
var captured ToolResult
|
|
_, err := Run(context.Background(), "go", provider, Options{
|
|
Registry: registry,
|
|
OnToolResult: func(r ToolResult) { captured = r },
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(captured.Output, secret) {
|
|
t.Fatalf("secret leaked into tool result output: %q", captured.Output)
|
|
}
|
|
if !captured.Redacted {
|
|
t.Error("expected Redacted=true when a secret was scrubbed")
|
|
}
|
|
if !strings.Contains(strings.ToLower(captured.Output), "redacted") {
|
|
t.Errorf("expected a redaction reminder, got %q", captured.Output)
|
|
}
|
|
if len(provider.requests) < 2 {
|
|
t.Fatalf("expected a second turn carrying the tool result")
|
|
}
|
|
for _, m := range provider.requests[1].Messages {
|
|
if strings.Contains(m.Content, secret) {
|
|
t.Fatalf("secret leaked into model message: %q", m.Content)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunDoesNotFlagCleanToolOutput(t *testing.T) {
|
|
registry := tools.NewRegistry()
|
|
registry.Register(secretEmittingTool{output: "perfectly ordinary output"})
|
|
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{
|
|
{
|
|
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "c1", ToolName: "leak"},
|
|
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "c1"},
|
|
{Type: zeroruntime.StreamEventDone},
|
|
},
|
|
{{Type: zeroruntime.StreamEventText, Content: "done"}, {Type: zeroruntime.StreamEventDone}},
|
|
}}
|
|
var captured ToolResult
|
|
if _, err := Run(context.Background(), "go", provider, Options{Registry: registry, OnToolResult: func(r ToolResult) { captured = r }}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if captured.Redacted {
|
|
t.Error("clean output should not be flagged Redacted")
|
|
}
|
|
if strings.Contains(strings.ToLower(captured.Output), "redacted") {
|
|
t.Errorf("clean output should not get a reminder, got %q", captured.Output)
|
|
}
|
|
}
|