zero/internal/agenteval/run_test.go

package agenteval

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"reflect"
	"testing"
)

func TestRunnerMapsPassingAndFailingCommandResults(t *testing.T) {
	workspace := t.TempDir()
	runner := Runner{
		RunCommand: func(_ context.Context, gotWorkspace string, command Command) CommandResult {
			if gotWorkspace != workspace {
				t.Fatalf("workspace = %q, want %q", gotWorkspace, workspace)
			}
			switch command.ID {
			case "pass":
				return CommandResult{ID: command.ID, ExitCode: 0, Stdout: "ok"}
			case "fail":
				return CommandResult{ID: command.ID, ExitCode: 2, Stderr: "nope"}
			default:
				t.Fatalf("unexpected command %q", command.ID)
			}
			return CommandResult{}
		},
		ChangedFiles: func(context.Context, string) ([]string, error) {
			return []string{"internal/reader/a.go"}, nil
		},
	}

	report := runner.Run(context.Background(), runnerSuite(), RunInput{
		TaskID:        "two-commands",
		WorkspacePath: workspace,
	})

	if report.Status != StatusFail || report.OK {
		t.Fatalf("status = %s ok = %v, want fail false", report.Status, report.OK)
	}
	if got := report.Results[0]; got.ID != "pass" || got.Status != StatusPass || got.Stdout != "ok" {
		t.Fatalf("passing command result = %#v", got)
	}
	if got := report.Results[1]; got.ID != "fail" || got.Status != StatusFail || got.Stderr != "nope" {
		t.Fatalf("failing command result = %#v", got)
	}
	if report.Summary.Failed != 1 || report.Summary.Passed != 2 {
		t.Fatalf("summary = %#v, want one failed command and two passing checks", report.Summary)
	}
}

func TestRunnerBoundsCommandsWithDefaultTimeout(t *testing.T) {
	workspace := t.TempDir()
	var hadDeadline bool
	runner := Runner{
		RunCommand: func(ctx context.Context, _ string, command Command) CommandResult {
			_, hadDeadline = ctx.Deadline()
			return CommandResult{ID: command.ID, ExitCode: 0}
		},
		ChangedFiles: func(context.Context, string) ([]string, error) {
			return []string{"internal/reader/a.go"}, nil
		},
	}

	// No CommandTimeout set: a per-command deadline must still apply by default so a
	// hung verification command cannot run unbounded.
	report := runner.Run(context.Background(), runnerSuite(), RunInput{
		TaskID:        "two-commands",
		WorkspacePath: workspace,
	})

	if report.Status == StatusError {
		t.Fatalf("unexpected error report: %#v", report)
	}
	if !hadDeadline {
		t.Fatal("verification commands must run under a per-command deadline by default")
	}
}

func TestRunnerBlocksWhenWorkspaceSetupFails(t *testing.T) {
	missing := filepath.Join(t.TempDir(), "missing")
	calledCommand := false
	calledChangedFiles := false
	runner := Runner{
		RunCommand: func(context.Context, string, Command) CommandResult {
			calledCommand = true
			return CommandResult{}
		},
		ChangedFiles: func(context.Context, string) ([]string, error) {
			calledChangedFiles = true
			return nil, nil
		},
	}

	report := runner.Run(context.Background(), runnerSuite(), RunInput{
		TaskID:        "single",
		WorkspacePath: missing,
	})

	if report.Status != StatusBlocked || report.OK {
		t.Fatalf("status = %s ok = %v, want blocked false", report.Status, report.OK)
	}
	if calledCommand || calledChangedFiles {
		t.Fatalf("blocked workspace should not execute commands or changed-file collection")
	}
	if report.Summary.Blocked != 2 {
		t.Fatalf("summary = %#v, want both checks blocked", report.Summary)
	}
	if got := report.Results[0].Message; got == "" {
		t.Fatal("blocked result should include a message")
	}
}

func TestRunnerCollectsChangedFilesFromInjectedFunc(t *testing.T) {
	workspace := t.TempDir()
	runner := Runner{
		RunCommand: func(_ context.Context, _ string, command Command) CommandResult {
			return CommandResult{ID: command.ID, ExitCode: 0}
		},
		ChangedFiles: func(_ context.Context, gotWorkspace string) ([]string, error) {
			if gotWorkspace != workspace {
				t.Fatalf("workspace = %q, want %q", gotWorkspace, workspace)
			}
			return []string{"z.go", "internal/reader/a.go", "internal/reader/a/../b.go"}, nil
		},
	}

	report := runner.Run(context.Background(), runnerSuite(), RunInput{
		TaskID:        "single",
		WorkspacePath: workspace,
	})

	want := []string{"internal/reader/a.go", "internal/reader/b.go", "z.go"}
	if !reflect.DeepEqual(report.ChangedFiles, want) {
		t.Fatalf("changed files = %#v, want %#v", report.ChangedFiles, want)
	}
	changed := report.Results[1]
	if !reflect.DeepEqual(changed.UnexpectedFiles, []string{"z.go"}) {
		t.Fatalf("unexpected files = %#v, want z.go", changed.UnexpectedFiles)
	}
}

func TestRunnerBlocksWhenChangedFileCollectionFails(t *testing.T) {
	workspace := t.TempDir()
	runner := Runner{
		RunCommand: func(_ context.Context, _ string, command Command) CommandResult {
			return CommandResult{ID: command.ID, ExitCode: 0}
		},
		ChangedFiles: func(context.Context, string) ([]string, error) {
			return nil, errors.New("git unavailable")
		},
	}

	report := runner.Run(context.Background(), runnerSuite(), RunInput{
		TaskID:        "single",
		WorkspacePath: workspace,
	})

	if report.Status != StatusBlocked || report.Summary.Blocked != 2 {
		t.Fatalf("report = %#v, want blocked command and changed-file checks", report)
	}
	if got := report.Results[1].Message; got == "" {
		t.Fatal("changed-file block should include a message")
	}
}

func TestRunnerSelectsRequestedTask(t *testing.T) {
	workspace := t.TempDir()
	var executed []string
	runner := Runner{
		RunCommand: func(_ context.Context, _ string, command Command) CommandResult {
			executed = append(executed, command.ID)
			return CommandResult{ID: command.ID, ExitCode: 0}
		},
		ChangedFiles: func(context.Context, string) ([]string, error) {
			return []string{"cmd/zero/main.go"}, nil
		},
	}

	report := runner.Run(context.Background(), runnerSuite(), RunInput{
		TaskID:        "other",
		WorkspacePath: workspace,
	})

	if report.TaskID != "other" || report.Status != StatusPass {
		t.Fatalf("report task/status = %q/%s, want other/pass", report.TaskID, report.Status)
	}
	if !reflect.DeepEqual(executed, []string{"other-test"}) {
		t.Fatalf("executed commands = %#v, want only selected task command", executed)
	}
}

func TestRunnerDoesNotRequireWorkspaceForTaskSelectionErrors(t *testing.T) {
	runner := Runner{}

	report := runner.Run(context.Background(), runnerSuite(), RunInput{})

	if report.Status != StatusError {
		t.Fatalf("status = %s, want task selection error", report.Status)
	}
	if report.Error == "" {
		t.Fatal("task selection error should be reported")
	}
}

func TestDefaultChangedFilesParsesGitStatusPorcelain(t *testing.T) {
	workspace := t.TempDir()
	runner := Runner{
		RunCommand: func(context.Context, string, Command) CommandResult {
			t.Fatal("command should not be needed")
			return CommandResult{}
		},
		runGit: func(_ context.Context, gotWorkspace string, args ...string) ([]byte, error) {
			if gotWorkspace != workspace {
				t.Fatalf("workspace = %q, want %q", gotWorkspace, workspace)
			}
			if !reflect.DeepEqual(args, []string{"status", "--porcelain", "--untracked-files=all"}) {
				t.Fatalf("git args = %#v", args)
			}
			return []byte(" M internal/reader/a.go\nR  old.go -> internal/reader/b.go\n?? z.go\n"), nil
		},
	}

	files, err := runner.changedFiles(context.Background(), workspace)
	if err != nil {
		t.Fatalf("changedFiles returned error: %v", err)
	}

	want := []string{"internal/reader/a.go", "internal/reader/b.go", "z.go"}
	if !reflect.DeepEqual(files, want) {
		t.Fatalf("files = %#v, want %#v", files, want)
	}
}

func TestRunnerBlocksWhenWorkspacePathIsAFile(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "file.txt")
	if err := os.WriteFile(path, []byte("not a directory"), 0o600); err != nil {
		t.Fatalf("write file workspace: %v", err)
	}

	report := (Runner{}).Run(context.Background(), runnerSuite(), RunInput{
		TaskID:        "single",
		WorkspacePath: path,
	})

	if report.Status != StatusBlocked {
		t.Fatalf("status = %s, want blocked", report.Status)
	}
}

func runnerSuite() Suite {
	return Suite{
		ID:   "runner-suite",
		Name: "Runner suite",
		Tasks: []Task{
			{
				ID:               "single",
				Name:             "Single command",
				Prompt:           "Change reader.",
				WorkspaceFixture: "fixtures/reader",
				VerificationCommands: []Command{{
					ID:      "test",
					Name:    "Tests",
					Command: []string{"go", "test", "./..."},
				}},
				ExpectedChangedFiles: []string{"internal/reader/a.go", "internal/reader/b.go"},
			},
			{
				ID:               "two-commands",
				Name:             "Two commands",
				Prompt:           "Run two commands.",
				WorkspaceFixture: "fixtures/reader",
				VerificationCommands: []Command{
					{ID: "pass", Name: "Passing", Command: []string{"go", "test", "./..."}},
					{ID: "fail", Name: "Failing", Command: []string{"go", "test", "./internal/nope"}},
				},
				ExpectedChangedFiles: []string{"internal/reader/a.go"},
			},
			{
				ID:               "other",
				Name:             "Other task",
				Prompt:           "Change CLI.",
				WorkspaceFixture: "fixtures/cli",
				VerificationCommands: []Command{{
					ID:      "other-test",
					Name:    "Other tests",
					Command: []string{"go", "test", "./internal/cli"},
				}},
				ExpectedChangedFiles: []string{"cmd/zero/main.go"},
			},
		},
	}
}