zero/internal/agenteval/run.go

package agenteval

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

// defaultCommandTimeout bounds a single verification command so a hung command
// (e.g. a test that waits on input) cannot run unbounded. Generous enough for
// real build/test commands; overridable per run via RunInput.CommandTimeout.
const defaultCommandTimeout = 10 * time.Minute

type RunInput struct {
	TaskID        string
	WorkspacePath string
	TraceStdout   string
	// CommandTimeout bounds each verification command. Non-positive applies
	// defaultCommandTimeout.
	CommandTimeout time.Duration
}

type CommandRunner func(context.Context, string, Command) CommandResult

type ChangedFilesFunc func(context.Context, string) ([]string, error)

type Runner struct {
	RunCommand   CommandRunner
	ChangedFiles ChangedFilesFunc

	runGit func(context.Context, string, ...string) ([]byte, error)
}

func (runner Runner) Run(ctx context.Context, suite Suite, input RunInput) Report {
	if ctx == nil {
		ctx = context.Background()
	}
	task, err := selectTask(suite, input.TaskID)
	if err != nil {
		return Score(suite, ScoreInput{TaskID: input.TaskID})
	}
	workspace, err := verifyWorkspace(input.WorkspacePath)
	if err != nil {
		return runner.blocked(suite, task.ID, fmt.Sprintf("workspace setup failed: %v", err), nil)
	}
	if err := ctx.Err(); err != nil {
		return runner.blocked(suite, task.ID, err.Error(), nil)
	}
	var contextResult *ContextCheckResult
	var contextError string
	if len(task.ContextChecks.RequiredFiles) > 0 || len(task.ContextChecks.ForbiddenFiles) > 0 {
		result, err := task.ContextChecks.CheckWorkspace(workspace)
		if err != nil {
			contextError = err.Error()
		} else {
			contextResult = &result
		}
	}

	timeout := input.CommandTimeout
	if timeout <= 0 {
		timeout = defaultCommandTimeout
	}
	results := make([]CommandResult, 0, len(task.VerificationCommands))
	for _, command := range task.VerificationCommands {
		if err := ctx.Err(); err != nil {
			return runner.blocked(suite, task.ID, err.Error(), results)
		}
		result := runner.runCommand(ctx, workspace, command, timeout)
		if result.ID == "" {
			result.ID = command.ID
		}
		results = append(results, result)
	}

	files, err := runner.changedFiles(ctx, workspace)
	if err != nil {
		reason := fmt.Sprintf("changed files collection failed: %v", err)
		return runner.blocked(suite, task.ID, reason, results)
	}

	return Score(suite, ScoreInput{
		TaskID:             task.ID,
		CommandResults:     results,
		ChangedFiles:       files,
		ContextCheckResult: contextResult,
		ContextCheckError:  contextError,
		TraceStdout:        input.TraceStdout,
	})
}

func (runner Runner) blocked(suite Suite, taskID string, reason string, results []CommandResult) Report {
	return Score(suite, ScoreInput{
		TaskID:         taskID,
		CommandResults: results,
		Blocked:        true,
		BlockReason:    reason,
	})
}

func (runner Runner) runCommand(ctx context.Context, workspace string, command Command, timeout time.Duration) CommandResult {
	if timeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, timeout)
		defer cancel()
	}
	if runner.RunCommand != nil {
		return runner.RunCommand(ctx, workspace, command)
	}
	return execCommand(ctx, workspace, command)
}

func (runner Runner) changedFiles(ctx context.Context, workspace string) ([]string, error) {
	if runner.ChangedFiles != nil {
		return runner.ChangedFiles(ctx, workspace)
	}
	runGit := runner.runGit
	if runGit == nil {
		runGit = defaultRunGit
	}
	output, err := runGit(ctx, workspace, "status", "--porcelain", "--untracked-files=all")
	if err != nil {
		return nil, err
	}
	return parseGitStatusPorcelain(output), nil
}

func execCommand(ctx context.Context, workspace string, command Command) CommandResult {
	result := CommandResult{ID: command.ID, ExitCode: -1}
	if emptyCommand(command.Command) {
		result.Error = "command must not be empty"
		return result
	}
	args := trimCommand(command.Command)
	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
	cmd.Dir = workspace
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	result.Stdout = stdout.String()
	result.Stderr = stderr.String()
	if err == nil {
		result.ExitCode = 0
		return result
	}
	var exitErr *exec.ExitError
	if errors.As(err, &exitErr) {
		result.ExitCode = exitErr.ExitCode()
		return result
	}
	if ctxErr := ctx.Err(); ctxErr != nil {
		result.Error = ctxErr.Error()
		return result
	}
	result.Error = err.Error()
	return result
}

func defaultRunGit(ctx context.Context, workspace string, args ...string) ([]byte, error) {
	allArgs := append([]string{"-C", workspace}, args...)
	cmd := exec.CommandContext(ctx, "git", allArgs...)
	output, err := cmd.Output()
	if err != nil {
		if ctxErr := ctx.Err(); ctxErr != nil {
			return nil, ctxErr
		}
		return nil, err
	}
	return output, nil
}

func parseGitStatusPorcelain(output []byte) []string {
	lines := strings.Split(string(output), "\n")
	files := make([]string, 0, len(lines))
	for _, line := range lines {
		if len(line) < 4 {
			continue
		}
		file := strings.TrimSpace(line[3:])
		if arrow := strings.LastIndex(file, " -> "); arrow >= 0 {
			file = file[arrow+4:]
		}
		files = append(files, file)
	}
	return normalizeFiles(files)
}

func verifyWorkspace(workspace string) (string, error) {
	workspace = strings.TrimSpace(workspace)
	if workspace == "" {
		return "", errors.New("workspace path is required")
	}
	clean, err := filepath.Abs(workspace)
	if err != nil {
		return "", err
	}
	info, err := os.Stat(clean)
	if err != nil {
		return "", err
	}
	if !info.IsDir() {
		return "", errors.New("workspace path must be a directory")
	}
	return clean, nil
}

func trimCommand(command []string) []string {
	trimmed := make([]string, 0, len(command))
	for _, part := range command {
		part = strings.TrimSpace(part)
		if part != "" {
			trimmed = append(trimmed, part)
		}
	}
	return trimmed
}