mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 09:49:34 -10:00
Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3518bfc64e | ||
|
|
fee94fc949 | ||
|
|
29e47d879d | ||
|
|
a3a19b26b6 | ||
|
|
9acb4113cc | ||
|
|
d66a9dda69 | ||
|
|
7d579996b4 | ||
|
|
abe24bcb10 | ||
|
|
75a78e715b | ||
|
|
8502449430 | ||
|
|
c0bcfdda41 | ||
|
|
1bb6b5745a |
No files matched your search
@@ -127,6 +127,9 @@ tagged. Until then, source builds report the version `dev`.
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- Shared multi-agent skills discovery: when present, `~/.agents/skills` is searched after the primary
|
||||
Zero skills dir (and before plugin skill roots). `zero skills list` / `info` and the runtime `skill`
|
||||
tool share one multi-root discovery path; install/remove/lock still target only the Zero skills directory.
|
||||
- `SECURITY.md` with a private vulnerability-reporting path, `CODE_OF_CONDUCT.md`, this changelog, and
|
||||
GitHub issue/PR templates.
|
||||
- Interactive `/theme` picker: bare `/theme` opens a popup that live-previews each palette as you move
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Zero build/test/lint targets. AGENTS.md says "Build with `make`" and "Run `make
|
||||
# lint` before opening a PR" — these targets back those instructions.
|
||||
.DEFAULT_GOAL := build
|
||||
.PHONY: build build-all test test-race vet fmt fmt-check lint tidy clean help
|
||||
.PHONY: build build-all test test-race vet fmt fmt-check lint tidy clean baseline help
|
||||
|
||||
# Build the main CLI binary into ./zero.
|
||||
build:
|
||||
@@ -40,5 +40,19 @@ clean:
|
||||
rm -f zero
|
||||
go clean ./...
|
||||
|
||||
# Run the per-turn benchmark harness over the checked-in baseline manifest and
|
||||
# write the JSON result to internal/perfbench/reports/baseline.json. Requires a
|
||||
# built `zero` binary and a model; set ZERO_BENCH_MODEL (required) and
|
||||
# ZERO_BENCH_BINARY (defaults to ./zero) to configure the run. The report is
|
||||
# machine-specific and regenerated, not hand-edited.
|
||||
baseline: build
|
||||
@if [ -z "$(ZERO_BENCH_MODEL)" ]; then echo "Set ZERO_BENCH_MODEL (and optionally ZERO_BENCH_BINARY) before running 'make baseline'"; exit 2; fi
|
||||
@ZERO_BIN="$${ZERO_BENCH_BINARY:-./zero}"; \
|
||||
go run ./cmd/zero-perf-bench turn \
|
||||
--suite internal/perfbench/manifests/baseline.json \
|
||||
--model $(ZERO_BENCH_MODEL) \
|
||||
--binary "$$ZERO_BIN" \
|
||||
--output internal/perfbench/reports/baseline.json
|
||||
|
||||
help:
|
||||
@echo "Targets: build (default), build-all, test, test-quick, vet, fmt, fmt-check, lint, tidy, clean"
|
||||
@echo "Targets: build (default), build-all, test, test-quick, vet, fmt, fmt-check, lint, tidy, clean, baseline"
|
||||
@@ -129,11 +129,18 @@ the key in the wizard:
|
||||
export OPENAI_API_KEY=sk-...
|
||||
export ANTHROPIC_API_KEY=...
|
||||
export GEMINI_API_KEY=...
|
||||
export AIMLAPI_API_KEY=...
|
||||
export LONGCAT_API_KEY=...
|
||||
export MINIMAX_API_KEY=...
|
||||
export MINIMAXI_API_KEY=...
|
||||
```
|
||||
|
||||
To configure AI/ML API directly, run:
|
||||
|
||||
```bash
|
||||
zero providers setup aimlapi --set-active
|
||||
```
|
||||
|
||||
To configure Meituan LongCat (LongCat-2.0) directly, run:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -39,6 +39,9 @@ func run(args []string, getenv func(string) string, stdout io.Writer, stderr io.
|
||||
if len(args) > 0 && args[0] == "tasks" {
|
||||
return runTasksCommand(args[1:], getenv, stdout, stderr)
|
||||
}
|
||||
if len(args) > 0 && args[0] == "turn" {
|
||||
return runTurnCommand(args[1:], getenv, stdout, stderr)
|
||||
}
|
||||
options, err := parseArgs(args, getenv)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, err.Error())
|
||||
@@ -206,6 +209,7 @@ func helpText() string {
|
||||
return strings.Join([]string{
|
||||
"Usage: zero-perf-bench [options]",
|
||||
" zero-perf-bench tasks [options] (Terminal-Bench-style task harness; see `tasks --help`)",
|
||||
" zero-perf-bench turn [options] (per-turn tracing benchmark; see `turn --help`)",
|
||||
"",
|
||||
"Options:",
|
||||
" --iterations <n> Measured samples to collect (default: 5)",
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/perfbench"
|
||||
)
|
||||
|
||||
// turnOptions configures the `zero-perf-bench turn` subcommand: the per-turn
|
||||
// benchmark harness that runs ZERO headlessly with --trace, parses each turn's
|
||||
// NDJSON trace, and records per-span latency plus the top controllable latency
|
||||
// sources — the Phase 0 baseline's "do not proceed until" criterion.
|
||||
type turnOptions struct {
|
||||
SuitePath string
|
||||
Model string
|
||||
Mode string
|
||||
SelfCorrect bool
|
||||
Binary string
|
||||
Iterations int
|
||||
Version string
|
||||
Commit string
|
||||
Output string
|
||||
JSON bool
|
||||
DryRun bool
|
||||
Help bool
|
||||
}
|
||||
|
||||
func runTurnCommand(args []string, getenv func(string) string, stdout io.Writer, stderr io.Writer) int {
|
||||
options, err := parseTurnArgs(args, getenv)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, err.Error())
|
||||
return 2
|
||||
}
|
||||
if options.Help {
|
||||
_, _ = fmt.Fprint(stdout, turnHelpText())
|
||||
return 0
|
||||
}
|
||||
|
||||
set, err := perfbench.LoadTaskSet(options.SuitePath)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 1
|
||||
}
|
||||
|
||||
// The dry-run path records a zero-iteration run without a binary, so the
|
||||
// manifest loads and the report path is exercised in CI without a model.
|
||||
if options.DryRun {
|
||||
_, _ = fmt.Fprintln(stdout, "[zero] turn benchmark: dry run (no agent invoked)")
|
||||
return 0
|
||||
}
|
||||
|
||||
binary, err := perfbench.ResolveBinary(options.Binary)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 2
|
||||
}
|
||||
|
||||
result, err := perfbench.RunTurnBench(context.Background(), set, perfbench.TurnBenchConfig{
|
||||
Model: options.Model,
|
||||
Mode: options.Mode,
|
||||
SelfCorrect: options.SelfCorrect,
|
||||
Version: options.Version,
|
||||
Commit: options.Commit,
|
||||
Iterations: options.Iterations,
|
||||
Runner: perfbench.NewTurnExecRunner(binary),
|
||||
})
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 1
|
||||
}
|
||||
|
||||
if options.Output != "" {
|
||||
if err := writeTurnReport(options.Output, result); err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 1
|
||||
}
|
||||
}
|
||||
if options.JSON {
|
||||
if err := perfbench.WriteTurnBenchJSON(stdout, result); err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
_, _ = fmt.Fprintln(stdout, perfbench.FormatTurnBenchSummary(result))
|
||||
return 0
|
||||
}
|
||||
|
||||
func parseTurnArgs(args []string, getenv func(string) string) (turnOptions, error) {
|
||||
options := turnOptions{
|
||||
Iterations: 1,
|
||||
Version: strings.TrimSpace(getenv("ZERO_BENCH_VERSION")),
|
||||
Commit: strings.TrimSpace(getenv("ZERO_BENCH_COMMIT")),
|
||||
}
|
||||
for index := 0; index < len(args); index++ {
|
||||
arg := args[index]
|
||||
flag, inlineValue := splitFlagValue(arg)
|
||||
switch flag {
|
||||
case "--suite":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.SuitePath = value
|
||||
index = next
|
||||
case "--model":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Model = value
|
||||
index = next
|
||||
case "--mode":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Mode = value
|
||||
index = next
|
||||
case "--binary":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Binary = value
|
||||
index = next
|
||||
case "--iterations":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
parsed, err := parsePositiveInteger(flag, value)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Iterations = parsed
|
||||
index = next
|
||||
case "--version":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Version = value
|
||||
index = next
|
||||
case "--commit":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Commit = value
|
||||
index = next
|
||||
case "--output":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Output = value
|
||||
index = next
|
||||
case "--self-correct":
|
||||
if strings.Contains(arg, "=") {
|
||||
return options, fmt.Errorf("%s does not accept a value", flag)
|
||||
}
|
||||
options.SelfCorrect = true
|
||||
case "--json":
|
||||
if strings.Contains(arg, "=") {
|
||||
return options, fmt.Errorf("%s does not accept a value", flag)
|
||||
}
|
||||
options.JSON = true
|
||||
case "--dry-run":
|
||||
if strings.Contains(arg, "=") {
|
||||
return options, fmt.Errorf("%s does not accept a value", flag)
|
||||
}
|
||||
options.DryRun = true
|
||||
case "-h", "--help":
|
||||
if strings.Contains(arg, "=") {
|
||||
return options, fmt.Errorf("%s does not accept a value", flag)
|
||||
}
|
||||
options.Help = true
|
||||
default:
|
||||
return options, fmt.Errorf("unknown option: %s", arg)
|
||||
}
|
||||
}
|
||||
if options.Help {
|
||||
return options, nil
|
||||
}
|
||||
if strings.TrimSpace(options.SuitePath) == "" {
|
||||
return options, fmt.Errorf("--suite is required")
|
||||
}
|
||||
if strings.TrimSpace(options.Model) == "" && !options.DryRun {
|
||||
return options, fmt.Errorf("--model is required (or pass --dry-run)")
|
||||
}
|
||||
return options, nil
|
||||
}
|
||||
|
||||
func writeTurnReport(path string, result perfbench.TurnBenchResult) error {
|
||||
if dir := filepath.Dir(path); dir != "" {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
if err := perfbench.WriteTurnBenchJSON(&buffer, result); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, buffer.Bytes(), 0o644)
|
||||
}
|
||||
|
||||
func turnHelpText() string {
|
||||
return strings.Join([]string{
|
||||
"Usage: zero-perf-bench turn [options]",
|
||||
"",
|
||||
"Runs ZERO headlessly with --trace against a per-turn benchmark task set and",
|
||||
"records per-span latency plus the top controllable latency sources (the",
|
||||
"Phase 0 baseline's \"do not proceed until\" criterion). Each task is a fresh",
|
||||
"`zero exec` process, so iterations are cold-start samples; a warm path needs",
|
||||
"an in-process runner (future work).",
|
||||
"",
|
||||
"Options:",
|
||||
" --suite <path> Task set JSON file (required)",
|
||||
" --model <model> Model to run (required unless --dry-run)",
|
||||
" --mode <name> Exec mode preset to apply",
|
||||
" --self-correct Enable the post-edit verify-and-correct loop",
|
||||
" --binary <path> Path to the `zero` binary (default: zero on PATH / repo root)",
|
||||
" --iterations <n> Times to run each task (default: 1)",
|
||||
" --version <v> Record the ZERO version (default: $ZERO_BENCH_VERSION)",
|
||||
" --commit <sha> Record the ZERO commit (default: $ZERO_BENCH_COMMIT)",
|
||||
" --output <path> Write the JSON result to path",
|
||||
" --json Print only the JSON result",
|
||||
" --dry-run Load the manifest and exit without invoking the agent",
|
||||
" -h, --help Show this help",
|
||||
}, "\n") + "\n"
|
||||
}
|
||||
+17
-4
@@ -102,14 +102,24 @@ The full format spec (frontmatter fields, tool scopes, prompt conventions) is in
|
||||
|
||||
Skills are markdown instruction packs the agent can pull in on demand. Each skill is a directory containing a `SKILL.md`. Standalone project skill directories are not supported in this version (shared project-wide skills must go in `AGENTS.md` or as a hook). However, project plugins (section 6) may bundle skills, which are merged into the active run.
|
||||
|
||||
Discovery root: `$ZERO_SKILLS_DIR` → `$XDG_DATA_HOME/zero/skills` → `~/.local/share/zero/skills/`. A missing directory is fine — Zero just reports "no skills".
|
||||
Discovery roots (earlier wins on name collisions):
|
||||
|
||||
1. **Primary Zero dir** — `$ZERO_SKILLS_DIR` if set, else `$XDG_DATA_HOME/zero/skills`, else `~/.local/share/zero/skills/`
|
||||
2. **Shared multi-agent dir** — `~/.agents/skills/` when present (read-only discovery; never an install target)
|
||||
3. **Plugin skill roots** — skills bundled by active plugins (section 6)
|
||||
|
||||
A missing directory is fine — Zero just omits it. Management commands (`zero skills add` / `remove` / `lock`) always write to the primary Zero dir only; `list` / `info` search primary + `~/.agents/skills`.
|
||||
|
||||
```text
|
||||
~/.local/share/zero/skills/
|
||||
~/.local/share/zero/skills/ # primary (install/remove/lock target)
|
||||
run-benchmarks/
|
||||
SKILL.md
|
||||
write-changelog/
|
||||
SKILL.md
|
||||
|
||||
~/.agents/skills/ # optional shared multi-agent root
|
||||
shared-review/
|
||||
SKILL.md
|
||||
```
|
||||
|
||||
`SKILL.md` format:
|
||||
@@ -126,9 +136,9 @@ description: Run the project's benchmark suite and summarize the deltas.
|
||||
3. Report any regression > 5% with the function name and the previous value.
|
||||
```
|
||||
|
||||
Only `name` and `description` are recognized in the frontmatter today. The `name` defaults to the directory name. Within a single skills root, duplicate names are resolved by lexicographic directory order. During an agent run, Zero loads the default skills directory before plugin skill roots; earlier roots win name collisions silently. Plugin-declared skills (section 6) are merged into the active agent run at plugin activation time, so bundled skills appear in the available skills list and can be loaded with the `skill` tool.
|
||||
Only `name` and `description` are recognized in the frontmatter today. The `name` defaults to the directory name. Within a single skills root, duplicate names are resolved by lexicographic directory order. Across roots, Zero loads the primary directory first, then `~/.agents/skills`, then plugin skill roots; earlier roots win name collisions. Plugin-declared skills (section 6) are merged into the active agent run at plugin activation time, so bundled skills appear in the available skills list and can be loaded with the `skill` tool.
|
||||
|
||||
The `skill` core tool lets the agent load any discovered skill by name.
|
||||
The `skill` tool lets the agent load any discovered skill by name (primary, agents, or plugin).
|
||||
|
||||
## 4. Hooks
|
||||
|
||||
@@ -240,10 +250,13 @@ OAuth server names ending in `.<32 hex chars>` are reserved for token storage.
|
||||
|
||||
```bash
|
||||
zero serve --mcp
|
||||
zero serve --mcp -C /path/to/workspace
|
||||
zero serve --mcp --add-dir /path/to/extra
|
||||
```
|
||||
|
||||
The server speaks MCP over stdio. Configure it from the receiving side as a `stdio` server whose command is `zero serve --mcp`.
|
||||
|
||||
`-C/--cwd` sets the workspace root exposed as MCP resources and used by core tools. `--add-dir` (repeatable) widens that resource/tool scope beyond the workspace without granting the sandbox temp roots used by interactive runs.
|
||||
## 6. Plugins
|
||||
|
||||
A plugin is a self-contained directory that bundles tools, hooks, and skills for one capability. Plugins live at:
|
||||
|
||||
@@ -733,9 +733,10 @@ flowchart TD
|
||||
- **Skills** are instruction packs the model can load on demand.
|
||||
- **Specialists** are sub-agents callable through the `Task` tool.
|
||||
- **MCP servers** contribute external tools.
|
||||
- **Plugins** can add tools, hooks, and skill roots. Bootstrap merges plugin skill
|
||||
roots into the runtime skill tool and prompt-visible skill list; the base
|
||||
`internal/skills` scanner still scans one root at a time.
|
||||
- **Plugins** can add tools, hooks, and skill roots. Bootstrap always registers a
|
||||
multi-root skill tool: primary Zero skills dir, optional `~/.agents/skills`,
|
||||
then plugin skill roots (earlier wins). `internal/skills` owns that merge via
|
||||
`LoadFromRoots` / `DiscoveryRoots`; single-root `Load` remains for install/write.
|
||||
- **Hooks** can observe or block tool lifecycle events.
|
||||
|
||||
## End-to-End Data Flow
|
||||
|
||||
@@ -3,9 +3,9 @@ module github.com/Gitlawb/zero
|
||||
go 1.26.5
|
||||
|
||||
require (
|
||||
charm.land/bubbles/v2 v2.1.0
|
||||
charm.land/bubbletea/v2 v2.0.7
|
||||
charm.land/lipgloss/v2 v2.0.4
|
||||
charm.land/bubbles/v2 v2.1.1
|
||||
charm.land/bubbletea/v2 v2.0.8
|
||||
charm.land/lipgloss/v2 v2.0.5
|
||||
github.com/alecthomas/chroma/v2 v2.27.0
|
||||
github.com/atotto/clipboard v0.1.4
|
||||
github.com/aymanbagabas/go-udiff v0.4.1
|
||||
@@ -14,18 +14,18 @@ require (
|
||||
github.com/charmbracelet/x/term v0.2.2
|
||||
github.com/coder/websocket v1.8.15
|
||||
github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
|
||||
golang.org/x/sys v0.46.0
|
||||
golang.org/x/sys v0.47.0
|
||||
mvdan.cc/sh/v3 v3.13.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260615092913-2399af76d5b1 // indirect
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260703014108-f5a850f9c2b7 // indirect
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20260615092313-b57e5e6d29bb // indirect
|
||||
github.com/charmbracelet/x/termios v0.1.1 // indirect
|
||||
github.com/charmbracelet/x/windows v0.2.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.11.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.5.0 // indirect
|
||||
github.com/go-quicktest/qt v1.102.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.24 // indirect
|
||||
@@ -34,5 +34,5 @@ require (
|
||||
github.com/rogpeppe/go-internal v1.15.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
|
||||
golang.org/x/sync v0.21.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
)
|
||||
@@ -1,9 +1,9 @@
|
||||
charm.land/bubbles/v2 v2.1.0 h1:YSnNh5cPYlYjPxRrzs5VEn3vwhtEn3jVGRBT3M7/I0g=
|
||||
charm.land/bubbles/v2 v2.1.0/go.mod h1:l97h4hym2hvWBVfmJDtrEHHCtkIKeTEb3TTJ4ZOB3wY=
|
||||
charm.land/bubbletea/v2 v2.0.7 h1:7qw2tTAVar7m7klOPBYfTB0mniv/RuexsYwMRNxSeL0=
|
||||
charm.land/bubbletea/v2 v2.0.7/go.mod h1:DGW2q8gvzHnOpMpZTORs0aySVHCox5C+2Svk0fci1qs=
|
||||
charm.land/lipgloss/v2 v2.0.4 h1:lcPeVtcp23SNra7lHy8iYE4UC2aIipVQ47sbGyyxR5Q=
|
||||
charm.land/lipgloss/v2 v2.0.4/go.mod h1:0653x8epbZSzdDfO/XPS1a/uYPOBeSsCssOpJOqDzik=
|
||||
charm.land/bubbles/v2 v2.1.1 h1:7r55WzBxpo/R3z98hGmY7KKPd3ET6vsf0Fb9sDHOV60=
|
||||
charm.land/bubbles/v2 v2.1.1/go.mod h1:GE6M31gaWZVXzGw73OeuTTgy4lX+OtkH0E5ymnNsHxo=
|
||||
charm.land/bubbletea/v2 v2.0.8 h1:SxTJMhCAI3lbPmy4SgX5LWZ24AdINr4I6UEqzZvYJuY=
|
||||
charm.land/bubbletea/v2 v2.0.8/go.mod h1:2SkdgoTXluXJHOUwAoRlRXF/28vklb1rFl6GcgV1/ss=
|
||||
charm.land/lipgloss/v2 v2.0.5 h1:kbNxgeeUOYv5J0YdpxFjfvf3dFvqH8Aci4zB6xqFtrY=
|
||||
charm.land/lipgloss/v2 v2.0.5/go.mod h1:9oqhxt4yxIMe6q5A4kHr44DremZk7J9UNh74GlWa5nc=
|
||||
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
|
||||
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r5BXYZs=
|
||||
@@ -16,8 +16,8 @@ github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ
|
||||
github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w=
|
||||
github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
|
||||
github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260615092913-2399af76d5b1 h1:4+r3uOJ69ueRBt4okgEfWZeXs3BD36HcDBmOIAUlETk=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260615092913-2399af76d5b1/go.mod h1:f/jRa757WUmaOZrbPspXymbg/GnbF+rwe4OLsG7aXYo=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260703014108-f5a850f9c2b7 h1:3FmWoGNWK4STvqg0O0Aeav2T7rodWJAPeF0QpH+8gFw=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260703014108-f5a850f9c2b7/go.mod h1:f/jRa757WUmaOZrbPspXymbg/GnbF+rwe4OLsG7aXYo=
|
||||
github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI=
|
||||
github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20260615092313-b57e5e6d29bb h1:m1Uub9uM7meZ8ofVyeRfK7M/knCi3sF2gKn9kldVW0I=
|
||||
@@ -34,8 +34,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/dlclark/regexp2/v2 v2.5.0 h1:liNiWIPCvCh5HBcYfsjd+P16AG79fwd6T1Toy2gOtEA=
|
||||
github.com/dlclark/regexp2/v2 v2.5.0/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
|
||||
github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
@@ -62,9 +62,9 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
mvdan.cc/sh/v3 v3.13.1 h1:DP3TfgZhDkT7lerUdnp6PTGKyxxzz6T+cOlY/xEvfWk=
|
||||
mvdan.cc/sh/v3 v3.13.1/go.mod h1:lXJ8SexMvEVcHCoDvAGLZgFJ9Wsm2sulmoNEXGhYZD0=
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -430,6 +431,12 @@ func (state *compactionState) maybeCompact(
|
||||
state.lowWaterMark = size
|
||||
return messages
|
||||
}
|
||||
// Only count a compaction when it actually shrank the history, so the
|
||||
// compaction counter reflects real context reductions rather than paid
|
||||
// no-ops that left the token budget untouched.
|
||||
if r := trace.FromContext(ctx); r != nil {
|
||||
r.Counter(trace.CounterCompactionCount, 1)
|
||||
}
|
||||
state.lowWaterMark = newSize
|
||||
return compacted
|
||||
}
|
||||
@@ -479,7 +486,11 @@ func (state *compactionState) recover(
|
||||
// one-shot budget now so a provider that keeps returning context-limit errors
|
||||
// after a successful compaction can't loop forever. Store the low-water mark in
|
||||
// the SAME combined (messages + tool-defs) domain maybeCompact uses, so the
|
||||
// proactive shrink-guard compares like with like.
|
||||
// proactive shrink-guard compares like with like. Count it now that the shrink
|
||||
// is confirmed, so the counter mirrors maybeCompact's real-reduction policy.
|
||||
if r := trace.FromContext(ctx); r != nil {
|
||||
r.Counter(trace.CounterCompactionCount, 1)
|
||||
}
|
||||
state.reactiveAttempted = true
|
||||
state.lowWaterMark = state.calibratedTokens(estimateTokens(result) + estimateToolDefTokens(tools))
|
||||
return result, true, nil
|
||||
|
||||
+82
-5
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/streamjson"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -114,6 +115,26 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
return Result{}, errors.New("agent provider is required")
|
||||
}
|
||||
|
||||
// Tracing is opt-in. When a recorder is wired, thread it into ctx so the
|
||||
// providerio seam and reconnect helper can reach it via trace.FromContext,
|
||||
// mark the run's start, and wrap OnUsage so token counters accumulate for
|
||||
// free alongside the existing per-request plumbing. nil recorder leaves the
|
||||
// loop byte-identical: FromContext returns nil, the no-op stamps cost
|
||||
// nothing, and the OnUsage wrapper is not installed.
|
||||
if options.Trace != nil {
|
||||
ctx = trace.WithContext(ctx, options.Trace)
|
||||
options.Trace.Start()
|
||||
originalOnUsage := options.OnUsage
|
||||
options.OnUsage = func(usage Usage) {
|
||||
if originalOnUsage != nil {
|
||||
originalOnUsage(usage)
|
||||
}
|
||||
options.Trace.Counter(trace.CounterInputTokens, int64(usage.InputTokens))
|
||||
options.Trace.Counter(trace.CounterCachedInputTokens, int64(usage.CachedInputTokens))
|
||||
options.Trace.Counter(trace.CounterOutputTokens, int64(usage.OutputTokens))
|
||||
}
|
||||
}
|
||||
|
||||
maxTurns := options.MaxTurns
|
||||
if maxTurns <= 0 {
|
||||
maxTurns = 12
|
||||
@@ -184,12 +205,20 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// the tool-definition tokens (they ride on every request) in its estimate.
|
||||
// partitionTools depends only on registry/permissions/options/loaded, not on
|
||||
// the messages, so computing it before compaction is safe.
|
||||
// Build the per-turn tool list first so proactive compaction can include
|
||||
// the tool-definition tokens (they ride on every request) in its estimate.
|
||||
// partitionTools depends only on registry/permissions/options/loaded, not on
|
||||
// the messages, so computing it before compaction is safe.
|
||||
toolPartitionSpan := options.Trace.Span(trace.SpanToolPartition)
|
||||
exposed, _ := partitionToolsCached(registry, permissionMode, options, loaded, toolDefCache)
|
||||
toolPartitionSpan.End()
|
||||
|
||||
// PROACTIVE compaction: if the history is approaching the model's
|
||||
// context window, summarize the oldest middle before building the
|
||||
// request. A no-op when ContextWindow == 0 (compaction disabled).
|
||||
compactionSpan := options.Trace.Span(trace.SpanCompaction)
|
||||
messages = compactor.maybeCompact(ctx, provider, messages, exposed)
|
||||
compactionSpan.End()
|
||||
request := zeroruntime.CompletionRequest{
|
||||
Messages: copyMessages(messages),
|
||||
Tools: exposed,
|
||||
@@ -255,11 +284,39 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// conversation-state duplication.
|
||||
forwardedVisibleText := false
|
||||
forwardingOpts := zeroruntime.CollectOptions{OnUsage: options.OnUsage}
|
||||
if options.OnText != nil {
|
||||
forwardingOpts.OnText = func(s string) { forwardedVisibleText = true; options.OnText(s) }
|
||||
// Install text/reasoning forwarding handlers whenever EITHER a user
|
||||
// callback OR a trace recorder is set. A headless traced run (e.g. `zero
|
||||
// exec --trace`) sets Trace but no OnText/OnReasoning; without these
|
||||
// handlers the stream is still collected, but FirstTokenAt would never
|
||||
// stamp and the trace would lose its TTFT signal. The trace recorder's
|
||||
// stamp methods are nil-safe, but we guard on `trace != nil` so a run with
|
||||
// a user callback but no recorder still works. forwardedVisibleText stays
|
||||
// tied to the USER callback only, preserving the stall-retry semantics
|
||||
// (a trace-only handler forwards nothing the user would see duplicated).
|
||||
rec := options.Trace
|
||||
onText := options.OnText
|
||||
if onText != nil || rec != nil {
|
||||
forwardingOpts.OnText = func(s string) {
|
||||
if rec != nil {
|
||||
rec.StampFirstVisibleEvent()
|
||||
rec.StampFirstToken()
|
||||
}
|
||||
if onText != nil {
|
||||
forwardedVisibleText = true
|
||||
onText(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.OnReasoning != nil {
|
||||
forwardingOpts.OnReasoning = func(s string) { options.OnReasoning(s) }
|
||||
onReasoning := options.OnReasoning
|
||||
if onReasoning != nil || rec != nil {
|
||||
forwardingOpts.OnReasoning = func(s string) {
|
||||
if rec != nil {
|
||||
rec.StampFirstToken()
|
||||
}
|
||||
if onReasoning != nil {
|
||||
onReasoning(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.OnToolCallStart != nil {
|
||||
forwardingOpts.OnToolCallStart = func(id, name string) { options.OnToolCallStart(id, name) }
|
||||
@@ -297,14 +354,18 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
if retryStreamErr != nil {
|
||||
return collected, retryStreamErr
|
||||
}
|
||||
genSpan := options.Trace.Span(trace.SpanGeneration)
|
||||
collected = zeroruntime.CollectStreamWithOptions(ctx, retryStream, zeroruntime.CollectOptions{
|
||||
OnUsage: options.OnUsage,
|
||||
})
|
||||
genSpan.End()
|
||||
}
|
||||
return collected, nil
|
||||
}
|
||||
|
||||
generationSpan := options.Trace.Span(trace.SpanGeneration)
|
||||
collected := zeroruntime.CollectStreamWithOptions(ctx, stream, forwardingOpts)
|
||||
generationSpan.End()
|
||||
if collected.Error != "" {
|
||||
updated, stop := recoverStreamError(collected)
|
||||
collected = updated
|
||||
@@ -357,7 +418,9 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
result.Messages = copyMessages(messages)
|
||||
return result, retryErr
|
||||
}
|
||||
stallGenSpan := options.Trace.Span(trace.SpanGeneration)
|
||||
collected = zeroruntime.CollectStreamWithOptions(ctx, retryStream, forwardingOpts)
|
||||
stallGenSpan.End()
|
||||
}
|
||||
if collected.Error != "" {
|
||||
// Route a reissued stream's non-stall error through the SAME recovery as
|
||||
@@ -459,6 +522,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
if cue || planPending {
|
||||
if continueNudges < maxContinueNudges {
|
||||
continueNudges++
|
||||
options.Trace.Counter(trace.CounterCompletionNudges, 1)
|
||||
reason := "your message ended mid-step"
|
||||
if !cue {
|
||||
reason = "pending plan items remain — finish them, or mark them complete with update_plan if you are done"
|
||||
@@ -488,6 +552,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// (no admission, no cue) then finalizes as success on the next turn.
|
||||
if options.SelfCorrect != nil && !acceptanceRequested {
|
||||
acceptanceRequested = true
|
||||
options.Trace.Counter(trace.CounterAcceptanceChecks, 1)
|
||||
messages = append(messages, zeroruntime.Message{
|
||||
Role: zeroruntime.MessageRoleUser,
|
||||
Content: acceptanceVerificationNudge(),
|
||||
@@ -548,20 +613,26 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
runEnd++
|
||||
}
|
||||
if runEnd-index >= 2 {
|
||||
batchSpan := options.Trace.Span(trace.SpanToolExecution)
|
||||
precomputed = executeParallelReadBatch(ctx, registry, collected.ToolCalls, index, runEnd, permissionMode, options)
|
||||
batchSpan.End()
|
||||
precomputedStart, precomputedEnd = index, runEnd
|
||||
}
|
||||
}
|
||||
if options.OnToolCall != nil {
|
||||
options.OnToolCall(call)
|
||||
}
|
||||
options.Trace.StampFirstUsefulAction()
|
||||
var toolResult ToolResult
|
||||
var abortErr error
|
||||
if index >= precomputedStart && index < precomputedEnd {
|
||||
toolResult, abortErr = precomputed[index-precomputedStart].result, precomputed[index-precomputedStart].abortErr
|
||||
} else {
|
||||
toolSpan := options.Trace.Span(trace.SpanToolExecution)
|
||||
toolResult, abortErr = executeToolCall(ctx, registry, call, permissionMode, options)
|
||||
toolSpan.End()
|
||||
}
|
||||
options.Trace.Counter(trace.CounterToolCalls, 1)
|
||||
if options.OnToolResult != nil {
|
||||
options.OnToolResult(toolResult)
|
||||
}
|
||||
@@ -666,6 +737,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// context-window sizing, and usage attribution follow the new model.
|
||||
provider = newProvider
|
||||
options.Model = turnRequestedModel
|
||||
options.Trace.Counter(trace.CounterModelSwitches, 1)
|
||||
// KNOWN LIMITATION (deferred): the compactor's context-window budget
|
||||
// is fixed at run start from options.ContextWindow and is NOT updated
|
||||
// here, so a switch to a model with a different window keeps compacting
|
||||
@@ -764,6 +836,7 @@ func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages [
|
||||
if err != nil {
|
||||
return "", messages, ""
|
||||
}
|
||||
finalGenSpan := options.Trace.Span(trace.SpanGeneration)
|
||||
collected := zeroruntime.CollectStreamWithOptions(ctx, stream, zeroruntime.CollectOptions{
|
||||
OnText: options.OnText,
|
||||
OnReasoning: options.OnReasoning,
|
||||
@@ -771,6 +844,7 @@ func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages [
|
||||
OnToolCallStart: options.OnToolCallStart,
|
||||
OnToolCallDelta: options.OnToolCallDelta,
|
||||
})
|
||||
finalGenSpan.End()
|
||||
if ctx.Err() != nil || collected.Error != "" || strings.TrimSpace(collected.Text) == "" {
|
||||
return "", messages, ""
|
||||
}
|
||||
@@ -1874,7 +1948,10 @@ func requestPermission(ctx context.Context, request PermissionRequest, options O
|
||||
if options.OnPermissionRequest == nil {
|
||||
return PermissionDecision{Action: PermissionDecisionDeny, Reason: request.Reason}, nil
|
||||
}
|
||||
return options.OnPermissionRequest(ctx, request)
|
||||
permSpan := options.Trace.Span(trace.SpanPermissionWait)
|
||||
decision, err := options.OnPermissionRequest(ctx, request)
|
||||
permSpan.End()
|
||||
return decision, err
|
||||
}
|
||||
|
||||
func normalizePermissionDecisionAction(action PermissionDecisionAction) PermissionDecisionAction {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/specmode"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -3461,3 +3462,73 @@ func TestRunDoesNotFlagCleanToolOutput(t *testing.T) {
|
||||
t.Errorf("clean output should not get a reminder, got %q", captured.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunTracingWrapperStampsUsage verifies the per-turn tracing setup in Run:
|
||||
// a wired recorder is Started, OnUsage is wrapped so it still forwards to the
|
||||
// caller's callback AND stamps token counters, and the run completes.
|
||||
func TestRunTracingWrapperStampsUsage(t *testing.T) {
|
||||
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{{
|
||||
{Type: zeroruntime.StreamEventUsage, Usage: zeroruntime.Usage{InputTokens: 100, CachedInputTokens: 20, OutputTokens: 40}},
|
||||
{Type: zeroruntime.StreamEventText, Content: "done"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
}}}
|
||||
onUsageCalls := 0
|
||||
rec := trace.NewRecorder("tracing-session", "run-1", "test")
|
||||
if _, err := Run(context.Background(), "hi", provider, Options{
|
||||
SessionID: "tracing-session",
|
||||
Cwd: t.TempDir(),
|
||||
ProviderName: "test-provider",
|
||||
Model: "test-model",
|
||||
Trace: rec,
|
||||
OnUsage: func(Usage) { onUsageCalls++ },
|
||||
}); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
tr := rec.Finish()
|
||||
if tr.StartedAt.IsZero() {
|
||||
t.Fatal("tracing wrapper did not Start the recorder")
|
||||
}
|
||||
// FirstTokenAt must stamp even though no OnText/OnReasoning user callback is
|
||||
// set: a headless traced run (e.g. `zero exec --trace`) sets Trace but no UI
|
||||
// callbacks, so the loop installs trace-only forwarding handlers to capture
|
||||
// TTFT. Without them FirstTokenAt stays zero and the trace loses its signal.
|
||||
if tr.FirstTokenAt.IsZero() {
|
||||
t.Fatal("FirstTokenAt not stamped for a traced run with no OnText/OnReasoning callbacks")
|
||||
}
|
||||
if got := tr.Counter(trace.CounterInputTokens); got != 100 {
|
||||
t.Fatalf("input token counter = %d, want 100", got)
|
||||
}
|
||||
if got := tr.Counter(trace.CounterCachedInputTokens); got != 20 {
|
||||
t.Fatalf("cached input token counter = %d, want 20", got)
|
||||
}
|
||||
if got := tr.Counter(trace.CounterOutputTokens); got != 40 {
|
||||
t.Fatalf("output token counter = %d, want 40", got)
|
||||
}
|
||||
if onUsageCalls == 0 {
|
||||
t.Fatal("wrapped OnUsage did not forward to the caller's callback")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunNilTraceForwardsUsage verifies a nil recorder leaves the loop
|
||||
// byte-identical: OnUsage is not wrapped, so the caller's callback still fires
|
||||
// and nothing panics.
|
||||
func TestRunNilTraceForwardsUsage(t *testing.T) {
|
||||
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{{
|
||||
{Type: zeroruntime.StreamEventUsage, Usage: zeroruntime.Usage{InputTokens: 7}},
|
||||
{Type: zeroruntime.StreamEventText, Content: "done"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
}}}
|
||||
onUsageCalls := 0
|
||||
if _, err := Run(context.Background(), "hi", provider, Options{
|
||||
SessionID: "nil-trace-session",
|
||||
Cwd: t.TempDir(),
|
||||
ProviderName: "test-provider",
|
||||
Model: "test-model",
|
||||
OnUsage: func(Usage) { onUsageCalls++ },
|
||||
}); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if onUsageCalls == 0 {
|
||||
t.Fatal("OnUsage not forwarded when Trace is nil")
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/errhint"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -74,14 +75,21 @@ func stallRetryNoticeFor(options Options) reconnectNotifier {
|
||||
// already past its deadline is returned immediately (no retry) — those have
|
||||
// their own handling (compaction for context-limit, image-rejection, etc.).
|
||||
func streamWithReconnect(ctx context.Context, provider Provider, request zeroruntime.CompletionRequest, notify reconnectNotifier) (<-chan zeroruntime.StreamEvent, error) {
|
||||
recorder := trace.FromContext(ctx)
|
||||
stream, err := provider.StreamCompletion(ctx, request)
|
||||
if err == nil {
|
||||
if recorder != nil {
|
||||
recorder.Counter(trace.CounterModelRequests, 1)
|
||||
}
|
||||
return stream, nil
|
||||
}
|
||||
for attempt := 1; attempt <= maxStreamReconnects; attempt++ {
|
||||
if !shouldReconnect(ctx, err) {
|
||||
return nil, err
|
||||
}
|
||||
if recorder != nil {
|
||||
recorder.Counter(trace.CounterReconnectCount, 1)
|
||||
}
|
||||
if notify != nil {
|
||||
notify(attempt, maxStreamReconnects)
|
||||
}
|
||||
@@ -90,6 +98,9 @@ func streamWithReconnect(ctx context.Context, provider Provider, request zerorun
|
||||
}
|
||||
stream, err = provider.StreamCompletion(ctx, request)
|
||||
if err == nil {
|
||||
if recorder != nil {
|
||||
recorder.Counter(trace.CounterModelRequests, 1)
|
||||
}
|
||||
return stream, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/Gitlawb/zero/internal/lsp"
|
||||
"github.com/Gitlawb/zero/internal/redaction"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/verify"
|
||||
)
|
||||
|
||||
@@ -133,7 +134,9 @@ func (sc *SelfCorrector) inspect(ctx context.Context, changedFiles []string) Cor
|
||||
}
|
||||
|
||||
if sc.cfg.IncludeTests && sc.verifier != nil {
|
||||
verifySpan := trace.FromContext(ctx).Span(trace.SpanVerification)
|
||||
vr, err := sc.verifier.Verify(ctx)
|
||||
verifySpan.End()
|
||||
if err != nil {
|
||||
// "No plan / no tests" is not an error (DetectPlan returns an empty
|
||||
// plan), so a non-nil error means verification could not run at all —
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/streamjson"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -279,6 +280,16 @@ type Options struct {
|
||||
// request), so every existing caller is unaffected. A returned error is
|
||||
// non-fatal: the run continues on the current model.
|
||||
ModelSwitcher func(ctx context.Context, modelID string) (Provider, error)
|
||||
// Trace, when set, records per-turn timing for the run: the loop stamps
|
||||
// spans (prompt build, generation, tool execution, permission wait,
|
||||
// compaction, provider connect) and counters (model requests, tool calls,
|
||||
// retries, tokens) into it. nil DISABLES tracing entirely — every stamp is
|
||||
// nil-safe and the loop is byte-identical to an untraced run. The caller
|
||||
// owns the recorder: Run stamps into it but does not Finish or emit it.
|
||||
// A fresh Recorder is required per Run — reusing one across runs merges
|
||||
// their spans, counters, and first-event timestamps, and Finish freezes a
|
||||
// recorder so no further stamps take.
|
||||
Trace *trace.Recorder
|
||||
// SelfCorrect, when set, runs a post-edit verify-and-correct cycle after a
|
||||
// mutating tool call: it verifies the changed files (LSP diagnostics + project
|
||||
// tests) and feeds failures back to the model to fix, bounded by an attempt
|
||||
|
||||
+23
-1
@@ -42,6 +42,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/worktrees"
|
||||
"github.com/Gitlawb/zero/internal/zerogit"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
"github.com/charmbracelet/x/term"
|
||||
)
|
||||
|
||||
var version = "dev"
|
||||
@@ -339,7 +340,17 @@ func runWithDeps(args []string, stdout io.Writer, stderr io.Writer, deps appDeps
|
||||
return 0
|
||||
}
|
||||
}
|
||||
if _, err := fmt.Fprintf(stdout, "zero %s\n", version); err != nil {
|
||||
// Pipes and redirects keep the machine-readable contract exactly as it
|
||||
// was: a single "zero <version>" line. Scripts, command substitutions,
|
||||
// and the NPM wrapper smoke check all parse that record, so the banner
|
||||
// is strictly a TTY affordance.
|
||||
if !stdoutIsTerminal(stdout) {
|
||||
if _, err := fmt.Fprintf(stdout, "zero %s\n", version); err != nil {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if _, err := fmt.Fprintf(stdout, "%s\n\nzero %s\n", tui.Wordmark(), version); err != nil {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
@@ -1115,6 +1126,17 @@ func closeSpecialistRuntime(stderr io.Writer, runtime *agentToolRuntime) {
|
||||
}
|
||||
}
|
||||
|
||||
// stdoutIsTerminal reports whether w is an interactive terminal. Tests pass
|
||||
// bytes.Buffer writers and pipes/redirects fail term.IsTerminal, so both take
|
||||
// the machine-readable path.
|
||||
func stdoutIsTerminal(w io.Writer) bool {
|
||||
f, ok := w.(*os.File)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return term.IsTerminal(f.Fd())
|
||||
}
|
||||
|
||||
func writeAppError(stderr io.Writer, message string, exitCode int) int {
|
||||
if _, err := fmt.Fprintf(stderr, "[zero] %s\n", message); err != nil {
|
||||
return 1
|
||||
|
||||
@@ -41,6 +41,8 @@ func TestRunPrintsVersion(t *testing.T) {
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected exit code 0, got %d", exitCode)
|
||||
}
|
||||
// Non-terminal stdout keeps the machine-readable contract: exactly one
|
||||
// "zero <version>" record, no wordmark banner, no ANSI.
|
||||
if got := stdout.String(); got != "zero dev\n" {
|
||||
t.Fatalf("expected version output, got %q", got)
|
||||
}
|
||||
@@ -49,6 +51,33 @@ func TestRunPrintsVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunVersionRedirectedFile covers the `zero --version > file` path: stdout
|
||||
// is a real *os.File but not a terminal, so the single-line contract must hold.
|
||||
func TestRunVersionRedirectedFile(t *testing.T) {
|
||||
stdout, err := os.CreateTemp(t.TempDir(), "version-stdout")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp stdout: %v", err)
|
||||
}
|
||||
defer stdout.Close()
|
||||
var stderr bytes.Buffer
|
||||
|
||||
exitCode := Run([]string{"--version"}, stdout, &stderr)
|
||||
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected exit code 0, got %d", exitCode)
|
||||
}
|
||||
content, err := os.ReadFile(stdout.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("read temp stdout: %v", err)
|
||||
}
|
||||
if got := string(content); got != "zero dev\n" {
|
||||
t.Fatalf("expected version output, got %q", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr, got %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPrintsHelp(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--help"},
|
||||
|
||||
@@ -118,7 +118,8 @@ func runSkillInfo(args []string, dir string, stdout io.Writer, stderr io.Writer)
|
||||
if name == "" {
|
||||
return writeExecUsageError(stderr, "usage: zero skill info <name> [--json]")
|
||||
}
|
||||
info, ok := skills.Info(dir, name)
|
||||
// Resolve across primary + ~/.agents/skills; lock metadata only from primary.
|
||||
info, ok := skills.InfoFromRoots(dir, skills.GlobalRoots(dir), name)
|
||||
if !ok {
|
||||
return writeAppError(stderr, fmt.Sprintf("skill %q not found", name), exitUsage)
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/specmode"
|
||||
"github.com/Gitlawb/zero/internal/streamjson"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/usage"
|
||||
"github.com/Gitlawb/zero/internal/worktrees"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
@@ -124,6 +125,12 @@ type execOptions struct {
|
||||
// additional write roots for this run. Unioned with
|
||||
// config.SandboxConfig.AdditionalWriteRoots at scope construction time.
|
||||
addDirs []string
|
||||
// tracePath, when set, writes a per-turn NDJSON trace (agenteval-compatible)
|
||||
// to the given file path — or to stderr when the value is "-". Falls back to
|
||||
// the ZERO_TRACE env var when the flag is absent. Off by default: a run
|
||||
// without it leaves agent.Options.Trace nil and is byte-identical to before.
|
||||
// The trace is pure observation — enabling it does not change agent behavior.
|
||||
tracePath string
|
||||
}
|
||||
|
||||
type execUsageError struct {
|
||||
@@ -422,6 +429,12 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
notify.MaybeAddWebhookSink(notifier, os.Getenv, func(format string, args ...any) {
|
||||
fmt.Fprintf(stderr, "[notify] "+format+"\n", args...)
|
||||
})
|
||||
// Spec-draft runs synthesize a prompt offline and never drive a model turn, so
|
||||
// there is no per-turn trace to emit. Reject --trace / ZERO_TRACE up front with
|
||||
// a clear error rather than silently accepting and writing nothing.
|
||||
if options.useSpec && resolveTracePath(options) != "" {
|
||||
return writeExecFormatUsageError(stdout, stderr, options.outputFormat, "--trace / ZERO_TRACE are not supported for spec-draft runs")
|
||||
}
|
||||
if options.useSpec {
|
||||
return runExecSpecDraft(execSpecDraftRun{
|
||||
options: options,
|
||||
@@ -474,6 +487,24 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
if err != nil {
|
||||
return writeAppError(stderr, "failed to create run id: "+err.Error(), exitCrash)
|
||||
}
|
||||
// Per-turn tracing is opt-in (--trace <path> or ZERO_TRACE=<path>). The
|
||||
// recorder is stamped throughout agent.Run and the providerio seam; the run
|
||||
// itself is byte-identical to an untraced run. We finish the recorder at the
|
||||
// agent.Run boundary (below) so the snapshot captures exactly one turn, then
|
||||
// serialize the snapshot on every exit path via the defer. "-" writes NDJSON
|
||||
// to stderr; otherwise the path is created/truncated. A failure to emit is
|
||||
// logged to stderr, never fatal.
|
||||
tracePath := resolveTracePath(options)
|
||||
var traceRecorder *trace.Recorder
|
||||
var traceSnapshot *trace.TurnTrace
|
||||
if tracePath != "" {
|
||||
traceRecorder = trace.NewRecorder(preparedSession.Session.SessionID, runID, "")
|
||||
defer func() {
|
||||
if err := writeTraceSnapshot(traceSnapshot, tracePath, stderr); err != nil {
|
||||
fmt.Fprintf(stderr, "[zero] failed to write trace: %s\n", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
writer := execEventWriter{
|
||||
stdout: stdout,
|
||||
stderr: stderr,
|
||||
@@ -556,6 +587,7 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
Model: resolved.Provider.Model,
|
||||
ModelSwitcher: modelSwitcher,
|
||||
ReasoningEffort: forwardEffort,
|
||||
Trace: traceRecorder,
|
||||
Cwd: workspaceRoot,
|
||||
Images: images,
|
||||
Registry: registry,
|
||||
@@ -625,6 +657,12 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
sessionRecorder.append(sessions.EventUsage, payload)
|
||||
},
|
||||
})
|
||||
// Finish the trace now that the turn is done, so the snapshot captures exactly
|
||||
// agent.Run's work and nothing the post-run cleanup stamps. The deferred
|
||||
// writer serializes the snapshot on every exit path.
|
||||
if traceRecorder != nil {
|
||||
traceSnapshot = traceRecorder.Finish()
|
||||
}
|
||||
notifier.Notify(notify.Completion, notify.DefaultMessage(notify.Completion))
|
||||
if writer.err != nil {
|
||||
return exitCrash
|
||||
@@ -1237,3 +1275,36 @@ func execNotifyMode(options execOptions, resolved config.ResolvedConfig) string
|
||||
}
|
||||
return resolved.Notify.Mode
|
||||
}
|
||||
|
||||
// resolveTracePath returns the trace destination for this run: the --trace flag
|
||||
// value when set, else the ZERO_TRACE env var, else "" (tracing off). A value of
|
||||
// "-" means "write to stderr".
|
||||
func resolveTracePath(options execOptions) string {
|
||||
if v := strings.TrimSpace(options.tracePath); v != "" {
|
||||
return v
|
||||
}
|
||||
if v := strings.TrimSpace(os.Getenv("ZERO_TRACE")); v != "" {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// writeTraceSnapshot writes an already-finished trace snapshot to dest ("-" =>
|
||||
// stderr, otherwise a file path created/truncated). A nil snapshot is a no-op
|
||||
// (tracing off, or the run exited before agent.Run produced a turn). The trace
|
||||
// is best-effort: a write error is returned to the caller, which logs it but
|
||||
// never fails the run.
|
||||
func writeTraceSnapshot(snapshot *trace.TurnTrace, dest string, stderr io.Writer) error {
|
||||
if snapshot == nil || dest == "" {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(dest) == "-" {
|
||||
return trace.WriteNDJSON(stderr, snapshot)
|
||||
}
|
||||
file, err := os.Create(dest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
return trace.WriteNDJSON(file, snapshot)
|
||||
}
|
||||
@@ -32,6 +32,15 @@ func parseExecArgs(args []string) (execOptions, bool, error) {
|
||||
options.noNotify = true
|
||||
case arg == "--no-completion-gate":
|
||||
options.noCompletionGate = true
|
||||
case arg == "--trace":
|
||||
value, next, err := nextFlagValue(args, index, arg)
|
||||
if err != nil {
|
||||
return options, false, err
|
||||
}
|
||||
options.tracePath = value
|
||||
index = next
|
||||
case strings.HasPrefix(arg, "--trace="):
|
||||
options.tracePath = strings.TrimSpace(strings.TrimPrefix(arg, "--trace="))
|
||||
case arg == "--notify":
|
||||
value, next, err := nextFlagValue(args, index, arg)
|
||||
if err != nil {
|
||||
|
||||
@@ -15,11 +15,11 @@ import (
|
||||
|
||||
// pluginActivation holds what plugin activation contributed to a bootstrap so the
|
||||
// later dispatcher + skill wiring can consume it: the plugin hook definitions and
|
||||
// the plugin skill search roots. The zero value (no plugins) is inert — the
|
||||
// dispatcher gets no extra hooks and the skill tool keeps only the default dir.
|
||||
// The embedded trustSkip reports whether the project plugin layer was dropped for
|
||||
// an untrusted workspace, so the caller can fold it into the single combined
|
||||
// trust notice.
|
||||
// the plugin skill search roots. The zero value (no plugins) still overlays the
|
||||
// multi-root skill tool (primary + ~/.agents/skills) so discovery is identical
|
||||
// with or without plugins. The embedded trustSkip reports whether the project
|
||||
// plugin layer was dropped for an untrusted workspace, so the caller can fold it
|
||||
// into the single combined trust notice.
|
||||
type pluginActivation struct {
|
||||
hooks []hooks.Definition
|
||||
skillRoots []string
|
||||
@@ -52,6 +52,9 @@ func activatePlugins(workspaceRoot string, registry *tools.Registry, deps appDep
|
||||
loaded, err := deps.loadPlugins(plugins.LoadOptions{Cwd: workspaceRoot, ExcludeProject: excludeProject})
|
||||
if err != nil {
|
||||
writePluginActivationWarning(stderr, "failed to load plugins: "+err.Error())
|
||||
// Still overlay multi-root discovery (primary + agents) so a plugin load
|
||||
// failure cannot hide shared skills.
|
||||
registry.Register(plugins.NewSkillTool(deps.skillsDir(), nil))
|
||||
return pluginActivation{trustSkip: skip}
|
||||
}
|
||||
|
||||
@@ -67,14 +70,11 @@ func activatePlugins(workspaceRoot string, registry *tools.Registry, deps appDep
|
||||
writePluginActivationWarning(stderr, warning)
|
||||
}
|
||||
|
||||
// Re-register the skill tool to also resolve plugin-declared skills. The core
|
||||
// skill tool only reads the default skills dir; the plugin-aware replacement
|
||||
// merges the default dir with the plugin skill roots (default dir wins a name
|
||||
// clash), so plugin skills appear in the agent's skill list. With no plugin
|
||||
// skill roots this is byte-equivalent to the default skills surface.
|
||||
if len(result.SkillRoots) > 0 {
|
||||
registry.Register(plugins.NewSkillTool(deps.skillsDir(), result.SkillRoots))
|
||||
}
|
||||
// Always overlay the multi-root skill tool so discovery is identical with or
|
||||
// without plugins: primary DefaultDir + optional ~/.agents/skills + plugin
|
||||
// roots (earlier wins). With empty SkillRoots and no agents dir this is
|
||||
// byte-equivalent to the core single-dir skill surface.
|
||||
registry.Register(plugins.NewSkillTool(deps.skillsDir(), result.SkillRoots))
|
||||
|
||||
return pluginActivation{hooks: result.Hooks, skillRoots: result.SkillRoots, trustSkip: skip}
|
||||
}
|
||||
@@ -91,8 +91,8 @@ func projectPluginsDirExists(workspaceRoot string) bool {
|
||||
}
|
||||
|
||||
// skillInfos resolves the reusable skills the model can load via the skill tool —
|
||||
// the default skills dir merged with any plugin skill roots, the same set the
|
||||
// plugin-aware skill tool resolves against — as plain data for the agent's system
|
||||
// primary dir + optional ~/.agents/skills + plugin skill roots, the same set the
|
||||
// multi-root skill tool resolves against — as plain data for the agent's system
|
||||
// prompt. It returns nil when no skills are installed, so a skill-less run leaves
|
||||
// the prompt byte-identical.
|
||||
func (a pluginActivation) skillInfos(defaultDir string) []agent.SkillInfo {
|
||||
|
||||
@@ -69,6 +69,11 @@ func TestActivatePluginsRegistersToolAndCollectsHooks(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestActivatePluginsRegistersPluginSkillTool(t *testing.T) {
|
||||
// Isolate host ~/.agents/skills so it cannot interfere with this test.
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
|
||||
pluginDir := t.TempDir()
|
||||
skillDir := filepath.Join(pluginDir, "skills", "demo-skill")
|
||||
if err := os.MkdirAll(skillDir, 0o755); err != nil {
|
||||
@@ -108,6 +113,42 @@ func TestActivatePluginsRegistersPluginSkillTool(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivatePluginsAlwaysRegistersMultiRootSkillTool(t *testing.T) {
|
||||
// Even with zero plugins / zero plugin skill roots, activation must overlay
|
||||
// the multi-root skill tool so ~/.agents/skills is loadable on bare runs.
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
skillDir := filepath.Join(agents, "agents-skill")
|
||||
if err := os.MkdirAll(skillDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("---\nname: agents-skill\n---\nagents body"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(tools.NewSkillTool(t.TempDir())) // core single-dir tool
|
||||
var stderr bytes.Buffer
|
||||
workspace := t.TempDir()
|
||||
activation := activatePlugins(workspace, registry, fakePluginDeps(t, nil), &stderr, workspace)
|
||||
if len(activation.skillRoots) != 0 {
|
||||
t.Fatalf("expected no plugin skill roots, got %#v", activation.skillRoots)
|
||||
}
|
||||
skillTool, ok := registry.Get("skill")
|
||||
if !ok {
|
||||
t.Fatal("skill tool missing after activation")
|
||||
}
|
||||
res := skillTool.Run(context.Background(), map[string]any{"name": "agents-skill"})
|
||||
if res.Status != tools.StatusOK || !bytes.Contains([]byte(res.Output), []byte("agents body")) {
|
||||
t.Fatalf("agents skill must load with zero plugin roots: %q (%s)", res.Status, res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivatePluginsSurfacesLoadDiagnostics(t *testing.T) {
|
||||
// Load succeeds (no top-level error) but reports a per-plugin diagnostic for a
|
||||
// plugin it had to skip. activatePlugins must forward that diagnostic to stderr
|
||||
|
||||
@@ -17,6 +17,7 @@ func TestProviderCatalogRuntimeProvidersShowOnboardingSetupHints(t *testing.T) {
|
||||
defaultModel string
|
||||
}{
|
||||
{id: "openai", name: "OpenAI", transport: "openai", defaultModel: "gpt-4.1"},
|
||||
{id: "aimlapi", name: "AI/ML API", transport: "openai-compatible", defaultModel: "openai/gpt-5-chat"},
|
||||
{id: "groq", name: "Groq", transport: "openai-compatible", defaultModel: "llama-3.3-70b-versatile"},
|
||||
{id: "longcat", name: "LongCat", transport: "openai-compatible", defaultModel: "LongCat-2.0"},
|
||||
{id: "ollama-cloud", name: "Ollama Cloud", transport: "openai-compatible", defaultModel: "qwen3-coder:480b"},
|
||||
@@ -47,6 +48,21 @@ func TestProviderCatalogRuntimeProvidersShowOnboardingSetupHints(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderCatalogAIMLAPIRequiresAPIKey(t *testing.T) {
|
||||
output := runProviderCatalogOnboarding(t)
|
||||
block := providerCatalogOnboardingBlock(t, output, "aimlapi")
|
||||
|
||||
for _, want := range []string{
|
||||
"requiresAuth=true",
|
||||
"authEnvVars=AIMLAPI_API_KEY",
|
||||
"setup: zero providers setup aimlapi --set-active",
|
||||
} {
|
||||
if !strings.Contains(block, want) {
|
||||
t.Fatalf("expected AIMLAPI catalog block to contain %q, got:\n%s", want, block)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderCatalogLongCatRequiresAPIKey(t *testing.T) {
|
||||
output := runProviderCatalogOnboarding(t)
|
||||
block := providerCatalogOnboardingBlock(t, output, "longcat")
|
||||
|
||||
+90
-5
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/mcp"
|
||||
@@ -14,9 +16,21 @@ import (
|
||||
type serveOptions struct {
|
||||
mcp bool
|
||||
cwd string
|
||||
addDirs []string
|
||||
allowUnsafeTools bool
|
||||
}
|
||||
|
||||
// serveRootsScope is a PathScope used only for MCP serve: the workspace root
|
||||
// plus optional --add-dir roots, without the sandbox temp write roots that
|
||||
// would otherwise pollute resources/list.
|
||||
type serveRootsScope []string
|
||||
|
||||
func (s serveRootsScope) Roots() []string {
|
||||
roots := make([]string, len(s))
|
||||
copy(roots, s)
|
||||
return roots
|
||||
}
|
||||
|
||||
func runServe(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int {
|
||||
options, help, err := parseServeArgs(args)
|
||||
if err != nil {
|
||||
@@ -36,7 +50,11 @@ func runServe(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) i
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
registry := newServeRegistry(workspaceRoot, options.allowUnsafeTools)
|
||||
scope, err := buildServeScope(workspaceRoot, options.addDirs)
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
registry := newServeRegistry(workspaceRoot, scope, options.allowUnsafeTools)
|
||||
if options.allowUnsafeTools {
|
||||
if _, err := fmt.Fprintln(stderr, "[zero] Unsafe MCP server tools enabled because --allow-unsafe-tools was passed."); err != nil {
|
||||
return exitCrash
|
||||
@@ -47,6 +65,8 @@ func runServe(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) i
|
||||
Name: "zero",
|
||||
Version: version,
|
||||
PermissionGranted: options.allowUnsafeTools,
|
||||
WorkspaceRoot: workspaceRoot,
|
||||
Scope: scope,
|
||||
})
|
||||
if err != nil {
|
||||
return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash)
|
||||
@@ -54,11 +74,11 @@ func runServe(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) i
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
func newServeRegistry(workspaceRoot string, allowUnsafeTools bool) *tools.Registry {
|
||||
func newServeRegistry(workspaceRoot string, scope tools.PathScope, allowUnsafeTools bool) *tools.Registry {
|
||||
registry := tools.NewRegistry()
|
||||
toolset := tools.CoreReadOnlyTools(workspaceRoot)
|
||||
toolset := tools.CoreReadOnlyToolsScoped(workspaceRoot, scope)
|
||||
if allowUnsafeTools {
|
||||
toolset = tools.CoreTools(workspaceRoot)
|
||||
toolset = tools.CoreToolsScoped(workspaceRoot, scope)
|
||||
}
|
||||
for _, tool := range toolset {
|
||||
registry.Register(tool)
|
||||
@@ -66,6 +86,58 @@ func newServeRegistry(workspaceRoot string, allowUnsafeTools bool) *tools.Regist
|
||||
return registry
|
||||
}
|
||||
|
||||
// buildServeScope returns nil when there are no extra roots (workspace-only),
|
||||
// otherwise a PathScope listing the workspace first followed by each --add-dir.
|
||||
func buildServeScope(workspaceRoot string, addDirs []string) (tools.PathScope, error) {
|
||||
if len(addDirs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
roots := make([]string, 0, 1+len(addDirs))
|
||||
workspaceRoot = filepath.Clean(workspaceRoot)
|
||||
if abs, err := filepath.Abs(workspaceRoot); err == nil {
|
||||
workspaceRoot = abs
|
||||
}
|
||||
|
||||
// Keep lexical absolute paths in roots so path checks stay aligned with the
|
||||
// un-evaluated WorkspaceRoot used when Scope is nil. Symlink-resolved forms
|
||||
// are only keys for duplicate detection.
|
||||
workspaceResolved := workspaceRoot
|
||||
if resolved, err := filepath.EvalSymlinks(workspaceRoot); err == nil {
|
||||
workspaceResolved = resolved
|
||||
}
|
||||
roots = append(roots, workspaceRoot)
|
||||
|
||||
seen := map[string]struct{}{workspaceResolved: {}}
|
||||
for _, dir := range addDirs {
|
||||
trimmed := strings.TrimSpace(dir)
|
||||
if trimmed == "" {
|
||||
return nil, execUsageError{"--add-dir requires a path"}
|
||||
}
|
||||
absolute, err := filepath.Abs(trimmed)
|
||||
if err != nil {
|
||||
return nil, execUsageError{fmt.Sprintf("--add-dir %q: %v", trimmed, err)}
|
||||
}
|
||||
info, err := os.Stat(absolute)
|
||||
if err != nil {
|
||||
return nil, execUsageError{fmt.Sprintf("--add-dir %q: %v", trimmed, err)}
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, execUsageError{fmt.Sprintf("--add-dir %q is not a directory", trimmed)}
|
||||
}
|
||||
|
||||
resolved := absolute
|
||||
if r, err := filepath.EvalSymlinks(absolute); err == nil {
|
||||
resolved = r
|
||||
}
|
||||
if _, ok := seen[resolved]; ok {
|
||||
continue
|
||||
}
|
||||
seen[resolved] = struct{}{}
|
||||
roots = append(roots, absolute)
|
||||
}
|
||||
return serveRootsScope(roots), nil
|
||||
}
|
||||
|
||||
func parseServeArgs(args []string) (serveOptions, bool, error) {
|
||||
options := serveOptions{}
|
||||
for index := 0; index < len(args); index++ {
|
||||
@@ -85,6 +157,18 @@ func parseServeArgs(args []string) (serveOptions, bool, error) {
|
||||
options.cwd = args[index]
|
||||
case strings.HasPrefix(arg, "--cwd="):
|
||||
options.cwd = strings.TrimPrefix(arg, "--cwd=")
|
||||
case arg == "--add-dir":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return options, false, execUsageError{"--add-dir requires a path"}
|
||||
}
|
||||
options.addDirs = append(options.addDirs, args[index])
|
||||
case strings.HasPrefix(arg, "--add-dir="):
|
||||
value := strings.TrimPrefix(arg, "--add-dir=")
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return options, false, execUsageError{"--add-dir requires a path"}
|
||||
}
|
||||
options.addDirs = append(options.addDirs, value)
|
||||
default:
|
||||
return options, false, execUsageError{fmt.Sprintf("unknown serve flag %q", arg)}
|
||||
}
|
||||
@@ -100,7 +184,8 @@ Starts Zero as an MCP stdio server.
|
||||
|
||||
Flags:
|
||||
--mcp Run the MCP stdio server
|
||||
-C, --cwd <path> Set the workspace directory
|
||||
-C, --cwd <path> Set the workspace directory (resources + tools)
|
||||
--add-dir <path> Extra resource/tool root (repeatable)
|
||||
--allow-unsafe-tools Expose write and shell tools to the MCP host
|
||||
-h, --help Show this help
|
||||
`)
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -80,6 +82,120 @@ func TestRunServeRequiresMCPMode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunServeMCPWiresWorkspaceRootAndAddDirIntoResources(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
extra := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(workspace, "workspace.txt"), []byte("ws\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(extra, "extra.txt"), []byte("ex\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
exitCode := runWithDeps(
|
||||
[]string{"serve", "--mcp", "-C", workspace, "--add-dir", extra},
|
||||
&stdout,
|
||||
&stderr,
|
||||
appDeps{stdin: bytes.NewReader(serveMCPResourcesInput(t))},
|
||||
)
|
||||
if exitCode != exitSuccess {
|
||||
t.Fatalf("exit=%d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "workspace.txt") {
|
||||
t.Fatalf("expected workspace resource, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "extra.txt") {
|
||||
t.Fatalf("expected --add-dir resource, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunServeMCPRejectsMissingAddDir(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
missing := filepath.Join(workspace, "does-not-exist")
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
exitCode := runWithDeps(
|
||||
[]string{"serve", "--mcp", "-C", workspace, "--add-dir", missing},
|
||||
&stdout,
|
||||
&stderr,
|
||||
appDeps{stdin: bytes.NewReader(nil)},
|
||||
)
|
||||
if exitCode != exitUsage {
|
||||
t.Fatalf("exit=%d want %d stderr=%q", exitCode, exitUsage, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "--add-dir") {
|
||||
t.Fatalf("expected --add-dir error, got %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseServeArgsCollectsAddDirs(t *testing.T) {
|
||||
options, help, err := parseServeArgs([]string{"--mcp", "--add-dir", "/one", "--add-dir=/two", "-C", "/ws"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if help {
|
||||
t.Fatal("unexpected help")
|
||||
}
|
||||
if !options.mcp || options.cwd != "/ws" {
|
||||
t.Fatalf("options=%#v", options)
|
||||
}
|
||||
if len(options.addDirs) != 2 || options.addDirs[0] != "/one" || options.addDirs[1] != "/two" {
|
||||
t.Fatalf("addDirs=%v", options.addDirs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildServeScopeNilWithoutExtras(t *testing.T) {
|
||||
scope, err := buildServeScope(t.TempDir(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scope != nil {
|
||||
t.Fatalf("expected nil scope, got %#v", scope)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildServeScopeKeepsLexicalPaths(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
realWorkspace := filepath.Join(base, "real-workspace")
|
||||
realExtra := filepath.Join(base, "real-extra")
|
||||
for _, dir := range []string{realWorkspace, realExtra} {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
linkWorkspace := filepath.Join(base, "link-workspace")
|
||||
linkExtra := filepath.Join(base, "link-extra")
|
||||
if err := os.Symlink(realWorkspace, linkWorkspace); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(realExtra, linkExtra); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
scope, err := buildServeScope(linkWorkspace, []string{linkExtra})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scope == nil {
|
||||
t.Fatal("expected non-nil scope")
|
||||
}
|
||||
roots := scope.Roots()
|
||||
wantWorkspace, err := filepath.Abs(linkWorkspace)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantExtra, err := filepath.Abs(linkExtra)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(roots) != 2 || roots[0] != wantWorkspace || roots[1] != wantExtra {
|
||||
t.Fatalf("roots=%v want [%q %q]", roots, wantWorkspace, wantExtra)
|
||||
}
|
||||
}
|
||||
|
||||
func serveMCPInput(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
@@ -104,6 +220,30 @@ func serveMCPInput(t *testing.T) []byte {
|
||||
return input.Bytes()
|
||||
}
|
||||
|
||||
func serveMCPResourcesInput(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
var input bytes.Buffer
|
||||
writeServeMCPMessage(t, &input, map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": map[string]any{},
|
||||
})
|
||||
writeServeMCPMessage(t, &input, map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/initialized",
|
||||
"params": map[string]any{},
|
||||
})
|
||||
writeServeMCPMessage(t, &input, map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"method": "resources/list",
|
||||
"params": map[string]any{},
|
||||
})
|
||||
return input.Bytes()
|
||||
}
|
||||
|
||||
func writeServeMCPMessage(t *testing.T, buffer *bytes.Buffer, message map[string]any) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(message)
|
||||
|
||||
+21
-22
@@ -59,25 +59,21 @@ func runSkills(args []string, stdout io.Writer, stderr io.Writer, deps appDeps)
|
||||
}
|
||||
|
||||
func runSkillsList(dir string, options skillListOptions, stdout io.Writer, stderr io.Writer) int {
|
||||
discovered, err := skills.List(dir)
|
||||
// Management CLI lists global roots only (primary + ~/.agents/skills); plugin
|
||||
// skills stay out of `zero skills list` and surface via the agent skill tool.
|
||||
roots := skills.GlobalRoots(dir)
|
||||
discovered, dups, err := skills.ListFromRoots(roots)
|
||||
if err != nil {
|
||||
return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash)
|
||||
}
|
||||
// Surface name collisions that List silently resolved (first directory wins),
|
||||
// so a shadowed same-named skill is reported instead of just disappearing.
|
||||
// Warnings go to stderr, keeping stdout (including --json) clean.
|
||||
if dups, derr := skills.Duplicates(dir); derr == nil {
|
||||
for _, dup := range dups {
|
||||
fmt.Fprintf(stderr, "warning: duplicate skill %q: using %s, ignoring %s\n",
|
||||
redaction.RedactString(dup.Name, redaction.Options{}),
|
||||
redaction.RedactString(dup.Winner, redaction.Options{}),
|
||||
redaction.RedactString(dup.Loser, redaction.Options{}))
|
||||
}
|
||||
} else {
|
||||
// Don't silently swallow a scan failure: "no warnings" would then be
|
||||
// ambiguous (no duplicates vs. detection broke). Surface it on stderr.
|
||||
fmt.Fprintf(stderr, "warning: could not check for duplicate skills: %s\n",
|
||||
redaction.ErrorMessage(derr, redaction.Options{}))
|
||||
// Surface name collisions that ListFromRoots silently resolved (earlier root
|
||||
// wins), so a shadowed same-named skill is reported instead of just
|
||||
// disappearing. Warnings go to stderr, keeping stdout (including --json) clean.
|
||||
for _, dup := range dups {
|
||||
fmt.Fprintf(stderr, "warning: duplicate skill %q: using %s, ignoring %s\n",
|
||||
redaction.RedactString(dup.Name, redaction.Options{}),
|
||||
redaction.RedactString(dup.Winner, redaction.Options{}),
|
||||
redaction.RedactString(dup.Loser, redaction.Options{}))
|
||||
}
|
||||
if options.json {
|
||||
payload := struct {
|
||||
@@ -88,16 +84,16 @@ func runSkillsList(dir string, options skillListOptions, stdout io.Writer, stder
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
output := redaction.RedactString(formatSkillList(discovered, dir), redaction.Options{})
|
||||
output := redaction.RedactString(formatSkillList(discovered), redaction.Options{})
|
||||
if _, err := fmt.Fprintln(stdout, output); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
func formatSkillList(discovered []skills.Skill, dir string) string {
|
||||
func formatSkillList(discovered []skills.Skill) string {
|
||||
if len(discovered) == 0 {
|
||||
return fmt.Sprintf("No Zero skills found in %s.", dir)
|
||||
return "No skills found."
|
||||
}
|
||||
lines := []string{"Zero Skills:"}
|
||||
for _, skill := range discovered {
|
||||
@@ -131,10 +127,13 @@ func writeSkillsHelp(w io.Writer) error {
|
||||
zero skills <command>
|
||||
|
||||
Commands:
|
||||
list List discovered Zero skills
|
||||
add <git-url|path> Install a skill (checksum-pinned in skills.lock)
|
||||
list List discovered skills (Zero dir + ~/.agents/skills)
|
||||
add <git-url|path> Install a skill into the Zero skills dir (checksum-pinned in skills.lock)
|
||||
info <name> Show a skill's frontmatter, source, and pinned hash
|
||||
remove <name> Remove an installed skill and its lockfile entry
|
||||
remove <name> Remove an installed skill and its lockfile entry from the Zero skills dir
|
||||
|
||||
list and info also search ~/.agents/skills when present (read-only).
|
||||
add and remove always target the Zero-specific skills directory.
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -23,7 +23,15 @@ func writeSkillFixture(t *testing.T, dir string, name string, content string) {
|
||||
}
|
||||
}
|
||||
|
||||
func isolateCLIAgentsHome(t *testing.T) {
|
||||
t.Helper()
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
}
|
||||
|
||||
func TestRunSkillsListText(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
dir := t.TempDir()
|
||||
writeSkillFixture(t, dir, "confirmation-policy", "---\nname: confirmation-policy\ndescription: Ask before risky actions.\n---\nbody")
|
||||
|
||||
@@ -43,6 +51,7 @@ func TestRunSkillsListText(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunSkillsListWarnsOnDuplicateNames(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
dir := t.TempDir()
|
||||
// Two directories declare the same frontmatter name; List keeps one and the
|
||||
// other is shadowed. The command must warn instead of silently dropping it.
|
||||
@@ -62,6 +71,7 @@ func TestRunSkillsListWarnsOnDuplicateNames(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunSkillsDefaultsToList(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
dir := t.TempDir()
|
||||
writeSkillFixture(t, dir, "demo", "body")
|
||||
|
||||
@@ -78,6 +88,7 @@ func TestRunSkillsDefaultsToList(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunSkillsListJSON(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
dir := t.TempDir()
|
||||
writeSkillFixture(t, dir, "demo", "---\nname: demo\ndescription: a demo\n---\nbody")
|
||||
|
||||
@@ -111,6 +122,7 @@ func TestRunSkillsListJSON(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunSkillsEmptyDir(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
exit := runWithDeps([]string{"skills", "list"}, &stdout, &stderr, appDeps{
|
||||
skillsDir: func() string { return filepath.Join(t.TempDir(), "missing") },
|
||||
@@ -118,11 +130,89 @@ func TestRunSkillsEmptyDir(t *testing.T) {
|
||||
if exit != 0 {
|
||||
t.Fatalf("exit = %d, stderr = %s", exit, stderr.String())
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(stdout.String()), "no") {
|
||||
if !strings.Contains(strings.ToLower(stdout.String()), "no skills found") {
|
||||
t.Fatalf("expected a no-skills message, got:\n%s", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkillsListIncludesAgentsOnlySkill(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
writeSkillFixture(t, agents, "agents-only", "---\nname: agents-only\ndescription: Shared multi-agent skill.\n---\nbody")
|
||||
|
||||
primary := t.TempDir()
|
||||
var stdout, stderr bytes.Buffer
|
||||
exit := runWithDeps([]string{"skills", "list"}, &stdout, &stderr, appDeps{
|
||||
skillsDir: func() string { return primary },
|
||||
})
|
||||
if exit != 0 {
|
||||
t.Fatalf("exit = %d, stderr = %s", exit, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
if !strings.Contains(out, "agents-only") {
|
||||
t.Fatalf("list should include agents-only skill:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, agents) {
|
||||
t.Fatalf("list should show agents path:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkillsListPrimaryShadowsAgents(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
primary := t.TempDir()
|
||||
writeSkillFixture(t, primary, "shared", "---\nname: shared\ndescription: From primary.\n---\nbody")
|
||||
writeSkillFixture(t, agents, "shared", "---\nname: shared\ndescription: From agents.\n---\nbody")
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
exit := runWithDeps([]string{"skills", "list"}, &stdout, &stderr, appDeps{
|
||||
skillsDir: func() string { return primary },
|
||||
})
|
||||
if exit != 0 {
|
||||
t.Fatalf("exit = %d, stderr = %s", exit, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "From primary") {
|
||||
t.Fatalf("primary should win list description:\n%s", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), primary) {
|
||||
t.Fatalf("primary path should be shown:\n%s", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `duplicate skill "shared"`) {
|
||||
t.Fatalf("expected cross-root duplicate warning, got: %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkillInfoResolvesAgentsOnly(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
writeSkillFixture(t, agents, "agents-info", "---\nname: agents-info\ndescription: Agents info.\n---\nbody")
|
||||
|
||||
primary := t.TempDir()
|
||||
var stdout, stderr bytes.Buffer
|
||||
exit := runWithDeps([]string{"skill", "info", "agents-info"}, &stdout, &stderr, appDeps{
|
||||
skillsDir: func() string { return primary },
|
||||
})
|
||||
if exit != 0 {
|
||||
t.Fatalf("exit = %d, stderr = %s", exit, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
if !strings.Contains(out, "agents-info") || !strings.Contains(out, "Agents info.") {
|
||||
t.Fatalf("info missing agents skill:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "source:") || strings.Contains(out, "hash:") {
|
||||
t.Fatalf("agents-only info must not invent lock metadata:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, agents) {
|
||||
t.Fatalf("info should show agents path:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkillsDoesNotLaunchTUI(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
launchCalled := false
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# Turn benchmark manifest
|
||||
|
||||
The baseline manifest (`manifests/baseline.json`) is the per-turn benchmark's
|
||||
program keystone: it defines the tasks the harness runs, the workspace each
|
||||
starts in, and — critically — what "pass" means for each task. Because `make
|
||||
baseline` is re-run on every perf change, the manifest's pass/fail contract
|
||||
matters for months, so the contract is written down here.
|
||||
|
||||
## Task count
|
||||
|
||||
48 tasks across seven classes:
|
||||
|
||||
| class | count | oracle | tier |
|
||||
|-----------|-------|-----------------------------|-------------|
|
||||
| nav | 10 | none | latency |
|
||||
| edit | 10 | substring grep | correctness |
|
||||
| fix | 8 | scoped `go test -run <name>` | correctness |
|
||||
| refactor | 6 | `go build ./...` | build |
|
||||
| longproc | 4 | none | latency |
|
||||
| longctx | 4 | none | latency |
|
||||
| parallel | 6 | none | latency |
|
||||
| **total** | **48**| | |
|
||||
|
||||
## Oracle tiers
|
||||
|
||||
Pass/fail is reported per tier so the report cannot be misread as a blanket
|
||||
correctness verdict. The tier is decided per task from oracle presence and the
|
||||
manifest's `buildOnlyClasses` list:
|
||||
|
||||
- **Correctness** (18 tasks: 10 edit + 8 fix) — a positive oracle (substring
|
||||
grep that the requested change landed, or a scoped `go test -run` that the
|
||||
bug fix passes). This is the only pass rate that can move with model quality:
|
||||
`tasksVerified` / `tasksPassed` / `correctnessPassRate`.
|
||||
- **Build-only** (6 tasks: refactor) — `go build ./...` proves the edit
|
||||
compiles, not that the refactor achieved its goal (a no-op refactor passes).
|
||||
The manifest declares `buildOnlyClasses: ["refactor"]`. Reported as
|
||||
`buildCheckedTasks` / `buildPassedTasks` / `buildPassRate`, never in
|
||||
`correctnessPassRate`.
|
||||
- **Latency-only** (24 tasks: nav, longproc, longctx, parallel) — no
|
||||
`verificationCommand`. An exit 0 only proves the turn ran, not that the answer
|
||||
was right. They contribute to latency and span attribution only and are
|
||||
counted in `latencyOnlyTasks`, never in any pass rate.
|
||||
|
||||
A task's tier is driven by oracle **presence** first: a task with no
|
||||
`verificationCommand` is latency-only even if its class is listed in
|
||||
`buildOnlyClasses`, so a missing oracle can never silently pass on exit 0.
|
||||
|
||||
## Known limitations (deferred)
|
||||
|
||||
These are accepted for the Phase 0 baseline and tracked in a follow-up issue;
|
||||
they do not block the baseline because the tier split keeps the report honest:
|
||||
|
||||
- The edit grep oracles are substring checks. A rename oracle asserts the new
|
||||
name is present AND the old name is gone (compound `bash -c`), but an
|
||||
add-field oracle (`grep -R Label .`) only proves the string appears, not that
|
||||
it landed on the right struct. Strengthening these to `go vet`/`go build` +
|
||||
structural assertions is follow-up work.
|
||||
- The refactor build oracle is non-positive (a no-op refactor passes). A
|
||||
structural verifier that proves the refactor happened is follow-up work;
|
||||
until then refactor is `buildPassRate`, not `correctnessPassRate`.
|
||||
- The 24 read-only tasks have no oracle. Adding deterministic oracles for nav
|
||||
(diff against an expected answer) and documenting longproc/longctx/parallel
|
||||
as permanently latency-only is follow-up work.
|
||||
|
||||
## Fixtures
|
||||
|
||||
Each task's `workspaceFixture` points at a small self-contained workspace under
|
||||
`testdata/` so the suite runs offline and repeatably. Mutating tasks (edit,
|
||||
fix, refactor) run against a per-invocation **copy** of their fixture, so the
|
||||
checked-in fixtures stay clean and one task's edits can't bleed into the next
|
||||
iteration or a later task.
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"id": "zero-baseline-turn",
|
||||
"name": "Zero per-turn baseline (Phase 0)",
|
||||
"description": "Latency-first baseline: measures where a turn spends wall time, with pass/fail reported per oracle tier so it cannot be misread as a blanket correctness verdict. Read-only classes (nav, longproc, longctx, parallel) carry no verification oracle — a zero-exit only proves the turn ran, so they count as latency-only and never in any pass rate. Correctness classes (edit, fix) carry a positive oracle (substring grep or a scoped `go test`) and are the only tasks in correctnessPassRate. Build-only classes (refactor) carry `go build ./...`, which proves the edit compiles but not that the refactor achieved its goal, so they are reported as buildPassRate and excluded from correctnessPassRate.",
|
||||
"buildOnlyClasses": ["refactor"],
|
||||
"tasks": [
|
||||
{"id": "nav-01", "name": "list repo files", "class": "nav", "prompt": "List the files in this repository and report their count. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-02", "name": "find a function", "class": "nav", "prompt": "Find the function named 'greet' and report the file and line it is defined on. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-03", "name": "summarize module", "class": "nav", "prompt": "Read main.go and summarize what this program does in two sentences. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-04", "name": "count tests", "class": "nav", "prompt": "Count the test functions (functions whose names start with Test) in this repository and report the number. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-05", "name": "locate a string", "class": "nav", "prompt": "Find every occurrence of the string 'TODO' in the repository and report the files. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-06", "name": "describe package", "class": "nav", "prompt": "Describe the exported API of this package by reading the source. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-07", "name": "trace a call", "class": "nav", "prompt": "Trace which function calls which other function starting from main. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-08", "name": "find imports", "class": "nav", "prompt": "List every third-party import used by this repository. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-09", "name": "report config", "class": "nav", "prompt": "Read config.json and report the keys it contains. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-10", "name": "doc lookup", "class": "nav", "prompt": "Find the doc comment on the greet function and quote it. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
|
||||
{"id": "edit-01", "name": "rename a constant", "class": "edit", "prompt": "Rename the constant MaxRetries to RetryLimit everywhere it appears.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "grep -R RetryLimit . && ! grep -R MaxRetries ."]},
|
||||
{"id": "edit-02", "name": "fix a typo", "class": "edit", "prompt": "Fix the typo in the comment above the greet function ('receieves' should be 'receives').", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "! grep -R receieves ."]},
|
||||
{"id": "edit-03", "name": "add a field", "class": "edit", "prompt": "Add a string field named Label to the Config struct in main.go.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["grep", "-R", "Label", "."]},
|
||||
{"id": "edit-04", "name": "bump a version", "class": "edit", "prompt": "Change the version string in version.go from 1.0.0 to 1.1.0.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "grep -R 1.1.0 . && ! grep -R 1.0.0 ."]},
|
||||
{"id": "edit-05", "name": "remove a log line", "class": "edit", "prompt": "Remove the line that prints 'debug: starting'.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "! grep -R 'debug: starting' ."]},
|
||||
{"id": "edit-06", "name": "add a license header", "class": "edit", "prompt": "Add a comment line '// SPDX-License-Identifier: MIT' at the top of main.go.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["grep", "-R", "SPDX-License-Identifier", "."]},
|
||||
{"id": "edit-07", "name": "change a default", "class": "edit", "prompt": "Change the default value of the Port constant from 8080 to 9090.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "grep -R 9090 . && ! grep -R 8080 ."]},
|
||||
{"id": "edit-08", "name": "add a getter", "class": "edit", "prompt": "Add a method func (c *Config) GetLabel() string that returns c.Label.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "grep -R 'func (c *Config) GetLabel' ."]},
|
||||
{"id": "edit-09", "name": "wrap an error", "class": "edit", "prompt": "In the load function, wrap the returned error with fmt.Errorf so it includes 'load failed'.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["grep", "-R", "load failed", "."]},
|
||||
{"id": "edit-10", "name": "update a message", "class": "edit", "prompt": "Change the printed greeting from 'hello' to 'hello, world'.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["grep", "-R", "hello, world", "."]},
|
||||
|
||||
{"id": "fix-01", "name": "off-by-one", "class": "fix", "prompt": "In bugs.go the Sum function has an off-by-one error that skips the last element; fix the loop so it sums all elements.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestSumIncludesLastElement", "./..."]},
|
||||
{"id": "fix-02", "name": "nil deref", "class": "fix", "prompt": "In bugs.go DefaultConfig returns nil and Port() dereferences it; make DefaultConfig return a valid Config with Port 8080.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestPortReturnsDefault", "./..."]},
|
||||
{"id": "fix-03", "name": "wrong operator", "class": "fix", "prompt": "In bugs.go Max uses the wrong inequality and returns the smaller value; fix it to return the larger.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestMaxReturnsLarger", "./..."]},
|
||||
{"id": "fix-04", "name": "truncated read", "class": "fix", "prompt": "In bugs.go ReadFirst returns only a single character instead of the full content; fix it to return the whole string.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestReadFirstReturnsFullContent", "./..."]},
|
||||
{"id": "fix-05", "name": "swapped args", "class": "fix", "prompt": "In bugs.go Divide swaps its operands; fix the order so Divide(10,2) returns 5.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestDivideOrder", "./..."]},
|
||||
{"id": "fix-06", "name": "missing return", "class": "fix", "prompt": "In bugs.go Classify is missing a return for the high case; add it so Classify(500) returns \"high\".", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestClassifyHigh", "./..."]},
|
||||
{"id": "fix-07", "name": "wrong label prefix", "class": "fix", "prompt": "In bugs.go Label returns 'admin-<name>' but the test expects 'user-<name>'; change the prefix from 'admin-' to 'user-'.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestLabelUsesString", "./..."]},
|
||||
{"id": "fix-08", "name": "race on counter", "class": "fix", "prompt": "In bugs.go the global counter is incremented without synchronization; guard it with a mutex.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-race", "-run", "TestCounterRace", "./..."]},
|
||||
|
||||
{"id": "refactor-01", "name": "extract helper", "class": "refactor", "prompt": "Extract the duplicated greeting logic into a single helper function and call it from both sites.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-02", "name": "split a file", "class": "refactor", "prompt": "Split main.go into two files: one for Config, one for the rest, keeping the package building.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-03", "name": "rename package", "class": "refactor", "prompt": "Rename the package from app to zeroapp everywhere and keep it building.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-04", "name": "introduce a type", "class": "refactor", "prompt": "Introduce a named type for the map[string]int used in main.go and update its users.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-05", "name": "inline a wrapper", "class": "refactor", "prompt": "Inline the thin wrapper function into its single caller and remove it.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-06", "name": "consolidate errors", "class": "refactor", "prompt": "Consolidate the repeated error-construction blocks into a single helper.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
|
||||
{"id": "longproc-01", "name": "long build", "class": "longproc", "prompt": "Run the build command and report whether it succeeds. Do not modify anything.", "workspaceFixture": "../testdata/longproc"},
|
||||
{"id": "longproc-02", "name": "long test", "class": "longproc", "prompt": "Run go test ./... and report the result. Do not modify anything.", "workspaceFixture": "../testdata/longproc"},
|
||||
{"id": "longproc-03", "name": "bench run", "class": "longproc", "prompt": "Run go test -bench=. -benchtime=1x and report the benchmark numbers. Do not modify anything.", "workspaceFixture": "../testdata/longproc"},
|
||||
{"id": "longproc-04", "name": "vet run", "class": "longproc", "prompt": "Run go vet ./... and report the output. Do not modify anything.", "workspaceFixture": "../testdata/longproc"},
|
||||
|
||||
{"id": "longctx-01", "name": "summarize large file", "class": "longctx", "prompt": "Read the large generated file big.go and summarize its structure. Do not modify anything.", "workspaceFixture": "../testdata/longctx"},
|
||||
{"id": "longctx-02", "name": "find in large file", "class": "longctx", "prompt": "In big.go, find every function that returns an error and list them. Do not modify anything.", "workspaceFixture": "../testdata/longctx"},
|
||||
{"id": "longctx-03", "name": "cross-file question", "class": "longctx", "prompt": "Read all .go files and report which two functions share the same name across files. Do not modify anything.", "workspaceFixture": "../testdata/longctx"},
|
||||
{"id": "longctx-04", "name": "trace through large file", "class": "longctx", "prompt": "Starting from Handle, trace the call path through big.go and report the chain. Do not modify anything.", "workspaceFixture": "../testdata/longctx"},
|
||||
|
||||
{"id": "parallel-01", "name": "read six files", "class": "parallel", "prompt": "Read a.txt, b.txt, c.txt, d.txt, e.txt, and f.txt, then report the first line of each. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-02", "name": "grep six patterns", "class": "parallel", "prompt": "Search for 'alpha', 'beta', 'gamma', 'delta', 'epsilon', and 'zeta' across the repo and report which files contain each. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-03", "name": "list six dirs", "class": "parallel", "prompt": "List the contents of dir1, dir2, dir3, dir4, dir5, and dir6. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-04", "name": "stat six files", "class": "parallel", "prompt": "Report the line count of a.txt, b.txt, c.txt, d.txt, e.txt, and f.txt. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-05", "name": "read configs", "class": "parallel", "prompt": "Read config1.json, config2.json, config3.json, config4.json, config5.json, and config6.json and report each top-level key. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-06", "name": "summarize six", "class": "parallel", "prompt": "Read the six .txt files and give a one-line summary of each. Do not modify anything.", "workspaceFixture": "../testdata/parallel"}
|
||||
]
|
||||
}
|
||||
Whitespace-only changes.
@@ -0,0 +1,84 @@
|
||||
# Per-turn benchmark reports
|
||||
|
||||
This directory holds generated per-turn benchmark reports. It is **not** a
|
||||
checked-in source of truth for performance — the numbers are machine- and
|
||||
model-specific, so a single snapshot here would mislead rather than inform.
|
||||
|
||||
## Generating a baseline
|
||||
|
||||
Run the harness over the checked-in manifest:
|
||||
|
||||
```sh
|
||||
make baseline ZERO_BENCH_MODEL=<model> # uses ./zero
|
||||
make baseline ZERO_BENCH_MODEL=<model> ZERO_BENCH_BINARY=/path/to/zero
|
||||
```
|
||||
|
||||
This builds `zero`, then runs `zero-perf-bench turn` over
|
||||
`internal/perfbench/manifests/baseline.json`, capturing each turn's trace
|
||||
(`zero exec --trace <tmpfile>`) and writing the aggregated result to
|
||||
`reports/baseline.json`.
|
||||
|
||||
The JSON report is self-describing: model, mode, self-correct flag, version,
|
||||
commit, date, per-span median/P95, the **top three controllable latency sources**
|
||||
ranked by **exclusive** time, per-class roll-ups, and token/count totals.
|
||||
That top-three list is the Phase 0 baseline's "do not proceed until" criterion —
|
||||
it names where a turn actually spends time so later optimization work is
|
||||
targeted, not guessed.
|
||||
|
||||
### Attribution model (honest by construction)
|
||||
|
||||
Spans record wall intervals and are **not** summed into each other. Each span's
|
||||
**exclusive** time is its own duration minus the union of its nested children's
|
||||
intervals, derived at finish by interval containment. So a `provider_connect`
|
||||
that runs concurrently inside `generation`, or a `permission_wait` nested inside
|
||||
`tool_execution`, each contributes only its own exclusive time — they no longer
|
||||
double-count the same wall. The top-latency shares therefore sum to ~1 for a
|
||||
well-instrumented run, and the ranking reflects where wall time is actually
|
||||
spent.
|
||||
|
||||
**Coverage** is the fraction of wall covered by the union of all span
|
||||
intervals (capped at 1.0) — the honest "≥95% of wall accounted for" metric. A
|
||||
run with `coverage < 0.95` has uninstrumented gaps, not an inflated attribution.
|
||||
|
||||
### Pass/fail is reported per oracle tier
|
||||
|
||||
Pass/fail is split into three tiers so it cannot be misread as a blanket
|
||||
correctness verdict (see `MANIFEST.md` for the class breakdown):
|
||||
|
||||
- **Correctness** (`tasksVerified` / `tasksPassed` / `correctnessPassRate`):
|
||||
tasks with a positive oracle — edit's substring grep, fix's scoped `go test`.
|
||||
This is the only pass rate that can move with model quality.
|
||||
- **Build-only** (`buildCheckedTasks` / `buildPassedTasks` / `buildPassRate`):
|
||||
refactor's `go build ./...`, which proves the edit compiles but not that the
|
||||
refactor achieved its goal. Reported separately, never in `correctnessPassRate`.
|
||||
- **Latency-only** (`latencyOnlyTasks`): the read-only classes (nav, longproc,
|
||||
longctx, parallel) carry no oracle — an exit 0 only proves the turn ran. They
|
||||
contribute to latency and span attribution and are excluded from every pass
|
||||
rate.
|
||||
|
||||
`tasksAttempted` is still the total across all three tiers. The tier class lists
|
||||
are echoed in the report so a consumer can see exactly which classes each rate
|
||||
is computed over.
|
||||
|
||||
> **Do not average the tier pass rates. Do not report a single "pass rate".**
|
||||
> `correctnessPassRate` and `buildPassRate` measure different things over
|
||||
> different task sets and must never be combined — a weighted or arithmetic
|
||||
> mean of the two is a number that means nothing. The schema offers no
|
||||
> headline pass rate on purpose: a consumer must name the tier it is reporting
|
||||
> (`correctness`, `build`, or `latency-only`) and quote that tier's fields. A
|
||||
> read-only run that exits 0 is a latency sample, not a pass.
|
||||
|
||||
## What to commit
|
||||
|
||||
Commit the manifest and fixtures (`manifests/`, `../testdata/`), not a generated
|
||||
`baseline.json`. A generated report belongs in a PR description or a shared
|
||||
dashboard as evidence for one configuration, not in the tree as a durable
|
||||
expectation. `.gitkeep` keeps the directory present between runs.
|
||||
|
||||
## Caveats
|
||||
|
||||
- Each task is a fresh `zero exec` process, so iterations are **cold-start**
|
||||
samples. A warm path (reusing an in-process agent) is future work.
|
||||
- Mutating tasks (edit/fix/refactor) run against a per-invocation **copy** of
|
||||
their fixture, so the checked-in fixtures stay clean and one task's edits
|
||||
can't bleed into the next iteration.
|
||||
@@ -25,17 +25,29 @@ const TaskSchemaVersion = 1
|
||||
// ZERO must satisfy. The set is recorded by ID with every result so a published
|
||||
// number is traceable to the exact tasks that produced it.
|
||||
type TaskSet struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Tasks []BenchTask `json:"tasks"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Tasks []BenchTask `json:"tasks"`
|
||||
// BuildOnlyClasses lists task classes whose verificationCommand is a
|
||||
// non-positive build check (e.g. refactor's `go build ./...`): it proves the
|
||||
// edit compiles, not that the refactor achieved its goal. The turn benchmark
|
||||
// reports these separately from correctness oracles so a build-pass cannot be
|
||||
// misread as a correctness pass. Classes with a verificationCommand that are
|
||||
// NOT listed here are treated as correctness classes; classes whose tasks
|
||||
// carry no verificationCommand are latency-only regardless of this list.
|
||||
BuildOnlyClasses []string `json:"buildOnlyClasses,omitempty"`
|
||||
}
|
||||
|
||||
// BenchTask is one benchmark task. WorkspaceFixture is the relative path of the
|
||||
// task's starting workspace; VerificationCommand (optional) is the command the
|
||||
// default runner executes to decide pass/fail after ZERO finishes.
|
||||
// default runner executes to decide pass/fail after ZERO finishes. Class groups
|
||||
// the task for the turn-benchmark's per-group latency breakdown (e.g. "nav",
|
||||
// "edit", "fix"); it is optional and ignored by the pass/fail runner.
|
||||
type BenchTask struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Class string `json:"class,omitempty"`
|
||||
Prompt string `json:"prompt"`
|
||||
WorkspaceFixture string `json:"workspaceFixture,omitempty"`
|
||||
VerificationCommand []string `json:"verificationCommand,omitempty"`
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
// Package edit is a small workspace for the edit-class benchmark tasks: each
|
||||
// task asks the agent to make one targeted edit, verified by a grep/build
|
||||
// command in the manifest.
|
||||
package edit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Port is the default listen port.
|
||||
const Port = 8080
|
||||
|
||||
// MaxRetries is the maximum number of retries before giving up.
|
||||
const MaxRetries = 3
|
||||
|
||||
// Config holds runtime configuration.
|
||||
type Config struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
// greet returns a greeting for the named user.
|
||||
//
|
||||
// This receieves a name and formats a hello message.
|
||||
func greet(name string) string {
|
||||
return "hello"
|
||||
}
|
||||
|
||||
// load reads a config file and returns its bytes, or an error on failure.
|
||||
func load(path string) ([]byte, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
fmt.Println("debug: starting")
|
||||
cfg := Config{Name: "demo"}
|
||||
fmt.Println(greet(cfg.Name))
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package edit
|
||||
|
||||
// Version is the build version string.
|
||||
const Version = "1.0.0"
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
// Package fix holds deliberately buggy functions for the bug-fix benchmark
|
||||
// tasks. Each function is covered by its own test (see bugs_test.go) and each
|
||||
// task verifies with `go test -run <TestName>`, so the tasks are independent:
|
||||
// the initial package compiles and the targeted test fails until the named bug
|
||||
// is fixed. Every bug is fixable without changing a function signature, so the
|
||||
// test files compile in both the buggy and the fixed state.
|
||||
package fix
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Sum returns the sum of all elements. BUG: off-by-one — skips the last element.
|
||||
func Sum(xs []int) int {
|
||||
total := 0
|
||||
for i := 0; i < len(xs)-1; i++ {
|
||||
total += xs[i]
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// DefaultConfig returns the default configuration. BUG: returns nil, which
|
||||
// Port() then dereferences.
|
||||
func DefaultConfig() *Config {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Config holds demo settings.
|
||||
type Config struct {
|
||||
Port int
|
||||
}
|
||||
|
||||
// Port returns the configured port. BUG: panics when DefaultConfig() is nil.
|
||||
func Port() int {
|
||||
return DefaultConfig().Port
|
||||
}
|
||||
|
||||
// Max returns the larger of a and b. BUG: uses the wrong inequality and returns
|
||||
// the smaller value.
|
||||
func Max(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// ReadFirst returns the first byte of the given file's content as a string. BUG:
|
||||
// returns only a single character instead of the full content.
|
||||
func ReadFirst(content string) string {
|
||||
if len(content) == 0 {
|
||||
return ""
|
||||
}
|
||||
return string(content[0])
|
||||
}
|
||||
|
||||
// Divide returns a divided by b. BUG: the operands are swapped.
|
||||
func Divide(a, b int) int {
|
||||
return b / a
|
||||
}
|
||||
|
||||
// Classify returns "low", "mid", or "high". BUG: the "high" branch is missing a
|
||||
// return, so high values fall through to the default "".
|
||||
func Classify(n int) string {
|
||||
if n < 10 {
|
||||
return "low"
|
||||
}
|
||||
if n < 100 {
|
||||
return "mid"
|
||||
}
|
||||
// BUG: missing return for the high case
|
||||
return ""
|
||||
}
|
||||
|
||||
// Label returns a display label for the given name. BUG: uses the wrong prefix
|
||||
// ("admin-" instead of "user-"), producing the wrong text.
|
||||
func Label(name string) string {
|
||||
return fmt.Sprintf("admin-%s", name)
|
||||
}
|
||||
|
||||
// counter is a shared, unsynchronized counter. BUG: Inc is not goroutine-safe.
|
||||
var counter int
|
||||
|
||||
// Inc increments the shared counter.
|
||||
func Inc() {
|
||||
counter++
|
||||
}
|
||||
|
||||
// Count returns the current counter value.
|
||||
func Count() int {
|
||||
return counter
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package fix
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSumIncludesLastElement(t *testing.T) {
|
||||
if got := Sum([]int{1, 2, 3, 4}); got != 10 {
|
||||
t.Fatalf("Sum = %d, want 10", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPortReturnsDefault(t *testing.T) {
|
||||
if got := Port(); got != 8080 {
|
||||
t.Fatalf("Port() = %d, want 8080 (DefaultConfig returned nil)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxReturnsLarger(t *testing.T) {
|
||||
if got := Max(3, 7); got != 7 {
|
||||
t.Fatalf("Max(3,7) = %d, want 7", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFirstReturnsFullContent(t *testing.T) {
|
||||
if got := ReadFirst("hello"); got != "hello" {
|
||||
t.Fatalf("ReadFirst = %q, want %q", got, "hello")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDivideOrder(t *testing.T) {
|
||||
if got := Divide(10, 2); got != 5 {
|
||||
t.Fatalf("Divide(10,2) = %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyHigh(t *testing.T) {
|
||||
if got := Classify(500); got != "high" {
|
||||
t.Fatalf("Classify(500) = %q, want %q", got, "high")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLabelUsesString(t *testing.T) {
|
||||
if got := Label("alice"); got != "user-alice" {
|
||||
t.Fatalf("Label = %q, want %q", got, "user-alice")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCounterRace exercises Inc concurrently. Run with `go test -race`:
|
||||
// the unsynchronized counter trips the race detector until a mutex is added.
|
||||
func TestCounterRace(t *testing.T) {
|
||||
counter = 0
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 200; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 100; j++ {
|
||||
Inc()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if got := Count(); got != 20000 {
|
||||
t.Fatalf("Count = %d, want 20000", got)
|
||||
}
|
||||
}
|
||||
+1287
File diff suppressed because it is too large.
Load diff
+21
@@ -0,0 +1,21 @@
|
||||
// Package longproc is a small buildable workspace for the long-running-process
|
||||
// benchmark tasks, which run build/test/vet/bench commands against it.
|
||||
package longproc
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Version is the demo build's version string.
|
||||
const Version = "1.0.0"
|
||||
|
||||
// Process simulates a bounded unit of work for the benchmark task.
|
||||
func Process(n int) int {
|
||||
total := 0
|
||||
for i := 0; i < n; i++ {
|
||||
total += i
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func main() {
|
||||
fmt.Println("longproc fixture", Version, Process(100))
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package longproc
|
||||
|
||||
import "testing"
|
||||
|
||||
// benchSink keeps Process(100) live across the benchmark loop so the compiler
|
||||
// can't optimize the call away and report an unrealistically fast benchmark.
|
||||
var benchSink int
|
||||
|
||||
func TestProcess(t *testing.T) {
|
||||
if got := Process(100); got != 4950 {
|
||||
t.Fatalf("Process(100) = %d, want 4950", got)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkProcess(b *testing.B) {
|
||||
var sink int
|
||||
for i := 0; i < b.N; i++ {
|
||||
sink = Process(100)
|
||||
}
|
||||
benchSink = sink
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
# nav fixture
|
||||
|
||||
Small read-only workspace for navigation/classification benchmark tasks.
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"port": 8080,
|
||||
"name": "nav-fixture",
|
||||
"retries": 3
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
// Package nav is a small read-only fixture for navigation tasks.
|
||||
package nav
|
||||
|
||||
import "fmt"
|
||||
|
||||
// MaxRetries is the configured retry limit for the demo client.
|
||||
const MaxRetries = 3
|
||||
|
||||
// Config holds the demo client's settings.
|
||||
type Config struct {
|
||||
Port int
|
||||
Name string
|
||||
}
|
||||
|
||||
// greet returns a greeting for the given name.
|
||||
// receieves is a deliberate typo present in the fixture (see edit tasks).
|
||||
func greet(name string) string {
|
||||
return fmt.Sprintf("hello, %s", name)
|
||||
}
|
||||
|
||||
// main is the fixture entry point.
|
||||
func main() {
|
||||
fmt.Println(greet("world"))
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of a.txt
|
||||
second line of a.txt
|
||||
third line of a.txt
|
||||
alpha alpha alpha
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of b.txt
|
||||
second line of b.txt
|
||||
third line of b.txt
|
||||
beta beta beta
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of c.txt
|
||||
second line of c.txt
|
||||
third line of c.txt
|
||||
gamma gamma gamma
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 1, "key": "value-1", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 2, "key": "value-2", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 3, "key": "value-3", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 4, "key": "value-4", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 5, "key": "value-5", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 6, "key": "value-6", "enabled": true }
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of d.txt
|
||||
second line of d.txt
|
||||
third line of d.txt
|
||||
delta delta delta
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir1/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir2/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir3/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir4/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir5/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir6/notes.md
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of e.txt
|
||||
second line of e.txt
|
||||
third line of e.txt
|
||||
epsilon epsilon epsilon
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of f.txt
|
||||
second line of f.txt
|
||||
third line of f.txt
|
||||
zeta zeta zeta
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
// Package refactor is the starting workspace for the multi-file refactor
|
||||
// benchmark tasks. It is intentionally a little messy (duplicated logic, a
|
||||
// bare map type, a thin wrapper) so the refactor prompts have real work to do.
|
||||
// It builds cleanly at all times; the tasks keep it building.
|
||||
package refactor
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Config holds demo settings.
|
||||
type Config struct {
|
||||
Name string
|
||||
Port int
|
||||
}
|
||||
|
||||
// GreetFromConfig returns a greeting using the config's name.
|
||||
func GreetFromConfig(c Config) string {
|
||||
return fmt.Sprintf("hello, %s", c.Name)
|
||||
}
|
||||
|
||||
// GreetByName returns a greeting using a bare name. BUG-note: this duplicates
|
||||
// GreetFromConfig's formatting — refactor-01 extracts a shared helper.
|
||||
func GreetByName(name string) string {
|
||||
return fmt.Sprintf("hello, %s", name)
|
||||
}
|
||||
|
||||
// Wrapper is a thin wrapper around GreetByName — refactor-05 inlines it.
|
||||
func Wrapper(name string) string {
|
||||
return GreetByName(name)
|
||||
}
|
||||
|
||||
// stats is a bare map used across the file — refactor-04 introduces a named type.
|
||||
var stats = map[string]int{}
|
||||
|
||||
// Record bumps a stat by name.
|
||||
func Record(name string) {
|
||||
stats[name]++
|
||||
}
|
||||
|
||||
// Lookup returns a stat by name.
|
||||
func Lookup(name string) int {
|
||||
return stats[name]
|
||||
}
|
||||
|
||||
// load failed wrapped error — refactor-06 consolidates these.
|
||||
func firstError() error {
|
||||
return fmt.Errorf("load failed: missing input")
|
||||
}
|
||||
|
||||
func secondError() error {
|
||||
return fmt.Errorf("load failed: bad format")
|
||||
}
|
||||
|
||||
func buildError(ctx string) error {
|
||||
return fmt.Errorf("load failed: %s", ctx)
|
||||
}
|
||||
@@ -0,0 +1,679 @@
|
||||
package perfbench
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
)
|
||||
|
||||
// TurnSchemaVersion is the schema version of a published turn-benchmark result.
|
||||
// Bump when the TurnBenchResult shape changes so consumers can detect drift.
|
||||
//
|
||||
// v2 splits pass/fail into three oracle tiers so the report cannot be misread as
|
||||
// a blanket correctness verdict: tasksVerified/tasksPassed/correctnessPassRate
|
||||
// cover only the positive-oracle classes (edit/fix); buildCheckedTasks/
|
||||
// buildPassedTasks/buildPassRate cover the non-positive build-check classes
|
||||
// (refactor); latencyOnlyTasks covers the no-oracle classes (nav/longproc/
|
||||
// longctx/parallel). tasksAttempted is still every task run.
|
||||
const TurnSchemaVersion = 2
|
||||
|
||||
// TurnRunner runs one benchmark task and reports its outcome plus the captured
|
||||
// per-turn trace. A non-nil Err means the run failed to execute (process crash);
|
||||
// Passed reflects the verification result. Trace is the parsed NDJSON trace
|
||||
// (nil when the run errored before emitting one).
|
||||
type TurnRunner func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome
|
||||
|
||||
// TurnTaskOutcome is what a TurnRunner reports for one task iteration.
|
||||
type TurnTaskOutcome struct {
|
||||
Passed bool
|
||||
VerifyErr string
|
||||
WallMs float64
|
||||
Trace *trace.TurnTrace
|
||||
// TraceIssue, when non-empty, explains why the per-turn trace could not be
|
||||
// parsed (file missing or malformed). The run still has a pass/fail verdict,
|
||||
// but an incomplete measurement is surfaced as a result warning so it can't
|
||||
// masquerade as a clean attribution sample.
|
||||
TraceIssue string
|
||||
Err error
|
||||
}
|
||||
|
||||
// TurnBenchConfig configures a turn-benchmark run.
|
||||
type TurnBenchConfig struct {
|
||||
Model string
|
||||
Mode string
|
||||
SelfCorrect bool
|
||||
Version string
|
||||
Commit string
|
||||
// Iterations is how many times each task is run. The per-process `zero exec`
|
||||
// runner is inherently cold-start, so this is the sample count for per-span
|
||||
// median/P95 — a genuine warm path needs an in-process runner (future work).
|
||||
Iterations int
|
||||
// Runner executes one task iteration. Required.
|
||||
Runner TurnRunner
|
||||
// Now overrides the clock for the recorded date (tests inject a fixed time).
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// SpanStats summarizes one span's duration across all measured task iterations.
|
||||
type SpanStats struct {
|
||||
Count int `json:"count"`
|
||||
TotalMs float64 `json:"totalMs"`
|
||||
MedianMs float64 `json:"medianMs"`
|
||||
P95Ms float64 `json:"p95Ms"`
|
||||
MaxMs float64 `json:"maxMs"`
|
||||
}
|
||||
|
||||
// LatencySource is one of the top controllable latency sources, ranked by total
|
||||
// exclusive time across the whole run. Share is its fraction of total exclusive
|
||||
// span time. Because exclusive time subtracts nested children, concurrent and
|
||||
// nested spans no longer double-count, so shares of the top sources sum to ~1
|
||||
// for a well-instrumented run.
|
||||
type LatencySource struct {
|
||||
Span string `json:"span"`
|
||||
TotalMs float64 `json:"totalMs"`
|
||||
Share float64 `json:"share"`
|
||||
}
|
||||
|
||||
// ClassSummary is the per-class (task group) roll-up.
|
||||
//
|
||||
// Passed is the count that passed THIS class's oracle: a correctness pass for
|
||||
// edit/fix, a build pass for refactor, and always 0 for the no-oracle classes
|
||||
// (nav/longproc/longctx/parallel). Verified is how many tasks in the class
|
||||
// carry an oracle (correctness or build); LatencyOnly is how many carry none.
|
||||
// A latency-only class therefore reports Passed=0, Verified=0, LatencyOnly=Tasks.
|
||||
type ClassSummary struct {
|
||||
Tasks int `json:"tasks"`
|
||||
Verified int `json:"verified"`
|
||||
Passed int `json:"passed"`
|
||||
LatencyOnly int `json:"latencyOnly"`
|
||||
WallMs NumericStats `json:"wallMs"`
|
||||
SpanTotals map[string]float64 `json:"spanTotals"`
|
||||
}
|
||||
|
||||
// TurnBenchResult is the publishable turn-benchmark record.
|
||||
//
|
||||
// Pass/fail is split into three oracle tiers so it cannot be misread as a
|
||||
// blanket correctness verdict:
|
||||
// - Correctness (tasksVerified / tasksPassed / correctnessPassRate): tasks
|
||||
// with a positive oracle (edit/fix). This is the only pass rate that can
|
||||
// move with model quality.
|
||||
// - Build-only (buildCheckedTasks / buildPassedTasks / buildPassRate): tasks
|
||||
// whose oracle is a non-positive build check (refactor `go build`). A pass
|
||||
// means it compiles, not that the refactor is correct.
|
||||
// - Latency-only (latencyOnlyTasks): tasks with no oracle (nav/longproc/
|
||||
// longctx/parallel). They ran for latency and span attribution only and are
|
||||
// excluded from every pass rate.
|
||||
//
|
||||
// tasksAttempted is still the total number of tasks run across all three tiers.
|
||||
// The tier class lists are echoed so a consumer can see exactly which classes
|
||||
// each rate is computed over.
|
||||
type TurnBenchResult struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Suite string `json:"suite"`
|
||||
Model string `json:"model"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
SelfCorrect bool `json:"selfCorrect"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Date string `json:"date"`
|
||||
TasksAttempted int `json:"tasksAttempted"`
|
||||
TasksVerified int `json:"tasksVerified"`
|
||||
TasksPassed int `json:"tasksPassed"`
|
||||
LatencyOnlyTasks int `json:"latencyOnlyTasks"`
|
||||
BuildCheckedTasks int `json:"buildCheckedTasks"`
|
||||
BuildPassedTasks int `json:"buildPassedTasks"`
|
||||
CorrectnessPassRate float64 `json:"correctnessPassRate"`
|
||||
BuildPassRate float64 `json:"buildPassRate"`
|
||||
CorrectnessClasses []string `json:"correctnessClasses,omitempty"`
|
||||
BuildOnlyClasses []string `json:"buildOnlyClasses,omitempty"`
|
||||
LatencyOnlyClasses []string `json:"latencyOnlyClasses,omitempty"`
|
||||
Iterations int `json:"iterations"`
|
||||
PerSpan map[string]SpanStats `json:"perSpan"`
|
||||
TopLatency []LatencySource `json:"topLatency"`
|
||||
PerClass map[string]ClassSummary `json:"perClass"`
|
||||
Totals TurnBenchTotals `json:"totals"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// TurnBenchTotals aggregates token and count totals across the whole run.
|
||||
type TurnBenchTotals struct {
|
||||
InputTokens int64 `json:"inputTokens"`
|
||||
CachedInputTokens int64 `json:"cachedInputTokens"`
|
||||
OutputTokens int64 `json:"outputTokens"`
|
||||
ModelRequests int64 `json:"modelRequests"`
|
||||
ToolCalls int64 `json:"toolCalls"`
|
||||
Retries int64 `json:"retries"`
|
||||
Reconnects int64 `json:"reconnects"`
|
||||
Compactions int64 `json:"compactions"`
|
||||
}
|
||||
|
||||
// RunTurnBench executes every task in the set with the configured runner and
|
||||
// returns a self-describing per-turn result. It never aborts on a single task
|
||||
// failure — every task is attempted and recorded. Per-span stats aggregate
|
||||
// across iterations; the top three controllable latency sources are ranked by
|
||||
// total attributed time.
|
||||
func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBenchResult, error) {
|
||||
if len(set.Tasks) == 0 {
|
||||
return TurnBenchResult{}, errors.New("task set has no tasks")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Model) == "" {
|
||||
return TurnBenchResult{}, errors.New("turn benchmark requires a model")
|
||||
}
|
||||
if cfg.Runner == nil {
|
||||
return TurnBenchResult{}, errors.New("turn benchmark requires a runner")
|
||||
}
|
||||
iterations := cfg.Iterations
|
||||
if iterations < 1 {
|
||||
iterations = 1
|
||||
}
|
||||
now := cfg.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
rc := RunContext{Model: cfg.Model, Mode: cfg.Mode, SelfCorrect: cfg.SelfCorrect}
|
||||
|
||||
perSpanSamples := map[string][]float64{}
|
||||
classWalls := map[string][]float64{}
|
||||
classSpanTotals := map[string]map[string]float64{}
|
||||
classTasks := map[string]int{}
|
||||
classVerified := map[string]int{}
|
||||
classPassed := map[string]int{}
|
||||
classLatencyOnly := map[string]int{}
|
||||
correctnessClasses := map[string]bool{}
|
||||
buildOnlyClasses := map[string]bool{}
|
||||
latencyOnlyClasses := map[string]bool{}
|
||||
var totals TurnBenchTotals
|
||||
|
||||
// A class is build-only when the manifest declares it in BuildOnlyClasses.
|
||||
// A task's tier is then decided per-task: no verificationCommand => latency-
|
||||
// only; otherwise build-only if its class is declared, else correctness. The
|
||||
// latency-only tier is always driven by oracle presence, never by the
|
||||
// declared list, so a declared build-only class with a missing oracle still
|
||||
// counts as latency-only rather than silently passing on exit 0.
|
||||
buildOnly := map[string]bool{}
|
||||
for _, c := range set.BuildOnlyClasses {
|
||||
buildOnly[strings.TrimSpace(c)] = true
|
||||
}
|
||||
|
||||
result := TurnBenchResult{
|
||||
SchemaVersion: TurnSchemaVersion,
|
||||
Suite: strings.TrimSpace(set.ID),
|
||||
Model: strings.TrimSpace(cfg.Model),
|
||||
Mode: strings.TrimSpace(cfg.Mode),
|
||||
SelfCorrect: cfg.SelfCorrect,
|
||||
Version: strings.TrimSpace(cfg.Version),
|
||||
Commit: strings.TrimSpace(cfg.Commit),
|
||||
Date: now().UTC().Format(time.RFC3339),
|
||||
Iterations: iterations,
|
||||
PerSpan: map[string]SpanStats{},
|
||||
PerClass: map[string]ClassSummary{},
|
||||
}
|
||||
|
||||
for _, task := range set.Tasks {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
class := strings.TrimSpace(task.Class)
|
||||
if class == "" {
|
||||
class = "default"
|
||||
}
|
||||
classTasks[class]++
|
||||
// A task counts as passed only when every iteration passed. The per-process
|
||||
// runner is cold-start, so a flaky pass on one iteration and a fail on
|
||||
// another is a real regression signal, not noise to average away.
|
||||
passedForTask := true
|
||||
for iter := 0; iter < iterations; iter++ {
|
||||
outcome := cfg.Runner(ctx, task, rc)
|
||||
if outcome.TraceIssue != "" {
|
||||
result.Warnings = append(result.Warnings, Warning{
|
||||
Metric: "trace",
|
||||
Message: fmt.Sprintf("task %s: %s", task.ID, outcome.TraceIssue),
|
||||
})
|
||||
}
|
||||
if outcome.Err != nil {
|
||||
// A crashed run must not look like a normal sample that's merely
|
||||
// absent — surface it as a warning so a run that died every
|
||||
// iteration can't pass as "fewer measurements."
|
||||
result.Warnings = append(result.Warnings, Warning{
|
||||
Metric: "run",
|
||||
Message: fmt.Sprintf("task %s: %v", task.ID, outcome.Err),
|
||||
})
|
||||
passedForTask = false
|
||||
continue
|
||||
}
|
||||
if !outcome.Passed {
|
||||
passedForTask = false
|
||||
}
|
||||
wall := outcome.WallMs
|
||||
if wall <= 0 && outcome.Trace != nil {
|
||||
wall = float64(outcome.Trace.WallDuration().Microseconds()) / 1000
|
||||
}
|
||||
if wall > 0 {
|
||||
classWalls[class] = append(classWalls[class], wall)
|
||||
}
|
||||
if outcome.Trace != nil {
|
||||
aggregateTotals(&totals, outcome.Trace)
|
||||
for _, span := range outcome.Trace.Spans {
|
||||
// Rank by exclusive time (duration minus nested children) so
|
||||
// concurrent/nested spans do not double-count: provider_connect
|
||||
// inside generation and permission_wait inside tool_execution
|
||||
// each contribute their own exclusive time, not their parent's.
|
||||
// A span whose exclusive is legitimately zero (a parent fully
|
||||
// covered by its children) contributes zero on purpose — do
|
||||
// NOT fall back to Duration, which would re-introduce the
|
||||
// double-count. ReadNDJSON preserves a written exclusive_ms:0
|
||||
// as 0 and only falls back to Duration when the key is absent,
|
||||
// so span.Exclusive is always populated here.
|
||||
ms := float64(span.Exclusive.Microseconds()) / 1000
|
||||
perSpanSamples[span.Name] = append(perSpanSamples[span.Name], ms)
|
||||
if classSpanTotals[class] == nil {
|
||||
classSpanTotals[class] = map[string]float64{}
|
||||
}
|
||||
classSpanTotals[class][span.Name] += ms
|
||||
}
|
||||
}
|
||||
}
|
||||
result.TasksAttempted++
|
||||
|
||||
// Classify the task into an oracle tier and update only that tier's
|
||||
// counters. A latency-only task (no verificationCommand) is never counted
|
||||
// in any pass rate even when the runner reports Passed — an exit-0
|
||||
// read-only run proves the turn ran, not that the answer was right.
|
||||
hasOracle := len(task.VerificationCommand) > 0
|
||||
switch {
|
||||
case !hasOracle:
|
||||
result.LatencyOnlyTasks++
|
||||
classLatencyOnly[class]++
|
||||
latencyOnlyClasses[class] = true
|
||||
case buildOnly[class]:
|
||||
result.BuildCheckedTasks++
|
||||
classVerified[class]++
|
||||
buildOnlyClasses[class] = true
|
||||
if passedForTask {
|
||||
result.BuildPassedTasks++
|
||||
classPassed[class]++
|
||||
}
|
||||
default:
|
||||
result.TasksVerified++
|
||||
classVerified[class]++
|
||||
correctnessClasses[class] = true
|
||||
if passedForTask {
|
||||
result.TasksPassed++
|
||||
classPassed[class]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for name, samples := range perSpanSamples {
|
||||
result.PerSpan[name] = summarizeSpan(samples)
|
||||
}
|
||||
result.TopLatency = topLatencySources(result.PerSpan, 3)
|
||||
for class := range classTasks {
|
||||
walls := classWalls[class]
|
||||
var wallStats NumericStats
|
||||
if len(walls) > 0 {
|
||||
wallStats = SummarizeSamples(walls)
|
||||
}
|
||||
result.PerClass[class] = ClassSummary{
|
||||
Tasks: classTasks[class],
|
||||
Verified: classVerified[class],
|
||||
Passed: classPassed[class],
|
||||
LatencyOnly: classLatencyOnly[class],
|
||||
WallMs: wallStats,
|
||||
SpanTotals: classSpanTotals[class],
|
||||
}
|
||||
}
|
||||
result.CorrectnessClasses = sortedSetKeys(correctnessClasses)
|
||||
result.BuildOnlyClasses = sortedSetKeys(buildOnlyClasses)
|
||||
result.LatencyOnlyClasses = sortedSetKeys(latencyOnlyClasses)
|
||||
result.CorrectnessPassRate = passRate(result.TasksPassed, result.TasksVerified)
|
||||
result.BuildPassRate = passRate(result.BuildPassedTasks, result.BuildCheckedTasks)
|
||||
result.Totals = totals
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// passRate is passed/total rounded to the benchmark's metric precision, or 0
|
||||
// when the denominator is zero (no tasks in that tier were run).
|
||||
func passRate(passed, total int) float64 {
|
||||
if total <= 0 {
|
||||
return 0
|
||||
}
|
||||
return RoundMetric(float64(passed) / float64(total))
|
||||
}
|
||||
|
||||
func sortedSetKeys(set map[string]bool) []string {
|
||||
if len(set) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func summarizeSpan(samples []float64) SpanStats {
|
||||
if len(samples) == 0 {
|
||||
return SpanStats{}
|
||||
}
|
||||
stats := SummarizeSamples(samples)
|
||||
total := 0.0
|
||||
for _, s := range samples {
|
||||
total += s
|
||||
}
|
||||
return SpanStats{
|
||||
Count: len(samples),
|
||||
TotalMs: RoundMetric(total),
|
||||
MedianMs: stats.Median,
|
||||
P95Ms: stats.P95,
|
||||
MaxMs: stats.Max,
|
||||
}
|
||||
}
|
||||
|
||||
func topLatencySources(perSpan map[string]SpanStats, top int) []LatencySource {
|
||||
sources := make([]LatencySource, 0, len(perSpan))
|
||||
totalAttributed := 0.0
|
||||
for _, s := range perSpan {
|
||||
totalAttributed += s.TotalMs
|
||||
}
|
||||
for name, s := range perSpan {
|
||||
share := 0.0
|
||||
if totalAttributed > 0 {
|
||||
share = s.TotalMs / totalAttributed
|
||||
}
|
||||
sources = append(sources, LatencySource{
|
||||
Span: name,
|
||||
TotalMs: s.TotalMs,
|
||||
Share: RoundMetric(share),
|
||||
})
|
||||
}
|
||||
sort.SliceStable(sources, func(i, j int) bool {
|
||||
if sources[i].TotalMs != sources[j].TotalMs {
|
||||
return sources[i].TotalMs > sources[j].TotalMs
|
||||
}
|
||||
return sources[i].Span < sources[j].Span
|
||||
})
|
||||
if top > 0 && len(sources) > top {
|
||||
sources = sources[:top]
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
func aggregateTotals(totals *TurnBenchTotals, tr *trace.TurnTrace) {
|
||||
totals.InputTokens += tr.Counter(trace.CounterInputTokens)
|
||||
totals.CachedInputTokens += tr.Counter(trace.CounterCachedInputTokens)
|
||||
totals.OutputTokens += tr.Counter(trace.CounterOutputTokens)
|
||||
totals.ModelRequests += tr.Counter(trace.CounterModelRequests)
|
||||
totals.ToolCalls += tr.Counter(trace.CounterToolCalls)
|
||||
totals.Retries += tr.Counter(trace.CounterRetryCount)
|
||||
totals.Reconnects += tr.Counter(trace.CounterReconnectCount)
|
||||
totals.Compactions += tr.Counter(trace.CounterCompactionCount)
|
||||
}
|
||||
|
||||
// FormatTurnBenchSummary renders a human-readable turn-benchmark summary that
|
||||
// names the top controllable latency sources — the baseline's "do not proceed
|
||||
// until" criterion.
|
||||
func FormatTurnBenchSummary(result TurnBenchResult) string {
|
||||
lines := []string{
|
||||
"Zero turn benchmark: " + displayOrUnknown(result.Suite),
|
||||
"model: " + displayOrUnknown(result.Model),
|
||||
// The headline separates the three oracle tiers so an exit-0 read-only
|
||||
// task can never inflate a "pass rate" that reads as correctness:
|
||||
// correctness (positive oracle, edit/fix), build (non-positive `go build`,
|
||||
// refactor), and latency-only (no oracle, nav/longproc/longctx/parallel).
|
||||
fmt.Sprintf("tasks: %d total | correctness %d/%d (%.0f%%) | build %d/%d (%.0f%%) | latency-only %d | %d iter",
|
||||
result.TasksAttempted,
|
||||
result.TasksPassed, result.TasksVerified, result.CorrectnessPassRate*100,
|
||||
result.BuildPassedTasks, result.BuildCheckedTasks, result.BuildPassRate*100,
|
||||
result.LatencyOnlyTasks, result.Iterations),
|
||||
}
|
||||
if result.Mode != "" {
|
||||
lines = append(lines, "mode: "+result.Mode)
|
||||
}
|
||||
if len(result.TopLatency) > 0 {
|
||||
lines = append(lines, "top latency sources:")
|
||||
for _, src := range result.TopLatency {
|
||||
lines = append(lines, fmt.Sprintf(" %-18s %10s %5.1f%%", src.Span, FormatMetric(src.TotalMs, "ms"), src.Share*100))
|
||||
}
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("totals: in=%d (cached %d) out=%d | requests=%d tools=%d retries=%d reconnects=%d compactions=%d",
|
||||
result.Totals.InputTokens, result.Totals.CachedInputTokens, result.Totals.OutputTokens,
|
||||
result.Totals.ModelRequests, result.Totals.ToolCalls, result.Totals.Retries,
|
||||
result.Totals.Reconnects, result.Totals.Compactions))
|
||||
for _, class := range sortedClasses(result.PerClass) {
|
||||
summary := result.PerClass[class]
|
||||
median := FormatMetric(summary.WallMs.Median, "ms")
|
||||
tier := classTier(result, class)
|
||||
lines = append(lines, fmt.Sprintf(" [%s/%s] %d/%d passed, %d latency-only, wall median %s",
|
||||
class, tier, summary.Passed, summary.Verified, summary.LatencyOnly, median))
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// classTier returns the oracle tier label a class was classified into, so the
|
||||
// per-class roll-up states explicitly which oracle (if any) its "passed" count
|
||||
// is measured against.
|
||||
func classTier(result TurnBenchResult, class string) string {
|
||||
for _, c := range result.BuildOnlyClasses {
|
||||
if c == class {
|
||||
return "build"
|
||||
}
|
||||
}
|
||||
for _, c := range result.LatencyOnlyClasses {
|
||||
if c == class {
|
||||
return "latency"
|
||||
}
|
||||
}
|
||||
return "correctness"
|
||||
}
|
||||
|
||||
func sortedClasses(perClass map[string]ClassSummary) []string {
|
||||
classes := make([]string, 0, len(perClass))
|
||||
for c := range perClass {
|
||||
classes = append(classes, c)
|
||||
}
|
||||
sort.Strings(classes)
|
||||
return classes
|
||||
}
|
||||
|
||||
// WriteTurnBenchJSON writes the indented JSON form of a turn-benchmark result.
|
||||
func WriteTurnBenchJSON(w io.Writer, result TurnBenchResult) error {
|
||||
encoder := json.NewEncoder(w)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(result)
|
||||
}
|
||||
|
||||
// NewTurnExecRunner builds the production turn-benchmark runner: it invokes
|
||||
// headless `zero exec` with stream-json output AND `--trace <tmpfile>`, then
|
||||
// parses the emitted NDJSON trace into a *trace.TurnTrace. binary is the path to
|
||||
// the `zero` binary; extraArgs are appended to every invocation. Pass/fail is
|
||||
// decided from the stream-json run_end exit code (and the task's
|
||||
// VerificationCommand when present), exactly like NewExecRunner.
|
||||
func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner {
|
||||
return func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome {
|
||||
// Isolate the workspace: copy the fixture into a fresh temp dir so a
|
||||
// mutating task (edit/fix/refactor) can't dirty the shared, checked-in
|
||||
// fixture or bleed into a later iteration of the same task. When no
|
||||
// fixture is configured the agent runs in the caller's cwd as before.
|
||||
if fixture := strings.TrimSpace(task.WorkspaceFixture); fixture != "" {
|
||||
copyDir, cerr := copyFixture(fixture)
|
||||
if cerr != nil {
|
||||
return TurnTaskOutcome{Err: fmt.Errorf("isolate fixture: %w", cerr)}
|
||||
}
|
||||
defer os.RemoveAll(copyDir)
|
||||
task.WorkspaceFixture = copyDir
|
||||
}
|
||||
|
||||
traceFile, err := os.CreateTemp("", "zero-turn-trace-*.ndjson")
|
||||
if err != nil {
|
||||
return TurnTaskOutcome{Err: fmt.Errorf("create trace file: %w", err)}
|
||||
}
|
||||
_ = traceFile.Close()
|
||||
tracePath := traceFile.Name()
|
||||
defer os.Remove(tracePath)
|
||||
|
||||
args := buildTurnExecArgs(task, rc, tracePath, extraArgs)
|
||||
cmd := exec.CommandContext(ctx, binary, args...)
|
||||
cmd.Env = appendNoColor(os.Environ())
|
||||
if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" {
|
||||
cmd.Dir = dir
|
||||
}
|
||||
var outBuf, errBuf bytes.Buffer
|
||||
cmd.Stdout = &outBuf
|
||||
cmd.Stderr = &errBuf
|
||||
start := time.Now()
|
||||
runErr := cmd.Run()
|
||||
wallMs := float64(time.Since(start).Microseconds()) / 1000
|
||||
|
||||
exitCode, haveExit := streamJSONExitCode(outBuf.Bytes())
|
||||
outcome := TurnTaskOutcome{WallMs: wallMs}
|
||||
if haveExit && exitCode != 0 {
|
||||
outcome.VerifyErr = fmt.Sprintf("agent run_end exit code %d", exitCode)
|
||||
} else if !haveExit {
|
||||
detail := strings.TrimSpace(errBuf.String())
|
||||
// If the process never produced a run_end event, the actual failure
|
||||
// is in runErr (binary missing, exec permission denied, etc.) —
|
||||
// prefer it over the generic fallback so the real reason survives.
|
||||
if detail == "" && runErr != nil {
|
||||
detail = runErr.Error()
|
||||
}
|
||||
if detail == "" {
|
||||
detail = "missing terminal run_end event"
|
||||
}
|
||||
outcome.Err = fmt.Errorf("zero exec failed: %s", detail)
|
||||
return outcome
|
||||
}
|
||||
|
||||
// Parse the captured trace. The trace is the attribution layer, so a
|
||||
// missing/malformed file is not fatal — but it IS surfaced as a warning
|
||||
// (via outcome.TraceIssue) so an incomplete measurement can't look valid.
|
||||
if f, ferr := os.Open(tracePath); ferr != nil {
|
||||
outcome.TraceIssue = fmt.Sprintf("trace open failed: %v", ferr)
|
||||
} else {
|
||||
tr, perr := trace.ReadNDJSON(f)
|
||||
_ = f.Close()
|
||||
if perr != nil {
|
||||
outcome.TraceIssue = fmt.Sprintf("trace parse failed: %v", perr)
|
||||
} else {
|
||||
outcome.Trace = tr
|
||||
}
|
||||
}
|
||||
|
||||
// A nonzero agent exit already decided failure; don't run verification or
|
||||
// mark the task passed.
|
||||
if outcome.VerifyErr != "" {
|
||||
return outcome
|
||||
}
|
||||
if len(task.VerificationCommand) > 0 {
|
||||
if vOutcome := runVerification(ctx, task); !vOutcome.Passed {
|
||||
outcome.VerifyErr = strings.TrimSpace(vOutcome.Detail)
|
||||
return outcome
|
||||
}
|
||||
// A positive oracle (grep/test/build) passed: this is the only path
|
||||
// that sets Passed. A task with no verificationCommand is latency-only
|
||||
// (read-only nav/longproc/longctx/parallel): its exit 0 proves the turn
|
||||
// ran, not that the answer was right, so it never reports Passed and the
|
||||
// harness counts it in latencyOnlyTasks rather than any pass rate.
|
||||
outcome.Passed = true
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
}
|
||||
|
||||
// copyFixture copies the fixture directory at src into a fresh temp dir and
|
||||
// returns the copy's path. Used to give each benchmark invocation an isolated
|
||||
// workspace so mutating tasks (edit/fix/refactor) can't dirty the checked-in
|
||||
// fixture or a later iteration.
|
||||
func copyFixture(src string) (string, error) {
|
||||
info, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("fixture %q is not a directory", src)
|
||||
}
|
||||
dst, err := os.MkdirTemp("", "zero-turn-fixture-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
err = filepath.WalkDir(src, func(path string, d fs.DirEntry, werr error) error {
|
||||
if werr != nil {
|
||||
return werr
|
||||
}
|
||||
rel, rerr := filepath.Rel(src, path)
|
||||
if rerr != nil {
|
||||
return rerr
|
||||
}
|
||||
target := filepath.Join(dst, rel)
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(target, 0o755)
|
||||
}
|
||||
data, rerr := os.ReadFile(path)
|
||||
if rerr != nil {
|
||||
return rerr
|
||||
}
|
||||
return os.WriteFile(target, data, 0o644)
|
||||
})
|
||||
if err != nil {
|
||||
os.RemoveAll(dst)
|
||||
return "", err
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
func buildTurnExecArgs(task BenchTask, rc RunContext, tracePath string, extraArgs []string) []string {
|
||||
args := []string{"exec", "--output-format", "stream-json", "--trace", tracePath}
|
||||
if model := strings.TrimSpace(rc.Model); model != "" {
|
||||
args = append(args, "--model", model)
|
||||
}
|
||||
if mode := strings.TrimSpace(rc.Mode); mode != "" {
|
||||
args = append(args, "--mode", mode)
|
||||
}
|
||||
if rc.SelfCorrect {
|
||||
args = append(args, "--self-correct")
|
||||
}
|
||||
args = append(args, extraArgs...)
|
||||
args = append(args, task.Prompt)
|
||||
return args
|
||||
}
|
||||
|
||||
// ResolveBinary locates the zero binary for a benchmark run: an explicit path
|
||||
// when provided, else a `zero` (or zero.exe) on PATH, else a binary built into
|
||||
// the repo root. Returns an error when none is found.
|
||||
func ResolveBinary(explicit string) (string, error) {
|
||||
if v := strings.TrimSpace(explicit); v != "" {
|
||||
if _, err := os.Stat(v); err != nil {
|
||||
return "", fmt.Errorf("trace binary not found: %w", err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
if path, err := exec.LookPath("zero"); err == nil {
|
||||
return path, nil
|
||||
}
|
||||
if path, err := exec.LookPath("zero.exe"); err == nil {
|
||||
return path, nil
|
||||
}
|
||||
cwd, err := os.Getwd()
|
||||
if err == nil {
|
||||
for _, name := range []string{"zero", "zero.exe"} {
|
||||
candidate := filepath.Join(cwd, name)
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", errors.New("zero binary not found; build it first or pass an explicit path")
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
package perfbench
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
)
|
||||
|
||||
// fakeTurnRunner returns a canned *trace.TurnTrace per task so the harness's
|
||||
// aggregation logic can be exercised without a model or a binary. Each task
|
||||
// yields one generation span (the dominant cost), one tool_execution span, and
|
||||
// the counters the totals aggregate.
|
||||
func fakeTurnRunner(canned map[string]*trace.TurnTrace) TurnRunner {
|
||||
return func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome {
|
||||
tr, ok := canned[task.ID]
|
||||
if !ok {
|
||||
return TurnTaskOutcome{Err: errNoCanned}
|
||||
}
|
||||
wallMs := float64(tr.WallDuration().Microseconds()) / 1000
|
||||
return TurnTaskOutcome{Passed: true, WallMs: wallMs, Trace: tr}
|
||||
}
|
||||
}
|
||||
|
||||
var errNoCanned = &cannedError{}
|
||||
|
||||
type cannedError struct{}
|
||||
|
||||
func (*cannedError) Error() string { return "no canned trace for task" }
|
||||
|
||||
// cannedTrace builds a deterministic *trace.TurnTrace with the given spans and
|
||||
// counters. Spans are recorded as fixed durations (no real timing) so the
|
||||
// aggregation math is predictable in assertions.
|
||||
func cannedTrace(genMs, toolMs int, tokens int64) *trace.TurnTrace {
|
||||
r := trace.NewRecorder("sess", "run-1", "test")
|
||||
r.Start()
|
||||
r.RecordSpan(trace.SpanGeneration, time.Duration(genMs)*time.Millisecond)
|
||||
r.RecordSpan(trace.SpanToolExecution, time.Duration(toolMs)*time.Millisecond)
|
||||
r.Counter(trace.CounterInputTokens, tokens)
|
||||
r.Counter(trace.CounterOutputTokens, tokens/2)
|
||||
r.Counter(trace.CounterModelRequests, 1)
|
||||
r.Counter(trace.CounterToolCalls, 1)
|
||||
r.StampFirstToken()
|
||||
return r.Finish()
|
||||
}
|
||||
|
||||
func TestRunTurnBenchAggregation(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "fake-suite",
|
||||
Tasks: []BenchTask{
|
||||
{ID: "t1", Class: "nav", Prompt: "p1"}, // latency-only
|
||||
{ID: "t2", Class: "nav", Prompt: "p2"}, // latency-only
|
||||
{ID: "t3", Class: "edit", Prompt: "p3", VerificationCommand: []string{"true"}}, // correctness
|
||||
{ID: "t4", Class: "refactor", Prompt: "p4", VerificationCommand: []string{"true"}}, // build-only
|
||||
},
|
||||
BuildOnlyClasses: []string{"refactor"},
|
||||
}
|
||||
canned := map[string]*trace.TurnTrace{
|
||||
"t1": cannedTrace(100, 10, 1000),
|
||||
"t2": cannedTrace(300, 10, 1000),
|
||||
"t3": cannedTrace(200, 50, 2000),
|
||||
"t4": cannedTrace(150, 20, 1500),
|
||||
}
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Iterations: 1,
|
||||
Runner: fakeTurnRunner(canned),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
}
|
||||
result, err := RunTurnBench(context.Background(), set, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
// 4 tasks attempted; the tier split is 2 latency-only, 1 correctness, 1 build.
|
||||
if result.TasksAttempted != 4 {
|
||||
t.Fatalf("attempted=%d, want 4", result.TasksAttempted)
|
||||
}
|
||||
if result.TasksVerified != 1 || result.TasksPassed != 1 {
|
||||
t.Fatalf("correctness verified=%d passed=%d, want 1/1", result.TasksVerified, result.TasksPassed)
|
||||
}
|
||||
if result.LatencyOnlyTasks != 2 {
|
||||
t.Fatalf("latencyOnly=%d, want 2", result.LatencyOnlyTasks)
|
||||
}
|
||||
if result.BuildCheckedTasks != 1 || result.BuildPassedTasks != 1 {
|
||||
t.Fatalf("build checked=%d passed=%d, want 1/1", result.BuildCheckedTasks, result.BuildPassedTasks)
|
||||
}
|
||||
if result.CorrectnessPassRate != 1.0 {
|
||||
t.Fatalf("correctnessPassRate=%v, want 1.0", result.CorrectnessPassRate)
|
||||
}
|
||||
if result.BuildPassRate != 1.0 {
|
||||
t.Fatalf("buildPassRate=%v, want 1.0", result.BuildPassRate)
|
||||
}
|
||||
if len(result.CorrectnessClasses) != 1 || result.CorrectnessClasses[0] != "edit" {
|
||||
t.Fatalf("correctnessClasses=%v, want [edit]", result.CorrectnessClasses)
|
||||
}
|
||||
if len(result.BuildOnlyClasses) != 1 || result.BuildOnlyClasses[0] != "refactor" {
|
||||
t.Fatalf("buildOnlyClasses=%v, want [refactor]", result.BuildOnlyClasses)
|
||||
}
|
||||
if len(result.LatencyOnlyClasses) != 1 || result.LatencyOnlyClasses[0] != "nav" {
|
||||
t.Fatalf("latencyOnlyClasses=%v, want [nav]", result.LatencyOnlyClasses)
|
||||
}
|
||||
if result.SchemaVersion != TurnSchemaVersion {
|
||||
t.Fatalf("schemaVersion = %d, want %d", result.SchemaVersion, TurnSchemaVersion)
|
||||
}
|
||||
if result.Date != "2026-01-02T03:04:05Z" {
|
||||
t.Fatalf("date = %q", result.Date)
|
||||
}
|
||||
|
||||
// Per-span: generation appears in all four (100+300+200+150=750ms), tool in
|
||||
// all four (10+10+50+20=90ms). Count must equal the number of tasks * iterations.
|
||||
gen := result.PerSpan[trace.SpanGeneration]
|
||||
if gen.Count != 4 {
|
||||
t.Fatalf("generation count = %d, want 4", gen.Count)
|
||||
}
|
||||
if gen.TotalMs != 750 {
|
||||
t.Fatalf("generation totalMs = %v, want 750", gen.TotalMs)
|
||||
}
|
||||
tool := result.PerSpan[trace.SpanToolExecution]
|
||||
if tool.TotalMs != 90 {
|
||||
t.Fatalf("tool totalMs = %v, want 90", tool.TotalMs)
|
||||
}
|
||||
|
||||
// Top latency: generation (750) ranks above tool (90). Exactly two spans
|
||||
// here, so both appear and generation is first.
|
||||
if len(result.TopLatency) != 2 || result.TopLatency[0].Span != trace.SpanGeneration {
|
||||
t.Fatalf("topLatency = %+v", result.TopLatency)
|
||||
}
|
||||
if result.TopLatency[0].Share <= result.TopLatency[1].Share {
|
||||
t.Fatalf("top latency not ranked by share: %+v", result.TopLatency)
|
||||
}
|
||||
|
||||
// Totals: 4 model requests, 4 tool calls, input tokens 1000+1000+2000+1500=5500.
|
||||
if result.Totals.ModelRequests != 4 {
|
||||
t.Fatalf("modelRequests = %d, want 4", result.Totals.ModelRequests)
|
||||
}
|
||||
if result.Totals.ToolCalls != 4 {
|
||||
t.Fatalf("toolCalls = %d, want 4", result.Totals.ToolCalls)
|
||||
}
|
||||
if result.Totals.InputTokens != 5500 {
|
||||
t.Fatalf("inputTokens = %d, want 5500", result.Totals.InputTokens)
|
||||
}
|
||||
if result.Totals.OutputTokens != 2750 {
|
||||
t.Fatalf("outputTokens = %d, want 2750", result.Totals.OutputTokens)
|
||||
}
|
||||
|
||||
// Per-class tier roll-up: nav is latency-only (0 verified, 2 latency-only),
|
||||
// edit is correctness (1/1 verified passed), refactor is build (1/1 passed).
|
||||
nav := result.PerClass["nav"]
|
||||
if nav.Tasks != 2 || nav.Verified != 0 || nav.Passed != 0 || nav.LatencyOnly != 2 {
|
||||
t.Fatalf("nav class = %+v", nav)
|
||||
}
|
||||
edit := result.PerClass["edit"]
|
||||
if edit.Tasks != 1 || edit.Verified != 1 || edit.Passed != 1 || edit.LatencyOnly != 0 {
|
||||
t.Fatalf("edit class = %+v", edit)
|
||||
}
|
||||
refactor := result.PerClass["refactor"]
|
||||
if refactor.Tasks != 1 || refactor.Verified != 1 || refactor.Passed != 1 || refactor.LatencyOnly != 0 {
|
||||
t.Fatalf("refactor class = %+v", refactor)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunTurnBenchLatencyOnlyNeverPassed asserts the honesty gate: a task with
|
||||
// no verificationCommand reports Passed=true from the (stub) runner, yet the
|
||||
// harness counts it ONLY in latencyOnlyTasks — never in tasksPassed or any pass
|
||||
// rate — so an exit-0 read-only run cannot inflate a correctness number.
|
||||
func TestRunTurnBenchLatencyOnlyNeverPassed(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "lo-suite",
|
||||
Tasks: []BenchTask{
|
||||
{ID: "n1", Class: "nav", Prompt: "p"}, // no oracle — runner still says Passed=true
|
||||
},
|
||||
}
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Runner: fakeTurnRunner(map[string]*trace.TurnTrace{"n1": cannedTrace(50, 5, 100)}),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
}
|
||||
result, err := RunTurnBench(context.Background(), set, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
if result.TasksPassed != 0 || result.TasksVerified != 0 {
|
||||
t.Fatalf("latency-only leaked into pass: passed=%d verified=%d, want 0/0",
|
||||
result.TasksPassed, result.TasksVerified)
|
||||
}
|
||||
if result.CorrectnessPassRate != 0 || result.BuildPassRate != 0 {
|
||||
t.Fatalf("pass rates should be 0 with no oracle tasks: c=%v b=%v",
|
||||
result.CorrectnessPassRate, result.BuildPassRate)
|
||||
}
|
||||
if result.LatencyOnlyTasks != 1 {
|
||||
t.Fatalf("latencyOnlyTasks=%d, want 1", result.LatencyOnlyTasks)
|
||||
}
|
||||
if result.PerClass["nav"].Passed != 0 || result.PerClass["nav"].LatencyOnly != 1 {
|
||||
t.Fatalf("nav class = %+v, want passed=0 latencyOnly=1", result.PerClass["nav"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTurnBenchIterationsAggregates(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "iter-suite",
|
||||
Tasks: []BenchTask{{ID: "t1", Class: "nav", Prompt: "p1"}},
|
||||
}
|
||||
canned := map[string]*trace.TurnTrace{"t1": cannedTrace(100, 10, 500)}
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Iterations: 3,
|
||||
Runner: fakeTurnRunner(canned),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
}
|
||||
result, err := RunTurnBench(context.Background(), set, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
if result.PerSpan[trace.SpanGeneration].Count != 3 {
|
||||
t.Fatalf("generation count = %d, want 3 (one per iteration)", result.PerSpan[trace.SpanGeneration].Count)
|
||||
}
|
||||
if result.PerSpan[trace.SpanGeneration].TotalMs != 300 {
|
||||
t.Fatalf("generation totalMs = %v, want 300", result.PerSpan[trace.SpanGeneration].TotalMs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTurnBenchRequiresModelAndRunner(t *testing.T) {
|
||||
set := TaskSet{ID: "s", Tasks: []BenchTask{{ID: "t", Prompt: "p"}}}
|
||||
if _, err := RunTurnBench(context.Background(), set, TurnBenchConfig{Runner: fakeTurnRunner(nil)}); err == nil {
|
||||
t.Fatal("expected error for missing model")
|
||||
}
|
||||
if _, err := RunTurnBench(context.Background(), set, TurnBenchConfig{Model: "m"}); err == nil {
|
||||
t.Fatal("expected error for missing runner")
|
||||
}
|
||||
if _, err := RunTurnBench(context.Background(), TaskSet{ID: "empty"}, TurnBenchConfig{Model: "m", Runner: fakeTurnRunner(nil)}); err == nil {
|
||||
t.Fatal("expected error for empty task set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopLatencySourcesRanksByTotalAndCapsTopN(t *testing.T) {
|
||||
perSpan := map[string]SpanStats{
|
||||
"a": {TotalMs: 100},
|
||||
"b": {TotalMs: 500},
|
||||
"c": {TotalMs: 300},
|
||||
"d": {TotalMs: 50},
|
||||
}
|
||||
top := topLatencySources(perSpan, 3)
|
||||
if len(top) != 3 {
|
||||
t.Fatalf("len = %d, want 3", len(top))
|
||||
}
|
||||
wantOrder := []string{"b", "c", "a"}
|
||||
for i, w := range wantOrder {
|
||||
if top[i].Span != w {
|
||||
t.Fatalf("top[%d] = %q, want %q", i, top[i].Span, w)
|
||||
}
|
||||
}
|
||||
// Shares sum to 1 across all four (100+500+300+50=950); the top-3 retain
|
||||
// their global share (not renormalized to the top-3).
|
||||
// Share is rounded to 2 decimals by RoundMetric (500/950 -> 0.53), so compare
|
||||
// against the rounded value with a small tolerance.
|
||||
if got, want := top[0].Share, RoundMetric(500.0/950.0); !approxEqual(got, want, 0.001) {
|
||||
t.Fatalf("top[0] share = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteTurnBenchJSONRoundTrip(t *testing.T) {
|
||||
set := TaskSet{ID: "json-suite", Tasks: []BenchTask{
|
||||
{ID: "t1", Class: "edit", Prompt: "p1", VerificationCommand: []string{"true"}},
|
||||
}}
|
||||
canned := map[string]*trace.TurnTrace{"t1": cannedTrace(150, 20, 800)}
|
||||
result, err := RunTurnBench(context.Background(), set, TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Runner: fakeTurnRunner(canned),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := WriteTurnBenchJSON(&buf, result); err != nil {
|
||||
t.Fatalf("WriteTurnBenchJSON: %v", err)
|
||||
}
|
||||
var decoded TurnBenchResult
|
||||
if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v\n%s", err, buf.String())
|
||||
}
|
||||
if decoded.SchemaVersion != TurnSchemaVersion {
|
||||
t.Fatalf("schemaVersion = %d, want %d", decoded.SchemaVersion, TurnSchemaVersion)
|
||||
}
|
||||
if decoded.TasksVerified != 1 || decoded.TasksPassed != 1 || decoded.LatencyOnlyTasks != 0 {
|
||||
t.Fatalf("decoded tier counts = verified=%d passed=%d latency=%d, want 1/1/0",
|
||||
decoded.TasksVerified, decoded.TasksPassed, decoded.LatencyOnlyTasks)
|
||||
}
|
||||
if decoded.CorrectnessPassRate != 1.0 {
|
||||
t.Fatalf("decoded correctnessPassRate = %v, want 1.0", decoded.CorrectnessPassRate)
|
||||
}
|
||||
if decoded.PerSpan[trace.SpanGeneration].TotalMs != 150 {
|
||||
t.Fatalf("decoded generation totalMs = %v, want 150", decoded.PerSpan[trace.SpanGeneration].TotalMs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatTurnBenchSummaryNamesTopSources(t *testing.T) {
|
||||
set := TaskSet{ID: "fmt-suite", Tasks: []BenchTask{{ID: "t1", Class: "nav", Prompt: "p1"}}}
|
||||
canned := map[string]*trace.TurnTrace{"t1": cannedTrace(150, 20, 800)}
|
||||
result, err := RunTurnBench(context.Background(), set, TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Runner: fakeTurnRunner(canned),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
summary := FormatTurnBenchSummary(result)
|
||||
if !strings.Contains(summary, "top latency sources") {
|
||||
t.Fatalf("summary missing top-latency header:\n%s", summary)
|
||||
}
|
||||
if !strings.Contains(summary, trace.SpanGeneration) {
|
||||
t.Fatalf("summary missing top span name %q:\n%s", trace.SpanGeneration, summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadBaselineManifest(t *testing.T) {
|
||||
path := filepath.Join("manifests", "baseline.json")
|
||||
set, err := LoadTaskSet(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadTaskSet: %v", err)
|
||||
}
|
||||
if set.ID == "" {
|
||||
t.Fatal("manifest has no id")
|
||||
}
|
||||
// The baseline must clear the "do not proceed until ≥30 tasks" gate.
|
||||
if len(set.Tasks) < 30 {
|
||||
t.Fatalf("baseline has %d tasks, want >= 30", len(set.Tasks))
|
||||
}
|
||||
// The six required classes must all be present and non-empty.
|
||||
wantClasses := map[string]bool{
|
||||
"nav": false, "edit": false, "fix": false,
|
||||
"refactor": false, "longproc": false, "longctx": false, "parallel": false,
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, task := range set.Tasks {
|
||||
class := strings.TrimSpace(task.Class)
|
||||
if class == "" {
|
||||
t.Fatalf("task %q has no class", task.ID)
|
||||
}
|
||||
if _, ok := wantClasses[class]; !ok {
|
||||
t.Fatalf("unexpected class %q on task %q", class, task.ID)
|
||||
}
|
||||
wantClasses[class] = true
|
||||
counts[class]++
|
||||
}
|
||||
for class, present := range wantClasses {
|
||||
if !present {
|
||||
t.Fatalf("manifest missing required class %q", class)
|
||||
}
|
||||
if counts[class] == 0 {
|
||||
t.Fatalf("class %q has zero tasks", class)
|
||||
}
|
||||
}
|
||||
// Every task must have a prompt and a workspace fixture pointing under testdata,
|
||||
// and that fixture must actually exist on disk — a manifest referencing a
|
||||
// missing fixture would make every task in its class error out at run time, so
|
||||
// catch it at load time instead.
|
||||
seen := map[string]bool{}
|
||||
for _, task := range set.Tasks {
|
||||
if strings.TrimSpace(task.Prompt) == "" {
|
||||
t.Fatalf("task %q has empty prompt", task.ID)
|
||||
}
|
||||
if strings.TrimSpace(task.WorkspaceFixture) == "" {
|
||||
t.Fatalf("task %q has no workspace fixture", task.ID)
|
||||
}
|
||||
if !strings.Contains(task.WorkspaceFixture, "testdata") {
|
||||
t.Fatalf("task %q fixture %q not under testdata", task.ID, task.WorkspaceFixture)
|
||||
}
|
||||
if seen[task.WorkspaceFixture] {
|
||||
continue
|
||||
}
|
||||
seen[task.WorkspaceFixture] = true
|
||||
info, err := os.Stat(task.WorkspaceFixture)
|
||||
if err != nil {
|
||||
t.Fatalf("task %q fixture %q does not exist: %v", task.ID, task.WorkspaceFixture, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
t.Fatalf("task %q fixture %q is not a directory", task.ID, task.WorkspaceFixture)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func approxEqual(a, b, tol float64) bool {
|
||||
d := a - b
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
return d < tol
|
||||
}
|
||||
|
||||
// TestCopyFixtureIsolatesSourceFromMutation asserts the property that lets a
|
||||
// mutating task run twice (or across iterations) without poisoning the next
|
||||
// sample: the runner operates on a per-invocation copy, so mutating the copy
|
||||
// leaves the checked-in source fixture byte-identical. This is verified by
|
||||
// reading the code today; a test makes it durable against future refactors of
|
||||
// the runner's isolation path.
|
||||
func TestCopyFixtureIsolatesSourceFromMutation(t *testing.T) {
|
||||
src, err := os.MkdirTemp("", "zero-fixture-src-*")
|
||||
if err != nil {
|
||||
t.Fatalf("mkdtemp src: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(src)
|
||||
orig := "package main\n\nfunc main() {}\n"
|
||||
if err := os.WriteFile(filepath.Join(src, "main.go"), []byte(orig), 0o644); err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(src, "sub"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir sub: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(src, "sub", "a.go"), []byte("package sub\n"), 0o644); err != nil {
|
||||
t.Fatalf("write sub: %v", err)
|
||||
}
|
||||
|
||||
// Snapshot the source tree before any mutation of the copy.
|
||||
wantMain, err := os.ReadFile(filepath.Join(src, "main.go"))
|
||||
if err != nil {
|
||||
t.Fatalf("read source main.go: %v", err)
|
||||
}
|
||||
|
||||
copyDir, err := copyFixture(src)
|
||||
if err != nil {
|
||||
t.Fatalf("copyFixture: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(copyDir)
|
||||
|
||||
// The copy must be a real copy, not a symlink/alias of the source, and
|
||||
// mutating it must not touch the source.
|
||||
if copyDir == src {
|
||||
t.Fatalf("copyFixture returned the source dir, not a copy: %s", copyDir)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(copyDir, "main.go"), []byte("package main\n\n// mutated\nfunc main() {}\n"), 0o644); err != nil {
|
||||
t.Fatalf("mutate copy: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(copyDir, "new.go"), []byte("package main\n"), 0o644); err != nil {
|
||||
t.Fatalf("add file to copy: %v", err)
|
||||
}
|
||||
|
||||
gotMain, err := os.ReadFile(filepath.Join(src, "main.go"))
|
||||
if err != nil {
|
||||
t.Fatalf("re-read source main.go: %v", err)
|
||||
}
|
||||
if string(gotMain) != string(wantMain) {
|
||||
t.Fatalf("source fixture mutated by copy: got %q, want %q", gotMain, wantMain)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(src, "new.go")); !os.IsNotExist(err) {
|
||||
t.Fatalf("file added to copy appeared in source: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -271,72 +271,55 @@ func skillSearchRoot(path string) string {
|
||||
// recognize a manifest path that points at the file rather than the directory.
|
||||
const skillFileName = "SKILL.md"
|
||||
|
||||
// MergedSkills loads the default skills directory plus the supplied plugin skill
|
||||
// roots and returns one merged, name-deduplicated list (Content stripped, like
|
||||
// skills.List) alongside the duplicate-name collisions across all roots. Earlier
|
||||
// roots win a name clash, matching skills.Load's first-wins rule; the default dir
|
||||
// is always considered first so a user skill shadows a same-named plugin skill.
|
||||
// A bad root simply yields no skills rather than failing the merge.
|
||||
// MergedSkills loads the default skills directory, the shared ~/.agents/skills
|
||||
// root when present, plus the supplied plugin skill roots and returns one
|
||||
// merged, name-deduplicated list (Content stripped, like skills.List) alongside
|
||||
// the duplicate-name collisions across all roots. Earlier roots win a name
|
||||
// clash, matching skills.Load's first-wins rule; the default dir is always
|
||||
// considered first so a user skill shadows agents and plugins. A bad root simply
|
||||
// yields no skills rather than failing the merge.
|
||||
func MergedSkills(defaultDir string, pluginRoots []string) ([]skills.Skill, []skills.DuplicateName) {
|
||||
return mergeSkills(defaultDir, pluginRoots, false)
|
||||
}
|
||||
|
||||
// mergeSkills merges the default skills dir and plugin roots into one sorted,
|
||||
// name-deduplicated list. keepContent retains each skill's body (skills.Load)
|
||||
// versus stripping it (skills.List). Roots are considered in order with the
|
||||
// default dir first, so an earlier root wins a name clash; collisions are recorded
|
||||
// rather than crashing, preserving the skills package's Duplicates behaviour.
|
||||
// mergeSkills merges the primary skills dir, optional ~/.agents/skills, and
|
||||
// plugin roots into one sorted, name-deduplicated list via skills.LoadFromRoots /
|
||||
// ListFromRoots. keepContent retains each skill's body versus stripping it.
|
||||
// Roots are considered in order with the default dir first, so an earlier root
|
||||
// wins a name clash; collisions are recorded rather than crashing.
|
||||
func mergeSkills(defaultDir string, pluginRoots []string, keepContent bool) ([]skills.Skill, []skills.DuplicateName) {
|
||||
merged := []skills.Skill{}
|
||||
dups := []skills.DuplicateName{}
|
||||
byName := map[string]skills.Skill{}
|
||||
|
||||
roots := append([]string{defaultDir}, pluginRoots...)
|
||||
for _, root := range roots {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
continue
|
||||
}
|
||||
var loaded []skills.Skill
|
||||
var err error
|
||||
if keepContent {
|
||||
loaded, err = skills.Load(root)
|
||||
} else {
|
||||
loaded, err = skills.List(root)
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, skill := range loaded {
|
||||
if winner, clash := byName[skill.Name]; clash {
|
||||
dups = append(dups, skills.DuplicateName{Name: skill.Name, Winner: winner.Path, Loser: skill.Path})
|
||||
continue
|
||||
}
|
||||
byName[skill.Name] = skill
|
||||
merged = append(merged, skill)
|
||||
// Keep defaultDir injectable for tests rather than always calling DefaultDir.
|
||||
roots := skills.GlobalRoots(defaultDir)
|
||||
// GlobalRoots already includes AgentsDir; append plugin roots only.
|
||||
for _, root := range pluginRoots {
|
||||
if root = strings.TrimSpace(root); root != "" {
|
||||
roots = append(roots, root)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(merged, func(left int, right int) bool {
|
||||
return merged[left].Name < merged[right].Name
|
||||
})
|
||||
if keepContent {
|
||||
merged, dups, _ := skills.LoadFromRoots(roots)
|
||||
return merged, dups
|
||||
}
|
||||
merged, dups, _ := skills.ListFromRoots(roots)
|
||||
return merged, dups
|
||||
}
|
||||
|
||||
// skillTool is a drop-in replacement for the core skill tool that resolves a
|
||||
// named skill across the default skills directory PLUS the plugin skill roots, so
|
||||
// plugin-declared skills surface in the agent's skill list. It keeps the core
|
||||
// tool's name, schema, and read-only/allow safety so registering it simply
|
||||
// overlays plugin skills onto the existing surface.
|
||||
// named skill across the default skills directory, the shared ~/.agents/skills
|
||||
// root, and plugin skill roots, so agents- and plugin-declared skills surface in
|
||||
// the agent's skill list. It keeps the core tool's name, schema, and
|
||||
// read-only/allow safety so registering it simply overlays multi-root discovery
|
||||
// onto the existing surface.
|
||||
type skillTool struct {
|
||||
defaultDir string
|
||||
pluginRoots []string
|
||||
}
|
||||
|
||||
// NewSkillTool builds the plugin-aware skill tool. defaultDir is the standard
|
||||
// NewSkillTool builds the multi-root skill tool. defaultDir is the standard
|
||||
// skills directory (skills.DefaultDir); pluginRoots are the plugin skill search
|
||||
// roots from an ActivationResult. The returned tool merges both, deterministically
|
||||
// deduplicating by name (default dir wins a clash) and listing all available
|
||||
// skills when an unknown name is requested.
|
||||
// roots from an ActivationResult. The returned tool merges primary + agents +
|
||||
// plugins, deterministically deduplicating by name (default dir wins a clash)
|
||||
// and listing all available skills when an unknown name is requested.
|
||||
func NewSkillTool(defaultDir string, pluginRoots []string) tools.Tool {
|
||||
return skillTool{defaultDir: defaultDir, pluginRoots: append([]string{}, pluginRoots...)}
|
||||
}
|
||||
@@ -345,7 +328,7 @@ func (tool skillTool) Name() string { return "skill" }
|
||||
|
||||
func (tool skillTool) Description() string {
|
||||
return "Load a named Zero skill and return its instructions as the tool output. " +
|
||||
"Skills are reusable, on-demand instruction sets (including any contributed by plugins). " +
|
||||
"Skills are reusable, on-demand instruction sets (including shared ~/.agents/skills and any contributed by plugins). " +
|
||||
"Call this when a relevant skill exists; an unknown name returns the list of available skills."
|
||||
}
|
||||
|
||||
|
||||
@@ -350,6 +350,7 @@ func TestActivateResolvesManifestRelativeSkillRoot(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNewSkillToolSurfacesPluginSkillInList(t *testing.T) {
|
||||
isolateAgentsHome(t)
|
||||
defaultDir := t.TempDir()
|
||||
writeTestSkill(t, defaultDir, "core-skill", "core body")
|
||||
pluginRoot := t.TempDir()
|
||||
@@ -384,6 +385,7 @@ func TestNewSkillToolSurfacesPluginSkillInList(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMergeSkillRootsListsPluginSkillInAgentList(t *testing.T) {
|
||||
isolateAgentsHome(t)
|
||||
defaultDir := t.TempDir()
|
||||
writeTestSkill(t, defaultDir, "core-skill", "core")
|
||||
|
||||
@@ -404,6 +406,9 @@ func TestMergeSkillRootsListsPluginSkillInAgentList(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMergedSkillsRecordsDuplicatesWithoutCrashing(t *testing.T) {
|
||||
// Isolate AgentsDir so a host ~/.agents/skills cannot add extra collisions.
|
||||
isolateAgentsHome(t)
|
||||
|
||||
defaultDir := t.TempDir()
|
||||
writeTestSkill(t, defaultDir, "shared", "default copy")
|
||||
|
||||
@@ -426,6 +431,90 @@ func TestMergedSkillsRecordsDuplicatesWithoutCrashing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// isolateAgentsHome points HOME/USERPROFILE at an empty temp home so AgentsDir
|
||||
// does not pick up the developer's real ~/.agents/skills during unit tests.
|
||||
func isolateAgentsHome(t *testing.T) {
|
||||
t.Helper()
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
}
|
||||
|
||||
func TestMergedSkillsIncludesAgentsWithoutPlugins(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeTestSkill(t, agents, "agents-skill", "from agents")
|
||||
|
||||
defaultDir := t.TempDir()
|
||||
listed, dups := MergedSkills(defaultDir, nil)
|
||||
if len(dups) != 0 {
|
||||
t.Fatalf("unexpected dups: %#v", dups)
|
||||
}
|
||||
if len(listed) != 1 || listed[0].Name != "agents-skill" {
|
||||
t.Fatalf("agents skill should surface with no plugins, got %#v", listed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedSkillsPriorityPrimaryAgentsPlugins(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
primary := t.TempDir()
|
||||
pluginRoot := t.TempDir()
|
||||
writeTestSkill(t, primary, "shared", "primary body")
|
||||
writeTestSkill(t, agents, "shared", "agents body")
|
||||
writeTestSkill(t, agents, "agents-plugin", "agents body")
|
||||
writeTestSkill(t, pluginRoot, "shared", "plugin body")
|
||||
writeTestSkill(t, pluginRoot, "agents-plugin", "plugin body")
|
||||
writeTestSkill(t, pluginRoot, "plugin-only", "plugin only")
|
||||
|
||||
listed, dups := MergedSkills(primary, []string{pluginRoot})
|
||||
byName := map[string]string{}
|
||||
for _, skill := range listed {
|
||||
// Content is stripped by MergedSkills; use path origin instead.
|
||||
byName[skill.Name] = skill.Path
|
||||
}
|
||||
if !strings.Contains(byName["shared"], primary) {
|
||||
t.Fatalf("primary should win shared, path=%q", byName["shared"])
|
||||
}
|
||||
if !strings.Contains(byName["agents-plugin"], agents) {
|
||||
t.Fatalf("agents should win over plugin for agents-plugin, path=%q", byName["agents-plugin"])
|
||||
}
|
||||
if !strings.Contains(byName["plugin-only"], pluginRoot) {
|
||||
t.Fatalf("plugin-only missing, path=%q", byName["plugin-only"])
|
||||
}
|
||||
if len(dups) < 2 {
|
||||
t.Fatalf("expected cross-layer dups, got %#v", dups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSkillToolLoadsAgentsSkillWithoutPluginRoots(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeTestSkill(t, agents, "agents-skill", "agents body")
|
||||
|
||||
tool := NewSkillTool(t.TempDir(), nil)
|
||||
got := tool.Run(context.Background(), map[string]any{"name": "agents-skill"})
|
||||
if got.Status != tools.StatusOK || !strings.Contains(got.Output, "agents body") {
|
||||
t.Fatalf("expected agents skill body with zero plugin roots, got %q (%s)", got.Status, got.Output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateSkipsMalformedPluginAndContinues(t *testing.T) {
|
||||
registry := tools.NewRegistry()
|
||||
good := toolPlugin(t.TempDir(), ToolExtension{Name: "good", Command: "good", Permission: PermissionPrompt})
|
||||
|
||||
@@ -34,6 +34,9 @@ const (
|
||||
|
||||
var ErrUnknownProvider = errors.New("unknown provider")
|
||||
|
||||
// AIMLAPIID is the provider catalog identifier for the AI/ML API preset.
|
||||
const AIMLAPIID = "aimlapi"
|
||||
|
||||
type Descriptor struct {
|
||||
ID string
|
||||
Name string
|
||||
@@ -107,6 +110,7 @@ var descriptors = []Descriptor{
|
||||
localOpenAI("ollama", "Ollama Local", "http://localhost:11434/v1", "llama3.1", "ollama local"),
|
||||
localOpenAI("lmstudio", "LM Studio", "http://localhost:1234/v1", "local-model", "lm-studio", "lm studio"),
|
||||
oauthProvider(openAICompat("openrouter", "OpenRouter", "https://openrouter.ai/api/v1", "openai/gpt-4.1", []string{"OPENROUTER_API_KEY"}), true, false),
|
||||
openAICompat(AIMLAPIID, "AI/ML API", "https://api.aimlapi.com/v1", "openai/gpt-5-chat", []string{"AIMLAPI_API_KEY"}, "aiml api", "ai/ml api"),
|
||||
// Hugging Face Inference Providers — OpenAI-compatible router at
|
||||
// https://router.huggingface.co/v1 exposes hundreds of OSS models. OAuth
|
||||
// requires a one-time app registration at huggingface.co/settings/applications/new
|
||||
|
||||
@@ -16,6 +16,7 @@ var expectedCatalogIDs = []string{
|
||||
"ollama",
|
||||
"lmstudio",
|
||||
"openrouter",
|
||||
"aimlapi",
|
||||
"huggingface",
|
||||
"chatgpt",
|
||||
"groq",
|
||||
@@ -103,6 +104,28 @@ func TestRecommendedProviderEndpoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAIMLAPIDescriptor(t *testing.T) {
|
||||
descriptor, err := Require("aimlapi")
|
||||
if err != nil {
|
||||
t.Fatalf("Require(aimlapi) error = %v", err)
|
||||
}
|
||||
if descriptor.Name != "AI/ML API" {
|
||||
t.Fatalf("Name = %q, want AI/ML API", descriptor.Name)
|
||||
}
|
||||
if descriptor.DefaultBaseURL != "https://api.aimlapi.com/v1" {
|
||||
t.Fatalf("DefaultBaseURL = %q, want AI/ML API endpoint", descriptor.DefaultBaseURL)
|
||||
}
|
||||
if descriptor.DefaultModel != "openai/gpt-5-chat" {
|
||||
t.Fatalf("DefaultModel = %q, want openai/gpt-5-chat", descriptor.DefaultModel)
|
||||
}
|
||||
if descriptor.Transport != TransportOpenAICompatible {
|
||||
t.Fatalf("Transport = %q, want %q", descriptor.Transport, TransportOpenAICompatible)
|
||||
}
|
||||
if !reflect.DeepEqual(descriptor.AuthEnvVars, []string{"AIMLAPI_API_KEY"}) {
|
||||
t.Fatalf("AuthEnvVars = %#v, want AIMLAPI_API_KEY", descriptor.AuthEnvVars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongCatDescriptor(t *testing.T) {
|
||||
descriptor, err := Require("longcat")
|
||||
if err != nil {
|
||||
@@ -293,7 +316,7 @@ func TestListByTransportPreservesCatalogOrder(t *testing.T) {
|
||||
TransportBedrock: {"bedrock"},
|
||||
TransportVertex: {"vertex"},
|
||||
TransportAnthropicCompat: {"minimax", "minimaxi-cn", "opencode-go-anthropic-compatible", "custom-anthropic-compatible"},
|
||||
TransportOpenAICompat: {"gitlawb-opengateway", "ollama-cloud", "ollama", "lmstudio", "openrouter", "huggingface", "chatgpt", "groq", "deepseek", "together", "dashscope", "moonshot", "longcat", "nvidia-nim", "mistral", "github", "xai", "venice", "xiaomi-mimo", "bankr", "zai", "zai-cn", "kilocode", "opencode", "opencode-go", "atomic-chat", "chatgpt-proxy", "custom-openai-compatible"},
|
||||
TransportOpenAICompat: {"gitlawb-opengateway", "ollama-cloud", "ollama", "lmstudio", "openrouter", "aimlapi", "huggingface", "chatgpt", "groq", "deepseek", "together", "dashscope", "moonshot", "longcat", "nvidia-nim", "mistral", "github", "xai", "venice", "xiaomi-mimo", "bankr", "zai", "zai-cn", "kilocode", "opencode", "opencode-go", "atomic-chat", "chatgpt-proxy", "custom-openai-compatible"},
|
||||
}
|
||||
|
||||
for transport, wantIDs := range cases {
|
||||
|
||||
@@ -195,7 +195,7 @@ func discoverOpenAIModels(ctx context.Context, profile config.ProviderProfile, o
|
||||
AuthHeader: profile.AuthHeader,
|
||||
AuthScheme: profile.AuthScheme,
|
||||
AuthHeaderValue: profile.AuthHeaderValue,
|
||||
CustomHeaders: profile.CustomHeaders,
|
||||
CustomHeaders: providerio.CopyHeaders(profile.CustomHeaders),
|
||||
}, nil)
|
||||
}
|
||||
|
||||
|
||||
@@ -52,6 +52,44 @@ func TestDiscoverOpenAICompatibleModelsFetchesModelsEndpoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverAIMLAPIModelsSendsAuthAndCustomHeadersWithoutAttribution(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
for header, want := range map[string]string{
|
||||
"Authorization": "Bearer test-key",
|
||||
"X-Trace": "test",
|
||||
} {
|
||||
if got := r.Header.Get(header); got != want {
|
||||
t.Errorf("%s = %q, want %q", header, got, want)
|
||||
}
|
||||
}
|
||||
// No first-party referral/attribution headers are injected for catalog
|
||||
// presets; aimlapi rides through CopyHeaders like every other provider.
|
||||
for _, header := range []string{
|
||||
"X-AIMLAPI-Partner-ID",
|
||||
"X-AIMLAPI-Integration-Repo",
|
||||
"X-AIMLAPI-Integration-Version",
|
||||
} {
|
||||
if got := r.Header.Get(header); got != "" {
|
||||
t.Errorf("%s = %q, want no attribution header", header, got)
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"openai/gpt-5-chat"}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := Discover(context.Background(), config.ProviderProfile{
|
||||
CatalogID: "aimlapi",
|
||||
ProviderKind: config.ProviderKindOpenAICompatible,
|
||||
BaseURL: server.URL + "/v1",
|
||||
APIKey: "test-key",
|
||||
CustomHeaders: map[string]string{"X-Trace": "test"},
|
||||
}, Options{HTTPClient: server.Client()})
|
||||
if err != nil {
|
||||
t.Fatalf("Discover returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverOpenAICompatibleModelsHonorsAuthHeaderValue(t *testing.T) {
|
||||
// A profile can authenticate via a raw auth-header value instead of APIKey;
|
||||
// discovery must send it rather than probe unauthenticated.
|
||||
|
||||
@@ -67,7 +67,7 @@ func New(profile config.ProviderProfile, options Options) (zeroruntime.Provider,
|
||||
AuthHeader: profile.AuthHeader,
|
||||
AuthScheme: profile.AuthScheme,
|
||||
AuthHeaderValue: profile.AuthHeaderValue,
|
||||
CustomHeaders: profile.CustomHeaders,
|
||||
CustomHeaders: providerio.CopyHeaders(profile.CustomHeaders),
|
||||
OAuthResolver: options.OAuthResolver,
|
||||
MaxTokens: resolved.maxOutputTokens,
|
||||
HTTPClient: options.HTTPClient,
|
||||
@@ -83,7 +83,7 @@ func New(profile config.ProviderProfile, options Options) (zeroruntime.Provider,
|
||||
AuthHeader: profile.AuthHeader,
|
||||
AuthScheme: profile.AuthScheme,
|
||||
AuthHeaderValue: profile.AuthHeaderValue,
|
||||
CustomHeaders: profile.CustomHeaders,
|
||||
CustomHeaders: providerio.CopyHeaders(profile.CustomHeaders),
|
||||
OAuthResolver: options.OAuthResolver,
|
||||
MaxTokens: resolved.maxOutputTokens,
|
||||
HTTPClient: options.HTTPClient,
|
||||
@@ -97,7 +97,7 @@ func New(profile config.ProviderProfile, options Options) (zeroruntime.Provider,
|
||||
AuthHeader: profile.AuthHeader,
|
||||
AuthScheme: profile.AuthScheme,
|
||||
AuthHeaderValue: profile.AuthHeaderValue,
|
||||
CustomHeaders: profile.CustomHeaders,
|
||||
CustomHeaders: providerio.CopyHeaders(profile.CustomHeaders),
|
||||
OAuthResolver: options.OAuthResolver,
|
||||
MaxTokens: resolved.maxOutputTokens,
|
||||
HTTPClient: options.HTTPClient,
|
||||
|
||||
@@ -220,6 +220,55 @@ func TestNewThreadsCustomProviderHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAIMLAPIProviderSendsEndpointAndAuthWithoutAttribution(t *testing.T) {
|
||||
transport := &captureTransport{responseBody: "data: [DONE]\n\n"}
|
||||
provider, err := New(config.ProviderProfile{
|
||||
Name: "aimlapi",
|
||||
CatalogID: "aimlapi",
|
||||
ProviderKind: config.ProviderKindOpenAICompatible,
|
||||
APIKey: "aimlapi-test-key",
|
||||
Model: "openai/gpt-5-chat",
|
||||
CustomHeaders: map[string]string{"X-Trace": "test"},
|
||||
}, Options{HTTPClient: &http.Client{Transport: transport}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
stream, err := provider.StreamCompletion(context.Background(), zeroruntime.CompletionRequest{
|
||||
Messages: []zeroruntime.Message{{Role: zeroruntime.MessageRoleUser, Content: "hello"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamCompletion() error = %v", err)
|
||||
}
|
||||
for range stream {
|
||||
}
|
||||
|
||||
if transport.request == nil {
|
||||
t.Fatal("HTTP client was not used")
|
||||
}
|
||||
if got := transport.request.URL.String(); got != "https://api.aimlapi.com/v1/chat/completions" {
|
||||
t.Fatalf("request URL = %q, want AI/ML API endpoint", got)
|
||||
}
|
||||
for header, want := range map[string]string{
|
||||
"Authorization": "Bearer aimlapi-test-key",
|
||||
"X-Trace": "test",
|
||||
} {
|
||||
if got := transport.request.Header.Get(header); got != want {
|
||||
t.Fatalf("%s = %q, want %q", header, got, want)
|
||||
}
|
||||
}
|
||||
// No first-party referral/attribution headers are injected for catalog
|
||||
// presets; aimlapi rides through CopyHeaders like every other provider.
|
||||
for _, header := range []string{
|
||||
"X-AIMLAPI-Partner-ID",
|
||||
"X-AIMLAPI-Integration-Repo",
|
||||
"X-AIMLAPI-Integration-Version",
|
||||
} {
|
||||
if got := transport.request.Header.Get(header); got != "" {
|
||||
t.Fatalf("%s = %q, want no attribution header", header, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSupportsOpenAIProviderKind(t *testing.T) {
|
||||
provider, err := New(config.ProviderProfile{
|
||||
Name: "openai",
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
)
|
||||
|
||||
// TokenResolver yields a fresh OAuth credential for one request, or ok=false to
|
||||
@@ -42,7 +44,12 @@ func SendWithAuthRetry(
|
||||
// request (leaking the path/body) before we return the error.
|
||||
headers := base
|
||||
if resolver != nil {
|
||||
// ProviderQueue captures pre-send auth-wait (OAuth token resolve). No
|
||||
// send-side semaphore exists today; a future request queue would also
|
||||
// accumulate here. nil recorder (untraced) is a no-op.
|
||||
queueSpan := trace.FromContext(ctx).Span(trace.SpanProviderQueue)
|
||||
header, value, ok, rerr := resolver(ctx, forceRefresh)
|
||||
queueSpan.End()
|
||||
if rerr != nil {
|
||||
return nil, rerr
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -371,9 +372,15 @@ func ScanSSEDataWithContext(
|
||||
// The provider asked to stop (e.g. it already emitted an error
|
||||
// for this payload). Abort the read and end like ScanSSEData:
|
||||
// return nil so callers fall through to their post-scan checks.
|
||||
// Do not stamp FirstToken — this payload was not accepted model
|
||||
// output, so counting it as first-token time would be misleading.
|
||||
cancel()
|
||||
return nil
|
||||
}
|
||||
// First accepted non-keepalive payload = first real model output. Stamp
|
||||
// once; later real payloads are no-ops. nil recorder (untraced run) is a
|
||||
// no-op.
|
||||
trace.FromContext(ctx).StampFirstToken()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
)
|
||||
|
||||
// Transient-failure retry, shared by every provider.
|
||||
@@ -67,7 +69,9 @@ func SendWithRetry(
|
||||
setHeader(request)
|
||||
}
|
||||
|
||||
connectSpan := trace.FromContext(ctx).Span(trace.SpanProviderConnect)
|
||||
response, err := client.Do(request)
|
||||
connectSpan.End()
|
||||
if err != nil {
|
||||
// A transport failure on a POST does NOT mean the server didn't receive
|
||||
// it — the request may have arrived and be generating a (billable,
|
||||
@@ -82,6 +86,9 @@ func SendWithRetry(
|
||||
}
|
||||
|
||||
if ShouldRetryStatus(response.StatusCode) && attempt < maxAttempts {
|
||||
if r := trace.FromContext(ctx); r != nil {
|
||||
r.Counter(trace.CounterRetryCount, 1)
|
||||
}
|
||||
wait := RetryAfter(response)
|
||||
_ = response.Body.Close()
|
||||
if Backoff(ctx, attempt, wait) {
|
||||
|
||||
@@ -221,6 +221,52 @@ func Info(dir string, name string) (SkillInfo, bool) {
|
||||
return info, true
|
||||
}
|
||||
|
||||
// InfoFromRoots resolves the named skill across discovery roots (earlier roots
|
||||
// win). Lock source/hash are attached only when the winning skill lives under
|
||||
// primaryDir and that dir's lockfile has an entry — agents-only skills return
|
||||
// frontmatter + path with empty Source/Hash. primaryDir is typically the Zero
|
||||
// write root (DefaultDir / skillsDir).
|
||||
func InfoFromRoots(primaryDir string, roots []string, name string) (SkillInfo, bool) {
|
||||
loaded, _, err := LoadFromRoots(roots)
|
||||
if err != nil {
|
||||
return SkillInfo{}, false
|
||||
}
|
||||
target := strings.TrimSpace(name)
|
||||
var skill Skill
|
||||
found := false
|
||||
for _, candidate := range loaded {
|
||||
if candidate.Name == target {
|
||||
skill = candidate
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return SkillInfo{}, false
|
||||
}
|
||||
info := SkillInfo{Skill: skill}
|
||||
primaryDir = strings.TrimSpace(primaryDir)
|
||||
if primaryDir == "" {
|
||||
return info, true
|
||||
}
|
||||
// Only attach lock metadata when the winner is from the primary write root.
|
||||
// Compare path prefixes after cleaning so agents-only skills never pick up a
|
||||
// Zero lock entry by name coincidence.
|
||||
skillPath := filepath.Clean(skill.Path)
|
||||
primaryRoot := filepath.Clean(primaryDir)
|
||||
rel, err := filepath.Rel(primaryRoot, skillPath)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||
return info, true
|
||||
}
|
||||
if lock, err := ReadLock(primaryDir); err == nil {
|
||||
if entry, found := lock[skill.Name]; found {
|
||||
info.Source = entry.Source
|
||||
info.Hash = entry.Hash
|
||||
}
|
||||
}
|
||||
return info, true
|
||||
}
|
||||
|
||||
// ReadLock loads the lockfile from dir. A missing lockfile yields an empty map
|
||||
// with no error so callers can treat "no lockfile" as "nothing installed".
|
||||
func ReadLock(dir string) (map[string]LockEntry, error) {
|
||||
|
||||
+158
-4
@@ -28,10 +28,18 @@ type Skill struct {
|
||||
|
||||
const skillFileName = "SKILL.md"
|
||||
|
||||
// errNotDirectory is returned when a skills root path exists but is not a
|
||||
// directory. On Windows, os.ReadDir reports that case as ErrNotExist
|
||||
// (ENOTDIR aliases ERROR_PATH_NOT_FOUND), so load reclassifies it explicitly.
|
||||
var errNotDirectory = errors.New("not a directory")
|
||||
|
||||
// DefaultDir resolves the skills directory, mirroring sessions.DefaultRoot. An
|
||||
// explicit ZERO_SKILLS_DIR override wins; otherwise it is
|
||||
// $XDG_DATA_HOME/zero/skills or ~/.local/share/zero/skills. The directory is
|
||||
// NOT created — a missing directory simply yields no skills.
|
||||
//
|
||||
// DefaultDir is the primary write root for install/remove/lock. Runtime discovery
|
||||
// also considers AgentsDir and plugin skill roots via DiscoveryRoots / LoadFromRoots.
|
||||
func DefaultDir(env map[string]string) string {
|
||||
if override := strings.TrimSpace(envValue(env, "ZERO_SKILLS_DIR")); override != "" {
|
||||
return override
|
||||
@@ -56,6 +64,66 @@ func DefaultDir(env map[string]string) string {
|
||||
return filepath.Join(base, "zero", "skills")
|
||||
}
|
||||
|
||||
// AgentsDir returns ~/.agents/skills when that path exists and is a directory.
|
||||
// It is a shared, read-only multi-agent skills root (Zero, Hermes, Claude Code,
|
||||
// etc.) and is never the target of install/remove/lock. Missing, non-directory,
|
||||
// or unresolvable home yields "" with no error and no directory creation.
|
||||
//
|
||||
// Home resolution matches other packages: HOME, then USERPROFILE, then
|
||||
// os.UserHomeDir(). ZERO_SKILLS_DIR is intentionally ignored — agents is a
|
||||
// pure convention path, not a Zero-specific override.
|
||||
func AgentsDir(env map[string]string) string {
|
||||
home := strings.TrimSpace(firstNonEmpty(
|
||||
envValue(env, "HOME"),
|
||||
envValue(env, "USERPROFILE"),
|
||||
))
|
||||
if home == "" {
|
||||
userHome, err := os.UserHomeDir()
|
||||
if err != nil || strings.TrimSpace(userHome) == "" {
|
||||
return ""
|
||||
}
|
||||
home = userHome
|
||||
}
|
||||
dir := filepath.Join(home, ".agents", "skills")
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil || !info.IsDir() {
|
||||
return ""
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// DiscoveryRoots returns ordered skill roots for runtime discovery: primary
|
||||
// DefaultDir, optional AgentsDir when present, then pluginRoots. Empty strings
|
||||
// are omitted. Earlier entries win on name clashes.
|
||||
func DiscoveryRoots(env map[string]string, pluginRoots []string) []string {
|
||||
return collectRoots(DefaultDir(env), AgentsDir(env), pluginRoots)
|
||||
}
|
||||
|
||||
// collectRoots assembles ordered non-empty skill roots. primary is typically
|
||||
// DefaultDir (or an injected test dir); agents is typically AgentsDir's result.
|
||||
func collectRoots(primary string, agents string, pluginRoots []string) []string {
|
||||
roots := make([]string, 0, 2+len(pluginRoots))
|
||||
if primary = strings.TrimSpace(primary); primary != "" {
|
||||
roots = append(roots, primary)
|
||||
}
|
||||
if agents = strings.TrimSpace(agents); agents != "" {
|
||||
roots = append(roots, agents)
|
||||
}
|
||||
for _, root := range pluginRoots {
|
||||
if root = strings.TrimSpace(root); root != "" {
|
||||
roots = append(roots, root)
|
||||
}
|
||||
}
|
||||
return roots
|
||||
}
|
||||
|
||||
// GlobalRoots returns discovery roots for management CLI list/info: an explicit
|
||||
// primary write/root dir (usually skillsDir / DefaultDir) plus AgentsDir when
|
||||
// present. Plugin roots are excluded from management UX.
|
||||
func GlobalRoots(primary string) []string {
|
||||
return collectRoots(primary, AgentsDir(nil), nil)
|
||||
}
|
||||
|
||||
// DuplicateName records two skills that resolved to the same frontmatter name.
|
||||
// Winner is the SKILL.md path of the skill that was kept (the one in the
|
||||
// lexicographically-first directory); Loser is the path that was dropped.
|
||||
@@ -77,14 +145,81 @@ type DuplicateName struct {
|
||||
// winner regardless of sort stability. Use Duplicates to surface a warning about
|
||||
// any such collisions.
|
||||
//
|
||||
// NOTE: Load scans one root. Agent startup merges plugin-declared skill roots
|
||||
// separately during plugin activation, so the runtime skill surface can include
|
||||
// both the default directory and skills bundled by active plugins.
|
||||
// NOTE: Load scans one root. Runtime discovery uses LoadFromRoots /
|
||||
// DiscoveryRoots (primary DefaultDir, optional ~/.agents/skills, then plugin
|
||||
// skill roots). Prefer those multi-root helpers for agent/CLI discovery; keep
|
||||
// Load for single-dir install/write call sites.
|
||||
func Load(dir string) ([]Skill, error) {
|
||||
skills, _, err := load(dir)
|
||||
return skills, err
|
||||
}
|
||||
|
||||
// LoadFromRoots loads and merges skills from the provided directories (earlier
|
||||
// entries win on name clashes). Empty roots are skipped. Missing directories are
|
||||
// treated as empty (same as Load). Intra-root and cross-root collisions are
|
||||
// reported as DuplicateName.
|
||||
//
|
||||
// The first non-empty root is the required primary: non-missing load failures
|
||||
// (permission, I/O, not a directory, etc.) are returned so callers do not
|
||||
// confuse a broken primary skills dir with "no skills". Later optional roots
|
||||
// (e.g. ~/.agents/skills, plugin roots) fail open so one bad optional directory
|
||||
// does not hide the rest.
|
||||
func LoadFromRoots(dirs []string) ([]Skill, []DuplicateName, error) {
|
||||
merged := make([]Skill, 0)
|
||||
duplicates := []DuplicateName{}
|
||||
byName := map[string]int{}
|
||||
primary := true
|
||||
|
||||
for _, dir := range dirs {
|
||||
if strings.TrimSpace(dir) == "" {
|
||||
continue
|
||||
}
|
||||
loaded, rootDups, err := load(dir)
|
||||
if err != nil {
|
||||
if primary {
|
||||
// Required primary root: surface real I/O failures instead of
|
||||
// collapsing to an empty skill list.
|
||||
return nil, nil, err
|
||||
}
|
||||
// Optional roots fail open: one bad directory must not hide the rest.
|
||||
continue
|
||||
}
|
||||
primary = false
|
||||
duplicates = append(duplicates, rootDups...)
|
||||
for _, skill := range loaded {
|
||||
if winnerIdx, clash := byName[skill.Name]; clash {
|
||||
duplicates = append(duplicates, DuplicateName{
|
||||
Name: skill.Name,
|
||||
Winner: merged[winnerIdx].Path,
|
||||
Loser: skill.Path,
|
||||
})
|
||||
continue
|
||||
}
|
||||
byName[skill.Name] = len(merged)
|
||||
merged = append(merged, skill)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(merged, func(left int, right int) bool {
|
||||
return merged[left].Name < merged[right].Name
|
||||
})
|
||||
return merged, duplicates, nil
|
||||
}
|
||||
|
||||
// ListFromRoots is like LoadFromRoots but strips Content (like List).
|
||||
func ListFromRoots(dirs []string) ([]Skill, []DuplicateName, error) {
|
||||
loaded, dups, err := LoadFromRoots(dirs)
|
||||
if err != nil {
|
||||
return nil, dups, err
|
||||
}
|
||||
listed := make([]Skill, 0, len(loaded))
|
||||
for _, skill := range loaded {
|
||||
skill.Content = ""
|
||||
listed = append(listed, skill)
|
||||
}
|
||||
return listed, dups, nil
|
||||
}
|
||||
|
||||
// Duplicates returns the duplicate-name collisions Load resolved by the
|
||||
// first-directory-wins rule, so a caller can warn the user that a shadowed skill
|
||||
// was dropped. A missing directory yields no duplicates and no error.
|
||||
@@ -131,6 +266,16 @@ func load(dir string) ([]Skill, []DuplicateName, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
// Windows maps ENOTDIR to ERROR_PATH_NOT_FOUND, which is also
|
||||
// ErrNotExist. A primary skills path that exists but is not a
|
||||
// directory must still surface as a load error, not "no skills".
|
||||
if info, statErr := os.Stat(dir); statErr == nil && !info.IsDir() {
|
||||
return nil, nil, &os.PathError{
|
||||
Op: "readdir",
|
||||
Path: dir,
|
||||
Err: errNotDirectory,
|
||||
}
|
||||
}
|
||||
return []Skill{}, nil, nil
|
||||
}
|
||||
return nil, nil, err
|
||||
@@ -281,7 +426,7 @@ func splitFrontmatter(normalized string) (string, string, bool) {
|
||||
// Matching is case-insensitive on the key; the first occurrence wins.
|
||||
func frontmatterValue(frontmatter string, key string) string {
|
||||
prefix := strings.ToLower(key) + ":"
|
||||
for _, line := range strings.Split(frontmatter, "\n") {
|
||||
for line := range strings.SplitSeq(frontmatter, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(strings.ToLower(trimmed), prefix) {
|
||||
value := strings.TrimSpace(trimmed[len(prefix):])
|
||||
@@ -297,3 +442,12 @@ func envValue(env map[string]string, key string) string {
|
||||
}
|
||||
return os.Getenv(key)
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package skills
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -147,6 +148,25 @@ func TestLoadMissingDirYieldsEmpty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadNotDirectoryErrors(t *testing.T) {
|
||||
// Portable across Unix and Windows: a regular file is not a skills root.
|
||||
// Windows reports ReadDir(file) as ErrNotExist; load must reclassify it.
|
||||
notDir := filepath.Join(t.TempDir(), "not-a-dir")
|
||||
if err := os.WriteFile(notDir, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := Load(notDir)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for non-directory skills root, got loaded=%#v", loaded)
|
||||
}
|
||||
if errors.Is(err, os.ErrNotExist) && !errors.Is(err, errNotDirectory) {
|
||||
t.Fatalf("non-directory skills root must not look missing: %v", err)
|
||||
}
|
||||
if loaded != nil {
|
||||
t.Fatalf("expected nil skills on non-directory root, got %#v", loaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSortsByName(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeSkill(t, dir, "zeta", "body")
|
||||
@@ -292,3 +312,273 @@ func TestDefaultDirFallsBackToHome(t *testing.T) {
|
||||
t.Fatalf("DefaultDir = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentsDirReturnsExistingDirectory(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := AgentsDir(map[string]string{"HOME": home})
|
||||
if got != agents {
|
||||
t.Fatalf("AgentsDir = %q, want %q", got, agents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentsDirMissingIsEmpty(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
got := AgentsDir(map[string]string{"HOME": home})
|
||||
if got != "" {
|
||||
t.Fatalf("AgentsDir for missing path = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentsDirFileNotDirIsEmpty(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
agentsParent := filepath.Join(home, ".agents")
|
||||
if err := os.MkdirAll(agentsParent, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// skills is a file, not a directory
|
||||
if err := os.WriteFile(filepath.Join(agentsParent, "skills"), []byte("not a dir"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := AgentsDir(map[string]string{"HOME": home})
|
||||
if got != "" {
|
||||
t.Fatalf("AgentsDir for file = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentsDirHonorsUserProfile(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := AgentsDir(map[string]string{"USERPROFILE": home})
|
||||
if got != agents {
|
||||
t.Fatalf("AgentsDir via USERPROFILE = %q, want %q", got, agents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentsDirIgnoresZeroSkillsDir(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ZERO_SKILLS_DIR must not redirect or suppress AgentsDir.
|
||||
got := AgentsDir(map[string]string{
|
||||
"HOME": home,
|
||||
"ZERO_SKILLS_DIR": filepath.Join(home, "zero-only"),
|
||||
})
|
||||
if got != agents {
|
||||
t.Fatalf("AgentsDir with ZERO_SKILLS_DIR set = %q, want %q", got, agents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentsDirUnresolvableHomeIsEmpty(t *testing.T) {
|
||||
// Empty env map values + no real UserHomeDir fallback is hard to force, but
|
||||
// empty HOME/USERPROFILE with an empty home should still not panic.
|
||||
// When UserHomeDir works this may return a host path; only assert no panic
|
||||
// and that a deliberately empty-looking override path is not invented from ZERO.
|
||||
_ = AgentsDir(map[string]string{"HOME": "", "USERPROFILE": ""})
|
||||
}
|
||||
|
||||
func TestDiscoveryRootsOrderAndOmission(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
primary := filepath.Join(home, "zero-skills")
|
||||
env := map[string]string{
|
||||
"HOME": home,
|
||||
"ZERO_SKILLS_DIR": primary,
|
||||
}
|
||||
roots := DiscoveryRoots(env, []string{"", " /plugin/a ", "plugin/b"})
|
||||
want := []string{primary, agents, "/plugin/a", "plugin/b"}
|
||||
if len(roots) != len(want) {
|
||||
t.Fatalf("DiscoveryRoots = %#v, want %#v", roots, want)
|
||||
}
|
||||
for i := range want {
|
||||
if roots[i] != want[i] {
|
||||
t.Fatalf("DiscoveryRoots[%d] = %q, want %q (full %#v)", i, roots[i], want[i], roots)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryRootsOmitsMissingAgents(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
primary := filepath.Join(home, "zero-skills")
|
||||
env := map[string]string{
|
||||
"HOME": home,
|
||||
"ZERO_SKILLS_DIR": primary,
|
||||
}
|
||||
roots := DiscoveryRoots(env, nil)
|
||||
if len(roots) != 1 || roots[0] != primary {
|
||||
t.Fatalf("DiscoveryRoots without agents = %#v, want only primary", roots)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromRootsPrimaryWinsOverAgents(t *testing.T) {
|
||||
primary := t.TempDir()
|
||||
agents := t.TempDir()
|
||||
writeSkill(t, primary, "shared", "---\nname: shared\n---\nprimary body\n")
|
||||
writeSkill(t, agents, "shared", "---\nname: shared\n---\nagents body\n")
|
||||
writeSkill(t, agents, "agents-only", "---\nname: agents-only\n---\nagents only\n")
|
||||
|
||||
loaded, dups, err := LoadFromRoots([]string{primary, agents})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromRoots: %v", err)
|
||||
}
|
||||
byName := map[string]Skill{}
|
||||
for _, skill := range loaded {
|
||||
byName[skill.Name] = skill
|
||||
}
|
||||
if byName["shared"].Content != "primary body" {
|
||||
t.Fatalf("primary should win shared, got %q", byName["shared"].Content)
|
||||
}
|
||||
if byName["agents-only"].Content != "agents only" {
|
||||
t.Fatalf("agents-only missing: %#v", loaded)
|
||||
}
|
||||
if len(dups) != 1 || dups[0].Name != "shared" {
|
||||
t.Fatalf("expected one shared duplicate, got %#v", dups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromRootsSkipsEmptyAndMissing(t *testing.T) {
|
||||
primary := t.TempDir()
|
||||
writeSkill(t, primary, "solo", "---\nname: solo\n---\nbody\n")
|
||||
loaded, dups, err := LoadFromRoots([]string{"", filepath.Join(t.TempDir(), "missing"), primary})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromRoots: %v", err)
|
||||
}
|
||||
if len(loaded) != 1 || loaded[0].Name != "solo" {
|
||||
t.Fatalf("expected only solo, got %#v", loaded)
|
||||
}
|
||||
if len(dups) != 0 {
|
||||
t.Fatalf("unexpected dups: %#v", dups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromRootsBubblesPrimaryError(t *testing.T) {
|
||||
// A regular file is not a directory. On Unix ReadDir fails with ENOTDIR; on
|
||||
// Windows that case is aliased to ErrNotExist, so load must reclassify an
|
||||
// existing non-directory and bubble it from the first non-empty root.
|
||||
notDir := filepath.Join(t.TempDir(), "not-a-dir")
|
||||
if err := os.WriteFile(notDir, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
optional := t.TempDir()
|
||||
writeSkill(t, optional, "fallback", "---\nname: fallback\n---\nbody\n")
|
||||
|
||||
loaded, dups, err := LoadFromRoots([]string{notDir, optional})
|
||||
if err == nil {
|
||||
t.Fatalf("expected primary root error, got loaded=%#v dups=%#v", loaded, dups)
|
||||
}
|
||||
// Unix returns ENOTDIR; Windows reclassifies via errNotDirectory because
|
||||
// ENOTDIR aliases ErrNotExist there. Either way it must not look missing.
|
||||
if errors.Is(err, os.ErrNotExist) && !errors.Is(err, errNotDirectory) {
|
||||
t.Fatalf("primary non-directory must not look missing: %v", err)
|
||||
}
|
||||
if loaded != nil {
|
||||
t.Fatalf("expected nil skills on primary error, got %#v", loaded)
|
||||
}
|
||||
if dups != nil {
|
||||
t.Fatalf("expected nil dups on primary error, got %#v", dups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromRootsOptionalRootFailOpen(t *testing.T) {
|
||||
primary := t.TempDir()
|
||||
writeSkill(t, primary, "keep", "---\nname: keep\n---\nbody\n")
|
||||
notDir := filepath.Join(t.TempDir(), "not-a-dir")
|
||||
if err := os.WriteFile(notDir, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
agents := t.TempDir()
|
||||
writeSkill(t, agents, "agents-only", "---\nname: agents-only\n---\nbody\n")
|
||||
|
||||
loaded, dups, err := LoadFromRoots([]string{primary, notDir, agents})
|
||||
if err != nil {
|
||||
t.Fatalf("optional root failure must not fail merge: %v", err)
|
||||
}
|
||||
byName := map[string]Skill{}
|
||||
for _, skill := range loaded {
|
||||
byName[skill.Name] = skill
|
||||
}
|
||||
if byName["keep"].Name != "keep" {
|
||||
t.Fatalf("primary skill missing: %#v", loaded)
|
||||
}
|
||||
if byName["agents-only"].Name != "agents-only" {
|
||||
t.Fatalf("later optional skill should still load: %#v", loaded)
|
||||
}
|
||||
if len(dups) != 0 {
|
||||
t.Fatalf("unexpected dups: %#v", dups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFromRootsStripsContent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeSkill(t, dir, "demo", "---\nname: demo\n---\nbody content\n")
|
||||
listed, _, err := ListFromRoots([]string{dir})
|
||||
if err != nil {
|
||||
t.Fatalf("ListFromRoots: %v", err)
|
||||
}
|
||||
if len(listed) != 1 || listed[0].Name != "demo" {
|
||||
t.Fatalf("unexpected listed: %#v", listed)
|
||||
}
|
||||
if listed[0].Content != "" {
|
||||
t.Fatalf("ListFromRoots must strip Content, got %q", listed[0].Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobalRootsIncludesAgents(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Point HOME so AgentsDir finds the temp agents root.
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
primary := filepath.Join(home, "primary")
|
||||
roots := GlobalRoots(primary)
|
||||
if len(roots) != 2 || roots[0] != primary || roots[1] != agents {
|
||||
t.Fatalf("GlobalRoots = %#v, want [primary, agents]", roots)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInfoFromRootsAgentsOnlyHasNoLock(t *testing.T) {
|
||||
primary := t.TempDir()
|
||||
agents := t.TempDir()
|
||||
writeSkill(t, agents, "shared-agents", "---\nname: agents-skill\ndescription: from agents\n---\nbody\n")
|
||||
info, ok := InfoFromRoots(primary, []string{primary, agents}, "agents-skill")
|
||||
if !ok {
|
||||
t.Fatal("expected agents skill to resolve")
|
||||
}
|
||||
if info.Skill.Name != "agents-skill" || info.Skill.Description != "from agents" {
|
||||
t.Fatalf("unexpected skill: %#v", info.Skill)
|
||||
}
|
||||
if info.Source != "" || info.Hash != "" {
|
||||
t.Fatalf("agents-only skill must not carry lock metadata, got source=%q hash=%q", info.Source, info.Hash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInfoFromRootsPrimaryLockMetadata(t *testing.T) {
|
||||
primary := t.TempDir()
|
||||
writeSkill(t, primary, "demo", "---\nname: demo\n---\nbody\n")
|
||||
// Write a lockfile entry the way install would.
|
||||
lockPath := filepath.Join(primary, LockFileName)
|
||||
if err := os.WriteFile(lockPath, []byte(`{"demo":{"source":"file:///src","hash":"sha256:abc"}}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, ok := InfoFromRoots(primary, []string{primary}, "demo")
|
||||
if !ok {
|
||||
t.Fatal("expected primary skill")
|
||||
}
|
||||
if info.Source != "file:///src" || info.Hash != "sha256:abc" {
|
||||
t.Fatalf("lock metadata missing: %#v", info)
|
||||
}
|
||||
}
|
||||
@@ -85,6 +85,14 @@ func TestSchedulerSkipsWhilePreviousRuns(t *testing.T) {
|
||||
waitFor(t, "first done", func() bool { return sw.Coordinator().Summarize().Done == 1 })
|
||||
ticks <- time.Time{}
|
||||
waitFor(t, "second spawn", func() bool { return len(l.recorded()) == 2 })
|
||||
// fireIfIdle's spawn (what "second spawn" observes via the launcher) and
|
||||
// run's subsequent job.incRuns() are sequential but distinct steps in the
|
||||
// scheduler's goroutine; wait for Runs itself rather than assuming the
|
||||
// launcher recording it means the job's counter is updated too.
|
||||
waitFor(t, "second run recorded", func() bool {
|
||||
j, ok := findJob(sched.List(), id)
|
||||
return ok && j.Runs == 2
|
||||
})
|
||||
|
||||
j, ok := findJob(sched.List(), id)
|
||||
if !ok {
|
||||
|
||||
@@ -34,7 +34,8 @@ func NewScopedApplyPatchTool(workspaceRoot string, scope PathScope) Tool {
|
||||
Required: []string{"patch"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: promptSafety(SideEffectWrite, "Applies patch hunks that can create, edit, or delete files."),
|
||||
safety: promptSafety(SideEffectWrite, "Applies patch hunks that can create, edit, or delete files."),
|
||||
capabilities: ToolCapabilities{Effect: EffectWorkspaceWrite, ThreadSafe: false, ResourceKeys: applyPatchResourceKeys},
|
||||
},
|
||||
workspaceRoot: normalizeWorkspaceRoot(workspaceRoot),
|
||||
scope: scope,
|
||||
|
||||
@@ -88,6 +88,8 @@ func NewAskUserTool() *askUserTool {
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: readOnlySafety("Asks the user clarifying questions; gathers input only."),
|
||||
// EffectInteractive: blocks on user input; ordering matters.
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,10 @@ func NewScopedBashTool(workspaceRoot string, scope PathScope) Tool {
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: promptSafety(SideEffectShell, "Shell commands can read, write, or execute programs."),
|
||||
// One-shot shell can rewrite the workspace (and more). Serialized
|
||||
// as WorkspaceWrite so external-mutator audits include bash; not
|
||||
// session-bound Interactive (that is reserved for PTY/terminal tools).
|
||||
capabilities: ToolCapabilities{Effect: EffectWorkspaceWrite, ThreadSafe: false, ResourceKeys: directoryResourceKeys},
|
||||
},
|
||||
workspaceRoot: normalizeWorkspaceRoot(workspaceRoot),
|
||||
scope: scope,
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package tools
|
||||
|
||||
import "github.com/Gitlawb/zero/internal/localcontrol"
|
||||
|
||||
// BuiltinCatalog returns every built-in tool that Zero can expose to the model
|
||||
// from this package (core + control + optional local-control helpers).
|
||||
//
|
||||
// It is the source of truth for metadata coverage tests: every tool returned
|
||||
// here must declare an explicit Effect other than EffectUnknown.
|
||||
//
|
||||
// Not included (fail-closed EffectUnknown by design):
|
||||
// - MCP tools (internal/mcp registryTool)
|
||||
// - Plugin tools (internal/plugins pluginTool)
|
||||
// - Specialist tools registered outside this package (Task, Generate, …)
|
||||
//
|
||||
// workspaceRoot is used only to construct path-scoped tools; it need not exist.
|
||||
func BuiltinCatalog(workspaceRoot string) []Tool {
|
||||
if workspaceRoot == "" {
|
||||
workspaceRoot = "."
|
||||
}
|
||||
var all []Tool
|
||||
all = append(all, CoreReadOnlyTools(workspaceRoot)...)
|
||||
all = append(all, CoreWriteTools(workspaceRoot)...)
|
||||
all = append(all, CoreShellTools(workspaceRoot)...)
|
||||
// Network tools: always include web_fetch; always include web_search for
|
||||
// classification even when no backend is configured at runtime (registration
|
||||
// remains conditional in CoreNetworkTools).
|
||||
all = append(all, NewWebFetchTool(), NewWebSearchTool())
|
||||
all = append(all, NewEscalateModelTool())
|
||||
|
||||
// tool_search needs a live registry of the tools above.
|
||||
reg := NewRegistry()
|
||||
for _, tool := range all {
|
||||
reg.Register(tool)
|
||||
}
|
||||
search := NewToolSearchTool(reg)
|
||||
reg.Register(search)
|
||||
all = append(all, search)
|
||||
|
||||
// Local-control tools are feature-gated at registration time but always
|
||||
// exist as built-in implementations with fixed classifications.
|
||||
all = append(all, NewLocalBrowserTools(localcontrol.BrowserOptions{})...)
|
||||
all = append(all, NewLocalDesktopTools(localcontrol.DesktopOptions{})...)
|
||||
all = append(all, NewLocalTerminalTools(localcontrol.TerminalOptions{})...)
|
||||
all = append(all, NewLocalControlArtifactTools(LocalControlArtifactOptions{})...)
|
||||
return all
|
||||
}
|
||||
|
||||
// ValidateBuiltinCatalog returns diagnostics for every catalog tool that is
|
||||
// missing or inconsistently classified. Empty means full coverage.
|
||||
//
|
||||
// Duplicate builtin names are always rejected: ResourceKeys is a function and
|
||||
// cannot be compared, so partial metadata comparison would miss conflict-scope
|
||||
// drift between two same-name tools.
|
||||
//
|
||||
// Validation uses constructor-declared capabilities (before ThreadSafe
|
||||
// normalization) so a mutator wired with ThreadSafe=true is still reported.
|
||||
// Runtime callers continue to use CapabilitiesOf, which normalizes.
|
||||
func ValidateBuiltinCatalog(workspaceRoot string) []string {
|
||||
var problems []string
|
||||
seen := map[string]struct{}{}
|
||||
for _, tool := range BuiltinCatalog(workspaceRoot) {
|
||||
name := tool.Name()
|
||||
if _, ok := seen[name]; ok {
|
||||
problems = append(problems, name+": duplicate builtin name in catalog")
|
||||
continue
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
|
||||
// Runtime view: must not remain Unknown after normalization.
|
||||
runtimeCaps := CapabilitiesOf(tool)
|
||||
if runtimeCaps.Effect == EffectUnknown {
|
||||
problems = append(problems, name+": built-in tool remains EffectUnknown (metadata required)")
|
||||
}
|
||||
|
||||
// Declaration view: catch invalid ThreadSafe on mutators that
|
||||
// CapabilitiesOf would silently clear.
|
||||
declared := declaredCapabilitiesOf(tool)
|
||||
problems = append(problems, ValidateCapabilities(name, declared)...)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// declaredCapabilitiesOf returns constructor metadata when available; otherwise
|
||||
// falls back to CapabilitiesOf (already normalized).
|
||||
func declaredCapabilitiesOf(tool Tool) ToolCapabilities {
|
||||
if tool == nil {
|
||||
return UnknownCapabilities()
|
||||
}
|
||||
if d, ok := tool.(declaredCapabilityProvider); ok {
|
||||
return d.declaredCapabilities()
|
||||
}
|
||||
return CapabilitiesOf(tool)
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package tools
|
||||
|
||||
import "strings"
|
||||
|
||||
// EffectClass classifies a tool's side effects for concurrency safety.
|
||||
// Concurrent execution (a later PR) must only consider tools that are both
|
||||
// ThreadSafe and EffectReadOnly; this package only declares metadata.
|
||||
// ThreadSafe is force-cleared for every non-ReadOnly effect at read time
|
||||
// (see normalizeCapabilities).
|
||||
type EffectClass int
|
||||
|
||||
const (
|
||||
// EffectUnknown is the fail-closed default for unclassified tools.
|
||||
// Never concurrency-eligible; always serialized.
|
||||
EffectUnknown EffectClass = iota
|
||||
// EffectReadOnly reads local or remote state without mutating it.
|
||||
// The only effect class that may set ThreadSafe=true after audit.
|
||||
EffectReadOnly
|
||||
// EffectProcessRead observes a retained process, PTY, or terminal.
|
||||
EffectProcessRead
|
||||
// EffectWorkspaceWrite mutates the local repository or workspace.
|
||||
// Includes one-shot shell commands that can rewrite the workspace.
|
||||
EffectWorkspaceWrite
|
||||
// EffectExternalWrite mutates state outside the local workspace.
|
||||
EffectExternalWrite
|
||||
// EffectInteractive uses browser, PTY/terminal, prompts, or other
|
||||
// session-bound interaction where ordering matters.
|
||||
EffectInteractive
|
||||
)
|
||||
|
||||
// String returns a stable diagnostic name for the effect class.
|
||||
func (e EffectClass) String() string {
|
||||
switch e {
|
||||
case EffectUnknown:
|
||||
return "unknown"
|
||||
case EffectReadOnly:
|
||||
return "read_only"
|
||||
case EffectProcessRead:
|
||||
return "process_read"
|
||||
case EffectWorkspaceWrite:
|
||||
return "workspace_write"
|
||||
case EffectExternalWrite:
|
||||
return "external_write"
|
||||
case EffectInteractive:
|
||||
return "interactive"
|
||||
default:
|
||||
return "invalid"
|
||||
}
|
||||
}
|
||||
|
||||
// Valid reports whether e is one of the defined effect classes.
|
||||
func (e EffectClass) Valid() bool {
|
||||
return e >= EffectUnknown && e <= EffectInteractive
|
||||
}
|
||||
|
||||
// ToolCapabilities is the concurrency-safety contract for a tool.
|
||||
// ResourceKeys, when set, returns deterministic conflict keys for a call's
|
||||
// arguments. Returning no keys does NOT make a tool parallel-safe.
|
||||
type ToolCapabilities struct {
|
||||
Effect EffectClass
|
||||
ThreadSafe bool
|
||||
ResourceKeys func(args map[string]any) []string
|
||||
}
|
||||
|
||||
// CapabilityProvider is the optional interface tools implement to declare
|
||||
// effect metadata. Tools that do not implement it are treated as EffectUnknown
|
||||
// and not thread-safe (fail-closed).
|
||||
type CapabilityProvider interface {
|
||||
Capabilities() ToolCapabilities
|
||||
}
|
||||
|
||||
// UnknownCapabilities is the fail-closed default for tools without metadata.
|
||||
func UnknownCapabilities() ToolCapabilities {
|
||||
return ToolCapabilities{
|
||||
Effect: EffectUnknown,
|
||||
ThreadSafe: false,
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeCapabilities is the single read path for capability metadata.
|
||||
// ThreadSafe is cleared unless Effect is a valid EffectReadOnly — mutators,
|
||||
// interactive tools, process readers, unknown, and invalid effects must never
|
||||
// appear concurrency-eligible to a future batch planner that only checks
|
||||
// ThreadSafe && Effect.
|
||||
func normalizeCapabilities(caps ToolCapabilities) ToolCapabilities {
|
||||
if !caps.Effect.Valid() || caps.Effect != EffectReadOnly {
|
||||
caps.ThreadSafe = false
|
||||
}
|
||||
return caps
|
||||
}
|
||||
|
||||
// CapabilitiesOf returns a tool's declared capabilities, or the fail-closed
|
||||
// unknown default when the tool does not implement CapabilityProvider.
|
||||
func CapabilitiesOf(tool Tool) ToolCapabilities {
|
||||
if tool == nil {
|
||||
return UnknownCapabilities()
|
||||
}
|
||||
if provider, ok := tool.(CapabilityProvider); ok {
|
||||
return normalizeCapabilities(provider.Capabilities())
|
||||
}
|
||||
return UnknownCapabilities()
|
||||
}
|
||||
|
||||
// (baseTool).Capabilities exposes the capabilities field set at construction,
|
||||
// normalized so ThreadSafe cannot surface on non-ReadOnly effects.
|
||||
// Tools that embed baseTool inherit this method.
|
||||
func (tool baseTool) Capabilities() ToolCapabilities {
|
||||
return normalizeCapabilities(tool.capabilities)
|
||||
}
|
||||
|
||||
// declaredCapabilities returns the constructor-set metadata without runtime
|
||||
// normalization. Catalog validation uses this so a mis-declared
|
||||
// ThreadSafe=true on a mutator is still reported even though CapabilitiesOf
|
||||
// clears it for callers.
|
||||
func (tool baseTool) declaredCapabilities() ToolCapabilities {
|
||||
return tool.capabilities
|
||||
}
|
||||
|
||||
// declaredCapabilityProvider is implemented by tools that can report their
|
||||
// raw constructor metadata (baseTool embedders).
|
||||
type declaredCapabilityProvider interface {
|
||||
declaredCapabilities() ToolCapabilities
|
||||
}
|
||||
|
||||
// Registry.Capabilities looks up a registered tool's capabilities by name.
|
||||
// Unknown tools return the fail-closed default (not an error) so callers can
|
||||
// treat missing tools as non-concurrent without panicking.
|
||||
func (registry *Registry) Capabilities(name string) ToolCapabilities {
|
||||
if registry == nil {
|
||||
return UnknownCapabilities()
|
||||
}
|
||||
tool, ok := registry.Get(name)
|
||||
if !ok {
|
||||
return UnknownCapabilities()
|
||||
}
|
||||
return CapabilitiesOf(tool)
|
||||
}
|
||||
|
||||
// ValidateCapabilities checks a single capability record for consistency.
|
||||
// It returns a non-empty slice of human-readable problems when invalid.
|
||||
// Note: CapabilitiesOf already normalizes ThreadSafe away for non-ReadOnly
|
||||
// effects; validation still rejects the raw combination so catalog tests
|
||||
// catch mis-declared constructors.
|
||||
func ValidateCapabilities(name string, caps ToolCapabilities) []string {
|
||||
var problems []string
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
name = "<unnamed>"
|
||||
}
|
||||
if !caps.Effect.Valid() {
|
||||
problems = append(problems, name+": invalid effect value")
|
||||
}
|
||||
if caps.Effect == EffectUnknown && caps.ThreadSafe {
|
||||
problems = append(problems, name+": ThreadSafe=true is forbidden with EffectUnknown")
|
||||
}
|
||||
// Mutating / interactive / process-read effects are never concurrency-eligible.
|
||||
switch caps.Effect {
|
||||
case EffectWorkspaceWrite, EffectExternalWrite, EffectInteractive, EffectProcessRead:
|
||||
if caps.ThreadSafe {
|
||||
problems = append(problems, name+": ThreadSafe=true is not permitted for effect "+caps.Effect.String())
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// ValidateToolCapabilities validates CapabilitiesOf(tool) for a registered tool.
|
||||
func ValidateToolCapabilities(tool Tool) []string {
|
||||
if tool == nil {
|
||||
return []string{"<nil tool>"}
|
||||
}
|
||||
return ValidateCapabilities(tool.Name(), CapabilitiesOf(tool))
|
||||
}
|
||||
@@ -0,0 +1,449 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// plainTool implements Tool without CapabilityProvider — fail-closed default.
|
||||
type plainTool struct {
|
||||
name string
|
||||
}
|
||||
|
||||
func (t plainTool) Name() string { return t.name }
|
||||
func (t plainTool) Description() string { return "plain" }
|
||||
func (t plainTool) Parameters() Schema { return Schema{Type: "object"} }
|
||||
func (t plainTool) Safety() Safety { return readOnlySafety("plain") }
|
||||
func (t plainTool) Run(context.Context, map[string]any) Result {
|
||||
return okResult("ok")
|
||||
}
|
||||
|
||||
func TestCapabilitiesOfDefaultsUnknown(t *testing.T) {
|
||||
caps := CapabilitiesOf(plainTool{name: "mystery"})
|
||||
if caps.Effect != EffectUnknown {
|
||||
t.Fatalf("Effect = %v, want Unknown", caps.Effect)
|
||||
}
|
||||
if caps.ThreadSafe {
|
||||
t.Fatal("ThreadSafe must be false for unclassified tools")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapabilitiesOfNilTool(t *testing.T) {
|
||||
caps := CapabilitiesOf(nil)
|
||||
if caps.Effect != EffectUnknown || caps.ThreadSafe {
|
||||
t.Fatalf("nil tool = %+v, want fail-closed unknown", caps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownNotThreadSafeEvenIfProviderLies(t *testing.T) {
|
||||
tool := baseTool{
|
||||
name: "liar",
|
||||
capabilities: ToolCapabilities{
|
||||
Effect: EffectUnknown,
|
||||
ThreadSafe: true, // invalid — Capabilities() must clear it
|
||||
},
|
||||
}
|
||||
caps := tool.Capabilities()
|
||||
if caps.ThreadSafe {
|
||||
t.Fatal("Capabilities() must force ThreadSafe=false for EffectUnknown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeClearsThreadSafeOnMutators(t *testing.T) {
|
||||
tool := baseTool{
|
||||
name: "writer",
|
||||
capabilities: ToolCapabilities{
|
||||
Effect: EffectWorkspaceWrite,
|
||||
ThreadSafe: true, // must be cleared on read
|
||||
},
|
||||
}
|
||||
if tool.Capabilities().ThreadSafe {
|
||||
t.Fatal("WorkspaceWrite must never surface ThreadSafe=true via Capabilities()")
|
||||
}
|
||||
// Wrap with Run so CapabilitiesOf's CapabilityProvider path is exercised.
|
||||
wrapped := deferredCapableTool{baseTool: tool}
|
||||
if CapabilitiesOf(wrapped).ThreadSafe {
|
||||
t.Fatal("CapabilitiesOf must normalize mutator ThreadSafe away")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsThreadSafeUnknown(t *testing.T) {
|
||||
problems := ValidateCapabilities("x", ToolCapabilities{Effect: EffectUnknown, ThreadSafe: true})
|
||||
if len(problems) == 0 {
|
||||
t.Fatal("expected validation problems for ThreadSafe+Unknown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsThreadSafeMutators(t *testing.T) {
|
||||
for _, effect := range []EffectClass{EffectWorkspaceWrite, EffectExternalWrite, EffectInteractive, EffectProcessRead} {
|
||||
problems := ValidateCapabilities("m", ToolCapabilities{Effect: effect, ThreadSafe: true})
|
||||
if len(problems) == 0 {
|
||||
t.Fatalf("effect %s: expected ThreadSafe rejection", effect)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryBuiltinHasExplicitMetadata(t *testing.T) {
|
||||
problems := ValidateBuiltinCatalog(t.TempDir())
|
||||
if len(problems) > 0 {
|
||||
t.Fatalf("builtin metadata coverage failed:\n %s", strings.Join(problems, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltinCatalogNoUnknown(t *testing.T) {
|
||||
counts := map[EffectClass]int{}
|
||||
for _, tool := range BuiltinCatalog(t.TempDir()) {
|
||||
caps := CapabilitiesOf(tool)
|
||||
if caps.Effect == EffectUnknown {
|
||||
t.Errorf("tool %q is EffectUnknown", tool.Name())
|
||||
}
|
||||
counts[caps.Effect]++
|
||||
}
|
||||
if counts[EffectUnknown] != 0 {
|
||||
t.Fatalf("Unknown built-ins = %d, want 0", counts[EffectUnknown])
|
||||
}
|
||||
t.Logf("classification counts: %+v", counts)
|
||||
}
|
||||
|
||||
func TestRegistryCapabilitiesLookup(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
reg.Register(NewScopedReadFileTool(t.TempDir(), nil))
|
||||
caps := reg.Capabilities("read_file")
|
||||
if caps.Effect != EffectReadOnly {
|
||||
t.Fatalf("read_file effect = %v, want ReadOnly", caps.Effect)
|
||||
}
|
||||
// Missing tool → fail-closed.
|
||||
missing := reg.Capabilities("does_not_exist")
|
||||
if missing.Effect != EffectUnknown || missing.ThreadSafe {
|
||||
t.Fatalf("missing tool = %+v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileResourceKeysNormalized(t *testing.T) {
|
||||
keys := fileResourceKeys(map[string]any{"path": `./src/../src/main.go`})
|
||||
if len(keys) != 1 {
|
||||
t.Fatalf("keys = %v", keys)
|
||||
}
|
||||
want := ResourceKeyFile + NormalizeResourcePath("./src/../src/main.go")
|
||||
if keys[0] != want {
|
||||
t.Fatalf("key = %q, want %q", keys[0], want)
|
||||
}
|
||||
// Missing args → nil, no panic.
|
||||
if fileResourceKeys(nil) != nil {
|
||||
t.Fatal("nil args should yield nil keys")
|
||||
}
|
||||
if fileResourceKeys(map[string]any{}) != nil {
|
||||
t.Fatal("empty args should yield nil keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePathNormalizePlatform(t *testing.T) {
|
||||
// Unix-style clean
|
||||
got := NormalizeResourcePath("a//b/../c")
|
||||
if got != "a/c" && got != filepath.ToSlash(filepath.Clean("a//b/../c")) {
|
||||
// After ToSlash clean should be a/c
|
||||
if NormalizeResourcePath("a//b/../c") != "a/c" {
|
||||
t.Fatalf("unix clean = %q", got)
|
||||
}
|
||||
}
|
||||
if runtime.GOOS == "windows" {
|
||||
// Case-insensitive
|
||||
if NormalizeResourcePath(`C:\Foo\Bar`) != NormalizeResourcePath(`c:\foo\bar`) {
|
||||
t.Fatalf("windows case fold failed: %q vs %q",
|
||||
NormalizeResourcePath(`C:\Foo\Bar`), NormalizeResourcePath(`c:\foo\bar`))
|
||||
}
|
||||
}
|
||||
// Empty / URL-shaped
|
||||
if NormalizeResourcePath("") != "" {
|
||||
t.Fatal("empty path")
|
||||
}
|
||||
if NormalizeResourcePath("https://user:pass@host/path") != "" {
|
||||
t.Fatal("URL must not become a file resource path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEndpointResourceKeysStripSecrets(t *testing.T) {
|
||||
keys := endpointResourceKeys(map[string]any{
|
||||
"url": "https://user:[email protected]:443/v1?token=abc",
|
||||
})
|
||||
if len(keys) != 1 {
|
||||
t.Fatalf("keys = %v", keys)
|
||||
}
|
||||
if strings.Contains(keys[0], "secret") || strings.Contains(keys[0], "token") || strings.Contains(keys[0], "user") {
|
||||
t.Fatalf("endpoint key leaked secret material: %q", keys[0])
|
||||
}
|
||||
if !strings.Contains(keys[0], "api.example.com") {
|
||||
t.Fatalf("endpoint key missing host: %q", keys[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEndpointResourceKeysAtInQueryDoesNotStealHost(t *testing.T) {
|
||||
// Path/query must be stripped BEFORE userinfo so ?x=@bad cannot become the host.
|
||||
keys := endpointResourceKeys(map[string]any{
|
||||
"url": "https://api.example.com/v1?token=@secret",
|
||||
})
|
||||
if len(keys) != 1 || keys[0] != ResourceKeyEndpoint+"api.example.com" {
|
||||
t.Fatalf("keys = %v, want endpoint:api.example.com", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessResourceKeysNumericSessionID(t *testing.T) {
|
||||
// write_stdin declares session_id as integer; JSON decodes as float64.
|
||||
keys := processResourceKeys(map[string]any{"session_id": float64(42)})
|
||||
if len(keys) != 1 || keys[0] != ResourceKeyProcess+"42" {
|
||||
t.Fatalf("keys = %v, want process:42", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessResourceKeysSessionArg(t *testing.T) {
|
||||
// terminal_session requires "session", not session_id.
|
||||
keys := processResourceKeys(map[string]any{"session": "term-7"})
|
||||
if len(keys) != 1 || keys[0] != ResourceKeyProcess+"term-7" {
|
||||
t.Fatalf("keys = %v, want process:term-7", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPatchResourceKeysFromDiffAndCwd(t *testing.T) {
|
||||
patch := "--- a/old.go\n+++ b/new.go\n@@ -1 +1 @@\n-x\n+y\n"
|
||||
keys := applyPatchResourceKeys(map[string]any{
|
||||
"patch": patch,
|
||||
"cwd": "pkg",
|
||||
})
|
||||
// Diff paths must be joined under cwd so the same basename under different
|
||||
// cwds does not collide for the future conflict planner.
|
||||
want := map[string]bool{
|
||||
ResourceKeyDirectory + "pkg": true,
|
||||
ResourceKeyFile + "pkg/old.go": true,
|
||||
ResourceKeyFile + "pkg/new.go": true,
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !want[k] {
|
||||
t.Fatalf("unexpected key %q in %v", k, keys)
|
||||
}
|
||||
delete(want, k)
|
||||
}
|
||||
if len(want) != 0 {
|
||||
t.Fatalf("missing keys %v from %v", want, keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeclaredMutatorThreadSafeIsReportedByCatalogValidation(t *testing.T) {
|
||||
// Runtime CapabilitiesOf clears ThreadSafe on mutators; catalog validation
|
||||
// must still see the raw declaration so miswired constructors fail the gate.
|
||||
tool := deferredCapableTool{baseTool: baseTool{
|
||||
name: "bad_writer",
|
||||
capabilities: ToolCapabilities{
|
||||
Effect: EffectWorkspaceWrite,
|
||||
ThreadSafe: true,
|
||||
},
|
||||
}}
|
||||
declared := declaredCapabilitiesOf(tool)
|
||||
if !declared.ThreadSafe || declared.Effect != EffectWorkspaceWrite {
|
||||
t.Fatalf("declared = %+v, want raw WorkspaceWrite+ThreadSafe", declared)
|
||||
}
|
||||
problems := ValidateCapabilities(tool.Name(), declared)
|
||||
if len(problems) == 0 {
|
||||
t.Fatal("expected ValidateCapabilities to reject raw WorkspaceWrite+ThreadSafe")
|
||||
}
|
||||
// Runtime view stays safe.
|
||||
if CapabilitiesOf(tool).ThreadSafe {
|
||||
t.Fatal("runtime CapabilitiesOf must still clear ThreadSafe")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPatchResourceKeysFallbackWorkspace(t *testing.T) {
|
||||
keys := applyPatchResourceKeys(map[string]any{"patch": "not a real diff"})
|
||||
if len(keys) != 1 || keys[0] != ResourceKeyWorkspace+"root" {
|
||||
t.Fatalf("keys = %v, want workspace:root fallback", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceWriteClassifications(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, tool := range []Tool{
|
||||
NewScopedWriteFileTool(root, nil),
|
||||
NewScopedEditFileTool(root, nil),
|
||||
NewScopedApplyPatchTool(root, nil),
|
||||
NewScopedBashTool(root, nil),
|
||||
} {
|
||||
if got := CapabilitiesOf(tool).Effect; got != EffectWorkspaceWrite {
|
||||
t.Errorf("%s effect = %v, want WorkspaceWrite", tool.Name(), got)
|
||||
}
|
||||
if CapabilitiesOf(tool).ThreadSafe {
|
||||
t.Errorf("%s must not be ThreadSafe", tool.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInteractiveClassifications(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, name := range []string{"ask_user", "exec_command", "write_stdin", "update_plan", "escalate_model", "request_permissions", "browser_open", "terminal_session", "desktop_action"} {
|
||||
var found Tool
|
||||
for _, tool := range BuiltinCatalog(root) {
|
||||
if tool.Name() == name {
|
||||
found = tool
|
||||
break
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
t.Fatalf("tool %q not in catalog", name)
|
||||
}
|
||||
if got := CapabilitiesOf(found).Effect; got != EffectInteractive {
|
||||
t.Errorf("%s effect = %v, want Interactive", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadOnlyClassifications(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, name := range []string{"read_file", "read_minified_file", "list_directory", "glob", "grep", "lsp_navigate", "skill", "web_fetch", "web_search", "tool_search"} {
|
||||
var found Tool
|
||||
for _, tool := range BuiltinCatalog(root) {
|
||||
if tool.Name() == name {
|
||||
found = tool
|
||||
break
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
t.Fatalf("tool %q not in catalog", name)
|
||||
}
|
||||
caps := CapabilitiesOf(found)
|
||||
if caps.Effect != EffectReadOnly {
|
||||
t.Errorf("%s effect = %v, want ReadOnly", name, caps.Effect)
|
||||
}
|
||||
// Current catalog intentionally keeps read tools non-thread-safe
|
||||
// until each is audited for shared mutable state (FileTracker, etc.).
|
||||
if caps.ThreadSafe {
|
||||
t.Errorf("%s ThreadSafe=true without explicit audit", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBuiltinCatalogRejectsUnknownRaw(t *testing.T) {
|
||||
// Prove the gate is not a rubber stamp: raw Unknown+ThreadSafe is rejected.
|
||||
problems := ValidateCapabilities("ghost", ToolCapabilities{Effect: EffectUnknown, ThreadSafe: true})
|
||||
if len(problems) == 0 {
|
||||
t.Fatal("expected ValidateCapabilities to reject Unknown+ThreadSafe")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureArtifactIsWorkspaceWrite(t *testing.T) {
|
||||
for _, tool := range NewLocalControlArtifactTools(LocalControlArtifactOptions{}) {
|
||||
if CapabilitiesOf(tool).Effect != EffectWorkspaceWrite {
|
||||
t.Fatalf("%s effect = %v, want WorkspaceWrite", tool.Name(), CapabilitiesOf(tool).Effect)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapabilitiesLookupRace(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
for _, tool := range BuiltinCatalog(t.TempDir()) {
|
||||
reg.Register(tool)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 32; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for _, tool := range reg.All() {
|
||||
_ = reg.Capabilities(tool.Name())
|
||||
_ = CapabilitiesOf(tool)
|
||||
caps := CapabilitiesOf(tool)
|
||||
if caps.ResourceKeys != nil {
|
||||
_ = caps.ResourceKeys(map[string]any{"path": "a/b.go", "url": "https://example.com"})
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestEffectStringAndValid(t *testing.T) {
|
||||
for _, e := range []EffectClass{EffectUnknown, EffectReadOnly, EffectProcessRead, EffectWorkspaceWrite, EffectExternalWrite, EffectInteractive} {
|
||||
if !e.Valid() {
|
||||
t.Fatalf("%v should be valid", e)
|
||||
}
|
||||
if e.String() == "invalid" || e.String() == "" {
|
||||
t.Fatalf("String for %d", e)
|
||||
}
|
||||
}
|
||||
if EffectClass(99).Valid() {
|
||||
t.Fatal("invalid class reported valid")
|
||||
}
|
||||
}
|
||||
|
||||
// deferredCapableTool models a deferred-eligible built-in that still carries
|
||||
// explicit capabilities (MCP tools do not implement CapabilityProvider and
|
||||
// stay Unknown by design).
|
||||
type deferredCapableTool struct {
|
||||
baseTool
|
||||
}
|
||||
|
||||
func (t deferredCapableTool) Deferred() bool { return true }
|
||||
|
||||
func (t deferredCapableTool) Run(context.Context, map[string]any) Result {
|
||||
return okResult("ok")
|
||||
}
|
||||
|
||||
func TestDeferredToolPreservesCapabilities(t *testing.T) {
|
||||
tool := deferredCapableTool{baseTool: baseTool{
|
||||
name: "deferred_reader",
|
||||
capabilities: ToolCapabilities{
|
||||
Effect: EffectReadOnly,
|
||||
ThreadSafe: false,
|
||||
ResourceKeys: fileResourceKeys,
|
||||
},
|
||||
}}
|
||||
if !IsDeferred(tool) {
|
||||
t.Fatal("expected deferred")
|
||||
}
|
||||
reg := NewRegistry()
|
||||
reg.Register(tool)
|
||||
got, ok := reg.Get("deferred_reader")
|
||||
if !ok {
|
||||
t.Fatal("missing registration")
|
||||
}
|
||||
caps := CapabilitiesOf(got)
|
||||
if caps.Effect != EffectReadOnly {
|
||||
t.Fatalf("effect = %v after deferred register", caps.Effect)
|
||||
}
|
||||
keys := caps.ResourceKeys(map[string]any{"path": "pkg/a.go"})
|
||||
if len(keys) != 1 || keys[0] != ResourceKeyFile+NormalizeResourcePath("pkg/a.go") {
|
||||
t.Fatalf("resource keys = %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginAndMCPStyleDefaultUnknown(t *testing.T) {
|
||||
// Plugin and MCP adapters do not implement CapabilityProvider; they must
|
||||
// remain fail-closed (serialized) until they opt into a trusted contract.
|
||||
for _, name := range []string{"mcp_github_create_issue", "plugin_custom_tool"} {
|
||||
caps := CapabilitiesOf(plainTool{name: name})
|
||||
if caps.Effect != EffectUnknown || caps.ThreadSafe {
|
||||
t.Fatalf("%s = %+v, want Unknown/not-thread-safe", name, caps)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchemasUnchangedForCoreTools(t *testing.T) {
|
||||
// Metadata must not alter tool names or parameter shapes.
|
||||
root := t.TempDir()
|
||||
read := NewScopedReadFileTool(root, nil)
|
||||
if read.Name() != "read_file" {
|
||||
t.Fatalf("name = %q", read.Name())
|
||||
}
|
||||
params := read.Parameters()
|
||||
if params.Type != "object" {
|
||||
t.Fatalf("params type = %q", params.Type)
|
||||
}
|
||||
if _, ok := params.Properties["path"]; !ok {
|
||||
t.Fatal("path property missing")
|
||||
}
|
||||
// Safety path unchanged.
|
||||
if read.Safety().SideEffect != SideEffectRead {
|
||||
t.Fatalf("safety side effect changed: %v", read.Safety().SideEffect)
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,8 @@ func NewScopedEditFileTool(workspaceRoot string, scope PathScope) Tool {
|
||||
Required: []string{"path", "old_string", "new_string"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: promptSafety(SideEffectWrite, "Edits files in place."),
|
||||
safety: promptSafety(SideEffectWrite, "Edits files in place."),
|
||||
capabilities: ToolCapabilities{Effect: EffectWorkspaceWrite, ThreadSafe: false, ResourceKeys: fileResourceKeys},
|
||||
},
|
||||
workspaceRoot: normalizeWorkspaceRoot(workspaceRoot),
|
||||
scope: scope,
|
||||
|
||||
@@ -57,6 +57,8 @@ func newEscalateModelTool(registry modelregistry.Registry) escalateModelTool {
|
||||
Reason: "Requests a mid-run switch to a stronger model; the agent loop performs the switch.",
|
||||
AdvertiseInAuto: true,
|
||||
},
|
||||
// Requests mid-run model switch; session-bound.
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false},
|
||||
},
|
||||
registry: registry,
|
||||
}
|
||||
|
||||
@@ -465,6 +465,8 @@ func NewScopedExecCommandTool(workspaceRoot string, scope PathScope, manager *ex
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: promptSafety(SideEffectShell, "Shell commands can read, write, or execute programs."),
|
||||
// PTY/shell session — never concurrent.
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: processResourceKeys},
|
||||
},
|
||||
workspaceRoot: normalizeWorkspaceRoot(workspaceRoot),
|
||||
scope: scope,
|
||||
@@ -717,6 +719,8 @@ func NewWriteStdinTool(manager *execSessionManager) Tool {
|
||||
Reason: "Sending stdin can drive an existing shell process beyond the original command; empty polling and Ctrl-C interrupts are allowed automatically.",
|
||||
AdvertiseInAuto: true,
|
||||
},
|
||||
// Writes to a retained process stdin — process interaction.
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: processResourceKeys},
|
||||
},
|
||||
manager: manager,
|
||||
}
|
||||
|
||||
@@ -38,7 +38,8 @@ func NewScopedGlobTool(workspaceRoot string, scope PathScope) Tool {
|
||||
Required: []string{"pattern"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: readOnlySafety("Finds matching paths without reading contents or modifying files."),
|
||||
safety: readOnlySafety("Finds matching paths without reading contents or modifying files."),
|
||||
capabilities: ToolCapabilities{Effect: EffectReadOnly, ThreadSafe: false, ResourceKeys: workspaceResourceKeys},
|
||||
},
|
||||
workspaceRoot: normalizeWorkspaceRoot(workspaceRoot),
|
||||
scope: scope,
|
||||
|
||||
@@ -53,7 +53,8 @@ func NewScopedGrepTool(workspaceRoot string, scope PathScope) Tool {
|
||||
Required: []string{"pattern"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: readOnlySafety("Searches file paths and matching lines without modifying files."),
|
||||
safety: readOnlySafety("Searches file paths and matching lines without modifying files."),
|
||||
capabilities: ToolCapabilities{Effect: EffectReadOnly, ThreadSafe: false, ResourceKeys: workspaceResourceKeys},
|
||||
},
|
||||
workspaceRoot: normalizeWorkspaceRoot(workspaceRoot),
|
||||
scope: scope,
|
||||
|
||||
@@ -33,7 +33,8 @@ func NewScopedListDirectoryTool(workspaceRoot string, scope PathScope) Tool {
|
||||
},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: readOnlySafety("Lists directory entries without modifying files."),
|
||||
safety: readOnlySafety("Lists directory entries without modifying files."),
|
||||
capabilities: ToolCapabilities{Effect: EffectReadOnly, ThreadSafe: false, ResourceKeys: directoryResourceKeys},
|
||||
},
|
||||
workspaceRoot: normalizeWorkspaceRoot(workspaceRoot),
|
||||
scope: scope,
|
||||
|
||||
@@ -53,7 +53,8 @@ func newBrowserInstallTool(options localcontrol.BrowserOptions) Tool {
|
||||
},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Downloads browser runtime files for local browser automation."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Downloads browser runtime files for local browser automation."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: sessionResourceKeys},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options),
|
||||
}
|
||||
@@ -98,7 +99,8 @@ func newBrowserLaunchToolWithLauncher(options localcontrol.BrowserOptions, launc
|
||||
Required: []string{"app"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Launches a supported local Chromium/Electron app with DevTools enabled and attaches local browser automation."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Launches a supported local Chromium/Electron app with DevTools enabled and attaches local browser automation."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: sessionResourceKeys},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options),
|
||||
launcher: launcher,
|
||||
@@ -172,7 +174,8 @@ func newBrowserConnectTool(options localcontrol.BrowserOptions) Tool {
|
||||
Required: []string{"target"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Attaches to a local browser or Electron app exposed over the Chrome DevTools Protocol."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Attaches to a local browser or Electron app exposed over the Chrome DevTools Protocol."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: sessionResourceKeys},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options),
|
||||
}
|
||||
@@ -211,7 +214,8 @@ func newBrowserOpenTool(options localcontrol.BrowserOptions) Tool {
|
||||
Required: []string{"url"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectNetwork, "Opens a URL through the local browser automation helper. For already-running Chrome/Electron apps, use browser_connect instead. If first use reports a missing browser runtime, run browser_install once."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectNetwork, "Opens a URL through the local browser automation helper. For already-running Chrome/Electron apps, use browser_connect instead. If first use reports a missing browser runtime, run browser_install once."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: sessionResourceKeys},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options),
|
||||
}
|
||||
@@ -253,7 +257,8 @@ func newBrowserSnapshotTool(options localcontrol.BrowserOptions) Tool {
|
||||
},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Reads the current local browser automation session state."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Reads the current local browser automation session state."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: sessionResourceKeys},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options),
|
||||
}
|
||||
@@ -292,7 +297,8 @@ func newBrowserClickTool(options localcontrol.BrowserOptions) Tool {
|
||||
Required: []string{"ref"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Clicks an element in the local browser automation session."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Clicks an element in the local browser automation session."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: sessionResourceKeys},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options),
|
||||
}
|
||||
@@ -332,7 +338,8 @@ func newBrowserTypeTool(options localcontrol.BrowserOptions) Tool {
|
||||
Required: []string{"ref", "text"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Types text into an element in the local browser automation session."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Types text into an element in the local browser automation session."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: sessionResourceKeys},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options),
|
||||
}
|
||||
@@ -371,7 +378,8 @@ func newBrowserPressTool(options localcontrol.BrowserOptions) Tool {
|
||||
Required: []string{"key"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Presses a key in the local browser automation session."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Presses a key in the local browser automation session."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: sessionResourceKeys},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options),
|
||||
}
|
||||
@@ -456,7 +464,8 @@ func newBrowserActionTool(options localcontrol.BrowserOptions) Tool {
|
||||
Required: []string{"command"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Interacts with the local browser automation session."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalBrowser, "Interacts with the local browser automation session."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: sessionResourceKeys},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options),
|
||||
}
|
||||
|
||||
@@ -58,6 +58,8 @@ func newCaptureArtifactTool(options LocalControlArtifactOptions) Tool {
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(enabled, SideEffectLocalControl, "Captures local browser, desktop, or terminal state into the configured artifact directory."),
|
||||
// Captures write artifacts under the configured directory.
|
||||
capabilities: ToolCapabilities{Effect: EffectWorkspaceWrite, ThreadSafe: false},
|
||||
},
|
||||
browser: localcontrol.NewBrowser(options.Browser),
|
||||
desktop: localcontrol.NewDesktop(options.Desktop),
|
||||
|
||||
@@ -35,7 +35,8 @@ func newDesktopWindowsTool(options localcontrol.DesktopOptions) Tool {
|
||||
},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalDesktop, "Reads native desktop window metadata."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalDesktop, "Reads native desktop window metadata."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false},
|
||||
},
|
||||
desktop: localcontrol.NewDesktop(options),
|
||||
}
|
||||
@@ -81,7 +82,8 @@ func newDesktopSnapshotTool(options localcontrol.DesktopOptions) Tool {
|
||||
Required: []string{"pid", "window_id"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalDesktop, "Reads a native desktop window state."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalDesktop, "Reads a native desktop window state."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false},
|
||||
},
|
||||
desktop: localcontrol.NewDesktop(options),
|
||||
}
|
||||
@@ -125,7 +127,8 @@ func newDesktopActionTool(options localcontrol.DesktopOptions) Tool {
|
||||
Required: []string{"command", "input"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalDesktop, "Interacts with native desktop windows."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalDesktop, "Interacts with native desktop windows."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false},
|
||||
},
|
||||
desktop: localcontrol.NewDesktop(options),
|
||||
}
|
||||
|
||||
@@ -45,7 +45,8 @@ func newTerminalSessionTool(options localcontrol.TerminalOptions) Tool {
|
||||
Required: []string{"action", "session"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalTerminal, "Launches or interacts with a local virtual terminal session."),
|
||||
safety: localControlSafety(options.Enabled, SideEffectLocalTerminal, "Launches or interacts with a local virtual terminal session."),
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false, ResourceKeys: processResourceKeys},
|
||||
},
|
||||
terminal: localcontrol.NewTerminal(options),
|
||||
}
|
||||
|
||||
@@ -58,7 +58,8 @@ func NewScopedLSPNavigateTool(workspaceRoot string, scope PathScope) Tool {
|
||||
Required: []string{"op", "path"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: readOnlySafety("Queries the language server for code navigation; reads files, modifies nothing."),
|
||||
safety: readOnlySafety("Queries the language server for code navigation; reads files, modifies nothing."),
|
||||
capabilities: ToolCapabilities{Effect: EffectReadOnly, ThreadSafe: false, ResourceKeys: fileResourceKeys},
|
||||
},
|
||||
workspaceRoot: root,
|
||||
scope: scope,
|
||||
|
||||
@@ -39,6 +39,8 @@ func NewScopedReadFileTool(workspaceRoot string, scope PathScope) Tool {
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: readOnlySafety("Reads file contents without modifying files."),
|
||||
// ThreadSafe=false: may update FileTracker session state on read.
|
||||
capabilities: ToolCapabilities{Effect: EffectReadOnly, ThreadSafe: false, ResourceKeys: fileResourceKeys},
|
||||
},
|
||||
workspaceRoot: normalizeWorkspaceRoot(workspaceRoot),
|
||||
scope: scope,
|
||||
|
||||
@@ -33,7 +33,8 @@ func NewScopedReadMinifiedFileTool(workspaceRoot string, scope PathScope) Tool {
|
||||
Required: []string{"path"},
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: readOnlySafety("Reads a minified view of file contents without modifying files."),
|
||||
safety: readOnlySafety("Reads a minified view of file contents without modifying files."),
|
||||
capabilities: ToolCapabilities{Effect: EffectReadOnly, ThreadSafe: false, ResourceKeys: fileResourceKeys},
|
||||
},
|
||||
workspaceRoot: normalizeWorkspaceRoot(workspaceRoot),
|
||||
scope: scope,
|
||||
|
||||
@@ -34,6 +34,8 @@ func NewRequestPermissionsTool() Tool {
|
||||
Permission: PermissionAllow,
|
||||
Reason: "Requests a user permission decision but does not read, write, run commands, or use the network.",
|
||||
},
|
||||
// Interactive permission escalation.
|
||||
capabilities: ToolCapabilities{Effect: EffectInteractive, ThreadSafe: false},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Resource-key prefixes used for future conflict detection. Keys are pure
|
||||
// metadata in this PR — no locks or scheduling are applied.
|
||||
//
|
||||
// Unused prefixes (repository/pty/browser) are reserved for follow-up
|
||||
// classifiers so PR6 conflict keys stay vocabulary-stable.
|
||||
const (
|
||||
ResourceKeyFile = "file:"
|
||||
ResourceKeyDirectory = "directory:"
|
||||
ResourceKeyRepository = "repository:" // reserved
|
||||
ResourceKeyProcess = "process:"
|
||||
ResourceKeyPTY = "pty:" // reserved
|
||||
ResourceKeyBrowser = "browser:" // reserved
|
||||
ResourceKeyEndpoint = "endpoint:"
|
||||
ResourceKeySession = "session:"
|
||||
ResourceKeyWorkspace = "workspace:"
|
||||
)
|
||||
|
||||
// NormalizeResourcePath produces a deterministic path token for resource keys.
|
||||
// It never touches the filesystem (no EvalSymlinks / Stat) so it cannot cause
|
||||
// side effects or panic on missing paths. Empty input returns "".
|
||||
//
|
||||
// Rules:
|
||||
// - Trim space
|
||||
// - filepath.Clean
|
||||
// - Convert separators to '/'
|
||||
// - On Windows, lower-case for case-insensitive comparison
|
||||
// - Strip a leading "./"
|
||||
// - Never include credentials or query fragments (paths only)
|
||||
func NormalizeResourcePath(path string) string {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return ""
|
||||
}
|
||||
// Reject URL-shaped values — those use endpoint: keys.
|
||||
if strings.Contains(path, "://") {
|
||||
return ""
|
||||
}
|
||||
cleaned := filepath.Clean(path)
|
||||
if cleaned == "." {
|
||||
return "."
|
||||
}
|
||||
normalized := filepath.ToSlash(cleaned)
|
||||
normalized = strings.TrimPrefix(normalized, "./")
|
||||
if runtime.GOOS == "windows" {
|
||||
normalized = strings.ToLower(normalized)
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
// fileResourceKeys extracts a file: resource key from common path argument names.
|
||||
// Missing or empty path returns nil (not an error); never panics.
|
||||
func fileResourceKeys(args map[string]any) []string {
|
||||
path := firstStringArg(args, "path", "file", "file_path", "filepath", "filename", "target")
|
||||
normalized := NormalizeResourcePath(path)
|
||||
if normalized == "" {
|
||||
return nil
|
||||
}
|
||||
return []string{ResourceKeyFile + normalized}
|
||||
}
|
||||
|
||||
// directoryResourceKeys extracts a directory: key from path/directory/cwd args.
|
||||
func directoryResourceKeys(args map[string]any) []string {
|
||||
path := firstStringArg(args, "path", "directory", "dir", "cwd", "workdir")
|
||||
normalized := NormalizeResourcePath(path)
|
||||
if normalized == "" {
|
||||
return nil
|
||||
}
|
||||
return []string{ResourceKeyDirectory + normalized}
|
||||
}
|
||||
|
||||
// endpointResourceKeys extracts an endpoint: key from url/endpoint args.
|
||||
// Host only is kept when a full URL is provided (no secrets/query/userinfo).
|
||||
func endpointResourceKeys(args map[string]any) []string {
|
||||
raw := firstStringArg(args, "url", "endpoint", "uri")
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
host := resourceHost(raw)
|
||||
if host == "" {
|
||||
return nil
|
||||
}
|
||||
return []string{ResourceKeyEndpoint + host}
|
||||
}
|
||||
|
||||
// sessionResourceKeys extracts a session: key from session / session_id args.
|
||||
// Numeric session ids (JSON numbers) are coerced to strings.
|
||||
func sessionResourceKeys(args map[string]any) []string {
|
||||
return scopedIDResourceKeys(ResourceKeySession, args, "session", "session_id", "sessionId")
|
||||
}
|
||||
|
||||
// processResourceKeys extracts process: keys for retained process / terminal
|
||||
// tools. Accepts session, session_id, pid, and numeric JSON values so
|
||||
// write_stdin (integer session_id) and terminal_session (string session)
|
||||
// both produce keys.
|
||||
func processResourceKeys(args map[string]any) []string {
|
||||
return scopedIDResourceKeys(ResourceKeyProcess, args,
|
||||
"session", "session_id", "sessionId", "process_id", "pid", "id")
|
||||
}
|
||||
|
||||
// scopedIDResourceKeys is the shared extract/trim/bound/prefix path for
|
||||
// session- and process-scoped resource keys.
|
||||
func scopedIDResourceKeys(prefix string, args map[string]any, keys ...string) []string {
|
||||
id := strings.TrimSpace(firstStringArg(args, keys...))
|
||||
if id == "" {
|
||||
return nil
|
||||
}
|
||||
// Never put free-form content in keys — IDs only, bounded length.
|
||||
if len(id) > 128 {
|
||||
id = id[:128]
|
||||
}
|
||||
return []string{prefix + id}
|
||||
}
|
||||
|
||||
// workspaceResourceKeys returns a single workspace: root marker when the call
|
||||
// affects the whole workspace (e.g. glob without a path root).
|
||||
func workspaceResourceKeys(_ map[string]any) []string {
|
||||
return []string{ResourceKeyWorkspace + "root"}
|
||||
}
|
||||
|
||||
// multiFileResourceKeys collects path and paths[] arguments into file: keys.
|
||||
func multiFileResourceKeys(args map[string]any) []string {
|
||||
var keys []string
|
||||
keys = append(keys, fileResourceKeys(args)...)
|
||||
if raw, ok := args["paths"]; ok {
|
||||
switch typed := raw.(type) {
|
||||
case []string:
|
||||
for _, p := range typed {
|
||||
if n := NormalizeResourcePath(p); n != "" {
|
||||
keys = append(keys, ResourceKeyFile+n)
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, item := range typed {
|
||||
if s, ok := item.(string); ok {
|
||||
if n := NormalizeResourcePath(s); n != "" {
|
||||
keys = append(keys, ResourceKeyFile+n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return uniqueKeys(keys)
|
||||
}
|
||||
|
||||
// applyPatchResourceKeys derives conflict keys for apply_patch: the cwd as a
|
||||
// directory key, optional path/paths args, plus any +++/--- file paths
|
||||
// parseable from the unified diff. Relative diff paths are joined under cwd
|
||||
// so the same file under different cwd values does not collide (e.g. cwd=pkg
|
||||
// + +++ b/new.go → file:pkg/new.go). Falls back to workspace:root when nothing
|
||||
// is available so a patch call is never "keyless" for the future conflict planner.
|
||||
func applyPatchResourceKeys(args map[string]any) []string {
|
||||
var keys []string
|
||||
keys = append(keys, directoryResourceKeys(args)...)
|
||||
keys = append(keys, multiFileResourceKeys(args)...)
|
||||
cwd := firstStringArg(args, "cwd", "workdir")
|
||||
patch := firstStringArg(args, "patch", "diff")
|
||||
for _, p := range pathsFromUnifiedDiff(patch) {
|
||||
joined := joinUnderResourceCwd(cwd, p)
|
||||
if n := NormalizeResourcePath(joined); n != "" {
|
||||
keys = append(keys, ResourceKeyFile+n)
|
||||
}
|
||||
}
|
||||
keys = uniqueKeys(keys)
|
||||
if len(keys) == 0 {
|
||||
return workspaceResourceKeys(args)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// joinUnderResourceCwd joins a relative path under cwd for resource keys only.
|
||||
// Absolute paths are left unchanged. Empty cwd or "." leaves rel as-is.
|
||||
// Never touches the filesystem.
|
||||
func joinUnderResourceCwd(cwd, rel string) string {
|
||||
rel = strings.TrimSpace(rel)
|
||||
if rel == "" {
|
||||
return ""
|
||||
}
|
||||
// Absolute (Unix or Windows) paths are already workspace-rooted or outside;
|
||||
// do not re-prefix.
|
||||
if filepath.IsAbs(rel) {
|
||||
return rel
|
||||
}
|
||||
cwd = strings.TrimSpace(cwd)
|
||||
if cwd == "" || cwd == "." {
|
||||
return rel
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join(cwd, rel))
|
||||
}
|
||||
|
||||
// pathsFromUnifiedDiff extracts a/ and b/ file paths from unified-diff headers.
|
||||
// Malformed input returns nil; never panics.
|
||||
func pathsFromUnifiedDiff(patch string) []string {
|
||||
if strings.TrimSpace(patch) == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, line := range strings.Split(patch, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
switch {
|
||||
case strings.HasPrefix(line, "+++ ") || strings.HasPrefix(line, "--- "):
|
||||
rest := strings.TrimSpace(line[4:])
|
||||
// Drop trailing tab timestamp if present.
|
||||
if i := strings.IndexByte(rest, '\t'); i >= 0 {
|
||||
rest = rest[:i]
|
||||
}
|
||||
if rest == "/dev/null" || rest == "" {
|
||||
continue
|
||||
}
|
||||
// Strip a/ or b/ prefix common in git diffs.
|
||||
if len(rest) > 2 && (rest[0] == 'a' || rest[0] == 'b') && rest[1] == '/' {
|
||||
rest = rest[2:]
|
||||
}
|
||||
out = append(out, rest)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// firstStringArg returns the first present argument as a string. Numeric JSON
|
||||
// values (float64/int from encoding/json) are coerced so tools that declare
|
||||
// integer session ids still yield resource keys.
|
||||
func firstStringArg(args map[string]any, keys ...string) string {
|
||||
if args == nil {
|
||||
return ""
|
||||
}
|
||||
for _, key := range keys {
|
||||
value, ok := args[key]
|
||||
if !ok || value == nil {
|
||||
continue
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return typed
|
||||
case float64:
|
||||
// JSON numbers decode as float64. Prefer integer form when exact.
|
||||
if typed == float64(int64(typed)) {
|
||||
return fmt.Sprintf("%d", int64(typed))
|
||||
}
|
||||
return fmt.Sprintf("%v", typed)
|
||||
case float32:
|
||||
return fmt.Sprintf("%v", typed)
|
||||
case int:
|
||||
return fmt.Sprintf("%d", typed)
|
||||
case int64:
|
||||
return fmt.Sprintf("%d", typed)
|
||||
case int32:
|
||||
return fmt.Sprintf("%d", typed)
|
||||
case jsonNumber:
|
||||
return typed.String()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// jsonNumber matches encoding/json.Number without importing encoding/json here
|
||||
// for a one-line type assert (tests may inject it).
|
||||
type jsonNumber interface {
|
||||
String() string
|
||||
}
|
||||
|
||||
// resourceHost returns a lower-cased host for endpoint keys, stripping
|
||||
// userinfo, path, query, and fragment. Returns "" when not parseable safely.
|
||||
//
|
||||
// Order matters: path/query/fragment are removed BEFORE userinfo so a query
|
||||
// value containing "@" cannot steal the authority (e.g.
|
||||
// https://api.example.com/v1?token=@secret → endpoint:api.example.com).
|
||||
// host:port is preserved so two services on different ports do not collide.
|
||||
func resourceHost(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
// Strip scheme.
|
||||
if idx := strings.Index(raw, "://"); idx >= 0 {
|
||||
raw = raw[idx+3:]
|
||||
}
|
||||
// Isolate authority: strip path/query/fragment first.
|
||||
if slash := strings.IndexAny(raw, "/?#"); slash >= 0 {
|
||||
raw = raw[:slash]
|
||||
}
|
||||
// Strip userinfo from the authority only.
|
||||
if at := strings.LastIndex(raw, "@"); at >= 0 {
|
||||
raw = raw[at+1:]
|
||||
}
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" || strings.Contains(raw, "@") {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(raw)
|
||||
}
|
||||
|
||||
func uniqueKeys(keys []string) []string {
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(keys))
|
||||
out := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[k]; ok {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -48,7 +48,8 @@ func NewSkillTool(dir string) *skillTool {
|
||||
// unknown keys (AdditionalProperties:false).
|
||||
AdditionalProperties: false,
|
||||
},
|
||||
safety: readOnlySafety("Reads a local skill file; gathers reusable instructions only."),
|
||||
safety: readOnlySafety("Reads a local skill file; gathers reusable instructions only."),
|
||||
capabilities: ToolCapabilities{Effect: EffectReadOnly, ThreadSafe: false},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Loaded 100 of 115 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user