mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 09:49:34 -10:00
Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e666395d47 | ||
|
|
9043baedcf | ||
|
|
4d8c31cf16 | ||
|
|
824ecdbcf9 | ||
|
|
6b9c2f0c08 | ||
|
|
8ef8576df7 | ||
|
|
1c5c6e78a8 | ||
|
|
7f669f4550 | ||
|
|
2b42cd567b | ||
|
|
665bd4b3d0 | ||
|
|
9acb4113cc | ||
|
|
d66a9dda69 | ||
|
|
7d579996b4 | ||
|
|
abe24bcb10 | ||
|
|
75a78e715b | ||
|
|
8502449430 | ||
|
|
c0bcfdda41 | ||
|
|
1bb6b5745a |
No files matched your search
@@ -1 +1 @@
|
||||
{".":"0.3.0"}
|
||||
{".":"0.4.0"}
|
||||
+63
-4
@@ -6,6 +6,60 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
aims to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html) once the first release is
|
||||
tagged. Until then, source builds report the version `dev`.
|
||||
|
||||
## [0.4.0](https://github.com/Gitlawb/zero/compare/v0.3.0...v0.4.0) (2026-07-17)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **aimlapi:** AI/ML API provider with guided onboarding (top-up + key issuance) ([#655](https://github.com/Gitlawb/zero/issues/655)) ([6b9c2f0](https://github.com/Gitlawb/zero/commit/6b9c2f0c083e02bcd3aad68fb7af0501a9e7cd61))
|
||||
* **cli:** show ZERO wordmark on --version ([#673](https://github.com/Gitlawb/zero/issues/673)) ([9acb411](https://github.com/Gitlawb/zero/commit/9acb4113cc3337b3f361a16278e9cf11ca105e34))
|
||||
* **cli:** wire MCP serve WorkspaceRoot and --add-dir scope ([#694](https://github.com/Gitlawb/zero/issues/694)) ([75a78e7](https://github.com/Gitlawb/zero/commit/75a78e715bf23154f69c8c79cb58eb4b535b2a2a))
|
||||
* **npm:** ship the native binary as platform optionalDependencies ([#626](https://github.com/Gitlawb/zero/issues/626)) ([5e1405d](https://github.com/Gitlawb/zero/commit/5e1405d0b7abff5b3ccb3cfdb66d64d6d3322922))
|
||||
* **perf:** emit prompt-prefix hash fingerprint per turn ([#704](https://github.com/Gitlawb/zero/issues/704)) ([1c5c6e7](https://github.com/Gitlawb/zero/commit/1c5c6e78a8a0e228bdf53d7be90934fbce9d98c3))
|
||||
* **providers:** add AI/ML API preset (rebased onto main) ([#621](https://github.com/Gitlawb/zero/issues/621)) ([d66a9dd](https://github.com/Gitlawb/zero/commit/d66a9dda69c32aa59f4bea903cfefe00d4b7adef))
|
||||
* **providers:** refresh MiniMax model coverage ([#665](https://github.com/Gitlawb/zero/issues/665)) ([fa3052a](https://github.com/Gitlawb/zero/commit/fa3052a1422a4ad30a3a6295829564f42dee31a8))
|
||||
* **skills:** discover shared ~/.agents/skills with multi-root skill loading ([#696](https://github.com/Gitlawb/zero/issues/696)) ([7d57999](https://github.com/Gitlawb/zero/commit/7d579996b43741a2e57e3e38fcdd8484fcfc34e9))
|
||||
* **tui:** Ctrl+X leader chords and emacs menu navigation ([#699](https://github.com/Gitlawb/zero/issues/699)) ([7f669f4](https://github.com/Gitlawb/zero/commit/7f669f455021be51319ee3b8298cd48a17f745c7))
|
||||
* **tui:** press up to edit queued messages ([#656](https://github.com/Gitlawb/zero/issues/656)) ([4c986d3](https://github.com/Gitlawb/zero/commit/4c986d327b5ed26338d295c23bea5839681db8d0))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **acp:** make truncateHint rune-safe ([#614](https://github.com/Gitlawb/zero/issues/614)) ([ddc4927](https://github.com/Gitlawb/zero/commit/ddc4927aac5544bf4dd2c46615aeda5a81c96576))
|
||||
* **agent,tui:** resolve git branch detection when starting Zero in subdirectories ([#613](https://github.com/Gitlawb/zero/issues/613)) ([0184581](https://github.com/Gitlawb/zero/commit/0184581ec234ea414e0a47bc33e8b0f4ddfb497b))
|
||||
* **agent:** raise default and deep-mode turn budgets ([#650](https://github.com/Gitlawb/zero/issues/650)) ([635c93a](https://github.com/Gitlawb/zero/commit/635c93af51ebc20f3e0917917e55a79edfe27c35))
|
||||
* **cli:** prevent consuming positional arguments as flag values ([#619](https://github.com/Gitlawb/zero/issues/619)) ([5b4f48d](https://github.com/Gitlawb/zero/commit/5b4f48d2dcb66402c13bde0c3cfe9c9371da19fb))
|
||||
* **config:** enforce MCP trust boundary so project config cannot override user disable ([#609](https://github.com/Gitlawb/zero/issues/609)) ([4d8c31c](https://github.com/Gitlawb/zero/commit/4d8c31cf16a3080344e3be7039408fcae71c075d)), closes [#512](https://github.com/Gitlawb/zero/issues/512)
|
||||
* **config:** surface unknown/typo'd config fields instead of silently dropping them ([#645](https://github.com/Gitlawb/zero/issues/645)) ([893b7b4](https://github.com/Gitlawb/zero/commit/893b7b424cc203a2fcf92327a4e25c84286a90e0))
|
||||
* **cron:** prevent cron job Mutate from clobbering concurrent updates ([#630](https://github.com/Gitlawb/zero/issues/630)) ([e4bd703](https://github.com/Gitlawb/zero/commit/e4bd703cfb28dab2dfa3c2ddba46237e1bb2e164))
|
||||
* **daemon:** handle os.ErrPermission as collision during O_EXCL lock creation ([#616](https://github.com/Gitlawb/zero/issues/616)) ([8ea5384](https://github.com/Gitlawb/zero/commit/8ea53841a5d54c37a153779ae86bea010659433c))
|
||||
* **exec:** stop false INCOMPLETE downgrades on conversational final messages ([#608](https://github.com/Gitlawb/zero/issues/608)) ([b6117af](https://github.com/Gitlawb/zero/commit/b6117af86d6bc87a4ee66910e99d76cb16b03fed))
|
||||
* harden MCP credential boundaries ([#597](https://github.com/Gitlawb/zero/issues/597)) ([fdddb05](https://github.com/Gitlawb/zero/commit/fdddb05ba84b1600ae6c3a20028bf83afe474c44))
|
||||
* **hooks:** fail closed on launch failures for beforeTool hooks ([#629](https://github.com/Gitlawb/zero/issues/629)) ([dc06fe7](https://github.com/Gitlawb/zero/commit/dc06fe72caf45f72d2cba1e8a835c0f5b405c1e8))
|
||||
* **hooks:** run sessionEnd hooks after Esc/Ctrl+C interrupts ([#606](https://github.com/Gitlawb/zero/issues/606)) ([824ecdb](https://github.com/Gitlawb/zero/commit/824ecdbcf9c467c35ef4e2666770fdadcb5bf402))
|
||||
* **keyring:** pass generic password via stdin on macOS ([#574](https://github.com/Gitlawb/zero/issues/574)) ([91ea6de](https://github.com/Gitlawb/zero/commit/91ea6ded7503538834a84d090f78670a363c62d3))
|
||||
* **lock:** prevent POSIX lock file overwrite and leak on Windows/Unix ([#628](https://github.com/Gitlawb/zero/issues/628)) ([da41c3a](https://github.com/Gitlawb/zero/commit/da41c3a75b782d6e0836fe13346321e40a90fbb4))
|
||||
* **openai:** omit prompt_cache_key for openai-compatible providers ([#636](https://github.com/Gitlawb/zero/issues/636)) ([1af5882](https://github.com/Gitlawb/zero/commit/1af58828eb3c22567599c000736c913a290959d2)), closes [#624](https://github.com/Gitlawb/zero/issues/624)
|
||||
* **plugins:** resolve relative executable paths against plugin root ([#627](https://github.com/Gitlawb/zero/issues/627)) ([2efe6d5](https://github.com/Gitlawb/zero/commit/2efe6d539e29374b3ef39c2290bdea81f33a228b))
|
||||
* **sandbox:** remove windowsWriteRestricted flag to fix DenyRead bypass ([#612](https://github.com/Gitlawb/zero/issues/612)) ([3d96ac7](https://github.com/Gitlawb/zero/commit/3d96ac7e55c760a97f28c0e6ceaf1ec3b4ab717a))
|
||||
* **sandbox:** scrub dynamic credential env vars ([#682](https://github.com/Gitlawb/zero/issues/682)) ([9043bae](https://github.com/Gitlawb/zero/commit/9043baedcff7776c7373645b57563cac06b31847))
|
||||
* **sandbox:** scrub sensitive credentials from sandbox environment ([#660](https://github.com/Gitlawb/zero/issues/660)) ([6fc1220](https://github.com/Gitlawb/zero/commit/6fc1220f6ac66fb3ae67b637cbbed7068d2213c0))
|
||||
* **sandbox:** unblock git fetch/commit/add under the write-restricted sandbox ([#654](https://github.com/Gitlawb/zero/issues/654)) ([5c4815a](https://github.com/Gitlawb/zero/commit/5c4815a66ed07d9cf90b825adfd936d3ac07639d))
|
||||
* **sandbox:** use WRITE_RESTRICTED token when no DenyRead paths are configured ([#658](https://github.com/Gitlawb/zero/issues/658)) ([a5d2e32](https://github.com/Gitlawb/zero/commit/a5d2e327c8681671aa8a9e5378801215b747edcf))
|
||||
* **securefile,credstore:** call Sync on temp file before close and rename ([#631](https://github.com/Gitlawb/zero/issues/631)) ([212734a](https://github.com/Gitlawb/zero/commit/212734adf3b5f982e22385161205aa1afe4634fe))
|
||||
* **securefile:** reclaim stale lock files to prevent permanent DOS ([#615](https://github.com/Gitlawb/zero/issues/615)) ([8536cc8](https://github.com/Gitlawb/zero/commit/8536cc87f7a885f9e436d6ef28f5f325201623dc))
|
||||
* **swarm:** wait for job.Runs directly in scheduler skip test ([#667](https://github.com/Gitlawb/zero/issues/667)) ([1bb6b57](https://github.com/Gitlawb/zero/commit/1bb6b5745af90321d1e657a12a1976cded5dd1bd))
|
||||
* **tools:** classify silent wrapped Windows command failures as sandbox denials ([#659](https://github.com/Gitlawb/zero/issues/659)) ([8bd9742](https://github.com/Gitlawb/zero/commit/8bd9742fa95c41b93ea4e718628aed0ff3ae9dd0))
|
||||
* **tools:** preserve SysProcAttr during PTY fallback ([#618](https://github.com/Gitlawb/zero/issues/618)) ([f78b36c](https://github.com/Gitlawb/zero/commit/f78b36c770daa4577a9f99265b18a354454e36eb))
|
||||
* **tui:** resolve pending askUser callbacks to prevent runner hangs ([#620](https://github.com/Gitlawb/zero/issues/620)) ([aa73a76](https://github.com/Gitlawb/zero/commit/aa73a76f1bd1b6fe97bac2fbff2d61b7474139f2))
|
||||
* **tui:** stop the composer cursor blinking while typing or unfocused ([#672](https://github.com/Gitlawb/zero/issues/672)) ([2b42cd5](https://github.com/Gitlawb/zero/commit/2b42cd567b96d6f7e0818594e53ff31cce1e42e9))
|
||||
* **windows:** resolve absolute path for taskkill to prevent hijacking ([#617](https://github.com/Gitlawb/zero/issues/617)) ([2db00ee](https://github.com/Gitlawb/zero/commit/2db00ee3d57db97e0fbe23cb8628e8bcb47f6f09))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* **tools:** add explicit effect metadata for safe concurrency ([#705](https://github.com/Gitlawb/zero/issues/705)) ([8ef8576](https://github.com/Gitlawb/zero/commit/8ef8576df7d0775a5b815bc0776a794cadb75c34))
|
||||
|
||||
## [0.3.0](https://github.com/Gitlawb/zero/compare/v0.2.0...v0.3.0) (2026-07-09)
|
||||
|
||||
|
||||
@@ -127,14 +181,19 @@ tagged. Until then, source builds report the version `dev`.
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- Shared multi-agent skills discovery: when present, `~/.agents/skills` is searched after the primary
|
||||
Zero skills dir (and before plugin skill roots). `zero skills list` / `info` and the runtime `skill`
|
||||
tool share one multi-root discovery path; install/remove/lock still target only the Zero skills directory.
|
||||
- `SECURITY.md` with a private vulnerability-reporting path, `CODE_OF_CONDUCT.md`, this changelog, and
|
||||
GitHub issue/PR templates.
|
||||
- Interactive `/theme` picker: bare `/theme` opens a popup that live-previews each palette as you move
|
||||
and applies on select (Esc reverts).
|
||||
- Ten built-in color themes alongside the `dark`/`light` built-ins — `dracula`, `nord`, `gruvbox`,
|
||||
`tokyo-night`, `catppuccin`, `one-dark`, `solarized-dark`, `rose-pine`, `everforest`, and
|
||||
`solarized-light` — selectable via `/theme <name>`, `--theme <name>`, or `ZERO_THEME`. Every palette
|
||||
is contrast-audited to WCAG AA. The built-in light theme was reworked for legibility.
|
||||
- Twelve built-in color themes alongside the `dark`/`light` built-ins — `dracula`, `nord`, `gruvbox`,
|
||||
`tokyo-night`, `catppuccin`, `one-dark`, `solarized-dark`, `rose-pine`, `everforest`,
|
||||
`solarized-light`, `dune`, and `neon` — selectable via `/theme <name>`, `--theme <name>`, or
|
||||
`ZERO_THEME`. Every palette is contrast-audited to WCAG AA, and the new presets are additionally
|
||||
audited after xterm-256 downsampling; see [docs/THEMES.md](docs/THEMES.md). The built-in light
|
||||
theme was reworked for legibility.
|
||||
- `--theme <name>` flag for the TUI, accepting `auto` or any registered theme (previously only the
|
||||
`ZERO_THEME` env var existed).
|
||||
- "Accessibility / Appearance" section in the README documenting `NO_COLOR`, `ZERO_THEME`, `/theme`,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Zero build/test/lint targets. AGENTS.md says "Build with `make`" and "Run `make
|
||||
# lint` before opening a PR" — these targets back those instructions.
|
||||
.DEFAULT_GOAL := build
|
||||
.PHONY: build build-all test test-race vet fmt fmt-check lint tidy clean help
|
||||
.PHONY: build build-all test test-race vet fmt fmt-check lint tidy clean baseline help
|
||||
|
||||
# Build the main CLI binary into ./zero.
|
||||
build:
|
||||
@@ -40,5 +40,19 @@ clean:
|
||||
rm -f zero
|
||||
go clean ./...
|
||||
|
||||
# Run the per-turn benchmark harness over the checked-in baseline manifest and
|
||||
# write the JSON result to internal/perfbench/reports/baseline.json. Requires a
|
||||
# built `zero` binary and a model; set ZERO_BENCH_MODEL (required) and
|
||||
# ZERO_BENCH_BINARY (defaults to ./zero) to configure the run. The report is
|
||||
# machine-specific and regenerated, not hand-edited.
|
||||
baseline: build
|
||||
@if [ -z "$(ZERO_BENCH_MODEL)" ]; then echo "Set ZERO_BENCH_MODEL (and optionally ZERO_BENCH_BINARY) before running 'make baseline'"; exit 2; fi
|
||||
@ZERO_BIN="$${ZERO_BENCH_BINARY:-./zero}"; \
|
||||
go run ./cmd/zero-perf-bench turn \
|
||||
--suite internal/perfbench/manifests/baseline.json \
|
||||
--model $(ZERO_BENCH_MODEL) \
|
||||
--binary "$$ZERO_BIN" \
|
||||
--output internal/perfbench/reports/baseline.json
|
||||
|
||||
help:
|
||||
@echo "Targets: build (default), build-all, test, test-quick, vet, fmt, fmt-check, lint, tidy, clean"
|
||||
@echo "Targets: build (default), build-all, test, test-quick, vet, fmt, fmt-check, lint, tidy, clean, baseline"
|
||||
@@ -129,11 +129,18 @@ the key in the wizard:
|
||||
export OPENAI_API_KEY=sk-...
|
||||
export ANTHROPIC_API_KEY=...
|
||||
export GEMINI_API_KEY=...
|
||||
export AIMLAPI_API_KEY=...
|
||||
export LONGCAT_API_KEY=...
|
||||
export MINIMAX_API_KEY=...
|
||||
export MINIMAXI_API_KEY=...
|
||||
```
|
||||
|
||||
To configure AI/ML API directly, run:
|
||||
|
||||
```bash
|
||||
zero providers setup aimlapi --set-active
|
||||
```
|
||||
|
||||
To configure Meituan LongCat (LongCat-2.0) directly, run:
|
||||
|
||||
```bash
|
||||
@@ -184,6 +191,8 @@ Useful controls:
|
||||
|---|---|
|
||||
| `Enter` | send the prompt |
|
||||
| `/` | open slash-command suggestions |
|
||||
| `Ctrl+X` then letter | common slash commands (e.g. `m` → `/model`; `Ctrl+X` `?` for full list) |
|
||||
| `Ctrl+P` / `Ctrl+N` | previous / next item in menus (arrows still work) |
|
||||
| `Shift+Tab` | cycle permission mode |
|
||||
| `Ctrl+B` | show/hide the sidebar |
|
||||
| `Ctrl+C` | cancel or exit |
|
||||
@@ -332,7 +341,7 @@ manifest.
|
||||
| Control | Effect |
|
||||
|---|---|
|
||||
| `NO_COLOR=<anything>` | disables color output |
|
||||
| `ZERO_THEME=<name>` | selects the startup theme (`auto`, `dark`, `light`, or a color theme like `dracula`, `nord`, `gruvbox`, `tokyo-night`, `catppuccin`, `one-dark`, `solarized-dark`, `rose-pine`, `everforest`, `solarized-light`) |
|
||||
| `ZERO_THEME=<name>` | selects the startup theme (`auto`, `dark`, `light`, or a color theme like `dracula`, `nord`, `gruvbox`, `tokyo-night`, `catppuccin`, `one-dark`, `solarized-dark`, `rose-pine`, `everforest`, `neon`, `solarized-light`, `dune`) |
|
||||
| `--theme <name>` | selects the TUI theme from the CLI (same names) |
|
||||
| `/theme` | opens the theme picker inside the TUI (live preview; `/theme <name>` switches directly) |
|
||||
| `ZERO_NO_FADE=1` | disables streaming fade animation |
|
||||
@@ -379,6 +388,7 @@ go run ./cmd/zero-release build --goos windows --goarch amd64 --output dist/zero
|
||||
|
||||
- [Install](docs/INSTALL.md)
|
||||
- [Update flow](docs/UPDATE.md)
|
||||
- [Themes](docs/THEMES.md)
|
||||
- [Stream-JSON protocol](docs/STREAM_JSON_PROTOCOL.md)
|
||||
- [Specialists](docs/SPECIALISTS.md)
|
||||
- [GitHub Action](docs/GITHUB_ACTION.md)
|
||||
|
||||
+2
-1
@@ -234,7 +234,7 @@ zero update 检查更新版本
|
||||
| 控制 | 效果 |
|
||||
|---|---|
|
||||
| `NO_COLOR=<任意值>` | 禁用颜色输出 |
|
||||
| `ZERO_THEME=<名称>` | 选择启动主题(`auto`、`dark`、`light`,或颜色主题如 `dracula`、`nord`、`gruvbox`、`tokyo-night`、`catppuccin`、`one-dark`、`solarized-dark`、`rose-pine`、`everforest`、`solarized-light`) |
|
||||
| `ZERO_THEME=<名称>` | 选择启动主题(`auto`、`dark`、`light`,或颜色主题如 `dracula`、`nord`、`gruvbox`、`tokyo-night`、`catppuccin`、`one-dark`、`solarized-dark`、`rose-pine`、`everforest`、`neon`、`solarized-light`、`dune`) |
|
||||
| `--theme <名称>` | 从 CLI 选择 TUI 主题(相同名称) |
|
||||
| `/theme` | 在 TUI 中打开主题选择器(实时预览;`/theme <名称>` 直接切换) |
|
||||
| `ZERO_NO_FADE=1` | 禁用流式淡入动画 |
|
||||
@@ -261,6 +261,7 @@ go run ./cmd/zero-release build --goos windows --goarch amd64 --output dist/zero
|
||||
|
||||
- [安装](docs/INSTALL.md)
|
||||
- [更新流程](docs/UPDATE.md)
|
||||
- [主题](docs/THEMES.md)
|
||||
- [Stream-JSON 协议](docs/STREAM_JSON_PROTOCOL.md)
|
||||
- [专家](docs/SPECIALISTS.md)
|
||||
- [GitHub Action](docs/GITHUB_ACTION.md)
|
||||
|
||||
@@ -39,6 +39,9 @@ func run(args []string, getenv func(string) string, stdout io.Writer, stderr io.
|
||||
if len(args) > 0 && args[0] == "tasks" {
|
||||
return runTasksCommand(args[1:], getenv, stdout, stderr)
|
||||
}
|
||||
if len(args) > 0 && args[0] == "turn" {
|
||||
return runTurnCommand(args[1:], getenv, stdout, stderr)
|
||||
}
|
||||
options, err := parseArgs(args, getenv)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, err.Error())
|
||||
@@ -206,6 +209,7 @@ func helpText() string {
|
||||
return strings.Join([]string{
|
||||
"Usage: zero-perf-bench [options]",
|
||||
" zero-perf-bench tasks [options] (Terminal-Bench-style task harness; see `tasks --help`)",
|
||||
" zero-perf-bench turn [options] (per-turn tracing benchmark; see `turn --help`)",
|
||||
"",
|
||||
"Options:",
|
||||
" --iterations <n> Measured samples to collect (default: 5)",
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/perfbench"
|
||||
)
|
||||
|
||||
// turnOptions configures the `zero-perf-bench turn` subcommand: the per-turn
|
||||
// benchmark harness that runs ZERO headlessly with --trace, parses each turn's
|
||||
// NDJSON trace, and records per-span latency plus the top controllable latency
|
||||
// sources — the Phase 0 baseline's "do not proceed until" criterion.
|
||||
type turnOptions struct {
|
||||
SuitePath string
|
||||
Model string
|
||||
Mode string
|
||||
SelfCorrect bool
|
||||
Binary string
|
||||
Iterations int
|
||||
Version string
|
||||
Commit string
|
||||
Output string
|
||||
JSON bool
|
||||
DryRun bool
|
||||
Help bool
|
||||
}
|
||||
|
||||
func runTurnCommand(args []string, getenv func(string) string, stdout io.Writer, stderr io.Writer) int {
|
||||
options, err := parseTurnArgs(args, getenv)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, err.Error())
|
||||
return 2
|
||||
}
|
||||
if options.Help {
|
||||
_, _ = fmt.Fprint(stdout, turnHelpText())
|
||||
return 0
|
||||
}
|
||||
|
||||
set, err := perfbench.LoadTaskSet(options.SuitePath)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 1
|
||||
}
|
||||
|
||||
// The dry-run path records a zero-iteration run without a binary, so the
|
||||
// manifest loads and the report path is exercised in CI without a model.
|
||||
if options.DryRun {
|
||||
_, _ = fmt.Fprintln(stdout, "[zero] turn benchmark: dry run (no agent invoked)")
|
||||
return 0
|
||||
}
|
||||
|
||||
binary, err := perfbench.ResolveBinary(options.Binary)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 2
|
||||
}
|
||||
|
||||
result, err := perfbench.RunTurnBench(context.Background(), set, perfbench.TurnBenchConfig{
|
||||
Model: options.Model,
|
||||
Mode: options.Mode,
|
||||
SelfCorrect: options.SelfCorrect,
|
||||
Version: options.Version,
|
||||
Commit: options.Commit,
|
||||
Iterations: options.Iterations,
|
||||
Runner: perfbench.NewTurnExecRunner(binary),
|
||||
})
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 1
|
||||
}
|
||||
|
||||
if options.Output != "" {
|
||||
if err := writeTurnReport(options.Output, result); err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 1
|
||||
}
|
||||
}
|
||||
if options.JSON {
|
||||
if err := perfbench.WriteTurnBenchJSON(stdout, result); err != nil {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
_, _ = fmt.Fprintln(stdout, perfbench.FormatTurnBenchSummary(result))
|
||||
return 0
|
||||
}
|
||||
|
||||
func parseTurnArgs(args []string, getenv func(string) string) (turnOptions, error) {
|
||||
options := turnOptions{
|
||||
Iterations: 1,
|
||||
Version: strings.TrimSpace(getenv("ZERO_BENCH_VERSION")),
|
||||
Commit: strings.TrimSpace(getenv("ZERO_BENCH_COMMIT")),
|
||||
}
|
||||
for index := 0; index < len(args); index++ {
|
||||
arg := args[index]
|
||||
flag, inlineValue := splitFlagValue(arg)
|
||||
switch flag {
|
||||
case "--suite":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.SuitePath = value
|
||||
index = next
|
||||
case "--model":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Model = value
|
||||
index = next
|
||||
case "--mode":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Mode = value
|
||||
index = next
|
||||
case "--binary":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Binary = value
|
||||
index = next
|
||||
case "--iterations":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
parsed, err := parsePositiveInteger(flag, value)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Iterations = parsed
|
||||
index = next
|
||||
case "--version":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Version = value
|
||||
index = next
|
||||
case "--commit":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Commit = value
|
||||
index = next
|
||||
case "--output":
|
||||
value, next, err := readOptionValue(args, inlineValue, index, flag)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Output = value
|
||||
index = next
|
||||
case "--self-correct":
|
||||
if strings.Contains(arg, "=") {
|
||||
return options, fmt.Errorf("%s does not accept a value", flag)
|
||||
}
|
||||
options.SelfCorrect = true
|
||||
case "--json":
|
||||
if strings.Contains(arg, "=") {
|
||||
return options, fmt.Errorf("%s does not accept a value", flag)
|
||||
}
|
||||
options.JSON = true
|
||||
case "--dry-run":
|
||||
if strings.Contains(arg, "=") {
|
||||
return options, fmt.Errorf("%s does not accept a value", flag)
|
||||
}
|
||||
options.DryRun = true
|
||||
case "-h", "--help":
|
||||
if strings.Contains(arg, "=") {
|
||||
return options, fmt.Errorf("%s does not accept a value", flag)
|
||||
}
|
||||
options.Help = true
|
||||
default:
|
||||
return options, fmt.Errorf("unknown option: %s", arg)
|
||||
}
|
||||
}
|
||||
if options.Help {
|
||||
return options, nil
|
||||
}
|
||||
if strings.TrimSpace(options.SuitePath) == "" {
|
||||
return options, fmt.Errorf("--suite is required")
|
||||
}
|
||||
if strings.TrimSpace(options.Model) == "" && !options.DryRun {
|
||||
return options, fmt.Errorf("--model is required (or pass --dry-run)")
|
||||
}
|
||||
return options, nil
|
||||
}
|
||||
|
||||
func writeTurnReport(path string, result perfbench.TurnBenchResult) error {
|
||||
if dir := filepath.Dir(path); dir != "" {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
if err := perfbench.WriteTurnBenchJSON(&buffer, result); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, buffer.Bytes(), 0o644)
|
||||
}
|
||||
|
||||
func turnHelpText() string {
|
||||
return strings.Join([]string{
|
||||
"Usage: zero-perf-bench turn [options]",
|
||||
"",
|
||||
"Runs ZERO headlessly with --trace against a per-turn benchmark task set and",
|
||||
"records per-span latency plus the top controllable latency sources (the",
|
||||
"Phase 0 baseline's \"do not proceed until\" criterion). Each task is a fresh",
|
||||
"`zero exec` process, so iterations are cold-start samples; a warm path needs",
|
||||
"an in-process runner (future work).",
|
||||
"",
|
||||
"Options:",
|
||||
" --suite <path> Task set JSON file (required)",
|
||||
" --model <model> Model to run (required unless --dry-run)",
|
||||
" --mode <name> Exec mode preset to apply",
|
||||
" --self-correct Enable the post-edit verify-and-correct loop",
|
||||
" --binary <path> Path to the `zero` binary (default: zero on PATH / repo root)",
|
||||
" --iterations <n> Times to run each task (default: 1)",
|
||||
" --version <v> Record the ZERO version (default: $ZERO_BENCH_VERSION)",
|
||||
" --commit <sha> Record the ZERO commit (default: $ZERO_BENCH_COMMIT)",
|
||||
" --output <path> Write the JSON result to path",
|
||||
" --json Print only the JSON result",
|
||||
" --dry-run Load the manifest and exit without invoking the agent",
|
||||
" -h, --help Show this help",
|
||||
}, "\n") + "\n"
|
||||
}
|
||||
+17
-4
@@ -102,14 +102,24 @@ The full format spec (frontmatter fields, tool scopes, prompt conventions) is in
|
||||
|
||||
Skills are markdown instruction packs the agent can pull in on demand. Each skill is a directory containing a `SKILL.md`. Standalone project skill directories are not supported in this version (shared project-wide skills must go in `AGENTS.md` or as a hook). However, project plugins (section 6) may bundle skills, which are merged into the active run.
|
||||
|
||||
Discovery root: `$ZERO_SKILLS_DIR` → `$XDG_DATA_HOME/zero/skills` → `~/.local/share/zero/skills/`. A missing directory is fine — Zero just reports "no skills".
|
||||
Discovery roots (earlier wins on name collisions):
|
||||
|
||||
1. **Primary Zero dir** — `$ZERO_SKILLS_DIR` if set, else `$XDG_DATA_HOME/zero/skills`, else `~/.local/share/zero/skills/`
|
||||
2. **Shared multi-agent dir** — `~/.agents/skills/` when present (read-only discovery; never an install target)
|
||||
3. **Plugin skill roots** — skills bundled by active plugins (section 6)
|
||||
|
||||
A missing directory is fine — Zero just omits it. Management commands (`zero skills add` / `remove` / `lock`) always write to the primary Zero dir only; `list` / `info` search primary + `~/.agents/skills`.
|
||||
|
||||
```text
|
||||
~/.local/share/zero/skills/
|
||||
~/.local/share/zero/skills/ # primary (install/remove/lock target)
|
||||
run-benchmarks/
|
||||
SKILL.md
|
||||
write-changelog/
|
||||
SKILL.md
|
||||
|
||||
~/.agents/skills/ # optional shared multi-agent root
|
||||
shared-review/
|
||||
SKILL.md
|
||||
```
|
||||
|
||||
`SKILL.md` format:
|
||||
@@ -126,9 +136,9 @@ description: Run the project's benchmark suite and summarize the deltas.
|
||||
3. Report any regression > 5% with the function name and the previous value.
|
||||
```
|
||||
|
||||
Only `name` and `description` are recognized in the frontmatter today. The `name` defaults to the directory name. Within a single skills root, duplicate names are resolved by lexicographic directory order. During an agent run, Zero loads the default skills directory before plugin skill roots; earlier roots win name collisions silently. Plugin-declared skills (section 6) are merged into the active agent run at plugin activation time, so bundled skills appear in the available skills list and can be loaded with the `skill` tool.
|
||||
Only `name` and `description` are recognized in the frontmatter today. The `name` defaults to the directory name. Within a single skills root, duplicate names are resolved by lexicographic directory order. Across roots, Zero loads the primary directory first, then `~/.agents/skills`, then plugin skill roots; earlier roots win name collisions. Plugin-declared skills (section 6) are merged into the active agent run at plugin activation time, so bundled skills appear in the available skills list and can be loaded with the `skill` tool.
|
||||
|
||||
The `skill` core tool lets the agent load any discovered skill by name.
|
||||
The `skill` tool lets the agent load any discovered skill by name (primary, agents, or plugin).
|
||||
|
||||
## 4. Hooks
|
||||
|
||||
@@ -240,10 +250,13 @@ OAuth server names ending in `.<32 hex chars>` are reserved for token storage.
|
||||
|
||||
```bash
|
||||
zero serve --mcp
|
||||
zero serve --mcp -C /path/to/workspace
|
||||
zero serve --mcp --add-dir /path/to/extra
|
||||
```
|
||||
|
||||
The server speaks MCP over stdio. Configure it from the receiving side as a `stdio` server whose command is `zero serve --mcp`.
|
||||
|
||||
`-C/--cwd` sets the workspace root exposed as MCP resources and used by core tools. `--add-dir` (repeatable) widens that resource/tool scope beyond the workspace without granting the sandbox temp roots used by interactive runs.
|
||||
## 6. Plugins
|
||||
|
||||
A plugin is a self-contained directory that bundles tools, hooks, and skills for one capability. Plugins live at:
|
||||
|
||||
@@ -733,9 +733,10 @@ flowchart TD
|
||||
- **Skills** are instruction packs the model can load on demand.
|
||||
- **Specialists** are sub-agents callable through the `Task` tool.
|
||||
- **MCP servers** contribute external tools.
|
||||
- **Plugins** can add tools, hooks, and skill roots. Bootstrap merges plugin skill
|
||||
roots into the runtime skill tool and prompt-visible skill list; the base
|
||||
`internal/skills` scanner still scans one root at a time.
|
||||
- **Plugins** can add tools, hooks, and skill roots. Bootstrap always registers a
|
||||
multi-root skill tool: primary Zero skills dir, optional `~/.agents/skills`,
|
||||
then plugin skill roots (earlier wins). `internal/skills` owns that merge via
|
||||
`LoadFromRoots` / `DiscoveryRoots`; single-root `Load` remains for install/write.
|
||||
- **Hooks** can observe or block tool lifecycle events.
|
||||
|
||||
## End-to-End Data Flow
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# TUI Themes
|
||||
|
||||
Zero's TUI ships a set of built-in color themes. Pick one with `--theme <name>`,
|
||||
the `ZERO_THEME` environment variable, or the `/theme <name>` command while
|
||||
running. `auto` (the default) follows the terminal's detected background.
|
||||
|
||||
Run `/theme` with no argument to open a picker: move through the list to
|
||||
live-preview each theme, press Enter to apply the highlighted one, or Esc to
|
||||
cancel and restore the previously active theme. Run `/theme list` to print
|
||||
the active theme and the registered names without opening the picker.
|
||||
|
||||
## Dune (`dune`)
|
||||
|
||||
A warm sand-and-cream palette: sand/cream surface, charcoal ink, and a soft
|
||||
amber accent.
|
||||
|
||||
## Neon (`neon`)
|
||||
|
||||
A neon-on-black palette: pitch-black surface, bright green ink, and a cyan accent.
|
||||
|
||||
## Adding a theme
|
||||
|
||||
Every theme is a `palette` (a table of color hex tokens) plus one entry in
|
||||
`themeRegistry`, both in `internal/tui/theme_palettes.go`. `buildTheme` in
|
||||
`internal/tui/theme.go` turns a `palette` into the resolved `lipgloss.Style`
|
||||
set every renderer reads from the active `zeroTheme`. Adding a theme means
|
||||
adding a new `palette{...}` literal, a `themeRegistry` entry, and test
|
||||
coverage for the new palette (see below).
|
||||
|
||||
Registry-wide tests in `internal/tui/theme_select_test.go` assert the basic
|
||||
WCAG AA text tokens, the gray-ramp order, the diff word-span pairs, and the
|
||||
selected-row band for every entry. The rendered-surface invariants beyond
|
||||
those (permission surfaces, selected-row secondary text, diff gutters, and
|
||||
the xterm-256 downsampling checks) are asserted per palette, not against the
|
||||
whole registry: `TestExtendedThemeContrastInvariants` and
|
||||
`TestExtendedThemeANSI256Contrast` enumerate the palettes they cover. A new
|
||||
theme must be added to those tests (or given equivalent palette-specific
|
||||
assertions), or CI can stay green while its permission, selected-row, and
|
||||
diff surfaces ship unreadable.
|
||||
@@ -3,9 +3,9 @@ module github.com/Gitlawb/zero
|
||||
go 1.26.5
|
||||
|
||||
require (
|
||||
charm.land/bubbles/v2 v2.1.0
|
||||
charm.land/bubbletea/v2 v2.0.7
|
||||
charm.land/lipgloss/v2 v2.0.4
|
||||
charm.land/bubbles/v2 v2.1.1
|
||||
charm.land/bubbletea/v2 v2.0.8
|
||||
charm.land/lipgloss/v2 v2.0.5
|
||||
github.com/alecthomas/chroma/v2 v2.27.0
|
||||
github.com/atotto/clipboard v0.1.4
|
||||
github.com/aymanbagabas/go-udiff v0.4.1
|
||||
@@ -14,18 +14,18 @@ require (
|
||||
github.com/charmbracelet/x/term v0.2.2
|
||||
github.com/coder/websocket v1.8.15
|
||||
github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
|
||||
golang.org/x/sys v0.46.0
|
||||
golang.org/x/sys v0.47.0
|
||||
mvdan.cc/sh/v3 v3.13.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260615092913-2399af76d5b1 // indirect
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260703014108-f5a850f9c2b7 // indirect
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20260615092313-b57e5e6d29bb // indirect
|
||||
github.com/charmbracelet/x/termios v0.1.1 // indirect
|
||||
github.com/charmbracelet/x/windows v0.2.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.11.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.5.0 // indirect
|
||||
github.com/go-quicktest/qt v1.102.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.24 // indirect
|
||||
@@ -34,5 +34,5 @@ require (
|
||||
github.com/rogpeppe/go-internal v1.15.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
|
||||
golang.org/x/sync v0.21.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
)
|
||||
@@ -1,9 +1,9 @@
|
||||
charm.land/bubbles/v2 v2.1.0 h1:YSnNh5cPYlYjPxRrzs5VEn3vwhtEn3jVGRBT3M7/I0g=
|
||||
charm.land/bubbles/v2 v2.1.0/go.mod h1:l97h4hym2hvWBVfmJDtrEHHCtkIKeTEb3TTJ4ZOB3wY=
|
||||
charm.land/bubbletea/v2 v2.0.7 h1:7qw2tTAVar7m7klOPBYfTB0mniv/RuexsYwMRNxSeL0=
|
||||
charm.land/bubbletea/v2 v2.0.7/go.mod h1:DGW2q8gvzHnOpMpZTORs0aySVHCox5C+2Svk0fci1qs=
|
||||
charm.land/lipgloss/v2 v2.0.4 h1:lcPeVtcp23SNra7lHy8iYE4UC2aIipVQ47sbGyyxR5Q=
|
||||
charm.land/lipgloss/v2 v2.0.4/go.mod h1:0653x8epbZSzdDfO/XPS1a/uYPOBeSsCssOpJOqDzik=
|
||||
charm.land/bubbles/v2 v2.1.1 h1:7r55WzBxpo/R3z98hGmY7KKPd3ET6vsf0Fb9sDHOV60=
|
||||
charm.land/bubbles/v2 v2.1.1/go.mod h1:GE6M31gaWZVXzGw73OeuTTgy4lX+OtkH0E5ymnNsHxo=
|
||||
charm.land/bubbletea/v2 v2.0.8 h1:SxTJMhCAI3lbPmy4SgX5LWZ24AdINr4I6UEqzZvYJuY=
|
||||
charm.land/bubbletea/v2 v2.0.8/go.mod h1:2SkdgoTXluXJHOUwAoRlRXF/28vklb1rFl6GcgV1/ss=
|
||||
charm.land/lipgloss/v2 v2.0.5 h1:kbNxgeeUOYv5J0YdpxFjfvf3dFvqH8Aci4zB6xqFtrY=
|
||||
charm.land/lipgloss/v2 v2.0.5/go.mod h1:9oqhxt4yxIMe6q5A4kHr44DremZk7J9UNh74GlWa5nc=
|
||||
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
|
||||
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r5BXYZs=
|
||||
@@ -16,8 +16,8 @@ github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ
|
||||
github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w=
|
||||
github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
|
||||
github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260615092913-2399af76d5b1 h1:4+r3uOJ69ueRBt4okgEfWZeXs3BD36HcDBmOIAUlETk=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260615092913-2399af76d5b1/go.mod h1:f/jRa757WUmaOZrbPspXymbg/GnbF+rwe4OLsG7aXYo=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260703014108-f5a850f9c2b7 h1:3FmWoGNWK4STvqg0O0Aeav2T7rodWJAPeF0QpH+8gFw=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260703014108-f5a850f9c2b7/go.mod h1:f/jRa757WUmaOZrbPspXymbg/GnbF+rwe4OLsG7aXYo=
|
||||
github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI=
|
||||
github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20260615092313-b57e5e6d29bb h1:m1Uub9uM7meZ8ofVyeRfK7M/knCi3sF2gKn9kldVW0I=
|
||||
@@ -34,8 +34,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/dlclark/regexp2/v2 v2.5.0 h1:liNiWIPCvCh5HBcYfsjd+P16AG79fwd6T1Toy2gOtEA=
|
||||
github.com/dlclark/regexp2/v2 v2.5.0/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
|
||||
github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
@@ -62,9 +62,9 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
mvdan.cc/sh/v3 v3.13.1 h1:DP3TfgZhDkT7lerUdnp6PTGKyxxzz6T+cOlY/xEvfWk=
|
||||
mvdan.cc/sh/v3 v3.13.1/go.mod h1:lXJ8SexMvEVcHCoDvAGLZgFJ9Wsm2sulmoNEXGhYZD0=
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -430,6 +431,12 @@ func (state *compactionState) maybeCompact(
|
||||
state.lowWaterMark = size
|
||||
return messages
|
||||
}
|
||||
// Only count a compaction when it actually shrank the history, so the
|
||||
// compaction counter reflects real context reductions rather than paid
|
||||
// no-ops that left the token budget untouched.
|
||||
if r := trace.FromContext(ctx); r != nil {
|
||||
r.Counter(trace.CounterCompactionCount, 1)
|
||||
}
|
||||
state.lowWaterMark = newSize
|
||||
return compacted
|
||||
}
|
||||
@@ -479,7 +486,11 @@ func (state *compactionState) recover(
|
||||
// one-shot budget now so a provider that keeps returning context-limit errors
|
||||
// after a successful compaction can't loop forever. Store the low-water mark in
|
||||
// the SAME combined (messages + tool-defs) domain maybeCompact uses, so the
|
||||
// proactive shrink-guard compares like with like.
|
||||
// proactive shrink-guard compares like with like. Count it now that the shrink
|
||||
// is confirmed, so the counter mirrors maybeCompact's real-reduction policy.
|
||||
if r := trace.FromContext(ctx); r != nil {
|
||||
r.Counter(trace.CounterCompactionCount, 1)
|
||||
}
|
||||
state.reactiveAttempted = true
|
||||
state.lowWaterMark = state.calibratedTokens(estimateTokens(result) + estimateToolDefTokens(tools))
|
||||
return result, true, nil
|
||||
|
||||
+107
-6
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/streamjson"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -114,6 +115,26 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
return Result{}, errors.New("agent provider is required")
|
||||
}
|
||||
|
||||
// Tracing is opt-in. When a recorder is wired, thread it into ctx so the
|
||||
// providerio seam and reconnect helper can reach it via trace.FromContext,
|
||||
// mark the run's start, and wrap OnUsage so token counters accumulate for
|
||||
// free alongside the existing per-request plumbing. nil recorder leaves the
|
||||
// loop byte-identical: FromContext returns nil, the no-op stamps cost
|
||||
// nothing, and the OnUsage wrapper is not installed.
|
||||
if options.Trace != nil {
|
||||
ctx = trace.WithContext(ctx, options.Trace)
|
||||
options.Trace.Start()
|
||||
originalOnUsage := options.OnUsage
|
||||
options.OnUsage = func(usage Usage) {
|
||||
if originalOnUsage != nil {
|
||||
originalOnUsage(usage)
|
||||
}
|
||||
options.Trace.Counter(trace.CounterInputTokens, int64(usage.InputTokens))
|
||||
options.Trace.Counter(trace.CounterCachedInputTokens, int64(usage.CachedInputTokens))
|
||||
options.Trace.Counter(trace.CounterOutputTokens, int64(usage.OutputTokens))
|
||||
}
|
||||
}
|
||||
|
||||
maxTurns := options.MaxTurns
|
||||
if maxTurns <= 0 {
|
||||
maxTurns = 12
|
||||
@@ -164,7 +185,11 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
result = Result{Messages: copyMessages(messages)}
|
||||
dispatchSessionStart(ctx, options)
|
||||
defer func() {
|
||||
dispatchSessionEnd(ctx, options, result, err)
|
||||
// Use a context detached from ctx's cancellation: on Esc/Ctrl+C the run
|
||||
// context is already canceled by the time this defer runs, and a canceled
|
||||
// parent would make Hooks.Dispatch's context.WithTimeout derive an
|
||||
// already-expired context, so the sessionEnd hook would never actually run.
|
||||
dispatchSessionEnd(context.WithoutCancel(ctx), options, result, err)
|
||||
}()
|
||||
for turn := 0; turn < maxTurns; turn++ {
|
||||
result.Turns = turn + 1
|
||||
@@ -184,12 +209,40 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// the tool-definition tokens (they ride on every request) in its estimate.
|
||||
// partitionTools depends only on registry/permissions/options/loaded, not on
|
||||
// the messages, so computing it before compaction is safe.
|
||||
// Build the per-turn tool list first so proactive compaction can include
|
||||
// the tool-definition tokens (they ride on every request) in its estimate.
|
||||
// partitionTools depends only on registry/permissions/options/loaded, not on
|
||||
// the messages, so computing it before compaction is safe.
|
||||
toolPartitionSpan := options.Trace.Span(trace.SpanToolPartition)
|
||||
exposed, _ := partitionToolsCached(registry, permissionMode, options, loaded, toolDefCache)
|
||||
toolPartitionSpan.End()
|
||||
|
||||
// Prompt-prefix fingerprint: hash the seven cacheable sub-components
|
||||
// of this turn's request (system prompt sections, project context,
|
||||
// skills, partitioned tool list, tool schemas) and emit one trace
|
||||
// event. Stable across turns => cacheable. Drift in any sub-hash
|
||||
// names the sub-component that broke the cache for this turn. The
|
||||
// computation is pure and cheap (seven SHA-256s of small strings)
|
||||
// and is a no-op when tracing is off.
|
||||
if options.Trace != nil {
|
||||
fp := ComputePrefixFingerprint(options, exposed)
|
||||
options.Trace.EmitPrefixHash(trace.PrefixHash{
|
||||
BaseInstructionsHash: fp.BaseInstructionsHash,
|
||||
ConfirmationPolicyHash: fp.ConfirmationPolicyHash,
|
||||
ProjectContextHash: fp.ProjectContextHash,
|
||||
SkillsHash: fp.SkillsHash,
|
||||
ToolsHash: fp.ToolsHash,
|
||||
SchemaHash: fp.SchemaHash,
|
||||
CompletePrefixHash: fp.CompletePrefixHash,
|
||||
})
|
||||
}
|
||||
|
||||
// PROACTIVE compaction: if the history is approaching the model's
|
||||
// context window, summarize the oldest middle before building the
|
||||
// request. A no-op when ContextWindow == 0 (compaction disabled).
|
||||
compactionSpan := options.Trace.Span(trace.SpanCompaction)
|
||||
messages = compactor.maybeCompact(ctx, provider, messages, exposed)
|
||||
compactionSpan.End()
|
||||
request := zeroruntime.CompletionRequest{
|
||||
Messages: copyMessages(messages),
|
||||
Tools: exposed,
|
||||
@@ -255,11 +308,39 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// conversation-state duplication.
|
||||
forwardedVisibleText := false
|
||||
forwardingOpts := zeroruntime.CollectOptions{OnUsage: options.OnUsage}
|
||||
if options.OnText != nil {
|
||||
forwardingOpts.OnText = func(s string) { forwardedVisibleText = true; options.OnText(s) }
|
||||
// Install text/reasoning forwarding handlers whenever EITHER a user
|
||||
// callback OR a trace recorder is set. A headless traced run (e.g. `zero
|
||||
// exec --trace`) sets Trace but no OnText/OnReasoning; without these
|
||||
// handlers the stream is still collected, but FirstTokenAt would never
|
||||
// stamp and the trace would lose its TTFT signal. The trace recorder's
|
||||
// stamp methods are nil-safe, but we guard on `trace != nil` so a run with
|
||||
// a user callback but no recorder still works. forwardedVisibleText stays
|
||||
// tied to the USER callback only, preserving the stall-retry semantics
|
||||
// (a trace-only handler forwards nothing the user would see duplicated).
|
||||
rec := options.Trace
|
||||
onText := options.OnText
|
||||
if onText != nil || rec != nil {
|
||||
forwardingOpts.OnText = func(s string) {
|
||||
if rec != nil {
|
||||
rec.StampFirstVisibleEvent()
|
||||
rec.StampFirstToken()
|
||||
}
|
||||
if onText != nil {
|
||||
forwardedVisibleText = true
|
||||
onText(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.OnReasoning != nil {
|
||||
forwardingOpts.OnReasoning = func(s string) { options.OnReasoning(s) }
|
||||
onReasoning := options.OnReasoning
|
||||
if onReasoning != nil || rec != nil {
|
||||
forwardingOpts.OnReasoning = func(s string) {
|
||||
if rec != nil {
|
||||
rec.StampFirstToken()
|
||||
}
|
||||
if onReasoning != nil {
|
||||
onReasoning(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.OnToolCallStart != nil {
|
||||
forwardingOpts.OnToolCallStart = func(id, name string) { options.OnToolCallStart(id, name) }
|
||||
@@ -297,14 +378,18 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
if retryStreamErr != nil {
|
||||
return collected, retryStreamErr
|
||||
}
|
||||
genSpan := options.Trace.Span(trace.SpanGeneration)
|
||||
collected = zeroruntime.CollectStreamWithOptions(ctx, retryStream, zeroruntime.CollectOptions{
|
||||
OnUsage: options.OnUsage,
|
||||
})
|
||||
genSpan.End()
|
||||
}
|
||||
return collected, nil
|
||||
}
|
||||
|
||||
generationSpan := options.Trace.Span(trace.SpanGeneration)
|
||||
collected := zeroruntime.CollectStreamWithOptions(ctx, stream, forwardingOpts)
|
||||
generationSpan.End()
|
||||
if collected.Error != "" {
|
||||
updated, stop := recoverStreamError(collected)
|
||||
collected = updated
|
||||
@@ -357,7 +442,9 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
result.Messages = copyMessages(messages)
|
||||
return result, retryErr
|
||||
}
|
||||
stallGenSpan := options.Trace.Span(trace.SpanGeneration)
|
||||
collected = zeroruntime.CollectStreamWithOptions(ctx, retryStream, forwardingOpts)
|
||||
stallGenSpan.End()
|
||||
}
|
||||
if collected.Error != "" {
|
||||
// Route a reissued stream's non-stall error through the SAME recovery as
|
||||
@@ -459,6 +546,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
if cue || planPending {
|
||||
if continueNudges < maxContinueNudges {
|
||||
continueNudges++
|
||||
options.Trace.Counter(trace.CounterCompletionNudges, 1)
|
||||
reason := "your message ended mid-step"
|
||||
if !cue {
|
||||
reason = "pending plan items remain — finish them, or mark them complete with update_plan if you are done"
|
||||
@@ -488,6 +576,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// (no admission, no cue) then finalizes as success on the next turn.
|
||||
if options.SelfCorrect != nil && !acceptanceRequested {
|
||||
acceptanceRequested = true
|
||||
options.Trace.Counter(trace.CounterAcceptanceChecks, 1)
|
||||
messages = append(messages, zeroruntime.Message{
|
||||
Role: zeroruntime.MessageRoleUser,
|
||||
Content: acceptanceVerificationNudge(),
|
||||
@@ -548,20 +637,26 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
runEnd++
|
||||
}
|
||||
if runEnd-index >= 2 {
|
||||
batchSpan := options.Trace.Span(trace.SpanToolExecution)
|
||||
precomputed = executeParallelReadBatch(ctx, registry, collected.ToolCalls, index, runEnd, permissionMode, options)
|
||||
batchSpan.End()
|
||||
precomputedStart, precomputedEnd = index, runEnd
|
||||
}
|
||||
}
|
||||
if options.OnToolCall != nil {
|
||||
options.OnToolCall(call)
|
||||
}
|
||||
options.Trace.StampFirstUsefulAction()
|
||||
var toolResult ToolResult
|
||||
var abortErr error
|
||||
if index >= precomputedStart && index < precomputedEnd {
|
||||
toolResult, abortErr = precomputed[index-precomputedStart].result, precomputed[index-precomputedStart].abortErr
|
||||
} else {
|
||||
toolSpan := options.Trace.Span(trace.SpanToolExecution)
|
||||
toolResult, abortErr = executeToolCall(ctx, registry, call, permissionMode, options)
|
||||
toolSpan.End()
|
||||
}
|
||||
options.Trace.Counter(trace.CounterToolCalls, 1)
|
||||
if options.OnToolResult != nil {
|
||||
options.OnToolResult(toolResult)
|
||||
}
|
||||
@@ -666,6 +761,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// context-window sizing, and usage attribution follow the new model.
|
||||
provider = newProvider
|
||||
options.Model = turnRequestedModel
|
||||
options.Trace.Counter(trace.CounterModelSwitches, 1)
|
||||
// KNOWN LIMITATION (deferred): the compactor's context-window budget
|
||||
// is fixed at run start from options.ContextWindow and is NOT updated
|
||||
// here, so a switch to a model with a different window keeps compacting
|
||||
@@ -764,6 +860,7 @@ func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages [
|
||||
if err != nil {
|
||||
return "", messages, ""
|
||||
}
|
||||
finalGenSpan := options.Trace.Span(trace.SpanGeneration)
|
||||
collected := zeroruntime.CollectStreamWithOptions(ctx, stream, zeroruntime.CollectOptions{
|
||||
OnText: options.OnText,
|
||||
OnReasoning: options.OnReasoning,
|
||||
@@ -771,6 +868,7 @@ func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages [
|
||||
OnToolCallStart: options.OnToolCallStart,
|
||||
OnToolCallDelta: options.OnToolCallDelta,
|
||||
})
|
||||
finalGenSpan.End()
|
||||
if ctx.Err() != nil || collected.Error != "" || strings.TrimSpace(collected.Text) == "" {
|
||||
return "", messages, ""
|
||||
}
|
||||
@@ -1874,7 +1972,10 @@ func requestPermission(ctx context.Context, request PermissionRequest, options O
|
||||
if options.OnPermissionRequest == nil {
|
||||
return PermissionDecision{Action: PermissionDecisionDeny, Reason: request.Reason}, nil
|
||||
}
|
||||
return options.OnPermissionRequest(ctx, request)
|
||||
permSpan := options.Trace.Span(trace.SpanPermissionWait)
|
||||
decision, err := options.OnPermissionRequest(ctx, request)
|
||||
permSpan.End()
|
||||
return decision, err
|
||||
}
|
||||
|
||||
func normalizePermissionDecisionAction(action PermissionDecisionAction) PermissionDecisionAction {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/specmode"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -92,6 +94,87 @@ func TestRunDispatchesSessionLifecycleHooks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// cancelingProvider simulates an Esc/Ctrl+C abort: it cancels the run's own
|
||||
// context mid-stream (as the TUI's interrupt handler would) and returns no
|
||||
// events, so Run observes ctx.Err() and returns early with an already-canceled
|
||||
// context in scope for the deferred sessionEnd dispatch.
|
||||
type cancelingProvider struct {
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (provider *cancelingProvider) StreamCompletion(ctx context.Context, request zeroruntime.CompletionRequest) (<-chan zeroruntime.StreamEvent, error) {
|
||||
provider.cancel()
|
||||
ch := make(chan zeroruntime.StreamEvent)
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// TestRunDispatchesSessionEndHookAfterContextCancellation is a regression test
|
||||
// for a bug where an interrupted run (Esc/Ctrl+C) canceled ctx before the
|
||||
// deferred sessionEnd dispatch ran, so Hooks.Dispatch's context.WithTimeout(ctx, ...)
|
||||
// derived an already-canceled context and the hook command never actually launched.
|
||||
func TestRunDispatchesSessionEndHookAfterContextCancellation(t *testing.T) {
|
||||
goBinary, err := exec.LookPath("go")
|
||||
if err != nil {
|
||||
// The test binary runs on the build machine, so its build-time GOROOT
|
||||
// still identifies the toolchain that produced it.
|
||||
goRoot := runtime.GOROOT() //nolint:staticcheck // Safe for this non-portable test binary.
|
||||
goBinary = filepath.Join(goRoot, "bin", "go")
|
||||
if runtime.GOOS == "windows" {
|
||||
goBinary += ".exe"
|
||||
}
|
||||
if _, statErr := os.Stat(goBinary); statErr != nil {
|
||||
t.Skipf("go binary unavailable on PATH or in GOROOT: %v", statErr)
|
||||
}
|
||||
}
|
||||
audit, err := hooks.NewAuditStore(hooks.AuditStoreOptions{AuditPath: filepath.Join(t.TempDir(), "audit.jsonl")})
|
||||
if err != nil {
|
||||
t.Fatalf("NewAuditStore: %v", err)
|
||||
}
|
||||
dispatcher := hooks.NewDispatcher(hooks.DispatcherOptions{
|
||||
Config: hooks.Config{
|
||||
Enabled: true,
|
||||
Hooks: []hooks.Definition{
|
||||
{ID: "zero.session-end", Event: hooks.EventSessionEnd, Command: goBinary, Args: []string{"version"}, Enabled: true},
|
||||
},
|
||||
},
|
||||
Audit: audit,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
provider := &cancelingProvider{cancel: cancel}
|
||||
|
||||
_, err = Run(ctx, "hi", provider, Options{
|
||||
SessionID: "session-cancel",
|
||||
Cwd: t.TempDir(),
|
||||
ProviderName: "test-provider",
|
||||
Model: "test-model",
|
||||
Hooks: dispatcher,
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Run err = %v, want context.Canceled", err)
|
||||
}
|
||||
|
||||
events, err := audit.ReadEvents()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadEvents: %v", err)
|
||||
}
|
||||
var completedStatus hooks.AuditStatus
|
||||
found := false
|
||||
for _, event := range events {
|
||||
if event.Type == "hook_execution_completed" && event.Event == hooks.EventSessionEnd {
|
||||
completedStatus = event.Status
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("no sessionEnd hook_execution_completed audit event, events=%#v", events)
|
||||
}
|
||||
if completedStatus != hooks.AuditCompleted {
|
||||
t.Fatalf("sessionEnd hook status = %q, want %q (hook must actually run despite ctx cancellation)", completedStatus, hooks.AuditCompleted)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingWebSearchTool struct {
|
||||
calls []map[string]any
|
||||
}
|
||||
@@ -3461,3 +3544,73 @@ func TestRunDoesNotFlagCleanToolOutput(t *testing.T) {
|
||||
t.Errorf("clean output should not get a reminder, got %q", captured.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunTracingWrapperStampsUsage verifies the per-turn tracing setup in Run:
|
||||
// a wired recorder is Started, OnUsage is wrapped so it still forwards to the
|
||||
// caller's callback AND stamps token counters, and the run completes.
|
||||
func TestRunTracingWrapperStampsUsage(t *testing.T) {
|
||||
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{{
|
||||
{Type: zeroruntime.StreamEventUsage, Usage: zeroruntime.Usage{InputTokens: 100, CachedInputTokens: 20, OutputTokens: 40}},
|
||||
{Type: zeroruntime.StreamEventText, Content: "done"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
}}}
|
||||
onUsageCalls := 0
|
||||
rec := trace.NewRecorder("tracing-session", "run-1", "test")
|
||||
if _, err := Run(context.Background(), "hi", provider, Options{
|
||||
SessionID: "tracing-session",
|
||||
Cwd: t.TempDir(),
|
||||
ProviderName: "test-provider",
|
||||
Model: "test-model",
|
||||
Trace: rec,
|
||||
OnUsage: func(Usage) { onUsageCalls++ },
|
||||
}); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
tr := rec.Finish()
|
||||
if tr.StartedAt.IsZero() {
|
||||
t.Fatal("tracing wrapper did not Start the recorder")
|
||||
}
|
||||
// FirstTokenAt must stamp even though no OnText/OnReasoning user callback is
|
||||
// set: a headless traced run (e.g. `zero exec --trace`) sets Trace but no UI
|
||||
// callbacks, so the loop installs trace-only forwarding handlers to capture
|
||||
// TTFT. Without them FirstTokenAt stays zero and the trace loses its signal.
|
||||
if tr.FirstTokenAt.IsZero() {
|
||||
t.Fatal("FirstTokenAt not stamped for a traced run with no OnText/OnReasoning callbacks")
|
||||
}
|
||||
if got := tr.Counter(trace.CounterInputTokens); got != 100 {
|
||||
t.Fatalf("input token counter = %d, want 100", got)
|
||||
}
|
||||
if got := tr.Counter(trace.CounterCachedInputTokens); got != 20 {
|
||||
t.Fatalf("cached input token counter = %d, want 20", got)
|
||||
}
|
||||
if got := tr.Counter(trace.CounterOutputTokens); got != 40 {
|
||||
t.Fatalf("output token counter = %d, want 40", got)
|
||||
}
|
||||
if onUsageCalls == 0 {
|
||||
t.Fatal("wrapped OnUsage did not forward to the caller's callback")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunNilTraceForwardsUsage verifies a nil recorder leaves the loop
|
||||
// byte-identical: OnUsage is not wrapped, so the caller's callback still fires
|
||||
// and nothing panics.
|
||||
func TestRunNilTraceForwardsUsage(t *testing.T) {
|
||||
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{{
|
||||
{Type: zeroruntime.StreamEventUsage, Usage: zeroruntime.Usage{InputTokens: 7}},
|
||||
{Type: zeroruntime.StreamEventText, Content: "done"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
}}}
|
||||
onUsageCalls := 0
|
||||
if _, err := Run(context.Background(), "hi", provider, Options{
|
||||
SessionID: "nil-trace-session",
|
||||
Cwd: t.TempDir(),
|
||||
ProviderName: "test-provider",
|
||||
Model: "test-model",
|
||||
OnUsage: func(Usage) { onUsageCalls++ },
|
||||
}); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if onUsageCalls == 0 {
|
||||
t.Fatal("OnUsage not forwarded when Trace is nil")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
// promptSubstrings are the seven cacheable sub-components of the prompt a run
|
||||
// sends to a model. Each is the literal content the sub-component would emit
|
||||
// into the system prompt or tool list, hashed so a downstream observer can
|
||||
// detect drift turn-over-turn. The substrings are produced by
|
||||
// buildPromptSubstrings and consumed by ComputePrefixFingerprint.
|
||||
type promptSubstrings struct {
|
||||
baseInstructions string
|
||||
confirmationPolicy string
|
||||
projectContext string
|
||||
skills string
|
||||
tools string
|
||||
schema string
|
||||
}
|
||||
|
||||
// buildPromptSubstrings assembles the seven cacheable sub-components of the
|
||||
// prompt without joining them. The system prompt sections are produced by the
|
||||
// same private builders that buildSystemPrompt uses, so the substrings are
|
||||
// byte-identical to what would have appeared in the joined prompt. The tools
|
||||
// and schema substrings are derived from the partitioned tool list (caller-
|
||||
// provided) so this helper does not need to re-run the partition.
|
||||
//
|
||||
// This is the seam ComputePrefixFingerprint reads from. It exists as a
|
||||
// separate function (rather than returning both prompt and substrings from
|
||||
// buildSystemPrompt) so existing callers of buildSystemPrompt are unaffected
|
||||
// and the substrings helper is independently testable.
|
||||
func buildPromptSubstrings(options Options, exposed []zeroruntime.ToolDefinition) promptSubstrings {
|
||||
core := strings.TrimSpace(options.SystemPrompt)
|
||||
if core == "" {
|
||||
core = strings.TrimSpace(coreSystemPrompt)
|
||||
}
|
||||
if core == "" {
|
||||
core = fallbackSystemPrompt
|
||||
}
|
||||
|
||||
// The confirmation policy is appended unconditionally in buildSystemPrompt;
|
||||
// the substring here is the same constant.
|
||||
policy := strings.TrimSpace(confirmationPolicy)
|
||||
|
||||
// projectContext is the joined output of workspaceContext, which itself
|
||||
// walks the AGENTS.md / ZERO.md / .zero/AGENTS.md chain. Hashing the
|
||||
// joined string captures every guideline file's contribution and the
|
||||
// ordering between them.
|
||||
project := workspaceContext(options.Cwd)
|
||||
|
||||
// skills is the joined output of skillsContext, with each skill on its own
|
||||
// line. The substring is the literal text the model would see.
|
||||
skills := skillsContext(options)
|
||||
|
||||
// tools and schema are derived from the partitioned tool list. The list
|
||||
// passed in is already in stable order (partitionToolsCached is the
|
||||
// canonical stable partitioner) so concatenating in slice order is
|
||||
// sufficient — we sort names again defensively in case a future caller
|
||||
// hands us a list from a different partitioner.
|
||||
toolsSubstr, schemaSubstr := toolSubstrings(exposed)
|
||||
|
||||
return promptSubstrings{
|
||||
baseInstructions: core,
|
||||
confirmationPolicy: policy,
|
||||
projectContext: project,
|
||||
skills: skills,
|
||||
tools: toolsSubstr,
|
||||
schema: schemaSubstr,
|
||||
}
|
||||
}
|
||||
|
||||
// toolSubstrings extracts a stable, name-sorted digest of the partitioned tool
|
||||
// list. The tools substring is "name1\nname2\n..." (sorted) so a reordering of
|
||||
// the partition produces a different hash. The schema substring is the
|
||||
// concatenation of each tool's JSON schema in name order, so a schema change
|
||||
// is detected independently of the tool ordering.
|
||||
func toolSubstrings(exposed []zeroruntime.ToolDefinition) (toolsSubstr, schemaSubstr string) {
|
||||
if len(exposed) == 0 {
|
||||
return "", ""
|
||||
}
|
||||
names := make([]string, 0, len(exposed))
|
||||
byName := make(map[string]zeroruntime.ToolDefinition, len(exposed))
|
||||
for _, def := range exposed {
|
||||
names = append(names, def.Name)
|
||||
byName[def.Name] = def
|
||||
}
|
||||
sort.Strings(names)
|
||||
var toolsSB, schemaSB strings.Builder
|
||||
for i, name := range names {
|
||||
if i > 0 {
|
||||
toolsSB.WriteByte('\n')
|
||||
}
|
||||
toolsSB.WriteString(name)
|
||||
def := byName[name]
|
||||
// Parameters is rendered to a canonical JSON string per tool. The
|
||||
// schema-render cache in internal/agent (used by partitionToolsCached)
|
||||
// guarantees the same render is byte-identical across turns for the
|
||||
// same tool, so this substring is a stable hash input.
|
||||
schemaSB.WriteString(name)
|
||||
schemaSB.WriteByte('\n')
|
||||
schemaSB.WriteString(canonicalSchemaString(def.Parameters))
|
||||
schemaSB.WriteByte('\n')
|
||||
}
|
||||
return toolsSB.String(), schemaSB.String()
|
||||
}
|
||||
|
||||
// canonicalSchemaString renders a tool's parameter schema to a stable string.
|
||||
// tools.ToolDef.Parameters is a map[string]any in the common case; Go's
|
||||
// fmt.Sprintf("%v", m) iterates a map in random order, which would produce
|
||||
// a different hash for the same Parameters value across calls and defeat
|
||||
// the fingerprint. encoding/json marshals maps with keys sorted
|
||||
// alphabetically, so json.Marshal is the primary stable render. The
|
||||
// fallback for the rare non-JSON-compatible value (functions, channels,
|
||||
// NaN/Inf floats, cyclic references) is a stable key-sorted stringifier
|
||||
// that walks the value with sorted keys for maps and a leading
|
||||
// "__non_json:" prefix so a future schema change is visible in the trace
|
||||
// (a SchemaHash collision from the same value) rather than a silent hash
|
||||
// drift.
|
||||
//
|
||||
// Note: SchemaHash is a stability signal, not a wire-identity signal. The
|
||||
// bytes json.Marshal produces for a Go map[string]any may differ from
|
||||
// the bytes the provider actually sends on the wire (provider encoders
|
||||
// use their own JSON conventions; an Anthropic schema may serialize with
|
||||
// different whitespace, key order, or number formatting). A consumer
|
||||
// must NOT assume SchemaHash matches the provider's wire schema — only
|
||||
// that two turns with the same Go-side Parameters produce the same
|
||||
// SchemaHash. This is sufficient for the trace's purpose (drift
|
||||
// detection) but not for a content-based equality check.
|
||||
func canonicalSchemaString(params map[string]any) string {
|
||||
if len(params) == 0 {
|
||||
return ""
|
||||
}
|
||||
data, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return "__non_json:" + stableStringify(params)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// stableStringify renders v to a deterministic string. Maps have their keys
|
||||
// sorted alphabetically; slices are walked in order; primitives use their
|
||||
// natural Go format. The function is recursive but bounded by the size of
|
||||
// the input, so a cyclic reference is not reachable here (the json.Marshal
|
||||
// fallback is hit when json.Marshal itself returns an error, which for the
|
||||
// common case is a non-JSON-compatible value, not a cycle — cycles are
|
||||
// rare in tool schemas and acceptable to mis-render in the fallback path
|
||||
// since the trace's contract is "the hash is stable for the same input,"
|
||||
// not "the fallback is lossless"). Used only when json.Marshal fails.
|
||||
func stableStringify(v any) string {
|
||||
var sb strings.Builder
|
||||
writeStable(&sb, v)
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func writeStable(sb *strings.Builder, v any) {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
sb.WriteString("null")
|
||||
case bool:
|
||||
if x {
|
||||
sb.WriteString("true")
|
||||
} else {
|
||||
sb.WriteString("false")
|
||||
}
|
||||
case string:
|
||||
sb.WriteByte('"')
|
||||
sb.WriteString(x)
|
||||
sb.WriteByte('"')
|
||||
case float64:
|
||||
// Use %g for compact, deterministic float rendering. NaN and Inf are
|
||||
// not JSON-encodable (which is why we are in the fallback path) and
|
||||
// render as "NaN" / "+Inf" / "-Inf" — distinct, stable strings.
|
||||
fmt.Fprintf(sb, "%g", x)
|
||||
case float32:
|
||||
fmt.Fprintf(sb, "%g", float64(x))
|
||||
case int:
|
||||
fmt.Fprintf(sb, "%d", x)
|
||||
case int64:
|
||||
fmt.Fprintf(sb, "%d", x)
|
||||
case int32:
|
||||
fmt.Fprintf(sb, "%d", x)
|
||||
case uint:
|
||||
fmt.Fprintf(sb, "%d", x)
|
||||
case uint64:
|
||||
fmt.Fprintf(sb, "%d", x)
|
||||
case uint32:
|
||||
fmt.Fprintf(sb, "%d", x)
|
||||
case map[string]any:
|
||||
sb.WriteByte('{')
|
||||
keys := make([]string, 0, len(x))
|
||||
for k := range x {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for i, k := range keys {
|
||||
if i > 0 {
|
||||
sb.WriteByte(',')
|
||||
}
|
||||
sb.WriteByte('"')
|
||||
sb.WriteString(k)
|
||||
sb.WriteString(`":`)
|
||||
writeStable(sb, x[k])
|
||||
}
|
||||
sb.WriteByte('}')
|
||||
case []any:
|
||||
sb.WriteByte('[')
|
||||
for i, item := range x {
|
||||
if i > 0 {
|
||||
sb.WriteByte(',')
|
||||
}
|
||||
writeStable(sb, item)
|
||||
}
|
||||
sb.WriteByte(']')
|
||||
default:
|
||||
// Last-resort: include the type so distinct values produce distinct
|
||||
// strings even if their default format collides. Without the type
|
||||
// tag, fmt.Sprintf("%v", x) for two different types could produce
|
||||
// the same bytes (rare, but the type prefix makes it impossible).
|
||||
fmt.Fprintf(sb, "<%T:%v>", x, x)
|
||||
}
|
||||
}
|
||||
|
||||
// ComputePrefixFingerprint returns a trace.PrefixHash (a 7-field fingerprint
|
||||
// of the prompt prefix) for one turn of a run. The seven sub-hashes are
|
||||
// independent SHA-256s of the corresponding sub-component; the complete-prefix
|
||||
// hash is a SHA-256 of the canonical concatenation of the other six, so any
|
||||
// sub-component drift is observable both individually and in aggregate.
|
||||
//
|
||||
// I/O side effects: buildPromptSubstrings calls workspaceContext, which runs
|
||||
// git (gitBranchForPrompt, FindProjectGitRoot) and reads the AGENTS.md /
|
||||
// ZERO.md / .zero/AGENTS.md chain plus the repo map. The same workspace-
|
||||
// context work is performed separately by buildSystemPrompt a few lines
|
||||
// later in the request-build path, so on every traced turn the file and
|
||||
// git reads happen twice. The duplication is intentional for this PR (a
|
||||
// follow-up will pass substrings through) and is a known perf cost only on
|
||||
// the opt-in trace path.
|
||||
//
|
||||
// A stable CompletePrefixHash across turns means the four captured
|
||||
// sub-components (baseInstructions, confirmationPolicy, projectContext,
|
||||
// skills) are byte-identical. It does NOT rule out drift in the seven
|
||||
// uncaptured sections of buildSystemPrompt (modelPromptAddendum,
|
||||
// sessionRuntimeContext, approvedCommandPrefixContext, workspaceSeedContext,
|
||||
// userGuidelines, specialistDelegationContext, responseStyleContext). For
|
||||
// default Options the four captured substrings are the full prompt; for
|
||||
// non-default Options the seven uncaptured sections may contribute, and a
|
||||
// consumer correlating CompletePrefix stability with cached_input_tokens
|
||||
// must cross-check the model_switches counter (modelPromptAddendum changes
|
||||
// on a model switch) to disambiguate "no drift" from "drift in an
|
||||
// uncaptured section."
|
||||
|
||||
func ComputePrefixFingerprint(options Options, exposed []zeroruntime.ToolDefinition) prefixFingerprint {
|
||||
subs := buildPromptSubstrings(options, exposed)
|
||||
base := sha256hex(subs.baseInstructions)
|
||||
policy := sha256hex(subs.confirmationPolicy)
|
||||
project := sha256hex(subs.projectContext)
|
||||
skills := sha256hex(subs.skills)
|
||||
toolsH := sha256hex(subs.tools)
|
||||
schema := sha256hex(subs.schema)
|
||||
complete := sha256hex(strings.Join([]string{
|
||||
base, policy, project, skills, toolsH, schema,
|
||||
}, "|"))
|
||||
return prefixFingerprint{
|
||||
BaseInstructionsHash: base,
|
||||
ConfirmationPolicyHash: policy,
|
||||
ProjectContextHash: project,
|
||||
SkillsHash: skills,
|
||||
ToolsHash: toolsH,
|
||||
SchemaHash: schema,
|
||||
CompletePrefixHash: complete,
|
||||
}
|
||||
}
|
||||
|
||||
// prefixFingerprint is the agent-side shape of a prompt-prefix fingerprint. It
|
||||
// is converted to a trace.PrefixHash at the loop boundary (see EmitPrefixHash
|
||||
// in loop.go) so the trace package does not need to import this one. The field
|
||||
// names match the trace.PrefixHash JSON tags 1:1.
|
||||
type prefixFingerprint struct {
|
||||
BaseInstructionsHash string
|
||||
ConfirmationPolicyHash string
|
||||
ProjectContextHash string
|
||||
SkillsHash string
|
||||
ToolsHash string
|
||||
SchemaHash string
|
||||
CompletePrefixHash string
|
||||
}
|
||||
|
||||
// sha256hex returns the hex-encoded SHA-256 of s. Empty input produces the
|
||||
// hash of the empty string, which is a constant; callers that want to
|
||||
// distinguish "absent" from "empty" should check s before calling.
|
||||
func sha256hex(s string) string {
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
// TestComputePrefixFingerprintStableAcrossCalls asserts the headline property
|
||||
// the trace is meant to expose: two ComputePrefixFingerprint calls over the
|
||||
// same Options and the same exposed tool list produce byte-identical hashes.
|
||||
// This is the regression catch for "did anyone introduce non-determinism into
|
||||
// the prompt prefix."
|
||||
func TestComputePrefixFingerprintStableAcrossCalls(t *testing.T) {
|
||||
opts := Options{Cwd: t.TempDir(), SystemPrompt: "core"}
|
||||
exposed := []zeroruntime.ToolDefinition{
|
||||
{Name: "read_file", Parameters: map[string]any{"type": "object"}},
|
||||
{Name: "grep", Parameters: map[string]any{"type": "object"}},
|
||||
}
|
||||
first := ComputePrefixFingerprint(opts, exposed)
|
||||
second := ComputePrefixFingerprint(opts, exposed)
|
||||
if first != second {
|
||||
t.Fatalf("fingerprint must be stable across calls with the same input:\n first=%+v\n second=%+v", first, second)
|
||||
}
|
||||
if first.CompletePrefixHash == "" {
|
||||
t.Fatalf("CompletePrefixHash must be non-empty for a non-trivial prompt")
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputePrefixFingerprintToolsHashIsNameSetSensitive asserts the
|
||||
// name-set sensitivity of the ToolsHash. The test name used to read as
|
||||
// "reorder changes hash" but the assertion is the opposite: the hash is
|
||||
// name-set sensitive, not name-order sensitive (toolSubstrings sorts
|
||||
// names before hashing). The partitioner is expected to produce a stable
|
||||
// order, so a name-order change is not a hash signal — a name-set change
|
||||
// is. The test asserts both: permutation of the same tools produces the
|
||||
// same hash, and adding a tool produces a different hash.
|
||||
func TestComputePrefixFingerprintToolsHashIsNameSetSensitive(t *testing.T) {
|
||||
opts := Options{Cwd: t.TempDir(), SystemPrompt: "core"}
|
||||
a := []zeroruntime.ToolDefinition{
|
||||
{Name: "read_file", Parameters: map[string]any{"schema": "schema-read"}},
|
||||
{Name: "grep", Parameters: map[string]any{"schema": "schema-grep"}},
|
||||
}
|
||||
b := []zeroruntime.ToolDefinition{
|
||||
{Name: "grep", Parameters: map[string]any{"schema": "schema-grep"}},
|
||||
{Name: "read_file", Parameters: map[string]any{"schema": "schema-read"}},
|
||||
}
|
||||
// toolSubstrings sorts names internally, so the ToolsHash is identical
|
||||
// for a permutation of the same tools. We assert that here to document
|
||||
// the contract: name-order independence, name-set sensitivity.
|
||||
fpA := ComputePrefixFingerprint(opts, a)
|
||||
fpB := ComputePrefixFingerprint(opts, b)
|
||||
if fpA.ToolsHash != fpB.ToolsHash {
|
||||
t.Fatalf("ToolsHash must be name-set sensitive, not name-order sensitive: a=%s b=%s", fpA.ToolsHash, fpB.ToolsHash)
|
||||
}
|
||||
// A different tool set must produce a different ToolsHash.
|
||||
c := append([]zeroruntime.ToolDefinition{}, a...)
|
||||
c = append(c, zeroruntime.ToolDefinition{Name: "bash", Parameters: map[string]any{"schema": "schema-bash"}})
|
||||
fpC := ComputePrefixFingerprint(opts, c)
|
||||
if fpA.ToolsHash == fpC.ToolsHash {
|
||||
t.Fatalf("ToolsHash must change when the tool set changes: a=%s c=%s", fpA.ToolsHash, fpC.ToolsHash)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputePrefixFingerprintSchemaChangeChangesSchemaHash asserts that a
|
||||
// change to a tool's Parameters schema is visible in the SchemaHash. A schema
|
||||
// edit that should be cache-stable (e.g. field reorder, doc tweak) will move
|
||||
// the hash and surface in the trace; a schema edit that legitimately
|
||||
// invalidates the cache also moves the hash, which is the right behavior.
|
||||
func TestComputePrefixFingerprintSchemaChangeChangesSchemaHash(t *testing.T) {
|
||||
opts := Options{Cwd: t.TempDir(), SystemPrompt: "core"}
|
||||
a := []zeroruntime.ToolDefinition{
|
||||
{Name: "read_file", Parameters: map[string]any{"version": "v1"}},
|
||||
}
|
||||
b := []zeroruntime.ToolDefinition{
|
||||
{Name: "read_file", Parameters: map[string]any{"version": "v2"}},
|
||||
}
|
||||
fpA := ComputePrefixFingerprint(opts, a)
|
||||
fpB := ComputePrefixFingerprint(opts, b)
|
||||
if fpA.SchemaHash == fpB.SchemaHash {
|
||||
t.Fatalf("SchemaHash must change when tool Parameters change: a=%s b=%s", fpA.SchemaHash, fpB.SchemaHash)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputePrefixFingerprintSystemPromptChangeChangesBaseHash asserts the
|
||||
// most common drift path: a system-prompt edit (e.g. an agent.md update, a
|
||||
// model addendum change) must move the base-instructions hash, which moves
|
||||
// the complete-prefix hash, which is what the trace is for.
|
||||
func TestComputePrefixFingerprintSystemPromptChangeChangesBaseHash(t *testing.T) {
|
||||
optsA := Options{Cwd: t.TempDir(), SystemPrompt: "core v1"}
|
||||
optsB := Options{Cwd: t.TempDir(), SystemPrompt: "core v2"}
|
||||
exposed := []zeroruntime.ToolDefinition{{Name: "noop", Parameters: map[string]any{}}}
|
||||
fpA := ComputePrefixFingerprint(optsA, exposed)
|
||||
fpB := ComputePrefixFingerprint(optsB, exposed)
|
||||
if fpA.BaseInstructionsHash == fpB.BaseInstructionsHash {
|
||||
t.Fatalf("BaseInstructionsHash must change when SystemPrompt changes: a=%s b=%s", fpA.BaseInstructionsHash, fpB.BaseInstructionsHash)
|
||||
}
|
||||
if fpA.CompletePrefixHash == fpB.CompletePrefixHash {
|
||||
t.Fatalf("CompletePrefixHash must change when any sub-hash changes: a=%s b=%s", fpA.CompletePrefixHash, fpB.CompletePrefixHash)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputePrefixFingerprintCompleteIsAggregateOfSubHashes asserts the
|
||||
// canonical-join property: CompletePrefixHash must depend on every sub-hash.
|
||||
// This is the regression catch for "did anyone reorder or drop a sub-hash
|
||||
// from the canonical join without updating CompletePrefixHash."
|
||||
func TestComputePrefixFingerprintCompleteIsAggregateOfSubHashes(t *testing.T) {
|
||||
opts := Options{Cwd: t.TempDir(), SystemPrompt: "core"}
|
||||
exposed := []zeroruntime.ToolDefinition{{Name: "read_file", Parameters: map[string]any{"k": "v"}}}
|
||||
fp := ComputePrefixFingerprint(opts, exposed)
|
||||
// Manually compute what the canonical join should be.
|
||||
expected := sha256hex(strings.Join([]string{
|
||||
fp.BaseInstructionsHash,
|
||||
fp.ConfirmationPolicyHash,
|
||||
fp.ProjectContextHash,
|
||||
fp.SkillsHash,
|
||||
fp.ToolsHash,
|
||||
fp.SchemaHash,
|
||||
}, "|"))
|
||||
if fp.CompletePrefixHash != expected {
|
||||
t.Fatalf("CompletePrefixHash must equal sha256hex of the canonical join of the other six:\n got: %s\n expected: %s", fp.CompletePrefixHash, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPromptSubstringsDefaultOptions asserts the invariants of the
|
||||
// substrings helper for default Options: which substrings are non-empty,
|
||||
// which are empty, and that the substring-to-hash round-trip is lossless.
|
||||
func TestBuildPromptSubstringsDefaultOptions(t *testing.T) {
|
||||
opts := Options{
|
||||
Cwd: t.TempDir(),
|
||||
SystemPrompt: "test core",
|
||||
}
|
||||
subs := buildPromptSubstrings(opts, nil)
|
||||
// Invariants the trace depends on (default Options):
|
||||
// 1. baseInstructions substring equals the core system prompt bytes.
|
||||
// 2. confirmationPolicy substring is non-empty (the embedded policy
|
||||
// is always present, post TrimSpace).
|
||||
// 3. skills, tools, and schema substrings are empty (no skills or
|
||||
// tools configured for default Options).
|
||||
if subs.baseInstructions != "test core" {
|
||||
t.Fatalf("baseInstructions substring must equal the core system prompt: got %q want %q", subs.baseInstructions, "test core")
|
||||
}
|
||||
if subs.confirmationPolicy == "" {
|
||||
t.Fatalf("confirmationPolicy substring must be non-empty (the embedded policy is always present)")
|
||||
}
|
||||
if subs.skills != "" {
|
||||
t.Fatalf("skills substring must be empty for default Options: got %q", subs.skills)
|
||||
}
|
||||
// Round-trip: the corresponding fingerprint hashes must equal
|
||||
// sha256hex of the substrings, with no truncation or reformatting
|
||||
// between the two layers.
|
||||
fp := ComputePrefixFingerprint(opts, nil)
|
||||
if fp.BaseInstructionsHash != sha256hex(subs.baseInstructions) {
|
||||
t.Fatalf("BaseInstructionsHash in fingerprint must equal sha256hex of the substring")
|
||||
}
|
||||
if fp.ConfirmationPolicyHash != sha256hex(subs.confirmationPolicy) {
|
||||
t.Fatalf("ConfirmationPolicyHash in fingerprint must equal sha256hex of the substring")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCanonicalSchemaStringStableForMapParameters asserts the headline
|
||||
// property the trace needs: the same map[string]any produces the same
|
||||
// canonical string across calls, even though Go's fmt.Sprintf("%v", m)
|
||||
// iterates a map in random order. json.Marshal sorts map keys
|
||||
// alphabetically, which is the fix.
|
||||
func TestCanonicalSchemaStringStableForMapParameters(t *testing.T) {
|
||||
params := map[string]any{
|
||||
"description": "read a file",
|
||||
"properties": map[string]any{
|
||||
"path": map[string]any{"type": "string"},
|
||||
},
|
||||
"required": []any{"path"},
|
||||
"type": "object",
|
||||
}
|
||||
first := canonicalSchemaString(params)
|
||||
second := canonicalSchemaString(params)
|
||||
if first != second {
|
||||
t.Fatalf("canonicalSchemaString must be stable across calls for the same Parameters:\n first=%s\n second=%s", first, second)
|
||||
}
|
||||
if first == "" {
|
||||
t.Fatal("canonicalSchemaString must produce a non-empty string for a non-empty map")
|
||||
}
|
||||
// Two maps with the same key/value pairs (different declaration
|
||||
// order in source) must produce identical canonical strings. This
|
||||
// is the property that defeats Go's map iteration randomization.
|
||||
reordered := map[string]any{
|
||||
"type": "object",
|
||||
"required": []any{"path"},
|
||||
"properties": map[string]any{
|
||||
"path": map[string]any{"type": "string"},
|
||||
},
|
||||
"description": "read a file",
|
||||
}
|
||||
if canonicalSchemaString(params) != canonicalSchemaString(reordered) {
|
||||
t.Fatal("canonicalSchemaString must produce identical output for maps with the same key/value pairs in different declaration orders")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCanonicalSchemaStringFallbackStableForNonJSONValue exercises the
|
||||
// non-JSON fallback in canonicalSchemaString: when Parameters contains a
|
||||
// value json.Marshal cannot encode (here, a channel), the function must
|
||||
// fall back to a key-sorted stringification that is stable across calls
|
||||
// and across map-iteration order. The previous fallback used
|
||||
// fmt.Sprintf("%v", params), which iterates the map in random order and
|
||||
// produced a different hash for the same value across calls — defeating
|
||||
// the fingerprint. The current fallback is stable by construction.
|
||||
func TestCanonicalSchemaStringFallbackStableForNonJSONValue(t *testing.T) {
|
||||
ch := make(chan int) // channels are not JSON-encodable
|
||||
params := map[string]any{
|
||||
"description": "a tool with a non-JSON value",
|
||||
"channel": ch,
|
||||
"z_first": 1,
|
||||
"a_second": 2,
|
||||
}
|
||||
first := canonicalSchemaString(params)
|
||||
second := canonicalSchemaString(params)
|
||||
if first == "" {
|
||||
t.Fatal("canonicalSchemaString must produce a non-empty string for the fallback path")
|
||||
}
|
||||
if first != second {
|
||||
t.Fatalf("canonicalSchemaString fallback must be stable across calls:\n first=%s\n second=%s", first, second)
|
||||
}
|
||||
// A re-ordered map (same key/value pairs, different declaration order)
|
||||
// must produce the same fallback string. This is the property the
|
||||
// key-sorted stringifier provides that fmt.Sprintf("%v", m) does not.
|
||||
reordered := map[string]any{
|
||||
"a_second": 2,
|
||||
"z_first": 1,
|
||||
"channel": ch,
|
||||
"description": "a tool with a non-JSON value",
|
||||
}
|
||||
if canonicalSchemaString(params) != canonicalSchemaString(reordered) {
|
||||
t.Fatalf("canonicalSchemaString fallback must produce identical output for maps with the same key/value pairs in different declaration orders:\n first=%s\n second=%s", first, canonicalSchemaString(reordered))
|
||||
}
|
||||
// The fallback must include the "__non_json:" prefix so a consumer
|
||||
// can tell the bytes are not a JSON-marshaled schema (a hash collision
|
||||
// between a json.Marshal result and a stableStringify result is
|
||||
// possible in theory; the prefix makes the source observable).
|
||||
if !strings.HasPrefix(first, "__non_json:") {
|
||||
t.Fatalf("canonicalSchemaString fallback must start with the __non_json: prefix, got: %s", first)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/errhint"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -74,14 +75,21 @@ func stallRetryNoticeFor(options Options) reconnectNotifier {
|
||||
// already past its deadline is returned immediately (no retry) — those have
|
||||
// their own handling (compaction for context-limit, image-rejection, etc.).
|
||||
func streamWithReconnect(ctx context.Context, provider Provider, request zeroruntime.CompletionRequest, notify reconnectNotifier) (<-chan zeroruntime.StreamEvent, error) {
|
||||
recorder := trace.FromContext(ctx)
|
||||
stream, err := provider.StreamCompletion(ctx, request)
|
||||
if err == nil {
|
||||
if recorder != nil {
|
||||
recorder.Counter(trace.CounterModelRequests, 1)
|
||||
}
|
||||
return stream, nil
|
||||
}
|
||||
for attempt := 1; attempt <= maxStreamReconnects; attempt++ {
|
||||
if !shouldReconnect(ctx, err) {
|
||||
return nil, err
|
||||
}
|
||||
if recorder != nil {
|
||||
recorder.Counter(trace.CounterReconnectCount, 1)
|
||||
}
|
||||
if notify != nil {
|
||||
notify(attempt, maxStreamReconnects)
|
||||
}
|
||||
@@ -90,6 +98,9 @@ func streamWithReconnect(ctx context.Context, provider Provider, request zerorun
|
||||
}
|
||||
stream, err = provider.StreamCompletion(ctx, request)
|
||||
if err == nil {
|
||||
if recorder != nil {
|
||||
recorder.Counter(trace.CounterModelRequests, 1)
|
||||
}
|
||||
return stream, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/Gitlawb/zero/internal/lsp"
|
||||
"github.com/Gitlawb/zero/internal/redaction"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/verify"
|
||||
)
|
||||
|
||||
@@ -133,7 +134,9 @@ func (sc *SelfCorrector) inspect(ctx context.Context, changedFiles []string) Cor
|
||||
}
|
||||
|
||||
if sc.cfg.IncludeTests && sc.verifier != nil {
|
||||
verifySpan := trace.FromContext(ctx).Span(trace.SpanVerification)
|
||||
vr, err := sc.verifier.Verify(ctx)
|
||||
verifySpan.End()
|
||||
if err != nil {
|
||||
// "No plan / no tests" is not an error (DetectPlan returns an empty
|
||||
// plan), so a non-nil error means verification could not run at all —
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/streamjson"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -279,6 +280,16 @@ type Options struct {
|
||||
// request), so every existing caller is unaffected. A returned error is
|
||||
// non-fatal: the run continues on the current model.
|
||||
ModelSwitcher func(ctx context.Context, modelID string) (Provider, error)
|
||||
// Trace, when set, records per-turn timing for the run: the loop stamps
|
||||
// spans (prompt build, generation, tool execution, permission wait,
|
||||
// compaction, provider connect) and counters (model requests, tool calls,
|
||||
// retries, tokens) into it. nil DISABLES tracing entirely — every stamp is
|
||||
// nil-safe and the loop is byte-identical to an untraced run. The caller
|
||||
// owns the recorder: Run stamps into it but does not Finish or emit it.
|
||||
// A fresh Recorder is required per Run — reusing one across runs merges
|
||||
// their spans, counters, and first-event timestamps, and Finish freezes a
|
||||
// recorder so no further stamps take.
|
||||
Trace *trace.Recorder
|
||||
// SelfCorrect, when set, runs a post-edit verify-and-correct cycle after a
|
||||
// mutating tool call: it verifies the changed files (LSP diagnostics + project
|
||||
// tests) and feeds failures back to the model to fix, bounded by an attempt
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
package aimlapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PaymentMethod selects how a partner-checkout top-up is paid for.
|
||||
type PaymentMethod string
|
||||
|
||||
const (
|
||||
PaymentMethodCard PaymentMethod = "card"
|
||||
PaymentMethodCrypto PaymentMethod = "crypto"
|
||||
)
|
||||
|
||||
// SessionStatus is the lifecycle state of a partner-checkout session, as reported
|
||||
// by the aimlapi.com backend while polling for payment.
|
||||
type SessionStatus string
|
||||
|
||||
const (
|
||||
SessionStatusPendingAuth SessionStatus = "pending_auth"
|
||||
SessionStatusPendingPayment SessionStatus = "pending_payment"
|
||||
SessionStatusPaid SessionStatus = "paid"
|
||||
SessionStatusExchanging SessionStatus = "exchanging"
|
||||
SessionStatusExchanged SessionStatus = "exchanged"
|
||||
SessionStatusCancelled SessionStatus = "cancelled"
|
||||
SessionStatusExpired SessionStatus = "expired"
|
||||
SessionStatusFailed SessionStatus = "failed"
|
||||
)
|
||||
|
||||
// AuthResult is the bearer token minted by a passwordless sign-in / sign-up.
|
||||
type AuthResult struct {
|
||||
Token string `json:"token"`
|
||||
Exp int64 `json:"exp"`
|
||||
}
|
||||
|
||||
// PartnerCheckoutSession is a partner-attributed top-up session; its SessionToken
|
||||
// addresses it on later pay/exchange/poll calls.
|
||||
type PartnerCheckoutSession struct {
|
||||
ID string `json:"id"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
PartnerID string `json:"partnerId"`
|
||||
PartnerName *string `json:"partnerName"`
|
||||
UserID *int64 `json:"userId"`
|
||||
AmountUSDMinor *int64 `json:"amountUsdMinor"`
|
||||
Status SessionStatus `json:"status"`
|
||||
IssuedKeyID *string `json:"issuedKeyId"`
|
||||
ReturnURL *string `json:"returnUrl"`
|
||||
}
|
||||
|
||||
// PaymentSession is the hosted payment-provider checkout; PayURL is the page the
|
||||
// user opens to pay.
|
||||
type PaymentSession struct {
|
||||
ProviderSessionID string `json:"providerSessionId"`
|
||||
PayURL string `json:"payUrl"`
|
||||
}
|
||||
|
||||
// PayResult pairs the hosted checkout with the updated partner-checkout session
|
||||
// returned when a top-up payment is initiated.
|
||||
type PayResult struct {
|
||||
Checkout PaymentSession `json:"checkout"`
|
||||
PartnerCheckout PartnerCheckoutSession `json:"partnerCheckout"`
|
||||
}
|
||||
|
||||
// ExchangeResult holds the API key minted when a paid session is exchanged.
|
||||
type ExchangeResult struct {
|
||||
APIKey string `json:"apiKey"`
|
||||
APIKeyID string `json:"apiKeyId"`
|
||||
}
|
||||
|
||||
// TopUpByKeyRequest funds a partner-checkout session with the API key that owns
|
||||
// the account (Path A / env key), instead of an email session.
|
||||
type TopUpByKeyRequest struct {
|
||||
SessionToken string
|
||||
AmountUSDMinor int
|
||||
PaymentSessionID string // idempotency: same id → same checkout, never a second charge
|
||||
AutoTopUp bool // enroll the account in automatic top-up at checkout (card, saved off-session)
|
||||
SuccessURL string
|
||||
CancelURL string
|
||||
}
|
||||
|
||||
// TopUpByKeyResult is the hosted checkout returned by TopUpByKey.
|
||||
type TopUpByKeyResult struct {
|
||||
Checkout PaymentSession `json:"checkout"`
|
||||
}
|
||||
|
||||
// APIError is a non-2xx HTTP response from the aimlapi.com API. Status carries the
|
||||
// code so callers can branch (e.g. 401 = bad key, 5xx = retryable).
|
||||
type APIError struct {
|
||||
Message string
|
||||
Status int
|
||||
Body string
|
||||
}
|
||||
|
||||
// Error renders the method, endpoint, status, and (when present) the response body.
|
||||
func (e APIError) Error() string {
|
||||
if strings.TrimSpace(e.Body) != "" {
|
||||
return fmt.Sprintf("%s: HTTP %d: %s", e.Message, e.Status, strings.TrimSpace(e.Body))
|
||||
}
|
||||
return fmt.Sprintf("%s: HTTP %d", e.Message, e.Status)
|
||||
}
|
||||
|
||||
// Client talks to the aimlapi.com auth, app, and inference APIs for the onboarding
|
||||
// and partner-checkout top-up flows.
|
||||
type Client struct {
|
||||
endpoints Endpoints
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewClient(endpoints Endpoints, httpClient *http.Client) *Client {
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: 60 * time.Second}
|
||||
}
|
||||
return &Client{endpoints: endpoints, httpClient: httpClient}
|
||||
}
|
||||
|
||||
// CreateSession opens a partner-checkout session attributed to partnerID; the
|
||||
// returned SessionToken addresses it on the pay/exchange/poll calls.
|
||||
func (c *Client) CreateSession(ctx context.Context, partnerID string, partnerName string, returnURL string) (PartnerCheckoutSession, error) {
|
||||
body := map[string]any{"partnerId": partnerID}
|
||||
if strings.TrimSpace(partnerName) != "" {
|
||||
body["partnerName"] = strings.TrimSpace(partnerName)
|
||||
}
|
||||
if strings.TrimSpace(returnURL) != "" {
|
||||
body["returnUrl"] = strings.TrimSpace(returnURL)
|
||||
}
|
||||
var result PartnerCheckoutSession
|
||||
err := c.request(ctx, http.MethodPost, strings.TrimRight(c.endpoints.AppBaseURL, "/")+"/v3/partner-checkout/sessions", "", body, &result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// GetSession fetches the current state of a partner-checkout session; used to poll
|
||||
// for payment completion.
|
||||
func (c *Client) GetSession(ctx context.Context, sessionToken string) (PartnerCheckoutSession, error) {
|
||||
var result PartnerCheckoutSession
|
||||
err := c.request(ctx, http.MethodGet, strings.TrimRight(c.endpoints.AppBaseURL, "/")+"/v3/partner-checkout/sessions/"+urlPathEscape(sessionToken), "", nil, &result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Pay initiates payment for a session and returns the hosted checkout URL.
|
||||
// autoTopUp enrolls the account in automatic top-up at checkout time.
|
||||
func (c *Client) Pay(ctx context.Context, bearer string, sessionToken string, amountUSDMinor int, paymentSessionID string, method PaymentMethod, successURL string, cancelURL string, autoTopUp bool) (PayResult, error) {
|
||||
body := map[string]any{
|
||||
"amountUsdMinor": amountUSDMinor,
|
||||
"paymentSessionId": paymentSessionID,
|
||||
"method": method,
|
||||
}
|
||||
if strings.TrimSpace(successURL) != "" {
|
||||
body["successUrl"] = strings.TrimSpace(successURL)
|
||||
}
|
||||
if strings.TrimSpace(cancelURL) != "" {
|
||||
body["cancelUrl"] = strings.TrimSpace(cancelURL)
|
||||
}
|
||||
// Only sent when enabled: enrolls the account in automatic top-up. The backend
|
||||
// honours this field at checkout time.
|
||||
if autoTopUp {
|
||||
body["autoTopUp"] = true
|
||||
}
|
||||
var result PayResult
|
||||
err := c.request(ctx, http.MethodPost, strings.TrimRight(c.endpoints.AppBaseURL, "/")+"/v3/partner-checkout/sessions/"+urlPathEscape(sessionToken)+"/pay", bearer, body, &result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Exchange converts a paid session into a freshly minted API key (new-account flow).
|
||||
func (c *Client) Exchange(ctx context.Context, bearer string, sessionToken string) (ExchangeResult, error) {
|
||||
var result ExchangeResult
|
||||
err := c.request(ctx, http.MethodPost, strings.TrimRight(c.endpoints.AppBaseURL, "/")+"/v3/partner-checkout/sessions/"+urlPathEscape(sessionToken)+"/exchange", bearer, nil, &result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// TopUpByKey funds a partner-checkout session using the raw API key that owns the
|
||||
// account (Path A / env key), returning the hosted checkout. It binds the session
|
||||
// to the key's account server-side, so no email session is needed and no key is
|
||||
// exchanged. req.PaymentSessionID makes a retry idempotent (same id → same
|
||||
// checkout, never a second charge).
|
||||
func (c *Client) TopUpByKey(ctx context.Context, apiKey string, req TopUpByKeyRequest) (TopUpByKeyResult, error) {
|
||||
body := map[string]any{
|
||||
"sessionToken": req.SessionToken,
|
||||
"amountUsdMinor": req.AmountUSDMinor,
|
||||
"paymentSessionId": req.PaymentSessionID,
|
||||
}
|
||||
if req.AutoTopUp {
|
||||
body["autoTopUp"] = true
|
||||
}
|
||||
if strings.TrimSpace(req.SuccessURL) != "" {
|
||||
body["successUrl"] = strings.TrimSpace(req.SuccessURL)
|
||||
}
|
||||
if strings.TrimSpace(req.CancelURL) != "" {
|
||||
body["cancelUrl"] = strings.TrimSpace(req.CancelURL)
|
||||
}
|
||||
var result TopUpByKeyResult
|
||||
err := c.request(ctx, http.MethodPost, c.billingV2URL("/billing/topup"), apiKey, body, &result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// billingV2URL builds a v2 billing API URL from the inference base. The top-up
|
||||
// endpoint lives next to the balance endpoint on the API gateway, one version up
|
||||
// (".../v1" → ".../v2/billing/..."), so it re-anchors the version while keeping any
|
||||
// host/prefix an override supplied.
|
||||
func (c *Client) billingV2URL(path string) string {
|
||||
base := strings.TrimRight(strings.TrimSpace(c.endpoints.InferenceBaseURL), "/")
|
||||
base = strings.TrimSuffix(base, "/v1")
|
||||
return strings.TrimRight(base, "/") + "/v2" + path
|
||||
}
|
||||
|
||||
func (c *Client) request(ctx context.Context, method string, endpoint string, bearer string, body any, out any) error {
|
||||
var reader io.Reader
|
||||
if body != nil {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reader = bytes.NewReader(data)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, method, endpoint, reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Header.Set("Accept", "application/json")
|
||||
if body != nil {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if strings.TrimSpace(bearer) != "" {
|
||||
request.Header.Set("Authorization", "Bearer "+strings.TrimSpace(bearer))
|
||||
}
|
||||
response, err := c.httpClient.Do(request)
|
||||
if err != nil {
|
||||
return fmt.Errorf("network request to %s failed: %w", endpoint, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
text, readErr := io.ReadAll(io.LimitReader(response.Body, 1<<20))
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return APIError{Message: method + " " + endpoint, Status: response.StatusCode, Body: string(text)}
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if len(bytes.TrimSpace(text)) == 0 {
|
||||
return APIError{Message: method + " " + endpoint + " returned empty body", Status: response.StatusCode}
|
||||
}
|
||||
if err := json.Unmarshal(text, out); err != nil {
|
||||
return APIError{Message: method + " " + endpoint + " returned non-JSON body", Status: response.StatusCode, Body: string(text)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func urlPathEscape(value string) string {
|
||||
return url.PathEscape(value)
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package aimlapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Additional client methods backing the interactive TUI onboarding flow. They
|
||||
// target the endpoints shipped in the aimlapi backend:
|
||||
// - GET {inference}/billing/balance (Path A: validate key + balance)
|
||||
// - PATCH{auth}/v1/auth/account (Path B: does the account exist?)
|
||||
// - POST {auth}/v1/auth/sign-in/code(/verify) (Path B existing: passwordless login)
|
||||
// - POST {auth}/v1/auth/account/passwordless (Path B new: email-only signup)
|
||||
// - POST {app}/v1/keys (mint a key once we hold a session)
|
||||
|
||||
// BalanceResult is the wallet balance returned by GetBalance, with the backend's
|
||||
// low-balance flag and threshold.
|
||||
type BalanceResult struct {
|
||||
Balance float64 `json:"balance"`
|
||||
LowBalance bool `json:"lowBalance"`
|
||||
LowBalanceThreshold float64 `json:"lowBalanceThreshold"`
|
||||
}
|
||||
|
||||
// CheckResult reports whether an email already has an account, driving the
|
||||
// sign-in vs sign-up branch of the onboarding flow.
|
||||
type CheckResult struct {
|
||||
// "sign-in" (account exists) or "sign-up" (no account yet).
|
||||
Action string `json:"action"`
|
||||
Provider *string `json:"provider"`
|
||||
}
|
||||
|
||||
// CreatedKey is an API key minted by CreateKey.
|
||||
type CreatedKey struct {
|
||||
Key string `json:"key"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
func (c *Client) authURL(path string) string {
|
||||
return strings.TrimRight(c.endpoints.AuthBaseURL, "/") + path
|
||||
}
|
||||
|
||||
func (c *Client) appURL(path string) string {
|
||||
return strings.TrimRight(c.endpoints.AppBaseURL, "/") + path
|
||||
}
|
||||
|
||||
func (c *Client) inferenceURL(path string) string {
|
||||
return strings.TrimRight(c.endpoints.InferenceBaseURL, "/") + path
|
||||
}
|
||||
|
||||
// GetBalance doubles as key validation: a 401 means the key is invalid, a 2xx
|
||||
// returns the wallet balance. Auth is the raw API key itself.
|
||||
func (c *Client) GetBalance(ctx context.Context, apiKey string) (BalanceResult, error) {
|
||||
var result BalanceResult
|
||||
err := c.request(ctx, http.MethodGet, c.inferenceURL("/billing/balance"), apiKey, nil, &result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// CheckAccount reports whether an email already has an aimlapi.com account
|
||||
// (Action "sign-in") or not ("sign-up").
|
||||
func (c *Client) CheckAccount(ctx context.Context, email string) (CheckResult, error) {
|
||||
var result CheckResult
|
||||
err := c.request(ctx, http.MethodPatch, c.authURL("/v1/auth/account"), "", map[string]any{"email": email}, &result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// SendSignInCode emails a 6-digit sign-in code to an existing account.
|
||||
func (c *Client) SendSignInCode(ctx context.Context, email string) error {
|
||||
return c.request(ctx, http.MethodPost, c.authURL("/v1/auth/sign-in/code"), "", map[string]any{"email": email}, nil)
|
||||
}
|
||||
|
||||
// VerifySignInCode exchanges an emailed code for a session bearer token.
|
||||
func (c *Client) VerifySignInCode(ctx context.Context, email string, code string) (AuthResult, error) {
|
||||
var result AuthResult
|
||||
err := c.request(ctx, http.MethodPost, c.authURL("/v1/auth/sign-in/code/verify"), "", map[string]any{"email": email, "code": code}, &result)
|
||||
if err == nil && strings.TrimSpace(result.Token) == "" {
|
||||
err = fmt.Errorf("aimlapi.com did not return an auth token")
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
// CreatePasswordlessAccount registers a new email-only account and returns its
|
||||
// session bearer token.
|
||||
func (c *Client) CreatePasswordlessAccount(ctx context.Context, email string) (AuthResult, error) {
|
||||
var result AuthResult
|
||||
err := c.request(ctx, http.MethodPost, c.authURL("/v1/auth/account/passwordless"), "", map[string]any{"email": email}, &result)
|
||||
if err == nil && strings.TrimSpace(result.Token) == "" {
|
||||
err = fmt.Errorf("aimlapi.com did not return an auth token")
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
// CreateKey mints a fresh API key for a session-holding user.
|
||||
func (c *Client) CreateKey(ctx context.Context, bearer string, name string) (CreatedKey, error) {
|
||||
body := map[string]any{}
|
||||
if strings.TrimSpace(name) != "" {
|
||||
body["name"] = strings.TrimSpace(name)
|
||||
}
|
||||
var result CreatedKey
|
||||
err := c.request(ctx, http.MethodPost, c.appURL("/v1/keys"), bearer, body, &result)
|
||||
if err == nil && strings.TrimSpace(result.Key) == "" {
|
||||
err = fmt.Errorf("aimlapi.com did not return an API key")
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package aimlapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCreateKeySendsAuthenticatedRequestAndDecodesKey(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/v1/keys" {
|
||||
t.Fatalf("request = %s %s, want POST /v1/keys", r.Method, r.URL.Path)
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer session-token" {
|
||||
t.Fatalf("Authorization = %q", got)
|
||||
}
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body["name"] != "Zero CLI" {
|
||||
t.Fatalf("name = %q", body["name"])
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"key":"sk-issued","id":"key-id"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(Endpoints{AppBaseURL: server.URL}, server.Client())
|
||||
key, err := client.CreateKey(context.Background(), " session-token ", " Zero CLI ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if key.Key != "sk-issued" || key.ID != "key-id" {
|
||||
t.Fatalf("key = %#v", key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateKeyRejectsEmptyKeyResponse(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"key-id"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(Endpoints{AppBaseURL: server.URL}, server.Client())
|
||||
if _, err := client.CreateKey(context.Background(), "session-token", "Zero"); err == nil {
|
||||
t.Fatal("CreateKey accepted a response without an API key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthMethodsRejectEmptyTokenResponses(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"token":""}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(Endpoints{AuthBaseURL: server.URL}, server.Client())
|
||||
tests := []struct {
|
||||
name string
|
||||
call func() error
|
||||
}{
|
||||
{
|
||||
name: "verify sign-in code",
|
||||
call: func() error {
|
||||
_, err := client.VerifySignInCode(context.Background(), "[email protected]", "123456")
|
||||
return err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "create passwordless account",
|
||||
call: func() error {
|
||||
_, err := client.CreatePasswordlessAccount(context.Background(), "[email protected]")
|
||||
return err
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := test.call(); err == nil {
|
||||
t.Fatal("accepted a response without an auth token")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientReturnsTypedAPIError(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, "invalid key", http.StatusUnauthorized)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(Endpoints{InferenceBaseURL: server.URL}, server.Client())
|
||||
_, err := client.GetBalance(context.Background(), "bad-key")
|
||||
var apiErr APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("error = %T %v, want APIError", err, err)
|
||||
}
|
||||
if apiErr.Status != http.StatusUnauthorized || apiErr.Body != "invalid key\n" {
|
||||
t.Fatalf("APIError = %#v", apiErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedResponseRejectsEmptySuccessBody(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(Endpoints{InferenceBaseURL: server.URL}, server.Client())
|
||||
if _, err := client.GetBalance(context.Background(), "key"); err == nil {
|
||||
t.Fatal("GetBalance accepted an empty successful response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBodylessOperationAcceptsEmptySuccessBody(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(Endpoints{AuthBaseURL: server.URL}, server.Client())
|
||||
if err := client.SendSignInCode(context.Background(), "[email protected]"); err != nil {
|
||||
t.Fatalf("SendSignInCode() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAmountUSD(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
value string
|
||||
want int
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "default", want: DefaultAmountUSDMinor},
|
||||
{name: "minimum", value: "20", want: 2000},
|
||||
{name: "cents", value: "25.49", want: 2549},
|
||||
{name: "below minimum", value: "19.99", wantErr: true},
|
||||
{name: "above maximum", value: "10001", wantErr: true},
|
||||
{name: "not a number", value: "twenty", wantErr: true},
|
||||
{name: "nan", value: "NaN", wantErr: true},
|
||||
{name: "infinity", value: "Inf", wantErr: true},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := ParseAmountUSD(test.value)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("ParseAmountUSD(%q) error = %v", test.value, err)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Fatalf("ParseAmountUSD(%q) = %d, want %d", test.value, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package aimlapi
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Endpoints struct {
|
||||
AuthBaseURL string
|
||||
AppBaseURL string
|
||||
InferenceBaseURL string
|
||||
PayBaseURL string
|
||||
// VerificationBaseURL is the aimlapi web-app frontend base. The browser is sent
|
||||
// back here after the partner checkout completes (see BuildPartnerReturnURL), so
|
||||
// it must point at the same environment's front as the one the CLI is talking to
|
||||
// to land the user there. Empty falls back to DefaultReturnURL.
|
||||
VerificationBaseURL string
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultPartnerID = "part_62yQoGYDq4Yqnrj2R1iGrDNJ"
|
||||
DefaultPartnerName = "Gitlawb"
|
||||
PartnerHeaderName = "X-AIMLAPI-Partner-ID"
|
||||
// DefaultReturnURL is the https fallback the browser is sent to after the
|
||||
// checkout / OAuth consent finishes when no frontend base is known. It must be
|
||||
// a real web page the browser can open — NOT a custom scheme like
|
||||
// zero://aimlapi/complete, which fails with "the scheme does not have a
|
||||
// registered handler" because the CLI registers no OS protocol handler. The
|
||||
// CLI learns of success by polling; this URL is purely the browser's landing.
|
||||
DefaultReturnURL = "https://aimlapi.com/app"
|
||||
DefaultModel = "anthropic/claude-sonnet-5"
|
||||
|
||||
MinAmountUSDMinor = 2000
|
||||
MaxAmountUSDMinor = 1000000
|
||||
DefaultAmountUSDMinor = 2500
|
||||
)
|
||||
|
||||
func ResolveEndpoints() Endpoints {
|
||||
return Endpoints{
|
||||
AuthBaseURL: envOrDefault("AIMLAPI_AUTH_URL", "https://auth.aimlapi.com"),
|
||||
AppBaseURL: envOrDefault("AIMLAPI_APP_URL", "https://app.aimlapi.com"),
|
||||
InferenceBaseURL: envOrDefault("AIMLAPI_INFERENCE_URL", "https://api.aimlapi.com/v1"),
|
||||
PayBaseURL: envOrDefault("AIMLAPI_PAY_URL", "https://pay.aimlapi.com"),
|
||||
VerificationBaseURL: envOrDefault("AIMLAPI_VERIFICATION_BASE_URL", "https://aimlapi.com/app"),
|
||||
}
|
||||
}
|
||||
|
||||
// ResolvePartnerID applies the same explicit-option, environment, and default
|
||||
// precedence everywhere partner attribution is used. Keeping checkout creation
|
||||
// and saved inference profiles on one resolver prevents them from attributing
|
||||
// the same user to different partners.
|
||||
func ResolvePartnerID(explicit string) string {
|
||||
if value := strings.TrimSpace(explicit); value != "" {
|
||||
return value
|
||||
}
|
||||
return envOrDefault("AIMLAPI_PARTNER_ID", DefaultPartnerID)
|
||||
}
|
||||
|
||||
// WithResolvedPartnerHeader returns a copy with the effective partner ID set.
|
||||
// Header matching is case-insensitive so an existing spelling is replaced rather
|
||||
// than duplicated. Callers remain responsible for applying catalog attribution
|
||||
// only to the canonical AIMLAPI endpoint.
|
||||
func WithResolvedPartnerHeader(headers map[string]string) map[string]string {
|
||||
resolved := make(map[string]string, len(headers)+1)
|
||||
for key, value := range headers {
|
||||
if strings.EqualFold(strings.TrimSpace(key), PartnerHeaderName) {
|
||||
continue
|
||||
}
|
||||
resolved[key] = value
|
||||
}
|
||||
resolved[PartnerHeaderName] = ResolvePartnerID("")
|
||||
return resolved
|
||||
}
|
||||
|
||||
func BuildPartnerCheckoutReturnURLs(appBaseURL string, sessionToken string) (successURL string, cancelURL string) {
|
||||
base := strings.TrimRight(strings.TrimSpace(appBaseURL), "/")
|
||||
if base == "" {
|
||||
// With no base there is nothing to anchor an absolute URL to. Return empty
|
||||
// so the caller omits the return URLs entirely (client.Pay skips empty ones,
|
||||
// leaving the backend's generic checkout) instead of emitting a broken
|
||||
// relative "/checkout?...".
|
||||
return "", ""
|
||||
}
|
||||
token := url.QueryEscape(sessionToken)
|
||||
return base + "/checkout?checkout=success&partnerCheckout=1&sessionToken=" + token,
|
||||
base + "/checkout?checkout=cancel&partnerCheckout=1&sessionToken=" + token
|
||||
}
|
||||
|
||||
// BuildPartnerReturnURL returns the https page the browser is sent back to once
|
||||
// the partner checkout or OAuth consent completes — the aimlapi web app the user
|
||||
// opened the flow from. frontendBaseURL is normally endpoints.VerificationBaseURL
|
||||
// (the front that hosts the login/consent page), so the return follows the same
|
||||
// environment. AIMLAPI_RETURN_URL overrides it outright; an empty base falls back
|
||||
// to DefaultReturnURL. It is deliberately NOT a custom scheme (zero://…): no
|
||||
// browser can hand a custom scheme off without an OS-registered handler, which a
|
||||
// CLI does not install.
|
||||
func BuildPartnerReturnURL(frontendBaseURL string) string {
|
||||
if override := strings.TrimSpace(os.Getenv("AIMLAPI_RETURN_URL")); override != "" {
|
||||
return override
|
||||
}
|
||||
if base := strings.TrimRight(strings.TrimSpace(frontendBaseURL), "/"); base != "" {
|
||||
return base
|
||||
}
|
||||
return DefaultReturnURL
|
||||
}
|
||||
|
||||
func envOrDefault(name string, fallback string) string {
|
||||
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package aimlapi
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestResolveEndpointsDefaults(t *testing.T) {
|
||||
t.Setenv("AIMLAPI_AUTH_URL", "")
|
||||
t.Setenv("AIMLAPI_APP_URL", "")
|
||||
t.Setenv("AIMLAPI_INFERENCE_URL", "")
|
||||
t.Setenv("AIMLAPI_PAY_URL", "")
|
||||
|
||||
endpoints := ResolveEndpoints()
|
||||
if endpoints.AuthBaseURL != "https://auth.aimlapi.com" ||
|
||||
endpoints.AppBaseURL != "https://app.aimlapi.com" ||
|
||||
endpoints.InferenceBaseURL != "https://api.aimlapi.com/v1" ||
|
||||
endpoints.PayBaseURL != "https://pay.aimlapi.com" {
|
||||
t.Fatalf("unexpected endpoints: %#v", endpoints)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPartnerHeaderUsesEnvironmentOverrideWithoutDuplicates(t *testing.T) {
|
||||
t.Setenv("AIMLAPI_PARTNER_ID", "part_override")
|
||||
|
||||
headers := WithResolvedPartnerHeader(map[string]string{
|
||||
"x-aimlapi-partner-id": "part_catalog",
|
||||
"X-AIMLAPI-Integration-Repo": "Gitlawb/zero",
|
||||
})
|
||||
|
||||
if got := headers[PartnerHeaderName]; got != "part_override" {
|
||||
t.Fatalf("partner header = %q, want part_override", got)
|
||||
}
|
||||
if _, ok := headers["x-aimlapi-partner-id"]; ok {
|
||||
t.Fatalf("case-insensitive partner header was duplicated: %#v", headers)
|
||||
}
|
||||
if got := headers["X-AIMLAPI-Integration-Repo"]; got != "Gitlawb/zero" {
|
||||
t.Fatalf("unrelated header changed: %#v", headers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPartnerReturnURL(t *testing.T) {
|
||||
t.Setenv("AIMLAPI_RETURN_URL", "")
|
||||
// The frontend base (the web app the flow was opened from) is returned as-is,
|
||||
// trailing slash trimmed — an https page the browser can actually land on.
|
||||
if got := BuildPartnerReturnURL("https://aimlapi.com/app/"); got != "https://aimlapi.com/app" {
|
||||
t.Fatalf("return URL = %q, want the frontend base", got)
|
||||
}
|
||||
// An empty base falls back to the https default, never a custom scheme.
|
||||
if got := BuildPartnerReturnURL(""); got != DefaultReturnURL {
|
||||
t.Fatalf("empty base return URL = %q, want %q", got, DefaultReturnURL)
|
||||
}
|
||||
|
||||
t.Setenv("AIMLAPI_RETURN_URL", "https://example.test/done")
|
||||
if got := BuildPartnerReturnURL("https://aimlapi.com/app"); got != "https://example.test/done" {
|
||||
t.Fatalf("env override return URL = %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package aimlapi
|
||||
|
||||
// UI strings shared by the first-run and provider-wizard onboarding surfaces.
|
||||
|
||||
const (
|
||||
// Path A — existing API key. Pick-path prompt + option labels.
|
||||
MsgAPIKeyInputPrompt = "Enter your aimlapi.com key."
|
||||
MsgAPIKeyInvalid = "API key is invalid. Please make sure you enter a valid aimlapi.com key."
|
||||
MsgPickPathPrompt = "Do you have aimlapi.com key?"
|
||||
MsgPickPathHaveKey = "I already have aimlapi.com key"
|
||||
MsgPickPathHaveHint = "Proceed to paste the key"
|
||||
MsgPickPathNewUser = "I am a new user"
|
||||
MsgPickPathNewHint = "One click set up"
|
||||
// Path B — email.
|
||||
MsgEnterEmail = "Enter your email.\nTo access aimlapi.com dashboard."
|
||||
MsgEmailInvalid = "Email format is incorrect."
|
||||
MsgAccountActionInvalid = "aimlapi.com returned an unsupported account action. Please try again."
|
||||
// %s = email
|
||||
MsgCodeSent = "We sent a 6-digit code to %s.\nEnter it below to continue."
|
||||
MsgCodeIncorrect = "Code you've entered is incorrect."
|
||||
|
||||
// Balance check.
|
||||
MsgLowBalance = "Your aimlapi.com balance is running low.\nIt is recommended to top up your balance."
|
||||
MsgEverythingRuns = "Everything is ready."
|
||||
// Low-balance choices (spec wording).
|
||||
MsgLowBalanceTopUp = "Sure, let's do that"
|
||||
MsgLowBalanceSkip = "I'll skip topping up the balance for now"
|
||||
|
||||
// Top-up workflow.
|
||||
MsgTopUpPrompt = "Add credits.\nEnter an amount (min $20)."
|
||||
MsgAmountRequired = "Please enter a top-up amount."
|
||||
// %s = checkout URL
|
||||
MsgTopUpBrowserFallback = "If the browser did not open automatically please use this link to top up your account: %s"
|
||||
MsgTopUpFailed = "Top up failed. Please try again."
|
||||
MsgTopUpSuccess = "Top-up successful."
|
||||
|
||||
// Success — key delivered.
|
||||
// %s = amount in dollars (e.g. "25")
|
||||
MsgTopUpAddedFmt = "$%s has been added to your balance."
|
||||
// %s = email
|
||||
MsgSuccessMagicLink = "We've emailed you a magic link to %s. Use it to access your aimlapi.com account and review your usage and balance."
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
package aimlapi
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidEmailRejectsIncompleteDomains(t *testing.T) {
|
||||
tests := []struct {
|
||||
value string
|
||||
valid bool
|
||||
}{
|
||||
{value: "[email protected]", valid: true},
|
||||
{value: "[email protected]", valid: true},
|
||||
{value: "123@123.", valid: false},
|
||||
{value: "123@123", valid: false},
|
||||
{value: "[email protected]", valid: false},
|
||||
{value: "[email protected]", valid: false},
|
||||
{value: "[email protected]", valid: false},
|
||||
{value: "[email protected]", valid: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.value, func(t *testing.T) {
|
||||
if got := ValidEmail(test.value); got != test.valid {
|
||||
t.Fatalf("ValidEmail(%q) = %v, want %v", test.value, got, test.valid)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
package aimlapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// StreamTopUpOptions drives a session-based partner-checkout top-up for the
|
||||
// interactive (TUI) onboarding. The caller already holds a user session —
|
||||
// obtained from an email-code sign-in (Path B existing) or a
|
||||
// passwordless new account (Path B sign-up) — so this skips authentication and
|
||||
// runs create-session → pay → wait-for-payment, optionally exchanging the paid
|
||||
// session into a fresh key (new accounts) versus keeping the caller's key.
|
||||
type StreamTopUpOptions struct {
|
||||
SessionToken string // user session bearer (required)
|
||||
AmountUSD string // dollars; parsed via ParseAmountUSD (min $20)
|
||||
// InferenceBaseURL pins key-bound billing and the returned provider profile to
|
||||
// the endpoint that validated the credential. Empty uses ResolveEndpoints.
|
||||
InferenceBaseURL string
|
||||
Method PaymentMethod
|
||||
AutoTopUp bool // enroll the account in automatic top-up at checkout time
|
||||
Exchange bool // exchange the paid session into a new key (new accounts)
|
||||
Model string
|
||||
PartnerID string
|
||||
PartnerName string
|
||||
NoOpen bool
|
||||
|
||||
// ResumeSessionToken, when set, resumes that partner-checkout session instead
|
||||
// of opening a new one, so a retry after an ambiguous failure (a lost Pay/
|
||||
// Exchange response) can never double-charge or strand an already-paid session.
|
||||
ResumeSessionToken string
|
||||
// PaymentSessionID is generated once per amount/auto-top-up intent and reused
|
||||
// on retry so an ambiguous Pay response cannot create a second checkout.
|
||||
PaymentSessionID string
|
||||
|
||||
HTTPClient *http.Client
|
||||
OpenBrowser func(string) error
|
||||
OnStatus func(Status, string)
|
||||
// OnSession reports the partner-checkout session token the moment it exists (a
|
||||
// fresh CreateSession, or the resumed one), so the caller can retain it and
|
||||
// resume on a later failure. It fires with "" when the session is dead and the
|
||||
// retained token should be dropped (a fresh run must start over).
|
||||
OnSession func(sessionToken string)
|
||||
}
|
||||
|
||||
// topupPhase is where StreamTopUp enters its create-checkout → poll → exchange
|
||||
// pipeline. A fresh run starts at phasePay; a resume skips ahead to whatever the
|
||||
// existing session's status makes safe.
|
||||
type topupPhase int
|
||||
|
||||
const (
|
||||
phasePay topupPhase = iota // (re-)open the Stripe checkout, then poll + exchange
|
||||
phasePoll // checkout already open; poll for payment, never re-pay
|
||||
phaseExchange // already paid; exchange only (recovers a lost key)
|
||||
phaseWaitExchange // an exchange claim is in flight; poll, never call Exchange twice
|
||||
)
|
||||
|
||||
// StreamTopUp performs the browser-checkout top-up and reports progress via
|
||||
// OnStatus.
|
||||
func StreamTopUp(ctx context.Context, options StreamTopUpOptions) (ProvisionedKey, error) {
|
||||
endpoints := ResolveEndpoints()
|
||||
if baseURL := strings.TrimSpace(options.InferenceBaseURL); baseURL != "" {
|
||||
endpoints.InferenceBaseURL = baseURL
|
||||
}
|
||||
client := NewClient(endpoints, options.HTTPClient)
|
||||
partnerID := ResolvePartnerID(options.PartnerID)
|
||||
partnerName := firstNonEmpty(options.PartnerName, DefaultPartnerName)
|
||||
method := options.Method
|
||||
if method != PaymentMethodCrypto {
|
||||
method = PaymentMethodCard
|
||||
}
|
||||
amount, err := ParseAmountUSD(options.AmountUSD)
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
if strings.TrimSpace(options.SessionToken) == "" {
|
||||
return ProvisionedKey{}, fmt.Errorf("a session is required to top up")
|
||||
}
|
||||
if strings.TrimSpace(options.PaymentSessionID) == "" {
|
||||
return ProvisionedKey{}, fmt.Errorf("a payment session id is required to top up")
|
||||
}
|
||||
|
||||
status(options.OnStatus, StatusCreatingSession, "")
|
||||
sessionToken, phase, err := resolveTopupSession(ctx, client, options, partnerID, partnerName, endpoints)
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
// Retain the live token so an ambiguous failure below can resume this session
|
||||
// rather than open a second checkout on the next attempt.
|
||||
reportSession(options.OnSession, sessionToken)
|
||||
|
||||
if phase <= phasePay {
|
||||
successURL, cancelURL := BuildPartnerCheckoutReturnURLs(endpoints.PayBaseURL, sessionToken)
|
||||
status(options.OnStatus, StatusOpeningCheckout, "")
|
||||
pay, err := client.Pay(ctx, options.SessionToken, sessionToken, amount, options.PaymentSessionID, method, successURL, cancelURL, options.AutoTopUp)
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
checkoutURL, err := validatedCheckoutURL(pay.Checkout.PayURL)
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
announceCheckout(options.OnStatus, options.OpenBrowser, options.NoOpen, checkoutURL)
|
||||
}
|
||||
|
||||
paidToken := sessionToken
|
||||
if phase <= phasePoll {
|
||||
status(options.OnStatus, StatusWaitingPayment, "")
|
||||
paid, err := pollUntilPaid(ctx, client, sessionToken, options.OnSession)
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
paidToken = paid.SessionToken
|
||||
if paid.Status == SessionStatusExchanging {
|
||||
// Another/lost request claimed exchange while we were polling payment.
|
||||
// Carry that fact into the provisioning branch so it follows the claim
|
||||
// instead of issuing a second single-flight Exchange request.
|
||||
phase = phaseWaitExchange
|
||||
}
|
||||
}
|
||||
|
||||
result := ProvisionedKey{
|
||||
BaseURL: endpoints.InferenceBaseURL,
|
||||
Model: firstNonEmpty(options.Model, DefaultModel),
|
||||
}
|
||||
if options.Exchange {
|
||||
status(options.OnStatus, StatusProvisioningKey, "")
|
||||
if phase == phaseWaitExchange {
|
||||
return ProvisionedKey{}, pollUntilExchangeSettled(ctx, client, sessionToken, options.OnSession)
|
||||
}
|
||||
exchange, err := client.Exchange(ctx, options.SessionToken, paidToken)
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
if strings.TrimSpace(exchange.APIKey) == "" {
|
||||
return ProvisionedKey{}, fmt.Errorf("aimlapi.com did not return an API key")
|
||||
}
|
||||
result.APIKey = exchange.APIKey
|
||||
result.APIKeyID = exchange.APIKeyID
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// resolveTopupSession returns the partner-checkout session token to drive and the
|
||||
// phase to enter. A fresh run opens a new session at phasePay. A resume inspects
|
||||
// the existing session and enters the earliest phase that cannot re-charge:
|
||||
// - pending_auth → phasePay: payment never started, so (re-)Pay is safe.
|
||||
// - pending_payment → phasePoll: a checkout is already open; poll, never re-Pay.
|
||||
// - paid → phaseExchange: exchange the paid session once.
|
||||
// - exchanging → phaseWaitExchange: another/lost request owns the atomic
|
||||
// exchange claim; poll it, never issue a second Exchange request.
|
||||
// - exchanged → terminal: the one-shot raw key is gone; point at dashboard
|
||||
// recovery and keep the token so a re-run repeats that guidance, not a charge.
|
||||
// - cancelled/expired/failed → terminal: drop the dead token so a re-run is fresh.
|
||||
func resolveTopupSession(ctx context.Context, client *Client, options StreamTopUpOptions, partnerID, partnerName string, endpoints Endpoints) (string, topupPhase, error) {
|
||||
resume := strings.TrimSpace(options.ResumeSessionToken)
|
||||
if resume == "" {
|
||||
session, err := client.CreateSession(ctx, partnerID, partnerName, BuildPartnerReturnURL(endpoints.VerificationBaseURL))
|
||||
if err != nil {
|
||||
return "", phasePay, err
|
||||
}
|
||||
return session.SessionToken, phasePay, nil
|
||||
}
|
||||
session, err := client.GetSession(ctx, resume)
|
||||
if err != nil {
|
||||
if isTerminalSessionAPIError(err) {
|
||||
reportSession(options.OnSession, "")
|
||||
}
|
||||
return "", phasePay, err
|
||||
}
|
||||
switch session.Status {
|
||||
case SessionStatusPendingAuth:
|
||||
return resume, phasePay, nil
|
||||
case SessionStatusPendingPayment:
|
||||
return resume, phasePoll, nil
|
||||
case SessionStatusPaid:
|
||||
return resume, phaseExchange, nil
|
||||
case SessionStatusExchanging:
|
||||
return resume, phaseWaitExchange, nil
|
||||
case SessionStatusExchanged:
|
||||
return "", phasePay, fmt.Errorf("session was already exchanged; rotate the key from the aimlapi.com dashboard")
|
||||
default:
|
||||
reportSession(options.OnSession, "") // drop the dead token → next run starts fresh
|
||||
return "", phasePay, fmt.Errorf("payment %s; re-run the top-up to try again", session.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func reportSession(callback func(string), sessionToken string) {
|
||||
if callback != nil {
|
||||
callback(sessionToken)
|
||||
}
|
||||
}
|
||||
|
||||
// announceCheckout opens the hosted checkout in the browser (unless suppressed)
|
||||
// and reports the URL either way, so the TUI can always show the direct link.
|
||||
func announceCheckout(onStatus func(Status, string), open func(string) error, noOpen bool, checkoutURL string) {
|
||||
if noOpen || open == nil {
|
||||
status(onStatus, StatusOpeningCheckout, checkoutURL)
|
||||
return
|
||||
}
|
||||
if err := open(checkoutURL); err != nil {
|
||||
status(onStatus, StatusOpeningCheckout, "Open manually: "+checkoutURL)
|
||||
return
|
||||
}
|
||||
status(onStatus, StatusOpeningCheckout, checkoutURL)
|
||||
}
|
||||
|
||||
func validatedCheckoutURL(value string) (string, error) {
|
||||
checkoutURL := strings.TrimSpace(value)
|
||||
parsed, err := url.Parse(checkoutURL)
|
||||
if err != nil || !parsed.IsAbs() || !strings.EqualFold(parsed.Scheme, "https") || parsed.Hostname() == "" || parsed.User != nil {
|
||||
return "", fmt.Errorf("payment provider did not return a valid HTTPS checkout URL")
|
||||
}
|
||||
return checkoutURL, nil
|
||||
}
|
||||
|
||||
// StreamTopUpByKeyOptions drives a top-up funded with the API key that already
|
||||
// owns the account (Path A pasted key, or AIMLAPI_API_KEY). Unlike StreamTopUp it
|
||||
// needs no email session and never exchanges a key — the caller already holds one,
|
||||
// so it only funds the wallet: create-session → pay-by-key → wait-for-payment.
|
||||
type StreamTopUpByKeyOptions struct {
|
||||
APIKey string // raw key that owns the account (required)
|
||||
AmountUSD string // dollars; parsed via ParseAmountUSD (min $20)
|
||||
InferenceBaseURL string // validated key endpoint; empty uses ResolveEndpoints
|
||||
AutoTopUp bool // enroll the account in automatic top-up at checkout time
|
||||
PartnerID string
|
||||
PartnerName string
|
||||
NoOpen bool
|
||||
|
||||
// ResumeSessionToken resumes an existing partner-checkout session instead of
|
||||
// opening a new one. PaymentSessionID is the idempotency handle carried across
|
||||
// retries so a re-issued pay returns the same checkout, never a second charge.
|
||||
ResumeSessionToken string
|
||||
PaymentSessionID string // required; generate once via NewPaymentSessionID, reuse on retry
|
||||
|
||||
HTTPClient *http.Client
|
||||
OpenBrowser func(string) error
|
||||
OnStatus func(Status, string)
|
||||
OnSession func(sessionToken string)
|
||||
}
|
||||
|
||||
// StreamTopUpByKey funds the account behind APIKey via the hosted checkout and
|
||||
// reports progress through OnStatus. It returns an empty ProvisionedKey: the
|
||||
// pasted/env key is unchanged, only the balance grows.
|
||||
func StreamTopUpByKey(ctx context.Context, options StreamTopUpByKeyOptions) (ProvisionedKey, error) {
|
||||
endpoints := ResolveEndpoints()
|
||||
if baseURL := strings.TrimSpace(options.InferenceBaseURL); baseURL != "" {
|
||||
endpoints.InferenceBaseURL = baseURL
|
||||
}
|
||||
client := NewClient(endpoints, options.HTTPClient)
|
||||
partnerID := ResolvePartnerID(options.PartnerID)
|
||||
partnerName := firstNonEmpty(options.PartnerName, DefaultPartnerName)
|
||||
amount, err := ParseAmountUSD(options.AmountUSD)
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
if strings.TrimSpace(options.APIKey) == "" {
|
||||
return ProvisionedKey{}, fmt.Errorf("an API key is required to top up")
|
||||
}
|
||||
if strings.TrimSpace(options.PaymentSessionID) == "" {
|
||||
return ProvisionedKey{}, fmt.Errorf("a payment session id is required to top up")
|
||||
}
|
||||
|
||||
status(options.OnStatus, StatusCreatingSession, "")
|
||||
sessionToken, phase, err := resolveByKeySession(ctx, client, options, partnerID, partnerName, endpoints)
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
reportSession(options.OnSession, sessionToken)
|
||||
|
||||
if phase <= phasePay {
|
||||
successURL, cancelURL := BuildPartnerCheckoutReturnURLs(endpoints.PayBaseURL, sessionToken)
|
||||
status(options.OnStatus, StatusOpeningCheckout, "")
|
||||
pay, err := client.TopUpByKey(ctx, options.APIKey, TopUpByKeyRequest{
|
||||
SessionToken: sessionToken,
|
||||
AmountUSDMinor: amount,
|
||||
PaymentSessionID: options.PaymentSessionID,
|
||||
AutoTopUp: options.AutoTopUp,
|
||||
SuccessURL: successURL,
|
||||
CancelURL: cancelURL,
|
||||
})
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
checkoutURL, err := validatedCheckoutURL(pay.Checkout.PayURL)
|
||||
if err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
announceCheckout(options.OnStatus, options.OpenBrowser, options.NoOpen, checkoutURL)
|
||||
}
|
||||
|
||||
if phase <= phasePoll {
|
||||
status(options.OnStatus, StatusWaitingPayment, "")
|
||||
if _, err := pollUntilPaid(ctx, client, sessionToken, options.OnSession); err != nil {
|
||||
return ProvisionedKey{}, err
|
||||
}
|
||||
}
|
||||
// No exchange: the account is funded and the caller keeps its existing key.
|
||||
return ProvisionedKey{}, nil
|
||||
}
|
||||
|
||||
// resolveByKeySession mirrors resolveTopupSession for the by-key flow. Because
|
||||
// by-key never exchanges, a paid/exchanging/exchanged session all mean "already
|
||||
// funded, done" (phaseExchange skips straight past pay + poll to the empty result).
|
||||
func resolveByKeySession(ctx context.Context, client *Client, options StreamTopUpByKeyOptions, partnerID, partnerName string, endpoints Endpoints) (string, topupPhase, error) {
|
||||
resume := strings.TrimSpace(options.ResumeSessionToken)
|
||||
if resume == "" {
|
||||
session, err := client.CreateSession(ctx, partnerID, partnerName, BuildPartnerReturnURL(endpoints.VerificationBaseURL))
|
||||
if err != nil {
|
||||
return "", phasePay, err
|
||||
}
|
||||
return session.SessionToken, phasePay, nil
|
||||
}
|
||||
session, err := client.GetSession(ctx, resume)
|
||||
if err != nil {
|
||||
if isTerminalSessionAPIError(err) {
|
||||
reportSession(options.OnSession, "")
|
||||
}
|
||||
return "", phasePay, err
|
||||
}
|
||||
switch session.Status {
|
||||
case SessionStatusPendingAuth:
|
||||
return resume, phasePay, nil
|
||||
case SessionStatusPendingPayment:
|
||||
return resume, phasePoll, nil
|
||||
case SessionStatusPaid, SessionStatusExchanging, SessionStatusExchanged:
|
||||
return resume, phaseExchange, nil
|
||||
default:
|
||||
reportSession(options.OnSession, "") // drop the dead token → next run starts fresh
|
||||
return "", phasePay, fmt.Errorf("payment %s; re-run the top-up to try again", session.Status)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
package aimlapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStreamTopUpByKeyFundsWithoutExchange(t *testing.T) {
|
||||
var topupCalls, exchangeCalls int
|
||||
var topupBody map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/partner-checkout/sessions"):
|
||||
_ = json.NewEncoder(w).Encode(PartnerCheckoutSession{SessionToken: "pcs_tok", Status: SessionStatusPendingAuth})
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/v2/billing/topup"):
|
||||
topupCalls++
|
||||
_ = json.NewDecoder(r.Body).Decode(&topupBody)
|
||||
_ = json.NewEncoder(w).Encode(TopUpByKeyResult{Checkout: PaymentSession{PayURL: "https://pay/checkout"}})
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/partner-checkout/sessions/"):
|
||||
_ = json.NewEncoder(w).Encode(PartnerCheckoutSession{SessionToken: "pcs_tok", Status: SessionStatusPaid})
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/exchange"):
|
||||
exchangeCalls++
|
||||
t.Errorf("by-key must never exchange a key")
|
||||
default:
|
||||
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("AIMLAPI_APP_URL", server.URL)
|
||||
t.Setenv("AIMLAPI_INFERENCE_URL", server.URL+"/v1")
|
||||
|
||||
var opened string
|
||||
result, err := StreamTopUpByKey(context.Background(), StreamTopUpByKeyOptions{
|
||||
APIKey: "sk-user",
|
||||
AmountUSD: "20",
|
||||
AutoTopUp: true,
|
||||
PaymentSessionID: "pay-1",
|
||||
OpenBrowser: func(u string) error { opened = u; return nil },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTopUpByKey = %v", err)
|
||||
}
|
||||
if result.APIKey != "" {
|
||||
t.Fatalf("by-key must not mint a key, got %q", result.APIKey)
|
||||
}
|
||||
if topupCalls != 1 || exchangeCalls != 0 {
|
||||
t.Fatalf("topup=%d exchange=%d, want 1/0", topupCalls, exchangeCalls)
|
||||
}
|
||||
if opened != "https://pay/checkout" {
|
||||
t.Fatalf("opened checkout = %q", opened)
|
||||
}
|
||||
if topupBody["autoTopUp"] != true || topupBody["paymentSessionId"] != "pay-1" {
|
||||
t.Fatalf("topup body missing autoTopUp/paymentSessionId: %#v", topupBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatedCheckoutURLRequiresAbsoluteHTTPS(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
value string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "https", value: " https://pay.example.test/checkout?id=1 ", want: "https://pay.example.test/checkout?id=1"},
|
||||
{name: "empty", wantErr: true},
|
||||
{name: "relative", value: "/checkout", wantErr: true},
|
||||
{name: "http", value: "http://pay.example.test/checkout", wantErr: true},
|
||||
{name: "file", value: "file:///tmp/checkout", wantErr: true},
|
||||
{name: "custom protocol", value: "wallet://checkout/123", wantErr: true},
|
||||
{name: "userinfo", value: "https://[email protected]/checkout", wantErr: true},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := validatedCheckoutURL(test.value)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("validatedCheckoutURL(%q) error = %v", test.value, err)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Fatalf("validatedCheckoutURL(%q) = %q, want %q", test.value, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamTopUpByKeyPinsValidatedInferenceEndpoint(t *testing.T) {
|
||||
validatedCalls := 0
|
||||
validated := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/partner-checkout/sessions"):
|
||||
_ = json.NewEncoder(w).Encode(PartnerCheckoutSession{SessionToken: "pcs_tok", Status: SessionStatusPendingAuth})
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/v2/billing/topup"):
|
||||
validatedCalls++
|
||||
_ = json.NewEncoder(w).Encode(TopUpByKeyResult{Checkout: PaymentSession{PayURL: "https://pay/checkout"}})
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/partner-checkout/sessions/"):
|
||||
_ = json.NewEncoder(w).Encode(PartnerCheckoutSession{SessionToken: "pcs_tok", Status: SessionStatusPaid})
|
||||
default:
|
||||
t.Errorf("unexpected validated-endpoint request: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer validated.Close()
|
||||
override := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Errorf("validated key leaked to process override: %s %s", r.Method, r.URL.Path)
|
||||
}))
|
||||
defer override.Close()
|
||||
t.Setenv("AIMLAPI_APP_URL", validated.URL)
|
||||
t.Setenv("AIMLAPI_INFERENCE_URL", override.URL)
|
||||
|
||||
_, err := StreamTopUpByKey(context.Background(), StreamTopUpByKeyOptions{
|
||||
APIKey: "production-key",
|
||||
AmountUSD: "20",
|
||||
InferenceBaseURL: validated.URL,
|
||||
PaymentSessionID: "payment-1",
|
||||
OpenBrowser: func(string) error { return nil },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTopUpByKey() error = %v", err)
|
||||
}
|
||||
if validatedCalls != 1 {
|
||||
t.Fatalf("validated endpoint top-up calls = %d, want 1", validatedCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamTopUpEmailRetrySendsStablePaymentSessionID(t *testing.T) {
|
||||
getCalls := 0
|
||||
var paymentSessionID string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/partner-checkout/sessions/"):
|
||||
status := SessionStatusPendingAuth
|
||||
if getCalls > 0 {
|
||||
status = SessionStatusPaid
|
||||
}
|
||||
getCalls++
|
||||
_ = json.NewEncoder(w).Encode(PartnerCheckoutSession{SessionToken: "pcs_retry", Status: status})
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/pay"):
|
||||
var body map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
paymentSessionID, _ = body["paymentSessionId"].(string)
|
||||
_ = json.NewEncoder(w).Encode(PayResult{Checkout: PaymentSession{PayURL: "https://pay/checkout"}})
|
||||
default:
|
||||
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("AIMLAPI_APP_URL", server.URL)
|
||||
|
||||
_, err := StreamTopUp(context.Background(), StreamTopUpOptions{
|
||||
SessionToken: "bearer",
|
||||
ResumeSessionToken: "pcs_retry",
|
||||
PaymentSessionID: "payment-stable",
|
||||
AmountUSD: "20",
|
||||
NoOpen: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTopUp() error = %v", err)
|
||||
}
|
||||
if paymentSessionID != "payment-stable" {
|
||||
t.Fatalf("paymentSessionId = %q, want stable retry id", paymentSessionID)
|
||||
}
|
||||
}
|
||||
|
||||
// resumeServer is a partner-checkout backend fake that fails the test if the
|
||||
// re-charging endpoints (create-session / pay) are ever hit, and serves a
|
||||
// scripted GetSession status sequence so a resume can be driven deterministically.
|
||||
type resumeServer struct {
|
||||
t *testing.T
|
||||
statuses []string // GetSession status per call, last value repeats
|
||||
getCalls int
|
||||
payCalls int
|
||||
createCall int
|
||||
exchanged int
|
||||
}
|
||||
|
||||
func (rs *resumeServer) handler() http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/partner-checkout/sessions"):
|
||||
rs.createCall++
|
||||
rs.t.Errorf("resume must not create a new session")
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/pay"):
|
||||
rs.payCalls++
|
||||
rs.t.Errorf("resume must not re-pay (double charge)")
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/exchange"):
|
||||
rs.exchanged++
|
||||
_ = json.NewEncoder(w).Encode(ExchangeResult{APIKey: "sk-resumed", APIKeyID: "key_1"})
|
||||
case r.Method == http.MethodGet:
|
||||
status := rs.statuses[len(rs.statuses)-1]
|
||||
if rs.getCalls < len(rs.statuses) {
|
||||
status = rs.statuses[rs.getCalls]
|
||||
}
|
||||
rs.getCalls++
|
||||
_ = json.NewEncoder(w).Encode(PartnerCheckoutSession{SessionToken: "pc_tok", Status: SessionStatus(status)})
|
||||
default:
|
||||
rs.t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestStreamTopUpResumePaidExchangesWithoutRecharge(t *testing.T) {
|
||||
rs := &resumeServer{t: t, statuses: []string{"paid"}}
|
||||
server := httptest.NewServer(rs.handler())
|
||||
defer server.Close()
|
||||
t.Setenv("AIMLAPI_APP_URL", server.URL)
|
||||
|
||||
var seen string
|
||||
result, err := StreamTopUp(context.Background(), StreamTopUpOptions{
|
||||
SessionToken: "bearer",
|
||||
ResumeSessionToken: "pc_tok",
|
||||
PaymentSessionID: "payment-1",
|
||||
AmountUSD: "20",
|
||||
Exchange: true,
|
||||
NoOpen: true,
|
||||
OnSession: func(token string) { seen = token },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTopUp resume = %v", err)
|
||||
}
|
||||
if result.APIKey != "sk-resumed" {
|
||||
t.Fatalf("APIKey = %q, want recovered key", result.APIKey)
|
||||
}
|
||||
if rs.exchanged != 1 {
|
||||
t.Fatalf("exchange calls = %d, want 1", rs.exchanged)
|
||||
}
|
||||
if seen != "pc_tok" {
|
||||
t.Fatalf("OnSession token = %q, want retained pc_tok", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamTopUpResumePendingPaymentPollsNeverRepays(t *testing.T) {
|
||||
// First GetSession (resume resolution) sees the still-open checkout; the next
|
||||
// (poll) sees it settle. Pay must never be called — the fake fails the test if
|
||||
// it is.
|
||||
rs := &resumeServer{t: t, statuses: []string{"pending_payment", "paid"}}
|
||||
server := httptest.NewServer(rs.handler())
|
||||
defer server.Close()
|
||||
t.Setenv("AIMLAPI_APP_URL", server.URL)
|
||||
|
||||
result, err := StreamTopUp(context.Background(), StreamTopUpOptions{
|
||||
SessionToken: "bearer",
|
||||
ResumeSessionToken: "pc_tok",
|
||||
PaymentSessionID: "payment-1",
|
||||
AmountUSD: "20",
|
||||
Exchange: true,
|
||||
NoOpen: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTopUp resume = %v", err)
|
||||
}
|
||||
if result.APIKey != "sk-resumed" {
|
||||
t.Fatalf("APIKey = %q, want recovered key", result.APIKey)
|
||||
}
|
||||
if rs.payCalls != 0 {
|
||||
t.Fatalf("pay calls = %d, want 0", rs.payCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamTopUpPollingDeadSessionClearsRetainedToken(t *testing.T) {
|
||||
rs := &resumeServer{t: t, statuses: []string{"pending_payment", "expired"}}
|
||||
server := httptest.NewServer(rs.handler())
|
||||
defer server.Close()
|
||||
t.Setenv("AIMLAPI_APP_URL", server.URL)
|
||||
|
||||
seen := []string{}
|
||||
_, err := StreamTopUp(context.Background(), StreamTopUpOptions{
|
||||
SessionToken: "bearer",
|
||||
ResumeSessionToken: "pc_tok",
|
||||
PaymentSessionID: "payment-1",
|
||||
AmountUSD: "20",
|
||||
Exchange: true,
|
||||
NoOpen: true,
|
||||
OnSession: func(token string) { seen = append(seen, token) },
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "payment expired") {
|
||||
t.Fatalf("err = %v, want expired payment error", err)
|
||||
}
|
||||
if len(seen) == 0 || seen[len(seen)-1] != "" {
|
||||
t.Fatalf("OnSession events = %#v, want terminal clear", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollUntilPaidTerminal4xxClearsRetainedToken(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "session rejected", http.StatusNotFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
seen := "retained"
|
||||
_, err := pollUntilPaid(context.Background(), NewClient(Endpoints{AppBaseURL: server.URL}, server.Client()), "pc_missing", func(token string) {
|
||||
seen = token
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("pollUntilPaid() error = nil, want terminal 4xx")
|
||||
}
|
||||
if seen != "" {
|
||||
t.Fatalf("retained session = %q, want cleared", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollUntilExchangeSettledClearsDeadSession(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
statusCode int
|
||||
status SessionStatus
|
||||
}{
|
||||
{name: "terminal API rejection", statusCode: http.StatusNotFound},
|
||||
{name: "expired exchange claim", status: SessionStatusExpired},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if test.statusCode != 0 {
|
||||
http.Error(w, "session rejected", test.statusCode)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(PartnerCheckoutSession{SessionToken: "pcs_dead", Status: test.status})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
seen := "retained"
|
||||
err := pollUntilExchangeSettled(context.Background(), NewClient(Endpoints{AppBaseURL: server.URL}, server.Client()), "pcs_dead", func(token string) {
|
||||
seen = token
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("pollUntilExchangeSettled() error = nil")
|
||||
}
|
||||
if seen != "" {
|
||||
t.Fatalf("retained session = %q, want cleared", seen)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeHelpersClearOnlyRejectedSessions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status int
|
||||
wantClear bool
|
||||
}{
|
||||
{name: "terminal 4xx", status: http.StatusNotFound, wantClear: true},
|
||||
{name: "retryable 5xx", status: http.StatusServiceUnavailable, wantClear: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, "session lookup failed", test.status)
|
||||
}))
|
||||
defer server.Close()
|
||||
client := NewClient(Endpoints{AppBaseURL: server.URL}, server.Client())
|
||||
|
||||
for _, flow := range []struct {
|
||||
name string
|
||||
resolve func(func(string)) error
|
||||
}{
|
||||
{
|
||||
name: "session bearer",
|
||||
resolve: func(onSession func(string)) error {
|
||||
_, _, err := resolveTopupSession(context.Background(), client, StreamTopUpOptions{
|
||||
ResumeSessionToken: "pc_rejected", OnSession: onSession,
|
||||
}, DefaultPartnerID, DefaultPartnerName, Endpoints{})
|
||||
return err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "api key",
|
||||
resolve: func(onSession func(string)) error {
|
||||
_, _, err := resolveByKeySession(context.Background(), client, StreamTopUpByKeyOptions{
|
||||
ResumeSessionToken: "pc_rejected", OnSession: onSession,
|
||||
}, DefaultPartnerID, DefaultPartnerName, Endpoints{})
|
||||
return err
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(flow.name, func(t *testing.T) {
|
||||
seen := "retained"
|
||||
if err := flow.resolve(func(token string) { seen = token }); err == nil {
|
||||
t.Fatal("resolve error = nil")
|
||||
}
|
||||
if cleared := seen == ""; cleared != test.wantClear {
|
||||
t.Fatalf("session cleared = %v, want %v", cleared, test.wantClear)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamTopUpResumeExchangedReturnsRecoveryError(t *testing.T) {
|
||||
rs := &resumeServer{t: t, statuses: []string{"exchanged"}}
|
||||
server := httptest.NewServer(rs.handler())
|
||||
defer server.Close()
|
||||
t.Setenv("AIMLAPI_APP_URL", server.URL)
|
||||
|
||||
_, err := StreamTopUp(context.Background(), StreamTopUpOptions{
|
||||
SessionToken: "bearer",
|
||||
ResumeSessionToken: "pc_tok",
|
||||
PaymentSessionID: "payment-1",
|
||||
AmountUSD: "20",
|
||||
Exchange: true,
|
||||
NoOpen: true,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "already exchanged") {
|
||||
t.Fatalf("err = %v, want an already-exchanged recovery error", err)
|
||||
}
|
||||
if rs.exchanged != 0 {
|
||||
t.Fatalf("exchange calls = %d, want 0 (no second key)", rs.exchanged)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamTopUpResumeExchangingPollsWithoutSecondExchange(t *testing.T) {
|
||||
rs := &resumeServer{t: t, statuses: []string{"exchanging", "exchanged"}}
|
||||
server := httptest.NewServer(rs.handler())
|
||||
defer server.Close()
|
||||
t.Setenv("AIMLAPI_APP_URL", server.URL)
|
||||
|
||||
_, err := StreamTopUp(context.Background(), StreamTopUpOptions{
|
||||
SessionToken: "bearer",
|
||||
ResumeSessionToken: "pc_tok",
|
||||
PaymentSessionID: "payment-1",
|
||||
AmountUSD: "20",
|
||||
Exchange: true,
|
||||
NoOpen: true,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "already exchanged") {
|
||||
t.Fatalf("err = %v, want dashboard recovery guidance", err)
|
||||
}
|
||||
if rs.exchanged != 0 {
|
||||
t.Fatalf("exchange calls = %d, want 0 while an exchange claim is in flight", rs.exchanged)
|
||||
}
|
||||
if rs.createCall != 0 || rs.payCalls != 0 {
|
||||
t.Fatalf("resume created/paid a new checkout: create=%d pay=%d", rs.createCall, rs.payCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamTopUpPollObservesExchangingWithoutSecondExchange(t *testing.T) {
|
||||
rs := &resumeServer{t: t, statuses: []string{"pending_payment", "exchanging", "exchanged"}}
|
||||
server := httptest.NewServer(rs.handler())
|
||||
defer server.Close()
|
||||
t.Setenv("AIMLAPI_APP_URL", server.URL)
|
||||
|
||||
_, err := StreamTopUp(context.Background(), StreamTopUpOptions{
|
||||
SessionToken: "bearer",
|
||||
ResumeSessionToken: "pc_tok",
|
||||
PaymentSessionID: "payment-1",
|
||||
AmountUSD: "20",
|
||||
Exchange: true,
|
||||
NoOpen: true,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "already exchanged") {
|
||||
t.Fatalf("err = %v, want dashboard recovery guidance", err)
|
||||
}
|
||||
if rs.exchanged != 0 {
|
||||
t.Fatalf("exchange calls = %d, want 0 after poll observed an existing claim", rs.exchanged)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package aimlapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Status is a progress phase reported by StreamTopUp through its OnStatus callback.
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusCreatingSession Status = "creating-session"
|
||||
StatusOpeningCheckout Status = "opening-checkout"
|
||||
StatusWaitingPayment Status = "waiting-payment"
|
||||
StatusProvisioningKey Status = "provisioning-key"
|
||||
)
|
||||
|
||||
// ProvisionedKey is the outcome of a top-up: the (optionally minted) API key plus
|
||||
// the inference base URL and model to write into the provider profile.
|
||||
type ProvisionedKey struct {
|
||||
APIKey string
|
||||
APIKeyID string
|
||||
BaseURL string
|
||||
Model string
|
||||
}
|
||||
|
||||
const (
|
||||
pollInterval = 3 * time.Second
|
||||
pollTimeout = 20 * time.Minute
|
||||
)
|
||||
|
||||
// ParseAmountUSD parses a dollar string into USD minor units (cents), enforcing the
|
||||
// min/max top-up bounds. An empty value yields the default amount.
|
||||
func ParseAmountUSD(value string) (int, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return DefaultAmountUSDMinor, nil
|
||||
}
|
||||
dollars, err := strconv.ParseFloat(value, 64)
|
||||
// NaN/±Inf parse without error and slip past the min/max comparisons below
|
||||
// (every ordered comparison with NaN is false), so reject them explicitly
|
||||
// before the cents conversion turns them into a garbage minor-unit amount.
|
||||
if err != nil || math.IsNaN(dollars) || math.IsInf(dollars, 0) || dollars <= 0 {
|
||||
return 0, fmt.Errorf("invalid amount %q; pass a positive USD amount", value)
|
||||
}
|
||||
minor := int(dollars*100 + 0.5)
|
||||
if minor < MinAmountUSDMinor {
|
||||
return 0, fmt.Errorf("minimum top-up is $%d", MinAmountUSDMinor/100)
|
||||
}
|
||||
if minor > MaxAmountUSDMinor {
|
||||
return 0, fmt.Errorf("maximum top-up is $%d", MaxAmountUSDMinor/100)
|
||||
}
|
||||
return minor, nil
|
||||
}
|
||||
|
||||
// NewPaymentSessionID returns a random UUIDv4-shaped idempotency id for a top-up.
|
||||
// The caller generates it once per top-up intent and reuses it on retry so the
|
||||
// backend (and Stripe) coalesce retries onto the same checkout instead of a
|
||||
// second charge.
|
||||
func NewPaymentSessionID() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40 // version 4
|
||||
b[8] = (b[8] & 0x3f) | 0x80 // variant 10
|
||||
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]), nil
|
||||
}
|
||||
|
||||
func pollUntilPaid(ctx context.Context, client *Client, sessionToken string, onSession func(string)) (PartnerCheckoutSession, error) {
|
||||
deadline := time.Now().Add(pollTimeout)
|
||||
for time.Now().Before(deadline) {
|
||||
session, err := client.GetSession(ctx, sessionToken)
|
||||
if err != nil {
|
||||
// Context cancellation/deadline is terminal (also covers an in-flight
|
||||
// request cancelled mid-poll, whose transport error wraps ctx.Err()).
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return PartnerCheckoutSession{}, err
|
||||
}
|
||||
// A 4xx API response is terminal. Everything else — a 5xx or a transient
|
||||
// transport error (non-APIError, e.g. a dropped connection during the
|
||||
// wait) — is retried until the poll deadline.
|
||||
if isTerminalSessionAPIError(err) {
|
||||
reportSession(onSession, "")
|
||||
return PartnerCheckoutSession{}, err
|
||||
}
|
||||
if err := sleepContext(ctx, pollInterval); err != nil {
|
||||
return PartnerCheckoutSession{}, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch session.Status {
|
||||
case SessionStatusPaid, SessionStatusExchanging:
|
||||
return session, nil
|
||||
case SessionStatusExchanged:
|
||||
return PartnerCheckoutSession{}, fmt.Errorf("session was already exchanged; rotate the key from the aimlapi.com dashboard")
|
||||
case SessionStatusCancelled, SessionStatusExpired, SessionStatusFailed:
|
||||
// The checkout cannot recover from these states. Drop the retained token
|
||||
// (and the TUI's paired by-key idempotency ID) now so the very next retry
|
||||
// creates a fresh payment intent.
|
||||
reportSession(onSession, "")
|
||||
return PartnerCheckoutSession{}, fmt.Errorf("payment %s; re-run the top-up to try again", session.Status)
|
||||
default:
|
||||
if err := sleepContext(ctx, pollInterval); err != nil {
|
||||
return PartnerCheckoutSession{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return PartnerCheckoutSession{}, fmt.Errorf("timed out waiting for payment; re-run once the payment clears")
|
||||
}
|
||||
|
||||
func isTerminalSessionAPIError(err error) bool {
|
||||
var apiErr APIError
|
||||
return errors.As(err, &apiErr) && apiErr.Status >= http.StatusBadRequest && apiErr.Status < http.StatusInternalServerError
|
||||
}
|
||||
|
||||
// pollUntilExchangeSettled follows an exchange claim that was already in flight
|
||||
// when a retry resumed the session. Exchange is single-flight and the raw key is
|
||||
// returned only to the winning request, so issuing Exchange again is never safe.
|
||||
// Once the claim completes, direct the user to the dashboard recovery path.
|
||||
func pollUntilExchangeSettled(ctx context.Context, client *Client, sessionToken string, onSession func(string)) error {
|
||||
deadline := time.Now().Add(pollTimeout)
|
||||
for time.Now().Before(deadline) {
|
||||
session, err := client.GetSession(ctx, sessionToken)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return err
|
||||
}
|
||||
if isTerminalSessionAPIError(err) {
|
||||
reportSession(onSession, "")
|
||||
return err
|
||||
}
|
||||
if err := sleepContext(ctx, pollInterval); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch session.Status {
|
||||
case SessionStatusExchanged:
|
||||
return fmt.Errorf("session was already exchanged; rotate the key from the aimlapi.com dashboard")
|
||||
case SessionStatusExchanging:
|
||||
if err := sleepContext(ctx, pollInterval); err != nil {
|
||||
return err
|
||||
}
|
||||
case SessionStatusCancelled, SessionStatusExpired, SessionStatusFailed:
|
||||
reportSession(onSession, "")
|
||||
return fmt.Errorf("key provisioning %s; rotate the key from the aimlapi.com dashboard", session.Status)
|
||||
default:
|
||||
return fmt.Errorf("key provisioning returned to %s; re-run the top-up", session.Status)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for key provisioning; retry to check the same session")
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, delay time.Duration) error {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func status(callback func(Status, string), value Status, detail string) {
|
||||
if callback != nil {
|
||||
callback(value, detail)
|
||||
}
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if value = strings.TrimSpace(value); value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package aimlapi
|
||||
|
||||
import (
|
||||
"net/mail"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ValidEmail performs the lightweight validation used before starting the
|
||||
// passwordless onboarding request.
|
||||
func ValidEmail(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
address, err := mail.ParseAddress(value)
|
||||
if err != nil || address.Address != value {
|
||||
return false
|
||||
}
|
||||
at := strings.LastIndexByte(value, '@')
|
||||
if at <= 0 || at == len(value)-1 {
|
||||
return false
|
||||
}
|
||||
domain := value[at+1:]
|
||||
if strings.HasPrefix(domain, ".") || strings.HasSuffix(domain, ".") || strings.Contains(domain, "..") {
|
||||
return false
|
||||
}
|
||||
dot := strings.LastIndexByte(domain, '.')
|
||||
if dot <= 0 || dot == len(domain)-1 {
|
||||
return false
|
||||
}
|
||||
tld := domain[dot+1:]
|
||||
if len(tld) < 2 {
|
||||
return false
|
||||
}
|
||||
for _, char := range tld {
|
||||
if char < 'A' || char > 'Z' && char < 'a' || char > 'z' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
+29
-6
@@ -42,6 +42,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/worktrees"
|
||||
"github.com/Gitlawb/zero/internal/zerogit"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
"github.com/charmbracelet/x/term"
|
||||
)
|
||||
|
||||
var version = "dev"
|
||||
@@ -339,7 +340,17 @@ func runWithDeps(args []string, stdout io.Writer, stderr io.Writer, deps appDeps
|
||||
return 0
|
||||
}
|
||||
}
|
||||
if _, err := fmt.Fprintf(stdout, "zero %s\n", version); err != nil {
|
||||
// Pipes and redirects keep the machine-readable contract exactly as it
|
||||
// was: a single "zero <version>" line. Scripts, command substitutions,
|
||||
// and the NPM wrapper smoke check all parse that record, so the banner
|
||||
// is strictly a TTY affordance.
|
||||
if !stdoutIsTerminal(stdout) {
|
||||
if _, err := fmt.Fprintf(stdout, "zero %s\n", version); err != nil {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if _, err := fmt.Fprintf(stdout, "%s\n\nzero %s\n", tui.Wordmark(), version); err != nil {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
@@ -755,11 +766,12 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
|
||||
}
|
||||
sandboxBackend := deps.selectSandboxBackend(sandbox.BackendOptions{})
|
||||
sandboxEngine := sandbox.NewEngine(sandbox.EngineOptions{
|
||||
WorkspaceRoot: workspaceRoot,
|
||||
Policy: applyConfiguredSandboxPolicy(sandbox.DefaultPolicy(), resolved.Sandbox),
|
||||
Store: sandboxStore,
|
||||
Backend: sandboxBackend,
|
||||
Scope: scope,
|
||||
WorkspaceRoot: workspaceRoot,
|
||||
Policy: applyConfiguredSandboxPolicy(sandbox.DefaultPolicy(), resolved.Sandbox),
|
||||
Store: sandboxStore,
|
||||
Backend: sandboxBackend,
|
||||
Scope: scope,
|
||||
SensitiveEnvKeys: providerSensitiveEnvKeys(resolved),
|
||||
})
|
||||
lastKnownMCPConfig := mcpConfig
|
||||
fileTracker := tools.NewFileTracker()
|
||||
@@ -1115,6 +1127,17 @@ func closeSpecialistRuntime(stderr io.Writer, runtime *agentToolRuntime) {
|
||||
}
|
||||
}
|
||||
|
||||
// stdoutIsTerminal reports whether w is an interactive terminal. Tests pass
|
||||
// bytes.Buffer writers and pipes/redirects fail term.IsTerminal, so both take
|
||||
// the machine-readable path.
|
||||
func stdoutIsTerminal(w io.Writer) bool {
|
||||
f, ok := w.(*os.File)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return term.IsTerminal(f.Fd())
|
||||
}
|
||||
|
||||
func writeAppError(stderr io.Writer, message string, exitCode int) int {
|
||||
if _, err := fmt.Fprintf(stderr, "[zero] %s\n", message); err != nil {
|
||||
return 1
|
||||
|
||||
@@ -41,6 +41,8 @@ func TestRunPrintsVersion(t *testing.T) {
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected exit code 0, got %d", exitCode)
|
||||
}
|
||||
// Non-terminal stdout keeps the machine-readable contract: exactly one
|
||||
// "zero <version>" record, no wordmark banner, no ANSI.
|
||||
if got := stdout.String(); got != "zero dev\n" {
|
||||
t.Fatalf("expected version output, got %q", got)
|
||||
}
|
||||
@@ -49,6 +51,33 @@ func TestRunPrintsVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunVersionRedirectedFile covers the `zero --version > file` path: stdout
|
||||
// is a real *os.File but not a terminal, so the single-line contract must hold.
|
||||
func TestRunVersionRedirectedFile(t *testing.T) {
|
||||
stdout, err := os.CreateTemp(t.TempDir(), "version-stdout")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp stdout: %v", err)
|
||||
}
|
||||
defer stdout.Close()
|
||||
var stderr bytes.Buffer
|
||||
|
||||
exitCode := Run([]string{"--version"}, stdout, &stderr)
|
||||
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected exit code 0, got %d", exitCode)
|
||||
}
|
||||
content, err := os.ReadFile(stdout.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("read temp stdout: %v", err)
|
||||
}
|
||||
if got := string(content); got != "zero dev\n" {
|
||||
t.Fatalf("expected version output, got %q", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr, got %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPrintsHelp(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--help"},
|
||||
|
||||
@@ -348,6 +348,83 @@ func TestRunProvidersAddWritesCatalogProfile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProvidersAddAimlapiWritesDefaultHeaders(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
configPath := filepath.Join(t.TempDir(), "zero", "config.json")
|
||||
|
||||
exitCode := runWithDeps([]string{"providers", "add", "aimlapi"}, &stdout, &stderr, providerSetupDeps(configPath))
|
||||
|
||||
if exitCode != exitSuccess {
|
||||
t.Fatalf("expected exit code %d, got %d: %s", exitSuccess, exitCode, stderr.String())
|
||||
}
|
||||
cfg := readFileConfig(t, configPath)
|
||||
if len(cfg.Providers) != 1 {
|
||||
t.Fatalf("providers = %#v, want one provider", cfg.Providers)
|
||||
}
|
||||
profile := cfg.Providers[0]
|
||||
if profile.Name != "aimlapi" ||
|
||||
profile.CatalogID != "aimlapi" ||
|
||||
profile.BaseURL != "https://api.aimlapi.com/v1" ||
|
||||
profile.Model != "anthropic/claude-sonnet-5" ||
|
||||
profile.APIKeyEnv != "AIMLAPI_API_KEY" {
|
||||
t.Fatalf("unexpected provider profile: %#v", profile)
|
||||
}
|
||||
if profile.CustomHeaders["X-AIMLAPI-Partner-ID"] != "part_62yQoGYDq4Yqnrj2R1iGrDNJ" ||
|
||||
profile.CustomHeaders["X-AIMLAPI-Integration-Repo"] != "Gitlawb/zero" {
|
||||
t.Fatalf("missing aimlapi.com default headers: %#v", profile.CustomHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProvidersAddAimlapiMixedCaseHeaderOverride(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
configPath := filepath.Join(t.TempDir(), "zero", "config.json")
|
||||
|
||||
// A differently-cased override must replace the catalog header in place, not
|
||||
// leave both spellings behind to race when request construction canonicalizes.
|
||||
exitCode := runWithDeps([]string{
|
||||
"providers", "add", "aimlapi",
|
||||
"--header", "x-aimlapi-partner-id=part_override",
|
||||
}, &stdout, &stderr, providerSetupDeps(configPath))
|
||||
|
||||
if exitCode != exitSuccess {
|
||||
t.Fatalf("expected exit code %d, got %d: %s", exitSuccess, exitCode, stderr.String())
|
||||
}
|
||||
profile := readFileConfig(t, configPath).Providers[0]
|
||||
if _, ok := profile.CustomHeaders["x-aimlapi-partner-id"]; ok {
|
||||
t.Fatalf("lowercase override left a duplicate key: %#v", profile.CustomHeaders)
|
||||
}
|
||||
if profile.CustomHeaders["X-AIMLAPI-Partner-ID"] != "part_override" {
|
||||
t.Fatalf("override did not win on the canonical key: %#v", profile.CustomHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProvidersAddAimlapiOverrideDropsCatalogHeaders(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
configPath := filepath.Join(t.TempDir(), "zero", "config.json")
|
||||
|
||||
exitCode := runWithDeps([]string{
|
||||
"providers", "add", "aimlapi",
|
||||
"--base-url", "https://staging.example/v1",
|
||||
"--header", "X-Environment=staging",
|
||||
}, &stdout, &stderr, providerSetupDeps(configPath))
|
||||
if exitCode != exitSuccess {
|
||||
t.Fatalf("expected exit code %d, got %d: %s", exitSuccess, exitCode, stderr.String())
|
||||
}
|
||||
profile := readFileConfig(t, configPath).Providers[0]
|
||||
if _, ok := profile.CustomHeaders["X-AIMLAPI-Partner-ID"]; ok {
|
||||
t.Fatalf("partner attribution leaked to overridden endpoint: %#v", profile.CustomHeaders)
|
||||
}
|
||||
if _, ok := profile.CustomHeaders["X-AIMLAPI-Integration-Repo"]; ok {
|
||||
t.Fatalf("integration attribution leaked to overridden endpoint: %#v", profile.CustomHeaders)
|
||||
}
|
||||
if profile.CustomHeaders["X-Environment"] != "staging" {
|
||||
t.Fatalf("explicit user header was dropped: %#v", profile.CustomHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProvidersAddWritesCustomHeaders(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
@@ -118,7 +118,8 @@ func runSkillInfo(args []string, dir string, stdout io.Writer, stderr io.Writer)
|
||||
if name == "" {
|
||||
return writeExecUsageError(stderr, "usage: zero skill info <name> [--json]")
|
||||
}
|
||||
info, ok := skills.Info(dir, name)
|
||||
// Resolve across primary + ~/.agents/skills; lock metadata only from primary.
|
||||
info, ok := skills.InfoFromRoots(dir, skills.GlobalRoots(dir), name)
|
||||
if !ok {
|
||||
return writeAppError(stderr, fmt.Sprintf("skill %q not found", name), exitUsage)
|
||||
}
|
||||
|
||||
+90
-5
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/specmode"
|
||||
"github.com/Gitlawb/zero/internal/streamjson"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/usage"
|
||||
"github.com/Gitlawb/zero/internal/worktrees"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
@@ -124,6 +125,12 @@ type execOptions struct {
|
||||
// additional write roots for this run. Unioned with
|
||||
// config.SandboxConfig.AdditionalWriteRoots at scope construction time.
|
||||
addDirs []string
|
||||
// tracePath, when set, writes a per-turn NDJSON trace (agenteval-compatible)
|
||||
// to the given file path — or to stderr when the value is "-". Falls back to
|
||||
// the ZERO_TRACE env var when the flag is absent. Off by default: a run
|
||||
// without it leaves agent.Options.Trace nil and is byte-identical to before.
|
||||
// The trace is pure observation — enabling it does not change agent behavior.
|
||||
tracePath string
|
||||
}
|
||||
|
||||
type execUsageError struct {
|
||||
@@ -422,6 +429,12 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
notify.MaybeAddWebhookSink(notifier, os.Getenv, func(format string, args ...any) {
|
||||
fmt.Fprintf(stderr, "[notify] "+format+"\n", args...)
|
||||
})
|
||||
// Spec-draft runs synthesize a prompt offline and never drive a model turn, so
|
||||
// there is no per-turn trace to emit. Reject --trace / ZERO_TRACE up front with
|
||||
// a clear error rather than silently accepting and writing nothing.
|
||||
if options.useSpec && resolveTracePath(options) != "" {
|
||||
return writeExecFormatUsageError(stdout, stderr, options.outputFormat, "--trace / ZERO_TRACE are not supported for spec-draft runs")
|
||||
}
|
||||
if options.useSpec {
|
||||
return runExecSpecDraft(execSpecDraftRun{
|
||||
options: options,
|
||||
@@ -474,6 +487,24 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
if err != nil {
|
||||
return writeAppError(stderr, "failed to create run id: "+err.Error(), exitCrash)
|
||||
}
|
||||
// Per-turn tracing is opt-in (--trace <path> or ZERO_TRACE=<path>). The
|
||||
// recorder is stamped throughout agent.Run and the providerio seam; the run
|
||||
// itself is byte-identical to an untraced run. We finish the recorder at the
|
||||
// agent.Run boundary (below) so the snapshot captures exactly one turn, then
|
||||
// serialize the snapshot on every exit path via the defer. "-" writes NDJSON
|
||||
// to stderr; otherwise the path is created/truncated. A failure to emit is
|
||||
// logged to stderr, never fatal.
|
||||
tracePath := resolveTracePath(options)
|
||||
var traceRecorder *trace.Recorder
|
||||
var traceSnapshot *trace.TurnTrace
|
||||
if tracePath != "" {
|
||||
traceRecorder = trace.NewRecorder(preparedSession.Session.SessionID, runID, "")
|
||||
defer func() {
|
||||
if err := writeTraceSnapshot(traceSnapshot, tracePath, stderr); err != nil {
|
||||
fmt.Fprintf(stderr, "[zero] failed to write trace: %s\n", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
writer := execEventWriter{
|
||||
stdout: stdout,
|
||||
stderr: stderr,
|
||||
@@ -556,6 +587,7 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
Model: resolved.Provider.Model,
|
||||
ModelSwitcher: modelSwitcher,
|
||||
ReasoningEffort: forwardEffort,
|
||||
Trace: traceRecorder,
|
||||
Cwd: workspaceRoot,
|
||||
Images: images,
|
||||
Registry: registry,
|
||||
@@ -625,6 +657,12 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
sessionRecorder.append(sessions.EventUsage, payload)
|
||||
},
|
||||
})
|
||||
// Finish the trace now that the turn is done, so the snapshot captures exactly
|
||||
// agent.Run's work and nothing the post-run cleanup stamps. The deferred
|
||||
// writer serializes the snapshot on every exit path.
|
||||
if traceRecorder != nil {
|
||||
traceSnapshot = traceRecorder.Finish()
|
||||
}
|
||||
notifier.Notify(notify.Completion, notify.DefaultMessage(notify.Completion))
|
||||
if writer.err != nil {
|
||||
return exitCrash
|
||||
@@ -815,14 +853,28 @@ func buildExecSandboxEngine(workspaceRoot string, resolved config.ResolvedConfig
|
||||
policy := applyConfiguredSandboxPolicy(sandbox.DefaultPolicy(), resolved.Sandbox)
|
||||
backend := deps.selectSandboxBackend(sandbox.BackendOptions{})
|
||||
return sandbox.NewEngine(sandbox.EngineOptions{
|
||||
WorkspaceRoot: workspaceRoot,
|
||||
Policy: policy,
|
||||
Store: store,
|
||||
Backend: backend,
|
||||
Scope: scope,
|
||||
WorkspaceRoot: workspaceRoot,
|
||||
Policy: policy,
|
||||
Store: store,
|
||||
Backend: backend,
|
||||
Scope: scope,
|
||||
SensitiveEnvKeys: providerSensitiveEnvKeys(resolved),
|
||||
}), nil
|
||||
}
|
||||
|
||||
func providerSensitiveEnvKeys(resolved config.ResolvedConfig) []string {
|
||||
keys := make([]string, 0, len(resolved.Providers)+1)
|
||||
for _, profile := range resolved.Providers {
|
||||
if key := strings.TrimSpace(profile.APIKeyEnv); key != "" {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
if key := strings.TrimSpace(resolved.Provider.APIKeyEnv); key != "" {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// applyConfiguredSandboxPolicy overlays every config-sourced sandbox knob onto
|
||||
// the default policy.
|
||||
func applyConfiguredSandboxPolicy(policy sandbox.Policy, cfg config.SandboxConfig) sandbox.Policy {
|
||||
@@ -1237,3 +1289,36 @@ func execNotifyMode(options execOptions, resolved config.ResolvedConfig) string
|
||||
}
|
||||
return resolved.Notify.Mode
|
||||
}
|
||||
|
||||
// resolveTracePath returns the trace destination for this run: the --trace flag
|
||||
// value when set, else the ZERO_TRACE env var, else "" (tracing off). A value of
|
||||
// "-" means "write to stderr".
|
||||
func resolveTracePath(options execOptions) string {
|
||||
if v := strings.TrimSpace(options.tracePath); v != "" {
|
||||
return v
|
||||
}
|
||||
if v := strings.TrimSpace(os.Getenv("ZERO_TRACE")); v != "" {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// writeTraceSnapshot writes an already-finished trace snapshot to dest ("-" =>
|
||||
// stderr, otherwise a file path created/truncated). A nil snapshot is a no-op
|
||||
// (tracing off, or the run exited before agent.Run produced a turn). The trace
|
||||
// is best-effort: a write error is returned to the caller, which logs it but
|
||||
// never fails the run.
|
||||
func writeTraceSnapshot(snapshot *trace.TurnTrace, dest string, stderr io.Writer) error {
|
||||
if snapshot == nil || dest == "" {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(dest) == "-" {
|
||||
return trace.WriteNDJSON(stderr, snapshot)
|
||||
}
|
||||
file, err := os.Create(dest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
return trace.WriteNDJSON(file, snapshot)
|
||||
}
|
||||
@@ -32,6 +32,15 @@ func parseExecArgs(args []string) (execOptions, bool, error) {
|
||||
options.noNotify = true
|
||||
case arg == "--no-completion-gate":
|
||||
options.noCompletionGate = true
|
||||
case arg == "--trace":
|
||||
value, next, err := nextFlagValue(args, index, arg)
|
||||
if err != nil {
|
||||
return options, false, err
|
||||
}
|
||||
options.tracePath = value
|
||||
index = next
|
||||
case strings.HasPrefix(arg, "--trace="):
|
||||
options.tracePath = strings.TrimSpace(strings.TrimPrefix(arg, "--trace="))
|
||||
case arg == "--notify":
|
||||
value, next, err := nextFlagValue(args, index, arg)
|
||||
if err != nil {
|
||||
|
||||
@@ -15,11 +15,11 @@ import (
|
||||
|
||||
// pluginActivation holds what plugin activation contributed to a bootstrap so the
|
||||
// later dispatcher + skill wiring can consume it: the plugin hook definitions and
|
||||
// the plugin skill search roots. The zero value (no plugins) is inert — the
|
||||
// dispatcher gets no extra hooks and the skill tool keeps only the default dir.
|
||||
// The embedded trustSkip reports whether the project plugin layer was dropped for
|
||||
// an untrusted workspace, so the caller can fold it into the single combined
|
||||
// trust notice.
|
||||
// the plugin skill search roots. The zero value (no plugins) still overlays the
|
||||
// multi-root skill tool (primary + ~/.agents/skills) so discovery is identical
|
||||
// with or without plugins. The embedded trustSkip reports whether the project
|
||||
// plugin layer was dropped for an untrusted workspace, so the caller can fold it
|
||||
// into the single combined trust notice.
|
||||
type pluginActivation struct {
|
||||
hooks []hooks.Definition
|
||||
skillRoots []string
|
||||
@@ -52,6 +52,9 @@ func activatePlugins(workspaceRoot string, registry *tools.Registry, deps appDep
|
||||
loaded, err := deps.loadPlugins(plugins.LoadOptions{Cwd: workspaceRoot, ExcludeProject: excludeProject})
|
||||
if err != nil {
|
||||
writePluginActivationWarning(stderr, "failed to load plugins: "+err.Error())
|
||||
// Still overlay multi-root discovery (primary + agents) so a plugin load
|
||||
// failure cannot hide shared skills.
|
||||
registry.Register(plugins.NewSkillTool(deps.skillsDir(), nil))
|
||||
return pluginActivation{trustSkip: skip}
|
||||
}
|
||||
|
||||
@@ -67,14 +70,11 @@ func activatePlugins(workspaceRoot string, registry *tools.Registry, deps appDep
|
||||
writePluginActivationWarning(stderr, warning)
|
||||
}
|
||||
|
||||
// Re-register the skill tool to also resolve plugin-declared skills. The core
|
||||
// skill tool only reads the default skills dir; the plugin-aware replacement
|
||||
// merges the default dir with the plugin skill roots (default dir wins a name
|
||||
// clash), so plugin skills appear in the agent's skill list. With no plugin
|
||||
// skill roots this is byte-equivalent to the default skills surface.
|
||||
if len(result.SkillRoots) > 0 {
|
||||
registry.Register(plugins.NewSkillTool(deps.skillsDir(), result.SkillRoots))
|
||||
}
|
||||
// Always overlay the multi-root skill tool so discovery is identical with or
|
||||
// without plugins: primary DefaultDir + optional ~/.agents/skills + plugin
|
||||
// roots (earlier wins). With empty SkillRoots and no agents dir this is
|
||||
// byte-equivalent to the core single-dir skill surface.
|
||||
registry.Register(plugins.NewSkillTool(deps.skillsDir(), result.SkillRoots))
|
||||
|
||||
return pluginActivation{hooks: result.Hooks, skillRoots: result.SkillRoots, trustSkip: skip}
|
||||
}
|
||||
@@ -91,8 +91,8 @@ func projectPluginsDirExists(workspaceRoot string) bool {
|
||||
}
|
||||
|
||||
// skillInfos resolves the reusable skills the model can load via the skill tool —
|
||||
// the default skills dir merged with any plugin skill roots, the same set the
|
||||
// plugin-aware skill tool resolves against — as plain data for the agent's system
|
||||
// primary dir + optional ~/.agents/skills + plugin skill roots, the same set the
|
||||
// multi-root skill tool resolves against — as plain data for the agent's system
|
||||
// prompt. It returns nil when no skills are installed, so a skill-less run leaves
|
||||
// the prompt byte-identical.
|
||||
func (a pluginActivation) skillInfos(defaultDir string) []agent.SkillInfo {
|
||||
|
||||
@@ -69,6 +69,11 @@ func TestActivatePluginsRegistersToolAndCollectsHooks(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestActivatePluginsRegistersPluginSkillTool(t *testing.T) {
|
||||
// Isolate host ~/.agents/skills so it cannot interfere with this test.
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
|
||||
pluginDir := t.TempDir()
|
||||
skillDir := filepath.Join(pluginDir, "skills", "demo-skill")
|
||||
if err := os.MkdirAll(skillDir, 0o755); err != nil {
|
||||
@@ -108,6 +113,42 @@ func TestActivatePluginsRegistersPluginSkillTool(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivatePluginsAlwaysRegistersMultiRootSkillTool(t *testing.T) {
|
||||
// Even with zero plugins / zero plugin skill roots, activation must overlay
|
||||
// the multi-root skill tool so ~/.agents/skills is loadable on bare runs.
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
skillDir := filepath.Join(agents, "agents-skill")
|
||||
if err := os.MkdirAll(skillDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("---\nname: agents-skill\n---\nagents body"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(tools.NewSkillTool(t.TempDir())) // core single-dir tool
|
||||
var stderr bytes.Buffer
|
||||
workspace := t.TempDir()
|
||||
activation := activatePlugins(workspace, registry, fakePluginDeps(t, nil), &stderr, workspace)
|
||||
if len(activation.skillRoots) != 0 {
|
||||
t.Fatalf("expected no plugin skill roots, got %#v", activation.skillRoots)
|
||||
}
|
||||
skillTool, ok := registry.Get("skill")
|
||||
if !ok {
|
||||
t.Fatal("skill tool missing after activation")
|
||||
}
|
||||
res := skillTool.Run(context.Background(), map[string]any{"name": "agents-skill"})
|
||||
if res.Status != tools.StatusOK || !bytes.Contains([]byte(res.Output), []byte("agents body")) {
|
||||
t.Fatalf("agents skill must load with zero plugin roots: %q (%s)", res.Status, res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivatePluginsSurfacesLoadDiagnostics(t *testing.T) {
|
||||
// Load succeeds (no top-level error) but reports a per-plugin diagnostic for a
|
||||
// plugin it had to skip. activatePlugins must forward that diagnostic to stderr
|
||||
|
||||
@@ -3,8 +3,10 @@ package cli
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/aimlapi"
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
"github.com/Gitlawb/zero/internal/providerhealth"
|
||||
@@ -424,22 +426,39 @@ func providerProfileForAdd(options providerAddOptions) (config.ProviderProfile,
|
||||
if apiKeyEnv == "" && len(descriptor.AuthEnvVars) > 0 {
|
||||
apiKeyEnv = descriptor.AuthEnvVars[0]
|
||||
}
|
||||
baseURL := firstNonEmptyCLI(options.baseURL, descriptor.DefaultBaseURL)
|
||||
var catalogHeaders map[string]string
|
||||
if sameProviderSetupBaseURL(baseURL, descriptor.DefaultBaseURL) {
|
||||
catalogHeaders = descriptor.CustomHeaders
|
||||
if strings.EqualFold(strings.TrimSpace(descriptor.ID), "aimlapi") {
|
||||
catalogHeaders = aimlapi.WithResolvedPartnerHeader(catalogHeaders)
|
||||
}
|
||||
}
|
||||
profile := config.ProviderProfile{
|
||||
Name: name,
|
||||
ProviderKind: providerKindForDescriptor(descriptor),
|
||||
CatalogID: descriptor.ID,
|
||||
BaseURL: firstNonEmptyCLI(options.baseURL, descriptor.DefaultBaseURL),
|
||||
BaseURL: baseURL,
|
||||
APIKeyEnv: apiKeyEnv,
|
||||
APIFormat: firstAPIFormat(descriptor),
|
||||
AuthHeader: strings.TrimSpace(options.authHeader),
|
||||
AuthScheme: normalizeAuthScheme(options.authScheme),
|
||||
AuthHeaderValue: strings.TrimSpace(options.authHeaderValue),
|
||||
CustomHeaders: copyProviderHeaders(options.customHeaders),
|
||||
Model: firstNonEmptyCLI(options.model, descriptor.DefaultModel),
|
||||
// Catalog attribution belongs only to the catalog endpoint. Explicit
|
||||
// user headers still apply to an override/staging/proxy endpoint.
|
||||
CustomHeaders: mergeProviderHeaders(catalogHeaders, options.customHeaders),
|
||||
Model: firstNonEmptyCLI(options.model, descriptor.DefaultModel),
|
||||
}
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
func sameProviderSetupBaseURL(left string, right string) bool {
|
||||
return strings.EqualFold(
|
||||
strings.TrimRight(strings.TrimSpace(left), "/"),
|
||||
strings.TrimRight(strings.TrimSpace(right), "/"),
|
||||
)
|
||||
}
|
||||
|
||||
func selectProviderForCheck(resolved config.ResolvedConfig, name string) (config.ProviderProfile, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
@@ -544,3 +563,43 @@ func copyProviderHeaders(headers map[string]string) map[string]string {
|
||||
}
|
||||
return copied
|
||||
}
|
||||
|
||||
func mergeProviderHeaders(base map[string]string, overrides map[string]string) map[string]string {
|
||||
merged := copyProviderHeaders(base)
|
||||
if len(overrides) == 0 {
|
||||
return merged
|
||||
}
|
||||
if merged == nil {
|
||||
merged = map[string]string{}
|
||||
}
|
||||
for key, value := range overrides {
|
||||
// HTTP header names are case-insensitive and request construction
|
||||
// canonicalizes them (http.Header.Set), so an override that differs from a
|
||||
// base key only in case must replace that key's value in place — otherwise
|
||||
// both survive as distinct map entries and race non-deterministically at
|
||||
// build time. Keep the existing spelling (the catalog's, which the resolver
|
||||
// re-merges verbatim); only a genuinely new header is added as typed.
|
||||
if existing, ok := headerKeyFold(merged, key); ok {
|
||||
merged[existing] = value
|
||||
continue
|
||||
}
|
||||
merged[key] = value
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
// headerKeyFold returns the key in headers whose canonical HTTP form matches key,
|
||||
// so a case-insensitive override lands on the existing entry instead of adding a
|
||||
// colliding duplicate.
|
||||
func headerKeyFold(headers map[string]string, key string) (string, bool) {
|
||||
if _, ok := headers[key]; ok {
|
||||
return key, true
|
||||
}
|
||||
canonical := http.CanonicalHeaderKey(key)
|
||||
for existing := range headers {
|
||||
if http.CanonicalHeaderKey(existing) == canonical {
|
||||
return existing, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
)
|
||||
|
||||
func TestProviderSensitiveEnvKeys(t *testing.T) {
|
||||
resolved := config.ResolvedConfig{
|
||||
Providers: []config.ProviderProfile{
|
||||
{Name: "custom", APIKeyEnv: " COMPANY_LLM_SECRET "},
|
||||
{Name: "catalog", APIKeyEnv: "OPENAI_API_KEY"},
|
||||
{Name: "inline"},
|
||||
},
|
||||
Provider: config.ProviderProfile{Name: "active", APIKeyEnv: "ACTIVE_PROVIDER_SECRET"},
|
||||
}
|
||||
want := []string{"COMPANY_LLM_SECRET", "OPENAI_API_KEY", "ACTIVE_PROVIDER_SECRET"}
|
||||
if got := providerSensitiveEnvKeys(resolved); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("providerSensitiveEnvKeys() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
+90
-5
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/mcp"
|
||||
@@ -14,9 +16,21 @@ import (
|
||||
type serveOptions struct {
|
||||
mcp bool
|
||||
cwd string
|
||||
addDirs []string
|
||||
allowUnsafeTools bool
|
||||
}
|
||||
|
||||
// serveRootsScope is a PathScope used only for MCP serve: the workspace root
|
||||
// plus optional --add-dir roots, without the sandbox temp write roots that
|
||||
// would otherwise pollute resources/list.
|
||||
type serveRootsScope []string
|
||||
|
||||
func (s serveRootsScope) Roots() []string {
|
||||
roots := make([]string, len(s))
|
||||
copy(roots, s)
|
||||
return roots
|
||||
}
|
||||
|
||||
func runServe(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int {
|
||||
options, help, err := parseServeArgs(args)
|
||||
if err != nil {
|
||||
@@ -36,7 +50,11 @@ func runServe(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) i
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
registry := newServeRegistry(workspaceRoot, options.allowUnsafeTools)
|
||||
scope, err := buildServeScope(workspaceRoot, options.addDirs)
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
registry := newServeRegistry(workspaceRoot, scope, options.allowUnsafeTools)
|
||||
if options.allowUnsafeTools {
|
||||
if _, err := fmt.Fprintln(stderr, "[zero] Unsafe MCP server tools enabled because --allow-unsafe-tools was passed."); err != nil {
|
||||
return exitCrash
|
||||
@@ -47,6 +65,8 @@ func runServe(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) i
|
||||
Name: "zero",
|
||||
Version: version,
|
||||
PermissionGranted: options.allowUnsafeTools,
|
||||
WorkspaceRoot: workspaceRoot,
|
||||
Scope: scope,
|
||||
})
|
||||
if err != nil {
|
||||
return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash)
|
||||
@@ -54,11 +74,11 @@ func runServe(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) i
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
func newServeRegistry(workspaceRoot string, allowUnsafeTools bool) *tools.Registry {
|
||||
func newServeRegistry(workspaceRoot string, scope tools.PathScope, allowUnsafeTools bool) *tools.Registry {
|
||||
registry := tools.NewRegistry()
|
||||
toolset := tools.CoreReadOnlyTools(workspaceRoot)
|
||||
toolset := tools.CoreReadOnlyToolsScoped(workspaceRoot, scope)
|
||||
if allowUnsafeTools {
|
||||
toolset = tools.CoreTools(workspaceRoot)
|
||||
toolset = tools.CoreToolsScoped(workspaceRoot, scope)
|
||||
}
|
||||
for _, tool := range toolset {
|
||||
registry.Register(tool)
|
||||
@@ -66,6 +86,58 @@ func newServeRegistry(workspaceRoot string, allowUnsafeTools bool) *tools.Regist
|
||||
return registry
|
||||
}
|
||||
|
||||
// buildServeScope returns nil when there are no extra roots (workspace-only),
|
||||
// otherwise a PathScope listing the workspace first followed by each --add-dir.
|
||||
func buildServeScope(workspaceRoot string, addDirs []string) (tools.PathScope, error) {
|
||||
if len(addDirs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
roots := make([]string, 0, 1+len(addDirs))
|
||||
workspaceRoot = filepath.Clean(workspaceRoot)
|
||||
if abs, err := filepath.Abs(workspaceRoot); err == nil {
|
||||
workspaceRoot = abs
|
||||
}
|
||||
|
||||
// Keep lexical absolute paths in roots so path checks stay aligned with the
|
||||
// un-evaluated WorkspaceRoot used when Scope is nil. Symlink-resolved forms
|
||||
// are only keys for duplicate detection.
|
||||
workspaceResolved := workspaceRoot
|
||||
if resolved, err := filepath.EvalSymlinks(workspaceRoot); err == nil {
|
||||
workspaceResolved = resolved
|
||||
}
|
||||
roots = append(roots, workspaceRoot)
|
||||
|
||||
seen := map[string]struct{}{workspaceResolved: {}}
|
||||
for _, dir := range addDirs {
|
||||
trimmed := strings.TrimSpace(dir)
|
||||
if trimmed == "" {
|
||||
return nil, execUsageError{"--add-dir requires a path"}
|
||||
}
|
||||
absolute, err := filepath.Abs(trimmed)
|
||||
if err != nil {
|
||||
return nil, execUsageError{fmt.Sprintf("--add-dir %q: %v", trimmed, err)}
|
||||
}
|
||||
info, err := os.Stat(absolute)
|
||||
if err != nil {
|
||||
return nil, execUsageError{fmt.Sprintf("--add-dir %q: %v", trimmed, err)}
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, execUsageError{fmt.Sprintf("--add-dir %q is not a directory", trimmed)}
|
||||
}
|
||||
|
||||
resolved := absolute
|
||||
if r, err := filepath.EvalSymlinks(absolute); err == nil {
|
||||
resolved = r
|
||||
}
|
||||
if _, ok := seen[resolved]; ok {
|
||||
continue
|
||||
}
|
||||
seen[resolved] = struct{}{}
|
||||
roots = append(roots, absolute)
|
||||
}
|
||||
return serveRootsScope(roots), nil
|
||||
}
|
||||
|
||||
func parseServeArgs(args []string) (serveOptions, bool, error) {
|
||||
options := serveOptions{}
|
||||
for index := 0; index < len(args); index++ {
|
||||
@@ -85,6 +157,18 @@ func parseServeArgs(args []string) (serveOptions, bool, error) {
|
||||
options.cwd = args[index]
|
||||
case strings.HasPrefix(arg, "--cwd="):
|
||||
options.cwd = strings.TrimPrefix(arg, "--cwd=")
|
||||
case arg == "--add-dir":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return options, false, execUsageError{"--add-dir requires a path"}
|
||||
}
|
||||
options.addDirs = append(options.addDirs, args[index])
|
||||
case strings.HasPrefix(arg, "--add-dir="):
|
||||
value := strings.TrimPrefix(arg, "--add-dir=")
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return options, false, execUsageError{"--add-dir requires a path"}
|
||||
}
|
||||
options.addDirs = append(options.addDirs, value)
|
||||
default:
|
||||
return options, false, execUsageError{fmt.Sprintf("unknown serve flag %q", arg)}
|
||||
}
|
||||
@@ -100,7 +184,8 @@ Starts Zero as an MCP stdio server.
|
||||
|
||||
Flags:
|
||||
--mcp Run the MCP stdio server
|
||||
-C, --cwd <path> Set the workspace directory
|
||||
-C, --cwd <path> Set the workspace directory (resources + tools)
|
||||
--add-dir <path> Extra resource/tool root (repeatable)
|
||||
--allow-unsafe-tools Expose write and shell tools to the MCP host
|
||||
-h, --help Show this help
|
||||
`)
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -80,6 +82,120 @@ func TestRunServeRequiresMCPMode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunServeMCPWiresWorkspaceRootAndAddDirIntoResources(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
extra := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(workspace, "workspace.txt"), []byte("ws\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(extra, "extra.txt"), []byte("ex\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
exitCode := runWithDeps(
|
||||
[]string{"serve", "--mcp", "-C", workspace, "--add-dir", extra},
|
||||
&stdout,
|
||||
&stderr,
|
||||
appDeps{stdin: bytes.NewReader(serveMCPResourcesInput(t))},
|
||||
)
|
||||
if exitCode != exitSuccess {
|
||||
t.Fatalf("exit=%d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "workspace.txt") {
|
||||
t.Fatalf("expected workspace resource, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "extra.txt") {
|
||||
t.Fatalf("expected --add-dir resource, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunServeMCPRejectsMissingAddDir(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
missing := filepath.Join(workspace, "does-not-exist")
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
exitCode := runWithDeps(
|
||||
[]string{"serve", "--mcp", "-C", workspace, "--add-dir", missing},
|
||||
&stdout,
|
||||
&stderr,
|
||||
appDeps{stdin: bytes.NewReader(nil)},
|
||||
)
|
||||
if exitCode != exitUsage {
|
||||
t.Fatalf("exit=%d want %d stderr=%q", exitCode, exitUsage, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "--add-dir") {
|
||||
t.Fatalf("expected --add-dir error, got %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseServeArgsCollectsAddDirs(t *testing.T) {
|
||||
options, help, err := parseServeArgs([]string{"--mcp", "--add-dir", "/one", "--add-dir=/two", "-C", "/ws"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if help {
|
||||
t.Fatal("unexpected help")
|
||||
}
|
||||
if !options.mcp || options.cwd != "/ws" {
|
||||
t.Fatalf("options=%#v", options)
|
||||
}
|
||||
if len(options.addDirs) != 2 || options.addDirs[0] != "/one" || options.addDirs[1] != "/two" {
|
||||
t.Fatalf("addDirs=%v", options.addDirs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildServeScopeNilWithoutExtras(t *testing.T) {
|
||||
scope, err := buildServeScope(t.TempDir(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scope != nil {
|
||||
t.Fatalf("expected nil scope, got %#v", scope)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildServeScopeKeepsLexicalPaths(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
realWorkspace := filepath.Join(base, "real-workspace")
|
||||
realExtra := filepath.Join(base, "real-extra")
|
||||
for _, dir := range []string{realWorkspace, realExtra} {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
linkWorkspace := filepath.Join(base, "link-workspace")
|
||||
linkExtra := filepath.Join(base, "link-extra")
|
||||
if err := os.Symlink(realWorkspace, linkWorkspace); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(realExtra, linkExtra); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
scope, err := buildServeScope(linkWorkspace, []string{linkExtra})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scope == nil {
|
||||
t.Fatal("expected non-nil scope")
|
||||
}
|
||||
roots := scope.Roots()
|
||||
wantWorkspace, err := filepath.Abs(linkWorkspace)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantExtra, err := filepath.Abs(linkExtra)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(roots) != 2 || roots[0] != wantWorkspace || roots[1] != wantExtra {
|
||||
t.Fatalf("roots=%v want [%q %q]", roots, wantWorkspace, wantExtra)
|
||||
}
|
||||
}
|
||||
|
||||
func serveMCPInput(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
@@ -104,6 +220,30 @@ func serveMCPInput(t *testing.T) []byte {
|
||||
return input.Bytes()
|
||||
}
|
||||
|
||||
func serveMCPResourcesInput(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
var input bytes.Buffer
|
||||
writeServeMCPMessage(t, &input, map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": map[string]any{},
|
||||
})
|
||||
writeServeMCPMessage(t, &input, map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/initialized",
|
||||
"params": map[string]any{},
|
||||
})
|
||||
writeServeMCPMessage(t, &input, map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"method": "resources/list",
|
||||
"params": map[string]any{},
|
||||
})
|
||||
return input.Bytes()
|
||||
}
|
||||
|
||||
func writeServeMCPMessage(t *testing.T, buffer *bytes.Buffer, message map[string]any) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(message)
|
||||
|
||||
@@ -200,6 +200,54 @@ func TestSaveSetupProviderStoresPastedAPIKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveSetupProviderAimlapiEnvReferenceAndOverrideHeaders(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "zero", "config.json")
|
||||
result, err := saveSetupProvider(appDeps{
|
||||
userConfigPath: func() (string, error) { return configPath, nil },
|
||||
}, tui.SetupSelection{
|
||||
CatalogID: "aimlapi",
|
||||
BaseURL: "https://staging.example/v1",
|
||||
Model: "anthropic/claude-sonnet-5",
|
||||
}, setupSaveOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("saveSetupProvider() error = %v", err)
|
||||
}
|
||||
if result.Provider.APIKey != "" || result.Provider.APIKeyEnv != "AIMLAPI_API_KEY" {
|
||||
t.Fatalf("runtime profile did not retain AIMLAPI_API_KEY reference: %+v", result.Provider)
|
||||
}
|
||||
if len(result.Provider.CustomHeaders) != 0 {
|
||||
t.Fatalf("catalog headers leaked to first-run override: %#v", result.Provider.CustomHeaders)
|
||||
}
|
||||
persisted := readFileConfig(t, configPath).Providers[0]
|
||||
if persisted.APIKey != "" || persisted.APIKeyEnv != "AIMLAPI_API_KEY" || persisted.APIKeyStored {
|
||||
t.Fatalf("persisted credential source changed: %+v", persisted)
|
||||
}
|
||||
if len(persisted.CustomHeaders) != 0 {
|
||||
t.Fatalf("persisted override contains catalog headers: %#v", persisted.CustomHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveSetupProviderAimlapiUsesResolvedPartnerOverride(t *testing.T) {
|
||||
t.Setenv("AIMLAPI_PARTNER_ID", "part_override")
|
||||
configPath := filepath.Join(t.TempDir(), "zero", "config.json")
|
||||
result, err := saveSetupProvider(appDeps{
|
||||
userConfigPath: func() (string, error) { return configPath, nil },
|
||||
}, tui.SetupSelection{
|
||||
CatalogID: "aimlapi",
|
||||
BaseURL: "https://api.aimlapi.com/v1",
|
||||
Model: "anthropic/claude-sonnet-5",
|
||||
}, setupSaveOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("saveSetupProvider() error = %v", err)
|
||||
}
|
||||
if got := result.Provider.CustomHeaders["X-AIMLAPI-Partner-ID"]; got != "part_override" {
|
||||
t.Fatalf("runtime partner header = %q, want part_override", got)
|
||||
}
|
||||
if got := readFileConfig(t, configPath).Providers[0].CustomHeaders["X-AIMLAPI-Partner-ID"]; got != "part_override" {
|
||||
t.Fatalf("persisted partner header = %q, want part_override", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveSetupProviderStoresCustomEndpointSelection(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "zero", "config.json")
|
||||
|
||||
|
||||
+21
-22
@@ -59,25 +59,21 @@ func runSkills(args []string, stdout io.Writer, stderr io.Writer, deps appDeps)
|
||||
}
|
||||
|
||||
func runSkillsList(dir string, options skillListOptions, stdout io.Writer, stderr io.Writer) int {
|
||||
discovered, err := skills.List(dir)
|
||||
// Management CLI lists global roots only (primary + ~/.agents/skills); plugin
|
||||
// skills stay out of `zero skills list` and surface via the agent skill tool.
|
||||
roots := skills.GlobalRoots(dir)
|
||||
discovered, dups, err := skills.ListFromRoots(roots)
|
||||
if err != nil {
|
||||
return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash)
|
||||
}
|
||||
// Surface name collisions that List silently resolved (first directory wins),
|
||||
// so a shadowed same-named skill is reported instead of just disappearing.
|
||||
// Warnings go to stderr, keeping stdout (including --json) clean.
|
||||
if dups, derr := skills.Duplicates(dir); derr == nil {
|
||||
for _, dup := range dups {
|
||||
fmt.Fprintf(stderr, "warning: duplicate skill %q: using %s, ignoring %s\n",
|
||||
redaction.RedactString(dup.Name, redaction.Options{}),
|
||||
redaction.RedactString(dup.Winner, redaction.Options{}),
|
||||
redaction.RedactString(dup.Loser, redaction.Options{}))
|
||||
}
|
||||
} else {
|
||||
// Don't silently swallow a scan failure: "no warnings" would then be
|
||||
// ambiguous (no duplicates vs. detection broke). Surface it on stderr.
|
||||
fmt.Fprintf(stderr, "warning: could not check for duplicate skills: %s\n",
|
||||
redaction.ErrorMessage(derr, redaction.Options{}))
|
||||
// Surface name collisions that ListFromRoots silently resolved (earlier root
|
||||
// wins), so a shadowed same-named skill is reported instead of just
|
||||
// disappearing. Warnings go to stderr, keeping stdout (including --json) clean.
|
||||
for _, dup := range dups {
|
||||
fmt.Fprintf(stderr, "warning: duplicate skill %q: using %s, ignoring %s\n",
|
||||
redaction.RedactString(dup.Name, redaction.Options{}),
|
||||
redaction.RedactString(dup.Winner, redaction.Options{}),
|
||||
redaction.RedactString(dup.Loser, redaction.Options{}))
|
||||
}
|
||||
if options.json {
|
||||
payload := struct {
|
||||
@@ -88,16 +84,16 @@ func runSkillsList(dir string, options skillListOptions, stdout io.Writer, stder
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
output := redaction.RedactString(formatSkillList(discovered, dir), redaction.Options{})
|
||||
output := redaction.RedactString(formatSkillList(discovered), redaction.Options{})
|
||||
if _, err := fmt.Fprintln(stdout, output); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
func formatSkillList(discovered []skills.Skill, dir string) string {
|
||||
func formatSkillList(discovered []skills.Skill) string {
|
||||
if len(discovered) == 0 {
|
||||
return fmt.Sprintf("No Zero skills found in %s.", dir)
|
||||
return "No skills found."
|
||||
}
|
||||
lines := []string{"Zero Skills:"}
|
||||
for _, skill := range discovered {
|
||||
@@ -131,10 +127,13 @@ func writeSkillsHelp(w io.Writer) error {
|
||||
zero skills <command>
|
||||
|
||||
Commands:
|
||||
list List discovered Zero skills
|
||||
add <git-url|path> Install a skill (checksum-pinned in skills.lock)
|
||||
list List discovered skills (Zero dir + ~/.agents/skills)
|
||||
add <git-url|path> Install a skill into the Zero skills dir (checksum-pinned in skills.lock)
|
||||
info <name> Show a skill's frontmatter, source, and pinned hash
|
||||
remove <name> Remove an installed skill and its lockfile entry
|
||||
remove <name> Remove an installed skill and its lockfile entry from the Zero skills dir
|
||||
|
||||
list and info also search ~/.agents/skills when present (read-only).
|
||||
add and remove always target the Zero-specific skills directory.
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -23,7 +23,15 @@ func writeSkillFixture(t *testing.T, dir string, name string, content string) {
|
||||
}
|
||||
}
|
||||
|
||||
func isolateCLIAgentsHome(t *testing.T) {
|
||||
t.Helper()
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
}
|
||||
|
||||
func TestRunSkillsListText(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
dir := t.TempDir()
|
||||
writeSkillFixture(t, dir, "confirmation-policy", "---\nname: confirmation-policy\ndescription: Ask before risky actions.\n---\nbody")
|
||||
|
||||
@@ -43,6 +51,7 @@ func TestRunSkillsListText(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunSkillsListWarnsOnDuplicateNames(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
dir := t.TempDir()
|
||||
// Two directories declare the same frontmatter name; List keeps one and the
|
||||
// other is shadowed. The command must warn instead of silently dropping it.
|
||||
@@ -62,6 +71,7 @@ func TestRunSkillsListWarnsOnDuplicateNames(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunSkillsDefaultsToList(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
dir := t.TempDir()
|
||||
writeSkillFixture(t, dir, "demo", "body")
|
||||
|
||||
@@ -78,6 +88,7 @@ func TestRunSkillsDefaultsToList(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunSkillsListJSON(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
dir := t.TempDir()
|
||||
writeSkillFixture(t, dir, "demo", "---\nname: demo\ndescription: a demo\n---\nbody")
|
||||
|
||||
@@ -111,6 +122,7 @@ func TestRunSkillsListJSON(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunSkillsEmptyDir(t *testing.T) {
|
||||
isolateCLIAgentsHome(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
exit := runWithDeps([]string{"skills", "list"}, &stdout, &stderr, appDeps{
|
||||
skillsDir: func() string { return filepath.Join(t.TempDir(), "missing") },
|
||||
@@ -118,11 +130,89 @@ func TestRunSkillsEmptyDir(t *testing.T) {
|
||||
if exit != 0 {
|
||||
t.Fatalf("exit = %d, stderr = %s", exit, stderr.String())
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(stdout.String()), "no") {
|
||||
if !strings.Contains(strings.ToLower(stdout.String()), "no skills found") {
|
||||
t.Fatalf("expected a no-skills message, got:\n%s", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkillsListIncludesAgentsOnlySkill(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
writeSkillFixture(t, agents, "agents-only", "---\nname: agents-only\ndescription: Shared multi-agent skill.\n---\nbody")
|
||||
|
||||
primary := t.TempDir()
|
||||
var stdout, stderr bytes.Buffer
|
||||
exit := runWithDeps([]string{"skills", "list"}, &stdout, &stderr, appDeps{
|
||||
skillsDir: func() string { return primary },
|
||||
})
|
||||
if exit != 0 {
|
||||
t.Fatalf("exit = %d, stderr = %s", exit, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
if !strings.Contains(out, "agents-only") {
|
||||
t.Fatalf("list should include agents-only skill:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, agents) {
|
||||
t.Fatalf("list should show agents path:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkillsListPrimaryShadowsAgents(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
primary := t.TempDir()
|
||||
writeSkillFixture(t, primary, "shared", "---\nname: shared\ndescription: From primary.\n---\nbody")
|
||||
writeSkillFixture(t, agents, "shared", "---\nname: shared\ndescription: From agents.\n---\nbody")
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
exit := runWithDeps([]string{"skills", "list"}, &stdout, &stderr, appDeps{
|
||||
skillsDir: func() string { return primary },
|
||||
})
|
||||
if exit != 0 {
|
||||
t.Fatalf("exit = %d, stderr = %s", exit, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "From primary") {
|
||||
t.Fatalf("primary should win list description:\n%s", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), primary) {
|
||||
t.Fatalf("primary path should be shown:\n%s", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `duplicate skill "shared"`) {
|
||||
t.Fatalf("expected cross-root duplicate warning, got: %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkillInfoResolvesAgentsOnly(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
writeSkillFixture(t, agents, "agents-info", "---\nname: agents-info\ndescription: Agents info.\n---\nbody")
|
||||
|
||||
primary := t.TempDir()
|
||||
var stdout, stderr bytes.Buffer
|
||||
exit := runWithDeps([]string{"skill", "info", "agents-info"}, &stdout, &stderr, appDeps{
|
||||
skillsDir: func() string { return primary },
|
||||
})
|
||||
if exit != 0 {
|
||||
t.Fatalf("exit = %d, stderr = %s", exit, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
if !strings.Contains(out, "agents-info") || !strings.Contains(out, "Agents info.") {
|
||||
t.Fatalf("info missing agents skill:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "source:") || strings.Contains(out, "hash:") {
|
||||
t.Fatalf("agents-only info must not invent lock metadata:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, agents) {
|
||||
t.Fatalf("info should show agents path:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkillsDoesNotLaunchTUI(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
launchCalled := false
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
func mergeMCPConfig(dst *MCPConfig, src MCPConfig) {
|
||||
func mergeMCPConfig(dst *MCPConfig, src MCPConfig, canReenable bool) {
|
||||
if len(src.Servers) == 0 {
|
||||
return
|
||||
}
|
||||
@@ -14,10 +14,14 @@ func mergeMCPConfig(dst *MCPConfig, src MCPConfig) {
|
||||
dst.Servers = map[string]MCPServerConfig{}
|
||||
}
|
||||
for name, server := range src.Servers {
|
||||
dst.Servers[name] = mergeMCPServer(dst.Servers[name], server)
|
||||
dst.Servers[name] = mergeMCPServer(dst.Servers[name], server, canReenable)
|
||||
}
|
||||
}
|
||||
|
||||
// mergeProjectMCPConfig merges a project-scoped (lower-trust) MCP config into
|
||||
// dst. It never re-enables a server the user explicitly disabled or disabled a
|
||||
// server the user explicitly enabled: it merges with canReenable=false so the
|
||||
// trust-boundary guard in mergeMCPServer keeps the user's decision authoritative.
|
||||
func mergeProjectMCPConfig(dst *MCPConfig, src MCPConfig) error {
|
||||
if len(src.Servers) == 0 {
|
||||
return nil
|
||||
@@ -27,7 +31,7 @@ func mergeProjectMCPConfig(dst *MCPConfig, src MCPConfig) error {
|
||||
}
|
||||
for name, server := range src.Servers {
|
||||
base := dst.Servers[name]
|
||||
candidate := mergeMCPServer(base, server)
|
||||
candidate := mergeMCPServer(base, server, false)
|
||||
if projectMCPServerTargetChanges(base, server) && hasInheritedMCPCredentialMaterial(server, candidate) {
|
||||
return fmt.Errorf("project MCP server %q cannot override target while inheriting user credentials; set headers/env/oauth explicitly or use a new server name", name)
|
||||
}
|
||||
@@ -37,7 +41,11 @@ func mergeProjectMCPConfig(dst *MCPConfig, src MCPConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeMCPServer(base MCPServerConfig, next MCPServerConfig) MCPServerConfig {
|
||||
// mergeMCPServer merges a later config layer's MCP server into the base. The
|
||||
// canReenable flag marks the CLI-override scope, the only layer that is allowed
|
||||
// to lift a sticky user-level disable/enable decision (see the disabled-handling
|
||||
// block).
|
||||
func mergeMCPServer(base MCPServerConfig, next MCPServerConfig, canReenable bool) MCPServerConfig {
|
||||
if strings.TrimSpace(next.Type) != "" {
|
||||
base.Type = next.Type
|
||||
}
|
||||
@@ -62,8 +70,26 @@ func mergeMCPServer(base MCPServerConfig, next MCPServerConfig) MCPServerConfig
|
||||
if next.OAuth != nil {
|
||||
base.OAuth = next.OAuth
|
||||
}
|
||||
// Capture the higher-trust scope's prior decision before folding in
|
||||
// next.disabledSet. The trust boundary must be evaluated against the state
|
||||
// the higher-trust layer actually left behind, not against next's own flag.
|
||||
baseDisabledSet := base.disabledSet
|
||||
baseDisabled := base.Disabled
|
||||
if next.disabledSet {
|
||||
base.disabledSet = true
|
||||
}
|
||||
if next.disabledSet || next.Disabled {
|
||||
base.Disabled = next.Disabled
|
||||
// Trust boundary: once a higher-trust scope has explicitly set the
|
||||
// disabled flag, a lower-trust layer (project config) cannot override
|
||||
// that decision in either direction — the user's choice (whether to
|
||||
// enable or disable a server) wins over the repo. This blocks a repo
|
||||
// from re-enabling a server the user disabled, and from disabling a
|
||||
// server the user explicitly enabled. The only layer allowed to
|
||||
// override an explicit higher-scope decision is the CLI override scope
|
||||
// (canReenable=true).
|
||||
if canReenable || !(baseDisabledSet && baseDisabled != next.Disabled) {
|
||||
base.Disabled = next.Disabled
|
||||
}
|
||||
}
|
||||
if next.configured {
|
||||
base.configured = true
|
||||
@@ -75,7 +101,7 @@ func mergeMCPServer(base MCPServerConfig, next MCPServerConfig) MCPServerConfig
|
||||
}
|
||||
|
||||
func projectMCPServerTargetChanges(base MCPServerConfig, next MCPServerConfig) bool {
|
||||
if strings.TrimSpace(next.Type) != "" && mcpServerTransportKind(base) != mcpServerTransportKind(mergeMCPServer(base, next)) {
|
||||
if strings.TrimSpace(next.Type) != "" && mcpServerTransportKind(base) != mcpServerTransportKind(mergeMCPServer(base, next, true)) {
|
||||
return true
|
||||
}
|
||||
if strings.TrimSpace(next.URL) != "" && strings.TrimSpace(next.URL) != strings.TrimSpace(base.URL) {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/aimlapi"
|
||||
"github.com/Gitlawb/zero/internal/modelregistry"
|
||||
"github.com/Gitlawb/zero/internal/notify"
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
@@ -170,7 +171,10 @@ func ResolveMCP(options ResolveOptions) (MCPConfig, error) {
|
||||
if err != nil {
|
||||
return MCPConfig{}, err
|
||||
}
|
||||
mergeMCPConfig(&cfg.MCP, fileConfig.MCP)
|
||||
// User config is higher-trust than project config: it may re-enable a
|
||||
// server the user disabled (a user-level disable is sticky, but the user
|
||||
// scope itself may lift it).
|
||||
mergeMCPConfig(&cfg.MCP, fileConfig.MCP, true)
|
||||
}
|
||||
// Drop the project layer when the workspace is untrusted, so a cloned repo's
|
||||
// ./.zero/config.json cannot register (and spawn) MCP servers. Fail-closed:
|
||||
@@ -184,7 +188,7 @@ func ResolveMCP(options ResolveOptions) (MCPConfig, error) {
|
||||
return MCPConfig{}, err
|
||||
}
|
||||
}
|
||||
mergeMCPConfig(&cfg.MCP, options.Overrides.MCP)
|
||||
mergeMCPConfig(&cfg.MCP, options.Overrides.MCP, true)
|
||||
return cfg.MCP, nil
|
||||
}
|
||||
|
||||
@@ -211,7 +215,7 @@ func mergeConfig(dst *FileConfig, src FileConfig) {
|
||||
for _, provider := range src.Providers {
|
||||
mergeProvider(dst, provider)
|
||||
}
|
||||
mergeMCPConfig(&dst.MCP, src.MCP)
|
||||
mergeMCPConfig(&dst.MCP, src.MCP, true)
|
||||
if network := strings.TrimSpace(src.Sandbox.Network); network != "" {
|
||||
dst.Sandbox.Network = network
|
||||
}
|
||||
@@ -709,7 +713,7 @@ func applyOverrides(cfg *FileConfig, overrides Overrides) {
|
||||
if hasProviderFields(overrides.Provider) {
|
||||
mergeProvider(cfg, overrides.Provider)
|
||||
}
|
||||
mergeMCPConfig(&cfg.MCP, overrides.MCP)
|
||||
mergeMCPConfig(&cfg.MCP, overrides.MCP, true)
|
||||
}
|
||||
|
||||
func mergeLocalControlConfig(dst *LocalControlConfig, src LocalControlConfig) {
|
||||
@@ -1039,6 +1043,45 @@ func applyCatalogDescriptor(profile *ProviderProfile, descriptor providercatalog
|
||||
if profile.APIKeyEnv == "" && len(descriptor.AuthEnvVars) > 0 && (!explicitBaseURL || sameBaseURL(profile.BaseURL, descriptor.DefaultBaseURL)) {
|
||||
profile.APIKeyEnv = descriptor.AuthEnvVars[0]
|
||||
}
|
||||
canonicalCatalogEndpoint := !explicitBaseURL || sameBaseURL(profile.BaseURL, descriptor.DefaultBaseURL)
|
||||
if len(descriptor.CustomHeaders) > 0 && canonicalCatalogEndpoint {
|
||||
catalogHeaders := descriptor.CustomHeaders
|
||||
if strings.EqualFold(strings.TrimSpace(descriptor.ID), "aimlapi") {
|
||||
catalogHeaders = aimlapi.WithResolvedPartnerHeader(catalogHeaders)
|
||||
}
|
||||
merged := copyStringMap(catalogHeaders)
|
||||
for key, value := range profile.CustomHeaders {
|
||||
// Header names are case-insensitive. Preserve the catalog spelling while
|
||||
// replacing its value so request construction cannot see two colliding
|
||||
// map entries whose eventual winner depends on iteration order.
|
||||
existingKey := ""
|
||||
for candidate := range merged {
|
||||
if strings.EqualFold(candidate, key) {
|
||||
existingKey = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if existingKey != "" {
|
||||
merged[existingKey] = value
|
||||
continue
|
||||
}
|
||||
merged[key] = value
|
||||
}
|
||||
profile.CustomHeaders = merged
|
||||
} else if strings.EqualFold(strings.TrimSpace(descriptor.ID), "aimlapi") && !canonicalCatalogEndpoint {
|
||||
// AIMLAPI attribution is owned by the catalog endpoint. A profile can retain
|
||||
// those generated headers after its base URL is edited; strip their names
|
||||
// before sending requests to an arbitrary staging/proxy host while preserving
|
||||
// unrelated headers explicitly supplied by the user.
|
||||
for profileKey := range profile.CustomHeaders {
|
||||
for catalogKey := range descriptor.CustomHeaders {
|
||||
if strings.EqualFold(profileKey, catalogKey) {
|
||||
delete(profile.CustomHeaders, profileKey)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sameBaseURL(left string, right string) bool {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/modelregistry"
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
)
|
||||
|
||||
func TestResolveAppliesLayerPrecedence(t *testing.T) {
|
||||
@@ -554,7 +555,7 @@ func TestResolveReplacesMCPServerOverlayCollections(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMCPServerLayersCanClearAndReenable(t *testing.T) {
|
||||
func TestResolveMCPServerLayersCannotReenableUserDisabled(t *testing.T) {
|
||||
userPath := writeConfig(t, `{
|
||||
"mcp": {
|
||||
"servers": {
|
||||
@@ -588,9 +589,11 @@ func TestResolveMCPServerLayersCanClearAndReenable(t *testing.T) {
|
||||
}
|
||||
|
||||
docs := resolved.MCP.Servers["docs"]
|
||||
if docs.Disabled {
|
||||
t.Fatalf("docs.Disabled = true, want project layer to re-enable")
|
||||
// A user-level disable is sticky: project config must not re-enable it.
|
||||
if !docs.Disabled {
|
||||
t.Fatal("docs.Disabled = false, want user-level disable to remain sticky")
|
||||
}
|
||||
// Non-disable fields from the project layer still merge normally.
|
||||
if len(docs.Args) != 0 {
|
||||
t.Fatalf("docs.Args = %#v, want project layer to clear inherited args", docs.Args)
|
||||
}
|
||||
@@ -599,6 +602,125 @@ func TestResolveMCPServerLayersCanClearAndReenable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeMCPStickyDisableReenable(t *testing.T) {
|
||||
disabled := MCPServerConfig{Disabled: true, disabledSet: true}
|
||||
|
||||
// A lower-trust (project) layer must not lift a sticky user disable.
|
||||
project := MCPServerConfig{Disabled: false, disabledSet: true}
|
||||
if got := mergeMCPServer(disabled, project, false); !got.Disabled {
|
||||
t.Fatal("project layer re-enabled a sticky user disable")
|
||||
}
|
||||
|
||||
// The user scope (explicit mcp enable command / CLI override) may re-enable.
|
||||
override := MCPServerConfig{Disabled: false, disabledSet: true}
|
||||
if got := mergeMCPServer(disabled, override, true); got.Disabled {
|
||||
t.Fatal("user-scope override must be able to re-enable")
|
||||
}
|
||||
|
||||
// A user-scope disable stacked on a project attempt still sticks.
|
||||
if got := mergeMCPServer(disabled, project, false); !got.Disabled {
|
||||
t.Fatal("sticky disable must survive a project re-enable attempt")
|
||||
}
|
||||
|
||||
// Baseline: a project layer may disable a default-enabled server when no
|
||||
// higher-trust scope has explicitly set disabled (the trust boundary must
|
||||
// not block legitimate project disables). Regression guard for the
|
||||
// disabledSet-before-check ordering bug.
|
||||
defaultEnabled := MCPServerConfig{Disabled: false, disabledSet: false}
|
||||
projectDisable := MCPServerConfig{Disabled: true, disabledSet: true}
|
||||
if got := mergeMCPServer(defaultEnabled, projectDisable, false); !got.Disabled {
|
||||
t.Fatal("project layer should be able to disable a default-enabled server")
|
||||
}
|
||||
|
||||
// And an empty project layer must not flip an unconfigured server to disabled.
|
||||
if got := mergeMCPServer(MCPServerConfig{}, MCPServerConfig{}, false); got.Disabled {
|
||||
t.Fatal("an empty project layer must not disable an unconfigured server")
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveMCPCannotReenableUserDisabled exercises the trust boundary at the
|
||||
// actual ResolveMCP entry point: a project config (project scope) must not lift
|
||||
// a disable the user set in their higher-trust user config.
|
||||
func TestResolveMCPCannotReenableUserDisabled(t *testing.T) {
|
||||
userPath := writeConfig(t, `{
|
||||
"mcp": {
|
||||
"servers": {
|
||||
"docs": {
|
||||
"type": "stdio",
|
||||
"command": "docs-mcp",
|
||||
"disabled": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}`)
|
||||
// Project config tries to re-enable it.
|
||||
projectPath := writeConfig(t, `{
|
||||
"mcpServers": {
|
||||
"docs": {
|
||||
"disabled": false
|
||||
}
|
||||
}
|
||||
}`)
|
||||
|
||||
resolved, err := ResolveMCP(ResolveOptions{
|
||||
UserConfigPath: userPath,
|
||||
ProjectConfigPath: projectPath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveMCP() error = %v", err)
|
||||
}
|
||||
|
||||
docs, ok := resolved.Servers["docs"]
|
||||
if !ok {
|
||||
t.Fatal("docs server missing from resolved config")
|
||||
}
|
||||
if !docs.Disabled {
|
||||
t.Fatalf("docs.Disabled = false, want user-level disable to remain sticky across project re-enable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveMCPUserLiftsProjectDisabled asserts the reverse direction is open:
|
||||
// a user config (higher-trust scope) re-enabling a server the project config
|
||||
// disabled must be permitted, since user > project in the trust hierarchy.
|
||||
func TestResolveMCPUserLiftsProjectDisabled(t *testing.T) {
|
||||
// Project config disables the server.
|
||||
projectPath := writeConfig(t, `{
|
||||
"mcpServers": {
|
||||
"docs": {
|
||||
"type": "stdio",
|
||||
"command": "docs-mcp",
|
||||
"disabled": true
|
||||
}
|
||||
}
|
||||
}`)
|
||||
// User config re-enables it.
|
||||
userPath := writeConfig(t, `{
|
||||
"mcp": {
|
||||
"servers": {
|
||||
"docs": {
|
||||
"disabled": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}`)
|
||||
|
||||
resolved, err := ResolveMCP(ResolveOptions{
|
||||
UserConfigPath: userPath,
|
||||
ProjectConfigPath: projectPath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveMCP() error = %v", err)
|
||||
}
|
||||
|
||||
docs, ok := resolved.Servers["docs"]
|
||||
if !ok {
|
||||
t.Fatal("docs server missing from resolved config")
|
||||
}
|
||||
if docs.Disabled {
|
||||
t.Fatalf("docs.Disabled = true, want user scope to lift the project-level disable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRejectsDuplicateMCPRootAliases(t *testing.T) {
|
||||
path := writeConfig(t, `{
|
||||
"mcpServers": {
|
||||
@@ -1374,6 +1496,71 @@ func TestResolveProviderProfileExtendedJSONAliases(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCatalogDescriptorFoldsCustomHeadersCaseInsensitively(t *testing.T) {
|
||||
t.Setenv("AIMLAPI_PARTNER_ID", "part_env")
|
||||
descriptor, err := providercatalog.Require("aimlapi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := ProviderProfile{
|
||||
BaseURL: descriptor.DefaultBaseURL,
|
||||
CustomHeaders: map[string]string{
|
||||
"x-aimlapi-partner-id": "part_override",
|
||||
},
|
||||
}
|
||||
|
||||
applyCatalogDescriptor(&profile, descriptor, true)
|
||||
|
||||
if got := profile.CustomHeaders["X-AIMLAPI-Partner-ID"]; got != "part_override" {
|
||||
t.Fatalf("canonical header value = %q, want override", got)
|
||||
}
|
||||
if _, duplicate := profile.CustomHeaders["x-aimlapi-partner-id"]; duplicate {
|
||||
t.Fatalf("case-colliding header survived merge: %#v", profile.CustomHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCatalogDescriptorUsesAimlapiPartnerEnvironmentOverride(t *testing.T) {
|
||||
t.Setenv("AIMLAPI_PARTNER_ID", "part_env")
|
||||
descriptor, err := providercatalog.Require("aimlapi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := ProviderProfile{BaseURL: descriptor.DefaultBaseURL}
|
||||
|
||||
applyCatalogDescriptor(&profile, descriptor, true)
|
||||
|
||||
if got := profile.CustomHeaders["X-AIMLAPI-Partner-ID"]; got != "part_env" {
|
||||
t.Fatalf("partner header = %q, want part_env", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCatalogDescriptorStripsAimlapiAttributionFromRetargetedProfile(t *testing.T) {
|
||||
descriptor, err := providercatalog.Require("aimlapi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := ProviderProfile{
|
||||
BaseURL: "https://proxy.example.test/v1",
|
||||
CustomHeaders: map[string]string{
|
||||
"x-aimlapi-partner-id": "persisted-partner",
|
||||
"X-AIMLAPI-Integration-Repo": "Gitlawb/zero",
|
||||
"X-AIMLAPI-Integration-Version": "zero",
|
||||
"X-Environment": "staging",
|
||||
},
|
||||
}
|
||||
|
||||
applyCatalogDescriptor(&profile, descriptor, true)
|
||||
|
||||
for key := range profile.CustomHeaders {
|
||||
if strings.HasPrefix(strings.ToLower(key), "x-aimlapi-") {
|
||||
t.Fatalf("catalog attribution survived retargeting: %#v", profile.CustomHeaders)
|
||||
}
|
||||
}
|
||||
if profile.CustomHeaders["X-Environment"] != "staging" {
|
||||
t.Fatalf("user header was removed: %#v", profile.CustomHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveProviderProfileParseThinkTagsFalseAlias(t *testing.T) {
|
||||
path := writeConfig(t, `{
|
||||
"activeProvider": "custom",
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# Turn benchmark manifest
|
||||
|
||||
The baseline manifest (`manifests/baseline.json`) is the per-turn benchmark's
|
||||
program keystone: it defines the tasks the harness runs, the workspace each
|
||||
starts in, and — critically — what "pass" means for each task. Because `make
|
||||
baseline` is re-run on every perf change, the manifest's pass/fail contract
|
||||
matters for months, so the contract is written down here.
|
||||
|
||||
## Task count
|
||||
|
||||
48 tasks across seven classes:
|
||||
|
||||
| class | count | oracle | tier |
|
||||
|-----------|-------|-----------------------------|-------------|
|
||||
| nav | 10 | none | latency |
|
||||
| edit | 10 | substring grep | correctness |
|
||||
| fix | 8 | scoped `go test -run <name>` | correctness |
|
||||
| refactor | 6 | `go build ./...` | build |
|
||||
| longproc | 4 | none | latency |
|
||||
| longctx | 4 | none | latency |
|
||||
| parallel | 6 | none | latency |
|
||||
| **total** | **48**| | |
|
||||
|
||||
## Oracle tiers
|
||||
|
||||
Pass/fail is reported per tier so the report cannot be misread as a blanket
|
||||
correctness verdict. The tier is decided per task from oracle presence and the
|
||||
manifest's `buildOnlyClasses` list:
|
||||
|
||||
- **Correctness** (18 tasks: 10 edit + 8 fix) — a positive oracle (substring
|
||||
grep that the requested change landed, or a scoped `go test -run` that the
|
||||
bug fix passes). This is the only pass rate that can move with model quality:
|
||||
`tasksVerified` / `tasksPassed` / `correctnessPassRate`.
|
||||
- **Build-only** (6 tasks: refactor) — `go build ./...` proves the edit
|
||||
compiles, not that the refactor achieved its goal (a no-op refactor passes).
|
||||
The manifest declares `buildOnlyClasses: ["refactor"]`. Reported as
|
||||
`buildCheckedTasks` / `buildPassedTasks` / `buildPassRate`, never in
|
||||
`correctnessPassRate`.
|
||||
- **Latency-only** (24 tasks: nav, longproc, longctx, parallel) — no
|
||||
`verificationCommand`. An exit 0 only proves the turn ran, not that the answer
|
||||
was right. They contribute to latency and span attribution only and are
|
||||
counted in `latencyOnlyTasks`, never in any pass rate.
|
||||
|
||||
A task's tier is driven by oracle **presence** first: a task with no
|
||||
`verificationCommand` is latency-only even if its class is listed in
|
||||
`buildOnlyClasses`, so a missing oracle can never silently pass on exit 0.
|
||||
|
||||
## Known limitations (deferred)
|
||||
|
||||
These are accepted for the Phase 0 baseline and tracked in a follow-up issue;
|
||||
they do not block the baseline because the tier split keeps the report honest:
|
||||
|
||||
- The edit grep oracles are substring checks. A rename oracle asserts the new
|
||||
name is present AND the old name is gone (compound `bash -c`), but an
|
||||
add-field oracle (`grep -R Label .`) only proves the string appears, not that
|
||||
it landed on the right struct. Strengthening these to `go vet`/`go build` +
|
||||
structural assertions is follow-up work.
|
||||
- The refactor build oracle is non-positive (a no-op refactor passes). A
|
||||
structural verifier that proves the refactor happened is follow-up work;
|
||||
until then refactor is `buildPassRate`, not `correctnessPassRate`.
|
||||
- The 24 read-only tasks have no oracle. Adding deterministic oracles for nav
|
||||
(diff against an expected answer) and documenting longproc/longctx/parallel
|
||||
as permanently latency-only is follow-up work.
|
||||
|
||||
## Fixtures
|
||||
|
||||
Each task's `workspaceFixture` points at a small self-contained workspace under
|
||||
`testdata/` so the suite runs offline and repeatably. Mutating tasks (edit,
|
||||
fix, refactor) run against a per-invocation **copy** of their fixture, so the
|
||||
checked-in fixtures stay clean and one task's edits can't bleed into the next
|
||||
iteration or a later task.
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"id": "zero-baseline-turn",
|
||||
"name": "Zero per-turn baseline (Phase 0)",
|
||||
"description": "Latency-first baseline: measures where a turn spends wall time, with pass/fail reported per oracle tier so it cannot be misread as a blanket correctness verdict. Read-only classes (nav, longproc, longctx, parallel) carry no verification oracle — a zero-exit only proves the turn ran, so they count as latency-only and never in any pass rate. Correctness classes (edit, fix) carry a positive oracle (substring grep or a scoped `go test`) and are the only tasks in correctnessPassRate. Build-only classes (refactor) carry `go build ./...`, which proves the edit compiles but not that the refactor achieved its goal, so they are reported as buildPassRate and excluded from correctnessPassRate.",
|
||||
"buildOnlyClasses": ["refactor"],
|
||||
"tasks": [
|
||||
{"id": "nav-01", "name": "list repo files", "class": "nav", "prompt": "List the files in this repository and report their count. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-02", "name": "find a function", "class": "nav", "prompt": "Find the function named 'greet' and report the file and line it is defined on. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-03", "name": "summarize module", "class": "nav", "prompt": "Read main.go and summarize what this program does in two sentences. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-04", "name": "count tests", "class": "nav", "prompt": "Count the test functions (functions whose names start with Test) in this repository and report the number. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-05", "name": "locate a string", "class": "nav", "prompt": "Find every occurrence of the string 'TODO' in the repository and report the files. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-06", "name": "describe package", "class": "nav", "prompt": "Describe the exported API of this package by reading the source. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-07", "name": "trace a call", "class": "nav", "prompt": "Trace which function calls which other function starting from main. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-08", "name": "find imports", "class": "nav", "prompt": "List every third-party import used by this repository. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-09", "name": "report config", "class": "nav", "prompt": "Read config.json and report the keys it contains. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
{"id": "nav-10", "name": "doc lookup", "class": "nav", "prompt": "Find the doc comment on the greet function and quote it. Do not modify anything.", "workspaceFixture": "../testdata/nav"},
|
||||
|
||||
{"id": "edit-01", "name": "rename a constant", "class": "edit", "prompt": "Rename the constant MaxRetries to RetryLimit everywhere it appears.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "grep -R RetryLimit . && ! grep -R MaxRetries ."]},
|
||||
{"id": "edit-02", "name": "fix a typo", "class": "edit", "prompt": "Fix the typo in the comment above the greet function ('receieves' should be 'receives').", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "! grep -R receieves ."]},
|
||||
{"id": "edit-03", "name": "add a field", "class": "edit", "prompt": "Add a string field named Label to the Config struct in main.go.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["grep", "-R", "Label", "."]},
|
||||
{"id": "edit-04", "name": "bump a version", "class": "edit", "prompt": "Change the version string in version.go from 1.0.0 to 1.1.0.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "grep -R 1.1.0 . && ! grep -R 1.0.0 ."]},
|
||||
{"id": "edit-05", "name": "remove a log line", "class": "edit", "prompt": "Remove the line that prints 'debug: starting'.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "! grep -R 'debug: starting' ."]},
|
||||
{"id": "edit-06", "name": "add a license header", "class": "edit", "prompt": "Add a comment line '// SPDX-License-Identifier: MIT' at the top of main.go.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["grep", "-R", "SPDX-License-Identifier", "."]},
|
||||
{"id": "edit-07", "name": "change a default", "class": "edit", "prompt": "Change the default value of the Port constant from 8080 to 9090.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "grep -R 9090 . && ! grep -R 8080 ."]},
|
||||
{"id": "edit-08", "name": "add a getter", "class": "edit", "prompt": "Add a method func (c *Config) GetLabel() string that returns c.Label.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["bash", "-c", "grep -R 'func (c *Config) GetLabel' ."]},
|
||||
{"id": "edit-09", "name": "wrap an error", "class": "edit", "prompt": "In the load function, wrap the returned error with fmt.Errorf so it includes 'load failed'.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["grep", "-R", "load failed", "."]},
|
||||
{"id": "edit-10", "name": "update a message", "class": "edit", "prompt": "Change the printed greeting from 'hello' to 'hello, world'.", "workspaceFixture": "../testdata/edit", "verificationCommand": ["grep", "-R", "hello, world", "."]},
|
||||
|
||||
{"id": "fix-01", "name": "off-by-one", "class": "fix", "prompt": "In bugs.go the Sum function has an off-by-one error that skips the last element; fix the loop so it sums all elements.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestSumIncludesLastElement", "./..."]},
|
||||
{"id": "fix-02", "name": "nil deref", "class": "fix", "prompt": "In bugs.go DefaultConfig returns nil and Port() dereferences it; make DefaultConfig return a valid Config with Port 8080.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestPortReturnsDefault", "./..."]},
|
||||
{"id": "fix-03", "name": "wrong operator", "class": "fix", "prompt": "In bugs.go Max uses the wrong inequality and returns the smaller value; fix it to return the larger.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestMaxReturnsLarger", "./..."]},
|
||||
{"id": "fix-04", "name": "truncated read", "class": "fix", "prompt": "In bugs.go ReadFirst returns only a single character instead of the full content; fix it to return the whole string.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestReadFirstReturnsFullContent", "./..."]},
|
||||
{"id": "fix-05", "name": "swapped args", "class": "fix", "prompt": "In bugs.go Divide swaps its operands; fix the order so Divide(10,2) returns 5.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestDivideOrder", "./..."]},
|
||||
{"id": "fix-06", "name": "missing return", "class": "fix", "prompt": "In bugs.go Classify is missing a return for the high case; add it so Classify(500) returns \"high\".", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestClassifyHigh", "./..."]},
|
||||
{"id": "fix-07", "name": "wrong label prefix", "class": "fix", "prompt": "In bugs.go Label returns 'admin-<name>' but the test expects 'user-<name>'; change the prefix from 'admin-' to 'user-'.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-run", "TestLabelUsesString", "./..."]},
|
||||
{"id": "fix-08", "name": "race on counter", "class": "fix", "prompt": "In bugs.go the global counter is incremented without synchronization; guard it with a mutex.", "workspaceFixture": "../testdata/fix", "verificationCommand": ["go", "test", "-race", "-run", "TestCounterRace", "./..."]},
|
||||
|
||||
{"id": "refactor-01", "name": "extract helper", "class": "refactor", "prompt": "Extract the duplicated greeting logic into a single helper function and call it from both sites.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-02", "name": "split a file", "class": "refactor", "prompt": "Split main.go into two files: one for Config, one for the rest, keeping the package building.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-03", "name": "rename package", "class": "refactor", "prompt": "Rename the package from app to zeroapp everywhere and keep it building.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-04", "name": "introduce a type", "class": "refactor", "prompt": "Introduce a named type for the map[string]int used in main.go and update its users.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-05", "name": "inline a wrapper", "class": "refactor", "prompt": "Inline the thin wrapper function into its single caller and remove it.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
{"id": "refactor-06", "name": "consolidate errors", "class": "refactor", "prompt": "Consolidate the repeated error-construction blocks into a single helper.", "workspaceFixture": "../testdata/refactor", "verificationCommand": ["go", "build", "./..."]},
|
||||
|
||||
{"id": "longproc-01", "name": "long build", "class": "longproc", "prompt": "Run the build command and report whether it succeeds. Do not modify anything.", "workspaceFixture": "../testdata/longproc"},
|
||||
{"id": "longproc-02", "name": "long test", "class": "longproc", "prompt": "Run go test ./... and report the result. Do not modify anything.", "workspaceFixture": "../testdata/longproc"},
|
||||
{"id": "longproc-03", "name": "bench run", "class": "longproc", "prompt": "Run go test -bench=. -benchtime=1x and report the benchmark numbers. Do not modify anything.", "workspaceFixture": "../testdata/longproc"},
|
||||
{"id": "longproc-04", "name": "vet run", "class": "longproc", "prompt": "Run go vet ./... and report the output. Do not modify anything.", "workspaceFixture": "../testdata/longproc"},
|
||||
|
||||
{"id": "longctx-01", "name": "summarize large file", "class": "longctx", "prompt": "Read the large generated file big.go and summarize its structure. Do not modify anything.", "workspaceFixture": "../testdata/longctx"},
|
||||
{"id": "longctx-02", "name": "find in large file", "class": "longctx", "prompt": "In big.go, find every function that returns an error and list them. Do not modify anything.", "workspaceFixture": "../testdata/longctx"},
|
||||
{"id": "longctx-03", "name": "cross-file question", "class": "longctx", "prompt": "Read all .go files and report which two functions share the same name across files. Do not modify anything.", "workspaceFixture": "../testdata/longctx"},
|
||||
{"id": "longctx-04", "name": "trace through large file", "class": "longctx", "prompt": "Starting from Handle, trace the call path through big.go and report the chain. Do not modify anything.", "workspaceFixture": "../testdata/longctx"},
|
||||
|
||||
{"id": "parallel-01", "name": "read six files", "class": "parallel", "prompt": "Read a.txt, b.txt, c.txt, d.txt, e.txt, and f.txt, then report the first line of each. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-02", "name": "grep six patterns", "class": "parallel", "prompt": "Search for 'alpha', 'beta', 'gamma', 'delta', 'epsilon', and 'zeta' across the repo and report which files contain each. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-03", "name": "list six dirs", "class": "parallel", "prompt": "List the contents of dir1, dir2, dir3, dir4, dir5, and dir6. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-04", "name": "stat six files", "class": "parallel", "prompt": "Report the line count of a.txt, b.txt, c.txt, d.txt, e.txt, and f.txt. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-05", "name": "read configs", "class": "parallel", "prompt": "Read config1.json, config2.json, config3.json, config4.json, config5.json, and config6.json and report each top-level key. Do not modify anything.", "workspaceFixture": "../testdata/parallel"},
|
||||
{"id": "parallel-06", "name": "summarize six", "class": "parallel", "prompt": "Read the six .txt files and give a one-line summary of each. Do not modify anything.", "workspaceFixture": "../testdata/parallel"}
|
||||
]
|
||||
}
|
||||
Whitespace-only changes.
@@ -0,0 +1,84 @@
|
||||
# Per-turn benchmark reports
|
||||
|
||||
This directory holds generated per-turn benchmark reports. It is **not** a
|
||||
checked-in source of truth for performance — the numbers are machine- and
|
||||
model-specific, so a single snapshot here would mislead rather than inform.
|
||||
|
||||
## Generating a baseline
|
||||
|
||||
Run the harness over the checked-in manifest:
|
||||
|
||||
```sh
|
||||
make baseline ZERO_BENCH_MODEL=<model> # uses ./zero
|
||||
make baseline ZERO_BENCH_MODEL=<model> ZERO_BENCH_BINARY=/path/to/zero
|
||||
```
|
||||
|
||||
This builds `zero`, then runs `zero-perf-bench turn` over
|
||||
`internal/perfbench/manifests/baseline.json`, capturing each turn's trace
|
||||
(`zero exec --trace <tmpfile>`) and writing the aggregated result to
|
||||
`reports/baseline.json`.
|
||||
|
||||
The JSON report is self-describing: model, mode, self-correct flag, version,
|
||||
commit, date, per-span median/P95, the **top three controllable latency sources**
|
||||
ranked by **exclusive** time, per-class roll-ups, and token/count totals.
|
||||
That top-three list is the Phase 0 baseline's "do not proceed until" criterion —
|
||||
it names where a turn actually spends time so later optimization work is
|
||||
targeted, not guessed.
|
||||
|
||||
### Attribution model (honest by construction)
|
||||
|
||||
Spans record wall intervals and are **not** summed into each other. Each span's
|
||||
**exclusive** time is its own duration minus the union of its nested children's
|
||||
intervals, derived at finish by interval containment. So a `provider_connect`
|
||||
that runs concurrently inside `generation`, or a `permission_wait` nested inside
|
||||
`tool_execution`, each contributes only its own exclusive time — they no longer
|
||||
double-count the same wall. The top-latency shares therefore sum to ~1 for a
|
||||
well-instrumented run, and the ranking reflects where wall time is actually
|
||||
spent.
|
||||
|
||||
**Coverage** is the fraction of wall covered by the union of all span
|
||||
intervals (capped at 1.0) — the honest "≥95% of wall accounted for" metric. A
|
||||
run with `coverage < 0.95` has uninstrumented gaps, not an inflated attribution.
|
||||
|
||||
### Pass/fail is reported per oracle tier
|
||||
|
||||
Pass/fail is split into three tiers so it cannot be misread as a blanket
|
||||
correctness verdict (see `MANIFEST.md` for the class breakdown):
|
||||
|
||||
- **Correctness** (`tasksVerified` / `tasksPassed` / `correctnessPassRate`):
|
||||
tasks with a positive oracle — edit's substring grep, fix's scoped `go test`.
|
||||
This is the only pass rate that can move with model quality.
|
||||
- **Build-only** (`buildCheckedTasks` / `buildPassedTasks` / `buildPassRate`):
|
||||
refactor's `go build ./...`, which proves the edit compiles but not that the
|
||||
refactor achieved its goal. Reported separately, never in `correctnessPassRate`.
|
||||
- **Latency-only** (`latencyOnlyTasks`): the read-only classes (nav, longproc,
|
||||
longctx, parallel) carry no oracle — an exit 0 only proves the turn ran. They
|
||||
contribute to latency and span attribution and are excluded from every pass
|
||||
rate.
|
||||
|
||||
`tasksAttempted` is still the total across all three tiers. The tier class lists
|
||||
are echoed in the report so a consumer can see exactly which classes each rate
|
||||
is computed over.
|
||||
|
||||
> **Do not average the tier pass rates. Do not report a single "pass rate".**
|
||||
> `correctnessPassRate` and `buildPassRate` measure different things over
|
||||
> different task sets and must never be combined — a weighted or arithmetic
|
||||
> mean of the two is a number that means nothing. The schema offers no
|
||||
> headline pass rate on purpose: a consumer must name the tier it is reporting
|
||||
> (`correctness`, `build`, or `latency-only`) and quote that tier's fields. A
|
||||
> read-only run that exits 0 is a latency sample, not a pass.
|
||||
|
||||
## What to commit
|
||||
|
||||
Commit the manifest and fixtures (`manifests/`, `../testdata/`), not a generated
|
||||
`baseline.json`. A generated report belongs in a PR description or a shared
|
||||
dashboard as evidence for one configuration, not in the tree as a durable
|
||||
expectation. `.gitkeep` keeps the directory present between runs.
|
||||
|
||||
## Caveats
|
||||
|
||||
- Each task is a fresh `zero exec` process, so iterations are **cold-start**
|
||||
samples. A warm path (reusing an in-process agent) is future work.
|
||||
- Mutating tasks (edit/fix/refactor) run against a per-invocation **copy** of
|
||||
their fixture, so the checked-in fixtures stay clean and one task's edits
|
||||
can't bleed into the next iteration.
|
||||
@@ -25,17 +25,29 @@ const TaskSchemaVersion = 1
|
||||
// ZERO must satisfy. The set is recorded by ID with every result so a published
|
||||
// number is traceable to the exact tasks that produced it.
|
||||
type TaskSet struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Tasks []BenchTask `json:"tasks"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Tasks []BenchTask `json:"tasks"`
|
||||
// BuildOnlyClasses lists task classes whose verificationCommand is a
|
||||
// non-positive build check (e.g. refactor's `go build ./...`): it proves the
|
||||
// edit compiles, not that the refactor achieved its goal. The turn benchmark
|
||||
// reports these separately from correctness oracles so a build-pass cannot be
|
||||
// misread as a correctness pass. Classes with a verificationCommand that are
|
||||
// NOT listed here are treated as correctness classes; classes whose tasks
|
||||
// carry no verificationCommand are latency-only regardless of this list.
|
||||
BuildOnlyClasses []string `json:"buildOnlyClasses,omitempty"`
|
||||
}
|
||||
|
||||
// BenchTask is one benchmark task. WorkspaceFixture is the relative path of the
|
||||
// task's starting workspace; VerificationCommand (optional) is the command the
|
||||
// default runner executes to decide pass/fail after ZERO finishes.
|
||||
// default runner executes to decide pass/fail after ZERO finishes. Class groups
|
||||
// the task for the turn-benchmark's per-group latency breakdown (e.g. "nav",
|
||||
// "edit", "fix"); it is optional and ignored by the pass/fail runner.
|
||||
type BenchTask struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Class string `json:"class,omitempty"`
|
||||
Prompt string `json:"prompt"`
|
||||
WorkspaceFixture string `json:"workspaceFixture,omitempty"`
|
||||
VerificationCommand []string `json:"verificationCommand,omitempty"`
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
// Package edit is a small workspace for the edit-class benchmark tasks: each
|
||||
// task asks the agent to make one targeted edit, verified by a grep/build
|
||||
// command in the manifest.
|
||||
package edit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Port is the default listen port.
|
||||
const Port = 8080
|
||||
|
||||
// MaxRetries is the maximum number of retries before giving up.
|
||||
const MaxRetries = 3
|
||||
|
||||
// Config holds runtime configuration.
|
||||
type Config struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
// greet returns a greeting for the named user.
|
||||
//
|
||||
// This receieves a name and formats a hello message.
|
||||
func greet(name string) string {
|
||||
return "hello"
|
||||
}
|
||||
|
||||
// load reads a config file and returns its bytes, or an error on failure.
|
||||
func load(path string) ([]byte, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
fmt.Println("debug: starting")
|
||||
cfg := Config{Name: "demo"}
|
||||
fmt.Println(greet(cfg.Name))
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package edit
|
||||
|
||||
// Version is the build version string.
|
||||
const Version = "1.0.0"
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
// Package fix holds deliberately buggy functions for the bug-fix benchmark
|
||||
// tasks. Each function is covered by its own test (see bugs_test.go) and each
|
||||
// task verifies with `go test -run <TestName>`, so the tasks are independent:
|
||||
// the initial package compiles and the targeted test fails until the named bug
|
||||
// is fixed. Every bug is fixable without changing a function signature, so the
|
||||
// test files compile in both the buggy and the fixed state.
|
||||
package fix
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Sum returns the sum of all elements. BUG: off-by-one — skips the last element.
|
||||
func Sum(xs []int) int {
|
||||
total := 0
|
||||
for i := 0; i < len(xs)-1; i++ {
|
||||
total += xs[i]
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// DefaultConfig returns the default configuration. BUG: returns nil, which
|
||||
// Port() then dereferences.
|
||||
func DefaultConfig() *Config {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Config holds demo settings.
|
||||
type Config struct {
|
||||
Port int
|
||||
}
|
||||
|
||||
// Port returns the configured port. BUG: panics when DefaultConfig() is nil.
|
||||
func Port() int {
|
||||
return DefaultConfig().Port
|
||||
}
|
||||
|
||||
// Max returns the larger of a and b. BUG: uses the wrong inequality and returns
|
||||
// the smaller value.
|
||||
func Max(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// ReadFirst returns the first byte of the given file's content as a string. BUG:
|
||||
// returns only a single character instead of the full content.
|
||||
func ReadFirst(content string) string {
|
||||
if len(content) == 0 {
|
||||
return ""
|
||||
}
|
||||
return string(content[0])
|
||||
}
|
||||
|
||||
// Divide returns a divided by b. BUG: the operands are swapped.
|
||||
func Divide(a, b int) int {
|
||||
return b / a
|
||||
}
|
||||
|
||||
// Classify returns "low", "mid", or "high". BUG: the "high" branch is missing a
|
||||
// return, so high values fall through to the default "".
|
||||
func Classify(n int) string {
|
||||
if n < 10 {
|
||||
return "low"
|
||||
}
|
||||
if n < 100 {
|
||||
return "mid"
|
||||
}
|
||||
// BUG: missing return for the high case
|
||||
return ""
|
||||
}
|
||||
|
||||
// Label returns a display label for the given name. BUG: uses the wrong prefix
|
||||
// ("admin-" instead of "user-"), producing the wrong text.
|
||||
func Label(name string) string {
|
||||
return fmt.Sprintf("admin-%s", name)
|
||||
}
|
||||
|
||||
// counter is a shared, unsynchronized counter. BUG: Inc is not goroutine-safe.
|
||||
var counter int
|
||||
|
||||
// Inc increments the shared counter.
|
||||
func Inc() {
|
||||
counter++
|
||||
}
|
||||
|
||||
// Count returns the current counter value.
|
||||
func Count() int {
|
||||
return counter
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package fix
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSumIncludesLastElement(t *testing.T) {
|
||||
if got := Sum([]int{1, 2, 3, 4}); got != 10 {
|
||||
t.Fatalf("Sum = %d, want 10", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPortReturnsDefault(t *testing.T) {
|
||||
if got := Port(); got != 8080 {
|
||||
t.Fatalf("Port() = %d, want 8080 (DefaultConfig returned nil)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxReturnsLarger(t *testing.T) {
|
||||
if got := Max(3, 7); got != 7 {
|
||||
t.Fatalf("Max(3,7) = %d, want 7", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFirstReturnsFullContent(t *testing.T) {
|
||||
if got := ReadFirst("hello"); got != "hello" {
|
||||
t.Fatalf("ReadFirst = %q, want %q", got, "hello")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDivideOrder(t *testing.T) {
|
||||
if got := Divide(10, 2); got != 5 {
|
||||
t.Fatalf("Divide(10,2) = %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyHigh(t *testing.T) {
|
||||
if got := Classify(500); got != "high" {
|
||||
t.Fatalf("Classify(500) = %q, want %q", got, "high")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLabelUsesString(t *testing.T) {
|
||||
if got := Label("alice"); got != "user-alice" {
|
||||
t.Fatalf("Label = %q, want %q", got, "user-alice")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCounterRace exercises Inc concurrently. Run with `go test -race`:
|
||||
// the unsynchronized counter trips the race detector until a mutex is added.
|
||||
func TestCounterRace(t *testing.T) {
|
||||
counter = 0
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 200; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 100; j++ {
|
||||
Inc()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if got := Count(); got != 20000 {
|
||||
t.Fatalf("Count = %d, want 20000", got)
|
||||
}
|
||||
}
|
||||
+1287
File diff suppressed because it is too large.
Load diff
+21
@@ -0,0 +1,21 @@
|
||||
// Package longproc is a small buildable workspace for the long-running-process
|
||||
// benchmark tasks, which run build/test/vet/bench commands against it.
|
||||
package longproc
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Version is the demo build's version string.
|
||||
const Version = "1.0.0"
|
||||
|
||||
// Process simulates a bounded unit of work for the benchmark task.
|
||||
func Process(n int) int {
|
||||
total := 0
|
||||
for i := 0; i < n; i++ {
|
||||
total += i
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func main() {
|
||||
fmt.Println("longproc fixture", Version, Process(100))
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package longproc
|
||||
|
||||
import "testing"
|
||||
|
||||
// benchSink keeps Process(100) live across the benchmark loop so the compiler
|
||||
// can't optimize the call away and report an unrealistically fast benchmark.
|
||||
var benchSink int
|
||||
|
||||
func TestProcess(t *testing.T) {
|
||||
if got := Process(100); got != 4950 {
|
||||
t.Fatalf("Process(100) = %d, want 4950", got)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkProcess(b *testing.B) {
|
||||
var sink int
|
||||
for i := 0; i < b.N; i++ {
|
||||
sink = Process(100)
|
||||
}
|
||||
benchSink = sink
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
# nav fixture
|
||||
|
||||
Small read-only workspace for navigation/classification benchmark tasks.
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"port": 8080,
|
||||
"name": "nav-fixture",
|
||||
"retries": 3
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
// Package nav is a small read-only fixture for navigation tasks.
|
||||
package nav
|
||||
|
||||
import "fmt"
|
||||
|
||||
// MaxRetries is the configured retry limit for the demo client.
|
||||
const MaxRetries = 3
|
||||
|
||||
// Config holds the demo client's settings.
|
||||
type Config struct {
|
||||
Port int
|
||||
Name string
|
||||
}
|
||||
|
||||
// greet returns a greeting for the given name.
|
||||
// receieves is a deliberate typo present in the fixture (see edit tasks).
|
||||
func greet(name string) string {
|
||||
return fmt.Sprintf("hello, %s", name)
|
||||
}
|
||||
|
||||
// main is the fixture entry point.
|
||||
func main() {
|
||||
fmt.Println(greet("world"))
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of a.txt
|
||||
second line of a.txt
|
||||
third line of a.txt
|
||||
alpha alpha alpha
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of b.txt
|
||||
second line of b.txt
|
||||
third line of b.txt
|
||||
beta beta beta
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of c.txt
|
||||
second line of c.txt
|
||||
third line of c.txt
|
||||
gamma gamma gamma
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 1, "key": "value-1", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 2, "key": "value-2", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 3, "key": "value-3", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 4, "key": "value-4", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 5, "key": "value-5", "enabled": true }
|
||||
@@ -0,0 +1 @@
|
||||
{ "id": 6, "key": "value-6", "enabled": true }
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of d.txt
|
||||
second line of d.txt
|
||||
third line of d.txt
|
||||
delta delta delta
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir1/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir2/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir3/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir4/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir5/notes.md
|
||||
@@ -0,0 +1 @@
|
||||
contents of dir6/notes.md
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of e.txt
|
||||
second line of e.txt
|
||||
third line of e.txt
|
||||
epsilon epsilon epsilon
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
first line of f.txt
|
||||
second line of f.txt
|
||||
third line of f.txt
|
||||
zeta zeta zeta
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
// Package refactor is the starting workspace for the multi-file refactor
|
||||
// benchmark tasks. It is intentionally a little messy (duplicated logic, a
|
||||
// bare map type, a thin wrapper) so the refactor prompts have real work to do.
|
||||
// It builds cleanly at all times; the tasks keep it building.
|
||||
package refactor
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Config holds demo settings.
|
||||
type Config struct {
|
||||
Name string
|
||||
Port int
|
||||
}
|
||||
|
||||
// GreetFromConfig returns a greeting using the config's name.
|
||||
func GreetFromConfig(c Config) string {
|
||||
return fmt.Sprintf("hello, %s", c.Name)
|
||||
}
|
||||
|
||||
// GreetByName returns a greeting using a bare name. BUG-note: this duplicates
|
||||
// GreetFromConfig's formatting — refactor-01 extracts a shared helper.
|
||||
func GreetByName(name string) string {
|
||||
return fmt.Sprintf("hello, %s", name)
|
||||
}
|
||||
|
||||
// Wrapper is a thin wrapper around GreetByName — refactor-05 inlines it.
|
||||
func Wrapper(name string) string {
|
||||
return GreetByName(name)
|
||||
}
|
||||
|
||||
// stats is a bare map used across the file — refactor-04 introduces a named type.
|
||||
var stats = map[string]int{}
|
||||
|
||||
// Record bumps a stat by name.
|
||||
func Record(name string) {
|
||||
stats[name]++
|
||||
}
|
||||
|
||||
// Lookup returns a stat by name.
|
||||
func Lookup(name string) int {
|
||||
return stats[name]
|
||||
}
|
||||
|
||||
// load failed wrapped error — refactor-06 consolidates these.
|
||||
func firstError() error {
|
||||
return fmt.Errorf("load failed: missing input")
|
||||
}
|
||||
|
||||
func secondError() error {
|
||||
return fmt.Errorf("load failed: bad format")
|
||||
}
|
||||
|
||||
func buildError(ctx string) error {
|
||||
return fmt.Errorf("load failed: %s", ctx)
|
||||
}
|
||||
@@ -0,0 +1,679 @@
|
||||
package perfbench
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
)
|
||||
|
||||
// TurnSchemaVersion is the schema version of a published turn-benchmark result.
|
||||
// Bump when the TurnBenchResult shape changes so consumers can detect drift.
|
||||
//
|
||||
// v2 splits pass/fail into three oracle tiers so the report cannot be misread as
|
||||
// a blanket correctness verdict: tasksVerified/tasksPassed/correctnessPassRate
|
||||
// cover only the positive-oracle classes (edit/fix); buildCheckedTasks/
|
||||
// buildPassedTasks/buildPassRate cover the non-positive build-check classes
|
||||
// (refactor); latencyOnlyTasks covers the no-oracle classes (nav/longproc/
|
||||
// longctx/parallel). tasksAttempted is still every task run.
|
||||
const TurnSchemaVersion = 2
|
||||
|
||||
// TurnRunner runs one benchmark task and reports its outcome plus the captured
|
||||
// per-turn trace. A non-nil Err means the run failed to execute (process crash);
|
||||
// Passed reflects the verification result. Trace is the parsed NDJSON trace
|
||||
// (nil when the run errored before emitting one).
|
||||
type TurnRunner func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome
|
||||
|
||||
// TurnTaskOutcome is what a TurnRunner reports for one task iteration.
|
||||
type TurnTaskOutcome struct {
|
||||
Passed bool
|
||||
VerifyErr string
|
||||
WallMs float64
|
||||
Trace *trace.TurnTrace
|
||||
// TraceIssue, when non-empty, explains why the per-turn trace could not be
|
||||
// parsed (file missing or malformed). The run still has a pass/fail verdict,
|
||||
// but an incomplete measurement is surfaced as a result warning so it can't
|
||||
// masquerade as a clean attribution sample.
|
||||
TraceIssue string
|
||||
Err error
|
||||
}
|
||||
|
||||
// TurnBenchConfig configures a turn-benchmark run.
|
||||
type TurnBenchConfig struct {
|
||||
Model string
|
||||
Mode string
|
||||
SelfCorrect bool
|
||||
Version string
|
||||
Commit string
|
||||
// Iterations is how many times each task is run. The per-process `zero exec`
|
||||
// runner is inherently cold-start, so this is the sample count for per-span
|
||||
// median/P95 — a genuine warm path needs an in-process runner (future work).
|
||||
Iterations int
|
||||
// Runner executes one task iteration. Required.
|
||||
Runner TurnRunner
|
||||
// Now overrides the clock for the recorded date (tests inject a fixed time).
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// SpanStats summarizes one span's duration across all measured task iterations.
|
||||
type SpanStats struct {
|
||||
Count int `json:"count"`
|
||||
TotalMs float64 `json:"totalMs"`
|
||||
MedianMs float64 `json:"medianMs"`
|
||||
P95Ms float64 `json:"p95Ms"`
|
||||
MaxMs float64 `json:"maxMs"`
|
||||
}
|
||||
|
||||
// LatencySource is one of the top controllable latency sources, ranked by total
|
||||
// exclusive time across the whole run. Share is its fraction of total exclusive
|
||||
// span time. Because exclusive time subtracts nested children, concurrent and
|
||||
// nested spans no longer double-count, so shares of the top sources sum to ~1
|
||||
// for a well-instrumented run.
|
||||
type LatencySource struct {
|
||||
Span string `json:"span"`
|
||||
TotalMs float64 `json:"totalMs"`
|
||||
Share float64 `json:"share"`
|
||||
}
|
||||
|
||||
// ClassSummary is the per-class (task group) roll-up.
|
||||
//
|
||||
// Passed is the count that passed THIS class's oracle: a correctness pass for
|
||||
// edit/fix, a build pass for refactor, and always 0 for the no-oracle classes
|
||||
// (nav/longproc/longctx/parallel). Verified is how many tasks in the class
|
||||
// carry an oracle (correctness or build); LatencyOnly is how many carry none.
|
||||
// A latency-only class therefore reports Passed=0, Verified=0, LatencyOnly=Tasks.
|
||||
type ClassSummary struct {
|
||||
Tasks int `json:"tasks"`
|
||||
Verified int `json:"verified"`
|
||||
Passed int `json:"passed"`
|
||||
LatencyOnly int `json:"latencyOnly"`
|
||||
WallMs NumericStats `json:"wallMs"`
|
||||
SpanTotals map[string]float64 `json:"spanTotals"`
|
||||
}
|
||||
|
||||
// TurnBenchResult is the publishable turn-benchmark record.
|
||||
//
|
||||
// Pass/fail is split into three oracle tiers so it cannot be misread as a
|
||||
// blanket correctness verdict:
|
||||
// - Correctness (tasksVerified / tasksPassed / correctnessPassRate): tasks
|
||||
// with a positive oracle (edit/fix). This is the only pass rate that can
|
||||
// move with model quality.
|
||||
// - Build-only (buildCheckedTasks / buildPassedTasks / buildPassRate): tasks
|
||||
// whose oracle is a non-positive build check (refactor `go build`). A pass
|
||||
// means it compiles, not that the refactor is correct.
|
||||
// - Latency-only (latencyOnlyTasks): tasks with no oracle (nav/longproc/
|
||||
// longctx/parallel). They ran for latency and span attribution only and are
|
||||
// excluded from every pass rate.
|
||||
//
|
||||
// tasksAttempted is still the total number of tasks run across all three tiers.
|
||||
// The tier class lists are echoed so a consumer can see exactly which classes
|
||||
// each rate is computed over.
|
||||
type TurnBenchResult struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Suite string `json:"suite"`
|
||||
Model string `json:"model"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
SelfCorrect bool `json:"selfCorrect"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Date string `json:"date"`
|
||||
TasksAttempted int `json:"tasksAttempted"`
|
||||
TasksVerified int `json:"tasksVerified"`
|
||||
TasksPassed int `json:"tasksPassed"`
|
||||
LatencyOnlyTasks int `json:"latencyOnlyTasks"`
|
||||
BuildCheckedTasks int `json:"buildCheckedTasks"`
|
||||
BuildPassedTasks int `json:"buildPassedTasks"`
|
||||
CorrectnessPassRate float64 `json:"correctnessPassRate"`
|
||||
BuildPassRate float64 `json:"buildPassRate"`
|
||||
CorrectnessClasses []string `json:"correctnessClasses,omitempty"`
|
||||
BuildOnlyClasses []string `json:"buildOnlyClasses,omitempty"`
|
||||
LatencyOnlyClasses []string `json:"latencyOnlyClasses,omitempty"`
|
||||
Iterations int `json:"iterations"`
|
||||
PerSpan map[string]SpanStats `json:"perSpan"`
|
||||
TopLatency []LatencySource `json:"topLatency"`
|
||||
PerClass map[string]ClassSummary `json:"perClass"`
|
||||
Totals TurnBenchTotals `json:"totals"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// TurnBenchTotals aggregates token and count totals across the whole run.
|
||||
type TurnBenchTotals struct {
|
||||
InputTokens int64 `json:"inputTokens"`
|
||||
CachedInputTokens int64 `json:"cachedInputTokens"`
|
||||
OutputTokens int64 `json:"outputTokens"`
|
||||
ModelRequests int64 `json:"modelRequests"`
|
||||
ToolCalls int64 `json:"toolCalls"`
|
||||
Retries int64 `json:"retries"`
|
||||
Reconnects int64 `json:"reconnects"`
|
||||
Compactions int64 `json:"compactions"`
|
||||
}
|
||||
|
||||
// RunTurnBench executes every task in the set with the configured runner and
|
||||
// returns a self-describing per-turn result. It never aborts on a single task
|
||||
// failure — every task is attempted and recorded. Per-span stats aggregate
|
||||
// across iterations; the top three controllable latency sources are ranked by
|
||||
// total attributed time.
|
||||
func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBenchResult, error) {
|
||||
if len(set.Tasks) == 0 {
|
||||
return TurnBenchResult{}, errors.New("task set has no tasks")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Model) == "" {
|
||||
return TurnBenchResult{}, errors.New("turn benchmark requires a model")
|
||||
}
|
||||
if cfg.Runner == nil {
|
||||
return TurnBenchResult{}, errors.New("turn benchmark requires a runner")
|
||||
}
|
||||
iterations := cfg.Iterations
|
||||
if iterations < 1 {
|
||||
iterations = 1
|
||||
}
|
||||
now := cfg.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
rc := RunContext{Model: cfg.Model, Mode: cfg.Mode, SelfCorrect: cfg.SelfCorrect}
|
||||
|
||||
perSpanSamples := map[string][]float64{}
|
||||
classWalls := map[string][]float64{}
|
||||
classSpanTotals := map[string]map[string]float64{}
|
||||
classTasks := map[string]int{}
|
||||
classVerified := map[string]int{}
|
||||
classPassed := map[string]int{}
|
||||
classLatencyOnly := map[string]int{}
|
||||
correctnessClasses := map[string]bool{}
|
||||
buildOnlyClasses := map[string]bool{}
|
||||
latencyOnlyClasses := map[string]bool{}
|
||||
var totals TurnBenchTotals
|
||||
|
||||
// A class is build-only when the manifest declares it in BuildOnlyClasses.
|
||||
// A task's tier is then decided per-task: no verificationCommand => latency-
|
||||
// only; otherwise build-only if its class is declared, else correctness. The
|
||||
// latency-only tier is always driven by oracle presence, never by the
|
||||
// declared list, so a declared build-only class with a missing oracle still
|
||||
// counts as latency-only rather than silently passing on exit 0.
|
||||
buildOnly := map[string]bool{}
|
||||
for _, c := range set.BuildOnlyClasses {
|
||||
buildOnly[strings.TrimSpace(c)] = true
|
||||
}
|
||||
|
||||
result := TurnBenchResult{
|
||||
SchemaVersion: TurnSchemaVersion,
|
||||
Suite: strings.TrimSpace(set.ID),
|
||||
Model: strings.TrimSpace(cfg.Model),
|
||||
Mode: strings.TrimSpace(cfg.Mode),
|
||||
SelfCorrect: cfg.SelfCorrect,
|
||||
Version: strings.TrimSpace(cfg.Version),
|
||||
Commit: strings.TrimSpace(cfg.Commit),
|
||||
Date: now().UTC().Format(time.RFC3339),
|
||||
Iterations: iterations,
|
||||
PerSpan: map[string]SpanStats{},
|
||||
PerClass: map[string]ClassSummary{},
|
||||
}
|
||||
|
||||
for _, task := range set.Tasks {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
class := strings.TrimSpace(task.Class)
|
||||
if class == "" {
|
||||
class = "default"
|
||||
}
|
||||
classTasks[class]++
|
||||
// A task counts as passed only when every iteration passed. The per-process
|
||||
// runner is cold-start, so a flaky pass on one iteration and a fail on
|
||||
// another is a real regression signal, not noise to average away.
|
||||
passedForTask := true
|
||||
for iter := 0; iter < iterations; iter++ {
|
||||
outcome := cfg.Runner(ctx, task, rc)
|
||||
if outcome.TraceIssue != "" {
|
||||
result.Warnings = append(result.Warnings, Warning{
|
||||
Metric: "trace",
|
||||
Message: fmt.Sprintf("task %s: %s", task.ID, outcome.TraceIssue),
|
||||
})
|
||||
}
|
||||
if outcome.Err != nil {
|
||||
// A crashed run must not look like a normal sample that's merely
|
||||
// absent — surface it as a warning so a run that died every
|
||||
// iteration can't pass as "fewer measurements."
|
||||
result.Warnings = append(result.Warnings, Warning{
|
||||
Metric: "run",
|
||||
Message: fmt.Sprintf("task %s: %v", task.ID, outcome.Err),
|
||||
})
|
||||
passedForTask = false
|
||||
continue
|
||||
}
|
||||
if !outcome.Passed {
|
||||
passedForTask = false
|
||||
}
|
||||
wall := outcome.WallMs
|
||||
if wall <= 0 && outcome.Trace != nil {
|
||||
wall = float64(outcome.Trace.WallDuration().Microseconds()) / 1000
|
||||
}
|
||||
if wall > 0 {
|
||||
classWalls[class] = append(classWalls[class], wall)
|
||||
}
|
||||
if outcome.Trace != nil {
|
||||
aggregateTotals(&totals, outcome.Trace)
|
||||
for _, span := range outcome.Trace.Spans {
|
||||
// Rank by exclusive time (duration minus nested children) so
|
||||
// concurrent/nested spans do not double-count: provider_connect
|
||||
// inside generation and permission_wait inside tool_execution
|
||||
// each contribute their own exclusive time, not their parent's.
|
||||
// A span whose exclusive is legitimately zero (a parent fully
|
||||
// covered by its children) contributes zero on purpose — do
|
||||
// NOT fall back to Duration, which would re-introduce the
|
||||
// double-count. ReadNDJSON preserves a written exclusive_ms:0
|
||||
// as 0 and only falls back to Duration when the key is absent,
|
||||
// so span.Exclusive is always populated here.
|
||||
ms := float64(span.Exclusive.Microseconds()) / 1000
|
||||
perSpanSamples[span.Name] = append(perSpanSamples[span.Name], ms)
|
||||
if classSpanTotals[class] == nil {
|
||||
classSpanTotals[class] = map[string]float64{}
|
||||
}
|
||||
classSpanTotals[class][span.Name] += ms
|
||||
}
|
||||
}
|
||||
}
|
||||
result.TasksAttempted++
|
||||
|
||||
// Classify the task into an oracle tier and update only that tier's
|
||||
// counters. A latency-only task (no verificationCommand) is never counted
|
||||
// in any pass rate even when the runner reports Passed — an exit-0
|
||||
// read-only run proves the turn ran, not that the answer was right.
|
||||
hasOracle := len(task.VerificationCommand) > 0
|
||||
switch {
|
||||
case !hasOracle:
|
||||
result.LatencyOnlyTasks++
|
||||
classLatencyOnly[class]++
|
||||
latencyOnlyClasses[class] = true
|
||||
case buildOnly[class]:
|
||||
result.BuildCheckedTasks++
|
||||
classVerified[class]++
|
||||
buildOnlyClasses[class] = true
|
||||
if passedForTask {
|
||||
result.BuildPassedTasks++
|
||||
classPassed[class]++
|
||||
}
|
||||
default:
|
||||
result.TasksVerified++
|
||||
classVerified[class]++
|
||||
correctnessClasses[class] = true
|
||||
if passedForTask {
|
||||
result.TasksPassed++
|
||||
classPassed[class]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for name, samples := range perSpanSamples {
|
||||
result.PerSpan[name] = summarizeSpan(samples)
|
||||
}
|
||||
result.TopLatency = topLatencySources(result.PerSpan, 3)
|
||||
for class := range classTasks {
|
||||
walls := classWalls[class]
|
||||
var wallStats NumericStats
|
||||
if len(walls) > 0 {
|
||||
wallStats = SummarizeSamples(walls)
|
||||
}
|
||||
result.PerClass[class] = ClassSummary{
|
||||
Tasks: classTasks[class],
|
||||
Verified: classVerified[class],
|
||||
Passed: classPassed[class],
|
||||
LatencyOnly: classLatencyOnly[class],
|
||||
WallMs: wallStats,
|
||||
SpanTotals: classSpanTotals[class],
|
||||
}
|
||||
}
|
||||
result.CorrectnessClasses = sortedSetKeys(correctnessClasses)
|
||||
result.BuildOnlyClasses = sortedSetKeys(buildOnlyClasses)
|
||||
result.LatencyOnlyClasses = sortedSetKeys(latencyOnlyClasses)
|
||||
result.CorrectnessPassRate = passRate(result.TasksPassed, result.TasksVerified)
|
||||
result.BuildPassRate = passRate(result.BuildPassedTasks, result.BuildCheckedTasks)
|
||||
result.Totals = totals
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// passRate is passed/total rounded to the benchmark's metric precision, or 0
|
||||
// when the denominator is zero (no tasks in that tier were run).
|
||||
func passRate(passed, total int) float64 {
|
||||
if total <= 0 {
|
||||
return 0
|
||||
}
|
||||
return RoundMetric(float64(passed) / float64(total))
|
||||
}
|
||||
|
||||
func sortedSetKeys(set map[string]bool) []string {
|
||||
if len(set) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func summarizeSpan(samples []float64) SpanStats {
|
||||
if len(samples) == 0 {
|
||||
return SpanStats{}
|
||||
}
|
||||
stats := SummarizeSamples(samples)
|
||||
total := 0.0
|
||||
for _, s := range samples {
|
||||
total += s
|
||||
}
|
||||
return SpanStats{
|
||||
Count: len(samples),
|
||||
TotalMs: RoundMetric(total),
|
||||
MedianMs: stats.Median,
|
||||
P95Ms: stats.P95,
|
||||
MaxMs: stats.Max,
|
||||
}
|
||||
}
|
||||
|
||||
func topLatencySources(perSpan map[string]SpanStats, top int) []LatencySource {
|
||||
sources := make([]LatencySource, 0, len(perSpan))
|
||||
totalAttributed := 0.0
|
||||
for _, s := range perSpan {
|
||||
totalAttributed += s.TotalMs
|
||||
}
|
||||
for name, s := range perSpan {
|
||||
share := 0.0
|
||||
if totalAttributed > 0 {
|
||||
share = s.TotalMs / totalAttributed
|
||||
}
|
||||
sources = append(sources, LatencySource{
|
||||
Span: name,
|
||||
TotalMs: s.TotalMs,
|
||||
Share: RoundMetric(share),
|
||||
})
|
||||
}
|
||||
sort.SliceStable(sources, func(i, j int) bool {
|
||||
if sources[i].TotalMs != sources[j].TotalMs {
|
||||
return sources[i].TotalMs > sources[j].TotalMs
|
||||
}
|
||||
return sources[i].Span < sources[j].Span
|
||||
})
|
||||
if top > 0 && len(sources) > top {
|
||||
sources = sources[:top]
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
func aggregateTotals(totals *TurnBenchTotals, tr *trace.TurnTrace) {
|
||||
totals.InputTokens += tr.Counter(trace.CounterInputTokens)
|
||||
totals.CachedInputTokens += tr.Counter(trace.CounterCachedInputTokens)
|
||||
totals.OutputTokens += tr.Counter(trace.CounterOutputTokens)
|
||||
totals.ModelRequests += tr.Counter(trace.CounterModelRequests)
|
||||
totals.ToolCalls += tr.Counter(trace.CounterToolCalls)
|
||||
totals.Retries += tr.Counter(trace.CounterRetryCount)
|
||||
totals.Reconnects += tr.Counter(trace.CounterReconnectCount)
|
||||
totals.Compactions += tr.Counter(trace.CounterCompactionCount)
|
||||
}
|
||||
|
||||
// FormatTurnBenchSummary renders a human-readable turn-benchmark summary that
|
||||
// names the top controllable latency sources — the baseline's "do not proceed
|
||||
// until" criterion.
|
||||
func FormatTurnBenchSummary(result TurnBenchResult) string {
|
||||
lines := []string{
|
||||
"Zero turn benchmark: " + displayOrUnknown(result.Suite),
|
||||
"model: " + displayOrUnknown(result.Model),
|
||||
// The headline separates the three oracle tiers so an exit-0 read-only
|
||||
// task can never inflate a "pass rate" that reads as correctness:
|
||||
// correctness (positive oracle, edit/fix), build (non-positive `go build`,
|
||||
// refactor), and latency-only (no oracle, nav/longproc/longctx/parallel).
|
||||
fmt.Sprintf("tasks: %d total | correctness %d/%d (%.0f%%) | build %d/%d (%.0f%%) | latency-only %d | %d iter",
|
||||
result.TasksAttempted,
|
||||
result.TasksPassed, result.TasksVerified, result.CorrectnessPassRate*100,
|
||||
result.BuildPassedTasks, result.BuildCheckedTasks, result.BuildPassRate*100,
|
||||
result.LatencyOnlyTasks, result.Iterations),
|
||||
}
|
||||
if result.Mode != "" {
|
||||
lines = append(lines, "mode: "+result.Mode)
|
||||
}
|
||||
if len(result.TopLatency) > 0 {
|
||||
lines = append(lines, "top latency sources:")
|
||||
for _, src := range result.TopLatency {
|
||||
lines = append(lines, fmt.Sprintf(" %-18s %10s %5.1f%%", src.Span, FormatMetric(src.TotalMs, "ms"), src.Share*100))
|
||||
}
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("totals: in=%d (cached %d) out=%d | requests=%d tools=%d retries=%d reconnects=%d compactions=%d",
|
||||
result.Totals.InputTokens, result.Totals.CachedInputTokens, result.Totals.OutputTokens,
|
||||
result.Totals.ModelRequests, result.Totals.ToolCalls, result.Totals.Retries,
|
||||
result.Totals.Reconnects, result.Totals.Compactions))
|
||||
for _, class := range sortedClasses(result.PerClass) {
|
||||
summary := result.PerClass[class]
|
||||
median := FormatMetric(summary.WallMs.Median, "ms")
|
||||
tier := classTier(result, class)
|
||||
lines = append(lines, fmt.Sprintf(" [%s/%s] %d/%d passed, %d latency-only, wall median %s",
|
||||
class, tier, summary.Passed, summary.Verified, summary.LatencyOnly, median))
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// classTier returns the oracle tier label a class was classified into, so the
|
||||
// per-class roll-up states explicitly which oracle (if any) its "passed" count
|
||||
// is measured against.
|
||||
func classTier(result TurnBenchResult, class string) string {
|
||||
for _, c := range result.BuildOnlyClasses {
|
||||
if c == class {
|
||||
return "build"
|
||||
}
|
||||
}
|
||||
for _, c := range result.LatencyOnlyClasses {
|
||||
if c == class {
|
||||
return "latency"
|
||||
}
|
||||
}
|
||||
return "correctness"
|
||||
}
|
||||
|
||||
func sortedClasses(perClass map[string]ClassSummary) []string {
|
||||
classes := make([]string, 0, len(perClass))
|
||||
for c := range perClass {
|
||||
classes = append(classes, c)
|
||||
}
|
||||
sort.Strings(classes)
|
||||
return classes
|
||||
}
|
||||
|
||||
// WriteTurnBenchJSON writes the indented JSON form of a turn-benchmark result.
|
||||
func WriteTurnBenchJSON(w io.Writer, result TurnBenchResult) error {
|
||||
encoder := json.NewEncoder(w)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(result)
|
||||
}
|
||||
|
||||
// NewTurnExecRunner builds the production turn-benchmark runner: it invokes
|
||||
// headless `zero exec` with stream-json output AND `--trace <tmpfile>`, then
|
||||
// parses the emitted NDJSON trace into a *trace.TurnTrace. binary is the path to
|
||||
// the `zero` binary; extraArgs are appended to every invocation. Pass/fail is
|
||||
// decided from the stream-json run_end exit code (and the task's
|
||||
// VerificationCommand when present), exactly like NewExecRunner.
|
||||
func NewTurnExecRunner(binary string, extraArgs ...string) TurnRunner {
|
||||
return func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome {
|
||||
// Isolate the workspace: copy the fixture into a fresh temp dir so a
|
||||
// mutating task (edit/fix/refactor) can't dirty the shared, checked-in
|
||||
// fixture or bleed into a later iteration of the same task. When no
|
||||
// fixture is configured the agent runs in the caller's cwd as before.
|
||||
if fixture := strings.TrimSpace(task.WorkspaceFixture); fixture != "" {
|
||||
copyDir, cerr := copyFixture(fixture)
|
||||
if cerr != nil {
|
||||
return TurnTaskOutcome{Err: fmt.Errorf("isolate fixture: %w", cerr)}
|
||||
}
|
||||
defer os.RemoveAll(copyDir)
|
||||
task.WorkspaceFixture = copyDir
|
||||
}
|
||||
|
||||
traceFile, err := os.CreateTemp("", "zero-turn-trace-*.ndjson")
|
||||
if err != nil {
|
||||
return TurnTaskOutcome{Err: fmt.Errorf("create trace file: %w", err)}
|
||||
}
|
||||
_ = traceFile.Close()
|
||||
tracePath := traceFile.Name()
|
||||
defer os.Remove(tracePath)
|
||||
|
||||
args := buildTurnExecArgs(task, rc, tracePath, extraArgs)
|
||||
cmd := exec.CommandContext(ctx, binary, args...)
|
||||
cmd.Env = appendNoColor(os.Environ())
|
||||
if dir := strings.TrimSpace(task.WorkspaceFixture); dir != "" {
|
||||
cmd.Dir = dir
|
||||
}
|
||||
var outBuf, errBuf bytes.Buffer
|
||||
cmd.Stdout = &outBuf
|
||||
cmd.Stderr = &errBuf
|
||||
start := time.Now()
|
||||
runErr := cmd.Run()
|
||||
wallMs := float64(time.Since(start).Microseconds()) / 1000
|
||||
|
||||
exitCode, haveExit := streamJSONExitCode(outBuf.Bytes())
|
||||
outcome := TurnTaskOutcome{WallMs: wallMs}
|
||||
if haveExit && exitCode != 0 {
|
||||
outcome.VerifyErr = fmt.Sprintf("agent run_end exit code %d", exitCode)
|
||||
} else if !haveExit {
|
||||
detail := strings.TrimSpace(errBuf.String())
|
||||
// If the process never produced a run_end event, the actual failure
|
||||
// is in runErr (binary missing, exec permission denied, etc.) —
|
||||
// prefer it over the generic fallback so the real reason survives.
|
||||
if detail == "" && runErr != nil {
|
||||
detail = runErr.Error()
|
||||
}
|
||||
if detail == "" {
|
||||
detail = "missing terminal run_end event"
|
||||
}
|
||||
outcome.Err = fmt.Errorf("zero exec failed: %s", detail)
|
||||
return outcome
|
||||
}
|
||||
|
||||
// Parse the captured trace. The trace is the attribution layer, so a
|
||||
// missing/malformed file is not fatal — but it IS surfaced as a warning
|
||||
// (via outcome.TraceIssue) so an incomplete measurement can't look valid.
|
||||
if f, ferr := os.Open(tracePath); ferr != nil {
|
||||
outcome.TraceIssue = fmt.Sprintf("trace open failed: %v", ferr)
|
||||
} else {
|
||||
tr, perr := trace.ReadNDJSON(f)
|
||||
_ = f.Close()
|
||||
if perr != nil {
|
||||
outcome.TraceIssue = fmt.Sprintf("trace parse failed: %v", perr)
|
||||
} else {
|
||||
outcome.Trace = tr
|
||||
}
|
||||
}
|
||||
|
||||
// A nonzero agent exit already decided failure; don't run verification or
|
||||
// mark the task passed.
|
||||
if outcome.VerifyErr != "" {
|
||||
return outcome
|
||||
}
|
||||
if len(task.VerificationCommand) > 0 {
|
||||
if vOutcome := runVerification(ctx, task); !vOutcome.Passed {
|
||||
outcome.VerifyErr = strings.TrimSpace(vOutcome.Detail)
|
||||
return outcome
|
||||
}
|
||||
// A positive oracle (grep/test/build) passed: this is the only path
|
||||
// that sets Passed. A task with no verificationCommand is latency-only
|
||||
// (read-only nav/longproc/longctx/parallel): its exit 0 proves the turn
|
||||
// ran, not that the answer was right, so it never reports Passed and the
|
||||
// harness counts it in latencyOnlyTasks rather than any pass rate.
|
||||
outcome.Passed = true
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
}
|
||||
|
||||
// copyFixture copies the fixture directory at src into a fresh temp dir and
|
||||
// returns the copy's path. Used to give each benchmark invocation an isolated
|
||||
// workspace so mutating tasks (edit/fix/refactor) can't dirty the checked-in
|
||||
// fixture or a later iteration.
|
||||
func copyFixture(src string) (string, error) {
|
||||
info, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("fixture %q is not a directory", src)
|
||||
}
|
||||
dst, err := os.MkdirTemp("", "zero-turn-fixture-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
err = filepath.WalkDir(src, func(path string, d fs.DirEntry, werr error) error {
|
||||
if werr != nil {
|
||||
return werr
|
||||
}
|
||||
rel, rerr := filepath.Rel(src, path)
|
||||
if rerr != nil {
|
||||
return rerr
|
||||
}
|
||||
target := filepath.Join(dst, rel)
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(target, 0o755)
|
||||
}
|
||||
data, rerr := os.ReadFile(path)
|
||||
if rerr != nil {
|
||||
return rerr
|
||||
}
|
||||
return os.WriteFile(target, data, 0o644)
|
||||
})
|
||||
if err != nil {
|
||||
os.RemoveAll(dst)
|
||||
return "", err
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
func buildTurnExecArgs(task BenchTask, rc RunContext, tracePath string, extraArgs []string) []string {
|
||||
args := []string{"exec", "--output-format", "stream-json", "--trace", tracePath}
|
||||
if model := strings.TrimSpace(rc.Model); model != "" {
|
||||
args = append(args, "--model", model)
|
||||
}
|
||||
if mode := strings.TrimSpace(rc.Mode); mode != "" {
|
||||
args = append(args, "--mode", mode)
|
||||
}
|
||||
if rc.SelfCorrect {
|
||||
args = append(args, "--self-correct")
|
||||
}
|
||||
args = append(args, extraArgs...)
|
||||
args = append(args, task.Prompt)
|
||||
return args
|
||||
}
|
||||
|
||||
// ResolveBinary locates the zero binary for a benchmark run: an explicit path
|
||||
// when provided, else a `zero` (or zero.exe) on PATH, else a binary built into
|
||||
// the repo root. Returns an error when none is found.
|
||||
func ResolveBinary(explicit string) (string, error) {
|
||||
if v := strings.TrimSpace(explicit); v != "" {
|
||||
if _, err := os.Stat(v); err != nil {
|
||||
return "", fmt.Errorf("trace binary not found: %w", err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
if path, err := exec.LookPath("zero"); err == nil {
|
||||
return path, nil
|
||||
}
|
||||
if path, err := exec.LookPath("zero.exe"); err == nil {
|
||||
return path, nil
|
||||
}
|
||||
cwd, err := os.Getwd()
|
||||
if err == nil {
|
||||
for _, name := range []string{"zero", "zero.exe"} {
|
||||
candidate := filepath.Join(cwd, name)
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", errors.New("zero binary not found; build it first or pass an explicit path")
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
package perfbench
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
)
|
||||
|
||||
// fakeTurnRunner returns a canned *trace.TurnTrace per task so the harness's
|
||||
// aggregation logic can be exercised without a model or a binary. Each task
|
||||
// yields one generation span (the dominant cost), one tool_execution span, and
|
||||
// the counters the totals aggregate.
|
||||
func fakeTurnRunner(canned map[string]*trace.TurnTrace) TurnRunner {
|
||||
return func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome {
|
||||
tr, ok := canned[task.ID]
|
||||
if !ok {
|
||||
return TurnTaskOutcome{Err: errNoCanned}
|
||||
}
|
||||
wallMs := float64(tr.WallDuration().Microseconds()) / 1000
|
||||
return TurnTaskOutcome{Passed: true, WallMs: wallMs, Trace: tr}
|
||||
}
|
||||
}
|
||||
|
||||
var errNoCanned = &cannedError{}
|
||||
|
||||
type cannedError struct{}
|
||||
|
||||
func (*cannedError) Error() string { return "no canned trace for task" }
|
||||
|
||||
// cannedTrace builds a deterministic *trace.TurnTrace with the given spans and
|
||||
// counters. Spans are recorded as fixed durations (no real timing) so the
|
||||
// aggregation math is predictable in assertions.
|
||||
func cannedTrace(genMs, toolMs int, tokens int64) *trace.TurnTrace {
|
||||
r := trace.NewRecorder("sess", "run-1", "test")
|
||||
r.Start()
|
||||
r.RecordSpan(trace.SpanGeneration, time.Duration(genMs)*time.Millisecond)
|
||||
r.RecordSpan(trace.SpanToolExecution, time.Duration(toolMs)*time.Millisecond)
|
||||
r.Counter(trace.CounterInputTokens, tokens)
|
||||
r.Counter(trace.CounterOutputTokens, tokens/2)
|
||||
r.Counter(trace.CounterModelRequests, 1)
|
||||
r.Counter(trace.CounterToolCalls, 1)
|
||||
r.StampFirstToken()
|
||||
return r.Finish()
|
||||
}
|
||||
|
||||
func TestRunTurnBenchAggregation(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "fake-suite",
|
||||
Tasks: []BenchTask{
|
||||
{ID: "t1", Class: "nav", Prompt: "p1"}, // latency-only
|
||||
{ID: "t2", Class: "nav", Prompt: "p2"}, // latency-only
|
||||
{ID: "t3", Class: "edit", Prompt: "p3", VerificationCommand: []string{"true"}}, // correctness
|
||||
{ID: "t4", Class: "refactor", Prompt: "p4", VerificationCommand: []string{"true"}}, // build-only
|
||||
},
|
||||
BuildOnlyClasses: []string{"refactor"},
|
||||
}
|
||||
canned := map[string]*trace.TurnTrace{
|
||||
"t1": cannedTrace(100, 10, 1000),
|
||||
"t2": cannedTrace(300, 10, 1000),
|
||||
"t3": cannedTrace(200, 50, 2000),
|
||||
"t4": cannedTrace(150, 20, 1500),
|
||||
}
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Iterations: 1,
|
||||
Runner: fakeTurnRunner(canned),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
}
|
||||
result, err := RunTurnBench(context.Background(), set, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
// 4 tasks attempted; the tier split is 2 latency-only, 1 correctness, 1 build.
|
||||
if result.TasksAttempted != 4 {
|
||||
t.Fatalf("attempted=%d, want 4", result.TasksAttempted)
|
||||
}
|
||||
if result.TasksVerified != 1 || result.TasksPassed != 1 {
|
||||
t.Fatalf("correctness verified=%d passed=%d, want 1/1", result.TasksVerified, result.TasksPassed)
|
||||
}
|
||||
if result.LatencyOnlyTasks != 2 {
|
||||
t.Fatalf("latencyOnly=%d, want 2", result.LatencyOnlyTasks)
|
||||
}
|
||||
if result.BuildCheckedTasks != 1 || result.BuildPassedTasks != 1 {
|
||||
t.Fatalf("build checked=%d passed=%d, want 1/1", result.BuildCheckedTasks, result.BuildPassedTasks)
|
||||
}
|
||||
if result.CorrectnessPassRate != 1.0 {
|
||||
t.Fatalf("correctnessPassRate=%v, want 1.0", result.CorrectnessPassRate)
|
||||
}
|
||||
if result.BuildPassRate != 1.0 {
|
||||
t.Fatalf("buildPassRate=%v, want 1.0", result.BuildPassRate)
|
||||
}
|
||||
if len(result.CorrectnessClasses) != 1 || result.CorrectnessClasses[0] != "edit" {
|
||||
t.Fatalf("correctnessClasses=%v, want [edit]", result.CorrectnessClasses)
|
||||
}
|
||||
if len(result.BuildOnlyClasses) != 1 || result.BuildOnlyClasses[0] != "refactor" {
|
||||
t.Fatalf("buildOnlyClasses=%v, want [refactor]", result.BuildOnlyClasses)
|
||||
}
|
||||
if len(result.LatencyOnlyClasses) != 1 || result.LatencyOnlyClasses[0] != "nav" {
|
||||
t.Fatalf("latencyOnlyClasses=%v, want [nav]", result.LatencyOnlyClasses)
|
||||
}
|
||||
if result.SchemaVersion != TurnSchemaVersion {
|
||||
t.Fatalf("schemaVersion = %d, want %d", result.SchemaVersion, TurnSchemaVersion)
|
||||
}
|
||||
if result.Date != "2026-01-02T03:04:05Z" {
|
||||
t.Fatalf("date = %q", result.Date)
|
||||
}
|
||||
|
||||
// Per-span: generation appears in all four (100+300+200+150=750ms), tool in
|
||||
// all four (10+10+50+20=90ms). Count must equal the number of tasks * iterations.
|
||||
gen := result.PerSpan[trace.SpanGeneration]
|
||||
if gen.Count != 4 {
|
||||
t.Fatalf("generation count = %d, want 4", gen.Count)
|
||||
}
|
||||
if gen.TotalMs != 750 {
|
||||
t.Fatalf("generation totalMs = %v, want 750", gen.TotalMs)
|
||||
}
|
||||
tool := result.PerSpan[trace.SpanToolExecution]
|
||||
if tool.TotalMs != 90 {
|
||||
t.Fatalf("tool totalMs = %v, want 90", tool.TotalMs)
|
||||
}
|
||||
|
||||
// Top latency: generation (750) ranks above tool (90). Exactly two spans
|
||||
// here, so both appear and generation is first.
|
||||
if len(result.TopLatency) != 2 || result.TopLatency[0].Span != trace.SpanGeneration {
|
||||
t.Fatalf("topLatency = %+v", result.TopLatency)
|
||||
}
|
||||
if result.TopLatency[0].Share <= result.TopLatency[1].Share {
|
||||
t.Fatalf("top latency not ranked by share: %+v", result.TopLatency)
|
||||
}
|
||||
|
||||
// Totals: 4 model requests, 4 tool calls, input tokens 1000+1000+2000+1500=5500.
|
||||
if result.Totals.ModelRequests != 4 {
|
||||
t.Fatalf("modelRequests = %d, want 4", result.Totals.ModelRequests)
|
||||
}
|
||||
if result.Totals.ToolCalls != 4 {
|
||||
t.Fatalf("toolCalls = %d, want 4", result.Totals.ToolCalls)
|
||||
}
|
||||
if result.Totals.InputTokens != 5500 {
|
||||
t.Fatalf("inputTokens = %d, want 5500", result.Totals.InputTokens)
|
||||
}
|
||||
if result.Totals.OutputTokens != 2750 {
|
||||
t.Fatalf("outputTokens = %d, want 2750", result.Totals.OutputTokens)
|
||||
}
|
||||
|
||||
// Per-class tier roll-up: nav is latency-only (0 verified, 2 latency-only),
|
||||
// edit is correctness (1/1 verified passed), refactor is build (1/1 passed).
|
||||
nav := result.PerClass["nav"]
|
||||
if nav.Tasks != 2 || nav.Verified != 0 || nav.Passed != 0 || nav.LatencyOnly != 2 {
|
||||
t.Fatalf("nav class = %+v", nav)
|
||||
}
|
||||
edit := result.PerClass["edit"]
|
||||
if edit.Tasks != 1 || edit.Verified != 1 || edit.Passed != 1 || edit.LatencyOnly != 0 {
|
||||
t.Fatalf("edit class = %+v", edit)
|
||||
}
|
||||
refactor := result.PerClass["refactor"]
|
||||
if refactor.Tasks != 1 || refactor.Verified != 1 || refactor.Passed != 1 || refactor.LatencyOnly != 0 {
|
||||
t.Fatalf("refactor class = %+v", refactor)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunTurnBenchLatencyOnlyNeverPassed asserts the honesty gate: a task with
|
||||
// no verificationCommand reports Passed=true from the (stub) runner, yet the
|
||||
// harness counts it ONLY in latencyOnlyTasks — never in tasksPassed or any pass
|
||||
// rate — so an exit-0 read-only run cannot inflate a correctness number.
|
||||
func TestRunTurnBenchLatencyOnlyNeverPassed(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "lo-suite",
|
||||
Tasks: []BenchTask{
|
||||
{ID: "n1", Class: "nav", Prompt: "p"}, // no oracle — runner still says Passed=true
|
||||
},
|
||||
}
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Runner: fakeTurnRunner(map[string]*trace.TurnTrace{"n1": cannedTrace(50, 5, 100)}),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
}
|
||||
result, err := RunTurnBench(context.Background(), set, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
if result.TasksPassed != 0 || result.TasksVerified != 0 {
|
||||
t.Fatalf("latency-only leaked into pass: passed=%d verified=%d, want 0/0",
|
||||
result.TasksPassed, result.TasksVerified)
|
||||
}
|
||||
if result.CorrectnessPassRate != 0 || result.BuildPassRate != 0 {
|
||||
t.Fatalf("pass rates should be 0 with no oracle tasks: c=%v b=%v",
|
||||
result.CorrectnessPassRate, result.BuildPassRate)
|
||||
}
|
||||
if result.LatencyOnlyTasks != 1 {
|
||||
t.Fatalf("latencyOnlyTasks=%d, want 1", result.LatencyOnlyTasks)
|
||||
}
|
||||
if result.PerClass["nav"].Passed != 0 || result.PerClass["nav"].LatencyOnly != 1 {
|
||||
t.Fatalf("nav class = %+v, want passed=0 latencyOnly=1", result.PerClass["nav"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTurnBenchIterationsAggregates(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "iter-suite",
|
||||
Tasks: []BenchTask{{ID: "t1", Class: "nav", Prompt: "p1"}},
|
||||
}
|
||||
canned := map[string]*trace.TurnTrace{"t1": cannedTrace(100, 10, 500)}
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Iterations: 3,
|
||||
Runner: fakeTurnRunner(canned),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
}
|
||||
result, err := RunTurnBench(context.Background(), set, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
if result.PerSpan[trace.SpanGeneration].Count != 3 {
|
||||
t.Fatalf("generation count = %d, want 3 (one per iteration)", result.PerSpan[trace.SpanGeneration].Count)
|
||||
}
|
||||
if result.PerSpan[trace.SpanGeneration].TotalMs != 300 {
|
||||
t.Fatalf("generation totalMs = %v, want 300", result.PerSpan[trace.SpanGeneration].TotalMs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTurnBenchRequiresModelAndRunner(t *testing.T) {
|
||||
set := TaskSet{ID: "s", Tasks: []BenchTask{{ID: "t", Prompt: "p"}}}
|
||||
if _, err := RunTurnBench(context.Background(), set, TurnBenchConfig{Runner: fakeTurnRunner(nil)}); err == nil {
|
||||
t.Fatal("expected error for missing model")
|
||||
}
|
||||
if _, err := RunTurnBench(context.Background(), set, TurnBenchConfig{Model: "m"}); err == nil {
|
||||
t.Fatal("expected error for missing runner")
|
||||
}
|
||||
if _, err := RunTurnBench(context.Background(), TaskSet{ID: "empty"}, TurnBenchConfig{Model: "m", Runner: fakeTurnRunner(nil)}); err == nil {
|
||||
t.Fatal("expected error for empty task set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopLatencySourcesRanksByTotalAndCapsTopN(t *testing.T) {
|
||||
perSpan := map[string]SpanStats{
|
||||
"a": {TotalMs: 100},
|
||||
"b": {TotalMs: 500},
|
||||
"c": {TotalMs: 300},
|
||||
"d": {TotalMs: 50},
|
||||
}
|
||||
top := topLatencySources(perSpan, 3)
|
||||
if len(top) != 3 {
|
||||
t.Fatalf("len = %d, want 3", len(top))
|
||||
}
|
||||
wantOrder := []string{"b", "c", "a"}
|
||||
for i, w := range wantOrder {
|
||||
if top[i].Span != w {
|
||||
t.Fatalf("top[%d] = %q, want %q", i, top[i].Span, w)
|
||||
}
|
||||
}
|
||||
// Shares sum to 1 across all four (100+500+300+50=950); the top-3 retain
|
||||
// their global share (not renormalized to the top-3).
|
||||
// Share is rounded to 2 decimals by RoundMetric (500/950 -> 0.53), so compare
|
||||
// against the rounded value with a small tolerance.
|
||||
if got, want := top[0].Share, RoundMetric(500.0/950.0); !approxEqual(got, want, 0.001) {
|
||||
t.Fatalf("top[0] share = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteTurnBenchJSONRoundTrip(t *testing.T) {
|
||||
set := TaskSet{ID: "json-suite", Tasks: []BenchTask{
|
||||
{ID: "t1", Class: "edit", Prompt: "p1", VerificationCommand: []string{"true"}},
|
||||
}}
|
||||
canned := map[string]*trace.TurnTrace{"t1": cannedTrace(150, 20, 800)}
|
||||
result, err := RunTurnBench(context.Background(), set, TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Runner: fakeTurnRunner(canned),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := WriteTurnBenchJSON(&buf, result); err != nil {
|
||||
t.Fatalf("WriteTurnBenchJSON: %v", err)
|
||||
}
|
||||
var decoded TurnBenchResult
|
||||
if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v\n%s", err, buf.String())
|
||||
}
|
||||
if decoded.SchemaVersion != TurnSchemaVersion {
|
||||
t.Fatalf("schemaVersion = %d, want %d", decoded.SchemaVersion, TurnSchemaVersion)
|
||||
}
|
||||
if decoded.TasksVerified != 1 || decoded.TasksPassed != 1 || decoded.LatencyOnlyTasks != 0 {
|
||||
t.Fatalf("decoded tier counts = verified=%d passed=%d latency=%d, want 1/1/0",
|
||||
decoded.TasksVerified, decoded.TasksPassed, decoded.LatencyOnlyTasks)
|
||||
}
|
||||
if decoded.CorrectnessPassRate != 1.0 {
|
||||
t.Fatalf("decoded correctnessPassRate = %v, want 1.0", decoded.CorrectnessPassRate)
|
||||
}
|
||||
if decoded.PerSpan[trace.SpanGeneration].TotalMs != 150 {
|
||||
t.Fatalf("decoded generation totalMs = %v, want 150", decoded.PerSpan[trace.SpanGeneration].TotalMs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatTurnBenchSummaryNamesTopSources(t *testing.T) {
|
||||
set := TaskSet{ID: "fmt-suite", Tasks: []BenchTask{{ID: "t1", Class: "nav", Prompt: "p1"}}}
|
||||
canned := map[string]*trace.TurnTrace{"t1": cannedTrace(150, 20, 800)}
|
||||
result, err := RunTurnBench(context.Background(), set, TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Runner: fakeTurnRunner(canned),
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
summary := FormatTurnBenchSummary(result)
|
||||
if !strings.Contains(summary, "top latency sources") {
|
||||
t.Fatalf("summary missing top-latency header:\n%s", summary)
|
||||
}
|
||||
if !strings.Contains(summary, trace.SpanGeneration) {
|
||||
t.Fatalf("summary missing top span name %q:\n%s", trace.SpanGeneration, summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadBaselineManifest(t *testing.T) {
|
||||
path := filepath.Join("manifests", "baseline.json")
|
||||
set, err := LoadTaskSet(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadTaskSet: %v", err)
|
||||
}
|
||||
if set.ID == "" {
|
||||
t.Fatal("manifest has no id")
|
||||
}
|
||||
// The baseline must clear the "do not proceed until ≥30 tasks" gate.
|
||||
if len(set.Tasks) < 30 {
|
||||
t.Fatalf("baseline has %d tasks, want >= 30", len(set.Tasks))
|
||||
}
|
||||
// The six required classes must all be present and non-empty.
|
||||
wantClasses := map[string]bool{
|
||||
"nav": false, "edit": false, "fix": false,
|
||||
"refactor": false, "longproc": false, "longctx": false, "parallel": false,
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, task := range set.Tasks {
|
||||
class := strings.TrimSpace(task.Class)
|
||||
if class == "" {
|
||||
t.Fatalf("task %q has no class", task.ID)
|
||||
}
|
||||
if _, ok := wantClasses[class]; !ok {
|
||||
t.Fatalf("unexpected class %q on task %q", class, task.ID)
|
||||
}
|
||||
wantClasses[class] = true
|
||||
counts[class]++
|
||||
}
|
||||
for class, present := range wantClasses {
|
||||
if !present {
|
||||
t.Fatalf("manifest missing required class %q", class)
|
||||
}
|
||||
if counts[class] == 0 {
|
||||
t.Fatalf("class %q has zero tasks", class)
|
||||
}
|
||||
}
|
||||
// Every task must have a prompt and a workspace fixture pointing under testdata,
|
||||
// and that fixture must actually exist on disk — a manifest referencing a
|
||||
// missing fixture would make every task in its class error out at run time, so
|
||||
// catch it at load time instead.
|
||||
seen := map[string]bool{}
|
||||
for _, task := range set.Tasks {
|
||||
if strings.TrimSpace(task.Prompt) == "" {
|
||||
t.Fatalf("task %q has empty prompt", task.ID)
|
||||
}
|
||||
if strings.TrimSpace(task.WorkspaceFixture) == "" {
|
||||
t.Fatalf("task %q has no workspace fixture", task.ID)
|
||||
}
|
||||
if !strings.Contains(task.WorkspaceFixture, "testdata") {
|
||||
t.Fatalf("task %q fixture %q not under testdata", task.ID, task.WorkspaceFixture)
|
||||
}
|
||||
if seen[task.WorkspaceFixture] {
|
||||
continue
|
||||
}
|
||||
seen[task.WorkspaceFixture] = true
|
||||
info, err := os.Stat(task.WorkspaceFixture)
|
||||
if err != nil {
|
||||
t.Fatalf("task %q fixture %q does not exist: %v", task.ID, task.WorkspaceFixture, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
t.Fatalf("task %q fixture %q is not a directory", task.ID, task.WorkspaceFixture)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func approxEqual(a, b, tol float64) bool {
|
||||
d := a - b
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
return d < tol
|
||||
}
|
||||
|
||||
// TestCopyFixtureIsolatesSourceFromMutation asserts the property that lets a
|
||||
// mutating task run twice (or across iterations) without poisoning the next
|
||||
// sample: the runner operates on a per-invocation copy, so mutating the copy
|
||||
// leaves the checked-in source fixture byte-identical. This is verified by
|
||||
// reading the code today; a test makes it durable against future refactors of
|
||||
// the runner's isolation path.
|
||||
func TestCopyFixtureIsolatesSourceFromMutation(t *testing.T) {
|
||||
src, err := os.MkdirTemp("", "zero-fixture-src-*")
|
||||
if err != nil {
|
||||
t.Fatalf("mkdtemp src: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(src)
|
||||
orig := "package main\n\nfunc main() {}\n"
|
||||
if err := os.WriteFile(filepath.Join(src, "main.go"), []byte(orig), 0o644); err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(src, "sub"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir sub: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(src, "sub", "a.go"), []byte("package sub\n"), 0o644); err != nil {
|
||||
t.Fatalf("write sub: %v", err)
|
||||
}
|
||||
|
||||
// Snapshot the source tree before any mutation of the copy.
|
||||
wantMain, err := os.ReadFile(filepath.Join(src, "main.go"))
|
||||
if err != nil {
|
||||
t.Fatalf("read source main.go: %v", err)
|
||||
}
|
||||
|
||||
copyDir, err := copyFixture(src)
|
||||
if err != nil {
|
||||
t.Fatalf("copyFixture: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(copyDir)
|
||||
|
||||
// The copy must be a real copy, not a symlink/alias of the source, and
|
||||
// mutating it must not touch the source.
|
||||
if copyDir == src {
|
||||
t.Fatalf("copyFixture returned the source dir, not a copy: %s", copyDir)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(copyDir, "main.go"), []byte("package main\n\n// mutated\nfunc main() {}\n"), 0o644); err != nil {
|
||||
t.Fatalf("mutate copy: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(copyDir, "new.go"), []byte("package main\n"), 0o644); err != nil {
|
||||
t.Fatalf("add file to copy: %v", err)
|
||||
}
|
||||
|
||||
gotMain, err := os.ReadFile(filepath.Join(src, "main.go"))
|
||||
if err != nil {
|
||||
t.Fatalf("re-read source main.go: %v", err)
|
||||
}
|
||||
if string(gotMain) != string(wantMain) {
|
||||
t.Fatalf("source fixture mutated by copy: got %q, want %q", gotMain, wantMain)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(src, "new.go")); !os.IsNotExist(err) {
|
||||
t.Fatalf("file added to copy appeared in source: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -271,72 +271,55 @@ func skillSearchRoot(path string) string {
|
||||
// recognize a manifest path that points at the file rather than the directory.
|
||||
const skillFileName = "SKILL.md"
|
||||
|
||||
// MergedSkills loads the default skills directory plus the supplied plugin skill
|
||||
// roots and returns one merged, name-deduplicated list (Content stripped, like
|
||||
// skills.List) alongside the duplicate-name collisions across all roots. Earlier
|
||||
// roots win a name clash, matching skills.Load's first-wins rule; the default dir
|
||||
// is always considered first so a user skill shadows a same-named plugin skill.
|
||||
// A bad root simply yields no skills rather than failing the merge.
|
||||
// MergedSkills loads the default skills directory, the shared ~/.agents/skills
|
||||
// root when present, plus the supplied plugin skill roots and returns one
|
||||
// merged, name-deduplicated list (Content stripped, like skills.List) alongside
|
||||
// the duplicate-name collisions across all roots. Earlier roots win a name
|
||||
// clash, matching skills.Load's first-wins rule; the default dir is always
|
||||
// considered first so a user skill shadows agents and plugins. A bad root simply
|
||||
// yields no skills rather than failing the merge.
|
||||
func MergedSkills(defaultDir string, pluginRoots []string) ([]skills.Skill, []skills.DuplicateName) {
|
||||
return mergeSkills(defaultDir, pluginRoots, false)
|
||||
}
|
||||
|
||||
// mergeSkills merges the default skills dir and plugin roots into one sorted,
|
||||
// name-deduplicated list. keepContent retains each skill's body (skills.Load)
|
||||
// versus stripping it (skills.List). Roots are considered in order with the
|
||||
// default dir first, so an earlier root wins a name clash; collisions are recorded
|
||||
// rather than crashing, preserving the skills package's Duplicates behaviour.
|
||||
// mergeSkills merges the primary skills dir, optional ~/.agents/skills, and
|
||||
// plugin roots into one sorted, name-deduplicated list via skills.LoadFromRoots /
|
||||
// ListFromRoots. keepContent retains each skill's body versus stripping it.
|
||||
// Roots are considered in order with the default dir first, so an earlier root
|
||||
// wins a name clash; collisions are recorded rather than crashing.
|
||||
func mergeSkills(defaultDir string, pluginRoots []string, keepContent bool) ([]skills.Skill, []skills.DuplicateName) {
|
||||
merged := []skills.Skill{}
|
||||
dups := []skills.DuplicateName{}
|
||||
byName := map[string]skills.Skill{}
|
||||
|
||||
roots := append([]string{defaultDir}, pluginRoots...)
|
||||
for _, root := range roots {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
continue
|
||||
}
|
||||
var loaded []skills.Skill
|
||||
var err error
|
||||
if keepContent {
|
||||
loaded, err = skills.Load(root)
|
||||
} else {
|
||||
loaded, err = skills.List(root)
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, skill := range loaded {
|
||||
if winner, clash := byName[skill.Name]; clash {
|
||||
dups = append(dups, skills.DuplicateName{Name: skill.Name, Winner: winner.Path, Loser: skill.Path})
|
||||
continue
|
||||
}
|
||||
byName[skill.Name] = skill
|
||||
merged = append(merged, skill)
|
||||
// Keep defaultDir injectable for tests rather than always calling DefaultDir.
|
||||
roots := skills.GlobalRoots(defaultDir)
|
||||
// GlobalRoots already includes AgentsDir; append plugin roots only.
|
||||
for _, root := range pluginRoots {
|
||||
if root = strings.TrimSpace(root); root != "" {
|
||||
roots = append(roots, root)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(merged, func(left int, right int) bool {
|
||||
return merged[left].Name < merged[right].Name
|
||||
})
|
||||
if keepContent {
|
||||
merged, dups, _ := skills.LoadFromRoots(roots)
|
||||
return merged, dups
|
||||
}
|
||||
merged, dups, _ := skills.ListFromRoots(roots)
|
||||
return merged, dups
|
||||
}
|
||||
|
||||
// skillTool is a drop-in replacement for the core skill tool that resolves a
|
||||
// named skill across the default skills directory PLUS the plugin skill roots, so
|
||||
// plugin-declared skills surface in the agent's skill list. It keeps the core
|
||||
// tool's name, schema, and read-only/allow safety so registering it simply
|
||||
// overlays plugin skills onto the existing surface.
|
||||
// named skill across the default skills directory, the shared ~/.agents/skills
|
||||
// root, and plugin skill roots, so agents- and plugin-declared skills surface in
|
||||
// the agent's skill list. It keeps the core tool's name, schema, and
|
||||
// read-only/allow safety so registering it simply overlays multi-root discovery
|
||||
// onto the existing surface.
|
||||
type skillTool struct {
|
||||
defaultDir string
|
||||
pluginRoots []string
|
||||
}
|
||||
|
||||
// NewSkillTool builds the plugin-aware skill tool. defaultDir is the standard
|
||||
// NewSkillTool builds the multi-root skill tool. defaultDir is the standard
|
||||
// skills directory (skills.DefaultDir); pluginRoots are the plugin skill search
|
||||
// roots from an ActivationResult. The returned tool merges both, deterministically
|
||||
// deduplicating by name (default dir wins a clash) and listing all available
|
||||
// skills when an unknown name is requested.
|
||||
// roots from an ActivationResult. The returned tool merges primary + agents +
|
||||
// plugins, deterministically deduplicating by name (default dir wins a clash)
|
||||
// and listing all available skills when an unknown name is requested.
|
||||
func NewSkillTool(defaultDir string, pluginRoots []string) tools.Tool {
|
||||
return skillTool{defaultDir: defaultDir, pluginRoots: append([]string{}, pluginRoots...)}
|
||||
}
|
||||
@@ -345,7 +328,7 @@ func (tool skillTool) Name() string { return "skill" }
|
||||
|
||||
func (tool skillTool) Description() string {
|
||||
return "Load a named Zero skill and return its instructions as the tool output. " +
|
||||
"Skills are reusable, on-demand instruction sets (including any contributed by plugins). " +
|
||||
"Skills are reusable, on-demand instruction sets (including shared ~/.agents/skills and any contributed by plugins). " +
|
||||
"Call this when a relevant skill exists; an unknown name returns the list of available skills."
|
||||
}
|
||||
|
||||
|
||||
@@ -350,6 +350,7 @@ func TestActivateResolvesManifestRelativeSkillRoot(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNewSkillToolSurfacesPluginSkillInList(t *testing.T) {
|
||||
isolateAgentsHome(t)
|
||||
defaultDir := t.TempDir()
|
||||
writeTestSkill(t, defaultDir, "core-skill", "core body")
|
||||
pluginRoot := t.TempDir()
|
||||
@@ -384,6 +385,7 @@ func TestNewSkillToolSurfacesPluginSkillInList(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMergeSkillRootsListsPluginSkillInAgentList(t *testing.T) {
|
||||
isolateAgentsHome(t)
|
||||
defaultDir := t.TempDir()
|
||||
writeTestSkill(t, defaultDir, "core-skill", "core")
|
||||
|
||||
@@ -404,6 +406,9 @@ func TestMergeSkillRootsListsPluginSkillInAgentList(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMergedSkillsRecordsDuplicatesWithoutCrashing(t *testing.T) {
|
||||
// Isolate AgentsDir so a host ~/.agents/skills cannot add extra collisions.
|
||||
isolateAgentsHome(t)
|
||||
|
||||
defaultDir := t.TempDir()
|
||||
writeTestSkill(t, defaultDir, "shared", "default copy")
|
||||
|
||||
@@ -426,6 +431,90 @@ func TestMergedSkillsRecordsDuplicatesWithoutCrashing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// isolateAgentsHome points HOME/USERPROFILE at an empty temp home so AgentsDir
|
||||
// does not pick up the developer's real ~/.agents/skills during unit tests.
|
||||
func isolateAgentsHome(t *testing.T) {
|
||||
t.Helper()
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
}
|
||||
|
||||
func TestMergedSkillsIncludesAgentsWithoutPlugins(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeTestSkill(t, agents, "agents-skill", "from agents")
|
||||
|
||||
defaultDir := t.TempDir()
|
||||
listed, dups := MergedSkills(defaultDir, nil)
|
||||
if len(dups) != 0 {
|
||||
t.Fatalf("unexpected dups: %#v", dups)
|
||||
}
|
||||
if len(listed) != 1 || listed[0].Name != "agents-skill" {
|
||||
t.Fatalf("agents skill should surface with no plugins, got %#v", listed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedSkillsPriorityPrimaryAgentsPlugins(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
primary := t.TempDir()
|
||||
pluginRoot := t.TempDir()
|
||||
writeTestSkill(t, primary, "shared", "primary body")
|
||||
writeTestSkill(t, agents, "shared", "agents body")
|
||||
writeTestSkill(t, agents, "agents-plugin", "agents body")
|
||||
writeTestSkill(t, pluginRoot, "shared", "plugin body")
|
||||
writeTestSkill(t, pluginRoot, "agents-plugin", "plugin body")
|
||||
writeTestSkill(t, pluginRoot, "plugin-only", "plugin only")
|
||||
|
||||
listed, dups := MergedSkills(primary, []string{pluginRoot})
|
||||
byName := map[string]string{}
|
||||
for _, skill := range listed {
|
||||
// Content is stripped by MergedSkills; use path origin instead.
|
||||
byName[skill.Name] = skill.Path
|
||||
}
|
||||
if !strings.Contains(byName["shared"], primary) {
|
||||
t.Fatalf("primary should win shared, path=%q", byName["shared"])
|
||||
}
|
||||
if !strings.Contains(byName["agents-plugin"], agents) {
|
||||
t.Fatalf("agents should win over plugin for agents-plugin, path=%q", byName["agents-plugin"])
|
||||
}
|
||||
if !strings.Contains(byName["plugin-only"], pluginRoot) {
|
||||
t.Fatalf("plugin-only missing, path=%q", byName["plugin-only"])
|
||||
}
|
||||
if len(dups) < 2 {
|
||||
t.Fatalf("expected cross-layer dups, got %#v", dups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSkillToolLoadsAgentsSkillWithoutPluginRoots(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
agents := filepath.Join(home, ".agents", "skills")
|
||||
if err := os.MkdirAll(agents, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeTestSkill(t, agents, "agents-skill", "agents body")
|
||||
|
||||
tool := NewSkillTool(t.TempDir(), nil)
|
||||
got := tool.Run(context.Background(), map[string]any{"name": "agents-skill"})
|
||||
if got.Status != tools.StatusOK || !strings.Contains(got.Output, "agents body") {
|
||||
t.Fatalf("expected agents skill body with zero plugin roots, got %q (%s)", got.Status, got.Output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateSkipsMalformedPluginAndContinues(t *testing.T) {
|
||||
registry := tools.NewRegistry()
|
||||
good := toolPlugin(t.TempDir(), ToolExtension{Name: "good", Command: "good", Permission: PermissionPrompt})
|
||||
|
||||
@@ -34,6 +34,9 @@ const (
|
||||
|
||||
var ErrUnknownProvider = errors.New("unknown provider")
|
||||
|
||||
// AIMLAPIID is the provider catalog identifier for the aimlapi.com preset.
|
||||
const AIMLAPIID = "aimlapi"
|
||||
|
||||
type Descriptor struct {
|
||||
ID string
|
||||
Name string
|
||||
@@ -41,6 +44,7 @@ type Descriptor struct {
|
||||
DefaultBaseURL string
|
||||
DefaultModel string
|
||||
AuthEnvVars []string
|
||||
CustomHeaders map[string]string
|
||||
RequiresAuth bool
|
||||
UsesAmbientAuth bool
|
||||
Public bool
|
||||
@@ -67,9 +71,9 @@ type Descriptor struct {
|
||||
// headless / SSH use) in addition to the browser flow.
|
||||
OAuthDeviceFlow bool
|
||||
|
||||
// Recommended marks the provider that should be surfaced first and badged
|
||||
// (★ … (recommended)) in every catalog-ordered list and picker. At most one
|
||||
// descriptor should set this.
|
||||
// Recommended marks a provider surfaced at the top and badged in
|
||||
// catalog-ordered lists and pickers. The recommended descriptors are the first
|
||||
// entries in the catalog, in order.
|
||||
Recommended bool
|
||||
}
|
||||
|
||||
@@ -100,6 +104,11 @@ var descriptors = []Descriptor{
|
||||
// minimax, qwen, google, nvidia, tencent, z-ai). Flat /v1/chat/completions
|
||||
// with a Bearer ogw_live_… key; listed first and badged in every picker.
|
||||
recommended(openAICompat("gitlawb-opengateway", "GitLawb OpenGateway", "https://opengateway.gitlawb.com/v1", "mimo-v2.5-pro", []string{"GITLAWB_OPENGATEWAY_API_KEY"}, "gitlawb opengateway", "opengateway")),
|
||||
// aimlapi.com — OpenAI-compatible aggregating gateway, also badged as
|
||||
// recommended (second, right after the OpenGateway default). The built-in TUI
|
||||
// onboarding can register/top up an account and save the issued key; the partner
|
||||
// headers attribute usage for the rebate.
|
||||
recommended(aimlapi()),
|
||||
openAI("openai", "OpenAI", "https://api.openai.com/v1", "gpt-4.1", []string{"OPENAI_API_KEY"}),
|
||||
anthropic("anthropic", "Anthropic", "https://api.anthropic.com", "claude-sonnet-4.5", []string{"ANTHROPIC_API_KEY"}),
|
||||
google("google", "Google", "https://generativelanguage.googleapis.com", "gemini-2.5-pro", []string{"GEMINI_API_KEY", "GOOGLE_API_KEY"}, "gemini"),
|
||||
@@ -298,8 +307,21 @@ func google(id string, name string, baseURL string, model string, env []string,
|
||||
}
|
||||
}
|
||||
|
||||
// recommended marks a descriptor as the recommended default so list/picker
|
||||
// surfaces sort it first and render the ★ … (recommended) badge.
|
||||
func aimlapi() Descriptor {
|
||||
descriptor := openAICompat(AIMLAPIID, "aimlapi.com", "https://api.aimlapi.com/v1", "anthropic/claude-sonnet-5", []string{"AIMLAPI_API_KEY"}, "aimlapi", "aiml api", "ai/ml api", "ai ml api")
|
||||
descriptor.CustomHeaders = map[string]string{
|
||||
"X-AIMLAPI-Partner-ID": "part_62yQoGYDq4Yqnrj2R1iGrDNJ",
|
||||
"X-AIMLAPI-Integration-Repo": "Gitlawb/zero",
|
||||
"X-AIMLAPI-Integration-Version": "zero",
|
||||
}
|
||||
// Onboarding is key-based only: the TUI sub-flow takes an existing key or an
|
||||
// email top-up (internal/aimlapi Paths A/B). Not flagged
|
||||
// OAuth-capable, so it does not appear in the wizard's "Sign in with OAuth" list.
|
||||
return descriptor
|
||||
}
|
||||
|
||||
// recommended marks a descriptor as a recommended default so list/picker
|
||||
// surfaces sort it to the top and render a provider-specific badge.
|
||||
func recommended(descriptor Descriptor) Descriptor {
|
||||
descriptor.Recommended = true
|
||||
return descriptor
|
||||
@@ -380,5 +402,19 @@ func cloneDescriptor(descriptor Descriptor) Descriptor {
|
||||
descriptor.AuthEnvVars = append([]string{}, descriptor.AuthEnvVars...)
|
||||
descriptor.SupportedAPIFormats = append([]APIFormat{}, descriptor.SupportedAPIFormats...)
|
||||
descriptor.Aliases = append([]string{}, descriptor.Aliases...)
|
||||
if descriptor.CustomHeaders != nil {
|
||||
descriptor.CustomHeaders = copyStringMap(descriptor.CustomHeaders)
|
||||
}
|
||||
return descriptor
|
||||
}
|
||||
|
||||
func copyStringMap(values map[string]string) map[string]string {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
copied := make(map[string]string, len(values))
|
||||
for key, value := range values {
|
||||
copied[key] = value
|
||||
}
|
||||
return copied
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
var expectedCatalogIDs = []string{
|
||||
"gitlawb-opengateway",
|
||||
"aimlapi",
|
||||
"openai",
|
||||
"anthropic",
|
||||
"google",
|
||||
@@ -68,25 +69,31 @@ func TestAllHasStableUniqueIDs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecommendedProviderIsFirstAndUnique(t *testing.T) {
|
||||
func TestRecommendedProvidersAreTopOfCatalog(t *testing.T) {
|
||||
descriptors := All()
|
||||
if len(descriptors) == 0 {
|
||||
t.Fatal("All() returned no descriptors")
|
||||
// The recommended providers are badged and pinned to the top of the catalog,
|
||||
// in this order (OpenGateway remains the default; aimlapi.com is also badged).
|
||||
wantTop := []string{"gitlawb-opengateway", "aimlapi"}
|
||||
if len(descriptors) < len(wantTop) {
|
||||
t.Fatalf("All() returned %d descriptors, want at least %d", len(descriptors), len(wantTop))
|
||||
}
|
||||
if !descriptors[0].Recommended {
|
||||
t.Fatalf("All()[0] = %q, want it to be the recommended provider", descriptors[0].ID)
|
||||
}
|
||||
if descriptors[0].ID != "gitlawb-opengateway" {
|
||||
t.Fatalf("recommended provider = %q, want %q", descriptors[0].ID, "gitlawb-opengateway")
|
||||
for index, id := range wantTop {
|
||||
if descriptors[index].ID != id {
|
||||
t.Fatalf("descriptors[%d] = %q, want %q", index, descriptors[index].ID, id)
|
||||
}
|
||||
if !descriptors[index].Recommended {
|
||||
t.Fatalf("descriptors[%d] (%q) should be recommended", index, id)
|
||||
}
|
||||
}
|
||||
// Exactly those are recommended, and they are contiguous at the top.
|
||||
recommended := 0
|
||||
for _, descriptor := range descriptors {
|
||||
if descriptor.Recommended {
|
||||
recommended++
|
||||
}
|
||||
}
|
||||
if recommended != 1 {
|
||||
t.Fatalf("recommended descriptor count = %d, want exactly 1", recommended)
|
||||
if recommended != len(wantTop) {
|
||||
t.Fatalf("recommended descriptor count = %d, want %d", recommended, len(wantTop))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,6 +110,28 @@ func TestRecommendedProviderEndpoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAIMLAPIDescriptor(t *testing.T) {
|
||||
descriptor, err := Require("aimlapi")
|
||||
if err != nil {
|
||||
t.Fatalf("Require(aimlapi) error = %v", err)
|
||||
}
|
||||
if descriptor.Name != "aimlapi.com" {
|
||||
t.Fatalf("Name = %q, want aimlapi.com", descriptor.Name)
|
||||
}
|
||||
if descriptor.DefaultBaseURL != "https://api.aimlapi.com/v1" {
|
||||
t.Fatalf("DefaultBaseURL = %q, want aimlapi.com endpoint", descriptor.DefaultBaseURL)
|
||||
}
|
||||
if descriptor.DefaultModel != "anthropic/claude-sonnet-5" {
|
||||
t.Fatalf("DefaultModel = %q, want anthropic/claude-sonnet-5", descriptor.DefaultModel)
|
||||
}
|
||||
if descriptor.Transport != TransportOpenAICompatible {
|
||||
t.Fatalf("Transport = %q, want %q", descriptor.Transport, TransportOpenAICompatible)
|
||||
}
|
||||
if !reflect.DeepEqual(descriptor.AuthEnvVars, []string{"AIMLAPI_API_KEY"}) {
|
||||
t.Fatalf("AuthEnvVars = %#v, want AIMLAPI_API_KEY", descriptor.AuthEnvVars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongCatDescriptor(t *testing.T) {
|
||||
descriptor, err := Require("longcat")
|
||||
if err != nil {
|
||||
@@ -293,7 +322,7 @@ func TestListByTransportPreservesCatalogOrder(t *testing.T) {
|
||||
TransportBedrock: {"bedrock"},
|
||||
TransportVertex: {"vertex"},
|
||||
TransportAnthropicCompat: {"minimax", "minimaxi-cn", "opencode-go-anthropic-compatible", "custom-anthropic-compatible"},
|
||||
TransportOpenAICompat: {"gitlawb-opengateway", "ollama-cloud", "ollama", "lmstudio", "openrouter", "huggingface", "chatgpt", "groq", "deepseek", "together", "dashscope", "moonshot", "longcat", "nvidia-nim", "mistral", "github", "xai", "venice", "xiaomi-mimo", "bankr", "zai", "zai-cn", "kilocode", "opencode", "opencode-go", "atomic-chat", "chatgpt-proxy", "custom-openai-compatible"},
|
||||
TransportOpenAICompat: {"gitlawb-opengateway", "aimlapi", "ollama-cloud", "ollama", "lmstudio", "openrouter", "huggingface", "chatgpt", "groq", "deepseek", "together", "dashscope", "moonshot", "longcat", "nvidia-nim", "mistral", "github", "xai", "venice", "xiaomi-mimo", "bankr", "zai", "zai-cn", "kilocode", "opencode", "opencode-go", "atomic-chat", "chatgpt-proxy", "custom-openai-compatible"},
|
||||
}
|
||||
|
||||
for transport, wantIDs := range cases {
|
||||
|
||||
@@ -61,6 +61,17 @@ var zaiCuratedModels = []Model{
|
||||
}
|
||||
|
||||
var curatedModels = map[string][]Model{
|
||||
// aimlapi.com aggregates every upstream provider behind one OpenAI-compatible
|
||||
// endpoint, so the curated defaults are a cross-provider "best of" flagship
|
||||
// list. The live /v1/models discovery (when a key is present) replaces this;
|
||||
// these are the offline/pre-key fallback and the picker's default order.
|
||||
"aimlapi": {
|
||||
{ID: "anthropic/claude-sonnet-5", Description: "catalog default (Anthropic Claude Sonnet 5)"},
|
||||
{ID: "google/gemini-3.5-flash", Description: "Google Gemini Flash 3.5"},
|
||||
{ID: "openai/gpt-5.5-2026-04-23", Description: "OpenAI GPT-5.5"},
|
||||
{ID: "qwen/qwen-3.7-max", Description: "Qwen 3.7 Max"},
|
||||
{ID: "deepseek/deepseek-v4-pro", Description: "DeepSeek V4"},
|
||||
},
|
||||
"ollama-cloud": {
|
||||
{ID: "qwen3-coder:480b", Description: "catalog default"},
|
||||
{ID: "gpt-oss:120b", Description: "agentic coding model"},
|
||||
|
||||
@@ -33,6 +33,11 @@ func TestModelsAreProviderScoped(t *testing.T) {
|
||||
want: []string{"gpt-5.5", "gpt-5.4", "gpt-5.4-mini", "gpt-5.3-codex-spark"},
|
||||
notWant: []string{"gpt-5", "gpt-4.1", "openai/gpt-4.1"},
|
||||
},
|
||||
{
|
||||
provider: "aimlapi",
|
||||
want: []string{"anthropic/claude-sonnet-5", "google/gemini-3.5-flash", "openai/gpt-5.5-2026-04-23", "qwen/qwen-3.7-max", "deepseek/deepseek-v4-pro"},
|
||||
notWant: []string{"gpt-4o", "openai/gpt-4o", "anthropic/claude-opus-4.8", "x-ai/grok-4-5"},
|
||||
},
|
||||
{
|
||||
provider: "mistral",
|
||||
want: []string{"mistral-large-latest", "codestral-latest"},
|
||||
|
||||
@@ -195,7 +195,7 @@ func discoverOpenAIModels(ctx context.Context, profile config.ProviderProfile, o
|
||||
AuthHeader: profile.AuthHeader,
|
||||
AuthScheme: profile.AuthScheme,
|
||||
AuthHeaderValue: profile.AuthHeaderValue,
|
||||
CustomHeaders: profile.CustomHeaders,
|
||||
CustomHeaders: providerio.CopyHeaders(profile.CustomHeaders),
|
||||
}, nil)
|
||||
}
|
||||
|
||||
|
||||
@@ -52,6 +52,44 @@ func TestDiscoverOpenAICompatibleModelsFetchesModelsEndpoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverAIMLAPIModelsSendsAuthAndCustomHeadersWithoutAttribution(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
for header, want := range map[string]string{
|
||||
"Authorization": "Bearer test-key",
|
||||
"X-Trace": "test",
|
||||
} {
|
||||
if got := r.Header.Get(header); got != want {
|
||||
t.Errorf("%s = %q, want %q", header, got, want)
|
||||
}
|
||||
}
|
||||
// No first-party referral/attribution headers are injected for catalog
|
||||
// presets; aimlapi rides through CopyHeaders like every other provider.
|
||||
for _, header := range []string{
|
||||
"X-AIMLAPI-Partner-ID",
|
||||
"X-AIMLAPI-Integration-Repo",
|
||||
"X-AIMLAPI-Integration-Version",
|
||||
} {
|
||||
if got := r.Header.Get(header); got != "" {
|
||||
t.Errorf("%s = %q, want no attribution header", header, got)
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"openai/gpt-5-chat"}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := Discover(context.Background(), config.ProviderProfile{
|
||||
CatalogID: "aimlapi",
|
||||
ProviderKind: config.ProviderKindOpenAICompatible,
|
||||
BaseURL: server.URL + "/v1",
|
||||
APIKey: "test-key",
|
||||
CustomHeaders: map[string]string{"X-Trace": "test"},
|
||||
}, Options{HTTPClient: server.Client()})
|
||||
if err != nil {
|
||||
t.Fatalf("Discover returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverOpenAICompatibleModelsHonorsAuthHeaderValue(t *testing.T) {
|
||||
// A profile can authenticate via a raw auth-header value instead of APIKey;
|
||||
// discovery must send it rather than probe unauthenticated.
|
||||
|
||||
@@ -67,7 +67,7 @@ func New(profile config.ProviderProfile, options Options) (zeroruntime.Provider,
|
||||
AuthHeader: profile.AuthHeader,
|
||||
AuthScheme: profile.AuthScheme,
|
||||
AuthHeaderValue: profile.AuthHeaderValue,
|
||||
CustomHeaders: profile.CustomHeaders,
|
||||
CustomHeaders: providerio.CopyHeaders(profile.CustomHeaders),
|
||||
OAuthResolver: options.OAuthResolver,
|
||||
MaxTokens: resolved.maxOutputTokens,
|
||||
HTTPClient: options.HTTPClient,
|
||||
@@ -83,7 +83,7 @@ func New(profile config.ProviderProfile, options Options) (zeroruntime.Provider,
|
||||
AuthHeader: profile.AuthHeader,
|
||||
AuthScheme: profile.AuthScheme,
|
||||
AuthHeaderValue: profile.AuthHeaderValue,
|
||||
CustomHeaders: profile.CustomHeaders,
|
||||
CustomHeaders: providerio.CopyHeaders(profile.CustomHeaders),
|
||||
OAuthResolver: options.OAuthResolver,
|
||||
MaxTokens: resolved.maxOutputTokens,
|
||||
HTTPClient: options.HTTPClient,
|
||||
@@ -97,7 +97,7 @@ func New(profile config.ProviderProfile, options Options) (zeroruntime.Provider,
|
||||
AuthHeader: profile.AuthHeader,
|
||||
AuthScheme: profile.AuthScheme,
|
||||
AuthHeaderValue: profile.AuthHeaderValue,
|
||||
CustomHeaders: profile.CustomHeaders,
|
||||
CustomHeaders: providerio.CopyHeaders(profile.CustomHeaders),
|
||||
OAuthResolver: options.OAuthResolver,
|
||||
MaxTokens: resolved.maxOutputTokens,
|
||||
HTTPClient: options.HTTPClient,
|
||||
|
||||
@@ -220,6 +220,55 @@ func TestNewThreadsCustomProviderHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAIMLAPIProviderSendsEndpointAndAuthWithoutAttribution(t *testing.T) {
|
||||
transport := &captureTransport{responseBody: "data: [DONE]\n\n"}
|
||||
provider, err := New(config.ProviderProfile{
|
||||
Name: "aimlapi",
|
||||
CatalogID: "aimlapi",
|
||||
ProviderKind: config.ProviderKindOpenAICompatible,
|
||||
APIKey: "aimlapi-test-key",
|
||||
Model: "openai/gpt-5-chat",
|
||||
CustomHeaders: map[string]string{"X-Trace": "test"},
|
||||
}, Options{HTTPClient: &http.Client{Transport: transport}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
stream, err := provider.StreamCompletion(context.Background(), zeroruntime.CompletionRequest{
|
||||
Messages: []zeroruntime.Message{{Role: zeroruntime.MessageRoleUser, Content: "hello"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamCompletion() error = %v", err)
|
||||
}
|
||||
for range stream {
|
||||
}
|
||||
|
||||
if transport.request == nil {
|
||||
t.Fatal("HTTP client was not used")
|
||||
}
|
||||
if got := transport.request.URL.String(); got != "https://api.aimlapi.com/v1/chat/completions" {
|
||||
t.Fatalf("request URL = %q, want AI/ML API endpoint", got)
|
||||
}
|
||||
for header, want := range map[string]string{
|
||||
"Authorization": "Bearer aimlapi-test-key",
|
||||
"X-Trace": "test",
|
||||
} {
|
||||
if got := transport.request.Header.Get(header); got != want {
|
||||
t.Fatalf("%s = %q, want %q", header, got, want)
|
||||
}
|
||||
}
|
||||
// No first-party referral/attribution headers are injected for catalog
|
||||
// presets; aimlapi rides through CopyHeaders like every other provider.
|
||||
for _, header := range []string{
|
||||
"X-AIMLAPI-Partner-ID",
|
||||
"X-AIMLAPI-Integration-Repo",
|
||||
"X-AIMLAPI-Integration-Version",
|
||||
} {
|
||||
if got := transport.request.Header.Get(header); got != "" {
|
||||
t.Fatalf("%s = %q, want no attribution header", header, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSupportsOpenAIProviderKind(t *testing.T) {
|
||||
provider, err := New(config.ProviderProfile{
|
||||
Name: "openai",
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
)
|
||||
|
||||
// TokenResolver yields a fresh OAuth credential for one request, or ok=false to
|
||||
@@ -42,7 +44,12 @@ func SendWithAuthRetry(
|
||||
// request (leaking the path/body) before we return the error.
|
||||
headers := base
|
||||
if resolver != nil {
|
||||
// ProviderQueue captures pre-send auth-wait (OAuth token resolve). No
|
||||
// send-side semaphore exists today; a future request queue would also
|
||||
// accumulate here. nil recorder (untraced) is a no-op.
|
||||
queueSpan := trace.FromContext(ctx).Span(trace.SpanProviderQueue)
|
||||
header, value, ok, rerr := resolver(ctx, forceRefresh)
|
||||
queueSpan.End()
|
||||
if rerr != nil {
|
||||
return nil, rerr
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -371,9 +372,15 @@ func ScanSSEDataWithContext(
|
||||
// The provider asked to stop (e.g. it already emitted an error
|
||||
// for this payload). Abort the read and end like ScanSSEData:
|
||||
// return nil so callers fall through to their post-scan checks.
|
||||
// Do not stamp FirstToken — this payload was not accepted model
|
||||
// output, so counting it as first-token time would be misleading.
|
||||
cancel()
|
||||
return nil
|
||||
}
|
||||
// First accepted non-keepalive payload = first real model output. Stamp
|
||||
// once; later real payloads are no-ops. nil recorder (untraced run) is a
|
||||
// no-op.
|
||||
trace.FromContext(ctx).StampFirstToken()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
)
|
||||
|
||||
// Transient-failure retry, shared by every provider.
|
||||
@@ -67,7 +69,9 @@ func SendWithRetry(
|
||||
setHeader(request)
|
||||
}
|
||||
|
||||
connectSpan := trace.FromContext(ctx).Span(trace.SpanProviderConnect)
|
||||
response, err := client.Do(request)
|
||||
connectSpan.End()
|
||||
if err != nil {
|
||||
// A transport failure on a POST does NOT mean the server didn't receive
|
||||
// it — the request may have arrived and be generating a (billable,
|
||||
@@ -82,6 +86,9 @@ func SendWithRetry(
|
||||
}
|
||||
|
||||
if ShouldRetryStatus(response.StatusCode) && attempt < maxAttempts {
|
||||
if r := trace.FromContext(ctx); r != nil {
|
||||
r.Counter(trace.CounterRetryCount, 1)
|
||||
}
|
||||
wait := RetryAfter(response)
|
||||
_ = response.Body.Close()
|
||||
if Backoff(ctx, attempt, wait) {
|
||||
|
||||
+23
-18
@@ -14,18 +14,22 @@ type EngineOptions struct {
|
||||
Store *GrantStore
|
||||
Backend Backend
|
||||
Scope *Scope
|
||||
// SensitiveEnvKeys adds config-derived credential variable names to the
|
||||
// catalog and namespace secrets scrubbed from sandboxed commands.
|
||||
SensitiveEnvKeys []string
|
||||
}
|
||||
|
||||
type Engine struct {
|
||||
workspaceRoot string
|
||||
policy Policy
|
||||
store *GrantStore
|
||||
backend Backend
|
||||
scope *Scope
|
||||
sessionGrants *memoryGrantSet
|
||||
sessionProfiles *permissionProfileGrantSet
|
||||
turnProfiles *permissionProfileGrantSet
|
||||
commandPrefixes *commandPrefixGrantSet
|
||||
workspaceRoot string
|
||||
policy Policy
|
||||
store *GrantStore
|
||||
backend Backend
|
||||
scope *Scope
|
||||
sensitiveEnvKeys []string
|
||||
sessionGrants *memoryGrantSet
|
||||
sessionProfiles *permissionProfileGrantSet
|
||||
turnProfiles *permissionProfileGrantSet
|
||||
commandPrefixes *commandPrefixGrantSet
|
||||
}
|
||||
|
||||
func NewEngine(options EngineOptions) *Engine {
|
||||
@@ -49,15 +53,16 @@ func NewEngine(options EngineOptions) *Engine {
|
||||
scope = newScopeBestEffort(workspaceRoot)
|
||||
}
|
||||
return &Engine{
|
||||
workspaceRoot: workspaceRoot,
|
||||
policy: policy,
|
||||
store: options.Store,
|
||||
backend: options.Backend,
|
||||
scope: scope,
|
||||
sessionGrants: newMemoryGrantSet(),
|
||||
sessionProfiles: newPermissionProfileGrantSet(),
|
||||
turnProfiles: newPermissionProfileGrantSet(),
|
||||
commandPrefixes: newCommandPrefixGrantSet(),
|
||||
workspaceRoot: workspaceRoot,
|
||||
policy: policy,
|
||||
store: options.Store,
|
||||
backend: options.Backend,
|
||||
scope: scope,
|
||||
sensitiveEnvKeys: normalizeSensitiveEnvKeys(options.SensitiveEnvKeys),
|
||||
sessionGrants: newMemoryGrantSet(),
|
||||
sessionProfiles: newPermissionProfileGrantSet(),
|
||||
turnProfiles: newPermissionProfileGrantSet(),
|
||||
commandPrefixes: newCommandPrefixGrantSet(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,9 @@ type CommandSpec struct {
|
||||
Args []string
|
||||
Dir string
|
||||
Env []string
|
||||
// sensitiveEnvKeys is populated by Engine from config-derived credential
|
||||
// variable names and carried through SandboxManager to the platform plan.
|
||||
sensitiveEnvKeys []string
|
||||
}
|
||||
|
||||
type CommandPlan struct {
|
||||
@@ -122,6 +125,7 @@ func (engine *Engine) BuildCommandPlan(spec CommandSpec) (CommandPlan, error) {
|
||||
return CommandPlan{}, errors.New("sandbox command name is required")
|
||||
}
|
||||
spec.Dir = commandDir
|
||||
spec.sensitiveEnvKeys = engine.sensitiveEnvKeys
|
||||
|
||||
backend := engine.backend
|
||||
if backend.Name == "" {
|
||||
@@ -207,7 +211,7 @@ func linuxSandboxHelperCommandPlan(execRequest SandboxExecutionRequest, policy P
|
||||
if err != nil {
|
||||
return CommandPlan{}, err
|
||||
}
|
||||
env := sandboxEnvironmentForCommand(spec.Env, policy, BackendLinuxBwrap, "")
|
||||
env := sandboxEnvironmentForCommandWithSensitiveEnv(spec.Env, policy, BackendLinuxBwrap, "", spec.sensitiveEnvKeys)
|
||||
planDir := spec.Dir
|
||||
if helper.Dir != "" {
|
||||
planDir = helper.Dir
|
||||
@@ -253,10 +257,27 @@ func directCommandPlan(spec CommandSpec, backend Backend, policy Policy, workspa
|
||||
Name: spec.Name,
|
||||
Args: cloneStrings(spec.Args),
|
||||
Dir: spec.Dir,
|
||||
Env: cloneStrings(spec.Env),
|
||||
Env: directCommandEnv(spec),
|
||||
}
|
||||
}
|
||||
|
||||
// directCommandEnv scrubs sensitive credentials from the environment for a
|
||||
// direct (unwrapped) command plan: the platform sandbox backend is
|
||||
// unavailable, disabled, or not required, so this is the actual environment
|
||||
// the child process inherits. Without scrubbing here, config-derived
|
||||
// apiKeyEnv and dynamic OAuth client-secret variables would leak into
|
||||
// commands that fall back to this path (e.g. EnforcementDegraded).
|
||||
func directCommandEnv(spec CommandSpec) []string {
|
||||
env := cloneStrings(spec.Env)
|
||||
if spec.Env == nil {
|
||||
// Match the wrapped-plan behavior: an unset spec.Env means "inherit the
|
||||
// caller's environment," so scrub a snapshot of it rather than passing
|
||||
// nil through to exec.Cmd, which would skip scrubbing entirely.
|
||||
env = os.Environ()
|
||||
}
|
||||
return scrubSensitiveEnv(env, spec.sensitiveEnvKeys...)
|
||||
}
|
||||
|
||||
func (engine *Engine) resolveCommandDir(dir string, policy Policy) (string, string, error) {
|
||||
workspaceRoot := strings.TrimSpace(engine.workspaceRoot)
|
||||
if workspaceRoot == "" {
|
||||
@@ -318,7 +339,7 @@ func seatbeltCommandPlanWithProfile(spec CommandSpec, workspaceRoot string, prof
|
||||
if envBackend == "" {
|
||||
envBackend = BackendMacOSSeatbelt
|
||||
}
|
||||
env := sandboxEnvironmentForCommand(spec.Env, policy, envBackend, "")
|
||||
env := sandboxEnvironmentForCommandWithSensitiveEnv(spec.Env, policy, envBackend, "", spec.sensitiveEnvKeys)
|
||||
plan := CommandPlan{
|
||||
Backend: backend,
|
||||
TargetBackend: backend.TargetBackend(),
|
||||
@@ -377,6 +398,10 @@ func sandboxEnvironment(policy Policy, backend BackendName, workspaceRoot string
|
||||
}
|
||||
|
||||
func sandboxEnvironmentForCommand(specEnv []string, policy Policy, backend BackendName, workspaceRoot string) []string {
|
||||
return sandboxEnvironmentForCommandWithSensitiveEnv(specEnv, policy, backend, workspaceRoot, nil)
|
||||
}
|
||||
|
||||
func sandboxEnvironmentForCommandWithSensitiveEnv(specEnv []string, policy Policy, backend BackendName, workspaceRoot string, sensitiveEnvKeys []string) []string {
|
||||
env := cloneStrings(specEnv)
|
||||
if specEnv == nil {
|
||||
// Preserve the caller environment for sandboxed commands. The sandbox
|
||||
@@ -384,7 +409,7 @@ func sandboxEnvironmentForCommand(specEnv []string, policy Policy, backend Backe
|
||||
// command env values still replace inherited values below.
|
||||
env = os.Environ()
|
||||
}
|
||||
env = scrubSensitiveEnv(env)
|
||||
env = scrubSensitiveEnv(env, sensitiveEnvKeys...)
|
||||
pathValue := envListValue(env, "PATH", defaultPath())
|
||||
if runtime.GOOS == "darwin" {
|
||||
// Preserve standard user tool locations so a bare `python3`/`node`
|
||||
@@ -967,7 +992,7 @@ func regexpQuoteMeta(value string) string {
|
||||
return replacer.Replace(value)
|
||||
}
|
||||
|
||||
func scrubSensitiveEnv(env []string) []string {
|
||||
func scrubSensitiveEnv(env []string, additionalKeys ...string) []string {
|
||||
// Secrets not covered by the provider catalog: cloud/VCS credentials and
|
||||
// providers Zero talks to through generic OpenAI-compatible endpoints.
|
||||
sensitiveKeys := []string{
|
||||
@@ -997,6 +1022,7 @@ func scrubSensitiveEnv(env []string) []string {
|
||||
sensitiveKeys = append(sensitiveKeys, key)
|
||||
}
|
||||
}
|
||||
sensitiveKeys = append(sensitiveKeys, normalizeSensitiveEnvKeys(additionalKeys)...)
|
||||
|
||||
out := make([]string, 0, len(env))
|
||||
for _, kv := range env {
|
||||
@@ -1005,7 +1031,7 @@ func scrubSensitiveEnv(env []string) []string {
|
||||
out = append(out, kv)
|
||||
continue
|
||||
}
|
||||
drop := false
|
||||
drop := isDynamicSensitiveEnvKey(key)
|
||||
for _, sensitive := range sensitiveKeys {
|
||||
if strings.EqualFold(key, sensitive) {
|
||||
drop = true
|
||||
@@ -1018,3 +1044,37 @@ func scrubSensitiveEnv(env []string) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeSensitiveEnvKeys(keys []string) []string {
|
||||
out := make([]string, 0, len(keys))
|
||||
seen := make(map[string]struct{}, len(keys))
|
||||
for _, key := range keys {
|
||||
key = strings.TrimSpace(key)
|
||||
// A misconfigured value like "COMPANY_LLM_SECRET=..." would never
|
||||
// match a real env key during scrubbing; keep only the name part.
|
||||
if name, _, found := strings.Cut(key, "="); found {
|
||||
key = strings.TrimSpace(name)
|
||||
}
|
||||
folded := strings.ToUpper(key)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[folded]; ok {
|
||||
continue
|
||||
}
|
||||
seen[folded] = struct{}{}
|
||||
out = append(out, key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isDynamicSensitiveEnvKey(key string) bool {
|
||||
const (
|
||||
prefix = "ZERO_OAUTH_"
|
||||
suffix = "_CLIENT_SECRET"
|
||||
)
|
||||
key = strings.ToUpper(strings.TrimSpace(key))
|
||||
return strings.HasPrefix(key, prefix) &&
|
||||
strings.HasSuffix(key, suffix) &&
|
||||
len(key) > len(prefix)+len(suffix)
|
||||
}
|
||||
Loaded 100 of 164 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user