mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 09:49:34 -10:00
Compare commits
38
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ab26dbf129 | ||
|
|
6eb4b89f3f | ||
|
|
6ec77d62f5 | ||
|
|
25653f686c | ||
|
|
c7346fbcf0 | ||
|
|
d0c4e62cd4 | ||
|
|
302f58bb5f | ||
|
|
ba99fa8d48 | ||
|
|
1805a8ce09 | ||
|
|
8b248f4e11 | ||
|
|
4f1f94f26a | ||
|
|
6247ee8f9c | ||
|
|
87158a1c90 | ||
|
|
16550569c1 | ||
|
|
008bc9b3f3 | ||
|
|
fd69233e33 | ||
|
|
3f0503bc23 | ||
|
|
f951aee65d | ||
|
|
82ecac5281 | ||
|
|
387fe67ee7 | ||
|
|
d185f3ef0d | ||
|
|
8c6c49ad52 | ||
|
|
809489a6e3 | ||
|
|
56147a708e | ||
|
|
81aad58d97 | ||
|
|
95ccd5bc32 | ||
|
|
284a6c86cb | ||
|
|
8f3b8f0d13 | ||
|
|
0f69d995e9 | ||
|
|
87e7e69afd | ||
|
|
f4c998ac30 | ||
|
|
30bff28efc | ||
|
|
7b10aab74b | ||
|
|
3c81fea228 | ||
|
|
f6c060631e | ||
|
|
563a6dbe91 | ||
|
|
2312abe5dd | ||
|
|
afb5010a46 |
No files matched your search
@@ -100,9 +100,10 @@ jobs:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
# govulncheck and deadcode resolve a toolchain from their own modules, which
|
||||
# can drop below the version go.mod requires and then fail to load our
|
||||
# packages. Pin GOTOOLCHAIN to the go.mod toolchain so both run under it.
|
||||
# govulncheck, deadcode, and golangci-lint each resolve a toolchain from
|
||||
# their own modules, which can drop below the version go.mod requires and
|
||||
# then fail to load our packages. Pin GOTOOLCHAIN to the go.mod toolchain
|
||||
# so all three run under it.
|
||||
- name: Pin toolchain from go.mod
|
||||
run: |
|
||||
toolchain="$(awk '/^toolchain /{print $2}' go.mod)"
|
||||
@@ -120,3 +121,13 @@ jobs:
|
||||
- name: deadcode (advisory)
|
||||
continue-on-error: true
|
||||
run: go run golang.org/x/tools/cmd/[email protected] -test=false ./...
|
||||
|
||||
# Advisory: catches what deadcode's whole-program reachability analysis
|
||||
# doesn't, unused private functions/assignments reachable within a
|
||||
# package but never actually called, plus staticcheck-style correctness
|
||||
# and readability issues. Scoped to a few linters rather than the full
|
||||
# default battery, and non-blocking, while the existing findings across
|
||||
# the repo are cleaned up incrementally (see #527).
|
||||
- name: golangci-lint (advisory)
|
||||
continue-on-error: true
|
||||
run: go run github.com/golangci/golangci-lint/v2/cmd/[email protected] run --enable-only unused,ineffassign,staticcheck ./...
|
||||
+13
@@ -27,6 +27,10 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
.cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Go build/module caches (GOCACHE/GOMODCACHE pointed here for local runs)
|
||||
.gocache/
|
||||
.gomodcache/
|
||||
|
||||
# IntelliJ based IDEs
|
||||
.idea
|
||||
|
||||
@@ -37,8 +41,17 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
/zero
|
||||
/zero.exe
|
||||
/zero-windows-command-runner.exe
|
||||
/zero-windows-sandbox-setup.exe
|
||||
/zero-skills
|
||||
/.zero-binary-version
|
||||
/snake-game.html
|
||||
|
||||
# Superpowers skill artifacts (brainstorm mockups, specs, plans) — local only
|
||||
.superpowers/
|
||||
docs/superpowers/
|
||||
|
||||
# Termux/Android build artifacts
|
||||
zero-termux-arm64
|
||||
zero-linux-sandbox
|
||||
zero-seccomp
|
||||
@@ -1 +1 @@
|
||||
{".":"0.1.0"}
|
||||
{".":"0.2.0"}
|
||||
@@ -47,6 +47,12 @@ Tips:
|
||||
- In a monorepo, drop a narrower `AGENTS.md` in each sub-tree (e.g. `services/api/AGENTS.md`). Zero picks those up automatically when you launch from inside the sub-tree.
|
||||
- A YAML frontmatter block (`---\n...\n---`) at the top is preserved verbatim in the injected prompt but is not parsed for `globs:` or `alwaysApply:` scoping today — keep the body self-contained.
|
||||
|
||||
### Personal guidelines, across every project
|
||||
|
||||
For preferences that follow *you*, not a specific repo (tone, tooling habits, workflow), drop a `ZERO.md` in your user config directory: `~/.config/zero/ZERO.md` on Linux/macOS, `%AppData%\Roaming\zero\ZERO.md` on Windows — the same directory as `config.json` and your personal specialists. Same format and 8 KiB cap as the project files above, and the same case-insensitive basename match.
|
||||
|
||||
This file is injected as its own `## User guidelines` section, before the project's `AGENTS.md`/`ZERO.md`, and is labeled as personal preference in the prompt: project guidelines are the later, more specific instruction and take precedence over it when the two conflict.
|
||||
|
||||
## 2. Custom specialists
|
||||
|
||||
Specialists are Zero's sub-agents. Three scopes, in priority order:
|
||||
@@ -126,7 +132,7 @@ description: Run the project's benchmark suite and summarize the deltas.
|
||||
3. Report any regression > 5% with the function name and the previous value.
|
||||
```
|
||||
|
||||
Only `name` and `description` are recognized in the frontmatter today. The `name` defaults to the directory name. If two skills declare the same name, the one in the lexicographically-first directory wins — duplicates are dropped silently. Plugin-declared skills (section 6) are not yet merged into the loader, so a `skills:` entry inside a plugin's `plugin.json` is not visible to the `skill` tool today.
|
||||
Only `name` and `description` are recognized in the frontmatter today. The `name` defaults to the directory name. Within a single skills root, duplicate names are resolved by lexicographic directory order. During an agent run, Zero loads the default skills directory before plugin skill roots; earlier roots win name collisions silently. Plugin-declared skills (section 6) are merged into the active agent run at plugin activation time, so bundled skills appear in the available skills list and can be loaded with the `skill` tool.
|
||||
|
||||
The `skill` core tool lets the agent load any discovered skill by name.
|
||||
|
||||
@@ -276,7 +282,7 @@ A plugin is enabled by being present in the plugins directory and disabled by re
|
||||
|
||||
Plugin commands run with the plugin directory as their working directory. Use relative paths; the loader resolves them at activation time.
|
||||
|
||||
> **Roadmap.** An in-UI plugins manager (browse, install, enable / disable) is on the backlog. Today you use the `zero plugins` CLI subcommands above. Skills declared inside a plugin's `plugin.json` are not yet merged into the `skill` tool's discovery (see section 3).
|
||||
> **Roadmap.** An in-UI plugins manager (browse, install, enable / disable) is on the backlog. Today you use the `zero plugins` CLI subcommands above.
|
||||
|
||||
## 7. Configuration locations
|
||||
|
||||
@@ -307,6 +313,7 @@ A team that wants every contributor's Zero to behave the same way commits:
|
||||
Each contributor adds only:
|
||||
|
||||
- `~/.config/zero/config.json` — their personal API keys, theme, default mode.
|
||||
- `~/.config/zero/ZERO.md` — personal preferences that follow them across every project (see section 1).
|
||||
- `~/.local/share/zero/skills/` — personal skills they keep across projects.
|
||||
|
||||
That's it. Run `zero` from the repo root and the agent has the team's full instruction set, every contributor's personal setup, and nothing else.
|
||||
|
||||
@@ -6,6 +6,71 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
aims to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html) once the first release is
|
||||
tagged. Until then, source builds report the version `dev`.
|
||||
|
||||
## [0.2.0](https://github.com/Gitlawb/zero/compare/v0.1.0...v0.2.0) (2026-07-06)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add --auto flag for LLM-generated commit messages ([#423](https://github.com/Gitlawb/zero/issues/423)) ([b0abde7](https://github.com/Gitlawb/zero/commit/b0abde7d0697e808480cd59d69a6f4d0c6320475))
|
||||
* add zero changes push and pr subcommands, and extra repo-info metrics ([#391](https://github.com/Gitlawb/zero/issues/391)) ([2312abe](https://github.com/Gitlawb/zero/commit/2312abe5ddd95f4c6ef373cfb61cc03092f48cdd))
|
||||
* agent quality, caching, retry, and tooling upgrades ([#506](https://github.com/Gitlawb/zero/issues/506)) ([3c81fea](https://github.com/Gitlawb/zero/commit/3c81fea22873ee3df7fc97b10cb4f77792706c4b))
|
||||
* **agent:** curb over-engineering the solution in the editing discipline ([#517](https://github.com/Gitlawb/zero/issues/517)) ([f4c998a](https://github.com/Gitlawb/zero/commit/f4c998ac30c4f07ff313a2d706791e857293be49)), closes [#516](https://github.com/Gitlawb/zero/issues/516)
|
||||
* **agent:** inject per-user config.UserConfigDir()/zero/ZERO.md guidelines into system prompt ([#475](https://github.com/Gitlawb/zero/issues/475)) ([7b10aab](https://github.com/Gitlawb/zero/commit/7b10aab74bf14a01166b2cea22deab79bba9850b))
|
||||
* **openai:** forward prompt_cache_key for server-side prefix cache routing ([#515](https://github.com/Gitlawb/zero/issues/515)) ([87e7e69](https://github.com/Gitlawb/zero/commit/87e7e69afd18b5539579856f3a61c6a95bc445ae))
|
||||
* **providers:** add `zero providers models` to discover a provider's models ([#386](https://github.com/Gitlawb/zero/issues/386)) ([0bc8074](https://github.com/Gitlawb/zero/commit/0bc8074c97b0310e4a9d70c3f967003ee5e8a59f))
|
||||
* **providers:** add KiloCode and OpenCode provider support ([#388](https://github.com/Gitlawb/zero/issues/388)) ([b1ccb6d](https://github.com/Gitlawb/zero/commit/b1ccb6d9c1875377f5e5ea81a1304edd1e41ab4f))
|
||||
* **providers:** add Meituan LongCat catalog preset ([#424](https://github.com/Gitlawb/zero/issues/424)) ([b4275e3](https://github.com/Gitlawb/zero/commit/b4275e350472b2490212bf814709819d354c1216))
|
||||
* **providers:** split minimax zai into intl cn ([#398](https://github.com/Gitlawb/zero/issues/398)) ([aaad4d2](https://github.com/Gitlawb/zero/commit/aaad4d271270f41af837b6f3b60ae80beba0c645))
|
||||
* require manual approval before npm publish + drop release-as pin ([#369](https://github.com/Gitlawb/zero/issues/369)) ([bd89a1f](https://github.com/Gitlawb/zero/commit/bd89a1f451643c1b65ec803070abc7b116631ebe))
|
||||
* **sandbox:** unelevated Windows fallback tier instead of prompts-only degrade ([#427](https://github.com/Gitlawb/zero/issues/427)) ([b9ddd6f](https://github.com/Gitlawb/zero/commit/b9ddd6f42138312a1fee8d8bb67c46c8eb1dea2f))
|
||||
* support shift enter for composer newlines ([#462](https://github.com/Gitlawb/zero/issues/462)) ([daf65e0](https://github.com/Gitlawb/zero/commit/daf65e0af9a040314d4ab337b0ad59c55416b7bc))
|
||||
* **tui:** /loop — repeat a prompt or command on an interval or self-paced ([#502](https://github.com/Gitlawb/zero/issues/502)) ([387fe67](https://github.com/Gitlawb/zero/commit/387fe67ee7cd81317c9c969f5906a4437080fea3))
|
||||
* **tui:** add search/filter to provider picker in setup wizard ([#400](https://github.com/Gitlawb/zero/issues/400)) ([2fcea71](https://github.com/Gitlawb/zero/commit/2fcea71778d23e050c93409c471aef45b68c1621))
|
||||
* **update:** add zero upgrade command to apply self-updates ([#461](https://github.com/Gitlawb/zero/issues/461)) ([5f36349](https://github.com/Gitlawb/zero/commit/5f36349c1884e81fa9bc66bb5fe813b627e897b7))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **action:** keep provider key scoped to zero step ([#448](https://github.com/Gitlawb/zero/issues/448)) ([407a927](https://github.com/Gitlawb/zero/commit/407a92739ff508cba32d2c12b3f36f0efcdd54c3))
|
||||
* add android platform support for Termux npm install ([#455](https://github.com/Gitlawb/zero/issues/455)) ([9bd93c6](https://github.com/Gitlawb/zero/commit/9bd93c62f8d57fb74057284aa66a1b6e1429dcdd)), closes [#449](https://github.com/Gitlawb/zero/issues/449)
|
||||
* **agent:** reject a malformed additional_permissions payload before prompting ([#453](https://github.com/Gitlawb/zero/issues/453)) ([e4f760e](https://github.com/Gitlawb/zero/commit/e4f760ee8bd57299cd2fcb37e8e23130037c2607))
|
||||
* allow non-TLS connections to private-network provider endpoints ([#444](https://github.com/Gitlawb/zero/issues/444)) ([1d86384](https://github.com/Gitlawb/zero/commit/1d8638466ca31517eb9db2b9353d3dce1cbeeabc))
|
||||
* **auth:** route zero auth login chatgpt to the dedicated ChatGPT flow ([#443](https://github.com/Gitlawb/zero/issues/443)) ([305a62c](https://github.com/Gitlawb/zero/commit/305a62c954ca6cec00bc58d5398f933415156aff))
|
||||
* **config:** fall back to a usable saved provider instead of forcing full re-onboarding ([#410](https://github.com/Gitlawb/zero/issues/410)) ([c60ad87](https://github.com/Gitlawb/zero/commit/c60ad8729f79bb841114d352ee2d2fe29d5d0e41))
|
||||
* **config:** let a gateway ANTHROPIC_BASE_URL resolve as anthropic-compatible ([#497](https://github.com/Gitlawb/zero/issues/497)) ([30dd7c3](https://github.com/Gitlawb/zero/commit/30dd7c3112ad22d42fa12b5addd4e38f4beda42a)), closes [#479](https://github.com/Gitlawb/zero/issues/479)
|
||||
* **config:** unbrick first-run setup — default google/anthropic models, enter setup on fixable config errors ([#385](https://github.com/Gitlawb/zero/issues/385)) ([72eed06](https://github.com/Gitlawb/zero/commit/72eed06b4f94c43d75d31fe54a58d2f566de059e))
|
||||
* **config:** use ~/.config on macOS and enter setup when no provider ([#371](https://github.com/Gitlawb/zero/issues/371)) ([#372](https://github.com/Gitlawb/zero/issues/372)) ([027a8f2](https://github.com/Gitlawb/zero/commit/027a8f2768b17b89f5c8270887f156e2ccda69ea))
|
||||
* **docs:** rename AGENTS.MD > AGENTS.md ([#438](https://github.com/Gitlawb/zero/issues/438)) ([4266baf](https://github.com/Gitlawb/zero/commit/4266baf222df583ed2078b776687f12d496475b5))
|
||||
* **gemini:** strip unsupported JSON Schema fields from tool declarations ([#374](https://github.com/Gitlawb/zero/issues/374)) ([39e7100](https://github.com/Gitlawb/zero/commit/39e7100674150144a1152e3110c64c7cf0321d64)), closes [#373](https://github.com/Gitlawb/zero/issues/373)
|
||||
* **install:** persist install dir to user PATH on Windows ([#407](https://github.com/Gitlawb/zero/issues/407)) ([bdb1b0e](https://github.com/Gitlawb/zero/commit/bdb1b0ecd15859b1712a6037d296dace7f9c3c3f))
|
||||
* **mcp:** block cross-origin credential redirects ([#396](https://github.com/Gitlawb/zero/issues/396)) ([f915f70](https://github.com/Gitlawb/zero/commit/f915f70e5a3096e2419fa8d961a0f84a626fa4a9))
|
||||
* **oauth:** treat Windows ERROR_ACCESS_DENIED as lock contention in createSecretFile ([#445](https://github.com/Gitlawb/zero/issues/445)) ([d05e914](https://github.com/Gitlawb/zero/commit/d05e9148a7f79f67d1d3c31fca2775f21fbd331e))
|
||||
* **openai:** handle Ollama reasoning stream deltas ([#486](https://github.com/Gitlawb/zero/issues/486)) ([f6c0606](https://github.com/Gitlawb/zero/commit/f6c060631e18e082dda24cc4dc0903c31c2120d6))
|
||||
* preserve conversation context in exec prompts ([#460](https://github.com/Gitlawb/zero/issues/460)) ([949ee43](https://github.com/Gitlawb/zero/commit/949ee43f71e5cb7fab4695c5cb7b442fe4ecfbf7))
|
||||
* **provider-wizard:** allow multiple custom OpenAI-compatible providers ([#403](https://github.com/Gitlawb/zero/issues/403)) ([3fbbd28](https://github.com/Gitlawb/zero/commit/3fbbd28e4c586822cc4312c86232d94befe56e87))
|
||||
* **sandbox:** fix nested pipe creation under the Windows restricted token ([#456](https://github.com/Gitlawb/zero/issues/456)) ([563a6db](https://github.com/Gitlawb/zero/commit/563a6dbe91e65d5daeefd7626e8a77e30a6d8fb2))
|
||||
* **sandbox:** gate /tmp test assertions on GOOS, not path existence ([#426](https://github.com/Gitlawb/zero/issues/426)) ([f653dca](https://github.com/Gitlawb/zero/commit/f653dcac363fb69ad7be5b35e6e0fa6d2bce476d))
|
||||
* **sandbox:** self-heal a corrupt unelevated setup marker ([#437](https://github.com/Gitlawb/zero/issues/437)) ([8d0c5fe](https://github.com/Gitlawb/zero/commit/8d0c5feccb8bdbfb015df0508aa6e3bcbd1fd0e8))
|
||||
* **specialist:** cap max specialist nesting depth ([#491](https://github.com/Gitlawb/zero/issues/491)) ([177442c](https://github.com/Gitlawb/zero/commit/177442cfe4015bd8df04cc9894f98b468ee796d4))
|
||||
* Termux/Android support — PRoot scroll, SIGSYS sandbox, build docs ([#509](https://github.com/Gitlawb/zero/issues/509)) ([0f69d99](https://github.com/Gitlawb/zero/commit/0f69d995e9b586b774f66c066b21abab5e03024a))
|
||||
* **tools:** Block MSYS coreutils under Windows sandbox ([#476](https://github.com/Gitlawb/zero/issues/476)) ([81aad58](https://github.com/Gitlawb/zero/commit/81aad58d97839e51c068e2f08907618991fdc3fb))
|
||||
* **tools:** CRLF line ending mismatch in edit_file tool on Windows ([#378](https://github.com/Gitlawb/zero/issues/378)) ([33dc7ae](https://github.com/Gitlawb/zero/commit/33dc7ae2cc82c5389675531e1416856dae7151ce))
|
||||
* **tools:** fix cmd.exe /S/C corrupting commands with embedded quotes ([#465](https://github.com/Gitlawb/zero/issues/465)) ([190241b](https://github.com/Gitlawb/zero/commit/190241bd593f43211b766e0b13c8e89802d4bb37))
|
||||
* **tools:** flag piped POSIX utilities before running on Windows ([#412](https://github.com/Gitlawb/zero/issues/412)) ([5658a36](https://github.com/Gitlawb/zero/commit/5658a366274fc59a9d5336b06a21019c9c25cbf1))
|
||||
* **tools:** make grep and glob respect run cancellation ([#464](https://github.com/Gitlawb/zero/issues/464)) ([ba6c026](https://github.com/Gitlawb/zero/commit/ba6c0264697b7d7ed479f6e782fba9700a481e3d))
|
||||
* **tools:** require permission before web_search requests ([#382](https://github.com/Gitlawb/zero/issues/382)) ([960db96](https://github.com/Gitlawb/zero/commit/960db9660e4e31dc588fe8f7d6f116ff5e225566))
|
||||
* **tui:** compose help overlay through the viewport overlay pipeline ([#421](https://github.com/Gitlawb/zero/issues/421)) ([5b2b4de](https://github.com/Gitlawb/zero/commit/5b2b4dea1aaf9e0f68baa25e97e83296fb17b1a2))
|
||||
* **tui:** keep the profile name on /model switch so the stored key resolves ([#441](https://github.com/Gitlawb/zero/issues/441)) ([9134148](https://github.com/Gitlawb/zero/commit/9134148f4df3e4e556fba6c2f8babfdf6fcfeee1)), closes [#440](https://github.com/Gitlawb/zero/issues/440)
|
||||
* **tui:** resolve every permission request so the agent can't deadlock ([#397](https://github.com/Gitlawb/zero/issues/397)) ([952788f](https://github.com/Gitlawb/zero/commit/952788f72d32957659fe004521fcc8372b9ba9b4))
|
||||
* **tui:** show an M suffix for million-scale token counts ([#457](https://github.com/Gitlawb/zero/issues/457)) ([0562e3b](https://github.com/Gitlawb/zero/commit/0562e3bef7df2328610a48a1e81632a8da4aec64))
|
||||
* **tui:** title /model rows by model name, not the catalog description ([#395](https://github.com/Gitlawb/zero/issues/395)) ([cdf9d83](https://github.com/Gitlawb/zero/commit/cdf9d839ae57a729f292f36f7c5b0c67b41b288d))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* cache TUI model registry ([#496](https://github.com/Gitlawb/zero/issues/496)) ([e7d88b4](https://github.com/Gitlawb/zero/commit/e7d88b4b518049733da25a8447c00144bd1da518))
|
||||
* universal tool-output ceiling with spill + async post-edit diagnostics ([#518](https://github.com/Gitlawb/zero/issues/518)) ([95ccd5b](https://github.com/Gitlawb/zero/commit/95ccd5bc327f6fb464ff0239f7229de789f578dc))
|
||||
|
||||
## 0.1.0 (2026-07-02)
|
||||
|
||||
|
||||
|
||||
@@ -186,6 +186,7 @@ Common slash commands:
|
||||
| `/spec`, `/plan` | draft and review a plan before building |
|
||||
| `/image` | attach an image for vision-capable models |
|
||||
| `/resume`, `/rewind` | continue or roll back local sessions |
|
||||
| `/loop` | repeat a prompt or custom `/command` on an interval (`/loop 5m /babysit-prs`) or self-paced |
|
||||
| `/compact`, `/context` | manage context usage |
|
||||
| `/permissions`, `/tools` | inspect available tools and policy |
|
||||
| `/add-dir` | allow an extra write directory for this session |
|
||||
@@ -283,6 +284,39 @@ zero cron scheduled agent jobs
|
||||
zero update check for newer releases
|
||||
```
|
||||
|
||||
## Extending Zero
|
||||
|
||||
### Project and personal instructions
|
||||
|
||||
Zero appends project-specific guidance to the system prompt from the first
|
||||
`AGENTS.md`, `ZERO.md`, or `.zero/AGENTS.md` file found in each directory from
|
||||
the git root down to your current working directory (checked in that order
|
||||
per directory). Files are injected general-to-specific, capped at 8 KiB per
|
||||
file and 32 KiB total.
|
||||
|
||||
A personal `ZERO.md` under `config.UserConfigDir()/zero/ZERO.md`
|
||||
(`$XDG_CONFIG_HOME/zero/ZERO.md` or `~/.config/zero/ZERO.md` on Linux/macOS,
|
||||
`%AppData%\Roaming\zero\ZERO.md` on Windows) applies across every workspace, ahead of any project guidelines.
|
||||
|
||||
### Plugins
|
||||
|
||||
Plugins are discovered from `~/.config/zero/plugins/<name>/plugin.json` (user
|
||||
scope — `$XDG_CONFIG_HOME` or `~/.config` on every OS, independent of the
|
||||
`config.UserConfigDir()` path used above) and `<cwd>/.zero/plugins/<name>/plugin.json`
|
||||
(project scope — resolved from the current working directory, not the repo
|
||||
root), and managed with `zero plugins`. A manifest can declare:
|
||||
|
||||
- `tools` — custom tools (`command`, `args`, `inputSchema`, and a
|
||||
`permission` of `prompt` or `deny`; `allow` is honored only when manifest tool
|
||||
auto-approval is enabled)
|
||||
- `hooks` — commands run on `beforeTool`, `afterTool`, `sessionStart`, or
|
||||
`sessionEnd`
|
||||
- `prompts` and `skills` — additional prompt/skill files
|
||||
|
||||
MCP servers (`zero mcp`) and standalone markdown skills (`zero skills`) use
|
||||
the same extension points and can also be wired up outside of a plugin
|
||||
manifest.
|
||||
|
||||
## Appearance And Accessibility
|
||||
|
||||
| Control | Effect |
|
||||
|
||||
+4
-4
@@ -180,6 +180,7 @@ runs:
|
||||
ZERO_INPUT_ADD_DIR: ${{ inputs.add-dir }}
|
||||
ZERO_INPUT_WORKTREE: ${{ inputs.worktree }}
|
||||
ZERO_INPUT_OUTPUT_FORMAT: ${{ inputs.output-format }}
|
||||
ACTION_PATH: ${{ github.action_path }}
|
||||
RUNNER_TEMP: ${{ runner.temp }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -247,10 +248,9 @@ runs:
|
||||
echo "exit-code=${code}" >> "${GITHUB_OUTPUT}"
|
||||
echo "output-file=${output_file}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Best-effort one-line summary: last non-empty line of the captured output.
|
||||
summary="$(grep -v '^[[:space:]]*$' "${output_file}" | tail -n 1 || true)"
|
||||
# Keep the summary single-line and bounded for the step output.
|
||||
summary="$(printf '%s' "${summary}" | cut -c1-280)"
|
||||
# Best-effort one-line summary. Structured output ends with lifecycle
|
||||
# metadata, so parse final/error events instead of tailing stdout.
|
||||
summary="$(node "${ACTION_PATH}/scripts/action-summary.mjs" "${ZERO_INPUT_OUTPUT_FORMAT}" "${output_file}" || true)"
|
||||
{
|
||||
echo "summary<<ZERO_EOF"
|
||||
printf '%s\n' "${summary}"
|
||||
|
||||
@@ -173,6 +173,61 @@ Linux native sandboxing also requires Bubblewrap to be installed.
|
||||
|
||||
macOS uses the system sandbox and does not need an extra helper binary.
|
||||
|
||||
### Termux (Android)
|
||||
|
||||
Zero can run natively on Android via [Termux](https://termux.dev/). Build with
|
||||
`GOOS=android` to avoid the `faccessat2` syscall that is blocked by Samsung's
|
||||
seccomp filter on Android:
|
||||
|
||||
```bash
|
||||
# Install Go in Termux
|
||||
pkg install golang
|
||||
|
||||
# Build Zero for Android
|
||||
git clone https://github.com/Gitlawb/zero.git
|
||||
cd zero
|
||||
CGO_ENABLED=0 GOOS=android GOARCH=arm64 go build -ldflags="-s -w" -o zero ./cmd/zero
|
||||
|
||||
# Move into PATH
|
||||
mv zero ~/.local/bin/
|
||||
```
|
||||
|
||||
> **Why `GOOS=android`?** Go 1.26+ detects `runtime.GOOS == "android"` and skips
|
||||
> the `faccessat2` syscall inside `os/exec.findExecutable`, falling back to
|
||||
> permission-bit checks. Without this flag, Android's seccomp sends SIGSYS and
|
||||
> kills the process whenever Zero looks up a binary on `PATH` (git, sh, etc.).
|
||||
|
||||
**DNS.** Android does not expose `/etc/resolv.conf`. Go's pure-Go DNS resolver
|
||||
needs one. Use `proot` to bind-mount Termux's resolver config:
|
||||
|
||||
```bash
|
||||
pkg install proot
|
||||
proot -b "$PREFIX/etc/resolv.conf:/etc/resolv.conf" zero
|
||||
```
|
||||
|
||||
Create a wrapper at `~/.local/bin/zero` to avoid typing proot every time:
|
||||
|
||||
```bash
|
||||
#!/data/data/com.termux/files/usr/bin/bash
|
||||
exec proot -b "$PREFIX/etc/resolv.conf:/etc/resolv.conf" ~/.local/bin/zero.bin "$@"
|
||||
```
|
||||
|
||||
**Scroll.** On native Termux (not under PRoot), mouse scrolling works out of the
|
||||
box. The TUI uses Bubble Tea's `AllMotion` mouse mode by default. If you run Zero
|
||||
inside PRoot (e.g. through proot-distro), the scroll fix activates `CellMotion`
|
||||
to avoid PRoot's ptrace interference with the 1003 escape sequence.
|
||||
|
||||
**Providers.** Zero works with any OpenAI-compatible provider on Termux. For
|
||||
example, to use OpenCode Zen's free tier:
|
||||
|
||||
```bash
|
||||
zero providers add opencode \
|
||||
--name opencode \
|
||||
--model deepseek-v4-flash-free \
|
||||
--base-url https://opencode.ai/zen/v1 \
|
||||
--set-active
|
||||
```
|
||||
|
||||
Windows source builds can use the main `zero.exe` as the command runner and setup
|
||||
helper through Zero's built-in self-dispatch path. If you want a release-style
|
||||
layout anyway, build the standalone helper executables next to `zero.exe`:
|
||||
|
||||
@@ -41,6 +41,7 @@ events when present.
|
||||
|
||||
```json
|
||||
{ "schemaVersion": 2, "type": "run_start", "runId": "run_20260603_abc123", "sessionId": "zero_20260603100000_abc123", "cwd": "/repo", "provider": "openai", "model": "gpt-4.1", "apiModel": "gpt-4.1" }
|
||||
{ "schemaVersion": 2, "type": "reasoning", "runId": "run_20260603_abc123", "delta": "Thinking..." }
|
||||
{ "schemaVersion": 2, "type": "text", "runId": "run_20260603_abc123", "delta": "..." }
|
||||
{ "schemaVersion": 2, "type": "tool_call", "runId": "run_20260603_abc123", "id": "call_1", "name": "read_file", "args": { "path": "README.md" }, "sideEffect": "read" }
|
||||
{ "schemaVersion": 2, "type": "permission_request", "runId": "run_20260603_abc123", "id": "call_2", "name": "write_file", "action": "prompt", "permission": "prompt", "permissionMode": "ask", "sideEffect": "write", "reason": "Creates or overwrites files." }
|
||||
@@ -51,6 +52,10 @@ events when present.
|
||||
{ "schemaVersion": 2, "type": "run_end", "runId": "run_20260603_abc123", "status": "success", "exitCode": 0 }
|
||||
```
|
||||
|
||||
`reasoning` events carry live model reasoning/status deltas for providers that
|
||||
stream them separately from answer text. They are liveness/progress events only:
|
||||
they are not folded into `text` or the final answer.
|
||||
|
||||
Permission events may include structured sandbox metadata:
|
||||
|
||||
```json
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
module github.com/Gitlawb/zero
|
||||
|
||||
go 1.25.0
|
||||
|
||||
toolchain go1.26.4
|
||||
go 1.26.4
|
||||
|
||||
require (
|
||||
charm.land/bubbles/v2 v2.1.0
|
||||
@@ -14,6 +12,7 @@ require (
|
||||
github.com/charmbracelet/colorprofile v0.4.3
|
||||
github.com/charmbracelet/x/ansi v0.11.7
|
||||
github.com/charmbracelet/x/term v0.2.2
|
||||
github.com/coder/websocket v1.8.15
|
||||
github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
|
||||
golang.org/x/sys v0.46.0
|
||||
mvdan.cc/sh/v3 v3.13.1
|
||||
|
||||
@@ -32,6 +32,8 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE
|
||||
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
|
||||
|
||||
+54
-21
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -159,8 +160,14 @@ func (a *Agent) handleSessionLoad(_ context.Context, params json.RawMessage) (an
|
||||
// Load history BEFORE publishing the session so no concurrent prompt observes
|
||||
// a half-initialized session (registerSession sets history under the lock and
|
||||
// reuses an already-live session rather than orphaning its in-flight turn).
|
||||
history := a.loadHistory(meta.SessionID)
|
||||
history, historyErr := a.loadHistory(meta.SessionID)
|
||||
sess := a.registerSession(meta.SessionID, root, history)
|
||||
a.warnPersistence(
|
||||
¬ifier{conn: a.conn, sessionID: sess.id},
|
||||
"load session history",
|
||||
"Could not load session history. The session is open, but earlier turns may be missing until storage recovers.",
|
||||
historyErr,
|
||||
)
|
||||
return LoadSessionResult{
|
||||
Modes: a.modeState(sess),
|
||||
}, nil
|
||||
@@ -253,7 +260,14 @@ func (a *Agent) runTurn(ctx context.Context, sess *acpSession, userText string,
|
||||
if stopErr != nil {
|
||||
return "", RPCError(codeInternalError, stopErr.Error())
|
||||
}
|
||||
a.persistTurn(sess, userText, result.FinalAnswer)
|
||||
if err := a.persistTurn(sess, userText, result.FinalAnswer); err != nil {
|
||||
a.warnPersistence(
|
||||
note,
|
||||
"save session history",
|
||||
"Could not save session history. This turn is available in memory, but future resume may miss it until storage recovers.",
|
||||
err,
|
||||
)
|
||||
}
|
||||
return reason, nil
|
||||
}
|
||||
|
||||
@@ -386,29 +400,34 @@ func (a *Agent) modeState(s *acpSession) *SessionModeState {
|
||||
|
||||
// ---- persistence + continuity ----
|
||||
|
||||
func (a *Agent) persistTurn(sess *acpSession, user, assistant string) {
|
||||
if a.deps.Store != nil {
|
||||
_, _ = a.deps.Store.AppendEvent(sess.id, sessions.AppendEventInput{
|
||||
Type: sessions.EventMessage,
|
||||
Payload: map[string]any{"role": "user", "content": user},
|
||||
})
|
||||
if assistant != "" {
|
||||
_, _ = a.deps.Store.AppendEvent(sess.id, sessions.AppendEventInput{
|
||||
Type: sessions.EventMessage,
|
||||
Payload: map[string]any{"role": "assistant", "content": assistant},
|
||||
})
|
||||
}
|
||||
}
|
||||
sess.appendHistory(turnRecord{user: user, assistant: assistant})
|
||||
}
|
||||
|
||||
func (a *Agent) loadHistory(sessionID string) []turnRecord {
|
||||
func (a *Agent) persistTurn(sess *acpSession, user, assistant string) error {
|
||||
defer sess.appendHistory(turnRecord{user: user, assistant: assistant})
|
||||
if a.deps.Store == nil {
|
||||
return nil
|
||||
}
|
||||
events := []sessions.AppendEventInput{
|
||||
{
|
||||
Type: sessions.EventMessage,
|
||||
Payload: map[string]any{"role": "user", "content": user},
|
||||
},
|
||||
}
|
||||
if assistant != "" {
|
||||
events = append(events, sessions.AppendEventInput{
|
||||
Type: sessions.EventMessage,
|
||||
Payload: map[string]any{"role": "assistant", "content": assistant},
|
||||
})
|
||||
}
|
||||
_, err := a.deps.Store.AppendEvents(sess.id, events)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *Agent) loadHistory(sessionID string) ([]turnRecord, error) {
|
||||
if a.deps.Store == nil {
|
||||
return nil, nil
|
||||
}
|
||||
events, err := a.deps.Store.ReadEvents(sessionID)
|
||||
if err != nil {
|
||||
return nil
|
||||
return nil, err
|
||||
}
|
||||
var records []turnRecord
|
||||
var pendingUser string
|
||||
@@ -444,7 +463,21 @@ func (a *Agent) loadHistory(sessionID string) []turnRecord {
|
||||
if havePending {
|
||||
records = append(records, turnRecord{user: pendingUser})
|
||||
}
|
||||
return records
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func (a *Agent) warnPersistence(note *notifier, action string, message string, err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
sessionID := ""
|
||||
if note != nil {
|
||||
sessionID = note.sessionID
|
||||
}
|
||||
log.Printf("zero acp: failed to %s for session %s: %v", action, sessionID, err)
|
||||
if note != nil {
|
||||
note.text("\n\n[zero warning] " + message + "\n")
|
||||
}
|
||||
}
|
||||
|
||||
// buildPrompt prepends prior conversation as context, since agent.Run drives a
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -242,8 +244,77 @@ func TestACPRejectsInvalidCwd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestACPPromptWarnsWhenTurnPersistenceFails(t *testing.T) {
|
||||
deps := testDeps(t)
|
||||
h := newHarness(t, deps)
|
||||
defer h.stop()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var newRes NewSessionResult
|
||||
if err := h.client.Call(ctx, MethodSessionNew, NewSessionParams{Cwd: t.TempDir(), McpServers: []McpServer{}}, &newRes); err != nil {
|
||||
t.Fatalf("session/new: %v", err)
|
||||
}
|
||||
metadataPath := filepath.Join(deps.Store.RootDir, newRes.SessionID, sessions.MetadataFile)
|
||||
if err := os.Remove(metadataPath); err != nil {
|
||||
t.Fatalf("remove metadata: %v", err)
|
||||
}
|
||||
|
||||
var promptRes PromptResult
|
||||
if err := h.client.Call(ctx, MethodSessionPrompt, PromptParams{
|
||||
SessionID: newRes.SessionID,
|
||||
Prompt: []ContentBlock{TextBlock("hi")},
|
||||
}, &promptRes); err != nil {
|
||||
t.Fatalf("session/prompt: %v", err)
|
||||
}
|
||||
if promptRes.StopReason != StopEndTurn {
|
||||
t.Fatalf("stopReason = %q, want %q", promptRes.StopReason, StopEndTurn)
|
||||
}
|
||||
got := drainTextUntil(t, h.updates, func(text string) bool {
|
||||
return strings.Contains(text, "Hello from ZERO") &&
|
||||
strings.Contains(text, "Could not save session history")
|
||||
})
|
||||
if !strings.Contains(got, "Could not save session history") {
|
||||
t.Fatalf("streamed text = %q, want persistence warning", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACPLoadWarnsWhenHistoryReadFails(t *testing.T) {
|
||||
deps := testDeps(t)
|
||||
cwd := t.TempDir()
|
||||
meta, err := deps.Store.Create(sessions.CreateInput{Title: "ACP session", Cwd: cwd})
|
||||
if err != nil {
|
||||
t.Fatalf("create session: %v", err)
|
||||
}
|
||||
eventsPath := filepath.Join(deps.Store.RootDir, meta.SessionID, sessions.EventsFile)
|
||||
if err := os.WriteFile(eventsPath, []byte("{bad json}\n"), 0o600); err != nil {
|
||||
t.Fatalf("write corrupt events: %v", err)
|
||||
}
|
||||
h := newHarness(t, deps)
|
||||
defer h.stop()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := h.client.Call(ctx, MethodSessionLoad, LoadSessionParams{SessionID: meta.SessionID, Cwd: cwd, McpServers: []McpServer{}}, &LoadSessionResult{}); err != nil {
|
||||
t.Fatalf("session/load: %v", err)
|
||||
}
|
||||
got := drainTextUntil(t, h.updates, func(text string) bool {
|
||||
return strings.Contains(text, "Could not load session history")
|
||||
})
|
||||
if !strings.Contains(got, "Could not load session history") {
|
||||
t.Fatalf("streamed text = %q, want load warning", got)
|
||||
}
|
||||
}
|
||||
|
||||
// drainText collects streamed chunks for a short window and concatenates them.
|
||||
func drainText(t *testing.T, ch <-chan string) string {
|
||||
t.Helper()
|
||||
return drainTextUntil(t, ch, func(text string) bool {
|
||||
return strings.Contains(text, "Hello from ZERO")
|
||||
})
|
||||
}
|
||||
|
||||
func drainTextUntil(t *testing.T, ch <-chan string, done func(string) bool) string {
|
||||
t.Helper()
|
||||
var b strings.Builder
|
||||
deadline := time.After(2 * time.Second)
|
||||
@@ -251,7 +322,7 @@ func drainText(t *testing.T, ch <-chan string) string {
|
||||
select {
|
||||
case s := <-ch:
|
||||
b.WriteString(s)
|
||||
if strings.Contains(b.String(), "Hello from ZERO") {
|
||||
if done(b.String()) {
|
||||
return b.String()
|
||||
}
|
||||
case <-deadline:
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Post-edit LSP diagnostics used to run synchronously inside edit_file /
|
||||
// write_file: every edit blocked its tool result on a fresh-publish wait with
|
||||
// a quiet-period debounce (≥300ms floor that reset on each publish, 10s cap) —
|
||||
// the largest recurring latency tax in an edit-heavy session. Collection is
|
||||
// now asynchronous: a file is enqueued the moment a mutating tool finishes, a
|
||||
// single worker checks it while the rest of the tool batch (and any
|
||||
// self-correct verification) runs, and the loop drains completed results just
|
||||
// before building the NEXT provider request, appending errors as a user-role
|
||||
// nudge. The model always issues another request after an edit turn — it has
|
||||
// to read its tool results — so it still sees introduced errors at the same
|
||||
// decision point as before, without any tool call stalling on the language
|
||||
// server.
|
||||
//
|
||||
// A file re-edited before its check runs is checked once in its final state
|
||||
// (the checker re-reads the file); a file re-edited after its check completed
|
||||
// is re-enqueued and the newer result wins. When the worker is still busy at
|
||||
// drain time, the drain gives up after a short wait and the results are
|
||||
// delivered on the following turn instead — they stay accurate because any
|
||||
// further edit re-enqueues the file.
|
||||
|
||||
// asyncDiagnosticsDrainTimeout bounds how long a turn waits on the in-flight
|
||||
// check before deferring delivery to the next turn. Most checks finish well
|
||||
// under it (debounce floor + analysis); a cold language server on a large
|
||||
// repo does not get to stall the loop the way the old inline 10s cap could.
|
||||
// A var so tests can shorten the wait.
|
||||
var asyncDiagnosticsDrainTimeout = 3 * time.Second
|
||||
|
||||
// asyncDiagnosticsFinalDrainTimeout is the wait used when the run is about to
|
||||
// FINALIZE (natural completion or the max-turns summary): there is no later
|
||||
// turn to defer to, so the gate waits out the old inline-era budget rather
|
||||
// than silently dropping an error the last edit introduced. A var so tests
|
||||
// can shorten the wait.
|
||||
var asyncDiagnosticsFinalDrainTimeout = 10 * time.Second
|
||||
|
||||
// asyncDiagnosticsNudge prefixes the drained diagnostics blocks; phrased like
|
||||
// the old inline block so the model reacts the same way.
|
||||
const asyncDiagnosticsNudge = "Diagnostics after your recent edits (fix any errors you introduced):\n"
|
||||
|
||||
type asyncDiagnostics struct {
|
||||
check func(context.Context, string) string
|
||||
workspaceRoot string
|
||||
|
||||
mu sync.Mutex
|
||||
queue []string
|
||||
queued map[string]bool
|
||||
results map[string]string
|
||||
// working is non-nil while the worker goroutine runs and is closed when it
|
||||
// exits, so drain can wait for quiescence without polling.
|
||||
working chan struct{}
|
||||
}
|
||||
|
||||
// newAsyncDiagnostics returns nil when check is nil, and every method on a nil
|
||||
// receiver is a no-op — callers wire it unconditionally.
|
||||
func newAsyncDiagnostics(check func(context.Context, string) string, workspaceRoot string) *asyncDiagnostics {
|
||||
if check == nil {
|
||||
return nil
|
||||
}
|
||||
return &asyncDiagnostics{
|
||||
check: check,
|
||||
workspaceRoot: workspaceRoot,
|
||||
queued: map[string]bool{},
|
||||
results: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
// enqueue schedules changed files for a background check. Paths are
|
||||
// workspace-relative from Result.ChangedFiles (absolute when under a granted
|
||||
// extra write root), mirroring SelfCorrector's resolution.
|
||||
func (diagnostics *asyncDiagnostics) enqueue(ctx context.Context, files []string) {
|
||||
if diagnostics == nil || len(files) == 0 {
|
||||
return
|
||||
}
|
||||
diagnostics.mu.Lock()
|
||||
defer diagnostics.mu.Unlock()
|
||||
for _, file := range files {
|
||||
path := diagnostics.absPath(file)
|
||||
if diagnostics.queued[path] {
|
||||
continue
|
||||
}
|
||||
diagnostics.queued[path] = true
|
||||
// A completed result for this file describes a pre-edit state now.
|
||||
delete(diagnostics.results, path)
|
||||
diagnostics.queue = append(diagnostics.queue, path)
|
||||
}
|
||||
if diagnostics.working == nil && len(diagnostics.queue) > 0 {
|
||||
done := make(chan struct{})
|
||||
diagnostics.working = done
|
||||
go diagnostics.work(ctx, done)
|
||||
}
|
||||
}
|
||||
|
||||
func (diagnostics *asyncDiagnostics) work(ctx context.Context, done chan struct{}) {
|
||||
defer close(done)
|
||||
for {
|
||||
diagnostics.mu.Lock()
|
||||
if len(diagnostics.queue) == 0 || ctx.Err() != nil {
|
||||
diagnostics.working = nil
|
||||
diagnostics.mu.Unlock()
|
||||
return
|
||||
}
|
||||
path := diagnostics.queue[0]
|
||||
diagnostics.queue = diagnostics.queue[1:]
|
||||
// Un-mark before checking so an edit that lands DURING the check
|
||||
// re-enqueues the file and it is checked again in its final state.
|
||||
delete(diagnostics.queued, path)
|
||||
diagnostics.mu.Unlock()
|
||||
|
||||
block := diagnostics.check(ctx, path)
|
||||
|
||||
diagnostics.mu.Lock()
|
||||
if block != "" {
|
||||
diagnostics.results[path] = block
|
||||
} else {
|
||||
delete(diagnostics.results, path)
|
||||
}
|
||||
diagnostics.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// drain waits briefly for the worker to go quiet, then formats and consumes
|
||||
// all completed error blocks as one nudge. Returns "" when there is nothing
|
||||
// to report or the worker is still busy (results then surface next turn).
|
||||
func (diagnostics *asyncDiagnostics) drain(ctx context.Context) string {
|
||||
return diagnostics.drainWithin(ctx, asyncDiagnosticsDrainTimeout)
|
||||
}
|
||||
|
||||
// drainFinal is drain with the finalization budget: called before the run's
|
||||
// last model request, where an undelivered result would otherwise be lost.
|
||||
func (diagnostics *asyncDiagnostics) drainFinal(ctx context.Context) string {
|
||||
return diagnostics.drainWithin(ctx, asyncDiagnosticsFinalDrainTimeout)
|
||||
}
|
||||
|
||||
func (diagnostics *asyncDiagnostics) drainWithin(ctx context.Context, timeout time.Duration) string {
|
||||
if diagnostics == nil {
|
||||
return ""
|
||||
}
|
||||
diagnostics.mu.Lock()
|
||||
busy := diagnostics.working
|
||||
diagnostics.mu.Unlock()
|
||||
if busy != nil {
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-busy:
|
||||
case <-timer.C:
|
||||
return ""
|
||||
case <-ctx.Done():
|
||||
return ""
|
||||
}
|
||||
}
|
||||
diagnostics.mu.Lock()
|
||||
defer diagnostics.mu.Unlock()
|
||||
if len(diagnostics.results) == 0 {
|
||||
return ""
|
||||
}
|
||||
paths := make([]string, 0, len(diagnostics.results))
|
||||
for path := range diagnostics.results {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
sort.Strings(paths)
|
||||
blocks := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
blocks = append(blocks, diagnostics.results[path])
|
||||
}
|
||||
diagnostics.results = map[string]string{}
|
||||
return asyncDiagnosticsNudge + strings.Join(blocks, "\n")
|
||||
}
|
||||
|
||||
func (diagnostics *asyncDiagnostics) absPath(path string) string {
|
||||
if filepath.IsAbs(path) {
|
||||
return path
|
||||
}
|
||||
return filepath.Join(diagnostics.workspaceRoot, path)
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
func TestAsyncDiagnosticsNilCollectorNoOps(t *testing.T) {
|
||||
var diagnostics *asyncDiagnostics
|
||||
if diagnostics != newAsyncDiagnostics(nil, "/ws") {
|
||||
t.Fatal("nil check must yield a nil collector")
|
||||
}
|
||||
diagnostics.enqueue(context.Background(), []string{"a.go"})
|
||||
if nudge := diagnostics.drain(context.Background()); nudge != "" {
|
||||
t.Fatalf("nil collector drain = %q, want empty", nudge)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAsyncDiagnosticsCollectsAndDrainsOnce(t *testing.T) {
|
||||
// A real absolute root: a literal "/ws" is not filepath.IsAbs on Windows
|
||||
// (no drive letter), which is exactly what the assertions below check.
|
||||
root := t.TempDir()
|
||||
var mu sync.Mutex
|
||||
var checked []string
|
||||
check := func(_ context.Context, absPath string) string {
|
||||
mu.Lock()
|
||||
checked = append(checked, absPath)
|
||||
mu.Unlock()
|
||||
if filepath.Base(absPath) == "clean.go" {
|
||||
return ""
|
||||
}
|
||||
return "ERR " + filepath.Base(absPath)
|
||||
}
|
||||
diagnostics := newAsyncDiagnostics(check, root)
|
||||
diagnostics.enqueue(context.Background(), []string{"broken.go", "clean.go"})
|
||||
|
||||
nudge := diagnostics.drain(context.Background())
|
||||
if !strings.HasPrefix(nudge, asyncDiagnosticsNudge) {
|
||||
t.Fatalf("nudge missing prefix: %q", nudge)
|
||||
}
|
||||
if !strings.Contains(nudge, "ERR broken.go") || strings.Contains(nudge, "clean.go") {
|
||||
t.Fatalf("nudge = %q, want broken.go errors only", nudge)
|
||||
}
|
||||
mu.Lock()
|
||||
for _, path := range checked {
|
||||
if !filepath.IsAbs(path) || !strings.HasPrefix(path, root+string(filepath.Separator)) {
|
||||
t.Fatalf("check received %q, want path under workspace root %q", path, root)
|
||||
}
|
||||
}
|
||||
mu.Unlock()
|
||||
if second := diagnostics.drain(context.Background()); second != "" {
|
||||
t.Fatalf("second drain = %q, want empty (results are consumed)", second)
|
||||
}
|
||||
}
|
||||
|
||||
// A check still running at drain time defers delivery to the next drain
|
||||
// instead of blocking the turn.
|
||||
func TestAsyncDiagnosticsSlowCheckDefersToNextDrain(t *testing.T) {
|
||||
previous := asyncDiagnosticsDrainTimeout
|
||||
asyncDiagnosticsDrainTimeout = 20 * time.Millisecond
|
||||
defer func() { asyncDiagnosticsDrainTimeout = previous }()
|
||||
|
||||
release := make(chan struct{})
|
||||
check := func(_ context.Context, absPath string) string {
|
||||
<-release
|
||||
return "ERR " + filepath.Base(absPath)
|
||||
}
|
||||
diagnostics := newAsyncDiagnostics(check, "/ws")
|
||||
diagnostics.enqueue(context.Background(), []string{"slow.go"})
|
||||
|
||||
start := time.Now()
|
||||
if nudge := diagnostics.drain(context.Background()); nudge != "" {
|
||||
t.Fatalf("drain during in-flight check = %q, want empty", nudge)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > time.Second {
|
||||
t.Fatalf("drain blocked %v, want bounded by the (shortened) timeout", elapsed)
|
||||
}
|
||||
|
||||
close(release)
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
if nudge := diagnostics.drain(context.Background()); nudge != "" {
|
||||
if !strings.Contains(nudge, "ERR slow.go") {
|
||||
t.Fatalf("deferred nudge = %q", nudge)
|
||||
}
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("deferred diagnostics never delivered")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// Re-enqueueing a file after its check completed replaces the stale result.
|
||||
func TestAsyncDiagnosticsReEditReplacesResult(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
response := "ERR old"
|
||||
check := func(context.Context, string) string {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return response
|
||||
}
|
||||
diagnostics := newAsyncDiagnostics(check, "/ws")
|
||||
|
||||
diagnostics.enqueue(context.Background(), []string{"a.go"})
|
||||
waitForIdle(t, diagnostics)
|
||||
mu.Lock()
|
||||
response = "ERR new"
|
||||
mu.Unlock()
|
||||
diagnostics.enqueue(context.Background(), []string{"a.go"})
|
||||
|
||||
nudge := diagnostics.drain(context.Background())
|
||||
if !strings.Contains(nudge, "ERR new") || strings.Contains(nudge, "ERR old") {
|
||||
t.Fatalf("nudge = %q, want only the re-check result", nudge)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForIdle(t *testing.T, diagnostics *asyncDiagnostics) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
diagnostics.mu.Lock()
|
||||
busy := diagnostics.working
|
||||
diagnostics.mu.Unlock()
|
||||
if busy == nil {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("worker never went idle")
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// changedFilesTool is a fake mutating tool reporting a changed file without
|
||||
// touching the filesystem.
|
||||
type changedFilesTool struct{}
|
||||
|
||||
func (changedFilesTool) Name() string { return "fake_edit" }
|
||||
func (changedFilesTool) Description() string { return "test mutating tool" }
|
||||
func (changedFilesTool) Parameters() tools.Schema {
|
||||
return tools.Schema{Type: "object", Properties: map[string]tools.PropertySchema{}}
|
||||
}
|
||||
func (changedFilesTool) Safety() tools.Safety {
|
||||
return tools.Safety{SideEffect: tools.SideEffectWrite, Permission: tools.PermissionAllow, Reason: "test"}
|
||||
}
|
||||
func (changedFilesTool) Run(context.Context, map[string]any) tools.Result {
|
||||
return tools.Result{Status: tools.StatusOK, Output: "edited", ChangedFiles: []string{"main.go"}}
|
||||
}
|
||||
|
||||
// End-to-end through Run: an edit turn does NOT carry diagnostics in the tool
|
||||
// result (the old inline path), but the next request contains the background
|
||||
// diagnostics nudge, before the model finalizes.
|
||||
func TestRunDeliversAsyncDiagnosticsNudgeNextTurn(t *testing.T) {
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(changedFilesTool{})
|
||||
provider := &mockProvider{
|
||||
turns: [][]zeroruntime.StreamEvent{
|
||||
{
|
||||
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "fake_edit"},
|
||||
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{}`},
|
||||
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
{
|
||||
{Type: zeroruntime.StreamEventText, Content: "done"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
},
|
||||
}
|
||||
root := t.TempDir()
|
||||
var checkedPath string
|
||||
var mu sync.Mutex
|
||||
|
||||
result, err := Run(context.Background(), "fix main.go", provider, Options{
|
||||
Registry: registry,
|
||||
Cwd: root,
|
||||
FileDiagnostics: func(_ context.Context, absPath string) string {
|
||||
mu.Lock()
|
||||
checkedPath = absPath
|
||||
mu.Unlock()
|
||||
return "main.go:1:1 error: boom"
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.FinalAnswer != "done" {
|
||||
t.Fatalf("final answer = %q", result.FinalAnswer)
|
||||
}
|
||||
if len(provider.requests) != 2 {
|
||||
t.Fatalf("expected 2 provider turns, got %d", len(provider.requests))
|
||||
}
|
||||
|
||||
// The tool result itself must NOT block on / embed diagnostics.
|
||||
for _, message := range provider.requests[1].Messages {
|
||||
if message.Role == zeroruntime.MessageRoleTool && strings.Contains(message.Content, "boom") {
|
||||
t.Fatalf("diagnostics leaked into the tool result: %q", message.Content)
|
||||
}
|
||||
}
|
||||
// The nudge must be present in the second request as a user message.
|
||||
found := false
|
||||
for _, message := range provider.requests[1].Messages {
|
||||
if message.Role == zeroruntime.MessageRoleUser && strings.HasPrefix(message.Content, asyncDiagnosticsNudge) {
|
||||
if !strings.Contains(message.Content, "boom") {
|
||||
t.Fatalf("nudge missing diagnostics: %q", message.Content)
|
||||
}
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("no async diagnostics nudge in the follow-up request")
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if want := filepath.Join(root, "main.go"); checkedPath != want {
|
||||
t.Fatalf("checked path = %q, want %q", checkedPath, want)
|
||||
}
|
||||
}
|
||||
|
||||
// When the run hits the maxTurns ceiling right after an edit turn, the
|
||||
// final-answer request must still carry the diagnostics nudge — otherwise an
|
||||
// error introduced by the last edit would go unreported in the summary.
|
||||
func TestRunDrainsAsyncDiagnosticsBeforeMaxTurnsFinalAnswer(t *testing.T) {
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(changedFilesTool{})
|
||||
provider := &mockProvider{
|
||||
turns: [][]zeroruntime.StreamEvent{
|
||||
{
|
||||
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "fake_edit"},
|
||||
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{}`},
|
||||
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
{
|
||||
{Type: zeroruntime.StreamEventText, Content: "summary"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := Run(context.Background(), "fix main.go", provider, Options{
|
||||
Registry: registry,
|
||||
Cwd: t.TempDir(),
|
||||
MaxTurns: 1,
|
||||
FileDiagnostics: func(context.Context, string) string {
|
||||
return "main.go:1:1 error: boom"
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.FinalAnswer != "summary" {
|
||||
t.Fatalf("final answer = %q", result.FinalAnswer)
|
||||
}
|
||||
if len(provider.requests) != 2 {
|
||||
t.Fatalf("expected the max-turns final-answer request, got %d requests", len(provider.requests))
|
||||
}
|
||||
found := false
|
||||
for _, message := range provider.requests[1].Messages {
|
||||
if message.Role == zeroruntime.MessageRoleUser && strings.HasPrefix(message.Content, asyncDiagnosticsNudge) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("max-turns final-answer request missing the diagnostics nudge")
|
||||
}
|
||||
}
|
||||
|
||||
// When the model finalizes while a slow check is still in flight (the
|
||||
// per-turn drain missed it), the finalization gate must wait it out and give
|
||||
// the model one more turn with the nudge instead of dropping the errors.
|
||||
func TestRunFinalizationGateDeliversLateDiagnostics(t *testing.T) {
|
||||
previous := asyncDiagnosticsDrainTimeout
|
||||
asyncDiagnosticsDrainTimeout = time.Millisecond
|
||||
defer func() { asyncDiagnosticsDrainTimeout = previous }()
|
||||
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(changedFilesTool{})
|
||||
provider := &mockProvider{
|
||||
turns: [][]zeroruntime.StreamEvent{
|
||||
{
|
||||
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-1", ToolName: "fake_edit"},
|
||||
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-1", ArgumentsFragment: `{}`},
|
||||
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-1"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
{
|
||||
{Type: zeroruntime.StreamEventText, Content: "done"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
{
|
||||
{Type: zeroruntime.StreamEventText, Content: "done after fix"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := Run(context.Background(), "fix main.go", provider, Options{
|
||||
Registry: registry,
|
||||
Cwd: t.TempDir(),
|
||||
FileDiagnostics: func(context.Context, string) string {
|
||||
// Slower than the (shortened) per-turn drain, well under the
|
||||
// finalization budget.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
return "main.go:1:1 error: boom"
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.FinalAnswer != "done after fix" {
|
||||
t.Fatalf("final answer = %q, want the post-nudge answer", result.FinalAnswer)
|
||||
}
|
||||
if len(provider.requests) != 3 {
|
||||
t.Fatalf("expected a third request carrying the late nudge, got %d requests", len(provider.requests))
|
||||
}
|
||||
found := false
|
||||
for _, message := range provider.requests[2].Messages {
|
||||
if message.Role == zeroruntime.MessageRoleUser && strings.HasPrefix(message.Content, asyncDiagnosticsNudge) {
|
||||
if !strings.Contains(message.Content, "boom") {
|
||||
t.Fatalf("nudge missing diagnostics: %q", message.Content)
|
||||
}
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("finalization gate did not deliver the late diagnostics nudge")
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
@@ -27,10 +28,20 @@ func proposedCommandPrefix(toolName string, args map[string]any) []string {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, tokens := range segments {
|
||||
if !knownSafeCommandSegment(tokens) && sandbox.ValidCommandPrefix(tokens) {
|
||||
return append([]string(nil), tokens...)
|
||||
// Only propose approving a prefix of one segment when every other segment
|
||||
// in the command is independently known-safe. Once a prefix is approved,
|
||||
// shellExecutionArgsForApproval escalates the whole command (every
|
||||
// segment, not just the approved one) to bypass the sandbox, so offering a
|
||||
// prefix that leaves an MSYS-prone (or otherwise unsafe) segment uncovered
|
||||
// would let that segment run unsandboxed without ever being reviewed.
|
||||
for index, tokens := range segments {
|
||||
if knownSafeCommandSegment(tokens) {
|
||||
continue
|
||||
}
|
||||
if !sandbox.ValidCommandPrefix(tokens) || !otherSegmentsKnownSafe(segments, index) {
|
||||
return nil
|
||||
}
|
||||
return append([]string(nil), tokens...)
|
||||
}
|
||||
if len(segments) == 0 || !sandbox.ValidCommandPrefix(segments[0]) {
|
||||
return nil
|
||||
@@ -38,6 +49,20 @@ func proposedCommandPrefix(toolName string, args map[string]any) []string {
|
||||
return append([]string(nil), segments[0]...)
|
||||
}
|
||||
|
||||
// otherSegmentsKnownSafe reports whether every segment other than the one at
|
||||
// skip is known-safe on its own.
|
||||
func otherSegmentsKnownSafe(segments [][]string, skip int) bool {
|
||||
for index, tokens := range segments {
|
||||
if index == skip {
|
||||
continue
|
||||
}
|
||||
if !knownSafeCommandSegment(tokens) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func matchCommandPrefix(toolName string, args map[string]any, options Options) (sandbox.CommandPrefixGrant, bool, bool) {
|
||||
if !isShellCommandTool(toolName) || options.Sandbox == nil {
|
||||
return sandbox.CommandPrefixGrant{}, false, false
|
||||
@@ -168,10 +193,6 @@ func cleanPrefixRule(prefix []string) []string {
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func safeRequestedPrefix(prefix []string, command []string) bool {
|
||||
return safeRequestedPrefixForSegments(prefix, [][]string{command})
|
||||
}
|
||||
|
||||
func safeRequestedPrefixForSegments(prefix []string, segments [][]string) bool {
|
||||
if len(prefix) == 0 || !sandbox.ValidCommandPrefix(prefix) {
|
||||
return false
|
||||
@@ -196,14 +217,6 @@ func safeRequestedPrefixForSegments(prefix []string, segments [][]string) bool {
|
||||
return matched
|
||||
}
|
||||
|
||||
func safeShellCommandTokens(command string) ([]string, bool) {
|
||||
segments, ok := safeShellCommandSegments(command)
|
||||
if !ok || len(segments) != 1 {
|
||||
return nil, false
|
||||
}
|
||||
return segments[0], true
|
||||
}
|
||||
|
||||
func safeShellCommandSegments(command string) ([][]string, bool) {
|
||||
command = strings.TrimSpace(command)
|
||||
if command == "" {
|
||||
@@ -275,7 +288,11 @@ func knownSafeCommandSegment(command []string) bool {
|
||||
if len(command) == 0 {
|
||||
return false
|
||||
}
|
||||
switch commandName(command[0]) {
|
||||
name := commandName(command[0])
|
||||
if runtime.GOOS == "windows" && tools.MsysProneCommandName(name) {
|
||||
return false
|
||||
}
|
||||
switch name {
|
||||
case "cat", "cd", "cut", "echo", "expr", "false", "grep", "head", "id",
|
||||
"ls", "nl", "paste", "pwd", "rev", "seq", "stat", "tail", "tr",
|
||||
"true", "uname", "uniq", "wc", "which", "whoami":
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
)
|
||||
|
||||
func TestProposedCommandPrefixUsesSafeSimpleCommands(t *testing.T) {
|
||||
@@ -35,12 +37,37 @@ func TestProposedCommandPrefixHonorsValidatedRequestedPrefix(t *testing.T) {
|
||||
|
||||
func TestProposedCommandPrefixSupportsSegmentedCommands(t *testing.T) {
|
||||
got := proposedCommandPrefix("bash", map[string]any{"command": "ps aux | head -5"})
|
||||
if runtime.GOOS == "windows" {
|
||||
// head is MSYS-prone on Windows (#458), so proposedCommandPrefix must
|
||||
// not offer "ps aux" as a reusable prefix here: approving it would
|
||||
// escalate the whole command, including the uncovered head segment,
|
||||
// to bypass the sandbox unreviewed. See
|
||||
// TestProposedCommandPrefixRejectsPrefixLeavingUnsafeTailUncovered for
|
||||
// the platform-independent regression coverage of this behavior.
|
||||
if got != nil {
|
||||
t.Fatalf("expected no prefix on Windows because head is MSYS-prone, got %#v", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
want := []string{"ps", "aux"}
|
||||
if !equalStringSlices(got, want) {
|
||||
t.Fatalf("prefix = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProposedCommandPrefixRejectsPrefixLeavingUnsafeTailUncovered guards
|
||||
// against proposedCommandPrefix offering to approve one segment of a
|
||||
// multi-segment command (e.g. "ps aux") while a different segment (e.g. "npm
|
||||
// install") is not known-safe. shellExecutionArgsForApproval escalates the
|
||||
// entire command once any prefix is approved, so an uncovered unsafe segment
|
||||
// would bypass the sandbox unreviewed. Uses npm, which is never known-safe on
|
||||
// any platform, so the assertion does not depend on runtime.GOOS.
|
||||
func TestProposedCommandPrefixRejectsPrefixLeavingUnsafeTailUncovered(t *testing.T) {
|
||||
if got := proposedCommandPrefix("bash", map[string]any{"command": "ps aux && npm install"}); got != nil {
|
||||
t.Fatalf("expected no prefix because npm segment is not known-safe, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposedCommandPrefixHonorsRequestedPrefixAcrossSegments(t *testing.T) {
|
||||
got := proposedCommandPrefix("bash", map[string]any{
|
||||
"command": "git status --short && git status --branch",
|
||||
@@ -120,13 +147,35 @@ func TestProposedCommandPrefixRejectsUnsafeLaunchers(t *testing.T) {
|
||||
func TestMatchCommandPrefixCoversSegmentedCommandWithSafeTail(t *testing.T) {
|
||||
engine := sandbox.NewEngine(sandbox.EngineOptions{WorkspaceRoot: t.TempDir()})
|
||||
engine.GrantCommandPrefixForSession("bash", []string{"ps", "aux"})
|
||||
// head is MSYS-prone on Windows (#458) and must not count as a known-safe tail.
|
||||
command := "ps aux | head -5"
|
||||
if runtime.GOOS == "windows" {
|
||||
command = "ps aux | echo ok"
|
||||
}
|
||||
|
||||
grant, ok, session := matchCommandPrefix("bash", map[string]any{"command": "ps aux | head -5"}, Options{Sandbox: engine})
|
||||
grant, ok, session := matchCommandPrefix("bash", map[string]any{"command": command}, Options{Sandbox: engine})
|
||||
if !ok || !session || !equalStringSlices(grant.Prefix, []string{"ps", "aux"}) {
|
||||
t.Fatalf("match = %#v ok=%v session=%v, want session ps aux prefix", grant, ok, session)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKnownSafeCommandSegmentRejectsMsysProneOnWindows(t *testing.T) {
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Skip("windows-only known-safe MSYS guard")
|
||||
}
|
||||
for _, command := range [][]string{{"head", "-5"}, {"cat", "file.txt"}, {"grep", "pat"}} {
|
||||
if knownSafeCommandSegment(command) {
|
||||
t.Fatalf("expected %q to be unsafe on Windows, got known-safe", command)
|
||||
}
|
||||
}
|
||||
if !knownSafeCommandSegment([]string{"echo", "ok"}) {
|
||||
t.Fatal("expected echo to remain known-safe on Windows")
|
||||
}
|
||||
if !tools.MsysProneCommandName("head") {
|
||||
t.Fatal("expected head to be MSYS-prone")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchCommandPrefixRejectsUncoveredSegment(t *testing.T) {
|
||||
engine := sandbox.NewEngine(sandbox.EngineOptions{WorkspaceRoot: t.TempDir()})
|
||||
engine.GrantCommandPrefixForSession("bash", []string{"ps", "aux"})
|
||||
|
||||
@@ -325,6 +325,43 @@ type compactionState struct {
|
||||
// OnText is deliberately NOT forwarded (compaction stays invisible to the user),
|
||||
// but its token COST must still be counted so usage reports and budgets include it.
|
||||
onUsage func(Usage)
|
||||
|
||||
// calibrationRatio scales the raw byte/4 token estimate toward the provider's
|
||||
// real prompt-token count. ApproxTextTokens over-counts code-heavy content by
|
||||
// ~15-20%, which would trip compaction early (at ~60% of true capacity). It
|
||||
// starts at 1.0 and converges via an EMA as each turn reports actual usage, so
|
||||
// later turns compact nearer to real capacity. Zero is treated as 1.0.
|
||||
calibrationRatio float64
|
||||
}
|
||||
|
||||
// calibrate folds one turn's (rawEstimate, actualPromptTokens) sample into the
|
||||
// running calibration ratio. A single sample is clamped to a sane band so an
|
||||
// outlier (a huge cache-read turn, a provider-overhead spike) can't skew the
|
||||
// estimate enough to disable or thrash compaction.
|
||||
func (state *compactionState) calibrate(rawEstimate int, actualPromptTokens int) {
|
||||
if !state.enabled || rawEstimate <= 0 || actualPromptTokens <= 0 {
|
||||
return
|
||||
}
|
||||
sample := float64(actualPromptTokens) / float64(rawEstimate)
|
||||
if sample < 0.5 {
|
||||
sample = 0.5
|
||||
} else if sample > 2.0 {
|
||||
sample = 2.0
|
||||
}
|
||||
if state.calibrationRatio <= 0 {
|
||||
state.calibrationRatio = 1.0
|
||||
}
|
||||
const alpha = 0.3 // weight on the newest sample; smooths jitter across turns
|
||||
state.calibrationRatio = state.calibrationRatio*(1-alpha) + sample*alpha
|
||||
}
|
||||
|
||||
// calibratedTokens applies the learned ratio to a raw estimate. Before any sample
|
||||
// arrives (ratio unset) it returns the raw estimate unchanged.
|
||||
func (state *compactionState) calibratedTokens(raw int) int {
|
||||
if state.calibrationRatio <= 0 {
|
||||
return raw
|
||||
}
|
||||
return int(float64(raw) * state.calibrationRatio)
|
||||
}
|
||||
|
||||
func newCompactionState(options Options) *compactionState {
|
||||
@@ -353,7 +390,7 @@ func (state *compactionState) maybeCompact(
|
||||
// the messages; both the threshold check and the shrink check below use the
|
||||
// same term so they stay consistent.
|
||||
toolTokens := estimateToolDefTokens(tools)
|
||||
size := estimateTokens(messages) + toolTokens
|
||||
size := state.calibratedTokens(estimateTokens(messages) + toolTokens)
|
||||
if size <= state.threshold {
|
||||
return messages
|
||||
}
|
||||
@@ -370,7 +407,7 @@ func (state *compactionState) maybeCompact(
|
||||
// entirely and preserve recent turns verbatim.
|
||||
if pruned, reclaimed := pruneStaleToolOutput(messages, state.preserveLast); reclaimed > 0 {
|
||||
messages = pruned
|
||||
size = estimateTokens(messages) + toolTokens
|
||||
size = state.calibratedTokens(estimateTokens(messages) + toolTokens)
|
||||
if size <= state.threshold {
|
||||
state.lowWaterMark = size
|
||||
return messages
|
||||
@@ -386,7 +423,7 @@ func (state *compactionState) maybeCompact(
|
||||
// later turn) can try again; we never drop messages on failure here.
|
||||
return messages
|
||||
}
|
||||
newSize := estimateTokens(compacted) + toolTokens
|
||||
newSize := state.calibratedTokens(estimateTokens(compacted) + toolTokens)
|
||||
if newSize >= size {
|
||||
// Compaction did not actually shrink anything (e.g. nothing to
|
||||
// summarize). Leave the history untouched and don't churn next turn.
|
||||
@@ -444,7 +481,7 @@ func (state *compactionState) recover(
|
||||
// the SAME combined (messages + tool-defs) domain maybeCompact uses, so the
|
||||
// proactive shrink-guard compares like with like.
|
||||
state.reactiveAttempted = true
|
||||
state.lowWaterMark = estimateTokens(result) + estimateToolDefTokens(tools)
|
||||
state.lowWaterMark = state.calibratedTokens(estimateTokens(result) + estimateToolDefTokens(tools))
|
||||
return result, true, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package agent
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCalibrationConvergesTowardActual(t *testing.T) {
|
||||
state := &compactionState{enabled: true}
|
||||
|
||||
// Before any sample, the estimate is unchanged.
|
||||
if got := state.calibratedTokens(1000); got != 1000 {
|
||||
t.Fatalf("uncalibrated tokens = %d, want 1000", got)
|
||||
}
|
||||
|
||||
// The estimator over-counts: our raw estimate is 1000 but the provider reports
|
||||
// 850 (a 0.85 ratio). Feeding that sample repeatedly should pull the ratio
|
||||
// toward 0.85, so a 1000-token raw estimate calibrates downward.
|
||||
for range 20 {
|
||||
state.calibrate(1000, 850)
|
||||
}
|
||||
got := state.calibratedTokens(1000)
|
||||
if got >= 1000 || got < 820 || got > 880 {
|
||||
t.Fatalf("calibrated tokens = %d, want ~850 after convergence", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalibrateIgnoresDegenerateSamples(t *testing.T) {
|
||||
state := &compactionState{enabled: true}
|
||||
state.calibrate(0, 500) // zero estimate
|
||||
state.calibrate(500, 0) // zero actual
|
||||
if state.calibrationRatio != 0 {
|
||||
t.Fatalf("degenerate samples must not move the ratio, got %v", state.calibrationRatio)
|
||||
}
|
||||
// Disabled state never calibrates.
|
||||
disabled := &compactionState{enabled: false}
|
||||
disabled.calibrate(1000, 850)
|
||||
if disabled.calibrationRatio != 0 {
|
||||
t.Fatalf("disabled compaction must not calibrate, got %v", disabled.calibrationRatio)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalibrateClampsOutliers(t *testing.T) {
|
||||
state := &compactionState{enabled: true}
|
||||
// A wild outlier (10x) is clamped to 2.0, so one bad sample can't blow up the
|
||||
// ratio and disable compaction.
|
||||
for range 50 {
|
||||
state.calibrate(1000, 10000)
|
||||
}
|
||||
if state.calibrationRatio > 2.0 {
|
||||
t.Fatalf("ratio should be clamped at 2.0, got %v", state.calibrationRatio)
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,16 @@ const (
|
||||
toolNameUpdatePlan = "update_plan"
|
||||
toolNameToolSearch = "tool_search"
|
||||
toolNameSkill = "skill"
|
||||
toolNameWriteFile = "write_file"
|
||||
toolNameEditFile = "edit_file"
|
||||
)
|
||||
|
||||
// maxRecentEdits caps how many edited files are carried across a compaction, and
|
||||
// maxEditNoteBytes caps each edit's one-line note, so a long editing run can't
|
||||
// bloat the preserved-state block.
|
||||
const (
|
||||
maxRecentEdits = 20
|
||||
maxEditNoteBytes = 160
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -100,12 +110,123 @@ func formatPlanArguments(arguments string) string {
|
||||
}
|
||||
|
||||
// skillEntry is a named preserved body. It began as loaded-skill state and is
|
||||
// reused for loaded tools and project instruction blocks.
|
||||
// reused for loaded tools, project instruction blocks, and recent file edits.
|
||||
type skillEntry struct {
|
||||
name string
|
||||
body string
|
||||
}
|
||||
|
||||
// recentEdits returns the files mutated by write_file/edit_file calls in messages
|
||||
// — latest note per path, in last-seen order — as skillEntry{name: path, body:
|
||||
// note}. After compaction elides the editing turns, this tells the model WHAT it
|
||||
// changed in each file (from the tool's result) so it need not re-read to
|
||||
// rediscover its own footprint. Capped at maxRecentEdits paths.
|
||||
//
|
||||
// Ordering is by LAST edit, not first: re-editing a file moves it to the newest
|
||||
// position so the tail cap below keeps the file the model most recently touched
|
||||
// rather than an earlier, now-stale entry.
|
||||
func recentEdits(messages []zeroruntime.Message) []skillEntry {
|
||||
pathByID := map[string]string{}
|
||||
sequence := make([]string, 0)
|
||||
for _, message := range messages {
|
||||
for _, call := range message.ToolCalls {
|
||||
if call.Name != toolNameWriteFile && call.Name != toolNameEditFile {
|
||||
continue
|
||||
}
|
||||
path := editPathFromArguments(call.Arguments)
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
if call.ID != "" {
|
||||
pathByID[call.ID] = path
|
||||
}
|
||||
sequence = append(sequence, path)
|
||||
}
|
||||
}
|
||||
order := lastSeenOrder(sequence)
|
||||
if len(order) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
noteByPath := map[string]string{}
|
||||
for _, message := range messages {
|
||||
if message.Role != zeroruntime.MessageRoleTool || message.ToolCallID == "" {
|
||||
continue
|
||||
}
|
||||
if path, ok := pathByID[message.ToolCallID]; ok {
|
||||
noteByPath[path] = editNote(message.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the most recent maxRecentEdits paths (the tail of last-seen order).
|
||||
if len(order) > maxRecentEdits {
|
||||
order = order[len(order)-maxRecentEdits:]
|
||||
}
|
||||
entries := make([]skillEntry, 0, len(order))
|
||||
for _, path := range order {
|
||||
entries = append(entries, skillEntry{name: path, body: noteByPath[path]})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// lastSeenOrder dedupes paths keeping each at its LAST occurrence, preserving
|
||||
// chronological order otherwise. A re-edited path therefore lands at the newest
|
||||
// position rather than staying pinned to where it first appeared.
|
||||
func lastSeenOrder(paths []string) []string {
|
||||
lastIdx := make(map[string]int, len(paths))
|
||||
for i, p := range paths {
|
||||
lastIdx[p] = i
|
||||
}
|
||||
order := make([]string, 0, len(lastIdx))
|
||||
for i, p := range paths {
|
||||
if lastIdx[p] == i {
|
||||
order = append(order, p)
|
||||
}
|
||||
}
|
||||
return order
|
||||
}
|
||||
|
||||
// editPathFromArguments pulls the target file path from a write_file/edit_file
|
||||
// call's JSON arguments (path/file/file_path/filename aliases). Returns "" on
|
||||
// malformed arguments or no path.
|
||||
func editPathFromArguments(arguments string) string {
|
||||
var parsed struct {
|
||||
Path string `json:"path"`
|
||||
File string `json:"file"`
|
||||
FilePath string `json:"file_path"`
|
||||
Filename string `json:"filename"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(arguments)), &parsed); err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, candidate := range []string{parsed.Path, parsed.File, parsed.FilePath, parsed.Filename} {
|
||||
if s := strings.TrimSpace(candidate); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// editNote reduces an edit tool's result to a single short line (its first
|
||||
// non-empty line, byte-capped on a rune boundary) for the preserved summary.
|
||||
func editNote(content string) string {
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
s := strings.TrimSpace(line)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if len(s) > maxEditNoteBytes {
|
||||
limit := maxEditNoteBytes
|
||||
for limit > 0 && !utf8.RuneStart(s[limit]) {
|
||||
limit--
|
||||
}
|
||||
s = s[:limit] + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// loadedSkills returns the skills loaded via the skill tool in messages — the
|
||||
// latest body per name, in first-seen order — matching each skill tool call to
|
||||
// its tool result by id.
|
||||
@@ -310,11 +431,17 @@ func capBody(body string) string {
|
||||
// preservedState is the JSON shape of the carried-across-compaction block.
|
||||
type preservedState struct {
|
||||
Plan string `json:"plan,omitempty"`
|
||||
RecentEdits []preservedEdit `json:"recent_edits,omitempty"`
|
||||
Tools []preservedTool `json:"tools,omitempty"`
|
||||
Skills []preservedSkill `json:"skills,omitempty"`
|
||||
ProjectInstructions []preservedInstruction `json:"project_instructions,omitempty"`
|
||||
}
|
||||
|
||||
type preservedEdit struct {
|
||||
Path string `json:"path"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
type preservedTool struct {
|
||||
Name string `json:"name"`
|
||||
Body string `json:"body"`
|
||||
@@ -347,6 +474,12 @@ func appendPreservedState(summary string, middle []zeroruntime.Message) string {
|
||||
plan = priorState.Plan
|
||||
}
|
||||
|
||||
// Recent edits: merge edits preserved earlier with fresh write_file/edit_file
|
||||
// results in middle (newer note per path wins AND moves to the newest
|
||||
// position), so the tail cap keeps the files the model most recently touched
|
||||
// across repeated compactions rather than dropping a just-re-edited file.
|
||||
edits := capRecentEdits(mergeRecentEdits(preservedEditsToEntries(priorState.RecentEdits), recentEdits(middle)))
|
||||
|
||||
// Tools: preserve deferred tool_search schemas from the transcript. Fresh
|
||||
// loads override older carried copies by name.
|
||||
tools := mergeSkillEntries(preservedToolsToEntries(priorState.Tools), loadedToolSchemas(middle))
|
||||
@@ -360,19 +493,76 @@ func appendPreservedState(summary string, middle []zeroruntime.Message) string {
|
||||
projectInstructionEntries(middle),
|
||||
)
|
||||
|
||||
if block := formatPreservedState(plan, tools, skills, instructions); block != "" {
|
||||
if block := formatPreservedState(plan, edits, tools, skills, instructions); block != "" {
|
||||
summary += "\n\n" + block
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
// mergeRecentEdits overlays fresh edits onto edits preserved by an earlier
|
||||
// compaction. Unlike mergeSkillEntries (which keeps refreshed entries in their
|
||||
// original slot), a path touched again by a fresh edit MOVES to the newest
|
||||
// position and takes the fresh note, so capRecentEdits — which keeps only the
|
||||
// most-recent tail — never drops a file the model just re-edited. Paths present
|
||||
// only in the older set keep their relative order, ahead of the fresh ones.
|
||||
func mergeRecentEdits(older, newer []skillEntry) []skillEntry {
|
||||
if len(newer) == 0 {
|
||||
return older
|
||||
}
|
||||
freshBody := make(map[string]string, len(newer))
|
||||
freshOrder := make([]string, 0, len(newer))
|
||||
for _, e := range newer {
|
||||
if _, ok := freshBody[e.name]; !ok {
|
||||
freshOrder = append(freshOrder, e.name)
|
||||
}
|
||||
freshBody[e.name] = e.body
|
||||
}
|
||||
merged := make([]skillEntry, 0, len(older)+len(freshOrder))
|
||||
// Older entries that were NOT re-edited keep their position, oldest first.
|
||||
for _, e := range older {
|
||||
if _, refreshed := freshBody[e.name]; refreshed {
|
||||
continue // re-appended at its newest position below
|
||||
}
|
||||
merged = append(merged, e)
|
||||
}
|
||||
// Fresh edits follow, in last-seen order, each carrying the fresh note — so
|
||||
// the most recently touched files sit at the tail the cap preserves.
|
||||
for _, name := range freshOrder {
|
||||
merged = append(merged, skillEntry{name: name, body: freshBody[name]})
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
// capRecentEdits bounds the preserved edit list to the most recent maxRecentEdits
|
||||
// entries after a merge, so repeated compactions can't grow it without limit.
|
||||
func capRecentEdits(entries []skillEntry) []skillEntry {
|
||||
if len(entries) > maxRecentEdits {
|
||||
return entries[len(entries)-maxRecentEdits:]
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func preservedEditsToEntries(edits []preservedEdit) []skillEntry {
|
||||
entries := make([]skillEntry, 0, len(edits))
|
||||
for _, e := range edits {
|
||||
if e.Path == "" {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, skillEntry{name: e.Path, body: e.Note})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// formatPreservedState renders state as the labelled, single-line
|
||||
// JSON block. Returns "" when there is nothing to preserve.
|
||||
func formatPreservedState(plan string, tools, skills, instructions []skillEntry) string {
|
||||
if plan == "" && len(tools) == 0 && len(skills) == 0 && len(instructions) == 0 {
|
||||
func formatPreservedState(plan string, edits, tools, skills, instructions []skillEntry) string {
|
||||
if plan == "" && len(edits) == 0 && len(tools) == 0 && len(skills) == 0 && len(instructions) == 0 {
|
||||
return ""
|
||||
}
|
||||
state := preservedState{Plan: plan}
|
||||
for _, e := range edits {
|
||||
state.RecentEdits = append(state.RecentEdits, preservedEdit{Path: e.name, Note: e.body})
|
||||
}
|
||||
for _, t := range tools {
|
||||
state.Tools = append(state.Tools, preservedTool{Name: t.name, Body: t.body})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
// recentEdits extracts each mutated file's path and a one-line note from the
|
||||
// matching tool result, latest note per path in last-seen order.
|
||||
func TestRecentEditsExtractsPathsAndNotes(t *testing.T) {
|
||||
messages := []zeroruntime.Message{
|
||||
{Role: zeroruntime.MessageRoleAssistant, ToolCalls: []zeroruntime.ToolCall{
|
||||
{ID: "e1", Name: "write_file", Arguments: `{"path":"internal/foo.go","content":"package foo"}`},
|
||||
}},
|
||||
{Role: zeroruntime.MessageRoleTool, ToolCallID: "e1", Content: "Wrote internal/foo.go (12 lines)"},
|
||||
{Role: zeroruntime.MessageRoleAssistant, ToolCalls: []zeroruntime.ToolCall{
|
||||
{ID: "e2", Name: "edit_file", Arguments: `{"path":"internal/bar.go","old_string":"a","new_string":"b"}`},
|
||||
}},
|
||||
{Role: zeroruntime.MessageRoleTool, ToolCallID: "e2", Content: "Applied edit to internal/bar.go"},
|
||||
}
|
||||
|
||||
edits := recentEdits(messages)
|
||||
if len(edits) != 2 {
|
||||
t.Fatalf("expected 2 edited files, got %d: %#v", len(edits), edits)
|
||||
}
|
||||
if edits[0].name != "internal/foo.go" || !strings.Contains(edits[0].body, "12 lines") {
|
||||
t.Fatalf("first edit = %#v, want foo.go with its note", edits[0])
|
||||
}
|
||||
if edits[1].name != "internal/bar.go" || !strings.Contains(edits[1].body, "Applied edit") {
|
||||
t.Fatalf("second edit = %#v, want bar.go with its note", edits[1])
|
||||
}
|
||||
}
|
||||
|
||||
// After compaction elides the editing turns, the preserved-state block still
|
||||
// names the edited files and what changed, so the model needn't re-read them.
|
||||
func TestCompactionPreservesRecentEdits(t *testing.T) {
|
||||
messages := []zeroruntime.Message{
|
||||
{Role: zeroruntime.MessageRoleSystem, Content: "system"},
|
||||
{Role: zeroruntime.MessageRoleUser, Content: "add a flag"},
|
||||
{Role: zeroruntime.MessageRoleAssistant, Content: "editing", ToolCalls: []zeroruntime.ToolCall{
|
||||
{ID: "e1", Name: "write_file", Arguments: `{"path":"cmd/main.go","content":"..."}`},
|
||||
}},
|
||||
{Role: zeroruntime.MessageRoleTool, ToolCallID: "e1", Content: "Wrote cmd/main.go (adds --version flag)"},
|
||||
{Role: zeroruntime.MessageRoleAssistant, Content: "done"},
|
||||
{Role: zeroruntime.MessageRoleUser, Content: "continue"},
|
||||
{Role: zeroruntime.MessageRoleAssistant, Content: "continuing"},
|
||||
}
|
||||
summary := compactStateConversation(t, messages)
|
||||
|
||||
state := parsePreservedStateBlock(summary)
|
||||
if len(state.RecentEdits) != 1 {
|
||||
t.Fatalf("expected 1 preserved edit, got %#v", state.RecentEdits)
|
||||
}
|
||||
if state.RecentEdits[0].Path != "cmd/main.go" || !strings.Contains(state.RecentEdits[0].Note, "--version") {
|
||||
t.Fatalf("preserved edit = %#v, want cmd/main.go + its note", state.RecentEdits[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A fresh edit note for a path overrides the one carried from an earlier
|
||||
// compaction (newer wins) and moves the path to the newest position rather than
|
||||
// duplicating it or leaving it pinned to its old slot.
|
||||
func TestRecentEditsMergeNewerWins(t *testing.T) {
|
||||
prior := preservedState{RecentEdits: []preservedEdit{{Path: "a.go", Note: "old note"}, {Path: "z.go", Note: "z"}}}
|
||||
older := preservedEditsToEntries(prior.RecentEdits)
|
||||
newer := []skillEntry{{name: "a.go", body: "new note"}, {name: "b.go", body: "added"}}
|
||||
|
||||
merged := mergeRecentEdits(older, newer)
|
||||
// z.go (untouched) keeps its lead; a.go is re-edited so it takes the new note
|
||||
// and moves behind z.go, ahead of the brand-new b.go: [z.go, a.go, b.go].
|
||||
if len(merged) != 3 {
|
||||
t.Fatalf("expected z.go + refreshed a.go + b.go, got %#v", merged)
|
||||
}
|
||||
if merged[0].name != "z.go" {
|
||||
t.Fatalf("untouched z.go should stay first, got %#v", merged)
|
||||
}
|
||||
if merged[1].name != "a.go" || merged[1].body != "new note" {
|
||||
t.Fatalf("re-edited a.go should move to the newest edits with the new note, got %#v", merged[1])
|
||||
}
|
||||
if merged[2].name != "b.go" {
|
||||
t.Fatalf("brand-new b.go should be last, got %#v", merged[2])
|
||||
}
|
||||
}
|
||||
|
||||
// Regression: once more than maxRecentEdits distinct files are tracked, re-editing
|
||||
// an early file must move it into the capped tail rather than leaving it to be
|
||||
// dropped — otherwise the next compaction omits exactly the file the model most
|
||||
// recently touched.
|
||||
func TestRecentEditsCapKeepsReeditedFile(t *testing.T) {
|
||||
// Earlier compaction preserved exactly maxRecentEdits files, f0 the oldest.
|
||||
older := make([]skillEntry, 0, maxRecentEdits)
|
||||
for i := 0; i < maxRecentEdits; i++ {
|
||||
older = append(older, skillEntry{name: fmt.Sprintf("f%d.go", i), body: "old"})
|
||||
}
|
||||
// Fresh window re-edits the OLDEST file (f0) and adds a brand-new file, so the
|
||||
// merged list exceeds the cap and something must be dropped.
|
||||
newer := []skillEntry{{name: "f0.go", body: "re-edited"}, {name: "fresh.go", body: "new"}}
|
||||
|
||||
merged := capRecentEdits(mergeRecentEdits(older, newer))
|
||||
if len(merged) != maxRecentEdits {
|
||||
t.Fatalf("cap should keep %d entries, got %d", maxRecentEdits, len(merged))
|
||||
}
|
||||
|
||||
var f0 *skillEntry
|
||||
for i := range merged {
|
||||
if merged[i].name == "f0.go" {
|
||||
f0 = &merged[i]
|
||||
}
|
||||
if merged[i].name == "f1.go" {
|
||||
t.Fatalf("f1.go (now the least-recently touched) should have been dropped, got %#v", merged)
|
||||
}
|
||||
}
|
||||
if f0 == nil {
|
||||
t.Fatalf("re-edited f0.go must survive the cap, got %#v", merged)
|
||||
}
|
||||
if f0.body != "re-edited" {
|
||||
t.Fatalf("surviving f0.go should carry the fresh note, got %q", f0.body)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package agent
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/lsp"
|
||||
@@ -18,9 +20,12 @@ const fileDiagnosticsTimeout = 10 * time.Second
|
||||
// checks it against the file's language server, and formats error-severity
|
||||
// diagnostics for the model. Warnings and hints are excluded — nagging about
|
||||
// style on every edit is noise, while a type error the edit just introduced is
|
||||
// exactly what the model should see before its next step. Returns nil when
|
||||
// manager is nil, disabling inline diagnostics entirely.
|
||||
func NewFileDiagnostics(manager *lsp.Manager) func(context.Context, string) string {
|
||||
// exactly what the model should see before its next step. Diagnostics are
|
||||
// rendered with workspace-relative paths: the absolute path would put the
|
||||
// local username/home directory into the model prompt and session transcript
|
||||
// on every edit. Returns nil when manager is nil, disabling inline diagnostics
|
||||
// entirely.
|
||||
func NewFileDiagnostics(manager *lsp.Manager, workspaceRoot string) func(context.Context, string) string {
|
||||
if manager == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -39,6 +44,19 @@ func NewFileDiagnostics(manager *lsp.Manager) func(context.Context, string) stri
|
||||
if len(errors) == 0 {
|
||||
return ""
|
||||
}
|
||||
return lsp.FormatDiagnostics(absPath, errors)
|
||||
return lsp.FormatDiagnostics(diagnosticsDisplayPath(workspaceRoot, absPath), errors)
|
||||
}
|
||||
}
|
||||
|
||||
// diagnosticsDisplayPath renders absPath relative to the workspace root for
|
||||
// model-facing output, falling back to the file's base name when the path is
|
||||
// outside the workspace (a bare name still identifies the file without
|
||||
// exposing the directory layout).
|
||||
func diagnosticsDisplayPath(workspaceRoot, absPath string) string {
|
||||
if workspaceRoot != "" {
|
||||
if rel, err := filepath.Rel(workspaceRoot, absPath); err == nil && !strings.HasPrefix(rel, "..") {
|
||||
return rel
|
||||
}
|
||||
}
|
||||
return filepath.Base(absPath)
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Diagnostics are model-facing: absolute paths would leak the local username
|
||||
// and directory layout into the prompt and session transcript on every edit.
|
||||
func TestDiagnosticsDisplayPath(t *testing.T) {
|
||||
root := filepath.Join("/Users", "someone", "project")
|
||||
cases := []struct {
|
||||
root, abs, want string
|
||||
}{
|
||||
{root, filepath.Join(root, "internal", "a.go"), filepath.Join("internal", "a.go")},
|
||||
{root, filepath.Join("/etc", "other.go"), "other.go"}, // outside root -> base name only
|
||||
{"", filepath.Join("/home", "user", "x.go"), "x.go"}, // no root -> base name only
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := diagnosticsDisplayPath(c.root, c.abs); got != c.want {
|
||||
t.Errorf("diagnosticsDisplayPath(%q, %q) = %q, want %q", c.root, c.abs, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,12 @@ const (
|
||||
// calls have executed since the last update_plan call.
|
||||
staleToolCallThreshold = 10
|
||||
|
||||
// stalePlanTurnThreshold injects the same one-shot reminder once this many
|
||||
// turns have passed since the last update_plan while plan items are still
|
||||
// pending — the turn-based complement to staleToolCallThreshold, catching a
|
||||
// plan that drifts stale across many turns that each make few tool calls.
|
||||
stalePlanTurnThreshold = 8
|
||||
|
||||
// toolOnlyProgressReminderAt injects a one-shot progress nudge after this
|
||||
// many consecutive turns contain tool calls but no visible assistant text.
|
||||
// It does not stop the run; it tells the model to synthesize what it already
|
||||
@@ -43,8 +49,12 @@ const (
|
||||
toolFailureHintAt = 2
|
||||
// toolFailureStopAt halts the run after a tool fails this many times in a row
|
||||
// with the same error, so NO model (weak or strong) burns turns looping on a
|
||||
// bad call.
|
||||
toolFailureStopAt = 4
|
||||
// bad call. Set to 6 (not 4): a corrective hint fires at toolFailureHintAt (2),
|
||||
// and a model iterating on a genuinely tricky edit can legitimately fail a few
|
||||
// times after the hint while converging — stopping at 4 cut those runs short.
|
||||
// The streak still resets the moment the tool succeeds or hits a different
|
||||
// error, so this only affects true same-error loops.
|
||||
toolFailureStopAt = 6
|
||||
|
||||
// maxContinueNudges bounds how many times the headless completion gate
|
||||
// (Options.RequireCompletionSignal) re-prompts a model that stopped without a
|
||||
@@ -354,8 +364,13 @@ type guardState struct {
|
||||
emptyTurns int
|
||||
totalToolCalls int
|
||||
toolCallsSincePlanUpdate int
|
||||
planEverCalled bool
|
||||
notCalledReminderSent bool
|
||||
// turnsSincePlanUpdate counts turns (not individual tool calls) since the last
|
||||
// update_plan, so a plan that goes stale across many low-tool-call turns is
|
||||
// still caught — the tool-call counter alone can take many turns to trip when
|
||||
// the model makes only one call per turn.
|
||||
turnsSincePlanUpdate int
|
||||
planEverCalled bool
|
||||
notCalledReminderSent bool
|
||||
// staleReminderSent records whether the stale reminder has already fired for
|
||||
// the current stale interval. It is cleared when a plan update opens a new
|
||||
// interval, making the reminder one-shot per interval rather than per turn.
|
||||
@@ -414,8 +429,9 @@ func (state *guardState) observeToolResult(name string, failed bool, output stri
|
||||
func (state *guardState) observeTurn(collected zeroruntime.CollectedStream) (stop bool) {
|
||||
hasToolCalls := len(collected.ToolCalls) > 0
|
||||
hasVisibleText := strings.TrimSpace(collected.Text) != ""
|
||||
hasReasoning := collected.HasReasoning || len(collected.ReasoningBlocks) > 0
|
||||
|
||||
if hasToolCalls || hasVisibleText {
|
||||
if hasToolCalls || hasVisibleText || hasReasoning {
|
||||
state.emptyTurns = 0
|
||||
} else {
|
||||
state.emptyTurns++
|
||||
@@ -427,11 +443,16 @@ func (state *guardState) observeTurn(collected zeroruntime.CollectedStream) (sto
|
||||
state.toolOnlyReminderSent = false
|
||||
}
|
||||
|
||||
// One turn has passed; the plan-update below resets this to 0 when the model
|
||||
// refreshes the plan this turn.
|
||||
state.turnsSincePlanUpdate++
|
||||
|
||||
for _, call := range collected.ToolCalls {
|
||||
state.totalToolCalls++
|
||||
if call.Name == planToolName {
|
||||
state.planEverCalled = true
|
||||
state.toolCallsSincePlanUpdate = 0
|
||||
state.turnsSincePlanUpdate = 0
|
||||
// A fresh plan update opens a new stale interval.
|
||||
state.staleReminderSent = false
|
||||
// Record how many items remain so the completion gate knows whether
|
||||
@@ -485,10 +506,12 @@ func (state *guardState) progressReminder() string {
|
||||
// number of turns completed so far).
|
||||
func (state *guardState) planReminder(turn int) string {
|
||||
// STALE reminder takes priority: a long run without a plan update is the
|
||||
// stronger signal. One-shot per stale interval.
|
||||
if state.planEverCalled &&
|
||||
!state.staleReminderSent &&
|
||||
state.toolCallsSincePlanUpdate >= staleToolCallThreshold {
|
||||
// stronger signal. Fires on either the tool-call streak OR a turn streak with
|
||||
// pending items (so a plan drifting stale across many low-call turns is caught,
|
||||
// while a fully-completed plan is left alone). One-shot per stale interval.
|
||||
if state.planEverCalled && !state.staleReminderSent &&
|
||||
(state.toolCallsSincePlanUpdate >= staleToolCallThreshold ||
|
||||
(state.turnsSincePlanUpdate >= stalePlanTurnThreshold && state.planItemsPending > 0)) {
|
||||
state.staleReminderSent = true
|
||||
return planStaleReminder(state.toolCallsSincePlanUpdate)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,14 @@ func textTurn(content string) []zeroruntime.StreamEvent {
|
||||
}
|
||||
}
|
||||
|
||||
// reasoningTurn produces live reasoning without visible assistant text.
|
||||
func reasoningTurn(content string) []zeroruntime.StreamEvent {
|
||||
return []zeroruntime.StreamEvent{
|
||||
{Type: zeroruntime.StreamEventReasoning, Content: content},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
}
|
||||
}
|
||||
|
||||
// toolTurn produces a turn that calls a named tool with the given args JSON.
|
||||
func toolTurn(callID string, toolName string, args string) []zeroruntime.StreamEvent {
|
||||
return []zeroruntime.StreamEvent{
|
||||
@@ -101,6 +109,31 @@ func TestRunResetsEmptyTurnCounterOnVisibleOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResetsEmptyTurnCounterOnReasoning(t *testing.T) {
|
||||
provider := &mockProvider{
|
||||
turns: [][]zeroruntime.StreamEvent{
|
||||
reasoningTurn("thinking 1"),
|
||||
reasoningTurn("thinking 2"),
|
||||
reasoningTurn("thinking 3"),
|
||||
textTurn("done"),
|
||||
},
|
||||
}
|
||||
|
||||
result, err := Run(context.Background(), "go", provider, Options{
|
||||
Registry: tools.NewRegistry(),
|
||||
MaxTurns: 12,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.FinalAnswer != "done" {
|
||||
t.Fatalf("expected reasoning-only turns to keep the run live until final answer, got %q", result.FinalAnswer)
|
||||
}
|
||||
if len(provider.requests) != 4 {
|
||||
t.Fatalf("expected 4 turns, got %d", len(provider.requests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResetsEmptyTurnCounterOnToolCall(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeAgentTestFile(t, root+"/notes.txt", "alpha")
|
||||
|
||||
+171
-15
@@ -25,7 +25,12 @@ const maxTurnsFinalAnswerPrompt = "You have reached the tool-turn limit. Do not
|
||||
// WITH NO OUTPUT yet is re-issued on a fresh connection before giving up. Only
|
||||
// the no-output case is retried (a partial turn would duplicate), so this is a
|
||||
// safe recovery for a stalled/dead pooled connection.
|
||||
const maxStreamStallRetries = 2
|
||||
//
|
||||
// Set to 1 (not 2): each attempt can itself idle for the full stream timeout
|
||||
// (~5min) before the stall is even detected, so 2 retries left an interactive
|
||||
// session frozen for ~15min. One retry keeps the common single-hiccup recovery
|
||||
// while bounding the worst case to ~2× the idle timeout.
|
||||
const maxStreamStallRetries = 1
|
||||
|
||||
const (
|
||||
toolResultMetaControl = "control"
|
||||
@@ -104,7 +109,7 @@ const escalationFailedNoticePrefix = "Note: could not switch to the requested mo
|
||||
// The system prompt (core coding-craft instructions + workspace context + safety
|
||||
// confirmation policy) is assembled in system_prompt.go via buildSystemPrompt.
|
||||
|
||||
func Run(ctx context.Context, prompt string, provider Provider, options Options) (Result, error) {
|
||||
func Run(ctx context.Context, prompt string, provider Provider, options Options) (result Result, err error) {
|
||||
if provider == nil {
|
||||
return Result{}, errors.New("agent provider is required")
|
||||
}
|
||||
@@ -132,6 +137,12 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
guards := newGuardState()
|
||||
compactor := newCompactionState(options)
|
||||
|
||||
// Background post-edit diagnostics: files changed by mutating tools are
|
||||
// checked off the tool-call critical path and any errors are appended as a
|
||||
// nudge before the NEXT request (see async_diagnostics.go). nil when no
|
||||
// FileDiagnostics callback is wired; every method no-ops on nil.
|
||||
postEditDiagnostics := newAsyncDiagnostics(options.FileDiagnostics, options.Cwd)
|
||||
|
||||
// loaded tracks deferred-eligible tools the model has pulled via tool_search
|
||||
// during THIS run. It is consulted by partitionTools each turn to expose a
|
||||
// loaded tool's full schema; it lives only for the run (v1 within-run scope).
|
||||
@@ -145,15 +156,35 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// has already been demanded this run, so it fires at most once.
|
||||
acceptanceRequested := false
|
||||
|
||||
result := Result{Messages: copyMessages(messages)}
|
||||
// toolDefCache memoizes each tool's rendered JSON-schema definition across
|
||||
// turns (a tool's advertised schema is stable for the run), so partitionTools
|
||||
// doesn't re-run the recursive schema→map conversion for every tool every turn.
|
||||
toolDefCache := map[string]zeroruntime.ToolDefinition{}
|
||||
|
||||
result = Result{Messages: copyMessages(messages)}
|
||||
dispatchSessionStart(ctx, options)
|
||||
defer func() {
|
||||
dispatchSessionEnd(ctx, options, result, err)
|
||||
}()
|
||||
for turn := 0; turn < maxTurns; turn++ {
|
||||
result.Turns = turn + 1
|
||||
|
||||
// Deliver background post-edit diagnostics from the previous turn's edits
|
||||
// BEFORE compaction so the nudge is part of the request being budgeted.
|
||||
// A brief wait at most (asyncDiagnosticsDrainTimeout); an unfinished check
|
||||
// simply delivers on a later turn.
|
||||
if nudge := postEditDiagnostics.drain(ctx); nudge != "" {
|
||||
messages = append(messages, zeroruntime.Message{
|
||||
Role: zeroruntime.MessageRoleUser,
|
||||
Content: nudge,
|
||||
})
|
||||
}
|
||||
|
||||
// Build the per-turn tool list first so proactive compaction can include
|
||||
// the tool-definition tokens (they ride on every request) in its estimate.
|
||||
// partitionTools depends only on registry/permissions/options/loaded, not on
|
||||
// the messages, so computing it before compaction is safe.
|
||||
exposed, _ := partitionTools(registry, permissionMode, options, loaded)
|
||||
exposed, _ := partitionToolsCached(registry, permissionMode, options, loaded, toolDefCache)
|
||||
|
||||
// PROACTIVE compaction: if the history is approaching the model's
|
||||
// context window, summarize the oldest middle before building the
|
||||
@@ -163,6 +194,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
Messages: copyMessages(messages),
|
||||
Tools: exposed,
|
||||
ReasoningEffort: options.ReasoningEffort,
|
||||
PromptCacheKey: options.SessionID,
|
||||
}
|
||||
|
||||
// Report the per-category context budget for this turn so a surface can
|
||||
@@ -198,6 +230,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
Messages: copyMessages(messages),
|
||||
Tools: exposed,
|
||||
ReasoningEffort: options.ReasoningEffort,
|
||||
PromptCacheKey: options.SessionID,
|
||||
}
|
||||
// Pre-content connect after a context-limit compaction: route through the
|
||||
// reconnect helper so a transient upstream hiccup here doesn't fail the
|
||||
@@ -258,6 +291,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
Messages: copyMessages(messages),
|
||||
Tools: exposed,
|
||||
ReasoningEffort: options.ReasoningEffort,
|
||||
PromptCacheKey: options.SessionID,
|
||||
}
|
||||
retryStream, retryStreamErr := streamWithReconnect(ctx, provider, retryRequest, reconnectNoticeFor(options))
|
||||
if retryStreamErr != nil {
|
||||
@@ -316,6 +350,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
Messages: copyMessages(messages),
|
||||
Tools: exposed,
|
||||
ReasoningEffort: options.ReasoningEffort,
|
||||
PromptCacheKey: options.SessionID,
|
||||
}
|
||||
retryStream, retryErr := streamWithReconnect(ctx, provider, retryRequest, reconnectNoticeFor(options))
|
||||
if retryErr != nil {
|
||||
@@ -340,6 +375,16 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
}
|
||||
}
|
||||
|
||||
// Calibrate the compaction token estimator against the provider's real
|
||||
// prompt-token count for the request we just sent, so later turns trigger
|
||||
// compaction near true capacity instead of ~15% early on code-heavy history.
|
||||
// Recompute the estimate HERE (not at request-build time): a reactive
|
||||
// compaction may have replaced `messages` with a smaller set and re-sent, so
|
||||
// this reflects the request that actually produced collected.Usage. The
|
||||
// assistant reply is appended below, after this, so `messages` is still the
|
||||
// sent request.
|
||||
compactor.calibrate(estimateTokens(messages)+estimateToolDefTokens(exposed), collected.Usage.InputTokens)
|
||||
|
||||
// Carry the turn's terminal stop reason so a final answer cut off at the
|
||||
// output token cap (or by a content filter) is reported as truncated. A
|
||||
// tool-call turn normalizes to "" and clears any prior reason.
|
||||
@@ -450,6 +495,19 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Finalization diagnostics gate: edits from this run may still have
|
||||
// checks in flight — the per-turn drain waits only briefly and defers
|
||||
// to "a later turn", but a final answer means there is no later turn.
|
||||
// Wait out the full inline-era budget once; an introduced error gives
|
||||
// the model one more turn to see (and fix) it instead of being lost.
|
||||
// Free for runs that never edited: an idle collector returns "".
|
||||
if nudge := postEditDiagnostics.drainFinal(ctx); nudge != "" {
|
||||
messages = append(messages, zeroruntime.Message{
|
||||
Role: zeroruntime.MessageRoleUser,
|
||||
Content: nudge,
|
||||
})
|
||||
continue
|
||||
}
|
||||
result.FinalAnswer = collected.Text
|
||||
result.Messages = copyMessages(messages)
|
||||
return result, nil
|
||||
@@ -566,11 +624,14 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
failureHint = toolFailureHint(call.Name, toolSchemaJSON(registry, call.Name), toolResult.Output)
|
||||
}
|
||||
|
||||
// Post-edit self-correction: collect the files this successful mutating
|
||||
// tool changed; verification runs once over the union after the batch.
|
||||
// A read-only tool (no ChangedFiles) never contributes.
|
||||
if options.SelfCorrect != nil && toolResult.Status == tools.StatusOK && len(toolResult.ChangedFiles) > 0 {
|
||||
// Collect the files this successful mutating tool changed. Self-correct
|
||||
// verification runs once over the union after the batch; background
|
||||
// diagnostics start NOW so the language server analyzes while the rest
|
||||
// of the batch executes. A read-only tool (no ChangedFiles) never
|
||||
// contributes.
|
||||
if toolResult.Status == tools.StatusOK && len(toolResult.ChangedFiles) > 0 {
|
||||
changedFilesThisBatch = append(changedFilesThisBatch, toolResult.ChangedFiles...)
|
||||
postEditDiagnostics.enqueue(ctx, toolResult.ChangedFiles)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -653,7 +714,19 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// mid-run, not a natural completion (a finished run returns via the no-tool-call
|
||||
// success path before this). Under the headless completion gate that is INCOMPLETE,
|
||||
// not success, so a run that loops to the turn limit isn't reported as done.
|
||||
if answer, finalMessages, finishReason := finalAnswerAfterMaxTurns(ctx, provider, messages, options); strings.TrimSpace(answer) != "" {
|
||||
//
|
||||
// The final-answer call is one more model request, so diagnostics from the
|
||||
// LAST turn's edits get a drain too — with the finalization budget, since
|
||||
// there is no later turn to defer to — otherwise an error introduced by
|
||||
// the final edit would go unreported in the summary.
|
||||
if nudge := postEditDiagnostics.drainFinal(ctx); nudge != "" {
|
||||
messages = append(messages, zeroruntime.Message{
|
||||
Role: zeroruntime.MessageRoleUser,
|
||||
Content: nudge,
|
||||
})
|
||||
}
|
||||
finalExposed, _ := partitionToolsCached(registry, permissionMode, options, loaded, toolDefCache)
|
||||
if answer, finalMessages, finishReason := finalAnswerAfterMaxTurns(ctx, provider, messages, finalExposed, options); strings.TrimSpace(answer) != "" {
|
||||
result.FinalAnswer = answer
|
||||
result.FinishReason = finishReason
|
||||
result.Messages = copyMessages(finalMessages)
|
||||
@@ -673,7 +746,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages []zeroruntime.Message, options Options) (string, []zeroruntime.Message, string) {
|
||||
func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages []zeroruntime.Message, toolDefs []zeroruntime.ToolDefinition, options Options) (string, []zeroruntime.Message, string) {
|
||||
finalMessages := copyMessages(messages)
|
||||
finalMessages = append(finalMessages, zeroruntime.Message{
|
||||
Role: zeroruntime.MessageRoleUser,
|
||||
@@ -684,7 +757,9 @@ func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages [
|
||||
// transient hiccup doesn't drop the final summary (AUDIT-L1).
|
||||
stream, err := streamWithReconnect(ctx, provider, zeroruntime.CompletionRequest{
|
||||
Messages: copyMessages(finalMessages),
|
||||
Tools: toolDefs,
|
||||
ReasoningEffort: options.ReasoningEffort,
|
||||
PromptCacheKey: options.SessionID,
|
||||
}, reconnectNoticeFor(options))
|
||||
if err != nil {
|
||||
return "", messages, ""
|
||||
@@ -1075,8 +1150,12 @@ func executeToolCall(ctx context.Context, registry *tools.Registry, call ToolCal
|
||||
// Per-session file version tracker so write_file/edit_file refuse to clobber
|
||||
// a file that changed on disk outside Zero since it was last read.
|
||||
FileTracker: options.FileTracker,
|
||||
// Inline post-edit diagnostics for mutating tools (nil = disabled).
|
||||
Diagnostics: options.FileDiagnostics,
|
||||
// Post-edit diagnostics are NOT forwarded to the tools: they used to run
|
||||
// synchronously inside edit_file/write_file and block every mutating tool
|
||||
// call on the language server (≥300ms debounce floor, 10s cap). The loop
|
||||
// now checks changed files in the background and nudges the model before
|
||||
// its next request instead (see async_diagnostics.go). The tools' inline
|
||||
// mechanism (tools.RunOptions.Diagnostics) stays for direct API callers.
|
||||
// Forward the run's operator tool filters so a filter-aware tool
|
||||
// (tool_search) never discloses or loads an operator-hidden deferred tool.
|
||||
EnabledTools: options.EnabledTools,
|
||||
@@ -1495,6 +1574,54 @@ func dispatchAfterTool(ctx context.Context, options Options, call ToolCall, args
|
||||
return strings.TrimSpace(strings.Join(outcome.Messages, "\n"))
|
||||
}
|
||||
|
||||
// dispatchSessionStart runs configured sessionStart hooks once before the first
|
||||
// model turn. Lifecycle hooks are advisory: dispatcher failures are audited but
|
||||
// never block the run.
|
||||
func dispatchSessionStart(ctx context.Context, options Options) {
|
||||
if options.Hooks == nil {
|
||||
return
|
||||
}
|
||||
options.Hooks.Dispatch(ctx, hooks.DispatchInput{
|
||||
Event: hooks.EventSessionStart,
|
||||
Payload: map[string]any{
|
||||
"event": string(hooks.EventSessionStart),
|
||||
"sessionId": options.SessionID,
|
||||
"cwd": options.Cwd,
|
||||
"provider": options.ProviderName,
|
||||
"model": options.Model,
|
||||
"depth": options.Depth,
|
||||
"tag": options.Tag,
|
||||
"sessionTitle": options.SessionTitle,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// dispatchSessionEnd runs configured sessionEnd hooks once when the agent run
|
||||
// exits, including early error returns. Lifecycle hooks are advisory.
|
||||
func dispatchSessionEnd(ctx context.Context, options Options, result Result, runErr error) {
|
||||
if options.Hooks == nil {
|
||||
return
|
||||
}
|
||||
payload := map[string]any{
|
||||
"event": string(hooks.EventSessionEnd),
|
||||
"sessionId": options.SessionID,
|
||||
"cwd": options.Cwd,
|
||||
"provider": options.ProviderName,
|
||||
"model": options.Model,
|
||||
"turns": result.Turns,
|
||||
"stopReason": string(result.StopReason),
|
||||
"finishReason": result.FinishReason,
|
||||
"incomplete": result.Incomplete,
|
||||
}
|
||||
if runErr != nil {
|
||||
payload["error"] = runErr.Error()
|
||||
}
|
||||
options.Hooks.Dispatch(ctx, hooks.DispatchInput{
|
||||
Event: hooks.EventSessionEnd,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
|
||||
// blockedByHookResult is the tool result for a call vetoed by a beforeTool hook.
|
||||
func blockedByHookResult(call ToolCall, outcome hooks.DispatchOutcome) ToolResult {
|
||||
// The hook reason (its stdout/stderr) is model-visible here, an intercepted
|
||||
@@ -2488,6 +2615,17 @@ func permissionActionFromSandbox(action sandbox.Action) PermissionAction {
|
||||
// exposed. The exposed slice is alpha-sorted by name, matching the legacy order
|
||||
// so the inactive path is stable.
|
||||
func partitionTools(registry *tools.Registry, permissionMode PermissionMode, options Options, loaded map[string]bool) ([]zeroruntime.ToolDefinition, string) {
|
||||
return partitionToolsCached(registry, permissionMode, options, loaded, nil)
|
||||
}
|
||||
|
||||
// partitionToolsCached is partitionTools with an optional per-tool definition
|
||||
// cache. The partitioning itself (visibility, deferral, ordering) is recomputed
|
||||
// every call — it must be, because a tool's deferred state can flip mid-run (e.g.
|
||||
// swarm tools un-defer once a swarm is active). Only the expensive part —
|
||||
// rendering each tool's JSON-schema parameters — is memoized by tool name, since a
|
||||
// tool's advertised name/description/schema is stable for the run. defCache nil
|
||||
// disables caching (used by tests and the plain partitionTools entrypoint).
|
||||
func partitionToolsCached(registry *tools.Registry, permissionMode PermissionMode, options Options, loaded map[string]bool, defCache map[string]zeroruntime.ToolDefinition) ([]zeroruntime.ToolDefinition, string) {
|
||||
registeredTools := registry.All()
|
||||
|
||||
visible := make([]tools.Tool, 0, len(registeredTools))
|
||||
@@ -2531,7 +2669,7 @@ func partitionTools(registry *tools.Registry, permissionMode PermissionMode, opt
|
||||
if tool.Name() == tools.ToolSearchToolName {
|
||||
continue
|
||||
}
|
||||
definitions = append(definitions, runtimeToolDefinition(tool))
|
||||
definitions = append(definitions, cachedRuntimeToolDefinition(defCache, tool))
|
||||
}
|
||||
sort.Slice(definitions, func(left int, right int) bool {
|
||||
return definitions[left].Name < definitions[right].Name
|
||||
@@ -2563,13 +2701,13 @@ func partitionTools(registry *tools.Registry, permissionMode PermissionMode, opt
|
||||
}
|
||||
if tools.IsDeferred(tool) {
|
||||
if loaded[name] {
|
||||
loadedTail = append(loadedTail, runtimeToolDefinition(tool))
|
||||
loadedTail = append(loadedTail, cachedRuntimeToolDefinition(defCache, tool))
|
||||
} else {
|
||||
hiddenTools = append(hiddenTools, tool)
|
||||
}
|
||||
continue
|
||||
}
|
||||
eager = append(eager, runtimeToolDefinition(tool))
|
||||
eager = append(eager, cachedRuntimeToolDefinition(defCache, tool))
|
||||
}
|
||||
sort.Slice(eager, func(left int, right int) bool {
|
||||
return eager[left].Name < eager[right].Name
|
||||
@@ -2603,6 +2741,24 @@ func partitionTools(registry *tools.Registry, permissionMode PermissionMode, opt
|
||||
return definitions, discovery
|
||||
}
|
||||
|
||||
// cachedRuntimeToolDefinition returns the tool's rendered definition, reusing a
|
||||
// cached render when defCache holds one for this tool name. A tool's advertised
|
||||
// definition is stable across a run, so caching skips the recursive schema→map
|
||||
// conversion (schemaToRuntimeMap) that would otherwise run for every tool on every
|
||||
// turn. tool_search is excluded by its callers (its description is dynamic), so it
|
||||
// never poisons the cache. A nil cache computes fresh.
|
||||
func cachedRuntimeToolDefinition(defCache map[string]zeroruntime.ToolDefinition, tool tools.Tool) zeroruntime.ToolDefinition {
|
||||
if defCache == nil {
|
||||
return runtimeToolDefinition(tool)
|
||||
}
|
||||
if def, ok := defCache[tool.Name()]; ok {
|
||||
return def
|
||||
}
|
||||
def := runtimeToolDefinition(tool)
|
||||
defCache[tool.Name()] = def
|
||||
return def
|
||||
}
|
||||
|
||||
// runtimeToolDefinition renders a tool's advertised definition (name, description,
|
||||
// JSON-schema parameters) as sent to the provider.
|
||||
func runtimeToolDefinition(tool tools.Tool) zeroruntime.ToolDefinition {
|
||||
|
||||
@@ -4,12 +4,14 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/hooks"
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/specmode"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
@@ -37,6 +39,59 @@ func (provider *mockProvider) StreamCompletion(ctx context.Context, request zero
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func TestRunDispatchesSessionLifecycleHooks(t *testing.T) {
|
||||
audit, err := hooks.NewAuditStore(hooks.AuditStoreOptions{AuditPath: filepath.Join(t.TempDir(), "audit.jsonl")})
|
||||
if err != nil {
|
||||
t.Fatalf("NewAuditStore: %v", err)
|
||||
}
|
||||
dispatcher := hooks.NewDispatcher(hooks.DispatcherOptions{
|
||||
Config: hooks.Config{
|
||||
Enabled: true,
|
||||
Hooks: []hooks.Definition{
|
||||
{ID: "zero.session-start", Event: hooks.EventSessionStart, Command: "zero-missing-hook-command", Enabled: true},
|
||||
{ID: "zero.session-end", Event: hooks.EventSessionEnd, Command: "zero-missing-hook-command", Enabled: true},
|
||||
},
|
||||
},
|
||||
Audit: audit,
|
||||
})
|
||||
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{{
|
||||
{Type: zeroruntime.StreamEventText, Content: "done"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
}}}
|
||||
|
||||
_, err = Run(context.Background(), "hi", provider, Options{
|
||||
SessionID: "session-123",
|
||||
Cwd: t.TempDir(),
|
||||
ProviderName: "test-provider",
|
||||
Model: "test-model",
|
||||
Hooks: dispatcher,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
|
||||
events, err := audit.ReadEvents()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadEvents: %v", err)
|
||||
}
|
||||
started := map[hooks.Event]int{}
|
||||
completed := map[hooks.Event]int{}
|
||||
for _, event := range events {
|
||||
switch event.Type {
|
||||
case "hook_execution_started":
|
||||
started[event.Event]++
|
||||
case "hook_execution_completed":
|
||||
completed[event.Event]++
|
||||
}
|
||||
}
|
||||
if started[hooks.EventSessionStart] != 1 || completed[hooks.EventSessionStart] != 1 {
|
||||
t.Fatalf("sessionStart audit counts started/completed = %d/%d, events=%#v", started[hooks.EventSessionStart], completed[hooks.EventSessionStart], events)
|
||||
}
|
||||
if started[hooks.EventSessionEnd] != 1 || completed[hooks.EventSessionEnd] != 1 {
|
||||
t.Fatalf("sessionEnd audit counts started/completed = %d/%d, events=%#v", started[hooks.EventSessionEnd], completed[hooks.EventSessionEnd], events)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingWebSearchTool struct {
|
||||
calls []map[string]any
|
||||
}
|
||||
@@ -1945,6 +2000,11 @@ func TestRunCommandPrefixApprovalBypassesSandboxForMatchingShellCalls(t *testing
|
||||
|
||||
func TestRunCommandPrefixApprovalCoversSegmentedShellWithSafeTail(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
segmentedCommand := `ps aux | head -5`
|
||||
if runtime.GOOS == "windows" {
|
||||
// head is MSYS-prone on Windows (#458) and no longer counts as a known-safe tail.
|
||||
segmentedCommand = `ps aux | echo ok`
|
||||
}
|
||||
retryTool := &sandboxDeniedRetryTool{}
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(retryTool)
|
||||
@@ -1958,7 +2018,7 @@ func TestRunCommandPrefixApprovalCoversSegmentedShellWithSafeTail(t *testing.T)
|
||||
},
|
||||
{
|
||||
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "call-2", ToolName: "bash"},
|
||||
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: `{"command":"ps aux | head -5"}`},
|
||||
{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "call-2", ArgumentsFragment: fmt.Sprintf(`{"command":%q}`, segmentedCommand)},
|
||||
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "call-2"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
@@ -2762,11 +2822,11 @@ func TestRunRequestsFinalAnswerAfterMaxTurns(t *testing.T) {
|
||||
t.Fatalf("expected final answer from finalization turn, got %q", result.FinalAnswer)
|
||||
}
|
||||
if len(provider.requests) != 2 {
|
||||
t.Fatalf("expected final no-tools request after max turns, got %d requests", len(provider.requests))
|
||||
t.Fatalf("expected finalization request after max turns, got %d requests", len(provider.requests))
|
||||
}
|
||||
finalRequest := provider.requests[1]
|
||||
if len(finalRequest.Tools) != 0 {
|
||||
t.Fatalf("finalization request must not advertise tools, got %#v", finalRequest.Tools)
|
||||
if toolDefinitionByName(finalRequest.Tools, "read_file") == nil {
|
||||
t.Fatalf("finalization request must keep prior tools exposed for strict provider replay, got %#v", finalRequest.Tools)
|
||||
}
|
||||
lastMessage := finalRequest.Messages[len(finalRequest.Messages)-1]
|
||||
if lastMessage.Role != zeroruntime.MessageRoleUser || !strings.Contains(lastMessage.Content, "tool-turn limit") {
|
||||
@@ -2907,8 +2967,10 @@ func TestBuildSystemPromptInjectsHostShellContext(t *testing.T) {
|
||||
t.Fatalf("expected operating system in environment block, got %q", prompt)
|
||||
}
|
||||
if runtime.GOOS == "windows" {
|
||||
if !strings.Contains(prompt, "Windows cmd.exe syntax") || !strings.Contains(prompt, "cwd argument") {
|
||||
t.Fatalf("expected Windows cmd.exe shell guidance in prompt, got %q", prompt)
|
||||
for _, want := range []string{"Windows cmd.exe syntax", "cwd argument", "MSYS binaries", "grep", "require_escalated"} {
|
||||
if !strings.Contains(prompt, want) {
|
||||
t.Fatalf("expected Windows shell guidance to mention %q, got %q", want, prompt)
|
||||
}
|
||||
}
|
||||
} else if !strings.Contains(prompt, "/bin/sh syntax") {
|
||||
t.Fatalf("expected POSIX shell guidance in prompt, got %q", prompt)
|
||||
|
||||
@@ -51,21 +51,33 @@ func parallelSafeToolCall(registry *tools.Registry, call ToolCall, options Optio
|
||||
}
|
||||
|
||||
// executeParallelReadBatch runs calls[start:end] concurrently (bounded by
|
||||
// maxParallelReadTools) and returns results indexed relative to start. Any
|
||||
// permission prompt that still occurs (a sandbox preflight can demand one even
|
||||
// for an auto-allowed read) is serialized so an interactive front-end never
|
||||
// sees two prompts at once.
|
||||
// maxParallelReadTools) and returns results indexed relative to start. All
|
||||
// execution-side callbacks that can fire inside executeToolCall are serialized
|
||||
// behind one mutex: a permission prompt (a sandbox preflight can demand one
|
||||
// even for an auto-allowed read) must never appear twice at once on an
|
||||
// interactive front-end, and OnPermission event handlers append to shared
|
||||
// session-recording state without their own locking — two batched reads under
|
||||
// a granted extra root would otherwise race (the pre-batch serial loop never
|
||||
// had two callbacks in flight at once).
|
||||
func executeParallelReadBatch(ctx context.Context, registry *tools.Registry, calls []ToolCall, start, end int, permissionMode PermissionMode, options Options) []precomputedToolResult {
|
||||
batchOptions := options
|
||||
var callbackMutex sync.Mutex
|
||||
if options.OnPermissionRequest != nil {
|
||||
var promptMutex sync.Mutex
|
||||
inner := options.OnPermissionRequest
|
||||
batchOptions.OnPermissionRequest = func(ctx context.Context, request PermissionRequest) (PermissionDecision, error) {
|
||||
promptMutex.Lock()
|
||||
defer promptMutex.Unlock()
|
||||
callbackMutex.Lock()
|
||||
defer callbackMutex.Unlock()
|
||||
return inner(ctx, request)
|
||||
}
|
||||
}
|
||||
if options.OnPermission != nil {
|
||||
inner := options.OnPermission
|
||||
batchOptions.OnPermission = func(event PermissionEvent) {
|
||||
callbackMutex.Lock()
|
||||
defer callbackMutex.Unlock()
|
||||
inner(event)
|
||||
}
|
||||
}
|
||||
|
||||
results := make([]precomputedToolResult, end-start)
|
||||
semaphore := make(chan struct{}, maxParallelReadTools)
|
||||
|
||||
@@ -17,6 +17,9 @@ type probeTool struct {
|
||||
name string
|
||||
sideEffect tools.SideEffect
|
||||
delay time.Duration
|
||||
// shared, when set, receives this probe's start/end entries too, so a test
|
||||
// can observe ordering ACROSS probes (e.g. reads vs a write barrier).
|
||||
shared *probeLog
|
||||
|
||||
mu sync.Mutex
|
||||
active int
|
||||
@@ -24,6 +27,24 @@ type probeTool struct {
|
||||
log []string
|
||||
}
|
||||
|
||||
// probeLog is a mutex-guarded event log shared between probes.
|
||||
type probeLog struct {
|
||||
mu sync.Mutex
|
||||
entries []string
|
||||
}
|
||||
|
||||
func (log *probeLog) append(entry string) {
|
||||
log.mu.Lock()
|
||||
defer log.mu.Unlock()
|
||||
log.entries = append(log.entries, entry)
|
||||
}
|
||||
|
||||
func (log *probeLog) snapshot() []string {
|
||||
log.mu.Lock()
|
||||
defer log.mu.Unlock()
|
||||
return append([]string(nil), log.entries...)
|
||||
}
|
||||
|
||||
func (tool *probeTool) Name() string { return tool.name }
|
||||
func (tool *probeTool) Description() string { return "test probe tool" }
|
||||
func (tool *probeTool) Parameters() tools.Schema {
|
||||
@@ -45,7 +66,13 @@ func (tool *probeTool) Run(_ context.Context, args map[string]any) tools.Result
|
||||
}
|
||||
tool.log = append(tool.log, "start:"+id)
|
||||
tool.mu.Unlock()
|
||||
if tool.shared != nil {
|
||||
tool.shared.append("start:" + id)
|
||||
}
|
||||
time.Sleep(tool.delay)
|
||||
if tool.shared != nil {
|
||||
tool.shared.append("end:" + id)
|
||||
}
|
||||
tool.mu.Lock()
|
||||
tool.active--
|
||||
tool.log = append(tool.log, "end:"+id)
|
||||
@@ -127,10 +154,9 @@ func TestRunExecutesConsecutiveReadsConcurrently(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunParallelReadsNeverSpanMutatingCall(t *testing.T) {
|
||||
read := &probeTool{name: "probe_read", sideEffect: tools.SideEffectRead, delay: 30 * time.Millisecond}
|
||||
write := &probeTool{name: "probe_write", sideEffect: tools.SideEffectWrite}
|
||||
// Shared log across both probes so relative ordering is observable.
|
||||
write.mu = sync.Mutex{}
|
||||
shared := &probeLog{}
|
||||
read := &probeTool{name: "probe_read", sideEffect: tools.SideEffectRead, delay: 30 * time.Millisecond, shared: shared}
|
||||
write := &probeTool{name: "probe_write", sideEffect: tools.SideEffectWrite, shared: shared}
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(read)
|
||||
registry.Register(write)
|
||||
@@ -151,32 +177,26 @@ func TestRunParallelReadsNeverSpanMutatingCall(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The write must start only after r1+r2 finished, and r3/r4 must start only
|
||||
// after the write finished: batches never cross a mutating call.
|
||||
index := func(log []string, entry string) int {
|
||||
|
||||
// Cross-probe ordering on the SHARED log: the write must start only after
|
||||
// both first-batch reads finished, and both second-batch reads must start
|
||||
// only after the write finished — batches never cross a mutating call.
|
||||
log := shared.snapshot()
|
||||
index := func(entry string) int {
|
||||
for i, e := range log {
|
||||
if e == entry {
|
||||
return i
|
||||
}
|
||||
}
|
||||
t.Fatalf("entry %q missing from shared log: %v", entry, log)
|
||||
return -1
|
||||
}
|
||||
readLog := func() []string { read.mu.Lock(); defer read.mu.Unlock(); return append([]string(nil), read.log...) }()
|
||||
writeLog := func() []string { write.mu.Lock(); defer write.mu.Unlock(); return append([]string(nil), write.log...) }()
|
||||
if index(writeLog, "start:w") == -1 {
|
||||
t.Fatalf("write probe never ran: %v", writeLog)
|
||||
writeStart, writeEnd := index("start:w"), index("end:w")
|
||||
if firstBatchMaxEnd := max(index("end:r1"), index("end:r2")); writeStart < firstBatchMaxEnd {
|
||||
t.Fatalf("write started before the first read batch finished: %v", log)
|
||||
}
|
||||
for _, entry := range []string{"end:r1", "end:r2"} {
|
||||
if index(readLog, entry) == -1 {
|
||||
t.Fatalf("first read batch incomplete: %v", readLog)
|
||||
}
|
||||
}
|
||||
// r3/r4 must appear strictly after r1/r2 in the read log (the write barrier
|
||||
// between the two batches forces full separation).
|
||||
firstBatchMaxEnd := max(index(readLog, "end:r1"), index(readLog, "end:r2"))
|
||||
secondBatchMinStart := min(index(readLog, "start:r3"), index(readLog, "start:r4"))
|
||||
if secondBatchMinStart < firstBatchMaxEnd {
|
||||
t.Fatalf("second read batch started before first batch ended: %v", readLog)
|
||||
if secondBatchMinStart := min(index("start:r3"), index("start:r4")); secondBatchMinStart < writeEnd {
|
||||
t.Fatalf("second read batch started before the write finished: %v", log)
|
||||
}
|
||||
if read.maxActive < 2 {
|
||||
t.Fatalf("reads within a batch must overlap, max concurrency was %d", read.maxActive)
|
||||
|
||||
@@ -34,7 +34,7 @@ func TestPartitionToolsActiveAppendsLoadedToolAfterEagerBlock(t *testing.T) {
|
||||
t.Fatalf("expected read_file, tool_search and loaded alpha all exposed, got %#v", exposed)
|
||||
}
|
||||
// The eager tools must precede the loaded deferred tool (appended, not inserted).
|
||||
if !(readPos < alphaPos && searchPos < alphaPos) {
|
||||
if readPos >= alphaPos || searchPos >= alphaPos {
|
||||
t.Fatalf("loaded deferred tool must be appended after the eager block; got read=%d search=%d alpha=%d", readPos, searchPos, alphaPos)
|
||||
}
|
||||
// beta was never loaded — it stays hidden.
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
// countingSchemaTool records how many times its Parameters() schema is read, so a
|
||||
// test can prove the definition cache avoids re-rendering across turns.
|
||||
type countingSchemaTool struct {
|
||||
name string
|
||||
calls *int
|
||||
}
|
||||
|
||||
func (t countingSchemaTool) Name() string { return t.name }
|
||||
func (t countingSchemaTool) Description() string { return "counts schema reads" }
|
||||
func (t countingSchemaTool) Parameters() tools.Schema {
|
||||
*t.calls++
|
||||
return tools.Schema{Type: "object", AdditionalProperties: false, Properties: map[string]tools.PropertySchema{
|
||||
"x": {Type: "string"},
|
||||
}}
|
||||
}
|
||||
func (t countingSchemaTool) Safety() tools.Safety {
|
||||
return tools.Safety{SideEffect: tools.SideEffectRead, Permission: tools.PermissionAllow}
|
||||
}
|
||||
func (t countingSchemaTool) Run(_ context.Context, _ map[string]any) tools.Result {
|
||||
return tools.Result{Status: tools.StatusOK}
|
||||
}
|
||||
|
||||
// The cache renders a tool's schema once and reuses it across calls, and its
|
||||
// output is identical to the uncached path.
|
||||
func TestPartitionToolsCacheRendersOnceAndMatchesUncached(t *testing.T) {
|
||||
calls := 0
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(countingSchemaTool{name: "alpha", calls: &calls})
|
||||
registry.Register(countingSchemaTool{name: "beta", calls: &calls})
|
||||
|
||||
uncached, _ := partitionTools(registry, PermissionModeAuto, Options{}, map[string]bool{})
|
||||
|
||||
base := calls
|
||||
cache := map[string]zeroruntime.ToolDefinition{}
|
||||
first, _ := partitionToolsCached(registry, PermissionModeAuto, Options{}, map[string]bool{}, cache)
|
||||
rendersAfterFirst := calls - base
|
||||
second, _ := partitionToolsCached(registry, PermissionModeAuto, Options{}, map[string]bool{}, cache)
|
||||
rendersAfterSecond := calls - base
|
||||
|
||||
if rendersAfterFirst != 2 {
|
||||
t.Fatalf("first cached call should render 2 tools once each, rendered %d times", rendersAfterFirst)
|
||||
}
|
||||
if rendersAfterSecond != 2 {
|
||||
t.Fatalf("second cached call must reuse the cache (no new schema reads), total renders = %d", rendersAfterSecond)
|
||||
}
|
||||
if !reflect.DeepEqual(first, second) {
|
||||
t.Fatal("cached calls must return identical definitions")
|
||||
}
|
||||
if !reflect.DeepEqual(first, uncached) {
|
||||
t.Fatal("cached output must match the uncached partitionTools output")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
func planUpdateTurn(status string) zeroruntime.CollectedStream {
|
||||
return zeroruntime.CollectedStream{ToolCalls: []zeroruntime.ToolCall{
|
||||
{Name: "update_plan", Arguments: `{"plan":[{"content":"step one","status":"` + status + `"}]}`},
|
||||
}}
|
||||
}
|
||||
|
||||
// Turn-based staleness: a plan that goes many turns without an update — while
|
||||
// items are still pending — draws the stale reminder even though few tool calls
|
||||
// have run (the tool-call streak alone would not have tripped).
|
||||
func TestPlanReminderFiresOnTurnStaleness(t *testing.T) {
|
||||
state := newGuardState()
|
||||
state.observeTurn(planUpdateTurn("in_progress"))
|
||||
|
||||
// Text-only turns advance the turn counter without adding tool calls.
|
||||
for range stalePlanTurnThreshold {
|
||||
state.observeTurn(zeroruntime.CollectedStream{Text: "still working…"})
|
||||
}
|
||||
|
||||
if state.toolCallsSincePlanUpdate >= staleToolCallThreshold {
|
||||
t.Fatalf("precondition: the tool-call trigger must not fire (%d calls)", state.toolCallsSincePlanUpdate)
|
||||
}
|
||||
if got := state.planReminder(stalePlanTurnThreshold + 2); !strings.Contains(got, planStaleReminderMarker) {
|
||||
t.Fatalf("expected a turn-based stale reminder, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A fully-completed plan is NOT stale: no pending items means the turn-based
|
||||
// trigger stays quiet even after many turns, so a finished plan isn't nagged.
|
||||
func TestPlanReminderSkipsTurnStalenessWhenComplete(t *testing.T) {
|
||||
state := newGuardState()
|
||||
state.observeTurn(planUpdateTurn("completed"))
|
||||
|
||||
for range stalePlanTurnThreshold + 3 {
|
||||
state.observeTurn(zeroruntime.CollectedStream{Text: "wrapping up…"})
|
||||
}
|
||||
|
||||
if got := state.planReminder(stalePlanTurnThreshold + 5); got != "" {
|
||||
t.Fatalf("a completed plan must not draw a stale reminder, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan update resets the turn counter, so the stale reminder does not fire
|
||||
// right after a fresh update.
|
||||
func TestPlanUpdateResetsTurnStaleness(t *testing.T) {
|
||||
state := newGuardState()
|
||||
state.observeTurn(planUpdateTurn("in_progress"))
|
||||
for range stalePlanTurnThreshold - 1 {
|
||||
state.observeTurn(zeroruntime.CollectedStream{Text: "working…"})
|
||||
}
|
||||
// Refresh the plan — resets the turn streak.
|
||||
state.observeTurn(planUpdateTurn("in_progress"))
|
||||
if got := state.planReminder(stalePlanTurnThreshold + 1); got != "" {
|
||||
t.Fatalf("a just-refreshed plan must not be stale, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -3,9 +3,11 @@ package agent
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/errhint"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
@@ -17,10 +19,20 @@ import (
|
||||
// times. We retry ONLY the connect (not a partially-consumed stream), so no
|
||||
// already-forwarded OnText is ever duplicated.
|
||||
const (
|
||||
maxStreamReconnects = 2
|
||||
streamReconnectBase = 500 * time.Millisecond
|
||||
// maxStreamReconnects is how many times the connect is re-issued after a
|
||||
// transient disconnect. 4 (not 2): with jittered exponential backoff this rides
|
||||
// out a multi-second network blip (~0.5s + 1s + 2s + 4s worst case) instead of
|
||||
// dying on a 2s hiccup and re-burning every token on a restart.
|
||||
maxStreamReconnects = 4
|
||||
// streamReconnectMax caps a single backoff so the tail attempts don't wait
|
||||
// minutes on a long outage.
|
||||
streamReconnectMax = 8 * time.Second
|
||||
)
|
||||
|
||||
// streamReconnectBase is the first-retry delay (doubled each subsequent attempt).
|
||||
// A var, not a const, so tests can shrink it to keep the exhaustion path fast.
|
||||
var streamReconnectBase = 500 * time.Millisecond
|
||||
|
||||
// reconnectNotifier is called before each retry with the 1-based attempt number
|
||||
// and the max, so the caller can surface a "Reconnecting N/max" notice. Nil is
|
||||
// fine.
|
||||
@@ -73,7 +85,7 @@ func streamWithReconnect(ctx context.Context, provider Provider, request zerorun
|
||||
if notify != nil {
|
||||
notify(attempt, maxStreamReconnects)
|
||||
}
|
||||
if waitErr := sleepWithContext(ctx, backoffFor(attempt)); waitErr != nil {
|
||||
if waitErr := sleepWithContext(ctx, jitteredBackoff(attempt)); waitErr != nil {
|
||||
return nil, err // ctx cancelled while waiting; surface the original error
|
||||
}
|
||||
stream, err = provider.StreamCompletion(ctx, request)
|
||||
@@ -96,6 +108,19 @@ func shouldReconnect(ctx context.Context, err error) bool {
|
||||
if isContextLimitError(msg) || isImageRejectionError(err) {
|
||||
return false
|
||||
}
|
||||
// HTTP 5xx statuses are non-reconnectable and must be excluded BEFORE the
|
||||
// transport substring match below — otherwise "504 Gateway Timeout" would slip
|
||||
// through on the generic "timeout" needle. 503 already exhausted
|
||||
// providerio.SendWithRetry (retrying is a redundant double-retry); 500/502/504
|
||||
// are non-idempotent by providerio's rule — the completion POST may already
|
||||
// have reached the model before the upstream/gateway gave up, so replaying the
|
||||
// connect risks duplicate billable work. Digit-boundary matched so an
|
||||
// incidental "504" inside a latency/id number is not mistaken for a status.
|
||||
if errhint.HasStatusCode(msg, "500", "502", "503", "504") {
|
||||
return false
|
||||
}
|
||||
// Transport-level disconnects only. A genuine transport failure (EOF, reset,
|
||||
// refused, timeout) means no response was received, which is safe to reconnect.
|
||||
for _, needle := range []string{
|
||||
"eof",
|
||||
"connection reset",
|
||||
@@ -106,8 +131,6 @@ func shouldReconnect(ctx context.Context, err error) bool {
|
||||
"timed out",
|
||||
"temporarily unavailable",
|
||||
"i/o timeout",
|
||||
"503",
|
||||
"502",
|
||||
"server closed",
|
||||
"unexpected end",
|
||||
} {
|
||||
@@ -118,14 +141,31 @@ func shouldReconnect(ctx context.Context, err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// backoffFor is the deterministic exponential base delay for a 1-based attempt,
|
||||
// capped at streamReconnectMax. Jitter is layered on separately (jitteredBackoff).
|
||||
func backoffFor(attempt int) time.Duration {
|
||||
d := streamReconnectBase
|
||||
for i := 1; i < attempt; i++ {
|
||||
if d >= streamReconnectMax {
|
||||
return streamReconnectMax
|
||||
}
|
||||
d *= 2
|
||||
}
|
||||
if d > streamReconnectMax {
|
||||
d = streamReconnectMax
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// jitteredBackoff adds up to 50% random jitter on top of backoffFor so concurrent
|
||||
// runs (swarm members, a cron fleet) that all trip on the same outage don't
|
||||
// reconnect in lockstep and hammer a recovering endpoint. Never shorter than the
|
||||
// deterministic base, so backoff still grows attempt over attempt.
|
||||
func jitteredBackoff(attempt int) time.Duration {
|
||||
base := backoffFor(attempt)
|
||||
return base + time.Duration(rand.Int63n(int64(base/2)+1))
|
||||
}
|
||||
|
||||
func sleepWithContext(ctx context.Context, d time.Duration) error {
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
|
||||
@@ -44,6 +44,10 @@ func TestStreamWithReconnectRecoversFromTransientDisconnect(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStreamWithReconnectGivesUpAfterMax(t *testing.T) {
|
||||
// Shrink the backoff so exhausting all retries stays fast (real base would
|
||||
// sleep ~7.5s across 4 attempts).
|
||||
defer func(orig time.Duration) { streamReconnectBase = orig }(streamReconnectBase)
|
||||
streamReconnectBase = time.Millisecond
|
||||
// Always fails with a disconnect error → exhausts retries and returns it.
|
||||
p := &flakyProvider{failBefore: 99, failErr: errors.New("connection reset by peer")}
|
||||
_, err := streamWithReconnect(context.Background(), p, zeroruntime.CompletionRequest{}, nil)
|
||||
@@ -87,7 +91,7 @@ func TestShouldReconnectClassification(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
disconnects := []string{
|
||||
"unexpected EOF", "connection reset by peer", "broken pipe",
|
||||
"i/o timeout", "503 Service Unavailable", "server closed the connection",
|
||||
"i/o timeout", "server closed the connection", "connection refused",
|
||||
}
|
||||
for _, m := range disconnects {
|
||||
if !shouldReconnect(ctx, errors.New(m)) {
|
||||
@@ -97,6 +101,13 @@ func TestShouldReconnectClassification(t *testing.T) {
|
||||
notDisconnects := []string{
|
||||
"context length exceeded", "invalid api key", "model not found",
|
||||
"400 bad request: unsupported parameter",
|
||||
// HTTP 5xx statuses are handled by providerio.SendWithRetry (503) or are
|
||||
// non-idempotent (500/502/504); the reconnect path must not double-retry
|
||||
// them. "504 Gateway Timeout" must NOT slip through on the generic "timeout"
|
||||
// transport signal — a gateway timeout is non-idempotent, since the upstream
|
||||
// may have already processed the completion POST before giving up.
|
||||
"503 Service Unavailable", "502 Bad Gateway",
|
||||
"504 Gateway Timeout", "500 Internal Server Error",
|
||||
}
|
||||
for _, m := range notDisconnects {
|
||||
if shouldReconnect(ctx, errors.New(m)) {
|
||||
@@ -112,6 +123,24 @@ func TestBackoffGrows(t *testing.T) {
|
||||
if backoffFor(2) != 2*streamReconnectBase {
|
||||
t.Fatalf("attempt 2 backoff = %v, want %v", backoffFor(2), 2*streamReconnectBase)
|
||||
}
|
||||
// The exponential base is capped so late attempts don't wait minutes.
|
||||
if got := backoffFor(20); got != streamReconnectMax {
|
||||
t.Fatalf("attempt 20 backoff = %v, want cap %v", got, streamReconnectMax)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJitteredBackoffStaysInBounds(t *testing.T) {
|
||||
// Jitter never drops below the deterministic base and never exceeds base*1.5,
|
||||
// so backoff still grows attempt over attempt while decorrelating retries.
|
||||
for attempt := 1; attempt <= 5; attempt++ {
|
||||
base := backoffFor(attempt)
|
||||
for i := 0; i < 200; i++ {
|
||||
got := jitteredBackoff(attempt)
|
||||
if got < base || got > base+base/2 {
|
||||
t.Fatalf("attempt %d jittered backoff %v out of [%v, %v]", attempt, got, base, base+base/2)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconnectNoticeRoutesThroughReasoning(t *testing.T) {
|
||||
|
||||
@@ -7,14 +7,18 @@ import (
|
||||
)
|
||||
|
||||
func TestSkillsContextCapsLongList(t *testing.T) {
|
||||
skills := make([]SkillInfo, 0, 40)
|
||||
for i := 0; i < 40; i++ {
|
||||
// 60 skills with realistic trigger-rich descriptions (~180 chars each, under
|
||||
// the 200-rune truncation) total well past the 4096-byte list budget, so the
|
||||
// overflow summary must kick in while the block stays bounded.
|
||||
longDesc := strings.Repeat("use when the request touches deployments, release notes, or verification; ", 2) + "number "
|
||||
skills := make([]SkillInfo, 0, 60)
|
||||
for i := 0; i < 60; i++ {
|
||||
n := strconv.Itoa(i)
|
||||
skills = append(skills, SkillInfo{Name: "skill-" + n, Description: "does something useful, number " + n})
|
||||
skills = append(skills, SkillInfo{Name: "skill-" + n, Description: longDesc + n})
|
||||
}
|
||||
got := skillsContext(Options{Skills: skills})
|
||||
if len(got) > 1200 {
|
||||
t.Fatalf("skills block should stay bounded, got %d bytes:\n%s", len(got), got)
|
||||
if len(got) > skillsContextListBudget+1000 {
|
||||
t.Fatalf("skills block should stay bounded near the %d-byte budget, got %d bytes:\n%s", skillsContextListBudget, len(got), got)
|
||||
}
|
||||
if !strings.Contains(got, "more (call skill") {
|
||||
t.Fatalf("expected an overflow summary line, got:\n%s", got)
|
||||
@@ -25,6 +29,27 @@ func TestSkillsContextCapsLongList(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A realistic mid-size skill set (20 skills, trigger-rich descriptions) must be
|
||||
// listed in FULL — no overflow summary. This pins the fix for the old 640-byte
|
||||
// budget, under which skills past ~#6 were invisible to the model and therefore
|
||||
// never triggered.
|
||||
func TestSkillsContextListsRealisticSetInFull(t *testing.T) {
|
||||
desc := "Use when the user asks about deployments, release notes, or pre-merge verification runs."
|
||||
skills := make([]SkillInfo, 0, 20)
|
||||
for i := 0; i < 20; i++ {
|
||||
skills = append(skills, SkillInfo{Name: "skill-" + strconv.Itoa(i), Description: desc})
|
||||
}
|
||||
got := skillsContext(Options{Skills: skills})
|
||||
if strings.Contains(got, "more (call skill") {
|
||||
t.Fatalf("20 described skills must all be listed without overflow, got:\n%s", got)
|
||||
}
|
||||
for i := 0; i < 20; i++ {
|
||||
if !strings.Contains(got, "- skill-"+strconv.Itoa(i)+":") {
|
||||
t.Fatalf("skill-%d missing from the list:\n%s", i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkillsContext(t *testing.T) {
|
||||
if got := skillsContext(Options{}); got != "" {
|
||||
t.Fatalf("no skills should yield an empty section, got %q", got)
|
||||
|
||||
+127
-27
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/repomap"
|
||||
"github.com/Gitlawb/zero/internal/workspaceseed"
|
||||
)
|
||||
@@ -38,6 +39,14 @@ const fallbackSystemPrompt = "You are Zero, a terminal coding agent. Help with t
|
||||
// general-to-specific order.
|
||||
var projectContextFiles = []string{"AGENTS.md", "ZERO.md", ".zero/AGENTS.md"}
|
||||
|
||||
// userContextFile is the per-user instruction file, resolved under
|
||||
// config.UserConfigDir()/zero/ alongside the rest of Zero's per-user config
|
||||
// (config.json, commands, specialists) so users can keep personal guidance out
|
||||
// of individual repositories.
|
||||
const userContextFile = "ZERO.md"
|
||||
|
||||
var userConfigDirForPrompt = config.UserConfigDir
|
||||
|
||||
// maxProjectContextBytes caps how much of a single project doc is injected so
|
||||
// a large guidelines file can't blow the context budget.
|
||||
const maxProjectContextBytes = 8 << 10 // 8 KiB per file
|
||||
@@ -81,6 +90,13 @@ func buildSystemPrompt(options Options) string {
|
||||
if seed := workspaceSeedContext(options.Cwd); seed != "" {
|
||||
sections = append(sections, seed)
|
||||
}
|
||||
// User guidelines are injected before workspace/project guidelines so the
|
||||
// project's AGENTS.md/ZERO.md is the later, more specific instruction
|
||||
// block. See userGuidelines for the explicit precedence note carried in
|
||||
// the section text itself.
|
||||
if user := userGuidelines(); user != "" {
|
||||
sections = append(sections, user)
|
||||
}
|
||||
if ws := workspaceContext(options.Cwd); ws != "" {
|
||||
sections = append(sections, ws)
|
||||
}
|
||||
@@ -175,10 +191,19 @@ func specialistDelegationContext(options Options) string {
|
||||
// when no skills are installed, so a skill-less run reproduces the previous prompt
|
||||
// byte-for-byte.
|
||||
// skillsContextListBudget bounds the bytes spent listing individual skills so a
|
||||
// workspace with dozens of them can't bloat every turn's prompt. Skills past the
|
||||
// budget are summarized as a count rather than dropped — the model can still load
|
||||
// any of them by name (an unknown name returns the full list from the skill tool).
|
||||
const skillsContextListBudget = 640
|
||||
// workspace with an extreme number of them can't bloat every turn's prompt.
|
||||
// Skills past the budget are summarized as a count rather than dropped — the
|
||||
// model can still load any of them by name (an unknown name returns the full
|
||||
// list from the skill tool).
|
||||
//
|
||||
// The budget is deliberately generous — roughly 18 skills with maximally long
|
||||
// (200-rune, ASCII) descriptions, or 40+ with typical terser ones; non-ASCII
|
||||
// descriptions consume it faster since the budget counts bytes. This list is
|
||||
// the model's ONLY discovery surface, so a skill squeezed out of it effectively
|
||||
// never triggers. The previous 640-byte cap silently broke invocation for
|
||||
// anyone with more than ~6 skills — the model cannot match a request against a
|
||||
// skill it cannot see.
|
||||
const skillsContextListBudget = 4096
|
||||
|
||||
func skillsContext(options Options) string {
|
||||
if len(options.Skills) == 0 {
|
||||
@@ -186,7 +211,7 @@ func skillsContext(options Options) string {
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("<available_skills>\n")
|
||||
b.WriteString("Reusable, on-demand instruction sets you can load with the skill tool. When a request matches one, call skill with its exact name to pull in its guidance before acting — do not guess names or skip a relevant skill.\n")
|
||||
b.WriteString("Reusable, on-demand instruction sets you can load with the skill tool. Before acting on a request, scan this list; when the request matches a skill's name or description, call skill with its exact name FIRST and follow the loaded guidance — do not guess names, do not skip a matching skill, and do not substitute your own approach for its instructions.\n")
|
||||
listed, spent, omitted := 0, 0, 0
|
||||
for _, info := range options.Skills {
|
||||
name := strings.TrimSpace(info.Name)
|
||||
@@ -216,9 +241,11 @@ func skillsContext(options Options) string {
|
||||
}
|
||||
|
||||
// truncateForSkillLine keeps a skill's one-line description short so a single
|
||||
// verbose description can't dominate the skills-list budget.
|
||||
// verbose description can't dominate the skills-list budget. The cap is roomy on
|
||||
// purpose: the description carries the skill's trigger conditions ("use when…"),
|
||||
// and truncating those is what stops the model from ever invoking the skill.
|
||||
func truncateForSkillLine(desc string) string {
|
||||
const maxDescRunes = 100
|
||||
const maxDescRunes = 200
|
||||
runes := []rune(desc)
|
||||
if len(runes) <= maxDescRunes {
|
||||
return desc
|
||||
@@ -259,7 +286,7 @@ func workspaceContext(cwd string) string {
|
||||
b.WriteString("Working directory: " + cwd + "\n")
|
||||
b.WriteString("Operating system: " + runtime.GOOS + "\n")
|
||||
if runtime.GOOS == "windows" {
|
||||
b.WriteString("Shell syntax: Windows cmd.exe syntax for exec_command/bash tools; prefer the workdir/cwd argument instead of cd when changing directories.\n")
|
||||
b.WriteString("Shell syntax: Windows cmd.exe syntax for exec_command/bash tools; prefer the workdir/cwd argument instead of cd when changing directories. Do not pipe to or invoke POSIX coreutils from Git for Windows (usr\\bin head/grep/tail/cat/...): they are MSYS binaries and fail under the write-restricted sandbox; use native Zero tools (grep, read_file, list_directory, glob) or cmd.exe findstr/more instead, or sandbox_permissions require_escalated only when host-level execution is truly required.\n")
|
||||
} else {
|
||||
b.WriteString("Shell syntax: /bin/sh syntax for exec_command/bash tools; prefer the workdir/cwd argument instead of cd when changing directories.\n")
|
||||
}
|
||||
@@ -312,13 +339,7 @@ func projectGuidelines(cwd, gitRoot string) string {
|
||||
if remaining := maxProjectContextTotalBytes - totalUsed; remaining < limit {
|
||||
limit = remaining
|
||||
}
|
||||
if len(content) > limit {
|
||||
cut := limit
|
||||
for cut > 0 && !utf8.RuneStart(content[cut]) {
|
||||
cut--
|
||||
}
|
||||
content = content[:cut] + "\n… (truncated)"
|
||||
}
|
||||
content = truncateGuidelineContent(content, limit)
|
||||
label := projectGuidelineLabel(match, gitRoot)
|
||||
b.WriteString("\n\n## Project guidelines (" + label + ")\n\n" + content)
|
||||
totalUsed += len(content)
|
||||
@@ -326,6 +347,92 @@ func projectGuidelines(cwd, gitRoot string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// userGuidelines returns the per-user ZERO.md instructions block, if present.
|
||||
// The file lives in config.UserConfigDir()/zero/ next to Zero's other
|
||||
// per-user config; the basename match is case-insensitive so a file saved as
|
||||
// zero.md still resolves on case-sensitive filesystems, mirroring the project
|
||||
// guideline loader. The section carries an explicit precedence note because
|
||||
// this is a global, personal preferences file: it is injected earlier in the
|
||||
// prompt than the project's AGENTS.md/ZERO.md (see buildSystemPrompt), and
|
||||
// the note keeps that precedence unambiguous even if a model weighs later
|
||||
// context more heavily than section order alone implies.
|
||||
func userGuidelines() string {
|
||||
configDir, err := userConfigDirForPrompt()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
configDir = strings.TrimSpace(configDir)
|
||||
if configDir == "" {
|
||||
return ""
|
||||
}
|
||||
match := findCaseInsensitiveFile(filepath.Join(configDir, "zero"), userContextFile)
|
||||
if match == "" {
|
||||
return ""
|
||||
}
|
||||
data, err := os.ReadFile(match)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
content := strings.TrimSpace(string(data))
|
||||
if content == "" {
|
||||
return ""
|
||||
}
|
||||
content = truncateGuidelineContent(content, maxProjectContextBytes)
|
||||
return "## User guidelines (" + filepath.Base(match) + ")\n\n" +
|
||||
"These are the operator's personal preferences, not project policy. " +
|
||||
"Where they conflict with a repository's project guidelines below (AGENTS.md/ZERO.md), the project guidelines take precedence.\n\n" +
|
||||
content
|
||||
}
|
||||
|
||||
// truncationMarker is appended to guideline content that was cut short.
|
||||
const truncationMarker = "\n… (truncated)"
|
||||
|
||||
// truncateGuidelineContent caps content at limit bytes without splitting a
|
||||
// UTF-8 rune, appending a truncation marker when anything was cut. Space for
|
||||
// the marker is reserved before choosing the cut point. When limit is too
|
||||
// small to fit the marker itself, the marker is dropped instead, so the
|
||||
// returned string never exceeds limit for any non-negative limit.
|
||||
func truncateGuidelineContent(content string, limit int) string {
|
||||
if len(content) <= limit {
|
||||
return content
|
||||
}
|
||||
if limit <= 0 {
|
||||
return ""
|
||||
}
|
||||
cut := limit - len(truncationMarker)
|
||||
if cut < 0 {
|
||||
cut = 0
|
||||
}
|
||||
for cut > 0 && !utf8.RuneStart(content[cut]) {
|
||||
cut--
|
||||
}
|
||||
if truncated := content[:cut] + truncationMarker; len(truncated) <= limit {
|
||||
return truncated
|
||||
}
|
||||
// limit is smaller than the marker itself: fall back to a hard,
|
||||
// rune-safe cut at limit bytes with no marker rather than exceeding it.
|
||||
end := limit
|
||||
for end > 0 && !utf8.RuneStart(content[end]) {
|
||||
end--
|
||||
}
|
||||
return content[:end]
|
||||
}
|
||||
|
||||
// findCaseInsensitiveFile returns the on-disk path of the regular file in dir
|
||||
// whose name matches basename case-insensitively, or "" when absent.
|
||||
func findCaseInsensitiveFile(dir, basename string) string {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.EqualFold(e.Name(), basename) {
|
||||
return filepath.Join(dir, e.Name())
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// projectGuidelineDirs returns the directory chain from gitRoot to cwd
|
||||
// (inclusive), in root-to-leaf order. If gitRoot is empty or unreachable
|
||||
// from cwd, the chain collapses to [cwd].
|
||||
@@ -378,23 +485,16 @@ func projectGuidelineLabel(match, gitRoot string) string {
|
||||
// Returns "" when nothing matches.
|
||||
func findProjectContextFile(dir string) string {
|
||||
for _, name := range projectContextFiles {
|
||||
baseLower := strings.ToLower(filepath.Base(name))
|
||||
// Walk to the file's parent through dir with case-insensitive segment
|
||||
// matching, then find a regular file with the same lowercased
|
||||
// basename. This works on both case-sensitive and case-insensitive
|
||||
// filesystems and always returns the on-disk filename.
|
||||
// matching, then find a regular file with the same basename. This works
|
||||
// on both case-sensitive and case-insensitive filesystems and always
|
||||
// returns the on-disk filename.
|
||||
parent, ok := resolveDirCaseInsensitive(filepath.Dir(filepath.Join(dir, name)), dir)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
entries, err := os.ReadDir(parent)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.ToLower(e.Name()) == baseLower {
|
||||
return filepath.Join(parent, e.Name())
|
||||
}
|
||||
if match := findCaseInsensitiveFile(parent, filepath.Base(name)); match != "" {
|
||||
return match
|
||||
}
|
||||
}
|
||||
return ""
|
||||
|
||||
@@ -63,6 +63,10 @@ work.
|
||||
targeted diffs. Match the existing indentation, imports, and idioms. Match the
|
||||
file's comment density: do not add explanatory comments unless the user asks or
|
||||
the code is already comment-dense.
|
||||
- Solve the problem as posed, not a more general version of it. Add no
|
||||
speculative abstraction, configurability, or handling for cases that cannot
|
||||
occur, and nothing the user did not ask for. A small diff can still be
|
||||
over-built; if a 200-line solution could be 50, rewrite it.
|
||||
- Preserve behavior you were not asked to change. Do not delete or rewrite code
|
||||
you did not author unless the task requires it; if you must, say so.
|
||||
|
||||
|
||||
@@ -9,6 +9,25 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
)
|
||||
|
||||
// TestMain points userConfigDirForPrompt at an empty, package-wide temp
|
||||
// directory for every test in this package. Without this, buildSystemPrompt
|
||||
// falls back to the real config.UserConfigDir, so any developer with a
|
||||
// personal ~/.config/zero/ZERO.md would get its content folded into
|
||||
// otherwise-unrelated prompt assertions, making test runs non-deterministic
|
||||
// across contributor machines. Tests that specifically exercise user
|
||||
// guidelines stub userConfigDirForPrompt themselves via
|
||||
// withSystemPromptTestUserConfigDir(Func) and restore this default on cleanup.
|
||||
func TestMain(m *testing.M) {
|
||||
dir, err := os.MkdirTemp("", "zero-agent-tests-*")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
userConfigDirForPrompt = func() (string, error) { return dir, nil }
|
||||
code := m.Run()
|
||||
os.RemoveAll(dir)
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestCoreSystemPromptIncludesCodingQualityRules(t *testing.T) {
|
||||
prompt := strings.ToLower(buildSystemPrompt(Options{}))
|
||||
|
||||
@@ -128,6 +147,88 @@ func systemPromptTestBlock(t *testing.T, prompt, start, end string) string {
|
||||
return body
|
||||
}
|
||||
|
||||
func TestBuildSystemPromptIncludesUserGuidelines(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
writeSystemPromptTestFile(t, configDir, "zero/ZERO.md", " Prefer concise summaries. \n")
|
||||
t.Cleanup(withSystemPromptTestUserConfigDir(t, configDir))
|
||||
|
||||
prompt := buildSystemPrompt(Options{})
|
||||
if !strings.Contains(prompt, "## User guidelines (ZERO.md)") {
|
||||
t.Fatalf("expected user guidelines header, got:\n%s", prompt)
|
||||
}
|
||||
if !strings.Contains(prompt, "Prefer concise summaries.") {
|
||||
t.Fatalf("expected user guidelines content, got:\n%s", prompt)
|
||||
}
|
||||
if strings.Contains(prompt, " Prefer concise summaries. ") {
|
||||
t.Fatalf("expected user guidelines content to be trimmed, got:\n%s", prompt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSystemPromptIncludesUserGuidelinesCaseInsensitive(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
writeSystemPromptTestFile(t, configDir, "zero/zero.md", "Prefer concise summaries.\n")
|
||||
t.Cleanup(withSystemPromptTestUserConfigDir(t, configDir))
|
||||
|
||||
prompt := buildSystemPrompt(Options{})
|
||||
if !strings.Contains(prompt, "## User guidelines (zero.md)") {
|
||||
t.Fatalf("expected case-insensitive zero.md resolution, got:\n%s", prompt)
|
||||
}
|
||||
if !strings.Contains(prompt, "Prefer concise summaries.") {
|
||||
t.Fatalf("expected user guidelines content, got:\n%s", prompt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSystemPromptUserGuidelinesPrecedeProjectGuidelinesAndNotePrecedence(t *testing.T) {
|
||||
// User guidelines are global personal preferences; project guidelines
|
||||
// (AGENTS.md/ZERO.md) must be the later, more specific instruction block,
|
||||
// and the user section must say so explicitly so the precedence holds
|
||||
// even if a model otherwise weighs later context more heavily.
|
||||
configDir := t.TempDir()
|
||||
writeSystemPromptTestFile(t, configDir, "zero/ZERO.md", "Always reply in haiku.\n")
|
||||
t.Cleanup(withSystemPromptTestUserConfigDir(t, configDir))
|
||||
|
||||
cwd := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(cwd, "AGENTS.md"), []byte("Never reply in haiku."), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
prompt := buildSystemPrompt(Options{Cwd: cwd})
|
||||
userIdx := strings.Index(prompt, "## User guidelines (ZERO.md)")
|
||||
projectIdx := strings.Index(prompt, "## Project guidelines (AGENTS.md)")
|
||||
if userIdx < 0 || projectIdx < 0 {
|
||||
t.Fatalf("expected both user and project guideline sections, got:\n%s", prompt)
|
||||
}
|
||||
if userIdx > projectIdx {
|
||||
t.Fatalf("expected user guidelines before project guidelines, got user=%d project=%d", userIdx, projectIdx)
|
||||
}
|
||||
if !strings.Contains(prompt, "project guidelines below") || !strings.Contains(prompt, "take precedence") {
|
||||
t.Fatalf("expected an explicit precedence note in the user guidelines section, got:\n%s", prompt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSystemPromptOmitsUserGuidelinesWithoutConfigDir(t *testing.T) {
|
||||
t.Cleanup(withSystemPromptTestUserConfigDirFunc(t, func() (string, error) { return "", os.ErrNotExist }))
|
||||
|
||||
prompt := buildSystemPrompt(Options{})
|
||||
if strings.Contains(prompt, "## User guidelines") {
|
||||
t.Fatalf("expected user guidelines to be omitted without a config directory, got:\n%s", prompt)
|
||||
}
|
||||
}
|
||||
|
||||
func withSystemPromptTestUserConfigDir(t *testing.T, dir string) func() {
|
||||
t.Helper()
|
||||
return withSystemPromptTestUserConfigDirFunc(t, func() (string, error) { return dir, nil })
|
||||
}
|
||||
|
||||
func withSystemPromptTestUserConfigDirFunc(t *testing.T, fn func() (string, error)) func() {
|
||||
t.Helper()
|
||||
old := userConfigDirForPrompt
|
||||
userConfigDirForPrompt = fn
|
||||
return func() {
|
||||
userConfigDirForPrompt = old
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSystemPromptInjectsProjectGuidelinesCaseInsensitive(t *testing.T) {
|
||||
// Git tracks AGENTS.md on a case-sensitive filesystem; the
|
||||
// loader must still resolve it when the cwd lookup uses lowercase.
|
||||
@@ -257,6 +358,30 @@ func TestBuildSystemPromptProjectGuidelinesTruncatesAtTotalCap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateGuidelineContentStaysWithinLimit(t *testing.T) {
|
||||
limit := 64
|
||||
content := strings.Repeat("x", limit+1)
|
||||
got := truncateGuidelineContent(content, limit)
|
||||
if len(got) > limit {
|
||||
t.Fatalf("truncated result is %d bytes, want at most %d", len(got), limit)
|
||||
}
|
||||
if !strings.Contains(got, "truncated") {
|
||||
t.Fatalf("expected truncation marker, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateGuidelineContentStaysWithinLimitForSmallBudgets(t *testing.T) {
|
||||
// Limits at or below the truncation marker's own length can't fit the
|
||||
// marker at all; the helper must still never exceed the requested limit.
|
||||
content := strings.Repeat("x", 100)
|
||||
for _, limit := range []int{0, 1, len(truncationMarker) - 1, len(truncationMarker), len(truncationMarker) + 1} {
|
||||
got := truncateGuidelineContent(content, limit)
|
||||
if len(got) > limit {
|
||||
t.Fatalf("truncateGuidelineContent(_, %d) = %q (%d bytes), want at most %d bytes", limit, got, len(got), limit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectGuidelineDirsOrdersRootToLeaf(t *testing.T) {
|
||||
root := filepath.Join("r")
|
||||
leaf := filepath.Join(root, "a", "b")
|
||||
|
||||
@@ -285,11 +285,13 @@ type Options struct {
|
||||
// ceiling and the autonomy gate. nil disables it entirely (the loop is
|
||||
// byte-identical to before). One instance per run — it holds attempt state.
|
||||
SelfCorrect *SelfCorrector
|
||||
// FileDiagnostics, when set, is passed to mutating tools so edit_file /
|
||||
// write_file append error-severity language diagnostics for the file they
|
||||
// just wrote directly to their tool output — the model sees an error it
|
||||
// introduced in the same turn instead of a later verification pass. Build
|
||||
// one with NewFileDiagnostics. nil disables inline diagnostics.
|
||||
// FileDiagnostics, when set, checks files changed by mutating tools for
|
||||
// error-severity language diagnostics IN THE BACKGROUND and appends any
|
||||
// errors as a nudge before the model's next request — the model still sees
|
||||
// an error it introduced at its next decision point, but no tool call ever
|
||||
// blocks on the language server (the old inline path stalled every edit on
|
||||
// a ≥300ms debounce, 10s cap). Build one with NewFileDiagnostics. nil
|
||||
// disables post-edit diagnostics.
|
||||
FileDiagnostics func(ctx context.Context, absPath string) string
|
||||
|
||||
// RequireCompletionSignal gates run completion for HEADLESS exec. Without it,
|
||||
|
||||
+70
-4
@@ -12,6 +12,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/agent"
|
||||
@@ -78,6 +79,8 @@ type appDeps struct {
|
||||
runAgentEval func(context.Context, agentEvalOptions) (agentEvalReport, error)
|
||||
inspectChanges func(context.Context, zerogit.InspectOptions) (zerogit.ChangeSummary, error)
|
||||
commitChanges func(context.Context, zerogit.CommitOptions) (zerogit.CommitResult, error)
|
||||
pushChanges func(context.Context, zerogit.PushOptions) (zerogit.PushResult, error)
|
||||
createPR func(context.Context, zerogit.PROptions) (zerogit.PRResult, error)
|
||||
runTUI func(context.Context, tui.Options) int
|
||||
runEditor func(string) error
|
||||
checkUpdate func(context.Context, update.Options) (update.Result, error)
|
||||
@@ -179,6 +182,8 @@ func defaultAppDeps() appDeps {
|
||||
runAgentEval: defaultRunAgentEval,
|
||||
inspectChanges: zerogit.Inspect,
|
||||
commitChanges: zerogit.Commit,
|
||||
pushChanges: zerogit.Push,
|
||||
createPR: zerogit.CreatePR,
|
||||
runTUI: tui.Run,
|
||||
runEditor: openEditor,
|
||||
checkUpdate: update.Check,
|
||||
@@ -517,6 +522,12 @@ func fillAppDeps(deps appDeps) appDeps {
|
||||
if deps.commitChanges == nil {
|
||||
deps.commitChanges = defaults.commitChanges
|
||||
}
|
||||
if deps.pushChanges == nil {
|
||||
deps.pushChanges = defaults.pushChanges
|
||||
}
|
||||
if deps.createPR == nil {
|
||||
deps.createPR = defaults.createPR
|
||||
}
|
||||
if deps.runTUI == nil {
|
||||
deps.runTUI = defaults.runTUI
|
||||
}
|
||||
@@ -675,8 +686,14 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
|
||||
defer closeMCPRuntime(stderr, mcpRuntime)
|
||||
// A server that could not be reached or validated is skipped, not fatal (one
|
||||
// bad MCP server must not abort startup) — surface each so a missing tool set is
|
||||
// explained rather than silently absent.
|
||||
// explained rather than silently absent. A built-in default the user never
|
||||
// configured (e.g. keyless Firecrawl with no credentials) is the exception: it
|
||||
// was never asked for, so its failure is not worth a startup warning — only
|
||||
// servers the user actually configured warn on failure (issue #552).
|
||||
for _, skipped := range mcpRuntime.Skipped() {
|
||||
if skipped.UnconfiguredDefault {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(stderr, "warning: MCP server %s unavailable, skipped: %s\n", skipped.Name, redaction.ErrorMessage(skipped.Err, redaction.Options{}))
|
||||
}
|
||||
// Make local plugins live: register their declared tools into the registry and
|
||||
@@ -718,8 +735,19 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
|
||||
Scope: scope,
|
||||
})
|
||||
lastKnownMCPConfig := mcpConfig
|
||||
sttServerManager := newDictationServerManager(resolved.STT)
|
||||
// Keep STT downloads in the SAME config tree the rest of the TUI uses. Deriving
|
||||
// from userConfigPath (rather than config.UserConfigDir()) matters when the config
|
||||
// root is overridden — e.g. in tests or a custom ZERO config dir — so the two
|
||||
// don't diverge. userConfigPath points at .../zero/config.json, so its dir is the
|
||||
// zero config dir.
|
||||
sttDownloadRoot := ""
|
||||
if userConfigPath != "" {
|
||||
sttDownloadRoot = filepath.Join(filepath.Dir(userConfigPath), "stt")
|
||||
}
|
||||
return deps.runTUI(context.Background(), tui.Options{
|
||||
Cwd: workspaceRoot,
|
||||
Version: version,
|
||||
Theme: theme,
|
||||
SavedTheme: resolved.Preferences.Theme,
|
||||
UserConfigPath: userConfigPath,
|
||||
@@ -730,6 +758,7 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
|
||||
ProviderProfile: resolved.Provider,
|
||||
SavedProviders: usableSavedProviders(resolved.Providers),
|
||||
FavoriteModels: resolved.Preferences.FavoriteModels,
|
||||
RecentModels: resolved.Preferences.RecentModels,
|
||||
RecapsEnabled: resolved.Preferences.RecapsEnabled(),
|
||||
Provider: provider,
|
||||
NewProvider: deps.newProvider,
|
||||
@@ -772,9 +801,23 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
|
||||
Specialists: specialistRuntime.specialists,
|
||||
Skills: pluginActivation.skillInfos(deps.skillsDir()),
|
||||
},
|
||||
PermissionMode: permissionMode,
|
||||
Notify: resolved.Notify,
|
||||
KeyBindings: resolved.KeyBindings,
|
||||
// LoadSkills backs /skills and direct /<skill-name> invocation in the TUI.
|
||||
// It resolves against the same merged set (default dir + plugin skill
|
||||
// roots) as the skill tool and the system-prompt list, re-read per use so
|
||||
// newly installed skills work without a restart.
|
||||
LoadSkills: cachedSkillsLoader(func() []skills.Skill {
|
||||
merged, _ := plugins.MergedSkillsLoaded(deps.skillsDir(), pluginActivation.skillRoots)
|
||||
return merged
|
||||
}),
|
||||
PermissionMode: permissionMode,
|
||||
Notify: resolved.Notify,
|
||||
KeyBindings: resolved.KeyBindings,
|
||||
STT: resolved.STT,
|
||||
BuildDictationTranscriber: newDictationTranscriberFactory(resolved, userConfigPath, sttServerManager),
|
||||
ShutdownDictationServer: sttServerManager.Shutdown,
|
||||
STTDownloadRoot: sttDownloadRoot,
|
||||
STTKeyStatus: newSTTKeyStatus(resolved, userConfigPath),
|
||||
SaveSTTKey: newSaveSTTKey(userConfigPath),
|
||||
Setup: tui.SetupOptions{
|
||||
Visible: setupVisible,
|
||||
Required: needsSetup,
|
||||
@@ -1267,3 +1310,26 @@ Flags:
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
// cachedSkillsLoader memoizes a skills loader for a short interval. The TUI
|
||||
// calls the loader from autocomplete on every "/x" keystroke; without a cache
|
||||
// each keystroke would re-read every SKILL.md body. Two seconds is fresh enough
|
||||
// that a newly installed skill still shows up "immediately" while typing.
|
||||
func cachedSkillsLoader(load func() []skills.Skill) func() []skills.Skill {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
cached []skills.Skill
|
||||
)
|
||||
const ttl = 2 * time.Second
|
||||
return func() []skills.Skill {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if cached != nil && time.Since(at) < ttl {
|
||||
return cached
|
||||
}
|
||||
cached = load()
|
||||
at = time.Now()
|
||||
return cached
|
||||
}
|
||||
}
|
||||
@@ -158,6 +158,86 @@ func TestRunNoArgsLaunchesSetupTUIWithNilProviderWhenNoProviderConfigured(t *tes
|
||||
assertAgentOptions(t, launchedOptions, 12, agent.PermissionModeAsk)
|
||||
}
|
||||
|
||||
type fakeMCPRuntimeWithSkips struct {
|
||||
skipped []mcp.SkippedServer
|
||||
}
|
||||
|
||||
func (r fakeMCPRuntimeWithSkips) Close() error { return nil }
|
||||
|
||||
func (r fakeMCPRuntimeWithSkips) Skipped() []mcp.SkippedServer { return r.skipped }
|
||||
|
||||
func TestTUIStartupSuppressesWarningForUnconfiguredDefaultServer(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cwd := t.TempDir()
|
||||
setCLIUserConfigRoot(t)
|
||||
|
||||
exitCode := runWithDeps([]string{}, &stdout, &stderr, appDeps{
|
||||
getwd: func() (string, error) {
|
||||
return cwd, nil
|
||||
},
|
||||
resolveConfig: func(workspaceRoot string, overrides config.Overrides) (config.ResolvedConfig, error) {
|
||||
return config.ResolvedConfig{MaxTurns: 8}, nil
|
||||
},
|
||||
resolveMCPConfig: func(workspaceRoot string) (config.MCPConfig, error) {
|
||||
return config.MCPConfig{Servers: map[string]config.MCPServerConfig{
|
||||
"firecrawl": config.DefaultMCPServers()["firecrawl"],
|
||||
}}, nil
|
||||
},
|
||||
registerMCPTools: func(context.Context, *tools.Registry, config.MCPConfig, mcp.RegisterOptions) (mcpToolRuntime, error) {
|
||||
return fakeMCPRuntimeWithSkips{skipped: []mcp.SkippedServer{
|
||||
{Name: "firecrawl", Err: errors.New("returned HTTP 401"), UnconfiguredDefault: true},
|
||||
}}, nil
|
||||
},
|
||||
runTUI: func(ctx context.Context, options tui.Options) int {
|
||||
return 0
|
||||
},
|
||||
})
|
||||
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("exitCode = %d stderr=%s", exitCode, stderr.String())
|
||||
}
|
||||
if strings.Contains(stderr.String(), "firecrawl") {
|
||||
t.Fatalf("stderr = %q, want no warning for an unconfigured default server", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTUIStartupWarnsForUserConfiguredServerSkip(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cwd := t.TempDir()
|
||||
setCLIUserConfigRoot(t)
|
||||
|
||||
exitCode := runWithDeps([]string{}, &stdout, &stderr, appDeps{
|
||||
getwd: func() (string, error) {
|
||||
return cwd, nil
|
||||
},
|
||||
resolveConfig: func(workspaceRoot string, overrides config.Overrides) (config.ResolvedConfig, error) {
|
||||
return config.ResolvedConfig{MaxTurns: 8}, nil
|
||||
},
|
||||
resolveMCPConfig: func(workspaceRoot string) (config.MCPConfig, error) {
|
||||
return config.MCPConfig{Servers: map[string]config.MCPServerConfig{
|
||||
"custom": {Type: "stdio", Command: "custom-mcp"},
|
||||
}}, nil
|
||||
},
|
||||
registerMCPTools: func(context.Context, *tools.Registry, config.MCPConfig, mcp.RegisterOptions) (mcpToolRuntime, error) {
|
||||
return fakeMCPRuntimeWithSkips{skipped: []mcp.SkippedServer{
|
||||
{Name: "custom", Err: errors.New("connect failed"), UnconfiguredDefault: false},
|
||||
}}, nil
|
||||
},
|
||||
runTUI: func(ctx context.Context, options tui.Options) int {
|
||||
return 0
|
||||
},
|
||||
})
|
||||
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("exitCode = %d stderr=%s", exitCode, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "warning: MCP server custom unavailable") {
|
||||
t.Fatalf("stderr = %q, want a warning for a user-configured server that failed", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNoArgsEntersSetupWhenResolveReportsNoActiveProvider(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
+45
-2
@@ -11,10 +11,46 @@ import (
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
"github.com/Gitlawb/zero/internal/provideroauth"
|
||||
"github.com/Gitlawb/zero/internal/redaction"
|
||||
)
|
||||
|
||||
// ensureLoginProviderProfile makes a freshly stored OAuth login visible as a
|
||||
// provider: without a profile in config.json the login shows up nowhere — not in
|
||||
// `zero providers list`, not in the TUI picker — and `zero providers use <name>`
|
||||
// fails "not found", so the user's working active provider looks broken while
|
||||
// the new login looks lost. Returns user-facing guidance lines ("" when the
|
||||
// login has no catalog entry to scaffold from). Best-effort by design: the
|
||||
// token is already stored, so a profile-write failure degrades to a warning
|
||||
// line instead of failing a login that succeeded.
|
||||
func ensureLoginProviderProfile(deps appDeps, provider string) string {
|
||||
if _, ok := providercatalog.Get(provider); !ok {
|
||||
// Custom OAuth server without a catalog entry — no endpoint/model defaults
|
||||
// to scaffold a profile from; the user wires their own profile.
|
||||
return ""
|
||||
}
|
||||
configPath, err := deps.userConfigPath()
|
||||
if err != nil {
|
||||
return "warning: login saved, but no provider profile was written: " + err.Error()
|
||||
}
|
||||
ensured, err := config.EnsureCatalogProvider(configPath, provider)
|
||||
if err != nil {
|
||||
return "warning: login saved, but no provider profile was written: " + err.Error()
|
||||
}
|
||||
active := strings.EqualFold(strings.TrimSpace(ensured.Active), strings.TrimSpace(ensured.Name))
|
||||
switch {
|
||||
case ensured.Created && active:
|
||||
return fmt.Sprintf("Added provider %q to your config and set it active.", ensured.Name)
|
||||
case ensured.Created:
|
||||
return fmt.Sprintf("Added provider %q to your config; the active provider is still %q.\nSwitch with: zero providers use %s", ensured.Name, ensured.Active, ensured.Name)
|
||||
case active:
|
||||
return fmt.Sprintf("Provider %q is configured and active.", ensured.Name)
|
||||
default:
|
||||
return fmt.Sprintf("Provider %q is already configured.\nSwitch with: zero providers use %s", ensured.Name, ensured.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// runAuth dispatches `zero auth <command>` for provider OAuth login. It is
|
||||
// additive and independent of `zero mcp oauth` (MCP server auth), which is
|
||||
// unchanged.
|
||||
@@ -138,8 +174,10 @@ func runAuthChatGPT(args []string, stdout io.Writer, stderr io.Writer, deps appD
|
||||
if _, err := fmt.Fprint(stdout, statuses); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
if _, err := fmt.Fprint(stdout, "\nUse it with zero, e.g.:\n zero --provider chatgpt --model gpt-5.5\n"); err != nil {
|
||||
return exitCrash
|
||||
if line := ensureLoginProviderProfile(deps, "chatgpt"); line != "" {
|
||||
if _, err := fmt.Fprintf(stdout, "\n%s\n", line); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
@@ -331,6 +369,11 @@ func runAuthLogin(args []string, stdout io.Writer, stderr io.Writer, deps appDep
|
||||
if _, err := fmt.Fprintf(stdout, "Logged in to %s.\n%s\n", parsed.positional[0], oauth.FormatStatuses([]oauth.Status{status})); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
if line := ensureLoginProviderProfile(deps, provider); line != "" {
|
||||
if _, err := fmt.Fprintln(stdout, line); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
|
||||
@@ -2,18 +2,24 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
)
|
||||
|
||||
// withAuthStore points the provider OAuth store at a temp file for the test.
|
||||
// withAuthStore points the provider OAuth store at a temp file for the test,
|
||||
// pinning the file backend so an inherited ZERO_OAUTH_STORAGE=keyring can't
|
||||
// ignore the temp path and hit the OS keychain.
|
||||
func withAuthStore(t *testing.T) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "oauth-tokens.json")
|
||||
t.Setenv("ZERO_OAUTH_TOKENS_PATH", path)
|
||||
t.Setenv("ZERO_OAUTH_STORAGE", "file")
|
||||
return path
|
||||
}
|
||||
|
||||
@@ -197,3 +203,81 @@ func TestRunAuthLoginChatGPTRejectsScope(t *testing.T) {
|
||||
t.Fatalf("stderr = %q, want the ChatGPT-specific --scope rejection", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLoginProviderProfileAddsProviderWithoutStealingActive(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
seed := `{"activeProvider":"opengateway","providers":[{"name":"opengateway","provider_kind":"openai-compatible","baseURL":"https://gateway.example.com/v1","apiKeyStored":true,"model":"some-model"}]}`
|
||||
if err := os.WriteFile(configPath, []byte(seed), 0o600); err != nil {
|
||||
t.Fatalf("seed config: %v", err)
|
||||
}
|
||||
deps := appDeps{userConfigPath: func() (string, error) { return configPath, nil }}
|
||||
|
||||
line := ensureLoginProviderProfile(deps, "chatgpt")
|
||||
if !strings.Contains(line, `Added provider "chatgpt"`) {
|
||||
t.Fatalf("expected added-provider guidance, got %q", line)
|
||||
}
|
||||
if !strings.Contains(line, "zero providers use chatgpt") {
|
||||
t.Fatalf("expected switch hint, got %q", line)
|
||||
}
|
||||
|
||||
cfg := readCLIConfigFixture(t, configPath)
|
||||
if cfg.ActiveProvider != "opengateway" {
|
||||
t.Fatalf("active provider changed to %q", cfg.ActiveProvider)
|
||||
}
|
||||
names := make([]string, 0, len(cfg.Providers))
|
||||
for _, provider := range cfg.Providers {
|
||||
names = append(names, provider.Name)
|
||||
}
|
||||
if len(cfg.Providers) != 2 || cfg.Providers[1].CatalogID != "chatgpt" {
|
||||
t.Fatalf("expected chatgpt profile appended, got %v", names)
|
||||
}
|
||||
|
||||
// A second login must be a no-op with switch guidance, not a duplicate.
|
||||
line = ensureLoginProviderProfile(deps, "chatgpt")
|
||||
if !strings.Contains(line, "already configured") {
|
||||
t.Fatalf("expected already-configured guidance, got %q", line)
|
||||
}
|
||||
cfg = readCLIConfigFixture(t, configPath)
|
||||
if len(cfg.Providers) != 2 {
|
||||
t.Fatalf("repeat login duplicated the profile: %d providers", len(cfg.Providers))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLoginProviderProfileActivatesOnFreshConfig(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
deps := appDeps{userConfigPath: func() (string, error) { return configPath, nil }}
|
||||
|
||||
line := ensureLoginProviderProfile(deps, "chatgpt")
|
||||
if !strings.Contains(line, "set it active") {
|
||||
t.Fatalf("fresh config should adopt the login as active, got %q", line)
|
||||
}
|
||||
cfg := readCLIConfigFixture(t, configPath)
|
||||
if cfg.ActiveProvider != "chatgpt" {
|
||||
t.Fatalf("active provider = %q, want chatgpt", cfg.ActiveProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLoginProviderProfileSkipsNonCatalogProviders(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
deps := appDeps{userConfigPath: func() (string, error) { return configPath, nil }}
|
||||
|
||||
if line := ensureLoginProviderProfile(deps, "my-custom-oauth-server"); line != "" {
|
||||
t.Fatalf("custom OAuth server must not scaffold a profile, got %q", line)
|
||||
}
|
||||
if _, err := os.Stat(configPath); err == nil {
|
||||
t.Fatalf("config must not be created for a non-catalog login")
|
||||
}
|
||||
}
|
||||
|
||||
func readCLIConfigFixture(t *testing.T, path string) config.FileConfig {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
var cfg config.FileConfig
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
t.Fatalf("decode config: %v", err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
@@ -75,6 +75,12 @@ func runProviders(args []string, stdout io.Writer, stderr io.Writer, deps appDep
|
||||
if command == "use" {
|
||||
return runProvidersUse(args, stdout, stderr, deps)
|
||||
}
|
||||
if command == "remove" || command == "rm" {
|
||||
return runProvidersRemove(args, stdout, stderr, deps)
|
||||
}
|
||||
if command == "rename" {
|
||||
return runProvidersRename(args, stdout, stderr, deps)
|
||||
}
|
||||
if command == "setup" {
|
||||
return runProvidersSetup(args, stdout, stderr, deps)
|
||||
}
|
||||
@@ -246,6 +252,20 @@ func parseCommandCenterArgs(args []string, allowModelFilters bool, allowProvider
|
||||
|
||||
func summarizeConfig(resolved config.ResolvedConfig) configSummary {
|
||||
summary := zerocommands.ConfigSnapshotFromResolved(resolved)
|
||||
// Mark keyless profiles that authenticate via a stored OAuth login (e.g.
|
||||
// ChatGPT) so the list shows "oauth login" instead of "api key: not set" —
|
||||
// the same candidate matching the runtime resolver uses.
|
||||
logins := oauthLoggedInProviders()
|
||||
if len(logins) > 0 {
|
||||
for index := range summary.Providers {
|
||||
for _, profile := range resolved.Providers {
|
||||
if profile.Name == summary.Providers[index].Name {
|
||||
summary.Providers[index].OAuthLogin = providerHasOAuthLogin(profile, logins)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.SliceStable(summary.Providers, func(i int, j int) bool {
|
||||
if summary.Providers[i].Active != summary.Providers[j].Active {
|
||||
return summary.Providers[i].Active
|
||||
@@ -323,7 +343,7 @@ func formatProviderSummaries(command string, providers []providerSummary) string
|
||||
"model: "+displayCLIValue(provider.Model, "none"),
|
||||
"api model: "+displayCLIValue(provider.APIModel, "unknown"),
|
||||
"base url: "+displayCLIValue(provider.BaseURL, "default"),
|
||||
"api key: "+apiKeyState(provider.APIKeySet),
|
||||
"api key: "+providerCredentialState(provider),
|
||||
)
|
||||
if provider.Message != "" {
|
||||
lines = append(lines, "status: "+provider.Status+" - "+provider.Message)
|
||||
@@ -340,7 +360,7 @@ func formatProviderLine(provider providerSummary) string {
|
||||
if provider.Active {
|
||||
marker = "*"
|
||||
}
|
||||
line := fmt.Sprintf("%s %s [%s] model=%s apiModel=%s api key: %s", marker, displayCLIValue(provider.Name, "none"), displayCLIValue(provider.ProviderKind, "unknown"), displayCLIValue(provider.Model, "none"), displayCLIValue(provider.APIModel, "unknown"), apiKeyState(provider.APIKeySet))
|
||||
line := fmt.Sprintf("%s %s [%s] model=%s apiModel=%s api key: %s", marker, displayCLIValue(provider.Name, "none"), displayCLIValue(provider.ProviderKind, "unknown"), displayCLIValue(provider.Model, "none"), displayCLIValue(provider.APIModel, "unknown"), providerCredentialState(provider))
|
||||
if provider.Message != "" {
|
||||
line += " (" + provider.Status + ": " + provider.Message + ")"
|
||||
}
|
||||
@@ -413,6 +433,15 @@ func apiKeyState(set bool) string {
|
||||
return "not set"
|
||||
}
|
||||
|
||||
// providerCredentialState renders the credential column: a configured key wins,
|
||||
// then a stored OAuth login, then "not set".
|
||||
func providerCredentialState(provider providerSummary) string {
|
||||
if !provider.APIKeySet && provider.OAuthLogin {
|
||||
return "oauth login"
|
||||
}
|
||||
return apiKeyState(provider.APIKeySet)
|
||||
}
|
||||
|
||||
func displayCLIValue(value string, fallback string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fallback
|
||||
@@ -466,6 +495,8 @@ func writeProvidersHelp(w io.Writer) error {
|
||||
zero providers add <catalog-id> [flags]
|
||||
zero providers check [name] [flags]
|
||||
zero providers use <name> [flags]
|
||||
zero providers remove <name> [flags]
|
||||
zero providers rename <old> <new> [flags]
|
||||
zero providers setup <catalog-id> [flags]
|
||||
zero providers detect [flags]
|
||||
zero providers models [name] [flags]
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
"github.com/Gitlawb/zero/internal/providerhealth"
|
||||
"github.com/Gitlawb/zero/internal/zerocommands"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
@@ -835,3 +836,57 @@ func (commandCenterProvider) StreamCompletion(context.Context, zeroruntime.Compl
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func TestProviderCredentialStateShowsOAuthLogin(t *testing.T) {
|
||||
if got := providerCredentialState(providerSummary{APIKeySet: true}); got != "set" {
|
||||
t.Fatalf("keyed provider = %q, want set", got)
|
||||
}
|
||||
if got := providerCredentialState(providerSummary{OAuthLogin: true}); got != "oauth login" {
|
||||
t.Fatalf("token-login provider = %q, want oauth login", got)
|
||||
}
|
||||
if got := providerCredentialState(providerSummary{}); got != "not set" {
|
||||
t.Fatalf("credential-less provider = %q, want not set", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProvidersListMarksOAuthLoginProviders: a keyless profile whose credential
|
||||
// is a stored OAuth login (the shape `zero auth chatgpt` now writes) must render
|
||||
// as "oauth login", not as a broken "api key: not set" entry.
|
||||
func TestProvidersListMarksOAuthLoginProviders(t *testing.T) {
|
||||
tokensPath := filepath.Join(t.TempDir(), "oauth-tokens.json")
|
||||
t.Setenv("ZERO_OAUTH_TOKENS_PATH", tokensPath)
|
||||
store, err := oauth.NewStore(oauth.StoreOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("oauth store: %v", err)
|
||||
}
|
||||
if err := store.Save(oauth.ProviderKey("chatgpt"), oauth.Token{AccessToken: "bearer-123"}); err != nil {
|
||||
t.Fatalf("save token: %v", err)
|
||||
}
|
||||
|
||||
resolved := config.ResolvedConfig{
|
||||
ActiveProvider: "opengateway",
|
||||
Providers: []config.ProviderProfile{
|
||||
{Name: "opengateway", ProviderKind: config.ProviderKindOpenAICompatible, BaseURL: "https://gateway.example.com/v1", APIKeyStored: true, Model: "some-model"},
|
||||
{Name: "chatgpt", CatalogID: "chatgpt", ProviderKind: config.ProviderKindOpenAICompatible, BaseURL: "https://chatgpt.com/backend-api/codex", Model: "gpt-5.5"},
|
||||
},
|
||||
}
|
||||
summary := summarizeConfig(resolved)
|
||||
byName := map[string]providerSummary{}
|
||||
for _, provider := range summary.Providers {
|
||||
byName[provider.Name] = provider
|
||||
}
|
||||
if !byName["opengateway"].APIKeySet {
|
||||
t.Fatalf("stored-key provider must report a credential: %+v", byName["opengateway"])
|
||||
}
|
||||
if byName["opengateway"].OAuthLogin {
|
||||
t.Fatalf("key-authed provider must not claim the OAuth login: %+v", byName["opengateway"])
|
||||
}
|
||||
if !byName["chatgpt"].OAuthLogin || byName["chatgpt"].APIKeySet {
|
||||
t.Fatalf("chatgpt must be marked oauth-login and keyless: %+v", byName["chatgpt"])
|
||||
}
|
||||
|
||||
rendered := formatProviderSummaries("list", summary.Providers)
|
||||
if !strings.Contains(rendered, "oauth login") {
|
||||
t.Fatalf("list should render the oauth login state, got:\n%s", rendered)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
neturl "net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/dictation"
|
||||
"github.com/Gitlawb/zero/internal/tui"
|
||||
)
|
||||
|
||||
// Default batch cloud models (§8). Overridable via stt.model.
|
||||
const (
|
||||
defaultGroqSTTModel = "whisper-large-v3-turbo"
|
||||
defaultOpenAISTTModel = "whisper-1"
|
||||
groqSTTBaseURL = "https://api.groq.com/openai/v1"
|
||||
)
|
||||
|
||||
// newDictationTranscriberFactory returns the closure the TUI calls to build a
|
||||
// transcriber when a recording starts. It lives here, not in the TUI, because it
|
||||
// resolves provider API keys (credstore + env + configured providers) — logic
|
||||
// that belongs with the rest of the CLI's credential handling.
|
||||
//
|
||||
// serverMgr is the shared, session-long sherpa-onnx streaming server; it is
|
||||
// spawned lazily on the first local-streaming recording and torn down on exit.
|
||||
//
|
||||
// preferStreaming asks for the streaming backend; the returned bool reports
|
||||
// whether streaming was actually built (false → the caller uses the batch
|
||||
// pipeline). A misconfigured streaming backend surfaces its own SetupError.
|
||||
func newDictationTranscriberFactory(resolved config.ResolvedConfig, userConfigPath string, serverMgr *dictation.ServerManager) func(config.STTConfig, bool) (tui.Transcriber, bool, error) {
|
||||
return func(stt config.STTConfig, preferStreaming bool) (tui.Transcriber, bool, error) {
|
||||
// stt is passed in (not captured) so a mid-session change — e.g. the F9
|
||||
// auto-download writing localModelPath — takes effect on the next call.
|
||||
if serverMgr != nil {
|
||||
serverMgr.SetModelPath(stt.LocalModelPath)
|
||||
}
|
||||
if preferStreaming {
|
||||
t, err := buildStreamingTranscriber(stt, resolved, userConfigPath, serverMgr)
|
||||
switch {
|
||||
case err == nil && t != nil:
|
||||
return t, true, nil
|
||||
case err != nil && stt.STTStreamProvider() != config.STTProviderLocal:
|
||||
// An explicitly-chosen cloud streaming backend that failed to build
|
||||
// (e.g. a missing key) should surface its own error, not silently
|
||||
// fall back to batch — the user asked for that backend specifically.
|
||||
return nil, true, err
|
||||
}
|
||||
// The default local streaming backend isn't set up: fall through to the
|
||||
// batch pipeline so a cloud-batch provider (or a batch-only setup) still
|
||||
// works instead of erroring on a streaming server the user never asked for.
|
||||
}
|
||||
t, err := buildBatchTranscriber(stt, resolved, userConfigPath)
|
||||
return t, false, err
|
||||
}
|
||||
}
|
||||
|
||||
// buildStreamingTranscriber constructs the configured streaming backend, or
|
||||
// returns (nil, nil) when streaming is not applicable so the caller falls back
|
||||
// to batch.
|
||||
func buildStreamingTranscriber(stt config.STTConfig, resolved config.ResolvedConfig, userConfigPath string, serverMgr *dictation.ServerManager) (dictation.Transcriber, error) {
|
||||
switch stt.STTStreamProvider() {
|
||||
case config.STTProviderDeepgram:
|
||||
key := resolveSTTKey(resolved, userConfigPath, dictation.StreamProviderDeepgram, "DEEPGRAM_API_KEY")
|
||||
return dictation.NewDeepgramTranscriber(dictation.DeepgramConfig{
|
||||
APIKey: key,
|
||||
Model: stt.StreamModel,
|
||||
Language: stt.Language,
|
||||
})
|
||||
case config.STTProviderOpenAI:
|
||||
key := resolveSTTKey(resolved, userConfigPath, dictation.ProviderOpenAI, "OPENAI_API_KEY")
|
||||
return dictation.NewOpenAIRealtimeTranscriber(dictation.OpenAIRealtimeConfig{
|
||||
APIKey: key,
|
||||
Model: stt.StreamModel,
|
||||
})
|
||||
default: // local streaming via the warm sherpa-onnx server
|
||||
// STTStreamProvider() defaults to "local" whenever streamProvider is unset —
|
||||
// including when the user picked a CLOUD batch provider (Groq/OpenAI) and
|
||||
// never touched streaming. Don't silently stream through the local sherpa
|
||||
// server in that case: local streaming is only appropriate when it was
|
||||
// explicitly requested (streamProvider: "local") OR the batch provider is
|
||||
// itself local. Otherwise report "not applicable" so the factory falls back
|
||||
// to the cloud batch pipeline the user actually chose.
|
||||
localWanted := stt.StreamProvider == config.STTProviderLocal || stt.STTProvider() == config.STTProviderLocal
|
||||
if !localWanted {
|
||||
return nil, nil
|
||||
}
|
||||
// The local streaming transcriber validates the model only lazily (at
|
||||
// connect time), so gate on the model path here — with none set, report
|
||||
// "not applicable" (nil, nil) so the factory falls back to the batch
|
||||
// pipeline instead of committing to a server that can't start.
|
||||
if serverMgr == nil || strings.TrimSpace(stt.LocalModelPath) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return dictation.NewLocalStreamingTranscriber(serverMgr), nil
|
||||
}
|
||||
}
|
||||
|
||||
// newDictationServerManager builds the warm sherpa-onnx streaming server manager
|
||||
// from config. Construction is side-effect free — the server spawns lazily on
|
||||
// the first local-streaming recording — so it is always safe to create.
|
||||
func newDictationServerManager(stt config.STTConfig) *dictation.ServerManager {
|
||||
return dictation.NewServerManager(dictation.ServerConfig{
|
||||
Binary: stt.LocalServerBinary,
|
||||
ModelPath: stt.LocalModelPath,
|
||||
Port: stt.LocalServerPort,
|
||||
NumThreads: stt.NumThreads,
|
||||
})
|
||||
}
|
||||
|
||||
// buildBatchTranscriber constructs the configured batch transcriber, resolving
|
||||
// credentials for the cloud providers.
|
||||
func buildBatchTranscriber(stt config.STTConfig, resolved config.ResolvedConfig, userConfigPath string) (dictation.Transcriber, error) {
|
||||
switch stt.STTProvider() {
|
||||
case config.STTProviderGroq:
|
||||
key := resolveSTTKey(resolved, userConfigPath, dictation.ProviderGroq, "GROQ_API_KEY")
|
||||
model := stt.Model
|
||||
if model == "" {
|
||||
model = defaultGroqSTTModel
|
||||
}
|
||||
return dictation.NewCloudTranscriber(dictation.CloudConfig{
|
||||
Provider: dictation.ProviderGroq,
|
||||
BaseURL: groqSTTBaseURL,
|
||||
APIKey: key,
|
||||
Model: model,
|
||||
Language: stt.Language,
|
||||
})
|
||||
case config.STTProviderOpenAI:
|
||||
key := resolveSTTKey(resolved, userConfigPath, dictation.ProviderOpenAI, "OPENAI_API_KEY")
|
||||
model := stt.Model
|
||||
if model == "" {
|
||||
model = defaultOpenAISTTModel
|
||||
}
|
||||
return dictation.NewCloudTranscriber(dictation.CloudConfig{
|
||||
Provider: dictation.ProviderOpenAI,
|
||||
BaseURL: config.OpenAIBaseURL,
|
||||
APIKey: key,
|
||||
Model: model,
|
||||
Language: stt.Language,
|
||||
})
|
||||
default: // local
|
||||
return dictation.NewLocalTranscriber(dictation.LocalConfig{
|
||||
Binary: stt.LocalBinary,
|
||||
ModelPath: stt.LocalModelPath,
|
||||
NumThreads: stt.NumThreads,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// sttEnvVar returns the conventional environment variable a cloud STT provider's
|
||||
// key is read from.
|
||||
func sttEnvVar(provider string) string {
|
||||
switch provider {
|
||||
case dictation.ProviderGroq:
|
||||
return "GROQ_API_KEY"
|
||||
case dictation.ProviderOpenAI: // == StreamProviderOpenAI ("openai")
|
||||
return "OPENAI_API_KEY"
|
||||
case dictation.StreamProviderDeepgram:
|
||||
return "DEEPGRAM_API_KEY"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// newSTTKeyStatus returns a predicate reporting whether a cloud STT provider
|
||||
// already has a resolvable API key (configured provider, credstore, or env).
|
||||
func newSTTKeyStatus(resolved config.ResolvedConfig, userConfigPath string) func(string) bool {
|
||||
return func(provider string) bool {
|
||||
return resolveSTTKey(resolved, userConfigPath, provider, sttEnvVar(provider)) != ""
|
||||
}
|
||||
}
|
||||
|
||||
// newSaveSTTKey returns a function that stores a cloud STT provider's API key in
|
||||
// the encrypted credential store, so the inline prompt can persist it.
|
||||
func newSaveSTTKey(userConfigPath string) func(string, string) error {
|
||||
return func(provider, key string) error {
|
||||
if strings.TrimSpace(userConfigPath) == "" {
|
||||
return fmt.Errorf("no config path to store the key")
|
||||
}
|
||||
store, err := config.ProviderKeyStoreAt(filepath.Dir(userConfigPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return store.Set(provider, strings.TrimSpace(key))
|
||||
}
|
||||
}
|
||||
|
||||
// resolveSTTKey finds an API key for an STT cloud provider, reusing Zero's
|
||||
// existing credentials (§2 — "zero new credential UI"). Precedence: a matching
|
||||
// configured provider's resolved key, then the encrypted credential store, then
|
||||
// the provider's conventional environment variable.
|
||||
func resolveSTTKey(resolved config.ResolvedConfig, userConfigPath, provider, envVar string) string {
|
||||
if key := keyFromConfiguredProviders(resolved, provider); key != "" {
|
||||
return key
|
||||
}
|
||||
if userConfigPath != "" {
|
||||
if store, err := config.ProviderKeyStoreAt(filepath.Dir(userConfigPath)); err == nil {
|
||||
if key, ok, _ := store.Get(provider); ok && strings.TrimSpace(key) != "" {
|
||||
return strings.TrimSpace(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(os.Getenv(envVar))
|
||||
}
|
||||
|
||||
// keyFromConfiguredProviders returns the resolved API key of a configured
|
||||
// provider that belongs to the given cloud provider (matched by catalog id,
|
||||
// provider name/kind, or base-URL host), or "" when none is configured.
|
||||
func keyFromConfiguredProviders(resolved config.ResolvedConfig, provider string) string {
|
||||
provider = strings.ToLower(provider)
|
||||
profiles := append([]config.ProviderProfile{resolved.Provider}, resolved.Providers...)
|
||||
for _, p := range profiles {
|
||||
if strings.TrimSpace(p.APIKey) == "" {
|
||||
continue
|
||||
}
|
||||
if providerMatches(p, provider) {
|
||||
return strings.TrimSpace(p.APIKey)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func providerMatches(p config.ProviderProfile, provider string) bool {
|
||||
provider = strings.ToLower(provider)
|
||||
// Match on stable identifiers (catalog id, provider name/kind, profile
|
||||
// name) rather than raw BaseURL substring matching: a Groq profile's
|
||||
// BaseURL (api.groq.com/openai/v1) contains "openai", so a substring
|
||||
// check can hand the wrong key to an OpenAI STT lookup and fail auth.
|
||||
for _, f := range []string{
|
||||
strings.ToLower(p.CatalogID),
|
||||
strings.ToLower(p.Provider),
|
||||
strings.ToLower(string(p.ProviderKind)),
|
||||
strings.ToLower(p.Name),
|
||||
} {
|
||||
if f != "" && strings.Contains(f, provider) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// BaseURL: compare the parsed host only, not a raw substring on the
|
||||
// full URL (which lets "openai" match api.groq.com/openai/v1).
|
||||
if u, err := neturl.Parse(p.BaseURL); err == nil && u.Host != "" {
|
||||
if host := strings.ToLower(u.Host); strings.Contains(host, provider) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/dictation"
|
||||
)
|
||||
|
||||
func TestFactoryFallsBackToBatchWhenLocalStreamingNotConfigured(t *testing.T) {
|
||||
t.Setenv("GROQ_API_KEY", "gsk-test")
|
||||
resolved := config.ResolvedConfig{STT: config.STTConfig{
|
||||
Provider: config.STTProviderGroq, // cloud batch
|
||||
// StreamProvider unset → defaults to local, which has no model configured.
|
||||
}}
|
||||
factory := newDictationTranscriberFactory(resolved, "", newDictationServerManager(resolved.STT))
|
||||
|
||||
tr, streaming, err := factory(resolved.STT, true) // prefer streaming
|
||||
if err != nil {
|
||||
t.Fatalf("expected batch fallback, got error: %v", err)
|
||||
}
|
||||
if streaming {
|
||||
t.Error("should have fallen back to batch (streaming=false)")
|
||||
}
|
||||
if tr == nil {
|
||||
t.Error("expected a Groq batch transcriber")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactoryCloudProviderIgnoresLeftoverLocalModel(t *testing.T) {
|
||||
// Regression: pick local once (LocalModelPath gets set), then switch to a
|
||||
// cloud batch provider. streamProvider stays unset (defaults to "local"), so
|
||||
// the streaming factory must NOT fire up the sherpa server on the leftover
|
||||
// path — it should fall back to the cloud batch pipeline the user chose.
|
||||
t.Setenv("GROQ_API_KEY", "gsk-test")
|
||||
stt := config.STTConfig{
|
||||
Provider: config.STTProviderGroq, // cloud batch, chosen now
|
||||
LocalModelPath: "/leftover/model/dir", // set by an earlier local selection
|
||||
}
|
||||
resolved := config.ResolvedConfig{STT: stt}
|
||||
factory := newDictationTranscriberFactory(resolved, "", newDictationServerManager(stt))
|
||||
|
||||
tr, streaming, err := factory(stt, true) // prefer streaming
|
||||
if err != nil {
|
||||
t.Fatalf("expected cloud batch fallback, got error: %v", err)
|
||||
}
|
||||
if streaming {
|
||||
t.Error("a cloud provider must not stream through the local sherpa server")
|
||||
}
|
||||
if tr == nil {
|
||||
t.Error("expected a Groq batch transcriber")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactorySurfacesExplicitCloudStreamingError(t *testing.T) {
|
||||
// Deepgram chosen explicitly but no key resolvable → surface the setup error.
|
||||
resolved := config.ResolvedConfig{STT: config.STTConfig{
|
||||
StreamProvider: config.STTProviderDeepgram,
|
||||
}}
|
||||
factory := newDictationTranscriberFactory(resolved, "", nil)
|
||||
|
||||
_, streaming, err := factory(resolved.STT, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected a setup error for missing Deepgram key")
|
||||
}
|
||||
if !streaming {
|
||||
t.Error("an explicit cloud streaming choice should report streaming=true even on error")
|
||||
}
|
||||
var setupErr *dictation.SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Errorf("want *SetupError, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactoryLocalBatchSetupErrorWhenNoModel(t *testing.T) {
|
||||
resolved := config.ResolvedConfig{STT: config.STTConfig{Provider: config.STTProviderLocal}}
|
||||
factory := newDictationTranscriberFactory(resolved, "", newDictationServerManager(resolved.STT))
|
||||
|
||||
_, _, err := factory(resolved.STT, true)
|
||||
var setupErr *dictation.SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError guiding local model setup, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSTTKeyPrefersConfiguredProvider(t *testing.T) {
|
||||
resolved := config.ResolvedConfig{
|
||||
Providers: []config.ProviderProfile{
|
||||
{Name: "groq", CatalogID: "groq", BaseURL: "https://api.groq.com/openai/v1", APIKey: "gsk-configured"},
|
||||
},
|
||||
}
|
||||
if got := resolveSTTKey(resolved, "", dictation.ProviderGroq, "GROQ_API_KEY"); got != "gsk-configured" {
|
||||
t.Errorf("key = %q, want the configured provider's key", got)
|
||||
}
|
||||
}
|
||||
+15
-4
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/Gitlawb/zero/internal/agent"
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/errhint"
|
||||
"github.com/Gitlawb/zero/internal/imageinput"
|
||||
"github.com/Gitlawb/zero/internal/lsp"
|
||||
"github.com/Gitlawb/zero/internal/modelregistry"
|
||||
@@ -543,6 +544,7 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
EnabledTools: options.enabledTools,
|
||||
DisabledTools: options.disabledTools,
|
||||
OnText: writer.text,
|
||||
OnReasoning: writer.reasoning,
|
||||
OnToolCall: func(call agent.ToolCall) {
|
||||
writer.toolCall(call, registry)
|
||||
sessionRecorder.append(sessions.EventToolCall, map[string]any{
|
||||
@@ -709,16 +711,17 @@ func runExec(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) in
|
||||
// sessions); it is a no-op when self-correct is off.
|
||||
func newExecSelfCorrector(enabled bool, workspaceRoot string, autonomy string) (*agent.SelfCorrector, func(context.Context, string) string, func()) {
|
||||
// The manager is created regardless of --self-correct: it also backs the
|
||||
// always-on inline post-edit diagnostics (agent.NewFileDiagnostics), and it
|
||||
// stays lazy — no language server is spawned unless an edited file's
|
||||
// language actually has one installed on PATH.
|
||||
// always-on background post-edit diagnostics (agent.NewFileDiagnostics,
|
||||
// collected off the tool-call path by the loop), and it stays lazy — no
|
||||
// language server is spawned unless an edited file's language actually has
|
||||
// one installed on PATH.
|
||||
manager := lsp.NewManager(workspaceRoot)
|
||||
cleanup := func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = manager.Shutdown(ctx)
|
||||
}
|
||||
fileDiagnostics := agent.NewFileDiagnostics(manager)
|
||||
fileDiagnostics := agent.NewFileDiagnostics(manager, workspaceRoot)
|
||||
if !enabled {
|
||||
return nil, fileDiagnostics, cleanup
|
||||
}
|
||||
@@ -961,6 +964,14 @@ func writeExecProviderError(stdout io.Writer, stderr io.Writer, format execOutpu
|
||||
if _, err := fmt.Fprintf(stderr, "[zero] %s\n", message); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
// Append a one-line next step for recognized provider failures (auth /
|
||||
// rate-limit / connectivity / …). The classifier gates on a provider-origin
|
||||
// marker, so non-provider codes (sandbox_error, mcp_error) never draw a hint.
|
||||
if hint := errhint.CLIHint(errors.New(message)); hint != "" {
|
||||
if _, err := fmt.Fprintf(stderr, "[zero] %s\n", hint); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
return exitProvider
|
||||
}
|
||||
|
||||
|
||||
@@ -529,6 +529,45 @@ func TestRunExecStreamJSONRunStartUsesResolvedAPIModel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunExecStreamJSONEmitsReasoningEvents(t *testing.T) {
|
||||
t.Setenv("XDG_DATA_HOME", t.TempDir())
|
||||
cwd := t.TempDir()
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
exitCode := runWithDeps([]string{"exec", "--output-format", "stream-json", "think then answer"}, &stdout, &stderr, appDeps{
|
||||
getwd: func() (string, error) {
|
||||
return cwd, nil
|
||||
},
|
||||
resolveConfig: func(_ string, _ config.Overrides) (config.ResolvedConfig, error) {
|
||||
return execResolvedConfig(), nil
|
||||
},
|
||||
newProvider: func(config.ProviderProfile) (zeroruntime.Provider, error) {
|
||||
return reasoningExecProvider{}, nil
|
||||
},
|
||||
})
|
||||
|
||||
if exitCode != exitSuccess {
|
||||
t.Fatalf("expected exit code %d, got %d: %s", exitSuccess, exitCode, stderr.String())
|
||||
}
|
||||
events := decodeJSONLines(t, stdout.String())
|
||||
reasoningIdx, reasoning := findJSONEventIndex(t, events, "reasoning")
|
||||
if reasoning["delta"] != "Thinking. " {
|
||||
t.Fatalf("unexpected reasoning event: %#v", reasoning)
|
||||
}
|
||||
textIdx, text := findJSONEventIndex(t, events, "text")
|
||||
if text["delta"] != "done" {
|
||||
t.Fatalf("unexpected text event: %#v", text)
|
||||
}
|
||||
if reasoningIdx >= textIdx {
|
||||
t.Fatalf("expected reasoning event before text event, got indices %d and %d", reasoningIdx, textIdx)
|
||||
}
|
||||
final := findJSONEvent(t, events, "final")
|
||||
if final["text"] != "done" {
|
||||
t.Fatalf("reasoning must not be folded into final answer: %#v", final)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecEventWriterTruncatesStreamJSONToolResults(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
writer := execEventWriter{
|
||||
@@ -751,6 +790,17 @@ type toolCallingExecProvider struct {
|
||||
answer string
|
||||
}
|
||||
|
||||
type reasoningExecProvider struct{}
|
||||
|
||||
func (reasoningExecProvider) StreamCompletion(context.Context, zeroruntime.CompletionRequest) (<-chan zeroruntime.StreamEvent, error) {
|
||||
ch := make(chan zeroruntime.StreamEvent, 3)
|
||||
ch <- zeroruntime.StreamEvent{Type: zeroruntime.StreamEventReasoning, Content: "Thinking. "}
|
||||
ch <- zeroruntime.StreamEvent{Type: zeroruntime.StreamEventText, Content: "done"}
|
||||
ch <- zeroruntime.StreamEvent{Type: zeroruntime.StreamEventDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (provider toolCallingExecProvider) StreamCompletion(ctx context.Context, request zeroruntime.CompletionRequest) (<-chan zeroruntime.StreamEvent, error) {
|
||||
for _, message := range request.Messages {
|
||||
if message.Role == zeroruntime.MessageRoleTool {
|
||||
@@ -777,13 +827,19 @@ func (provider toolCallingExecProvider) StreamCompletion(ctx context.Context, re
|
||||
|
||||
func findJSONEvent(t *testing.T, events []map[string]any, eventType string) map[string]any {
|
||||
t.Helper()
|
||||
for _, event := range events {
|
||||
_, event := findJSONEventIndex(t, events, eventType)
|
||||
return event
|
||||
}
|
||||
|
||||
func findJSONEventIndex(t *testing.T, events []map[string]any, eventType string) (int, map[string]any) {
|
||||
t.Helper()
|
||||
for idx, event := range events {
|
||||
if event["type"] == eventType {
|
||||
return event
|
||||
return idx, event
|
||||
}
|
||||
}
|
||||
t.Fatalf("event %q not found in %#v", eventType, events)
|
||||
return nil
|
||||
return -1, nil
|
||||
}
|
||||
|
||||
func findSessionEvent(t *testing.T, events []sessions.Event, eventType sessions.EventType) sessions.Event {
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A recognized provider error in text mode prints the raw message plus a one-line
|
||||
// actionable hint; a non-provider error (or JSON mode) prints no hint.
|
||||
func TestWriteExecProviderErrorAppendsHint(t *testing.T) {
|
||||
t.Run("provider auth error gets a hint", func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := writeExecProviderError(&stdout, &stderr, execOutputText, "provider_error",
|
||||
"auth error: your API key is missing or invalid")
|
||||
if code != exitProvider {
|
||||
t.Fatalf("exit code = %d, want exitProvider", code)
|
||||
}
|
||||
out := stderr.String()
|
||||
if !strings.Contains(out, "auth error:") {
|
||||
t.Fatalf("expected raw message, got %q", out)
|
||||
}
|
||||
if !strings.Contains(out, "zero auth") {
|
||||
t.Fatalf("expected an actionable hint referencing `zero auth`, got %q", out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-provider error gets no hint", func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
writeExecProviderError(&stdout, &stderr, execOutputText, "sandbox_error",
|
||||
"sandbox setup failed: permission denied")
|
||||
out := stderr.String()
|
||||
// Exactly one "[zero]" line — no spurious hint attached to a local error.
|
||||
if n := strings.Count(out, "[zero]"); n != 1 {
|
||||
t.Fatalf("expected exactly one [zero] line for a non-provider error, got %d:\n%s", n, out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("json mode never appends a hint line", func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
writeExecProviderError(&stdout, &stderr, execOutputJSON, "provider_error",
|
||||
"auth error: bad key")
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("json mode must not write to stderr, got %q", stderr.String())
|
||||
}
|
||||
if strings.Contains(stdout.String(), "zero auth") {
|
||||
t.Fatalf("json mode must not inject a hint into the structured payload, got %q", stdout.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -80,6 +80,16 @@ func (writer *execEventWriter) text(delta string) {
|
||||
writer.writeStdout(delta)
|
||||
}
|
||||
|
||||
func (writer *execEventWriter) reasoning(delta string) {
|
||||
if writer.format == execOutputJSON {
|
||||
writer.writeJSON(map[string]any{"type": "reasoning", "delta": delta})
|
||||
return
|
||||
}
|
||||
if writer.format == execOutputStreamJSON {
|
||||
writer.writeStreamJSON(streamjson.Event{Type: streamjson.EventReasoning, RunID: writer.runID, Delta: delta})
|
||||
}
|
||||
}
|
||||
|
||||
func (writer *execEventWriter) toolCall(call agent.ToolCall, registry *tools.Registry) {
|
||||
if writer.format == execOutputJSON {
|
||||
writer.writeJSON(map[string]any{
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/providermodelcatalog"
|
||||
"github.com/Gitlawb/zero/internal/providermodeldiscovery"
|
||||
)
|
||||
|
||||
@@ -51,6 +52,19 @@ func runProvidersModels(args []string, stdout io.Writer, stderr io.Writer, deps
|
||||
return writeAppError(stderr, err.Error(), exitProvider)
|
||||
}
|
||||
|
||||
// Apply provider-specific model filtering for catalog-backed profiles.
|
||||
// For example, opencode-go-anthropic-compatible only permits Qwen and
|
||||
// MiniMax model IDs; the raw /zen/go/v1/models endpoint returns many more.
|
||||
if profile.CatalogID != "" {
|
||||
filtered := make([]providermodeldiscovery.Model, 0, len(models))
|
||||
for _, model := range models {
|
||||
if providermodelcatalog.ModelIDAllowedForProvider(profile.CatalogID, model.ID) {
|
||||
filtered = append(filtered, model)
|
||||
}
|
||||
}
|
||||
models = filtered
|
||||
}
|
||||
|
||||
if options.json {
|
||||
items := make([]map[string]any, 0, len(models))
|
||||
for _, model := range models {
|
||||
|
||||
@@ -132,6 +132,62 @@ func TestRunProvidersModelsUnknownProviderFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProvidersModelsFiltersForCatalogProfile(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
deps := commandCenterDeps(t)
|
||||
// Override the resolved config to include a catalog-backed profile.
|
||||
deps.resolveConfig = func(_ string, _ config.Overrides) (config.ResolvedConfig, error) {
|
||||
profile := config.ProviderProfile{
|
||||
Name: "opencode-go",
|
||||
ProviderKind: config.ProviderKindAnthropicCompat,
|
||||
CatalogID: "opencode-go-anthropic-compatible",
|
||||
BaseURL: "https://api.example.com/zen/go/v1",
|
||||
APIKey: "sk-test",
|
||||
Model: "minimax-max",
|
||||
}
|
||||
return config.ResolvedConfig{
|
||||
ActiveProvider: "opencode-go",
|
||||
Providers: []config.ProviderProfile{profile},
|
||||
Provider: profile,
|
||||
}, nil
|
||||
}
|
||||
deps.discoverProviderModels = func(_ context.Context, _ config.ProviderProfile) ([]providermodeldiscovery.Model, error) {
|
||||
return []providermodeldiscovery.Model{
|
||||
{ID: "deepseek-chat", Description: "DeepSeek Chat"},
|
||||
{ID: "qwen-max", Description: "Qwen Max"},
|
||||
{ID: "minimax-max-01", Description: "MiniMax Max"},
|
||||
{ID: "kimi-k2", Description: "Kimi K2"},
|
||||
{ID: "glm-4", Description: "GLM-4"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
exitCode := runWithDeps([]string{"providers", "models"}, &stdout, &stderr, deps)
|
||||
|
||||
if exitCode != exitSuccess {
|
||||
t.Fatalf("exit = %d, want %d: %s", exitCode, exitSuccess, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
// Only Qwen and MiniMax model IDs should survive the filter.
|
||||
if strings.Contains(out, "deepseek-chat") {
|
||||
t.Error("output should not contain deepseek-chat (blocked by opencode-go-anthropic filter)")
|
||||
}
|
||||
if strings.Contains(out, "kimi-k2") {
|
||||
t.Error("output should not contain kimi-k2 (blocked by opencode-go-anthropic filter)")
|
||||
}
|
||||
if strings.Contains(out, "glm-4") {
|
||||
t.Error("output should not contain glm-4 (blocked by opencode-go-anthropic filter)")
|
||||
}
|
||||
for _, want := range []string{"qwen-max", "minimax-max-01"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("output missing allowed model %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(out, "2 models discovered") {
|
||||
t.Fatalf("output should show 2 models:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProvidersModelsSurfacesDiscoveryError(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
@@ -306,3 +307,143 @@ func providerCommandArg(value string) string {
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type providerNamesOptions struct {
|
||||
names []string
|
||||
json bool
|
||||
}
|
||||
|
||||
func parseProviderNamesArgs(args []string, want int, usage string) (providerNamesOptions, bool, error) {
|
||||
options := providerNamesOptions{}
|
||||
for _, arg := range args {
|
||||
switch {
|
||||
case arg == "-h" || arg == "--help" || arg == "help":
|
||||
return options, true, nil
|
||||
case arg == "--json":
|
||||
options.json = true
|
||||
case strings.HasPrefix(arg, "-"):
|
||||
return options, false, execUsageError{fmt.Sprintf("unknown flag %q", arg)}
|
||||
default:
|
||||
options.names = append(options.names, arg)
|
||||
}
|
||||
}
|
||||
if len(options.names) != want {
|
||||
return options, false, execUsageError{usage}
|
||||
}
|
||||
return options, false, nil
|
||||
}
|
||||
|
||||
// runProvidersRemove deletes a saved provider profile and its stored API key.
|
||||
// The OAuth token (if any) is kept — logins outlive profiles so re-adding the
|
||||
// provider needs no new browser round-trip; `zero auth logout <name>` removes it.
|
||||
func runProvidersRemove(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int {
|
||||
options, help, err := parseProviderNamesArgs(args, 1, "usage: zero providers remove <name>")
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
if help {
|
||||
if err := writeProvidersHelp(stdout); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
configPath, err := deps.userConfigPath()
|
||||
if err != nil {
|
||||
return writeAppError(stderr, err.Error(), exitCrash)
|
||||
}
|
||||
name := options.names[0]
|
||||
cfg, err := config.RemoveProvider(configPath, name)
|
||||
if err != nil {
|
||||
return writeAppError(stderr, err.Error(), exitCrash)
|
||||
}
|
||||
// Delete the key from the store BESIDE the config being edited — the same
|
||||
// store setup/rename write to — not the default-path store, so a
|
||||
// non-default config path cannot leave the encrypted key behind.
|
||||
keyRemoved, keyErr := removeStoredProviderKeyAt(configPath, name)
|
||||
if options.json {
|
||||
payload := map[string]any{
|
||||
"removed": name,
|
||||
"keyRemoved": keyRemoved,
|
||||
"activeProvider": cfg.ActiveProvider,
|
||||
"configPath": configPath,
|
||||
}
|
||||
if keyErr != nil {
|
||||
// A lingering secret must not read as a clean removal.
|
||||
payload["keyError"] = keyErr.Error()
|
||||
}
|
||||
if err := writePrettyJSON(stdout, payload); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
if _, err := fmt.Fprintf(stdout, "Removed provider %s\n", name); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
if keyErr != nil {
|
||||
if _, err := fmt.Fprintf(stderr, "warning: its stored API key could not be deleted and remains in the credential store: %v\n", keyErr); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
} else if keyRemoved {
|
||||
if _, err := fmt.Fprintln(stdout, "Deleted its stored API key."); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
if active := strings.TrimSpace(cfg.ActiveProvider); active != "" {
|
||||
if _, err := fmt.Fprintf(stdout, "Active provider: %s\n", active); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
} else {
|
||||
if _, err := fmt.Fprintln(stdout, "No providers remain — run zero setup to add one."); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
// removeStoredProviderKeyAt deletes a provider's API key from the credential
|
||||
// store co-located with configPath (the store SecureProviderProfile captured
|
||||
// it into and RenameProvider migrates within).
|
||||
func removeStoredProviderKeyAt(configPath string, provider string) (bool, error) {
|
||||
store, err := config.ProviderKeyStoreAt(filepath.Dir(configPath))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return store.Delete(provider)
|
||||
}
|
||||
|
||||
// runProvidersRename renames a saved provider profile, migrating its stored
|
||||
// API key and the activeProvider pointer along with it (config.RenameProvider).
|
||||
func runProvidersRename(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int {
|
||||
options, help, err := parseProviderNamesArgs(args, 2, "usage: zero providers rename <old> <new>")
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
if help {
|
||||
if err := writeProvidersHelp(stdout); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
configPath, err := deps.userConfigPath()
|
||||
if err != nil {
|
||||
return writeAppError(stderr, err.Error(), exitCrash)
|
||||
}
|
||||
cfg, err := config.RenameProvider(configPath, options.names[0], options.names[1])
|
||||
if err != nil {
|
||||
return writeAppError(stderr, err.Error(), exitCrash)
|
||||
}
|
||||
if options.json {
|
||||
if err := writePrettyJSON(stdout, map[string]any{
|
||||
"renamed": map[string]string{"from": options.names[0], "to": options.names[1]},
|
||||
"activeProvider": cfg.ActiveProvider,
|
||||
"configPath": configPath,
|
||||
}); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
if _, err := fmt.Fprintf(stdout, "Renamed provider %s to %s\n", options.names[0], options.names[1]); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
@@ -234,3 +234,48 @@ func writeProviderOnboardingConfig(t *testing.T, path string, cfg config.FileCon
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunProvidersRemoveDeletesKeyBesideConfig: the stored key must be deleted
|
||||
// from the credential store CO-LOCATED with the config being edited (where
|
||||
// SecureProviderProfile captured it), not the default-path store.
|
||||
func TestRunProvidersRemoveDeletesKeyBesideConfig(t *testing.T) {
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.json")
|
||||
seed := `{"activeProvider":"gw","providers":[{"name":"gw","provider_kind":"openai-compatible","baseURL":"https://gw.example.com/v1","apiKeyStored":true,"model":"m1"},{"name":"other","provider_kind":"openai-compatible","baseURL":"https://o.example.com/v1","model":"m2"}]}`
|
||||
if err := os.WriteFile(configPath, []byte(seed), 0o600); err != nil {
|
||||
t.Fatalf("seed config: %v", err)
|
||||
}
|
||||
store, err := config.ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("gw", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed key: %v", err)
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
deps := appDeps{userConfigPath: func() (string, error) { return configPath, nil }}
|
||||
if code := runWithDeps([]string{"providers", "remove", "gw", "--json"}, &stdout, &stderr, deps); code != exitSuccess {
|
||||
t.Fatalf("remove failed: code=%d stderr=%s", code, stderr.String())
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Removed string `json:"removed"`
|
||||
KeyRemoved bool `json:"keyRemoved"`
|
||||
KeyError string `json:"keyError"`
|
||||
ActiveProvider string `json:"activeProvider"`
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("decode JSON: %v\n%s", err, stdout.String())
|
||||
}
|
||||
if payload.Removed != "gw" || !payload.KeyRemoved || payload.KeyError != "" {
|
||||
t.Fatalf("unexpected payload: %+v", payload)
|
||||
}
|
||||
if payload.ActiveProvider != "other" {
|
||||
t.Fatalf("active must hand off, got %q", payload.ActiveProvider)
|
||||
}
|
||||
if _, ok, _ := store.Get("gw"); ok {
|
||||
t.Fatalf("stored key must be deleted from the store beside the config")
|
||||
}
|
||||
}
|
||||
@@ -457,7 +457,9 @@ func selectProviderForCheck(resolved config.ResolvedConfig, name string) (config
|
||||
}
|
||||
|
||||
func validateProviderRuntimeReady(profile config.ProviderProfile) error {
|
||||
hasCredential := providerProfileHasCredential(profile)
|
||||
// A stored OAuth login (e.g. `zero auth chatgpt`) is a credential too: a
|
||||
// keyless token-login profile must pass the readiness check instead of being
|
||||
// told to set an API key it will never have.
|
||||
if profile.CatalogID != "" {
|
||||
descriptor, err := providercatalog.Require(profile.CatalogID)
|
||||
if err != nil {
|
||||
@@ -466,23 +468,19 @@ func validateProviderRuntimeReady(profile config.ProviderProfile) error {
|
||||
if !providercatalog.RuntimeSupported(descriptor) {
|
||||
return fmt.Errorf("provider %q uses transport %q: %s", descriptor.ID, descriptor.Transport, providercatalog.RuntimeUnsupportedReason(descriptor))
|
||||
}
|
||||
if descriptor.RequiresAuth && !hasCredential {
|
||||
apiKeyEnv := strings.TrimSpace(profile.APIKeyEnv)
|
||||
if apiKeyEnv == "" && len(descriptor.AuthEnvVars) > 0 {
|
||||
apiKeyEnv = descriptor.AuthEnvVars[0]
|
||||
}
|
||||
if apiKeyEnv != "" {
|
||||
return fmt.Errorf("provider %s requires API key; set %s", profile.Name, apiKeyEnv)
|
||||
}
|
||||
return fmt.Errorf("provider %s requires API key", profile.Name)
|
||||
}
|
||||
}
|
||||
if providerHasOAuthLogin(profile, oauthLoggedInProviders()) {
|
||||
return nil
|
||||
}
|
||||
switch profile.ProviderKind {
|
||||
case config.ProviderKindOpenAI, config.ProviderKindAnthropic, config.ProviderKindGoogle:
|
||||
if !hasCredential {
|
||||
return fmt.Errorf("provider %s requires API key", profile.Name)
|
||||
// Delegates to the single source of truth shared with the TUI and the
|
||||
// setup wizard, so `providers check` never disagrees with them about
|
||||
// whether a saved provider (including a no-auth custom endpoint) is
|
||||
// actually missing a credential.
|
||||
if envVar, missing := profile.MissingCredentialEnv(); missing {
|
||||
if envVar != "" {
|
||||
return fmt.Errorf("provider %s requires API key; set %s", profile.Name, envVar)
|
||||
}
|
||||
return fmt.Errorf("provider %s requires API key", profile.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
)
|
||||
|
||||
// Regression for issue #555's follow-up: `zero providers check` must not
|
||||
// error that a no-auth custom endpoint requires an API key, matching what
|
||||
// /model and /providers already treat as usable.
|
||||
func TestValidateProviderRuntimeReadyCustomEndpoint(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
profile config.ProviderProfile
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "custom openai compatible with no credential configured",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
Model: "custom-model",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "custom openai compatible with stale legacy default env",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
APIKeyEnv: "OPENAI_API_KEY",
|
||||
Model: "custom-model",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "custom openai compatible with explicit non-default env still requires it",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
APIKeyEnv: "LLAMA_CPP_API_KEY",
|
||||
Model: "custom-model",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "catalog provider missing key still errors",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "groq",
|
||||
CatalogID: "groq",
|
||||
Model: "llama-3.3-70b-versatile",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
err := validateProviderRuntimeReady(c.profile)
|
||||
if (err != nil) != c.wantErr {
|
||||
t.Fatalf("validateProviderRuntimeReady() error = %v, wantErr %v", err, c.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -88,6 +88,15 @@ func formatRepoInfo(info repoinfo.Info) string {
|
||||
fmt.Fprintf(&b, "Files: %d\n", info.FileCount)
|
||||
fmt.Fprintf(&b, "Directories: %d (max depth %d)\n", info.DirectoryCount, info.MaxDepth)
|
||||
fmt.Fprintf(&b, "Est. LOC: %d\n", info.LOCEstimate)
|
||||
if info.CommitCount != nil {
|
||||
fmt.Fprintf(&b, "Commits: %d\n", *info.CommitCount)
|
||||
}
|
||||
if info.BranchCount != nil {
|
||||
fmt.Fprintf(&b, "Branches: %d\n", *info.BranchCount)
|
||||
}
|
||||
if info.TagCount != nil {
|
||||
fmt.Fprintf(&b, "Tags: %d\n", *info.TagCount)
|
||||
}
|
||||
if info.PrimaryLanguage != "" {
|
||||
fmt.Fprintf(&b, "Primary lang: %s\n", info.PrimaryLanguage)
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestFormatRepoInfoText(t *testing.T) {
|
||||
contrib := 3
|
||||
info := repoInfoTestData(age, contrib)
|
||||
out := formatRepoInfo(info)
|
||||
for _, want := range []string{"Files", "TypeScript", "main", "GitHub Actions", "42", "3"} {
|
||||
for _, want := range []string{"Files", "TypeScript", "main", "GitHub Actions", "42", "3", "Commits: 100", "Branches: 5", "Tags: 10"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("text output missing %q:\n%s", want, out)
|
||||
}
|
||||
@@ -113,12 +113,16 @@ func initTempGitRepo(t *testing.T) string {
|
||||
}
|
||||
|
||||
func repoInfoTestData(age, contrib int) repoinfo.Info {
|
||||
commits := 100
|
||||
branches := 5
|
||||
tags := 10
|
||||
return repoinfo.Info{
|
||||
FileCount: 10, DirectoryCount: 3, MaxDepth: 2, LOCEstimate: 500,
|
||||
Languages: []repoinfo.LangStat{{Name: "TypeScript", LOCEstimate: 400, FileCount: 4}},
|
||||
PrimaryLanguage: "TypeScript", LanguageCount: 1,
|
||||
WorkspaceType: "none", CICD: []string{"GitHub Actions"},
|
||||
HasGit: true, Branch: "main", AgeDays: &age, Contributors90d: &contrib,
|
||||
CommitCount: &commits, BranchCount: &branches, TagCount: &tags,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+21
-41
@@ -322,7 +322,9 @@ func setupRequired(resolved config.ResolvedConfig) bool {
|
||||
// the candidate set is permissive (profile name + catalog ID).
|
||||
func providerHasOAuthLogin(profile config.ProviderProfile, oauthLogins map[string]bool) bool {
|
||||
for _, name := range profile.OAuthLoginCandidates() {
|
||||
if oauthLogins[name] {
|
||||
// Store keys are normalized to lower case (oauth.ProviderKey), so the
|
||||
// presence check must compare the same way.
|
||||
if oauthLogins[strings.ToLower(strings.TrimSpace(name))] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -344,18 +346,19 @@ func oauthLoggedInProviders() map[string]bool {
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status.HasToken {
|
||||
out[strings.TrimPrefix(status.Key, oauth.KeyPrefixProvider)] = true
|
||||
out[strings.ToLower(strings.TrimPrefix(status.Key, oauth.KeyPrefixProvider))] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// firstUsableProvider returns the saved provider best suited to run without
|
||||
// onboarding: the first usable (inline credential present, or no-auth/local)
|
||||
// non-local provider, else the first usable local one. It lets the CLI fall back
|
||||
// to an already-configured login when the active provider happens to lack a
|
||||
// credential, instead of re-running onboarding every launch.
|
||||
// onboarding: the first usable (inline credential, stored OAuth login, or
|
||||
// no-auth/local) non-local provider, else the first usable local one. It lets
|
||||
// the CLI fall back to an already-configured login when the active provider
|
||||
// happens to lack a credential, instead of re-running onboarding every launch.
|
||||
func firstUsableProvider(providers []config.ProviderProfile) (config.ProviderProfile, bool) {
|
||||
logins := oauthLoggedInProviders()
|
||||
var localFallback config.ProviderProfile
|
||||
haveLocal := false
|
||||
for _, profile := range providers {
|
||||
@@ -371,7 +374,11 @@ func firstUsableProvider(providers []config.ProviderProfile) (config.ProviderPro
|
||||
continue
|
||||
}
|
||||
}
|
||||
if _, missing := setupMissingCredentialEnv(profile); missing {
|
||||
// A stored OAuth login (e.g. `zero auth login xai`) is a credential too, even
|
||||
// when the profile has no inline key / env var — mirrors setupRequired and
|
||||
// usableSavedProviders so this fallback doesn't force onboarding for a
|
||||
// provider the user is already authenticated with.
|
||||
if _, missing := setupMissingCredentialEnv(profile); missing && !providerHasOAuthLogin(profile, logins) {
|
||||
continue
|
||||
}
|
||||
if providerProfileIsLocal(profile) {
|
||||
@@ -496,41 +503,14 @@ func setupTryThisExample(profile config.ProviderProfile) string {
|
||||
return example
|
||||
}
|
||||
|
||||
// setupMissingCredentialEnv reports the environment variable a saved profile
|
||||
// is expected to read an API key from, and whether that expectation is
|
||||
// unmet. It delegates to config.ProviderProfile.MissingCredentialEnv, the
|
||||
// single source of truth shared with the TUI's provider status/wizard
|
||||
// surfaces and the CLI's `providers check` readiness gate, so all three never
|
||||
// disagree about whether a saved provider is missing a credential.
|
||||
func setupMissingCredentialEnv(profile config.ProviderProfile) (string, bool) {
|
||||
if providerProfileHasCredential(profile) {
|
||||
return "", false
|
||||
}
|
||||
if catalogID := strings.TrimSpace(profile.CatalogID); catalogID != "" {
|
||||
descriptor, err := providercatalog.Require(catalogID)
|
||||
if err != nil || !descriptor.RequiresAuth {
|
||||
return "", false
|
||||
}
|
||||
return firstNonEmptyCLI(profile.APIKeyEnv, setupProviderEnvVar(descriptor)), true
|
||||
}
|
||||
|
||||
switch normalizedSetupProviderKind(profile) {
|
||||
case config.ProviderKindOpenAI, config.ProviderKindOpenAICompatible:
|
||||
return firstNonEmptyCLI(profile.APIKeyEnv, "OPENAI_API_KEY"), true
|
||||
case config.ProviderKindAnthropic, config.ProviderKindAnthropicCompat:
|
||||
return firstNonEmptyCLI(profile.APIKeyEnv, "ANTHROPIC_API_KEY"), true
|
||||
case config.ProviderKindGoogle:
|
||||
return firstNonEmptyCLI(profile.APIKeyEnv, "GEMINI_API_KEY"), true
|
||||
default:
|
||||
if strings.TrimSpace(profile.APIKeyEnv) != "" {
|
||||
return strings.TrimSpace(profile.APIKeyEnv), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func normalizedSetupProviderKind(profile config.ProviderProfile) config.ProviderKind {
|
||||
if kind := strings.TrimSpace(string(profile.ProviderKind)); kind != "" {
|
||||
return config.ProviderKind(strings.ToLower(kind))
|
||||
}
|
||||
if provider := strings.TrimSpace(profile.Provider); provider != "" {
|
||||
return config.ProviderKind(strings.ToLower(provider))
|
||||
}
|
||||
return ""
|
||||
return profile.MissingCredentialEnv()
|
||||
}
|
||||
|
||||
func setupCheckCommand(name string) string {
|
||||
|
||||
@@ -2,11 +2,14 @@ package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
)
|
||||
|
||||
func TestFirstUsableProviderPrefersRemoteKeyed(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "ollama", CatalogID: "ollama", BaseURL: "http://localhost:11434/v1", APIKey: "k"}, // usable but local
|
||||
{Name: "moonshot", CatalogID: "moonshot", BaseURL: "https://api.moonshot.ai/v1", APIKey: "k"}, // usable, remote
|
||||
@@ -19,6 +22,7 @@ func TestFirstUsableProviderPrefersRemoteKeyed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFirstUsableProviderFallsBackToLocal(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "xai", CatalogID: "xai", APIKeyEnv: "XAI_API_KEY"}, // not usable
|
||||
{Name: "ollama", CatalogID: "ollama", BaseURL: "http://localhost:11434/v1", APIKey: "k"}, // local, usable
|
||||
@@ -30,6 +34,7 @@ func TestFirstUsableProviderFallsBackToLocal(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFirstUsableProviderNoneUsable(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "xai", CatalogID: "xai", APIKeyEnv: "XAI_API_KEY"},
|
||||
{Name: "openai", CatalogID: "openai", APIKeyEnv: "OPENAI_API_KEY"},
|
||||
@@ -42,6 +47,10 @@ func TestFirstUsableProviderNoneUsable(t *testing.T) {
|
||||
// A keyless local proxy (chatgpt-proxy, RequiresAuth=false) is usable without a
|
||||
// credential, so it can serve as a fallback rather than forcing onboarding.
|
||||
func TestFirstUsableProviderAcceptsKeylessLocalProxy(t *testing.T) {
|
||||
// Isolate the OAuth token store: on a developer machine with a real xai
|
||||
// login, the env-only xai profile would count as usable and win over the
|
||||
// local proxy this test is about.
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "xai", CatalogID: "xai", APIKeyEnv: "XAI_API_KEY"},
|
||||
{Name: "chatgpt", CatalogID: "chatgpt-proxy", BaseURL: "http://localhost:10531/v1"},
|
||||
@@ -56,6 +65,7 @@ func TestFirstUsableProviderAcceptsKeylessLocalProxy(t *testing.T) {
|
||||
// BaseURL has no endpoint, so it must be skipped rather than selected as a
|
||||
// fallback that fails at first use. A stale CatalogID with a BaseURL still works.
|
||||
func TestFirstUsableProviderSkipsUnresolvableCatalogWithoutBaseURL(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "ghost", CatalogID: "no-such-catalog-entry", APIKey: "k"}, // unusable: no endpoint
|
||||
{Name: "custom", CatalogID: "no-such-catalog-entry", BaseURL: "https://api.custom.test/v1", APIKey: "k"},
|
||||
@@ -66,6 +76,29 @@ func TestFirstUsableProviderSkipsUnresolvableCatalogWithoutBaseURL(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
// An OAuth-only provider (no inline key, no env var) must be selectable as a
|
||||
// fallback, matching setupRequired/usableSavedProviders — otherwise a fully
|
||||
// authenticated user gets forced back into onboarding when activeProvider
|
||||
// goes stale.
|
||||
func TestFirstUsableProviderRecognizesOAuthLogin(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
store, err := oauth.NewStore(oauth.StoreOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore: %v", err)
|
||||
}
|
||||
if err := store.Save(oauth.ProviderKey("xai"), oauth.Token{AccessToken: "tok", ExpiresAt: time.Now().Add(time.Hour)}); err != nil {
|
||||
t.Fatalf("seed token: %v", err)
|
||||
}
|
||||
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "xai", CatalogID: "xai", APIKeyEnv: "XAI_API_KEY"}, // no inline key/env, but logged in via OAuth
|
||||
}
|
||||
got, ok := firstUsableProvider(providers)
|
||||
if !ok || got.Name != "xai" {
|
||||
t.Fatalf("want OAuth-logged-in provider (xai), got %q ok=%v", got.Name, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderProfileIsLocal(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -89,3 +122,32 @@ func TestProviderProfileIsLocal(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFirstUsableProviderAcceptsOAuthLoginProfile: a keyless catalog profile
|
||||
// whose credential is a stored OAuth login (the shape the login flows persist)
|
||||
// must be usable during active-pointer recovery — the same rule setupRequired
|
||||
// and usableSavedProviders apply — instead of falling through to onboarding.
|
||||
func TestFirstUsableProviderAcceptsOAuthLoginProfile(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "chatgpt", CatalogID: "chatgpt", BaseURL: "https://chatgpt.com/backend-api/codex", Model: "gpt-5.5"},
|
||||
}
|
||||
|
||||
// No login stored: still not usable.
|
||||
if got, ok := firstUsableProvider(providers); ok {
|
||||
t.Fatalf("keyless profile without a login must not be usable, got %q", got.Name)
|
||||
}
|
||||
|
||||
store, err := oauth.NewStore(oauth.StoreOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("oauth store: %v", err)
|
||||
}
|
||||
if err := store.Save(oauth.ProviderKey("chatgpt"), oauth.Token{AccessToken: "bearer-123"}); err != nil {
|
||||
t.Fatalf("save token: %v", err)
|
||||
}
|
||||
|
||||
got, ok := firstUsableProvider(providers)
|
||||
if !ok || got.Name != "chatgpt" {
|
||||
t.Fatalf("want OAuth-login profile to be usable, got %q ok=%v", got.Name, ok)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,54 @@ func TestSetupMissingCredentialEnv(t *testing.T) {
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// Regression for issue #555: a custom OpenAI-compatible profile saved
|
||||
// with no credential (e.g. a local llama.cpp server with no auth) must
|
||||
// not be reported as missing one, or it gets filtered out of /model on
|
||||
// the next run even though it was configured exactly as intended.
|
||||
name: "custom openai compatible with no credential configured",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "custom openai compatible with explicit api key env",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
APIKeyEnv: "LLAMA_CPP_API_KEY",
|
||||
},
|
||||
wantEnv: "LLAMA_CPP_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// Self-heal regression: profiles saved by the pre-fix wizard were
|
||||
// stamped with the catalog's own guessed default (OPENAI_API_KEY)
|
||||
// even when the endpoint needed no auth. That stale value must be
|
||||
// treated the same as unset so already-broken saved profiles start
|
||||
// working again without a config.json rewrite.
|
||||
name: "custom openai compatible with stale legacy default env",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
APIKeyEnv: "OPENAI_API_KEY",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "custom anthropic compatible with no credential configured",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-proxy",
|
||||
CatalogID: "custom-anthropic-compatible",
|
||||
BaseURL: "http://localhost:9000/anthropic",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
+195
-2
@@ -41,6 +41,13 @@ type changesCommandOptions struct {
|
||||
hasMessage bool
|
||||
dryRun bool
|
||||
maxDiffBytes int
|
||||
remote string
|
||||
force bool
|
||||
title string
|
||||
body string
|
||||
fill bool
|
||||
draft bool
|
||||
yes bool
|
||||
auto bool
|
||||
}
|
||||
|
||||
@@ -266,6 +273,10 @@ func runChanges(args []string, stdout io.Writer, stderr io.Writer, deps appDeps)
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
case "push":
|
||||
return runChangesPush(args, stdout, stderr, deps)
|
||||
case "pr", "pull-request":
|
||||
return runChangesPR(args, stdout, stderr, deps)
|
||||
default:
|
||||
return writeExecUsageError(stderr, fmt.Sprintf("unknown changes command %q", command))
|
||||
}
|
||||
@@ -465,6 +476,41 @@ func parseChangesArgs(args []string, command string) (changesCommandOptions, boo
|
||||
return options, false, err
|
||||
}
|
||||
options.maxDiffBytes = maxDiffBytes
|
||||
case arg == "--remote":
|
||||
value, next, err := nextFlagValue(args, index, arg)
|
||||
if err != nil {
|
||||
return options, false, err
|
||||
}
|
||||
options.remote = strings.TrimSpace(value)
|
||||
index = next
|
||||
case strings.HasPrefix(arg, "--remote="):
|
||||
options.remote = strings.TrimSpace(strings.TrimPrefix(arg, "--remote="))
|
||||
case arg == "--force":
|
||||
options.force = true
|
||||
case arg == "--title":
|
||||
value, next, err := nextFlagValue(args, index, arg)
|
||||
if err != nil {
|
||||
return options, false, err
|
||||
}
|
||||
options.title = value
|
||||
index = next
|
||||
case strings.HasPrefix(arg, "--title="):
|
||||
options.title = strings.TrimPrefix(arg, "--title=")
|
||||
case arg == "--body":
|
||||
value, next, err := nextFlagValue(args, index, arg)
|
||||
if err != nil {
|
||||
return options, false, err
|
||||
}
|
||||
options.body = value
|
||||
index = next
|
||||
case strings.HasPrefix(arg, "--body="):
|
||||
options.body = strings.TrimPrefix(arg, "--body=")
|
||||
case arg == "--fill":
|
||||
options.fill = true
|
||||
case arg == "--draft":
|
||||
options.draft = true
|
||||
case arg == "--yes":
|
||||
options.yes = true
|
||||
case arg == "-a" || arg == "--auto":
|
||||
options.auto = true
|
||||
case strings.HasPrefix(arg, "-"):
|
||||
@@ -473,15 +519,30 @@ func parseChangesArgs(args []string, command string) (changesCommandOptions, boo
|
||||
return options, false, execUsageError{fmt.Sprintf("unexpected changes argument %q", arg)}
|
||||
}
|
||||
}
|
||||
if command != "commit" && options.message != "" {
|
||||
return options, false, execUsageError{"--message is only valid with `zero changes commit`"}
|
||||
}
|
||||
if command != "commit" && (options.hasMessage || options.dryRun || options.auto) {
|
||||
return options, false, execUsageError{"--message, --dry-run, and --auto are only valid with `zero changes commit`"}
|
||||
}
|
||||
if command == "commit" && options.hasMessage && options.auto {
|
||||
return options, false, execUsageError{"cannot specify both --message and --auto"}
|
||||
}
|
||||
if command == "commit" && options.baseRef != "" {
|
||||
if command != "commit" && command != "push" && options.dryRun {
|
||||
return options, false, execUsageError{"--dry-run is only valid with commit or push"}
|
||||
}
|
||||
if command != "inspect" && options.baseRef != "" {
|
||||
return options, false, execUsageError{"--base is only valid with `zero changes inspect`"}
|
||||
}
|
||||
if command != "push" && command != "pr" && (options.remote != "" || options.force) {
|
||||
return options, false, execUsageError{"--remote and --force are only valid with push or pr"}
|
||||
}
|
||||
if command != "pr" && (options.title != "" || options.body != "" || options.fill || options.draft) {
|
||||
return options, false, execUsageError{"--title, --body, --fill, and --draft are only valid with `zero changes pr`"}
|
||||
}
|
||||
if command != "push" && command != "pr" && options.yes {
|
||||
return options, false, execUsageError{"--yes is only valid with push or pr"}
|
||||
}
|
||||
return options, false, nil
|
||||
}
|
||||
|
||||
@@ -759,14 +820,24 @@ func writeChangesHelp(w io.Writer) error {
|
||||
_, err := fmt.Fprint(w, `Usage:
|
||||
zero changes inspect [flags]
|
||||
zero changes commit [flags]
|
||||
zero changes push [flags]
|
||||
zero changes pr [flags]
|
||||
|
||||
Inspects local git changes and optionally creates a commit.
|
||||
Inspects, commits, pushes, and creates pull requests for local git changes.
|
||||
|
||||
Flags:
|
||||
-C, --cwd <path> Workspace directory
|
||||
--base <ref> Diff against <ref>...HEAD instead of the working tree
|
||||
--diff-bytes <n> Maximum diff bytes to include
|
||||
-m, --message <text> Commit message for `+"`zero changes commit`"+`
|
||||
--dry-run Preview commit metadata / push without mutating git state
|
||||
--remote <name> Remote to push to (defaults to upstream tracked branch or origin)
|
||||
--force Use force-with-lease when pushing
|
||||
--title <text> PR title
|
||||
--body <text> PR body
|
||||
--fill Automatically populate PR title and body from commits
|
||||
--draft Create PR as a draft
|
||||
--yes Confirm pushing to a default/protected branch
|
||||
-a, --auto Auto-generate commit message using LLM (use --dry-run to preview)
|
||||
--dry-run Preview commit metadata without mutating git state
|
||||
--json Print JSON output
|
||||
@@ -775,6 +846,128 @@ Flags:
|
||||
return err
|
||||
}
|
||||
|
||||
func runChangesPush(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int {
|
||||
options, help, err := parseChangesArgs(args, "push")
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
if help {
|
||||
if err := writeChangesHelp(stdout); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
workspaceRoot, err := resolveWorkspaceRoot(options.cwd, deps)
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
|
||||
result, err := deps.pushChanges(context.Background(), zerogit.PushOptions{
|
||||
Cwd: workspaceRoot,
|
||||
Remote: options.remote,
|
||||
Force: options.force,
|
||||
DryRun: options.dryRun,
|
||||
AllowPushDefaultBranch: options.yes,
|
||||
})
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
|
||||
if options.json {
|
||||
if err := writePrettyJSON(stdout, result); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
dryRunStr := ""
|
||||
if options.dryRun {
|
||||
dryRunStr = " (dry run)"
|
||||
}
|
||||
if _, err := fmt.Fprintf(stdout, "Pushed branch %s to remote %s%s\n", result.Branch, result.Remote, dryRunStr); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
if result.Output != "" {
|
||||
if _, err := fmt.Fprintln(stdout, result.Output); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
func runChangesPR(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int {
|
||||
options, help, err := parseChangesArgs(args, "pr")
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
if help {
|
||||
if err := writeChangesHelp(stdout); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
if !options.fill && options.title == "" {
|
||||
return writeExecUsageError(stderr, "must provide either --fill or --title to run non-interactively")
|
||||
}
|
||||
workspaceRoot, err := resolveWorkspaceRoot(options.cwd, deps)
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
|
||||
if !options.json {
|
||||
if _, err := fmt.Fprintln(stdout, "Pushing current branch to set upstream..."); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
pushResult, err := deps.pushChanges(context.Background(), zerogit.PushOptions{
|
||||
Cwd: workspaceRoot,
|
||||
Remote: options.remote,
|
||||
Force: options.force,
|
||||
AllowPushDefaultBranch: options.yes,
|
||||
})
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, fmt.Sprintf("auto-push failed: %v", err))
|
||||
}
|
||||
if !options.json {
|
||||
if _, err := fmt.Fprintf(stdout, "Pushed branch %s to remote %s\n", pushResult.Branch, pushResult.Remote); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
|
||||
prResult, err := deps.createPR(context.Background(), zerogit.PROptions{
|
||||
Cwd: workspaceRoot,
|
||||
Fill: options.fill,
|
||||
Draft: options.draft,
|
||||
Title: options.title,
|
||||
Body: options.body,
|
||||
})
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
|
||||
if options.json {
|
||||
type prJSONResult struct {
|
||||
Branch string `json:"branch"`
|
||||
Remote string `json:"remote"`
|
||||
Output string `json:"output"`
|
||||
}
|
||||
res := prJSONResult{
|
||||
Branch: pushResult.Branch,
|
||||
Remote: pushResult.Remote,
|
||||
Output: strings.TrimSpace(prResult.Output),
|
||||
}
|
||||
if err := writePrettyJSON(stdout, res); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintln(stdout, strings.TrimSpace(prResult.Output)); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
func generateAutoCommitMessage(ctx context.Context, provider zeroruntime.Provider, model string, summary zerogit.ChangeSummary) (string, error) {
|
||||
var promptBuilder strings.Builder
|
||||
promptBuilder.WriteString("Analyze the following git diff and generate a concise, conventional commit message.\n")
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/credstore"
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
)
|
||||
|
||||
// ProviderKeyStoreAt opens the encrypted credential store whose file backend lives
|
||||
@@ -168,6 +169,86 @@ func (profile ProviderProfile) HasConfiguredCredential() bool {
|
||||
strings.TrimSpace(profile.AuthHeaderValue) != ""
|
||||
}
|
||||
|
||||
// MissingCredentialEnv reports the environment variable this profile is
|
||||
// expected to read an API key from, and whether that expectation is
|
||||
// currently unmet. It is the single source of truth for "does this saved
|
||||
// provider need a credential it doesn't have" — the CLI (`providers check`,
|
||||
// `usableSavedProviders`) and TUI (`/providers` status, provider wizard
|
||||
// summary) surfaces all delegate here so they never disagree about a
|
||||
// profile's status.
|
||||
//
|
||||
// For the catalog's two "bring your own endpoint" descriptors
|
||||
// (custom-openai-compatible, custom-anthropic-compatible), RequiresAuth is
|
||||
// just a wizard template default, not a fact about the user's actual
|
||||
// endpoint: a custom target may need no auth at all. So for those two only,
|
||||
// a credential is expected exclusively when this profile carries its own
|
||||
// explicit APIKeyEnv that differs from the catalog's guessed default — a
|
||||
// value matching that default is indistinguishable from the wizard's old
|
||||
// auto-stamp bug (issue #555) and is treated the same as unset, which also
|
||||
// self-heals profiles saved before that bug was fixed.
|
||||
func (profile ProviderProfile) MissingCredentialEnv() (string, bool) {
|
||||
if profile.HasConfiguredCredential() {
|
||||
return "", false
|
||||
}
|
||||
if catalogID := strings.TrimSpace(profile.CatalogID); catalogID != "" {
|
||||
descriptor, err := providercatalog.Require(catalogID)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
defaultEnvVar := firstNonEmptyTrimmed(descriptor.AuthEnvVars...)
|
||||
if descriptor.Custom {
|
||||
envVar := strings.TrimSpace(profile.APIKeyEnv)
|
||||
if envVar == "" || envVar == defaultEnvVar {
|
||||
return "", false
|
||||
}
|
||||
return envVar, true
|
||||
}
|
||||
if !descriptor.RequiresAuth {
|
||||
return "", false
|
||||
}
|
||||
if envVar := strings.TrimSpace(profile.APIKeyEnv); envVar != "" {
|
||||
return envVar, true
|
||||
}
|
||||
return defaultEnvVar, true
|
||||
}
|
||||
|
||||
switch profile.normalizedProviderKind() {
|
||||
case ProviderKindOpenAI, ProviderKindOpenAICompatible:
|
||||
return firstNonEmptyTrimmed(profile.APIKeyEnv, "OPENAI_API_KEY"), true
|
||||
case ProviderKindAnthropic, ProviderKindAnthropicCompat:
|
||||
return firstNonEmptyTrimmed(profile.APIKeyEnv, "ANTHROPIC_API_KEY"), true
|
||||
case ProviderKindGoogle:
|
||||
return firstNonEmptyTrimmed(profile.APIKeyEnv, "GEMINI_API_KEY"), true
|
||||
default:
|
||||
if envVar := strings.TrimSpace(profile.APIKeyEnv); envVar != "" {
|
||||
return envVar, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// normalizedProviderKind resolves this profile's kind, falling back to the
|
||||
// legacy Provider string field (lowercased) when ProviderKind is unset —
|
||||
// matching how the resolver (resolver.go) treats the two fields as synonyms.
|
||||
func (profile ProviderProfile) normalizedProviderKind() ProviderKind {
|
||||
if kind := strings.TrimSpace(string(profile.ProviderKind)); kind != "" {
|
||||
return ProviderKind(strings.ToLower(kind))
|
||||
}
|
||||
if provider := strings.TrimSpace(profile.Provider); provider != "" {
|
||||
return ProviderKind(strings.ToLower(provider))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstNonEmptyTrimmed(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// OAuthLoginCandidates returns the login names to try, in order, when resolving
|
||||
// this profile's OAuth token — used by the runtime bearer resolver, the Codex
|
||||
// account-header resolver, and the onboarding presence check so all three agree.
|
||||
|
||||
@@ -238,3 +238,82 @@ func TestOAuthLoginCandidates(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderProfileMissingCredentialEnv(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
profile ProviderProfile
|
||||
wantEnv string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "catalog provider requiring auth",
|
||||
profile: ProviderProfile{Name: "groq", CatalogID: "groq"},
|
||||
wantEnv: "GROQ_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "local catalog provider",
|
||||
profile: ProviderProfile{Name: "local", CatalogID: "ollama"},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "credential resolved via inline key",
|
||||
profile: ProviderProfile{Name: "openai", ProviderKind: ProviderKindOpenAI, APIKey: "sk-test"},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// issue #555: a custom endpoint left with no credential means "no
|
||||
// auth needed" (e.g. a local llama.cpp server), not "missing".
|
||||
name: "custom openai compatible with no credential configured",
|
||||
profile: ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "custom openai compatible with explicit non-default api key env",
|
||||
profile: ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
APIKeyEnv: "LLAMA_CPP_API_KEY",
|
||||
},
|
||||
wantEnv: "LLAMA_CPP_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// Self-heal: a profile stamped by the pre-fix wizard with the
|
||||
// catalog's own guessed default is indistinguishable from that bug
|
||||
// and must not stay filtered out after upgrading.
|
||||
name: "custom openai compatible with stale legacy default env",
|
||||
profile: ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
APIKeyEnv: "OPENAI_API_KEY",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "openai compatible without catalog falls back to provider kind",
|
||||
profile: ProviderProfile{Name: "custom", ProviderKind: ProviderKindOpenAICompatible},
|
||||
wantEnv: "OPENAI_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "legacy Provider string field fallback",
|
||||
profile: ProviderProfile{Name: "legacy", Provider: "anthropic"},
|
||||
wantEnv: "ANTHROPIC_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
gotEnv, got := c.profile.MissingCredentialEnv()
|
||||
if got != c.want || gotEnv != c.wantEnv {
|
||||
t.Fatalf("MissingCredentialEnv() = (%q, %v), want (%q, %v)", gotEnv, got, c.wantEnv, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
package config
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultMCPServers returns the MCP servers Zero ships ENABLED by default so web
|
||||
// search and scraping work out of the box with no setup and no API key. They are
|
||||
@@ -29,3 +32,29 @@ func IsDefaultMCPServer(name string) bool {
|
||||
_, ok := DefaultMCPServers()[strings.TrimSpace(name)]
|
||||
return ok
|
||||
}
|
||||
|
||||
// IsUnconfiguredDefault reports whether server is one of Zero's built-in
|
||||
// defaults that the user never wrote an entry for in their config — i.e. it is
|
||||
// running with whatever Zero ships (e.g. keyless Firecrawl, no credentials).
|
||||
//
|
||||
// Both conditions below must hold:
|
||||
// - !server.configured: the user's JSON never declared an object for this
|
||||
// server key at all (set by MCPServerConfig.UnmarshalJSON only when it
|
||||
// actually ran for this key). Any explicit action — including a
|
||||
// disable/enable toggle like `zero mcp enable firecrawl` that leaves the
|
||||
// resolved value unchanged — sets configured, so it always counts as
|
||||
// user-configured, even though the value comparison below could not tell
|
||||
// the difference on its own.
|
||||
// - reflect.DeepEqual(def, server): the value still matches the default.
|
||||
// This is the fallback for callers that construct MCPServerConfig
|
||||
// directly rather than through the JSON/merge pipeline (server.configured
|
||||
// is then always false) — without it, any hand-built config with
|
||||
// different field values would be misreported as unconfigured.
|
||||
//
|
||||
// Callers use this to tell "server we turned on for the user" apart from
|
||||
// "server the user configured themselves," e.g. to avoid warning loudly when
|
||||
// an out-of-the-box default that was never given credentials fails to connect.
|
||||
func IsUnconfiguredDefault(name string, server MCPServerConfig) bool {
|
||||
def, ok := DefaultMCPServers()[strings.TrimSpace(name)]
|
||||
return ok && !server.configured && reflect.DeepEqual(def, server)
|
||||
}
|
||||
@@ -49,6 +49,69 @@ func TestResolveMCPUserCanDisableDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsUnconfiguredDefault(t *testing.T) {
|
||||
if !IsUnconfiguredDefault("firecrawl", DefaultMCPServers()["firecrawl"]) {
|
||||
t.Fatal("an untouched firecrawl default should be reported as unconfigured")
|
||||
}
|
||||
if IsUnconfiguredDefault("firecrawl", MCPServerConfig{Type: "http", URL: "http://localhost:3002/mcp"}) {
|
||||
t.Fatal("a server overriding the default URL is no longer unconfigured")
|
||||
}
|
||||
if IsUnconfiguredDefault("firecrawl", MCPServerConfig{Type: "http", URL: "https://mcp.firecrawl.dev/v2/mcp", Auth: "bearer"}) {
|
||||
t.Fatal("a server with credentials added is no longer unconfigured")
|
||||
}
|
||||
if IsUnconfiguredDefault("not-a-default", MCPServerConfig{}) {
|
||||
t.Fatal("a server with no matching default can never be unconfigured-default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMCPExplicitReenableIsNotUnconfiguredDefault(t *testing.T) {
|
||||
// `zero mcp enable firecrawl` after a prior disable writes {"disabled":false}
|
||||
// explicitly. The resolved value is identical to the untouched default (both
|
||||
// enabled, no credentials), but the user DID take an explicit action here, so
|
||||
// IsUnconfiguredDefault must not treat it as untouched (issue #563 review).
|
||||
path := filepath.Join(t.TempDir(), "config.json")
|
||||
if err := os.WriteFile(path, []byte(`{"mcp":{"servers":{"firecrawl":{"disabled":false}}}}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := ResolveMCP(ResolveOptions{UserConfigPath: path})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveMCP: %v", err)
|
||||
}
|
||||
firecrawl := cfg.Servers["firecrawl"]
|
||||
if firecrawl.Disabled {
|
||||
t.Fatalf("explicit re-enable should leave the server enabled: %#v", firecrawl)
|
||||
}
|
||||
if IsUnconfiguredDefault("firecrawl", firecrawl) {
|
||||
t.Fatal("an explicit enable/disable toggle must count as user-configured, even though the resolved value matches the default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMCPExplicitRedeclareOfDefaultValuesIsNotUnconfiguredDefault(t *testing.T) {
|
||||
// A user who copies firecrawl's exact default type/url into their config
|
||||
// (e.g. from an example file, planning to add credentials later) produces a
|
||||
// resolved value byte-identical to DefaultMCPServers()["firecrawl"] — the
|
||||
// same trap TestResolveMCPExplicitReenableIsNotUnconfiguredDefault covers for
|
||||
// the disabled toggle. IsUnconfiguredDefault must still treat this as
|
||||
// user-configured because the user's JSON declared an entry for it, even
|
||||
// though a plain resolved-value comparison could not tell the difference.
|
||||
path := filepath.Join(t.TempDir(), "config.json")
|
||||
if err := os.WriteFile(path, []byte(`{"mcp":{"servers":{"firecrawl":{"type":"http","url":"https://mcp.firecrawl.dev/v2/mcp"}}}}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := ResolveMCP(ResolveOptions{UserConfigPath: path})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveMCP: %v", err)
|
||||
}
|
||||
firecrawl := cfg.Servers["firecrawl"]
|
||||
want := DefaultMCPServers()["firecrawl"]
|
||||
if firecrawl.Type != want.Type || firecrawl.URL != want.URL || firecrawl.Disabled != want.Disabled {
|
||||
t.Fatalf("expected the resolved value to match the default's fields exactly: %#v", firecrawl)
|
||||
}
|
||||
if IsUnconfiguredDefault("firecrawl", firecrawl) {
|
||||
t.Fatal("redeclaring the default's exact values is still an explicit user configuration, not an untouched default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMCPUserCanOverrideDefaultURLKeepingOtherFields(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.json")
|
||||
// Point firecrawl at a self-hosted instance; the default's Type must survive.
|
||||
|
||||
@@ -127,6 +127,9 @@ func Resolve(options ResolveOptions) (ResolvedConfig, error) {
|
||||
return ResolvedConfig{}, fmt.Errorf("invalid notify.focusMode %q: expected unfocused, always, or focused", focusMode)
|
||||
}
|
||||
}
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return ResolvedConfig{}, err
|
||||
}
|
||||
|
||||
providers, active, err := normalizeProviders(cfg.Providers, cfg.ActiveProvider, options.Env)
|
||||
if err != nil {
|
||||
@@ -150,6 +153,7 @@ func Resolve(options ResolveOptions) (ResolvedConfig, error) {
|
||||
Preferences: cfg.Preferences,
|
||||
KeyBindings: cfg.KeyBindings,
|
||||
LocalControl: cfg.LocalControl,
|
||||
STT: cfg.STT,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -223,6 +227,9 @@ func mergeConfig(dst *FileConfig, src FileConfig) {
|
||||
if src.Preferences.FavoriteModels != nil {
|
||||
dst.Preferences.FavoriteModels = normalizeFavoriteModels(src.Preferences.FavoriteModels)
|
||||
}
|
||||
if src.Preferences.RecentModels != nil {
|
||||
dst.Preferences.RecentModels = NormalizeRecentModels(src.Preferences.RecentModels)
|
||||
}
|
||||
if src.Preferences.Recaps != nil {
|
||||
dst.Preferences.Recaps = src.Preferences.Recaps
|
||||
}
|
||||
@@ -231,6 +238,7 @@ func mergeConfig(dst *FileConfig, src FileConfig) {
|
||||
}
|
||||
mergeLocalControlConfig(&dst.LocalControl, src.LocalControl)
|
||||
mergeKeyBindings(&dst.KeyBindings, src.KeyBindings)
|
||||
mergeSTTConfig(&dst.STT, src.STT)
|
||||
}
|
||||
|
||||
func mergeProjectConfig(dst *FileConfig, src FileConfig) error {
|
||||
@@ -681,6 +689,7 @@ func applyOverrides(cfg *FileConfig, overrides Overrides) {
|
||||
}
|
||||
mergeLocalControlConfig(&cfg.LocalControl, overrides.LocalControl)
|
||||
mergeKeyBindings(&cfg.KeyBindings, overrides.KeyBindings)
|
||||
mergeSTTConfig(&cfg.STT, overrides.STT)
|
||||
for _, provider := range overrides.Providers {
|
||||
mergeProvider(cfg, provider)
|
||||
}
|
||||
@@ -734,6 +743,91 @@ func mergeKeyBindings(dst *KeyBindingsConfig, src KeyBindingsConfig) {
|
||||
}
|
||||
}
|
||||
|
||||
// mergeSTTConfig overlays any set field of src onto dst. Each field carries its
|
||||
// own "unset" sentinel (empty string, 0, or nil *bool), matching how the other
|
||||
// section mergers detect intent.
|
||||
func mergeSTTConfig(dst *STTConfig, src STTConfig) {
|
||||
if src.Provider != "" {
|
||||
dst.Provider = src.Provider
|
||||
}
|
||||
if src.StreamProvider != "" {
|
||||
dst.StreamProvider = src.StreamProvider
|
||||
}
|
||||
if src.Streaming != nil {
|
||||
dst.Streaming = src.Streaming
|
||||
}
|
||||
if src.Model != "" {
|
||||
dst.Model = src.Model
|
||||
}
|
||||
if src.StreamModel != "" {
|
||||
dst.StreamModel = src.StreamModel
|
||||
}
|
||||
if src.LocalModelPath != "" {
|
||||
dst.LocalModelPath = src.LocalModelPath
|
||||
}
|
||||
if src.LocalBinary != "" {
|
||||
dst.LocalBinary = src.LocalBinary
|
||||
}
|
||||
if src.LocalServerBinary != "" {
|
||||
dst.LocalServerBinary = src.LocalServerBinary
|
||||
}
|
||||
if src.LocalServerPort != 0 {
|
||||
dst.LocalServerPort = src.LocalServerPort
|
||||
}
|
||||
if src.EngineVersion != "" {
|
||||
dst.EngineVersion = src.EngineVersion
|
||||
}
|
||||
if src.NumThreads != 0 {
|
||||
dst.NumThreads = src.NumThreads
|
||||
}
|
||||
if src.Language != "" {
|
||||
dst.Language = src.Language
|
||||
}
|
||||
if src.MaxDurationSeconds != 0 {
|
||||
dst.MaxDurationSeconds = src.MaxDurationSeconds
|
||||
}
|
||||
if src.SilenceAutoStop != nil {
|
||||
dst.SilenceAutoStop = src.SilenceAutoStop
|
||||
}
|
||||
if src.AutoSubmit != nil {
|
||||
dst.AutoSubmit = src.AutoSubmit
|
||||
}
|
||||
if src.WindowsAudioDevice != "" {
|
||||
dst.WindowsAudioDevice = src.WindowsAudioDevice
|
||||
}
|
||||
}
|
||||
|
||||
// validateSTTConfig rejects unknown provider/streamProvider values and a
|
||||
// negative maxDurationSeconds at load time — a clear startup error naming the
|
||||
// bad value and the valid options, never a silent fallback the user did not ask
|
||||
// for (§11a).
|
||||
func validateSTTConfig(cfg STTConfig) error {
|
||||
if cfg.Provider != "" {
|
||||
switch cfg.Provider {
|
||||
case STTProviderLocal, STTProviderGroq, STTProviderOpenAI:
|
||||
default:
|
||||
return fmt.Errorf("invalid stt.provider %q: expected local, groq, or openai", cfg.Provider)
|
||||
}
|
||||
}
|
||||
if cfg.StreamProvider != "" {
|
||||
switch cfg.StreamProvider {
|
||||
case STTProviderLocal, STTProviderDeepgram, STTProviderOpenAI:
|
||||
default:
|
||||
return fmt.Errorf("invalid stt.streamProvider %q: expected local, deepgram, or openai", cfg.StreamProvider)
|
||||
}
|
||||
}
|
||||
if cfg.MaxDurationSeconds < 0 {
|
||||
return fmt.Errorf("invalid stt.maxDurationSeconds %d: must be >= 0 (0 uses the default)", cfg.MaxDurationSeconds)
|
||||
}
|
||||
if cfg.NumThreads < 0 {
|
||||
return fmt.Errorf("invalid stt.numThreads %d: must be >= 0 (0 uses the engine default)", cfg.NumThreads)
|
||||
}
|
||||
if cfg.LocalServerPort < 0 || cfg.LocalServerPort > 65535 {
|
||||
return fmt.Errorf("invalid stt.localServerPort %d: must be between 1 and 65535 (0 uses the default)", cfg.LocalServerPort)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeMCPConfig(dst *MCPConfig, src MCPConfig) {
|
||||
if len(src.Servers) == 0 {
|
||||
return
|
||||
@@ -774,6 +868,9 @@ func mergeMCPServer(base MCPServerConfig, next MCPServerConfig) MCPServerConfig
|
||||
if next.disabledSet || next.Disabled {
|
||||
base.Disabled = next.Disabled
|
||||
}
|
||||
if next.configured {
|
||||
base.configured = true
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
|
||||
@@ -127,6 +127,79 @@ func TestResolveLoadsFavoriteModelsFromUserConfigOnly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveLoadsRecentModelsFromUserConfigOnly(t *testing.T) {
|
||||
userPath := writeConfig(t, `{
|
||||
"activeProvider": "user",
|
||||
"providers": [{
|
||||
"name": "user",
|
||||
"provider": "openai",
|
||||
"apiKey": "sk-user",
|
||||
"model": "gpt-user"
|
||||
}],
|
||||
"preferences": {
|
||||
"recentModels": [
|
||||
{"provider": "openrouter", "model": " google/gemini-2.5-pro "},
|
||||
{"provider": "openrouter", "model": "minimax/minimax-m2.1"}
|
||||
]
|
||||
}
|
||||
}`)
|
||||
projectPath := writeConfig(t, `{
|
||||
"preferences": {
|
||||
"recentModels": [{"provider": "project", "model": "project-model"}]
|
||||
}
|
||||
}`)
|
||||
|
||||
resolved, err := Resolve(ResolveOptions{
|
||||
UserConfigPath: userPath,
|
||||
ProjectConfigPath: projectPath,
|
||||
Env: map[string]string{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
|
||||
want := []RecentModelEntry{
|
||||
{Provider: "openrouter", Model: "google/gemini-2.5-pro"},
|
||||
{Provider: "openrouter", Model: "minimax/minimax-m2.1"},
|
||||
}
|
||||
if !reflect.DeepEqual(resolved.Preferences.RecentModels, want) {
|
||||
t.Fatalf("RecentModels = %#v, want %#v", resolved.Preferences.RecentModels, want)
|
||||
}
|
||||
}
|
||||
|
||||
// recentModels must never be merged from project config (same posture as
|
||||
// favoriteModels): a project setting it with no corresponding user setting
|
||||
// should leave the resolved value empty, not pick up the project's list.
|
||||
func TestResolveIgnoresProjectOnlyRecentModels(t *testing.T) {
|
||||
userPath := writeConfig(t, `{
|
||||
"activeProvider": "user",
|
||||
"providers": [{
|
||||
"name": "user",
|
||||
"provider": "openai",
|
||||
"apiKey": "sk-user",
|
||||
"model": "gpt-user"
|
||||
}]
|
||||
}`)
|
||||
projectPath := writeConfig(t, `{
|
||||
"preferences": {
|
||||
"recentModels": [{"provider": "project", "model": "project-model"}]
|
||||
}
|
||||
}`)
|
||||
|
||||
resolved, err := Resolve(ResolveOptions{
|
||||
UserConfigPath: userPath,
|
||||
ProjectConfigPath: projectPath,
|
||||
Env: map[string]string{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
|
||||
if len(resolved.Preferences.RecentModels) != 0 {
|
||||
t.Fatalf("RecentModels = %#v, want empty (project-only recentModels must not be merged)", resolved.Preferences.RecentModels)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveLoadsProviderCatalogSnakeAndCamelJSONFields(t *testing.T) {
|
||||
path := writeConfig(t, `{
|
||||
"activeProvider": "snake",
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSetSTTLocalEngineResetsStreamProvider(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := dir + "/config.json"
|
||||
// Pre-seed a cloud streaming provider — the situation the fix guards against.
|
||||
if err := os.WriteFile(path, []byte(`{"stt":{"streamProvider":"deepgram"}}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := SetSTTLocalEngine(path, "sherpa-onnx-offline", "sherpa-onnx-online-websocket-server", "/models/kroko", true)
|
||||
if err != nil {
|
||||
t.Fatalf("SetSTTLocalEngine: %v", err)
|
||||
}
|
||||
// Switching to a local model must point BOTH pipelines at the local engine, or
|
||||
// the leftover cloud streamProvider keeps the live transcript on the cloud.
|
||||
if cfg.STT.Provider != STTProviderLocal || cfg.STT.StreamProvider != STTProviderLocal {
|
||||
t.Errorf("both providers should be local, got provider=%q streamProvider=%q", cfg.STT.Provider, cfg.STT.StreamProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSTTConfigRejectsUnknownProvider(t *testing.T) {
|
||||
_, issues := ValidateBytes([]byte(`{"stt":{"provider":"assemblyai"}}`))
|
||||
if len(issues) == 0 {
|
||||
t.Fatal("expected an issue for unknown stt.provider")
|
||||
}
|
||||
if !strings.Contains(issues[0].Message, "assemblyai") || !strings.Contains(issues[0].Message, "local, groq, or openai") {
|
||||
t.Errorf("issue message = %q, want it to name the bad value and valid options", issues[0].Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSTTConfigRejectsUnknownStreamProvider(t *testing.T) {
|
||||
_, issues := ValidateBytes([]byte(`{"stt":{"streamProvider":"groq"}}`))
|
||||
if len(issues) == 0 {
|
||||
t.Fatal("expected an issue: groq has no streaming product, not a valid streamProvider")
|
||||
}
|
||||
if !strings.Contains(issues[0].Message, "deepgram") {
|
||||
t.Errorf("issue message = %q", issues[0].Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSTTConfigAcceptsValid(t *testing.T) {
|
||||
_, issues := ValidateBytes([]byte(`{"stt":{"provider":"groq","streamProvider":"deepgram","maxDurationSeconds":120}}`))
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("unexpected issues: %+v", issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSTTConfigRejectsNegativeDuration(t *testing.T) {
|
||||
_, issues := ValidateBytes([]byte(`{"stt":{"maxDurationSeconds":-5}}`))
|
||||
if len(issues) == 0 {
|
||||
t.Fatal("expected an issue for negative maxDurationSeconds")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTTConfigDefaults(t *testing.T) {
|
||||
cfg := STTConfig{}
|
||||
if cfg.STTProvider() != STTProviderLocal {
|
||||
t.Errorf("default provider = %q, want local", cfg.STTProvider())
|
||||
}
|
||||
if cfg.STTStreamProvider() != STTProviderLocal {
|
||||
t.Errorf("default streamProvider = %q, want local", cfg.STTStreamProvider())
|
||||
}
|
||||
if !cfg.StreamingEnabled() {
|
||||
t.Error("streaming should default on")
|
||||
}
|
||||
if !cfg.SilenceAutoStopEnabled() {
|
||||
t.Error("silence auto-stop should default on")
|
||||
}
|
||||
if cfg.AutoSubmitEnabled() {
|
||||
t.Error("auto-submit should default off (insert-for-review is the safety net)")
|
||||
}
|
||||
if !cfg.Empty() {
|
||||
t.Error("zero-value STTConfig should be Empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTTConfigTriStateBools(t *testing.T) {
|
||||
cfg, issues := ValidateBytes([]byte(`{"stt":{"autoSubmit":true,"streaming":false,"silenceAutoStop":false}}`))
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("unexpected issues: %+v", issues)
|
||||
}
|
||||
if !cfg.STT.AutoSubmitEnabled() {
|
||||
t.Error("autoSubmit:true should be enabled")
|
||||
}
|
||||
if cfg.STT.StreamingEnabled() {
|
||||
t.Error("streaming:false should be disabled")
|
||||
}
|
||||
if cfg.STT.SilenceAutoStopEnabled() {
|
||||
t.Error("silenceAutoStop:false should be disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTTConfigRoundTripsThroughMarshal(t *testing.T) {
|
||||
cfg, _ := ValidateBytes([]byte(`{"stt":{"provider":"groq","model":"whisper-large-v3-turbo","localServerPort":6007}}`))
|
||||
data, err := cfg.MarshalJSON()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), `"stt"`) {
|
||||
t.Errorf("marshaled config dropped stt section: %s", data)
|
||||
}
|
||||
reparsed, issues := ValidateBytes(data)
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("re-parse issues: %+v", issues)
|
||||
}
|
||||
if reparsed.STT.Provider != STTProviderGroq || reparsed.STT.Model != "whisper-large-v3-turbo" || reparsed.STT.LocalServerPort != 6007 {
|
||||
t.Errorf("round-trip lost fields: %+v", reparsed.STT)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetSTTModelPersists(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := dir + "/config.json"
|
||||
// Cloud provider stores under stt.model.
|
||||
if _, err := SetSTTModel(path, STTProviderGroq, "whisper-large-v3-turbo"); err != nil {
|
||||
t.Fatalf("SetSTTModel: %v", err)
|
||||
}
|
||||
cfg, issues := ValidateFile(path)
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("issues: %+v", issues)
|
||||
}
|
||||
if cfg.STT.Provider != STTProviderGroq || cfg.STT.Model != "whisper-large-v3-turbo" {
|
||||
t.Errorf("persisted STT = %+v", cfg.STT)
|
||||
}
|
||||
// Local provider stores under stt.localModelPath.
|
||||
if _, err := SetSTTModel(path, STTProviderLocal, "/models/moonshine"); err != nil {
|
||||
t.Fatalf("SetSTTModel local: %v", err)
|
||||
}
|
||||
cfg, _ = ValidateFile(path)
|
||||
if cfg.STT.LocalModelPath != "/models/moonshine" {
|
||||
t.Errorf("local model path = %q", cfg.STT.LocalModelPath)
|
||||
}
|
||||
}
|
||||
@@ -97,6 +97,11 @@ type ToolsConfig struct {
|
||||
|
||||
type PreferencesConfig struct {
|
||||
FavoriteModels []string `json:"favoriteModels,omitempty"`
|
||||
// RecentModels is the short automatic history of provider+model pairs the
|
||||
// user has switched to via /model, newest first. Unlike FavoriteModels
|
||||
// (manual pins), this list is maintained automatically on every switch and
|
||||
// capped to MaxRecentModels entries. See RecentModelEntry.
|
||||
RecentModels []RecentModelEntry `json:"recentModels,omitempty"`
|
||||
// Theme is the persisted TUI palette preference — "auto" or a registered theme
|
||||
// name (e.g. "dracula"). Applied at startup below the --theme flag and
|
||||
// ZERO_THEME, so a /theme choice survives restart. Empty = unset (defaults auto).
|
||||
@@ -107,6 +112,18 @@ type PreferencesConfig struct {
|
||||
Recaps *bool `json:"recaps,omitempty"`
|
||||
}
|
||||
|
||||
// RecentModelEntry is one provider-qualified model selection recorded in
|
||||
// Preferences.RecentModels. Provider is the saved provider profile's Name (not
|
||||
// a display label), so a recent entry can be resolved back to a concrete
|
||||
// profile the same way the /model picker's cross-provider rows are.
|
||||
type RecentModelEntry struct {
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// MaxRecentModels caps the persisted/displayed recent-selection history.
|
||||
const MaxRecentModels = 5
|
||||
|
||||
// RecapsEnabled reports whether post-turn recaps are on. Unset defaults to ON.
|
||||
func (p PreferencesConfig) RecapsEnabled() bool {
|
||||
return p.Recaps == nil || *p.Recaps
|
||||
@@ -131,6 +148,115 @@ type KeyBindingsConfig struct {
|
||||
ToggleSidebar KeyBindingDef `json:"toggleSidebar,omitempty"`
|
||||
}
|
||||
|
||||
// STTProviderKind is the batch (or streaming) transcription backend selector.
|
||||
// Validated at config load time against the known set, so a typo fails loudly
|
||||
// with the valid options rather than silently falling back to a default.
|
||||
type STTProviderKind string
|
||||
|
||||
const (
|
||||
STTProviderLocal STTProviderKind = "local"
|
||||
STTProviderGroq STTProviderKind = "groq"
|
||||
STTProviderOpenAI STTProviderKind = "openai"
|
||||
STTProviderDeepgram STTProviderKind = "deepgram"
|
||||
)
|
||||
|
||||
// STTConfig configures speech-to-text dictation. All fields are optional; empty
|
||||
// values take the documented defaults. Booleans that need a real tri-state
|
||||
// (distinguishing "unset" from "false") use *bool, matching PreferencesConfig.Recaps.
|
||||
type STTConfig struct {
|
||||
// Provider is the batch transcription backend: "local" (sherpa-onnx-offline,
|
||||
// the default), "groq", or "openai". Termux and the fallback everywhere use
|
||||
// this path.
|
||||
Provider STTProviderKind `json:"provider,omitempty"`
|
||||
// StreamProvider is the streaming backend on desktop: "local" (sherpa-onnx
|
||||
// websocket server, default), "deepgram", or "openai" (Realtime).
|
||||
StreamProvider STTProviderKind `json:"streamProvider,omitempty"`
|
||||
// Streaming enables the live-transcript pipeline on platforms that support
|
||||
// it (desktop). Defaults to on; set false to always use the batch pipeline.
|
||||
Streaming *bool `json:"streaming,omitempty"`
|
||||
// Model overrides the cloud batch model (e.g. "whisper-large-v3-turbo" for
|
||||
// Groq, "whisper-1" for OpenAI). Empty uses the provider default.
|
||||
Model string `json:"model,omitempty"`
|
||||
// StreamModel overrides the cloud streaming model (Deepgram/OpenAI Realtime).
|
||||
StreamModel string `json:"streamModel,omitempty"`
|
||||
// LocalModelPath is the sherpa-onnx model directory for local transcription
|
||||
// (batch and streaming). Required to use a local provider.
|
||||
LocalModelPath string `json:"localModelPath,omitempty"`
|
||||
// LocalBinary overrides the offline binary name/path (default
|
||||
// "sherpa-onnx-offline"); LocalServerBinary overrides the streaming server
|
||||
// (default "sherpa-onnx-online-websocket-server"). Both looked up on PATH.
|
||||
LocalBinary string `json:"localBinary,omitempty"`
|
||||
LocalServerBinary string `json:"localServerBinary,omitempty"`
|
||||
// LocalServerPort is the localhost port for the sherpa-onnx websocket server
|
||||
// (default 6006).
|
||||
LocalServerPort int `json:"localServerPort,omitempty"`
|
||||
// EngineVersion selects the sherpa-onnx release the auto-download fetches
|
||||
// ("" → a pinned known-good default; "latest" or any release tag also works,
|
||||
// so a newer engine needs no Zero update). Verified against the release's
|
||||
// published SHA256 digest either way.
|
||||
EngineVersion string `json:"engineVersion,omitempty"`
|
||||
// NumThreads sets the local engine's thread count (0 = engine default).
|
||||
NumThreads int `json:"numThreads,omitempty"`
|
||||
// Language optionally constrains recognition (ISO-639-1, e.g. "en"). Empty =
|
||||
// auto-detect.
|
||||
Language string `json:"language,omitempty"`
|
||||
// MaxDurationSeconds is the hard runaway-recording cap on every platform
|
||||
// (0 = default 300s).
|
||||
MaxDurationSeconds int `json:"maxDurationSeconds,omitempty"`
|
||||
// SilenceAutoStop ends a recording ~2s after the signal goes quiet, as a
|
||||
// backstop for a forgotten manual stop (not VAD-triggered start). Defaults on.
|
||||
SilenceAutoStop *bool `json:"silenceAutoStop,omitempty"`
|
||||
// AutoSubmit fires the transcript at the agent instead of inserting it for
|
||||
// review. Defaults OFF — insert-for-review is the safety net for a misheard
|
||||
// prompt (§3).
|
||||
AutoSubmit *bool `json:"autoSubmit,omitempty"`
|
||||
// WindowsAudioDevice names the dshow capture device (auto-detected when empty).
|
||||
WindowsAudioDevice string `json:"windowsAudioDevice,omitempty"`
|
||||
}
|
||||
|
||||
// STTProvider returns the configured batch provider, defaulting to local.
|
||||
func (c STTConfig) STTProvider() STTProviderKind {
|
||||
if c.Provider == "" {
|
||||
return STTProviderLocal
|
||||
}
|
||||
return c.Provider
|
||||
}
|
||||
|
||||
// STTStreamProvider returns the configured streaming provider, defaulting to local.
|
||||
func (c STTConfig) STTStreamProvider() STTProviderKind {
|
||||
if c.StreamProvider == "" {
|
||||
return STTProviderLocal
|
||||
}
|
||||
return c.StreamProvider
|
||||
}
|
||||
|
||||
// StreamingEnabled reports whether the live pipeline is on. Unset defaults ON.
|
||||
func (c STTConfig) StreamingEnabled() bool {
|
||||
return c.Streaming == nil || *c.Streaming
|
||||
}
|
||||
|
||||
// SilenceAutoStopEnabled reports whether trailing-silence auto-stop is on.
|
||||
// Unset defaults ON.
|
||||
func (c STTConfig) SilenceAutoStopEnabled() bool {
|
||||
return c.SilenceAutoStop == nil || *c.SilenceAutoStop
|
||||
}
|
||||
|
||||
// AutoSubmitEnabled reports whether transcripts are auto-fired. Unset defaults OFF.
|
||||
func (c STTConfig) AutoSubmitEnabled() bool {
|
||||
return c.AutoSubmit != nil && *c.AutoSubmit
|
||||
}
|
||||
|
||||
// Empty reports whether the STT config carries no user-set values (so it can be
|
||||
// omitted from a marshaled config).
|
||||
func (c STTConfig) Empty() bool {
|
||||
return c.Provider == "" && c.StreamProvider == "" && c.Streaming == nil &&
|
||||
c.Model == "" && c.StreamModel == "" && c.LocalModelPath == "" &&
|
||||
c.LocalBinary == "" && c.LocalServerBinary == "" && c.LocalServerPort == 0 &&
|
||||
c.EngineVersion == "" && c.NumThreads == 0 && c.Language == "" &&
|
||||
c.MaxDurationSeconds == 0 && c.SilenceAutoStop == nil && c.AutoSubmit == nil &&
|
||||
c.WindowsAudioDevice == ""
|
||||
}
|
||||
|
||||
// LocalControlConfig controls local browser/desktop/terminal automation helpers.
|
||||
// Helpers are discovered lazily by the tool that needs them; no setup command or
|
||||
// background probe is required during startup.
|
||||
@@ -227,6 +353,7 @@ type FileConfig struct {
|
||||
Preferences PreferencesConfig `json:"preferences,omitempty"`
|
||||
KeyBindings KeyBindingsConfig `json:"keybindings,omitempty"`
|
||||
LocalControl LocalControlConfig `json:"localControl,omitempty"`
|
||||
STT STTConfig `json:"stt,omitempty"`
|
||||
}
|
||||
|
||||
func (cfg FileConfig) MarshalJSON() ([]byte, error) {
|
||||
@@ -242,6 +369,7 @@ func (cfg FileConfig) MarshalJSON() ([]byte, error) {
|
||||
Preferences PreferencesConfig `json:"preferences,omitempty"`
|
||||
KeyBindings KeyBindingsConfig `json:"keybindings,omitempty"`
|
||||
LocalControl *LocalControlConfig `json:"localControl,omitempty"`
|
||||
STT *STTConfig `json:"stt,omitempty"`
|
||||
}
|
||||
raw := rawConfig{
|
||||
ActiveProvider: cfg.ActiveProvider,
|
||||
@@ -258,6 +386,9 @@ func (cfg FileConfig) MarshalJSON() ([]byte, error) {
|
||||
if !cfg.LocalControl.Empty() {
|
||||
raw.LocalControl = &cfg.LocalControl
|
||||
}
|
||||
if !cfg.STT.Empty() {
|
||||
raw.STT = &cfg.STT
|
||||
}
|
||||
return json.Marshal(raw)
|
||||
}
|
||||
|
||||
@@ -280,6 +411,7 @@ type Overrides struct {
|
||||
Tools ToolsConfig
|
||||
KeyBindings KeyBindingsConfig
|
||||
LocalControl LocalControlConfig
|
||||
STT STTConfig
|
||||
}
|
||||
|
||||
type ResolvedConfig struct {
|
||||
@@ -295,6 +427,7 @@ type ResolvedConfig struct {
|
||||
Preferences PreferencesConfig
|
||||
KeyBindings KeyBindingsConfig
|
||||
LocalControl LocalControlConfig
|
||||
STT STTConfig
|
||||
}
|
||||
|
||||
type MCPConfig struct {
|
||||
@@ -312,6 +445,16 @@ type MCPServerConfig struct {
|
||||
OAuth *MCPOAuthConfig `json:"oauth,omitempty"`
|
||||
Disabled bool `json:"disabled,omitempty"`
|
||||
disabledSet bool
|
||||
// configured is true when the user's config JSON declared an object for
|
||||
// this server at all (i.e. UnmarshalJSON ran for it), regardless of which
|
||||
// fields it set or what values they hold. A built-in default seeded by
|
||||
// DefaultMCPServers() is never unmarshaled from JSON, so it starts false;
|
||||
// any explicit entry in the user/project file — even one that happens to
|
||||
// repeat a default's exact field values (e.g. re-declaring firecrawl's
|
||||
// default URL) — sets it true. IsUnconfiguredDefault checks this alongside
|
||||
// a resolved-value comparison, so redeclaring default values verbatim still
|
||||
// counts as user-configured.
|
||||
configured bool
|
||||
}
|
||||
|
||||
// MCPOAuthConfig describes how to authenticate to a remote MCP server using an
|
||||
@@ -342,6 +485,7 @@ func (cfg *FileConfig) UnmarshalJSON(data []byte) error {
|
||||
Preferences PreferencesConfig `json:"preferences"`
|
||||
KeyBindings KeyBindingsConfig `json:"keybindings"`
|
||||
LocalControl LocalControlConfig `json:"localControl"`
|
||||
STT STTConfig `json:"stt"`
|
||||
MCPServers map[string]MCPServerConfig `json:"mcpServers"`
|
||||
MCPServersSnake map[string]MCPServerConfig `json:"mcp_servers"`
|
||||
}
|
||||
@@ -369,6 +513,7 @@ func (cfg *FileConfig) UnmarshalJSON(data []byte) error {
|
||||
cfg.Preferences = raw.Preferences
|
||||
cfg.KeyBindings = raw.KeyBindings
|
||||
cfg.LocalControl = raw.LocalControl
|
||||
cfg.STT = raw.STT
|
||||
if cfg.MCP.Servers == nil && (len(raw.MCPServers) > 0 || len(raw.MCPServersSnake) > 0) {
|
||||
cfg.MCP.Servers = map[string]MCPServerConfig{}
|
||||
}
|
||||
@@ -503,6 +648,10 @@ func (server *MCPServerConfig) UnmarshalJSON(data []byte) error {
|
||||
server.Disabled = *raw.Disabled
|
||||
server.disabledSet = true
|
||||
}
|
||||
// This method only runs when the user's JSON actually has an object for this
|
||||
// server key (built-in defaults are seeded as Go struct literals, never
|
||||
// unmarshaled), so reaching here at all means the user configured it.
|
||||
server.configured = true
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -54,5 +54,8 @@ func validateSemantics(cfg FileConfig) []Issue {
|
||||
// normalizeProviders already redacts secrets via providerError.
|
||||
return []Issue{{FieldPath: "providers", Message: err.Error()}}
|
||||
}
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return []Issue{{FieldPath: "stt", Message: err.Error()}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
)
|
||||
|
||||
func UpsertProvider(path string, profile ProviderProfile, setActive bool) (FileConfig, error) {
|
||||
@@ -29,6 +31,20 @@ func UpsertProvider(path string, profile ProviderProfile, setActive bool) (FileC
|
||||
}
|
||||
|
||||
mergeProvider(&cfg, profile)
|
||||
// mergeProfile deliberately ignores APIKeyStored — during resolve-time
|
||||
// layering a project config must not be able to claim the user's stored
|
||||
// keys. This user-config WRITE path re-applies the marker: capturing a key
|
||||
// via SecureProviderProfile onto a previously env/no-key profile must
|
||||
// persist apiKeyStored, or the secret sits in the credential store while
|
||||
// every ApplyStoredAPIKey gate skips it (PR #560 review).
|
||||
if profile.APIKeyStored {
|
||||
for index := range cfg.Providers {
|
||||
if cfg.Providers[index].Name == profile.Name {
|
||||
cfg.Providers[index].APIKeyStored = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if setActive || strings.TrimSpace(cfg.ActiveProvider) == "" {
|
||||
cfg.ActiveProvider = profile.Name
|
||||
}
|
||||
@@ -39,6 +55,62 @@ func UpsertProvider(path string, profile ProviderProfile, setActive bool) (FileC
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// EnsuredProvider reports the outcome of EnsureCatalogProvider: the profile name
|
||||
// that serves the catalog entry, whether it was newly created, and which provider
|
||||
// is active after the call (unchanged unless it was blank).
|
||||
type EnsuredProvider struct {
|
||||
Name string
|
||||
Created bool
|
||||
Active string
|
||||
}
|
||||
|
||||
// EnsureCatalogProvider guarantees a provider profile exists in the config at
|
||||
// path for the given catalog entry. OAuth login flows call this right after
|
||||
// storing a token: a login is only reachable from the provider list and
|
||||
// `zero providers use` when a profile exists, but a login must never replace or
|
||||
// deactivate the user's current active provider — so an existing profile whose
|
||||
// Name or CatalogID already matches is left completely untouched (its name,
|
||||
// credentials, and model are the user's), and a created profile is NOT marked
|
||||
// active unless no provider was active at all.
|
||||
func EnsureCatalogProvider(path string, catalogID string) (EnsuredProvider, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return EnsuredProvider{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
descriptor, err := providercatalog.Require(catalogID)
|
||||
if err != nil {
|
||||
return EnsuredProvider{}, err
|
||||
}
|
||||
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return EnsuredProvider{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return EnsuredProvider{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
for _, provider := range cfg.Providers {
|
||||
if strings.EqualFold(strings.TrimSpace(provider.CatalogID), descriptor.ID) ||
|
||||
strings.EqualFold(strings.TrimSpace(provider.Name), descriptor.ID) {
|
||||
return EnsuredProvider{Name: provider.Name, Active: cfg.ActiveProvider}, nil
|
||||
}
|
||||
}
|
||||
|
||||
profile := ProviderProfile{
|
||||
Name: descriptor.ID,
|
||||
ProviderKind: providerKindForCatalogTransport(descriptor.Transport),
|
||||
CatalogID: descriptor.ID,
|
||||
BaseURL: descriptor.DefaultBaseURL,
|
||||
Model: descriptor.DefaultModel,
|
||||
}
|
||||
written, err := UpsertProvider(path, profile, false)
|
||||
if err != nil {
|
||||
return EnsuredProvider{}, err
|
||||
}
|
||||
return EnsuredProvider{Name: profile.Name, Created: true, Active: written.ActiveProvider}, nil
|
||||
}
|
||||
|
||||
func SetActiveProvider(path string, name string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
@@ -72,6 +144,296 @@ func SetActiveProvider(path string, name string) (FileConfig, error) {
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", name)
|
||||
}
|
||||
|
||||
// RemoveProvider deletes the named provider profile from the config at path.
|
||||
// When the removed profile was active, activeProvider hands off to the first
|
||||
// remaining provider (or clears when none remain) so the config never points at
|
||||
// a profile that no longer exists. The caller owns cleaning up the credential
|
||||
// store entry — config stays pure of secret I/O on the read path, and remove
|
||||
// keeps that symmetry by only touching config.json.
|
||||
func RemoveProvider(path string, name string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return FileConfig{}, fmt.Errorf("provider name is required")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
|
||||
index := -1
|
||||
for i, provider := range cfg.Providers {
|
||||
if strings.EqualFold(strings.TrimSpace(provider.Name), name) {
|
||||
index = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", name)
|
||||
}
|
||||
removed := cfg.Providers[index]
|
||||
cfg.Providers = append(cfg.Providers[:index], cfg.Providers[index+1:]...)
|
||||
if strings.EqualFold(strings.TrimSpace(cfg.ActiveProvider), strings.TrimSpace(removed.Name)) {
|
||||
cfg.ActiveProvider = ""
|
||||
if len(cfg.Providers) > 0 {
|
||||
cfg.ActiveProvider = cfg.Providers[0].Name
|
||||
}
|
||||
}
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// RenameProvider renames a provider profile, keeping everything keyed by the
|
||||
// profile name consistent: the activeProvider pointer follows the rename, and a
|
||||
// key in the encrypted credential store (APIKeyStored) is migrated to the new
|
||||
// name BEFORE the config is rewritten — the store write must succeed first so a
|
||||
// failed migration never strands the config pointing at a key that no longer
|
||||
// resolves. OAuth tokens are deliberately not migrated: the runtime's login
|
||||
// candidates fall back to the profile's CatalogID, which every OAuth-capable
|
||||
// catalog profile carries, so a rename keeps the login reachable.
|
||||
func RenameProvider(path string, oldName string, newName string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
oldName = strings.TrimSpace(oldName)
|
||||
newName = strings.TrimSpace(newName)
|
||||
if oldName == "" || newName == "" {
|
||||
return FileConfig{}, fmt.Errorf("provider names are required")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
|
||||
index := -1
|
||||
for i, provider := range cfg.Providers {
|
||||
providerName := strings.TrimSpace(provider.Name)
|
||||
if strings.EqualFold(providerName, oldName) {
|
||||
index = i
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(providerName, newName) {
|
||||
return FileConfig{}, fmt.Errorf("provider %q already exists", newName)
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", oldName)
|
||||
}
|
||||
if strings.EqualFold(oldName, newName) && cfg.Providers[index].Name == newName {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
previousName := cfg.Providers[index].Name
|
||||
keyMigrated := false
|
||||
if cfg.Providers[index].APIKeyStored {
|
||||
if err := migrateStoredProviderKey(path, previousName, newName); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("migrate stored key for %q: %w", oldName, err)
|
||||
}
|
||||
keyMigrated = true
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(cfg.ActiveProvider), strings.TrimSpace(previousName)) {
|
||||
cfg.ActiveProvider = newName
|
||||
}
|
||||
cfg.Providers[index].Name = newName
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
if keyMigrated {
|
||||
// Compensate best-effort: config.json still names the OLD profile, so
|
||||
// move the key back where that config can find it — otherwise a failed
|
||||
// rewrite strands the key under a name no profile carries.
|
||||
_ = migrateStoredProviderKey(path, newName, previousName)
|
||||
}
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// ProviderEdit is a field-level edit of one saved provider, applied by
|
||||
// EditProvider in a single atomic write. Name is the CURRENT profile name
|
||||
// (matched case-insensitively); NewName renames (case-only renames included).
|
||||
// Empty BaseURL/Model/APIKey mean "leave unchanged"; Description is applied
|
||||
// VERBATIM (the editor always knows the full desired text, so clearing works).
|
||||
type ProviderEdit struct {
|
||||
Name string
|
||||
NewName string
|
||||
BaseURL string
|
||||
Model string
|
||||
APIKey string
|
||||
APIKeyStored bool
|
||||
Description string
|
||||
}
|
||||
|
||||
// EditProvider applies a provider edit in ONE read-modify-write: rename
|
||||
// (activeProvider follows; a stored key migrates, with a best-effort rollback
|
||||
// if the config write fails), field updates, the stored-key marker, and the
|
||||
// verbatim description. A single write keeps the operation atomic — the
|
||||
// previous rename+upsert+describe sequence could fail halfway and leave
|
||||
// config.json renamed while every in-memory consumer still held the old name —
|
||||
// and, unlike UpsertProvider's exact-name merge, the case-insensitive match
|
||||
// here makes a case-only rename (groq -> Groq) an in-place update instead of
|
||||
// an appended duplicate profile.
|
||||
func EditProvider(path string, edit ProviderEdit) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
oldName := strings.TrimSpace(edit.Name)
|
||||
if oldName == "" {
|
||||
return FileConfig{}, fmt.Errorf("provider name is required")
|
||||
}
|
||||
newName := strings.TrimSpace(edit.NewName)
|
||||
if newName == "" {
|
||||
newName = oldName
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
|
||||
index := -1
|
||||
for i, provider := range cfg.Providers {
|
||||
providerName := strings.TrimSpace(provider.Name)
|
||||
if strings.EqualFold(providerName, oldName) {
|
||||
index = i
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(providerName, newName) {
|
||||
return FileConfig{}, fmt.Errorf("provider %q already exists", newName)
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", oldName)
|
||||
}
|
||||
|
||||
previousName := cfg.Providers[index].Name
|
||||
renamed := previousName != newName
|
||||
keyMigrated := false
|
||||
// A rename moves the stored key along: either the profile's existing entry,
|
||||
// or a replacement key the caller just captured — the contract is that a
|
||||
// captured key is stored under the CURRENT name before EditProvider runs, so
|
||||
// one migration covers both. migrateStoredProviderKey no-ops on case-only
|
||||
// renames (the store normalizes names), so it cannot delete the key it just
|
||||
// moved.
|
||||
if renamed && (cfg.Providers[index].APIKeyStored || edit.APIKeyStored) {
|
||||
if err := migrateStoredProviderKey(path, previousName, newName); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("migrate stored key for %q: %w", oldName, err)
|
||||
}
|
||||
keyMigrated = true
|
||||
}
|
||||
if renamed && strings.EqualFold(strings.TrimSpace(cfg.ActiveProvider), strings.TrimSpace(previousName)) {
|
||||
cfg.ActiveProvider = newName
|
||||
}
|
||||
|
||||
profile := &cfg.Providers[index]
|
||||
profile.Name = newName
|
||||
if baseURL := strings.TrimSpace(edit.BaseURL); baseURL != "" {
|
||||
profile.BaseURL = baseURL
|
||||
}
|
||||
if model := strings.TrimSpace(edit.Model); model != "" {
|
||||
profile.Model = model
|
||||
}
|
||||
if apiKey := strings.TrimSpace(edit.APIKey); apiKey != "" {
|
||||
profile.APIKey = apiKey
|
||||
}
|
||||
if edit.APIKeyStored {
|
||||
profile.APIKeyStored = true
|
||||
}
|
||||
profile.Description = strings.TrimSpace(edit.Description)
|
||||
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
if keyMigrated {
|
||||
// Compensate best-effort: config.json still names the OLD profile, so
|
||||
// move the key back where that config can find it.
|
||||
_ = migrateStoredProviderKey(path, newName, previousName)
|
||||
}
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetProviderDescription sets a provider's description VERBATIM — including to
|
||||
// empty. The generic UpsertProvider merge treats empty fields as "leave
|
||||
// unchanged", so clearing a description needs this dedicated setter.
|
||||
func SetProviderDescription(path string, name string, description string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return FileConfig{}, fmt.Errorf("provider name is required")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
for index := range cfg.Providers {
|
||||
if strings.EqualFold(strings.TrimSpace(cfg.Providers[index].Name), name) {
|
||||
cfg.Providers[index].Description = strings.TrimSpace(description)
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
}
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", name)
|
||||
}
|
||||
|
||||
// migrateStoredProviderKey moves a credential-store entry to a new provider
|
||||
// name: write-new-then-delete-old, so an interruption can leave a duplicate but
|
||||
// never a missing key. A missing source entry is a no-op (the marker may be
|
||||
// stale); only a failed WRITE aborts the rename.
|
||||
func migrateStoredProviderKey(configPath string, oldName string, newName string) error {
|
||||
// The store normalizes names case-insensitively, so a case-only rename
|
||||
// (groq -> Groq) targets ONE entry: Set(new) rewrites it in place and
|
||||
// Delete(old) would then remove the key that was just "moved". Nothing to
|
||||
// migrate — the existing entry already serves the new name.
|
||||
if strings.EqualFold(strings.TrimSpace(oldName), strings.TrimSpace(newName)) {
|
||||
return nil
|
||||
}
|
||||
store, err := ProviderKeyStoreAt(filepath.Dir(configPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, ok, err := store.Get(oldName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok || strings.TrimSpace(key) == "" {
|
||||
return nil
|
||||
}
|
||||
if err := store.Set(newName, key); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = store.Delete(oldName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetProviderModel(path string, name string, model string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
@@ -131,6 +493,32 @@ func SetFavoriteModels(path string, models []string) (FileConfig, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetRecentModels persists the automatic recent-model-switch history,
|
||||
// mirroring SetFavoriteModels (read-modify-atomic-write). Unlike favorites,
|
||||
// order is preserved (newest first) rather than sorted, since it reflects
|
||||
// switch recency, not an alphabetical preference list.
|
||||
func SetRecentModels(path string, entries []RecentModelEntry) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
|
||||
cfg.Preferences.RecentModels = NormalizeRecentModels(entries)
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetRecapsEnabled persists the post-turn recap preference, mirroring
|
||||
// SetFavoriteModels (read-modify-atomic-write).
|
||||
func SetRecapsEnabled(path string, enabled bool) (FileConfig, error) {
|
||||
@@ -176,6 +564,106 @@ func SetTheme(path string, theme string) (FileConfig, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetSTTModel persists the dictation model and its provider, mirroring
|
||||
// SetTheme (read-modify-atomic-write). provider must be one of the known STT
|
||||
// provider kinds; a local provider stores the model as stt.localModelPath,
|
||||
// otherwise as stt.model. A blank model clears the stored value for that slot.
|
||||
func SetSTTModel(path string, provider STTProviderKind, model string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
if provider != "" {
|
||||
cfg.STT.Provider = provider
|
||||
}
|
||||
model = strings.TrimSpace(model)
|
||||
if cfg.STT.STTProvider() == STTProviderLocal {
|
||||
cfg.STT.LocalModelPath = model
|
||||
} else {
|
||||
cfg.STT.Model = model
|
||||
}
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetSTTLocalEngine persists the paths of an auto-downloaded local engine +
|
||||
// model and switches dictation to the local provider, mirroring SetTheme
|
||||
// (read-modify-atomic-write). streaming selects the pipeline matching the
|
||||
// downloaded model (a streaming transducer vs a batch model). Called after a
|
||||
// download completes.
|
||||
func SetSTTLocalEngine(path, binary, serverBinary, modelPath string, streaming bool) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg.STT.Provider = STTProviderLocal
|
||||
// Point BOTH pipelines at the local engine. Without resetting StreamProvider, a
|
||||
// previously-chosen cloud value (deepgram/openai) would still win in
|
||||
// buildStreamingTranscriber, so the live transcript would keep hitting the cloud
|
||||
// after the user switched to a downloaded local model.
|
||||
cfg.STT.StreamProvider = STTProviderLocal
|
||||
cfg.STT.LocalBinary = strings.TrimSpace(binary)
|
||||
cfg.STT.LocalServerBinary = strings.TrimSpace(serverBinary)
|
||||
cfg.STT.LocalModelPath = strings.TrimSpace(modelPath)
|
||||
// Match the pipeline to the downloaded model: a streaming transducer drives
|
||||
// the websocket server for a live transcript; a batch model uses the offline
|
||||
// binary.
|
||||
s := streaming
|
||||
cfg.STT.Streaming = &s
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetSTTProvider persists just the dictation batch provider, mirroring SetTheme.
|
||||
func SetSTTProvider(path string, provider STTProviderKind) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg.STT.Provider = provider
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func normalizeFavoriteModels(models []string) []string {
|
||||
seen := map[string]bool{}
|
||||
favorites := make([]string, 0, len(models))
|
||||
@@ -191,6 +679,35 @@ func normalizeFavoriteModels(models []string) []string {
|
||||
return favorites
|
||||
}
|
||||
|
||||
// NormalizeRecentModels trims, drops entries with no model id, de-duplicates
|
||||
// by provider+model pair (keeping the first/newest occurrence), and caps the
|
||||
// result to MaxRecentModels. Order is preserved — the caller is responsible
|
||||
// for passing entries newest-first. Exported so callers outside this package
|
||||
// (e.g. the TUI, which keeps its own in-memory copy of recent history) apply
|
||||
// the exact same normalization rules as the persisted config, instead of
|
||||
// maintaining a second, independently-drifting copy of this logic.
|
||||
func NormalizeRecentModels(entries []RecentModelEntry) []RecentModelEntry {
|
||||
seen := map[string]bool{}
|
||||
recent := make([]RecentModelEntry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
provider := strings.TrimSpace(entry.Provider)
|
||||
model := strings.TrimSpace(entry.Model)
|
||||
if model == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(provider) + "\x00" + model
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
recent = append(recent, RecentModelEntry{Provider: provider, Model: model})
|
||||
if len(recent) >= MaxRecentModels {
|
||||
break
|
||||
}
|
||||
}
|
||||
return recent
|
||||
}
|
||||
|
||||
func writeConfigFile(path string, cfg FileConfig) error {
|
||||
dir := filepath.Dir(path)
|
||||
if dir != "." && dir != "" {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
@@ -272,6 +273,69 @@ func TestSetFavoriteModelsPersistsUserPreferences(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetRecentModelsPersistsOrderDedupesAndCaps(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "openai",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "openai", ProviderKind: ProviderKindOpenAI, Model: "gpt-4.1"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
cfg, err := SetRecentModels(path, []RecentModelEntry{
|
||||
{Provider: " openrouter ", Model: " google/gemini-2.5-pro "},
|
||||
{Provider: "openrouter", Model: "minimax/minimax-m2.1"},
|
||||
{Provider: "openrouter", Model: "google/gemini-2.5-pro"}, // duplicate pair, older: dropped
|
||||
{Provider: "", Model: ""}, // blank model: dropped
|
||||
{Provider: "openrouter", Model: "a"},
|
||||
{Provider: "openrouter", Model: "b"},
|
||||
{Provider: "openrouter", Model: "c"},
|
||||
{Provider: "openrouter", Model: "d"}, // beyond MaxRecentModels (5): dropped
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SetRecentModels() error = %v", err)
|
||||
}
|
||||
|
||||
want := []RecentModelEntry{
|
||||
{Provider: "openrouter", Model: "google/gemini-2.5-pro"},
|
||||
{Provider: "openrouter", Model: "minimax/minimax-m2.1"},
|
||||
{Provider: "openrouter", Model: "a"},
|
||||
{Provider: "openrouter", Model: "b"},
|
||||
{Provider: "openrouter", Model: "c"},
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.Preferences.RecentModels, want) {
|
||||
t.Fatalf("RecentModels = %#v, want %#v", cfg.Preferences.RecentModels, want)
|
||||
}
|
||||
persisted := readConfigFixture(t, path)
|
||||
if !reflect.DeepEqual(persisted.Preferences.RecentModels, want) {
|
||||
t.Fatalf("persisted RecentModels = %#v, want %#v", persisted.Preferences.RecentModels, want)
|
||||
}
|
||||
if persisted.ActiveProvider != "openai" || len(persisted.Providers) != 1 {
|
||||
t.Fatalf("provider config was not preserved: %#v", persisted)
|
||||
}
|
||||
}
|
||||
|
||||
// Two providers offering the same model id must both survive normalization —
|
||||
// recent history de-duplicates by provider+model pair, not model id alone.
|
||||
func TestSetRecentModelsDedupesByProviderAndModelPair(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
|
||||
cfg, err := SetRecentModels(path, []RecentModelEntry{
|
||||
{Provider: "provider-a", Model: "shared-model"},
|
||||
{Provider: "provider-b", Model: "shared-model"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SetRecentModels() error = %v", err)
|
||||
}
|
||||
want := []RecentModelEntry{
|
||||
{Provider: "provider-a", Model: "shared-model"},
|
||||
{Provider: "provider-b", Model: "shared-model"},
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.Preferences.RecentModels, want) {
|
||||
t.Fatalf("RecentModels = %#v, want both providers preserved: %#v", cfg.Preferences.RecentModels, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetThemePersistsUserPreference(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
@@ -385,3 +449,545 @@ func readConfigFixture(t *testing.T, path string) FileConfig {
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func TestEnsureCatalogProviderCreatesProfileWithoutStealingActive(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "opengateway",
|
||||
Providers: []ProviderProfile{
|
||||
{
|
||||
Name: "opengateway",
|
||||
ProviderKind: ProviderKindOpenAICompatible,
|
||||
BaseURL: "https://gateway.example.com/v1",
|
||||
APIKeyStored: true,
|
||||
Model: "some-model",
|
||||
},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
ensured, err := EnsureCatalogProvider(path, "chatgpt")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureCatalogProvider: %v", err)
|
||||
}
|
||||
if !ensured.Created {
|
||||
t.Fatalf("expected profile to be created")
|
||||
}
|
||||
if ensured.Name != "chatgpt" {
|
||||
t.Fatalf("expected profile name chatgpt, got %q", ensured.Name)
|
||||
}
|
||||
if ensured.Active != "opengateway" {
|
||||
t.Fatalf("active provider must stay opengateway, got %q", ensured.Active)
|
||||
}
|
||||
|
||||
cfg := readConfigFixture(t, path)
|
||||
if cfg.ActiveProvider != "opengateway" {
|
||||
t.Fatalf("persisted active provider changed to %q", cfg.ActiveProvider)
|
||||
}
|
||||
if len(cfg.Providers) != 2 {
|
||||
t.Fatalf("expected 2 providers, got %d", len(cfg.Providers))
|
||||
}
|
||||
chatgpt := cfg.Providers[1]
|
||||
if chatgpt.Name != "chatgpt" || chatgpt.CatalogID != "chatgpt" {
|
||||
t.Fatalf("unexpected created profile: %+v", chatgpt)
|
||||
}
|
||||
if chatgpt.Model == "" || chatgpt.BaseURL == "" {
|
||||
t.Fatalf("created profile must carry catalog defaults, got %+v", chatgpt)
|
||||
}
|
||||
if chatgpt.APIKey != "" || chatgpt.APIKeyStored {
|
||||
t.Fatalf("OAuth-created profile must stay keyless, got %+v", chatgpt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCatalogProviderLeavesExistingProfileUntouched(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
original := FileConfig{
|
||||
ActiveProvider: "opengateway",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "opengateway", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://gateway.example.com/v1", Model: "some-model"},
|
||||
// Renamed profile that already serves the chatgpt catalog entry.
|
||||
{Name: "codex", CatalogID: "chatgpt", Model: "gpt-5.5"},
|
||||
},
|
||||
}
|
||||
data := writeConfigFixture(t, path, original, 0o600)
|
||||
|
||||
ensured, err := EnsureCatalogProvider(path, "chatgpt")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureCatalogProvider: %v", err)
|
||||
}
|
||||
if ensured.Created {
|
||||
t.Fatalf("existing profile must not be recreated")
|
||||
}
|
||||
if ensured.Name != "codex" {
|
||||
t.Fatalf("expected existing profile name codex, got %q", ensured.Name)
|
||||
}
|
||||
after, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
if string(after) != string(data) {
|
||||
t.Fatalf("config rewritten for a no-op ensure:\nbefore: %s\nafter: %s", data, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCatalogProviderActivatesOnEmptyConfig(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
|
||||
ensured, err := EnsureCatalogProvider(path, "chatgpt")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureCatalogProvider: %v", err)
|
||||
}
|
||||
if !ensured.Created {
|
||||
t.Fatalf("expected profile to be created")
|
||||
}
|
||||
if ensured.Active != "chatgpt" {
|
||||
t.Fatalf("blank active must adopt the new provider, got %q", ensured.Active)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCatalogProviderRejectsUnknownCatalogID(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
if _, err := EnsureCatalogProvider(path, "no-such-provider"); err == nil {
|
||||
t.Fatalf("expected unknown catalog id to error")
|
||||
}
|
||||
if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("config must not be written for an unknown catalog id")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveProviderDeletesAndHandsOffActive(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "beta",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"},
|
||||
{Name: "beta", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
cfg, err := RemoveProvider(path, " BETA ")
|
||||
if err != nil {
|
||||
t.Fatalf("RemoveProvider() error = %v", err)
|
||||
}
|
||||
if len(cfg.Providers) != 1 || cfg.Providers[0].Name != "alpha" {
|
||||
t.Fatalf("expected only alpha to remain, got %+v", cfg.Providers)
|
||||
}
|
||||
if cfg.ActiveProvider != "alpha" {
|
||||
t.Fatalf("active must hand off to the first remaining provider, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
|
||||
persisted := readConfigFixture(t, path)
|
||||
if len(persisted.Providers) != 1 || persisted.ActiveProvider != "alpha" {
|
||||
t.Fatalf("persisted config wrong: %+v", persisted)
|
||||
}
|
||||
|
||||
// Removing the last provider clears the active pointer entirely.
|
||||
cfg, err = RemoveProvider(path, "alpha")
|
||||
if err != nil {
|
||||
t.Fatalf("RemoveProvider(last) error = %v", err)
|
||||
}
|
||||
if len(cfg.Providers) != 0 || cfg.ActiveProvider != "" {
|
||||
t.Fatalf("expected empty providers and no active, got %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveProviderKeepsActiveWhenOtherRemoved(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"},
|
||||
{Name: "beta", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
cfg, err := RemoveProvider(path, "beta")
|
||||
if err != nil {
|
||||
t.Fatalf("RemoveProvider() error = %v", err)
|
||||
}
|
||||
if cfg.ActiveProvider != "alpha" {
|
||||
t.Fatalf("active provider must be untouched, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveProviderRejectsUnknownWithoutRewriting(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
before := writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"}},
|
||||
}, 0o600)
|
||||
|
||||
if _, err := RemoveProvider(path, "ghost"); err == nil {
|
||||
t.Fatal("RemoveProvider() error = nil, want not-found error")
|
||||
}
|
||||
after, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
if string(after) != string(before) {
|
||||
t.Fatalf("config was rewritten for unknown provider")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameProviderFollowsActiveAndMigratesStoredKey(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// Force the file credential backend so the test never touches the real OS
|
||||
// keyring regardless of platform.
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "oldname",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "oldname", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", APIKeyStored: true, Model: "m1"},
|
||||
{Name: "other", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("oldname", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := RenameProvider(path, "oldname", "newname")
|
||||
if err != nil {
|
||||
t.Fatalf("RenameProvider() error = %v", err)
|
||||
}
|
||||
if cfg.ActiveProvider != "newname" {
|
||||
t.Fatalf("active must follow the rename, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
if cfg.Providers[0].Name != "newname" || !cfg.Providers[0].APIKeyStored {
|
||||
t.Fatalf("renamed profile wrong: %+v", cfg.Providers[0])
|
||||
}
|
||||
if cfg.Providers[1].Name != "other" {
|
||||
t.Fatalf("unrelated profile changed: %+v", cfg.Providers[1])
|
||||
}
|
||||
|
||||
key, ok, err := store.Get("newname")
|
||||
if err != nil || !ok || key != "sk-secret" {
|
||||
t.Fatalf("stored key must migrate to the new name, got key=%q ok=%v err=%v", key, ok, err)
|
||||
}
|
||||
if _, ok, _ := store.Get("oldname"); ok {
|
||||
t.Fatalf("old credential entry must be deleted after migration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameProviderRejectsCollisionAndUnknown(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
before := writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"},
|
||||
{Name: "beta", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
if _, err := RenameProvider(path, "alpha", "BETA"); err == nil {
|
||||
t.Fatal("rename onto an existing name must fail")
|
||||
}
|
||||
if _, err := RenameProvider(path, "ghost", "gamma"); err == nil {
|
||||
t.Fatal("renaming an unknown provider must fail")
|
||||
}
|
||||
after, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
if string(after) != string(before) {
|
||||
t.Fatalf("config was rewritten by a rejected rename")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertProviderPreservesStoredKeyMarkerOnExistingProfile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
// An env-keyed profile with NO stored-key marker — the shape a provider has
|
||||
// before its key is captured into the credential store.
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "groq",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "groq", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://api.groq.com/openai/v1", APIKeyEnv: "GROQ_API_KEY", Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
// The manager/setup edit paths persist a SecureProviderProfile-shaped
|
||||
// profile: key already in the store, marker set, inline key cleared.
|
||||
cfg, err := UpsertProvider(path, ProviderProfile{Name: "groq", APIKeyStored: true}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertProvider() error = %v", err)
|
||||
}
|
||||
if !cfg.Providers[0].APIKeyStored {
|
||||
t.Fatalf("APIKeyStored marker must survive the merge, got %+v", cfg.Providers[0])
|
||||
}
|
||||
if cfg.Providers[0].APIKeyEnv != "GROQ_API_KEY" || cfg.Providers[0].BaseURL == "" {
|
||||
t.Fatalf("other fields must be preserved: %+v", cfg.Providers[0])
|
||||
}
|
||||
persisted := readConfigFixture(t, path)
|
||||
if !persisted.Providers[0].APIKeyStored {
|
||||
t.Fatalf("marker not persisted to disk: %+v", persisted.Providers[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetProviderDescriptionSetsAndClears(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1", Description: "old text"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
cfg, err := SetProviderDescription(path, " ALPHA ", "new text")
|
||||
if err != nil {
|
||||
t.Fatalf("SetProviderDescription() error = %v", err)
|
||||
}
|
||||
if cfg.Providers[0].Description != "new text" {
|
||||
t.Fatalf("description not set: %+v", cfg.Providers[0])
|
||||
}
|
||||
|
||||
// Clearing must persist too — the reason this setter exists (UpsertProvider's
|
||||
// merge treats an empty description as "leave unchanged").
|
||||
cfg, err = SetProviderDescription(path, "alpha", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("SetProviderDescription(clear) error = %v", err)
|
||||
}
|
||||
if cfg.Providers[0].Description != "" {
|
||||
t.Fatalf("description not cleared: %+v", cfg.Providers[0])
|
||||
}
|
||||
persisted := readConfigFixture(t, path)
|
||||
if persisted.Providers[0].Description != "" {
|
||||
t.Fatalf("cleared description not persisted: %+v", persisted.Providers[0])
|
||||
}
|
||||
|
||||
if _, err := SetProviderDescription(path, "ghost", "x"); err == nil {
|
||||
t.Fatal("unknown provider must error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameProviderRollsBackKeyMigrationWhenConfigWriteFails(t *testing.T) {
|
||||
if runtime.GOOS != "darwin" {
|
||||
t.Skip("uses chflags uchg to force the config write to fail; macOS only")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "oldname",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "oldname", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", APIKeyStored: true, Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("oldname", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
// Immutable flag: temp-file creation and store writes in the directory keep
|
||||
// working, but the final rename over config.json fails — the exact window
|
||||
// where a migrated key would otherwise strand under the new name.
|
||||
if out, err := exec.Command("chflags", "uchg", path).CombinedOutput(); err != nil {
|
||||
t.Skipf("chflags uchg unavailable: %v (%s)", err, out)
|
||||
}
|
||||
t.Cleanup(func() { _ = exec.Command("chflags", "nouchg", path).Run() })
|
||||
|
||||
if _, err := RenameProvider(path, "oldname", "newname"); err == nil {
|
||||
t.Fatal("expected the config write to fail under the immutable flag")
|
||||
}
|
||||
|
||||
key, ok, err := store.Get("oldname")
|
||||
if err != nil || !ok || key != "sk-secret" {
|
||||
t.Fatalf("key must be rolled back to the old name, got key=%q ok=%v err=%v", key, ok, err)
|
||||
}
|
||||
if _, ok, _ := store.Get("newname"); ok {
|
||||
t.Fatalf("rolled-back migration must not leave a key under the new name")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenameProviderCaseOnlyKeepsStoredKey: the credential store normalizes
|
||||
// names case-insensitively, so a case-only rename targets ONE store entry —
|
||||
// migrating would Set and then Delete the same key, losing it (PR #560 review).
|
||||
func TestRenameProviderCaseOnlyKeepsStoredKey(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "groq",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "groq", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://api.groq.com/openai/v1", APIKeyStored: true, Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("groq", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := RenameProvider(path, "groq", "Groq")
|
||||
if err != nil {
|
||||
t.Fatalf("RenameProvider() error = %v", err)
|
||||
}
|
||||
if cfg.Providers[0].Name != "Groq" || cfg.ActiveProvider != "Groq" {
|
||||
t.Fatalf("case-only rename must still apply to config: %+v", cfg)
|
||||
}
|
||||
// The store is case-insensitive: the key must remain retrievable under the
|
||||
// new casing (same entry), not deleted by a same-entry "migration".
|
||||
if key, ok, err := store.Get("Groq"); err != nil || !ok || key != "sk-secret" {
|
||||
t.Fatalf("stored key lost on case-only rename: key=%q ok=%v err=%v", key, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEditProviderAppliesRenameFieldsAndDescriptionAtomically(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "groq",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "groq", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://api.groq.com/openai/v1", APIKeyStored: true, Model: "m1", Description: "old text"},
|
||||
{Name: "other", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://o.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("groq", "sk-old"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := EditProvider(path, ProviderEdit{
|
||||
Name: "groq",
|
||||
NewName: "grok-main",
|
||||
Model: "m1-pro",
|
||||
Description: "", // cleared — must persist verbatim
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("EditProvider() error = %v", err)
|
||||
}
|
||||
edited := cfg.Providers[0]
|
||||
if edited.Name != "grok-main" || edited.Model != "m1-pro" || edited.Description != "" {
|
||||
t.Fatalf("edit not applied: %+v", edited)
|
||||
}
|
||||
if edited.BaseURL != "https://api.groq.com/openai/v1" || !edited.APIKeyStored {
|
||||
t.Fatalf("untouched fields must survive: %+v", edited)
|
||||
}
|
||||
if cfg.ActiveProvider != "grok-main" {
|
||||
t.Fatalf("active must follow the rename, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
if key, ok, _ := store.Get("grok-main"); !ok || key != "sk-old" {
|
||||
t.Fatalf("stored key must migrate with the rename, got %q ok=%v", key, ok)
|
||||
}
|
||||
if len(cfg.Providers) != 2 || cfg.Providers[1].Name != "other" {
|
||||
t.Fatalf("unrelated profile changed: %+v", cfg.Providers)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEditProviderCaseOnlyRenameUpdatesInPlace: the manager previously skipped
|
||||
// RenameProvider on case-insensitively-equal names and fell into UpsertProvider,
|
||||
// whose case-SENSITIVE merge appended a duplicate profile. EditProvider matches
|
||||
// case-insensitively, so a case-only rename is an in-place update and the store
|
||||
// entry (case-normalized) survives.
|
||||
func TestEditProviderCaseOnlyRenameUpdatesInPlace(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "groq",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "groq", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://api.groq.com/openai/v1", APIKeyStored: true, Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("groq", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := EditProvider(path, ProviderEdit{Name: "groq", NewName: "Groq"})
|
||||
if err != nil {
|
||||
t.Fatalf("EditProvider() error = %v", err)
|
||||
}
|
||||
if len(cfg.Providers) != 1 {
|
||||
t.Fatalf("case-only rename must not duplicate the profile: %+v", cfg.Providers)
|
||||
}
|
||||
if cfg.Providers[0].Name != "Groq" || !cfg.Providers[0].APIKeyStored {
|
||||
t.Fatalf("in-place update wrong: %+v", cfg.Providers[0])
|
||||
}
|
||||
if cfg.Providers[0].BaseURL != "https://api.groq.com/openai/v1" {
|
||||
t.Fatalf("fields must survive a case-only rename: %+v", cfg.Providers[0])
|
||||
}
|
||||
if cfg.ActiveProvider != "Groq" {
|
||||
t.Fatalf("active must follow, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
if key, ok, _ := store.Get("Groq"); !ok || key != "sk-secret" {
|
||||
t.Fatalf("stored key lost on case-only rename: %q ok=%v", key, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEditProviderRenameMigratesFreshlyCapturedKey: replacing the key AND
|
||||
// renaming in one edit — the caller captures under the CURRENT name and the
|
||||
// rename migration moves it, so the new key lands under the new name.
|
||||
func TestEditProviderRenameMigratesFreshlyCapturedKey(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "gw",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "gw", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://gw.example.com/v1", APIKeyEnv: "GW_KEY", Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
// The caller's contract: a replacement key is stored under the CURRENT name
|
||||
// before EditProvider runs (what SecureProviderProfile does).
|
||||
if err := store.Set("gw", "sk-new"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := EditProvider(path, ProviderEdit{Name: "gw", NewName: "gateway", APIKeyStored: true})
|
||||
if err != nil {
|
||||
t.Fatalf("EditProvider() error = %v", err)
|
||||
}
|
||||
if !cfg.Providers[0].APIKeyStored || cfg.Providers[0].Name != "gateway" {
|
||||
t.Fatalf("marker/rename wrong: %+v", cfg.Providers[0])
|
||||
}
|
||||
if key, ok, _ := store.Get("gateway"); !ok || key != "sk-new" {
|
||||
t.Fatalf("captured key must migrate to the new name, got %q ok=%v", key, ok)
|
||||
}
|
||||
if _, ok, _ := store.Get("gw"); ok {
|
||||
t.Fatalf("old entry must be cleaned up after migration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEditProviderRejectsCollisionAndUnknown(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.json")
|
||||
before := writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"},
|
||||
{Name: "beta", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
if _, err := EditProvider(path, ProviderEdit{Name: "alpha", NewName: "BETA"}); err == nil {
|
||||
t.Fatal("rename onto an existing name must fail")
|
||||
}
|
||||
if _, err := EditProvider(path, ProviderEdit{Name: "ghost", Model: "x"}); err == nil {
|
||||
t.Fatal("editing an unknown provider must fail")
|
||||
}
|
||||
after, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
if string(after) != string(before) {
|
||||
t.Fatalf("config was rewritten by a rejected edit")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// Package dictation implements speech-to-text capture and transcription for
|
||||
// the composer: press a key, talk, and the transcript lands in the input for
|
||||
// review — never auto-fired at the agent.
|
||||
//
|
||||
// Two pipelines share a keybinding and a composer-insertion target:
|
||||
//
|
||||
// - Batch: record to a temp WAV file (Recorder.Start/Stop), then transcribe
|
||||
// in one shot (Transcriber.Transcribe) — a local sherpa-onnx-offline exec
|
||||
// or a single HTTPS POST to Groq/OpenAI. The only pipeline on Termux.
|
||||
// - Streaming: capture raw PCM from the mic tool's stdout
|
||||
// (Recorder.StartStreaming), feed 50ms chunks to a websocket
|
||||
// (Transcriber.StreamTranscribe), and render partial transcripts live.
|
||||
//
|
||||
// Every OS integration shells out via exec.Command with discrete argv — never
|
||||
// a shell string, never CGO — matching internal/imageinput/clipboard.go. The
|
||||
// external binaries (arecord/sox/ffmpeg/termux-microphone-record, sherpa-onnx)
|
||||
// are looked up on PATH and degrade to a clear setup message when missing,
|
||||
// matching internal/lsp's philosophy for language servers.
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Batch transcription providers (config value stt.provider).
|
||||
const (
|
||||
ProviderLocal = "local"
|
||||
ProviderGroq = "groq"
|
||||
ProviderOpenAI = "openai"
|
||||
)
|
||||
|
||||
// Streaming transcription providers (config value stt.streamProvider).
|
||||
const (
|
||||
StreamProviderLocal = "local"
|
||||
StreamProviderDeepgram = "deepgram"
|
||||
StreamProviderOpenAI = "openai"
|
||||
)
|
||||
|
||||
// BatchProviders lists valid stt.provider values, for config validation.
|
||||
func BatchProviders() []string {
|
||||
return []string{ProviderLocal, ProviderGroq, ProviderOpenAI}
|
||||
}
|
||||
|
||||
// StreamProviders lists valid stt.streamProvider values, for config validation.
|
||||
func StreamProviders() []string {
|
||||
return []string{StreamProviderLocal, StreamProviderDeepgram, StreamProviderOpenAI}
|
||||
}
|
||||
|
||||
// AudioFormat identifies a recorded container format, sniffed from the bytes —
|
||||
// desktop recorders produce WAV, Termux's recorder produces M4A/AAC.
|
||||
type AudioFormat string
|
||||
|
||||
const (
|
||||
FormatWAV AudioFormat = "wav"
|
||||
FormatM4A AudioFormat = "m4a"
|
||||
FormatUnknown AudioFormat = ""
|
||||
)
|
||||
|
||||
// SniffFormat detects the audio container from magic bytes. WAV is
|
||||
// "RIFF....WAVE"; MP4-family (Termux's .m4a) has "ftyp" at offset 4.
|
||||
func SniffFormat(data []byte) AudioFormat {
|
||||
if len(data) >= 12 && bytes.Equal(data[:4], []byte("RIFF")) && bytes.Equal(data[8:12], []byte("WAVE")) {
|
||||
return FormatWAV
|
||||
}
|
||||
if len(data) >= 8 && bytes.Equal(data[4:8], []byte("ftyp")) {
|
||||
return FormatM4A
|
||||
}
|
||||
return FormatUnknown
|
||||
}
|
||||
|
||||
// FileName returns a transcription-upload filename for the format.
|
||||
func (f AudioFormat) FileName() string {
|
||||
switch f {
|
||||
case FormatM4A:
|
||||
return "audio.m4a"
|
||||
default:
|
||||
return "audio.wav"
|
||||
}
|
||||
}
|
||||
|
||||
// SetupError reports a missing external tool with install guidance. Missing
|
||||
// binaries are an expected condition (the user hasn't set dictation up yet),
|
||||
// not a bug — callers show the hint instead of a raw exec error.
|
||||
type SetupError struct {
|
||||
Tool string
|
||||
Hint string
|
||||
}
|
||||
|
||||
func (e *SetupError) Error() string {
|
||||
return fmt.Sprintf("%s not found — %s", e.Tool, e.Hint)
|
||||
}
|
||||
|
||||
// AuthError reports that a cloud provider rejected a transcription request for
|
||||
// authentication reasons (HTTP 401/403) — a missing or invalid API key. It is a
|
||||
// fixable credential problem, so callers (the TUI) can offer an inline key
|
||||
// prompt for Provider instead of a dead-end error. Message is the already-
|
||||
// classified, secret-redacted user-facing text.
|
||||
type AuthError struct {
|
||||
Provider string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *AuthError) Error() string { return e.Message }
|
||||
@@ -0,0 +1,893 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/bzip2"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Auto-download of the local engine + a default model (opt-in, behind a confirm
|
||||
// in the F9 flow). Assets come from the official k2-fsa/sherpa-onnx GitHub
|
||||
// releases and are verified against a SHA256 digest before use — the same trust
|
||||
// boundary as installing them yourself, automated. Auto-downloading a native
|
||||
// executable is a real trust decision, which is why it is opt-in and verified.
|
||||
//
|
||||
// Rather than freeze a single hardcoded version, the downloader resolves the
|
||||
// release ASSETS via the GitHub API at download time and verifies each download
|
||||
// against the API's per-asset digest — so a newer sherpa-onnx release (or
|
||||
// "latest") works with no code change. A known-good version is still the DEFAULT
|
||||
// (reproducible, smoke-tested), and for that default the resolved digest is
|
||||
// cross-checked against a pinned value so a silently-changed release is caught.
|
||||
|
||||
// DefaultSherpaVersion is the pinned, smoke-tested release used unless
|
||||
// stt.engineVersion overrides it. "latest" or any tag also works.
|
||||
const DefaultSherpaVersion = "v1.13.3"
|
||||
|
||||
const (
|
||||
sherpaRepo = "k2-fsa/sherpa-onnx"
|
||||
modelReleaseTag = "asr-models" // rolling release the ASR models live in
|
||||
modelAssetName = "sherpa-onnx-moonshine-tiny-en-int8.tar.bz2"
|
||||
defaultAPIBase = "https://api.github.com"
|
||||
)
|
||||
|
||||
// engineAssetSuffix maps GOOS-GOARCH to the version-independent suffix of the
|
||||
// sherpa-onnx executable bundle for that platform. Matching by suffix (not a
|
||||
// full version-embedded name) is what lets any release resolve. The bundles are
|
||||
// self-contained (RPATH $ORIGIN/../lib), so the extracted bin/ executables run
|
||||
// with no library-path setup.
|
||||
var engineAssetSuffix = map[string]string{
|
||||
"linux-amd64": "linux-x64-shared-no-tts.tar.bz2",
|
||||
"darwin-arm64": "osx-arm64-shared-no-tts.tar.bz2",
|
||||
"darwin-amd64": "osx-x64-shared-no-tts.tar.bz2",
|
||||
"windows-amd64": "win-x64-shared-MT-Release-no-tts.tar.bz2",
|
||||
}
|
||||
|
||||
// pinnedEngineDigest cross-checks the DefaultSherpaVersion engine assets (a
|
||||
// silently-changed pinned release is refused). Empty for non-default versions,
|
||||
// which are verified against the API digest only.
|
||||
var pinnedEngineDigest = map[string]string{
|
||||
"linux-amd64": "e5cbfd96f3666c25d2272d468e337764fce6a4d50ed074e4b8e5021bfd7fefc1",
|
||||
"darwin-arm64": "ea73233bc250019ccf1f8a16feb29460efcd3425b50cac16980bcb76281d60bf",
|
||||
"darwin-amd64": "563423887d63d8a831c473e0ab6815139fe2e7a3482936716c2a128071dd217f",
|
||||
"windows-amd64": "343c6c39cb4fe1880a9fb1abc8401d5174629ccbb147fca1fd53c440b2664570",
|
||||
}
|
||||
|
||||
// pinnedModelDigest cross-checks the default Moonshine-tiny int8 model.
|
||||
const pinnedModelDigest = "d5fe6ec4334fef36255b2a4010412cad4c007e33103fec62fb5d17cad88086f2"
|
||||
|
||||
// ModelVariant is a downloadable model the /stt-model picker offers, with a
|
||||
// pinned SHA256 (the model release predates GitHub's per-asset digests, so these
|
||||
// must be pinned). Batch variants use the offline binary; the streaming variant
|
||||
// (a transducer) drives the websocket server for a live transcript.
|
||||
type ModelVariant struct {
|
||||
ID string
|
||||
Label string
|
||||
Description string
|
||||
Bytes int64
|
||||
AssetName string
|
||||
Digest string
|
||||
DirName string
|
||||
Streaming bool
|
||||
// Recommended marks a curated, checksum-pinned, validated model (shown first).
|
||||
Recommended bool
|
||||
}
|
||||
|
||||
// ModelVariants returns the curated English models the download picker offers.
|
||||
// This is a hand-picked shortlist (each pinned by SHA256 since the model release
|
||||
// predates GitHub's per-asset digests), NOT the whole sherpa-onnx model zoo —
|
||||
// many more models (other languages, sizes, and streaming/batch families) work
|
||||
// via a manual stt.localModelPath. Streaming (live) options come first.
|
||||
func ModelVariants() []ModelVariant {
|
||||
return []ModelVariant{
|
||||
// Live / streaming — transcript builds up as you speak.
|
||||
// DirName MUST match what ListModels derives from the asset ("model-"+id, id =
|
||||
// asset minus "sherpa-onnx-" prefix and ".tar.bz2" suffix, dates included) so a
|
||||
// model extracts to the same directory whether picked from the curated shortlist
|
||||
// or the full browse list — otherwise installed-detection disagrees between the
|
||||
// two views and the same model can be downloaded twice. Recommended matches the
|
||||
// full list too (recommendedModelIDs marks every curated variant), so the ★ and
|
||||
// ordering don't change when the background fetch swaps the curated list out.
|
||||
{
|
||||
ID: "streaming-zipformer-en-kroko", Label: "Kroko (newest, fastest)",
|
||||
Description: "smallest live model", Bytes: 54 << 20,
|
||||
AssetName: "sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06.tar.bz2",
|
||||
Digest: "c8676e5ff9ac2a85296e53ee0fd4d5fb1db6770e7a7647166eeafe349ade6834",
|
||||
DirName: "model-streaming-zipformer-en-kroko-2025-08-06",
|
||||
Streaming: true,
|
||||
Recommended: true,
|
||||
},
|
||||
{
|
||||
ID: "streaming-zipformer-20m", Label: "Zipformer 20M",
|
||||
Description: "small, well-tested", Bytes: 121 << 20,
|
||||
AssetName: "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2",
|
||||
Digest: "9c559283e8498d3fe95913c79ca1cb454bb26281ac2b102b41306c7d752765d9",
|
||||
DirName: "model-streaming-zipformer-en-20M-2023-02-17",
|
||||
Streaming: true,
|
||||
},
|
||||
{
|
||||
ID: "streaming-zipformer", Label: "Zipformer",
|
||||
Description: "larger, more accurate", Bytes: 297 << 20,
|
||||
AssetName: "sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2",
|
||||
Digest: "639e25b578e9e997131402199419c13a941f8e4e198e2da1ce57dbf5cf401282",
|
||||
DirName: "model-streaming-zipformer-en-2023-06-26",
|
||||
Streaming: true,
|
||||
},
|
||||
{
|
||||
ID: "nemo-streaming-fast-conformer-transducer-en-80ms-int8", Label: "NeMo streaming (fast)",
|
||||
Description: "low-latency English", Bytes: 98 << 20,
|
||||
AssetName: "sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms-int8.tar.bz2",
|
||||
Digest: "7bd33a914e93370a1ba9c2066d9e841bdcad8613fa2a00537c1ae15d851a14d8",
|
||||
DirName: "model-nemo-streaming-fast-conformer-transducer-en-80ms-int8",
|
||||
Streaming: true,
|
||||
Recommended: true,
|
||||
},
|
||||
// Batch — transcribes once when you stop.
|
||||
{
|
||||
ID: "moonshine-tiny", Label: "Moonshine tiny",
|
||||
Description: "fast, smallest", Bytes: 103 << 20,
|
||||
AssetName: "sherpa-onnx-moonshine-tiny-en-int8.tar.bz2",
|
||||
Digest: pinnedModelDigest,
|
||||
DirName: "model-moonshine-tiny-en-int8",
|
||||
},
|
||||
{
|
||||
ID: "moonshine-base", Label: "Moonshine base",
|
||||
Description: "more accurate, larger", Bytes: 239 << 20,
|
||||
AssetName: "sherpa-onnx-moonshine-base-en-int8.tar.bz2",
|
||||
Digest: "21870cecaa2e44e4e2bf63e02d1072bed183ccd10284871353bd9d24dad14e5e",
|
||||
DirName: "model-moonshine-base-en-int8",
|
||||
Recommended: true,
|
||||
},
|
||||
{
|
||||
ID: "sense-voice-zh-en-ja-ko-yue-int8-2025-09-09", Label: "SenseVoice",
|
||||
Description: "multilingual, accurate", Bytes: 158 << 20,
|
||||
AssetName: "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2",
|
||||
Digest: "7305f7905bfcf77fa0b39388a313f3da35c68d971661a65475b56fb2162c8e63",
|
||||
DirName: "model-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09",
|
||||
Recommended: true,
|
||||
},
|
||||
{
|
||||
// Heavy (~1 GB) but the best transcription quality — recommended when
|
||||
// accuracy matters more than speed/size. The asr-models release publishes
|
||||
// no digest for it, so (like every browse-listed model) it downloads with
|
||||
// TLS-only verification.
|
||||
ID: "whisper-large-v3", Label: "Whisper large v3",
|
||||
Description: "best accuracy, heavy", Bytes: 1019 << 20,
|
||||
AssetName: "sherpa-onnx-whisper-large-v3.tar.bz2",
|
||||
DirName: "model-whisper-large-v3",
|
||||
Recommended: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// recommendedModelIDs marks the curated variants flagged Recommended so the full
|
||||
// browse list stars (and surfaces) exactly the ones the curated shortlist does —
|
||||
// keyed off the same Recommended flag so the two views never disagree.
|
||||
var recommendedModelIDs = func() map[string]bool {
|
||||
m := map[string]bool{}
|
||||
for _, v := range ModelVariants() {
|
||||
if v.Recommended {
|
||||
m[v.AssetName] = true
|
||||
}
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// pinnedModelDigests maps a model asset name to its pinned SHA256 (the curated
|
||||
// variants), so browse-listed models we happen to know get verified too.
|
||||
var pinnedModelDigests = func() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, v := range ModelVariants() {
|
||||
m[v.AssetName] = v.Digest
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// curatedLabels maps a curated model's asset name to its friendly label, so the
|
||||
// recommended rows read nicely in the full browse list too.
|
||||
var curatedLabels = func() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, v := range ModelVariants() {
|
||||
m[v.AssetName] = v.Label
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// browseLabel turns a raw asset id into a shorter display label: it drops the
|
||||
// redundant "-int8"/quantization noise and the "streaming-" prefix (the group
|
||||
// header already says live vs batch).
|
||||
func browseLabel(id string) string {
|
||||
label := strings.TrimPrefix(id, "streaming-")
|
||||
for _, drop := range []string{"-int8", ".int8", "-quantized", "-fp16"} {
|
||||
label = strings.ReplaceAll(label, drop, "")
|
||||
}
|
||||
return label
|
||||
}
|
||||
|
||||
// ListModels fetches every transcription model in the sherpa-onnx asr-models
|
||||
// release and returns them as variants (the curated ones flagged Recommended and
|
||||
// carrying a pinned digest; the rest with an empty digest, downloaded with
|
||||
// TLS-only verification since that release predates GitHub's per-asset digests).
|
||||
// Non-ASR assets (VAD, TTS, punctuation, speaker/keyword/…): filtered out.
|
||||
func ListModels(ctx context.Context, client *http.Client, apiBase string) ([]ModelVariant, error) {
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
if apiBase == "" {
|
||||
apiBase = defaultAPIBase
|
||||
}
|
||||
// Resolve the release id, then page its assets (a release can have hundreds).
|
||||
relURL := fmt.Sprintf("%s/repos/%s/releases/tags/%s", strings.TrimRight(apiBase, "/"), sherpaRepo, modelReleaseTag)
|
||||
var rel struct {
|
||||
ID int64 `json:"id"`
|
||||
Assets []githubAsset
|
||||
}
|
||||
if err := getJSON(ctx, client, relURL, &rel); err != nil {
|
||||
return nil, fmt.Errorf("listing dictation models: %w", err)
|
||||
}
|
||||
assets := rel.Assets
|
||||
// If the inline assets look capped, page the assets endpoint for the rest.
|
||||
for page := 1; len(assets)%100 == 0 && len(assets) > 0; page++ {
|
||||
var more []githubAsset
|
||||
u := fmt.Sprintf("%s/repos/%s/releases/%d/assets?per_page=100&page=%d", strings.TrimRight(apiBase, "/"), sherpaRepo, rel.ID, page+1)
|
||||
if err := getJSON(ctx, client, u, &more); err != nil || len(more) == 0 {
|
||||
break
|
||||
}
|
||||
assets = append(assets, more...)
|
||||
if len(more) < 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
var out []ModelVariant
|
||||
for _, a := range assets {
|
||||
if seen[a.Name] || !isTranscriptionModelAsset(a.Name) {
|
||||
continue
|
||||
}
|
||||
seen[a.Name] = true
|
||||
id := strings.TrimSuffix(strings.TrimPrefix(a.Name, "sherpa-onnx-"), ".tar.bz2")
|
||||
digest := strings.TrimPrefix(a.Digest, "sha256:")
|
||||
if digest == "" {
|
||||
digest = pinnedModelDigests[a.Name] // may still be empty (browse/unverified)
|
||||
}
|
||||
label := browseLabel(id)
|
||||
desc := modelFamily(a.Name) // readable language, e.g. "French"
|
||||
if nice, ok := curatedLabels[a.Name]; ok {
|
||||
label = nice
|
||||
desc = curatedDescriptions[a.Name] // curated blurb ("fast, smallest")
|
||||
}
|
||||
out = append(out, ModelVariant{
|
||||
ID: id,
|
||||
Label: label,
|
||||
Description: desc,
|
||||
Bytes: a.Size,
|
||||
AssetName: a.Name,
|
||||
Digest: digest,
|
||||
DirName: "model-" + id,
|
||||
Streaming: strings.Contains(strings.ToLower(a.Name), "streaming-"),
|
||||
Recommended: recommendedModelIDs[a.Name],
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type githubAsset struct {
|
||||
Name string `json:"name"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
func getJSON(ctx context.Context, client *http.Client, url string, v any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("GitHub API HTTP %d", resp.StatusCode)
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(v)
|
||||
}
|
||||
|
||||
// isTranscriptionModelAsset reports whether an asset is a speech-to-text model
|
||||
// (as opposed to a VAD/TTS/punctuation/speaker/keyword/enhancement tool).
|
||||
func isTranscriptionModelAsset(name string) bool {
|
||||
n := strings.ToLower(name)
|
||||
if !strings.HasPrefix(n, "sherpa-onnx-") || !strings.HasSuffix(n, ".tar.bz2") {
|
||||
return false
|
||||
}
|
||||
for _, bad := range []string{
|
||||
"-vad", "vad-", "tts", "punct", "speaker", "diariz", "kws", "keyword",
|
||||
"audio-tagging", "denois", "source-separation", "speech-enhancement",
|
||||
"spoken-language", "language-identification", "-lid", "melo", "kokoro",
|
||||
"vits", "matcha", "piper", "wespeaker",
|
||||
// Rockchip NPU (RKNN) builds for specific ARM boards — not CPU/ONNX, so
|
||||
// the standard sherpa-onnx binary can't run them.
|
||||
"rknn", "rk3566", "rk3568", "rk3562", "rk3588", "rk3576", "rk3528", "rk3399",
|
||||
} {
|
||||
if strings.Contains(n, bad) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, fam := range []string{
|
||||
"zipformer", "moonshine", "whisper", "paraformer", "sense-voice",
|
||||
"sensevoice", "conformer", "nemo", "canary", "parakeet", "fire-red",
|
||||
"fireredasr", "dolphin", "telespeech", "wenet", "omnilingual", "funasr",
|
||||
"medasr", "cohere", "qwen", "citrinet", "transducer", "-ctc-", "zipvoice",
|
||||
} {
|
||||
if strings.Contains(n, fam) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// langNames maps ISO-ish language tokens in a model name to readable names.
|
||||
var langNames = map[string]string{
|
||||
"en": "English", "zh": "Chinese", "yue": "Cantonese", "fr": "French",
|
||||
"de": "German", "es": "Spanish", "ru": "Russian", "ja": "Japanese",
|
||||
"ko": "Korean", "bn": "Bengali", "ar": "Arabic", "hi": "Hindi",
|
||||
"pt": "Portuguese", "nl": "Dutch", "vi": "Vietnamese", "th": "Thai",
|
||||
"uk": "Ukrainian", "fa": "Persian", "pl": "Polish", "tr": "Turkish",
|
||||
"cs": "Czech", "sv": "Swedish", "fil": "Filipino", "tl": "Tagalog",
|
||||
"fi": "Finnish", "el": "Greek", "he": "Hebrew", "ro": "Romanian",
|
||||
}
|
||||
|
||||
// modelFamily is a short readable descriptor for a model, favoring the language
|
||||
// (the most useful thing not already obvious from the group header) and falling
|
||||
// back to the family. E.g. "French", "Chinese", "Multilingual".
|
||||
func modelFamily(name string) string {
|
||||
n := strings.ToLower(name)
|
||||
if strings.Contains(n, "multi") || strings.Contains(n, "bilingual") ||
|
||||
strings.Contains(n, "trilingual") || strings.Contains(n, "polyglot") ||
|
||||
strings.Contains(n, "omnilingual") {
|
||||
return "Multilingual"
|
||||
}
|
||||
var langs []string
|
||||
seen := map[string]bool{}
|
||||
add := func(name string) {
|
||||
if name != "" && !seen[name] {
|
||||
seen[name] = true
|
||||
langs = append(langs, name)
|
||||
}
|
||||
}
|
||||
for _, tok := range strings.Split(strings.TrimSuffix(n, ".tar.bz2"), "-") {
|
||||
add(langNames[tok])
|
||||
}
|
||||
// Full language words spelled out in some names (e.g. "-cantonese-").
|
||||
for word, name := range map[string]string{
|
||||
"cantonese": "Cantonese", "mandarin": "Chinese", "english": "English",
|
||||
"chinese": "Chinese", "korean": "Korean", "japanese": "Japanese",
|
||||
"russian": "Russian", "arabic": "Arabic", "spanish": "Spanish",
|
||||
"french": "French", "german": "German", "vietnamese": "Vietnamese",
|
||||
"thai": "Thai", "farsi": "Persian", "persian": "Persian",
|
||||
"portuguese": "Portuguese", "italian": "Italian", "dutch": "Dutch",
|
||||
"ukrainian": "Ukrainian", "polish": "Polish", "turkish": "Turkish",
|
||||
} {
|
||||
if strings.Contains(n, word) {
|
||||
add(name)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(langs) == 0:
|
||||
return ""
|
||||
case len(langs) > 2:
|
||||
return "Multilingual"
|
||||
default:
|
||||
return strings.Join(langs, " + ")
|
||||
}
|
||||
}
|
||||
|
||||
// curatedDescriptions maps a curated model's asset name to its quality blurb.
|
||||
var curatedDescriptions = func() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, v := range ModelVariants() {
|
||||
m[v.AssetName] = v.Description
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// EngineComponents identifies the resolved local-engine paths.
|
||||
type EngineComponents struct {
|
||||
BinaryPath string // extracted sherpa-onnx-offline
|
||||
ServerPath string // extracted sherpa-onnx-online-websocket-server
|
||||
ModelPath string // extracted model directory
|
||||
}
|
||||
|
||||
// DownloadOptions configures EnsureLocalEngine.
|
||||
type DownloadOptions struct {
|
||||
// DestRoot is where extracted engines/models live (e.g. ~/.config/zero/stt).
|
||||
DestRoot string
|
||||
// EngineVersion selects the sherpa-onnx release tag ("" → DefaultSherpaVersion;
|
||||
// "latest" and any tag also work).
|
||||
EngineVersion string
|
||||
// HTTPClient is injectable for tests; nil uses a plain client.
|
||||
HTTPClient *http.Client
|
||||
// APIBase overrides the GitHub API base (tests). "" → api.github.com.
|
||||
APIBase string
|
||||
// Model selection (default: the Moonshine-tiny int8 batch model). Set these
|
||||
// from a ModelVariant to download a different model.
|
||||
ModelAssetName string
|
||||
ModelPinnedDigest string
|
||||
ModelDirName string
|
||||
// ModelLabel is the friendly model name shown in progress ("Zipformer 20M").
|
||||
ModelLabel string
|
||||
// Progress receives live status strings for the UI, including a percentage
|
||||
// while downloading (e.g. "Engine 45% · 57/126 MB").
|
||||
Progress func(status string)
|
||||
// platformKey overrides runtime detection (tests).
|
||||
platformKey string
|
||||
// skipPinned disables the pinned-digest cross-check (tests, which use fixture
|
||||
// assets); the API-digest verification still runs.
|
||||
skipPinned bool
|
||||
}
|
||||
|
||||
// AutoDownloadSupported reports whether the current platform has a known engine
|
||||
// asset (so the F9 flow can offer the download). Termux/Android and unusual
|
||||
// arches install manually or use a cloud provider.
|
||||
func AutoDownloadSupported() bool {
|
||||
_, ok := engineAssetSuffix[platformKey()]
|
||||
return ok
|
||||
}
|
||||
|
||||
func platformKey() string { return runtime.GOOS + "-" + runtime.GOARCH }
|
||||
|
||||
// EnsureLocalEngine downloads (if absent) and verifies the sherpa-onnx engine and
|
||||
// the default model into DestRoot, returning the resolved paths. Idempotent: an
|
||||
// already-extracted, still-present engine/model is reused without re-downloading.
|
||||
// Every archive is SHA256-verified against the GitHub API digest before extraction.
|
||||
func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineComponents, error) {
|
||||
if strings.TrimSpace(opts.DestRoot) == "" {
|
||||
return EngineComponents{}, fmt.Errorf("dictation download: DestRoot is required")
|
||||
}
|
||||
key := opts.platformKey
|
||||
if key == "" {
|
||||
key = platformKey()
|
||||
}
|
||||
suffix, ok := engineAssetSuffix[key]
|
||||
if !ok {
|
||||
return EngineComponents{}, &SetupError{
|
||||
Tool: "sherpa-onnx auto-download",
|
||||
Hint: fmt.Sprintf("no prebuilt engine is known for %s — install sherpa-onnx manually or use a cloud provider (see docs/dictation.md)", key),
|
||||
}
|
||||
}
|
||||
version := strings.TrimSpace(opts.EngineVersion)
|
||||
if version == "" {
|
||||
version = DefaultSherpaVersion
|
||||
}
|
||||
progress := opts.Progress
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
}
|
||||
client := opts.HTTPClient
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
apiBase := opts.APIBase
|
||||
if apiBase == "" {
|
||||
apiBase = defaultAPIBase
|
||||
}
|
||||
|
||||
engineDir := filepath.Join(opts.DestRoot, "engine-"+version+"-"+key)
|
||||
targetWindows := strings.HasPrefix(key, "windows-")
|
||||
// Resolve through the tarball's flattened subdir so an ALREADY-extracted
|
||||
// engine is found and not needlessly re-downloaded (the idempotency check).
|
||||
binPath, serverPath := resolveEnginePaths(engineDir, targetWindows)
|
||||
if !fileExists(binPath) {
|
||||
pinned := ""
|
||||
if version == DefaultSherpaVersion && !opts.skipPinned {
|
||||
pinned = pinnedEngineDigest[key]
|
||||
}
|
||||
asset, err := resolveAsset(ctx, client, apiBase, version, "sherpa-onnx-", suffix)
|
||||
if err != nil {
|
||||
return EngineComponents{}, err
|
||||
}
|
||||
if err := downloadVerifyExtract(ctx, client, asset, pinned, false, "Engine", engineDir, progress); err != nil {
|
||||
return EngineComponents{}, err
|
||||
}
|
||||
binPath, serverPath = resolveEnginePaths(engineDir, targetWindows)
|
||||
}
|
||||
if !fileExists(binPath) {
|
||||
return EngineComponents{}, fmt.Errorf("dictation download: engine binary not found after extraction in %s", engineDir)
|
||||
}
|
||||
|
||||
modelName := opts.ModelAssetName
|
||||
if modelName == "" {
|
||||
modelName = modelAssetName
|
||||
}
|
||||
modelDirName := opts.ModelDirName
|
||||
if modelDirName == "" {
|
||||
modelDirName = "model-moonshine-tiny-en-int8"
|
||||
}
|
||||
modelDir := filepath.Join(opts.DestRoot, modelDirName)
|
||||
if !dirHasModel(modelDir) {
|
||||
asset, err := resolveAsset(ctx, client, apiBase, modelReleaseTag, modelName, "")
|
||||
if err != nil {
|
||||
return EngineComponents{}, err
|
||||
}
|
||||
modelPinned := opts.ModelPinnedDigest
|
||||
// Only fall back to the built-in digest for the DEFAULT model — a different
|
||||
// (browse-listed) model must never be checked against the default's digest.
|
||||
if modelPinned == "" && opts.ModelAssetName == "" {
|
||||
modelPinned = pinnedModelDigest
|
||||
}
|
||||
if opts.skipPinned {
|
||||
modelPinned = ""
|
||||
}
|
||||
modelLabel := opts.ModelLabel
|
||||
if modelLabel == "" {
|
||||
modelLabel = "Model"
|
||||
}
|
||||
// Models are data files from the official release: verify against a digest
|
||||
// when we have one (curated/pinned or API-provided), else allow a TLS-only
|
||||
// download (the model release predates GitHub's per-asset digests). The
|
||||
// engine BINARY above never allows this — a native executable is always
|
||||
// digest-verified.
|
||||
if err := downloadVerifyExtract(ctx, client, asset, modelPinned, true, modelLabel, modelDir, progress); err != nil {
|
||||
return EngineComponents{}, err
|
||||
}
|
||||
}
|
||||
resolvedModel := modelDir
|
||||
if !hasTokensFile(resolvedModel) {
|
||||
if child, err := flattenSingleChild(modelDir); err == nil {
|
||||
resolvedModel = child
|
||||
}
|
||||
}
|
||||
if !hasTokensFile(resolvedModel) {
|
||||
return EngineComponents{}, fmt.Errorf("dictation download: model files not found after extraction in %s", modelDir)
|
||||
}
|
||||
return EngineComponents{BinaryPath: binPath, ServerPath: serverPath, ModelPath: resolvedModel}, nil
|
||||
}
|
||||
|
||||
type resolvedAsset struct {
|
||||
url string
|
||||
sha256 string
|
||||
size int64
|
||||
name string
|
||||
}
|
||||
|
||||
// resolveAsset queries the GitHub release for tag and returns the asset whose
|
||||
// name has the given prefix and suffix, plus its SHA256 digest and size.
|
||||
func resolveAsset(ctx context.Context, client *http.Client, apiBase, tag, namePrefix, nameSuffix string) (resolvedAsset, error) {
|
||||
tagPath := "tags/" + tag
|
||||
if tag == "" || tag == "latest" {
|
||||
tagPath = "latest"
|
||||
}
|
||||
url := fmt.Sprintf("%s/repos/%s/releases/%s", strings.TrimRight(apiBase, "/"), sherpaRepo, tagPath)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return resolvedAsset{}, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return resolvedAsset{}, fmt.Errorf("resolving sherpa-onnx release %s: %w", tag, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return resolvedAsset{}, fmt.Errorf("resolving sherpa-onnx release %s: GitHub API HTTP %d", tag, resp.StatusCode)
|
||||
}
|
||||
var release struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Assets []struct {
|
||||
Name string `json:"name"`
|
||||
DownloadURL string `json:"browser_download_url"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
} `json:"assets"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
|
||||
return resolvedAsset{}, fmt.Errorf("parsing sherpa-onnx release %s: %w", tag, err)
|
||||
}
|
||||
for _, a := range release.Assets {
|
||||
if !strings.HasPrefix(a.Name, namePrefix) || !strings.HasSuffix(a.Name, nameSuffix) {
|
||||
continue
|
||||
}
|
||||
// The API digest may be empty for older assets (GitHub only populates it for
|
||||
// assets uploaded after ~2024 — the ASR models predate that). An empty
|
||||
// digest is fine here; downloadVerifyExtract requires the API digest OR a
|
||||
// pinned one and refuses only when neither exists.
|
||||
digest := strings.TrimPrefix(a.Digest, "sha256:")
|
||||
return resolvedAsset{url: a.DownloadURL, sha256: digest, size: a.Size, name: a.Name}, nil
|
||||
}
|
||||
return resolvedAsset{}, &SetupError{
|
||||
Tool: "sherpa-onnx auto-download",
|
||||
Hint: fmt.Sprintf("release %s has no asset matching %s*%s — try a different stt.engineVersion or install manually", tag, namePrefix, nameSuffix),
|
||||
}
|
||||
}
|
||||
|
||||
// downloadVerifyExtract fetches the asset to a temp file, verifies its SHA256
|
||||
// (against the API digest, and — when pinned is set — a cross-check that the
|
||||
// resolved digest equals the audited value), and extracts the tar.bz2. A
|
||||
// mismatch aborts before anything is extracted or run.
|
||||
func downloadVerifyExtract(ctx context.Context, client *http.Client, asset resolvedAsset, pinned string, allowUnverified bool, label, destDir string, progress func(string)) error {
|
||||
// Determine the digest to verify against: the API digest and the pinned digest
|
||||
// must agree when both exist; at least one must exist unless allowUnverified
|
||||
// (models are data files from the official release — TLS-only is acceptable;
|
||||
// a native executable never sets allowUnverified).
|
||||
expected := asset.sha256
|
||||
if pinned != "" {
|
||||
if expected != "" && !strings.EqualFold(expected, pinned) {
|
||||
return fmt.Errorf("dictation download: %s digest %s does not match the pinned known-good value %s — refusing (the release may have changed)", asset.name, asset.sha256, pinned)
|
||||
}
|
||||
expected = pinned
|
||||
}
|
||||
if expected == "" && !allowUnverified {
|
||||
return fmt.Errorf("dictation download: no SHA256 available for %s (neither the GitHub API nor a pinned value) — refusing an unverifiable download", asset.name)
|
||||
}
|
||||
// Ensure the parent of destDir exists; the final destDir is created by
|
||||
// the atomic rename at the end of the function.
|
||||
if err := os.MkdirAll(filepath.Dir(destDir), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
// Stage the extraction into a sibling temp dir and only swap it into
|
||||
// destDir on success. An interrupted run that left a truncated engine
|
||||
// binary in destDir would otherwise pass fileExists() on the next start
|
||||
// and break the local recorder silently.
|
||||
stageDir, err := os.MkdirTemp(filepath.Dir(destDir), filepath.Base(destDir)+".stage-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cleanupStage := true
|
||||
defer func() {
|
||||
if cleanupStage {
|
||||
_ = os.RemoveAll(stageDir)
|
||||
}
|
||||
}()
|
||||
|
||||
tmp, err := os.CreateTemp(filepath.Dir(destDir), "stt-dl-*.tar.bz2")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, asset.url, nil)
|
||||
if err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("downloading %s: %w", asset.name, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("downloading %s: HTTP %d", asset.name, resp.StatusCode)
|
||||
}
|
||||
hasher := sha256.New()
|
||||
total := asset.size
|
||||
if total <= 0 {
|
||||
total = resp.ContentLength
|
||||
}
|
||||
src := io.Reader(resp.Body)
|
||||
if total > 0 {
|
||||
src = &progressReader{r: resp.Body, total: total, label: label, report: progress, lastPct: -1}
|
||||
}
|
||||
if _, err := io.Copy(io.MultiWriter(tmp, hasher), src); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("downloading %s: %w", asset.name, err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
got := hex.EncodeToString(hasher.Sum(nil))
|
||||
if expected != "" && !strings.EqualFold(got, expected) {
|
||||
return fmt.Errorf("dictation download: checksum mismatch for %s (want %s, got %s) — refusing to extract", asset.name, expected, got)
|
||||
}
|
||||
if err := extractTarBz2(tmpPath, stageDir, "Extracting "+label, progress); err != nil {
|
||||
return err
|
||||
}
|
||||
// Remove any pre-existing destDir so Rename is atomic on the same
|
||||
// filesystem (os.Rename refuses to overwrite a non-empty dir).
|
||||
if err := os.RemoveAll(destDir); err != nil {
|
||||
return fmt.Errorf("clearing previous %s install: %w", label, err)
|
||||
}
|
||||
if err := os.Rename(stageDir, destDir); err != nil {
|
||||
return fmt.Errorf("promoting staged %s: %w", label, err)
|
||||
}
|
||||
cleanupStage = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// progressReader reports download progress as a percentage on each whole-percent
|
||||
// change, formatted "<label> NN% · X/Y MB".
|
||||
type progressReader struct {
|
||||
r io.Reader
|
||||
total int64
|
||||
read int64
|
||||
label string
|
||||
report func(string)
|
||||
lastPct int
|
||||
}
|
||||
|
||||
func (p *progressReader) Read(b []byte) (int, error) {
|
||||
n, err := p.r.Read(b)
|
||||
p.read += int64(n)
|
||||
if p.total > 0 {
|
||||
pct := int(p.read * 100 / p.total)
|
||||
if pct != p.lastPct {
|
||||
p.lastPct = pct
|
||||
p.report(fmt.Sprintf("%s %d%% · %d/%d MB", p.label, pct, p.read>>20, p.total>>20))
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// enginePaths builds the engine executable paths. The ".exe" suffix follows the
|
||||
// TARGET platform (the one the engine was downloaded for), not the host — they
|
||||
// match in production (you download your own platform), but keying off the target
|
||||
// keeps resolution correct when the two differ (e.g. a cross-platform test).
|
||||
func enginePaths(engineDir string, targetWindows bool) (bin, server string) {
|
||||
ext := ""
|
||||
if targetWindows {
|
||||
ext = ".exe"
|
||||
}
|
||||
return filepath.Join(engineDir, "bin", "sherpa-onnx-offline"+ext),
|
||||
filepath.Join(engineDir, "bin", "sherpa-onnx-online-websocket-server"+ext)
|
||||
}
|
||||
|
||||
// resolveEnginePaths finds the executables, flattening the tarball's single
|
||||
// top-level directory when present.
|
||||
func resolveEnginePaths(engineDir string, targetWindows bool) (bin, server string) {
|
||||
bin, server = enginePaths(engineDir, targetWindows)
|
||||
if fileExists(bin) {
|
||||
return bin, server
|
||||
}
|
||||
if child, err := flattenSingleChild(engineDir); err == nil && child != engineDir {
|
||||
return enginePaths(child, targetWindows)
|
||||
}
|
||||
return bin, server
|
||||
}
|
||||
|
||||
// extractTarBz2 unpacks a bzip2-compressed tar into destDir, guarding against
|
||||
// path-traversal entries. It reports extraction progress by how much of the
|
||||
// (compressed) archive it has consumed — the same MB scale as the download, so
|
||||
// "Extracting … 50% · 520/1041 MB" lines up with the download that preceded it.
|
||||
func extractTarBz2(archivePath, destDir, label string, progress func(string)) error {
|
||||
f, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
var src io.Reader = f
|
||||
if progress != nil {
|
||||
if info, statErr := f.Stat(); statErr == nil && info.Size() > 0 {
|
||||
src = &progressReader{r: f, total: info.Size(), label: label, report: progress, lastPct: -1}
|
||||
}
|
||||
}
|
||||
tr := tar.NewReader(bzip2.NewReader(src))
|
||||
destAbs, err := filepath.Abs(destDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target := filepath.Join(destDir, hdr.Name)
|
||||
absTarget, err := filepath.Abs(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if absTarget != destAbs && !strings.HasPrefix(absTarget, destAbs+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("dictation download: archive entry %q escapes destination", hdr.Name)
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(hdr.Mode)&0o777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, tr); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// flattenSingleChild returns the sole subdirectory of dir (the tarball's
|
||||
// top-level folder) when dir contains exactly one directory entry.
|
||||
func flattenSingleChild(dir string) (string, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var dirs []string
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
dirs = append(dirs, e.Name())
|
||||
}
|
||||
}
|
||||
if len(dirs) == 1 {
|
||||
return filepath.Join(dir, dirs[0]), nil
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// ModelDownloaded reports whether a model variant is already extracted under
|
||||
// destRoot (so the picker can mark it as installed). dirName is the variant's
|
||||
// DirName.
|
||||
func ModelDownloaded(destRoot, dirName string) bool {
|
||||
if destRoot == "" || dirName == "" {
|
||||
return false
|
||||
}
|
||||
return dirHasModel(filepath.Join(destRoot, dirName))
|
||||
}
|
||||
|
||||
// EngineDownloaded reports whether the shared sherpa-onnx engine is already on
|
||||
// disk for this platform (one engine serves every model), so the picker can drop
|
||||
// the engine's size from a model's download total once it's been fetched.
|
||||
func EngineDownloaded(destRoot, version string) bool {
|
||||
if destRoot == "" {
|
||||
return false
|
||||
}
|
||||
if version == "" {
|
||||
version = DefaultSherpaVersion
|
||||
}
|
||||
engineDir := filepath.Join(destRoot, "engine-"+version+"-"+platformKey())
|
||||
bin, _ := resolveEnginePaths(engineDir, runtime.GOOS == "windows")
|
||||
return fileExists(bin)
|
||||
}
|
||||
|
||||
func dirHasModel(dir string) bool {
|
||||
if hasTokensFile(dir) {
|
||||
return true
|
||||
}
|
||||
if child, err := flattenSingleChild(dir); err == nil && child != dir {
|
||||
return hasTokensFile(child)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hasTokensFile reports whether dir holds a sherpa tokens file — usually
|
||||
// "tokens.txt", but some families (Whisper) prefix it as "<model>-tokens.txt".
|
||||
func hasTokensFile(dir string) bool {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.HasSuffix(e.Name(), "tokens.txt") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fileExists(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && !info.IsDir()
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestEnsureLocalEngineRealDownload exercises the full auto-download against the
|
||||
// REAL k2-fsa/sherpa-onnx GitHub API (resolve → verify digest → extract → run).
|
||||
// It downloads ~130MB, so it is opt-in via ZERO_STT_DOWNLOAD_TEST=1.
|
||||
//
|
||||
// ZERO_STT_DOWNLOAD_TEST=1 go test ./internal/dictation/ -run RealDownload -v -timeout 10m
|
||||
func TestEnsureLocalEngineRealDownload(t *testing.T) {
|
||||
if os.Getenv("ZERO_STT_DOWNLOAD_TEST") == "" {
|
||||
t.Skip("set ZERO_STT_DOWNLOAD_TEST=1 to run the real ~130MB download smoke test")
|
||||
}
|
||||
dest := t.TempDir()
|
||||
comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: dest,
|
||||
Progress: func(s string) { t.Logf("stage: %s", s) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureLocalEngine: %v", err)
|
||||
}
|
||||
if !fileExists(comp.BinaryPath) || !fileExists(filepath.Join(comp.ModelPath, "tokens.txt")) {
|
||||
t.Fatalf("missing components: %+v", comp)
|
||||
}
|
||||
|
||||
// The downloaded engine + model must actually transcribe.
|
||||
wav := filepath.Join(comp.ModelPath, "test_wavs", "0.wav")
|
||||
if !fileExists(wav) {
|
||||
t.Skipf("no bundled test wav at %s", wav)
|
||||
}
|
||||
audio, err := os.ReadFile(wav)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tr, err := NewLocalTranscriber(LocalConfig{Binary: comp.BinaryPath, ModelPath: comp.ModelPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text, err := tr.Transcribe(context.Background(), audio)
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe with downloaded engine: %v", err)
|
||||
}
|
||||
if text == "" {
|
||||
t.Fatal("downloaded engine returned empty transcript")
|
||||
}
|
||||
t.Logf("downloaded-engine transcript: %q", text)
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tiny tar.bz2 fixtures (generated in-repo): the engine has top/bin/sherpa-onnx-
|
||||
// offline + server; the model has top/tokens.txt.
|
||||
const (
|
||||
engineTarB64 = "QlpoOTFBWSZTWW+fEfAAAW1/hMGRAIBAA/+AAgAgRH9v3+AAAQCoMAEYAYA00NGjEZANABoYYA00NGjEZANABoYIpE0TRoTGkA2poGgxqepl1H6w5anW3tY2OggUi1AW/d29pO0GLUoOtnQsrGFNowIEh0cImtQ9gyHqYH7unm+/wZSLXsBuBDDCfHiahCsGEgGL9/pUG+RQZo76xlaDIGKWDErPKuyhXYOlzJMG3iNa56kiluctzUqgZ2A1AGwgcB3jtB5PB7pYYlDHH8OvOznM6B6MHPRqa7sTE1vwDBc0LjmG4ah5F+YYaXHTZpGmcemWW1yMh1adhdrB6A3nAQP8XckU4UJBvnxHwA=="
|
||||
engineSHA = "97b5cb6a417705860f8940e1bf9ffec8cc60e8666a0d228cd60f1de7607d503d"
|
||||
modelTarB64 = "QlpoOTFBWSZTWTtAoX4AANp/hMOQBIBAAf+AAAIQBHoJ3mAAAQAIMAC5sG1SNMhNqGmQwIyeKEagam1DQ0AAABFRT0T0mg0aeoDQHpH7Nzrs24wsbVW8hMQHClNI0LuLhKMGY4sEQi6N2qQWP1id8o1KtprGd4hll4uQkMGA1nFMXAwohEZOnIMIPQKQl2k0GbSGwkC0lJb2IWOQ57ETYbmxFGspaRp1QyT6q4sKj855st7GzOGj5eIwmWt6wRajEQl/F3JFOFCQO0Chfg=="
|
||||
modelSHA = "16c6b1dcb40f6b21c0de70bc1a8edab95375e4de90dac7c076fd2cc7a34fc59a"
|
||||
)
|
||||
|
||||
// fakeReleaseServer serves the GitHub release API + asset bytes for the engine
|
||||
// and model, exercising resolveAsset + downloadVerifyExtract against a real HTTP
|
||||
// round-trip (with the real coder/websocket-free download path).
|
||||
func fakeReleaseServer(t *testing.T, engineDigest, modelDigest string) *httptest.Server {
|
||||
t.Helper()
|
||||
engineBytes, _ := base64.StdEncoding.DecodeString(engineTarB64)
|
||||
modelBytes, _ := base64.StdEncoding.DecodeString(modelTarB64)
|
||||
mux := http.NewServeMux()
|
||||
base := ""
|
||||
release := func(assetName, digest, dlPath string) string {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"tag_name": "test",
|
||||
"assets": []map[string]any{{
|
||||
"name": assetName,
|
||||
"browser_download_url": base + dlPath,
|
||||
"digest": "sha256:" + digest,
|
||||
"size": 123,
|
||||
}},
|
||||
})
|
||||
return string(body)
|
||||
}
|
||||
var srv *httptest.Server
|
||||
// Any release tag under this prefix resolves: asr-models → the model, every
|
||||
// other tag → the engine (so the default-version pinned-check test works too).
|
||||
mux.HandleFunc("/repos/k2-fsa/sherpa-onnx/releases/tags/", func(w http.ResponseWriter, r *http.Request) {
|
||||
base = srv.URL
|
||||
if strings.HasSuffix(r.URL.Path, "/asr-models") {
|
||||
fmt.Fprint(w, release(modelAssetName, modelDigest, "/model.tar.bz2"))
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, release("sherpa-onnx-test-linux-x64-shared-no-tts.tar.bz2", engineDigest, "/engine.tar.bz2"))
|
||||
})
|
||||
mux.HandleFunc("/engine.tar.bz2", func(w http.ResponseWriter, r *http.Request) { w.Write(engineBytes) })
|
||||
mux.HandleFunc("/model.tar.bz2", func(w http.ResponseWriter, r *http.Request) { w.Write(modelBytes) })
|
||||
srv = httptest.NewServer(mux)
|
||||
base = srv.URL
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestResolveEnginePathsHonorsTargetPlatform(t *testing.T) {
|
||||
// The engine binary name follows the TARGET platform, not the host: a Windows
|
||||
// engine has ".exe", a Linux one doesn't. Resolution must find each regardless
|
||||
// of the OS the test runs on (this is what makes the cross-platform download
|
||||
// test pass on Windows CI).
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
target bool
|
||||
binName string
|
||||
}{
|
||||
{"linux target", false, "sherpa-onnx-offline"},
|
||||
{"windows target", true, "sherpa-onnx-offline.exe"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binDir := filepath.Join(dir, "bin")
|
||||
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(binDir, tc.binName), []byte("x"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bin, _ := resolveEnginePaths(dir, tc.target)
|
||||
if !fileExists(bin) {
|
||||
t.Errorf("resolveEnginePaths(target-windows=%v) = %q; not found", tc.target, bin)
|
||||
}
|
||||
if !strings.HasSuffix(bin, tc.binName) {
|
||||
t.Errorf("bin = %q, want suffix %q", bin, tc.binName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLocalEngineDownloadsAndExtracts(t *testing.T) {
|
||||
srv := fakeReleaseServer(t, engineSHA, modelSHA)
|
||||
dest := t.TempDir()
|
||||
var stages []string
|
||||
comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: dest,
|
||||
EngineVersion: "test",
|
||||
APIBase: srv.URL,
|
||||
platformKey: "linux-amd64",
|
||||
skipPinned: true,
|
||||
Progress: func(s string) { stages = append(stages, s) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureLocalEngine: %v", err)
|
||||
}
|
||||
if !fileExists(comp.BinaryPath) || !strings.HasSuffix(comp.BinaryPath, "sherpa-onnx-offline") {
|
||||
t.Errorf("binary path wrong: %q", comp.BinaryPath)
|
||||
}
|
||||
if !fileExists(comp.ServerPath) {
|
||||
t.Errorf("server path missing: %q", comp.ServerPath)
|
||||
}
|
||||
if !fileExists(filepath.Join(comp.ModelPath, "tokens.txt")) {
|
||||
t.Errorf("model tokens.txt missing under %q", comp.ModelPath)
|
||||
}
|
||||
// The extracted binary keeps its exec bit — Unix only; Windows has no
|
||||
// executable permission bit, so os.FileMode never reports 0o100 there.
|
||||
if runtime.GOOS != "windows" {
|
||||
if info, err := os.Stat(comp.BinaryPath); err == nil && info.Mode()&0o100 == 0 {
|
||||
t.Error("engine binary should be executable")
|
||||
}
|
||||
}
|
||||
if len(stages) == 0 {
|
||||
t.Error("expected progress stages")
|
||||
}
|
||||
|
||||
// Idempotent: a second call reuses the extracted engine + model and downloads
|
||||
// NOTHING (no progress stages) — the fix for re-downloading an already-present
|
||||
// engine whose binary lives in the tarball's flattened subdir.
|
||||
var stages2 []string
|
||||
comp2, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: dest, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true,
|
||||
Progress: func(s string) { stages2 = append(stages2, s) },
|
||||
})
|
||||
if err != nil || comp2.BinaryPath != comp.BinaryPath {
|
||||
t.Errorf("second call should reuse: %v / %q vs %q", err, comp2.BinaryPath, comp.BinaryPath)
|
||||
}
|
||||
if len(stages2) != 0 {
|
||||
t.Errorf("a fully-cached second call must not download/extract anything, got stages: %v", stages2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLocalEngineRejectsChecksumMismatch(t *testing.T) {
|
||||
// Serve the engine with a wrong digest → verification must refuse to extract.
|
||||
srv := fakeReleaseServer(t, strings.Repeat("0", 64), modelSHA)
|
||||
_, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: t.TempDir(), EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "checksum mismatch") {
|
||||
t.Fatalf("expected checksum-mismatch error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLocalEnginePinnedCrossCheckRefusesChangedRelease(t *testing.T) {
|
||||
// Default version + a digest that doesn't match the pinned value → refuse,
|
||||
// even though the download itself would verify against the API digest.
|
||||
srv := fakeReleaseServer(t, engineSHA, modelSHA)
|
||||
_, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: t.TempDir(),
|
||||
EngineVersion: DefaultSherpaVersion, // triggers the pinned cross-check
|
||||
APIBase: srv.URL,
|
||||
platformKey: "linux-amd64",
|
||||
// skipPinned false: the pinned digest for linux-amd64 != engineSHA fixture.
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "pinned known-good") {
|
||||
t.Fatalf("expected pinned cross-check refusal, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAssetMissingDigestAllowed(t *testing.T) {
|
||||
// An older asset with no API digest resolves fine (verification against a
|
||||
// pinned value happens later); resolveAsset no longer refuses here.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]any{
|
||||
"assets": []map[string]any{{"name": "x-linux-x64-shared-no-tts.tar.bz2", "browser_download_url": "http://x", "digest": ""}},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
asset, err := resolveAsset(context.Background(), http.DefaultClient, srv.URL, "test", "x-", "linux-x64-shared-no-tts.tar.bz2")
|
||||
if err != nil {
|
||||
t.Fatalf("resolveAsset should tolerate a missing digest, got %v", err)
|
||||
}
|
||||
if asset.sha256 != "" {
|
||||
t.Errorf("expected empty digest, got %q", asset.sha256)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnverifiableDownloadRefused(t *testing.T) {
|
||||
// No API digest AND no pinned digest → refuse to download/extract.
|
||||
srv := fakeReleaseServer(t, "", "") // both assets served with empty digest
|
||||
_, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: t.TempDir(), EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "unverifiable") {
|
||||
t.Fatalf("expected refusal of an unverifiable download, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressReaderReportsPercent(t *testing.T) {
|
||||
data := make([]byte, 1000)
|
||||
var last string
|
||||
pr := &progressReader{r: bytesReader(data), total: 1000, label: "Engine", report: func(s string) { last = s }, lastPct: -1}
|
||||
buf := make([]byte, 100)
|
||||
for {
|
||||
if _, err := pr.Read(buf); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !strings.Contains(last, "Engine") || !strings.Contains(last, "100%") {
|
||||
t.Errorf("expected a final 'Engine 100%%' report, got %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
func bytesReader(b []byte) *sliceReader { return &sliceReader{b: b} }
|
||||
|
||||
type sliceReader struct {
|
||||
b []byte
|
||||
pos int
|
||||
}
|
||||
|
||||
func (s *sliceReader) Read(p []byte) (int, error) {
|
||||
if s.pos >= len(s.b) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, s.b[s.pos:])
|
||||
s.pos += n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func TestUnsupportedPlatformSetupError(t *testing.T) {
|
||||
_, err := EnsureLocalEngine(context.Background(), DownloadOptions{DestRoot: t.TempDir(), platformKey: "plan9-mips"})
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for unsupported platform, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestListModelsFiltersAndFlags(t *testing.T) {
|
||||
assets := []map[string]any{
|
||||
{"name": "sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2", "size": 1 << 20, "digest": "sha256:abc"},
|
||||
{"name": "sherpa-onnx-whisper-tiny.en.tar.bz2", "size": 2 << 20},
|
||||
{"name": "sherpa-onnx-moonshine-tiny-en-int8.tar.bz2", "size": 3 << 20}, // curated but not flagged recommended
|
||||
{"name": "sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06.tar.bz2", "size": 1 << 20}, // curated → recommended
|
||||
{"name": "sherpa-onnx-paraformer-zh-2024-03-09.tar.bz2", "size": 4 << 20},
|
||||
// Non-ASR tools that MUST be filtered out:
|
||||
{"name": "sherpa-onnx-vad-silero.tar.bz2", "size": 1},
|
||||
{"name": "vits-piper-en_US-amy.tar.bz2", "size": 1},
|
||||
{"name": "sherpa-onnx-punct-en.tar.bz2", "size": 1},
|
||||
{"name": "sherpa-onnx-kws-zipformer.tar.bz2", "size": 1},
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]any{"id": 1, "assets": assets})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
vs, err := ListModels(context.Background(), http.DefaultClient, srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(vs) != 5 {
|
||||
t.Fatalf("expected 5 transcription models (tools filtered), got %d", len(vs))
|
||||
}
|
||||
byID := map[string]ModelVariant{}
|
||||
for _, v := range vs {
|
||||
byID[v.ID] = v
|
||||
}
|
||||
if !byID["streaming-zipformer-en-2023-06-26"].Streaming {
|
||||
t.Error("streaming model should be flagged Streaming")
|
||||
}
|
||||
if byID["whisper-tiny.en"].Streaming {
|
||||
t.Error("whisper is batch, not streaming")
|
||||
}
|
||||
if !byID["streaming-zipformer-en-kroko-2025-08-06"].Recommended {
|
||||
t.Error("a curated model flagged Recommended should be Recommended")
|
||||
}
|
||||
if byID["moonshine-tiny-en-int8"].Recommended {
|
||||
t.Error("moonshine tiny is curated but no longer flagged Recommended (no star)")
|
||||
}
|
||||
if byID["paraformer-zh-2024-03-09"].Recommended {
|
||||
t.Error("an uncurated model should not be Recommended")
|
||||
}
|
||||
// A curated model with no API digest still gets its pinned digest.
|
||||
if byID["moonshine-tiny-en-int8"].Digest == "" {
|
||||
t.Error("curated model should carry a pinned digest")
|
||||
}
|
||||
// An uncurated model with no API digest has an empty digest (TLS-only download).
|
||||
if byID["whisper-tiny.en"].Digest != "" {
|
||||
t.Error("uncurated model without an API digest should have an empty digest")
|
||||
}
|
||||
// The API-provided digest is used when present.
|
||||
if byID["streaming-zipformer-en-2023-06-26"].Digest != "abc" {
|
||||
t.Errorf("expected the API digest, got %q", byID["streaming-zipformer-en-2023-06-26"].Digest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestLocalTranscribeRealBinary exercises the full local batch path against a
|
||||
// REAL sherpa-onnx-offline binary and model — the manual smoke test §14 calls
|
||||
// out as not CI-able. Skipped unless both env vars point at a real install:
|
||||
//
|
||||
// ZERO_STT_TEST_BINARY=/path/to/sherpa-onnx-offline \
|
||||
// ZERO_STT_TEST_MODEL=/path/to/model-dir \
|
||||
// ZERO_STT_TEST_WAV=/path/to/16k-mono.wav \
|
||||
// go test ./internal/dictation/ -run TestLocalTranscribeRealBinary -v
|
||||
func TestLocalTranscribeRealBinary(t *testing.T) {
|
||||
binary := os.Getenv("ZERO_STT_TEST_BINARY")
|
||||
model := os.Getenv("ZERO_STT_TEST_MODEL")
|
||||
wav := os.Getenv("ZERO_STT_TEST_WAV")
|
||||
if binary == "" || model == "" || wav == "" {
|
||||
t.Skip("set ZERO_STT_TEST_BINARY, ZERO_STT_TEST_MODEL, ZERO_STT_TEST_WAV to run the real-engine smoke test")
|
||||
}
|
||||
audio, err := os.ReadFile(wav)
|
||||
if err != nil {
|
||||
t.Fatalf("read wav: %v", err)
|
||||
}
|
||||
tr, err := NewLocalTranscriber(LocalConfig{Binary: binary, ModelPath: model})
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalTranscriber: %v", err)
|
||||
}
|
||||
text, err := tr.Transcribe(context.Background(), audio)
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if text == "" {
|
||||
t.Fatal("real engine returned an empty transcript")
|
||||
}
|
||||
t.Logf("transcript: %q", text)
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
//go:build !windows
|
||||
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// checkAudioDirOwner rejects a dictation temp directory not owned by the
|
||||
// current user — on a shared /tmp another user could have pre-created the
|
||||
// path and would then control its lifetime. Mirrors
|
||||
// internal/tools/spill_owner_unix.go.
|
||||
func checkAudioDirOwner(info os.FileInfo) error {
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if int(stat.Uid) != os.Geteuid() {
|
||||
return fmt.Errorf("dictation temp directory is owned by uid %d, not the current user", stat.Uid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
//go:build windows
|
||||
|
||||
package dictation
|
||||
|
||||
import "os"
|
||||
|
||||
// checkAudioDirOwner is a no-op on Windows: %TEMP% is per-user by default and
|
||||
// there is no portable uid to compare. The Lstat symlink/dir check in
|
||||
// audioSpillDir still applies. Mirrors internal/tools/spill_owner_windows.go.
|
||||
func checkAudioDirOwner(os.FileInfo) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,581 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Recorder captures microphone audio. Start/Stop is the batch path (used on
|
||||
// Termux, and as the fallback everywhere); StartStreaming is the desktop-only
|
||||
// continuous-PCM path.
|
||||
type Recorder interface {
|
||||
Start() error
|
||||
// Stop ends a batch recording and returns the recorded audio bytes (WAV on
|
||||
// desktop, M4A on Termux — SniffFormat distinguishes them).
|
||||
Stop() ([]byte, error)
|
||||
// StartStreaming begins continuous capture of raw little-endian PCM16 mono
|
||||
// at the configured sample rate. Chunks arrive on the channel in ~50ms
|
||||
// units; the channel closes when capture ends (stop called, max duration
|
||||
// hit, or trailing-silence auto-stop). stop is idempotent.
|
||||
StartStreaming() (chunks <-chan []byte, stop func() error, err error)
|
||||
}
|
||||
|
||||
// Platform selects the audio-capture toolchain (§9 of the design doc).
|
||||
type Platform string
|
||||
|
||||
const (
|
||||
PlatformLinux Platform = "linux"
|
||||
PlatformDarwin Platform = "darwin"
|
||||
PlatformWindows Platform = "windows"
|
||||
PlatformTermux Platform = "termux"
|
||||
PlatformUnsupported Platform = ""
|
||||
)
|
||||
|
||||
// DetectPlatform picks the capture toolchain for the current host. Termux is
|
||||
// detected by its environment (it reports GOOS=linux/android): TERMUX_VERSION
|
||||
// is exported to every Termux shell, and PREFIX carries the app package path.
|
||||
func DetectPlatform() Platform {
|
||||
if os.Getenv("TERMUX_VERSION") != "" || strings.Contains(os.Getenv("PREFIX"), "com.termux") {
|
||||
return PlatformTermux
|
||||
}
|
||||
switch runtime.GOOS {
|
||||
case "linux":
|
||||
return PlatformLinux
|
||||
case "darwin":
|
||||
return PlatformDarwin
|
||||
case "windows":
|
||||
return PlatformWindows
|
||||
}
|
||||
return PlatformUnsupported
|
||||
}
|
||||
|
||||
// StreamingSupported reports whether the platform's capture tool can write raw
|
||||
// PCM to stdout. Termux's termux-microphone-record requires a regular output
|
||||
// file — a hard tool limitation, so Termux is batch-only permanently.
|
||||
func (p Platform) StreamingSupported() bool {
|
||||
switch p {
|
||||
case PlatformLinux, PlatformDarwin, PlatformWindows:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const (
|
||||
// DefaultSampleRate is what the local engine and Deepgram consume.
|
||||
// OpenAI Realtime's pcm16 requires 24kHz; its transcriber constructs the
|
||||
// recorder accordingly.
|
||||
DefaultSampleRate = 16000
|
||||
// DefaultMaxDuration is the hard runaway-recording cap (§12), applied on
|
||||
// every platform, batch and streaming. Overridable via
|
||||
// stt.maxDurationSeconds.
|
||||
DefaultMaxDuration = 5 * time.Minute
|
||||
|
||||
// Trailing-silence auto-stop (§9): stop ~2s after the signal falls below a
|
||||
// 3% amplitude threshold — SoX's own `silence 1 0.1 3% 1 2.0 3%` numbers.
|
||||
silenceThresholdPct = 3
|
||||
silenceHold = 2 * time.Second
|
||||
|
||||
chunkInterval = 50 * time.Millisecond
|
||||
stopGrace = 3 * time.Second
|
||||
)
|
||||
|
||||
// RecorderOptions configures a Recorder. Zero values pick sensible defaults.
|
||||
type RecorderOptions struct {
|
||||
Platform Platform
|
||||
SampleRate int
|
||||
MaxDuration time.Duration
|
||||
SilenceAutoStop bool
|
||||
// WindowsAudioDevice is the dshow capture device name; auto-detected from
|
||||
// `ffmpeg -list_devices` when empty.
|
||||
WindowsAudioDevice string
|
||||
|
||||
// Test seams (unexported): fake the OS boundary, mirroring how
|
||||
// internal/lsp/manager_test.go stubs serverStarter.
|
||||
starter processStarter
|
||||
runOutput commandOutputRunner
|
||||
tempDir func() (string, error)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (o RecorderOptions) withDefaults() RecorderOptions {
|
||||
if o.Platform == PlatformUnsupported {
|
||||
o.Platform = DetectPlatform()
|
||||
}
|
||||
if o.SampleRate <= 0 {
|
||||
o.SampleRate = DefaultSampleRate
|
||||
}
|
||||
if o.MaxDuration <= 0 {
|
||||
o.MaxDuration = DefaultMaxDuration
|
||||
}
|
||||
if o.starter == nil {
|
||||
o.starter = startProcess
|
||||
}
|
||||
if o.runOutput == nil {
|
||||
o.runOutput = runCommandOutput
|
||||
}
|
||||
if o.tempDir == nil {
|
||||
o.tempDir = audioSpillDir
|
||||
}
|
||||
if o.now == nil {
|
||||
o.now = time.Now
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// NewRecorder builds the platform recorder. Construction is cheap and
|
||||
// side-effect free; tools are only exec'd on Start/StartStreaming.
|
||||
func NewRecorder(opts RecorderOptions) Recorder {
|
||||
o := opts.withDefaults()
|
||||
return &recorder{opts: o}
|
||||
}
|
||||
|
||||
type recorder struct {
|
||||
opts RecorderOptions
|
||||
|
||||
mu sync.Mutex
|
||||
proc processHandle
|
||||
filePath string
|
||||
recording bool
|
||||
}
|
||||
|
||||
var errAlreadyRecording = errors.New("a recording is already in progress")
|
||||
|
||||
// Start begins a batch (record-to-file) capture.
|
||||
func (r *recorder) Start() error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.recording {
|
||||
return errAlreadyRecording
|
||||
}
|
||||
if r.opts.Platform == PlatformUnsupported {
|
||||
return fmt.Errorf("dictation is not supported on %s", runtime.GOOS)
|
||||
}
|
||||
|
||||
dir, err := r.opts.tempDir()
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing dictation temp dir: %w", err)
|
||||
}
|
||||
ext := ".wav"
|
||||
if r.opts.Platform == PlatformTermux {
|
||||
ext = ".m4a"
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, "rec-*"+ext)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating dictation temp file: %w", err)
|
||||
}
|
||||
path := tmp.Name()
|
||||
_ = tmp.Close()
|
||||
|
||||
if r.opts.Platform == PlatformTermux {
|
||||
// termux-microphone-record starts the recording in the Termux:API
|
||||
// companion app and exits immediately; there is no long-lived process
|
||||
// to hold. Stop is a separate `-q` invocation.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := r.opts.runOutput(ctx, "termux-microphone-record",
|
||||
"-l", strconv.Itoa(maxSeconds(r.opts.MaxDuration)), "-f", path)
|
||||
if err != nil {
|
||||
_ = os.Remove(path)
|
||||
return startError("termux-microphone-record", r.opts.Platform, err, out)
|
||||
}
|
||||
r.filePath = path
|
||||
r.recording = true
|
||||
return nil
|
||||
}
|
||||
|
||||
spec, err := batchCommand(r.opts, path)
|
||||
if err != nil {
|
||||
_ = os.Remove(path)
|
||||
return err
|
||||
}
|
||||
proc, _, err := r.opts.starter(spec)
|
||||
if err != nil {
|
||||
_ = os.Remove(path)
|
||||
return startError(spec.name, r.opts.Platform, err, nil)
|
||||
}
|
||||
r.proc = proc
|
||||
r.filePath = path
|
||||
r.recording = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop ends a batch capture, reads the recorded file, deletes it, and returns
|
||||
// the audio bytes.
|
||||
func (r *recorder) Stop() ([]byte, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if !r.recording {
|
||||
return nil, errors.New("no recording in progress")
|
||||
}
|
||||
proc, path := r.proc, r.filePath
|
||||
// Termux: Start's termux-microphone-record branch leaves r.proc nil because
|
||||
// there's no long-lived process to hold (Stop is a separate `-q` invocation
|
||||
// via runOutput). Treat that as legitimate; only reject a nil proc that
|
||||
// came from StartStreaming (which owns its process via the returned
|
||||
// closure).
|
||||
if proc == nil && r.opts.Platform != PlatformTermux {
|
||||
return nil, errors.New("recording was started with StartStreaming; call the stop closure returned from StartStreaming instead of Stop()")
|
||||
}
|
||||
r.proc, r.filePath, r.recording = nil, "", false
|
||||
defer os.Remove(path)
|
||||
|
||||
if r.opts.Platform == PlatformTermux {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if out, err := r.opts.runOutput(ctx, "termux-microphone-record", "-q"); err != nil {
|
||||
return nil, fmt.Errorf("stopping termux recording: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return readRecordingWithRetry(path, r.opts.now)
|
||||
}
|
||||
|
||||
if err := proc.StopGracefully(); err != nil {
|
||||
_ = proc.Kill()
|
||||
}
|
||||
// A signal-terminated capture tool reports a non-zero exit — expected, not
|
||||
// an error. The recorded file's contents are the real success signal.
|
||||
_ = waitWithTimeout(proc, stopGrace)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading recorded audio: %w", err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, errors.New("no audio was captured (is the microphone available?)")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// readRecordingWithRetry reads the Termux recording, retrying briefly: the
|
||||
// companion app finalizes the file asynchronously after `-q` returns.
|
||||
func readRecordingWithRetry(path string, now func() time.Time) ([]byte, error) {
|
||||
deadline := now().Add(2 * time.Second)
|
||||
for {
|
||||
data, err := os.ReadFile(path)
|
||||
if err == nil && len(data) > 0 {
|
||||
return data, nil
|
||||
}
|
||||
if now().After(deadline) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading recorded audio: %w", err)
|
||||
}
|
||||
return nil, errors.New("no audio was captured (check Termux:API app and microphone permission)")
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// batchCommand builds the record-to-file argv for desktop platforms (§9).
|
||||
// The max-duration cap is enforced by the tool itself so a runaway recording
|
||||
// ends even if Zero never calls Stop.
|
||||
func batchCommand(opts RecorderOptions, path string) (commandSpec, error) {
|
||||
rate := strconv.Itoa(opts.SampleRate)
|
||||
maxSec := strconv.Itoa(maxSeconds(opts.MaxDuration))
|
||||
switch opts.Platform {
|
||||
case PlatformLinux:
|
||||
return commandSpec{
|
||||
name: "arecord",
|
||||
args: []string{"-q", "-f", "S16_LE", "-r", rate, "-c", "1", "-t", "wav", "-d", maxSec, path},
|
||||
}, nil
|
||||
case PlatformDarwin:
|
||||
args := []string{"-q", "-d", "-r", rate, "-c", "1", "-b", "16", "-e", "signed-integer", path, "trim", "0", maxSec}
|
||||
if opts.SilenceAutoStop {
|
||||
// SoX's built-in trailing-silence auto-stop: end ~2s after the
|
||||
// signal falls below 3% amplitude. A backstop for a forgotten
|
||||
// manual stop, not VAD — recording still starts on an explicit key.
|
||||
args = append(args, "silence", "1", "0.1", "3%", "1", "2.0", "3%")
|
||||
}
|
||||
return commandSpec{name: "sox", args: args}, nil
|
||||
case PlatformWindows:
|
||||
device := opts.WindowsAudioDevice
|
||||
if device == "" {
|
||||
detected, err := detectWindowsAudioDevice(opts.runOutput)
|
||||
if err != nil {
|
||||
return commandSpec{}, err
|
||||
}
|
||||
device = detected
|
||||
}
|
||||
return commandSpec{
|
||||
name: "ffmpeg",
|
||||
args: []string{"-hide_banner", "-loglevel", "error", "-f", "dshow", "-i", "audio=" + device,
|
||||
"-ac", "1", "-ar", rate, "-c:a", "pcm_s16le", "-t", maxSec, "-y", path},
|
||||
stopViaStdin: "q",
|
||||
}, nil
|
||||
}
|
||||
return commandSpec{}, fmt.Errorf("no batch capture tool for platform %q", opts.Platform)
|
||||
}
|
||||
|
||||
// streamCommand builds the raw-PCM-to-stdout argv for desktop platforms (§9).
|
||||
// Same tools as batch, invoked with their stdout flag — Termux has no stdout
|
||||
// mode, which is why it is batch-only.
|
||||
func streamCommand(opts RecorderOptions) (commandSpec, error) {
|
||||
rate := strconv.Itoa(opts.SampleRate)
|
||||
switch opts.Platform {
|
||||
case PlatformLinux:
|
||||
return commandSpec{
|
||||
name: "arecord",
|
||||
args: []string{"-q", "-t", "raw", "-f", "S16_LE", "-r", rate, "-c", "1", "-"},
|
||||
wantStdout: true,
|
||||
}, nil
|
||||
case PlatformDarwin:
|
||||
return commandSpec{
|
||||
name: "sox",
|
||||
args: []string{"-q", "-d", "-t", "raw", "-r", rate, "-e", "signed", "-b", "16", "-c", "1", "-"},
|
||||
wantStdout: true,
|
||||
}, nil
|
||||
case PlatformWindows:
|
||||
device := opts.WindowsAudioDevice
|
||||
if device == "" {
|
||||
detected, err := detectWindowsAudioDevice(opts.runOutput)
|
||||
if err != nil {
|
||||
return commandSpec{}, err
|
||||
}
|
||||
device = detected
|
||||
}
|
||||
return commandSpec{
|
||||
name: "ffmpeg",
|
||||
args: []string{"-hide_banner", "-loglevel", "error", "-f", "dshow", "-i", "audio=" + device,
|
||||
"-ac", "1", "-ar", rate, "-f", "s16le", "-"},
|
||||
wantStdout: true,
|
||||
stopViaStdin: "q",
|
||||
}, nil
|
||||
}
|
||||
return commandSpec{}, fmt.Errorf("streaming capture is not supported on platform %q", opts.Platform)
|
||||
}
|
||||
|
||||
// StartStreaming begins continuous PCM capture with the §12 safety nets: a
|
||||
// hard max-duration cap and Go-side trailing-silence auto-stop (the streaming
|
||||
// analog of SoX's silence effect — we see the samples, so we measure them).
|
||||
func (r *recorder) StartStreaming() (<-chan []byte, func() error, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.recording {
|
||||
return nil, nil, errAlreadyRecording
|
||||
}
|
||||
spec, err := streamCommand(r.opts)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
proc, stdout, err := r.opts.starter(spec)
|
||||
if err != nil {
|
||||
return nil, nil, startError(spec.name, r.opts.Platform, err, nil)
|
||||
}
|
||||
if stdout == nil {
|
||||
_ = proc.Kill()
|
||||
return nil, nil, errors.New("capture process has no stdout pipe")
|
||||
}
|
||||
r.recording = true
|
||||
|
||||
chunkBytes := chunkSizeBytes(r.opts.SampleRate)
|
||||
chunks := make(chan []byte, 8)
|
||||
done := make(chan struct{})
|
||||
var stopOnce sync.Once
|
||||
stop := func() error {
|
||||
stopOnce.Do(func() {
|
||||
close(done)
|
||||
if err := proc.StopGracefully(); err != nil {
|
||||
_ = proc.Kill()
|
||||
}
|
||||
_ = waitWithTimeout(proc, stopGrace)
|
||||
_ = stdout.Close()
|
||||
r.mu.Lock()
|
||||
r.recording = false
|
||||
r.mu.Unlock()
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Max-duration cap: streaming has no tool-side -d/-t equivalent that also
|
||||
// flushes stdout cleanly everywhere, so the cap runs on our side of the pipe.
|
||||
capTimer := time.AfterFunc(r.opts.MaxDuration, func() { _ = stop() })
|
||||
|
||||
go func() {
|
||||
defer close(chunks)
|
||||
defer capTimer.Stop()
|
||||
// Tear down on ANY exit — including the capture process ending on its own
|
||||
// (EOF/crash), which the loop below returns on without going through an
|
||||
// explicit stop(). stop() is idempotent (sync.Once), so this is safe even
|
||||
// when the session was already stopped externally; it clears r.recording and
|
||||
// closes proc/stdout so a self-terminating device doesn't wedge the recorder.
|
||||
defer func() { _ = stop() }()
|
||||
threshold := int16(32767 * silenceThresholdPct / 100)
|
||||
silentChunks, speechSeen := 0, false
|
||||
silenceLimit := int(silenceHold / chunkInterval)
|
||||
for {
|
||||
buf := make([]byte, chunkBytes)
|
||||
n, err := readFullOrEOF(stdout, buf)
|
||||
if n > 0 {
|
||||
chunk := buf[:n]
|
||||
if r.opts.SilenceAutoStop {
|
||||
if chunkPeak(chunk) >= threshold {
|
||||
speechSeen = true
|
||||
silentChunks = 0
|
||||
} else if speechSeen {
|
||||
silentChunks++
|
||||
if silentChunks >= silenceLimit {
|
||||
go func() { _ = stop() }()
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case chunks <- chunk:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return chunks, stop, nil
|
||||
}
|
||||
|
||||
// chunkSizeBytes is 50ms of PCM16 mono at the given rate — small chunks mean
|
||||
// faster time-to-first-partial (§6a).
|
||||
func chunkSizeBytes(sampleRate int) int {
|
||||
return sampleRate * 2 / int(time.Second/chunkInterval)
|
||||
}
|
||||
|
||||
// readFullOrEOF fills buf from r, returning what it read and any terminal
|
||||
// error. A short final read (pipe closing mid-chunk) still returns its bytes.
|
||||
func readFullOrEOF(r interface{ Read([]byte) (int, error) }, buf []byte) (int, error) {
|
||||
total := 0
|
||||
for total < len(buf) {
|
||||
n, err := r.Read(buf[total:])
|
||||
total += n
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// chunkPeak returns the loudest absolute sample in a little-endian PCM16 chunk.
|
||||
func chunkPeak(chunk []byte) int16 {
|
||||
var peak int16
|
||||
for i := 0; i+1 < len(chunk); i += 2 {
|
||||
s := int16(uint16(chunk[i]) | uint16(chunk[i+1])<<8)
|
||||
if s == -32768 {
|
||||
return 32767
|
||||
}
|
||||
if s < 0 {
|
||||
s = -s
|
||||
}
|
||||
if s > peak {
|
||||
peak = s
|
||||
}
|
||||
}
|
||||
return peak
|
||||
}
|
||||
|
||||
// levelFloorDB and levelCeilDB bound the RMS→meter mapping. Speech RMS is small
|
||||
// (roughly -45 dBFS quiet to -15 dBFS loud), so a LINEAR scale barely moves the
|
||||
// bars — a dB (logarithmic) scale, like real audio meters, is what makes normal
|
||||
// speech span the meter.
|
||||
const (
|
||||
levelFloorDB = -50.0 // ≈ near-silence → empty
|
||||
levelCeilDB = -15.0 // loud speech → full
|
||||
)
|
||||
|
||||
// ChunkLevel returns a perceptual 0..1 loudness for a little-endian PCM16 chunk,
|
||||
// driving the live recording waveform from the actual microphone input. It uses
|
||||
// an RMS→dBFS→linear mapping over [levelFloorDB, levelCeilDB].
|
||||
func ChunkLevel(chunk []byte) float64 {
|
||||
var sumSq float64
|
||||
n := 0
|
||||
for i := 0; i+1 < len(chunk); i += 2 {
|
||||
s := int16(uint16(chunk[i]) | uint16(chunk[i+1])<<8)
|
||||
f := float64(s) / 32768.0
|
||||
sumSq += f * f
|
||||
n++
|
||||
}
|
||||
if n == 0 {
|
||||
return 0
|
||||
}
|
||||
rms := math.Sqrt(sumSq / float64(n))
|
||||
if rms <= 0 {
|
||||
return 0
|
||||
}
|
||||
db := 20 * math.Log10(rms) // dBFS (negative)
|
||||
level := (db - levelFloorDB) / (levelCeilDB - levelFloorDB)
|
||||
if level < 0 {
|
||||
level = 0
|
||||
}
|
||||
if level > 1 {
|
||||
level = 1
|
||||
}
|
||||
return level
|
||||
}
|
||||
|
||||
func maxSeconds(d time.Duration) int {
|
||||
sec := int(d / time.Second)
|
||||
if sec < 1 {
|
||||
sec = 1
|
||||
}
|
||||
return sec
|
||||
}
|
||||
|
||||
// detectWindowsAudioDevice picks the first dshow audio capture device from
|
||||
// `ffmpeg -list_devices`. dshow has no "default" alias, so a concrete device
|
||||
// name is required; stt.windowsAudioDevice overrides the auto-pick.
|
||||
func detectWindowsAudioDevice(runOutput commandOutputRunner) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
out, _ := runOutput(ctx, "ffmpeg", "-hide_banner", "-list_devices", "true", "-f", "dshow", "-i", "dummy")
|
||||
// Listing always "fails" (dummy isn't a real input); the device table is
|
||||
// in the output regardless. Lines look like: [dshow @ ...] "Mic name" (audio)
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
if !strings.Contains(line, "(audio)") {
|
||||
continue
|
||||
}
|
||||
start := strings.Index(line, `"`)
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
rest := line[start+1:]
|
||||
end := strings.Index(rest, `"`)
|
||||
if end <= 0 {
|
||||
continue
|
||||
}
|
||||
return rest[:end], nil
|
||||
}
|
||||
return "", &SetupError{
|
||||
Tool: "ffmpeg",
|
||||
Hint: "no dshow audio capture device found — set stt.windowsAudioDevice to your microphone's name (list with: ffmpeg -list_devices true -f dshow -i dummy)",
|
||||
}
|
||||
}
|
||||
|
||||
// startError maps a failed capture-tool launch to setup guidance: a missing
|
||||
// binary is an expected unconfigured state (matching internal/lsp's
|
||||
// missing-server philosophy), not an internal error.
|
||||
func startError(tool string, platform Platform, err error, output []byte) error {
|
||||
if errors.Is(err, exec.ErrNotFound) {
|
||||
return &SetupError{Tool: tool, Hint: installHint(platform)}
|
||||
}
|
||||
if len(output) > 0 {
|
||||
return fmt.Errorf("starting %s: %w (%s)", tool, err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
return fmt.Errorf("starting %s: %w", tool, err)
|
||||
}
|
||||
|
||||
func installHint(platform Platform) string {
|
||||
switch platform {
|
||||
case PlatformLinux:
|
||||
return "install alsa-utils to enable dictation (e.g. sudo dnf install alsa-utils / sudo apt install alsa-utils)"
|
||||
case PlatformDarwin:
|
||||
return "install sox to enable dictation (brew install sox)"
|
||||
case PlatformWindows:
|
||||
return "install ffmpeg and ensure it is on PATH to enable dictation"
|
||||
case PlatformTermux:
|
||||
return "run `pkg install termux-api` and install the Termux:API companion app to enable dictation"
|
||||
}
|
||||
return "dictation is not supported on this platform"
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeProcess is a processHandle whose stdout is driven from an in-memory
|
||||
// buffer, mirroring how internal/lsp/manager_test.go stubs its subprocess over
|
||||
// in-memory pipes rather than spawning a real one.
|
||||
type fakeProcess struct {
|
||||
stopped bool
|
||||
killed bool
|
||||
waitErr error
|
||||
stopErr error
|
||||
closeOnce sync.Once
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newFakeProcess() *fakeProcess { return &fakeProcess{done: make(chan struct{})} }
|
||||
|
||||
func (f *fakeProcess) StopGracefully() error {
|
||||
f.stopped = true
|
||||
f.closeOnce.Do(func() { close(f.done) })
|
||||
return f.stopErr
|
||||
}
|
||||
func (f *fakeProcess) Wait() error {
|
||||
<-f.done
|
||||
return f.waitErr
|
||||
}
|
||||
func (f *fakeProcess) Kill() error {
|
||||
f.killed = true
|
||||
f.closeOnce.Do(func() { close(f.done) })
|
||||
return nil
|
||||
}
|
||||
|
||||
// captured records the last spec a starter was asked to launch, so tests can
|
||||
// assert the exact discrete argv per platform.
|
||||
type captured struct {
|
||||
spec commandSpec
|
||||
}
|
||||
|
||||
// fileWritingStarter simulates a record-to-file batch tool: on Start it writes
|
||||
// fixture bytes to the output path (last positional arg for arecord/sox, or the
|
||||
// -y target for ffmpeg) and returns a process that "records" until stopped.
|
||||
func fileWritingStarter(t *testing.T, cap *captured, fixture []byte) processStarter {
|
||||
t.Helper()
|
||||
return func(spec commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
cap.spec = spec
|
||||
path := outputPath(spec)
|
||||
if path == "" {
|
||||
t.Fatalf("could not determine output path from argv: %v", spec.args)
|
||||
}
|
||||
if err := os.WriteFile(path, fixture, 0o600); err != nil {
|
||||
t.Fatalf("fixture write: %v", err)
|
||||
}
|
||||
return newFakeProcess(), nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
func outputPath(spec commandSpec) string {
|
||||
for i, a := range spec.args {
|
||||
if a == "-y" && i+1 < len(spec.args) {
|
||||
return spec.args[i+1]
|
||||
}
|
||||
}
|
||||
// arecord/sox: the path is the last arg (sox appends trim/silence effects
|
||||
// after it, so scan from the front for the first existing-dir target).
|
||||
last := spec.args[len(spec.args)-1]
|
||||
if filepath.IsAbs(last) {
|
||||
return last
|
||||
}
|
||||
for _, a := range spec.args {
|
||||
if filepath.IsAbs(a) && filepath.Ext(a) != "" {
|
||||
return a
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func wavFixture() []byte {
|
||||
// Minimal valid RIFF/WAVE header + a little data, enough for SniffFormat.
|
||||
var b bytes.Buffer
|
||||
b.WriteString("RIFF")
|
||||
binary.Write(&b, binary.LittleEndian, uint32(36))
|
||||
b.WriteString("WAVE")
|
||||
b.WriteString("fmt ")
|
||||
binary.Write(&b, binary.LittleEndian, uint32(16))
|
||||
b.Write(make([]byte, 16))
|
||||
b.WriteString("data")
|
||||
binary.Write(&b, binary.LittleEndian, uint32(4))
|
||||
b.Write([]byte{1, 2, 3, 4})
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func testTempDir(t *testing.T) func() (string, error) {
|
||||
dir := t.TempDir()
|
||||
return func() (string, error) { return dir, nil }
|
||||
}
|
||||
|
||||
func TestBatchRecordLinuxArgv(t *testing.T) {
|
||||
cap := &captured{}
|
||||
fixture := wavFixture()
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: fileWritingStarter(t, cap, fixture),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
got, err := r.Stop()
|
||||
if err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, fixture) {
|
||||
t.Fatalf("audio mismatch: got %d bytes", len(got))
|
||||
}
|
||||
if cap.spec.name != "arecord" {
|
||||
t.Errorf("tool = %q, want arecord", cap.spec.name)
|
||||
}
|
||||
assertArgs(t, cap.spec.args, "-f", "S16_LE")
|
||||
assertArgs(t, cap.spec.args, "-r", "16000")
|
||||
assertArgs(t, cap.spec.args, "-c", "1")
|
||||
}
|
||||
|
||||
func TestBatchRecordDarwinArgv(t *testing.T) {
|
||||
cap := &captured{}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformDarwin,
|
||||
SilenceAutoStop: true,
|
||||
starter: fileWritingStarter(t, cap, wavFixture()),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if _, err := r.Stop(); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if cap.spec.name != "sox" {
|
||||
t.Errorf("tool = %q, want sox", cap.spec.name)
|
||||
}
|
||||
if !containsSubseq(cap.spec.args, []string{"silence", "1", "0.1", "3%", "1", "2.0", "3%"}) {
|
||||
t.Errorf("expected SoX silence auto-stop effect in argv: %v", cap.spec.args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchRecordWindowsArgv(t *testing.T) {
|
||||
cap := &captured{}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformWindows,
|
||||
WindowsAudioDevice: "Microphone (Realtek)",
|
||||
starter: fileWritingStarter(t, cap, wavFixture()),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if _, err := r.Stop(); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if cap.spec.name != "ffmpeg" {
|
||||
t.Errorf("tool = %q, want ffmpeg", cap.spec.name)
|
||||
}
|
||||
assertArgs(t, cap.spec.args, "-i", "audio=Microphone (Realtek)")
|
||||
if cap.spec.stopViaStdin != "q" {
|
||||
t.Errorf("ffmpeg should stop via stdin 'q', got %q", cap.spec.stopViaStdin)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchRecordTermuxUsesCommandRunner(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var calls [][]string
|
||||
runOutput := func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
calls = append(calls, append([]string{name}, args...))
|
||||
if len(args) > 0 && args[0] == "-l" {
|
||||
// The -f target is the last arg; write the fixture there.
|
||||
path := args[len(args)-1]
|
||||
_ = os.WriteFile(path, m4aFixture(), 0o600)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformTermux,
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
t.Fatal("termux must not use the process starter")
|
||||
return nil, nil, nil
|
||||
},
|
||||
runOutput: runOutput,
|
||||
tempDir: func() (string, error) { return dir, nil },
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
got, err := r.Stop()
|
||||
if err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if SniffFormat(got) != FormatM4A {
|
||||
t.Errorf("expected m4a audio, got format %q", SniffFormat(got))
|
||||
}
|
||||
if len(calls) != 2 || calls[0][0] != "termux-microphone-record" || calls[1][len(calls[1])-1] != "-q" {
|
||||
t.Errorf("unexpected termux calls: %v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartMissingBinaryReturnsSetupError(t *testing.T) {
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
return nil, nil, exec.ErrNotFound
|
||||
},
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
err := r.Start()
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing binary, got %v", err)
|
||||
}
|
||||
if setupErr.Tool != "arecord" {
|
||||
t.Errorf("SetupError.Tool = %q, want arecord", setupErr.Tool)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoubleStartRejected(t *testing.T) {
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: fileWritingStarter(t, &captured{}, wavFixture()),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if err := r.Start(); !errors.Is(err, errAlreadyRecording) {
|
||||
t.Errorf("second Start should be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingChunksAndStop(t *testing.T) {
|
||||
// 300ms of PCM16 @16kHz = 6 chunks of 50ms. Feed a fixed buffer, then EOF.
|
||||
pcm := make([]byte, chunkSizeBytes(16000)*6)
|
||||
starter := func(spec commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
if !spec.wantStdout {
|
||||
t.Fatal("streaming spec must request stdout")
|
||||
}
|
||||
return newFakeProcess(), io.NopCloser(bytes.NewReader(pcm)), nil
|
||||
}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: starter,
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
chunks, stop, err := r.StartStreaming()
|
||||
if err != nil {
|
||||
t.Fatalf("StartStreaming: %v", err)
|
||||
}
|
||||
var total int
|
||||
for c := range chunks {
|
||||
total += len(c)
|
||||
}
|
||||
_ = stop()
|
||||
if total != len(pcm) {
|
||||
t.Errorf("streamed %d bytes, want %d", total, len(pcm))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingStopIdempotent(t *testing.T) {
|
||||
// A blocking reader so the capture goroutine parks until we stop it.
|
||||
pr, pw := io.Pipe()
|
||||
defer pw.Close()
|
||||
starter := func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
return newFakeProcess(), pr, nil
|
||||
}
|
||||
r := NewRecorder(RecorderOptions{Platform: PlatformLinux, starter: starter, tempDir: testTempDir(t)})
|
||||
chunks, stop, err := r.StartStreaming()
|
||||
if err != nil {
|
||||
t.Fatalf("StartStreaming: %v", err)
|
||||
}
|
||||
if err := stop(); err != nil {
|
||||
t.Fatalf("stop: %v", err)
|
||||
}
|
||||
if err := stop(); err != nil {
|
||||
t.Fatalf("second stop should be a no-op, got %v", err)
|
||||
}
|
||||
// Draining the channel must terminate now that we've stopped.
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
for range chunks {
|
||||
}
|
||||
close(drained)
|
||||
}()
|
||||
select {
|
||||
case <-drained:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("chunks channel did not close after stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchMaxDurationCapInArgv(t *testing.T) {
|
||||
cap := &captured{}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
MaxDuration: 90 * time.Second,
|
||||
starter: fileWritingStarter(t, cap, wavFixture()),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if _, err := r.Stop(); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
// arecord's -d bounds a runaway recording even if Stop is never called.
|
||||
assertArgs(t, cap.spec.args, "-d", "90")
|
||||
}
|
||||
|
||||
func TestRecordedTempFileDeletedAfterStop(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cap := &captured{}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: fileWritingStarter(t, cap, wavFixture()),
|
||||
tempDir: func() (string, error) { return dir, nil },
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if _, err := r.Stop(); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("recorded audio should be deleted after Stop; found %d files", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingSilenceAutoStop(t *testing.T) {
|
||||
// Speech (loud) then a run of silence longer than silenceHold must auto-stop.
|
||||
rate := 16000
|
||||
chunk := chunkSizeBytes(rate)
|
||||
loud := make([]byte, chunk)
|
||||
for i := 0; i+1 < len(loud); i += 2 {
|
||||
loud[i], loud[i+1] = 0x00, 0x40 // +16384, well above threshold
|
||||
}
|
||||
silence := make([]byte, chunk) // zeros
|
||||
var buf []byte
|
||||
buf = append(buf, loud...)
|
||||
silentChunks := int(silenceHold/chunkInterval) + 3
|
||||
for i := 0; i < silentChunks; i++ {
|
||||
buf = append(buf, silence...)
|
||||
}
|
||||
starter := func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
return newFakeProcess(), io.NopCloser(bytes.NewReader(buf)), nil
|
||||
}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
SampleRate: rate,
|
||||
SilenceAutoStop: true,
|
||||
starter: starter,
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
chunks, stop, err := r.StartStreaming()
|
||||
if err != nil {
|
||||
t.Fatalf("StartStreaming: %v", err)
|
||||
}
|
||||
defer stop()
|
||||
// Draining must terminate on its own once silence auto-stop fires.
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
for range chunks {
|
||||
}
|
||||
close(drained)
|
||||
}()
|
||||
select {
|
||||
case <-drained:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("silence auto-stop did not end the stream")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkPeak(t *testing.T) {
|
||||
loud := []byte{0x00, 0x40} // +16384
|
||||
if got := chunkPeak(loud); got != 16384 {
|
||||
t.Errorf("peak = %d, want 16384", got)
|
||||
}
|
||||
quiet := []byte{0x10, 0x00} // +16
|
||||
if got := chunkPeak(quiet); got != 16 {
|
||||
t.Errorf("peak = %d, want 16", got)
|
||||
}
|
||||
min := []byte{0x00, 0x80} // -32768 -> clamped to 32767
|
||||
if got := chunkPeak(min); got != 32767 {
|
||||
t.Errorf("peak = %d, want 32767 (clamp)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectWindowsAudioDevice(t *testing.T) {
|
||||
listing := `[dshow @ 0000] DirectShow audio devices
|
||||
[dshow @ 0000] "Microphone (USB Audio)" (audio)
|
||||
[dshow @ 0000] Alternative name "@device_cm_..."`
|
||||
runOutput := func(context.Context, string, ...string) ([]byte, error) { return []byte(listing), nil }
|
||||
got, err := detectWindowsAudioDevice(runOutput)
|
||||
if err != nil {
|
||||
t.Fatalf("detect: %v", err)
|
||||
}
|
||||
if got != "Microphone (USB Audio)" {
|
||||
t.Errorf("device = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSniffFormat(t *testing.T) {
|
||||
if SniffFormat(wavFixture()) != FormatWAV {
|
||||
t.Error("wav not detected")
|
||||
}
|
||||
if SniffFormat(m4aFixture()) != FormatM4A {
|
||||
t.Error("m4a not detected")
|
||||
}
|
||||
if SniffFormat([]byte("nope")) != FormatUnknown {
|
||||
t.Error("unknown not detected")
|
||||
}
|
||||
}
|
||||
|
||||
func m4aFixture() []byte {
|
||||
b := make([]byte, 16)
|
||||
copy(b[4:], []byte("ftyp"))
|
||||
copy(b[8:], []byte("M4A "))
|
||||
return b
|
||||
}
|
||||
|
||||
func assertArgs(t *testing.T, args []string, want ...string) {
|
||||
t.Helper()
|
||||
if !containsSubseq(args, want) {
|
||||
t.Errorf("argv %v missing subsequence %v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func containsSubseq(hay, needle []string) bool {
|
||||
if len(needle) == 0 {
|
||||
return true
|
||||
}
|
||||
for i := 0; i+len(needle) <= len(hay); i++ {
|
||||
if equalSlice(hay[i:i+len(needle)], needle) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func equalSlice(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestChunkLevel(t *testing.T) {
|
||||
// Silence → ~0.
|
||||
silence := make([]byte, 640)
|
||||
if lvl := ChunkLevel(silence); lvl > 0.01 {
|
||||
t.Errorf("silence level = %f, want ~0", lvl)
|
||||
}
|
||||
// A loud full-scale tone → clamped to 1.
|
||||
loud := make([]byte, 640)
|
||||
for i := 0; i+1 < len(loud); i += 2 {
|
||||
loud[i], loud[i+1] = 0xFF, 0x7F // +32767
|
||||
}
|
||||
if lvl := ChunkLevel(loud); lvl < 0.99 {
|
||||
t.Errorf("full-scale level = %f, want ~1", lvl)
|
||||
}
|
||||
// A normal speech-level signal (~-30 dBFS) should land mid-meter, not pinned
|
||||
// to the bottom (the whole point of the dB mapping).
|
||||
mid := make([]byte, 640)
|
||||
for i := 0; i+1 < len(mid); i += 2 {
|
||||
mid[i], mid[i+1] = byte(1000&0xFF), byte(1000>>8) // ≈ -30 dBFS
|
||||
}
|
||||
if lvl := ChunkLevel(mid); lvl < 0.4 || lvl > 0.75 {
|
||||
t.Errorf("normal-speech level = %f, want a visible mid-range value", lvl)
|
||||
}
|
||||
// A quiet signal (~-45 dBFS) should be low but non-zero.
|
||||
quiet := make([]byte, 640)
|
||||
for i := 0; i+1 < len(quiet); i += 2 {
|
||||
quiet[i], quiet[i+1] = byte(180&0xFF), byte(180>>8)
|
||||
}
|
||||
if lvl := ChunkLevel(quiet); lvl <= 0 || lvl > 0.35 {
|
||||
t.Errorf("quiet level = %f, want a small non-zero value", lvl)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
// commandSpec describes one capture-process invocation. Argv is always
|
||||
// discrete — never a shell string (the same invariant documented in
|
||||
// internal/imageinput/clipboard.go).
|
||||
type commandSpec struct {
|
||||
name string
|
||||
args []string
|
||||
// wantStdout wires up a stdout pipe (streaming capture reads raw PCM).
|
||||
wantStdout bool
|
||||
// stopViaStdin, when non-empty, makes graceful stop write this to the
|
||||
// process's stdin instead of signaling — ffmpeg finalizes cleanly on "q",
|
||||
// and Windows has no os.Interrupt delivery anyway.
|
||||
stopViaStdin string
|
||||
}
|
||||
|
||||
// processHandle is a started capture process. Injectable via processStarter so
|
||||
// unit tests fake OS-boundary behavior, mirroring internal/lsp's serverStarter
|
||||
// seam (this package's clipboard.go-style shell-outs get the test seam that
|
||||
// file lacks).
|
||||
type processHandle interface {
|
||||
// StopGracefully asks the tool to finish and flush output (SIGINT, or the
|
||||
// spec's stdin stop text). The recorder falls back to Kill on timeout.
|
||||
StopGracefully() error
|
||||
Wait() error
|
||||
Kill() error
|
||||
}
|
||||
|
||||
// processStarter launches a capture process, returning its handle and — when
|
||||
// the spec asks for it — its stdout pipe.
|
||||
type processStarter func(spec commandSpec) (processHandle, io.ReadCloser, error)
|
||||
|
||||
// commandOutputRunner runs a short one-shot command to completion and returns
|
||||
// its combined output. Used for helper invocations that aren't recordings
|
||||
// (Termux start/stop, Windows ffmpeg device listing) and the offline
|
||||
// transcription exec — ctx lets a caller cancel/timeout a wedged process.
|
||||
type commandOutputRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
|
||||
|
||||
func startProcess(spec commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
cmd := exec.Command(spec.name, spec.args...)
|
||||
var stdout io.ReadCloser
|
||||
if spec.wantStdout {
|
||||
pipe, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
stdout = pipe
|
||||
}
|
||||
var stdin io.WriteCloser
|
||||
if spec.stopViaStdin != "" {
|
||||
pipe, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
stdin = pipe
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
// StdoutPipe/StdinPipe allocate fds the runtime only reclaims via
|
||||
// Wait; on Start failure the process is never established, so close
|
||||
// any pipes we already opened to avoid leaking them.
|
||||
if stdout != nil {
|
||||
_ = stdout.Close()
|
||||
}
|
||||
if stdin != nil {
|
||||
_ = stdin.Close()
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
return &realProcess{cmd: cmd, stdin: stdin, stopText: spec.stopViaStdin}, stdout, nil
|
||||
}
|
||||
|
||||
type realProcess struct {
|
||||
cmd *exec.Cmd
|
||||
stdin io.WriteCloser
|
||||
stopText string
|
||||
}
|
||||
|
||||
func (p *realProcess) StopGracefully() error {
|
||||
if p.stopText != "" && p.stdin != nil {
|
||||
_, err := io.WriteString(p.stdin, p.stopText)
|
||||
closeErr := p.stdin.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
// os.Interrupt is how arecord/sox finalize their WAV header. Unsupported
|
||||
// on Windows, but the only Windows tool (ffmpeg) stops via stdin above.
|
||||
return p.cmd.Process.Signal(os.Interrupt)
|
||||
}
|
||||
|
||||
func (p *realProcess) Wait() error { return p.cmd.Wait() }
|
||||
|
||||
func (p *realProcess) Kill() error { return p.cmd.Process.Kill() }
|
||||
|
||||
func runCommandOutput(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
err := cmd.Run()
|
||||
return out.Bytes(), err
|
||||
}
|
||||
|
||||
// waitWithTimeout waits for the process, killing it if it ignores the graceful
|
||||
// stop for longer than the grace window. Capture tools exit on SIGINT/"q"
|
||||
// almost instantly; a hang here means a wedged device driver, and a leaked
|
||||
// recording process holds the microphone open.
|
||||
func waitWithTimeout(proc processHandle, grace time.Duration) error {
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- proc.Wait() }()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-time.After(grace):
|
||||
_ = proc.Kill()
|
||||
return <-done
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultServerPort is the localhost port the sherpa-onnx streaming server binds.
|
||||
const DefaultServerPort = 6006
|
||||
|
||||
// ServerConfig configures the warm sherpa-onnx streaming server (§6a).
|
||||
type ServerConfig struct {
|
||||
// Binary is the streaming server executable (default
|
||||
// "sherpa-onnx-online-websocket-server"); looked up on PATH.
|
||||
Binary string
|
||||
// ModelPath is the sherpa-onnx streaming model directory (transducer:
|
||||
// encoder/decoder/joiner + tokens).
|
||||
ModelPath string
|
||||
Port int
|
||||
NumThreads int
|
||||
|
||||
// Test seams.
|
||||
starter processStarter
|
||||
healthCheck func(ctx context.Context, port int) error
|
||||
aliveCheck func(port int) bool
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// ServerManager owns one long-lived sherpa-onnx streaming server, spawned lazily
|
||||
// on first streaming use and kept warm for the session — a websocket server with
|
||||
// ~1-2s startup latency can't be respawned per utterance (§6a). Its lifecycle
|
||||
// mirrors internal/lsp's Manager: lazy start, health-checked reuse, one restart
|
||||
// on a crashed process, torn down on exit. Safe for concurrent use.
|
||||
type ServerManager struct {
|
||||
cfg ServerConfig
|
||||
|
||||
mu sync.Mutex
|
||||
proc processHandle
|
||||
url string
|
||||
}
|
||||
|
||||
// NewServerManager builds a manager. Construction is side-effect free; the
|
||||
// server is only spawned on the first EnsureRunning.
|
||||
func NewServerManager(cfg ServerConfig) *ServerManager {
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = "sherpa-onnx-online-websocket-server"
|
||||
}
|
||||
if cfg.Port == 0 {
|
||||
cfg.Port = DefaultServerPort
|
||||
}
|
||||
if cfg.starter == nil {
|
||||
cfg.starter = startProcess
|
||||
}
|
||||
if cfg.healthCheck == nil {
|
||||
cfg.healthCheck = dialHealthCheck
|
||||
}
|
||||
if cfg.aliveCheck == nil {
|
||||
cfg.aliveCheck = func(port int) bool { return portListening(port) }
|
||||
}
|
||||
if cfg.now == nil {
|
||||
cfg.now = time.Now
|
||||
}
|
||||
return &ServerManager{cfg: cfg}
|
||||
}
|
||||
|
||||
// EnsureRunning returns the websocket URL of a live server, starting one if
|
||||
// needed. A previously-started server whose process has since died is reaped and
|
||||
// replaced (mirroring the LSP manager's crashed-session eviction).
|
||||
func (m *ServerManager) EnsureRunning(ctx context.Context) (string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.proc != nil {
|
||||
if m.processAlive() {
|
||||
return m.url, nil
|
||||
}
|
||||
// The cached server died; reap it so a fresh one starts below.
|
||||
_ = m.proc.Kill()
|
||||
m.proc = nil
|
||||
m.url = ""
|
||||
}
|
||||
|
||||
spec, err := onlineServerCommand(m.cfg)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
proc, _, err := m.cfg.starter(spec)
|
||||
if err != nil {
|
||||
if errors.Is(err, exec.ErrNotFound) {
|
||||
return "", &SetupError{
|
||||
Tool: m.cfg.Binary,
|
||||
Hint: "install sherpa-onnx and put its online-websocket-server binary on PATH (see docs/dictation.md)",
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("starting sherpa-onnx server: %w", err)
|
||||
}
|
||||
|
||||
// Health-check the port before handing back a URL — the server takes ~1-2s to
|
||||
// bind, and a client connecting too early would fail (§6a).
|
||||
if err := m.cfg.healthCheck(ctx, m.cfg.Port); err != nil {
|
||||
_ = proc.Kill()
|
||||
return "", fmt.Errorf("sherpa-onnx server did not become ready: %w", err)
|
||||
}
|
||||
m.proc = proc
|
||||
m.url = fmt.Sprintf("ws://127.0.0.1:%d", m.cfg.Port)
|
||||
return m.url, nil
|
||||
}
|
||||
|
||||
// processAlive reports whether the tracked server is still reachable. Probing
|
||||
// the port (rather than the process handle) reflects real reachability and is
|
||||
// the seam tests replace.
|
||||
func (m *ServerManager) processAlive() bool {
|
||||
return m.cfg.aliveCheck(m.cfg.Port)
|
||||
}
|
||||
|
||||
// portListening reports whether something accepts TCP connections on the port.
|
||||
func portListening(port int) bool {
|
||||
conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), 200*time.Millisecond)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
// Shutdown stops the running server. Called alongside the LSP manager's shutdown
|
||||
// on exit — a leaked warm server holds a port and CPU.
|
||||
func (m *ServerManager) Shutdown(ctx context.Context) error {
|
||||
m.mu.Lock()
|
||||
proc := m.proc
|
||||
m.proc = nil
|
||||
m.url = ""
|
||||
m.mu.Unlock()
|
||||
if proc == nil {
|
||||
return nil
|
||||
}
|
||||
if err := proc.StopGracefully(); err != nil {
|
||||
_ = proc.Kill()
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- waitWithTimeout(proc, stopGrace) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
_ = proc.Kill()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// URL returns the current server URL without starting one ("" when not running).
|
||||
func (m *ServerManager) URL() string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.url
|
||||
}
|
||||
|
||||
// SetModelPath updates the model directory the server launches with, for a
|
||||
// mid-session config change (e.g. an F9 auto-download). It takes effect the next
|
||||
// time the server (re)starts; a running server is left as-is.
|
||||
func (m *ServerManager) SetModelPath(path string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if path != "" {
|
||||
m.cfg.ModelPath = path
|
||||
}
|
||||
}
|
||||
|
||||
// onlineServerCommand builds the streaming-server argv from the model directory.
|
||||
// The online server uses a streaming transducer (encoder/decoder/joiner) plus
|
||||
// tokens — the shape the sherpa-onnx docs' example uses (§6a).
|
||||
func onlineServerCommand(cfg ServerConfig) (commandSpec, error) {
|
||||
if strings.TrimSpace(cfg.ModelPath) == "" {
|
||||
return commandSpec{}, &SetupError{
|
||||
Tool: "local STT streaming model",
|
||||
Hint: "set stt.localModelPath to a sherpa-onnx streaming model directory (see docs/dictation.md)",
|
||||
}
|
||||
}
|
||||
files, err := onnxFilesIn(cfg.ModelPath)
|
||||
if err != nil {
|
||||
return commandSpec{}, &SetupError{Tool: "local STT streaming model", Hint: err.Error()}
|
||||
}
|
||||
tokens := findTokensFile(files)
|
||||
// A streaming model ships encoder/decoder/joiner, often BOTH an int8 and an
|
||||
// fp32 copy of each. Pick a CONSISTENT set — all int8 or all fp32 — because
|
||||
// mixing (int8 encoder + fp32 decoder) is invalid. Prefer int8 (smaller,
|
||||
// faster) when a full int8 set is present.
|
||||
set := consistentQuantizedSet(files, "encoder", "decoder", "joiner")
|
||||
var encoder, decoder, joiner string
|
||||
if set != nil {
|
||||
encoder, decoder, joiner = set[0], set[1], set[2]
|
||||
}
|
||||
if tokens == "" || encoder == "" || decoder == "" || joiner == "" {
|
||||
return commandSpec{}, &SetupError{
|
||||
Tool: "local STT streaming model",
|
||||
Hint: fmt.Sprintf("streaming needs a transducer model (encoder/decoder/joiner .onnx + tokens.txt) in %q; see docs/dictation.md", cfg.ModelPath),
|
||||
}
|
||||
}
|
||||
args := []string{
|
||||
"--port=" + strconv.Itoa(cfg.Port),
|
||||
"--tokens=" + tokens,
|
||||
"--encoder=" + encoder,
|
||||
"--decoder=" + decoder,
|
||||
"--joiner=" + joiner,
|
||||
"--decoding-method=greedy_search",
|
||||
}
|
||||
if cfg.NumThreads > 0 {
|
||||
args = append(args, "--num-threads="+strconv.Itoa(cfg.NumThreads))
|
||||
}
|
||||
return commandSpec{name: cfg.Binary, args: args}, nil
|
||||
}
|
||||
|
||||
func onnxFilesIn(dir string) (map[string]string, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stt.localModelPath %q is not readable: %v", dir, err)
|
||||
}
|
||||
files := map[string]string{}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
files[strings.ToLower(e.Name())] = dir + string(os.PathSeparator) + e.Name()
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// dialHealthCheck polls the server's TCP port until it accepts a connection or
|
||||
// the deadline passes.
|
||||
func dialHealthCheck(ctx context.Context, port int) error {
|
||||
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
conn, err := net.DialTimeout("tcp", addr, 500*time.Millisecond)
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("port %d not listening after 5s", port)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func streamingModelDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for _, f := range []string{"encoder.onnx", "decoder.onnx", "joiner.onnx", "tokens.txt"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestServerManagerLazyStartAndReuse(t *testing.T) {
|
||||
var startCount int
|
||||
var mu sync.Mutex
|
||||
starter := func(spec commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
mu.Lock()
|
||||
startCount++
|
||||
mu.Unlock()
|
||||
if spec.name != "sherpa-onnx-online-websocket-server" {
|
||||
t.Errorf("binary = %q", spec.name)
|
||||
}
|
||||
return newFakeProcess(), nil, nil
|
||||
}
|
||||
alive := true
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
Port: 6006,
|
||||
starter: starter,
|
||||
healthCheck: func(context.Context, int) error { return nil },
|
||||
aliveCheck: func(int) bool { return alive },
|
||||
})
|
||||
|
||||
url, err := m.EnsureRunning(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureRunning: %v", err)
|
||||
}
|
||||
if url != "ws://127.0.0.1:6006" {
|
||||
t.Errorf("url = %q", url)
|
||||
}
|
||||
// Second call reuses the warm server (alive) — no new start.
|
||||
if _, err := m.EnsureRunning(context.Background()); err != nil {
|
||||
t.Fatalf("EnsureRunning reuse: %v", err)
|
||||
}
|
||||
if startCount != 1 {
|
||||
t.Errorf("expected 1 start (warm reuse), got %d", startCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerManagerRestartsCrashedServer(t *testing.T) {
|
||||
var startCount int
|
||||
starter := func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
startCount++
|
||||
return newFakeProcess(), nil, nil
|
||||
}
|
||||
alive := true
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
starter: starter,
|
||||
healthCheck: func(context.Context, int) error { return nil },
|
||||
aliveCheck: func(int) bool { return alive },
|
||||
})
|
||||
if _, err := m.EnsureRunning(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
alive = false // simulate a crashed server
|
||||
if _, err := m.EnsureRunning(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if startCount != 2 {
|
||||
t.Errorf("expected a restart (2 starts), got %d", startCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerManagerMissingBinaryIsSetupError(t *testing.T) {
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) { return nil, nil, exec.ErrNotFound },
|
||||
})
|
||||
_, err := m.EnsureRunning(context.Background())
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerManagerMissingModelIsSetupError(t *testing.T) {
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: t.TempDir(), // empty dir, no onnx/tokens
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) { return newFakeProcess(), nil, nil },
|
||||
})
|
||||
_, err := m.EnsureRunning(context.Background())
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing streaming model, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerManagerShutdown(t *testing.T) {
|
||||
proc := newFakeProcess()
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) { return proc, nil, nil },
|
||||
healthCheck: func(context.Context, int) error { return nil },
|
||||
aliveCheck: func(int) bool { return true },
|
||||
})
|
||||
if _, err := m.EnsureRunning(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Shutdown(context.Background()); err != nil {
|
||||
t.Fatalf("Shutdown: %v", err)
|
||||
}
|
||||
if !proc.stopped && !proc.killed {
|
||||
t.Error("Shutdown should stop the server process")
|
||||
}
|
||||
if m.URL() != "" {
|
||||
t.Error("URL should be empty after shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlineServerCommandBuildsTransducerArgs(t *testing.T) {
|
||||
spec, err := onlineServerCommand(ServerConfig{Binary: "sherpa-onnx-online-websocket-server", ModelPath: streamingModelDir(t), Port: 6007})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := ""
|
||||
for _, a := range spec.args {
|
||||
joined += a + " "
|
||||
}
|
||||
for _, want := range []string{"--port=6007", "--tokens=", "--encoder=", "--decoder=", "--joiner=", "--decoding-method=greedy_search"} {
|
||||
if !contains(joined, want) {
|
||||
t.Errorf("missing %q in args: %s", want, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPCM16ToFloat32LE(t *testing.T) {
|
||||
// int16 max (32767) and min (-32768), little-endian.
|
||||
pcm := []byte{0xFF, 0x7F, 0x00, 0x80}
|
||||
out := pcm16ToFloat32LE(pcm)
|
||||
if len(out) != 8 {
|
||||
t.Fatalf("expected 8 bytes (2 float32), got %d", len(out))
|
||||
}
|
||||
f0 := math.Float32frombits(binary.LittleEndian.Uint32(out[0:4]))
|
||||
f1 := math.Float32frombits(binary.LittleEndian.Uint32(out[4:8]))
|
||||
if f0 < 0.99 || f0 > 1.0 {
|
||||
t.Errorf("32767 → %f, want ~1.0", f0)
|
||||
}
|
||||
if f1 != -1.0 {
|
||||
t.Errorf("-32768 → %f, want -1.0", f1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSherpaStreamResult(t *testing.T) {
|
||||
text, segment, final, ok := parseSherpaStreamResult([]byte(`{"text":" hello world ","segment":2,"is_final":true}`))
|
||||
if !ok || text != "hello world" || segment != 2 || !final {
|
||||
t.Errorf("parsed = (%q, %d, %v, %v)", text, segment, final, ok)
|
||||
}
|
||||
if _, _, _, ok := parseSherpaStreamResult([]byte("not json")); ok {
|
||||
t.Error("non-JSON should not parse")
|
||||
}
|
||||
}
|
||||
|
||||
func contains(hay, needle string) bool {
|
||||
return len(needle) == 0 || (len(hay) >= len(needle) && indexOf(hay, needle) >= 0)
|
||||
}
|
||||
|
||||
func indexOf(hay, needle string) int {
|
||||
for i := 0; i+len(needle) <= len(hay); i++ {
|
||||
if hay[i:i+len(needle)] == needle {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Recorded audio is staged in a hardened per-user temp directory, mirroring
|
||||
// internal/tools/spill.go (PR #506): uid-suffixed name so users on a shared
|
||||
// /tmp cannot collide, 0700 perms, and a pre-existing path is only accepted
|
||||
// when it is a real directory (not a symlink that would redirect recordings)
|
||||
// owned by the current user. Files are removed as soon as their bytes are
|
||||
// read back after Stop — audio never lingers on disk.
|
||||
|
||||
func audioSpillRoot() string {
|
||||
name := "zero-dictation"
|
||||
if uid := os.Getuid(); uid >= 0 {
|
||||
name = fmt.Sprintf("zero-dictation-%d", uid)
|
||||
}
|
||||
return filepath.Join(os.TempDir(), name)
|
||||
}
|
||||
|
||||
func audioSpillDir() (string, error) {
|
||||
dir := audioSpillRoot()
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// MkdirAll follows symlinks and leaves an existing directory untouched, so
|
||||
// verify what is actually at the path.
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("dictation temp path %s is not a directory", dir)
|
||||
}
|
||||
// MkdirAll only applies 0o700 to dirs it creates; an existing directory
|
||||
// (e.g. one created under a loose umask in a prior run) can be group- or
|
||||
// world-readable. Tighten it so staged audio (PII) is never visible to
|
||||
// other users on a shared /tmp.
|
||||
if info.Mode().Perm() != 0o700 {
|
||||
if err := os.Chmod(dir, 0o700); err != nil {
|
||||
return "", fmt.Errorf("tightening dictation temp dir permissions: %w", err)
|
||||
}
|
||||
}
|
||||
if err := checkAudioDirOwner(info); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestLocalStreamingRealServer drives the full local streaming path against a
|
||||
// REAL sherpa-onnx-online-websocket-server + streaming model: spawn the server,
|
||||
// connect, feed a wav's PCM in 50ms chunks, and check the transcript. Opt-in:
|
||||
//
|
||||
// ZERO_STT_STREAM_SERVER=/path/to/sherpa-onnx-online-websocket-server \
|
||||
// ZERO_STT_STREAM_MODEL=/path/to/streaming-model-dir \
|
||||
// ZERO_STT_STREAM_WAV=/path/to/16k-mono.wav \
|
||||
// go test ./internal/dictation/ -run RealServer -v -timeout 5m
|
||||
func TestLocalStreamingRealServer(t *testing.T) {
|
||||
server := os.Getenv("ZERO_STT_STREAM_SERVER")
|
||||
model := os.Getenv("ZERO_STT_STREAM_MODEL")
|
||||
wav := os.Getenv("ZERO_STT_STREAM_WAV")
|
||||
if server == "" || model == "" || wav == "" {
|
||||
t.Skip("set ZERO_STT_STREAM_SERVER, ZERO_STT_STREAM_MODEL, ZERO_STT_STREAM_WAV to run the streaming smoke test")
|
||||
}
|
||||
raw, err := os.ReadFile(wav)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pcm := stripWavHeader(raw)
|
||||
|
||||
mgr := NewServerManager(ServerConfig{Binary: server, ModelPath: model, Port: 6100})
|
||||
defer mgr.Shutdown(context.Background())
|
||||
tr := NewLocalStreamingTranscriber(mgr)
|
||||
|
||||
chunks := make(chan []byte)
|
||||
go func() {
|
||||
defer close(chunks)
|
||||
size := chunkSizeBytes(DefaultSampleRate) // 50ms
|
||||
for off := 0; off < len(pcm); off += size {
|
||||
end := off + size
|
||||
if end > len(pcm) {
|
||||
end = len(pcm)
|
||||
}
|
||||
chunks <- pcm[off:end]
|
||||
time.Sleep(10 * time.Millisecond) // pace it a little, like a mic
|
||||
}
|
||||
}()
|
||||
|
||||
var partials int
|
||||
final, err := tr.StreamTranscribe(context.Background(), chunks, func(string, bool) { partials++ })
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTranscribe: %v", err)
|
||||
}
|
||||
if final == "" {
|
||||
t.Fatal("streaming returned empty transcript")
|
||||
}
|
||||
t.Logf("streaming transcript (%d partials): %q", partials, final)
|
||||
}
|
||||
|
||||
// stripWavHeader drops a standard 44-byte PCM WAV header, returning the raw
|
||||
// int16 samples. Good enough for the canonical 16kHz/mono/16-bit test wavs.
|
||||
func stripWavHeader(b []byte) []byte {
|
||||
if len(b) > 44 && string(b[:4]) == "RIFF" {
|
||||
return b[44:]
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package dictation
|
||||
|
||||
import "context"
|
||||
|
||||
// Transcriber turns captured audio into text. Batch (Transcribe) and streaming
|
||||
// (StreamTranscribe) are one interface but two independent code paths — a
|
||||
// batch-only provider (Groq/OpenAI, sherpa-onnx-offline) leaves
|
||||
// StreamTranscribe returning ErrStreamingUnsupported, and a streaming provider
|
||||
// still implements Transcribe for the batch/Termux fallback.
|
||||
type Transcriber interface {
|
||||
// Transcribe returns the final text for one recorded clip.
|
||||
Transcribe(ctx context.Context, audio []byte) (string, error)
|
||||
|
||||
// StreamTranscribe consumes PCM16 chunks and reports incremental results
|
||||
// via onPartial(text, final) as decoding proceeds, returning the complete
|
||||
// final transcript when the chunk channel closes.
|
||||
//
|
||||
// onPartial's text is the CURRENT best transcript of the utterance so far
|
||||
// (cumulative, not a delta) so the composer can replace the live region
|
||||
// wholesale; final marks a segment the provider considers settled.
|
||||
//
|
||||
// On a mid-stream failure (network drop for cloud, subprocess crash for
|
||||
// local) StreamTranscribe returns the best-effort text accumulated from
|
||||
// onPartial calls made so far PLUS a non-nil error — already-streamed text
|
||||
// is never discarded, only the untranscribed tail after the failure is lost.
|
||||
StreamTranscribe(ctx context.Context, chunks <-chan []byte, onPartial func(text string, final bool)) (string, error)
|
||||
}
|
||||
|
||||
// SampleRate lets a Transcriber declare the capture rate its wire format
|
||||
// expects, so the recorder can be built to match (OpenAI Realtime wants 24kHz;
|
||||
// everything else uses 16kHz). A Transcriber that does not implement this uses
|
||||
// DefaultSampleRate.
|
||||
type SampleRate interface {
|
||||
SampleRate() int
|
||||
}
|
||||
|
||||
// RequiredSampleRate returns the capture rate a Transcriber needs.
|
||||
func RequiredSampleRate(t Transcriber) int {
|
||||
if sr, ok := t.(SampleRate); ok {
|
||||
if rate := sr.SampleRate(); rate > 0 {
|
||||
return rate
|
||||
}
|
||||
}
|
||||
return DefaultSampleRate
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/providers/providerio"
|
||||
)
|
||||
|
||||
// CloudConfig configures a batch cloud transcriber. Groq and OpenAI share the
|
||||
// OpenAI-compatible /audio/transcriptions multipart endpoint — the design's
|
||||
// "each owns its endpoint/model specifics while sharing the request shape"
|
||||
// (§6). Key and base URL are resolved by the caller (TUI layer) from
|
||||
// config/credstore, keeping this type decoupled and unit-testable.
|
||||
type CloudConfig struct {
|
||||
Provider string // ProviderGroq or ProviderOpenAI, for error messages
|
||||
BaseURL string // e.g. https://api.groq.com/openai/v1
|
||||
APIKey string
|
||||
Model string // e.g. whisper-large-v3-turbo, whisper-1
|
||||
// Language optionally constrains recognition (ISO-639-1). Empty = auto.
|
||||
Language string
|
||||
|
||||
// HTTPClient is injectable for tests; nil uses providerio's shared,
|
||||
// stall-hardened client.
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type cloudTranscriber struct {
|
||||
cfg CloudConfig
|
||||
}
|
||||
|
||||
// NewCloudTranscriber builds a batch transcriber for Groq or OpenAI.
|
||||
func NewCloudTranscriber(cfg CloudConfig) (Transcriber, error) {
|
||||
if strings.TrimSpace(cfg.APIKey) == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: cfg.Provider + " API key",
|
||||
Hint: fmt.Sprintf("set a %s API key (run `zero auth`) to use cloud dictation", cfg.Provider),
|
||||
}
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
return nil, fmt.Errorf("dictation: no model configured for provider %q", cfg.Provider)
|
||||
}
|
||||
base, err := providerio.NormalizeBaseURL(cfg.BaseURL, cfg.BaseURL, cfg.Provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.BaseURL = base
|
||||
return &cloudTranscriber{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (c *cloudTranscriber) Transcribe(ctx context.Context, audio []byte) (string, error) {
|
||||
if len(audio) == 0 {
|
||||
return "", errors.New("no audio to transcribe")
|
||||
}
|
||||
body, contentType, err := c.buildMultipart(audio)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
endpoint := c.cfg.BaseURL + "/audio/transcriptions"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
|
||||
|
||||
resp, err := providerio.HTTPClient(c.cfg.HTTPClient).Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s transcription request failed: %w", c.cfg.Provider, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
msg := providerio.ClassifiedError(resp.StatusCode, strings.TrimSpace(string(respBody)), c.cfg.APIKey)
|
||||
// A 401/403 is a fixable credential problem: surface it as a typed AuthError
|
||||
// so the TUI can offer an inline key prompt instead of a dead-end message.
|
||||
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
|
||||
return "", &AuthError{Provider: c.cfg.Provider, Message: msg}
|
||||
}
|
||||
return "", errors.New(msg)
|
||||
}
|
||||
var parsed struct {
|
||||
Text string `json:"text"`
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(respBody, &parsed); err != nil {
|
||||
return "", fmt.Errorf("%s transcription: could not parse response: %w", c.cfg.Provider, err)
|
||||
}
|
||||
if parsed.Error.Message != "" {
|
||||
return "", fmt.Errorf("%s transcription error: %s", c.cfg.Provider, providerio.Redact(parsed.Error.Message, c.cfg.APIKey))
|
||||
}
|
||||
return strings.TrimSpace(parsed.Text), nil
|
||||
}
|
||||
|
||||
// buildMultipart assembles the transcription upload. The file field name and
|
||||
// the "model"/"response_format" fields match the OpenAI audio API that Groq
|
||||
// mirrors. The filename's extension tells the server the container type —
|
||||
// SniffFormat keeps WAV/M4A honest so Termux's AAC uploads work.
|
||||
func (c *cloudTranscriber) buildMultipart(audio []byte) ([]byte, string, error) {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
fw, err := w.CreateFormFile("file", SniffFormat(audio).FileName())
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if _, err := fw.Write(audio); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := w.WriteField("model", c.cfg.Model); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := w.WriteField("response_format", "json"); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if c.cfg.Language != "" {
|
||||
if err := w.WriteField("language", c.cfg.Language); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return buf.Bytes(), w.FormDataContentType(), nil
|
||||
}
|
||||
|
||||
// StreamTranscribe is not supported for the batch cloud providers (Groq has no
|
||||
// streaming product; OpenAI batch is one-shot). Streaming uses the dedicated
|
||||
// Deepgram/OpenAI-Realtime transcribers instead.
|
||||
func (c *cloudTranscriber) StreamTranscribe(context.Context, <-chan []byte, func(string, bool)) (string, error) {
|
||||
return "", ErrStreamingUnsupported
|
||||
}
|
||||
|
||||
// ErrStreamingUnsupported reports that a batch-only Transcriber cannot stream;
|
||||
// the caller falls back to the batch pipeline.
|
||||
var ErrStreamingUnsupported = errors.New("this transcription provider does not support streaming")
|
||||
@@ -0,0 +1,142 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCloudTranscribeBuildsMultipartAndParsesText(t *testing.T) {
|
||||
var gotModel, gotAuth, gotFilename string
|
||||
var gotFileBytes []byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
if r.URL.Path != "/audio/transcriptions" {
|
||||
t.Errorf("unexpected path %q", r.URL.Path)
|
||||
}
|
||||
mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil || !strings.HasPrefix(mediaType, "multipart/") {
|
||||
t.Fatalf("bad content type %q: %v", r.Header.Get("Content-Type"), err)
|
||||
}
|
||||
mr := multipart.NewReader(r.Body, params["boundary"])
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("multipart: %v", err)
|
||||
}
|
||||
switch part.FormName() {
|
||||
case "model":
|
||||
b, _ := io.ReadAll(part)
|
||||
gotModel = string(b)
|
||||
case "file":
|
||||
gotFilename = part.FileName()
|
||||
gotFileBytes, _ = io.ReadAll(part)
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
io.WriteString(w, `{"text":" hello world "}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tr, err := NewCloudTranscriber(CloudConfig{
|
||||
Provider: ProviderGroq,
|
||||
BaseURL: srv.URL,
|
||||
APIKey: "secret-key",
|
||||
Model: "whisper-large-v3-turbo",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewCloudTranscriber: %v", err)
|
||||
}
|
||||
audio := wavFixture()
|
||||
text, err := tr.Transcribe(context.Background(), audio)
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if text != "hello world" {
|
||||
t.Errorf("text = %q, want trimmed 'hello world'", text)
|
||||
}
|
||||
if gotModel != "whisper-large-v3-turbo" {
|
||||
t.Errorf("model field = %q", gotModel)
|
||||
}
|
||||
if gotAuth != "Bearer secret-key" {
|
||||
t.Errorf("auth header = %q", gotAuth)
|
||||
}
|
||||
if gotFilename != "audio.wav" {
|
||||
t.Errorf("filename = %q, want audio.wav", gotFilename)
|
||||
}
|
||||
if len(gotFileBytes) != len(audio) {
|
||||
t.Errorf("uploaded %d file bytes, want %d", len(gotFileBytes), len(audio))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudTranscribeM4AFilename(t *testing.T) {
|
||||
var gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, params, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
mr := multipart.NewReader(r.Body, params["boundary"])
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if part.FormName() == "file" {
|
||||
gotFilename = part.FileName()
|
||||
}
|
||||
}
|
||||
io.WriteString(w, `{"text":"ok"}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tr, _ := NewCloudTranscriber(CloudConfig{Provider: ProviderOpenAI, BaseURL: srv.URL, APIKey: "k", Model: "whisper-1"})
|
||||
if _, err := tr.Transcribe(context.Background(), m4aFixture()); err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if gotFilename != "audio.m4a" {
|
||||
t.Errorf("filename = %q, want audio.m4a", gotFilename)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudTranscribeAuthErrorClassified(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
io.WriteString(w, `{"error":{"message":"invalid api key sk-secret-key"}}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tr, _ := NewCloudTranscriber(CloudConfig{Provider: ProviderGroq, BaseURL: srv.URL, APIKey: "sk-secret-key", Model: "whisper-large-v3-turbo"})
|
||||
_, err := tr.Transcribe(context.Background(), wavFixture())
|
||||
if err == nil {
|
||||
t.Fatal("expected error on 401")
|
||||
}
|
||||
if strings.Contains(err.Error(), "sk-secret-key") {
|
||||
t.Errorf("API key leaked into error: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "auth error") {
|
||||
t.Errorf("expected auth-error classification, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCloudTranscriberRequiresKey(t *testing.T) {
|
||||
_, err := NewCloudTranscriber(CloudConfig{Provider: ProviderGroq, BaseURL: "https://api.groq.com/openai/v1", Model: "m"})
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing key, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudTranscribeStreamingUnsupported(t *testing.T) {
|
||||
tr, _ := NewCloudTranscriber(CloudConfig{Provider: ProviderGroq, BaseURL: "https://x", APIKey: "k", Model: "m"})
|
||||
_, err := tr.StreamTranscribe(context.Background(), nil, nil)
|
||||
if !errors.Is(err, ErrStreamingUnsupported) {
|
||||
t.Errorf("want ErrStreamingUnsupported, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// DeepgramConfig configures the Deepgram streaming transcriber (§6b, default
|
||||
// cloud streaming provider). The key is resolved by the caller.
|
||||
type DeepgramConfig struct {
|
||||
APIKey string
|
||||
Model string // default "nova-3"
|
||||
Language string
|
||||
SampleRate int // capture rate; default DefaultSampleRate (16kHz)
|
||||
// BaseURL overrides the wss endpoint (tests point it at a fake server).
|
||||
BaseURL string
|
||||
}
|
||||
|
||||
type deepgramTranscriber struct {
|
||||
cfg DeepgramConfig
|
||||
}
|
||||
|
||||
// NewDeepgramTranscriber builds the Deepgram streaming transcriber.
|
||||
func NewDeepgramTranscriber(cfg DeepgramConfig) (Transcriber, error) {
|
||||
if strings.TrimSpace(cfg.APIKey) == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: "Deepgram API key",
|
||||
Hint: "set a Deepgram API key (DEEPGRAM_API_KEY, or `zero auth`) to use Deepgram streaming dictation",
|
||||
}
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
cfg.Model = "nova-3"
|
||||
}
|
||||
if cfg.SampleRate <= 0 {
|
||||
cfg.SampleRate = DefaultSampleRate
|
||||
}
|
||||
if cfg.BaseURL == "" {
|
||||
cfg.BaseURL = "wss://api.deepgram.com/v1/listen"
|
||||
}
|
||||
return &deepgramTranscriber{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
// Transcribe is unsupported — Deepgram is used only on the streaming path here.
|
||||
func (d *deepgramTranscriber) Transcribe(context.Context, []byte) (string, error) {
|
||||
return "", ErrStreamingUnsupported
|
||||
}
|
||||
|
||||
func (d *deepgramTranscriber) SampleRate() int { return d.cfg.SampleRate }
|
||||
|
||||
func (d *deepgramTranscriber) StreamTranscribe(ctx context.Context, chunks <-chan []byte, onPartial func(string, bool)) (string, error) {
|
||||
endpoint := d.buildURL()
|
||||
conn, _, err := websocket.Dial(ctx, endpoint, &websocket.DialOptions{
|
||||
HTTPHeader: http.Header{"Authorization": {"Token " + d.cfg.APIKey}},
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("connecting to Deepgram: %w", err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
|
||||
// Deepgram consumes raw linear16 (int16) PCM — the recorder's native format,
|
||||
// so no conversion (unlike sherpa-onnx, which wants float32).
|
||||
writeErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
for chunk := range chunks {
|
||||
if err := conn.Write(ctx, websocket.MessageBinary, chunk); err != nil {
|
||||
writeErrCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
// CloseStream flushes any buffered audio and returns final results before
|
||||
// Deepgram closes the socket.
|
||||
writeErrCh <- conn.Write(ctx, websocket.MessageText, []byte(`{"type":"CloseStream"}`))
|
||||
}()
|
||||
|
||||
// Deepgram results are per-utterance segments, not cumulative: accumulate the
|
||||
// is_final segments and append the current interim to build the full text.
|
||||
var finals []string
|
||||
var interim string
|
||||
compose := func() string {
|
||||
return strings.TrimSpace(strings.Join(append(finals, interim), " "))
|
||||
}
|
||||
for {
|
||||
typ, data, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
// A clean close after CloseStream is success, not failure.
|
||||
if websocket.CloseStatus(err) == websocket.StatusNormalClosure {
|
||||
return compose(), nil
|
||||
}
|
||||
select {
|
||||
case werr := <-writeErrCh:
|
||||
if werr != nil {
|
||||
err = werr
|
||||
}
|
||||
default:
|
||||
}
|
||||
return compose(), fmt.Errorf("Deepgram stream error: %w", err)
|
||||
}
|
||||
if typ != websocket.MessageText {
|
||||
continue
|
||||
}
|
||||
transcript, isFinal, ok := parseDeepgramResult(data)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if isFinal {
|
||||
if transcript != "" {
|
||||
finals = append(finals, transcript)
|
||||
}
|
||||
interim = ""
|
||||
} else {
|
||||
interim = transcript
|
||||
}
|
||||
if onPartial != nil {
|
||||
onPartial(compose(), isFinal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deepgramTranscriber) buildURL() string {
|
||||
q := url.Values{}
|
||||
q.Set("encoding", "linear16")
|
||||
q.Set("sample_rate", strconv.Itoa(d.cfg.SampleRate))
|
||||
q.Set("channels", "1")
|
||||
q.Set("interim_results", "true")
|
||||
q.Set("punctuate", "true")
|
||||
q.Set("model", d.cfg.Model)
|
||||
if d.cfg.Language != "" {
|
||||
q.Set("language", d.cfg.Language)
|
||||
}
|
||||
return d.cfg.BaseURL + "?" + q.Encode()
|
||||
}
|
||||
|
||||
// parseDeepgramResult extracts the transcript and is_final flag from a Deepgram
|
||||
// "Results" message: channel.alternatives[0].transcript.
|
||||
func parseDeepgramResult(data []byte) (transcript string, isFinal bool, ok bool) {
|
||||
var msg struct {
|
||||
Type string `json:"type"`
|
||||
IsFinal bool `json:"is_final"`
|
||||
Channel struct {
|
||||
Alternatives []struct {
|
||||
Transcript string `json:"transcript"`
|
||||
} `json:"alternatives"`
|
||||
} `json:"channel"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &msg); err != nil {
|
||||
return "", false, false
|
||||
}
|
||||
// Only "Results" messages carry transcripts; ignore Metadata/UtteranceEnd/etc.
|
||||
if msg.Type != "" && msg.Type != "Results" {
|
||||
return "", false, false
|
||||
}
|
||||
if len(msg.Channel.Alternatives) == 0 {
|
||||
return "", false, false
|
||||
}
|
||||
return strings.TrimSpace(msg.Channel.Alternatives[0].Transcript), msg.IsFinal, true
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// LocalConfig configures the offline sherpa-onnx batch transcriber. Every tier
|
||||
// (default/standard/high-end, §7) is just a different ModelPath — there are no
|
||||
// tier-specific code paths; the model family is detected from the files present.
|
||||
type LocalConfig struct {
|
||||
// Binary is the sherpa-onnx-offline executable; looked up on PATH when a
|
||||
// bare name. Empty defaults to "sherpa-onnx-offline".
|
||||
Binary string
|
||||
// ModelPath is the directory holding the model's .onnx + tokens.txt files
|
||||
// (stt.localModelPath).
|
||||
ModelPath string
|
||||
NumThreads int
|
||||
|
||||
// Test seams.
|
||||
runOutput commandOutputRunner
|
||||
tempDir func() (string, error)
|
||||
}
|
||||
|
||||
type localTranscriber struct {
|
||||
cfg LocalConfig
|
||||
}
|
||||
|
||||
// NewLocalTranscriber builds the offline transcriber. It does not touch the
|
||||
// filesystem or PATH until Transcribe runs, so a missing binary/model degrades
|
||||
// to a clear setup message at use time (matching internal/lsp).
|
||||
func NewLocalTranscriber(cfg LocalConfig) (Transcriber, error) {
|
||||
if strings.TrimSpace(cfg.ModelPath) == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: "local STT model",
|
||||
Hint: "set stt.localModelPath to a sherpa-onnx model directory (see docs/dictation.md)",
|
||||
}
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = "sherpa-onnx-offline"
|
||||
}
|
||||
if cfg.runOutput == nil {
|
||||
cfg.runOutput = runCommandOutput
|
||||
}
|
||||
if cfg.tempDir == nil {
|
||||
cfg.tempDir = audioSpillDir
|
||||
}
|
||||
return &localTranscriber{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (l *localTranscriber) Transcribe(ctx context.Context, audio []byte) (string, error) {
|
||||
if len(audio) == 0 {
|
||||
return "", errors.New("no audio to transcribe")
|
||||
}
|
||||
modelArgs, err := detectModelArgs(l.cfg.ModelPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
dir, err := l.cfg.tempDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("preparing dictation temp dir: %w", err)
|
||||
}
|
||||
ext := ".wav"
|
||||
if SniffFormat(audio) == FormatM4A {
|
||||
ext = ".m4a"
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, "stt-*"+ext)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("staging audio for transcription: %w", err)
|
||||
}
|
||||
path := tmp.Name()
|
||||
defer os.Remove(path) // audio never lingers on disk (§12)
|
||||
if _, err := tmp.Write(audio); err != nil {
|
||||
tmp.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
args := make([]string, 0, len(modelArgs)+3)
|
||||
args = append(args, modelArgs...)
|
||||
if l.cfg.NumThreads > 0 {
|
||||
args = append(args, "--num-threads="+strconv.Itoa(l.cfg.NumThreads))
|
||||
}
|
||||
args = append(args, path)
|
||||
|
||||
// Fail fast if already cancelled; ctx also propagates into the exec below so a
|
||||
// wedged sherpa-onnx-offline is killed on cancel/timeout rather than hanging.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := l.cfg.runOutput(ctx, l.cfg.Binary, args...)
|
||||
if err != nil {
|
||||
if errors.Is(err, exec.ErrNotFound) {
|
||||
return "", &SetupError{
|
||||
Tool: l.cfg.Binary,
|
||||
Hint: "install sherpa-onnx and put its offline binary on PATH (see docs/dictation.md)",
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("sherpa-onnx transcription failed: %w (%s)", err, lastLines(out, 3))
|
||||
}
|
||||
text := parseSherpaOutput(out)
|
||||
if text == "" {
|
||||
return "", fmt.Errorf("sherpa-onnx produced no transcript (output: %s)", lastLines(out, 3))
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// detectModelArgs inspects a sherpa-onnx model directory and builds the
|
||||
// family-specific --*=path flags. Supports the families sherpa-onnx-offline
|
||||
// accepts; Moonshine (the design's default/standard/high-end tiers) is the
|
||||
// primary target. tokens.txt is required by every family.
|
||||
func detectModelArgs(dir string) ([]string, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, &SetupError{
|
||||
Tool: "local STT model",
|
||||
Hint: fmt.Sprintf("stt.localModelPath %q is not readable: %v", dir, err),
|
||||
}
|
||||
}
|
||||
files := map[string]string{} // lowercased basename -> absolute path
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
files[strings.ToLower(e.Name())] = filepath.Join(dir, e.Name())
|
||||
}
|
||||
|
||||
tokens := findTokensFile(files)
|
||||
if tokens == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: "local STT model",
|
||||
Hint: fmt.Sprintf("no tokens.txt found in stt.localModelPath %q — is it a sherpa-onnx model directory?", dir),
|
||||
}
|
||||
}
|
||||
|
||||
dirName := strings.ToLower(filepath.Base(dir))
|
||||
|
||||
// Order matters — check the most specific signatures first so a model is
|
||||
// never mis-classified as a family it only partially resembles.
|
||||
|
||||
// Moonshine: preprocessor + encoder + cached/uncached decoders. Its files are
|
||||
// "encode"/"decode" (no trailing "r"), so they never collide with the
|
||||
// "encoder"/"decoder" families below.
|
||||
if pre := findContains(files, "preprocess"); pre != "" {
|
||||
enc := findContains(files, "encode")
|
||||
// "uncached_decode" CONTAINS "cached_decode", so match the cached decoder
|
||||
// as "cached" AND not "uncached" to disambiguate (map order is random).
|
||||
unc := findOnnxMatching(files, "uncached", "")
|
||||
cac := findOnnxMatching(files, "cached", "uncached")
|
||||
if enc != "" && unc != "" && cac != "" {
|
||||
return []string{
|
||||
"--moonshine-preprocessor=" + pre,
|
||||
"--moonshine-encoder=" + enc,
|
||||
"--moonshine-uncached-decoder=" + unc,
|
||||
"--moonshine-cached-decoder=" + cac,
|
||||
"--tokens=" + tokens,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Transducer (Zipformer/Conformer): encoder + decoder + joiner, matched as a
|
||||
// consistent int8-or-fp32 set (mixing quantizations is invalid).
|
||||
if set := consistentQuantizedSet(files, "encoder", "decoder", "joiner"); set != nil {
|
||||
return []string{"--encoder=" + set[0], "--decoder=" + set[1], "--joiner=" + set[2], "--tokens=" + tokens}, nil
|
||||
}
|
||||
|
||||
// Encoder+decoder families that share Whisper's file shape (encoder + decoder,
|
||||
// no joiner) but need their own flags — FireRedASR-AED, Canary, Cohere. They are
|
||||
// indistinguishable from Whisper by files alone, so disambiguate by name FIRST;
|
||||
// anything unnamed falls through to the Whisper default below.
|
||||
for _, fam := range encoderDecoderFamilies {
|
||||
if nameMatchesFamily(dirName, files, fam.match) {
|
||||
if set := consistentQuantizedSet(files, "encoder", "decoder"); set != nil {
|
||||
return []string{fam.encoderFlag + "=" + set[0], fam.decoderFlag + "=" + set[1], "--tokens=" + tokens}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Whisper: encoder + decoder, NO joiner. Sherpa's Whisper files are named
|
||||
// "<model>-encoder.onnx" / "<model>-decoder.onnx" (NOT "whisper-encoder"), so
|
||||
// match on the encoder/decoder roles — a consistent pair — after transducer
|
||||
// (which owns the encoder/decoder/joiner shape) has already been ruled out.
|
||||
if set := consistentQuantizedSet(files, "encoder", "decoder"); set != nil {
|
||||
return []string{"--whisper-encoder=" + set[0], "--whisper-decoder=" + set[1], "--tokens=" + tokens}, nil
|
||||
}
|
||||
|
||||
// Single-model families: one generic model .onnx + tokens, indistinguishable by
|
||||
// file shape, so they are told apart by the model directory/file name. This
|
||||
// covers SenseVoice, Paraformer, and the CTC families (FireRedASR-CTC, NeMo,
|
||||
// TeleSpeech, WeNet, Zipformer-CTC, Dolphin, Omnilingual). Ordered most-specific
|
||||
// first so a compound name (e.g. "fire-red-asr2-ctc") matches its own family
|
||||
// before a generic token in it (e.g. bare "ctc") could steer it elsewhere.
|
||||
if m := singleModelOnnx(files); m != "" {
|
||||
for _, fam := range singleModelFamilies {
|
||||
if nameMatchesFamily(dirName, files, fam.match) {
|
||||
return []string{fam.flag + "=" + m, "--tokens=" + tokens}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, &SetupError{
|
||||
Tool: "local STT model",
|
||||
Hint: fmt.Sprintf("could not recognize the model family in %q (supported: Moonshine, Whisper, transducer/Zipformer, Paraformer, SenseVoice, and the CTC families FireRedASR/NeMo/TeleSpeech/WeNet/Zipformer-CTC/Dolphin/Omnilingual). See docs/dictation.md", dir),
|
||||
}
|
||||
}
|
||||
|
||||
// singleModelFamilies maps a model's directory/file-name signature to the
|
||||
// sherpa-onnx-offline flag for that single-file family. Each family ships exactly
|
||||
// one generic model .onnx (+ tokens), so only the name tells them apart. Ordered
|
||||
// so a specific compound name wins before a broader one.
|
||||
var singleModelFamilies = []struct {
|
||||
match []string // name substrings (any hit); most-specific spellings first
|
||||
flag string
|
||||
}{
|
||||
{[]string{"fire-red-asr2-ctc", "fire-red-asr-ctc", "fire-red-asr", "firered"}, "--fire-red-asr-ctc"},
|
||||
{[]string{"nemo-ctc", "nemo_ctc"}, "--nemo-ctc-model"},
|
||||
{[]string{"zipformer2-ctc", "zipformer-ctc"}, "--zipformer-ctc-model"},
|
||||
{[]string{"telespeech"}, "--telespeech-ctc"},
|
||||
{[]string{"wenet"}, "--wenet-ctc-model"},
|
||||
{[]string{"dolphin"}, "--dolphin-model"},
|
||||
{[]string{"omnilingual"}, "--omnilingual-asr-model"},
|
||||
{[]string{"medasr"}, "--medasr"},
|
||||
{[]string{"tdnn"}, "--tdnn-model"},
|
||||
{[]string{"sense-voice", "sensevoice"}, "--sense-voice-model"},
|
||||
{[]string{"paraformer"}, "--paraformer"},
|
||||
}
|
||||
|
||||
// encoderDecoderFamilies are offline families shaped like Whisper (an encoder +
|
||||
// decoder .onnx, no joiner) but needing their own flags; matched by name before
|
||||
// the Whisper fallback claims the shape.
|
||||
var encoderDecoderFamilies = []struct {
|
||||
match []string
|
||||
encoderFlag string
|
||||
decoderFlag string
|
||||
}{
|
||||
{[]string{"fire-red-asr"}, "--fire-red-asr-encoder", "--fire-red-asr-decoder"},
|
||||
{[]string{"canary"}, "--canary-encoder", "--canary-decoder"},
|
||||
{[]string{"cohere"}, "--cohere-transcribe-encoder", "--cohere-transcribe-decoder"},
|
||||
}
|
||||
|
||||
// nameMatchesFamily reports whether the model directory name or any of its .onnx
|
||||
// filenames contain one of the family's signature substrings.
|
||||
func nameMatchesFamily(dirName string, files map[string]string, subs []string) bool {
|
||||
for _, sub := range subs {
|
||||
if strings.Contains(dirName, sub) {
|
||||
return true
|
||||
}
|
||||
for name := range files {
|
||||
if strings.HasSuffix(name, ".onnx") && strings.Contains(name, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// consistentQuantizedSet returns one file per role (in order), preferring an
|
||||
// int8 build of each but falling back to fp32 when a role has no int8 file.
|
||||
// Returns nil when any role is entirely missing.
|
||||
//
|
||||
// Models legitimately SHIP mixed quantization — e.g. the x-asr streaming
|
||||
// zipformers ship an int8 encoder and joiner with an fp32 decoder — and
|
||||
// sherpa-onnx accepts that. So this must NOT require every role to share one
|
||||
// quantization; it just needs one file per role, favoring the smaller int8 when
|
||||
// a choice exists. (When a model ships both int8 and fp32 for every role, this
|
||||
// still picks all-int8, since int8 is available for each.)
|
||||
func consistentQuantizedSet(files map[string]string, roles ...string) []string {
|
||||
pick := func(role string, int8Only bool) string {
|
||||
for name, path := range files {
|
||||
if !strings.HasSuffix(name, ".onnx") || !strings.Contains(name, role) {
|
||||
continue
|
||||
}
|
||||
if int8Only && !strings.Contains(name, "int8") {
|
||||
continue
|
||||
}
|
||||
return path
|
||||
}
|
||||
return ""
|
||||
}
|
||||
out := make([]string, len(roles))
|
||||
for i, role := range roles {
|
||||
p := pick(role, true) // prefer int8 (smaller, faster)
|
||||
if p == "" {
|
||||
p = pick(role, false) // fall back to fp32 for this role
|
||||
}
|
||||
if p == "" {
|
||||
return nil // this role is genuinely absent
|
||||
}
|
||||
out[i] = p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// findTokensFile returns the model's tokens file. It is usually "tokens.txt",
|
||||
// but some families (e.g. Whisper) prefix it as "<model>-tokens.txt".
|
||||
func findTokensFile(files map[string]string) string {
|
||||
if p := files["tokens.txt"]; p != "" {
|
||||
return p
|
||||
}
|
||||
for name, path := range files {
|
||||
if strings.HasSuffix(name, "tokens.txt") {
|
||||
return path
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// singleModelOnnx returns the model's single .onnx file (preferring an int8
|
||||
// build), for families like SenseVoice/Paraformer that ship exactly one.
|
||||
func singleModelOnnx(files map[string]string) string {
|
||||
best := ""
|
||||
for name, path := range files {
|
||||
if !strings.HasSuffix(name, ".onnx") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(name, "int8") {
|
||||
return path
|
||||
}
|
||||
best = path
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// findOnnxMatching returns an .onnx file whose name contains `must` and, when
|
||||
// `exclude` is non-empty, does NOT contain `exclude` — used to tell the cached
|
||||
// from the uncached Moonshine decoder.
|
||||
func findOnnxMatching(files map[string]string, must, exclude string) string {
|
||||
for name, path := range files {
|
||||
if !strings.HasSuffix(name, ".onnx") || !strings.Contains(name, must) {
|
||||
continue
|
||||
}
|
||||
if exclude != "" && strings.Contains(name, exclude) {
|
||||
continue
|
||||
}
|
||||
return path
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func findContains(files map[string]string, substr string) string {
|
||||
for name, path := range files {
|
||||
if strings.HasSuffix(name, ".onnx") && strings.Contains(name, substr) {
|
||||
return path
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseSherpaOutput extracts the transcript from sherpa-onnx-offline's output.
|
||||
// The binary prints a JSON object with a "text" field (amid diagnostic lines);
|
||||
// scan for the first line that parses as such.
|
||||
func parseSherpaOutput(out []byte) string {
|
||||
for _, line := range bytes.Split(out, []byte("\n")) {
|
||||
trimmed := bytes.TrimSpace(line)
|
||||
if len(trimmed) == 0 || trimmed[0] != '{' {
|
||||
continue
|
||||
}
|
||||
var parsed struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := json.Unmarshal(trimmed, &parsed); err == nil && strings.TrimSpace(parsed.Text) != "" {
|
||||
return strings.TrimSpace(parsed.Text)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func lastLines(out []byte, n int) string {
|
||||
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, " | ")
|
||||
}
|
||||
|
||||
// StreamTranscribe is unsupported for the offline binary; local streaming uses
|
||||
// the sherpa-onnx websocket server transcriber instead.
|
||||
func (l *localTranscriber) StreamTranscribe(context.Context, <-chan []byte, func(string, bool)) (string, error) {
|
||||
return "", ErrStreamingUnsupported
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// localStreamingTranscriber streams audio to the warm sherpa-onnx websocket
|
||||
// server (§6a). The server keeps encoder state across chunks (a true streaming
|
||||
// transducer), so partials arrive as decoding proceeds — no redundant
|
||||
// re-inference.
|
||||
//
|
||||
// Wire protocol (verified against sherpa-onnx's reference client, NOT the
|
||||
// plan's "16-bit PCM" description, which was imprecise): the client sends
|
||||
// binary frames of little-endian float32 samples normalized to [-1, 1]
|
||||
// (int16/32768), then the text "Done" to end; the server replies with JSON
|
||||
// result messages and finally the text "Done!" to signal completion.
|
||||
type localStreamingTranscriber struct {
|
||||
manager *ServerManager
|
||||
}
|
||||
|
||||
// NewLocalStreamingTranscriber builds a streaming transcriber backed by the
|
||||
// shared, session-long sherpa-onnx server manager.
|
||||
func NewLocalStreamingTranscriber(manager *ServerManager) Transcriber {
|
||||
return &localStreamingTranscriber{manager: manager}
|
||||
}
|
||||
|
||||
// Transcribe is not this transcriber's job — the streaming path uses
|
||||
// StreamTranscribe; the batch/Termux fallback uses the offline transcriber.
|
||||
func (l *localStreamingTranscriber) Transcribe(context.Context, []byte) (string, error) {
|
||||
return "", errors.New("local streaming transcriber does not support batch transcription")
|
||||
}
|
||||
|
||||
func (l *localStreamingTranscriber) StreamTranscribe(ctx context.Context, chunks <-chan []byte, onPartial func(string, bool)) (string, error) {
|
||||
url, err := l.manager.EnsureRunning(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conn, err := dialWithOneRestart(ctx, l.manager, url)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("connecting to sherpa-onnx server: %w", err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
|
||||
// Writer goroutine: forward converted chunks, then signal end. A write error
|
||||
// (server crashed) cancels the read side via writeErr.
|
||||
writeErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
for chunk := range chunks {
|
||||
if err := conn.Write(ctx, websocket.MessageBinary, pcm16ToFloat32LE(chunk)); err != nil {
|
||||
writeErrCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
writeErrCh <- conn.Write(ctx, websocket.MessageText, []byte("Done"))
|
||||
}()
|
||||
|
||||
// The server reports "text" PER SEGMENT, not cumulatively for the whole
|
||||
// utterance: when it detects an endpoint (a pause between phrases, or the
|
||||
// trailing silence when recording stops) it finalizes the current segment, bumps
|
||||
// "segment", and resets "text" to "" for the next one. So we accumulate the text
|
||||
// of every segment — otherwise a new segment's (often empty) result would
|
||||
// overwrite everything already transcribed, and the transcript would vanish on
|
||||
// stop. Displayed/returned text is all segments joined in order.
|
||||
var segmentTexts []string
|
||||
joinSegments := func() string {
|
||||
parts := make([]string, 0, len(segmentTexts))
|
||||
for _, s := range segmentTexts {
|
||||
if t := strings.TrimSpace(s); t != "" {
|
||||
parts = append(parts, t)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
lastText := ""
|
||||
for {
|
||||
typ, data, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
// Mid-stream failure: return the best-effort text accumulated so far
|
||||
// plus the error — already-streamed partials are never discarded (§6).
|
||||
select {
|
||||
case werr := <-writeErrCh:
|
||||
if werr != nil {
|
||||
err = werr
|
||||
}
|
||||
default:
|
||||
}
|
||||
return lastText, fmt.Errorf("sherpa-onnx stream error: %w", err)
|
||||
}
|
||||
if typ == websocket.MessageText && strings.TrimSpace(string(data)) == "Done!" {
|
||||
break
|
||||
}
|
||||
text, segment, final, ok := parseSherpaStreamResult(data)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if segment < 0 {
|
||||
segment = 0
|
||||
}
|
||||
for len(segmentTexts) <= segment {
|
||||
segmentTexts = append(segmentTexts, "")
|
||||
}
|
||||
segmentTexts[segment] = text
|
||||
lastText = joinSegments()
|
||||
if onPartial != nil {
|
||||
onPartial(lastText, final)
|
||||
}
|
||||
}
|
||||
_ = conn.Close(websocket.StatusNormalClosure, "")
|
||||
return lastText, nil
|
||||
}
|
||||
|
||||
// dialWithOneRestart dials the server, and on failure restarts it once and
|
||||
// retries — mirroring the LSP manager's single crashed-server recovery (§6/§6a).
|
||||
func dialWithOneRestart(ctx context.Context, manager *ServerManager, url string) (*websocket.Conn, error) {
|
||||
conn, _, err := websocket.Dial(ctx, url, nil)
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
// The warm server may have died between EnsureRunning and Dial; reap and
|
||||
// restart once.
|
||||
_ = manager.Shutdown(ctx)
|
||||
url, rerr := manager.EnsureRunning(ctx)
|
||||
if rerr != nil {
|
||||
return nil, rerr
|
||||
}
|
||||
conn, _, err = websocket.Dial(ctx, url, nil)
|
||||
return conn, err
|
||||
}
|
||||
|
||||
// parseSherpaStreamResult extracts one server JSON result. "text" is the
|
||||
// hypothesis for the CURRENT segment (segment index in "segment"), which resets
|
||||
// each time the recognizer endpoints — so the caller must accumulate per segment,
|
||||
// not treat "text" as the whole-utterance transcript. "is_final" marks a settled
|
||||
// segment.
|
||||
func parseSherpaStreamResult(data []byte) (text string, segment int, final bool, ok bool) {
|
||||
var parsed struct {
|
||||
Text string `json:"text"`
|
||||
Segment int `json:"segment"`
|
||||
IsFinal bool `json:"is_final"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &parsed); err != nil {
|
||||
return "", 0, false, false
|
||||
}
|
||||
return strings.TrimSpace(parsed.Text), parsed.Segment, parsed.IsFinal, true
|
||||
}
|
||||
|
||||
// pcm16ToFloat32LE converts a little-endian int16 PCM chunk to the little-endian
|
||||
// float32 samples (normalized to [-1, 1]) the sherpa-onnx server expects.
|
||||
func pcm16ToFloat32LE(pcm []byte) []byte {
|
||||
n := len(pcm) / 2
|
||||
out := make([]byte, n*4)
|
||||
for i := 0; i < n; i++ {
|
||||
s := int16(uint16(pcm[2*i]) | uint16(pcm[2*i+1])<<8)
|
||||
bits := math.Float32bits(float32(s) / 32768.0)
|
||||
binary.LittleEndian.PutUint32(out[4*i:], bits)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeModelDir(t *testing.T, files ...string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for _, f := range files {
|
||||
if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestLocalTranscribeMoonshineArgs(t *testing.T) {
|
||||
dir := writeModelDir(t,
|
||||
"preprocess.onnx", "encode.onnx", "uncached_decode.onnx", "cached_decode.onnx", "tokens.txt")
|
||||
var gotBin string
|
||||
var gotArgs []string
|
||||
runOutput := func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
gotBin, gotArgs = name, args
|
||||
return []byte("Started\nDone!\n\n/tmp/a.wav\n{\"text\":\" transcribed speech \"}\n----"), nil
|
||||
}
|
||||
tr, err := NewLocalTranscriber(LocalConfig{ModelPath: dir, runOutput: runOutput, tempDir: func() (string, error) { return t.TempDir(), nil }})
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalTranscriber: %v", err)
|
||||
}
|
||||
text, err := tr.Transcribe(context.Background(), wavFixture())
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if text != "transcribed speech" {
|
||||
t.Errorf("text = %q", text)
|
||||
}
|
||||
if gotBin != "sherpa-onnx-offline" {
|
||||
t.Errorf("binary = %q", gotBin)
|
||||
}
|
||||
joined := strings.Join(gotArgs, " ")
|
||||
for _, want := range []string{"--moonshine-preprocessor=", "--moonshine-encoder=", "--moonshine-uncached-decoder=", "--moonshine-cached-decoder=", "--tokens="} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("missing flag %q in %q", want, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalTranscribeTransducerArgs(t *testing.T) {
|
||||
dir := writeModelDir(t, "encoder-epoch-99.onnx", "decoder-epoch-99.onnx", "joiner-epoch-99.onnx", "tokens.txt")
|
||||
var gotArgs []string
|
||||
runOutput := func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
gotArgs = args
|
||||
return []byte(`{"text":"hi"}`), nil
|
||||
}
|
||||
tr, _ := NewLocalTranscriber(LocalConfig{ModelPath: dir, runOutput: runOutput, tempDir: func() (string, error) { return t.TempDir(), nil }})
|
||||
if _, err := tr.Transcribe(context.Background(), wavFixture()); err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
joined := strings.Join(gotArgs, " ")
|
||||
for _, want := range []string{"--encoder=", "--decoder=", "--joiner=", "--tokens="} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("missing flag %q in %q", want, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectWhisperArgs(t *testing.T) {
|
||||
// Sherpa Whisper models are named "<model>-encoder.onnx" (NOT "whisper-encoder")
|
||||
// and prefix the tokens file as "<model>-tokens.txt".
|
||||
dir := writeModelDir(t, "tiny.en-encoder.onnx", "tiny.en-encoder.int8.onnx",
|
||||
"tiny.en-decoder.onnx", "tiny.en-decoder.int8.onnx", "tiny.en-tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Whisper detection failed: %v", err)
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, "--whisper-encoder=") || !strings.Contains(joined, "--whisper-decoder=") {
|
||||
t.Errorf("expected whisper flags, got %q", joined)
|
||||
}
|
||||
// Must not emit transducer flags (Whisper has no joiner).
|
||||
if strings.Contains(joined, "--joiner=") || strings.Contains(joined, "--encoder=") {
|
||||
t.Errorf("Whisper must not be detected as a transducer: %q", joined)
|
||||
}
|
||||
// Consistent quantization: both int8 (preferred) or both fp32, never mixed.
|
||||
if strings.Contains(joined, "encoder.int8.onnx") != strings.Contains(joined, "decoder.int8.onnx") {
|
||||
t.Errorf("mixed encoder/decoder quantization: %q", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectTransducerPrefersConsistentInt8(t *testing.T) {
|
||||
dir := writeModelDir(t,
|
||||
"encoder-epoch-99.onnx", "encoder-epoch-99.int8.onnx",
|
||||
"decoder-epoch-99.onnx", "decoder-epoch-99.int8.onnx",
|
||||
"joiner-epoch-99.onnx", "joiner-epoch-99.int8.onnx", "tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
// All three int8 (preferred), consistently.
|
||||
for _, want := range []string{"--encoder=", "encoder-epoch-99.int8.onnx", "joiner-epoch-99.int8.onnx"} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("missing %q in %q", want, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectTransducerMixedQuantization(t *testing.T) {
|
||||
// Some models ship an int8 encoder+joiner with an fp32 decoder (e.g. the
|
||||
// x-asr streaming zipformers). Detection must accept that mix, not reject it.
|
||||
dir := writeModelDir(t, "encoder.int8.onnx", "decoder.onnx", "joiner.int8.onnx", "tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("mixed-quantization transducer should be detected, got %v", err)
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, "encoder.int8.onnx") || !strings.Contains(joined, "decoder.onnx") || !strings.Contains(joined, "joiner.int8.onnx") {
|
||||
t.Errorf("expected the actual mixed files, got %q", joined)
|
||||
}
|
||||
// The fp32 decoder must not be mistaken for an int8 one.
|
||||
if strings.Contains(joined, "decoder.int8") {
|
||||
t.Errorf("no int8 decoder exists; should use decoder.onnx: %q", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectSenseVoiceByDirName(t *testing.T) {
|
||||
dir := t.TempDir() // rename won't help; use a dir whose base contains sense-voice
|
||||
sv := dir + "/sherpa-onnx-sense-voice-zh-en"
|
||||
if err := os.MkdirAll(sv, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range []string{"model.int8.onnx", "tokens.txt"} {
|
||||
os.WriteFile(sv+"/"+f, []byte("x"), 0o600)
|
||||
}
|
||||
args, err := detectModelArgs(sv)
|
||||
if err != nil {
|
||||
t.Fatalf("SenseVoice detection failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(strings.Join(args, " "), "--sense-voice-model=") {
|
||||
t.Errorf("expected sense-voice flag, got %v", args)
|
||||
}
|
||||
}
|
||||
|
||||
// writeNamedModelDir creates a model directory whose base name carries the family
|
||||
// signature (how sherpa-onnx distributes single-file / CTC models), with the given
|
||||
// files, and returns its path.
|
||||
func writeNamedModelDir(t *testing.T, base string, files ...string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir() + "/" + base
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
if err := os.WriteFile(dir+"/"+f, []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestDetectSingleFileCTCFamilies(t *testing.T) {
|
||||
// One generic model.onnx + tokens, family told apart by the directory name.
|
||||
cases := []struct{ base, wantFlag string }{
|
||||
{"sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25", "--fire-red-asr-ctc="},
|
||||
{"sherpa-onnx-nemo-ctc-en-conformer-medium", "--nemo-ctc-model="},
|
||||
{"sherpa-onnx-zipformer-ctc-zh-2024", "--zipformer-ctc-model="},
|
||||
{"sherpa-onnx-telespeech-ctc-int8", "--telespeech-ctc="},
|
||||
{"sherpa-onnx-wenet-ctc-en", "--wenet-ctc-model="},
|
||||
{"sherpa-onnx-dolphin-base-ctc-multi-lang", "--dolphin-model="},
|
||||
{"sherpa-onnx-omnilingual-asr-ctc", "--omnilingual-asr-model="},
|
||||
{"sherpa-onnx-tdnn-yesno", "--tdnn-model="},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
dir := writeNamedModelDir(t, tc.base, "model.int8.onnx", "tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Errorf("%s: detection failed: %v", tc.base, err)
|
||||
continue
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, tc.wantFlag) || !strings.Contains(joined, "--tokens=") {
|
||||
t.Errorf("%s: got %q, want %s", tc.base, joined, tc.wantFlag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectEncoderDecoderFamiliesBeforeWhisper(t *testing.T) {
|
||||
// FireRedASR-AED / Canary share Whisper's encoder+decoder shape, so they must be
|
||||
// disambiguated by name — not fall through to --whisper-encoder.
|
||||
cases := []struct{ base, wantFlag string }{
|
||||
{"sherpa-onnx-fire-red-asr-large-zh_en", "--fire-red-asr-encoder="},
|
||||
{"sherpa-onnx-canary-180m-flash", "--canary-encoder="},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
dir := writeNamedModelDir(t, tc.base, "encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Errorf("%s: detection failed: %v", tc.base, err)
|
||||
continue
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, tc.wantFlag) {
|
||||
t.Errorf("%s: got %q, want %s", tc.base, joined, tc.wantFlag)
|
||||
}
|
||||
if strings.Contains(joined, "--whisper-") {
|
||||
t.Errorf("%s must not be detected as Whisper: %q", tc.base, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectUnknownFamilyFailsCleanly(t *testing.T) {
|
||||
// A model with tokens but no recognizable family → a clear SetupError, not a
|
||||
// crash or wrong flags.
|
||||
dir := writeModelDir(t, "some-random-model.onnx", "tokens.txt")
|
||||
_, err := detectModelArgs(dir)
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want a clean *SetupError, got %v", err)
|
||||
}
|
||||
if !strings.Contains(setupErr.Error(), "could not recognize") {
|
||||
t.Errorf("expected a recognizable failure message, got %q", setupErr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalTranscribeMissingTokens(t *testing.T) {
|
||||
dir := writeModelDir(t, "encoder.onnx", "decoder.onnx", "joiner.onnx") // no tokens.txt
|
||||
tr, _ := NewLocalTranscriber(LocalConfig{ModelPath: dir, runOutput: func(context.Context, string, ...string) ([]byte, error) { return nil, nil }, tempDir: func() (string, error) { return t.TempDir(), nil }})
|
||||
_, err := tr.Transcribe(context.Background(), wavFixture())
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing tokens, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalTranscribeMissingBinary(t *testing.T) {
|
||||
dir := writeModelDir(t, "preprocess.onnx", "encode.onnx", "uncached_decode.onnx", "cached_decode.onnx", "tokens.txt")
|
||||
runOutput := func(context.Context, string, ...string) ([]byte, error) { return nil, exec.ErrNotFound }
|
||||
tr, _ := NewLocalTranscriber(LocalConfig{ModelPath: dir, runOutput: runOutput, tempDir: func() (string, error) { return t.TempDir(), nil }})
|
||||
_, err := tr.Transcribe(context.Background(), wavFixture())
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing binary, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSherpaOutput(t *testing.T) {
|
||||
out := []byte("Started\nDone!\n\naudio.wav\n{\"lang\":\"\",\"text\":\" the quick brown fox \",\"tokens\":[]}\n----\nnum threads: 2")
|
||||
if got := parseSherpaOutput(out); got != "the quick brown fox" {
|
||||
t.Errorf("parsed %q", got)
|
||||
}
|
||||
if got := parseSherpaOutput([]byte("no json here")); got != "" {
|
||||
t.Errorf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLocalTranscriberRequiresModelPath(t *testing.T) {
|
||||
_, err := NewLocalTranscriber(LocalConfig{})
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// OpenAI Realtime transcription-only wants 24kHz PCM16 (§6b); the recorder is
|
||||
// built to match via the SampleRate hint.
|
||||
const openAIRealtimeSampleRate = 24000
|
||||
|
||||
// OpenAIRealtimeConfig configures the OpenAI Realtime transcription transcriber
|
||||
// (§6b, the credential-reuse streaming alternative). Reuses the OpenAI key.
|
||||
type OpenAIRealtimeConfig struct {
|
||||
APIKey string
|
||||
Model string // default "gpt-4o-transcribe"
|
||||
// BaseURL overrides the wss endpoint (tests point it at a fake server).
|
||||
BaseURL string
|
||||
}
|
||||
|
||||
type openAIRealtimeTranscriber struct {
|
||||
cfg OpenAIRealtimeConfig
|
||||
}
|
||||
|
||||
// NewOpenAIRealtimeTranscriber builds the OpenAI Realtime streaming transcriber.
|
||||
func NewOpenAIRealtimeTranscriber(cfg OpenAIRealtimeConfig) (Transcriber, error) {
|
||||
if strings.TrimSpace(cfg.APIKey) == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: "OpenAI API key",
|
||||
Hint: "set an OpenAI API key (OPENAI_API_KEY, or `zero auth`) to use OpenAI Realtime streaming dictation",
|
||||
}
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
cfg.Model = "gpt-4o-transcribe"
|
||||
}
|
||||
if cfg.BaseURL == "" {
|
||||
cfg.BaseURL = "wss://api.openai.com/v1/realtime?intent=transcription"
|
||||
}
|
||||
return &openAIRealtimeTranscriber{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (o *openAIRealtimeTranscriber) Transcribe(context.Context, []byte) (string, error) {
|
||||
return "", ErrStreamingUnsupported
|
||||
}
|
||||
|
||||
func (o *openAIRealtimeTranscriber) SampleRate() int { return openAIRealtimeSampleRate }
|
||||
|
||||
func (o *openAIRealtimeTranscriber) StreamTranscribe(ctx context.Context, chunks <-chan []byte, onPartial func(string, bool)) (string, error) {
|
||||
conn, _, err := websocket.Dial(ctx, o.cfg.BaseURL, &websocket.DialOptions{
|
||||
HTTPHeader: http.Header{
|
||||
"Authorization": {"Bearer " + o.cfg.APIKey},
|
||||
"OpenAI-Beta": {"realtime=v1"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("connecting to OpenAI Realtime: %w", err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
|
||||
// Configure a transcription-only session (pcm16 in, gpt-4o-transcribe).
|
||||
sessionUpdate := map[string]any{
|
||||
"type": "transcription_session.update",
|
||||
"session": map[string]any{
|
||||
"input_audio_format": "pcm16",
|
||||
"input_audio_transcription": map[string]any{
|
||||
"model": o.cfg.Model,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := writeJSON(ctx, conn, sessionUpdate); err != nil {
|
||||
return "", fmt.Errorf("configuring OpenAI Realtime session: %w", err)
|
||||
}
|
||||
|
||||
writeErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
for chunk := range chunks {
|
||||
appendMsg := map[string]any{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.StdEncoding.EncodeToString(chunk),
|
||||
}
|
||||
if err := writeJSON(ctx, conn, appendMsg); err != nil {
|
||||
writeErrCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
// Commit the buffered audio to force final transcription of the utterance.
|
||||
writeErrCh <- writeJSON(ctx, conn, map[string]any{"type": "input_audio_buffer.commit"})
|
||||
}()
|
||||
|
||||
// OpenAI deltas are incremental (append), unlike Deepgram/sherpa's cumulative
|
||||
// text. Accumulate completed segments plus the in-progress delta buffer.
|
||||
var finalized, current strings.Builder
|
||||
compose := func() string {
|
||||
return strings.TrimSpace(finalized.String() + current.String())
|
||||
}
|
||||
committed := false
|
||||
for {
|
||||
typ, data, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
if websocket.CloseStatus(err) == websocket.StatusNormalClosure {
|
||||
return compose(), nil
|
||||
}
|
||||
select {
|
||||
case werr := <-writeErrCh:
|
||||
if werr != nil {
|
||||
err = werr
|
||||
}
|
||||
default:
|
||||
}
|
||||
return compose(), fmt.Errorf("OpenAI Realtime stream error: %w", err)
|
||||
}
|
||||
if typ != websocket.MessageText {
|
||||
continue
|
||||
}
|
||||
evt := parseRealtimeEvent(data)
|
||||
switch evt.kind {
|
||||
case realtimeDelta:
|
||||
current.WriteString(evt.text)
|
||||
if onPartial != nil {
|
||||
onPartial(compose(), false)
|
||||
}
|
||||
case realtimeCompleted:
|
||||
// A completed item replaces the in-progress delta buffer with the
|
||||
// server's authoritative transcript for that segment.
|
||||
if evt.text != "" {
|
||||
if finalized.Len() > 0 {
|
||||
finalized.WriteString(" ")
|
||||
}
|
||||
finalized.WriteString(strings.TrimSpace(evt.text))
|
||||
}
|
||||
current.Reset()
|
||||
if onPartial != nil {
|
||||
onPartial(compose(), true)
|
||||
}
|
||||
// Once we've committed the buffer (user stopped) and the server has
|
||||
// returned a completed transcription, the utterance is done.
|
||||
if committed {
|
||||
_ = conn.Close(websocket.StatusNormalClosure, "")
|
||||
return compose(), nil
|
||||
}
|
||||
case realtimeCommitted:
|
||||
committed = true
|
||||
case realtimeError:
|
||||
return compose(), fmt.Errorf("OpenAI Realtime error: %s", evt.text)
|
||||
}
|
||||
// The writer signals commit completion out of band; observe it so a
|
||||
// stop with no further audio still flips `committed`.
|
||||
select {
|
||||
case werr := <-writeErrCh:
|
||||
if werr == nil {
|
||||
committed = true
|
||||
}
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type realtimeEventKind int
|
||||
|
||||
const (
|
||||
realtimeOther realtimeEventKind = iota
|
||||
realtimeDelta
|
||||
realtimeCompleted
|
||||
realtimeCommitted
|
||||
realtimeError
|
||||
)
|
||||
|
||||
type realtimeEvent struct {
|
||||
kind realtimeEventKind
|
||||
text string
|
||||
}
|
||||
|
||||
func parseRealtimeEvent(data []byte) realtimeEvent {
|
||||
var msg struct {
|
||||
Type string `json:"type"`
|
||||
Delta string `json:"delta"`
|
||||
Transcript string `json:"transcript"`
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &msg); err != nil {
|
||||
return realtimeEvent{kind: realtimeOther}
|
||||
}
|
||||
switch msg.Type {
|
||||
case "conversation.item.input_audio_transcription.delta":
|
||||
return realtimeEvent{kind: realtimeDelta, text: msg.Delta}
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
return realtimeEvent{kind: realtimeCompleted, text: msg.Transcript}
|
||||
case "input_audio_buffer.committed":
|
||||
return realtimeEvent{kind: realtimeCommitted}
|
||||
case "error":
|
||||
return realtimeEvent{kind: realtimeError, text: msg.Error.Message}
|
||||
}
|
||||
return realtimeEvent{kind: realtimeOther}
|
||||
}
|
||||
|
||||
func writeJSON(ctx context.Context, conn *websocket.Conn, v any) error {
|
||||
data, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conn.Write(ctx, websocket.MessageText, data)
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// wsTestServer runs handler over a real localhost websocket, returning a wss->ws
|
||||
// base URL the transcribers can dial. Exercising the real coder/websocket client
|
||||
// against a real server is the closest unit test to the wire protocol.
|
||||
func wsTestServer(t *testing.T, handler func(ctx context.Context, c *websocket.Conn)) string {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := websocket.Accept(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
handler(r.Context(), c)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return "ws" + strings.TrimPrefix(srv.URL, "http")
|
||||
}
|
||||
|
||||
func TestDeepgramStreamTranscribe(t *testing.T) {
|
||||
var gotAuth string
|
||||
url := wsTestServer(t, func(ctx context.Context, c *websocket.Conn) {
|
||||
defer c.Close(websocket.StatusNormalClosure, "")
|
||||
// Read audio frames until CloseStream, then emit interim + final results.
|
||||
for {
|
||||
typ, data, err := c.Read(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if typ == websocket.MessageText && strings.Contains(string(data), "CloseStream") {
|
||||
break
|
||||
}
|
||||
}
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"Results","is_final":false,"channel":{"alternatives":[{"transcript":"hello"}]}}`))
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"Results","is_final":true,"channel":{"alternatives":[{"transcript":"hello world"}]}}`))
|
||||
})
|
||||
|
||||
tr, err := NewDeepgramTranscriber(DeepgramConfig{APIKey: "dg-key", BaseURL: url})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = gotAuth
|
||||
|
||||
var partials []string
|
||||
chunks := make(chan []byte, 2)
|
||||
chunks <- make([]byte, 320)
|
||||
close(chunks)
|
||||
final, err := tr.StreamTranscribe(context.Background(), chunks, func(text string, _ bool) {
|
||||
partials = append(partials, text)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTranscribe: %v", err)
|
||||
}
|
||||
if final != "hello world" {
|
||||
t.Errorf("final = %q, want 'hello world'", final)
|
||||
}
|
||||
if len(partials) == 0 || partials[0] != "hello" {
|
||||
t.Errorf("expected an interim 'hello' partial, got %v", partials)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepgramSampleRateDefault(t *testing.T) {
|
||||
tr, _ := NewDeepgramTranscriber(DeepgramConfig{APIKey: "k"})
|
||||
if RequiredSampleRate(tr) != DefaultSampleRate {
|
||||
t.Errorf("Deepgram sample rate = %d, want %d", RequiredSampleRate(tr), DefaultSampleRate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepgramMissingKey(t *testing.T) {
|
||||
if _, err := NewDeepgramTranscriber(DeepgramConfig{}); err == nil {
|
||||
t.Fatal("expected setup error for missing key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIRealtimeStreamTranscribe(t *testing.T) {
|
||||
url := wsTestServer(t, func(ctx context.Context, c *websocket.Conn) {
|
||||
defer c.Close(websocket.StatusNormalClosure, "")
|
||||
committed := false
|
||||
for {
|
||||
typ, data, err := c.Read(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if typ != websocket.MessageText {
|
||||
continue
|
||||
}
|
||||
var msg struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
_ = json.Unmarshal(data, &msg)
|
||||
switch msg.Type {
|
||||
case "transcription_session.update":
|
||||
// Emit incremental deltas as if audio were flowing.
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"conversation.item.input_audio_transcription.delta","delta":"the "}`))
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"conversation.item.input_audio_transcription.delta","delta":"answer"}`))
|
||||
case "input_audio_buffer.commit":
|
||||
committed = true
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"input_audio_buffer.committed"}`))
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"conversation.item.input_audio_transcription.completed","transcript":"the answer"}`))
|
||||
}
|
||||
_ = committed
|
||||
}
|
||||
})
|
||||
|
||||
tr, err := NewOpenAIRealtimeTranscriber(OpenAIRealtimeConfig{APIKey: "sk-key", BaseURL: url})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if RequiredSampleRate(tr) != openAIRealtimeSampleRate {
|
||||
t.Errorf("OpenAI Realtime sample rate = %d, want %d", RequiredSampleRate(tr), openAIRealtimeSampleRate)
|
||||
}
|
||||
|
||||
chunks := make(chan []byte, 1)
|
||||
chunks <- make([]byte, 480)
|
||||
close(chunks)
|
||||
done := make(chan struct{})
|
||||
var final string
|
||||
var ferr error
|
||||
go func() {
|
||||
final, ferr = tr.StreamTranscribe(context.Background(), chunks, func(string, bool) {})
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("StreamTranscribe did not complete")
|
||||
}
|
||||
if ferr != nil {
|
||||
t.Fatalf("StreamTranscribe: %v", ferr)
|
||||
}
|
||||
if final != "the answer" {
|
||||
t.Errorf("final = %q, want 'the answer'", final)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalStreamingAccumulatesSegments is the regression for the "text vanishes
|
||||
// on stop" bug: the sherpa server reports text PER SEGMENT and resets it to "" when
|
||||
// it endpoints, so the transcriber must accumulate across segments rather than let
|
||||
// a new (empty) segment overwrite everything already recognized.
|
||||
func TestLocalStreamingAccumulatesSegments(t *testing.T) {
|
||||
url := wsTestServer(t, func(ctx context.Context, c *websocket.Conn) {
|
||||
defer c.Close(websocket.StatusNormalClosure, "")
|
||||
go func() { // drain the client's audio + "Done"
|
||||
for {
|
||||
if _, _, err := c.Read(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, m := range []string{
|
||||
`{"text":" hello","segment":0,"is_final":false}`,
|
||||
`{"text":" hello world","segment":0,"is_final":true}`,
|
||||
`{"text":"","segment":1,"is_final":false}`, // endpoint reset — must NOT wipe seg 0
|
||||
`{"text":" foo bar","segment":1,"is_final":true}`,
|
||||
`{"text":"","segment":2,"is_final":true}`, // trailing empty segment at stop
|
||||
} {
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(m))
|
||||
}
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte("Done!"))
|
||||
})
|
||||
|
||||
colon := strings.LastIndex(url, ":")
|
||||
port, err := strconv.Atoi(url[colon+1:])
|
||||
if err != nil {
|
||||
t.Fatalf("parsing test server port from %q: %v", url, err)
|
||||
}
|
||||
mgr := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
Port: port,
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) { return newFakeProcess(), nil, nil },
|
||||
healthCheck: func(context.Context, int) error { return nil },
|
||||
aliveCheck: func(int) bool { return false },
|
||||
})
|
||||
tr := NewLocalStreamingTranscriber(mgr)
|
||||
|
||||
chunks := make(chan []byte)
|
||||
close(chunks) // no audio; the writer immediately sends "Done"
|
||||
var partials []string
|
||||
text, err := tr.StreamTranscribe(context.Background(), chunks, func(s string, _ bool) {
|
||||
partials = append(partials, s)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTranscribe: %v", err)
|
||||
}
|
||||
if text != "hello world foo bar" {
|
||||
t.Errorf("final text = %q, want %q", text, "hello world foo bar")
|
||||
}
|
||||
// Once text has appeared, no later partial may go empty — that would be the
|
||||
// segment-reset wiping the transcript, the exact bug this guards.
|
||||
seen := false
|
||||
for _, p := range partials {
|
||||
if p != "" {
|
||||
seen = true
|
||||
}
|
||||
if seen && p == "" {
|
||||
t.Errorf("transcript went empty after text appeared; partials=%v", partials)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDeepgramResultIgnoresNonResults(t *testing.T) {
|
||||
if _, _, ok := parseDeepgramResult([]byte(`{"type":"Metadata"}`)); ok {
|
||||
t.Error("Metadata messages should be ignored")
|
||||
}
|
||||
tr, final, ok := parseDeepgramResult([]byte(`{"type":"Results","is_final":true,"channel":{"alternatives":[{"transcript":"hi"}]}}`))
|
||||
if !ok || tr != "hi" || !final {
|
||||
t.Errorf("parse = (%q,%v,%v)", tr, final, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRealtimeEvent(t *testing.T) {
|
||||
if e := parseRealtimeEvent([]byte(`{"type":"conversation.item.input_audio_transcription.delta","delta":"x"}`)); e.kind != realtimeDelta || e.text != "x" {
|
||||
t.Errorf("delta parse: %+v", e)
|
||||
}
|
||||
if e := parseRealtimeEvent([]byte(`{"type":"error","error":{"message":"boom"}}`)); e.kind != realtimeError || e.text != "boom" {
|
||||
t.Errorf("error parse: %+v", e)
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/modelregistry"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
"github.com/Gitlawb/zero/internal/providerhealth"
|
||||
"github.com/Gitlawb/zero/internal/redaction"
|
||||
)
|
||||
@@ -143,12 +144,19 @@ func providerConfigCheck(profile config.ProviderProfile) Check {
|
||||
// Report credential PRESENCE, never the value. Reported under a non-sensitive
|
||||
// key ("credentialConfigured"): the prior "apiKey" key was itself sensitive, so
|
||||
// check()'s redaction scrubbed the indicator to [REDACTED] — making "set"/"not
|
||||
// set" invisible. A profile can authenticate via a raw auth-header value instead
|
||||
// of APIKey, and both are trimmed so a whitespace-only value reads "not set"
|
||||
// (matching ProviderSnapshot.APIKeySet).
|
||||
// set" invisible. HasConfiguredCredential is the shared definition of
|
||||
// "key-authed" (inline key, raw auth header, or a key in the encrypted
|
||||
// credential store), matching ProviderSnapshot.APIKeySet — checking only the
|
||||
// inline fields made doctor and `zero providers list` disagree about the
|
||||
// same stored-key profile. A stored OAuth login counts too: a keyless
|
||||
// token-login provider (e.g. ChatGPT) is fully able to make requests and
|
||||
// must not fail the one tool meant to verify setup.
|
||||
credential := "not set"
|
||||
if strings.TrimSpace(profile.APIKey) != "" || strings.TrimSpace(profile.AuthHeaderValue) != "" {
|
||||
switch {
|
||||
case profile.HasConfiguredCredential():
|
||||
credential = "set"
|
||||
case providerHasStoredOAuthLogin(profile):
|
||||
credential = "oauth login"
|
||||
}
|
||||
details := map[string]any{
|
||||
"name": profile.Name,
|
||||
@@ -169,6 +177,22 @@ func providerConfigCheck(profile config.ProviderProfile) Check {
|
||||
return check("provider.config", "Provider config", StatusPass, fmt.Sprintf("Provider config loaded for %s.", providerName(profile)), details)
|
||||
}
|
||||
|
||||
// providerHasStoredOAuthLogin reports whether the token store holds a login for
|
||||
// any of the profile's candidates — the same candidate set and normalized-key
|
||||
// matching the runtime resolver uses. Errors degrade to false.
|
||||
func providerHasStoredOAuthLogin(profile config.ProviderProfile) bool {
|
||||
candidates := profile.OAuthLoginCandidates()
|
||||
if len(candidates) == 0 {
|
||||
return false
|
||||
}
|
||||
store, err := oauth.NewStore(oauth.StoreOptions{})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, _, ok := oauth.FirstStored(store, candidates)
|
||||
return ok
|
||||
}
|
||||
|
||||
// localProviderBaseURL reports whether the configured base_url is a loopback host
|
||||
// (a keyless local provider like Ollama/LM Studio), which needs no API key.
|
||||
func localProviderBaseURL(baseURL string) bool {
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
// Package errhint classifies provider/model failures into a few user-actionable
|
||||
// categories and turns them into a one-line "next step" hint.
|
||||
//
|
||||
// Provider errors already arrive with a classified string prefix from
|
||||
// providerio.ClassifiedError ("auth error:", "rate limit error:", …); lower-level
|
||||
// failures (DNS, TLS, timeouts, context-length) arrive as raw driver or library
|
||||
// messages. Classify matches both so the interactive error row (TUI) and the
|
||||
// `zero exec` provider-error path (CLI) can append one concrete next step instead
|
||||
// of dumping an identical red blob for every failure mode.
|
||||
package errhint
|
||||
|
||||
import "strings"
|
||||
|
||||
// Category buckets a provider/model failure into a small set of classes that each
|
||||
// map to a distinct recovery action.
|
||||
type Category int
|
||||
|
||||
const (
|
||||
// Unknown means the error didn't match any known signature; callers should
|
||||
// emit no hint rather than guess.
|
||||
Unknown Category = iota
|
||||
Auth
|
||||
RateLimit
|
||||
Connectivity
|
||||
ModelNotFound
|
||||
ContextOverflow
|
||||
)
|
||||
|
||||
// providerMarkers are the prefixes the provider layer attaches to every
|
||||
// provider-originated failure: providerio.ClassifiedError emits "auth error:",
|
||||
// "rate limit error:", "provider error:", and "provider request error:", and the
|
||||
// streaming paths wrap transport/decoding failures as "provider stream error:".
|
||||
// A UI surface's error can also be a *local* failure (a tool's "permission
|
||||
// denied", a "file does not exist", a config error), so Classify only proceeds
|
||||
// past this gate for messages that are recognizably from the provider — otherwise
|
||||
// a broad substring like "does not exist" would attach a bogus /model hint to an
|
||||
// unrelated local error.
|
||||
var providerMarkers = []string{
|
||||
"auth error:",
|
||||
"rate limit error:",
|
||||
"provider error:",
|
||||
"provider request error:",
|
||||
"provider stream error:",
|
||||
}
|
||||
|
||||
// Classify buckets err by scanning its message for known signatures. It is a
|
||||
// deliberately conservative string heuristic — the provider layer has already
|
||||
// discarded the numeric HTTP status by the time the error reaches a UI surface,
|
||||
// so the message is all we have. It first gates on a provider-origin marker (see
|
||||
// providerMarkers) so local failures never draw a provider hint, then sub-classifies.
|
||||
// Order matters: more specific signatures are tested before broader ones (e.g.
|
||||
// "context length" as overflow before the generic "timeout" as connectivity).
|
||||
func Classify(err error) Category {
|
||||
if err == nil {
|
||||
return Unknown
|
||||
}
|
||||
m := strings.ToLower(err.Error())
|
||||
if !containsAny(m, providerMarkers...) {
|
||||
return Unknown
|
||||
}
|
||||
switch {
|
||||
case containsAny(m, "auth error:", "unauthorized", "api key", "api_key", "invalid_api_key",
|
||||
"authentication", "permission denied", "forbidden") || containsStatusCode(m, "401", "403"):
|
||||
return Auth
|
||||
case containsAny(m, "rate limit", "rate_limit", "too many requests", "quota",
|
||||
"resource_exhausted", "overloaded") || containsStatusCode(m, "429", "529"):
|
||||
return RateLimit
|
||||
case containsAny(m, "context length", "context window", "maximum context", "context_length_exceeded",
|
||||
"too many tokens", "prompt is too long", "reduce the length", "maximum context length"):
|
||||
return ContextOverflow
|
||||
case containsAny(m, "model not found", "model_not_found", "does not exist", "unknown model",
|
||||
"no such model", "unsupported model", "invalid model", "model is not"):
|
||||
return ModelNotFound
|
||||
case containsAny(m, "dial tcp", "no such host", "connection refused", "network is unreachable",
|
||||
"i/o timeout", "context deadline exceeded", "tls handshake", "connection reset",
|
||||
"unexpected eof", "lookup ", "timeout"):
|
||||
return Connectivity
|
||||
default:
|
||||
return Unknown
|
||||
}
|
||||
}
|
||||
|
||||
// TUIHint returns a one-line hint referencing interactive slash commands, or ""
|
||||
// when the category is Unknown. Meant to sit under the raw error in the live
|
||||
// error row.
|
||||
func TUIHint(err error) string {
|
||||
switch Classify(err) {
|
||||
case Auth:
|
||||
return "API key rejected — run /provider to re-check your credentials"
|
||||
case RateLimit:
|
||||
return "Rate limited — wait a moment, or switch model with /model"
|
||||
case Connectivity:
|
||||
return "Can't reach the provider — run /doctor --connectivity"
|
||||
case ModelNotFound:
|
||||
return "Model unavailable — pick another with /model"
|
||||
case ContextOverflow:
|
||||
return "Context window full — run /compact to free space"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// CLIHint returns a one-line hint referencing `zero …` subcommands, or "" when the
|
||||
// category is Unknown. Meant for the non-interactive `zero exec` error path, where
|
||||
// slash commands don't apply.
|
||||
func CLIHint(err error) string {
|
||||
switch Classify(err) {
|
||||
case Auth:
|
||||
return "API key rejected — run `zero auth` or set the provider's API key, then retry"
|
||||
case RateLimit:
|
||||
return "Rate limited — wait a moment, or switch model with --model"
|
||||
case Connectivity:
|
||||
return "Can't reach the provider — run `zero doctor`"
|
||||
case ModelNotFound:
|
||||
return "Model unavailable — run `zero doctor` or pick another with --model"
|
||||
case ContextOverflow:
|
||||
return "Context window full — shorten the prompt or start a fresh session"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func containsAny(haystack string, needles ...string) bool {
|
||||
for _, n := range needles {
|
||||
if strings.Contains(haystack, n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// containsStatusCode reports whether haystack contains any of the given HTTP
|
||||
// status codes as a standalone number — not embedded in a longer digit run like
|
||||
// "completed in 4290ms" or "request id 14015" — so an incidental number can't be
|
||||
// mis-bucketed as an auth/rate-limit failure.
|
||||
func containsStatusCode(haystack string, codes ...string) bool {
|
||||
return HasStatusCode(haystack, codes...)
|
||||
}
|
||||
|
||||
// HasStatusCode reports whether haystack contains any of the given HTTP status
|
||||
// codes as a standalone number — not embedded in a longer digit run like
|
||||
// "completed in 4290ms" or "request id 14015". Exported so other packages (e.g.
|
||||
// the reconnect classifier) can gate on a status code without re-implementing
|
||||
// the digit-boundary check.
|
||||
func HasStatusCode(haystack string, codes ...string) bool {
|
||||
for _, code := range codes {
|
||||
for from := 0; ; {
|
||||
rel := strings.Index(haystack[from:], code)
|
||||
if rel < 0 {
|
||||
break
|
||||
}
|
||||
pos := from + rel
|
||||
beforeOK := pos == 0 || !isASCIIDigit(haystack[pos-1])
|
||||
end := pos + len(code)
|
||||
afterOK := end >= len(haystack) || !isASCIIDigit(haystack[end])
|
||||
if beforeOK && afterOK {
|
||||
return true
|
||||
}
|
||||
from = pos + 1
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isASCIIDigit(b byte) bool {
|
||||
return b >= '0' && b <= '9'
|
||||
}
|
||||
Loaded 100 of 273 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user