mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 09:49:34 -10:00
Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ab26dbf129 | ||
|
|
6eb4b89f3f | ||
|
|
6ec77d62f5 | ||
|
|
25653f686c | ||
|
|
c7346fbcf0 | ||
|
|
d0c4e62cd4 | ||
|
|
302f58bb5f | ||
|
|
ba99fa8d48 | ||
|
|
1805a8ce09 | ||
|
|
8b248f4e11 | ||
|
|
4f1f94f26a | ||
|
|
6247ee8f9c | ||
|
|
87158a1c90 | ||
|
|
16550569c1 | ||
|
|
008bc9b3f3 | ||
|
|
fd69233e33 | ||
|
|
3f0503bc23 | ||
|
|
f951aee65d | ||
|
|
82ecac5281 | ||
|
|
387fe67ee7 |
No files matched your search
@@ -27,6 +27,10 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
.cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Go build/module caches (GOCACHE/GOMODCACHE pointed here for local runs)
|
||||
.gocache/
|
||||
.gomodcache/
|
||||
|
||||
# IntelliJ based IDEs
|
||||
.idea
|
||||
|
||||
@@ -37,7 +41,10 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
/zero
|
||||
/zero.exe
|
||||
/zero-windows-command-runner.exe
|
||||
/zero-windows-sandbox-setup.exe
|
||||
/zero-skills
|
||||
/.zero-binary-version
|
||||
/snake-game.html
|
||||
|
||||
# Superpowers skill artifacts (brainstorm mockups, specs, plans) — local only
|
||||
@@ -47,3 +54,4 @@ docs/superpowers/
|
||||
# Termux/Android build artifacts
|
||||
zero-termux-arm64
|
||||
zero-linux-sandbox
|
||||
zero-seccomp
|
||||
@@ -1 +1 @@
|
||||
{".":"0.1.0"}
|
||||
{".":"0.2.0"}
|
||||
@@ -132,7 +132,7 @@ description: Run the project's benchmark suite and summarize the deltas.
|
||||
3. Report any regression > 5% with the function name and the previous value.
|
||||
```
|
||||
|
||||
Only `name` and `description` are recognized in the frontmatter today. The `name` defaults to the directory name. If two skills declare the same name, the one in the lexicographically-first directory wins — duplicates are dropped silently. Plugin-declared skills (section 6) are not yet merged into the loader, so a `skills:` entry inside a plugin's `plugin.json` is not visible to the `skill` tool today.
|
||||
Only `name` and `description` are recognized in the frontmatter today. The `name` defaults to the directory name. Within a single skills root, duplicate names are resolved by lexicographic directory order. During an agent run, Zero loads the default skills directory before plugin skill roots; earlier roots win name collisions silently. Plugin-declared skills (section 6) are merged into the active agent run at plugin activation time, so bundled skills appear in the available skills list and can be loaded with the `skill` tool.
|
||||
|
||||
The `skill` core tool lets the agent load any discovered skill by name.
|
||||
|
||||
@@ -282,7 +282,7 @@ A plugin is enabled by being present in the plugins directory and disabled by re
|
||||
|
||||
Plugin commands run with the plugin directory as their working directory. Use relative paths; the loader resolves them at activation time.
|
||||
|
||||
> **Roadmap.** An in-UI plugins manager (browse, install, enable / disable) is on the backlog. Today you use the `zero plugins` CLI subcommands above. Skills declared inside a plugin's `plugin.json` are not yet merged into the `skill` tool's discovery (see section 3).
|
||||
> **Roadmap.** An in-UI plugins manager (browse, install, enable / disable) is on the backlog. Today you use the `zero plugins` CLI subcommands above.
|
||||
|
||||
## 7. Configuration locations
|
||||
|
||||
|
||||
@@ -6,6 +6,71 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
aims to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html) once the first release is
|
||||
tagged. Until then, source builds report the version `dev`.
|
||||
|
||||
## [0.2.0](https://github.com/Gitlawb/zero/compare/v0.1.0...v0.2.0) (2026-07-06)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add --auto flag for LLM-generated commit messages ([#423](https://github.com/Gitlawb/zero/issues/423)) ([b0abde7](https://github.com/Gitlawb/zero/commit/b0abde7d0697e808480cd59d69a6f4d0c6320475))
|
||||
* add zero changes push and pr subcommands, and extra repo-info metrics ([#391](https://github.com/Gitlawb/zero/issues/391)) ([2312abe](https://github.com/Gitlawb/zero/commit/2312abe5ddd95f4c6ef373cfb61cc03092f48cdd))
|
||||
* agent quality, caching, retry, and tooling upgrades ([#506](https://github.com/Gitlawb/zero/issues/506)) ([3c81fea](https://github.com/Gitlawb/zero/commit/3c81fea22873ee3df7fc97b10cb4f77792706c4b))
|
||||
* **agent:** curb over-engineering the solution in the editing discipline ([#517](https://github.com/Gitlawb/zero/issues/517)) ([f4c998a](https://github.com/Gitlawb/zero/commit/f4c998ac30c4f07ff313a2d706791e857293be49)), closes [#516](https://github.com/Gitlawb/zero/issues/516)
|
||||
* **agent:** inject per-user config.UserConfigDir()/zero/ZERO.md guidelines into system prompt ([#475](https://github.com/Gitlawb/zero/issues/475)) ([7b10aab](https://github.com/Gitlawb/zero/commit/7b10aab74bf14a01166b2cea22deab79bba9850b))
|
||||
* **openai:** forward prompt_cache_key for server-side prefix cache routing ([#515](https://github.com/Gitlawb/zero/issues/515)) ([87e7e69](https://github.com/Gitlawb/zero/commit/87e7e69afd18b5539579856f3a61c6a95bc445ae))
|
||||
* **providers:** add `zero providers models` to discover a provider's models ([#386](https://github.com/Gitlawb/zero/issues/386)) ([0bc8074](https://github.com/Gitlawb/zero/commit/0bc8074c97b0310e4a9d70c3f967003ee5e8a59f))
|
||||
* **providers:** add KiloCode and OpenCode provider support ([#388](https://github.com/Gitlawb/zero/issues/388)) ([b1ccb6d](https://github.com/Gitlawb/zero/commit/b1ccb6d9c1875377f5e5ea81a1304edd1e41ab4f))
|
||||
* **providers:** add Meituan LongCat catalog preset ([#424](https://github.com/Gitlawb/zero/issues/424)) ([b4275e3](https://github.com/Gitlawb/zero/commit/b4275e350472b2490212bf814709819d354c1216))
|
||||
* **providers:** split minimax zai into intl cn ([#398](https://github.com/Gitlawb/zero/issues/398)) ([aaad4d2](https://github.com/Gitlawb/zero/commit/aaad4d271270f41af837b6f3b60ae80beba0c645))
|
||||
* require manual approval before npm publish + drop release-as pin ([#369](https://github.com/Gitlawb/zero/issues/369)) ([bd89a1f](https://github.com/Gitlawb/zero/commit/bd89a1f451643c1b65ec803070abc7b116631ebe))
|
||||
* **sandbox:** unelevated Windows fallback tier instead of prompts-only degrade ([#427](https://github.com/Gitlawb/zero/issues/427)) ([b9ddd6f](https://github.com/Gitlawb/zero/commit/b9ddd6f42138312a1fee8d8bb67c46c8eb1dea2f))
|
||||
* support shift enter for composer newlines ([#462](https://github.com/Gitlawb/zero/issues/462)) ([daf65e0](https://github.com/Gitlawb/zero/commit/daf65e0af9a040314d4ab337b0ad59c55416b7bc))
|
||||
* **tui:** /loop — repeat a prompt or command on an interval or self-paced ([#502](https://github.com/Gitlawb/zero/issues/502)) ([387fe67](https://github.com/Gitlawb/zero/commit/387fe67ee7cd81317c9c969f5906a4437080fea3))
|
||||
* **tui:** add search/filter to provider picker in setup wizard ([#400](https://github.com/Gitlawb/zero/issues/400)) ([2fcea71](https://github.com/Gitlawb/zero/commit/2fcea71778d23e050c93409c471aef45b68c1621))
|
||||
* **update:** add zero upgrade command to apply self-updates ([#461](https://github.com/Gitlawb/zero/issues/461)) ([5f36349](https://github.com/Gitlawb/zero/commit/5f36349c1884e81fa9bc66bb5fe813b627e897b7))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **action:** keep provider key scoped to zero step ([#448](https://github.com/Gitlawb/zero/issues/448)) ([407a927](https://github.com/Gitlawb/zero/commit/407a92739ff508cba32d2c12b3f36f0efcdd54c3))
|
||||
* add android platform support for Termux npm install ([#455](https://github.com/Gitlawb/zero/issues/455)) ([9bd93c6](https://github.com/Gitlawb/zero/commit/9bd93c62f8d57fb74057284aa66a1b6e1429dcdd)), closes [#449](https://github.com/Gitlawb/zero/issues/449)
|
||||
* **agent:** reject a malformed additional_permissions payload before prompting ([#453](https://github.com/Gitlawb/zero/issues/453)) ([e4f760e](https://github.com/Gitlawb/zero/commit/e4f760ee8bd57299cd2fcb37e8e23130037c2607))
|
||||
* allow non-TLS connections to private-network provider endpoints ([#444](https://github.com/Gitlawb/zero/issues/444)) ([1d86384](https://github.com/Gitlawb/zero/commit/1d8638466ca31517eb9db2b9353d3dce1cbeeabc))
|
||||
* **auth:** route zero auth login chatgpt to the dedicated ChatGPT flow ([#443](https://github.com/Gitlawb/zero/issues/443)) ([305a62c](https://github.com/Gitlawb/zero/commit/305a62c954ca6cec00bc58d5398f933415156aff))
|
||||
* **config:** fall back to a usable saved provider instead of forcing full re-onboarding ([#410](https://github.com/Gitlawb/zero/issues/410)) ([c60ad87](https://github.com/Gitlawb/zero/commit/c60ad8729f79bb841114d352ee2d2fe29d5d0e41))
|
||||
* **config:** let a gateway ANTHROPIC_BASE_URL resolve as anthropic-compatible ([#497](https://github.com/Gitlawb/zero/issues/497)) ([30dd7c3](https://github.com/Gitlawb/zero/commit/30dd7c3112ad22d42fa12b5addd4e38f4beda42a)), closes [#479](https://github.com/Gitlawb/zero/issues/479)
|
||||
* **config:** unbrick first-run setup — default google/anthropic models, enter setup on fixable config errors ([#385](https://github.com/Gitlawb/zero/issues/385)) ([72eed06](https://github.com/Gitlawb/zero/commit/72eed06b4f94c43d75d31fe54a58d2f566de059e))
|
||||
* **config:** use ~/.config on macOS and enter setup when no provider ([#371](https://github.com/Gitlawb/zero/issues/371)) ([#372](https://github.com/Gitlawb/zero/issues/372)) ([027a8f2](https://github.com/Gitlawb/zero/commit/027a8f2768b17b89f5c8270887f156e2ccda69ea))
|
||||
* **docs:** rename AGENTS.MD > AGENTS.md ([#438](https://github.com/Gitlawb/zero/issues/438)) ([4266baf](https://github.com/Gitlawb/zero/commit/4266baf222df583ed2078b776687f12d496475b5))
|
||||
* **gemini:** strip unsupported JSON Schema fields from tool declarations ([#374](https://github.com/Gitlawb/zero/issues/374)) ([39e7100](https://github.com/Gitlawb/zero/commit/39e7100674150144a1152e3110c64c7cf0321d64)), closes [#373](https://github.com/Gitlawb/zero/issues/373)
|
||||
* **install:** persist install dir to user PATH on Windows ([#407](https://github.com/Gitlawb/zero/issues/407)) ([bdb1b0e](https://github.com/Gitlawb/zero/commit/bdb1b0ecd15859b1712a6037d296dace7f9c3c3f))
|
||||
* **mcp:** block cross-origin credential redirects ([#396](https://github.com/Gitlawb/zero/issues/396)) ([f915f70](https://github.com/Gitlawb/zero/commit/f915f70e5a3096e2419fa8d961a0f84a626fa4a9))
|
||||
* **oauth:** treat Windows ERROR_ACCESS_DENIED as lock contention in createSecretFile ([#445](https://github.com/Gitlawb/zero/issues/445)) ([d05e914](https://github.com/Gitlawb/zero/commit/d05e9148a7f79f67d1d3c31fca2775f21fbd331e))
|
||||
* **openai:** handle Ollama reasoning stream deltas ([#486](https://github.com/Gitlawb/zero/issues/486)) ([f6c0606](https://github.com/Gitlawb/zero/commit/f6c060631e18e082dda24cc4dc0903c31c2120d6))
|
||||
* preserve conversation context in exec prompts ([#460](https://github.com/Gitlawb/zero/issues/460)) ([949ee43](https://github.com/Gitlawb/zero/commit/949ee43f71e5cb7fab4695c5cb7b442fe4ecfbf7))
|
||||
* **provider-wizard:** allow multiple custom OpenAI-compatible providers ([#403](https://github.com/Gitlawb/zero/issues/403)) ([3fbbd28](https://github.com/Gitlawb/zero/commit/3fbbd28e4c586822cc4312c86232d94befe56e87))
|
||||
* **sandbox:** fix nested pipe creation under the Windows restricted token ([#456](https://github.com/Gitlawb/zero/issues/456)) ([563a6db](https://github.com/Gitlawb/zero/commit/563a6dbe91e65d5daeefd7626e8a77e30a6d8fb2))
|
||||
* **sandbox:** gate /tmp test assertions on GOOS, not path existence ([#426](https://github.com/Gitlawb/zero/issues/426)) ([f653dca](https://github.com/Gitlawb/zero/commit/f653dcac363fb69ad7be5b35e6e0fa6d2bce476d))
|
||||
* **sandbox:** self-heal a corrupt unelevated setup marker ([#437](https://github.com/Gitlawb/zero/issues/437)) ([8d0c5fe](https://github.com/Gitlawb/zero/commit/8d0c5feccb8bdbfb015df0508aa6e3bcbd1fd0e8))
|
||||
* **specialist:** cap max specialist nesting depth ([#491](https://github.com/Gitlawb/zero/issues/491)) ([177442c](https://github.com/Gitlawb/zero/commit/177442cfe4015bd8df04cc9894f98b468ee796d4))
|
||||
* Termux/Android support — PRoot scroll, SIGSYS sandbox, build docs ([#509](https://github.com/Gitlawb/zero/issues/509)) ([0f69d99](https://github.com/Gitlawb/zero/commit/0f69d995e9b586b774f66c066b21abab5e03024a))
|
||||
* **tools:** Block MSYS coreutils under Windows sandbox ([#476](https://github.com/Gitlawb/zero/issues/476)) ([81aad58](https://github.com/Gitlawb/zero/commit/81aad58d97839e51c068e2f08907618991fdc3fb))
|
||||
* **tools:** CRLF line ending mismatch in edit_file tool on Windows ([#378](https://github.com/Gitlawb/zero/issues/378)) ([33dc7ae](https://github.com/Gitlawb/zero/commit/33dc7ae2cc82c5389675531e1416856dae7151ce))
|
||||
* **tools:** fix cmd.exe /S/C corrupting commands with embedded quotes ([#465](https://github.com/Gitlawb/zero/issues/465)) ([190241b](https://github.com/Gitlawb/zero/commit/190241bd593f43211b766e0b13c8e89802d4bb37))
|
||||
* **tools:** flag piped POSIX utilities before running on Windows ([#412](https://github.com/Gitlawb/zero/issues/412)) ([5658a36](https://github.com/Gitlawb/zero/commit/5658a366274fc59a9d5336b06a21019c9c25cbf1))
|
||||
* **tools:** make grep and glob respect run cancellation ([#464](https://github.com/Gitlawb/zero/issues/464)) ([ba6c026](https://github.com/Gitlawb/zero/commit/ba6c0264697b7d7ed479f6e782fba9700a481e3d))
|
||||
* **tools:** require permission before web_search requests ([#382](https://github.com/Gitlawb/zero/issues/382)) ([960db96](https://github.com/Gitlawb/zero/commit/960db9660e4e31dc588fe8f7d6f116ff5e225566))
|
||||
* **tui:** compose help overlay through the viewport overlay pipeline ([#421](https://github.com/Gitlawb/zero/issues/421)) ([5b2b4de](https://github.com/Gitlawb/zero/commit/5b2b4dea1aaf9e0f68baa25e97e83296fb17b1a2))
|
||||
* **tui:** keep the profile name on /model switch so the stored key resolves ([#441](https://github.com/Gitlawb/zero/issues/441)) ([9134148](https://github.com/Gitlawb/zero/commit/9134148f4df3e4e556fba6c2f8babfdf6fcfeee1)), closes [#440](https://github.com/Gitlawb/zero/issues/440)
|
||||
* **tui:** resolve every permission request so the agent can't deadlock ([#397](https://github.com/Gitlawb/zero/issues/397)) ([952788f](https://github.com/Gitlawb/zero/commit/952788f72d32957659fe004521fcc8372b9ba9b4))
|
||||
* **tui:** show an M suffix for million-scale token counts ([#457](https://github.com/Gitlawb/zero/issues/457)) ([0562e3b](https://github.com/Gitlawb/zero/commit/0562e3bef7df2328610a48a1e81632a8da4aec64))
|
||||
* **tui:** title /model rows by model name, not the catalog description ([#395](https://github.com/Gitlawb/zero/issues/395)) ([cdf9d83](https://github.com/Gitlawb/zero/commit/cdf9d839ae57a729f292f36f7c5b0c67b41b288d))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* cache TUI model registry ([#496](https://github.com/Gitlawb/zero/issues/496)) ([e7d88b4](https://github.com/Gitlawb/zero/commit/e7d88b4b518049733da25a8447c00144bd1da518))
|
||||
* universal tool-output ceiling with spill + async post-edit diagnostics ([#518](https://github.com/Gitlawb/zero/issues/518)) ([95ccd5b](https://github.com/Gitlawb/zero/commit/95ccd5bc327f6fb464ff0239f7229de789f578dc))
|
||||
|
||||
## 0.1.0 (2026-07-02)
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
module github.com/Gitlawb/zero
|
||||
|
||||
go 1.25.0
|
||||
|
||||
toolchain go1.26.4
|
||||
go 1.26.4
|
||||
|
||||
require (
|
||||
charm.land/bubbles/v2 v2.1.0
|
||||
@@ -14,6 +12,7 @@ require (
|
||||
github.com/charmbracelet/colorprofile v0.4.3
|
||||
github.com/charmbracelet/x/ansi v0.11.7
|
||||
github.com/charmbracelet/x/term v0.2.2
|
||||
github.com/coder/websocket v1.8.15
|
||||
github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
|
||||
golang.org/x/sys v0.46.0
|
||||
mvdan.cc/sh/v3 v3.13.1
|
||||
|
||||
@@ -32,6 +32,8 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE
|
||||
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
|
||||
|
||||
+54
-21
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -159,8 +160,14 @@ func (a *Agent) handleSessionLoad(_ context.Context, params json.RawMessage) (an
|
||||
// Load history BEFORE publishing the session so no concurrent prompt observes
|
||||
// a half-initialized session (registerSession sets history under the lock and
|
||||
// reuses an already-live session rather than orphaning its in-flight turn).
|
||||
history := a.loadHistory(meta.SessionID)
|
||||
history, historyErr := a.loadHistory(meta.SessionID)
|
||||
sess := a.registerSession(meta.SessionID, root, history)
|
||||
a.warnPersistence(
|
||||
¬ifier{conn: a.conn, sessionID: sess.id},
|
||||
"load session history",
|
||||
"Could not load session history. The session is open, but earlier turns may be missing until storage recovers.",
|
||||
historyErr,
|
||||
)
|
||||
return LoadSessionResult{
|
||||
Modes: a.modeState(sess),
|
||||
}, nil
|
||||
@@ -253,7 +260,14 @@ func (a *Agent) runTurn(ctx context.Context, sess *acpSession, userText string,
|
||||
if stopErr != nil {
|
||||
return "", RPCError(codeInternalError, stopErr.Error())
|
||||
}
|
||||
a.persistTurn(sess, userText, result.FinalAnswer)
|
||||
if err := a.persistTurn(sess, userText, result.FinalAnswer); err != nil {
|
||||
a.warnPersistence(
|
||||
note,
|
||||
"save session history",
|
||||
"Could not save session history. This turn is available in memory, but future resume may miss it until storage recovers.",
|
||||
err,
|
||||
)
|
||||
}
|
||||
return reason, nil
|
||||
}
|
||||
|
||||
@@ -386,29 +400,34 @@ func (a *Agent) modeState(s *acpSession) *SessionModeState {
|
||||
|
||||
// ---- persistence + continuity ----
|
||||
|
||||
func (a *Agent) persistTurn(sess *acpSession, user, assistant string) {
|
||||
if a.deps.Store != nil {
|
||||
_, _ = a.deps.Store.AppendEvent(sess.id, sessions.AppendEventInput{
|
||||
Type: sessions.EventMessage,
|
||||
Payload: map[string]any{"role": "user", "content": user},
|
||||
})
|
||||
if assistant != "" {
|
||||
_, _ = a.deps.Store.AppendEvent(sess.id, sessions.AppendEventInput{
|
||||
Type: sessions.EventMessage,
|
||||
Payload: map[string]any{"role": "assistant", "content": assistant},
|
||||
})
|
||||
}
|
||||
}
|
||||
sess.appendHistory(turnRecord{user: user, assistant: assistant})
|
||||
}
|
||||
|
||||
func (a *Agent) loadHistory(sessionID string) []turnRecord {
|
||||
func (a *Agent) persistTurn(sess *acpSession, user, assistant string) error {
|
||||
defer sess.appendHistory(turnRecord{user: user, assistant: assistant})
|
||||
if a.deps.Store == nil {
|
||||
return nil
|
||||
}
|
||||
events := []sessions.AppendEventInput{
|
||||
{
|
||||
Type: sessions.EventMessage,
|
||||
Payload: map[string]any{"role": "user", "content": user},
|
||||
},
|
||||
}
|
||||
if assistant != "" {
|
||||
events = append(events, sessions.AppendEventInput{
|
||||
Type: sessions.EventMessage,
|
||||
Payload: map[string]any{"role": "assistant", "content": assistant},
|
||||
})
|
||||
}
|
||||
_, err := a.deps.Store.AppendEvents(sess.id, events)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *Agent) loadHistory(sessionID string) ([]turnRecord, error) {
|
||||
if a.deps.Store == nil {
|
||||
return nil, nil
|
||||
}
|
||||
events, err := a.deps.Store.ReadEvents(sessionID)
|
||||
if err != nil {
|
||||
return nil
|
||||
return nil, err
|
||||
}
|
||||
var records []turnRecord
|
||||
var pendingUser string
|
||||
@@ -444,7 +463,21 @@ func (a *Agent) loadHistory(sessionID string) []turnRecord {
|
||||
if havePending {
|
||||
records = append(records, turnRecord{user: pendingUser})
|
||||
}
|
||||
return records
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func (a *Agent) warnPersistence(note *notifier, action string, message string, err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
sessionID := ""
|
||||
if note != nil {
|
||||
sessionID = note.sessionID
|
||||
}
|
||||
log.Printf("zero acp: failed to %s for session %s: %v", action, sessionID, err)
|
||||
if note != nil {
|
||||
note.text("\n\n[zero warning] " + message + "\n")
|
||||
}
|
||||
}
|
||||
|
||||
// buildPrompt prepends prior conversation as context, since agent.Run drives a
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -242,8 +244,77 @@ func TestACPRejectsInvalidCwd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestACPPromptWarnsWhenTurnPersistenceFails(t *testing.T) {
|
||||
deps := testDeps(t)
|
||||
h := newHarness(t, deps)
|
||||
defer h.stop()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var newRes NewSessionResult
|
||||
if err := h.client.Call(ctx, MethodSessionNew, NewSessionParams{Cwd: t.TempDir(), McpServers: []McpServer{}}, &newRes); err != nil {
|
||||
t.Fatalf("session/new: %v", err)
|
||||
}
|
||||
metadataPath := filepath.Join(deps.Store.RootDir, newRes.SessionID, sessions.MetadataFile)
|
||||
if err := os.Remove(metadataPath); err != nil {
|
||||
t.Fatalf("remove metadata: %v", err)
|
||||
}
|
||||
|
||||
var promptRes PromptResult
|
||||
if err := h.client.Call(ctx, MethodSessionPrompt, PromptParams{
|
||||
SessionID: newRes.SessionID,
|
||||
Prompt: []ContentBlock{TextBlock("hi")},
|
||||
}, &promptRes); err != nil {
|
||||
t.Fatalf("session/prompt: %v", err)
|
||||
}
|
||||
if promptRes.StopReason != StopEndTurn {
|
||||
t.Fatalf("stopReason = %q, want %q", promptRes.StopReason, StopEndTurn)
|
||||
}
|
||||
got := drainTextUntil(t, h.updates, func(text string) bool {
|
||||
return strings.Contains(text, "Hello from ZERO") &&
|
||||
strings.Contains(text, "Could not save session history")
|
||||
})
|
||||
if !strings.Contains(got, "Could not save session history") {
|
||||
t.Fatalf("streamed text = %q, want persistence warning", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACPLoadWarnsWhenHistoryReadFails(t *testing.T) {
|
||||
deps := testDeps(t)
|
||||
cwd := t.TempDir()
|
||||
meta, err := deps.Store.Create(sessions.CreateInput{Title: "ACP session", Cwd: cwd})
|
||||
if err != nil {
|
||||
t.Fatalf("create session: %v", err)
|
||||
}
|
||||
eventsPath := filepath.Join(deps.Store.RootDir, meta.SessionID, sessions.EventsFile)
|
||||
if err := os.WriteFile(eventsPath, []byte("{bad json}\n"), 0o600); err != nil {
|
||||
t.Fatalf("write corrupt events: %v", err)
|
||||
}
|
||||
h := newHarness(t, deps)
|
||||
defer h.stop()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := h.client.Call(ctx, MethodSessionLoad, LoadSessionParams{SessionID: meta.SessionID, Cwd: cwd, McpServers: []McpServer{}}, &LoadSessionResult{}); err != nil {
|
||||
t.Fatalf("session/load: %v", err)
|
||||
}
|
||||
got := drainTextUntil(t, h.updates, func(text string) bool {
|
||||
return strings.Contains(text, "Could not load session history")
|
||||
})
|
||||
if !strings.Contains(got, "Could not load session history") {
|
||||
t.Fatalf("streamed text = %q, want load warning", got)
|
||||
}
|
||||
}
|
||||
|
||||
// drainText collects streamed chunks for a short window and concatenates them.
|
||||
func drainText(t *testing.T, ch <-chan string) string {
|
||||
t.Helper()
|
||||
return drainTextUntil(t, ch, func(text string) bool {
|
||||
return strings.Contains(text, "Hello from ZERO")
|
||||
})
|
||||
}
|
||||
|
||||
func drainTextUntil(t *testing.T, ch <-chan string, done func(string) bool) string {
|
||||
t.Helper()
|
||||
var b strings.Builder
|
||||
deadline := time.After(2 * time.Second)
|
||||
@@ -251,7 +322,7 @@ func drainText(t *testing.T, ch <-chan string) string {
|
||||
select {
|
||||
case s := <-ch:
|
||||
b.WriteString(s)
|
||||
if strings.Contains(b.String(), "Hello from ZERO") {
|
||||
if done(b.String()) {
|
||||
return b.String()
|
||||
}
|
||||
case <-deadline:
|
||||
|
||||
+58
-4
@@ -109,7 +109,7 @@ const escalationFailedNoticePrefix = "Note: could not switch to the requested mo
|
||||
// The system prompt (core coding-craft instructions + workspace context + safety
|
||||
// confirmation policy) is assembled in system_prompt.go via buildSystemPrompt.
|
||||
|
||||
func Run(ctx context.Context, prompt string, provider Provider, options Options) (Result, error) {
|
||||
func Run(ctx context.Context, prompt string, provider Provider, options Options) (result Result, err error) {
|
||||
if provider == nil {
|
||||
return Result{}, errors.New("agent provider is required")
|
||||
}
|
||||
@@ -161,7 +161,11 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// doesn't re-run the recursive schema→map conversion for every tool every turn.
|
||||
toolDefCache := map[string]zeroruntime.ToolDefinition{}
|
||||
|
||||
result := Result{Messages: copyMessages(messages)}
|
||||
result = Result{Messages: copyMessages(messages)}
|
||||
dispatchSessionStart(ctx, options)
|
||||
defer func() {
|
||||
dispatchSessionEnd(ctx, options, result, err)
|
||||
}()
|
||||
for turn := 0; turn < maxTurns; turn++ {
|
||||
result.Turns = turn + 1
|
||||
|
||||
@@ -721,7 +725,8 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
Content: nudge,
|
||||
})
|
||||
}
|
||||
if answer, finalMessages, finishReason := finalAnswerAfterMaxTurns(ctx, provider, messages, options); strings.TrimSpace(answer) != "" {
|
||||
finalExposed, _ := partitionToolsCached(registry, permissionMode, options, loaded, toolDefCache)
|
||||
if answer, finalMessages, finishReason := finalAnswerAfterMaxTurns(ctx, provider, messages, finalExposed, options); strings.TrimSpace(answer) != "" {
|
||||
result.FinalAnswer = answer
|
||||
result.FinishReason = finishReason
|
||||
result.Messages = copyMessages(finalMessages)
|
||||
@@ -741,7 +746,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages []zeroruntime.Message, options Options) (string, []zeroruntime.Message, string) {
|
||||
func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages []zeroruntime.Message, toolDefs []zeroruntime.ToolDefinition, options Options) (string, []zeroruntime.Message, string) {
|
||||
finalMessages := copyMessages(messages)
|
||||
finalMessages = append(finalMessages, zeroruntime.Message{
|
||||
Role: zeroruntime.MessageRoleUser,
|
||||
@@ -752,6 +757,7 @@ func finalAnswerAfterMaxTurns(ctx context.Context, provider Provider, messages [
|
||||
// transient hiccup doesn't drop the final summary (AUDIT-L1).
|
||||
stream, err := streamWithReconnect(ctx, provider, zeroruntime.CompletionRequest{
|
||||
Messages: copyMessages(finalMessages),
|
||||
Tools: toolDefs,
|
||||
ReasoningEffort: options.ReasoningEffort,
|
||||
PromptCacheKey: options.SessionID,
|
||||
}, reconnectNoticeFor(options))
|
||||
@@ -1568,6 +1574,54 @@ func dispatchAfterTool(ctx context.Context, options Options, call ToolCall, args
|
||||
return strings.TrimSpace(strings.Join(outcome.Messages, "\n"))
|
||||
}
|
||||
|
||||
// dispatchSessionStart runs configured sessionStart hooks once before the first
|
||||
// model turn. Lifecycle hooks are advisory: dispatcher failures are audited but
|
||||
// never block the run.
|
||||
func dispatchSessionStart(ctx context.Context, options Options) {
|
||||
if options.Hooks == nil {
|
||||
return
|
||||
}
|
||||
options.Hooks.Dispatch(ctx, hooks.DispatchInput{
|
||||
Event: hooks.EventSessionStart,
|
||||
Payload: map[string]any{
|
||||
"event": string(hooks.EventSessionStart),
|
||||
"sessionId": options.SessionID,
|
||||
"cwd": options.Cwd,
|
||||
"provider": options.ProviderName,
|
||||
"model": options.Model,
|
||||
"depth": options.Depth,
|
||||
"tag": options.Tag,
|
||||
"sessionTitle": options.SessionTitle,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// dispatchSessionEnd runs configured sessionEnd hooks once when the agent run
|
||||
// exits, including early error returns. Lifecycle hooks are advisory.
|
||||
func dispatchSessionEnd(ctx context.Context, options Options, result Result, runErr error) {
|
||||
if options.Hooks == nil {
|
||||
return
|
||||
}
|
||||
payload := map[string]any{
|
||||
"event": string(hooks.EventSessionEnd),
|
||||
"sessionId": options.SessionID,
|
||||
"cwd": options.Cwd,
|
||||
"provider": options.ProviderName,
|
||||
"model": options.Model,
|
||||
"turns": result.Turns,
|
||||
"stopReason": string(result.StopReason),
|
||||
"finishReason": result.FinishReason,
|
||||
"incomplete": result.Incomplete,
|
||||
}
|
||||
if runErr != nil {
|
||||
payload["error"] = runErr.Error()
|
||||
}
|
||||
options.Hooks.Dispatch(ctx, hooks.DispatchInput{
|
||||
Event: hooks.EventSessionEnd,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
|
||||
// blockedByHookResult is the tool result for a call vetoed by a beforeTool hook.
|
||||
func blockedByHookResult(call ToolCall, outcome hooks.DispatchOutcome) ToolResult {
|
||||
// The hook reason (its stdout/stderr) is model-visible here, an intercepted
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/hooks"
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/specmode"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
@@ -38,6 +39,59 @@ func (provider *mockProvider) StreamCompletion(ctx context.Context, request zero
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func TestRunDispatchesSessionLifecycleHooks(t *testing.T) {
|
||||
audit, err := hooks.NewAuditStore(hooks.AuditStoreOptions{AuditPath: filepath.Join(t.TempDir(), "audit.jsonl")})
|
||||
if err != nil {
|
||||
t.Fatalf("NewAuditStore: %v", err)
|
||||
}
|
||||
dispatcher := hooks.NewDispatcher(hooks.DispatcherOptions{
|
||||
Config: hooks.Config{
|
||||
Enabled: true,
|
||||
Hooks: []hooks.Definition{
|
||||
{ID: "zero.session-start", Event: hooks.EventSessionStart, Command: "zero-missing-hook-command", Enabled: true},
|
||||
{ID: "zero.session-end", Event: hooks.EventSessionEnd, Command: "zero-missing-hook-command", Enabled: true},
|
||||
},
|
||||
},
|
||||
Audit: audit,
|
||||
})
|
||||
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{{
|
||||
{Type: zeroruntime.StreamEventText, Content: "done"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
}}}
|
||||
|
||||
_, err = Run(context.Background(), "hi", provider, Options{
|
||||
SessionID: "session-123",
|
||||
Cwd: t.TempDir(),
|
||||
ProviderName: "test-provider",
|
||||
Model: "test-model",
|
||||
Hooks: dispatcher,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
|
||||
events, err := audit.ReadEvents()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadEvents: %v", err)
|
||||
}
|
||||
started := map[hooks.Event]int{}
|
||||
completed := map[hooks.Event]int{}
|
||||
for _, event := range events {
|
||||
switch event.Type {
|
||||
case "hook_execution_started":
|
||||
started[event.Event]++
|
||||
case "hook_execution_completed":
|
||||
completed[event.Event]++
|
||||
}
|
||||
}
|
||||
if started[hooks.EventSessionStart] != 1 || completed[hooks.EventSessionStart] != 1 {
|
||||
t.Fatalf("sessionStart audit counts started/completed = %d/%d, events=%#v", started[hooks.EventSessionStart], completed[hooks.EventSessionStart], events)
|
||||
}
|
||||
if started[hooks.EventSessionEnd] != 1 || completed[hooks.EventSessionEnd] != 1 {
|
||||
t.Fatalf("sessionEnd audit counts started/completed = %d/%d, events=%#v", started[hooks.EventSessionEnd], completed[hooks.EventSessionEnd], events)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingWebSearchTool struct {
|
||||
calls []map[string]any
|
||||
}
|
||||
@@ -2768,11 +2822,11 @@ func TestRunRequestsFinalAnswerAfterMaxTurns(t *testing.T) {
|
||||
t.Fatalf("expected final answer from finalization turn, got %q", result.FinalAnswer)
|
||||
}
|
||||
if len(provider.requests) != 2 {
|
||||
t.Fatalf("expected final no-tools request after max turns, got %d requests", len(provider.requests))
|
||||
t.Fatalf("expected finalization request after max turns, got %d requests", len(provider.requests))
|
||||
}
|
||||
finalRequest := provider.requests[1]
|
||||
if len(finalRequest.Tools) != 0 {
|
||||
t.Fatalf("finalization request must not advertise tools, got %#v", finalRequest.Tools)
|
||||
if toolDefinitionByName(finalRequest.Tools, "read_file") == nil {
|
||||
t.Fatalf("finalization request must keep prior tools exposed for strict provider replay, got %#v", finalRequest.Tools)
|
||||
}
|
||||
lastMessage := finalRequest.Messages[len(finalRequest.Messages)-1]
|
||||
if lastMessage.Role != zeroruntime.MessageRoleUser || !strings.Contains(lastMessage.Content, "tool-turn limit") {
|
||||
|
||||
+28
-4
@@ -686,8 +686,14 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
|
||||
defer closeMCPRuntime(stderr, mcpRuntime)
|
||||
// A server that could not be reached or validated is skipped, not fatal (one
|
||||
// bad MCP server must not abort startup) — surface each so a missing tool set is
|
||||
// explained rather than silently absent.
|
||||
// explained rather than silently absent. A built-in default the user never
|
||||
// configured (e.g. keyless Firecrawl with no credentials) is the exception: it
|
||||
// was never asked for, so its failure is not worth a startup warning — only
|
||||
// servers the user actually configured warn on failure (issue #552).
|
||||
for _, skipped := range mcpRuntime.Skipped() {
|
||||
if skipped.UnconfiguredDefault {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(stderr, "warning: MCP server %s unavailable, skipped: %s\n", skipped.Name, redaction.ErrorMessage(skipped.Err, redaction.Options{}))
|
||||
}
|
||||
// Make local plugins live: register their declared tools into the registry and
|
||||
@@ -729,8 +735,19 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
|
||||
Scope: scope,
|
||||
})
|
||||
lastKnownMCPConfig := mcpConfig
|
||||
sttServerManager := newDictationServerManager(resolved.STT)
|
||||
// Keep STT downloads in the SAME config tree the rest of the TUI uses. Deriving
|
||||
// from userConfigPath (rather than config.UserConfigDir()) matters when the config
|
||||
// root is overridden — e.g. in tests or a custom ZERO config dir — so the two
|
||||
// don't diverge. userConfigPath points at .../zero/config.json, so its dir is the
|
||||
// zero config dir.
|
||||
sttDownloadRoot := ""
|
||||
if userConfigPath != "" {
|
||||
sttDownloadRoot = filepath.Join(filepath.Dir(userConfigPath), "stt")
|
||||
}
|
||||
return deps.runTUI(context.Background(), tui.Options{
|
||||
Cwd: workspaceRoot,
|
||||
Version: version,
|
||||
Theme: theme,
|
||||
SavedTheme: resolved.Preferences.Theme,
|
||||
UserConfigPath: userConfigPath,
|
||||
@@ -741,6 +758,7 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
|
||||
ProviderProfile: resolved.Provider,
|
||||
SavedProviders: usableSavedProviders(resolved.Providers),
|
||||
FavoriteModels: resolved.Preferences.FavoriteModels,
|
||||
RecentModels: resolved.Preferences.RecentModels,
|
||||
RecapsEnabled: resolved.Preferences.RecapsEnabled(),
|
||||
Provider: provider,
|
||||
NewProvider: deps.newProvider,
|
||||
@@ -791,9 +809,15 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
|
||||
merged, _ := plugins.MergedSkillsLoaded(deps.skillsDir(), pluginActivation.skillRoots)
|
||||
return merged
|
||||
}),
|
||||
PermissionMode: permissionMode,
|
||||
Notify: resolved.Notify,
|
||||
KeyBindings: resolved.KeyBindings,
|
||||
PermissionMode: permissionMode,
|
||||
Notify: resolved.Notify,
|
||||
KeyBindings: resolved.KeyBindings,
|
||||
STT: resolved.STT,
|
||||
BuildDictationTranscriber: newDictationTranscriberFactory(resolved, userConfigPath, sttServerManager),
|
||||
ShutdownDictationServer: sttServerManager.Shutdown,
|
||||
STTDownloadRoot: sttDownloadRoot,
|
||||
STTKeyStatus: newSTTKeyStatus(resolved, userConfigPath),
|
||||
SaveSTTKey: newSaveSTTKey(userConfigPath),
|
||||
Setup: tui.SetupOptions{
|
||||
Visible: setupVisible,
|
||||
Required: needsSetup,
|
||||
|
||||
@@ -158,6 +158,86 @@ func TestRunNoArgsLaunchesSetupTUIWithNilProviderWhenNoProviderConfigured(t *tes
|
||||
assertAgentOptions(t, launchedOptions, 12, agent.PermissionModeAsk)
|
||||
}
|
||||
|
||||
type fakeMCPRuntimeWithSkips struct {
|
||||
skipped []mcp.SkippedServer
|
||||
}
|
||||
|
||||
func (r fakeMCPRuntimeWithSkips) Close() error { return nil }
|
||||
|
||||
func (r fakeMCPRuntimeWithSkips) Skipped() []mcp.SkippedServer { return r.skipped }
|
||||
|
||||
func TestTUIStartupSuppressesWarningForUnconfiguredDefaultServer(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cwd := t.TempDir()
|
||||
setCLIUserConfigRoot(t)
|
||||
|
||||
exitCode := runWithDeps([]string{}, &stdout, &stderr, appDeps{
|
||||
getwd: func() (string, error) {
|
||||
return cwd, nil
|
||||
},
|
||||
resolveConfig: func(workspaceRoot string, overrides config.Overrides) (config.ResolvedConfig, error) {
|
||||
return config.ResolvedConfig{MaxTurns: 8}, nil
|
||||
},
|
||||
resolveMCPConfig: func(workspaceRoot string) (config.MCPConfig, error) {
|
||||
return config.MCPConfig{Servers: map[string]config.MCPServerConfig{
|
||||
"firecrawl": config.DefaultMCPServers()["firecrawl"],
|
||||
}}, nil
|
||||
},
|
||||
registerMCPTools: func(context.Context, *tools.Registry, config.MCPConfig, mcp.RegisterOptions) (mcpToolRuntime, error) {
|
||||
return fakeMCPRuntimeWithSkips{skipped: []mcp.SkippedServer{
|
||||
{Name: "firecrawl", Err: errors.New("returned HTTP 401"), UnconfiguredDefault: true},
|
||||
}}, nil
|
||||
},
|
||||
runTUI: func(ctx context.Context, options tui.Options) int {
|
||||
return 0
|
||||
},
|
||||
})
|
||||
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("exitCode = %d stderr=%s", exitCode, stderr.String())
|
||||
}
|
||||
if strings.Contains(stderr.String(), "firecrawl") {
|
||||
t.Fatalf("stderr = %q, want no warning for an unconfigured default server", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTUIStartupWarnsForUserConfiguredServerSkip(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cwd := t.TempDir()
|
||||
setCLIUserConfigRoot(t)
|
||||
|
||||
exitCode := runWithDeps([]string{}, &stdout, &stderr, appDeps{
|
||||
getwd: func() (string, error) {
|
||||
return cwd, nil
|
||||
},
|
||||
resolveConfig: func(workspaceRoot string, overrides config.Overrides) (config.ResolvedConfig, error) {
|
||||
return config.ResolvedConfig{MaxTurns: 8}, nil
|
||||
},
|
||||
resolveMCPConfig: func(workspaceRoot string) (config.MCPConfig, error) {
|
||||
return config.MCPConfig{Servers: map[string]config.MCPServerConfig{
|
||||
"custom": {Type: "stdio", Command: "custom-mcp"},
|
||||
}}, nil
|
||||
},
|
||||
registerMCPTools: func(context.Context, *tools.Registry, config.MCPConfig, mcp.RegisterOptions) (mcpToolRuntime, error) {
|
||||
return fakeMCPRuntimeWithSkips{skipped: []mcp.SkippedServer{
|
||||
{Name: "custom", Err: errors.New("connect failed"), UnconfiguredDefault: false},
|
||||
}}, nil
|
||||
},
|
||||
runTUI: func(ctx context.Context, options tui.Options) int {
|
||||
return 0
|
||||
},
|
||||
})
|
||||
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("exitCode = %d stderr=%s", exitCode, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "warning: MCP server custom unavailable") {
|
||||
t.Fatalf("stderr = %q, want a warning for a user-configured server that failed", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNoArgsEntersSetupWhenResolveReportsNoActiveProvider(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
+45
-2
@@ -11,10 +11,46 @@ import (
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
"github.com/Gitlawb/zero/internal/provideroauth"
|
||||
"github.com/Gitlawb/zero/internal/redaction"
|
||||
)
|
||||
|
||||
// ensureLoginProviderProfile makes a freshly stored OAuth login visible as a
|
||||
// provider: without a profile in config.json the login shows up nowhere — not in
|
||||
// `zero providers list`, not in the TUI picker — and `zero providers use <name>`
|
||||
// fails "not found", so the user's working active provider looks broken while
|
||||
// the new login looks lost. Returns user-facing guidance lines ("" when the
|
||||
// login has no catalog entry to scaffold from). Best-effort by design: the
|
||||
// token is already stored, so a profile-write failure degrades to a warning
|
||||
// line instead of failing a login that succeeded.
|
||||
func ensureLoginProviderProfile(deps appDeps, provider string) string {
|
||||
if _, ok := providercatalog.Get(provider); !ok {
|
||||
// Custom OAuth server without a catalog entry — no endpoint/model defaults
|
||||
// to scaffold a profile from; the user wires their own profile.
|
||||
return ""
|
||||
}
|
||||
configPath, err := deps.userConfigPath()
|
||||
if err != nil {
|
||||
return "warning: login saved, but no provider profile was written: " + err.Error()
|
||||
}
|
||||
ensured, err := config.EnsureCatalogProvider(configPath, provider)
|
||||
if err != nil {
|
||||
return "warning: login saved, but no provider profile was written: " + err.Error()
|
||||
}
|
||||
active := strings.EqualFold(strings.TrimSpace(ensured.Active), strings.TrimSpace(ensured.Name))
|
||||
switch {
|
||||
case ensured.Created && active:
|
||||
return fmt.Sprintf("Added provider %q to your config and set it active.", ensured.Name)
|
||||
case ensured.Created:
|
||||
return fmt.Sprintf("Added provider %q to your config; the active provider is still %q.\nSwitch with: zero providers use %s", ensured.Name, ensured.Active, ensured.Name)
|
||||
case active:
|
||||
return fmt.Sprintf("Provider %q is configured and active.", ensured.Name)
|
||||
default:
|
||||
return fmt.Sprintf("Provider %q is already configured.\nSwitch with: zero providers use %s", ensured.Name, ensured.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// runAuth dispatches `zero auth <command>` for provider OAuth login. It is
|
||||
// additive and independent of `zero mcp oauth` (MCP server auth), which is
|
||||
// unchanged.
|
||||
@@ -138,8 +174,10 @@ func runAuthChatGPT(args []string, stdout io.Writer, stderr io.Writer, deps appD
|
||||
if _, err := fmt.Fprint(stdout, statuses); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
if _, err := fmt.Fprint(stdout, "\nUse it with zero, e.g.:\n zero --provider chatgpt --model gpt-5.5\n"); err != nil {
|
||||
return exitCrash
|
||||
if line := ensureLoginProviderProfile(deps, "chatgpt"); line != "" {
|
||||
if _, err := fmt.Fprintf(stdout, "\n%s\n", line); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
@@ -331,6 +369,11 @@ func runAuthLogin(args []string, stdout io.Writer, stderr io.Writer, deps appDep
|
||||
if _, err := fmt.Fprintf(stdout, "Logged in to %s.\n%s\n", parsed.positional[0], oauth.FormatStatuses([]oauth.Status{status})); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
if line := ensureLoginProviderProfile(deps, provider); line != "" {
|
||||
if _, err := fmt.Fprintln(stdout, line); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
|
||||
@@ -2,18 +2,24 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
)
|
||||
|
||||
// withAuthStore points the provider OAuth store at a temp file for the test.
|
||||
// withAuthStore points the provider OAuth store at a temp file for the test,
|
||||
// pinning the file backend so an inherited ZERO_OAUTH_STORAGE=keyring can't
|
||||
// ignore the temp path and hit the OS keychain.
|
||||
func withAuthStore(t *testing.T) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "oauth-tokens.json")
|
||||
t.Setenv("ZERO_OAUTH_TOKENS_PATH", path)
|
||||
t.Setenv("ZERO_OAUTH_STORAGE", "file")
|
||||
return path
|
||||
}
|
||||
|
||||
@@ -197,3 +203,81 @@ func TestRunAuthLoginChatGPTRejectsScope(t *testing.T) {
|
||||
t.Fatalf("stderr = %q, want the ChatGPT-specific --scope rejection", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLoginProviderProfileAddsProviderWithoutStealingActive(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
seed := `{"activeProvider":"opengateway","providers":[{"name":"opengateway","provider_kind":"openai-compatible","baseURL":"https://gateway.example.com/v1","apiKeyStored":true,"model":"some-model"}]}`
|
||||
if err := os.WriteFile(configPath, []byte(seed), 0o600); err != nil {
|
||||
t.Fatalf("seed config: %v", err)
|
||||
}
|
||||
deps := appDeps{userConfigPath: func() (string, error) { return configPath, nil }}
|
||||
|
||||
line := ensureLoginProviderProfile(deps, "chatgpt")
|
||||
if !strings.Contains(line, `Added provider "chatgpt"`) {
|
||||
t.Fatalf("expected added-provider guidance, got %q", line)
|
||||
}
|
||||
if !strings.Contains(line, "zero providers use chatgpt") {
|
||||
t.Fatalf("expected switch hint, got %q", line)
|
||||
}
|
||||
|
||||
cfg := readCLIConfigFixture(t, configPath)
|
||||
if cfg.ActiveProvider != "opengateway" {
|
||||
t.Fatalf("active provider changed to %q", cfg.ActiveProvider)
|
||||
}
|
||||
names := make([]string, 0, len(cfg.Providers))
|
||||
for _, provider := range cfg.Providers {
|
||||
names = append(names, provider.Name)
|
||||
}
|
||||
if len(cfg.Providers) != 2 || cfg.Providers[1].CatalogID != "chatgpt" {
|
||||
t.Fatalf("expected chatgpt profile appended, got %v", names)
|
||||
}
|
||||
|
||||
// A second login must be a no-op with switch guidance, not a duplicate.
|
||||
line = ensureLoginProviderProfile(deps, "chatgpt")
|
||||
if !strings.Contains(line, "already configured") {
|
||||
t.Fatalf("expected already-configured guidance, got %q", line)
|
||||
}
|
||||
cfg = readCLIConfigFixture(t, configPath)
|
||||
if len(cfg.Providers) != 2 {
|
||||
t.Fatalf("repeat login duplicated the profile: %d providers", len(cfg.Providers))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLoginProviderProfileActivatesOnFreshConfig(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
deps := appDeps{userConfigPath: func() (string, error) { return configPath, nil }}
|
||||
|
||||
line := ensureLoginProviderProfile(deps, "chatgpt")
|
||||
if !strings.Contains(line, "set it active") {
|
||||
t.Fatalf("fresh config should adopt the login as active, got %q", line)
|
||||
}
|
||||
cfg := readCLIConfigFixture(t, configPath)
|
||||
if cfg.ActiveProvider != "chatgpt" {
|
||||
t.Fatalf("active provider = %q, want chatgpt", cfg.ActiveProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLoginProviderProfileSkipsNonCatalogProviders(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.json")
|
||||
deps := appDeps{userConfigPath: func() (string, error) { return configPath, nil }}
|
||||
|
||||
if line := ensureLoginProviderProfile(deps, "my-custom-oauth-server"); line != "" {
|
||||
t.Fatalf("custom OAuth server must not scaffold a profile, got %q", line)
|
||||
}
|
||||
if _, err := os.Stat(configPath); err == nil {
|
||||
t.Fatalf("config must not be created for a non-catalog login")
|
||||
}
|
||||
}
|
||||
|
||||
func readCLIConfigFixture(t *testing.T, path string) config.FileConfig {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
var cfg config.FileConfig
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
t.Fatalf("decode config: %v", err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
@@ -75,6 +75,12 @@ func runProviders(args []string, stdout io.Writer, stderr io.Writer, deps appDep
|
||||
if command == "use" {
|
||||
return runProvidersUse(args, stdout, stderr, deps)
|
||||
}
|
||||
if command == "remove" || command == "rm" {
|
||||
return runProvidersRemove(args, stdout, stderr, deps)
|
||||
}
|
||||
if command == "rename" {
|
||||
return runProvidersRename(args, stdout, stderr, deps)
|
||||
}
|
||||
if command == "setup" {
|
||||
return runProvidersSetup(args, stdout, stderr, deps)
|
||||
}
|
||||
@@ -246,6 +252,20 @@ func parseCommandCenterArgs(args []string, allowModelFilters bool, allowProvider
|
||||
|
||||
func summarizeConfig(resolved config.ResolvedConfig) configSummary {
|
||||
summary := zerocommands.ConfigSnapshotFromResolved(resolved)
|
||||
// Mark keyless profiles that authenticate via a stored OAuth login (e.g.
|
||||
// ChatGPT) so the list shows "oauth login" instead of "api key: not set" —
|
||||
// the same candidate matching the runtime resolver uses.
|
||||
logins := oauthLoggedInProviders()
|
||||
if len(logins) > 0 {
|
||||
for index := range summary.Providers {
|
||||
for _, profile := range resolved.Providers {
|
||||
if profile.Name == summary.Providers[index].Name {
|
||||
summary.Providers[index].OAuthLogin = providerHasOAuthLogin(profile, logins)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.SliceStable(summary.Providers, func(i int, j int) bool {
|
||||
if summary.Providers[i].Active != summary.Providers[j].Active {
|
||||
return summary.Providers[i].Active
|
||||
@@ -323,7 +343,7 @@ func formatProviderSummaries(command string, providers []providerSummary) string
|
||||
"model: "+displayCLIValue(provider.Model, "none"),
|
||||
"api model: "+displayCLIValue(provider.APIModel, "unknown"),
|
||||
"base url: "+displayCLIValue(provider.BaseURL, "default"),
|
||||
"api key: "+apiKeyState(provider.APIKeySet),
|
||||
"api key: "+providerCredentialState(provider),
|
||||
)
|
||||
if provider.Message != "" {
|
||||
lines = append(lines, "status: "+provider.Status+" - "+provider.Message)
|
||||
@@ -340,7 +360,7 @@ func formatProviderLine(provider providerSummary) string {
|
||||
if provider.Active {
|
||||
marker = "*"
|
||||
}
|
||||
line := fmt.Sprintf("%s %s [%s] model=%s apiModel=%s api key: %s", marker, displayCLIValue(provider.Name, "none"), displayCLIValue(provider.ProviderKind, "unknown"), displayCLIValue(provider.Model, "none"), displayCLIValue(provider.APIModel, "unknown"), apiKeyState(provider.APIKeySet))
|
||||
line := fmt.Sprintf("%s %s [%s] model=%s apiModel=%s api key: %s", marker, displayCLIValue(provider.Name, "none"), displayCLIValue(provider.ProviderKind, "unknown"), displayCLIValue(provider.Model, "none"), displayCLIValue(provider.APIModel, "unknown"), providerCredentialState(provider))
|
||||
if provider.Message != "" {
|
||||
line += " (" + provider.Status + ": " + provider.Message + ")"
|
||||
}
|
||||
@@ -413,6 +433,15 @@ func apiKeyState(set bool) string {
|
||||
return "not set"
|
||||
}
|
||||
|
||||
// providerCredentialState renders the credential column: a configured key wins,
|
||||
// then a stored OAuth login, then "not set".
|
||||
func providerCredentialState(provider providerSummary) string {
|
||||
if !provider.APIKeySet && provider.OAuthLogin {
|
||||
return "oauth login"
|
||||
}
|
||||
return apiKeyState(provider.APIKeySet)
|
||||
}
|
||||
|
||||
func displayCLIValue(value string, fallback string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fallback
|
||||
@@ -466,6 +495,8 @@ func writeProvidersHelp(w io.Writer) error {
|
||||
zero providers add <catalog-id> [flags]
|
||||
zero providers check [name] [flags]
|
||||
zero providers use <name> [flags]
|
||||
zero providers remove <name> [flags]
|
||||
zero providers rename <old> <new> [flags]
|
||||
zero providers setup <catalog-id> [flags]
|
||||
zero providers detect [flags]
|
||||
zero providers models [name] [flags]
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
"github.com/Gitlawb/zero/internal/providerhealth"
|
||||
"github.com/Gitlawb/zero/internal/zerocommands"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
@@ -835,3 +836,57 @@ func (commandCenterProvider) StreamCompletion(context.Context, zeroruntime.Compl
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func TestProviderCredentialStateShowsOAuthLogin(t *testing.T) {
|
||||
if got := providerCredentialState(providerSummary{APIKeySet: true}); got != "set" {
|
||||
t.Fatalf("keyed provider = %q, want set", got)
|
||||
}
|
||||
if got := providerCredentialState(providerSummary{OAuthLogin: true}); got != "oauth login" {
|
||||
t.Fatalf("token-login provider = %q, want oauth login", got)
|
||||
}
|
||||
if got := providerCredentialState(providerSummary{}); got != "not set" {
|
||||
t.Fatalf("credential-less provider = %q, want not set", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProvidersListMarksOAuthLoginProviders: a keyless profile whose credential
|
||||
// is a stored OAuth login (the shape `zero auth chatgpt` now writes) must render
|
||||
// as "oauth login", not as a broken "api key: not set" entry.
|
||||
func TestProvidersListMarksOAuthLoginProviders(t *testing.T) {
|
||||
tokensPath := filepath.Join(t.TempDir(), "oauth-tokens.json")
|
||||
t.Setenv("ZERO_OAUTH_TOKENS_PATH", tokensPath)
|
||||
store, err := oauth.NewStore(oauth.StoreOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("oauth store: %v", err)
|
||||
}
|
||||
if err := store.Save(oauth.ProviderKey("chatgpt"), oauth.Token{AccessToken: "bearer-123"}); err != nil {
|
||||
t.Fatalf("save token: %v", err)
|
||||
}
|
||||
|
||||
resolved := config.ResolvedConfig{
|
||||
ActiveProvider: "opengateway",
|
||||
Providers: []config.ProviderProfile{
|
||||
{Name: "opengateway", ProviderKind: config.ProviderKindOpenAICompatible, BaseURL: "https://gateway.example.com/v1", APIKeyStored: true, Model: "some-model"},
|
||||
{Name: "chatgpt", CatalogID: "chatgpt", ProviderKind: config.ProviderKindOpenAICompatible, BaseURL: "https://chatgpt.com/backend-api/codex", Model: "gpt-5.5"},
|
||||
},
|
||||
}
|
||||
summary := summarizeConfig(resolved)
|
||||
byName := map[string]providerSummary{}
|
||||
for _, provider := range summary.Providers {
|
||||
byName[provider.Name] = provider
|
||||
}
|
||||
if !byName["opengateway"].APIKeySet {
|
||||
t.Fatalf("stored-key provider must report a credential: %+v", byName["opengateway"])
|
||||
}
|
||||
if byName["opengateway"].OAuthLogin {
|
||||
t.Fatalf("key-authed provider must not claim the OAuth login: %+v", byName["opengateway"])
|
||||
}
|
||||
if !byName["chatgpt"].OAuthLogin || byName["chatgpt"].APIKeySet {
|
||||
t.Fatalf("chatgpt must be marked oauth-login and keyless: %+v", byName["chatgpt"])
|
||||
}
|
||||
|
||||
rendered := formatProviderSummaries("list", summary.Providers)
|
||||
if !strings.Contains(rendered, "oauth login") {
|
||||
t.Fatalf("list should render the oauth login state, got:\n%s", rendered)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
neturl "net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/dictation"
|
||||
"github.com/Gitlawb/zero/internal/tui"
|
||||
)
|
||||
|
||||
// Default batch cloud models (§8). Overridable via stt.model.
|
||||
const (
|
||||
defaultGroqSTTModel = "whisper-large-v3-turbo"
|
||||
defaultOpenAISTTModel = "whisper-1"
|
||||
groqSTTBaseURL = "https://api.groq.com/openai/v1"
|
||||
)
|
||||
|
||||
// newDictationTranscriberFactory returns the closure the TUI calls to build a
|
||||
// transcriber when a recording starts. It lives here, not in the TUI, because it
|
||||
// resolves provider API keys (credstore + env + configured providers) — logic
|
||||
// that belongs with the rest of the CLI's credential handling.
|
||||
//
|
||||
// serverMgr is the shared, session-long sherpa-onnx streaming server; it is
|
||||
// spawned lazily on the first local-streaming recording and torn down on exit.
|
||||
//
|
||||
// preferStreaming asks for the streaming backend; the returned bool reports
|
||||
// whether streaming was actually built (false → the caller uses the batch
|
||||
// pipeline). A misconfigured streaming backend surfaces its own SetupError.
|
||||
func newDictationTranscriberFactory(resolved config.ResolvedConfig, userConfigPath string, serverMgr *dictation.ServerManager) func(config.STTConfig, bool) (tui.Transcriber, bool, error) {
|
||||
return func(stt config.STTConfig, preferStreaming bool) (tui.Transcriber, bool, error) {
|
||||
// stt is passed in (not captured) so a mid-session change — e.g. the F9
|
||||
// auto-download writing localModelPath — takes effect on the next call.
|
||||
if serverMgr != nil {
|
||||
serverMgr.SetModelPath(stt.LocalModelPath)
|
||||
}
|
||||
if preferStreaming {
|
||||
t, err := buildStreamingTranscriber(stt, resolved, userConfigPath, serverMgr)
|
||||
switch {
|
||||
case err == nil && t != nil:
|
||||
return t, true, nil
|
||||
case err != nil && stt.STTStreamProvider() != config.STTProviderLocal:
|
||||
// An explicitly-chosen cloud streaming backend that failed to build
|
||||
// (e.g. a missing key) should surface its own error, not silently
|
||||
// fall back to batch — the user asked for that backend specifically.
|
||||
return nil, true, err
|
||||
}
|
||||
// The default local streaming backend isn't set up: fall through to the
|
||||
// batch pipeline so a cloud-batch provider (or a batch-only setup) still
|
||||
// works instead of erroring on a streaming server the user never asked for.
|
||||
}
|
||||
t, err := buildBatchTranscriber(stt, resolved, userConfigPath)
|
||||
return t, false, err
|
||||
}
|
||||
}
|
||||
|
||||
// buildStreamingTranscriber constructs the configured streaming backend, or
|
||||
// returns (nil, nil) when streaming is not applicable so the caller falls back
|
||||
// to batch.
|
||||
func buildStreamingTranscriber(stt config.STTConfig, resolved config.ResolvedConfig, userConfigPath string, serverMgr *dictation.ServerManager) (dictation.Transcriber, error) {
|
||||
switch stt.STTStreamProvider() {
|
||||
case config.STTProviderDeepgram:
|
||||
key := resolveSTTKey(resolved, userConfigPath, dictation.StreamProviderDeepgram, "DEEPGRAM_API_KEY")
|
||||
return dictation.NewDeepgramTranscriber(dictation.DeepgramConfig{
|
||||
APIKey: key,
|
||||
Model: stt.StreamModel,
|
||||
Language: stt.Language,
|
||||
})
|
||||
case config.STTProviderOpenAI:
|
||||
key := resolveSTTKey(resolved, userConfigPath, dictation.ProviderOpenAI, "OPENAI_API_KEY")
|
||||
return dictation.NewOpenAIRealtimeTranscriber(dictation.OpenAIRealtimeConfig{
|
||||
APIKey: key,
|
||||
Model: stt.StreamModel,
|
||||
})
|
||||
default: // local streaming via the warm sherpa-onnx server
|
||||
// STTStreamProvider() defaults to "local" whenever streamProvider is unset —
|
||||
// including when the user picked a CLOUD batch provider (Groq/OpenAI) and
|
||||
// never touched streaming. Don't silently stream through the local sherpa
|
||||
// server in that case: local streaming is only appropriate when it was
|
||||
// explicitly requested (streamProvider: "local") OR the batch provider is
|
||||
// itself local. Otherwise report "not applicable" so the factory falls back
|
||||
// to the cloud batch pipeline the user actually chose.
|
||||
localWanted := stt.StreamProvider == config.STTProviderLocal || stt.STTProvider() == config.STTProviderLocal
|
||||
if !localWanted {
|
||||
return nil, nil
|
||||
}
|
||||
// The local streaming transcriber validates the model only lazily (at
|
||||
// connect time), so gate on the model path here — with none set, report
|
||||
// "not applicable" (nil, nil) so the factory falls back to the batch
|
||||
// pipeline instead of committing to a server that can't start.
|
||||
if serverMgr == nil || strings.TrimSpace(stt.LocalModelPath) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return dictation.NewLocalStreamingTranscriber(serverMgr), nil
|
||||
}
|
||||
}
|
||||
|
||||
// newDictationServerManager builds the warm sherpa-onnx streaming server manager
|
||||
// from config. Construction is side-effect free — the server spawns lazily on
|
||||
// the first local-streaming recording — so it is always safe to create.
|
||||
func newDictationServerManager(stt config.STTConfig) *dictation.ServerManager {
|
||||
return dictation.NewServerManager(dictation.ServerConfig{
|
||||
Binary: stt.LocalServerBinary,
|
||||
ModelPath: stt.LocalModelPath,
|
||||
Port: stt.LocalServerPort,
|
||||
NumThreads: stt.NumThreads,
|
||||
})
|
||||
}
|
||||
|
||||
// buildBatchTranscriber constructs the configured batch transcriber, resolving
|
||||
// credentials for the cloud providers.
|
||||
func buildBatchTranscriber(stt config.STTConfig, resolved config.ResolvedConfig, userConfigPath string) (dictation.Transcriber, error) {
|
||||
switch stt.STTProvider() {
|
||||
case config.STTProviderGroq:
|
||||
key := resolveSTTKey(resolved, userConfigPath, dictation.ProviderGroq, "GROQ_API_KEY")
|
||||
model := stt.Model
|
||||
if model == "" {
|
||||
model = defaultGroqSTTModel
|
||||
}
|
||||
return dictation.NewCloudTranscriber(dictation.CloudConfig{
|
||||
Provider: dictation.ProviderGroq,
|
||||
BaseURL: groqSTTBaseURL,
|
||||
APIKey: key,
|
||||
Model: model,
|
||||
Language: stt.Language,
|
||||
})
|
||||
case config.STTProviderOpenAI:
|
||||
key := resolveSTTKey(resolved, userConfigPath, dictation.ProviderOpenAI, "OPENAI_API_KEY")
|
||||
model := stt.Model
|
||||
if model == "" {
|
||||
model = defaultOpenAISTTModel
|
||||
}
|
||||
return dictation.NewCloudTranscriber(dictation.CloudConfig{
|
||||
Provider: dictation.ProviderOpenAI,
|
||||
BaseURL: config.OpenAIBaseURL,
|
||||
APIKey: key,
|
||||
Model: model,
|
||||
Language: stt.Language,
|
||||
})
|
||||
default: // local
|
||||
return dictation.NewLocalTranscriber(dictation.LocalConfig{
|
||||
Binary: stt.LocalBinary,
|
||||
ModelPath: stt.LocalModelPath,
|
||||
NumThreads: stt.NumThreads,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// sttEnvVar returns the conventional environment variable a cloud STT provider's
|
||||
// key is read from.
|
||||
func sttEnvVar(provider string) string {
|
||||
switch provider {
|
||||
case dictation.ProviderGroq:
|
||||
return "GROQ_API_KEY"
|
||||
case dictation.ProviderOpenAI: // == StreamProviderOpenAI ("openai")
|
||||
return "OPENAI_API_KEY"
|
||||
case dictation.StreamProviderDeepgram:
|
||||
return "DEEPGRAM_API_KEY"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// newSTTKeyStatus returns a predicate reporting whether a cloud STT provider
|
||||
// already has a resolvable API key (configured provider, credstore, or env).
|
||||
func newSTTKeyStatus(resolved config.ResolvedConfig, userConfigPath string) func(string) bool {
|
||||
return func(provider string) bool {
|
||||
return resolveSTTKey(resolved, userConfigPath, provider, sttEnvVar(provider)) != ""
|
||||
}
|
||||
}
|
||||
|
||||
// newSaveSTTKey returns a function that stores a cloud STT provider's API key in
|
||||
// the encrypted credential store, so the inline prompt can persist it.
|
||||
func newSaveSTTKey(userConfigPath string) func(string, string) error {
|
||||
return func(provider, key string) error {
|
||||
if strings.TrimSpace(userConfigPath) == "" {
|
||||
return fmt.Errorf("no config path to store the key")
|
||||
}
|
||||
store, err := config.ProviderKeyStoreAt(filepath.Dir(userConfigPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return store.Set(provider, strings.TrimSpace(key))
|
||||
}
|
||||
}
|
||||
|
||||
// resolveSTTKey finds an API key for an STT cloud provider, reusing Zero's
|
||||
// existing credentials (§2 — "zero new credential UI"). Precedence: a matching
|
||||
// configured provider's resolved key, then the encrypted credential store, then
|
||||
// the provider's conventional environment variable.
|
||||
func resolveSTTKey(resolved config.ResolvedConfig, userConfigPath, provider, envVar string) string {
|
||||
if key := keyFromConfiguredProviders(resolved, provider); key != "" {
|
||||
return key
|
||||
}
|
||||
if userConfigPath != "" {
|
||||
if store, err := config.ProviderKeyStoreAt(filepath.Dir(userConfigPath)); err == nil {
|
||||
if key, ok, _ := store.Get(provider); ok && strings.TrimSpace(key) != "" {
|
||||
return strings.TrimSpace(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(os.Getenv(envVar))
|
||||
}
|
||||
|
||||
// keyFromConfiguredProviders returns the resolved API key of a configured
|
||||
// provider that belongs to the given cloud provider (matched by catalog id,
|
||||
// provider name/kind, or base-URL host), or "" when none is configured.
|
||||
func keyFromConfiguredProviders(resolved config.ResolvedConfig, provider string) string {
|
||||
provider = strings.ToLower(provider)
|
||||
profiles := append([]config.ProviderProfile{resolved.Provider}, resolved.Providers...)
|
||||
for _, p := range profiles {
|
||||
if strings.TrimSpace(p.APIKey) == "" {
|
||||
continue
|
||||
}
|
||||
if providerMatches(p, provider) {
|
||||
return strings.TrimSpace(p.APIKey)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func providerMatches(p config.ProviderProfile, provider string) bool {
|
||||
provider = strings.ToLower(provider)
|
||||
// Match on stable identifiers (catalog id, provider name/kind, profile
|
||||
// name) rather than raw BaseURL substring matching: a Groq profile's
|
||||
// BaseURL (api.groq.com/openai/v1) contains "openai", so a substring
|
||||
// check can hand the wrong key to an OpenAI STT lookup and fail auth.
|
||||
for _, f := range []string{
|
||||
strings.ToLower(p.CatalogID),
|
||||
strings.ToLower(p.Provider),
|
||||
strings.ToLower(string(p.ProviderKind)),
|
||||
strings.ToLower(p.Name),
|
||||
} {
|
||||
if f != "" && strings.Contains(f, provider) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// BaseURL: compare the parsed host only, not a raw substring on the
|
||||
// full URL (which lets "openai" match api.groq.com/openai/v1).
|
||||
if u, err := neturl.Parse(p.BaseURL); err == nil && u.Host != "" {
|
||||
if host := strings.ToLower(u.Host); strings.Contains(host, provider) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/dictation"
|
||||
)
|
||||
|
||||
func TestFactoryFallsBackToBatchWhenLocalStreamingNotConfigured(t *testing.T) {
|
||||
t.Setenv("GROQ_API_KEY", "gsk-test")
|
||||
resolved := config.ResolvedConfig{STT: config.STTConfig{
|
||||
Provider: config.STTProviderGroq, // cloud batch
|
||||
// StreamProvider unset → defaults to local, which has no model configured.
|
||||
}}
|
||||
factory := newDictationTranscriberFactory(resolved, "", newDictationServerManager(resolved.STT))
|
||||
|
||||
tr, streaming, err := factory(resolved.STT, true) // prefer streaming
|
||||
if err != nil {
|
||||
t.Fatalf("expected batch fallback, got error: %v", err)
|
||||
}
|
||||
if streaming {
|
||||
t.Error("should have fallen back to batch (streaming=false)")
|
||||
}
|
||||
if tr == nil {
|
||||
t.Error("expected a Groq batch transcriber")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactoryCloudProviderIgnoresLeftoverLocalModel(t *testing.T) {
|
||||
// Regression: pick local once (LocalModelPath gets set), then switch to a
|
||||
// cloud batch provider. streamProvider stays unset (defaults to "local"), so
|
||||
// the streaming factory must NOT fire up the sherpa server on the leftover
|
||||
// path — it should fall back to the cloud batch pipeline the user chose.
|
||||
t.Setenv("GROQ_API_KEY", "gsk-test")
|
||||
stt := config.STTConfig{
|
||||
Provider: config.STTProviderGroq, // cloud batch, chosen now
|
||||
LocalModelPath: "/leftover/model/dir", // set by an earlier local selection
|
||||
}
|
||||
resolved := config.ResolvedConfig{STT: stt}
|
||||
factory := newDictationTranscriberFactory(resolved, "", newDictationServerManager(stt))
|
||||
|
||||
tr, streaming, err := factory(stt, true) // prefer streaming
|
||||
if err != nil {
|
||||
t.Fatalf("expected cloud batch fallback, got error: %v", err)
|
||||
}
|
||||
if streaming {
|
||||
t.Error("a cloud provider must not stream through the local sherpa server")
|
||||
}
|
||||
if tr == nil {
|
||||
t.Error("expected a Groq batch transcriber")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactorySurfacesExplicitCloudStreamingError(t *testing.T) {
|
||||
// Deepgram chosen explicitly but no key resolvable → surface the setup error.
|
||||
resolved := config.ResolvedConfig{STT: config.STTConfig{
|
||||
StreamProvider: config.STTProviderDeepgram,
|
||||
}}
|
||||
factory := newDictationTranscriberFactory(resolved, "", nil)
|
||||
|
||||
_, streaming, err := factory(resolved.STT, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected a setup error for missing Deepgram key")
|
||||
}
|
||||
if !streaming {
|
||||
t.Error("an explicit cloud streaming choice should report streaming=true even on error")
|
||||
}
|
||||
var setupErr *dictation.SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Errorf("want *SetupError, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactoryLocalBatchSetupErrorWhenNoModel(t *testing.T) {
|
||||
resolved := config.ResolvedConfig{STT: config.STTConfig{Provider: config.STTProviderLocal}}
|
||||
factory := newDictationTranscriberFactory(resolved, "", newDictationServerManager(resolved.STT))
|
||||
|
||||
_, _, err := factory(resolved.STT, true)
|
||||
var setupErr *dictation.SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError guiding local model setup, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSTTKeyPrefersConfiguredProvider(t *testing.T) {
|
||||
resolved := config.ResolvedConfig{
|
||||
Providers: []config.ProviderProfile{
|
||||
{Name: "groq", CatalogID: "groq", BaseURL: "https://api.groq.com/openai/v1", APIKey: "gsk-configured"},
|
||||
},
|
||||
}
|
||||
if got := resolveSTTKey(resolved, "", dictation.ProviderGroq, "GROQ_API_KEY"); got != "gsk-configured" {
|
||||
t.Errorf("key = %q, want the configured provider's key", got)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/providermodelcatalog"
|
||||
"github.com/Gitlawb/zero/internal/providermodeldiscovery"
|
||||
)
|
||||
|
||||
@@ -51,6 +52,19 @@ func runProvidersModels(args []string, stdout io.Writer, stderr io.Writer, deps
|
||||
return writeAppError(stderr, err.Error(), exitProvider)
|
||||
}
|
||||
|
||||
// Apply provider-specific model filtering for catalog-backed profiles.
|
||||
// For example, opencode-go-anthropic-compatible only permits Qwen and
|
||||
// MiniMax model IDs; the raw /zen/go/v1/models endpoint returns many more.
|
||||
if profile.CatalogID != "" {
|
||||
filtered := make([]providermodeldiscovery.Model, 0, len(models))
|
||||
for _, model := range models {
|
||||
if providermodelcatalog.ModelIDAllowedForProvider(profile.CatalogID, model.ID) {
|
||||
filtered = append(filtered, model)
|
||||
}
|
||||
}
|
||||
models = filtered
|
||||
}
|
||||
|
||||
if options.json {
|
||||
items := make([]map[string]any, 0, len(models))
|
||||
for _, model := range models {
|
||||
|
||||
@@ -132,6 +132,62 @@ func TestRunProvidersModelsUnknownProviderFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProvidersModelsFiltersForCatalogProfile(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
deps := commandCenterDeps(t)
|
||||
// Override the resolved config to include a catalog-backed profile.
|
||||
deps.resolveConfig = func(_ string, _ config.Overrides) (config.ResolvedConfig, error) {
|
||||
profile := config.ProviderProfile{
|
||||
Name: "opencode-go",
|
||||
ProviderKind: config.ProviderKindAnthropicCompat,
|
||||
CatalogID: "opencode-go-anthropic-compatible",
|
||||
BaseURL: "https://api.example.com/zen/go/v1",
|
||||
APIKey: "sk-test",
|
||||
Model: "minimax-max",
|
||||
}
|
||||
return config.ResolvedConfig{
|
||||
ActiveProvider: "opencode-go",
|
||||
Providers: []config.ProviderProfile{profile},
|
||||
Provider: profile,
|
||||
}, nil
|
||||
}
|
||||
deps.discoverProviderModels = func(_ context.Context, _ config.ProviderProfile) ([]providermodeldiscovery.Model, error) {
|
||||
return []providermodeldiscovery.Model{
|
||||
{ID: "deepseek-chat", Description: "DeepSeek Chat"},
|
||||
{ID: "qwen-max", Description: "Qwen Max"},
|
||||
{ID: "minimax-max-01", Description: "MiniMax Max"},
|
||||
{ID: "kimi-k2", Description: "Kimi K2"},
|
||||
{ID: "glm-4", Description: "GLM-4"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
exitCode := runWithDeps([]string{"providers", "models"}, &stdout, &stderr, deps)
|
||||
|
||||
if exitCode != exitSuccess {
|
||||
t.Fatalf("exit = %d, want %d: %s", exitCode, exitSuccess, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
// Only Qwen and MiniMax model IDs should survive the filter.
|
||||
if strings.Contains(out, "deepseek-chat") {
|
||||
t.Error("output should not contain deepseek-chat (blocked by opencode-go-anthropic filter)")
|
||||
}
|
||||
if strings.Contains(out, "kimi-k2") {
|
||||
t.Error("output should not contain kimi-k2 (blocked by opencode-go-anthropic filter)")
|
||||
}
|
||||
if strings.Contains(out, "glm-4") {
|
||||
t.Error("output should not contain glm-4 (blocked by opencode-go-anthropic filter)")
|
||||
}
|
||||
for _, want := range []string{"qwen-max", "minimax-max-01"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("output missing allowed model %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(out, "2 models discovered") {
|
||||
t.Fatalf("output should show 2 models:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProvidersModelsSurfacesDiscoveryError(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
@@ -306,3 +307,143 @@ func providerCommandArg(value string) string {
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type providerNamesOptions struct {
|
||||
names []string
|
||||
json bool
|
||||
}
|
||||
|
||||
func parseProviderNamesArgs(args []string, want int, usage string) (providerNamesOptions, bool, error) {
|
||||
options := providerNamesOptions{}
|
||||
for _, arg := range args {
|
||||
switch {
|
||||
case arg == "-h" || arg == "--help" || arg == "help":
|
||||
return options, true, nil
|
||||
case arg == "--json":
|
||||
options.json = true
|
||||
case strings.HasPrefix(arg, "-"):
|
||||
return options, false, execUsageError{fmt.Sprintf("unknown flag %q", arg)}
|
||||
default:
|
||||
options.names = append(options.names, arg)
|
||||
}
|
||||
}
|
||||
if len(options.names) != want {
|
||||
return options, false, execUsageError{usage}
|
||||
}
|
||||
return options, false, nil
|
||||
}
|
||||
|
||||
// runProvidersRemove deletes a saved provider profile and its stored API key.
|
||||
// The OAuth token (if any) is kept — logins outlive profiles so re-adding the
|
||||
// provider needs no new browser round-trip; `zero auth logout <name>` removes it.
|
||||
func runProvidersRemove(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int {
|
||||
options, help, err := parseProviderNamesArgs(args, 1, "usage: zero providers remove <name>")
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
if help {
|
||||
if err := writeProvidersHelp(stdout); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
configPath, err := deps.userConfigPath()
|
||||
if err != nil {
|
||||
return writeAppError(stderr, err.Error(), exitCrash)
|
||||
}
|
||||
name := options.names[0]
|
||||
cfg, err := config.RemoveProvider(configPath, name)
|
||||
if err != nil {
|
||||
return writeAppError(stderr, err.Error(), exitCrash)
|
||||
}
|
||||
// Delete the key from the store BESIDE the config being edited — the same
|
||||
// store setup/rename write to — not the default-path store, so a
|
||||
// non-default config path cannot leave the encrypted key behind.
|
||||
keyRemoved, keyErr := removeStoredProviderKeyAt(configPath, name)
|
||||
if options.json {
|
||||
payload := map[string]any{
|
||||
"removed": name,
|
||||
"keyRemoved": keyRemoved,
|
||||
"activeProvider": cfg.ActiveProvider,
|
||||
"configPath": configPath,
|
||||
}
|
||||
if keyErr != nil {
|
||||
// A lingering secret must not read as a clean removal.
|
||||
payload["keyError"] = keyErr.Error()
|
||||
}
|
||||
if err := writePrettyJSON(stdout, payload); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
if _, err := fmt.Fprintf(stdout, "Removed provider %s\n", name); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
if keyErr != nil {
|
||||
if _, err := fmt.Fprintf(stderr, "warning: its stored API key could not be deleted and remains in the credential store: %v\n", keyErr); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
} else if keyRemoved {
|
||||
if _, err := fmt.Fprintln(stdout, "Deleted its stored API key."); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
if active := strings.TrimSpace(cfg.ActiveProvider); active != "" {
|
||||
if _, err := fmt.Fprintf(stdout, "Active provider: %s\n", active); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
} else {
|
||||
if _, err := fmt.Fprintln(stdout, "No providers remain — run zero setup to add one."); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
|
||||
// removeStoredProviderKeyAt deletes a provider's API key from the credential
|
||||
// store co-located with configPath (the store SecureProviderProfile captured
|
||||
// it into and RenameProvider migrates within).
|
||||
func removeStoredProviderKeyAt(configPath string, provider string) (bool, error) {
|
||||
store, err := config.ProviderKeyStoreAt(filepath.Dir(configPath))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return store.Delete(provider)
|
||||
}
|
||||
|
||||
// runProvidersRename renames a saved provider profile, migrating its stored
|
||||
// API key and the activeProvider pointer along with it (config.RenameProvider).
|
||||
func runProvidersRename(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int {
|
||||
options, help, err := parseProviderNamesArgs(args, 2, "usage: zero providers rename <old> <new>")
|
||||
if err != nil {
|
||||
return writeExecUsageError(stderr, err.Error())
|
||||
}
|
||||
if help {
|
||||
if err := writeProvidersHelp(stdout); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
configPath, err := deps.userConfigPath()
|
||||
if err != nil {
|
||||
return writeAppError(stderr, err.Error(), exitCrash)
|
||||
}
|
||||
cfg, err := config.RenameProvider(configPath, options.names[0], options.names[1])
|
||||
if err != nil {
|
||||
return writeAppError(stderr, err.Error(), exitCrash)
|
||||
}
|
||||
if options.json {
|
||||
if err := writePrettyJSON(stdout, map[string]any{
|
||||
"renamed": map[string]string{"from": options.names[0], "to": options.names[1]},
|
||||
"activeProvider": cfg.ActiveProvider,
|
||||
"configPath": configPath,
|
||||
}); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
if _, err := fmt.Fprintf(stdout, "Renamed provider %s to %s\n", options.names[0], options.names[1]); err != nil {
|
||||
return exitCrash
|
||||
}
|
||||
return exitSuccess
|
||||
}
|
||||
@@ -234,3 +234,48 @@ func writeProviderOnboardingConfig(t *testing.T, path string, cfg config.FileCon
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunProvidersRemoveDeletesKeyBesideConfig: the stored key must be deleted
|
||||
// from the credential store CO-LOCATED with the config being edited (where
|
||||
// SecureProviderProfile captured it), not the default-path store.
|
||||
func TestRunProvidersRemoveDeletesKeyBesideConfig(t *testing.T) {
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.json")
|
||||
seed := `{"activeProvider":"gw","providers":[{"name":"gw","provider_kind":"openai-compatible","baseURL":"https://gw.example.com/v1","apiKeyStored":true,"model":"m1"},{"name":"other","provider_kind":"openai-compatible","baseURL":"https://o.example.com/v1","model":"m2"}]}`
|
||||
if err := os.WriteFile(configPath, []byte(seed), 0o600); err != nil {
|
||||
t.Fatalf("seed config: %v", err)
|
||||
}
|
||||
store, err := config.ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("gw", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed key: %v", err)
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
deps := appDeps{userConfigPath: func() (string, error) { return configPath, nil }}
|
||||
if code := runWithDeps([]string{"providers", "remove", "gw", "--json"}, &stdout, &stderr, deps); code != exitSuccess {
|
||||
t.Fatalf("remove failed: code=%d stderr=%s", code, stderr.String())
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Removed string `json:"removed"`
|
||||
KeyRemoved bool `json:"keyRemoved"`
|
||||
KeyError string `json:"keyError"`
|
||||
ActiveProvider string `json:"activeProvider"`
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("decode JSON: %v\n%s", err, stdout.String())
|
||||
}
|
||||
if payload.Removed != "gw" || !payload.KeyRemoved || payload.KeyError != "" {
|
||||
t.Fatalf("unexpected payload: %+v", payload)
|
||||
}
|
||||
if payload.ActiveProvider != "other" {
|
||||
t.Fatalf("active must hand off, got %q", payload.ActiveProvider)
|
||||
}
|
||||
if _, ok, _ := store.Get("gw"); ok {
|
||||
t.Fatalf("stored key must be deleted from the store beside the config")
|
||||
}
|
||||
}
|
||||
@@ -457,7 +457,9 @@ func selectProviderForCheck(resolved config.ResolvedConfig, name string) (config
|
||||
}
|
||||
|
||||
func validateProviderRuntimeReady(profile config.ProviderProfile) error {
|
||||
hasCredential := providerProfileHasCredential(profile)
|
||||
// A stored OAuth login (e.g. `zero auth chatgpt`) is a credential too: a
|
||||
// keyless token-login profile must pass the readiness check instead of being
|
||||
// told to set an API key it will never have.
|
||||
if profile.CatalogID != "" {
|
||||
descriptor, err := providercatalog.Require(profile.CatalogID)
|
||||
if err != nil {
|
||||
@@ -466,23 +468,19 @@ func validateProviderRuntimeReady(profile config.ProviderProfile) error {
|
||||
if !providercatalog.RuntimeSupported(descriptor) {
|
||||
return fmt.Errorf("provider %q uses transport %q: %s", descriptor.ID, descriptor.Transport, providercatalog.RuntimeUnsupportedReason(descriptor))
|
||||
}
|
||||
if descriptor.RequiresAuth && !hasCredential {
|
||||
apiKeyEnv := strings.TrimSpace(profile.APIKeyEnv)
|
||||
if apiKeyEnv == "" && len(descriptor.AuthEnvVars) > 0 {
|
||||
apiKeyEnv = descriptor.AuthEnvVars[0]
|
||||
}
|
||||
if apiKeyEnv != "" {
|
||||
return fmt.Errorf("provider %s requires API key; set %s", profile.Name, apiKeyEnv)
|
||||
}
|
||||
return fmt.Errorf("provider %s requires API key", profile.Name)
|
||||
}
|
||||
}
|
||||
if providerHasOAuthLogin(profile, oauthLoggedInProviders()) {
|
||||
return nil
|
||||
}
|
||||
switch profile.ProviderKind {
|
||||
case config.ProviderKindOpenAI, config.ProviderKindAnthropic, config.ProviderKindGoogle:
|
||||
if !hasCredential {
|
||||
return fmt.Errorf("provider %s requires API key", profile.Name)
|
||||
// Delegates to the single source of truth shared with the TUI and the
|
||||
// setup wizard, so `providers check` never disagrees with them about
|
||||
// whether a saved provider (including a no-auth custom endpoint) is
|
||||
// actually missing a credential.
|
||||
if envVar, missing := profile.MissingCredentialEnv(); missing {
|
||||
if envVar != "" {
|
||||
return fmt.Errorf("provider %s requires API key; set %s", profile.Name, envVar)
|
||||
}
|
||||
return fmt.Errorf("provider %s requires API key", profile.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
)
|
||||
|
||||
// Regression for issue #555's follow-up: `zero providers check` must not
|
||||
// error that a no-auth custom endpoint requires an API key, matching what
|
||||
// /model and /providers already treat as usable.
|
||||
func TestValidateProviderRuntimeReadyCustomEndpoint(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
profile config.ProviderProfile
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "custom openai compatible with no credential configured",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
Model: "custom-model",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "custom openai compatible with stale legacy default env",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
APIKeyEnv: "OPENAI_API_KEY",
|
||||
Model: "custom-model",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "custom openai compatible with explicit non-default env still requires it",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
APIKeyEnv: "LLAMA_CPP_API_KEY",
|
||||
Model: "custom-model",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "catalog provider missing key still errors",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "groq",
|
||||
CatalogID: "groq",
|
||||
Model: "llama-3.3-70b-versatile",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
err := validateProviderRuntimeReady(c.profile)
|
||||
if (err != nil) != c.wantErr {
|
||||
t.Fatalf("validateProviderRuntimeReady() error = %v, wantErr %v", err, c.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+21
-41
@@ -322,7 +322,9 @@ func setupRequired(resolved config.ResolvedConfig) bool {
|
||||
// the candidate set is permissive (profile name + catalog ID).
|
||||
func providerHasOAuthLogin(profile config.ProviderProfile, oauthLogins map[string]bool) bool {
|
||||
for _, name := range profile.OAuthLoginCandidates() {
|
||||
if oauthLogins[name] {
|
||||
// Store keys are normalized to lower case (oauth.ProviderKey), so the
|
||||
// presence check must compare the same way.
|
||||
if oauthLogins[strings.ToLower(strings.TrimSpace(name))] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -344,18 +346,19 @@ func oauthLoggedInProviders() map[string]bool {
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status.HasToken {
|
||||
out[strings.TrimPrefix(status.Key, oauth.KeyPrefixProvider)] = true
|
||||
out[strings.ToLower(strings.TrimPrefix(status.Key, oauth.KeyPrefixProvider))] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// firstUsableProvider returns the saved provider best suited to run without
|
||||
// onboarding: the first usable (inline credential present, or no-auth/local)
|
||||
// non-local provider, else the first usable local one. It lets the CLI fall back
|
||||
// to an already-configured login when the active provider happens to lack a
|
||||
// credential, instead of re-running onboarding every launch.
|
||||
// onboarding: the first usable (inline credential, stored OAuth login, or
|
||||
// no-auth/local) non-local provider, else the first usable local one. It lets
|
||||
// the CLI fall back to an already-configured login when the active provider
|
||||
// happens to lack a credential, instead of re-running onboarding every launch.
|
||||
func firstUsableProvider(providers []config.ProviderProfile) (config.ProviderProfile, bool) {
|
||||
logins := oauthLoggedInProviders()
|
||||
var localFallback config.ProviderProfile
|
||||
haveLocal := false
|
||||
for _, profile := range providers {
|
||||
@@ -371,7 +374,11 @@ func firstUsableProvider(providers []config.ProviderProfile) (config.ProviderPro
|
||||
continue
|
||||
}
|
||||
}
|
||||
if _, missing := setupMissingCredentialEnv(profile); missing {
|
||||
// A stored OAuth login (e.g. `zero auth login xai`) is a credential too, even
|
||||
// when the profile has no inline key / env var — mirrors setupRequired and
|
||||
// usableSavedProviders so this fallback doesn't force onboarding for a
|
||||
// provider the user is already authenticated with.
|
||||
if _, missing := setupMissingCredentialEnv(profile); missing && !providerHasOAuthLogin(profile, logins) {
|
||||
continue
|
||||
}
|
||||
if providerProfileIsLocal(profile) {
|
||||
@@ -496,41 +503,14 @@ func setupTryThisExample(profile config.ProviderProfile) string {
|
||||
return example
|
||||
}
|
||||
|
||||
// setupMissingCredentialEnv reports the environment variable a saved profile
|
||||
// is expected to read an API key from, and whether that expectation is
|
||||
// unmet. It delegates to config.ProviderProfile.MissingCredentialEnv, the
|
||||
// single source of truth shared with the TUI's provider status/wizard
|
||||
// surfaces and the CLI's `providers check` readiness gate, so all three never
|
||||
// disagree about whether a saved provider is missing a credential.
|
||||
func setupMissingCredentialEnv(profile config.ProviderProfile) (string, bool) {
|
||||
if providerProfileHasCredential(profile) {
|
||||
return "", false
|
||||
}
|
||||
if catalogID := strings.TrimSpace(profile.CatalogID); catalogID != "" {
|
||||
descriptor, err := providercatalog.Require(catalogID)
|
||||
if err != nil || !descriptor.RequiresAuth {
|
||||
return "", false
|
||||
}
|
||||
return firstNonEmptyCLI(profile.APIKeyEnv, setupProviderEnvVar(descriptor)), true
|
||||
}
|
||||
|
||||
switch normalizedSetupProviderKind(profile) {
|
||||
case config.ProviderKindOpenAI, config.ProviderKindOpenAICompatible:
|
||||
return firstNonEmptyCLI(profile.APIKeyEnv, "OPENAI_API_KEY"), true
|
||||
case config.ProviderKindAnthropic, config.ProviderKindAnthropicCompat:
|
||||
return firstNonEmptyCLI(profile.APIKeyEnv, "ANTHROPIC_API_KEY"), true
|
||||
case config.ProviderKindGoogle:
|
||||
return firstNonEmptyCLI(profile.APIKeyEnv, "GEMINI_API_KEY"), true
|
||||
default:
|
||||
if strings.TrimSpace(profile.APIKeyEnv) != "" {
|
||||
return strings.TrimSpace(profile.APIKeyEnv), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func normalizedSetupProviderKind(profile config.ProviderProfile) config.ProviderKind {
|
||||
if kind := strings.TrimSpace(string(profile.ProviderKind)); kind != "" {
|
||||
return config.ProviderKind(strings.ToLower(kind))
|
||||
}
|
||||
if provider := strings.TrimSpace(profile.Provider); provider != "" {
|
||||
return config.ProviderKind(strings.ToLower(provider))
|
||||
}
|
||||
return ""
|
||||
return profile.MissingCredentialEnv()
|
||||
}
|
||||
|
||||
func setupCheckCommand(name string) string {
|
||||
|
||||
@@ -2,11 +2,14 @@ package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
)
|
||||
|
||||
func TestFirstUsableProviderPrefersRemoteKeyed(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "ollama", CatalogID: "ollama", BaseURL: "http://localhost:11434/v1", APIKey: "k"}, // usable but local
|
||||
{Name: "moonshot", CatalogID: "moonshot", BaseURL: "https://api.moonshot.ai/v1", APIKey: "k"}, // usable, remote
|
||||
@@ -19,6 +22,7 @@ func TestFirstUsableProviderPrefersRemoteKeyed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFirstUsableProviderFallsBackToLocal(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "xai", CatalogID: "xai", APIKeyEnv: "XAI_API_KEY"}, // not usable
|
||||
{Name: "ollama", CatalogID: "ollama", BaseURL: "http://localhost:11434/v1", APIKey: "k"}, // local, usable
|
||||
@@ -30,6 +34,7 @@ func TestFirstUsableProviderFallsBackToLocal(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFirstUsableProviderNoneUsable(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "xai", CatalogID: "xai", APIKeyEnv: "XAI_API_KEY"},
|
||||
{Name: "openai", CatalogID: "openai", APIKeyEnv: "OPENAI_API_KEY"},
|
||||
@@ -42,6 +47,10 @@ func TestFirstUsableProviderNoneUsable(t *testing.T) {
|
||||
// A keyless local proxy (chatgpt-proxy, RequiresAuth=false) is usable without a
|
||||
// credential, so it can serve as a fallback rather than forcing onboarding.
|
||||
func TestFirstUsableProviderAcceptsKeylessLocalProxy(t *testing.T) {
|
||||
// Isolate the OAuth token store: on a developer machine with a real xai
|
||||
// login, the env-only xai profile would count as usable and win over the
|
||||
// local proxy this test is about.
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "xai", CatalogID: "xai", APIKeyEnv: "XAI_API_KEY"},
|
||||
{Name: "chatgpt", CatalogID: "chatgpt-proxy", BaseURL: "http://localhost:10531/v1"},
|
||||
@@ -56,6 +65,7 @@ func TestFirstUsableProviderAcceptsKeylessLocalProxy(t *testing.T) {
|
||||
// BaseURL has no endpoint, so it must be skipped rather than selected as a
|
||||
// fallback that fails at first use. A stale CatalogID with a BaseURL still works.
|
||||
func TestFirstUsableProviderSkipsUnresolvableCatalogWithoutBaseURL(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "ghost", CatalogID: "no-such-catalog-entry", APIKey: "k"}, // unusable: no endpoint
|
||||
{Name: "custom", CatalogID: "no-such-catalog-entry", BaseURL: "https://api.custom.test/v1", APIKey: "k"},
|
||||
@@ -66,6 +76,29 @@ func TestFirstUsableProviderSkipsUnresolvableCatalogWithoutBaseURL(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
// An OAuth-only provider (no inline key, no env var) must be selectable as a
|
||||
// fallback, matching setupRequired/usableSavedProviders — otherwise a fully
|
||||
// authenticated user gets forced back into onboarding when activeProvider
|
||||
// goes stale.
|
||||
func TestFirstUsableProviderRecognizesOAuthLogin(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
store, err := oauth.NewStore(oauth.StoreOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore: %v", err)
|
||||
}
|
||||
if err := store.Save(oauth.ProviderKey("xai"), oauth.Token{AccessToken: "tok", ExpiresAt: time.Now().Add(time.Hour)}); err != nil {
|
||||
t.Fatalf("seed token: %v", err)
|
||||
}
|
||||
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "xai", CatalogID: "xai", APIKeyEnv: "XAI_API_KEY"}, // no inline key/env, but logged in via OAuth
|
||||
}
|
||||
got, ok := firstUsableProvider(providers)
|
||||
if !ok || got.Name != "xai" {
|
||||
t.Fatalf("want OAuth-logged-in provider (xai), got %q ok=%v", got.Name, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderProfileIsLocal(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -89,3 +122,32 @@ func TestProviderProfileIsLocal(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFirstUsableProviderAcceptsOAuthLoginProfile: a keyless catalog profile
|
||||
// whose credential is a stored OAuth login (the shape the login flows persist)
|
||||
// must be usable during active-pointer recovery — the same rule setupRequired
|
||||
// and usableSavedProviders apply — instead of falling through to onboarding.
|
||||
func TestFirstUsableProviderAcceptsOAuthLoginProfile(t *testing.T) {
|
||||
withAuthStore(t)
|
||||
providers := []config.ProviderProfile{
|
||||
{Name: "chatgpt", CatalogID: "chatgpt", BaseURL: "https://chatgpt.com/backend-api/codex", Model: "gpt-5.5"},
|
||||
}
|
||||
|
||||
// No login stored: still not usable.
|
||||
if got, ok := firstUsableProvider(providers); ok {
|
||||
t.Fatalf("keyless profile without a login must not be usable, got %q", got.Name)
|
||||
}
|
||||
|
||||
store, err := oauth.NewStore(oauth.StoreOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("oauth store: %v", err)
|
||||
}
|
||||
if err := store.Save(oauth.ProviderKey("chatgpt"), oauth.Token{AccessToken: "bearer-123"}); err != nil {
|
||||
t.Fatalf("save token: %v", err)
|
||||
}
|
||||
|
||||
got, ok := firstUsableProvider(providers)
|
||||
if !ok || got.Name != "chatgpt" {
|
||||
t.Fatalf("want OAuth-login profile to be usable, got %q ok=%v", got.Name, ok)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,54 @@ func TestSetupMissingCredentialEnv(t *testing.T) {
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// Regression for issue #555: a custom OpenAI-compatible profile saved
|
||||
// with no credential (e.g. a local llama.cpp server with no auth) must
|
||||
// not be reported as missing one, or it gets filtered out of /model on
|
||||
// the next run even though it was configured exactly as intended.
|
||||
name: "custom openai compatible with no credential configured",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "custom openai compatible with explicit api key env",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
APIKeyEnv: "LLAMA_CPP_API_KEY",
|
||||
},
|
||||
wantEnv: "LLAMA_CPP_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// Self-heal regression: profiles saved by the pre-fix wizard were
|
||||
// stamped with the catalog's own guessed default (OPENAI_API_KEY)
|
||||
// even when the endpoint needed no auth. That stale value must be
|
||||
// treated the same as unset so already-broken saved profiles start
|
||||
// working again without a config.json rewrite.
|
||||
name: "custom openai compatible with stale legacy default env",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
APIKeyEnv: "OPENAI_API_KEY",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "custom anthropic compatible with no credential configured",
|
||||
profile: config.ProviderProfile{
|
||||
Name: "local-proxy",
|
||||
CatalogID: "custom-anthropic-compatible",
|
||||
BaseURL: "http://localhost:9000/anthropic",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/credstore"
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
)
|
||||
|
||||
// ProviderKeyStoreAt opens the encrypted credential store whose file backend lives
|
||||
@@ -168,6 +169,86 @@ func (profile ProviderProfile) HasConfiguredCredential() bool {
|
||||
strings.TrimSpace(profile.AuthHeaderValue) != ""
|
||||
}
|
||||
|
||||
// MissingCredentialEnv reports the environment variable this profile is
|
||||
// expected to read an API key from, and whether that expectation is
|
||||
// currently unmet. It is the single source of truth for "does this saved
|
||||
// provider need a credential it doesn't have" — the CLI (`providers check`,
|
||||
// `usableSavedProviders`) and TUI (`/providers` status, provider wizard
|
||||
// summary) surfaces all delegate here so they never disagree about a
|
||||
// profile's status.
|
||||
//
|
||||
// For the catalog's two "bring your own endpoint" descriptors
|
||||
// (custom-openai-compatible, custom-anthropic-compatible), RequiresAuth is
|
||||
// just a wizard template default, not a fact about the user's actual
|
||||
// endpoint: a custom target may need no auth at all. So for those two only,
|
||||
// a credential is expected exclusively when this profile carries its own
|
||||
// explicit APIKeyEnv that differs from the catalog's guessed default — a
|
||||
// value matching that default is indistinguishable from the wizard's old
|
||||
// auto-stamp bug (issue #555) and is treated the same as unset, which also
|
||||
// self-heals profiles saved before that bug was fixed.
|
||||
func (profile ProviderProfile) MissingCredentialEnv() (string, bool) {
|
||||
if profile.HasConfiguredCredential() {
|
||||
return "", false
|
||||
}
|
||||
if catalogID := strings.TrimSpace(profile.CatalogID); catalogID != "" {
|
||||
descriptor, err := providercatalog.Require(catalogID)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
defaultEnvVar := firstNonEmptyTrimmed(descriptor.AuthEnvVars...)
|
||||
if descriptor.Custom {
|
||||
envVar := strings.TrimSpace(profile.APIKeyEnv)
|
||||
if envVar == "" || envVar == defaultEnvVar {
|
||||
return "", false
|
||||
}
|
||||
return envVar, true
|
||||
}
|
||||
if !descriptor.RequiresAuth {
|
||||
return "", false
|
||||
}
|
||||
if envVar := strings.TrimSpace(profile.APIKeyEnv); envVar != "" {
|
||||
return envVar, true
|
||||
}
|
||||
return defaultEnvVar, true
|
||||
}
|
||||
|
||||
switch profile.normalizedProviderKind() {
|
||||
case ProviderKindOpenAI, ProviderKindOpenAICompatible:
|
||||
return firstNonEmptyTrimmed(profile.APIKeyEnv, "OPENAI_API_KEY"), true
|
||||
case ProviderKindAnthropic, ProviderKindAnthropicCompat:
|
||||
return firstNonEmptyTrimmed(profile.APIKeyEnv, "ANTHROPIC_API_KEY"), true
|
||||
case ProviderKindGoogle:
|
||||
return firstNonEmptyTrimmed(profile.APIKeyEnv, "GEMINI_API_KEY"), true
|
||||
default:
|
||||
if envVar := strings.TrimSpace(profile.APIKeyEnv); envVar != "" {
|
||||
return envVar, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// normalizedProviderKind resolves this profile's kind, falling back to the
|
||||
// legacy Provider string field (lowercased) when ProviderKind is unset —
|
||||
// matching how the resolver (resolver.go) treats the two fields as synonyms.
|
||||
func (profile ProviderProfile) normalizedProviderKind() ProviderKind {
|
||||
if kind := strings.TrimSpace(string(profile.ProviderKind)); kind != "" {
|
||||
return ProviderKind(strings.ToLower(kind))
|
||||
}
|
||||
if provider := strings.TrimSpace(profile.Provider); provider != "" {
|
||||
return ProviderKind(strings.ToLower(provider))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstNonEmptyTrimmed(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// OAuthLoginCandidates returns the login names to try, in order, when resolving
|
||||
// this profile's OAuth token — used by the runtime bearer resolver, the Codex
|
||||
// account-header resolver, and the onboarding presence check so all three agree.
|
||||
|
||||
@@ -238,3 +238,82 @@ func TestOAuthLoginCandidates(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderProfileMissingCredentialEnv(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
profile ProviderProfile
|
||||
wantEnv string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "catalog provider requiring auth",
|
||||
profile: ProviderProfile{Name: "groq", CatalogID: "groq"},
|
||||
wantEnv: "GROQ_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "local catalog provider",
|
||||
profile: ProviderProfile{Name: "local", CatalogID: "ollama"},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "credential resolved via inline key",
|
||||
profile: ProviderProfile{Name: "openai", ProviderKind: ProviderKindOpenAI, APIKey: "sk-test"},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// issue #555: a custom endpoint left with no credential means "no
|
||||
// auth needed" (e.g. a local llama.cpp server), not "missing".
|
||||
name: "custom openai compatible with no credential configured",
|
||||
profile: ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
BaseURL: "http://192.168.1.50:8080/v1",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "custom openai compatible with explicit non-default api key env",
|
||||
profile: ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
APIKeyEnv: "LLAMA_CPP_API_KEY",
|
||||
},
|
||||
wantEnv: "LLAMA_CPP_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// Self-heal: a profile stamped by the pre-fix wizard with the
|
||||
// catalog's own guessed default is indistinguishable from that bug
|
||||
// and must not stay filtered out after upgrading.
|
||||
name: "custom openai compatible with stale legacy default env",
|
||||
profile: ProviderProfile{
|
||||
Name: "local-llama",
|
||||
CatalogID: "custom-openai-compatible",
|
||||
APIKeyEnv: "OPENAI_API_KEY",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "openai compatible without catalog falls back to provider kind",
|
||||
profile: ProviderProfile{Name: "custom", ProviderKind: ProviderKindOpenAICompatible},
|
||||
wantEnv: "OPENAI_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "legacy Provider string field fallback",
|
||||
profile: ProviderProfile{Name: "legacy", Provider: "anthropic"},
|
||||
wantEnv: "ANTHROPIC_API_KEY",
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
gotEnv, got := c.profile.MissingCredentialEnv()
|
||||
if got != c.want || gotEnv != c.wantEnv {
|
||||
t.Fatalf("MissingCredentialEnv() = (%q, %v), want (%q, %v)", gotEnv, got, c.wantEnv, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
package config
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultMCPServers returns the MCP servers Zero ships ENABLED by default so web
|
||||
// search and scraping work out of the box with no setup and no API key. They are
|
||||
@@ -29,3 +32,29 @@ func IsDefaultMCPServer(name string) bool {
|
||||
_, ok := DefaultMCPServers()[strings.TrimSpace(name)]
|
||||
return ok
|
||||
}
|
||||
|
||||
// IsUnconfiguredDefault reports whether server is one of Zero's built-in
|
||||
// defaults that the user never wrote an entry for in their config — i.e. it is
|
||||
// running with whatever Zero ships (e.g. keyless Firecrawl, no credentials).
|
||||
//
|
||||
// Both conditions below must hold:
|
||||
// - !server.configured: the user's JSON never declared an object for this
|
||||
// server key at all (set by MCPServerConfig.UnmarshalJSON only when it
|
||||
// actually ran for this key). Any explicit action — including a
|
||||
// disable/enable toggle like `zero mcp enable firecrawl` that leaves the
|
||||
// resolved value unchanged — sets configured, so it always counts as
|
||||
// user-configured, even though the value comparison below could not tell
|
||||
// the difference on its own.
|
||||
// - reflect.DeepEqual(def, server): the value still matches the default.
|
||||
// This is the fallback for callers that construct MCPServerConfig
|
||||
// directly rather than through the JSON/merge pipeline (server.configured
|
||||
// is then always false) — without it, any hand-built config with
|
||||
// different field values would be misreported as unconfigured.
|
||||
//
|
||||
// Callers use this to tell "server we turned on for the user" apart from
|
||||
// "server the user configured themselves," e.g. to avoid warning loudly when
|
||||
// an out-of-the-box default that was never given credentials fails to connect.
|
||||
func IsUnconfiguredDefault(name string, server MCPServerConfig) bool {
|
||||
def, ok := DefaultMCPServers()[strings.TrimSpace(name)]
|
||||
return ok && !server.configured && reflect.DeepEqual(def, server)
|
||||
}
|
||||
@@ -49,6 +49,69 @@ func TestResolveMCPUserCanDisableDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsUnconfiguredDefault(t *testing.T) {
|
||||
if !IsUnconfiguredDefault("firecrawl", DefaultMCPServers()["firecrawl"]) {
|
||||
t.Fatal("an untouched firecrawl default should be reported as unconfigured")
|
||||
}
|
||||
if IsUnconfiguredDefault("firecrawl", MCPServerConfig{Type: "http", URL: "http://localhost:3002/mcp"}) {
|
||||
t.Fatal("a server overriding the default URL is no longer unconfigured")
|
||||
}
|
||||
if IsUnconfiguredDefault("firecrawl", MCPServerConfig{Type: "http", URL: "https://mcp.firecrawl.dev/v2/mcp", Auth: "bearer"}) {
|
||||
t.Fatal("a server with credentials added is no longer unconfigured")
|
||||
}
|
||||
if IsUnconfiguredDefault("not-a-default", MCPServerConfig{}) {
|
||||
t.Fatal("a server with no matching default can never be unconfigured-default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMCPExplicitReenableIsNotUnconfiguredDefault(t *testing.T) {
|
||||
// `zero mcp enable firecrawl` after a prior disable writes {"disabled":false}
|
||||
// explicitly. The resolved value is identical to the untouched default (both
|
||||
// enabled, no credentials), but the user DID take an explicit action here, so
|
||||
// IsUnconfiguredDefault must not treat it as untouched (issue #563 review).
|
||||
path := filepath.Join(t.TempDir(), "config.json")
|
||||
if err := os.WriteFile(path, []byte(`{"mcp":{"servers":{"firecrawl":{"disabled":false}}}}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := ResolveMCP(ResolveOptions{UserConfigPath: path})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveMCP: %v", err)
|
||||
}
|
||||
firecrawl := cfg.Servers["firecrawl"]
|
||||
if firecrawl.Disabled {
|
||||
t.Fatalf("explicit re-enable should leave the server enabled: %#v", firecrawl)
|
||||
}
|
||||
if IsUnconfiguredDefault("firecrawl", firecrawl) {
|
||||
t.Fatal("an explicit enable/disable toggle must count as user-configured, even though the resolved value matches the default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMCPExplicitRedeclareOfDefaultValuesIsNotUnconfiguredDefault(t *testing.T) {
|
||||
// A user who copies firecrawl's exact default type/url into their config
|
||||
// (e.g. from an example file, planning to add credentials later) produces a
|
||||
// resolved value byte-identical to DefaultMCPServers()["firecrawl"] — the
|
||||
// same trap TestResolveMCPExplicitReenableIsNotUnconfiguredDefault covers for
|
||||
// the disabled toggle. IsUnconfiguredDefault must still treat this as
|
||||
// user-configured because the user's JSON declared an entry for it, even
|
||||
// though a plain resolved-value comparison could not tell the difference.
|
||||
path := filepath.Join(t.TempDir(), "config.json")
|
||||
if err := os.WriteFile(path, []byte(`{"mcp":{"servers":{"firecrawl":{"type":"http","url":"https://mcp.firecrawl.dev/v2/mcp"}}}}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := ResolveMCP(ResolveOptions{UserConfigPath: path})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveMCP: %v", err)
|
||||
}
|
||||
firecrawl := cfg.Servers["firecrawl"]
|
||||
want := DefaultMCPServers()["firecrawl"]
|
||||
if firecrawl.Type != want.Type || firecrawl.URL != want.URL || firecrawl.Disabled != want.Disabled {
|
||||
t.Fatalf("expected the resolved value to match the default's fields exactly: %#v", firecrawl)
|
||||
}
|
||||
if IsUnconfiguredDefault("firecrawl", firecrawl) {
|
||||
t.Fatal("redeclaring the default's exact values is still an explicit user configuration, not an untouched default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMCPUserCanOverrideDefaultURLKeepingOtherFields(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.json")
|
||||
// Point firecrawl at a self-hosted instance; the default's Type must survive.
|
||||
|
||||
@@ -127,6 +127,9 @@ func Resolve(options ResolveOptions) (ResolvedConfig, error) {
|
||||
return ResolvedConfig{}, fmt.Errorf("invalid notify.focusMode %q: expected unfocused, always, or focused", focusMode)
|
||||
}
|
||||
}
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return ResolvedConfig{}, err
|
||||
}
|
||||
|
||||
providers, active, err := normalizeProviders(cfg.Providers, cfg.ActiveProvider, options.Env)
|
||||
if err != nil {
|
||||
@@ -150,6 +153,7 @@ func Resolve(options ResolveOptions) (ResolvedConfig, error) {
|
||||
Preferences: cfg.Preferences,
|
||||
KeyBindings: cfg.KeyBindings,
|
||||
LocalControl: cfg.LocalControl,
|
||||
STT: cfg.STT,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -223,6 +227,9 @@ func mergeConfig(dst *FileConfig, src FileConfig) {
|
||||
if src.Preferences.FavoriteModels != nil {
|
||||
dst.Preferences.FavoriteModels = normalizeFavoriteModels(src.Preferences.FavoriteModels)
|
||||
}
|
||||
if src.Preferences.RecentModels != nil {
|
||||
dst.Preferences.RecentModels = NormalizeRecentModels(src.Preferences.RecentModels)
|
||||
}
|
||||
if src.Preferences.Recaps != nil {
|
||||
dst.Preferences.Recaps = src.Preferences.Recaps
|
||||
}
|
||||
@@ -231,6 +238,7 @@ func mergeConfig(dst *FileConfig, src FileConfig) {
|
||||
}
|
||||
mergeLocalControlConfig(&dst.LocalControl, src.LocalControl)
|
||||
mergeKeyBindings(&dst.KeyBindings, src.KeyBindings)
|
||||
mergeSTTConfig(&dst.STT, src.STT)
|
||||
}
|
||||
|
||||
func mergeProjectConfig(dst *FileConfig, src FileConfig) error {
|
||||
@@ -681,6 +689,7 @@ func applyOverrides(cfg *FileConfig, overrides Overrides) {
|
||||
}
|
||||
mergeLocalControlConfig(&cfg.LocalControl, overrides.LocalControl)
|
||||
mergeKeyBindings(&cfg.KeyBindings, overrides.KeyBindings)
|
||||
mergeSTTConfig(&cfg.STT, overrides.STT)
|
||||
for _, provider := range overrides.Providers {
|
||||
mergeProvider(cfg, provider)
|
||||
}
|
||||
@@ -734,6 +743,91 @@ func mergeKeyBindings(dst *KeyBindingsConfig, src KeyBindingsConfig) {
|
||||
}
|
||||
}
|
||||
|
||||
// mergeSTTConfig overlays any set field of src onto dst. Each field carries its
|
||||
// own "unset" sentinel (empty string, 0, or nil *bool), matching how the other
|
||||
// section mergers detect intent.
|
||||
func mergeSTTConfig(dst *STTConfig, src STTConfig) {
|
||||
if src.Provider != "" {
|
||||
dst.Provider = src.Provider
|
||||
}
|
||||
if src.StreamProvider != "" {
|
||||
dst.StreamProvider = src.StreamProvider
|
||||
}
|
||||
if src.Streaming != nil {
|
||||
dst.Streaming = src.Streaming
|
||||
}
|
||||
if src.Model != "" {
|
||||
dst.Model = src.Model
|
||||
}
|
||||
if src.StreamModel != "" {
|
||||
dst.StreamModel = src.StreamModel
|
||||
}
|
||||
if src.LocalModelPath != "" {
|
||||
dst.LocalModelPath = src.LocalModelPath
|
||||
}
|
||||
if src.LocalBinary != "" {
|
||||
dst.LocalBinary = src.LocalBinary
|
||||
}
|
||||
if src.LocalServerBinary != "" {
|
||||
dst.LocalServerBinary = src.LocalServerBinary
|
||||
}
|
||||
if src.LocalServerPort != 0 {
|
||||
dst.LocalServerPort = src.LocalServerPort
|
||||
}
|
||||
if src.EngineVersion != "" {
|
||||
dst.EngineVersion = src.EngineVersion
|
||||
}
|
||||
if src.NumThreads != 0 {
|
||||
dst.NumThreads = src.NumThreads
|
||||
}
|
||||
if src.Language != "" {
|
||||
dst.Language = src.Language
|
||||
}
|
||||
if src.MaxDurationSeconds != 0 {
|
||||
dst.MaxDurationSeconds = src.MaxDurationSeconds
|
||||
}
|
||||
if src.SilenceAutoStop != nil {
|
||||
dst.SilenceAutoStop = src.SilenceAutoStop
|
||||
}
|
||||
if src.AutoSubmit != nil {
|
||||
dst.AutoSubmit = src.AutoSubmit
|
||||
}
|
||||
if src.WindowsAudioDevice != "" {
|
||||
dst.WindowsAudioDevice = src.WindowsAudioDevice
|
||||
}
|
||||
}
|
||||
|
||||
// validateSTTConfig rejects unknown provider/streamProvider values and a
|
||||
// negative maxDurationSeconds at load time — a clear startup error naming the
|
||||
// bad value and the valid options, never a silent fallback the user did not ask
|
||||
// for (§11a).
|
||||
func validateSTTConfig(cfg STTConfig) error {
|
||||
if cfg.Provider != "" {
|
||||
switch cfg.Provider {
|
||||
case STTProviderLocal, STTProviderGroq, STTProviderOpenAI:
|
||||
default:
|
||||
return fmt.Errorf("invalid stt.provider %q: expected local, groq, or openai", cfg.Provider)
|
||||
}
|
||||
}
|
||||
if cfg.StreamProvider != "" {
|
||||
switch cfg.StreamProvider {
|
||||
case STTProviderLocal, STTProviderDeepgram, STTProviderOpenAI:
|
||||
default:
|
||||
return fmt.Errorf("invalid stt.streamProvider %q: expected local, deepgram, or openai", cfg.StreamProvider)
|
||||
}
|
||||
}
|
||||
if cfg.MaxDurationSeconds < 0 {
|
||||
return fmt.Errorf("invalid stt.maxDurationSeconds %d: must be >= 0 (0 uses the default)", cfg.MaxDurationSeconds)
|
||||
}
|
||||
if cfg.NumThreads < 0 {
|
||||
return fmt.Errorf("invalid stt.numThreads %d: must be >= 0 (0 uses the engine default)", cfg.NumThreads)
|
||||
}
|
||||
if cfg.LocalServerPort < 0 || cfg.LocalServerPort > 65535 {
|
||||
return fmt.Errorf("invalid stt.localServerPort %d: must be between 1 and 65535 (0 uses the default)", cfg.LocalServerPort)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeMCPConfig(dst *MCPConfig, src MCPConfig) {
|
||||
if len(src.Servers) == 0 {
|
||||
return
|
||||
@@ -774,6 +868,9 @@ func mergeMCPServer(base MCPServerConfig, next MCPServerConfig) MCPServerConfig
|
||||
if next.disabledSet || next.Disabled {
|
||||
base.Disabled = next.Disabled
|
||||
}
|
||||
if next.configured {
|
||||
base.configured = true
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
|
||||
@@ -127,6 +127,79 @@ func TestResolveLoadsFavoriteModelsFromUserConfigOnly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveLoadsRecentModelsFromUserConfigOnly(t *testing.T) {
|
||||
userPath := writeConfig(t, `{
|
||||
"activeProvider": "user",
|
||||
"providers": [{
|
||||
"name": "user",
|
||||
"provider": "openai",
|
||||
"apiKey": "sk-user",
|
||||
"model": "gpt-user"
|
||||
}],
|
||||
"preferences": {
|
||||
"recentModels": [
|
||||
{"provider": "openrouter", "model": " google/gemini-2.5-pro "},
|
||||
{"provider": "openrouter", "model": "minimax/minimax-m2.1"}
|
||||
]
|
||||
}
|
||||
}`)
|
||||
projectPath := writeConfig(t, `{
|
||||
"preferences": {
|
||||
"recentModels": [{"provider": "project", "model": "project-model"}]
|
||||
}
|
||||
}`)
|
||||
|
||||
resolved, err := Resolve(ResolveOptions{
|
||||
UserConfigPath: userPath,
|
||||
ProjectConfigPath: projectPath,
|
||||
Env: map[string]string{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
|
||||
want := []RecentModelEntry{
|
||||
{Provider: "openrouter", Model: "google/gemini-2.5-pro"},
|
||||
{Provider: "openrouter", Model: "minimax/minimax-m2.1"},
|
||||
}
|
||||
if !reflect.DeepEqual(resolved.Preferences.RecentModels, want) {
|
||||
t.Fatalf("RecentModels = %#v, want %#v", resolved.Preferences.RecentModels, want)
|
||||
}
|
||||
}
|
||||
|
||||
// recentModels must never be merged from project config (same posture as
|
||||
// favoriteModels): a project setting it with no corresponding user setting
|
||||
// should leave the resolved value empty, not pick up the project's list.
|
||||
func TestResolveIgnoresProjectOnlyRecentModels(t *testing.T) {
|
||||
userPath := writeConfig(t, `{
|
||||
"activeProvider": "user",
|
||||
"providers": [{
|
||||
"name": "user",
|
||||
"provider": "openai",
|
||||
"apiKey": "sk-user",
|
||||
"model": "gpt-user"
|
||||
}]
|
||||
}`)
|
||||
projectPath := writeConfig(t, `{
|
||||
"preferences": {
|
||||
"recentModels": [{"provider": "project", "model": "project-model"}]
|
||||
}
|
||||
}`)
|
||||
|
||||
resolved, err := Resolve(ResolveOptions{
|
||||
UserConfigPath: userPath,
|
||||
ProjectConfigPath: projectPath,
|
||||
Env: map[string]string{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
|
||||
if len(resolved.Preferences.RecentModels) != 0 {
|
||||
t.Fatalf("RecentModels = %#v, want empty (project-only recentModels must not be merged)", resolved.Preferences.RecentModels)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveLoadsProviderCatalogSnakeAndCamelJSONFields(t *testing.T) {
|
||||
path := writeConfig(t, `{
|
||||
"activeProvider": "snake",
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSetSTTLocalEngineResetsStreamProvider(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := dir + "/config.json"
|
||||
// Pre-seed a cloud streaming provider — the situation the fix guards against.
|
||||
if err := os.WriteFile(path, []byte(`{"stt":{"streamProvider":"deepgram"}}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := SetSTTLocalEngine(path, "sherpa-onnx-offline", "sherpa-onnx-online-websocket-server", "/models/kroko", true)
|
||||
if err != nil {
|
||||
t.Fatalf("SetSTTLocalEngine: %v", err)
|
||||
}
|
||||
// Switching to a local model must point BOTH pipelines at the local engine, or
|
||||
// the leftover cloud streamProvider keeps the live transcript on the cloud.
|
||||
if cfg.STT.Provider != STTProviderLocal || cfg.STT.StreamProvider != STTProviderLocal {
|
||||
t.Errorf("both providers should be local, got provider=%q streamProvider=%q", cfg.STT.Provider, cfg.STT.StreamProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSTTConfigRejectsUnknownProvider(t *testing.T) {
|
||||
_, issues := ValidateBytes([]byte(`{"stt":{"provider":"assemblyai"}}`))
|
||||
if len(issues) == 0 {
|
||||
t.Fatal("expected an issue for unknown stt.provider")
|
||||
}
|
||||
if !strings.Contains(issues[0].Message, "assemblyai") || !strings.Contains(issues[0].Message, "local, groq, or openai") {
|
||||
t.Errorf("issue message = %q, want it to name the bad value and valid options", issues[0].Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSTTConfigRejectsUnknownStreamProvider(t *testing.T) {
|
||||
_, issues := ValidateBytes([]byte(`{"stt":{"streamProvider":"groq"}}`))
|
||||
if len(issues) == 0 {
|
||||
t.Fatal("expected an issue: groq has no streaming product, not a valid streamProvider")
|
||||
}
|
||||
if !strings.Contains(issues[0].Message, "deepgram") {
|
||||
t.Errorf("issue message = %q", issues[0].Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSTTConfigAcceptsValid(t *testing.T) {
|
||||
_, issues := ValidateBytes([]byte(`{"stt":{"provider":"groq","streamProvider":"deepgram","maxDurationSeconds":120}}`))
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("unexpected issues: %+v", issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSTTConfigRejectsNegativeDuration(t *testing.T) {
|
||||
_, issues := ValidateBytes([]byte(`{"stt":{"maxDurationSeconds":-5}}`))
|
||||
if len(issues) == 0 {
|
||||
t.Fatal("expected an issue for negative maxDurationSeconds")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTTConfigDefaults(t *testing.T) {
|
||||
cfg := STTConfig{}
|
||||
if cfg.STTProvider() != STTProviderLocal {
|
||||
t.Errorf("default provider = %q, want local", cfg.STTProvider())
|
||||
}
|
||||
if cfg.STTStreamProvider() != STTProviderLocal {
|
||||
t.Errorf("default streamProvider = %q, want local", cfg.STTStreamProvider())
|
||||
}
|
||||
if !cfg.StreamingEnabled() {
|
||||
t.Error("streaming should default on")
|
||||
}
|
||||
if !cfg.SilenceAutoStopEnabled() {
|
||||
t.Error("silence auto-stop should default on")
|
||||
}
|
||||
if cfg.AutoSubmitEnabled() {
|
||||
t.Error("auto-submit should default off (insert-for-review is the safety net)")
|
||||
}
|
||||
if !cfg.Empty() {
|
||||
t.Error("zero-value STTConfig should be Empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTTConfigTriStateBools(t *testing.T) {
|
||||
cfg, issues := ValidateBytes([]byte(`{"stt":{"autoSubmit":true,"streaming":false,"silenceAutoStop":false}}`))
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("unexpected issues: %+v", issues)
|
||||
}
|
||||
if !cfg.STT.AutoSubmitEnabled() {
|
||||
t.Error("autoSubmit:true should be enabled")
|
||||
}
|
||||
if cfg.STT.StreamingEnabled() {
|
||||
t.Error("streaming:false should be disabled")
|
||||
}
|
||||
if cfg.STT.SilenceAutoStopEnabled() {
|
||||
t.Error("silenceAutoStop:false should be disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTTConfigRoundTripsThroughMarshal(t *testing.T) {
|
||||
cfg, _ := ValidateBytes([]byte(`{"stt":{"provider":"groq","model":"whisper-large-v3-turbo","localServerPort":6007}}`))
|
||||
data, err := cfg.MarshalJSON()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), `"stt"`) {
|
||||
t.Errorf("marshaled config dropped stt section: %s", data)
|
||||
}
|
||||
reparsed, issues := ValidateBytes(data)
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("re-parse issues: %+v", issues)
|
||||
}
|
||||
if reparsed.STT.Provider != STTProviderGroq || reparsed.STT.Model != "whisper-large-v3-turbo" || reparsed.STT.LocalServerPort != 6007 {
|
||||
t.Errorf("round-trip lost fields: %+v", reparsed.STT)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetSTTModelPersists(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := dir + "/config.json"
|
||||
// Cloud provider stores under stt.model.
|
||||
if _, err := SetSTTModel(path, STTProviderGroq, "whisper-large-v3-turbo"); err != nil {
|
||||
t.Fatalf("SetSTTModel: %v", err)
|
||||
}
|
||||
cfg, issues := ValidateFile(path)
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("issues: %+v", issues)
|
||||
}
|
||||
if cfg.STT.Provider != STTProviderGroq || cfg.STT.Model != "whisper-large-v3-turbo" {
|
||||
t.Errorf("persisted STT = %+v", cfg.STT)
|
||||
}
|
||||
// Local provider stores under stt.localModelPath.
|
||||
if _, err := SetSTTModel(path, STTProviderLocal, "/models/moonshine"); err != nil {
|
||||
t.Fatalf("SetSTTModel local: %v", err)
|
||||
}
|
||||
cfg, _ = ValidateFile(path)
|
||||
if cfg.STT.LocalModelPath != "/models/moonshine" {
|
||||
t.Errorf("local model path = %q", cfg.STT.LocalModelPath)
|
||||
}
|
||||
}
|
||||
@@ -97,6 +97,11 @@ type ToolsConfig struct {
|
||||
|
||||
type PreferencesConfig struct {
|
||||
FavoriteModels []string `json:"favoriteModels,omitempty"`
|
||||
// RecentModels is the short automatic history of provider+model pairs the
|
||||
// user has switched to via /model, newest first. Unlike FavoriteModels
|
||||
// (manual pins), this list is maintained automatically on every switch and
|
||||
// capped to MaxRecentModels entries. See RecentModelEntry.
|
||||
RecentModels []RecentModelEntry `json:"recentModels,omitempty"`
|
||||
// Theme is the persisted TUI palette preference — "auto" or a registered theme
|
||||
// name (e.g. "dracula"). Applied at startup below the --theme flag and
|
||||
// ZERO_THEME, so a /theme choice survives restart. Empty = unset (defaults auto).
|
||||
@@ -107,6 +112,18 @@ type PreferencesConfig struct {
|
||||
Recaps *bool `json:"recaps,omitempty"`
|
||||
}
|
||||
|
||||
// RecentModelEntry is one provider-qualified model selection recorded in
|
||||
// Preferences.RecentModels. Provider is the saved provider profile's Name (not
|
||||
// a display label), so a recent entry can be resolved back to a concrete
|
||||
// profile the same way the /model picker's cross-provider rows are.
|
||||
type RecentModelEntry struct {
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// MaxRecentModels caps the persisted/displayed recent-selection history.
|
||||
const MaxRecentModels = 5
|
||||
|
||||
// RecapsEnabled reports whether post-turn recaps are on. Unset defaults to ON.
|
||||
func (p PreferencesConfig) RecapsEnabled() bool {
|
||||
return p.Recaps == nil || *p.Recaps
|
||||
@@ -131,6 +148,115 @@ type KeyBindingsConfig struct {
|
||||
ToggleSidebar KeyBindingDef `json:"toggleSidebar,omitempty"`
|
||||
}
|
||||
|
||||
// STTProviderKind is the batch (or streaming) transcription backend selector.
|
||||
// Validated at config load time against the known set, so a typo fails loudly
|
||||
// with the valid options rather than silently falling back to a default.
|
||||
type STTProviderKind string
|
||||
|
||||
const (
|
||||
STTProviderLocal STTProviderKind = "local"
|
||||
STTProviderGroq STTProviderKind = "groq"
|
||||
STTProviderOpenAI STTProviderKind = "openai"
|
||||
STTProviderDeepgram STTProviderKind = "deepgram"
|
||||
)
|
||||
|
||||
// STTConfig configures speech-to-text dictation. All fields are optional; empty
|
||||
// values take the documented defaults. Booleans that need a real tri-state
|
||||
// (distinguishing "unset" from "false") use *bool, matching PreferencesConfig.Recaps.
|
||||
type STTConfig struct {
|
||||
// Provider is the batch transcription backend: "local" (sherpa-onnx-offline,
|
||||
// the default), "groq", or "openai". Termux and the fallback everywhere use
|
||||
// this path.
|
||||
Provider STTProviderKind `json:"provider,omitempty"`
|
||||
// StreamProvider is the streaming backend on desktop: "local" (sherpa-onnx
|
||||
// websocket server, default), "deepgram", or "openai" (Realtime).
|
||||
StreamProvider STTProviderKind `json:"streamProvider,omitempty"`
|
||||
// Streaming enables the live-transcript pipeline on platforms that support
|
||||
// it (desktop). Defaults to on; set false to always use the batch pipeline.
|
||||
Streaming *bool `json:"streaming,omitempty"`
|
||||
// Model overrides the cloud batch model (e.g. "whisper-large-v3-turbo" for
|
||||
// Groq, "whisper-1" for OpenAI). Empty uses the provider default.
|
||||
Model string `json:"model,omitempty"`
|
||||
// StreamModel overrides the cloud streaming model (Deepgram/OpenAI Realtime).
|
||||
StreamModel string `json:"streamModel,omitempty"`
|
||||
// LocalModelPath is the sherpa-onnx model directory for local transcription
|
||||
// (batch and streaming). Required to use a local provider.
|
||||
LocalModelPath string `json:"localModelPath,omitempty"`
|
||||
// LocalBinary overrides the offline binary name/path (default
|
||||
// "sherpa-onnx-offline"); LocalServerBinary overrides the streaming server
|
||||
// (default "sherpa-onnx-online-websocket-server"). Both looked up on PATH.
|
||||
LocalBinary string `json:"localBinary,omitempty"`
|
||||
LocalServerBinary string `json:"localServerBinary,omitempty"`
|
||||
// LocalServerPort is the localhost port for the sherpa-onnx websocket server
|
||||
// (default 6006).
|
||||
LocalServerPort int `json:"localServerPort,omitempty"`
|
||||
// EngineVersion selects the sherpa-onnx release the auto-download fetches
|
||||
// ("" → a pinned known-good default; "latest" or any release tag also works,
|
||||
// so a newer engine needs no Zero update). Verified against the release's
|
||||
// published SHA256 digest either way.
|
||||
EngineVersion string `json:"engineVersion,omitempty"`
|
||||
// NumThreads sets the local engine's thread count (0 = engine default).
|
||||
NumThreads int `json:"numThreads,omitempty"`
|
||||
// Language optionally constrains recognition (ISO-639-1, e.g. "en"). Empty =
|
||||
// auto-detect.
|
||||
Language string `json:"language,omitempty"`
|
||||
// MaxDurationSeconds is the hard runaway-recording cap on every platform
|
||||
// (0 = default 300s).
|
||||
MaxDurationSeconds int `json:"maxDurationSeconds,omitempty"`
|
||||
// SilenceAutoStop ends a recording ~2s after the signal goes quiet, as a
|
||||
// backstop for a forgotten manual stop (not VAD-triggered start). Defaults on.
|
||||
SilenceAutoStop *bool `json:"silenceAutoStop,omitempty"`
|
||||
// AutoSubmit fires the transcript at the agent instead of inserting it for
|
||||
// review. Defaults OFF — insert-for-review is the safety net for a misheard
|
||||
// prompt (§3).
|
||||
AutoSubmit *bool `json:"autoSubmit,omitempty"`
|
||||
// WindowsAudioDevice names the dshow capture device (auto-detected when empty).
|
||||
WindowsAudioDevice string `json:"windowsAudioDevice,omitempty"`
|
||||
}
|
||||
|
||||
// STTProvider returns the configured batch provider, defaulting to local.
|
||||
func (c STTConfig) STTProvider() STTProviderKind {
|
||||
if c.Provider == "" {
|
||||
return STTProviderLocal
|
||||
}
|
||||
return c.Provider
|
||||
}
|
||||
|
||||
// STTStreamProvider returns the configured streaming provider, defaulting to local.
|
||||
func (c STTConfig) STTStreamProvider() STTProviderKind {
|
||||
if c.StreamProvider == "" {
|
||||
return STTProviderLocal
|
||||
}
|
||||
return c.StreamProvider
|
||||
}
|
||||
|
||||
// StreamingEnabled reports whether the live pipeline is on. Unset defaults ON.
|
||||
func (c STTConfig) StreamingEnabled() bool {
|
||||
return c.Streaming == nil || *c.Streaming
|
||||
}
|
||||
|
||||
// SilenceAutoStopEnabled reports whether trailing-silence auto-stop is on.
|
||||
// Unset defaults ON.
|
||||
func (c STTConfig) SilenceAutoStopEnabled() bool {
|
||||
return c.SilenceAutoStop == nil || *c.SilenceAutoStop
|
||||
}
|
||||
|
||||
// AutoSubmitEnabled reports whether transcripts are auto-fired. Unset defaults OFF.
|
||||
func (c STTConfig) AutoSubmitEnabled() bool {
|
||||
return c.AutoSubmit != nil && *c.AutoSubmit
|
||||
}
|
||||
|
||||
// Empty reports whether the STT config carries no user-set values (so it can be
|
||||
// omitted from a marshaled config).
|
||||
func (c STTConfig) Empty() bool {
|
||||
return c.Provider == "" && c.StreamProvider == "" && c.Streaming == nil &&
|
||||
c.Model == "" && c.StreamModel == "" && c.LocalModelPath == "" &&
|
||||
c.LocalBinary == "" && c.LocalServerBinary == "" && c.LocalServerPort == 0 &&
|
||||
c.EngineVersion == "" && c.NumThreads == 0 && c.Language == "" &&
|
||||
c.MaxDurationSeconds == 0 && c.SilenceAutoStop == nil && c.AutoSubmit == nil &&
|
||||
c.WindowsAudioDevice == ""
|
||||
}
|
||||
|
||||
// LocalControlConfig controls local browser/desktop/terminal automation helpers.
|
||||
// Helpers are discovered lazily by the tool that needs them; no setup command or
|
||||
// background probe is required during startup.
|
||||
@@ -227,6 +353,7 @@ type FileConfig struct {
|
||||
Preferences PreferencesConfig `json:"preferences,omitempty"`
|
||||
KeyBindings KeyBindingsConfig `json:"keybindings,omitempty"`
|
||||
LocalControl LocalControlConfig `json:"localControl,omitempty"`
|
||||
STT STTConfig `json:"stt,omitempty"`
|
||||
}
|
||||
|
||||
func (cfg FileConfig) MarshalJSON() ([]byte, error) {
|
||||
@@ -242,6 +369,7 @@ func (cfg FileConfig) MarshalJSON() ([]byte, error) {
|
||||
Preferences PreferencesConfig `json:"preferences,omitempty"`
|
||||
KeyBindings KeyBindingsConfig `json:"keybindings,omitempty"`
|
||||
LocalControl *LocalControlConfig `json:"localControl,omitempty"`
|
||||
STT *STTConfig `json:"stt,omitempty"`
|
||||
}
|
||||
raw := rawConfig{
|
||||
ActiveProvider: cfg.ActiveProvider,
|
||||
@@ -258,6 +386,9 @@ func (cfg FileConfig) MarshalJSON() ([]byte, error) {
|
||||
if !cfg.LocalControl.Empty() {
|
||||
raw.LocalControl = &cfg.LocalControl
|
||||
}
|
||||
if !cfg.STT.Empty() {
|
||||
raw.STT = &cfg.STT
|
||||
}
|
||||
return json.Marshal(raw)
|
||||
}
|
||||
|
||||
@@ -280,6 +411,7 @@ type Overrides struct {
|
||||
Tools ToolsConfig
|
||||
KeyBindings KeyBindingsConfig
|
||||
LocalControl LocalControlConfig
|
||||
STT STTConfig
|
||||
}
|
||||
|
||||
type ResolvedConfig struct {
|
||||
@@ -295,6 +427,7 @@ type ResolvedConfig struct {
|
||||
Preferences PreferencesConfig
|
||||
KeyBindings KeyBindingsConfig
|
||||
LocalControl LocalControlConfig
|
||||
STT STTConfig
|
||||
}
|
||||
|
||||
type MCPConfig struct {
|
||||
@@ -312,6 +445,16 @@ type MCPServerConfig struct {
|
||||
OAuth *MCPOAuthConfig `json:"oauth,omitempty"`
|
||||
Disabled bool `json:"disabled,omitempty"`
|
||||
disabledSet bool
|
||||
// configured is true when the user's config JSON declared an object for
|
||||
// this server at all (i.e. UnmarshalJSON ran for it), regardless of which
|
||||
// fields it set or what values they hold. A built-in default seeded by
|
||||
// DefaultMCPServers() is never unmarshaled from JSON, so it starts false;
|
||||
// any explicit entry in the user/project file — even one that happens to
|
||||
// repeat a default's exact field values (e.g. re-declaring firecrawl's
|
||||
// default URL) — sets it true. IsUnconfiguredDefault checks this alongside
|
||||
// a resolved-value comparison, so redeclaring default values verbatim still
|
||||
// counts as user-configured.
|
||||
configured bool
|
||||
}
|
||||
|
||||
// MCPOAuthConfig describes how to authenticate to a remote MCP server using an
|
||||
@@ -342,6 +485,7 @@ func (cfg *FileConfig) UnmarshalJSON(data []byte) error {
|
||||
Preferences PreferencesConfig `json:"preferences"`
|
||||
KeyBindings KeyBindingsConfig `json:"keybindings"`
|
||||
LocalControl LocalControlConfig `json:"localControl"`
|
||||
STT STTConfig `json:"stt"`
|
||||
MCPServers map[string]MCPServerConfig `json:"mcpServers"`
|
||||
MCPServersSnake map[string]MCPServerConfig `json:"mcp_servers"`
|
||||
}
|
||||
@@ -369,6 +513,7 @@ func (cfg *FileConfig) UnmarshalJSON(data []byte) error {
|
||||
cfg.Preferences = raw.Preferences
|
||||
cfg.KeyBindings = raw.KeyBindings
|
||||
cfg.LocalControl = raw.LocalControl
|
||||
cfg.STT = raw.STT
|
||||
if cfg.MCP.Servers == nil && (len(raw.MCPServers) > 0 || len(raw.MCPServersSnake) > 0) {
|
||||
cfg.MCP.Servers = map[string]MCPServerConfig{}
|
||||
}
|
||||
@@ -503,6 +648,10 @@ func (server *MCPServerConfig) UnmarshalJSON(data []byte) error {
|
||||
server.Disabled = *raw.Disabled
|
||||
server.disabledSet = true
|
||||
}
|
||||
// This method only runs when the user's JSON actually has an object for this
|
||||
// server key (built-in defaults are seeded as Go struct literals, never
|
||||
// unmarshaled), so reaching here at all means the user configured it.
|
||||
server.configured = true
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -54,5 +54,8 @@ func validateSemantics(cfg FileConfig) []Issue {
|
||||
// normalizeProviders already redacts secrets via providerError.
|
||||
return []Issue{{FieldPath: "providers", Message: err.Error()}}
|
||||
}
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return []Issue{{FieldPath: "stt", Message: err.Error()}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
)
|
||||
|
||||
func UpsertProvider(path string, profile ProviderProfile, setActive bool) (FileConfig, error) {
|
||||
@@ -29,6 +31,20 @@ func UpsertProvider(path string, profile ProviderProfile, setActive bool) (FileC
|
||||
}
|
||||
|
||||
mergeProvider(&cfg, profile)
|
||||
// mergeProfile deliberately ignores APIKeyStored — during resolve-time
|
||||
// layering a project config must not be able to claim the user's stored
|
||||
// keys. This user-config WRITE path re-applies the marker: capturing a key
|
||||
// via SecureProviderProfile onto a previously env/no-key profile must
|
||||
// persist apiKeyStored, or the secret sits in the credential store while
|
||||
// every ApplyStoredAPIKey gate skips it (PR #560 review).
|
||||
if profile.APIKeyStored {
|
||||
for index := range cfg.Providers {
|
||||
if cfg.Providers[index].Name == profile.Name {
|
||||
cfg.Providers[index].APIKeyStored = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if setActive || strings.TrimSpace(cfg.ActiveProvider) == "" {
|
||||
cfg.ActiveProvider = profile.Name
|
||||
}
|
||||
@@ -39,6 +55,62 @@ func UpsertProvider(path string, profile ProviderProfile, setActive bool) (FileC
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// EnsuredProvider reports the outcome of EnsureCatalogProvider: the profile name
|
||||
// that serves the catalog entry, whether it was newly created, and which provider
|
||||
// is active after the call (unchanged unless it was blank).
|
||||
type EnsuredProvider struct {
|
||||
Name string
|
||||
Created bool
|
||||
Active string
|
||||
}
|
||||
|
||||
// EnsureCatalogProvider guarantees a provider profile exists in the config at
|
||||
// path for the given catalog entry. OAuth login flows call this right after
|
||||
// storing a token: a login is only reachable from the provider list and
|
||||
// `zero providers use` when a profile exists, but a login must never replace or
|
||||
// deactivate the user's current active provider — so an existing profile whose
|
||||
// Name or CatalogID already matches is left completely untouched (its name,
|
||||
// credentials, and model are the user's), and a created profile is NOT marked
|
||||
// active unless no provider was active at all.
|
||||
func EnsureCatalogProvider(path string, catalogID string) (EnsuredProvider, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return EnsuredProvider{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
descriptor, err := providercatalog.Require(catalogID)
|
||||
if err != nil {
|
||||
return EnsuredProvider{}, err
|
||||
}
|
||||
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return EnsuredProvider{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return EnsuredProvider{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
for _, provider := range cfg.Providers {
|
||||
if strings.EqualFold(strings.TrimSpace(provider.CatalogID), descriptor.ID) ||
|
||||
strings.EqualFold(strings.TrimSpace(provider.Name), descriptor.ID) {
|
||||
return EnsuredProvider{Name: provider.Name, Active: cfg.ActiveProvider}, nil
|
||||
}
|
||||
}
|
||||
|
||||
profile := ProviderProfile{
|
||||
Name: descriptor.ID,
|
||||
ProviderKind: providerKindForCatalogTransport(descriptor.Transport),
|
||||
CatalogID: descriptor.ID,
|
||||
BaseURL: descriptor.DefaultBaseURL,
|
||||
Model: descriptor.DefaultModel,
|
||||
}
|
||||
written, err := UpsertProvider(path, profile, false)
|
||||
if err != nil {
|
||||
return EnsuredProvider{}, err
|
||||
}
|
||||
return EnsuredProvider{Name: profile.Name, Created: true, Active: written.ActiveProvider}, nil
|
||||
}
|
||||
|
||||
func SetActiveProvider(path string, name string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
@@ -72,6 +144,296 @@ func SetActiveProvider(path string, name string) (FileConfig, error) {
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", name)
|
||||
}
|
||||
|
||||
// RemoveProvider deletes the named provider profile from the config at path.
|
||||
// When the removed profile was active, activeProvider hands off to the first
|
||||
// remaining provider (or clears when none remain) so the config never points at
|
||||
// a profile that no longer exists. The caller owns cleaning up the credential
|
||||
// store entry — config stays pure of secret I/O on the read path, and remove
|
||||
// keeps that symmetry by only touching config.json.
|
||||
func RemoveProvider(path string, name string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return FileConfig{}, fmt.Errorf("provider name is required")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
|
||||
index := -1
|
||||
for i, provider := range cfg.Providers {
|
||||
if strings.EqualFold(strings.TrimSpace(provider.Name), name) {
|
||||
index = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", name)
|
||||
}
|
||||
removed := cfg.Providers[index]
|
||||
cfg.Providers = append(cfg.Providers[:index], cfg.Providers[index+1:]...)
|
||||
if strings.EqualFold(strings.TrimSpace(cfg.ActiveProvider), strings.TrimSpace(removed.Name)) {
|
||||
cfg.ActiveProvider = ""
|
||||
if len(cfg.Providers) > 0 {
|
||||
cfg.ActiveProvider = cfg.Providers[0].Name
|
||||
}
|
||||
}
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// RenameProvider renames a provider profile, keeping everything keyed by the
|
||||
// profile name consistent: the activeProvider pointer follows the rename, and a
|
||||
// key in the encrypted credential store (APIKeyStored) is migrated to the new
|
||||
// name BEFORE the config is rewritten — the store write must succeed first so a
|
||||
// failed migration never strands the config pointing at a key that no longer
|
||||
// resolves. OAuth tokens are deliberately not migrated: the runtime's login
|
||||
// candidates fall back to the profile's CatalogID, which every OAuth-capable
|
||||
// catalog profile carries, so a rename keeps the login reachable.
|
||||
func RenameProvider(path string, oldName string, newName string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
oldName = strings.TrimSpace(oldName)
|
||||
newName = strings.TrimSpace(newName)
|
||||
if oldName == "" || newName == "" {
|
||||
return FileConfig{}, fmt.Errorf("provider names are required")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
|
||||
index := -1
|
||||
for i, provider := range cfg.Providers {
|
||||
providerName := strings.TrimSpace(provider.Name)
|
||||
if strings.EqualFold(providerName, oldName) {
|
||||
index = i
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(providerName, newName) {
|
||||
return FileConfig{}, fmt.Errorf("provider %q already exists", newName)
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", oldName)
|
||||
}
|
||||
if strings.EqualFold(oldName, newName) && cfg.Providers[index].Name == newName {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
previousName := cfg.Providers[index].Name
|
||||
keyMigrated := false
|
||||
if cfg.Providers[index].APIKeyStored {
|
||||
if err := migrateStoredProviderKey(path, previousName, newName); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("migrate stored key for %q: %w", oldName, err)
|
||||
}
|
||||
keyMigrated = true
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(cfg.ActiveProvider), strings.TrimSpace(previousName)) {
|
||||
cfg.ActiveProvider = newName
|
||||
}
|
||||
cfg.Providers[index].Name = newName
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
if keyMigrated {
|
||||
// Compensate best-effort: config.json still names the OLD profile, so
|
||||
// move the key back where that config can find it — otherwise a failed
|
||||
// rewrite strands the key under a name no profile carries.
|
||||
_ = migrateStoredProviderKey(path, newName, previousName)
|
||||
}
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// ProviderEdit is a field-level edit of one saved provider, applied by
|
||||
// EditProvider in a single atomic write. Name is the CURRENT profile name
|
||||
// (matched case-insensitively); NewName renames (case-only renames included).
|
||||
// Empty BaseURL/Model/APIKey mean "leave unchanged"; Description is applied
|
||||
// VERBATIM (the editor always knows the full desired text, so clearing works).
|
||||
type ProviderEdit struct {
|
||||
Name string
|
||||
NewName string
|
||||
BaseURL string
|
||||
Model string
|
||||
APIKey string
|
||||
APIKeyStored bool
|
||||
Description string
|
||||
}
|
||||
|
||||
// EditProvider applies a provider edit in ONE read-modify-write: rename
|
||||
// (activeProvider follows; a stored key migrates, with a best-effort rollback
|
||||
// if the config write fails), field updates, the stored-key marker, and the
|
||||
// verbatim description. A single write keeps the operation atomic — the
|
||||
// previous rename+upsert+describe sequence could fail halfway and leave
|
||||
// config.json renamed while every in-memory consumer still held the old name —
|
||||
// and, unlike UpsertProvider's exact-name merge, the case-insensitive match
|
||||
// here makes a case-only rename (groq -> Groq) an in-place update instead of
|
||||
// an appended duplicate profile.
|
||||
func EditProvider(path string, edit ProviderEdit) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
oldName := strings.TrimSpace(edit.Name)
|
||||
if oldName == "" {
|
||||
return FileConfig{}, fmt.Errorf("provider name is required")
|
||||
}
|
||||
newName := strings.TrimSpace(edit.NewName)
|
||||
if newName == "" {
|
||||
newName = oldName
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
|
||||
index := -1
|
||||
for i, provider := range cfg.Providers {
|
||||
providerName := strings.TrimSpace(provider.Name)
|
||||
if strings.EqualFold(providerName, oldName) {
|
||||
index = i
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(providerName, newName) {
|
||||
return FileConfig{}, fmt.Errorf("provider %q already exists", newName)
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", oldName)
|
||||
}
|
||||
|
||||
previousName := cfg.Providers[index].Name
|
||||
renamed := previousName != newName
|
||||
keyMigrated := false
|
||||
// A rename moves the stored key along: either the profile's existing entry,
|
||||
// or a replacement key the caller just captured — the contract is that a
|
||||
// captured key is stored under the CURRENT name before EditProvider runs, so
|
||||
// one migration covers both. migrateStoredProviderKey no-ops on case-only
|
||||
// renames (the store normalizes names), so it cannot delete the key it just
|
||||
// moved.
|
||||
if renamed && (cfg.Providers[index].APIKeyStored || edit.APIKeyStored) {
|
||||
if err := migrateStoredProviderKey(path, previousName, newName); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("migrate stored key for %q: %w", oldName, err)
|
||||
}
|
||||
keyMigrated = true
|
||||
}
|
||||
if renamed && strings.EqualFold(strings.TrimSpace(cfg.ActiveProvider), strings.TrimSpace(previousName)) {
|
||||
cfg.ActiveProvider = newName
|
||||
}
|
||||
|
||||
profile := &cfg.Providers[index]
|
||||
profile.Name = newName
|
||||
if baseURL := strings.TrimSpace(edit.BaseURL); baseURL != "" {
|
||||
profile.BaseURL = baseURL
|
||||
}
|
||||
if model := strings.TrimSpace(edit.Model); model != "" {
|
||||
profile.Model = model
|
||||
}
|
||||
if apiKey := strings.TrimSpace(edit.APIKey); apiKey != "" {
|
||||
profile.APIKey = apiKey
|
||||
}
|
||||
if edit.APIKeyStored {
|
||||
profile.APIKeyStored = true
|
||||
}
|
||||
profile.Description = strings.TrimSpace(edit.Description)
|
||||
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
if keyMigrated {
|
||||
// Compensate best-effort: config.json still names the OLD profile, so
|
||||
// move the key back where that config can find it.
|
||||
_ = migrateStoredProviderKey(path, newName, previousName)
|
||||
}
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetProviderDescription sets a provider's description VERBATIM — including to
|
||||
// empty. The generic UpsertProvider merge treats empty fields as "leave
|
||||
// unchanged", so clearing a description needs this dedicated setter.
|
||||
func SetProviderDescription(path string, name string, description string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return FileConfig{}, fmt.Errorf("provider name is required")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
for index := range cfg.Providers {
|
||||
if strings.EqualFold(strings.TrimSpace(cfg.Providers[index].Name), name) {
|
||||
cfg.Providers[index].Description = strings.TrimSpace(description)
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
}
|
||||
return FileConfig{}, fmt.Errorf("provider %q not found", name)
|
||||
}
|
||||
|
||||
// migrateStoredProviderKey moves a credential-store entry to a new provider
|
||||
// name: write-new-then-delete-old, so an interruption can leave a duplicate but
|
||||
// never a missing key. A missing source entry is a no-op (the marker may be
|
||||
// stale); only a failed WRITE aborts the rename.
|
||||
func migrateStoredProviderKey(configPath string, oldName string, newName string) error {
|
||||
// The store normalizes names case-insensitively, so a case-only rename
|
||||
// (groq -> Groq) targets ONE entry: Set(new) rewrites it in place and
|
||||
// Delete(old) would then remove the key that was just "moved". Nothing to
|
||||
// migrate — the existing entry already serves the new name.
|
||||
if strings.EqualFold(strings.TrimSpace(oldName), strings.TrimSpace(newName)) {
|
||||
return nil
|
||||
}
|
||||
store, err := ProviderKeyStoreAt(filepath.Dir(configPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, ok, err := store.Get(oldName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok || strings.TrimSpace(key) == "" {
|
||||
return nil
|
||||
}
|
||||
if err := store.Set(newName, key); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = store.Delete(oldName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetProviderModel(path string, name string, model string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
@@ -131,6 +493,32 @@ func SetFavoriteModels(path string, models []string) (FileConfig, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetRecentModels persists the automatic recent-model-switch history,
|
||||
// mirroring SetFavoriteModels (read-modify-atomic-write). Unlike favorites,
|
||||
// order is preserved (newest first) rather than sorted, since it reflects
|
||||
// switch recency, not an alphabetical preference list.
|
||||
func SetRecentModels(path string, entries []RecentModelEntry) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
|
||||
cfg.Preferences.RecentModels = NormalizeRecentModels(entries)
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetRecapsEnabled persists the post-turn recap preference, mirroring
|
||||
// SetFavoriteModels (read-modify-atomic-write).
|
||||
func SetRecapsEnabled(path string, enabled bool) (FileConfig, error) {
|
||||
@@ -176,6 +564,106 @@ func SetTheme(path string, theme string) (FileConfig, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetSTTModel persists the dictation model and its provider, mirroring
|
||||
// SetTheme (read-modify-atomic-write). provider must be one of the known STT
|
||||
// provider kinds; a local provider stores the model as stt.localModelPath,
|
||||
// otherwise as stt.model. A blank model clears the stored value for that slot.
|
||||
func SetSTTModel(path string, provider STTProviderKind, model string) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
if provider != "" {
|
||||
cfg.STT.Provider = provider
|
||||
}
|
||||
model = strings.TrimSpace(model)
|
||||
if cfg.STT.STTProvider() == STTProviderLocal {
|
||||
cfg.STT.LocalModelPath = model
|
||||
} else {
|
||||
cfg.STT.Model = model
|
||||
}
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetSTTLocalEngine persists the paths of an auto-downloaded local engine +
|
||||
// model and switches dictation to the local provider, mirroring SetTheme
|
||||
// (read-modify-atomic-write). streaming selects the pipeline matching the
|
||||
// downloaded model (a streaming transducer vs a batch model). Called after a
|
||||
// download completes.
|
||||
func SetSTTLocalEngine(path, binary, serverBinary, modelPath string, streaming bool) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg.STT.Provider = STTProviderLocal
|
||||
// Point BOTH pipelines at the local engine. Without resetting StreamProvider, a
|
||||
// previously-chosen cloud value (deepgram/openai) would still win in
|
||||
// buildStreamingTranscriber, so the live transcript would keep hitting the cloud
|
||||
// after the user switched to a downloaded local model.
|
||||
cfg.STT.StreamProvider = STTProviderLocal
|
||||
cfg.STT.LocalBinary = strings.TrimSpace(binary)
|
||||
cfg.STT.LocalServerBinary = strings.TrimSpace(serverBinary)
|
||||
cfg.STT.LocalModelPath = strings.TrimSpace(modelPath)
|
||||
// Match the pipeline to the downloaded model: a streaming transducer drives
|
||||
// the websocket server for a live transcript; a batch model uses the offline
|
||||
// binary.
|
||||
s := streaming
|
||||
cfg.STT.Streaming = &s
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SetSTTProvider persists just the dictation batch provider, mirroring SetTheme.
|
||||
func SetSTTProvider(path string, provider STTProviderKind) (FileConfig, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return FileConfig{}, fmt.Errorf("config path is required")
|
||||
}
|
||||
cfg := FileConfig{}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("invalid config JSON %s: %w", path, err)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return FileConfig{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
cfg.STT.Provider = provider
|
||||
if err := validateSTTConfig(cfg.STT); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
if err := writeConfigFile(path, cfg); err != nil {
|
||||
return FileConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func normalizeFavoriteModels(models []string) []string {
|
||||
seen := map[string]bool{}
|
||||
favorites := make([]string, 0, len(models))
|
||||
@@ -191,6 +679,35 @@ func normalizeFavoriteModels(models []string) []string {
|
||||
return favorites
|
||||
}
|
||||
|
||||
// NormalizeRecentModels trims, drops entries with no model id, de-duplicates
|
||||
// by provider+model pair (keeping the first/newest occurrence), and caps the
|
||||
// result to MaxRecentModels. Order is preserved — the caller is responsible
|
||||
// for passing entries newest-first. Exported so callers outside this package
|
||||
// (e.g. the TUI, which keeps its own in-memory copy of recent history) apply
|
||||
// the exact same normalization rules as the persisted config, instead of
|
||||
// maintaining a second, independently-drifting copy of this logic.
|
||||
func NormalizeRecentModels(entries []RecentModelEntry) []RecentModelEntry {
|
||||
seen := map[string]bool{}
|
||||
recent := make([]RecentModelEntry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
provider := strings.TrimSpace(entry.Provider)
|
||||
model := strings.TrimSpace(entry.Model)
|
||||
if model == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(provider) + "\x00" + model
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
recent = append(recent, RecentModelEntry{Provider: provider, Model: model})
|
||||
if len(recent) >= MaxRecentModels {
|
||||
break
|
||||
}
|
||||
}
|
||||
return recent
|
||||
}
|
||||
|
||||
func writeConfigFile(path string, cfg FileConfig) error {
|
||||
dir := filepath.Dir(path)
|
||||
if dir != "." && dir != "" {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
@@ -272,6 +273,69 @@ func TestSetFavoriteModelsPersistsUserPreferences(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetRecentModelsPersistsOrderDedupesAndCaps(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "openai",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "openai", ProviderKind: ProviderKindOpenAI, Model: "gpt-4.1"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
cfg, err := SetRecentModels(path, []RecentModelEntry{
|
||||
{Provider: " openrouter ", Model: " google/gemini-2.5-pro "},
|
||||
{Provider: "openrouter", Model: "minimax/minimax-m2.1"},
|
||||
{Provider: "openrouter", Model: "google/gemini-2.5-pro"}, // duplicate pair, older: dropped
|
||||
{Provider: "", Model: ""}, // blank model: dropped
|
||||
{Provider: "openrouter", Model: "a"},
|
||||
{Provider: "openrouter", Model: "b"},
|
||||
{Provider: "openrouter", Model: "c"},
|
||||
{Provider: "openrouter", Model: "d"}, // beyond MaxRecentModels (5): dropped
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SetRecentModels() error = %v", err)
|
||||
}
|
||||
|
||||
want := []RecentModelEntry{
|
||||
{Provider: "openrouter", Model: "google/gemini-2.5-pro"},
|
||||
{Provider: "openrouter", Model: "minimax/minimax-m2.1"},
|
||||
{Provider: "openrouter", Model: "a"},
|
||||
{Provider: "openrouter", Model: "b"},
|
||||
{Provider: "openrouter", Model: "c"},
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.Preferences.RecentModels, want) {
|
||||
t.Fatalf("RecentModels = %#v, want %#v", cfg.Preferences.RecentModels, want)
|
||||
}
|
||||
persisted := readConfigFixture(t, path)
|
||||
if !reflect.DeepEqual(persisted.Preferences.RecentModels, want) {
|
||||
t.Fatalf("persisted RecentModels = %#v, want %#v", persisted.Preferences.RecentModels, want)
|
||||
}
|
||||
if persisted.ActiveProvider != "openai" || len(persisted.Providers) != 1 {
|
||||
t.Fatalf("provider config was not preserved: %#v", persisted)
|
||||
}
|
||||
}
|
||||
|
||||
// Two providers offering the same model id must both survive normalization —
|
||||
// recent history de-duplicates by provider+model pair, not model id alone.
|
||||
func TestSetRecentModelsDedupesByProviderAndModelPair(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
|
||||
cfg, err := SetRecentModels(path, []RecentModelEntry{
|
||||
{Provider: "provider-a", Model: "shared-model"},
|
||||
{Provider: "provider-b", Model: "shared-model"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SetRecentModels() error = %v", err)
|
||||
}
|
||||
want := []RecentModelEntry{
|
||||
{Provider: "provider-a", Model: "shared-model"},
|
||||
{Provider: "provider-b", Model: "shared-model"},
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.Preferences.RecentModels, want) {
|
||||
t.Fatalf("RecentModels = %#v, want both providers preserved: %#v", cfg.Preferences.RecentModels, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetThemePersistsUserPreference(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
@@ -385,3 +449,545 @@ func readConfigFixture(t *testing.T, path string) FileConfig {
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func TestEnsureCatalogProviderCreatesProfileWithoutStealingActive(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "opengateway",
|
||||
Providers: []ProviderProfile{
|
||||
{
|
||||
Name: "opengateway",
|
||||
ProviderKind: ProviderKindOpenAICompatible,
|
||||
BaseURL: "https://gateway.example.com/v1",
|
||||
APIKeyStored: true,
|
||||
Model: "some-model",
|
||||
},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
ensured, err := EnsureCatalogProvider(path, "chatgpt")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureCatalogProvider: %v", err)
|
||||
}
|
||||
if !ensured.Created {
|
||||
t.Fatalf("expected profile to be created")
|
||||
}
|
||||
if ensured.Name != "chatgpt" {
|
||||
t.Fatalf("expected profile name chatgpt, got %q", ensured.Name)
|
||||
}
|
||||
if ensured.Active != "opengateway" {
|
||||
t.Fatalf("active provider must stay opengateway, got %q", ensured.Active)
|
||||
}
|
||||
|
||||
cfg := readConfigFixture(t, path)
|
||||
if cfg.ActiveProvider != "opengateway" {
|
||||
t.Fatalf("persisted active provider changed to %q", cfg.ActiveProvider)
|
||||
}
|
||||
if len(cfg.Providers) != 2 {
|
||||
t.Fatalf("expected 2 providers, got %d", len(cfg.Providers))
|
||||
}
|
||||
chatgpt := cfg.Providers[1]
|
||||
if chatgpt.Name != "chatgpt" || chatgpt.CatalogID != "chatgpt" {
|
||||
t.Fatalf("unexpected created profile: %+v", chatgpt)
|
||||
}
|
||||
if chatgpt.Model == "" || chatgpt.BaseURL == "" {
|
||||
t.Fatalf("created profile must carry catalog defaults, got %+v", chatgpt)
|
||||
}
|
||||
if chatgpt.APIKey != "" || chatgpt.APIKeyStored {
|
||||
t.Fatalf("OAuth-created profile must stay keyless, got %+v", chatgpt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCatalogProviderLeavesExistingProfileUntouched(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
original := FileConfig{
|
||||
ActiveProvider: "opengateway",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "opengateway", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://gateway.example.com/v1", Model: "some-model"},
|
||||
// Renamed profile that already serves the chatgpt catalog entry.
|
||||
{Name: "codex", CatalogID: "chatgpt", Model: "gpt-5.5"},
|
||||
},
|
||||
}
|
||||
data := writeConfigFixture(t, path, original, 0o600)
|
||||
|
||||
ensured, err := EnsureCatalogProvider(path, "chatgpt")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureCatalogProvider: %v", err)
|
||||
}
|
||||
if ensured.Created {
|
||||
t.Fatalf("existing profile must not be recreated")
|
||||
}
|
||||
if ensured.Name != "codex" {
|
||||
t.Fatalf("expected existing profile name codex, got %q", ensured.Name)
|
||||
}
|
||||
after, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
if string(after) != string(data) {
|
||||
t.Fatalf("config rewritten for a no-op ensure:\nbefore: %s\nafter: %s", data, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCatalogProviderActivatesOnEmptyConfig(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
|
||||
ensured, err := EnsureCatalogProvider(path, "chatgpt")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureCatalogProvider: %v", err)
|
||||
}
|
||||
if !ensured.Created {
|
||||
t.Fatalf("expected profile to be created")
|
||||
}
|
||||
if ensured.Active != "chatgpt" {
|
||||
t.Fatalf("blank active must adopt the new provider, got %q", ensured.Active)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCatalogProviderRejectsUnknownCatalogID(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
if _, err := EnsureCatalogProvider(path, "no-such-provider"); err == nil {
|
||||
t.Fatalf("expected unknown catalog id to error")
|
||||
}
|
||||
if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("config must not be written for an unknown catalog id")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveProviderDeletesAndHandsOffActive(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "beta",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"},
|
||||
{Name: "beta", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
cfg, err := RemoveProvider(path, " BETA ")
|
||||
if err != nil {
|
||||
t.Fatalf("RemoveProvider() error = %v", err)
|
||||
}
|
||||
if len(cfg.Providers) != 1 || cfg.Providers[0].Name != "alpha" {
|
||||
t.Fatalf("expected only alpha to remain, got %+v", cfg.Providers)
|
||||
}
|
||||
if cfg.ActiveProvider != "alpha" {
|
||||
t.Fatalf("active must hand off to the first remaining provider, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
|
||||
persisted := readConfigFixture(t, path)
|
||||
if len(persisted.Providers) != 1 || persisted.ActiveProvider != "alpha" {
|
||||
t.Fatalf("persisted config wrong: %+v", persisted)
|
||||
}
|
||||
|
||||
// Removing the last provider clears the active pointer entirely.
|
||||
cfg, err = RemoveProvider(path, "alpha")
|
||||
if err != nil {
|
||||
t.Fatalf("RemoveProvider(last) error = %v", err)
|
||||
}
|
||||
if len(cfg.Providers) != 0 || cfg.ActiveProvider != "" {
|
||||
t.Fatalf("expected empty providers and no active, got %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveProviderKeepsActiveWhenOtherRemoved(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"},
|
||||
{Name: "beta", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
cfg, err := RemoveProvider(path, "beta")
|
||||
if err != nil {
|
||||
t.Fatalf("RemoveProvider() error = %v", err)
|
||||
}
|
||||
if cfg.ActiveProvider != "alpha" {
|
||||
t.Fatalf("active provider must be untouched, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveProviderRejectsUnknownWithoutRewriting(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
before := writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"}},
|
||||
}, 0o600)
|
||||
|
||||
if _, err := RemoveProvider(path, "ghost"); err == nil {
|
||||
t.Fatal("RemoveProvider() error = nil, want not-found error")
|
||||
}
|
||||
after, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
if string(after) != string(before) {
|
||||
t.Fatalf("config was rewritten for unknown provider")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameProviderFollowsActiveAndMigratesStoredKey(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// Force the file credential backend so the test never touches the real OS
|
||||
// keyring regardless of platform.
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "oldname",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "oldname", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", APIKeyStored: true, Model: "m1"},
|
||||
{Name: "other", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("oldname", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := RenameProvider(path, "oldname", "newname")
|
||||
if err != nil {
|
||||
t.Fatalf("RenameProvider() error = %v", err)
|
||||
}
|
||||
if cfg.ActiveProvider != "newname" {
|
||||
t.Fatalf("active must follow the rename, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
if cfg.Providers[0].Name != "newname" || !cfg.Providers[0].APIKeyStored {
|
||||
t.Fatalf("renamed profile wrong: %+v", cfg.Providers[0])
|
||||
}
|
||||
if cfg.Providers[1].Name != "other" {
|
||||
t.Fatalf("unrelated profile changed: %+v", cfg.Providers[1])
|
||||
}
|
||||
|
||||
key, ok, err := store.Get("newname")
|
||||
if err != nil || !ok || key != "sk-secret" {
|
||||
t.Fatalf("stored key must migrate to the new name, got key=%q ok=%v err=%v", key, ok, err)
|
||||
}
|
||||
if _, ok, _ := store.Get("oldname"); ok {
|
||||
t.Fatalf("old credential entry must be deleted after migration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameProviderRejectsCollisionAndUnknown(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
before := writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"},
|
||||
{Name: "beta", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
if _, err := RenameProvider(path, "alpha", "BETA"); err == nil {
|
||||
t.Fatal("rename onto an existing name must fail")
|
||||
}
|
||||
if _, err := RenameProvider(path, "ghost", "gamma"); err == nil {
|
||||
t.Fatal("renaming an unknown provider must fail")
|
||||
}
|
||||
after, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
if string(after) != string(before) {
|
||||
t.Fatalf("config was rewritten by a rejected rename")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertProviderPreservesStoredKeyMarkerOnExistingProfile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
// An env-keyed profile with NO stored-key marker — the shape a provider has
|
||||
// before its key is captured into the credential store.
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "groq",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "groq", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://api.groq.com/openai/v1", APIKeyEnv: "GROQ_API_KEY", Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
// The manager/setup edit paths persist a SecureProviderProfile-shaped
|
||||
// profile: key already in the store, marker set, inline key cleared.
|
||||
cfg, err := UpsertProvider(path, ProviderProfile{Name: "groq", APIKeyStored: true}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertProvider() error = %v", err)
|
||||
}
|
||||
if !cfg.Providers[0].APIKeyStored {
|
||||
t.Fatalf("APIKeyStored marker must survive the merge, got %+v", cfg.Providers[0])
|
||||
}
|
||||
if cfg.Providers[0].APIKeyEnv != "GROQ_API_KEY" || cfg.Providers[0].BaseURL == "" {
|
||||
t.Fatalf("other fields must be preserved: %+v", cfg.Providers[0])
|
||||
}
|
||||
persisted := readConfigFixture(t, path)
|
||||
if !persisted.Providers[0].APIKeyStored {
|
||||
t.Fatalf("marker not persisted to disk: %+v", persisted.Providers[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetProviderDescriptionSetsAndClears(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "zero.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1", Description: "old text"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
cfg, err := SetProviderDescription(path, " ALPHA ", "new text")
|
||||
if err != nil {
|
||||
t.Fatalf("SetProviderDescription() error = %v", err)
|
||||
}
|
||||
if cfg.Providers[0].Description != "new text" {
|
||||
t.Fatalf("description not set: %+v", cfg.Providers[0])
|
||||
}
|
||||
|
||||
// Clearing must persist too — the reason this setter exists (UpsertProvider's
|
||||
// merge treats an empty description as "leave unchanged").
|
||||
cfg, err = SetProviderDescription(path, "alpha", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("SetProviderDescription(clear) error = %v", err)
|
||||
}
|
||||
if cfg.Providers[0].Description != "" {
|
||||
t.Fatalf("description not cleared: %+v", cfg.Providers[0])
|
||||
}
|
||||
persisted := readConfigFixture(t, path)
|
||||
if persisted.Providers[0].Description != "" {
|
||||
t.Fatalf("cleared description not persisted: %+v", persisted.Providers[0])
|
||||
}
|
||||
|
||||
if _, err := SetProviderDescription(path, "ghost", "x"); err == nil {
|
||||
t.Fatal("unknown provider must error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameProviderRollsBackKeyMigrationWhenConfigWriteFails(t *testing.T) {
|
||||
if runtime.GOOS != "darwin" {
|
||||
t.Skip("uses chflags uchg to force the config write to fail; macOS only")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "oldname",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "oldname", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", APIKeyStored: true, Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("oldname", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
// Immutable flag: temp-file creation and store writes in the directory keep
|
||||
// working, but the final rename over config.json fails — the exact window
|
||||
// where a migrated key would otherwise strand under the new name.
|
||||
if out, err := exec.Command("chflags", "uchg", path).CombinedOutput(); err != nil {
|
||||
t.Skipf("chflags uchg unavailable: %v (%s)", err, out)
|
||||
}
|
||||
t.Cleanup(func() { _ = exec.Command("chflags", "nouchg", path).Run() })
|
||||
|
||||
if _, err := RenameProvider(path, "oldname", "newname"); err == nil {
|
||||
t.Fatal("expected the config write to fail under the immutable flag")
|
||||
}
|
||||
|
||||
key, ok, err := store.Get("oldname")
|
||||
if err != nil || !ok || key != "sk-secret" {
|
||||
t.Fatalf("key must be rolled back to the old name, got key=%q ok=%v err=%v", key, ok, err)
|
||||
}
|
||||
if _, ok, _ := store.Get("newname"); ok {
|
||||
t.Fatalf("rolled-back migration must not leave a key under the new name")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenameProviderCaseOnlyKeepsStoredKey: the credential store normalizes
|
||||
// names case-insensitively, so a case-only rename targets ONE store entry —
|
||||
// migrating would Set and then Delete the same key, losing it (PR #560 review).
|
||||
func TestRenameProviderCaseOnlyKeepsStoredKey(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "groq",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "groq", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://api.groq.com/openai/v1", APIKeyStored: true, Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("groq", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := RenameProvider(path, "groq", "Groq")
|
||||
if err != nil {
|
||||
t.Fatalf("RenameProvider() error = %v", err)
|
||||
}
|
||||
if cfg.Providers[0].Name != "Groq" || cfg.ActiveProvider != "Groq" {
|
||||
t.Fatalf("case-only rename must still apply to config: %+v", cfg)
|
||||
}
|
||||
// The store is case-insensitive: the key must remain retrievable under the
|
||||
// new casing (same entry), not deleted by a same-entry "migration".
|
||||
if key, ok, err := store.Get("Groq"); err != nil || !ok || key != "sk-secret" {
|
||||
t.Fatalf("stored key lost on case-only rename: key=%q ok=%v err=%v", key, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEditProviderAppliesRenameFieldsAndDescriptionAtomically(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "groq",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "groq", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://api.groq.com/openai/v1", APIKeyStored: true, Model: "m1", Description: "old text"},
|
||||
{Name: "other", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://o.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("groq", "sk-old"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := EditProvider(path, ProviderEdit{
|
||||
Name: "groq",
|
||||
NewName: "grok-main",
|
||||
Model: "m1-pro",
|
||||
Description: "", // cleared — must persist verbatim
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("EditProvider() error = %v", err)
|
||||
}
|
||||
edited := cfg.Providers[0]
|
||||
if edited.Name != "grok-main" || edited.Model != "m1-pro" || edited.Description != "" {
|
||||
t.Fatalf("edit not applied: %+v", edited)
|
||||
}
|
||||
if edited.BaseURL != "https://api.groq.com/openai/v1" || !edited.APIKeyStored {
|
||||
t.Fatalf("untouched fields must survive: %+v", edited)
|
||||
}
|
||||
if cfg.ActiveProvider != "grok-main" {
|
||||
t.Fatalf("active must follow the rename, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
if key, ok, _ := store.Get("grok-main"); !ok || key != "sk-old" {
|
||||
t.Fatalf("stored key must migrate with the rename, got %q ok=%v", key, ok)
|
||||
}
|
||||
if len(cfg.Providers) != 2 || cfg.Providers[1].Name != "other" {
|
||||
t.Fatalf("unrelated profile changed: %+v", cfg.Providers)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEditProviderCaseOnlyRenameUpdatesInPlace: the manager previously skipped
|
||||
// RenameProvider on case-insensitively-equal names and fell into UpsertProvider,
|
||||
// whose case-SENSITIVE merge appended a duplicate profile. EditProvider matches
|
||||
// case-insensitively, so a case-only rename is an in-place update and the store
|
||||
// entry (case-normalized) survives.
|
||||
func TestEditProviderCaseOnlyRenameUpdatesInPlace(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "groq",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "groq", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://api.groq.com/openai/v1", APIKeyStored: true, Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if err := store.Set("groq", "sk-secret"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := EditProvider(path, ProviderEdit{Name: "groq", NewName: "Groq"})
|
||||
if err != nil {
|
||||
t.Fatalf("EditProvider() error = %v", err)
|
||||
}
|
||||
if len(cfg.Providers) != 1 {
|
||||
t.Fatalf("case-only rename must not duplicate the profile: %+v", cfg.Providers)
|
||||
}
|
||||
if cfg.Providers[0].Name != "Groq" || !cfg.Providers[0].APIKeyStored {
|
||||
t.Fatalf("in-place update wrong: %+v", cfg.Providers[0])
|
||||
}
|
||||
if cfg.Providers[0].BaseURL != "https://api.groq.com/openai/v1" {
|
||||
t.Fatalf("fields must survive a case-only rename: %+v", cfg.Providers[0])
|
||||
}
|
||||
if cfg.ActiveProvider != "Groq" {
|
||||
t.Fatalf("active must follow, got %q", cfg.ActiveProvider)
|
||||
}
|
||||
if key, ok, _ := store.Get("Groq"); !ok || key != "sk-secret" {
|
||||
t.Fatalf("stored key lost on case-only rename: %q ok=%v", key, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEditProviderRenameMigratesFreshlyCapturedKey: replacing the key AND
|
||||
// renaming in one edit — the caller captures under the CURRENT name and the
|
||||
// rename migration moves it, so the new key lands under the new name.
|
||||
func TestEditProviderRenameMigratesFreshlyCapturedKey(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ZERO_CRED_STORAGE", "encrypted-file")
|
||||
path := filepath.Join(dir, "config.json")
|
||||
writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "gw",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "gw", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://gw.example.com/v1", APIKeyEnv: "GW_KEY", Model: "m1"},
|
||||
},
|
||||
}, 0o600)
|
||||
store, err := ProviderKeyStoreAt(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
// The caller's contract: a replacement key is stored under the CURRENT name
|
||||
// before EditProvider runs (what SecureProviderProfile does).
|
||||
if err := store.Set("gw", "sk-new"); err != nil {
|
||||
t.Fatalf("seed store: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := EditProvider(path, ProviderEdit{Name: "gw", NewName: "gateway", APIKeyStored: true})
|
||||
if err != nil {
|
||||
t.Fatalf("EditProvider() error = %v", err)
|
||||
}
|
||||
if !cfg.Providers[0].APIKeyStored || cfg.Providers[0].Name != "gateway" {
|
||||
t.Fatalf("marker/rename wrong: %+v", cfg.Providers[0])
|
||||
}
|
||||
if key, ok, _ := store.Get("gateway"); !ok || key != "sk-new" {
|
||||
t.Fatalf("captured key must migrate to the new name, got %q ok=%v", key, ok)
|
||||
}
|
||||
if _, ok, _ := store.Get("gw"); ok {
|
||||
t.Fatalf("old entry must be cleaned up after migration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEditProviderRejectsCollisionAndUnknown(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.json")
|
||||
before := writeConfigFixture(t, path, FileConfig{
|
||||
ActiveProvider: "alpha",
|
||||
Providers: []ProviderProfile{
|
||||
{Name: "alpha", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://a.example.com/v1", Model: "m1"},
|
||||
{Name: "beta", ProviderKind: ProviderKindOpenAICompatible, BaseURL: "https://b.example.com/v1", Model: "m2"},
|
||||
},
|
||||
}, 0o600)
|
||||
|
||||
if _, err := EditProvider(path, ProviderEdit{Name: "alpha", NewName: "BETA"}); err == nil {
|
||||
t.Fatal("rename onto an existing name must fail")
|
||||
}
|
||||
if _, err := EditProvider(path, ProviderEdit{Name: "ghost", Model: "x"}); err == nil {
|
||||
t.Fatal("editing an unknown provider must fail")
|
||||
}
|
||||
after, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
if string(after) != string(before) {
|
||||
t.Fatalf("config was rewritten by a rejected edit")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// Package dictation implements speech-to-text capture and transcription for
|
||||
// the composer: press a key, talk, and the transcript lands in the input for
|
||||
// review — never auto-fired at the agent.
|
||||
//
|
||||
// Two pipelines share a keybinding and a composer-insertion target:
|
||||
//
|
||||
// - Batch: record to a temp WAV file (Recorder.Start/Stop), then transcribe
|
||||
// in one shot (Transcriber.Transcribe) — a local sherpa-onnx-offline exec
|
||||
// or a single HTTPS POST to Groq/OpenAI. The only pipeline on Termux.
|
||||
// - Streaming: capture raw PCM from the mic tool's stdout
|
||||
// (Recorder.StartStreaming), feed 50ms chunks to a websocket
|
||||
// (Transcriber.StreamTranscribe), and render partial transcripts live.
|
||||
//
|
||||
// Every OS integration shells out via exec.Command with discrete argv — never
|
||||
// a shell string, never CGO — matching internal/imageinput/clipboard.go. The
|
||||
// external binaries (arecord/sox/ffmpeg/termux-microphone-record, sherpa-onnx)
|
||||
// are looked up on PATH and degrade to a clear setup message when missing,
|
||||
// matching internal/lsp's philosophy for language servers.
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Batch transcription providers (config value stt.provider).
|
||||
const (
|
||||
ProviderLocal = "local"
|
||||
ProviderGroq = "groq"
|
||||
ProviderOpenAI = "openai"
|
||||
)
|
||||
|
||||
// Streaming transcription providers (config value stt.streamProvider).
|
||||
const (
|
||||
StreamProviderLocal = "local"
|
||||
StreamProviderDeepgram = "deepgram"
|
||||
StreamProviderOpenAI = "openai"
|
||||
)
|
||||
|
||||
// BatchProviders lists valid stt.provider values, for config validation.
|
||||
func BatchProviders() []string {
|
||||
return []string{ProviderLocal, ProviderGroq, ProviderOpenAI}
|
||||
}
|
||||
|
||||
// StreamProviders lists valid stt.streamProvider values, for config validation.
|
||||
func StreamProviders() []string {
|
||||
return []string{StreamProviderLocal, StreamProviderDeepgram, StreamProviderOpenAI}
|
||||
}
|
||||
|
||||
// AudioFormat identifies a recorded container format, sniffed from the bytes —
|
||||
// desktop recorders produce WAV, Termux's recorder produces M4A/AAC.
|
||||
type AudioFormat string
|
||||
|
||||
const (
|
||||
FormatWAV AudioFormat = "wav"
|
||||
FormatM4A AudioFormat = "m4a"
|
||||
FormatUnknown AudioFormat = ""
|
||||
)
|
||||
|
||||
// SniffFormat detects the audio container from magic bytes. WAV is
|
||||
// "RIFF....WAVE"; MP4-family (Termux's .m4a) has "ftyp" at offset 4.
|
||||
func SniffFormat(data []byte) AudioFormat {
|
||||
if len(data) >= 12 && bytes.Equal(data[:4], []byte("RIFF")) && bytes.Equal(data[8:12], []byte("WAVE")) {
|
||||
return FormatWAV
|
||||
}
|
||||
if len(data) >= 8 && bytes.Equal(data[4:8], []byte("ftyp")) {
|
||||
return FormatM4A
|
||||
}
|
||||
return FormatUnknown
|
||||
}
|
||||
|
||||
// FileName returns a transcription-upload filename for the format.
|
||||
func (f AudioFormat) FileName() string {
|
||||
switch f {
|
||||
case FormatM4A:
|
||||
return "audio.m4a"
|
||||
default:
|
||||
return "audio.wav"
|
||||
}
|
||||
}
|
||||
|
||||
// SetupError reports a missing external tool with install guidance. Missing
|
||||
// binaries are an expected condition (the user hasn't set dictation up yet),
|
||||
// not a bug — callers show the hint instead of a raw exec error.
|
||||
type SetupError struct {
|
||||
Tool string
|
||||
Hint string
|
||||
}
|
||||
|
||||
func (e *SetupError) Error() string {
|
||||
return fmt.Sprintf("%s not found — %s", e.Tool, e.Hint)
|
||||
}
|
||||
|
||||
// AuthError reports that a cloud provider rejected a transcription request for
|
||||
// authentication reasons (HTTP 401/403) — a missing or invalid API key. It is a
|
||||
// fixable credential problem, so callers (the TUI) can offer an inline key
|
||||
// prompt for Provider instead of a dead-end error. Message is the already-
|
||||
// classified, secret-redacted user-facing text.
|
||||
type AuthError struct {
|
||||
Provider string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *AuthError) Error() string { return e.Message }
|
||||
@@ -0,0 +1,893 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/bzip2"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Auto-download of the local engine + a default model (opt-in, behind a confirm
|
||||
// in the F9 flow). Assets come from the official k2-fsa/sherpa-onnx GitHub
|
||||
// releases and are verified against a SHA256 digest before use — the same trust
|
||||
// boundary as installing them yourself, automated. Auto-downloading a native
|
||||
// executable is a real trust decision, which is why it is opt-in and verified.
|
||||
//
|
||||
// Rather than freeze a single hardcoded version, the downloader resolves the
|
||||
// release ASSETS via the GitHub API at download time and verifies each download
|
||||
// against the API's per-asset digest — so a newer sherpa-onnx release (or
|
||||
// "latest") works with no code change. A known-good version is still the DEFAULT
|
||||
// (reproducible, smoke-tested), and for that default the resolved digest is
|
||||
// cross-checked against a pinned value so a silently-changed release is caught.
|
||||
|
||||
// DefaultSherpaVersion is the pinned, smoke-tested release used unless
|
||||
// stt.engineVersion overrides it. "latest" or any tag also works.
|
||||
const DefaultSherpaVersion = "v1.13.3"
|
||||
|
||||
const (
|
||||
sherpaRepo = "k2-fsa/sherpa-onnx"
|
||||
modelReleaseTag = "asr-models" // rolling release the ASR models live in
|
||||
modelAssetName = "sherpa-onnx-moonshine-tiny-en-int8.tar.bz2"
|
||||
defaultAPIBase = "https://api.github.com"
|
||||
)
|
||||
|
||||
// engineAssetSuffix maps GOOS-GOARCH to the version-independent suffix of the
|
||||
// sherpa-onnx executable bundle for that platform. Matching by suffix (not a
|
||||
// full version-embedded name) is what lets any release resolve. The bundles are
|
||||
// self-contained (RPATH $ORIGIN/../lib), so the extracted bin/ executables run
|
||||
// with no library-path setup.
|
||||
var engineAssetSuffix = map[string]string{
|
||||
"linux-amd64": "linux-x64-shared-no-tts.tar.bz2",
|
||||
"darwin-arm64": "osx-arm64-shared-no-tts.tar.bz2",
|
||||
"darwin-amd64": "osx-x64-shared-no-tts.tar.bz2",
|
||||
"windows-amd64": "win-x64-shared-MT-Release-no-tts.tar.bz2",
|
||||
}
|
||||
|
||||
// pinnedEngineDigest cross-checks the DefaultSherpaVersion engine assets (a
|
||||
// silently-changed pinned release is refused). Empty for non-default versions,
|
||||
// which are verified against the API digest only.
|
||||
var pinnedEngineDigest = map[string]string{
|
||||
"linux-amd64": "e5cbfd96f3666c25d2272d468e337764fce6a4d50ed074e4b8e5021bfd7fefc1",
|
||||
"darwin-arm64": "ea73233bc250019ccf1f8a16feb29460efcd3425b50cac16980bcb76281d60bf",
|
||||
"darwin-amd64": "563423887d63d8a831c473e0ab6815139fe2e7a3482936716c2a128071dd217f",
|
||||
"windows-amd64": "343c6c39cb4fe1880a9fb1abc8401d5174629ccbb147fca1fd53c440b2664570",
|
||||
}
|
||||
|
||||
// pinnedModelDigest cross-checks the default Moonshine-tiny int8 model.
|
||||
const pinnedModelDigest = "d5fe6ec4334fef36255b2a4010412cad4c007e33103fec62fb5d17cad88086f2"
|
||||
|
||||
// ModelVariant is a downloadable model the /stt-model picker offers, with a
|
||||
// pinned SHA256 (the model release predates GitHub's per-asset digests, so these
|
||||
// must be pinned). Batch variants use the offline binary; the streaming variant
|
||||
// (a transducer) drives the websocket server for a live transcript.
|
||||
type ModelVariant struct {
|
||||
ID string
|
||||
Label string
|
||||
Description string
|
||||
Bytes int64
|
||||
AssetName string
|
||||
Digest string
|
||||
DirName string
|
||||
Streaming bool
|
||||
// Recommended marks a curated, checksum-pinned, validated model (shown first).
|
||||
Recommended bool
|
||||
}
|
||||
|
||||
// ModelVariants returns the curated English models the download picker offers.
|
||||
// This is a hand-picked shortlist (each pinned by SHA256 since the model release
|
||||
// predates GitHub's per-asset digests), NOT the whole sherpa-onnx model zoo —
|
||||
// many more models (other languages, sizes, and streaming/batch families) work
|
||||
// via a manual stt.localModelPath. Streaming (live) options come first.
|
||||
func ModelVariants() []ModelVariant {
|
||||
return []ModelVariant{
|
||||
// Live / streaming — transcript builds up as you speak.
|
||||
// DirName MUST match what ListModels derives from the asset ("model-"+id, id =
|
||||
// asset minus "sherpa-onnx-" prefix and ".tar.bz2" suffix, dates included) so a
|
||||
// model extracts to the same directory whether picked from the curated shortlist
|
||||
// or the full browse list — otherwise installed-detection disagrees between the
|
||||
// two views and the same model can be downloaded twice. Recommended matches the
|
||||
// full list too (recommendedModelIDs marks every curated variant), so the ★ and
|
||||
// ordering don't change when the background fetch swaps the curated list out.
|
||||
{
|
||||
ID: "streaming-zipformer-en-kroko", Label: "Kroko (newest, fastest)",
|
||||
Description: "smallest live model", Bytes: 54 << 20,
|
||||
AssetName: "sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06.tar.bz2",
|
||||
Digest: "c8676e5ff9ac2a85296e53ee0fd4d5fb1db6770e7a7647166eeafe349ade6834",
|
||||
DirName: "model-streaming-zipformer-en-kroko-2025-08-06",
|
||||
Streaming: true,
|
||||
Recommended: true,
|
||||
},
|
||||
{
|
||||
ID: "streaming-zipformer-20m", Label: "Zipformer 20M",
|
||||
Description: "small, well-tested", Bytes: 121 << 20,
|
||||
AssetName: "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2",
|
||||
Digest: "9c559283e8498d3fe95913c79ca1cb454bb26281ac2b102b41306c7d752765d9",
|
||||
DirName: "model-streaming-zipformer-en-20M-2023-02-17",
|
||||
Streaming: true,
|
||||
},
|
||||
{
|
||||
ID: "streaming-zipformer", Label: "Zipformer",
|
||||
Description: "larger, more accurate", Bytes: 297 << 20,
|
||||
AssetName: "sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2",
|
||||
Digest: "639e25b578e9e997131402199419c13a941f8e4e198e2da1ce57dbf5cf401282",
|
||||
DirName: "model-streaming-zipformer-en-2023-06-26",
|
||||
Streaming: true,
|
||||
},
|
||||
{
|
||||
ID: "nemo-streaming-fast-conformer-transducer-en-80ms-int8", Label: "NeMo streaming (fast)",
|
||||
Description: "low-latency English", Bytes: 98 << 20,
|
||||
AssetName: "sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms-int8.tar.bz2",
|
||||
Digest: "7bd33a914e93370a1ba9c2066d9e841bdcad8613fa2a00537c1ae15d851a14d8",
|
||||
DirName: "model-nemo-streaming-fast-conformer-transducer-en-80ms-int8",
|
||||
Streaming: true,
|
||||
Recommended: true,
|
||||
},
|
||||
// Batch — transcribes once when you stop.
|
||||
{
|
||||
ID: "moonshine-tiny", Label: "Moonshine tiny",
|
||||
Description: "fast, smallest", Bytes: 103 << 20,
|
||||
AssetName: "sherpa-onnx-moonshine-tiny-en-int8.tar.bz2",
|
||||
Digest: pinnedModelDigest,
|
||||
DirName: "model-moonshine-tiny-en-int8",
|
||||
},
|
||||
{
|
||||
ID: "moonshine-base", Label: "Moonshine base",
|
||||
Description: "more accurate, larger", Bytes: 239 << 20,
|
||||
AssetName: "sherpa-onnx-moonshine-base-en-int8.tar.bz2",
|
||||
Digest: "21870cecaa2e44e4e2bf63e02d1072bed183ccd10284871353bd9d24dad14e5e",
|
||||
DirName: "model-moonshine-base-en-int8",
|
||||
Recommended: true,
|
||||
},
|
||||
{
|
||||
ID: "sense-voice-zh-en-ja-ko-yue-int8-2025-09-09", Label: "SenseVoice",
|
||||
Description: "multilingual, accurate", Bytes: 158 << 20,
|
||||
AssetName: "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2",
|
||||
Digest: "7305f7905bfcf77fa0b39388a313f3da35c68d971661a65475b56fb2162c8e63",
|
||||
DirName: "model-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09",
|
||||
Recommended: true,
|
||||
},
|
||||
{
|
||||
// Heavy (~1 GB) but the best transcription quality — recommended when
|
||||
// accuracy matters more than speed/size. The asr-models release publishes
|
||||
// no digest for it, so (like every browse-listed model) it downloads with
|
||||
// TLS-only verification.
|
||||
ID: "whisper-large-v3", Label: "Whisper large v3",
|
||||
Description: "best accuracy, heavy", Bytes: 1019 << 20,
|
||||
AssetName: "sherpa-onnx-whisper-large-v3.tar.bz2",
|
||||
DirName: "model-whisper-large-v3",
|
||||
Recommended: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// recommendedModelIDs marks the curated variants flagged Recommended so the full
|
||||
// browse list stars (and surfaces) exactly the ones the curated shortlist does —
|
||||
// keyed off the same Recommended flag so the two views never disagree.
|
||||
var recommendedModelIDs = func() map[string]bool {
|
||||
m := map[string]bool{}
|
||||
for _, v := range ModelVariants() {
|
||||
if v.Recommended {
|
||||
m[v.AssetName] = true
|
||||
}
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// pinnedModelDigests maps a model asset name to its pinned SHA256 (the curated
|
||||
// variants), so browse-listed models we happen to know get verified too.
|
||||
var pinnedModelDigests = func() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, v := range ModelVariants() {
|
||||
m[v.AssetName] = v.Digest
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// curatedLabels maps a curated model's asset name to its friendly label, so the
|
||||
// recommended rows read nicely in the full browse list too.
|
||||
var curatedLabels = func() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, v := range ModelVariants() {
|
||||
m[v.AssetName] = v.Label
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// browseLabel turns a raw asset id into a shorter display label: it drops the
|
||||
// redundant "-int8"/quantization noise and the "streaming-" prefix (the group
|
||||
// header already says live vs batch).
|
||||
func browseLabel(id string) string {
|
||||
label := strings.TrimPrefix(id, "streaming-")
|
||||
for _, drop := range []string{"-int8", ".int8", "-quantized", "-fp16"} {
|
||||
label = strings.ReplaceAll(label, drop, "")
|
||||
}
|
||||
return label
|
||||
}
|
||||
|
||||
// ListModels fetches every transcription model in the sherpa-onnx asr-models
|
||||
// release and returns them as variants (the curated ones flagged Recommended and
|
||||
// carrying a pinned digest; the rest with an empty digest, downloaded with
|
||||
// TLS-only verification since that release predates GitHub's per-asset digests).
|
||||
// Non-ASR assets (VAD, TTS, punctuation, speaker/keyword/…): filtered out.
|
||||
func ListModels(ctx context.Context, client *http.Client, apiBase string) ([]ModelVariant, error) {
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
if apiBase == "" {
|
||||
apiBase = defaultAPIBase
|
||||
}
|
||||
// Resolve the release id, then page its assets (a release can have hundreds).
|
||||
relURL := fmt.Sprintf("%s/repos/%s/releases/tags/%s", strings.TrimRight(apiBase, "/"), sherpaRepo, modelReleaseTag)
|
||||
var rel struct {
|
||||
ID int64 `json:"id"`
|
||||
Assets []githubAsset
|
||||
}
|
||||
if err := getJSON(ctx, client, relURL, &rel); err != nil {
|
||||
return nil, fmt.Errorf("listing dictation models: %w", err)
|
||||
}
|
||||
assets := rel.Assets
|
||||
// If the inline assets look capped, page the assets endpoint for the rest.
|
||||
for page := 1; len(assets)%100 == 0 && len(assets) > 0; page++ {
|
||||
var more []githubAsset
|
||||
u := fmt.Sprintf("%s/repos/%s/releases/%d/assets?per_page=100&page=%d", strings.TrimRight(apiBase, "/"), sherpaRepo, rel.ID, page+1)
|
||||
if err := getJSON(ctx, client, u, &more); err != nil || len(more) == 0 {
|
||||
break
|
||||
}
|
||||
assets = append(assets, more...)
|
||||
if len(more) < 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
var out []ModelVariant
|
||||
for _, a := range assets {
|
||||
if seen[a.Name] || !isTranscriptionModelAsset(a.Name) {
|
||||
continue
|
||||
}
|
||||
seen[a.Name] = true
|
||||
id := strings.TrimSuffix(strings.TrimPrefix(a.Name, "sherpa-onnx-"), ".tar.bz2")
|
||||
digest := strings.TrimPrefix(a.Digest, "sha256:")
|
||||
if digest == "" {
|
||||
digest = pinnedModelDigests[a.Name] // may still be empty (browse/unverified)
|
||||
}
|
||||
label := browseLabel(id)
|
||||
desc := modelFamily(a.Name) // readable language, e.g. "French"
|
||||
if nice, ok := curatedLabels[a.Name]; ok {
|
||||
label = nice
|
||||
desc = curatedDescriptions[a.Name] // curated blurb ("fast, smallest")
|
||||
}
|
||||
out = append(out, ModelVariant{
|
||||
ID: id,
|
||||
Label: label,
|
||||
Description: desc,
|
||||
Bytes: a.Size,
|
||||
AssetName: a.Name,
|
||||
Digest: digest,
|
||||
DirName: "model-" + id,
|
||||
Streaming: strings.Contains(strings.ToLower(a.Name), "streaming-"),
|
||||
Recommended: recommendedModelIDs[a.Name],
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type githubAsset struct {
|
||||
Name string `json:"name"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
func getJSON(ctx context.Context, client *http.Client, url string, v any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("GitHub API HTTP %d", resp.StatusCode)
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(v)
|
||||
}
|
||||
|
||||
// isTranscriptionModelAsset reports whether an asset is a speech-to-text model
|
||||
// (as opposed to a VAD/TTS/punctuation/speaker/keyword/enhancement tool).
|
||||
func isTranscriptionModelAsset(name string) bool {
|
||||
n := strings.ToLower(name)
|
||||
if !strings.HasPrefix(n, "sherpa-onnx-") || !strings.HasSuffix(n, ".tar.bz2") {
|
||||
return false
|
||||
}
|
||||
for _, bad := range []string{
|
||||
"-vad", "vad-", "tts", "punct", "speaker", "diariz", "kws", "keyword",
|
||||
"audio-tagging", "denois", "source-separation", "speech-enhancement",
|
||||
"spoken-language", "language-identification", "-lid", "melo", "kokoro",
|
||||
"vits", "matcha", "piper", "wespeaker",
|
||||
// Rockchip NPU (RKNN) builds for specific ARM boards — not CPU/ONNX, so
|
||||
// the standard sherpa-onnx binary can't run them.
|
||||
"rknn", "rk3566", "rk3568", "rk3562", "rk3588", "rk3576", "rk3528", "rk3399",
|
||||
} {
|
||||
if strings.Contains(n, bad) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, fam := range []string{
|
||||
"zipformer", "moonshine", "whisper", "paraformer", "sense-voice",
|
||||
"sensevoice", "conformer", "nemo", "canary", "parakeet", "fire-red",
|
||||
"fireredasr", "dolphin", "telespeech", "wenet", "omnilingual", "funasr",
|
||||
"medasr", "cohere", "qwen", "citrinet", "transducer", "-ctc-", "zipvoice",
|
||||
} {
|
||||
if strings.Contains(n, fam) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// langNames maps ISO-ish language tokens in a model name to readable names.
|
||||
var langNames = map[string]string{
|
||||
"en": "English", "zh": "Chinese", "yue": "Cantonese", "fr": "French",
|
||||
"de": "German", "es": "Spanish", "ru": "Russian", "ja": "Japanese",
|
||||
"ko": "Korean", "bn": "Bengali", "ar": "Arabic", "hi": "Hindi",
|
||||
"pt": "Portuguese", "nl": "Dutch", "vi": "Vietnamese", "th": "Thai",
|
||||
"uk": "Ukrainian", "fa": "Persian", "pl": "Polish", "tr": "Turkish",
|
||||
"cs": "Czech", "sv": "Swedish", "fil": "Filipino", "tl": "Tagalog",
|
||||
"fi": "Finnish", "el": "Greek", "he": "Hebrew", "ro": "Romanian",
|
||||
}
|
||||
|
||||
// modelFamily is a short readable descriptor for a model, favoring the language
|
||||
// (the most useful thing not already obvious from the group header) and falling
|
||||
// back to the family. E.g. "French", "Chinese", "Multilingual".
|
||||
func modelFamily(name string) string {
|
||||
n := strings.ToLower(name)
|
||||
if strings.Contains(n, "multi") || strings.Contains(n, "bilingual") ||
|
||||
strings.Contains(n, "trilingual") || strings.Contains(n, "polyglot") ||
|
||||
strings.Contains(n, "omnilingual") {
|
||||
return "Multilingual"
|
||||
}
|
||||
var langs []string
|
||||
seen := map[string]bool{}
|
||||
add := func(name string) {
|
||||
if name != "" && !seen[name] {
|
||||
seen[name] = true
|
||||
langs = append(langs, name)
|
||||
}
|
||||
}
|
||||
for _, tok := range strings.Split(strings.TrimSuffix(n, ".tar.bz2"), "-") {
|
||||
add(langNames[tok])
|
||||
}
|
||||
// Full language words spelled out in some names (e.g. "-cantonese-").
|
||||
for word, name := range map[string]string{
|
||||
"cantonese": "Cantonese", "mandarin": "Chinese", "english": "English",
|
||||
"chinese": "Chinese", "korean": "Korean", "japanese": "Japanese",
|
||||
"russian": "Russian", "arabic": "Arabic", "spanish": "Spanish",
|
||||
"french": "French", "german": "German", "vietnamese": "Vietnamese",
|
||||
"thai": "Thai", "farsi": "Persian", "persian": "Persian",
|
||||
"portuguese": "Portuguese", "italian": "Italian", "dutch": "Dutch",
|
||||
"ukrainian": "Ukrainian", "polish": "Polish", "turkish": "Turkish",
|
||||
} {
|
||||
if strings.Contains(n, word) {
|
||||
add(name)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(langs) == 0:
|
||||
return ""
|
||||
case len(langs) > 2:
|
||||
return "Multilingual"
|
||||
default:
|
||||
return strings.Join(langs, " + ")
|
||||
}
|
||||
}
|
||||
|
||||
// curatedDescriptions maps a curated model's asset name to its quality blurb.
|
||||
var curatedDescriptions = func() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, v := range ModelVariants() {
|
||||
m[v.AssetName] = v.Description
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// EngineComponents identifies the resolved local-engine paths.
|
||||
type EngineComponents struct {
|
||||
BinaryPath string // extracted sherpa-onnx-offline
|
||||
ServerPath string // extracted sherpa-onnx-online-websocket-server
|
||||
ModelPath string // extracted model directory
|
||||
}
|
||||
|
||||
// DownloadOptions configures EnsureLocalEngine.
|
||||
type DownloadOptions struct {
|
||||
// DestRoot is where extracted engines/models live (e.g. ~/.config/zero/stt).
|
||||
DestRoot string
|
||||
// EngineVersion selects the sherpa-onnx release tag ("" → DefaultSherpaVersion;
|
||||
// "latest" and any tag also work).
|
||||
EngineVersion string
|
||||
// HTTPClient is injectable for tests; nil uses a plain client.
|
||||
HTTPClient *http.Client
|
||||
// APIBase overrides the GitHub API base (tests). "" → api.github.com.
|
||||
APIBase string
|
||||
// Model selection (default: the Moonshine-tiny int8 batch model). Set these
|
||||
// from a ModelVariant to download a different model.
|
||||
ModelAssetName string
|
||||
ModelPinnedDigest string
|
||||
ModelDirName string
|
||||
// ModelLabel is the friendly model name shown in progress ("Zipformer 20M").
|
||||
ModelLabel string
|
||||
// Progress receives live status strings for the UI, including a percentage
|
||||
// while downloading (e.g. "Engine 45% · 57/126 MB").
|
||||
Progress func(status string)
|
||||
// platformKey overrides runtime detection (tests).
|
||||
platformKey string
|
||||
// skipPinned disables the pinned-digest cross-check (tests, which use fixture
|
||||
// assets); the API-digest verification still runs.
|
||||
skipPinned bool
|
||||
}
|
||||
|
||||
// AutoDownloadSupported reports whether the current platform has a known engine
|
||||
// asset (so the F9 flow can offer the download). Termux/Android and unusual
|
||||
// arches install manually or use a cloud provider.
|
||||
func AutoDownloadSupported() bool {
|
||||
_, ok := engineAssetSuffix[platformKey()]
|
||||
return ok
|
||||
}
|
||||
|
||||
func platformKey() string { return runtime.GOOS + "-" + runtime.GOARCH }
|
||||
|
||||
// EnsureLocalEngine downloads (if absent) and verifies the sherpa-onnx engine and
|
||||
// the default model into DestRoot, returning the resolved paths. Idempotent: an
|
||||
// already-extracted, still-present engine/model is reused without re-downloading.
|
||||
// Every archive is SHA256-verified against the GitHub API digest before extraction.
|
||||
func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineComponents, error) {
|
||||
if strings.TrimSpace(opts.DestRoot) == "" {
|
||||
return EngineComponents{}, fmt.Errorf("dictation download: DestRoot is required")
|
||||
}
|
||||
key := opts.platformKey
|
||||
if key == "" {
|
||||
key = platformKey()
|
||||
}
|
||||
suffix, ok := engineAssetSuffix[key]
|
||||
if !ok {
|
||||
return EngineComponents{}, &SetupError{
|
||||
Tool: "sherpa-onnx auto-download",
|
||||
Hint: fmt.Sprintf("no prebuilt engine is known for %s — install sherpa-onnx manually or use a cloud provider (see docs/dictation.md)", key),
|
||||
}
|
||||
}
|
||||
version := strings.TrimSpace(opts.EngineVersion)
|
||||
if version == "" {
|
||||
version = DefaultSherpaVersion
|
||||
}
|
||||
progress := opts.Progress
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
}
|
||||
client := opts.HTTPClient
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
apiBase := opts.APIBase
|
||||
if apiBase == "" {
|
||||
apiBase = defaultAPIBase
|
||||
}
|
||||
|
||||
engineDir := filepath.Join(opts.DestRoot, "engine-"+version+"-"+key)
|
||||
targetWindows := strings.HasPrefix(key, "windows-")
|
||||
// Resolve through the tarball's flattened subdir so an ALREADY-extracted
|
||||
// engine is found and not needlessly re-downloaded (the idempotency check).
|
||||
binPath, serverPath := resolveEnginePaths(engineDir, targetWindows)
|
||||
if !fileExists(binPath) {
|
||||
pinned := ""
|
||||
if version == DefaultSherpaVersion && !opts.skipPinned {
|
||||
pinned = pinnedEngineDigest[key]
|
||||
}
|
||||
asset, err := resolveAsset(ctx, client, apiBase, version, "sherpa-onnx-", suffix)
|
||||
if err != nil {
|
||||
return EngineComponents{}, err
|
||||
}
|
||||
if err := downloadVerifyExtract(ctx, client, asset, pinned, false, "Engine", engineDir, progress); err != nil {
|
||||
return EngineComponents{}, err
|
||||
}
|
||||
binPath, serverPath = resolveEnginePaths(engineDir, targetWindows)
|
||||
}
|
||||
if !fileExists(binPath) {
|
||||
return EngineComponents{}, fmt.Errorf("dictation download: engine binary not found after extraction in %s", engineDir)
|
||||
}
|
||||
|
||||
modelName := opts.ModelAssetName
|
||||
if modelName == "" {
|
||||
modelName = modelAssetName
|
||||
}
|
||||
modelDirName := opts.ModelDirName
|
||||
if modelDirName == "" {
|
||||
modelDirName = "model-moonshine-tiny-en-int8"
|
||||
}
|
||||
modelDir := filepath.Join(opts.DestRoot, modelDirName)
|
||||
if !dirHasModel(modelDir) {
|
||||
asset, err := resolveAsset(ctx, client, apiBase, modelReleaseTag, modelName, "")
|
||||
if err != nil {
|
||||
return EngineComponents{}, err
|
||||
}
|
||||
modelPinned := opts.ModelPinnedDigest
|
||||
// Only fall back to the built-in digest for the DEFAULT model — a different
|
||||
// (browse-listed) model must never be checked against the default's digest.
|
||||
if modelPinned == "" && opts.ModelAssetName == "" {
|
||||
modelPinned = pinnedModelDigest
|
||||
}
|
||||
if opts.skipPinned {
|
||||
modelPinned = ""
|
||||
}
|
||||
modelLabel := opts.ModelLabel
|
||||
if modelLabel == "" {
|
||||
modelLabel = "Model"
|
||||
}
|
||||
// Models are data files from the official release: verify against a digest
|
||||
// when we have one (curated/pinned or API-provided), else allow a TLS-only
|
||||
// download (the model release predates GitHub's per-asset digests). The
|
||||
// engine BINARY above never allows this — a native executable is always
|
||||
// digest-verified.
|
||||
if err := downloadVerifyExtract(ctx, client, asset, modelPinned, true, modelLabel, modelDir, progress); err != nil {
|
||||
return EngineComponents{}, err
|
||||
}
|
||||
}
|
||||
resolvedModel := modelDir
|
||||
if !hasTokensFile(resolvedModel) {
|
||||
if child, err := flattenSingleChild(modelDir); err == nil {
|
||||
resolvedModel = child
|
||||
}
|
||||
}
|
||||
if !hasTokensFile(resolvedModel) {
|
||||
return EngineComponents{}, fmt.Errorf("dictation download: model files not found after extraction in %s", modelDir)
|
||||
}
|
||||
return EngineComponents{BinaryPath: binPath, ServerPath: serverPath, ModelPath: resolvedModel}, nil
|
||||
}
|
||||
|
||||
type resolvedAsset struct {
|
||||
url string
|
||||
sha256 string
|
||||
size int64
|
||||
name string
|
||||
}
|
||||
|
||||
// resolveAsset queries the GitHub release for tag and returns the asset whose
|
||||
// name has the given prefix and suffix, plus its SHA256 digest and size.
|
||||
func resolveAsset(ctx context.Context, client *http.Client, apiBase, tag, namePrefix, nameSuffix string) (resolvedAsset, error) {
|
||||
tagPath := "tags/" + tag
|
||||
if tag == "" || tag == "latest" {
|
||||
tagPath = "latest"
|
||||
}
|
||||
url := fmt.Sprintf("%s/repos/%s/releases/%s", strings.TrimRight(apiBase, "/"), sherpaRepo, tagPath)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return resolvedAsset{}, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return resolvedAsset{}, fmt.Errorf("resolving sherpa-onnx release %s: %w", tag, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return resolvedAsset{}, fmt.Errorf("resolving sherpa-onnx release %s: GitHub API HTTP %d", tag, resp.StatusCode)
|
||||
}
|
||||
var release struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Assets []struct {
|
||||
Name string `json:"name"`
|
||||
DownloadURL string `json:"browser_download_url"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
} `json:"assets"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
|
||||
return resolvedAsset{}, fmt.Errorf("parsing sherpa-onnx release %s: %w", tag, err)
|
||||
}
|
||||
for _, a := range release.Assets {
|
||||
if !strings.HasPrefix(a.Name, namePrefix) || !strings.HasSuffix(a.Name, nameSuffix) {
|
||||
continue
|
||||
}
|
||||
// The API digest may be empty for older assets (GitHub only populates it for
|
||||
// assets uploaded after ~2024 — the ASR models predate that). An empty
|
||||
// digest is fine here; downloadVerifyExtract requires the API digest OR a
|
||||
// pinned one and refuses only when neither exists.
|
||||
digest := strings.TrimPrefix(a.Digest, "sha256:")
|
||||
return resolvedAsset{url: a.DownloadURL, sha256: digest, size: a.Size, name: a.Name}, nil
|
||||
}
|
||||
return resolvedAsset{}, &SetupError{
|
||||
Tool: "sherpa-onnx auto-download",
|
||||
Hint: fmt.Sprintf("release %s has no asset matching %s*%s — try a different stt.engineVersion or install manually", tag, namePrefix, nameSuffix),
|
||||
}
|
||||
}
|
||||
|
||||
// downloadVerifyExtract fetches the asset to a temp file, verifies its SHA256
|
||||
// (against the API digest, and — when pinned is set — a cross-check that the
|
||||
// resolved digest equals the audited value), and extracts the tar.bz2. A
|
||||
// mismatch aborts before anything is extracted or run.
|
||||
func downloadVerifyExtract(ctx context.Context, client *http.Client, asset resolvedAsset, pinned string, allowUnverified bool, label, destDir string, progress func(string)) error {
|
||||
// Determine the digest to verify against: the API digest and the pinned digest
|
||||
// must agree when both exist; at least one must exist unless allowUnverified
|
||||
// (models are data files from the official release — TLS-only is acceptable;
|
||||
// a native executable never sets allowUnverified).
|
||||
expected := asset.sha256
|
||||
if pinned != "" {
|
||||
if expected != "" && !strings.EqualFold(expected, pinned) {
|
||||
return fmt.Errorf("dictation download: %s digest %s does not match the pinned known-good value %s — refusing (the release may have changed)", asset.name, asset.sha256, pinned)
|
||||
}
|
||||
expected = pinned
|
||||
}
|
||||
if expected == "" && !allowUnverified {
|
||||
return fmt.Errorf("dictation download: no SHA256 available for %s (neither the GitHub API nor a pinned value) — refusing an unverifiable download", asset.name)
|
||||
}
|
||||
// Ensure the parent of destDir exists; the final destDir is created by
|
||||
// the atomic rename at the end of the function.
|
||||
if err := os.MkdirAll(filepath.Dir(destDir), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
// Stage the extraction into a sibling temp dir and only swap it into
|
||||
// destDir on success. An interrupted run that left a truncated engine
|
||||
// binary in destDir would otherwise pass fileExists() on the next start
|
||||
// and break the local recorder silently.
|
||||
stageDir, err := os.MkdirTemp(filepath.Dir(destDir), filepath.Base(destDir)+".stage-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cleanupStage := true
|
||||
defer func() {
|
||||
if cleanupStage {
|
||||
_ = os.RemoveAll(stageDir)
|
||||
}
|
||||
}()
|
||||
|
||||
tmp, err := os.CreateTemp(filepath.Dir(destDir), "stt-dl-*.tar.bz2")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, asset.url, nil)
|
||||
if err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("downloading %s: %w", asset.name, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("downloading %s: HTTP %d", asset.name, resp.StatusCode)
|
||||
}
|
||||
hasher := sha256.New()
|
||||
total := asset.size
|
||||
if total <= 0 {
|
||||
total = resp.ContentLength
|
||||
}
|
||||
src := io.Reader(resp.Body)
|
||||
if total > 0 {
|
||||
src = &progressReader{r: resp.Body, total: total, label: label, report: progress, lastPct: -1}
|
||||
}
|
||||
if _, err := io.Copy(io.MultiWriter(tmp, hasher), src); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("downloading %s: %w", asset.name, err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
got := hex.EncodeToString(hasher.Sum(nil))
|
||||
if expected != "" && !strings.EqualFold(got, expected) {
|
||||
return fmt.Errorf("dictation download: checksum mismatch for %s (want %s, got %s) — refusing to extract", asset.name, expected, got)
|
||||
}
|
||||
if err := extractTarBz2(tmpPath, stageDir, "Extracting "+label, progress); err != nil {
|
||||
return err
|
||||
}
|
||||
// Remove any pre-existing destDir so Rename is atomic on the same
|
||||
// filesystem (os.Rename refuses to overwrite a non-empty dir).
|
||||
if err := os.RemoveAll(destDir); err != nil {
|
||||
return fmt.Errorf("clearing previous %s install: %w", label, err)
|
||||
}
|
||||
if err := os.Rename(stageDir, destDir); err != nil {
|
||||
return fmt.Errorf("promoting staged %s: %w", label, err)
|
||||
}
|
||||
cleanupStage = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// progressReader reports download progress as a percentage on each whole-percent
|
||||
// change, formatted "<label> NN% · X/Y MB".
|
||||
type progressReader struct {
|
||||
r io.Reader
|
||||
total int64
|
||||
read int64
|
||||
label string
|
||||
report func(string)
|
||||
lastPct int
|
||||
}
|
||||
|
||||
func (p *progressReader) Read(b []byte) (int, error) {
|
||||
n, err := p.r.Read(b)
|
||||
p.read += int64(n)
|
||||
if p.total > 0 {
|
||||
pct := int(p.read * 100 / p.total)
|
||||
if pct != p.lastPct {
|
||||
p.lastPct = pct
|
||||
p.report(fmt.Sprintf("%s %d%% · %d/%d MB", p.label, pct, p.read>>20, p.total>>20))
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// enginePaths builds the engine executable paths. The ".exe" suffix follows the
|
||||
// TARGET platform (the one the engine was downloaded for), not the host — they
|
||||
// match in production (you download your own platform), but keying off the target
|
||||
// keeps resolution correct when the two differ (e.g. a cross-platform test).
|
||||
func enginePaths(engineDir string, targetWindows bool) (bin, server string) {
|
||||
ext := ""
|
||||
if targetWindows {
|
||||
ext = ".exe"
|
||||
}
|
||||
return filepath.Join(engineDir, "bin", "sherpa-onnx-offline"+ext),
|
||||
filepath.Join(engineDir, "bin", "sherpa-onnx-online-websocket-server"+ext)
|
||||
}
|
||||
|
||||
// resolveEnginePaths finds the executables, flattening the tarball's single
|
||||
// top-level directory when present.
|
||||
func resolveEnginePaths(engineDir string, targetWindows bool) (bin, server string) {
|
||||
bin, server = enginePaths(engineDir, targetWindows)
|
||||
if fileExists(bin) {
|
||||
return bin, server
|
||||
}
|
||||
if child, err := flattenSingleChild(engineDir); err == nil && child != engineDir {
|
||||
return enginePaths(child, targetWindows)
|
||||
}
|
||||
return bin, server
|
||||
}
|
||||
|
||||
// extractTarBz2 unpacks a bzip2-compressed tar into destDir, guarding against
|
||||
// path-traversal entries. It reports extraction progress by how much of the
|
||||
// (compressed) archive it has consumed — the same MB scale as the download, so
|
||||
// "Extracting … 50% · 520/1041 MB" lines up with the download that preceded it.
|
||||
func extractTarBz2(archivePath, destDir, label string, progress func(string)) error {
|
||||
f, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
var src io.Reader = f
|
||||
if progress != nil {
|
||||
if info, statErr := f.Stat(); statErr == nil && info.Size() > 0 {
|
||||
src = &progressReader{r: f, total: info.Size(), label: label, report: progress, lastPct: -1}
|
||||
}
|
||||
}
|
||||
tr := tar.NewReader(bzip2.NewReader(src))
|
||||
destAbs, err := filepath.Abs(destDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target := filepath.Join(destDir, hdr.Name)
|
||||
absTarget, err := filepath.Abs(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if absTarget != destAbs && !strings.HasPrefix(absTarget, destAbs+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("dictation download: archive entry %q escapes destination", hdr.Name)
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(hdr.Mode)&0o777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, tr); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// flattenSingleChild returns the sole subdirectory of dir (the tarball's
|
||||
// top-level folder) when dir contains exactly one directory entry.
|
||||
func flattenSingleChild(dir string) (string, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var dirs []string
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
dirs = append(dirs, e.Name())
|
||||
}
|
||||
}
|
||||
if len(dirs) == 1 {
|
||||
return filepath.Join(dir, dirs[0]), nil
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// ModelDownloaded reports whether a model variant is already extracted under
|
||||
// destRoot (so the picker can mark it as installed). dirName is the variant's
|
||||
// DirName.
|
||||
func ModelDownloaded(destRoot, dirName string) bool {
|
||||
if destRoot == "" || dirName == "" {
|
||||
return false
|
||||
}
|
||||
return dirHasModel(filepath.Join(destRoot, dirName))
|
||||
}
|
||||
|
||||
// EngineDownloaded reports whether the shared sherpa-onnx engine is already on
|
||||
// disk for this platform (one engine serves every model), so the picker can drop
|
||||
// the engine's size from a model's download total once it's been fetched.
|
||||
func EngineDownloaded(destRoot, version string) bool {
|
||||
if destRoot == "" {
|
||||
return false
|
||||
}
|
||||
if version == "" {
|
||||
version = DefaultSherpaVersion
|
||||
}
|
||||
engineDir := filepath.Join(destRoot, "engine-"+version+"-"+platformKey())
|
||||
bin, _ := resolveEnginePaths(engineDir, runtime.GOOS == "windows")
|
||||
return fileExists(bin)
|
||||
}
|
||||
|
||||
func dirHasModel(dir string) bool {
|
||||
if hasTokensFile(dir) {
|
||||
return true
|
||||
}
|
||||
if child, err := flattenSingleChild(dir); err == nil && child != dir {
|
||||
return hasTokensFile(child)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hasTokensFile reports whether dir holds a sherpa tokens file — usually
|
||||
// "tokens.txt", but some families (Whisper) prefix it as "<model>-tokens.txt".
|
||||
func hasTokensFile(dir string) bool {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.HasSuffix(e.Name(), "tokens.txt") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fileExists(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && !info.IsDir()
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestEnsureLocalEngineRealDownload exercises the full auto-download against the
|
||||
// REAL k2-fsa/sherpa-onnx GitHub API (resolve → verify digest → extract → run).
|
||||
// It downloads ~130MB, so it is opt-in via ZERO_STT_DOWNLOAD_TEST=1.
|
||||
//
|
||||
// ZERO_STT_DOWNLOAD_TEST=1 go test ./internal/dictation/ -run RealDownload -v -timeout 10m
|
||||
func TestEnsureLocalEngineRealDownload(t *testing.T) {
|
||||
if os.Getenv("ZERO_STT_DOWNLOAD_TEST") == "" {
|
||||
t.Skip("set ZERO_STT_DOWNLOAD_TEST=1 to run the real ~130MB download smoke test")
|
||||
}
|
||||
dest := t.TempDir()
|
||||
comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: dest,
|
||||
Progress: func(s string) { t.Logf("stage: %s", s) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureLocalEngine: %v", err)
|
||||
}
|
||||
if !fileExists(comp.BinaryPath) || !fileExists(filepath.Join(comp.ModelPath, "tokens.txt")) {
|
||||
t.Fatalf("missing components: %+v", comp)
|
||||
}
|
||||
|
||||
// The downloaded engine + model must actually transcribe.
|
||||
wav := filepath.Join(comp.ModelPath, "test_wavs", "0.wav")
|
||||
if !fileExists(wav) {
|
||||
t.Skipf("no bundled test wav at %s", wav)
|
||||
}
|
||||
audio, err := os.ReadFile(wav)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tr, err := NewLocalTranscriber(LocalConfig{Binary: comp.BinaryPath, ModelPath: comp.ModelPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text, err := tr.Transcribe(context.Background(), audio)
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe with downloaded engine: %v", err)
|
||||
}
|
||||
if text == "" {
|
||||
t.Fatal("downloaded engine returned empty transcript")
|
||||
}
|
||||
t.Logf("downloaded-engine transcript: %q", text)
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tiny tar.bz2 fixtures (generated in-repo): the engine has top/bin/sherpa-onnx-
|
||||
// offline + server; the model has top/tokens.txt.
|
||||
const (
|
||||
engineTarB64 = "QlpoOTFBWSZTWW+fEfAAAW1/hMGRAIBAA/+AAgAgRH9v3+AAAQCoMAEYAYA00NGjEZANABoYYA00NGjEZANABoYIpE0TRoTGkA2poGgxqepl1H6w5anW3tY2OggUi1AW/d29pO0GLUoOtnQsrGFNowIEh0cImtQ9gyHqYH7unm+/wZSLXsBuBDDCfHiahCsGEgGL9/pUG+RQZo76xlaDIGKWDErPKuyhXYOlzJMG3iNa56kiluctzUqgZ2A1AGwgcB3jtB5PB7pYYlDHH8OvOznM6B6MHPRqa7sTE1vwDBc0LjmG4ah5F+YYaXHTZpGmcemWW1yMh1adhdrB6A3nAQP8XckU4UJBvnxHwA=="
|
||||
engineSHA = "97b5cb6a417705860f8940e1bf9ffec8cc60e8666a0d228cd60f1de7607d503d"
|
||||
modelTarB64 = "QlpoOTFBWSZTWTtAoX4AANp/hMOQBIBAAf+AAAIQBHoJ3mAAAQAIMAC5sG1SNMhNqGmQwIyeKEagam1DQ0AAABFRT0T0mg0aeoDQHpH7Nzrs24wsbVW8hMQHClNI0LuLhKMGY4sEQi6N2qQWP1id8o1KtprGd4hll4uQkMGA1nFMXAwohEZOnIMIPQKQl2k0GbSGwkC0lJb2IWOQ57ETYbmxFGspaRp1QyT6q4sKj855st7GzOGj5eIwmWt6wRajEQl/F3JFOFCQO0Chfg=="
|
||||
modelSHA = "16c6b1dcb40f6b21c0de70bc1a8edab95375e4de90dac7c076fd2cc7a34fc59a"
|
||||
)
|
||||
|
||||
// fakeReleaseServer serves the GitHub release API + asset bytes for the engine
|
||||
// and model, exercising resolveAsset + downloadVerifyExtract against a real HTTP
|
||||
// round-trip (with the real coder/websocket-free download path).
|
||||
func fakeReleaseServer(t *testing.T, engineDigest, modelDigest string) *httptest.Server {
|
||||
t.Helper()
|
||||
engineBytes, _ := base64.StdEncoding.DecodeString(engineTarB64)
|
||||
modelBytes, _ := base64.StdEncoding.DecodeString(modelTarB64)
|
||||
mux := http.NewServeMux()
|
||||
base := ""
|
||||
release := func(assetName, digest, dlPath string) string {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"tag_name": "test",
|
||||
"assets": []map[string]any{{
|
||||
"name": assetName,
|
||||
"browser_download_url": base + dlPath,
|
||||
"digest": "sha256:" + digest,
|
||||
"size": 123,
|
||||
}},
|
||||
})
|
||||
return string(body)
|
||||
}
|
||||
var srv *httptest.Server
|
||||
// Any release tag under this prefix resolves: asr-models → the model, every
|
||||
// other tag → the engine (so the default-version pinned-check test works too).
|
||||
mux.HandleFunc("/repos/k2-fsa/sherpa-onnx/releases/tags/", func(w http.ResponseWriter, r *http.Request) {
|
||||
base = srv.URL
|
||||
if strings.HasSuffix(r.URL.Path, "/asr-models") {
|
||||
fmt.Fprint(w, release(modelAssetName, modelDigest, "/model.tar.bz2"))
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, release("sherpa-onnx-test-linux-x64-shared-no-tts.tar.bz2", engineDigest, "/engine.tar.bz2"))
|
||||
})
|
||||
mux.HandleFunc("/engine.tar.bz2", func(w http.ResponseWriter, r *http.Request) { w.Write(engineBytes) })
|
||||
mux.HandleFunc("/model.tar.bz2", func(w http.ResponseWriter, r *http.Request) { w.Write(modelBytes) })
|
||||
srv = httptest.NewServer(mux)
|
||||
base = srv.URL
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestResolveEnginePathsHonorsTargetPlatform(t *testing.T) {
|
||||
// The engine binary name follows the TARGET platform, not the host: a Windows
|
||||
// engine has ".exe", a Linux one doesn't. Resolution must find each regardless
|
||||
// of the OS the test runs on (this is what makes the cross-platform download
|
||||
// test pass on Windows CI).
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
target bool
|
||||
binName string
|
||||
}{
|
||||
{"linux target", false, "sherpa-onnx-offline"},
|
||||
{"windows target", true, "sherpa-onnx-offline.exe"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binDir := filepath.Join(dir, "bin")
|
||||
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(binDir, tc.binName), []byte("x"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bin, _ := resolveEnginePaths(dir, tc.target)
|
||||
if !fileExists(bin) {
|
||||
t.Errorf("resolveEnginePaths(target-windows=%v) = %q; not found", tc.target, bin)
|
||||
}
|
||||
if !strings.HasSuffix(bin, tc.binName) {
|
||||
t.Errorf("bin = %q, want suffix %q", bin, tc.binName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLocalEngineDownloadsAndExtracts(t *testing.T) {
|
||||
srv := fakeReleaseServer(t, engineSHA, modelSHA)
|
||||
dest := t.TempDir()
|
||||
var stages []string
|
||||
comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: dest,
|
||||
EngineVersion: "test",
|
||||
APIBase: srv.URL,
|
||||
platformKey: "linux-amd64",
|
||||
skipPinned: true,
|
||||
Progress: func(s string) { stages = append(stages, s) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureLocalEngine: %v", err)
|
||||
}
|
||||
if !fileExists(comp.BinaryPath) || !strings.HasSuffix(comp.BinaryPath, "sherpa-onnx-offline") {
|
||||
t.Errorf("binary path wrong: %q", comp.BinaryPath)
|
||||
}
|
||||
if !fileExists(comp.ServerPath) {
|
||||
t.Errorf("server path missing: %q", comp.ServerPath)
|
||||
}
|
||||
if !fileExists(filepath.Join(comp.ModelPath, "tokens.txt")) {
|
||||
t.Errorf("model tokens.txt missing under %q", comp.ModelPath)
|
||||
}
|
||||
// The extracted binary keeps its exec bit — Unix only; Windows has no
|
||||
// executable permission bit, so os.FileMode never reports 0o100 there.
|
||||
if runtime.GOOS != "windows" {
|
||||
if info, err := os.Stat(comp.BinaryPath); err == nil && info.Mode()&0o100 == 0 {
|
||||
t.Error("engine binary should be executable")
|
||||
}
|
||||
}
|
||||
if len(stages) == 0 {
|
||||
t.Error("expected progress stages")
|
||||
}
|
||||
|
||||
// Idempotent: a second call reuses the extracted engine + model and downloads
|
||||
// NOTHING (no progress stages) — the fix for re-downloading an already-present
|
||||
// engine whose binary lives in the tarball's flattened subdir.
|
||||
var stages2 []string
|
||||
comp2, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: dest, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true,
|
||||
Progress: func(s string) { stages2 = append(stages2, s) },
|
||||
})
|
||||
if err != nil || comp2.BinaryPath != comp.BinaryPath {
|
||||
t.Errorf("second call should reuse: %v / %q vs %q", err, comp2.BinaryPath, comp.BinaryPath)
|
||||
}
|
||||
if len(stages2) != 0 {
|
||||
t.Errorf("a fully-cached second call must not download/extract anything, got stages: %v", stages2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLocalEngineRejectsChecksumMismatch(t *testing.T) {
|
||||
// Serve the engine with a wrong digest → verification must refuse to extract.
|
||||
srv := fakeReleaseServer(t, strings.Repeat("0", 64), modelSHA)
|
||||
_, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: t.TempDir(), EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "checksum mismatch") {
|
||||
t.Fatalf("expected checksum-mismatch error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLocalEnginePinnedCrossCheckRefusesChangedRelease(t *testing.T) {
|
||||
// Default version + a digest that doesn't match the pinned value → refuse,
|
||||
// even though the download itself would verify against the API digest.
|
||||
srv := fakeReleaseServer(t, engineSHA, modelSHA)
|
||||
_, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: t.TempDir(),
|
||||
EngineVersion: DefaultSherpaVersion, // triggers the pinned cross-check
|
||||
APIBase: srv.URL,
|
||||
platformKey: "linux-amd64",
|
||||
// skipPinned false: the pinned digest for linux-amd64 != engineSHA fixture.
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "pinned known-good") {
|
||||
t.Fatalf("expected pinned cross-check refusal, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAssetMissingDigestAllowed(t *testing.T) {
|
||||
// An older asset with no API digest resolves fine (verification against a
|
||||
// pinned value happens later); resolveAsset no longer refuses here.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]any{
|
||||
"assets": []map[string]any{{"name": "x-linux-x64-shared-no-tts.tar.bz2", "browser_download_url": "http://x", "digest": ""}},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
asset, err := resolveAsset(context.Background(), http.DefaultClient, srv.URL, "test", "x-", "linux-x64-shared-no-tts.tar.bz2")
|
||||
if err != nil {
|
||||
t.Fatalf("resolveAsset should tolerate a missing digest, got %v", err)
|
||||
}
|
||||
if asset.sha256 != "" {
|
||||
t.Errorf("expected empty digest, got %q", asset.sha256)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnverifiableDownloadRefused(t *testing.T) {
|
||||
// No API digest AND no pinned digest → refuse to download/extract.
|
||||
srv := fakeReleaseServer(t, "", "") // both assets served with empty digest
|
||||
_, err := EnsureLocalEngine(context.Background(), DownloadOptions{
|
||||
DestRoot: t.TempDir(), EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "unverifiable") {
|
||||
t.Fatalf("expected refusal of an unverifiable download, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressReaderReportsPercent(t *testing.T) {
|
||||
data := make([]byte, 1000)
|
||||
var last string
|
||||
pr := &progressReader{r: bytesReader(data), total: 1000, label: "Engine", report: func(s string) { last = s }, lastPct: -1}
|
||||
buf := make([]byte, 100)
|
||||
for {
|
||||
if _, err := pr.Read(buf); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !strings.Contains(last, "Engine") || !strings.Contains(last, "100%") {
|
||||
t.Errorf("expected a final 'Engine 100%%' report, got %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
func bytesReader(b []byte) *sliceReader { return &sliceReader{b: b} }
|
||||
|
||||
type sliceReader struct {
|
||||
b []byte
|
||||
pos int
|
||||
}
|
||||
|
||||
func (s *sliceReader) Read(p []byte) (int, error) {
|
||||
if s.pos >= len(s.b) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, s.b[s.pos:])
|
||||
s.pos += n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func TestUnsupportedPlatformSetupError(t *testing.T) {
|
||||
_, err := EnsureLocalEngine(context.Background(), DownloadOptions{DestRoot: t.TempDir(), platformKey: "plan9-mips"})
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for unsupported platform, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestListModelsFiltersAndFlags(t *testing.T) {
|
||||
assets := []map[string]any{
|
||||
{"name": "sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2", "size": 1 << 20, "digest": "sha256:abc"},
|
||||
{"name": "sherpa-onnx-whisper-tiny.en.tar.bz2", "size": 2 << 20},
|
||||
{"name": "sherpa-onnx-moonshine-tiny-en-int8.tar.bz2", "size": 3 << 20}, // curated but not flagged recommended
|
||||
{"name": "sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06.tar.bz2", "size": 1 << 20}, // curated → recommended
|
||||
{"name": "sherpa-onnx-paraformer-zh-2024-03-09.tar.bz2", "size": 4 << 20},
|
||||
// Non-ASR tools that MUST be filtered out:
|
||||
{"name": "sherpa-onnx-vad-silero.tar.bz2", "size": 1},
|
||||
{"name": "vits-piper-en_US-amy.tar.bz2", "size": 1},
|
||||
{"name": "sherpa-onnx-punct-en.tar.bz2", "size": 1},
|
||||
{"name": "sherpa-onnx-kws-zipformer.tar.bz2", "size": 1},
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]any{"id": 1, "assets": assets})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
vs, err := ListModels(context.Background(), http.DefaultClient, srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(vs) != 5 {
|
||||
t.Fatalf("expected 5 transcription models (tools filtered), got %d", len(vs))
|
||||
}
|
||||
byID := map[string]ModelVariant{}
|
||||
for _, v := range vs {
|
||||
byID[v.ID] = v
|
||||
}
|
||||
if !byID["streaming-zipformer-en-2023-06-26"].Streaming {
|
||||
t.Error("streaming model should be flagged Streaming")
|
||||
}
|
||||
if byID["whisper-tiny.en"].Streaming {
|
||||
t.Error("whisper is batch, not streaming")
|
||||
}
|
||||
if !byID["streaming-zipformer-en-kroko-2025-08-06"].Recommended {
|
||||
t.Error("a curated model flagged Recommended should be Recommended")
|
||||
}
|
||||
if byID["moonshine-tiny-en-int8"].Recommended {
|
||||
t.Error("moonshine tiny is curated but no longer flagged Recommended (no star)")
|
||||
}
|
||||
if byID["paraformer-zh-2024-03-09"].Recommended {
|
||||
t.Error("an uncurated model should not be Recommended")
|
||||
}
|
||||
// A curated model with no API digest still gets its pinned digest.
|
||||
if byID["moonshine-tiny-en-int8"].Digest == "" {
|
||||
t.Error("curated model should carry a pinned digest")
|
||||
}
|
||||
// An uncurated model with no API digest has an empty digest (TLS-only download).
|
||||
if byID["whisper-tiny.en"].Digest != "" {
|
||||
t.Error("uncurated model without an API digest should have an empty digest")
|
||||
}
|
||||
// The API-provided digest is used when present.
|
||||
if byID["streaming-zipformer-en-2023-06-26"].Digest != "abc" {
|
||||
t.Errorf("expected the API digest, got %q", byID["streaming-zipformer-en-2023-06-26"].Digest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestLocalTranscribeRealBinary exercises the full local batch path against a
|
||||
// REAL sherpa-onnx-offline binary and model — the manual smoke test §14 calls
|
||||
// out as not CI-able. Skipped unless both env vars point at a real install:
|
||||
//
|
||||
// ZERO_STT_TEST_BINARY=/path/to/sherpa-onnx-offline \
|
||||
// ZERO_STT_TEST_MODEL=/path/to/model-dir \
|
||||
// ZERO_STT_TEST_WAV=/path/to/16k-mono.wav \
|
||||
// go test ./internal/dictation/ -run TestLocalTranscribeRealBinary -v
|
||||
func TestLocalTranscribeRealBinary(t *testing.T) {
|
||||
binary := os.Getenv("ZERO_STT_TEST_BINARY")
|
||||
model := os.Getenv("ZERO_STT_TEST_MODEL")
|
||||
wav := os.Getenv("ZERO_STT_TEST_WAV")
|
||||
if binary == "" || model == "" || wav == "" {
|
||||
t.Skip("set ZERO_STT_TEST_BINARY, ZERO_STT_TEST_MODEL, ZERO_STT_TEST_WAV to run the real-engine smoke test")
|
||||
}
|
||||
audio, err := os.ReadFile(wav)
|
||||
if err != nil {
|
||||
t.Fatalf("read wav: %v", err)
|
||||
}
|
||||
tr, err := NewLocalTranscriber(LocalConfig{Binary: binary, ModelPath: model})
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalTranscriber: %v", err)
|
||||
}
|
||||
text, err := tr.Transcribe(context.Background(), audio)
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if text == "" {
|
||||
t.Fatal("real engine returned an empty transcript")
|
||||
}
|
||||
t.Logf("transcript: %q", text)
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
//go:build !windows
|
||||
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// checkAudioDirOwner rejects a dictation temp directory not owned by the
|
||||
// current user — on a shared /tmp another user could have pre-created the
|
||||
// path and would then control its lifetime. Mirrors
|
||||
// internal/tools/spill_owner_unix.go.
|
||||
func checkAudioDirOwner(info os.FileInfo) error {
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if int(stat.Uid) != os.Geteuid() {
|
||||
return fmt.Errorf("dictation temp directory is owned by uid %d, not the current user", stat.Uid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
//go:build windows
|
||||
|
||||
package dictation
|
||||
|
||||
import "os"
|
||||
|
||||
// checkAudioDirOwner is a no-op on Windows: %TEMP% is per-user by default and
|
||||
// there is no portable uid to compare. The Lstat symlink/dir check in
|
||||
// audioSpillDir still applies. Mirrors internal/tools/spill_owner_windows.go.
|
||||
func checkAudioDirOwner(os.FileInfo) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,581 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Recorder captures microphone audio. Start/Stop is the batch path (used on
|
||||
// Termux, and as the fallback everywhere); StartStreaming is the desktop-only
|
||||
// continuous-PCM path.
|
||||
type Recorder interface {
|
||||
Start() error
|
||||
// Stop ends a batch recording and returns the recorded audio bytes (WAV on
|
||||
// desktop, M4A on Termux — SniffFormat distinguishes them).
|
||||
Stop() ([]byte, error)
|
||||
// StartStreaming begins continuous capture of raw little-endian PCM16 mono
|
||||
// at the configured sample rate. Chunks arrive on the channel in ~50ms
|
||||
// units; the channel closes when capture ends (stop called, max duration
|
||||
// hit, or trailing-silence auto-stop). stop is idempotent.
|
||||
StartStreaming() (chunks <-chan []byte, stop func() error, err error)
|
||||
}
|
||||
|
||||
// Platform selects the audio-capture toolchain (§9 of the design doc).
|
||||
type Platform string
|
||||
|
||||
const (
|
||||
PlatformLinux Platform = "linux"
|
||||
PlatformDarwin Platform = "darwin"
|
||||
PlatformWindows Platform = "windows"
|
||||
PlatformTermux Platform = "termux"
|
||||
PlatformUnsupported Platform = ""
|
||||
)
|
||||
|
||||
// DetectPlatform picks the capture toolchain for the current host. Termux is
|
||||
// detected by its environment (it reports GOOS=linux/android): TERMUX_VERSION
|
||||
// is exported to every Termux shell, and PREFIX carries the app package path.
|
||||
func DetectPlatform() Platform {
|
||||
if os.Getenv("TERMUX_VERSION") != "" || strings.Contains(os.Getenv("PREFIX"), "com.termux") {
|
||||
return PlatformTermux
|
||||
}
|
||||
switch runtime.GOOS {
|
||||
case "linux":
|
||||
return PlatformLinux
|
||||
case "darwin":
|
||||
return PlatformDarwin
|
||||
case "windows":
|
||||
return PlatformWindows
|
||||
}
|
||||
return PlatformUnsupported
|
||||
}
|
||||
|
||||
// StreamingSupported reports whether the platform's capture tool can write raw
|
||||
// PCM to stdout. Termux's termux-microphone-record requires a regular output
|
||||
// file — a hard tool limitation, so Termux is batch-only permanently.
|
||||
func (p Platform) StreamingSupported() bool {
|
||||
switch p {
|
||||
case PlatformLinux, PlatformDarwin, PlatformWindows:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const (
|
||||
// DefaultSampleRate is what the local engine and Deepgram consume.
|
||||
// OpenAI Realtime's pcm16 requires 24kHz; its transcriber constructs the
|
||||
// recorder accordingly.
|
||||
DefaultSampleRate = 16000
|
||||
// DefaultMaxDuration is the hard runaway-recording cap (§12), applied on
|
||||
// every platform, batch and streaming. Overridable via
|
||||
// stt.maxDurationSeconds.
|
||||
DefaultMaxDuration = 5 * time.Minute
|
||||
|
||||
// Trailing-silence auto-stop (§9): stop ~2s after the signal falls below a
|
||||
// 3% amplitude threshold — SoX's own `silence 1 0.1 3% 1 2.0 3%` numbers.
|
||||
silenceThresholdPct = 3
|
||||
silenceHold = 2 * time.Second
|
||||
|
||||
chunkInterval = 50 * time.Millisecond
|
||||
stopGrace = 3 * time.Second
|
||||
)
|
||||
|
||||
// RecorderOptions configures a Recorder. Zero values pick sensible defaults.
|
||||
type RecorderOptions struct {
|
||||
Platform Platform
|
||||
SampleRate int
|
||||
MaxDuration time.Duration
|
||||
SilenceAutoStop bool
|
||||
// WindowsAudioDevice is the dshow capture device name; auto-detected from
|
||||
// `ffmpeg -list_devices` when empty.
|
||||
WindowsAudioDevice string
|
||||
|
||||
// Test seams (unexported): fake the OS boundary, mirroring how
|
||||
// internal/lsp/manager_test.go stubs serverStarter.
|
||||
starter processStarter
|
||||
runOutput commandOutputRunner
|
||||
tempDir func() (string, error)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (o RecorderOptions) withDefaults() RecorderOptions {
|
||||
if o.Platform == PlatformUnsupported {
|
||||
o.Platform = DetectPlatform()
|
||||
}
|
||||
if o.SampleRate <= 0 {
|
||||
o.SampleRate = DefaultSampleRate
|
||||
}
|
||||
if o.MaxDuration <= 0 {
|
||||
o.MaxDuration = DefaultMaxDuration
|
||||
}
|
||||
if o.starter == nil {
|
||||
o.starter = startProcess
|
||||
}
|
||||
if o.runOutput == nil {
|
||||
o.runOutput = runCommandOutput
|
||||
}
|
||||
if o.tempDir == nil {
|
||||
o.tempDir = audioSpillDir
|
||||
}
|
||||
if o.now == nil {
|
||||
o.now = time.Now
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// NewRecorder builds the platform recorder. Construction is cheap and
|
||||
// side-effect free; tools are only exec'd on Start/StartStreaming.
|
||||
func NewRecorder(opts RecorderOptions) Recorder {
|
||||
o := opts.withDefaults()
|
||||
return &recorder{opts: o}
|
||||
}
|
||||
|
||||
type recorder struct {
|
||||
opts RecorderOptions
|
||||
|
||||
mu sync.Mutex
|
||||
proc processHandle
|
||||
filePath string
|
||||
recording bool
|
||||
}
|
||||
|
||||
var errAlreadyRecording = errors.New("a recording is already in progress")
|
||||
|
||||
// Start begins a batch (record-to-file) capture.
|
||||
func (r *recorder) Start() error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.recording {
|
||||
return errAlreadyRecording
|
||||
}
|
||||
if r.opts.Platform == PlatformUnsupported {
|
||||
return fmt.Errorf("dictation is not supported on %s", runtime.GOOS)
|
||||
}
|
||||
|
||||
dir, err := r.opts.tempDir()
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing dictation temp dir: %w", err)
|
||||
}
|
||||
ext := ".wav"
|
||||
if r.opts.Platform == PlatformTermux {
|
||||
ext = ".m4a"
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, "rec-*"+ext)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating dictation temp file: %w", err)
|
||||
}
|
||||
path := tmp.Name()
|
||||
_ = tmp.Close()
|
||||
|
||||
if r.opts.Platform == PlatformTermux {
|
||||
// termux-microphone-record starts the recording in the Termux:API
|
||||
// companion app and exits immediately; there is no long-lived process
|
||||
// to hold. Stop is a separate `-q` invocation.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := r.opts.runOutput(ctx, "termux-microphone-record",
|
||||
"-l", strconv.Itoa(maxSeconds(r.opts.MaxDuration)), "-f", path)
|
||||
if err != nil {
|
||||
_ = os.Remove(path)
|
||||
return startError("termux-microphone-record", r.opts.Platform, err, out)
|
||||
}
|
||||
r.filePath = path
|
||||
r.recording = true
|
||||
return nil
|
||||
}
|
||||
|
||||
spec, err := batchCommand(r.opts, path)
|
||||
if err != nil {
|
||||
_ = os.Remove(path)
|
||||
return err
|
||||
}
|
||||
proc, _, err := r.opts.starter(spec)
|
||||
if err != nil {
|
||||
_ = os.Remove(path)
|
||||
return startError(spec.name, r.opts.Platform, err, nil)
|
||||
}
|
||||
r.proc = proc
|
||||
r.filePath = path
|
||||
r.recording = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop ends a batch capture, reads the recorded file, deletes it, and returns
|
||||
// the audio bytes.
|
||||
func (r *recorder) Stop() ([]byte, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if !r.recording {
|
||||
return nil, errors.New("no recording in progress")
|
||||
}
|
||||
proc, path := r.proc, r.filePath
|
||||
// Termux: Start's termux-microphone-record branch leaves r.proc nil because
|
||||
// there's no long-lived process to hold (Stop is a separate `-q` invocation
|
||||
// via runOutput). Treat that as legitimate; only reject a nil proc that
|
||||
// came from StartStreaming (which owns its process via the returned
|
||||
// closure).
|
||||
if proc == nil && r.opts.Platform != PlatformTermux {
|
||||
return nil, errors.New("recording was started with StartStreaming; call the stop closure returned from StartStreaming instead of Stop()")
|
||||
}
|
||||
r.proc, r.filePath, r.recording = nil, "", false
|
||||
defer os.Remove(path)
|
||||
|
||||
if r.opts.Platform == PlatformTermux {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if out, err := r.opts.runOutput(ctx, "termux-microphone-record", "-q"); err != nil {
|
||||
return nil, fmt.Errorf("stopping termux recording: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return readRecordingWithRetry(path, r.opts.now)
|
||||
}
|
||||
|
||||
if err := proc.StopGracefully(); err != nil {
|
||||
_ = proc.Kill()
|
||||
}
|
||||
// A signal-terminated capture tool reports a non-zero exit — expected, not
|
||||
// an error. The recorded file's contents are the real success signal.
|
||||
_ = waitWithTimeout(proc, stopGrace)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading recorded audio: %w", err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, errors.New("no audio was captured (is the microphone available?)")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// readRecordingWithRetry reads the Termux recording, retrying briefly: the
|
||||
// companion app finalizes the file asynchronously after `-q` returns.
|
||||
func readRecordingWithRetry(path string, now func() time.Time) ([]byte, error) {
|
||||
deadline := now().Add(2 * time.Second)
|
||||
for {
|
||||
data, err := os.ReadFile(path)
|
||||
if err == nil && len(data) > 0 {
|
||||
return data, nil
|
||||
}
|
||||
if now().After(deadline) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading recorded audio: %w", err)
|
||||
}
|
||||
return nil, errors.New("no audio was captured (check Termux:API app and microphone permission)")
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// batchCommand builds the record-to-file argv for desktop platforms (§9).
|
||||
// The max-duration cap is enforced by the tool itself so a runaway recording
|
||||
// ends even if Zero never calls Stop.
|
||||
func batchCommand(opts RecorderOptions, path string) (commandSpec, error) {
|
||||
rate := strconv.Itoa(opts.SampleRate)
|
||||
maxSec := strconv.Itoa(maxSeconds(opts.MaxDuration))
|
||||
switch opts.Platform {
|
||||
case PlatformLinux:
|
||||
return commandSpec{
|
||||
name: "arecord",
|
||||
args: []string{"-q", "-f", "S16_LE", "-r", rate, "-c", "1", "-t", "wav", "-d", maxSec, path},
|
||||
}, nil
|
||||
case PlatformDarwin:
|
||||
args := []string{"-q", "-d", "-r", rate, "-c", "1", "-b", "16", "-e", "signed-integer", path, "trim", "0", maxSec}
|
||||
if opts.SilenceAutoStop {
|
||||
// SoX's built-in trailing-silence auto-stop: end ~2s after the
|
||||
// signal falls below 3% amplitude. A backstop for a forgotten
|
||||
// manual stop, not VAD — recording still starts on an explicit key.
|
||||
args = append(args, "silence", "1", "0.1", "3%", "1", "2.0", "3%")
|
||||
}
|
||||
return commandSpec{name: "sox", args: args}, nil
|
||||
case PlatformWindows:
|
||||
device := opts.WindowsAudioDevice
|
||||
if device == "" {
|
||||
detected, err := detectWindowsAudioDevice(opts.runOutput)
|
||||
if err != nil {
|
||||
return commandSpec{}, err
|
||||
}
|
||||
device = detected
|
||||
}
|
||||
return commandSpec{
|
||||
name: "ffmpeg",
|
||||
args: []string{"-hide_banner", "-loglevel", "error", "-f", "dshow", "-i", "audio=" + device,
|
||||
"-ac", "1", "-ar", rate, "-c:a", "pcm_s16le", "-t", maxSec, "-y", path},
|
||||
stopViaStdin: "q",
|
||||
}, nil
|
||||
}
|
||||
return commandSpec{}, fmt.Errorf("no batch capture tool for platform %q", opts.Platform)
|
||||
}
|
||||
|
||||
// streamCommand builds the raw-PCM-to-stdout argv for desktop platforms (§9).
|
||||
// Same tools as batch, invoked with their stdout flag — Termux has no stdout
|
||||
// mode, which is why it is batch-only.
|
||||
func streamCommand(opts RecorderOptions) (commandSpec, error) {
|
||||
rate := strconv.Itoa(opts.SampleRate)
|
||||
switch opts.Platform {
|
||||
case PlatformLinux:
|
||||
return commandSpec{
|
||||
name: "arecord",
|
||||
args: []string{"-q", "-t", "raw", "-f", "S16_LE", "-r", rate, "-c", "1", "-"},
|
||||
wantStdout: true,
|
||||
}, nil
|
||||
case PlatformDarwin:
|
||||
return commandSpec{
|
||||
name: "sox",
|
||||
args: []string{"-q", "-d", "-t", "raw", "-r", rate, "-e", "signed", "-b", "16", "-c", "1", "-"},
|
||||
wantStdout: true,
|
||||
}, nil
|
||||
case PlatformWindows:
|
||||
device := opts.WindowsAudioDevice
|
||||
if device == "" {
|
||||
detected, err := detectWindowsAudioDevice(opts.runOutput)
|
||||
if err != nil {
|
||||
return commandSpec{}, err
|
||||
}
|
||||
device = detected
|
||||
}
|
||||
return commandSpec{
|
||||
name: "ffmpeg",
|
||||
args: []string{"-hide_banner", "-loglevel", "error", "-f", "dshow", "-i", "audio=" + device,
|
||||
"-ac", "1", "-ar", rate, "-f", "s16le", "-"},
|
||||
wantStdout: true,
|
||||
stopViaStdin: "q",
|
||||
}, nil
|
||||
}
|
||||
return commandSpec{}, fmt.Errorf("streaming capture is not supported on platform %q", opts.Platform)
|
||||
}
|
||||
|
||||
// StartStreaming begins continuous PCM capture with the §12 safety nets: a
|
||||
// hard max-duration cap and Go-side trailing-silence auto-stop (the streaming
|
||||
// analog of SoX's silence effect — we see the samples, so we measure them).
|
||||
func (r *recorder) StartStreaming() (<-chan []byte, func() error, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.recording {
|
||||
return nil, nil, errAlreadyRecording
|
||||
}
|
||||
spec, err := streamCommand(r.opts)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
proc, stdout, err := r.opts.starter(spec)
|
||||
if err != nil {
|
||||
return nil, nil, startError(spec.name, r.opts.Platform, err, nil)
|
||||
}
|
||||
if stdout == nil {
|
||||
_ = proc.Kill()
|
||||
return nil, nil, errors.New("capture process has no stdout pipe")
|
||||
}
|
||||
r.recording = true
|
||||
|
||||
chunkBytes := chunkSizeBytes(r.opts.SampleRate)
|
||||
chunks := make(chan []byte, 8)
|
||||
done := make(chan struct{})
|
||||
var stopOnce sync.Once
|
||||
stop := func() error {
|
||||
stopOnce.Do(func() {
|
||||
close(done)
|
||||
if err := proc.StopGracefully(); err != nil {
|
||||
_ = proc.Kill()
|
||||
}
|
||||
_ = waitWithTimeout(proc, stopGrace)
|
||||
_ = stdout.Close()
|
||||
r.mu.Lock()
|
||||
r.recording = false
|
||||
r.mu.Unlock()
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Max-duration cap: streaming has no tool-side -d/-t equivalent that also
|
||||
// flushes stdout cleanly everywhere, so the cap runs on our side of the pipe.
|
||||
capTimer := time.AfterFunc(r.opts.MaxDuration, func() { _ = stop() })
|
||||
|
||||
go func() {
|
||||
defer close(chunks)
|
||||
defer capTimer.Stop()
|
||||
// Tear down on ANY exit — including the capture process ending on its own
|
||||
// (EOF/crash), which the loop below returns on without going through an
|
||||
// explicit stop(). stop() is idempotent (sync.Once), so this is safe even
|
||||
// when the session was already stopped externally; it clears r.recording and
|
||||
// closes proc/stdout so a self-terminating device doesn't wedge the recorder.
|
||||
defer func() { _ = stop() }()
|
||||
threshold := int16(32767 * silenceThresholdPct / 100)
|
||||
silentChunks, speechSeen := 0, false
|
||||
silenceLimit := int(silenceHold / chunkInterval)
|
||||
for {
|
||||
buf := make([]byte, chunkBytes)
|
||||
n, err := readFullOrEOF(stdout, buf)
|
||||
if n > 0 {
|
||||
chunk := buf[:n]
|
||||
if r.opts.SilenceAutoStop {
|
||||
if chunkPeak(chunk) >= threshold {
|
||||
speechSeen = true
|
||||
silentChunks = 0
|
||||
} else if speechSeen {
|
||||
silentChunks++
|
||||
if silentChunks >= silenceLimit {
|
||||
go func() { _ = stop() }()
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case chunks <- chunk:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return chunks, stop, nil
|
||||
}
|
||||
|
||||
// chunkSizeBytes is 50ms of PCM16 mono at the given rate — small chunks mean
|
||||
// faster time-to-first-partial (§6a).
|
||||
func chunkSizeBytes(sampleRate int) int {
|
||||
return sampleRate * 2 / int(time.Second/chunkInterval)
|
||||
}
|
||||
|
||||
// readFullOrEOF fills buf from r, returning what it read and any terminal
|
||||
// error. A short final read (pipe closing mid-chunk) still returns its bytes.
|
||||
func readFullOrEOF(r interface{ Read([]byte) (int, error) }, buf []byte) (int, error) {
|
||||
total := 0
|
||||
for total < len(buf) {
|
||||
n, err := r.Read(buf[total:])
|
||||
total += n
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// chunkPeak returns the loudest absolute sample in a little-endian PCM16 chunk.
|
||||
func chunkPeak(chunk []byte) int16 {
|
||||
var peak int16
|
||||
for i := 0; i+1 < len(chunk); i += 2 {
|
||||
s := int16(uint16(chunk[i]) | uint16(chunk[i+1])<<8)
|
||||
if s == -32768 {
|
||||
return 32767
|
||||
}
|
||||
if s < 0 {
|
||||
s = -s
|
||||
}
|
||||
if s > peak {
|
||||
peak = s
|
||||
}
|
||||
}
|
||||
return peak
|
||||
}
|
||||
|
||||
// levelFloorDB and levelCeilDB bound the RMS→meter mapping. Speech RMS is small
|
||||
// (roughly -45 dBFS quiet to -15 dBFS loud), so a LINEAR scale barely moves the
|
||||
// bars — a dB (logarithmic) scale, like real audio meters, is what makes normal
|
||||
// speech span the meter.
|
||||
const (
|
||||
levelFloorDB = -50.0 // ≈ near-silence → empty
|
||||
levelCeilDB = -15.0 // loud speech → full
|
||||
)
|
||||
|
||||
// ChunkLevel returns a perceptual 0..1 loudness for a little-endian PCM16 chunk,
|
||||
// driving the live recording waveform from the actual microphone input. It uses
|
||||
// an RMS→dBFS→linear mapping over [levelFloorDB, levelCeilDB].
|
||||
func ChunkLevel(chunk []byte) float64 {
|
||||
var sumSq float64
|
||||
n := 0
|
||||
for i := 0; i+1 < len(chunk); i += 2 {
|
||||
s := int16(uint16(chunk[i]) | uint16(chunk[i+1])<<8)
|
||||
f := float64(s) / 32768.0
|
||||
sumSq += f * f
|
||||
n++
|
||||
}
|
||||
if n == 0 {
|
||||
return 0
|
||||
}
|
||||
rms := math.Sqrt(sumSq / float64(n))
|
||||
if rms <= 0 {
|
||||
return 0
|
||||
}
|
||||
db := 20 * math.Log10(rms) // dBFS (negative)
|
||||
level := (db - levelFloorDB) / (levelCeilDB - levelFloorDB)
|
||||
if level < 0 {
|
||||
level = 0
|
||||
}
|
||||
if level > 1 {
|
||||
level = 1
|
||||
}
|
||||
return level
|
||||
}
|
||||
|
||||
func maxSeconds(d time.Duration) int {
|
||||
sec := int(d / time.Second)
|
||||
if sec < 1 {
|
||||
sec = 1
|
||||
}
|
||||
return sec
|
||||
}
|
||||
|
||||
// detectWindowsAudioDevice picks the first dshow audio capture device from
|
||||
// `ffmpeg -list_devices`. dshow has no "default" alias, so a concrete device
|
||||
// name is required; stt.windowsAudioDevice overrides the auto-pick.
|
||||
func detectWindowsAudioDevice(runOutput commandOutputRunner) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
out, _ := runOutput(ctx, "ffmpeg", "-hide_banner", "-list_devices", "true", "-f", "dshow", "-i", "dummy")
|
||||
// Listing always "fails" (dummy isn't a real input); the device table is
|
||||
// in the output regardless. Lines look like: [dshow @ ...] "Mic name" (audio)
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
if !strings.Contains(line, "(audio)") {
|
||||
continue
|
||||
}
|
||||
start := strings.Index(line, `"`)
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
rest := line[start+1:]
|
||||
end := strings.Index(rest, `"`)
|
||||
if end <= 0 {
|
||||
continue
|
||||
}
|
||||
return rest[:end], nil
|
||||
}
|
||||
return "", &SetupError{
|
||||
Tool: "ffmpeg",
|
||||
Hint: "no dshow audio capture device found — set stt.windowsAudioDevice to your microphone's name (list with: ffmpeg -list_devices true -f dshow -i dummy)",
|
||||
}
|
||||
}
|
||||
|
||||
// startError maps a failed capture-tool launch to setup guidance: a missing
|
||||
// binary is an expected unconfigured state (matching internal/lsp's
|
||||
// missing-server philosophy), not an internal error.
|
||||
func startError(tool string, platform Platform, err error, output []byte) error {
|
||||
if errors.Is(err, exec.ErrNotFound) {
|
||||
return &SetupError{Tool: tool, Hint: installHint(platform)}
|
||||
}
|
||||
if len(output) > 0 {
|
||||
return fmt.Errorf("starting %s: %w (%s)", tool, err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
return fmt.Errorf("starting %s: %w", tool, err)
|
||||
}
|
||||
|
||||
func installHint(platform Platform) string {
|
||||
switch platform {
|
||||
case PlatformLinux:
|
||||
return "install alsa-utils to enable dictation (e.g. sudo dnf install alsa-utils / sudo apt install alsa-utils)"
|
||||
case PlatformDarwin:
|
||||
return "install sox to enable dictation (brew install sox)"
|
||||
case PlatformWindows:
|
||||
return "install ffmpeg and ensure it is on PATH to enable dictation"
|
||||
case PlatformTermux:
|
||||
return "run `pkg install termux-api` and install the Termux:API companion app to enable dictation"
|
||||
}
|
||||
return "dictation is not supported on this platform"
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeProcess is a processHandle whose stdout is driven from an in-memory
|
||||
// buffer, mirroring how internal/lsp/manager_test.go stubs its subprocess over
|
||||
// in-memory pipes rather than spawning a real one.
|
||||
type fakeProcess struct {
|
||||
stopped bool
|
||||
killed bool
|
||||
waitErr error
|
||||
stopErr error
|
||||
closeOnce sync.Once
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newFakeProcess() *fakeProcess { return &fakeProcess{done: make(chan struct{})} }
|
||||
|
||||
func (f *fakeProcess) StopGracefully() error {
|
||||
f.stopped = true
|
||||
f.closeOnce.Do(func() { close(f.done) })
|
||||
return f.stopErr
|
||||
}
|
||||
func (f *fakeProcess) Wait() error {
|
||||
<-f.done
|
||||
return f.waitErr
|
||||
}
|
||||
func (f *fakeProcess) Kill() error {
|
||||
f.killed = true
|
||||
f.closeOnce.Do(func() { close(f.done) })
|
||||
return nil
|
||||
}
|
||||
|
||||
// captured records the last spec a starter was asked to launch, so tests can
|
||||
// assert the exact discrete argv per platform.
|
||||
type captured struct {
|
||||
spec commandSpec
|
||||
}
|
||||
|
||||
// fileWritingStarter simulates a record-to-file batch tool: on Start it writes
|
||||
// fixture bytes to the output path (last positional arg for arecord/sox, or the
|
||||
// -y target for ffmpeg) and returns a process that "records" until stopped.
|
||||
func fileWritingStarter(t *testing.T, cap *captured, fixture []byte) processStarter {
|
||||
t.Helper()
|
||||
return func(spec commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
cap.spec = spec
|
||||
path := outputPath(spec)
|
||||
if path == "" {
|
||||
t.Fatalf("could not determine output path from argv: %v", spec.args)
|
||||
}
|
||||
if err := os.WriteFile(path, fixture, 0o600); err != nil {
|
||||
t.Fatalf("fixture write: %v", err)
|
||||
}
|
||||
return newFakeProcess(), nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
func outputPath(spec commandSpec) string {
|
||||
for i, a := range spec.args {
|
||||
if a == "-y" && i+1 < len(spec.args) {
|
||||
return spec.args[i+1]
|
||||
}
|
||||
}
|
||||
// arecord/sox: the path is the last arg (sox appends trim/silence effects
|
||||
// after it, so scan from the front for the first existing-dir target).
|
||||
last := spec.args[len(spec.args)-1]
|
||||
if filepath.IsAbs(last) {
|
||||
return last
|
||||
}
|
||||
for _, a := range spec.args {
|
||||
if filepath.IsAbs(a) && filepath.Ext(a) != "" {
|
||||
return a
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func wavFixture() []byte {
|
||||
// Minimal valid RIFF/WAVE header + a little data, enough for SniffFormat.
|
||||
var b bytes.Buffer
|
||||
b.WriteString("RIFF")
|
||||
binary.Write(&b, binary.LittleEndian, uint32(36))
|
||||
b.WriteString("WAVE")
|
||||
b.WriteString("fmt ")
|
||||
binary.Write(&b, binary.LittleEndian, uint32(16))
|
||||
b.Write(make([]byte, 16))
|
||||
b.WriteString("data")
|
||||
binary.Write(&b, binary.LittleEndian, uint32(4))
|
||||
b.Write([]byte{1, 2, 3, 4})
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func testTempDir(t *testing.T) func() (string, error) {
|
||||
dir := t.TempDir()
|
||||
return func() (string, error) { return dir, nil }
|
||||
}
|
||||
|
||||
func TestBatchRecordLinuxArgv(t *testing.T) {
|
||||
cap := &captured{}
|
||||
fixture := wavFixture()
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: fileWritingStarter(t, cap, fixture),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
got, err := r.Stop()
|
||||
if err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, fixture) {
|
||||
t.Fatalf("audio mismatch: got %d bytes", len(got))
|
||||
}
|
||||
if cap.spec.name != "arecord" {
|
||||
t.Errorf("tool = %q, want arecord", cap.spec.name)
|
||||
}
|
||||
assertArgs(t, cap.spec.args, "-f", "S16_LE")
|
||||
assertArgs(t, cap.spec.args, "-r", "16000")
|
||||
assertArgs(t, cap.spec.args, "-c", "1")
|
||||
}
|
||||
|
||||
func TestBatchRecordDarwinArgv(t *testing.T) {
|
||||
cap := &captured{}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformDarwin,
|
||||
SilenceAutoStop: true,
|
||||
starter: fileWritingStarter(t, cap, wavFixture()),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if _, err := r.Stop(); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if cap.spec.name != "sox" {
|
||||
t.Errorf("tool = %q, want sox", cap.spec.name)
|
||||
}
|
||||
if !containsSubseq(cap.spec.args, []string{"silence", "1", "0.1", "3%", "1", "2.0", "3%"}) {
|
||||
t.Errorf("expected SoX silence auto-stop effect in argv: %v", cap.spec.args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchRecordWindowsArgv(t *testing.T) {
|
||||
cap := &captured{}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformWindows,
|
||||
WindowsAudioDevice: "Microphone (Realtek)",
|
||||
starter: fileWritingStarter(t, cap, wavFixture()),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if _, err := r.Stop(); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if cap.spec.name != "ffmpeg" {
|
||||
t.Errorf("tool = %q, want ffmpeg", cap.spec.name)
|
||||
}
|
||||
assertArgs(t, cap.spec.args, "-i", "audio=Microphone (Realtek)")
|
||||
if cap.spec.stopViaStdin != "q" {
|
||||
t.Errorf("ffmpeg should stop via stdin 'q', got %q", cap.spec.stopViaStdin)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchRecordTermuxUsesCommandRunner(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var calls [][]string
|
||||
runOutput := func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
calls = append(calls, append([]string{name}, args...))
|
||||
if len(args) > 0 && args[0] == "-l" {
|
||||
// The -f target is the last arg; write the fixture there.
|
||||
path := args[len(args)-1]
|
||||
_ = os.WriteFile(path, m4aFixture(), 0o600)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformTermux,
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
t.Fatal("termux must not use the process starter")
|
||||
return nil, nil, nil
|
||||
},
|
||||
runOutput: runOutput,
|
||||
tempDir: func() (string, error) { return dir, nil },
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
got, err := r.Stop()
|
||||
if err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if SniffFormat(got) != FormatM4A {
|
||||
t.Errorf("expected m4a audio, got format %q", SniffFormat(got))
|
||||
}
|
||||
if len(calls) != 2 || calls[0][0] != "termux-microphone-record" || calls[1][len(calls[1])-1] != "-q" {
|
||||
t.Errorf("unexpected termux calls: %v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartMissingBinaryReturnsSetupError(t *testing.T) {
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
return nil, nil, exec.ErrNotFound
|
||||
},
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
err := r.Start()
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing binary, got %v", err)
|
||||
}
|
||||
if setupErr.Tool != "arecord" {
|
||||
t.Errorf("SetupError.Tool = %q, want arecord", setupErr.Tool)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoubleStartRejected(t *testing.T) {
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: fileWritingStarter(t, &captured{}, wavFixture()),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if err := r.Start(); !errors.Is(err, errAlreadyRecording) {
|
||||
t.Errorf("second Start should be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingChunksAndStop(t *testing.T) {
|
||||
// 300ms of PCM16 @16kHz = 6 chunks of 50ms. Feed a fixed buffer, then EOF.
|
||||
pcm := make([]byte, chunkSizeBytes(16000)*6)
|
||||
starter := func(spec commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
if !spec.wantStdout {
|
||||
t.Fatal("streaming spec must request stdout")
|
||||
}
|
||||
return newFakeProcess(), io.NopCloser(bytes.NewReader(pcm)), nil
|
||||
}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: starter,
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
chunks, stop, err := r.StartStreaming()
|
||||
if err != nil {
|
||||
t.Fatalf("StartStreaming: %v", err)
|
||||
}
|
||||
var total int
|
||||
for c := range chunks {
|
||||
total += len(c)
|
||||
}
|
||||
_ = stop()
|
||||
if total != len(pcm) {
|
||||
t.Errorf("streamed %d bytes, want %d", total, len(pcm))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingStopIdempotent(t *testing.T) {
|
||||
// A blocking reader so the capture goroutine parks until we stop it.
|
||||
pr, pw := io.Pipe()
|
||||
defer pw.Close()
|
||||
starter := func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
return newFakeProcess(), pr, nil
|
||||
}
|
||||
r := NewRecorder(RecorderOptions{Platform: PlatformLinux, starter: starter, tempDir: testTempDir(t)})
|
||||
chunks, stop, err := r.StartStreaming()
|
||||
if err != nil {
|
||||
t.Fatalf("StartStreaming: %v", err)
|
||||
}
|
||||
if err := stop(); err != nil {
|
||||
t.Fatalf("stop: %v", err)
|
||||
}
|
||||
if err := stop(); err != nil {
|
||||
t.Fatalf("second stop should be a no-op, got %v", err)
|
||||
}
|
||||
// Draining the channel must terminate now that we've stopped.
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
for range chunks {
|
||||
}
|
||||
close(drained)
|
||||
}()
|
||||
select {
|
||||
case <-drained:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("chunks channel did not close after stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchMaxDurationCapInArgv(t *testing.T) {
|
||||
cap := &captured{}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
MaxDuration: 90 * time.Second,
|
||||
starter: fileWritingStarter(t, cap, wavFixture()),
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if _, err := r.Stop(); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
// arecord's -d bounds a runaway recording even if Stop is never called.
|
||||
assertArgs(t, cap.spec.args, "-d", "90")
|
||||
}
|
||||
|
||||
func TestRecordedTempFileDeletedAfterStop(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cap := &captured{}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
starter: fileWritingStarter(t, cap, wavFixture()),
|
||||
tempDir: func() (string, error) { return dir, nil },
|
||||
})
|
||||
if err := r.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if _, err := r.Stop(); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("recorded audio should be deleted after Stop; found %d files", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingSilenceAutoStop(t *testing.T) {
|
||||
// Speech (loud) then a run of silence longer than silenceHold must auto-stop.
|
||||
rate := 16000
|
||||
chunk := chunkSizeBytes(rate)
|
||||
loud := make([]byte, chunk)
|
||||
for i := 0; i+1 < len(loud); i += 2 {
|
||||
loud[i], loud[i+1] = 0x00, 0x40 // +16384, well above threshold
|
||||
}
|
||||
silence := make([]byte, chunk) // zeros
|
||||
var buf []byte
|
||||
buf = append(buf, loud...)
|
||||
silentChunks := int(silenceHold/chunkInterval) + 3
|
||||
for i := 0; i < silentChunks; i++ {
|
||||
buf = append(buf, silence...)
|
||||
}
|
||||
starter := func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
return newFakeProcess(), io.NopCloser(bytes.NewReader(buf)), nil
|
||||
}
|
||||
r := NewRecorder(RecorderOptions{
|
||||
Platform: PlatformLinux,
|
||||
SampleRate: rate,
|
||||
SilenceAutoStop: true,
|
||||
starter: starter,
|
||||
tempDir: testTempDir(t),
|
||||
})
|
||||
chunks, stop, err := r.StartStreaming()
|
||||
if err != nil {
|
||||
t.Fatalf("StartStreaming: %v", err)
|
||||
}
|
||||
defer stop()
|
||||
// Draining must terminate on its own once silence auto-stop fires.
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
for range chunks {
|
||||
}
|
||||
close(drained)
|
||||
}()
|
||||
select {
|
||||
case <-drained:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("silence auto-stop did not end the stream")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkPeak(t *testing.T) {
|
||||
loud := []byte{0x00, 0x40} // +16384
|
||||
if got := chunkPeak(loud); got != 16384 {
|
||||
t.Errorf("peak = %d, want 16384", got)
|
||||
}
|
||||
quiet := []byte{0x10, 0x00} // +16
|
||||
if got := chunkPeak(quiet); got != 16 {
|
||||
t.Errorf("peak = %d, want 16", got)
|
||||
}
|
||||
min := []byte{0x00, 0x80} // -32768 -> clamped to 32767
|
||||
if got := chunkPeak(min); got != 32767 {
|
||||
t.Errorf("peak = %d, want 32767 (clamp)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectWindowsAudioDevice(t *testing.T) {
|
||||
listing := `[dshow @ 0000] DirectShow audio devices
|
||||
[dshow @ 0000] "Microphone (USB Audio)" (audio)
|
||||
[dshow @ 0000] Alternative name "@device_cm_..."`
|
||||
runOutput := func(context.Context, string, ...string) ([]byte, error) { return []byte(listing), nil }
|
||||
got, err := detectWindowsAudioDevice(runOutput)
|
||||
if err != nil {
|
||||
t.Fatalf("detect: %v", err)
|
||||
}
|
||||
if got != "Microphone (USB Audio)" {
|
||||
t.Errorf("device = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSniffFormat(t *testing.T) {
|
||||
if SniffFormat(wavFixture()) != FormatWAV {
|
||||
t.Error("wav not detected")
|
||||
}
|
||||
if SniffFormat(m4aFixture()) != FormatM4A {
|
||||
t.Error("m4a not detected")
|
||||
}
|
||||
if SniffFormat([]byte("nope")) != FormatUnknown {
|
||||
t.Error("unknown not detected")
|
||||
}
|
||||
}
|
||||
|
||||
func m4aFixture() []byte {
|
||||
b := make([]byte, 16)
|
||||
copy(b[4:], []byte("ftyp"))
|
||||
copy(b[8:], []byte("M4A "))
|
||||
return b
|
||||
}
|
||||
|
||||
func assertArgs(t *testing.T, args []string, want ...string) {
|
||||
t.Helper()
|
||||
if !containsSubseq(args, want) {
|
||||
t.Errorf("argv %v missing subsequence %v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func containsSubseq(hay, needle []string) bool {
|
||||
if len(needle) == 0 {
|
||||
return true
|
||||
}
|
||||
for i := 0; i+len(needle) <= len(hay); i++ {
|
||||
if equalSlice(hay[i:i+len(needle)], needle) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func equalSlice(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestChunkLevel(t *testing.T) {
|
||||
// Silence → ~0.
|
||||
silence := make([]byte, 640)
|
||||
if lvl := ChunkLevel(silence); lvl > 0.01 {
|
||||
t.Errorf("silence level = %f, want ~0", lvl)
|
||||
}
|
||||
// A loud full-scale tone → clamped to 1.
|
||||
loud := make([]byte, 640)
|
||||
for i := 0; i+1 < len(loud); i += 2 {
|
||||
loud[i], loud[i+1] = 0xFF, 0x7F // +32767
|
||||
}
|
||||
if lvl := ChunkLevel(loud); lvl < 0.99 {
|
||||
t.Errorf("full-scale level = %f, want ~1", lvl)
|
||||
}
|
||||
// A normal speech-level signal (~-30 dBFS) should land mid-meter, not pinned
|
||||
// to the bottom (the whole point of the dB mapping).
|
||||
mid := make([]byte, 640)
|
||||
for i := 0; i+1 < len(mid); i += 2 {
|
||||
mid[i], mid[i+1] = byte(1000&0xFF), byte(1000>>8) // ≈ -30 dBFS
|
||||
}
|
||||
if lvl := ChunkLevel(mid); lvl < 0.4 || lvl > 0.75 {
|
||||
t.Errorf("normal-speech level = %f, want a visible mid-range value", lvl)
|
||||
}
|
||||
// A quiet signal (~-45 dBFS) should be low but non-zero.
|
||||
quiet := make([]byte, 640)
|
||||
for i := 0; i+1 < len(quiet); i += 2 {
|
||||
quiet[i], quiet[i+1] = byte(180&0xFF), byte(180>>8)
|
||||
}
|
||||
if lvl := ChunkLevel(quiet); lvl <= 0 || lvl > 0.35 {
|
||||
t.Errorf("quiet level = %f, want a small non-zero value", lvl)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
// commandSpec describes one capture-process invocation. Argv is always
|
||||
// discrete — never a shell string (the same invariant documented in
|
||||
// internal/imageinput/clipboard.go).
|
||||
type commandSpec struct {
|
||||
name string
|
||||
args []string
|
||||
// wantStdout wires up a stdout pipe (streaming capture reads raw PCM).
|
||||
wantStdout bool
|
||||
// stopViaStdin, when non-empty, makes graceful stop write this to the
|
||||
// process's stdin instead of signaling — ffmpeg finalizes cleanly on "q",
|
||||
// and Windows has no os.Interrupt delivery anyway.
|
||||
stopViaStdin string
|
||||
}
|
||||
|
||||
// processHandle is a started capture process. Injectable via processStarter so
|
||||
// unit tests fake OS-boundary behavior, mirroring internal/lsp's serverStarter
|
||||
// seam (this package's clipboard.go-style shell-outs get the test seam that
|
||||
// file lacks).
|
||||
type processHandle interface {
|
||||
// StopGracefully asks the tool to finish and flush output (SIGINT, or the
|
||||
// spec's stdin stop text). The recorder falls back to Kill on timeout.
|
||||
StopGracefully() error
|
||||
Wait() error
|
||||
Kill() error
|
||||
}
|
||||
|
||||
// processStarter launches a capture process, returning its handle and — when
|
||||
// the spec asks for it — its stdout pipe.
|
||||
type processStarter func(spec commandSpec) (processHandle, io.ReadCloser, error)
|
||||
|
||||
// commandOutputRunner runs a short one-shot command to completion and returns
|
||||
// its combined output. Used for helper invocations that aren't recordings
|
||||
// (Termux start/stop, Windows ffmpeg device listing) and the offline
|
||||
// transcription exec — ctx lets a caller cancel/timeout a wedged process.
|
||||
type commandOutputRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
|
||||
|
||||
func startProcess(spec commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
cmd := exec.Command(spec.name, spec.args...)
|
||||
var stdout io.ReadCloser
|
||||
if spec.wantStdout {
|
||||
pipe, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
stdout = pipe
|
||||
}
|
||||
var stdin io.WriteCloser
|
||||
if spec.stopViaStdin != "" {
|
||||
pipe, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
stdin = pipe
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
// StdoutPipe/StdinPipe allocate fds the runtime only reclaims via
|
||||
// Wait; on Start failure the process is never established, so close
|
||||
// any pipes we already opened to avoid leaking them.
|
||||
if stdout != nil {
|
||||
_ = stdout.Close()
|
||||
}
|
||||
if stdin != nil {
|
||||
_ = stdin.Close()
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
return &realProcess{cmd: cmd, stdin: stdin, stopText: spec.stopViaStdin}, stdout, nil
|
||||
}
|
||||
|
||||
type realProcess struct {
|
||||
cmd *exec.Cmd
|
||||
stdin io.WriteCloser
|
||||
stopText string
|
||||
}
|
||||
|
||||
func (p *realProcess) StopGracefully() error {
|
||||
if p.stopText != "" && p.stdin != nil {
|
||||
_, err := io.WriteString(p.stdin, p.stopText)
|
||||
closeErr := p.stdin.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
// os.Interrupt is how arecord/sox finalize their WAV header. Unsupported
|
||||
// on Windows, but the only Windows tool (ffmpeg) stops via stdin above.
|
||||
return p.cmd.Process.Signal(os.Interrupt)
|
||||
}
|
||||
|
||||
func (p *realProcess) Wait() error { return p.cmd.Wait() }
|
||||
|
||||
func (p *realProcess) Kill() error { return p.cmd.Process.Kill() }
|
||||
|
||||
func runCommandOutput(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
err := cmd.Run()
|
||||
return out.Bytes(), err
|
||||
}
|
||||
|
||||
// waitWithTimeout waits for the process, killing it if it ignores the graceful
|
||||
// stop for longer than the grace window. Capture tools exit on SIGINT/"q"
|
||||
// almost instantly; a hang here means a wedged device driver, and a leaked
|
||||
// recording process holds the microphone open.
|
||||
func waitWithTimeout(proc processHandle, grace time.Duration) error {
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- proc.Wait() }()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-time.After(grace):
|
||||
_ = proc.Kill()
|
||||
return <-done
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultServerPort is the localhost port the sherpa-onnx streaming server binds.
|
||||
const DefaultServerPort = 6006
|
||||
|
||||
// ServerConfig configures the warm sherpa-onnx streaming server (§6a).
|
||||
type ServerConfig struct {
|
||||
// Binary is the streaming server executable (default
|
||||
// "sherpa-onnx-online-websocket-server"); looked up on PATH.
|
||||
Binary string
|
||||
// ModelPath is the sherpa-onnx streaming model directory (transducer:
|
||||
// encoder/decoder/joiner + tokens).
|
||||
ModelPath string
|
||||
Port int
|
||||
NumThreads int
|
||||
|
||||
// Test seams.
|
||||
starter processStarter
|
||||
healthCheck func(ctx context.Context, port int) error
|
||||
aliveCheck func(port int) bool
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// ServerManager owns one long-lived sherpa-onnx streaming server, spawned lazily
|
||||
// on first streaming use and kept warm for the session — a websocket server with
|
||||
// ~1-2s startup latency can't be respawned per utterance (§6a). Its lifecycle
|
||||
// mirrors internal/lsp's Manager: lazy start, health-checked reuse, one restart
|
||||
// on a crashed process, torn down on exit. Safe for concurrent use.
|
||||
type ServerManager struct {
|
||||
cfg ServerConfig
|
||||
|
||||
mu sync.Mutex
|
||||
proc processHandle
|
||||
url string
|
||||
}
|
||||
|
||||
// NewServerManager builds a manager. Construction is side-effect free; the
|
||||
// server is only spawned on the first EnsureRunning.
|
||||
func NewServerManager(cfg ServerConfig) *ServerManager {
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = "sherpa-onnx-online-websocket-server"
|
||||
}
|
||||
if cfg.Port == 0 {
|
||||
cfg.Port = DefaultServerPort
|
||||
}
|
||||
if cfg.starter == nil {
|
||||
cfg.starter = startProcess
|
||||
}
|
||||
if cfg.healthCheck == nil {
|
||||
cfg.healthCheck = dialHealthCheck
|
||||
}
|
||||
if cfg.aliveCheck == nil {
|
||||
cfg.aliveCheck = func(port int) bool { return portListening(port) }
|
||||
}
|
||||
if cfg.now == nil {
|
||||
cfg.now = time.Now
|
||||
}
|
||||
return &ServerManager{cfg: cfg}
|
||||
}
|
||||
|
||||
// EnsureRunning returns the websocket URL of a live server, starting one if
|
||||
// needed. A previously-started server whose process has since died is reaped and
|
||||
// replaced (mirroring the LSP manager's crashed-session eviction).
|
||||
func (m *ServerManager) EnsureRunning(ctx context.Context) (string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.proc != nil {
|
||||
if m.processAlive() {
|
||||
return m.url, nil
|
||||
}
|
||||
// The cached server died; reap it so a fresh one starts below.
|
||||
_ = m.proc.Kill()
|
||||
m.proc = nil
|
||||
m.url = ""
|
||||
}
|
||||
|
||||
spec, err := onlineServerCommand(m.cfg)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
proc, _, err := m.cfg.starter(spec)
|
||||
if err != nil {
|
||||
if errors.Is(err, exec.ErrNotFound) {
|
||||
return "", &SetupError{
|
||||
Tool: m.cfg.Binary,
|
||||
Hint: "install sherpa-onnx and put its online-websocket-server binary on PATH (see docs/dictation.md)",
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("starting sherpa-onnx server: %w", err)
|
||||
}
|
||||
|
||||
// Health-check the port before handing back a URL — the server takes ~1-2s to
|
||||
// bind, and a client connecting too early would fail (§6a).
|
||||
if err := m.cfg.healthCheck(ctx, m.cfg.Port); err != nil {
|
||||
_ = proc.Kill()
|
||||
return "", fmt.Errorf("sherpa-onnx server did not become ready: %w", err)
|
||||
}
|
||||
m.proc = proc
|
||||
m.url = fmt.Sprintf("ws://127.0.0.1:%d", m.cfg.Port)
|
||||
return m.url, nil
|
||||
}
|
||||
|
||||
// processAlive reports whether the tracked server is still reachable. Probing
|
||||
// the port (rather than the process handle) reflects real reachability and is
|
||||
// the seam tests replace.
|
||||
func (m *ServerManager) processAlive() bool {
|
||||
return m.cfg.aliveCheck(m.cfg.Port)
|
||||
}
|
||||
|
||||
// portListening reports whether something accepts TCP connections on the port.
|
||||
func portListening(port int) bool {
|
||||
conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), 200*time.Millisecond)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
// Shutdown stops the running server. Called alongside the LSP manager's shutdown
|
||||
// on exit — a leaked warm server holds a port and CPU.
|
||||
func (m *ServerManager) Shutdown(ctx context.Context) error {
|
||||
m.mu.Lock()
|
||||
proc := m.proc
|
||||
m.proc = nil
|
||||
m.url = ""
|
||||
m.mu.Unlock()
|
||||
if proc == nil {
|
||||
return nil
|
||||
}
|
||||
if err := proc.StopGracefully(); err != nil {
|
||||
_ = proc.Kill()
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- waitWithTimeout(proc, stopGrace) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
_ = proc.Kill()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// URL returns the current server URL without starting one ("" when not running).
|
||||
func (m *ServerManager) URL() string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.url
|
||||
}
|
||||
|
||||
// SetModelPath updates the model directory the server launches with, for a
|
||||
// mid-session config change (e.g. an F9 auto-download). It takes effect the next
|
||||
// time the server (re)starts; a running server is left as-is.
|
||||
func (m *ServerManager) SetModelPath(path string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if path != "" {
|
||||
m.cfg.ModelPath = path
|
||||
}
|
||||
}
|
||||
|
||||
// onlineServerCommand builds the streaming-server argv from the model directory.
|
||||
// The online server uses a streaming transducer (encoder/decoder/joiner) plus
|
||||
// tokens — the shape the sherpa-onnx docs' example uses (§6a).
|
||||
func onlineServerCommand(cfg ServerConfig) (commandSpec, error) {
|
||||
if strings.TrimSpace(cfg.ModelPath) == "" {
|
||||
return commandSpec{}, &SetupError{
|
||||
Tool: "local STT streaming model",
|
||||
Hint: "set stt.localModelPath to a sherpa-onnx streaming model directory (see docs/dictation.md)",
|
||||
}
|
||||
}
|
||||
files, err := onnxFilesIn(cfg.ModelPath)
|
||||
if err != nil {
|
||||
return commandSpec{}, &SetupError{Tool: "local STT streaming model", Hint: err.Error()}
|
||||
}
|
||||
tokens := findTokensFile(files)
|
||||
// A streaming model ships encoder/decoder/joiner, often BOTH an int8 and an
|
||||
// fp32 copy of each. Pick a CONSISTENT set — all int8 or all fp32 — because
|
||||
// mixing (int8 encoder + fp32 decoder) is invalid. Prefer int8 (smaller,
|
||||
// faster) when a full int8 set is present.
|
||||
set := consistentQuantizedSet(files, "encoder", "decoder", "joiner")
|
||||
var encoder, decoder, joiner string
|
||||
if set != nil {
|
||||
encoder, decoder, joiner = set[0], set[1], set[2]
|
||||
}
|
||||
if tokens == "" || encoder == "" || decoder == "" || joiner == "" {
|
||||
return commandSpec{}, &SetupError{
|
||||
Tool: "local STT streaming model",
|
||||
Hint: fmt.Sprintf("streaming needs a transducer model (encoder/decoder/joiner .onnx + tokens.txt) in %q; see docs/dictation.md", cfg.ModelPath),
|
||||
}
|
||||
}
|
||||
args := []string{
|
||||
"--port=" + strconv.Itoa(cfg.Port),
|
||||
"--tokens=" + tokens,
|
||||
"--encoder=" + encoder,
|
||||
"--decoder=" + decoder,
|
||||
"--joiner=" + joiner,
|
||||
"--decoding-method=greedy_search",
|
||||
}
|
||||
if cfg.NumThreads > 0 {
|
||||
args = append(args, "--num-threads="+strconv.Itoa(cfg.NumThreads))
|
||||
}
|
||||
return commandSpec{name: cfg.Binary, args: args}, nil
|
||||
}
|
||||
|
||||
func onnxFilesIn(dir string) (map[string]string, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stt.localModelPath %q is not readable: %v", dir, err)
|
||||
}
|
||||
files := map[string]string{}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
files[strings.ToLower(e.Name())] = dir + string(os.PathSeparator) + e.Name()
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// dialHealthCheck polls the server's TCP port until it accepts a connection or
|
||||
// the deadline passes.
|
||||
func dialHealthCheck(ctx context.Context, port int) error {
|
||||
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
conn, err := net.DialTimeout("tcp", addr, 500*time.Millisecond)
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("port %d not listening after 5s", port)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func streamingModelDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for _, f := range []string{"encoder.onnx", "decoder.onnx", "joiner.onnx", "tokens.txt"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestServerManagerLazyStartAndReuse(t *testing.T) {
|
||||
var startCount int
|
||||
var mu sync.Mutex
|
||||
starter := func(spec commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
mu.Lock()
|
||||
startCount++
|
||||
mu.Unlock()
|
||||
if spec.name != "sherpa-onnx-online-websocket-server" {
|
||||
t.Errorf("binary = %q", spec.name)
|
||||
}
|
||||
return newFakeProcess(), nil, nil
|
||||
}
|
||||
alive := true
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
Port: 6006,
|
||||
starter: starter,
|
||||
healthCheck: func(context.Context, int) error { return nil },
|
||||
aliveCheck: func(int) bool { return alive },
|
||||
})
|
||||
|
||||
url, err := m.EnsureRunning(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureRunning: %v", err)
|
||||
}
|
||||
if url != "ws://127.0.0.1:6006" {
|
||||
t.Errorf("url = %q", url)
|
||||
}
|
||||
// Second call reuses the warm server (alive) — no new start.
|
||||
if _, err := m.EnsureRunning(context.Background()); err != nil {
|
||||
t.Fatalf("EnsureRunning reuse: %v", err)
|
||||
}
|
||||
if startCount != 1 {
|
||||
t.Errorf("expected 1 start (warm reuse), got %d", startCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerManagerRestartsCrashedServer(t *testing.T) {
|
||||
var startCount int
|
||||
starter := func(commandSpec) (processHandle, io.ReadCloser, error) {
|
||||
startCount++
|
||||
return newFakeProcess(), nil, nil
|
||||
}
|
||||
alive := true
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
starter: starter,
|
||||
healthCheck: func(context.Context, int) error { return nil },
|
||||
aliveCheck: func(int) bool { return alive },
|
||||
})
|
||||
if _, err := m.EnsureRunning(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
alive = false // simulate a crashed server
|
||||
if _, err := m.EnsureRunning(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if startCount != 2 {
|
||||
t.Errorf("expected a restart (2 starts), got %d", startCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerManagerMissingBinaryIsSetupError(t *testing.T) {
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) { return nil, nil, exec.ErrNotFound },
|
||||
})
|
||||
_, err := m.EnsureRunning(context.Background())
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerManagerMissingModelIsSetupError(t *testing.T) {
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: t.TempDir(), // empty dir, no onnx/tokens
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) { return newFakeProcess(), nil, nil },
|
||||
})
|
||||
_, err := m.EnsureRunning(context.Background())
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing streaming model, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerManagerShutdown(t *testing.T) {
|
||||
proc := newFakeProcess()
|
||||
m := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) { return proc, nil, nil },
|
||||
healthCheck: func(context.Context, int) error { return nil },
|
||||
aliveCheck: func(int) bool { return true },
|
||||
})
|
||||
if _, err := m.EnsureRunning(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Shutdown(context.Background()); err != nil {
|
||||
t.Fatalf("Shutdown: %v", err)
|
||||
}
|
||||
if !proc.stopped && !proc.killed {
|
||||
t.Error("Shutdown should stop the server process")
|
||||
}
|
||||
if m.URL() != "" {
|
||||
t.Error("URL should be empty after shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlineServerCommandBuildsTransducerArgs(t *testing.T) {
|
||||
spec, err := onlineServerCommand(ServerConfig{Binary: "sherpa-onnx-online-websocket-server", ModelPath: streamingModelDir(t), Port: 6007})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := ""
|
||||
for _, a := range spec.args {
|
||||
joined += a + " "
|
||||
}
|
||||
for _, want := range []string{"--port=6007", "--tokens=", "--encoder=", "--decoder=", "--joiner=", "--decoding-method=greedy_search"} {
|
||||
if !contains(joined, want) {
|
||||
t.Errorf("missing %q in args: %s", want, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPCM16ToFloat32LE(t *testing.T) {
|
||||
// int16 max (32767) and min (-32768), little-endian.
|
||||
pcm := []byte{0xFF, 0x7F, 0x00, 0x80}
|
||||
out := pcm16ToFloat32LE(pcm)
|
||||
if len(out) != 8 {
|
||||
t.Fatalf("expected 8 bytes (2 float32), got %d", len(out))
|
||||
}
|
||||
f0 := math.Float32frombits(binary.LittleEndian.Uint32(out[0:4]))
|
||||
f1 := math.Float32frombits(binary.LittleEndian.Uint32(out[4:8]))
|
||||
if f0 < 0.99 || f0 > 1.0 {
|
||||
t.Errorf("32767 → %f, want ~1.0", f0)
|
||||
}
|
||||
if f1 != -1.0 {
|
||||
t.Errorf("-32768 → %f, want -1.0", f1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSherpaStreamResult(t *testing.T) {
|
||||
text, segment, final, ok := parseSherpaStreamResult([]byte(`{"text":" hello world ","segment":2,"is_final":true}`))
|
||||
if !ok || text != "hello world" || segment != 2 || !final {
|
||||
t.Errorf("parsed = (%q, %d, %v, %v)", text, segment, final, ok)
|
||||
}
|
||||
if _, _, _, ok := parseSherpaStreamResult([]byte("not json")); ok {
|
||||
t.Error("non-JSON should not parse")
|
||||
}
|
||||
}
|
||||
|
||||
func contains(hay, needle string) bool {
|
||||
return len(needle) == 0 || (len(hay) >= len(needle) && indexOf(hay, needle) >= 0)
|
||||
}
|
||||
|
||||
func indexOf(hay, needle string) int {
|
||||
for i := 0; i+len(needle) <= len(hay); i++ {
|
||||
if hay[i:i+len(needle)] == needle {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Recorded audio is staged in a hardened per-user temp directory, mirroring
|
||||
// internal/tools/spill.go (PR #506): uid-suffixed name so users on a shared
|
||||
// /tmp cannot collide, 0700 perms, and a pre-existing path is only accepted
|
||||
// when it is a real directory (not a symlink that would redirect recordings)
|
||||
// owned by the current user. Files are removed as soon as their bytes are
|
||||
// read back after Stop — audio never lingers on disk.
|
||||
|
||||
func audioSpillRoot() string {
|
||||
name := "zero-dictation"
|
||||
if uid := os.Getuid(); uid >= 0 {
|
||||
name = fmt.Sprintf("zero-dictation-%d", uid)
|
||||
}
|
||||
return filepath.Join(os.TempDir(), name)
|
||||
}
|
||||
|
||||
func audioSpillDir() (string, error) {
|
||||
dir := audioSpillRoot()
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// MkdirAll follows symlinks and leaves an existing directory untouched, so
|
||||
// verify what is actually at the path.
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("dictation temp path %s is not a directory", dir)
|
||||
}
|
||||
// MkdirAll only applies 0o700 to dirs it creates; an existing directory
|
||||
// (e.g. one created under a loose umask in a prior run) can be group- or
|
||||
// world-readable. Tighten it so staged audio (PII) is never visible to
|
||||
// other users on a shared /tmp.
|
||||
if info.Mode().Perm() != 0o700 {
|
||||
if err := os.Chmod(dir, 0o700); err != nil {
|
||||
return "", fmt.Errorf("tightening dictation temp dir permissions: %w", err)
|
||||
}
|
||||
}
|
||||
if err := checkAudioDirOwner(info); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestLocalStreamingRealServer drives the full local streaming path against a
|
||||
// REAL sherpa-onnx-online-websocket-server + streaming model: spawn the server,
|
||||
// connect, feed a wav's PCM in 50ms chunks, and check the transcript. Opt-in:
|
||||
//
|
||||
// ZERO_STT_STREAM_SERVER=/path/to/sherpa-onnx-online-websocket-server \
|
||||
// ZERO_STT_STREAM_MODEL=/path/to/streaming-model-dir \
|
||||
// ZERO_STT_STREAM_WAV=/path/to/16k-mono.wav \
|
||||
// go test ./internal/dictation/ -run RealServer -v -timeout 5m
|
||||
func TestLocalStreamingRealServer(t *testing.T) {
|
||||
server := os.Getenv("ZERO_STT_STREAM_SERVER")
|
||||
model := os.Getenv("ZERO_STT_STREAM_MODEL")
|
||||
wav := os.Getenv("ZERO_STT_STREAM_WAV")
|
||||
if server == "" || model == "" || wav == "" {
|
||||
t.Skip("set ZERO_STT_STREAM_SERVER, ZERO_STT_STREAM_MODEL, ZERO_STT_STREAM_WAV to run the streaming smoke test")
|
||||
}
|
||||
raw, err := os.ReadFile(wav)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pcm := stripWavHeader(raw)
|
||||
|
||||
mgr := NewServerManager(ServerConfig{Binary: server, ModelPath: model, Port: 6100})
|
||||
defer mgr.Shutdown(context.Background())
|
||||
tr := NewLocalStreamingTranscriber(mgr)
|
||||
|
||||
chunks := make(chan []byte)
|
||||
go func() {
|
||||
defer close(chunks)
|
||||
size := chunkSizeBytes(DefaultSampleRate) // 50ms
|
||||
for off := 0; off < len(pcm); off += size {
|
||||
end := off + size
|
||||
if end > len(pcm) {
|
||||
end = len(pcm)
|
||||
}
|
||||
chunks <- pcm[off:end]
|
||||
time.Sleep(10 * time.Millisecond) // pace it a little, like a mic
|
||||
}
|
||||
}()
|
||||
|
||||
var partials int
|
||||
final, err := tr.StreamTranscribe(context.Background(), chunks, func(string, bool) { partials++ })
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTranscribe: %v", err)
|
||||
}
|
||||
if final == "" {
|
||||
t.Fatal("streaming returned empty transcript")
|
||||
}
|
||||
t.Logf("streaming transcript (%d partials): %q", partials, final)
|
||||
}
|
||||
|
||||
// stripWavHeader drops a standard 44-byte PCM WAV header, returning the raw
|
||||
// int16 samples. Good enough for the canonical 16kHz/mono/16-bit test wavs.
|
||||
func stripWavHeader(b []byte) []byte {
|
||||
if len(b) > 44 && string(b[:4]) == "RIFF" {
|
||||
return b[44:]
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package dictation
|
||||
|
||||
import "context"
|
||||
|
||||
// Transcriber turns captured audio into text. Batch (Transcribe) and streaming
|
||||
// (StreamTranscribe) are one interface but two independent code paths — a
|
||||
// batch-only provider (Groq/OpenAI, sherpa-onnx-offline) leaves
|
||||
// StreamTranscribe returning ErrStreamingUnsupported, and a streaming provider
|
||||
// still implements Transcribe for the batch/Termux fallback.
|
||||
type Transcriber interface {
|
||||
// Transcribe returns the final text for one recorded clip.
|
||||
Transcribe(ctx context.Context, audio []byte) (string, error)
|
||||
|
||||
// StreamTranscribe consumes PCM16 chunks and reports incremental results
|
||||
// via onPartial(text, final) as decoding proceeds, returning the complete
|
||||
// final transcript when the chunk channel closes.
|
||||
//
|
||||
// onPartial's text is the CURRENT best transcript of the utterance so far
|
||||
// (cumulative, not a delta) so the composer can replace the live region
|
||||
// wholesale; final marks a segment the provider considers settled.
|
||||
//
|
||||
// On a mid-stream failure (network drop for cloud, subprocess crash for
|
||||
// local) StreamTranscribe returns the best-effort text accumulated from
|
||||
// onPartial calls made so far PLUS a non-nil error — already-streamed text
|
||||
// is never discarded, only the untranscribed tail after the failure is lost.
|
||||
StreamTranscribe(ctx context.Context, chunks <-chan []byte, onPartial func(text string, final bool)) (string, error)
|
||||
}
|
||||
|
||||
// SampleRate lets a Transcriber declare the capture rate its wire format
|
||||
// expects, so the recorder can be built to match (OpenAI Realtime wants 24kHz;
|
||||
// everything else uses 16kHz). A Transcriber that does not implement this uses
|
||||
// DefaultSampleRate.
|
||||
type SampleRate interface {
|
||||
SampleRate() int
|
||||
}
|
||||
|
||||
// RequiredSampleRate returns the capture rate a Transcriber needs.
|
||||
func RequiredSampleRate(t Transcriber) int {
|
||||
if sr, ok := t.(SampleRate); ok {
|
||||
if rate := sr.SampleRate(); rate > 0 {
|
||||
return rate
|
||||
}
|
||||
}
|
||||
return DefaultSampleRate
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/providers/providerio"
|
||||
)
|
||||
|
||||
// CloudConfig configures a batch cloud transcriber. Groq and OpenAI share the
|
||||
// OpenAI-compatible /audio/transcriptions multipart endpoint — the design's
|
||||
// "each owns its endpoint/model specifics while sharing the request shape"
|
||||
// (§6). Key and base URL are resolved by the caller (TUI layer) from
|
||||
// config/credstore, keeping this type decoupled and unit-testable.
|
||||
type CloudConfig struct {
|
||||
Provider string // ProviderGroq or ProviderOpenAI, for error messages
|
||||
BaseURL string // e.g. https://api.groq.com/openai/v1
|
||||
APIKey string
|
||||
Model string // e.g. whisper-large-v3-turbo, whisper-1
|
||||
// Language optionally constrains recognition (ISO-639-1). Empty = auto.
|
||||
Language string
|
||||
|
||||
// HTTPClient is injectable for tests; nil uses providerio's shared,
|
||||
// stall-hardened client.
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type cloudTranscriber struct {
|
||||
cfg CloudConfig
|
||||
}
|
||||
|
||||
// NewCloudTranscriber builds a batch transcriber for Groq or OpenAI.
|
||||
func NewCloudTranscriber(cfg CloudConfig) (Transcriber, error) {
|
||||
if strings.TrimSpace(cfg.APIKey) == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: cfg.Provider + " API key",
|
||||
Hint: fmt.Sprintf("set a %s API key (run `zero auth`) to use cloud dictation", cfg.Provider),
|
||||
}
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
return nil, fmt.Errorf("dictation: no model configured for provider %q", cfg.Provider)
|
||||
}
|
||||
base, err := providerio.NormalizeBaseURL(cfg.BaseURL, cfg.BaseURL, cfg.Provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.BaseURL = base
|
||||
return &cloudTranscriber{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (c *cloudTranscriber) Transcribe(ctx context.Context, audio []byte) (string, error) {
|
||||
if len(audio) == 0 {
|
||||
return "", errors.New("no audio to transcribe")
|
||||
}
|
||||
body, contentType, err := c.buildMultipart(audio)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
endpoint := c.cfg.BaseURL + "/audio/transcriptions"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
|
||||
|
||||
resp, err := providerio.HTTPClient(c.cfg.HTTPClient).Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s transcription request failed: %w", c.cfg.Provider, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
msg := providerio.ClassifiedError(resp.StatusCode, strings.TrimSpace(string(respBody)), c.cfg.APIKey)
|
||||
// A 401/403 is a fixable credential problem: surface it as a typed AuthError
|
||||
// so the TUI can offer an inline key prompt instead of a dead-end message.
|
||||
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
|
||||
return "", &AuthError{Provider: c.cfg.Provider, Message: msg}
|
||||
}
|
||||
return "", errors.New(msg)
|
||||
}
|
||||
var parsed struct {
|
||||
Text string `json:"text"`
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(respBody, &parsed); err != nil {
|
||||
return "", fmt.Errorf("%s transcription: could not parse response: %w", c.cfg.Provider, err)
|
||||
}
|
||||
if parsed.Error.Message != "" {
|
||||
return "", fmt.Errorf("%s transcription error: %s", c.cfg.Provider, providerio.Redact(parsed.Error.Message, c.cfg.APIKey))
|
||||
}
|
||||
return strings.TrimSpace(parsed.Text), nil
|
||||
}
|
||||
|
||||
// buildMultipart assembles the transcription upload. The file field name and
|
||||
// the "model"/"response_format" fields match the OpenAI audio API that Groq
|
||||
// mirrors. The filename's extension tells the server the container type —
|
||||
// SniffFormat keeps WAV/M4A honest so Termux's AAC uploads work.
|
||||
func (c *cloudTranscriber) buildMultipart(audio []byte) ([]byte, string, error) {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
fw, err := w.CreateFormFile("file", SniffFormat(audio).FileName())
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if _, err := fw.Write(audio); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := w.WriteField("model", c.cfg.Model); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := w.WriteField("response_format", "json"); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if c.cfg.Language != "" {
|
||||
if err := w.WriteField("language", c.cfg.Language); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return buf.Bytes(), w.FormDataContentType(), nil
|
||||
}
|
||||
|
||||
// StreamTranscribe is not supported for the batch cloud providers (Groq has no
|
||||
// streaming product; OpenAI batch is one-shot). Streaming uses the dedicated
|
||||
// Deepgram/OpenAI-Realtime transcribers instead.
|
||||
func (c *cloudTranscriber) StreamTranscribe(context.Context, <-chan []byte, func(string, bool)) (string, error) {
|
||||
return "", ErrStreamingUnsupported
|
||||
}
|
||||
|
||||
// ErrStreamingUnsupported reports that a batch-only Transcriber cannot stream;
|
||||
// the caller falls back to the batch pipeline.
|
||||
var ErrStreamingUnsupported = errors.New("this transcription provider does not support streaming")
|
||||
@@ -0,0 +1,142 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCloudTranscribeBuildsMultipartAndParsesText(t *testing.T) {
|
||||
var gotModel, gotAuth, gotFilename string
|
||||
var gotFileBytes []byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
if r.URL.Path != "/audio/transcriptions" {
|
||||
t.Errorf("unexpected path %q", r.URL.Path)
|
||||
}
|
||||
mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil || !strings.HasPrefix(mediaType, "multipart/") {
|
||||
t.Fatalf("bad content type %q: %v", r.Header.Get("Content-Type"), err)
|
||||
}
|
||||
mr := multipart.NewReader(r.Body, params["boundary"])
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("multipart: %v", err)
|
||||
}
|
||||
switch part.FormName() {
|
||||
case "model":
|
||||
b, _ := io.ReadAll(part)
|
||||
gotModel = string(b)
|
||||
case "file":
|
||||
gotFilename = part.FileName()
|
||||
gotFileBytes, _ = io.ReadAll(part)
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
io.WriteString(w, `{"text":" hello world "}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tr, err := NewCloudTranscriber(CloudConfig{
|
||||
Provider: ProviderGroq,
|
||||
BaseURL: srv.URL,
|
||||
APIKey: "secret-key",
|
||||
Model: "whisper-large-v3-turbo",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewCloudTranscriber: %v", err)
|
||||
}
|
||||
audio := wavFixture()
|
||||
text, err := tr.Transcribe(context.Background(), audio)
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if text != "hello world" {
|
||||
t.Errorf("text = %q, want trimmed 'hello world'", text)
|
||||
}
|
||||
if gotModel != "whisper-large-v3-turbo" {
|
||||
t.Errorf("model field = %q", gotModel)
|
||||
}
|
||||
if gotAuth != "Bearer secret-key" {
|
||||
t.Errorf("auth header = %q", gotAuth)
|
||||
}
|
||||
if gotFilename != "audio.wav" {
|
||||
t.Errorf("filename = %q, want audio.wav", gotFilename)
|
||||
}
|
||||
if len(gotFileBytes) != len(audio) {
|
||||
t.Errorf("uploaded %d file bytes, want %d", len(gotFileBytes), len(audio))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudTranscribeM4AFilename(t *testing.T) {
|
||||
var gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, params, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
mr := multipart.NewReader(r.Body, params["boundary"])
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if part.FormName() == "file" {
|
||||
gotFilename = part.FileName()
|
||||
}
|
||||
}
|
||||
io.WriteString(w, `{"text":"ok"}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tr, _ := NewCloudTranscriber(CloudConfig{Provider: ProviderOpenAI, BaseURL: srv.URL, APIKey: "k", Model: "whisper-1"})
|
||||
if _, err := tr.Transcribe(context.Background(), m4aFixture()); err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if gotFilename != "audio.m4a" {
|
||||
t.Errorf("filename = %q, want audio.m4a", gotFilename)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudTranscribeAuthErrorClassified(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
io.WriteString(w, `{"error":{"message":"invalid api key sk-secret-key"}}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tr, _ := NewCloudTranscriber(CloudConfig{Provider: ProviderGroq, BaseURL: srv.URL, APIKey: "sk-secret-key", Model: "whisper-large-v3-turbo"})
|
||||
_, err := tr.Transcribe(context.Background(), wavFixture())
|
||||
if err == nil {
|
||||
t.Fatal("expected error on 401")
|
||||
}
|
||||
if strings.Contains(err.Error(), "sk-secret-key") {
|
||||
t.Errorf("API key leaked into error: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "auth error") {
|
||||
t.Errorf("expected auth-error classification, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCloudTranscriberRequiresKey(t *testing.T) {
|
||||
_, err := NewCloudTranscriber(CloudConfig{Provider: ProviderGroq, BaseURL: "https://api.groq.com/openai/v1", Model: "m"})
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing key, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudTranscribeStreamingUnsupported(t *testing.T) {
|
||||
tr, _ := NewCloudTranscriber(CloudConfig{Provider: ProviderGroq, BaseURL: "https://x", APIKey: "k", Model: "m"})
|
||||
_, err := tr.StreamTranscribe(context.Background(), nil, nil)
|
||||
if !errors.Is(err, ErrStreamingUnsupported) {
|
||||
t.Errorf("want ErrStreamingUnsupported, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// DeepgramConfig configures the Deepgram streaming transcriber (§6b, default
|
||||
// cloud streaming provider). The key is resolved by the caller.
|
||||
type DeepgramConfig struct {
|
||||
APIKey string
|
||||
Model string // default "nova-3"
|
||||
Language string
|
||||
SampleRate int // capture rate; default DefaultSampleRate (16kHz)
|
||||
// BaseURL overrides the wss endpoint (tests point it at a fake server).
|
||||
BaseURL string
|
||||
}
|
||||
|
||||
type deepgramTranscriber struct {
|
||||
cfg DeepgramConfig
|
||||
}
|
||||
|
||||
// NewDeepgramTranscriber builds the Deepgram streaming transcriber.
|
||||
func NewDeepgramTranscriber(cfg DeepgramConfig) (Transcriber, error) {
|
||||
if strings.TrimSpace(cfg.APIKey) == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: "Deepgram API key",
|
||||
Hint: "set a Deepgram API key (DEEPGRAM_API_KEY, or `zero auth`) to use Deepgram streaming dictation",
|
||||
}
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
cfg.Model = "nova-3"
|
||||
}
|
||||
if cfg.SampleRate <= 0 {
|
||||
cfg.SampleRate = DefaultSampleRate
|
||||
}
|
||||
if cfg.BaseURL == "" {
|
||||
cfg.BaseURL = "wss://api.deepgram.com/v1/listen"
|
||||
}
|
||||
return &deepgramTranscriber{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
// Transcribe is unsupported — Deepgram is used only on the streaming path here.
|
||||
func (d *deepgramTranscriber) Transcribe(context.Context, []byte) (string, error) {
|
||||
return "", ErrStreamingUnsupported
|
||||
}
|
||||
|
||||
func (d *deepgramTranscriber) SampleRate() int { return d.cfg.SampleRate }
|
||||
|
||||
func (d *deepgramTranscriber) StreamTranscribe(ctx context.Context, chunks <-chan []byte, onPartial func(string, bool)) (string, error) {
|
||||
endpoint := d.buildURL()
|
||||
conn, _, err := websocket.Dial(ctx, endpoint, &websocket.DialOptions{
|
||||
HTTPHeader: http.Header{"Authorization": {"Token " + d.cfg.APIKey}},
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("connecting to Deepgram: %w", err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
|
||||
// Deepgram consumes raw linear16 (int16) PCM — the recorder's native format,
|
||||
// so no conversion (unlike sherpa-onnx, which wants float32).
|
||||
writeErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
for chunk := range chunks {
|
||||
if err := conn.Write(ctx, websocket.MessageBinary, chunk); err != nil {
|
||||
writeErrCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
// CloseStream flushes any buffered audio and returns final results before
|
||||
// Deepgram closes the socket.
|
||||
writeErrCh <- conn.Write(ctx, websocket.MessageText, []byte(`{"type":"CloseStream"}`))
|
||||
}()
|
||||
|
||||
// Deepgram results are per-utterance segments, not cumulative: accumulate the
|
||||
// is_final segments and append the current interim to build the full text.
|
||||
var finals []string
|
||||
var interim string
|
||||
compose := func() string {
|
||||
return strings.TrimSpace(strings.Join(append(finals, interim), " "))
|
||||
}
|
||||
for {
|
||||
typ, data, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
// A clean close after CloseStream is success, not failure.
|
||||
if websocket.CloseStatus(err) == websocket.StatusNormalClosure {
|
||||
return compose(), nil
|
||||
}
|
||||
select {
|
||||
case werr := <-writeErrCh:
|
||||
if werr != nil {
|
||||
err = werr
|
||||
}
|
||||
default:
|
||||
}
|
||||
return compose(), fmt.Errorf("Deepgram stream error: %w", err)
|
||||
}
|
||||
if typ != websocket.MessageText {
|
||||
continue
|
||||
}
|
||||
transcript, isFinal, ok := parseDeepgramResult(data)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if isFinal {
|
||||
if transcript != "" {
|
||||
finals = append(finals, transcript)
|
||||
}
|
||||
interim = ""
|
||||
} else {
|
||||
interim = transcript
|
||||
}
|
||||
if onPartial != nil {
|
||||
onPartial(compose(), isFinal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deepgramTranscriber) buildURL() string {
|
||||
q := url.Values{}
|
||||
q.Set("encoding", "linear16")
|
||||
q.Set("sample_rate", strconv.Itoa(d.cfg.SampleRate))
|
||||
q.Set("channels", "1")
|
||||
q.Set("interim_results", "true")
|
||||
q.Set("punctuate", "true")
|
||||
q.Set("model", d.cfg.Model)
|
||||
if d.cfg.Language != "" {
|
||||
q.Set("language", d.cfg.Language)
|
||||
}
|
||||
return d.cfg.BaseURL + "?" + q.Encode()
|
||||
}
|
||||
|
||||
// parseDeepgramResult extracts the transcript and is_final flag from a Deepgram
|
||||
// "Results" message: channel.alternatives[0].transcript.
|
||||
func parseDeepgramResult(data []byte) (transcript string, isFinal bool, ok bool) {
|
||||
var msg struct {
|
||||
Type string `json:"type"`
|
||||
IsFinal bool `json:"is_final"`
|
||||
Channel struct {
|
||||
Alternatives []struct {
|
||||
Transcript string `json:"transcript"`
|
||||
} `json:"alternatives"`
|
||||
} `json:"channel"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &msg); err != nil {
|
||||
return "", false, false
|
||||
}
|
||||
// Only "Results" messages carry transcripts; ignore Metadata/UtteranceEnd/etc.
|
||||
if msg.Type != "" && msg.Type != "Results" {
|
||||
return "", false, false
|
||||
}
|
||||
if len(msg.Channel.Alternatives) == 0 {
|
||||
return "", false, false
|
||||
}
|
||||
return strings.TrimSpace(msg.Channel.Alternatives[0].Transcript), msg.IsFinal, true
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// LocalConfig configures the offline sherpa-onnx batch transcriber. Every tier
|
||||
// (default/standard/high-end, §7) is just a different ModelPath — there are no
|
||||
// tier-specific code paths; the model family is detected from the files present.
|
||||
type LocalConfig struct {
|
||||
// Binary is the sherpa-onnx-offline executable; looked up on PATH when a
|
||||
// bare name. Empty defaults to "sherpa-onnx-offline".
|
||||
Binary string
|
||||
// ModelPath is the directory holding the model's .onnx + tokens.txt files
|
||||
// (stt.localModelPath).
|
||||
ModelPath string
|
||||
NumThreads int
|
||||
|
||||
// Test seams.
|
||||
runOutput commandOutputRunner
|
||||
tempDir func() (string, error)
|
||||
}
|
||||
|
||||
type localTranscriber struct {
|
||||
cfg LocalConfig
|
||||
}
|
||||
|
||||
// NewLocalTranscriber builds the offline transcriber. It does not touch the
|
||||
// filesystem or PATH until Transcribe runs, so a missing binary/model degrades
|
||||
// to a clear setup message at use time (matching internal/lsp).
|
||||
func NewLocalTranscriber(cfg LocalConfig) (Transcriber, error) {
|
||||
if strings.TrimSpace(cfg.ModelPath) == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: "local STT model",
|
||||
Hint: "set stt.localModelPath to a sherpa-onnx model directory (see docs/dictation.md)",
|
||||
}
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = "sherpa-onnx-offline"
|
||||
}
|
||||
if cfg.runOutput == nil {
|
||||
cfg.runOutput = runCommandOutput
|
||||
}
|
||||
if cfg.tempDir == nil {
|
||||
cfg.tempDir = audioSpillDir
|
||||
}
|
||||
return &localTranscriber{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (l *localTranscriber) Transcribe(ctx context.Context, audio []byte) (string, error) {
|
||||
if len(audio) == 0 {
|
||||
return "", errors.New("no audio to transcribe")
|
||||
}
|
||||
modelArgs, err := detectModelArgs(l.cfg.ModelPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
dir, err := l.cfg.tempDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("preparing dictation temp dir: %w", err)
|
||||
}
|
||||
ext := ".wav"
|
||||
if SniffFormat(audio) == FormatM4A {
|
||||
ext = ".m4a"
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, "stt-*"+ext)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("staging audio for transcription: %w", err)
|
||||
}
|
||||
path := tmp.Name()
|
||||
defer os.Remove(path) // audio never lingers on disk (§12)
|
||||
if _, err := tmp.Write(audio); err != nil {
|
||||
tmp.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
args := make([]string, 0, len(modelArgs)+3)
|
||||
args = append(args, modelArgs...)
|
||||
if l.cfg.NumThreads > 0 {
|
||||
args = append(args, "--num-threads="+strconv.Itoa(l.cfg.NumThreads))
|
||||
}
|
||||
args = append(args, path)
|
||||
|
||||
// Fail fast if already cancelled; ctx also propagates into the exec below so a
|
||||
// wedged sherpa-onnx-offline is killed on cancel/timeout rather than hanging.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := l.cfg.runOutput(ctx, l.cfg.Binary, args...)
|
||||
if err != nil {
|
||||
if errors.Is(err, exec.ErrNotFound) {
|
||||
return "", &SetupError{
|
||||
Tool: l.cfg.Binary,
|
||||
Hint: "install sherpa-onnx and put its offline binary on PATH (see docs/dictation.md)",
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("sherpa-onnx transcription failed: %w (%s)", err, lastLines(out, 3))
|
||||
}
|
||||
text := parseSherpaOutput(out)
|
||||
if text == "" {
|
||||
return "", fmt.Errorf("sherpa-onnx produced no transcript (output: %s)", lastLines(out, 3))
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// detectModelArgs inspects a sherpa-onnx model directory and builds the
|
||||
// family-specific --*=path flags. Supports the families sherpa-onnx-offline
|
||||
// accepts; Moonshine (the design's default/standard/high-end tiers) is the
|
||||
// primary target. tokens.txt is required by every family.
|
||||
func detectModelArgs(dir string) ([]string, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, &SetupError{
|
||||
Tool: "local STT model",
|
||||
Hint: fmt.Sprintf("stt.localModelPath %q is not readable: %v", dir, err),
|
||||
}
|
||||
}
|
||||
files := map[string]string{} // lowercased basename -> absolute path
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
files[strings.ToLower(e.Name())] = filepath.Join(dir, e.Name())
|
||||
}
|
||||
|
||||
tokens := findTokensFile(files)
|
||||
if tokens == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: "local STT model",
|
||||
Hint: fmt.Sprintf("no tokens.txt found in stt.localModelPath %q — is it a sherpa-onnx model directory?", dir),
|
||||
}
|
||||
}
|
||||
|
||||
dirName := strings.ToLower(filepath.Base(dir))
|
||||
|
||||
// Order matters — check the most specific signatures first so a model is
|
||||
// never mis-classified as a family it only partially resembles.
|
||||
|
||||
// Moonshine: preprocessor + encoder + cached/uncached decoders. Its files are
|
||||
// "encode"/"decode" (no trailing "r"), so they never collide with the
|
||||
// "encoder"/"decoder" families below.
|
||||
if pre := findContains(files, "preprocess"); pre != "" {
|
||||
enc := findContains(files, "encode")
|
||||
// "uncached_decode" CONTAINS "cached_decode", so match the cached decoder
|
||||
// as "cached" AND not "uncached" to disambiguate (map order is random).
|
||||
unc := findOnnxMatching(files, "uncached", "")
|
||||
cac := findOnnxMatching(files, "cached", "uncached")
|
||||
if enc != "" && unc != "" && cac != "" {
|
||||
return []string{
|
||||
"--moonshine-preprocessor=" + pre,
|
||||
"--moonshine-encoder=" + enc,
|
||||
"--moonshine-uncached-decoder=" + unc,
|
||||
"--moonshine-cached-decoder=" + cac,
|
||||
"--tokens=" + tokens,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Transducer (Zipformer/Conformer): encoder + decoder + joiner, matched as a
|
||||
// consistent int8-or-fp32 set (mixing quantizations is invalid).
|
||||
if set := consistentQuantizedSet(files, "encoder", "decoder", "joiner"); set != nil {
|
||||
return []string{"--encoder=" + set[0], "--decoder=" + set[1], "--joiner=" + set[2], "--tokens=" + tokens}, nil
|
||||
}
|
||||
|
||||
// Encoder+decoder families that share Whisper's file shape (encoder + decoder,
|
||||
// no joiner) but need their own flags — FireRedASR-AED, Canary, Cohere. They are
|
||||
// indistinguishable from Whisper by files alone, so disambiguate by name FIRST;
|
||||
// anything unnamed falls through to the Whisper default below.
|
||||
for _, fam := range encoderDecoderFamilies {
|
||||
if nameMatchesFamily(dirName, files, fam.match) {
|
||||
if set := consistentQuantizedSet(files, "encoder", "decoder"); set != nil {
|
||||
return []string{fam.encoderFlag + "=" + set[0], fam.decoderFlag + "=" + set[1], "--tokens=" + tokens}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Whisper: encoder + decoder, NO joiner. Sherpa's Whisper files are named
|
||||
// "<model>-encoder.onnx" / "<model>-decoder.onnx" (NOT "whisper-encoder"), so
|
||||
// match on the encoder/decoder roles — a consistent pair — after transducer
|
||||
// (which owns the encoder/decoder/joiner shape) has already been ruled out.
|
||||
if set := consistentQuantizedSet(files, "encoder", "decoder"); set != nil {
|
||||
return []string{"--whisper-encoder=" + set[0], "--whisper-decoder=" + set[1], "--tokens=" + tokens}, nil
|
||||
}
|
||||
|
||||
// Single-model families: one generic model .onnx + tokens, indistinguishable by
|
||||
// file shape, so they are told apart by the model directory/file name. This
|
||||
// covers SenseVoice, Paraformer, and the CTC families (FireRedASR-CTC, NeMo,
|
||||
// TeleSpeech, WeNet, Zipformer-CTC, Dolphin, Omnilingual). Ordered most-specific
|
||||
// first so a compound name (e.g. "fire-red-asr2-ctc") matches its own family
|
||||
// before a generic token in it (e.g. bare "ctc") could steer it elsewhere.
|
||||
if m := singleModelOnnx(files); m != "" {
|
||||
for _, fam := range singleModelFamilies {
|
||||
if nameMatchesFamily(dirName, files, fam.match) {
|
||||
return []string{fam.flag + "=" + m, "--tokens=" + tokens}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, &SetupError{
|
||||
Tool: "local STT model",
|
||||
Hint: fmt.Sprintf("could not recognize the model family in %q (supported: Moonshine, Whisper, transducer/Zipformer, Paraformer, SenseVoice, and the CTC families FireRedASR/NeMo/TeleSpeech/WeNet/Zipformer-CTC/Dolphin/Omnilingual). See docs/dictation.md", dir),
|
||||
}
|
||||
}
|
||||
|
||||
// singleModelFamilies maps a model's directory/file-name signature to the
|
||||
// sherpa-onnx-offline flag for that single-file family. Each family ships exactly
|
||||
// one generic model .onnx (+ tokens), so only the name tells them apart. Ordered
|
||||
// so a specific compound name wins before a broader one.
|
||||
var singleModelFamilies = []struct {
|
||||
match []string // name substrings (any hit); most-specific spellings first
|
||||
flag string
|
||||
}{
|
||||
{[]string{"fire-red-asr2-ctc", "fire-red-asr-ctc", "fire-red-asr", "firered"}, "--fire-red-asr-ctc"},
|
||||
{[]string{"nemo-ctc", "nemo_ctc"}, "--nemo-ctc-model"},
|
||||
{[]string{"zipformer2-ctc", "zipformer-ctc"}, "--zipformer-ctc-model"},
|
||||
{[]string{"telespeech"}, "--telespeech-ctc"},
|
||||
{[]string{"wenet"}, "--wenet-ctc-model"},
|
||||
{[]string{"dolphin"}, "--dolphin-model"},
|
||||
{[]string{"omnilingual"}, "--omnilingual-asr-model"},
|
||||
{[]string{"medasr"}, "--medasr"},
|
||||
{[]string{"tdnn"}, "--tdnn-model"},
|
||||
{[]string{"sense-voice", "sensevoice"}, "--sense-voice-model"},
|
||||
{[]string{"paraformer"}, "--paraformer"},
|
||||
}
|
||||
|
||||
// encoderDecoderFamilies are offline families shaped like Whisper (an encoder +
|
||||
// decoder .onnx, no joiner) but needing their own flags; matched by name before
|
||||
// the Whisper fallback claims the shape.
|
||||
var encoderDecoderFamilies = []struct {
|
||||
match []string
|
||||
encoderFlag string
|
||||
decoderFlag string
|
||||
}{
|
||||
{[]string{"fire-red-asr"}, "--fire-red-asr-encoder", "--fire-red-asr-decoder"},
|
||||
{[]string{"canary"}, "--canary-encoder", "--canary-decoder"},
|
||||
{[]string{"cohere"}, "--cohere-transcribe-encoder", "--cohere-transcribe-decoder"},
|
||||
}
|
||||
|
||||
// nameMatchesFamily reports whether the model directory name or any of its .onnx
|
||||
// filenames contain one of the family's signature substrings.
|
||||
func nameMatchesFamily(dirName string, files map[string]string, subs []string) bool {
|
||||
for _, sub := range subs {
|
||||
if strings.Contains(dirName, sub) {
|
||||
return true
|
||||
}
|
||||
for name := range files {
|
||||
if strings.HasSuffix(name, ".onnx") && strings.Contains(name, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// consistentQuantizedSet returns one file per role (in order), preferring an
|
||||
// int8 build of each but falling back to fp32 when a role has no int8 file.
|
||||
// Returns nil when any role is entirely missing.
|
||||
//
|
||||
// Models legitimately SHIP mixed quantization — e.g. the x-asr streaming
|
||||
// zipformers ship an int8 encoder and joiner with an fp32 decoder — and
|
||||
// sherpa-onnx accepts that. So this must NOT require every role to share one
|
||||
// quantization; it just needs one file per role, favoring the smaller int8 when
|
||||
// a choice exists. (When a model ships both int8 and fp32 for every role, this
|
||||
// still picks all-int8, since int8 is available for each.)
|
||||
func consistentQuantizedSet(files map[string]string, roles ...string) []string {
|
||||
pick := func(role string, int8Only bool) string {
|
||||
for name, path := range files {
|
||||
if !strings.HasSuffix(name, ".onnx") || !strings.Contains(name, role) {
|
||||
continue
|
||||
}
|
||||
if int8Only && !strings.Contains(name, "int8") {
|
||||
continue
|
||||
}
|
||||
return path
|
||||
}
|
||||
return ""
|
||||
}
|
||||
out := make([]string, len(roles))
|
||||
for i, role := range roles {
|
||||
p := pick(role, true) // prefer int8 (smaller, faster)
|
||||
if p == "" {
|
||||
p = pick(role, false) // fall back to fp32 for this role
|
||||
}
|
||||
if p == "" {
|
||||
return nil // this role is genuinely absent
|
||||
}
|
||||
out[i] = p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// findTokensFile returns the model's tokens file. It is usually "tokens.txt",
|
||||
// but some families (e.g. Whisper) prefix it as "<model>-tokens.txt".
|
||||
func findTokensFile(files map[string]string) string {
|
||||
if p := files["tokens.txt"]; p != "" {
|
||||
return p
|
||||
}
|
||||
for name, path := range files {
|
||||
if strings.HasSuffix(name, "tokens.txt") {
|
||||
return path
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// singleModelOnnx returns the model's single .onnx file (preferring an int8
|
||||
// build), for families like SenseVoice/Paraformer that ship exactly one.
|
||||
func singleModelOnnx(files map[string]string) string {
|
||||
best := ""
|
||||
for name, path := range files {
|
||||
if !strings.HasSuffix(name, ".onnx") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(name, "int8") {
|
||||
return path
|
||||
}
|
||||
best = path
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// findOnnxMatching returns an .onnx file whose name contains `must` and, when
|
||||
// `exclude` is non-empty, does NOT contain `exclude` — used to tell the cached
|
||||
// from the uncached Moonshine decoder.
|
||||
func findOnnxMatching(files map[string]string, must, exclude string) string {
|
||||
for name, path := range files {
|
||||
if !strings.HasSuffix(name, ".onnx") || !strings.Contains(name, must) {
|
||||
continue
|
||||
}
|
||||
if exclude != "" && strings.Contains(name, exclude) {
|
||||
continue
|
||||
}
|
||||
return path
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func findContains(files map[string]string, substr string) string {
|
||||
for name, path := range files {
|
||||
if strings.HasSuffix(name, ".onnx") && strings.Contains(name, substr) {
|
||||
return path
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseSherpaOutput extracts the transcript from sherpa-onnx-offline's output.
|
||||
// The binary prints a JSON object with a "text" field (amid diagnostic lines);
|
||||
// scan for the first line that parses as such.
|
||||
func parseSherpaOutput(out []byte) string {
|
||||
for _, line := range bytes.Split(out, []byte("\n")) {
|
||||
trimmed := bytes.TrimSpace(line)
|
||||
if len(trimmed) == 0 || trimmed[0] != '{' {
|
||||
continue
|
||||
}
|
||||
var parsed struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := json.Unmarshal(trimmed, &parsed); err == nil && strings.TrimSpace(parsed.Text) != "" {
|
||||
return strings.TrimSpace(parsed.Text)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func lastLines(out []byte, n int) string {
|
||||
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, " | ")
|
||||
}
|
||||
|
||||
// StreamTranscribe is unsupported for the offline binary; local streaming uses
|
||||
// the sherpa-onnx websocket server transcriber instead.
|
||||
func (l *localTranscriber) StreamTranscribe(context.Context, <-chan []byte, func(string, bool)) (string, error) {
|
||||
return "", ErrStreamingUnsupported
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// localStreamingTranscriber streams audio to the warm sherpa-onnx websocket
|
||||
// server (§6a). The server keeps encoder state across chunks (a true streaming
|
||||
// transducer), so partials arrive as decoding proceeds — no redundant
|
||||
// re-inference.
|
||||
//
|
||||
// Wire protocol (verified against sherpa-onnx's reference client, NOT the
|
||||
// plan's "16-bit PCM" description, which was imprecise): the client sends
|
||||
// binary frames of little-endian float32 samples normalized to [-1, 1]
|
||||
// (int16/32768), then the text "Done" to end; the server replies with JSON
|
||||
// result messages and finally the text "Done!" to signal completion.
|
||||
type localStreamingTranscriber struct {
|
||||
manager *ServerManager
|
||||
}
|
||||
|
||||
// NewLocalStreamingTranscriber builds a streaming transcriber backed by the
|
||||
// shared, session-long sherpa-onnx server manager.
|
||||
func NewLocalStreamingTranscriber(manager *ServerManager) Transcriber {
|
||||
return &localStreamingTranscriber{manager: manager}
|
||||
}
|
||||
|
||||
// Transcribe is not this transcriber's job — the streaming path uses
|
||||
// StreamTranscribe; the batch/Termux fallback uses the offline transcriber.
|
||||
func (l *localStreamingTranscriber) Transcribe(context.Context, []byte) (string, error) {
|
||||
return "", errors.New("local streaming transcriber does not support batch transcription")
|
||||
}
|
||||
|
||||
func (l *localStreamingTranscriber) StreamTranscribe(ctx context.Context, chunks <-chan []byte, onPartial func(string, bool)) (string, error) {
|
||||
url, err := l.manager.EnsureRunning(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conn, err := dialWithOneRestart(ctx, l.manager, url)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("connecting to sherpa-onnx server: %w", err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
|
||||
// Writer goroutine: forward converted chunks, then signal end. A write error
|
||||
// (server crashed) cancels the read side via writeErr.
|
||||
writeErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
for chunk := range chunks {
|
||||
if err := conn.Write(ctx, websocket.MessageBinary, pcm16ToFloat32LE(chunk)); err != nil {
|
||||
writeErrCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
writeErrCh <- conn.Write(ctx, websocket.MessageText, []byte("Done"))
|
||||
}()
|
||||
|
||||
// The server reports "text" PER SEGMENT, not cumulatively for the whole
|
||||
// utterance: when it detects an endpoint (a pause between phrases, or the
|
||||
// trailing silence when recording stops) it finalizes the current segment, bumps
|
||||
// "segment", and resets "text" to "" for the next one. So we accumulate the text
|
||||
// of every segment — otherwise a new segment's (often empty) result would
|
||||
// overwrite everything already transcribed, and the transcript would vanish on
|
||||
// stop. Displayed/returned text is all segments joined in order.
|
||||
var segmentTexts []string
|
||||
joinSegments := func() string {
|
||||
parts := make([]string, 0, len(segmentTexts))
|
||||
for _, s := range segmentTexts {
|
||||
if t := strings.TrimSpace(s); t != "" {
|
||||
parts = append(parts, t)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
lastText := ""
|
||||
for {
|
||||
typ, data, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
// Mid-stream failure: return the best-effort text accumulated so far
|
||||
// plus the error — already-streamed partials are never discarded (§6).
|
||||
select {
|
||||
case werr := <-writeErrCh:
|
||||
if werr != nil {
|
||||
err = werr
|
||||
}
|
||||
default:
|
||||
}
|
||||
return lastText, fmt.Errorf("sherpa-onnx stream error: %w", err)
|
||||
}
|
||||
if typ == websocket.MessageText && strings.TrimSpace(string(data)) == "Done!" {
|
||||
break
|
||||
}
|
||||
text, segment, final, ok := parseSherpaStreamResult(data)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if segment < 0 {
|
||||
segment = 0
|
||||
}
|
||||
for len(segmentTexts) <= segment {
|
||||
segmentTexts = append(segmentTexts, "")
|
||||
}
|
||||
segmentTexts[segment] = text
|
||||
lastText = joinSegments()
|
||||
if onPartial != nil {
|
||||
onPartial(lastText, final)
|
||||
}
|
||||
}
|
||||
_ = conn.Close(websocket.StatusNormalClosure, "")
|
||||
return lastText, nil
|
||||
}
|
||||
|
||||
// dialWithOneRestart dials the server, and on failure restarts it once and
|
||||
// retries — mirroring the LSP manager's single crashed-server recovery (§6/§6a).
|
||||
func dialWithOneRestart(ctx context.Context, manager *ServerManager, url string) (*websocket.Conn, error) {
|
||||
conn, _, err := websocket.Dial(ctx, url, nil)
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
// The warm server may have died between EnsureRunning and Dial; reap and
|
||||
// restart once.
|
||||
_ = manager.Shutdown(ctx)
|
||||
url, rerr := manager.EnsureRunning(ctx)
|
||||
if rerr != nil {
|
||||
return nil, rerr
|
||||
}
|
||||
conn, _, err = websocket.Dial(ctx, url, nil)
|
||||
return conn, err
|
||||
}
|
||||
|
||||
// parseSherpaStreamResult extracts one server JSON result. "text" is the
|
||||
// hypothesis for the CURRENT segment (segment index in "segment"), which resets
|
||||
// each time the recognizer endpoints — so the caller must accumulate per segment,
|
||||
// not treat "text" as the whole-utterance transcript. "is_final" marks a settled
|
||||
// segment.
|
||||
func parseSherpaStreamResult(data []byte) (text string, segment int, final bool, ok bool) {
|
||||
var parsed struct {
|
||||
Text string `json:"text"`
|
||||
Segment int `json:"segment"`
|
||||
IsFinal bool `json:"is_final"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &parsed); err != nil {
|
||||
return "", 0, false, false
|
||||
}
|
||||
return strings.TrimSpace(parsed.Text), parsed.Segment, parsed.IsFinal, true
|
||||
}
|
||||
|
||||
// pcm16ToFloat32LE converts a little-endian int16 PCM chunk to the little-endian
|
||||
// float32 samples (normalized to [-1, 1]) the sherpa-onnx server expects.
|
||||
func pcm16ToFloat32LE(pcm []byte) []byte {
|
||||
n := len(pcm) / 2
|
||||
out := make([]byte, n*4)
|
||||
for i := 0; i < n; i++ {
|
||||
s := int16(uint16(pcm[2*i]) | uint16(pcm[2*i+1])<<8)
|
||||
bits := math.Float32bits(float32(s) / 32768.0)
|
||||
binary.LittleEndian.PutUint32(out[4*i:], bits)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeModelDir(t *testing.T, files ...string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for _, f := range files {
|
||||
if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestLocalTranscribeMoonshineArgs(t *testing.T) {
|
||||
dir := writeModelDir(t,
|
||||
"preprocess.onnx", "encode.onnx", "uncached_decode.onnx", "cached_decode.onnx", "tokens.txt")
|
||||
var gotBin string
|
||||
var gotArgs []string
|
||||
runOutput := func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
gotBin, gotArgs = name, args
|
||||
return []byte("Started\nDone!\n\n/tmp/a.wav\n{\"text\":\" transcribed speech \"}\n----"), nil
|
||||
}
|
||||
tr, err := NewLocalTranscriber(LocalConfig{ModelPath: dir, runOutput: runOutput, tempDir: func() (string, error) { return t.TempDir(), nil }})
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalTranscriber: %v", err)
|
||||
}
|
||||
text, err := tr.Transcribe(context.Background(), wavFixture())
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if text != "transcribed speech" {
|
||||
t.Errorf("text = %q", text)
|
||||
}
|
||||
if gotBin != "sherpa-onnx-offline" {
|
||||
t.Errorf("binary = %q", gotBin)
|
||||
}
|
||||
joined := strings.Join(gotArgs, " ")
|
||||
for _, want := range []string{"--moonshine-preprocessor=", "--moonshine-encoder=", "--moonshine-uncached-decoder=", "--moonshine-cached-decoder=", "--tokens="} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("missing flag %q in %q", want, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalTranscribeTransducerArgs(t *testing.T) {
|
||||
dir := writeModelDir(t, "encoder-epoch-99.onnx", "decoder-epoch-99.onnx", "joiner-epoch-99.onnx", "tokens.txt")
|
||||
var gotArgs []string
|
||||
runOutput := func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
gotArgs = args
|
||||
return []byte(`{"text":"hi"}`), nil
|
||||
}
|
||||
tr, _ := NewLocalTranscriber(LocalConfig{ModelPath: dir, runOutput: runOutput, tempDir: func() (string, error) { return t.TempDir(), nil }})
|
||||
if _, err := tr.Transcribe(context.Background(), wavFixture()); err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
joined := strings.Join(gotArgs, " ")
|
||||
for _, want := range []string{"--encoder=", "--decoder=", "--joiner=", "--tokens="} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("missing flag %q in %q", want, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectWhisperArgs(t *testing.T) {
|
||||
// Sherpa Whisper models are named "<model>-encoder.onnx" (NOT "whisper-encoder")
|
||||
// and prefix the tokens file as "<model>-tokens.txt".
|
||||
dir := writeModelDir(t, "tiny.en-encoder.onnx", "tiny.en-encoder.int8.onnx",
|
||||
"tiny.en-decoder.onnx", "tiny.en-decoder.int8.onnx", "tiny.en-tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Whisper detection failed: %v", err)
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, "--whisper-encoder=") || !strings.Contains(joined, "--whisper-decoder=") {
|
||||
t.Errorf("expected whisper flags, got %q", joined)
|
||||
}
|
||||
// Must not emit transducer flags (Whisper has no joiner).
|
||||
if strings.Contains(joined, "--joiner=") || strings.Contains(joined, "--encoder=") {
|
||||
t.Errorf("Whisper must not be detected as a transducer: %q", joined)
|
||||
}
|
||||
// Consistent quantization: both int8 (preferred) or both fp32, never mixed.
|
||||
if strings.Contains(joined, "encoder.int8.onnx") != strings.Contains(joined, "decoder.int8.onnx") {
|
||||
t.Errorf("mixed encoder/decoder quantization: %q", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectTransducerPrefersConsistentInt8(t *testing.T) {
|
||||
dir := writeModelDir(t,
|
||||
"encoder-epoch-99.onnx", "encoder-epoch-99.int8.onnx",
|
||||
"decoder-epoch-99.onnx", "decoder-epoch-99.int8.onnx",
|
||||
"joiner-epoch-99.onnx", "joiner-epoch-99.int8.onnx", "tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
// All three int8 (preferred), consistently.
|
||||
for _, want := range []string{"--encoder=", "encoder-epoch-99.int8.onnx", "joiner-epoch-99.int8.onnx"} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("missing %q in %q", want, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectTransducerMixedQuantization(t *testing.T) {
|
||||
// Some models ship an int8 encoder+joiner with an fp32 decoder (e.g. the
|
||||
// x-asr streaming zipformers). Detection must accept that mix, not reject it.
|
||||
dir := writeModelDir(t, "encoder.int8.onnx", "decoder.onnx", "joiner.int8.onnx", "tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("mixed-quantization transducer should be detected, got %v", err)
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, "encoder.int8.onnx") || !strings.Contains(joined, "decoder.onnx") || !strings.Contains(joined, "joiner.int8.onnx") {
|
||||
t.Errorf("expected the actual mixed files, got %q", joined)
|
||||
}
|
||||
// The fp32 decoder must not be mistaken for an int8 one.
|
||||
if strings.Contains(joined, "decoder.int8") {
|
||||
t.Errorf("no int8 decoder exists; should use decoder.onnx: %q", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectSenseVoiceByDirName(t *testing.T) {
|
||||
dir := t.TempDir() // rename won't help; use a dir whose base contains sense-voice
|
||||
sv := dir + "/sherpa-onnx-sense-voice-zh-en"
|
||||
if err := os.MkdirAll(sv, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range []string{"model.int8.onnx", "tokens.txt"} {
|
||||
os.WriteFile(sv+"/"+f, []byte("x"), 0o600)
|
||||
}
|
||||
args, err := detectModelArgs(sv)
|
||||
if err != nil {
|
||||
t.Fatalf("SenseVoice detection failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(strings.Join(args, " "), "--sense-voice-model=") {
|
||||
t.Errorf("expected sense-voice flag, got %v", args)
|
||||
}
|
||||
}
|
||||
|
||||
// writeNamedModelDir creates a model directory whose base name carries the family
|
||||
// signature (how sherpa-onnx distributes single-file / CTC models), with the given
|
||||
// files, and returns its path.
|
||||
func writeNamedModelDir(t *testing.T, base string, files ...string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir() + "/" + base
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
if err := os.WriteFile(dir+"/"+f, []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestDetectSingleFileCTCFamilies(t *testing.T) {
|
||||
// One generic model.onnx + tokens, family told apart by the directory name.
|
||||
cases := []struct{ base, wantFlag string }{
|
||||
{"sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25", "--fire-red-asr-ctc="},
|
||||
{"sherpa-onnx-nemo-ctc-en-conformer-medium", "--nemo-ctc-model="},
|
||||
{"sherpa-onnx-zipformer-ctc-zh-2024", "--zipformer-ctc-model="},
|
||||
{"sherpa-onnx-telespeech-ctc-int8", "--telespeech-ctc="},
|
||||
{"sherpa-onnx-wenet-ctc-en", "--wenet-ctc-model="},
|
||||
{"sherpa-onnx-dolphin-base-ctc-multi-lang", "--dolphin-model="},
|
||||
{"sherpa-onnx-omnilingual-asr-ctc", "--omnilingual-asr-model="},
|
||||
{"sherpa-onnx-tdnn-yesno", "--tdnn-model="},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
dir := writeNamedModelDir(t, tc.base, "model.int8.onnx", "tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Errorf("%s: detection failed: %v", tc.base, err)
|
||||
continue
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, tc.wantFlag) || !strings.Contains(joined, "--tokens=") {
|
||||
t.Errorf("%s: got %q, want %s", tc.base, joined, tc.wantFlag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectEncoderDecoderFamiliesBeforeWhisper(t *testing.T) {
|
||||
// FireRedASR-AED / Canary share Whisper's encoder+decoder shape, so they must be
|
||||
// disambiguated by name — not fall through to --whisper-encoder.
|
||||
cases := []struct{ base, wantFlag string }{
|
||||
{"sherpa-onnx-fire-red-asr-large-zh_en", "--fire-red-asr-encoder="},
|
||||
{"sherpa-onnx-canary-180m-flash", "--canary-encoder="},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
dir := writeNamedModelDir(t, tc.base, "encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt")
|
||||
args, err := detectModelArgs(dir)
|
||||
if err != nil {
|
||||
t.Errorf("%s: detection failed: %v", tc.base, err)
|
||||
continue
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, tc.wantFlag) {
|
||||
t.Errorf("%s: got %q, want %s", tc.base, joined, tc.wantFlag)
|
||||
}
|
||||
if strings.Contains(joined, "--whisper-") {
|
||||
t.Errorf("%s must not be detected as Whisper: %q", tc.base, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectUnknownFamilyFailsCleanly(t *testing.T) {
|
||||
// A model with tokens but no recognizable family → a clear SetupError, not a
|
||||
// crash or wrong flags.
|
||||
dir := writeModelDir(t, "some-random-model.onnx", "tokens.txt")
|
||||
_, err := detectModelArgs(dir)
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want a clean *SetupError, got %v", err)
|
||||
}
|
||||
if !strings.Contains(setupErr.Error(), "could not recognize") {
|
||||
t.Errorf("expected a recognizable failure message, got %q", setupErr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalTranscribeMissingTokens(t *testing.T) {
|
||||
dir := writeModelDir(t, "encoder.onnx", "decoder.onnx", "joiner.onnx") // no tokens.txt
|
||||
tr, _ := NewLocalTranscriber(LocalConfig{ModelPath: dir, runOutput: func(context.Context, string, ...string) ([]byte, error) { return nil, nil }, tempDir: func() (string, error) { return t.TempDir(), nil }})
|
||||
_, err := tr.Transcribe(context.Background(), wavFixture())
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing tokens, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalTranscribeMissingBinary(t *testing.T) {
|
||||
dir := writeModelDir(t, "preprocess.onnx", "encode.onnx", "uncached_decode.onnx", "cached_decode.onnx", "tokens.txt")
|
||||
runOutput := func(context.Context, string, ...string) ([]byte, error) { return nil, exec.ErrNotFound }
|
||||
tr, _ := NewLocalTranscriber(LocalConfig{ModelPath: dir, runOutput: runOutput, tempDir: func() (string, error) { return t.TempDir(), nil }})
|
||||
_, err := tr.Transcribe(context.Background(), wavFixture())
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError for missing binary, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSherpaOutput(t *testing.T) {
|
||||
out := []byte("Started\nDone!\n\naudio.wav\n{\"lang\":\"\",\"text\":\" the quick brown fox \",\"tokens\":[]}\n----\nnum threads: 2")
|
||||
if got := parseSherpaOutput(out); got != "the quick brown fox" {
|
||||
t.Errorf("parsed %q", got)
|
||||
}
|
||||
if got := parseSherpaOutput([]byte("no json here")); got != "" {
|
||||
t.Errorf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLocalTranscriberRequiresModelPath(t *testing.T) {
|
||||
_, err := NewLocalTranscriber(LocalConfig{})
|
||||
var setupErr *SetupError
|
||||
if !errors.As(err, &setupErr) {
|
||||
t.Fatalf("want *SetupError, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// OpenAI Realtime transcription-only wants 24kHz PCM16 (§6b); the recorder is
|
||||
// built to match via the SampleRate hint.
|
||||
const openAIRealtimeSampleRate = 24000
|
||||
|
||||
// OpenAIRealtimeConfig configures the OpenAI Realtime transcription transcriber
|
||||
// (§6b, the credential-reuse streaming alternative). Reuses the OpenAI key.
|
||||
type OpenAIRealtimeConfig struct {
|
||||
APIKey string
|
||||
Model string // default "gpt-4o-transcribe"
|
||||
// BaseURL overrides the wss endpoint (tests point it at a fake server).
|
||||
BaseURL string
|
||||
}
|
||||
|
||||
type openAIRealtimeTranscriber struct {
|
||||
cfg OpenAIRealtimeConfig
|
||||
}
|
||||
|
||||
// NewOpenAIRealtimeTranscriber builds the OpenAI Realtime streaming transcriber.
|
||||
func NewOpenAIRealtimeTranscriber(cfg OpenAIRealtimeConfig) (Transcriber, error) {
|
||||
if strings.TrimSpace(cfg.APIKey) == "" {
|
||||
return nil, &SetupError{
|
||||
Tool: "OpenAI API key",
|
||||
Hint: "set an OpenAI API key (OPENAI_API_KEY, or `zero auth`) to use OpenAI Realtime streaming dictation",
|
||||
}
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
cfg.Model = "gpt-4o-transcribe"
|
||||
}
|
||||
if cfg.BaseURL == "" {
|
||||
cfg.BaseURL = "wss://api.openai.com/v1/realtime?intent=transcription"
|
||||
}
|
||||
return &openAIRealtimeTranscriber{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (o *openAIRealtimeTranscriber) Transcribe(context.Context, []byte) (string, error) {
|
||||
return "", ErrStreamingUnsupported
|
||||
}
|
||||
|
||||
func (o *openAIRealtimeTranscriber) SampleRate() int { return openAIRealtimeSampleRate }
|
||||
|
||||
func (o *openAIRealtimeTranscriber) StreamTranscribe(ctx context.Context, chunks <-chan []byte, onPartial func(string, bool)) (string, error) {
|
||||
conn, _, err := websocket.Dial(ctx, o.cfg.BaseURL, &websocket.DialOptions{
|
||||
HTTPHeader: http.Header{
|
||||
"Authorization": {"Bearer " + o.cfg.APIKey},
|
||||
"OpenAI-Beta": {"realtime=v1"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("connecting to OpenAI Realtime: %w", err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
|
||||
// Configure a transcription-only session (pcm16 in, gpt-4o-transcribe).
|
||||
sessionUpdate := map[string]any{
|
||||
"type": "transcription_session.update",
|
||||
"session": map[string]any{
|
||||
"input_audio_format": "pcm16",
|
||||
"input_audio_transcription": map[string]any{
|
||||
"model": o.cfg.Model,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := writeJSON(ctx, conn, sessionUpdate); err != nil {
|
||||
return "", fmt.Errorf("configuring OpenAI Realtime session: %w", err)
|
||||
}
|
||||
|
||||
writeErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
for chunk := range chunks {
|
||||
appendMsg := map[string]any{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.StdEncoding.EncodeToString(chunk),
|
||||
}
|
||||
if err := writeJSON(ctx, conn, appendMsg); err != nil {
|
||||
writeErrCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
// Commit the buffered audio to force final transcription of the utterance.
|
||||
writeErrCh <- writeJSON(ctx, conn, map[string]any{"type": "input_audio_buffer.commit"})
|
||||
}()
|
||||
|
||||
// OpenAI deltas are incremental (append), unlike Deepgram/sherpa's cumulative
|
||||
// text. Accumulate completed segments plus the in-progress delta buffer.
|
||||
var finalized, current strings.Builder
|
||||
compose := func() string {
|
||||
return strings.TrimSpace(finalized.String() + current.String())
|
||||
}
|
||||
committed := false
|
||||
for {
|
||||
typ, data, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
if websocket.CloseStatus(err) == websocket.StatusNormalClosure {
|
||||
return compose(), nil
|
||||
}
|
||||
select {
|
||||
case werr := <-writeErrCh:
|
||||
if werr != nil {
|
||||
err = werr
|
||||
}
|
||||
default:
|
||||
}
|
||||
return compose(), fmt.Errorf("OpenAI Realtime stream error: %w", err)
|
||||
}
|
||||
if typ != websocket.MessageText {
|
||||
continue
|
||||
}
|
||||
evt := parseRealtimeEvent(data)
|
||||
switch evt.kind {
|
||||
case realtimeDelta:
|
||||
current.WriteString(evt.text)
|
||||
if onPartial != nil {
|
||||
onPartial(compose(), false)
|
||||
}
|
||||
case realtimeCompleted:
|
||||
// A completed item replaces the in-progress delta buffer with the
|
||||
// server's authoritative transcript for that segment.
|
||||
if evt.text != "" {
|
||||
if finalized.Len() > 0 {
|
||||
finalized.WriteString(" ")
|
||||
}
|
||||
finalized.WriteString(strings.TrimSpace(evt.text))
|
||||
}
|
||||
current.Reset()
|
||||
if onPartial != nil {
|
||||
onPartial(compose(), true)
|
||||
}
|
||||
// Once we've committed the buffer (user stopped) and the server has
|
||||
// returned a completed transcription, the utterance is done.
|
||||
if committed {
|
||||
_ = conn.Close(websocket.StatusNormalClosure, "")
|
||||
return compose(), nil
|
||||
}
|
||||
case realtimeCommitted:
|
||||
committed = true
|
||||
case realtimeError:
|
||||
return compose(), fmt.Errorf("OpenAI Realtime error: %s", evt.text)
|
||||
}
|
||||
// The writer signals commit completion out of band; observe it so a
|
||||
// stop with no further audio still flips `committed`.
|
||||
select {
|
||||
case werr := <-writeErrCh:
|
||||
if werr == nil {
|
||||
committed = true
|
||||
}
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type realtimeEventKind int
|
||||
|
||||
const (
|
||||
realtimeOther realtimeEventKind = iota
|
||||
realtimeDelta
|
||||
realtimeCompleted
|
||||
realtimeCommitted
|
||||
realtimeError
|
||||
)
|
||||
|
||||
type realtimeEvent struct {
|
||||
kind realtimeEventKind
|
||||
text string
|
||||
}
|
||||
|
||||
func parseRealtimeEvent(data []byte) realtimeEvent {
|
||||
var msg struct {
|
||||
Type string `json:"type"`
|
||||
Delta string `json:"delta"`
|
||||
Transcript string `json:"transcript"`
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &msg); err != nil {
|
||||
return realtimeEvent{kind: realtimeOther}
|
||||
}
|
||||
switch msg.Type {
|
||||
case "conversation.item.input_audio_transcription.delta":
|
||||
return realtimeEvent{kind: realtimeDelta, text: msg.Delta}
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
return realtimeEvent{kind: realtimeCompleted, text: msg.Transcript}
|
||||
case "input_audio_buffer.committed":
|
||||
return realtimeEvent{kind: realtimeCommitted}
|
||||
case "error":
|
||||
return realtimeEvent{kind: realtimeError, text: msg.Error.Message}
|
||||
}
|
||||
return realtimeEvent{kind: realtimeOther}
|
||||
}
|
||||
|
||||
func writeJSON(ctx context.Context, conn *websocket.Conn, v any) error {
|
||||
data, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conn.Write(ctx, websocket.MessageText, data)
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package dictation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// wsTestServer runs handler over a real localhost websocket, returning a wss->ws
|
||||
// base URL the transcribers can dial. Exercising the real coder/websocket client
|
||||
// against a real server is the closest unit test to the wire protocol.
|
||||
func wsTestServer(t *testing.T, handler func(ctx context.Context, c *websocket.Conn)) string {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := websocket.Accept(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
handler(r.Context(), c)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return "ws" + strings.TrimPrefix(srv.URL, "http")
|
||||
}
|
||||
|
||||
func TestDeepgramStreamTranscribe(t *testing.T) {
|
||||
var gotAuth string
|
||||
url := wsTestServer(t, func(ctx context.Context, c *websocket.Conn) {
|
||||
defer c.Close(websocket.StatusNormalClosure, "")
|
||||
// Read audio frames until CloseStream, then emit interim + final results.
|
||||
for {
|
||||
typ, data, err := c.Read(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if typ == websocket.MessageText && strings.Contains(string(data), "CloseStream") {
|
||||
break
|
||||
}
|
||||
}
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"Results","is_final":false,"channel":{"alternatives":[{"transcript":"hello"}]}}`))
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"Results","is_final":true,"channel":{"alternatives":[{"transcript":"hello world"}]}}`))
|
||||
})
|
||||
|
||||
tr, err := NewDeepgramTranscriber(DeepgramConfig{APIKey: "dg-key", BaseURL: url})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = gotAuth
|
||||
|
||||
var partials []string
|
||||
chunks := make(chan []byte, 2)
|
||||
chunks <- make([]byte, 320)
|
||||
close(chunks)
|
||||
final, err := tr.StreamTranscribe(context.Background(), chunks, func(text string, _ bool) {
|
||||
partials = append(partials, text)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTranscribe: %v", err)
|
||||
}
|
||||
if final != "hello world" {
|
||||
t.Errorf("final = %q, want 'hello world'", final)
|
||||
}
|
||||
if len(partials) == 0 || partials[0] != "hello" {
|
||||
t.Errorf("expected an interim 'hello' partial, got %v", partials)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepgramSampleRateDefault(t *testing.T) {
|
||||
tr, _ := NewDeepgramTranscriber(DeepgramConfig{APIKey: "k"})
|
||||
if RequiredSampleRate(tr) != DefaultSampleRate {
|
||||
t.Errorf("Deepgram sample rate = %d, want %d", RequiredSampleRate(tr), DefaultSampleRate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepgramMissingKey(t *testing.T) {
|
||||
if _, err := NewDeepgramTranscriber(DeepgramConfig{}); err == nil {
|
||||
t.Fatal("expected setup error for missing key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIRealtimeStreamTranscribe(t *testing.T) {
|
||||
url := wsTestServer(t, func(ctx context.Context, c *websocket.Conn) {
|
||||
defer c.Close(websocket.StatusNormalClosure, "")
|
||||
committed := false
|
||||
for {
|
||||
typ, data, err := c.Read(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if typ != websocket.MessageText {
|
||||
continue
|
||||
}
|
||||
var msg struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
_ = json.Unmarshal(data, &msg)
|
||||
switch msg.Type {
|
||||
case "transcription_session.update":
|
||||
// Emit incremental deltas as if audio were flowing.
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"conversation.item.input_audio_transcription.delta","delta":"the "}`))
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"conversation.item.input_audio_transcription.delta","delta":"answer"}`))
|
||||
case "input_audio_buffer.commit":
|
||||
committed = true
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"input_audio_buffer.committed"}`))
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(`{"type":"conversation.item.input_audio_transcription.completed","transcript":"the answer"}`))
|
||||
}
|
||||
_ = committed
|
||||
}
|
||||
})
|
||||
|
||||
tr, err := NewOpenAIRealtimeTranscriber(OpenAIRealtimeConfig{APIKey: "sk-key", BaseURL: url})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if RequiredSampleRate(tr) != openAIRealtimeSampleRate {
|
||||
t.Errorf("OpenAI Realtime sample rate = %d, want %d", RequiredSampleRate(tr), openAIRealtimeSampleRate)
|
||||
}
|
||||
|
||||
chunks := make(chan []byte, 1)
|
||||
chunks <- make([]byte, 480)
|
||||
close(chunks)
|
||||
done := make(chan struct{})
|
||||
var final string
|
||||
var ferr error
|
||||
go func() {
|
||||
final, ferr = tr.StreamTranscribe(context.Background(), chunks, func(string, bool) {})
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("StreamTranscribe did not complete")
|
||||
}
|
||||
if ferr != nil {
|
||||
t.Fatalf("StreamTranscribe: %v", ferr)
|
||||
}
|
||||
if final != "the answer" {
|
||||
t.Errorf("final = %q, want 'the answer'", final)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalStreamingAccumulatesSegments is the regression for the "text vanishes
|
||||
// on stop" bug: the sherpa server reports text PER SEGMENT and resets it to "" when
|
||||
// it endpoints, so the transcriber must accumulate across segments rather than let
|
||||
// a new (empty) segment overwrite everything already recognized.
|
||||
func TestLocalStreamingAccumulatesSegments(t *testing.T) {
|
||||
url := wsTestServer(t, func(ctx context.Context, c *websocket.Conn) {
|
||||
defer c.Close(websocket.StatusNormalClosure, "")
|
||||
go func() { // drain the client's audio + "Done"
|
||||
for {
|
||||
if _, _, err := c.Read(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, m := range []string{
|
||||
`{"text":" hello","segment":0,"is_final":false}`,
|
||||
`{"text":" hello world","segment":0,"is_final":true}`,
|
||||
`{"text":"","segment":1,"is_final":false}`, // endpoint reset — must NOT wipe seg 0
|
||||
`{"text":" foo bar","segment":1,"is_final":true}`,
|
||||
`{"text":"","segment":2,"is_final":true}`, // trailing empty segment at stop
|
||||
} {
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte(m))
|
||||
}
|
||||
_ = c.Write(ctx, websocket.MessageText, []byte("Done!"))
|
||||
})
|
||||
|
||||
colon := strings.LastIndex(url, ":")
|
||||
port, err := strconv.Atoi(url[colon+1:])
|
||||
if err != nil {
|
||||
t.Fatalf("parsing test server port from %q: %v", url, err)
|
||||
}
|
||||
mgr := NewServerManager(ServerConfig{
|
||||
ModelPath: streamingModelDir(t),
|
||||
Port: port,
|
||||
starter: func(commandSpec) (processHandle, io.ReadCloser, error) { return newFakeProcess(), nil, nil },
|
||||
healthCheck: func(context.Context, int) error { return nil },
|
||||
aliveCheck: func(int) bool { return false },
|
||||
})
|
||||
tr := NewLocalStreamingTranscriber(mgr)
|
||||
|
||||
chunks := make(chan []byte)
|
||||
close(chunks) // no audio; the writer immediately sends "Done"
|
||||
var partials []string
|
||||
text, err := tr.StreamTranscribe(context.Background(), chunks, func(s string, _ bool) {
|
||||
partials = append(partials, s)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamTranscribe: %v", err)
|
||||
}
|
||||
if text != "hello world foo bar" {
|
||||
t.Errorf("final text = %q, want %q", text, "hello world foo bar")
|
||||
}
|
||||
// Once text has appeared, no later partial may go empty — that would be the
|
||||
// segment-reset wiping the transcript, the exact bug this guards.
|
||||
seen := false
|
||||
for _, p := range partials {
|
||||
if p != "" {
|
||||
seen = true
|
||||
}
|
||||
if seen && p == "" {
|
||||
t.Errorf("transcript went empty after text appeared; partials=%v", partials)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDeepgramResultIgnoresNonResults(t *testing.T) {
|
||||
if _, _, ok := parseDeepgramResult([]byte(`{"type":"Metadata"}`)); ok {
|
||||
t.Error("Metadata messages should be ignored")
|
||||
}
|
||||
tr, final, ok := parseDeepgramResult([]byte(`{"type":"Results","is_final":true,"channel":{"alternatives":[{"transcript":"hi"}]}}`))
|
||||
if !ok || tr != "hi" || !final {
|
||||
t.Errorf("parse = (%q,%v,%v)", tr, final, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRealtimeEvent(t *testing.T) {
|
||||
if e := parseRealtimeEvent([]byte(`{"type":"conversation.item.input_audio_transcription.delta","delta":"x"}`)); e.kind != realtimeDelta || e.text != "x" {
|
||||
t.Errorf("delta parse: %+v", e)
|
||||
}
|
||||
if e := parseRealtimeEvent([]byte(`{"type":"error","error":{"message":"boom"}}`)); e.kind != realtimeError || e.text != "boom" {
|
||||
t.Errorf("error parse: %+v", e)
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/Gitlawb/zero/internal/config"
|
||||
"github.com/Gitlawb/zero/internal/modelregistry"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
"github.com/Gitlawb/zero/internal/providerhealth"
|
||||
"github.com/Gitlawb/zero/internal/redaction"
|
||||
)
|
||||
@@ -143,12 +144,19 @@ func providerConfigCheck(profile config.ProviderProfile) Check {
|
||||
// Report credential PRESENCE, never the value. Reported under a non-sensitive
|
||||
// key ("credentialConfigured"): the prior "apiKey" key was itself sensitive, so
|
||||
// check()'s redaction scrubbed the indicator to [REDACTED] — making "set"/"not
|
||||
// set" invisible. A profile can authenticate via a raw auth-header value instead
|
||||
// of APIKey, and both are trimmed so a whitespace-only value reads "not set"
|
||||
// (matching ProviderSnapshot.APIKeySet).
|
||||
// set" invisible. HasConfiguredCredential is the shared definition of
|
||||
// "key-authed" (inline key, raw auth header, or a key in the encrypted
|
||||
// credential store), matching ProviderSnapshot.APIKeySet — checking only the
|
||||
// inline fields made doctor and `zero providers list` disagree about the
|
||||
// same stored-key profile. A stored OAuth login counts too: a keyless
|
||||
// token-login provider (e.g. ChatGPT) is fully able to make requests and
|
||||
// must not fail the one tool meant to verify setup.
|
||||
credential := "not set"
|
||||
if strings.TrimSpace(profile.APIKey) != "" || strings.TrimSpace(profile.AuthHeaderValue) != "" {
|
||||
switch {
|
||||
case profile.HasConfiguredCredential():
|
||||
credential = "set"
|
||||
case providerHasStoredOAuthLogin(profile):
|
||||
credential = "oauth login"
|
||||
}
|
||||
details := map[string]any{
|
||||
"name": profile.Name,
|
||||
@@ -169,6 +177,22 @@ func providerConfigCheck(profile config.ProviderProfile) Check {
|
||||
return check("provider.config", "Provider config", StatusPass, fmt.Sprintf("Provider config loaded for %s.", providerName(profile)), details)
|
||||
}
|
||||
|
||||
// providerHasStoredOAuthLogin reports whether the token store holds a login for
|
||||
// any of the profile's candidates — the same candidate set and normalized-key
|
||||
// matching the runtime resolver uses. Errors degrade to false.
|
||||
func providerHasStoredOAuthLogin(profile config.ProviderProfile) bool {
|
||||
candidates := profile.OAuthLoginCandidates()
|
||||
if len(candidates) == 0 {
|
||||
return false
|
||||
}
|
||||
store, err := oauth.NewStore(oauth.StoreOptions{})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, _, ok := oauth.FirstStored(store, candidates)
|
||||
return ok
|
||||
}
|
||||
|
||||
// localProviderBaseURL reports whether the configured base_url is a loopback host
|
||||
// (a keyless local provider like Ollama/LM Studio), which needs no API key.
|
||||
func localProviderBaseURL(baseURL string) bool {
|
||||
|
||||
+15
-9
@@ -31,6 +31,11 @@ type Server struct {
|
||||
Auth string
|
||||
OAuth *OAuthConfig
|
||||
Identity string
|
||||
// UnconfiguredDefault is true when this server is one of Zero's built-in
|
||||
// defaults (e.g. keyless Firecrawl) that the user never touched in their
|
||||
// config — no credentials, no overrides. Callers use it to avoid warning
|
||||
// loudly when a server nobody configured fails to connect.
|
||||
UnconfiguredDefault bool
|
||||
}
|
||||
|
||||
func NormalizeConfig(cfg config.MCPConfig) ([]Server, error) {
|
||||
@@ -74,15 +79,16 @@ func normalizeServer(name string, raw config.MCPServerConfig) (Server, error) {
|
||||
|
||||
auth := strings.ToLower(strings.TrimSpace(raw.Auth))
|
||||
server := Server{
|
||||
Name: name,
|
||||
Type: serverType,
|
||||
Command: strings.TrimSpace(raw.Command),
|
||||
Args: trimStringSlice(raw.Args),
|
||||
Env: copyStringMap(raw.Env),
|
||||
URL: strings.TrimSpace(raw.URL),
|
||||
Headers: copyStringMap(raw.Headers),
|
||||
Auth: auth,
|
||||
OAuth: normalizeOAuthConfig(raw.OAuth),
|
||||
Name: name,
|
||||
Type: serverType,
|
||||
Command: strings.TrimSpace(raw.Command),
|
||||
Args: trimStringSlice(raw.Args),
|
||||
Env: copyStringMap(raw.Env),
|
||||
URL: strings.TrimSpace(raw.URL),
|
||||
Headers: copyStringMap(raw.Headers),
|
||||
Auth: auth,
|
||||
OAuth: normalizeOAuthConfig(raw.OAuth),
|
||||
UnconfiguredDefault: config.IsUnconfiguredDefault(name, raw),
|
||||
}
|
||||
|
||||
switch server.Type {
|
||||
|
||||
@@ -88,6 +88,33 @@ func TestNormalizeConfigValidatesTransportBoundaries(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeConfigFlagsUnconfiguredDefault(t *testing.T) {
|
||||
cfg := config.MCPConfig{Servers: map[string]config.MCPServerConfig{
|
||||
"firecrawl": config.DefaultMCPServers()["firecrawl"],
|
||||
"web": {
|
||||
Type: "http",
|
||||
URL: "https://example.com/mcp",
|
||||
},
|
||||
}}
|
||||
|
||||
servers, err := NormalizeConfig(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizeConfig() error = %v", err)
|
||||
}
|
||||
|
||||
byName := make(map[string]Server, len(servers))
|
||||
for _, server := range servers {
|
||||
byName[server.Name] = server
|
||||
}
|
||||
|
||||
if !byName["firecrawl"].UnconfiguredDefault {
|
||||
t.Fatal("an untouched firecrawl default should be flagged UnconfiguredDefault")
|
||||
}
|
||||
if byName["web"].UnconfiguredDefault {
|
||||
t.Fatal("a server the user configured must not be flagged UnconfiguredDefault")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerIdentityChangesWithTransportFields(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
|
||||
@@ -35,6 +35,10 @@ type RegisterOptions struct {
|
||||
type SkippedServer struct {
|
||||
Name string
|
||||
Err error
|
||||
// UnconfiguredDefault mirrors Server.UnconfiguredDefault: true when this
|
||||
// server is an out-of-the-box default the user never configured, so a
|
||||
// caller can skip warning loudly about it.
|
||||
UnconfiguredDefault bool
|
||||
}
|
||||
|
||||
type Runtime struct {
|
||||
@@ -148,7 +152,7 @@ func RegisterTools(ctx context.Context, registry *tools.Registry, cfg config.MCP
|
||||
for index, server := range servers {
|
||||
res := results[index]
|
||||
if res.err != nil {
|
||||
runtime.skipped = append(runtime.skipped, SkippedServer{Name: server.Name, Err: res.err})
|
||||
runtime.skipped = append(runtime.skipped, SkippedServer{Name: server.Name, Err: res.err, UnconfiguredDefault: server.UnconfiguredDefault})
|
||||
continue
|
||||
}
|
||||
serverTools, validateErr := buildServerTools(registry, server, res.remote, res.client, options, stagedNames)
|
||||
@@ -157,7 +161,7 @@ func RegisterTools(ctx context.Context, registry *tools.Registry, cfg config.MCP
|
||||
res.cancel()
|
||||
}
|
||||
_ = res.client.Close()
|
||||
runtime.skipped = append(runtime.skipped, SkippedServer{Name: server.Name, Err: validateErr})
|
||||
runtime.skipped = append(runtime.skipped, SkippedServer{Name: server.Name, Err: validateErr, UnconfiguredDefault: server.UnconfiguredDefault})
|
||||
continue
|
||||
}
|
||||
runtime.clients = append(runtime.clients, res.client)
|
||||
|
||||
@@ -153,6 +153,47 @@ func TestRegisterToolsSkipsUnreachableServerAndKeepsOthers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterToolsFlagsUnconfiguredDefaultOnSkip(t *testing.T) {
|
||||
// firecrawl is seeded by config.DefaultMCPServers with no credentials; when a
|
||||
// user never configures it, a connect failure (e.g. the real HTTP 401 from
|
||||
// issue #552) must be recorded as UnconfiguredDefault so callers can avoid
|
||||
// warning about a server the user never asked for. "custom" is configured by
|
||||
// the user and must NOT be flagged even though it also fails to connect.
|
||||
registry := tools.NewRegistry()
|
||||
|
||||
runtime, err := RegisterTools(context.Background(), registry, config.MCPConfig{Servers: map[string]config.MCPServerConfig{
|
||||
"firecrawl": config.DefaultMCPServers()["firecrawl"],
|
||||
"custom": {Type: "stdio", Command: "custom-mcp"},
|
||||
}}, RegisterOptions{
|
||||
ClientFactory: func(_ context.Context, server Server) (ToolClient, error) {
|
||||
return nil, errors.New(server.Name + " connect failed")
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RegisterTools returned error, want a skip: %v", err)
|
||||
}
|
||||
defer runtime.Close()
|
||||
|
||||
byName := make(map[string]SkippedServer)
|
||||
for _, skipped := range runtime.Skipped() {
|
||||
byName[skipped.Name] = skipped
|
||||
}
|
||||
firecrawl, ok := byName["firecrawl"]
|
||||
if !ok {
|
||||
t.Fatalf("Skipped() = %#v, want an entry for firecrawl", runtime.Skipped())
|
||||
}
|
||||
if !firecrawl.UnconfiguredDefault {
|
||||
t.Fatalf("Skipped()[firecrawl] = %#v, want UnconfiguredDefault", firecrawl)
|
||||
}
|
||||
custom, ok := byName["custom"]
|
||||
if !ok {
|
||||
t.Fatalf("Skipped() = %#v, want an entry for custom", runtime.Skipped())
|
||||
}
|
||||
if custom.UnconfiguredDefault {
|
||||
t.Fatalf("Skipped()[custom] = %#v, want a user-configured server not flagged", custom)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterToolsKeepsPriorStateAndReachableServersWhenOneIsSkipped(t *testing.T) {
|
||||
// A tool that predates registration must survive, the reachable server's tools
|
||||
// must register, and the unreachable server is skipped (recorded), not fatal.
|
||||
|
||||
@@ -122,17 +122,17 @@ func createSecretFile(path string) ([]byte, error) {
|
||||
if !errors.Is(err, os.ErrExist) && !errors.Is(err, os.ErrPermission) {
|
||||
return nil, fmt.Errorf("oauth: create token secret lock: %w", err)
|
||||
}
|
||||
// Remember the lock-creation error itself rather than a subsequent
|
||||
// "secret file doesn't exist yet" read error -- otherwise a real
|
||||
// contention/ACL problem is masked by the expected-while-waiting
|
||||
// ErrNotExist once the retries are exhausted.
|
||||
lastErr = err
|
||||
if data, rerr := readSecretFileRetry(path); rerr == nil {
|
||||
return data, nil
|
||||
} else {
|
||||
lastErr = rerr
|
||||
}
|
||||
time.Sleep(secretRetryDelay)
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, fmt.Errorf("oauth: timed out waiting for token secret %s: %w", path, lastErr)
|
||||
}
|
||||
return nil, fmt.Errorf("oauth: timed out waiting for token secret lock %s", lockPath)
|
||||
return nil, fmt.Errorf("oauth: timed out waiting for token secret %s: %w", path, lastErr)
|
||||
}
|
||||
|
||||
func writeNewSecretFile(path string) ([]byte, error) {
|
||||
|
||||
+10
-2
@@ -38,8 +38,16 @@ func ValidateKey(key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProviderKey builds the store key for a provider login.
|
||||
func ProviderKey(name string) string { return KeyPrefixProvider + name }
|
||||
// ProviderKey builds the store key for a provider login, normalizing the name
|
||||
// to lower case. Every write (Manager.Login, the ChatGPT flow) and every
|
||||
// lookup (FirstStored, GetFresh, logout, status filters) funnels through here,
|
||||
// so normalizing at this one choke point keeps them symmetric: without it,
|
||||
// `zero auth login xAI` stored "provider:xAI" while the profile scaffolded for
|
||||
// it looked up "provider:xai" case-sensitively — a fresh, successful login
|
||||
// that was invisible to the runtime.
|
||||
func ProviderKey(name string) string {
|
||||
return KeyPrefixProvider + strings.ToLower(strings.TrimSpace(name))
|
||||
}
|
||||
|
||||
// FirstStored returns the token and its ProviderKey for the FIRST candidate name
|
||||
// that has a token in the store, with ok=false when none do. Callers pass
|
||||
|
||||
@@ -126,3 +126,20 @@ func TestResolveStorePathHonorsOverride(t *testing.T) {
|
||||
t.Fatalf("path = %q, want %q", path, override)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderKeyNormalizesCase: every provider-login write and lookup funnels
|
||||
// through ProviderKey, so `zero auth login xAI` and the scaffolded profile's
|
||||
// candidate "xai" must resolve to the SAME store entry — the raw-typed casing
|
||||
// previously produced an invisible login.
|
||||
func TestProviderKeyNormalizesCase(t *testing.T) {
|
||||
if got := ProviderKey(" xAI "); got != "provider:xai" {
|
||||
t.Fatalf("ProviderKey must trim and lower-case, got %q", got)
|
||||
}
|
||||
store, _ := newTestStore(t)
|
||||
if err := store.Save(ProviderKey("xAI"), Token{AccessToken: "tok"}); err != nil {
|
||||
t.Fatalf("save: %v", err)
|
||||
}
|
||||
if _, key, ok := FirstStored(store, []string{"xai"}); !ok || key != "provider:xai" {
|
||||
t.Fatalf("candidate \"xai\" must find the mixed-case login, ok=%v key=%q", ok, key)
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,14 @@ type Descriptor struct {
|
||||
SupportedAPIFormats []APIFormat
|
||||
Aliases []string
|
||||
|
||||
// Custom marks the "bring your own endpoint" catalog entries
|
||||
// (custom-openai-compatible, custom-anthropic-compatible). Unlike every other
|
||||
// descriptor, RequiresAuth here is just a template default for the credential
|
||||
// wizard step, not a fact about the user's actual endpoint — a custom target may
|
||||
// or may not need auth, so callers deciding whether a *saved profile* is missing
|
||||
// a credential must not treat RequiresAuth as authoritative for these entries.
|
||||
Custom bool
|
||||
|
||||
// OAuth reports that this provider offers an in-app OAuth login that yields a
|
||||
// credential usable for model calls (browser PKCE and/or device code). Only
|
||||
// set for providers where this actually works (not subscription-via-proxy).
|
||||
@@ -89,8 +97,8 @@ func RuntimeUnsupportedReason(descriptor Descriptor) string {
|
||||
var descriptors = []Descriptor{
|
||||
// GitLawb OpenGateway — the recommended default. An OpenAI-compatible gateway
|
||||
// that smart-routes by model id across upstream providers (xiaomi-mimo,
|
||||
// minimax, qwen, google, nvidia, z-ai). Flat /v1/chat/completions with a
|
||||
// Bearer ogw_live_… key; listed first and badged in every picker.
|
||||
// minimax, qwen, google, nvidia, tencent, z-ai). Flat /v1/chat/completions
|
||||
// with a Bearer ogw_live_… key; listed first and badged in every picker.
|
||||
recommended(openAICompat("gitlawb-opengateway", "GitLawb OpenGateway", "https://opengateway.gitlawb.com/v1", "mimo-v2.5-pro", []string{"GITLAWB_OPENGATEWAY_API_KEY"}, "gitlawb opengateway", "opengateway")),
|
||||
openAI("openai", "OpenAI", "https://api.openai.com/v1", "gpt-4.1", []string{"OPENAI_API_KEY"}),
|
||||
anthropic("anthropic", "Anthropic", "https://api.anthropic.com", "claude-sonnet-4.5", []string{"ANTHROPIC_API_KEY"}),
|
||||
@@ -140,6 +148,7 @@ var descriptors = []Descriptor{
|
||||
openAICompat("kilocode", "KiloCode", "https://api.kilo.ai/api/gateway", "anthropic/claude-sonnet-4.6", []string{"KILO_API_KEY"}, "kilo", "kilo gateway"),
|
||||
openAICompat("opencode", "OpenCode Zen", "https://opencode.ai/zen/v1", "deepseek-v4-flash", []string{"OPENCODE_API_KEY"}, "opencode zen"),
|
||||
openAICompat("opencode-go", "OpenCode Go", "https://opencode.ai/zen/go/v1", "deepseek-v4-pro", []string{"OPENCODE_API_KEY"}, "opencode go"),
|
||||
anthropicCompat("opencode-go-anthropic-compatible", "OpenCode Go Anthropic-compatible", "https://opencode.ai/zen/go", "minimax-m3", []string{"OPENCODE_API_KEY"}),
|
||||
openAICompat("atomic-chat", "Atomic Chat", "https://api.atomic.chat/v1", "gpt-4.1", []string{"ATOMIC_CHAT_API_KEY"}),
|
||||
// ChatGPT subscription via a local OAuth proxy. A ChatGPT (Plus/Pro) OAuth
|
||||
// token only works against ChatGPT's own backend (which is Cloudflare-gated to
|
||||
@@ -148,8 +157,16 @@ var descriptors = []Descriptor{
|
||||
// endpoint. Local (no API key — the proxy authenticates); override the base URL
|
||||
// for your proxy's port. See docs/oauth-subscriptions.md.
|
||||
localOpenAI("chatgpt-proxy", "ChatGPT (local OAuth proxy)", "http://localhost:10531/v1", "gpt-5", "chatgpt"),
|
||||
openAICompat("custom-openai-compatible", "Custom OpenAI-compatible", "https://example.invalid/v1", "custom-model", []string{"OPENAI_API_KEY"}, "custom openai compatible"),
|
||||
anthropicCompat("custom-anthropic-compatible", "Custom Anthropic-compatible", "https://example.invalid/anthropic", "custom-model", []string{"ANTHROPIC_API_KEY"}, "custom anthropic compatible"),
|
||||
func() Descriptor {
|
||||
d := openAICompat("custom-openai-compatible", "Custom OpenAI-compatible", "https://example.invalid/v1", "custom-model", []string{"OPENAI_API_KEY"}, "custom openai compatible")
|
||||
d.Custom = true
|
||||
return d
|
||||
}(),
|
||||
func() Descriptor {
|
||||
d := anthropicCompat("custom-anthropic-compatible", "Custom Anthropic-compatible", "https://example.invalid/anthropic", "custom-model", []string{"ANTHROPIC_API_KEY"}, "custom anthropic compatible")
|
||||
d.Custom = true
|
||||
return d
|
||||
}(),
|
||||
}
|
||||
|
||||
func All() []Descriptor {
|
||||
|
||||
@@ -40,6 +40,7 @@ var expectedCatalogIDs = []string{
|
||||
"kilocode",
|
||||
"opencode",
|
||||
"opencode-go",
|
||||
"opencode-go-anthropic-compatible",
|
||||
"atomic-chat",
|
||||
"chatgpt-proxy",
|
||||
"custom-openai-compatible",
|
||||
@@ -291,7 +292,7 @@ func TestListByTransportPreservesCatalogOrder(t *testing.T) {
|
||||
TransportGoogle: {"google"},
|
||||
TransportBedrock: {"bedrock"},
|
||||
TransportVertex: {"vertex"},
|
||||
TransportAnthropicCompat: {"minimax", "minimaxi-cn", "custom-anthropic-compatible"},
|
||||
TransportAnthropicCompat: {"minimax", "minimaxi-cn", "opencode-go-anthropic-compatible", "custom-anthropic-compatible"},
|
||||
TransportOpenAICompat: {"gitlawb-opengateway", "ollama-cloud", "ollama", "lmstudio", "openrouter", "huggingface", "chatgpt", "groq", "deepseek", "together", "dashscope", "moonshot", "longcat", "nvidia-nim", "mistral", "github", "xai", "venice", "xiaomi-mimo", "bankr", "zai", "zai-cn", "kilocode", "opencode", "opencode-go", "atomic-chat", "chatgpt-proxy", "custom-openai-compatible"},
|
||||
}
|
||||
|
||||
|
||||
@@ -140,12 +140,13 @@ var curatedModels = map[string][]Model{
|
||||
"zai": zaiCuratedModels,
|
||||
"zai-cn": zaiCuratedModels,
|
||||
// OpenGateway smart-routes by model id across its upstream providers
|
||||
// (see /health: xiaomi-mimo, minimax, qwen, google, nvidia, z-ai). These are
|
||||
// the curated coding defaults; the gateway accepts any model its upstreams
|
||||
// expose, so users can also type an id the picker doesn't list.
|
||||
// (see /health: xiaomi-mimo, minimax, qwen, google, nvidia, tencent, z-ai).
|
||||
// These are the curated coding defaults; the gateway accepts any model its
|
||||
// upstreams expose, so users can also type an id the picker doesn't list.
|
||||
"gitlawb-opengateway": {
|
||||
{ID: "mimo-v2.5-pro", Description: "catalog default (Xiaomi MiMo)"},
|
||||
{ID: "mimo-v2.5-pro-ultraspeed", Description: "fast model (Xiaomi MiMo)"},
|
||||
{ID: "tencent/hy3", Description: "free Tencent HY3 model"},
|
||||
{ID: "MiniMax-M3", Description: "MiniMax model"},
|
||||
{ID: "qwen-plus", Description: "Qwen model"},
|
||||
{ID: "gemini-2.5-pro", Description: "long-context model (Google)"},
|
||||
@@ -156,6 +157,12 @@ var curatedModels = map[string][]Model{
|
||||
{ID: "gpt-4.1", Description: "catalog default"},
|
||||
{ID: "gpt-4o-mini", Description: "fast model"},
|
||||
},
|
||||
"opencode-go-anthropic-compatible": {
|
||||
{ID: "minimax-m3", Description: "MiniMax M3: default"},
|
||||
{ID: "minimax-m2.7", Description: "MiniMax M2.7: coding model"},
|
||||
{ID: "qwen3.7-plus", Description: "Qwen 3.7 Plus: balanced model"},
|
||||
{ID: "qwen3.7-max", Description: "Qwen 3.7 Max: strong model"},
|
||||
},
|
||||
"custom-openai-compatible": {
|
||||
{ID: "custom-model", Description: "custom endpoint model"},
|
||||
},
|
||||
@@ -166,13 +173,13 @@ var curatedModels = map[string][]Model{
|
||||
|
||||
func Models(provider providercatalog.Descriptor) []Model {
|
||||
if models, ok := curatedModels[provider.ID]; ok {
|
||||
return dedupeModels(provider.DefaultModel, models)
|
||||
return FilterModelsForProvider(provider.ID, dedupeModels(provider.DefaultModel, models))
|
||||
}
|
||||
models := registryModels(provider)
|
||||
if len(models) > 0 {
|
||||
return dedupeModels(provider.DefaultModel, models)
|
||||
return FilterModelsForProvider(provider.ID, dedupeModels(provider.DefaultModel, models))
|
||||
}
|
||||
return dedupeModels(provider.DefaultModel, nil)
|
||||
return FilterModelsForProvider(provider.ID, dedupeModels(provider.DefaultModel, nil))
|
||||
}
|
||||
|
||||
func registryModels(provider providercatalog.Descriptor) []Model {
|
||||
|
||||
@@ -47,6 +47,16 @@ func TestModelsAreProviderScoped(t *testing.T) {
|
||||
want: []string{"glm-4.5", "glm-4.6", "glm-4.5-air", "glm-z1-air"},
|
||||
notWant: []string{"gpt-4.1", "claude-sonnet-4.5"},
|
||||
},
|
||||
{
|
||||
provider: "gitlawb-opengateway",
|
||||
want: []string{"mimo-v2.5-pro", "tencent/hy3"},
|
||||
notWant: []string{"openai/gpt-4.1", "claude-sonnet-4.5"},
|
||||
},
|
||||
{
|
||||
provider: "opencode-go-anthropic-compatible",
|
||||
want: []string{"minimax-m3", "minimax-m2.7", "qwen3.7-plus", "qwen3.7-max"},
|
||||
notWant: []string{"gpt-4.1", "claude-sonnet-4.5", "deepseek-chat"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -1,6 +1,38 @@
|
||||
package providermodelcatalog
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
)
|
||||
|
||||
// ModelIDAllowedForProvider reports whether a model ID is permitted for the
|
||||
// given provider under provider-specific allow/block rules.
|
||||
// For opencode-go-anthropic-compatible: only Qwen and MiniMax model IDs are allowed.
|
||||
// Allows ANY model by default
|
||||
func ModelIDAllowedForProvider(providerID, modelID string) bool {
|
||||
modelID = strings.ToLower(strings.TrimSpace(modelID))
|
||||
if modelID == "" {
|
||||
return false
|
||||
}
|
||||
switch providercatalog.NormalizeID(providerID) {
|
||||
case "opencode-go-anthropic-compatible":
|
||||
return strings.Contains(modelID, "qwen") || strings.Contains(modelID, "minimax")
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// FilterModelsForProvider filters a model slice by ModelIDAllowedForProvider.
|
||||
func FilterModelsForProvider(providerID string, models []Model) []Model {
|
||||
result := make([]Model, 0, len(models))
|
||||
for _, model := range models {
|
||||
if ModelIDAllowedForProvider(providerID, model.ID) {
|
||||
result = append(result, model)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func IsCodingModel(model Model) bool {
|
||||
if IsKnownNonCodingModelID(model.ID) {
|
||||
@@ -31,7 +63,7 @@ func LooksLikeCodingModelID(id string) bool {
|
||||
"mistral", "codestral", "devstral", "magistral", "ministral",
|
||||
"grok", "glm", "command", "nemotron", "mixtral", "coder",
|
||||
"code", "chat", "instruct", "reasoner", "reasoning", "mimo",
|
||||
"maverick", "scout", "bankr",
|
||||
"hy3", "tencent", "maverick", "scout", "bankr",
|
||||
} {
|
||||
if strings.Contains(normalized, term) {
|
||||
return true
|
||||
@@ -57,6 +89,67 @@ func IsKnownNonCodingModelID(id string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsSTTModel reports whether a model transcribes speech to text — the models
|
||||
// IsKnownNonCodingModelID filters OUT of the coding picker, filtered back IN for
|
||||
// the /stt-model picker.
|
||||
//
|
||||
// The reliable signal is the model's declared modalities, not its name: a
|
||||
// transcriber consumes audio and emits text. Anything that emits audio is
|
||||
// text-to-speech or speech-to-speech, never a transcriber, so audio output is a
|
||||
// hard disqualifier. Only when a catalog entry carries no usable modality
|
||||
// metadata (common for the Groq/OpenAI transcription models, which models.dev
|
||||
// lists sparsely) do we fall back to a name heuristic.
|
||||
func IsSTTModel(model Model) bool {
|
||||
audioIn := containsFold(model.InputModalities, "audio")
|
||||
audioOut := containsFold(model.OutputModalities, "audio")
|
||||
textOut := containsFold(model.OutputModalities, "text")
|
||||
|
||||
// Emits audio → TTS or speech-to-speech, regardless of a transcription-like
|
||||
// name. This is what stops gpt-4o-audio-preview / *-realtime from slipping in.
|
||||
if audioOut {
|
||||
return false
|
||||
}
|
||||
// Consumes audio and emits text (or emits nothing explicit) → transcriber.
|
||||
if audioIn && (textOut || len(model.OutputModalities) == 0) {
|
||||
return true
|
||||
}
|
||||
// Declares input modalities but none is audio → cannot be speech-to-text,
|
||||
// whatever it is called.
|
||||
if len(model.InputModalities) > 0 && !audioIn {
|
||||
return false
|
||||
}
|
||||
// No usable modality metadata: fall back to the name heuristic.
|
||||
return IsSTTModelID(model.ID)
|
||||
}
|
||||
|
||||
// IsSTTModelID is the name-only heuristic used when a catalog entry declares no
|
||||
// modalities. It matches the known transcription model families and explicitly
|
||||
// rejects text-to-speech names, so it never confuses a TTS model for a
|
||||
// transcriber. Weaker than the modality signal in IsSTTModel — a fallback, not
|
||||
// the primary test.
|
||||
func IsSTTModelID(id string) bool {
|
||||
normalized := strings.ToLower(strings.TrimSpace(id))
|
||||
if normalized == "" {
|
||||
return false
|
||||
}
|
||||
// Text-to-speech / speech-synthesis names are never transcription.
|
||||
for _, tts := range []string{"tts", "text-to-speech"} {
|
||||
if strings.Contains(normalized, tts) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, term := range []string{
|
||||
"whisper", "transcribe", "transcription",
|
||||
"scribe", "parakeet", "canary", "moonshine",
|
||||
"sense-voice", "sensevoice", "wav2vec", "conformer",
|
||||
} {
|
||||
if strings.Contains(normalized, term) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasCodingTag(tags []string) bool {
|
||||
for _, tag := range tags {
|
||||
normalized := strings.ToLower(strings.TrimSpace(tag))
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package providermodelcatalog
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
)
|
||||
|
||||
func TestModelIDAllowedForProvider(t *testing.T) {
|
||||
cases := []struct {
|
||||
testName string
|
||||
providerID string
|
||||
modelID string
|
||||
want bool
|
||||
}{
|
||||
// Empty model ID is always rejected.
|
||||
{"empty model", "any-provider", "", false},
|
||||
{"empty model after trim", "any-provider", " ", false},
|
||||
|
||||
// opencode-go-anthropic-compatible: only qwen and minimax allowed.
|
||||
{"opencode-anthropic qwen", "opencode-go-anthropic-compatible", "qwen3.7-plus", true},
|
||||
{"opencode-anthropic minimax", "opencode-go-anthropic-compatible", "MiniMax-M3", true},
|
||||
{"opencode-anthropic non-allowed", "opencode-go-anthropic-compatible", "claude-sonnet-4.5", false},
|
||||
{"opencode-anthropic deepseek", "opencode-go-anthropic-compatible", "deepseek-chat", false},
|
||||
|
||||
// Case insensitivity and trimming for the blocked provider.
|
||||
{"opencode-anthropic case insensitive", "opencode-go-anthropic-compatible", "QWEN3.7-PLUS", true},
|
||||
{"opencode-anthropic whitespace", "opencode-go-anthropic-compatible", " minimax-m3 ", true},
|
||||
|
||||
// Normalized provider ID — e.g. NormalizeID converts spaces/dots to dashes.
|
||||
{"opencode-anthropic normalized provider", "opencode go anthropic compatible", "qwen3.7-plus", true},
|
||||
|
||||
// Default branch: all model IDs allowed.
|
||||
{"default provider allows anything", "openai", "claude-sonnet-4.5", true},
|
||||
{"default provider allows any name", "openai", "x", true},
|
||||
{"default provider groq", "groq", "deepseek-chat", true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.testName, func(t *testing.T) {
|
||||
got := ModelIDAllowedForProvider(tc.providerID, tc.modelID)
|
||||
if got != tc.want {
|
||||
t.Errorf("ModelIDAllowedForProvider(%q, %q) = %v, want %v",
|
||||
tc.providerID, tc.modelID, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterModelsForProvider verifies that filtering delegates correctly.
|
||||
func TestFilterModelsForProvider(t *testing.T) {
|
||||
models := []Model{
|
||||
{ID: "qwen3.7-plus"},
|
||||
{ID: "MiniMax-M3"},
|
||||
{ID: "claude-sonnet-4.5"},
|
||||
{ID: "deepseek-chat"},
|
||||
}
|
||||
|
||||
filtered := FilterModelsForProvider("opencode-go-anthropic-compatible", models)
|
||||
if len(filtered) != 2 {
|
||||
t.Fatalf("FilterModelsForProvider returned %d models, want 2: %#v", len(filtered), modelIDs(filtered))
|
||||
}
|
||||
if filtered[0].ID != "qwen3.7-plus" || filtered[1].ID != "MiniMax-M3" {
|
||||
t.Errorf("FilterModelsForProvider returned %#v, want [qwen3.7-plus MiniMax-M3]", modelIDs(filtered))
|
||||
}
|
||||
|
||||
// With a default provider nothing is filtered out.
|
||||
all := FilterModelsForProvider("openai", models)
|
||||
if len(all) != len(models) {
|
||||
t.Fatalf("FilterModelsForProvider(openai) returned %d, want %d", len(all), len(models))
|
||||
}
|
||||
|
||||
// Empty input.
|
||||
empty := FilterModelsForProvider("opencode-go-anthropic-compatible", nil)
|
||||
if len(empty) != 0 {
|
||||
t.Fatal("expected empty result for nil input")
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelIDAllowedForProvider_NormalizedProviderIDs verifies that various
|
||||
// raw provider ID formats that NormalizeID maps to the same canonical form
|
||||
// are handled consistently.
|
||||
func TestModelIDAllowedForProvider_NormalizedProviderIDs(t *testing.T) {
|
||||
canonical := providercatalog.NormalizeID("opencode-go-anthropic-compatible")
|
||||
variants := []string{
|
||||
"opencode-go-anthropic-compatible",
|
||||
"opencode_go_anthropic_compatible",
|
||||
"opencode go anthropic compatible",
|
||||
"OpenCode-Go-Anthropic-Compatible",
|
||||
" opencode-go-anthropic-compatible ",
|
||||
"opencode.go.anthropic.compatible",
|
||||
}
|
||||
for _, variant := range variants {
|
||||
if n := providercatalog.NormalizeID(variant); n != canonical {
|
||||
t.Fatalf("NormalizeID(%q) = %q, want %q", variant, n, canonical)
|
||||
}
|
||||
}
|
||||
|
||||
// All variants should produce the same filter result.
|
||||
for _, variant := range variants {
|
||||
if !ModelIDAllowedForProvider(variant, "qwen3.7-plus") {
|
||||
t.Errorf("ModelIDAllowedForProvider(%q, qwen3.7-plus) = false, want true", variant)
|
||||
}
|
||||
if ModelIDAllowedForProvider(variant, "claude-sonnet-4.5") {
|
||||
t.Errorf("ModelIDAllowedForProvider(%q, claude-sonnet-4.5) = true, want false", variant)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
func TestLooksLikeCodingModelID(t *testing.T) {
|
||||
coding := []string{"gpt-4o", "claude-sonnet-4.5", "o1", "o3-mini", "o4-mini", "qwen2.5-coder", "deepseek-chat", "kimi-k2", "codestral-latest", "grok-code"}
|
||||
coding := []string{"gpt-4o", "claude-sonnet-4.5", "o1", "o3-mini", "o4-mini", "qwen2.5-coder", "deepseek-chat", "kimi-k2", "codestral-latest", "grok-code", "tencent/hy3"}
|
||||
for _, id := range coding {
|
||||
if !LooksLikeCodingModelID(id) {
|
||||
t.Errorf("LooksLikeCodingModelID(%q) = false, want true", id)
|
||||
|
||||
@@ -80,6 +80,14 @@ func TestParseOpenGatewayCatalogSupportsRichModelJSON(t *testing.T) {
|
||||
"tags": ["coding", "free"],
|
||||
"cost": {"input": 0, "output": 0}
|
||||
},
|
||||
{
|
||||
"id": "tencent/hy3",
|
||||
"name": "Tencent HY3",
|
||||
"description": "free Tencent chat route",
|
||||
"context_window": 262144,
|
||||
"tools": true,
|
||||
"tags": ["free"]
|
||||
},
|
||||
{
|
||||
"id": "image-route",
|
||||
"name": "Image Route",
|
||||
@@ -92,8 +100,8 @@ func TestParseOpenGatewayCatalogSupportsRichModelJSON(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("ParseOpenGatewayCatalog returned error: %v", err)
|
||||
}
|
||||
if got := strings.Join(modelIDs(models), ","); got != "minimax-m3" {
|
||||
t.Fatalf("models = %#v, want one gateway coding model", got)
|
||||
if got := strings.Join(modelIDs(models), ","); got != "minimax-m3,tencent/hy3" {
|
||||
t.Fatalf("models = %#v, want gateway coding models", got)
|
||||
}
|
||||
model := models[0]
|
||||
if model.ID != "minimax-m3" || model.Description != "agentic coding route" {
|
||||
@@ -108,6 +116,9 @@ func TestParseOpenGatewayCatalogSupportsRichModelJSON(t *testing.T) {
|
||||
if model.Source != "opengateway" {
|
||||
t.Fatalf("gateway source = %q, want opengateway", model.Source)
|
||||
}
|
||||
if models[1].ID != "tencent/hy3" || models[1].ContextWindow != 262144 || !models[1].ToolCall {
|
||||
t.Fatalf("gateway HY3 model = %#v, want Tencent HY3 with context/tools", models[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelsDevProviderIDMapsZeroAliases(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package providermodelcatalog
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsSTTModelID(t *testing.T) {
|
||||
in := []string{"whisper-1", "whisper-large-v3-turbo", "distil-whisper-large-v3-en",
|
||||
"gpt-4o-transcribe", "gpt-4o-mini-transcribe", "nova-3-scribe", "parakeet-tdt-1.1b", "canary-1b"}
|
||||
for _, id := range in {
|
||||
if !IsSTTModelID(id) {
|
||||
t.Errorf("%q should be an STT model by name", id)
|
||||
}
|
||||
}
|
||||
out := []string{"gpt-4o", "claude-sonnet-4", "tts-1", "gpt-4o-mini-tts",
|
||||
"dall-e-3", "text-to-speech-1", ""}
|
||||
for _, id := range out {
|
||||
if IsSTTModelID(id) {
|
||||
t.Errorf("%q should NOT be an STT model by name", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSTTModelUsesModalities(t *testing.T) {
|
||||
// Audio in, text out → transcriber, even with an unfamiliar name.
|
||||
if !IsSTTModel(Model{ID: "acme-speech-v2", InputModalities: []string{"audio"}, OutputModalities: []string{"text"}}) {
|
||||
t.Error("audio-in/text-out model should be classified STT regardless of name")
|
||||
}
|
||||
// Audio out → TTS or speech-to-speech, never STT, even if the name says "transcribe".
|
||||
if IsSTTModel(Model{ID: "weird-transcribe-tts", InputModalities: []string{"text"}, OutputModalities: []string{"audio"}}) {
|
||||
t.Error("an audio-OUTPUT model must never be classified STT")
|
||||
}
|
||||
// Realtime speech-to-speech: audio in AND audio out → not a transcriber.
|
||||
if IsSTTModel(Model{ID: "gpt-4o-realtime", InputModalities: []string{"audio", "text"}, OutputModalities: []string{"audio", "text"}}) {
|
||||
t.Error("audio-in AND audio-out (speech-to-speech) must not be classified STT")
|
||||
}
|
||||
// Text-only LLM: input declared, no audio → not STT.
|
||||
if IsSTTModel(Model{ID: "gpt-4o", InputModalities: []string{"text", "image"}, OutputModalities: []string{"text"}}) {
|
||||
t.Error("a text/image-in model must not be classified STT")
|
||||
}
|
||||
// No modality metadata: fall back to the name heuristic.
|
||||
if !IsSTTModel(Model{ID: "whisper-large-v3-turbo"}) {
|
||||
t.Error("with no modalities, a whisper id should fall back to STT-true")
|
||||
}
|
||||
if IsSTTModel(Model{ID: "gpt-4o-mini-tts"}) {
|
||||
t.Error("with no modalities, a tts id should be STT-false")
|
||||
}
|
||||
}
|
||||
@@ -405,6 +405,9 @@ func mergeLiveModels(provider providercatalog.Descriptor, liveModels []Model, ca
|
||||
}
|
||||
|
||||
func liveModelAllowedWithoutCatalog(provider providercatalog.Descriptor, id string) bool {
|
||||
if !providermodelcatalog.ModelIDAllowedForProvider(provider.ID, id) {
|
||||
return false
|
||||
}
|
||||
if providermodelcatalog.IsKnownNonCodingModelID(id) {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -261,6 +261,47 @@ func TestDiscoverCatalogMergesLiveModelsWithModelsDevMetadata(t *testing.T) {
|
||||
t.Fatal("missing gpt-4.1")
|
||||
}
|
||||
|
||||
func TestLiveModelAllowedWithoutCatalogChecksProviderGateFirst(t *testing.T) {
|
||||
// The ModelIDAllowedForProvider check runs before the others.
|
||||
// For the restricted provider (opencode-go-anthropic-compatible) a
|
||||
// non-allowed model returns false immediately, without reaching the
|
||||
// IsKnownNonCodingModelID, Local, or LooksLikeCodingModelID checks.
|
||||
restricted := providercatalog.Descriptor{
|
||||
ID: "opencode-go-anthropic-compatible",
|
||||
Local: true, // would pass the Local check if we got past the gate
|
||||
}
|
||||
|
||||
// A model that isn't qwen/minimax is blocked at the gate, even though
|
||||
// Local=true would let any model through on its own.
|
||||
if got := liveModelAllowedWithoutCatalog(restricted, "claude-sonnet-4"); got != false {
|
||||
t.Fatal("liveModelAllowedWithoutCatalog: want false for claude-sonnet-4 on opencode-go-anthropic-compatible (blocked by ModelIDAllowedForProvider)")
|
||||
}
|
||||
|
||||
// A qwen model passes the gate and continues to the remaining checks;
|
||||
// it's not a known non-coding model and looks like a coding model, so
|
||||
// the result is true.
|
||||
if got := liveModelAllowedWithoutCatalog(restricted, "qwen-max"); got != true {
|
||||
t.Fatal("liveModelAllowedWithoutCatalog: want true for qwen-max on opencode-go-anthropic-compatible (passes all checks)")
|
||||
}
|
||||
|
||||
// A minimax model also passes the gate.
|
||||
if got := liveModelAllowedWithoutCatalog(restricted, "minimax-text-01"); got != true {
|
||||
t.Fatal("liveModelAllowedWithoutCatalog: want true for minimax-text-01 on opencode-go-anthropic-compatible (passes all checks)")
|
||||
}
|
||||
|
||||
// Unrestricted provider: all models pass the gate, so the other checks
|
||||
// decide the result. claude-sonnet-4 looks like a coding model → true.
|
||||
openAI := providercatalog.Descriptor{ID: "openai"}
|
||||
if got := liveModelAllowedWithoutCatalog(openAI, "claude-sonnet-4"); got != true {
|
||||
t.Fatal("liveModelAllowedWithoutCatalog: want true for claude-sonnet-4 on openai (unrestricted)")
|
||||
}
|
||||
|
||||
// Non-coding model still filtered on an unrestricted provider.
|
||||
if got := liveModelAllowedWithoutCatalog(openAI, "text-embedding-3-large"); got != false {
|
||||
t.Fatal("liveModelAllowedWithoutCatalog: want false for embedding model on openai")
|
||||
}
|
||||
}
|
||||
|
||||
func modelIDs(models []Model) []string {
|
||||
ids := make([]string, 0, len(models))
|
||||
for _, model := range models {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/modelregistry"
|
||||
"github.com/Gitlawb/zero/internal/oauth"
|
||||
"github.com/Gitlawb/zero/internal/providercatalog"
|
||||
"github.com/Gitlawb/zero/internal/providermodelcatalog"
|
||||
"github.com/Gitlawb/zero/internal/providers/anthropic"
|
||||
"github.com/Gitlawb/zero/internal/providers/gemini"
|
||||
"github.com/Gitlawb/zero/internal/providers/openai"
|
||||
@@ -205,6 +206,9 @@ func resolveProfile(profile config.ProviderProfile, options Options) (resolvedPr
|
||||
} else if providerKind != modelProvider {
|
||||
return resolvedProfile{}, fmt.Errorf("zero model %s belongs to %s, not %s", entry.ID, entry.Provider, providerKind)
|
||||
}
|
||||
if err := validateModelAllowedForProvider(profile, entry.ID); err != nil {
|
||||
return resolvedProfile{}, err
|
||||
}
|
||||
return resolvedProfile{
|
||||
providerKind: providerKind,
|
||||
apiModel: entry.APIModel,
|
||||
@@ -216,6 +220,9 @@ func resolveProfile(profile config.ProviderProfile, options Options) (resolvedPr
|
||||
if providerKind == "" {
|
||||
providerKind = config.ProviderKindOpenAI
|
||||
}
|
||||
if err := validateModelAllowedForProvider(profile, model); err != nil {
|
||||
return resolvedProfile{}, err
|
||||
}
|
||||
return resolvedProfile{
|
||||
providerKind: providerKind,
|
||||
apiModel: model,
|
||||
@@ -223,6 +230,29 @@ func resolveProfile(profile config.ProviderProfile, options Options) (resolvedPr
|
||||
}, nil
|
||||
}
|
||||
|
||||
// validateModelAllowedForProvider enforces provider-scoped model allowlists at
|
||||
// runtime resolution time. It delegates to
|
||||
// providermodelcatalog.ModelIDAllowedForProvider, which encodes the per-provider
|
||||
// rules in one place; restricted providers (such as
|
||||
// opencode-go-anthropic-compatible) only accept a curated subset of model
|
||||
// families, while the default branch makes unrestricted providers a no-op. This
|
||||
// guards both New() (TUI /model, headless --model overrides) and
|
||||
// ResolveRuntimeMetadata (the read-only metadata path) so a catalog-backed
|
||||
// profile cannot persist or send a model outside its scoped allowlist.
|
||||
func validateModelAllowedForProvider(profile config.ProviderProfile, model string) error {
|
||||
providerID := strings.TrimSpace(profile.CatalogID)
|
||||
if providermodelcatalog.ModelIDAllowedForProvider(providerID, model) {
|
||||
return nil
|
||||
}
|
||||
if providerID == "" {
|
||||
providerID = strings.TrimSpace(profile.Name)
|
||||
}
|
||||
if providerID == "" {
|
||||
providerID = "provider"
|
||||
}
|
||||
return fmt.Errorf("provider %s does not allow model %s", providerID, strings.TrimSpace(model))
|
||||
}
|
||||
|
||||
func explicitProviderKind(profile config.ProviderProfile) (config.ProviderKind, bool) {
|
||||
providerKind := config.ProviderKind(strings.TrimSpace(strings.ToLower(string(profile.ProviderKind))))
|
||||
if providerKind != "" {
|
||||
|
||||
@@ -56,6 +56,56 @@ func TestNewCreatesOpenAIProviderWithFactoryOptions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewPassesOpenGatewayHY3ModelThrough(t *testing.T) {
|
||||
transport := &captureTransport{
|
||||
responseBody: "data: [DONE]\n\n",
|
||||
}
|
||||
client := &http.Client{Transport: transport}
|
||||
|
||||
provider, err := New(config.ProviderProfile{
|
||||
Name: "opengateway",
|
||||
CatalogID: "gitlawb-opengateway",
|
||||
ProviderKind: config.ProviderKindOpenAICompatible,
|
||||
APIKey: "ogw_live_test",
|
||||
Model: "tencent/hy3",
|
||||
}, Options{HTTPClient: client})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
stream, err := provider.StreamCompletion(context.Background(), zeroruntime.CompletionRequest{
|
||||
Messages: []zeroruntime.Message{{Role: zeroruntime.MessageRoleUser, Content: "hello"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StreamCompletion() error = %v", err)
|
||||
}
|
||||
for range stream {
|
||||
}
|
||||
|
||||
if transport.request.URL.String() != "https://opengateway.gitlawb.com/v1/chat/completions" {
|
||||
t.Fatalf("request URL = %q, want OpenGateway chat completions", transport.request.URL.String())
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(transport.body()).Decode(&body); err != nil {
|
||||
t.Fatalf("decode request body: %v", err)
|
||||
}
|
||||
if body["model"] != "tencent/hy3" {
|
||||
t.Fatalf("model = %q, want tencent/hy3 passthrough", body["model"])
|
||||
}
|
||||
|
||||
metadata, err := ResolveRuntimeMetadata(config.ProviderProfile{
|
||||
Name: "opengateway",
|
||||
CatalogID: "gitlawb-opengateway",
|
||||
ProviderKind: config.ProviderKindOpenAICompatible,
|
||||
Model: "tencent/hy3",
|
||||
}, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveRuntimeMetadata() error = %v", err)
|
||||
}
|
||||
if metadata.ProviderKind != config.ProviderKindOpenAICompatible || metadata.APIModel != "tencent/hy3" {
|
||||
t.Fatalf("runtime metadata = %#v, want OpenAI-compatible tencent/hy3", metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewThreadsCustomProviderHeaders(t *testing.T) {
|
||||
transport := &captureTransport{
|
||||
responseBody: "data: [DONE]\n\n",
|
||||
@@ -466,3 +516,101 @@ func TestCodexAccountForKey(t *testing.T) {
|
||||
t.Fatalf("account = %q, want acc-rotated (in-place refresh must be picked up)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewRejectsModelOutsideProviderAllowlist guards the PR #544 P2 fix:
|
||||
// opencode-go-anthropic-compatible only accepts Qwen and MiniMax model IDs,
|
||||
// so a Claude model routed through this catalog-backed profile must be rejected
|
||||
// at runtime, not silently forwarded to https://opencode.ai/zen/go/v1/messages.
|
||||
func TestNewRejectsModelOutsideProviderAllowlist(t *testing.T) {
|
||||
_, err := New(config.ProviderProfile{
|
||||
Name: "opencode-go-anthropic",
|
||||
CatalogID: "opencode-go-anthropic-compatible",
|
||||
APIKey: "sk-test",
|
||||
Model: "claude-sonnet-4.5",
|
||||
}, Options{})
|
||||
if err == nil {
|
||||
t.Fatal("New() error = nil, want claude-sonnet-4.5 rejected for opencode-go-anthropic-compatible")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "opencode-go-anthropic-compatible") {
|
||||
t.Fatalf("error = %q, want it to name the provider", err.Error())
|
||||
}
|
||||
if !strings.Contains(err.Error(), "claude-sonnet-4.5") {
|
||||
t.Fatalf("error = %q, want it to name the rejected model", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewRejectsRawModelOutsideProviderAllowlist covers the registry-miss
|
||||
// branch of resolveProfile: an unknown raw model id (not in the model registry)
|
||||
// typed by the user into a restricted-provider profile must still be rejected
|
||||
// rather than passthrough-sent.
|
||||
func TestNewRejectsRawModelOutsideProviderAllowlist(t *testing.T) {
|
||||
_, err := New(config.ProviderProfile{
|
||||
Name: "opencode-go-anthropic",
|
||||
CatalogID: "opencode-go-anthropic-compatible",
|
||||
APIKey: "sk-test",
|
||||
Model: "totally-custom-not-in-allowlist-12345",
|
||||
}, Options{})
|
||||
if err == nil {
|
||||
t.Fatal("New() error = nil, want unknown raw model rejected for opencode-go-anthropic-compatible")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "does not allow model") {
|
||||
t.Fatalf("error = %q, want allowlist rejection", err.Error())
|
||||
}
|
||||
if !strings.Contains(err.Error(), "totally-custom-not-in-allowlist-12345") {
|
||||
t.Fatalf("error = %q, want the raw model name to appear in the error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewAllowsCuratedModelsForRestrictedProvider proves the gate is not
|
||||
// over-eager: Qwen and MiniMax curated models pass through unchanged for
|
||||
// opencode-go-anthropic-compatible.
|
||||
func TestNewAllowsCuratedModelsForRestrictedProvider(t *testing.T) {
|
||||
for _, model := range []string{"minimax-m3", "qwen3.7-plus", "qwen3.7-max", "minimax-m2.7"} {
|
||||
_, err := New(config.ProviderProfile{
|
||||
Name: "opencode-go-anthropic",
|
||||
CatalogID: "opencode-go-anthropic-compatible",
|
||||
APIKey: "sk-test",
|
||||
Model: model,
|
||||
}, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v, want curated model %q to be allowed", err, model)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewAllowsAnyModelForUnrestrictedProvider is the regression guard:
|
||||
// unrestricted providers (no catalog or default catalog id) must continue to
|
||||
// accept any model id, including those outside the opencode allowlist.
|
||||
func TestNewAllowsAnyModelForUnrestrictedProvider(t *testing.T) {
|
||||
_, err := New(config.ProviderProfile{
|
||||
Name: "anthropic",
|
||||
ProviderKind: config.ProviderKindAnthropic,
|
||||
APIKey: "sk-ant",
|
||||
Model: "claude-sonnet-4.5",
|
||||
}, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v, want unrestricted anthropic provider to accept claude-sonnet-4.5", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRuntimeMetadataRejectsModelOutsideProviderAllowlist guards the
|
||||
// read-only metadata path (called from TUI command_center, exec, provider
|
||||
// health, context report). The metadata path must enforce the same gate as
|
||||
// New() so a config-time override cannot escape through it.
|
||||
func TestResolveRuntimeMetadataRejectsModelOutsideProviderAllowlist(t *testing.T) {
|
||||
_, err := ResolveRuntimeMetadata(config.ProviderProfile{
|
||||
Name: "opencode-go-anthropic",
|
||||
CatalogID: "opencode-go-anthropic-compatible",
|
||||
APIKey: "sk-test",
|
||||
Model: "claude-sonnet-4.5",
|
||||
}, Options{})
|
||||
if err == nil {
|
||||
t.Fatal("ResolveRuntimeMetadata error = nil, want claude-sonnet-4.5 rejected")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "opencode-go-anthropic-compatible") {
|
||||
t.Fatalf("error = %q, want it to name the provider", err.Error())
|
||||
}
|
||||
if !strings.Contains(err.Error(), "claude-sonnet-4.5") {
|
||||
t.Fatalf("error = %q, want it to name the rejected model", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -200,20 +200,23 @@ func normalizeHostScope(raw string) string {
|
||||
// directory itself or any descendant; a host grant matches its exact normalized
|
||||
// host. A narrower grant never covers a tool-wide request (reqScope == ""), so
|
||||
// such a request re-prompts (fail-safe).
|
||||
//
|
||||
// File and dir comparisons go through filepath.Rel (via scopePathEqual and the
|
||||
// shared pathWithinRoot helper) rather than a plain string comparison, so they
|
||||
// apply the same case-folding as the workspace-boundary check on Windows --
|
||||
// otherwise a persisted grant (including a deny) could be silently bypassed by
|
||||
// spelling the same path with different case.
|
||||
func grantCovers(grant Grant, reqScope string) bool {
|
||||
switch grant.ScopeKind {
|
||||
case ScopeToolWide:
|
||||
return true
|
||||
case ScopeFile:
|
||||
return reqScope != "" && reqScope == grant.Scope
|
||||
return reqScope != "" && scopePathEqual(grant.Scope, reqScope)
|
||||
case ScopeDir:
|
||||
if reqScope == "" || grant.Scope == "" {
|
||||
return false
|
||||
}
|
||||
if reqScope == grant.Scope {
|
||||
return true
|
||||
}
|
||||
return strings.HasPrefix(reqScope, grant.Scope+string(filepath.Separator))
|
||||
return pathWithinRoot(grant.Scope, reqScope)
|
||||
case ScopeHost:
|
||||
return reqScope != "" && normalizeHostScope(reqScope) == normalizeHostScope(grant.Scope)
|
||||
default:
|
||||
@@ -221,6 +224,14 @@ func grantCovers(grant Grant, reqScope string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// scopePathEqual reports whether two absolute, cleaned scope paths refer to
|
||||
// the same file, applying the same case-folding filepath.Rel already uses for
|
||||
// directory-boundary checks (case-insensitive path components on Windows).
|
||||
func scopePathEqual(a, b string) bool {
|
||||
rel, err := filepath.Rel(a, b)
|
||||
return err == nil && rel == "."
|
||||
}
|
||||
|
||||
// scopeSpecificity ranks scope kinds so the most precise covering allow wins when
|
||||
// several grants match the same request.
|
||||
func scopeSpecificity(kind ScopeKind) int {
|
||||
|
||||
@@ -2,6 +2,8 @@ package sandbox
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -234,3 +236,25 @@ func TestGrantCovers(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrantCoversCaseInsensitiveOnWindows guards against a persisted grant
|
||||
// (including a deny) being silently bypassed by spelling the same path with
|
||||
// different case, since Windows path components are case-insensitive.
|
||||
func TestGrantCoversCaseInsensitiveOnWindows(t *testing.T) {
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Skip("path case-insensitivity is a Windows-specific filesystem property")
|
||||
}
|
||||
dir := filepath.Join(string(filepath.Separator)+"proj", "src")
|
||||
file := filepath.Join(dir, "main.go")
|
||||
descendant := filepath.Join(dir, "api", "z.go")
|
||||
|
||||
fileGrant := Grant{Scope: file, ScopeKind: ScopeFile}
|
||||
dirGrant := Grant{Scope: dir, ScopeKind: ScopeDir}
|
||||
|
||||
if !grantCovers(fileGrant, strings.ToUpper(file)) {
|
||||
t.Fatalf("file grant %q should cover differently-cased request %q on Windows", file, strings.ToUpper(file))
|
||||
}
|
||||
if !grantCovers(dirGrant, strings.ToUpper(descendant)) {
|
||||
t.Fatalf("dir grant %q should cover differently-cased descendant %q on Windows", dir, strings.ToUpper(descendant))
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,10 @@ type interactiveProgram struct {
|
||||
suggestion string
|
||||
// windowsOnly limits the match to GOOS == "windows" (e.g. notepad).
|
||||
windowsOnly bool
|
||||
// windowsSuggestion overrides suggestion on GOOS == "windows", for cases
|
||||
// where the default suggestion names POSIX-only tools (cat/head/tail)
|
||||
// that cmd.exe does not have.
|
||||
windowsSuggestion string
|
||||
}
|
||||
|
||||
// interactivePrograms maps a bare command name to its non-interactive guidance.
|
||||
@@ -39,9 +43,21 @@ var interactivePrograms = map[string]interactiveProgram{
|
||||
"emacs": {reason: "emacs opens an interactive session", suggestion: "Use `emacs --batch` for scripting, or the edit_file/write_file tools."},
|
||||
"pico": {reason: "pico is a full-screen editor that waits for keystrokes", suggestion: "Use the edit_file/write_file tools or `sed -i`."},
|
||||
// Pagers.
|
||||
"less": {reason: "less is a pager that waits for navigation keys", suggestion: "Use `cat`, `head`, or `tail -n N` to print file contents non-interactively."},
|
||||
"more": {reason: "more is a pager that waits for navigation keys", suggestion: "Use `cat`, `head`, or `tail -n N` to print file contents non-interactively."},
|
||||
"most": {reason: "most is a pager that waits for navigation keys", suggestion: "Use `cat`, `head`, or `tail -n N` to print file contents non-interactively."},
|
||||
"less": {
|
||||
reason: "less is a pager that waits for navigation keys",
|
||||
suggestion: "Use `cat`, `head`, or `tail -n N` to print file contents non-interactively.",
|
||||
windowsSuggestion: "Use `type` to print file contents non-interactively, or the read_file tool with offset/limit for a partial view.",
|
||||
},
|
||||
"more": {
|
||||
reason: "more is a pager that waits for navigation keys",
|
||||
suggestion: "Use `cat`, `head`, or `tail -n N` to print file contents non-interactively.",
|
||||
windowsSuggestion: "Use `type` to print file contents non-interactively, or the read_file tool with offset/limit for a partial view.",
|
||||
},
|
||||
"most": {
|
||||
reason: "most is a pager that waits for navigation keys",
|
||||
suggestion: "Use `cat`, `head`, or `tail -n N` to print file contents non-interactively.",
|
||||
windowsSuggestion: "Use `type` to print file contents non-interactively, or the read_file tool with offset/limit for a partial view.",
|
||||
},
|
||||
// Process/system monitors.
|
||||
"top": {reason: "top runs a live full-screen dashboard until you quit it", suggestion: "Use `ps aux` (optionally `| head`) for a one-shot snapshot."},
|
||||
"htop": {reason: "htop runs a live full-screen dashboard until you quit it", suggestion: "Use `ps aux` (optionally `| head`) for a one-shot snapshot."},
|
||||
@@ -186,11 +202,15 @@ func DetectInteractiveCommand(command string, goos string) InteractiveCommandRes
|
||||
if hasNonInteractiveFlag(first, fields) {
|
||||
continue
|
||||
}
|
||||
suggestion := program.suggestion
|
||||
if goos == "windows" && program.windowsSuggestion != "" {
|
||||
suggestion = program.windowsSuggestion
|
||||
}
|
||||
return InteractiveCommandResult{
|
||||
Interactive: true,
|
||||
Command: first,
|
||||
Reason: program.reason,
|
||||
Suggestion: program.suggestion,
|
||||
Suggestion: suggestion,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -88,6 +88,28 @@ func TestDetectInteractiveCommandHonorsWindows(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDetectInteractiveCommandPagerSuggestionIsPlatformSpecific covers a fix
|
||||
// for suggesting POSIX-only tools (cat/head/tail) as the escape hatch for a
|
||||
// blocked pager on Windows, where cmd.exe has none of them.
|
||||
func TestDetectInteractiveCommandPagerSuggestionIsPlatformSpecific(t *testing.T) {
|
||||
for _, pager := range []string{"less", "more", "most"} {
|
||||
linux := DetectInteractiveCommand(pager+" notes.txt", "linux")
|
||||
if !linux.Interactive || !strings.Contains(linux.Suggestion, "cat") {
|
||||
t.Fatalf("%s on linux: expected cat/head/tail suggestion, got %+v", pager, linux)
|
||||
}
|
||||
windows := DetectInteractiveCommand(pager+" notes.txt", "windows")
|
||||
if !windows.Interactive {
|
||||
t.Fatalf("%s on windows: expected interactive block, got %+v", pager, windows)
|
||||
}
|
||||
if strings.Contains(windows.Suggestion, "cat") || strings.Contains(windows.Suggestion, "tail") {
|
||||
t.Fatalf("%s on windows: suggestion should not name POSIX-only tools, got %q", pager, windows.Suggestion)
|
||||
}
|
||||
if !strings.Contains(windows.Suggestion, "type") {
|
||||
t.Fatalf("%s on windows: expected type suggestion, got %q", pager, windows.Suggestion)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectInteractiveCommandFindsAcrossSeparators(t *testing.T) {
|
||||
// Interactive commands hidden after a shell operator should still be caught.
|
||||
for _, command := range []string{
|
||||
|
||||
@@ -26,6 +26,8 @@ const (
|
||||
secretRetryDelay = 2 * time.Millisecond
|
||||
)
|
||||
|
||||
var openSecretLockFile = os.OpenFile
|
||||
|
||||
// Crypter encrypts a blob at rest with AES-256-GCM under a per-user random secret
|
||||
// persisted (0600) at secretPath.
|
||||
type Crypter struct {
|
||||
@@ -107,7 +109,7 @@ func createSecretFile(path string) ([]byte, error) {
|
||||
lockPath := path + ".lock"
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < secretRetryAttempts; attempt++ {
|
||||
lock, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
lock, err := openSecretLockFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err == nil {
|
||||
_ = lock.Close()
|
||||
defer os.Remove(lockPath)
|
||||
@@ -118,20 +120,25 @@ func createSecretFile(path string) ([]byte, error) {
|
||||
}
|
||||
return writeNewSecretFile(path)
|
||||
}
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
// On Windows a concurrent holder's os.Remove leaves the lock file in a
|
||||
// "delete pending" state, so an O_EXCL create races it with
|
||||
// ERROR_ACCESS_DENIED (os.ErrPermission) rather than ErrExist. Treat that
|
||||
// as contention and retry too -- mirroring oauth's createSecretFile --
|
||||
// otherwise concurrent secret creation spuriously fails on Windows.
|
||||
if !errors.Is(err, os.ErrExist) && !errors.Is(err, os.ErrPermission) {
|
||||
return nil, fmt.Errorf("securefile: create secret lock: %w", err)
|
||||
}
|
||||
// Remember the lock-creation error itself rather than a subsequent
|
||||
// "secret file doesn't exist yet" read error -- otherwise a real
|
||||
// contention/ACL problem is masked by the expected-while-waiting
|
||||
// ErrNotExist once the retries are exhausted.
|
||||
lastErr = err
|
||||
if data, rerr := readSecretFileRetry(path); rerr == nil {
|
||||
return data, nil
|
||||
} else {
|
||||
lastErr = rerr
|
||||
}
|
||||
time.Sleep(secretRetryDelay)
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, fmt.Errorf("securefile: timed out waiting for secret %s: %w", path, lastErr)
|
||||
}
|
||||
return nil, fmt.Errorf("securefile: timed out waiting for secret lock %s", lockPath)
|
||||
return nil, fmt.Errorf("securefile: timed out waiting for secret %s: %w", path, lastErr)
|
||||
}
|
||||
|
||||
func writeNewSecretFile(path string) ([]byte, error) {
|
||||
|
||||
@@ -7,6 +7,33 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCreateSecretFileRetriesOnPermissionContention(t *testing.T) {
|
||||
secretPath := filepath.Join(t.TempDir(), "k.secret")
|
||||
attempts := 0
|
||||
origOpen := openSecretLockFile
|
||||
openSecretLockFile = func(path string, flag int, perm os.FileMode) (*os.File, error) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
return nil, os.ErrPermission
|
||||
}
|
||||
return os.OpenFile(path, flag, perm)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
openSecretLockFile = origOpen
|
||||
})
|
||||
|
||||
secret, err := createSecretFile(secretPath)
|
||||
if err != nil {
|
||||
t.Fatalf("createSecretFile should retry on permission contention: %v", err)
|
||||
}
|
||||
if len(secret) != secretBytes {
|
||||
t.Fatalf("secret length = %d, want %d", len(secret), secretBytes)
|
||||
}
|
||||
if attempts < 2 {
|
||||
t.Fatalf("expected at least 2 lock attempts, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSealOpenRoundTrip(t *testing.T) {
|
||||
secret := filepath.Join(t.TempDir(), "k.secret")
|
||||
c := NewCrypter(secret)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type ExecMode string
|
||||
@@ -221,11 +222,24 @@ func summarizePayload(payload any) string {
|
||||
text = string(data)
|
||||
}
|
||||
if len(text) > 500 {
|
||||
return text[:500]
|
||||
return truncateUTF8(text, 500)
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
// truncateUTF8 returns the longest prefix of s that is at most n bytes,
|
||||
// backing off to the nearest rune boundary so a multi-byte character isn't
|
||||
// split — a split rune here would embed invalid UTF-8 into the exec prompt.
|
||||
func truncateUTF8(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
for n > 0 && !utf8.RuneStart(s[n]) {
|
||||
n--
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
func extractText(value any) string {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestStoreCreatesAppendsListsAndReadsEvents(t *testing.T) {
|
||||
@@ -640,6 +641,23 @@ func TestFormatExecPromptTruncatesConversationMessagesAfterFilteringNoise(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
// A truncation cut at a raw byte offset can land in the middle of a
|
||||
// multi-byte UTF-8 rune (e.g. CJK text), embedding invalid UTF-8 into the
|
||||
// exec prompt. summarizePayload must back off to a rune boundary instead.
|
||||
func TestSummarizePayloadTruncatesOnRuneBoundary(t *testing.T) {
|
||||
content := strings.Repeat("中文", 300) // 900 bytes of 3-byte runes
|
||||
payload := json.RawMessage(fmt.Sprintf(`{"role":"user","content":%q}`, content))
|
||||
|
||||
got := summarizePayload(payload)
|
||||
|
||||
if !utf8.ValidString(got) {
|
||||
t.Fatalf("summarizePayload produced invalid UTF-8: %q", got)
|
||||
}
|
||||
if len(got) > 500 {
|
||||
t.Fatalf("expected summary to be at most 500 bytes, got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareExecPersistsSpecialistMetadataForNewSession(t *testing.T) {
|
||||
store := NewStore(StoreOptions{RootDir: t.TempDir(), Now: fixedClock("2026-06-04T14:30:00Z")})
|
||||
|
||||
|
||||
@@ -77,9 +77,9 @@ type DuplicateName struct {
|
||||
// winner regardless of sort stability. Use Duplicates to surface a warning about
|
||||
// any such collisions.
|
||||
//
|
||||
// NOTE: Load currently scans a single root (ZERO_SKILLS_DIR / the data dir).
|
||||
// Plugin-declared skill paths (the plugins manifest "skills" array) are NOT yet
|
||||
// merged into this lookup; multi-root loading is tracked as a separate feature.
|
||||
// NOTE: Load scans one root. Agent startup merges plugin-declared skill roots
|
||||
// separately during plugin activation, so the runtime skill surface can include
|
||||
// both the default directory and skills bundled by active plugins.
|
||||
func Load(dir string) ([]Skill, error) {
|
||||
skills, _, err := load(dir)
|
||||
return skills, err
|
||||
|
||||
@@ -152,7 +152,7 @@ func TestDetectShellCommandIssueAllowsWindowsCDSwitch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectShellCommandIssueRequiresActualLSCommand(t *testing.T) {
|
||||
func TestDetectShellCommandIssueFlagsWindowsBadCommands(t *testing.T) {
|
||||
for _, command := range []string{
|
||||
`echo false ls -la`,
|
||||
`echo list -items`,
|
||||
@@ -172,6 +172,22 @@ func TestDetectShellCommandIssueRequiresActualLSCommand(t *testing.T) {
|
||||
t.Fatalf("expected actual ls command to be flagged for %q", command)
|
||||
}
|
||||
}
|
||||
|
||||
// Common Unix filters used for truncation (e.g. gh output) must be caught.
|
||||
for _, command := range []string{
|
||||
`gh pr view 465 | head -200`,
|
||||
`some-cmd | tail -n 10`,
|
||||
`cat file | head -5`,
|
||||
`ps aux | head`,
|
||||
// Issue #467's exact reported case: `| cat` alone (no other POSIX
|
||||
// utility in the pipeline) must also be flagged before execution.
|
||||
`gh pr view 465 --json reviews,comments -q '.reviews | length' 2>&1 | cat`,
|
||||
`some-cmd | cat`,
|
||||
} {
|
||||
if issue := detectShellCommandIssue(command, "windows"); issue == nil {
|
||||
t.Fatalf("expected POSIX filter command to be flagged for %q", command)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectShellCommandIssueFlagsPipedPosixUtilities(t *testing.T) {
|
||||
@@ -191,6 +207,55 @@ func TestDetectShellCommandIssueFlagsPipedPosixUtilities(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectShellCommandIssueIgnoresQuotedPipeText(t *testing.T) {
|
||||
for _, command := range []string{
|
||||
`gh pr comment --body "please do not use | cat here"`,
|
||||
`git commit -m "grep for TODO later"`,
|
||||
`git commit -m "fix | cat pipe"`,
|
||||
} {
|
||||
if issue := detectShellCommandIssue(command, "windows"); issue != nil {
|
||||
t.Fatalf("expected POSIX-utility text inside a quoted argument to pass for %q, got %#v", command, issue)
|
||||
}
|
||||
}
|
||||
|
||||
// The same utility names, actually piped (unquoted), must still be flagged.
|
||||
for _, command := range []string{
|
||||
`echo test | cat`,
|
||||
`echo test | grep TODO`,
|
||||
} {
|
||||
if issue := detectShellCommandIssue(command, "windows"); issue == nil {
|
||||
t.Fatalf("expected unquoted POSIX utility pipeline to still be flagged for %q", command)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectShellCommandIssueIgnoresCaretEscapedMetachars(t *testing.T) {
|
||||
for _, command := range []string{
|
||||
`foo^|cat`,
|
||||
`echo a^&cat`,
|
||||
`echo a^;cat`,
|
||||
} {
|
||||
if issue := detectShellCommandIssue(command, "windows"); issue != nil {
|
||||
t.Fatalf("expected caret-escaped metachar to read as a literal for %q, got %#v", command, issue)
|
||||
}
|
||||
}
|
||||
|
||||
// The same shape, unescaped, must still be flagged.
|
||||
if issue := detectShellCommandIssue(`foo|cat`, "windows"); issue == nil {
|
||||
t.Fatal("expected unescaped pipe into cat to still be flagged")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectShellCommandIssueSuggestionOmitsBlockedMore(t *testing.T) {
|
||||
issue := detectShellCommandIssue(`some_command | wc -l`, "windows")
|
||||
if issue == nil {
|
||||
t.Fatal("expected POSIX utility pipeline to be flagged")
|
||||
}
|
||||
if strings.Contains(issue.Suggestion, "more") {
|
||||
t.Fatalf("suggestion should not recommend more, which is itself blocked as an interactive pager on Windows: %q", issue.Suggestion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectShellCommandIssueAllowsUnrelatedCommands(t *testing.T) {
|
||||
for _, command := range []string{
|
||||
`git log --oneline`,
|
||||
@@ -207,6 +272,31 @@ func TestDetectShellCommandIssueAllowsUnrelatedCommands(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Operator/utility-name text that only appears inside a quoted argument must
|
||||
// not be mistaken for real shell syntax — e.g. `echo "foo; ls -la"` is a
|
||||
// harmless echo, not a bash-style `ls` invocation, and quoted `head`/`grep`
|
||||
// text isn't a piped POSIX utility.
|
||||
func TestDetectShellCommandIssueIgnoresQuotedContent(t *testing.T) {
|
||||
for _, command := range []string{
|
||||
`echo "foo; ls -la"`,
|
||||
`echo "run head over the file"`,
|
||||
`echo 'ls -la is a common bash command'`,
|
||||
`echo "grep for the pattern"`,
|
||||
} {
|
||||
if issue := detectShellCommandIssue(command, "windows"); issue != nil {
|
||||
t.Fatalf("expected quoted content to be ignored for %q, got %#v", command, issue)
|
||||
}
|
||||
}
|
||||
|
||||
// Sanity: the same utility names still get flagged when they're NOT quoted.
|
||||
// cmd.exe (unlike bash) does not treat ; as a statement separator, so
|
||||
// `echo "foo" ; ls -la` is a single echo invocation with literal
|
||||
// arguments, not a real ls invocation, and must not be flagged.
|
||||
if issue := detectShellCommandIssue(`git log | head`, "windows"); issue == nil {
|
||||
t.Fatalf("expected unquoted shell syntax to still be flagged for %q", `git log | head`)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectShellOutputIssueAddsWindowsSyntaxHint(t *testing.T) {
|
||||
issue := detectShellOutputIssue("The syntax of the command is incorrect.", "windows")
|
||||
if issue == nil {
|
||||
@@ -668,8 +758,15 @@ func TestBashToolBlocksInteractiveCommandThroughSandbox(t *testing.T) {
|
||||
if result.Status != StatusError {
|
||||
t.Fatalf("expected error status, got %s: %s", result.Status, result.Output)
|
||||
}
|
||||
if !strings.Contains(result.Output, "interactive") || !strings.Contains(result.Output, "cat") {
|
||||
t.Fatalf("expected pager guard message with cat suggestion, got %q", result.Output)
|
||||
// The suggested non-interactive alternative is platform-specific: cat
|
||||
// doesn't exist on Windows cmd.exe, so the guard suggests `type` there
|
||||
// instead (see safe_command.go's windowsSuggestion).
|
||||
wantSuggestion := "cat"
|
||||
if runtime.GOOS == "windows" {
|
||||
wantSuggestion = "type"
|
||||
}
|
||||
if !strings.Contains(result.Output, "interactive") || !strings.Contains(result.Output, wantSuggestion) {
|
||||
t.Fatalf("expected pager guard message with %q suggestion, got %q", wantSuggestion, result.Output)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -76,6 +76,45 @@ func TestReadFileToolAppliesByteBudget(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFileToolTracksWholeFileHashWhenReadingSmallRange(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
var builder strings.Builder
|
||||
for i := 1; i <= 2000; i++ {
|
||||
builder.WriteString("line ")
|
||||
builder.WriteString(strings.Repeat("x", 16))
|
||||
builder.WriteString("\n")
|
||||
}
|
||||
content := builder.String()
|
||||
path := filepath.Join(root, "large.txt")
|
||||
writeTestFile(t, path, content)
|
||||
|
||||
tracker := NewFileTracker()
|
||||
tool := NewReadFileTool(root).(optionsAwareTool)
|
||||
result := tool.RunWithOptions(context.Background(), map[string]any{
|
||||
"path": "large.txt",
|
||||
"start_line": 1000,
|
||||
"max_lines": 2,
|
||||
}, RunOptions{FileTracker: tracker})
|
||||
|
||||
if result.Status != StatusOK {
|
||||
t.Fatalf("expected ok status, got %s: %s", result.Status, result.Output)
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
version, ok := tracker.Version(resolved)
|
||||
if !ok {
|
||||
t.Fatal("expected read_file to record a file version")
|
||||
}
|
||||
if version.Hash != HashContent([]byte(content)) {
|
||||
t.Fatalf("hash = %q, want %q", version.Hash, HashContent([]byte(content)))
|
||||
}
|
||||
if err := tracker.CheckConflict(resolved, []byte(content+"changed")); err != ErrFileChangedOnDisk {
|
||||
t.Fatalf("changed content should conflict, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFileToolRejectsOutsideWorkspace(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := filepath.Join(t.TempDir(), "secret.txt")
|
||||
@@ -205,6 +244,33 @@ func TestGrepToolMakesHeadLimitTruncationVisible(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrepToolCountsAllLinesWhileKeepingHeadLimitSmall(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
var builder strings.Builder
|
||||
for i := 0; i < 100; i++ {
|
||||
builder.WriteString("needle ")
|
||||
builder.WriteString(strings.Repeat("x", 8))
|
||||
builder.WriteString("\n")
|
||||
}
|
||||
writeTestFile(t, filepath.Join(root, "notes.txt"), builder.String())
|
||||
|
||||
result := NewGrepTool(root).Run(context.Background(), map[string]any{
|
||||
"pattern": "needle",
|
||||
"path": ".",
|
||||
"head_limit": 3,
|
||||
})
|
||||
|
||||
if result.Status != StatusOK || !result.Truncated {
|
||||
t.Fatalf("expected ok+truncated, got status=%s truncated=%v output=%q", result.Status, result.Truncated, result.Output)
|
||||
}
|
||||
if !strings.Contains(result.Output, "[truncated: showing first 3 of 100 matches") {
|
||||
t.Fatalf("expected exact total match count, got %q", result.Output)
|
||||
}
|
||||
if strings.Contains(result.Output, "notes.txt:4:") {
|
||||
t.Fatalf("head_limit leaked fourth result: %q", result.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// A glob is matched relative to the search directory, not the workspace root, so
|
||||
// `path=subdir glob=*.go` finds subdir/a.go (as "a.go"). Previously the glob was
|
||||
// matched against the workspace-relative "subdir/a.go", so a non-recursive "*.go"
|
||||
@@ -255,6 +321,22 @@ func TestGrepToolSupportsFilesAndCountModes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrepCountModeCountsMultipleHitsPerLine(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeTestFile(t, filepath.Join(root, "notes.txt"), "needle needle\nneedle\n")
|
||||
|
||||
count := NewGrepTool(root).Run(context.Background(), map[string]any{
|
||||
"pattern": "needle",
|
||||
"output_mode": "count",
|
||||
})
|
||||
if count.Status != StatusOK {
|
||||
t.Fatalf("expected count result, got %s: %s", count.Status, count.Output)
|
||||
}
|
||||
if count.Output != "3 matches found" {
|
||||
t.Fatalf("expected count output, got %q", count.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// Finding 1: grep must not follow an in-workspace symlink that points to a file
|
||||
// OUTSIDE the workspace (confinement bypass). The symlinked secret must not be
|
||||
// searched or returned, mirroring read_file's EvalSymlinks confinement.
|
||||
|
||||
@@ -42,10 +42,16 @@ func NewFileTracker() *FileTracker {
|
||||
|
||||
// Record stores the version of absPath given its content and optional stat info.
|
||||
func (tracker *FileTracker) Record(absPath string, content []byte, info os.FileInfo) {
|
||||
tracker.RecordHash(absPath, HashContent(content), info)
|
||||
}
|
||||
|
||||
// RecordHash stores the version of absPath when the caller already computed
|
||||
// the same HashContent-compatible SHA-256 while reading the file.
|
||||
func (tracker *FileTracker) RecordHash(absPath string, hash string, info os.FileInfo) {
|
||||
if tracker == nil {
|
||||
return
|
||||
}
|
||||
version := FileVersion{Hash: HashContent(content)}
|
||||
version := FileVersion{Hash: hash}
|
||||
if info != nil {
|
||||
version.Size = info.Size()
|
||||
version.MTime = info.ModTime()
|
||||
|
||||
@@ -34,6 +34,18 @@ func TestFileTrackerRecordsAndReadsBackVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileTrackerRecordHashMatchesRecord(t *testing.T) {
|
||||
tracker := NewFileTracker()
|
||||
content := []byte("streamed content")
|
||||
tracker.RecordHash("/repo/x.txt", HashContent(content), nil)
|
||||
if err := tracker.CheckConflict("/repo/x.txt", content); err != nil {
|
||||
t.Fatalf("recorded hash should match content, got %v", err)
|
||||
}
|
||||
if err := tracker.CheckConflict("/repo/x.txt", []byte("changed")); err != ErrFileChangedOnDisk {
|
||||
t.Fatalf("changed content should conflict, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckConflictAllowsUntrackedPath(t *testing.T) {
|
||||
tracker := NewFileTracker()
|
||||
// No Record call: a first-touch write has no baseline to conflict against.
|
||||
|
||||
+203
-124
@@ -1,8 +1,10 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
@@ -138,84 +140,35 @@ func (tool grepTool) runWith(ctx context.Context, args map[string]any, exclude r
|
||||
}
|
||||
}
|
||||
|
||||
files, err := grepFiles(ctx, resolvedRoot, target, globMatcher, exclude)
|
||||
if err != nil {
|
||||
if res, ok := searchCancelledResult("grep", err); ok {
|
||||
return res
|
||||
}
|
||||
return errorResult("Error running grep: " + err.Error())
|
||||
}
|
||||
|
||||
// resolveScopedPath returns an absolute displayRoot when the target resolved
|
||||
// to an extra (non-workspace) granted root; emit absolute match paths there so
|
||||
// they survive a round-trip through read_file/edit_file (mirrors glob).
|
||||
matches, err := collectGrepMatches(ctx, resolvedRoot, filepath.IsAbs(displayRoot), files, compiled)
|
||||
if err != nil {
|
||||
if res, ok := searchCancelledResult("grep", err); ok {
|
||||
return res
|
||||
}
|
||||
return errorResult("Error running grep: " + err.Error())
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
if outputMode == "count" {
|
||||
return okResult("0 matches found")
|
||||
}
|
||||
return okResult("No matches found.")
|
||||
}
|
||||
|
||||
absolutePaths := filepath.IsAbs(displayRoot)
|
||||
switch outputMode {
|
||||
case "count":
|
||||
total := 0
|
||||
for _, match := range matches {
|
||||
total += match.hits
|
||||
collector := &grepCountCollector{}
|
||||
if err := scanGrepMatches(ctx, resolvedRoot, target, globMatcher, exclude, absolutePaths, exactGrepLineMatcher(compiled), collector.collect); err != nil {
|
||||
if res, ok := searchCancelledResult("grep", err); ok {
|
||||
return res
|
||||
}
|
||||
return errorResult("Error running grep: " + err.Error())
|
||||
}
|
||||
return okResult(fmt.Sprintf("%d matches found", total))
|
||||
return collector.result()
|
||||
case "files_with_matches":
|
||||
seen := map[string]bool{}
|
||||
files := []string{}
|
||||
for _, match := range matches {
|
||||
if !seen[match.file] {
|
||||
seen[match.file] = true
|
||||
files = append(files, match.file)
|
||||
collector := &grepFileListCollector{}
|
||||
if err := scanGrepMatches(ctx, resolvedRoot, target, globMatcher, exclude, absolutePaths, presenceGrepLineMatcher(compiled), collector.collect); err != nil {
|
||||
if res, ok := searchCancelledResult("grep", err); ok {
|
||||
return res
|
||||
}
|
||||
return errorResult("Error running grep: " + err.Error())
|
||||
}
|
||||
sort.Strings(files)
|
||||
budgeted := applyOutputBudget(strings.Join(files, "\n"), searchOutputBudgetBytes, "narrow path/glob/pattern to continue")
|
||||
meta := outputBudgetMeta(budgeted)
|
||||
if budgeted.Truncated {
|
||||
meta["truncated"] = "true"
|
||||
meta["truncation_reason"] = "byte_budget"
|
||||
}
|
||||
return Result{Status: StatusOK, Output: budgeted.Output, Truncated: budgeted.Truncated, Meta: meta}
|
||||
return collector.result()
|
||||
default:
|
||||
lines := make([]string, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
if len(lines) >= headLimit {
|
||||
break
|
||||
collector := &grepContentCollector{headLimit: headLimit}
|
||||
if err := scanGrepMatches(ctx, resolvedRoot, target, globMatcher, exclude, absolutePaths, presenceGrepLineMatcher(compiled), collector.collect); err != nil {
|
||||
if res, ok := searchCancelledResult("grep", err); ok {
|
||||
return res
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("%s:%d: %s", match.file, match.line, match.text))
|
||||
}
|
||||
truncated := len(matches) > headLimit
|
||||
output := strings.Join(lines, "\n")
|
||||
if truncated {
|
||||
output += fmt.Sprintf("\n\n[truncated: showing first %d of %d matches; narrow path/glob/pattern or increase head_limit]", len(lines), len(matches))
|
||||
}
|
||||
budgeted := applyOutputBudget(output, searchOutputBudgetBytes, "narrow path/glob/pattern or increase head_limit")
|
||||
meta := outputBudgetMeta(budgeted)
|
||||
if truncated || budgeted.Truncated {
|
||||
meta["truncated"] = "true"
|
||||
if budgeted.Truncated {
|
||||
meta["truncation_reason"] = "byte_budget"
|
||||
} else {
|
||||
meta["truncation_reason"] = "head_limit"
|
||||
}
|
||||
}
|
||||
return Result{
|
||||
Status: StatusOK,
|
||||
Output: budgeted.Output,
|
||||
Truncated: truncated || budgeted.Truncated,
|
||||
Meta: meta,
|
||||
return errorResult("Error running grep: " + err.Error())
|
||||
}
|
||||
return collector.result()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,33 +223,32 @@ func confineGrepFile(resolvedRoot string, path string) (string, string, bool) {
|
||||
return filepath.ToSlash(relative), resolved, true
|
||||
}
|
||||
|
||||
func grepFiles(ctx context.Context, resolvedRoot string, target string, globMatcher *regexp.Regexp, exclude readExcluder) ([]string, error) {
|
||||
func walkGrepFiles(ctx context.Context, resolvedRoot string, target string, globMatcher *regexp.Regexp, exclude readExcluder, visit func(string) error) error {
|
||||
info, err := os.Stat(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
relative, _, ok := confineGrepFile(resolvedRoot, target)
|
||||
if !ok {
|
||||
return []string{}, nil
|
||||
return nil
|
||||
}
|
||||
if shouldSkipWorkspaceFile(relative) {
|
||||
return []string{}, nil
|
||||
return nil
|
||||
}
|
||||
if exclude.fileExcluded(target) {
|
||||
return []string{}, nil
|
||||
return nil
|
||||
}
|
||||
// A single explicit file is matched by its base name so a pattern like
|
||||
// "*.go" applies regardless of how deep the file sits under the workspace.
|
||||
if globMatcher == nil || globMatcher.MatchString(filepath.Base(target)) {
|
||||
return []string{target}, nil
|
||||
return visit(target)
|
||||
}
|
||||
return []string{}, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
files := []string{}
|
||||
err = filepath.WalkDir(target, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
return filepath.WalkDir(target, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
// Checked first, ahead of walkErr: an unscoped search over a large tree
|
||||
// (e.g. a broad parent directory, not just the workspace) can run long
|
||||
// enough that cancelling the run must stop the walk promptly rather than
|
||||
@@ -347,62 +299,189 @@ func grepFiles(ctx context.Context, resolvedRoot string, target string, globMatc
|
||||
globPath = filepath.ToSlash(rel)
|
||||
}
|
||||
if globMatcher == nil || globMatcher.MatchString(globPath) {
|
||||
files = append(files, path)
|
||||
if err := visit(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(files)
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func collectGrepMatches(ctx context.Context, resolvedRoot string, absolutePaths bool, files []string, compiled *regexp.Regexp) ([]grepMatch, error) {
|
||||
matches := []grepMatch{}
|
||||
for _, file := range files {
|
||||
// Checked per file, matching the walk's per-entry check in grepFiles: a
|
||||
// broad search can already have collected many files before the run is
|
||||
// cancelled, and reading/regex-scanning each one is real work that must
|
||||
// stop promptly rather than running to completion.
|
||||
type grepLineMatcher func(string) (int, bool)
|
||||
|
||||
func presenceGrepLineMatcher(compiled *regexp.Regexp) grepLineMatcher {
|
||||
return func(line string) (int, bool) {
|
||||
if !compiled.MatchString(line) {
|
||||
return 0, false
|
||||
}
|
||||
return 1, true
|
||||
}
|
||||
}
|
||||
|
||||
func exactGrepLineMatcher(compiled *regexp.Regexp) grepLineMatcher {
|
||||
return func(line string) (int, bool) {
|
||||
matches := compiled.FindAllStringIndex(line, -1)
|
||||
if len(matches) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return len(matches), true
|
||||
}
|
||||
}
|
||||
|
||||
func scanGrepMatches(ctx context.Context, resolvedRoot string, target string, globMatcher *regexp.Regexp, exclude readExcluder, absolutePaths bool, matcher grepLineMatcher, emit func(grepMatch)) error {
|
||||
return walkGrepFiles(ctx, resolvedRoot, target, globMatcher, exclude, func(file string) error {
|
||||
return scanGrepFile(ctx, resolvedRoot, absolutePaths, file, matcher, emit)
|
||||
})
|
||||
}
|
||||
|
||||
func scanGrepFile(ctx context.Context, resolvedRoot string, absolutePaths bool, file string, matcher grepLineMatcher, emit func(grepMatch)) error {
|
||||
// Re-confine at read time (defense-in-depth) AND to compute the clean
|
||||
// workspace-relative path used in output.
|
||||
relative, resolvedPath, ok := confineGrepFile(resolvedRoot, file)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if shouldSkipWorkspaceFile(relative) {
|
||||
return nil
|
||||
}
|
||||
fileLabel := relative
|
||||
if absolutePaths {
|
||||
// Extra-root search: report the absolute, symlink-resolved path
|
||||
// confineGrepFile already validated, so a bare workspace-relative
|
||||
// name can't resolve under the workspace and hit the wrong file when
|
||||
// the same name exists in both roots.
|
||||
fileLabel = filepath.ToSlash(resolvedPath)
|
||||
}
|
||||
|
||||
// Read the symlink-RESOLVED path that confineGrepFile validated, not the
|
||||
// raw candidate, so a symlink swapped in after the check can't escape.
|
||||
handle, err := os.Open(resolvedPath)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer handle.Close()
|
||||
|
||||
reader := bufio.NewReader(handle)
|
||||
lineNumber := 1
|
||||
sawLine := false
|
||||
lastEnded := false
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return matches, err
|
||||
return err
|
||||
}
|
||||
// Re-confine at read time (defense-in-depth) AND to compute the clean
|
||||
// workspace-relative path used in output.
|
||||
relative, resolvedPath, ok := confineGrepFile(resolvedRoot, file)
|
||||
if !ok {
|
||||
continue
|
||||
raw, ended, err := readRawLine(reader)
|
||||
if err == io.EOF {
|
||||
if !sawLine || lastEnded {
|
||||
emitGrepLine(matcher, fileLabel, lineNumber, "", emit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if shouldSkipWorkspaceFile(relative) {
|
||||
continue
|
||||
}
|
||||
// Read the symlink-RESOLVED path that confineGrepFile validated, not the
|
||||
// raw candidate, so a symlink swapped in after the check can't escape.
|
||||
content, err := os.ReadFile(resolvedPath)
|
||||
if err != nil {
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
for index, line := range strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n") {
|
||||
lineMatches := compiled.FindAllStringIndex(line, -1)
|
||||
if len(lineMatches) == 0 {
|
||||
continue
|
||||
}
|
||||
fileLabel := relative
|
||||
if absolutePaths {
|
||||
// Extra-root search: report the absolute, symlink-resolved path
|
||||
// confineGrepFile already validated, so a bare workspace-relative
|
||||
// name can't resolve under the workspace and hit the wrong file when
|
||||
// the same name exists in both roots.
|
||||
fileLabel = filepath.ToSlash(resolvedPath)
|
||||
}
|
||||
matches = append(matches, grepMatch{
|
||||
file: fileLabel,
|
||||
line: index + 1,
|
||||
text: strings.TrimRight(line, "\r"),
|
||||
hits: len(lineMatches),
|
||||
})
|
||||
sawLine = true
|
||||
lastEnded = ended
|
||||
line := strings.TrimRight(string(trimLineBreak(raw, ended)), "\r")
|
||||
emitGrepLine(matcher, fileLabel, lineNumber, line, emit)
|
||||
lineNumber++
|
||||
}
|
||||
}
|
||||
|
||||
func emitGrepLine(matcher grepLineMatcher, fileLabel string, lineNumber int, line string, emit func(grepMatch)) {
|
||||
hits, ok := matcher(line)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
emit(grepMatch{
|
||||
file: fileLabel,
|
||||
line: lineNumber,
|
||||
text: line,
|
||||
hits: hits,
|
||||
})
|
||||
}
|
||||
|
||||
type grepCountCollector struct {
|
||||
hits int
|
||||
}
|
||||
|
||||
func (collector *grepCountCollector) collect(match grepMatch) {
|
||||
collector.hits += match.hits
|
||||
}
|
||||
|
||||
func (collector *grepCountCollector) result() Result {
|
||||
return okResult(fmt.Sprintf("%d matches found", collector.hits))
|
||||
}
|
||||
|
||||
type grepFileListCollector struct {
|
||||
files []string
|
||||
seen map[string]bool
|
||||
}
|
||||
|
||||
func (collector *grepFileListCollector) collect(match grepMatch) {
|
||||
if collector.seen == nil {
|
||||
collector.seen = map[string]bool{}
|
||||
}
|
||||
if collector.seen[match.file] {
|
||||
return
|
||||
}
|
||||
collector.seen[match.file] = true
|
||||
collector.files = append(collector.files, match.file)
|
||||
}
|
||||
|
||||
func (collector *grepFileListCollector) result() Result {
|
||||
if len(collector.files) == 0 {
|
||||
return okResult("No matches found.")
|
||||
}
|
||||
sort.Strings(collector.files)
|
||||
budgeted := applyOutputBudget(strings.Join(collector.files, "\n"), searchOutputBudgetBytes, "narrow path/glob/pattern to continue")
|
||||
meta := outputBudgetMeta(budgeted)
|
||||
if budgeted.Truncated {
|
||||
meta["truncated"] = "true"
|
||||
meta["truncation_reason"] = "byte_budget"
|
||||
}
|
||||
return Result{Status: StatusOK, Output: budgeted.Output, Truncated: budgeted.Truncated, Meta: meta}
|
||||
}
|
||||
|
||||
type grepContentCollector struct {
|
||||
headLimit int
|
||||
matches []grepMatch
|
||||
matchingLines int
|
||||
}
|
||||
|
||||
func (collector *grepContentCollector) collect(match grepMatch) {
|
||||
collector.matchingLines++
|
||||
if len(collector.matches) < collector.headLimit {
|
||||
collector.matches = append(collector.matches, match)
|
||||
}
|
||||
}
|
||||
|
||||
func (collector *grepContentCollector) result() Result {
|
||||
if collector.matchingLines == 0 {
|
||||
return okResult("No matches found.")
|
||||
}
|
||||
lines := make([]string, 0, len(collector.matches))
|
||||
for _, match := range collector.matches {
|
||||
lines = append(lines, fmt.Sprintf("%s:%d: %s", match.file, match.line, match.text))
|
||||
}
|
||||
truncated := collector.matchingLines > collector.headLimit
|
||||
output := strings.Join(lines, "\n")
|
||||
if truncated {
|
||||
output += fmt.Sprintf("\n\n[truncated: showing first %d of %d matches; narrow path/glob/pattern or increase head_limit]", len(lines), collector.matchingLines)
|
||||
}
|
||||
budgeted := applyOutputBudget(output, searchOutputBudgetBytes, "narrow path/glob/pattern or increase head_limit")
|
||||
meta := outputBudgetMeta(budgeted)
|
||||
if truncated || budgeted.Truncated {
|
||||
meta["truncated"] = "true"
|
||||
if budgeted.Truncated {
|
||||
meta["truncation_reason"] = "byte_budget"
|
||||
} else {
|
||||
meta["truncation_reason"] = "head_limit"
|
||||
}
|
||||
}
|
||||
return matches, nil
|
||||
return Result{
|
||||
Status: StatusOK,
|
||||
Output: budgeted.Output,
|
||||
Truncated: truncated || budgeted.Truncated,
|
||||
Meta: meta,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
)
|
||||
|
||||
func readRawLine(reader *bufio.Reader) ([]byte, bool, error) {
|
||||
var line []byte
|
||||
for {
|
||||
fragment, err := reader.ReadSlice('\n')
|
||||
if len(fragment) > 0 {
|
||||
if line != nil || err == bufio.ErrBufferFull {
|
||||
line = append(line, fragment...)
|
||||
} else {
|
||||
line = fragment
|
||||
}
|
||||
}
|
||||
switch err {
|
||||
case nil:
|
||||
return line, true, nil
|
||||
case bufio.ErrBufferFull:
|
||||
continue
|
||||
case io.EOF:
|
||||
if len(line) > 0 {
|
||||
return line, false, nil
|
||||
}
|
||||
return nil, false, io.EOF
|
||||
default:
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func trimLineBreak(raw []byte, ended bool) []byte {
|
||||
if !ended || len(raw) == 0 || raw[len(raw)-1] != '\n' {
|
||||
return raw
|
||||
}
|
||||
raw = raw[:len(raw)-1]
|
||||
if len(raw) > 0 && raw[len(raw)-1] == '\r' {
|
||||
raw = raw[:len(raw)-1]
|
||||
}
|
||||
return raw
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package tools
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -52,3 +53,45 @@ func estimatedTokensFromBytes(bytes int) int {
|
||||
}
|
||||
return (bytes + 3) / 4
|
||||
}
|
||||
|
||||
type outputBudgetBuilder struct {
|
||||
builder strings.Builder
|
||||
rawBytes int
|
||||
maxBytes int
|
||||
hint string
|
||||
}
|
||||
|
||||
func newOutputBudgetBuilder(maxBytes int, hint string) *outputBudgetBuilder {
|
||||
return &outputBudgetBuilder{maxBytes: maxBytes, hint: hint}
|
||||
}
|
||||
|
||||
func (builder *outputBudgetBuilder) WriteString(value string) {
|
||||
builder.rawBytes += len(value)
|
||||
if builder.maxBytes <= 0 || builder.builder.Len() >= builder.maxBytes {
|
||||
return
|
||||
}
|
||||
remaining := builder.maxBytes - builder.builder.Len()
|
||||
builder.builder.WriteString(utf8Prefix(value, remaining))
|
||||
}
|
||||
|
||||
func (builder *outputBudgetBuilder) Result() outputBudgetResult {
|
||||
output := builder.builder.String()
|
||||
result := outputBudgetResult{
|
||||
Output: output,
|
||||
RawBytes: builder.rawBytes,
|
||||
EmittedBytes: len(output),
|
||||
}
|
||||
if builder.maxBytes <= 0 || builder.rawBytes <= builder.maxBytes {
|
||||
return result
|
||||
}
|
||||
|
||||
marker := fmt.Sprintf("\n\n[truncated: output exceeded %d bytes; %s]", builder.maxBytes, builder.hint)
|
||||
budget := builder.maxBytes - len(marker)
|
||||
if budget < 0 {
|
||||
budget = 0
|
||||
}
|
||||
result.Output = utf8Prefix(output, budget) + marker
|
||||
result.Truncated = true
|
||||
result.EmittedBytes = len(result.Output)
|
||||
return result
|
||||
}
|
||||
+96
-23
@@ -1,8 +1,12 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -68,7 +72,7 @@ func (tool readFileTool) RunWithOptions(_ context.Context, args map[string]any,
|
||||
return errorResult("Error reading file " + requestedPath + ": " + err.Error())
|
||||
}
|
||||
|
||||
content, err := os.ReadFile(absolutePath)
|
||||
stats, err := scanReadFileStats(absolutePath)
|
||||
if err != nil {
|
||||
return errorResult("Error reading file " + relativePath + ": " + err.Error())
|
||||
}
|
||||
@@ -76,17 +80,15 @@ func (tool readFileTool) RunWithOptions(_ context.Context, args map[string]any,
|
||||
// write_file read) so a later write can detect an out-of-Zero modification.
|
||||
// Stat is best-effort: a missing FileInfo only drops the diagnostic size/mtime,
|
||||
// not the authoritative content hash.
|
||||
info, _ := os.Stat(absolutePath)
|
||||
options.FileTracker.Record(absolutePath, content, info)
|
||||
options.FileTracker.RecordHash(absolutePath, stats.hash, stats.info)
|
||||
|
||||
normalizedContent := strings.ReplaceAll(string(content), "\r\n", "\n")
|
||||
normalizedContent = strings.TrimSuffix(normalizedContent, "\n")
|
||||
lines := strings.Split(normalizedContent, "\n")
|
||||
total := len(lines)
|
||||
return renderReadFileRange(absolutePath, relativePath, stats.lines, startLine, endLine, maxLines)
|
||||
}
|
||||
|
||||
func renderReadFileRange(absolutePath string, relativePath string, total int, startLine int, endLine int, maxLines int) Result {
|
||||
if startLine > total {
|
||||
return okResult(fmt.Sprintf("File: %s\n(start_line %d is past the end of the file, which has %d lines)", relativePath, startLine, total))
|
||||
}
|
||||
|
||||
if endLine == 0 || endLine > total {
|
||||
endLine = total
|
||||
}
|
||||
@@ -94,42 +96,39 @@ func (tool readFileTool) RunWithOptions(_ context.Context, args map[string]any,
|
||||
return errorResult("Error: Invalid arguments for read_file: end_line must be greater than or equal to start_line")
|
||||
}
|
||||
|
||||
selected := lines[startLine-1 : endLine]
|
||||
truncated := false
|
||||
if maxLines > 0 && len(selected) > maxLines {
|
||||
selected = selected[:maxLines]
|
||||
selectedLines := endLine - startLine + 1
|
||||
if maxLines > 0 && selectedLines > maxLines {
|
||||
selectedLines = maxLines
|
||||
truncated = true
|
||||
}
|
||||
|
||||
lastLine := startLine + len(selected) - 1
|
||||
lastLine := startLine + selectedLines - 1
|
||||
width := len(strconv.Itoa(lastLine))
|
||||
numbered := make([]string, 0, len(selected))
|
||||
for index, line := range selected {
|
||||
lineNumber := strconv.Itoa(startLine + index)
|
||||
numbered = append(numbered, strings.Repeat(" ", width-len(lineNumber))+lineNumber+" | "+line)
|
||||
}
|
||||
|
||||
header := fmt.Sprintf("File: %s (%d lines)", relativePath, total)
|
||||
if startLine != 1 || endLine != total || maxLines > 0 {
|
||||
header = fmt.Sprintf("File: %s (lines %d-%d of %d)", relativePath, startLine, lastLine, total)
|
||||
}
|
||||
|
||||
body := strings.Join(numbered, "\n")
|
||||
budgetedOutput := newOutputBudgetBuilder(readOutputBudgetBytes, "use start_line/end_line or max_lines to continue with a smaller range")
|
||||
budgetedOutput.WriteString(header)
|
||||
budgetedOutput.WriteString("\n\n")
|
||||
if err := appendReadFileRange(budgetedOutput, absolutePath, startLine, selectedLines, width); err != nil {
|
||||
return errorResult("Error reading file " + relativePath + ": " + err.Error())
|
||||
}
|
||||
if truncated {
|
||||
// The Truncated flag alone is invisible to the model in the rendered
|
||||
// output, so it cannot tell a max_lines cut from a complete read. Make the
|
||||
// cut explicit and tell it how to continue.
|
||||
body += fmt.Sprintf("\n\n[truncated: %d more line(s) in the requested range not shown; set start_line=%d to continue]", endLine-lastLine, lastLine+1)
|
||||
budgetedOutput.WriteString(fmt.Sprintf("\n\n[truncated: %d more line(s) in the requested range not shown; set start_line=%d to continue]", endLine-lastLine, lastLine+1))
|
||||
}
|
||||
|
||||
output := header + "\n\n" + body
|
||||
budgeted := applyOutputBudget(output, readOutputBudgetBytes, "use start_line/end_line or max_lines to continue with a smaller range")
|
||||
budgeted := budgetedOutput.Result()
|
||||
meta := outputBudgetMeta(budgeted)
|
||||
if budgeted.Truncated {
|
||||
meta["truncated"] = "true"
|
||||
meta["truncation_reason"] = "byte_budget"
|
||||
}
|
||||
|
||||
return Result{
|
||||
Status: StatusOK,
|
||||
Output: budgeted.Output,
|
||||
@@ -137,3 +136,77 @@ func (tool readFileTool) RunWithOptions(_ context.Context, args map[string]any,
|
||||
Meta: meta,
|
||||
}
|
||||
}
|
||||
|
||||
type readFileStats struct {
|
||||
lines int
|
||||
hash string
|
||||
info os.FileInfo
|
||||
}
|
||||
|
||||
func scanReadFileStats(path string) (readFileStats, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return readFileStats{}, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
reader := bufio.NewReader(file)
|
||||
lines := 0
|
||||
for {
|
||||
raw, _, err := readRawLine(reader)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return readFileStats{}, err
|
||||
}
|
||||
if _, err := hasher.Write(raw); err != nil {
|
||||
return readFileStats{}, err
|
||||
}
|
||||
lines++
|
||||
}
|
||||
if lines == 0 {
|
||||
lines = 1
|
||||
}
|
||||
info, _ := file.Stat()
|
||||
return readFileStats{lines: lines, hash: hex.EncodeToString(hasher.Sum(nil)), info: info}, nil
|
||||
}
|
||||
|
||||
func appendReadFileRange(output *outputBudgetBuilder, path string, startLine int, selectedLines int, width int) error {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
reader := bufio.NewReader(file)
|
||||
lineNumber := 1
|
||||
emitted := 0
|
||||
for emitted < selectedLines {
|
||||
raw, ended, err := readRawLine(reader)
|
||||
if err == io.EOF {
|
||||
raw = nil
|
||||
ended = false
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if lineNumber >= startLine {
|
||||
if emitted > 0 {
|
||||
output.WriteString("\n")
|
||||
}
|
||||
number := strconv.Itoa(lineNumber)
|
||||
output.WriteString(strings.Repeat(" ", width-len(number)))
|
||||
output.WriteString(number)
|
||||
output.WriteString(" | ")
|
||||
output.WriteString(string(trimLineBreak(raw, ended)))
|
||||
emitted++
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
lineNumber++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loaded 100 of 147 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user