mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 01:09:46 -10:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
844ceafc18 | ||
|
|
03024756e9 |
No files matched your search
@@ -1,9 +1,11 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
// These ceilings are a deliberate ratchet on Zero's fixed per-turn overhead — the
|
||||
@@ -33,7 +35,24 @@ const (
|
||||
// The ratchet did its job: it caught the creep and forced a decision instead
|
||||
// of a drift. That is the point of it, so keep raising it deliberately rather
|
||||
// than reflexively.
|
||||
//
|
||||
// THIS COUNTS THE SCHEMAS ONLY, NOT THE HOST'S SHELL GUIDANCE. Until
|
||||
// 2026-09-07 it counted both, and that made it a different measurement on
|
||||
// every machine: exec_command appends shell guidance whose size depends on
|
||||
// which shell was detected, so the same tree measured 3498 on Linux, ~3600 on
|
||||
// a Windows host with pwsh 7, and 3653 on Windows PowerShell 5.1, which is
|
||||
// stock Windows. The 5.1 case has been over this ceiling since view_image
|
||||
// landed, so `go test ./...` failed for anyone on a stock Windows box while
|
||||
// CI stayed green, because the runners have pwsh 7 and take the cheaper
|
||||
// branch. A budget that passes or fails on which PowerShell happens to be
|
||||
// installed is not ratcheting anything.
|
||||
maxEagerToolSchemaTokens = 3650
|
||||
// The host-specific half, ratcheted on its own so it cannot creep either.
|
||||
// Measured 2026-09-07: Windows PowerShell 5.1 is the most expensive host at
|
||||
// 155 tokens (722 chars), pwsh 7 is 110, cmd.exe is 87, and every POSIX host
|
||||
// is 0 because no guidance is appended there. Set above the worst case, not
|
||||
// above whatever this machine happens to be.
|
||||
maxExecCommandShellGuidanceTokens = 200
|
||||
)
|
||||
|
||||
func TestSystemPromptTokenBudget(t *testing.T) {
|
||||
@@ -56,8 +75,35 @@ func TestEagerToolSchemaTokenBudget(t *testing.T) {
|
||||
// Options{} keeps DeferThreshold at 0, so deferral is inactive and every core
|
||||
// tool is exposed eagerly — exactly what a plugin-free session sends each turn.
|
||||
exposed, _ := partitionTools(registry, PermissionModeAuto, Options{}, map[string]bool{})
|
||||
got := estimateToolDefTokens(exposed)
|
||||
t.Logf("eager core tool schemas: %d tokens across %d tools", got, len(exposed))
|
||||
total := estimateToolDefTokens(exposed)
|
||||
// exec_command's description carries this host's shell guidance, and that
|
||||
// guidance is the one part of the eager set that is not the same everywhere.
|
||||
// Charged separately below rather than folded in here, or this ceiling would
|
||||
// mean a different thing on every machine. See the constants above.
|
||||
//
|
||||
// Removed from the description BEFORE estimating rather than subtracted from
|
||||
// the total afterwards. ApproxTextTokens is an integer division, so
|
||||
// floor((schema+guidance)/4) - floor(guidance/4) comes out one token high
|
||||
// whenever the two remainders add past four, and which side of that line a
|
||||
// host lands on depends on the length of its guidance: exactly the
|
||||
// host-dependence this number is supposed to be rid of.
|
||||
guidance := tools.HostExecCommandShellGuidance()
|
||||
schemas := make([]zeroruntime.ToolDefinition, len(exposed))
|
||||
copy(schemas, exposed)
|
||||
stripped := 0
|
||||
for index := range schemas {
|
||||
without := strings.TrimSuffix(schemas[index].Description, guidance)
|
||||
if without != schemas[index].Description {
|
||||
stripped++
|
||||
}
|
||||
schemas[index].Description = without
|
||||
}
|
||||
if guidance != "" && stripped != 1 {
|
||||
t.Fatalf("SETUP INVALID: this host's shell guidance came off %d descriptions, want exactly exec_command's; internal/tools pins it as that description's tail", stripped)
|
||||
}
|
||||
got := estimateToolDefTokens(schemas)
|
||||
guidanceTokens := ApproxTextTokens(guidance)
|
||||
t.Logf("eager core tool schemas: %d tokens across %d tools (%d total, %d host shell guidance)", got, len(exposed), total, guidanceTokens)
|
||||
if got > maxEagerToolSchemaTokens {
|
||||
// NOT "defer a tool": this test pins DeferThreshold at 0, so marking a
|
||||
// tool deferred leaves it exposed here and changes nothing. Deferral is
|
||||
@@ -66,6 +112,9 @@ func TestEagerToolSchemaTokenBudget(t *testing.T) {
|
||||
// deliberate raise.
|
||||
t.Fatalf("eager tool schemas are %d tokens, over the %d ceiling — trim a schema, drop a core tool, or raise the ceiling deliberately (deferring will NOT help: this test disables deferral)", got, maxEagerToolSchemaTokens)
|
||||
}
|
||||
if guidanceTokens > maxExecCommandShellGuidanceTokens {
|
||||
t.Fatalf("this host's exec_command shell guidance is %d tokens, over the %d ceiling — trim the guidance in internal/tools/shell_runtime.go or raise the ceiling deliberately", guidanceTokens, maxExecCommandShellGuidanceTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentAdvertisesBothEditTools(t *testing.T) {
|
||||
|
||||
@@ -75,14 +75,10 @@ func NewScopedExecCommandTool(workspaceRoot string, scope PathScope, manager *ex
|
||||
if manager == nil {
|
||||
manager = defaultExecSessionManager
|
||||
}
|
||||
description := "Runs a command in a PTY, returning output or a session ID for ongoing interaction."
|
||||
if runtimeGOOS() == "windows" {
|
||||
description += "\n\n" + shellGuidanceForGOOS(runtimeGOOS())
|
||||
}
|
||||
return execCommandTool{
|
||||
baseTool: baseTool{
|
||||
name: ExecCommandToolName,
|
||||
description: description,
|
||||
description: execCommandDescription(runtimeGOOS()),
|
||||
parameters: Schema{
|
||||
Type: "object",
|
||||
Properties: map[string]PropertySchema{
|
||||
@@ -674,3 +670,19 @@ func execDisplaySummary(commandText string, sessionID int, exited bool, exitCode
|
||||
func runtimeGOOS() string {
|
||||
return runtime.GOOS
|
||||
}
|
||||
|
||||
// execCommandDescription is exec_command's description as a host running goos
|
||||
// would see it. The shell guidance is appended on Windows only.
|
||||
//
|
||||
// Taking goos as an argument rather than reading it is what lets the guidance
|
||||
// contract be checked for every platform from any one of them. internal/agent's
|
||||
// token ratchet removes exactly this appended text before charging the schemas,
|
||||
// so the condition here and hostExecCommandShellGuidance have to agree on all
|
||||
// three platforms, not only on whichever one the test happens to run on.
|
||||
func execCommandDescription(goos string) string {
|
||||
description := "Runs a command in a PTY, returning output or a session ID for ongoing interaction."
|
||||
if goos == "windows" {
|
||||
description += "\n\n" + shellGuidanceForGOOS(goos)
|
||||
}
|
||||
return description
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// THE BUDGET TEST REMOVES THIS TEXT, SO IT HAS TO BE EXACTLY WHAT IS ADDED.
|
||||
//
|
||||
// internal/agent's eager-schema ratchet charges the tool schemas and this
|
||||
// host's shell guidance separately, and it separates them by stripping
|
||||
// HostExecCommandShellGuidance off exec_command's description before
|
||||
// estimating. That is only valid while the guidance is appended verbatim as the
|
||||
// tail of that description, and only while a platform that reports no guidance
|
||||
// really appends none. If the two drift apart the ratchet keeps reporting a
|
||||
// number, just not the one it claims.
|
||||
//
|
||||
// Checked for all three platforms rather than for the running one: the
|
||||
// appending is itself conditional on GOOS, so a host-only check is the shape of
|
||||
// bug this pins. Off Windows it would pass while reporting nothing at all.
|
||||
func TestExecCommandDescriptionMatchesReportedGuidance(t *testing.T) {
|
||||
for _, goos := range []string{"windows", "linux", "darwin"} {
|
||||
t.Run(goos, func(t *testing.T) {
|
||||
description := execCommandDescription(goos)
|
||||
guidance := hostExecCommandShellGuidance(goos)
|
||||
|
||||
if goos == "windows" {
|
||||
if guidance == "" {
|
||||
t.Fatal("exec_command appends shell guidance on Windows, but the accessor reports none; the budget would charge it to the schemas")
|
||||
}
|
||||
if !strings.HasSuffix(description, guidance) {
|
||||
t.Fatalf("the guidance is not the tail of the description, so removing it from the schema total is wrong.\nguidance: %q\ndescription tail: %q",
|
||||
guidance, description[max(0, len(description)-len(guidance)-40):])
|
||||
}
|
||||
} else if guidance != "" {
|
||||
t.Fatalf("no shell guidance is appended off Windows, but the accessor reported %q", guidance)
|
||||
}
|
||||
|
||||
// Whatever guidance THIS platform would produce, none of it may be
|
||||
// left in the description once the reported guidance is removed.
|
||||
// Off Windows the accessor reports nothing, so this is the check
|
||||
// that the description really carries nothing: drop the GOOS
|
||||
// condition in execCommandDescription and a POSIX host starts
|
||||
// appending its own guidance (about 150 characters on darwin) with
|
||||
// the ratchet still charging every byte of it to the schemas.
|
||||
platformGuidance := shellGuidanceForGOOS(goos)
|
||||
if platformGuidance == "" {
|
||||
t.Fatalf("SETUP INVALID: %s produces no shell guidance at all, so the check below cannot fail", goos)
|
||||
}
|
||||
if remainder := strings.TrimSuffix(description, guidance); strings.Contains(remainder, platformGuidance) {
|
||||
t.Fatalf("exec_command's description still carries %s shell guidance after removing the %d characters the accessor reports, so the ratchet charges the rest to the tool schemas.\nremainder: %q",
|
||||
goos, len(guidance), remainder)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// And the running host agrees with the table above, so the exported accessor
|
||||
// and the real constructor cannot drift away from the platform contract.
|
||||
func TestHostExecCommandShellGuidanceMatchesTheDescription(t *testing.T) {
|
||||
tool := NewExecCommandTool(t.TempDir(), nil)
|
||||
if got, want := tool.Description(), execCommandDescription(runtime.GOOS); got != want {
|
||||
t.Fatalf("the constructed tool's description is not what the platform contract says.\ngot: %q\nwant: %q", got, want)
|
||||
}
|
||||
if got, want := HostExecCommandShellGuidance(), hostExecCommandShellGuidance(runtime.GOOS); got != want {
|
||||
t.Fatalf("the exported accessor disagrees with the platform contract.\ngot: %q\nwant: %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// And the guidance a Windows host gets is never empty, whichever shell was
|
||||
// detected, since an empty one would mean the model gets no syntax rule at all.
|
||||
func TestWindowsShellGuidanceIsNeverEmpty(t *testing.T) {
|
||||
for _, shell := range []shellRuntime{
|
||||
{GOOS: "windows", Executable: `C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe`, Kind: shellKindPowerShell, Syntax: "PowerShell"},
|
||||
{GOOS: "windows", Executable: `C:\Program Files\PowerShell\7\pwsh.exe`, Kind: shellKindPowerShell, Syntax: "PowerShell"},
|
||||
{GOOS: "windows", Executable: `C:\Windows\System32\cmd.exe`, Kind: shellKindCmd, Syntax: "cmd.exe"},
|
||||
} {
|
||||
if guidance := shellGuidanceForRuntime(shell); strings.TrimSpace(guidance) == "" {
|
||||
t.Errorf("%s produced no shell guidance", shell.Executable)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -181,6 +181,28 @@ func shellGuidanceForRuntime(shell shellRuntime) string {
|
||||
return guidance
|
||||
}
|
||||
|
||||
// HostExecCommandShellGuidance returns the shell guidance appended to
|
||||
// exec_command's description on this host, or "" where none is appended.
|
||||
//
|
||||
// Exported for the per-turn token ratchet in internal/agent. That budget has to
|
||||
// separate what every host pays for the tool schemas from what THIS host pays
|
||||
// for its shell guidance, because the guidance varies by shell and by
|
||||
// PowerShell version, and a single number covering both silently means a
|
||||
// different thing on each machine.
|
||||
func HostExecCommandShellGuidance() string {
|
||||
return hostExecCommandShellGuidance(runtimeGOOS())
|
||||
}
|
||||
|
||||
// hostExecCommandShellGuidance is the guidance a host running goos appends. It
|
||||
// is the pair of execCommandDescription, so the two can be checked against each
|
||||
// other for every platform rather than only for the running one.
|
||||
func hostExecCommandShellGuidance(goos string) string {
|
||||
if goos != "windows" {
|
||||
return ""
|
||||
}
|
||||
return shellGuidanceForGOOS(goos)
|
||||
}
|
||||
|
||||
// HostShellEnvironmentGuidance returns the concise, model-facing shell rule
|
||||
// for the current host.
|
||||
func HostShellEnvironmentGuidance() string {
|
||||
|
||||
Reference in new issue
Block a user