mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-11 18:00:54 -10:00
KRATOS
03784268ac
* token efficiency: lossless cost accounting + minified file reads Two complementary improvements to how Zero handles tokens. ACCOUNTING — make cost measurement lossless and correct: - Carry all five token dimensions end-to-end (input, cache-read, cache-write, output, reasoning) in one Usage; add CacheWriteTokens (cache creation), which the Anthropic adapter previously parsed then discarded. - Price cache-writes at their premium rate (Anthropic 1.25x input, derived per model). Regression-safe: a model without the rate bills cache-write at the input rate exactly as before. - Persist cache-read/cache-write/reasoning in EventUsage so `zero usage report` reconstructs the SAME cost as the live tracker instead of over-pricing cache-heavy sessions. A single EventUsagePayload writer is paired with the reader so the JSON keys can't drift. - Add a live context-window gauge to the status bar (used/window + percent, graded green/amber/red) using the last request's input tokens. REDUCTION — read_minified_file, a dense token-cheap file view: - New internal/minify package + read_minified_file tool returning a comment-free, line-number-free view: Go via the real go/parser AST; C/Java/CSS/SCSS/LESS/ Python via a string-aware comment lexer (handles strings, char literals, triple-quotes, Java text blocks, Python f-strings incl. 3.12 same-quote nesting). Every other language falls back to whitespace-only, so the stripper can never corrupt content. - ~32% fewer tokens on a realistic multi-file read; +~102 fixed tokens for the tool schema, recovered many times over by a single subsystem read. Tests: cache-write pricing + fallback, lossless persistence round-trip, the context gauge, comment-stripping golden cases per language (comment-chars in strings, Java text blocks, Python f-string nesting, CSS has no //), and the unsupported-language safe fallback. gofmt / vet / build / -race all green. * specialist: register read_minified_file in the tool registry + read-only sets Smoke caught TestKnownToolNamesMatchCoreRegistry drifting once read_minified_file joined the core read-only tools: the specialist tool allowlist (knownToolNames), the read-only/edit/execute categories, and the read-only auto-approve set did not list it. It is a read-only companion to read_file, so it belongs alongside read_file in all of them; updated the default-allowlist test accordingly.
65 lines
1.8 KiB
Go
65 lines
1.8 KiB
Go
package minify
|
|
|
|
import (
|
|
"go/parser"
|
|
"go/token"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestMinifyGoStripsCommentsKeepsValidCode(t *testing.T) {
|
|
src := `package demo
|
|
|
|
import "fmt"
|
|
|
|
// Greet returns a greeting. This doc comment MUST be removed.
|
|
func Greet(name string) string {
|
|
// inline comment to drop
|
|
msg := fmt.Sprintf("hi %s", name) // trailing comment
|
|
return msg
|
|
}
|
|
`
|
|
r := File("x.go", []byte(src))
|
|
if !r.Applied || r.Language != "go" {
|
|
t.Fatalf("expected go minification, got %+v", r)
|
|
}
|
|
for _, c := range []string{"MUST be removed", "inline comment", "trailing comment", "//"} {
|
|
if strings.Contains(r.Content, c) {
|
|
t.Errorf("comment text leaked: %q\n%s", c, r.Content)
|
|
}
|
|
}
|
|
for _, code := range []string{"func Greet", "fmt.Sprintf", "return msg"} {
|
|
if !strings.Contains(r.Content, code) {
|
|
t.Errorf("code dropped: %q\n%s", code, r.Content)
|
|
}
|
|
}
|
|
// The minified output must still be valid Go.
|
|
if _, err := parser.ParseFile(token.NewFileSet(), "", r.Content, 0); err != nil {
|
|
t.Fatalf("minified Go does not parse: %v\n%s", err, r.Content)
|
|
}
|
|
}
|
|
|
|
func TestMinifyGoFallsBackOnUnparsableGo(t *testing.T) {
|
|
// A snippet (no package clause) cannot parse as a file -> safe generic path.
|
|
r := File("snippet.go", []byte("x := 1 \n\n\n// keep me\ny := 2\n"))
|
|
if r.Applied {
|
|
t.Fatalf("expected fallback for unparsable Go, got %+v", r)
|
|
}
|
|
if strings.Contains(r.Content, "\n\n\n") {
|
|
t.Errorf("generic should collapse blank runs:\n%q", r.Content)
|
|
}
|
|
if !strings.Contains(r.Content, "// keep me") {
|
|
t.Errorf("generic must NOT strip comments (unsafe without a parser): %q", r.Content)
|
|
}
|
|
}
|
|
|
|
func TestMinifyGenericCollapsesBlanksAndTrims(t *testing.T) {
|
|
r := File("notes.txt", []byte("a \n\n\n\nb\t\n"))
|
|
if r.Applied {
|
|
t.Fatalf("text is not 'applied' minification")
|
|
}
|
|
if r.Content != "a\n\nb" {
|
|
t.Fatalf("generic = %q, want %q", r.Content, "a\n\nb")
|
|
}
|
|
}
|