Files
zero/internal/minify/comments_test.go
KRATOS 03784268ac token efficiency: lossless cost accounting + minified file reads (read_minified_file) (#256)
* token efficiency: lossless cost accounting + minified file reads

Two complementary improvements to how Zero handles tokens.

ACCOUNTING — make cost measurement lossless and correct:
- Carry all five token dimensions end-to-end (input, cache-read, cache-write,
  output, reasoning) in one Usage; add CacheWriteTokens (cache creation), which
  the Anthropic adapter previously parsed then discarded.
- Price cache-writes at their premium rate (Anthropic 1.25x input, derived per
  model). Regression-safe: a model without the rate bills cache-write at the
  input rate exactly as before.
- Persist cache-read/cache-write/reasoning in EventUsage so `zero usage report`
  reconstructs the SAME cost as the live tracker instead of over-pricing
  cache-heavy sessions. A single EventUsagePayload writer is paired with the
  reader so the JSON keys can't drift.
- Add a live context-window gauge to the status bar (used/window + percent,
  graded green/amber/red) using the last request's input tokens.

REDUCTION — read_minified_file, a dense token-cheap file view:
- New internal/minify package + read_minified_file tool returning a comment-free,
  line-number-free view: Go via the real go/parser AST; C/Java/CSS/SCSS/LESS/
  Python via a string-aware comment lexer (handles strings, char literals,
  triple-quotes, Java text blocks, Python f-strings incl. 3.12 same-quote
  nesting). Every other language falls back to whitespace-only, so the stripper
  can never corrupt content.
- ~32% fewer tokens on a realistic multi-file read; +~102 fixed tokens for the
  tool schema, recovered many times over by a single subsystem read.

Tests: cache-write pricing + fallback, lossless persistence round-trip, the
context gauge, comment-stripping golden cases per language (comment-chars in
strings, Java text blocks, Python f-string nesting, CSS has no //), and the
unsupported-language safe fallback. gofmt / vet / build / -race all green.

* specialist: register read_minified_file in the tool registry + read-only sets

Smoke caught TestKnownToolNamesMatchCoreRegistry drifting once read_minified_file
joined the core read-only tools: the specialist tool allowlist (knownToolNames),
the read-only/edit/execute categories, and the read-only auto-approve set did not
list it. It is a read-only companion to read_file, so it belongs alongside
read_file in all of them; updated the default-allowlist test accordingly.
2026-06-19 12:03:07 +05:30

106 lines
3.4 KiB
Go

package minify
import (
"strings"
"testing"
)
func mustContain(t *testing.T, s string, subs ...string) {
t.Helper()
for _, sub := range subs {
if !strings.Contains(s, sub) {
t.Errorf("expected to CONTAIN %q in:\n%s", sub, s)
}
}
}
func mustNotContain(t *testing.T, s string, subs ...string) {
t.Helper()
for _, sub := range subs {
if strings.Contains(s, sub) {
t.Errorf("expected NOT to contain %q in:\n%s", sub, s)
}
}
}
func TestStripC(t *testing.T) {
src := "int x = 1; // DROPLINE\n" +
"/* DROPBLOCK\n spanning */\n" +
"int y = 2;\n" +
"char *u = \"http://e.com/a//b\"; // DROPURLNOTE\n" +
"char c = '\\''; /* DROPCHARNOTE */\n"
r := File("f.c", []byte(src))
if !r.Applied || r.Language != "c" {
t.Fatalf("got %+v", r)
}
mustNotContain(t, r.Content, "DROPLINE", "DROPBLOCK", "spanning", "DROPURLNOTE", "DROPCHARNOTE")
mustContain(t, r.Content, "int x = 1;", "int y = 2;", `"http://e.com/a//b"`, `'\''`)
}
func TestStripCSSHasNoLineComments(t *testing.T) {
// CSS: only /* */ are comments. A //-looking value must survive untouched.
src := "a { color: red; /* DROPBLOCK */ }\n" +
"b { background: url(http://x//y); }\n" +
"/* DROPWHOLE */\n" +
"c { content: \"/* KEEPSTR */\"; }\n"
r := File("s.css", []byte(src))
mustNotContain(t, r.Content, "DROPBLOCK", "DROPWHOLE")
mustContain(t, r.Content, "color: red;", "url(http://x//y)", `"/* KEEPSTR */"`)
}
func TestStripSCSSKeepsInterpolationAndHex(t *testing.T) {
src := "$c: #fff; // DROPLINE\n.x { color: #{$c}; } /* DROPBLOCK */\n"
r := File("s.scss", []byte(src))
mustNotContain(t, r.Content, "DROPLINE", "DROPBLOCK")
mustContain(t, r.Content, "#fff", "#{$c}", "color:")
}
func TestStripJavaTextBlock(t *testing.T) {
src := "class A {\n" +
" // DROPLINE\n" +
" String s = \"\"\"\n" +
" KEEP // this and /* this */ and \"\" too\n" +
" \"\"\";\n" +
" int x = 1; /* DROPBLOCK */\n" +
"}\n"
r := File("A.java", []byte(src))
mustNotContain(t, r.Content, "DROPLINE", "DROPBLOCK")
mustContain(t, r.Content, "KEEP // this and /* this */ and \"\" too", "int x = 1;")
}
func TestStripPython(t *testing.T) {
src := "x = 1 # DROPCOMMENT\n" +
"s = 'has # KEEPHASH inside'\n" +
"t = \"\"\"\ntriple # KEEPTRIPLE\n\"\"\"\n" +
"u = f\"{d['k']}# KEEPFSTR\"\n" +
"r = r'raw\\# KEEPRAW'\n" +
"# DROPLINE\n" +
"y = 2\n"
r := File("m.py", []byte(src))
if !r.Applied || r.Language != "python" {
t.Fatalf("got %+v", r)
}
mustNotContain(t, r.Content, "DROPCOMMENT", "DROPLINE")
mustContain(t, r.Content, "x = 1", "KEEPHASH", "KEEPTRIPLE", "KEEPFSTR", "KEEPRAW", "y = 2")
}
func TestStripPythonFStringSameQuoteNesting(t *testing.T) {
// Python 3.12 same-quote-nested f-string with a # inside must be preserved.
src := "v = f\"{d[\"k\"]}# KEEPINNER\"\nz = 3 # DROPREAL\n"
r := File("m.py", []byte(src))
mustNotContain(t, r.Content, "DROPREAL")
mustContain(t, r.Content, "# KEEPINNER", "z = 3")
}
func TestUnsupportedLangFallsBackToWhitespaceOnly(t *testing.T) {
// JS/TS/C++/Rust/shell are intentionally NOT comment-stripped (hazards).
for _, f := range []string{"a.js", "a.ts", "a.cpp", "a.rs", "a.sh"} {
r := File(f, []byte("x // KEEPJSCOMMENT\n\n\n y\n"))
if r.Applied {
t.Errorf("%s: expected whitespace-only (Applied=false), got %+v", f, r)
}
if !strings.Contains(r.Content, "KEEPJSCOMMENT") {
t.Errorf("%s: comment must be preserved for unsupported langs:\n%s", f, r.Content)
}
}
}