diff --git a/.agents/skills/benchmark-model/SKILL.md b/.agents/skills/benchmark-model/SKILL.md new file mode 100644 index 00000000..dfc74d72 --- /dev/null +++ b/.agents/skills/benchmark-model/SKILL.md @@ -0,0 +1,294 @@ +--- +name: benchmark-model +description: Measure every retained number in docs/models//BENCHMARKS.md for one Gufo model on Strix Halo gfx1151 over HTTP, run the same workload against the reference project's OpenAI-compatible server (llama.cpp, audio.cpp, ...), and report the gain per cell. +metadata: + origin: gufo +--- + +# Benchmark a model + +Fill or refresh `docs/models//BENCHMARKS.md` for one model. Which +tables exist and which external project is the comparison baseline depend on +the model category below. Read the model's `README.md`, `BENCHMARKS.md` and +`EVALUATION.md` first, then [docs/BENCHMARKS.md](../../../docs/BENCHMARKS.md) +for methodology. Follow the user's machine, time and Git instructions. + +## Start + +Ask the user before measuring anything: + +1. Which model (`docs/models/`) to benchmark. +2. Where the model files are: target GGUF or safetensors directory per + variant, draft/MTP/DSpark support files, mmproj, audio fixtures. Model + files live wherever the user keeps them; nothing in Git records paths. +3. Full benchmark (every table, every cell, both targets) or only the cells + currently `TODO` in `BENCHMARKS.md`. + +Then confirm the time budget: a full LLM sweep with the reference is hours. + +## Measurement model + +Everything is measured **over HTTP on both sides** so the prompt, timed scope +and transport are identical: Gufo through `gufo serve`, the reference through +its own OpenAI-compatible server. `gufo bench` and the standalone CLIs are +kernel-iteration tools; their numbers are not published in the comparison +tables. + +The driver is `tools/bench/model-bench.py`. `docs/models//artifacts/bench.json` +declares the category, variant ids, sweep grid, workloads and reference server +flags; model file paths are passed on the command line. The driver launches +the right server itself on a private port, runs one target at a time, writes +one JSON per table (and per mode for concurrency tables) into +`docs/models//artifacts/`, and renders the Markdown tables between +`` markers in `BENCHMARKS.md`: + +```sh +nix build # production Gufo binary at result/bin/gufo +FILES="--gguf q4=/path/to/target-q4.gguf --draft q4=/path/to/draft-q4.gguf --mmproj /path/to/mmproj.gguf" +nix develop -c python3 tools/bench/model-bench.py --model tables # ids and existing artifacts +nix develop -c python3 tools/bench/model-bench.py --model $FILES run --target gufo +nix develop -c python3 tools/bench/model-bench.py --model $FILES run --target reference +nix develop -c python3 tools/bench/model-bench.py --model render # or render --check +``` + +`run` flags: `--todo` measures only rows whose target-owned cells are `TODO` +in `BENCHMARKS.md`; `--table [,]` restricts to named tables; +`--drop-caches ""` is needed by the loading table on hosts +without passwordless sudo/doas (`sync; echo 3 > /proc/sys/vm/drop_caches`); +`--config` and `--artifacts-dir` point at an alternative `bench.json` and +output directory for experiments that must not touch the retained artifacts. +Server logs go to the ignored `artifacts/model-bench/`. `render` keeps +hand-entered cells that no artifact covers, so tables can hold both, and it +draws one SVG chart per table with data into +`docs/models//artifacts/charts/` (matplotlib, deterministic output), +placed as an image line right after the table. Tables stay the source of +truth for agents; charts are an addition for readers. `--no-charts` skips them. + +`bench.json` (`schema: gufo-model-bench/1`) holds: `category`; `model.id`; +`files` (the file roles the CLI must supply, with a description each); +`variants` (id, label, required roles; single-variant models use `default` +and unsuffixed table ids); `sampling`; `speculative` (mode, label, Gufo +arguments with `{role}` placeholders, and which reference mode it is compared +against); `tables` keyed by table id with that table's workload, and +`gufo.serve` / `reference.server` + `args` for the two servers. Readiness is +`GET /ready` on Gufo and `GET /health` on llama-server. + +Reference servers (llama.cpp, audio.cpp, stable-diffusion.cpp) come from this +repository's `flake.nix`; do not use ad-hoc installs. + +Request-level timing comes from the response, not the client clock, whenever +the server provides it: Gufo and llama-server both return llama.cpp-compatible +`timings` (`prompt_n` = newly processed tokens, `cache_n` = reused tokens, +`prompt_ms`, `predicted_n`, `predicted_ms`); Gufo adds `usage.gufo` stage +metrics. Servers without timings get client-measured whole-request wall time, +and the table says so. + +## Rules for every category + +1. Production binary: `nix build`, `./result/bin/gufo`. Record the binary + SHA-256 prefix, Git revision, model artifact identities (repo, revision, + quantization, hashes), reference tool version/commit, backend (ROCm/Vulkan), + full server command lines and the measurement date. Server flags go into + the artifact `notes`; hosts, prompts, generated text and paths never do. +2. Same input, same artifact, same timed scope on both sides: same GGUF or + safetensors, same prompt or audio file, same output length, same + seed/temperature, same context depth, same concurrency, fresh server per + point, no prompt-cache hits unless the cell says so. If the reference cannot + match (different quantization, no equivalent speculative mode, no batching), + measure the closest configuration and state the mismatch under the table. +3. Warm once, then time. One warmed sample per point is acceptable for a + sweep; add repetitions when two runs differ by more than about 2% or a + regression is suspected, and report mean ± sd. Run model jobs sequentially; + nothing else on the GPU. +4. Run the model's quality checks in `EVALUATION.md` before publishing speed. + Greedy speculative output must match AR token IDs; the reference's greedy + output is compared by completion hash and the match count is reported. A + mismatch or device fault is not a result. +5. A cell without a current qualified measurement is `TODO`, never a stale + number. Keep dates per table; do not sum stage medians into a headline. +6. Retained JSON goes to `docs/models//artifacts/`; raw samples, + traces, audio, images and videos stay in the ignored top-level `artifacts/`. + +## Comparison columns + +Every headline table carries the reference project next to Gufo: + +``` +| ... | Gufo | | Gain | ... | +``` + +- `Gain` is a percentage, positive when Gufo is better: + - throughput (tok/s, audio s per wall s): `(Gufo / ref − 1) × 100` + - latency, RTF, wall time, memory: `(ref / Gufo − 1) × 100` +- No difference column: both raw values are visible, and the extra column + makes wide tables unreadable. When a cell is `mean ± sd`, Gain uses means. +- Name the reference in the header (`llama.cpp pp`, `audio.cpp RTF`). +- `TODO` on either side leaves `Gain` as `TODO`. +- When Gufo runs a mode the reference lacks (DFlash2, DSpark, MTP without a + matching draft), compare against the reference's AR number and label the + column `Gain vs AR`. +- One table per quantization and per mode; never pack `pp / tg` pairs or two + quantizations into one cell. + +Rendered tables sit between `` and +`` markers so `render` can replace them in place; the prose +around them (dates, acceptance notes, caveats) is hand-maintained. + +## LLM (GGUF text and vision models) + +Models: `deepseek-v4-flash`, `qwen3.8-27b`, `qwen3.8-flash-next`. +Reference: **llama.cpp** `llama-server`, same GGUF, ROCm build, +`-ngl 999 -fa on --cache-reuse 0 --jinja --reasoning off`, `-np` equal to the +largest concurrency, `-c` covering the deepest sweep point plus 2048 + 128. +Gufo: `gufo serve --sessions 8 --max-pending-per-client 8`, thinking off, +greedy, seed 1. The sweep parameters are identical across the three models so +the documents stay comparable. + +Tables, in order (table ids in parentheses; append `-` when the +model has several quantizations, e.g. `single-ar-q4`): + +1. **Loading** (`loading`). Cold-file-cache launch to `/health` readiness + (drop caches, start the server, poll), then Gufo snapshot size and restore + time for the documented prompt. Reference: `llama-server` readiness with + the same GGUF. `Target | Gufo ready | llama.cpp ready | Gain`. +2. **Single user, autoregressive** (`single-ar`). pp2048 / tg128, C1, depth + `0,4096,8192,12288,16384,32768`; add `65536,131072` when the model's + context allows. Depth `d` is a cached prefix: send the `d`-token prefix once + with `cache_prompt=true`, then prefix + 2048 new tokens with 128 output + tokens. Neither server exposes a tokenizer endpoint, so the driver + calibrates the synthetic prefix text (`prefix.generator`, seeded) against + each server's reported `prompt_n`/`cache_n` and accepts a point only when + `cache_n` is within `depth_tolerance` of `d` and `prompt_n` within it of + 2048; the actual counts are stored in the artifact. The same text is sent + to both servers. + `Depth | Gufo pp | llama.cpp pp | Gain | Gufo tg | llama.cpp tg | Gain`. +3. **Single user, speculative** (`single-`, DFlash2 / DSpark / MTP as + the model supports). Same grid plus `Acceptance` from `usage.gufo`. + Reference: `llama-server --model-draft` when an equivalent draft exists + (`llama.cpp draft tg | Gain`), otherwise llama.cpp AR + (`llama.cpp AR tg | Gain vs llama.cpp AR`); the note under the table says + which. `Depth | Gufo pp | Gufo tg | Acceptance | llama.cpp ... tg | Gain ...`. +4. **Multiple users** (`multi-mixed`, `multi-repetition`). `C = 1,2,4,6,8`, + context capacity 4096, 128 output tokens, fresh server per point, + `cache_prompt=false`. Two workloads from + `docs/models/qwen3.8-27b/artifacts/speculative-corpus.json`: `repetition` + and `mixed` (distinct layout). Metric: aggregate delivered output tok/s + (`aggregate.output_tokens_per_second.overall`). `Exact` is the count of + llama.cpp completions whose hash matches the Gufo AR C1 reference. Report + C8 median / p95 latency and acceptance under the table. + `Users | Gufo AR | llama.cpp AR | Gain | Gufo | Gain vs llama.cpp AR | Exact`. +5. **Memory** (`memory`). GPU-visible allocation at the documented context + capacity for pp2048+tg128 and a 16K-prefix pp4096+tg128 (`usage.gufo` + memory fields / `gufo diagnose`). Reference: `llama-server` resident + device memory at the same `-c`, read from the ROCm SMI while the request + runs. `Workload | Gufo GiB | llama.cpp GiB | Gain`. +6. **Image encoder** (`image-encoder`, vision models only). Warm encode + latency for 256×256 and 1024×1024 RGB through the chat endpoint with the + BF16 projector, isolated from language prefill via `usage.gufo` stages. + Reference: `llama-server --mmproj` with the same file; whole-request wall + when it reports no encode stage. + `RGB image | Merged tokens | Gufo ms | llama.cpp ms | Gain`. + +Concurrency artifacts are `gufo-serving-bench` corpus reports named +`-gufo-ar.json`, `
-gufo-.json` and `
-reference.json`; +the other tables use the compact `model-bench-table` schema with one entry per +row and the actual `cache_n`/`prompt_n` counts. The image-encoder table is not +automated yet; leave its cells `TODO` or fill them by hand from a documented +measurement. + +## ASR (audio to text) + +Model: `qwen3-asr`. Reference: **audio.cpp** server (`qwen3_asr` model spec, +ROCm), `/v1/audio/transcriptions`, same WAV, greedy. Both transcripts are +compared against the official reference transcript. + +1. **Loading.** Cold-file-cache readiness; first-request and warm latency for + a short clip. +2. **Single request.** The documented fixed recording (15.05 s English), one + warmup, three timed requests, median. Rows: request time, RTF (request / + input duration, lower is better), audio seconds per wall second; Gufo-only + rows for audio-encoder time and text prefill+generation from `usage.gufo`. + Columns: `Metric | Gufo | audio.cpp | Gain`. +3. **Concurrent HTTP.** C = 1,2,4,6,8, same recording: per-request latency + median / p95 and aggregate audio seconds per wall second, both servers. +4. **Long-form.** One recording over ten minutes: wall time, RTF, transcript + diff between the two implementations and against the reference. +5. **Profile** (Gufo only). Dispatches, GPU work, GPU-busy share, projection + share from `tools/prof/prof.py --stages qwen-asr`, run separately. + +## TTS (text and reference audio to speech) + +Model: `qwen3-tts`. Reference: **audio.cpp** server (`qwen3_tts` model spec, +ROCm), `/v1/audio/speech`, same text, seed, sampling and variant +(CustomVoice / VoiceDesign / Base ICL); variants audio.cpp does not expose +are `n/a`. + +1. **Loading.** Cold-file-cache readiness per variant; first-request and warm + latency for a short sentence. +2. **Single request.** The documented 37-word paragraph, 64 codec frames + (5.12 s), one warmup, three timed, median. Per variant: + `Variant | Gufo WAV | audio.cpp WAV | Gain | Gufo RTF | audio.cpp RTF | Gain | First PCM | Complete PCM`. + RTF = request time / generated audio duration. Gufo streamed and buffered + PCM must be identical; seeded runs must repeat exactly. First-audio latency + is measured on the streaming endpoint of each server when it has one. +3. **Transports** (Gufo only). PCM HTTP, SSE, WebSocket first-audio and + complete times; byte-identical PCM across transports. +4. **Concurrency.** C = 1,2,4,6,8 completion times on both servers; + execution is serialized, report the queue effect honestly. +5. **Profile** (Gufo only). Dispatches, GPU work, GPU busy, projection share + per variant from `tools/prof/prof.py --stages qwen-tts`. + +Natural-EOS duration and intelligibility belong to `EVALUATION.md`. + +## Image generation and editing + +Model: `qwen-image-2.1`. Reference: **stable-diffusion.cpp** server on ROCm +(`/v1/images/generations`) with the same checkpoint when it supports the +model; otherwise an OpenAI-compatible wrapper around the official Diffusers +pipeline on ROCm, labelled `Diffusers (ROCm)`. Same prompt, seed, size, +steps, scheduler and guidance. + +1. **Loading.** Metadata readiness, first generation including component + loading, and the same request warm. +2. **Generation and edit, C1.** 1024×1024, the default 40 steps, seed 42. + Rows: generation; edit with one reference image (edit only where the + reference supports it). Columns: + `Mode | Steps | Prompt (s) | Denoising (s) | VAE (s) | Gufo total | total | Gain`; + stage columns are Gufo `usage.gufo` values. Add 512×512 / 20 steps as a + second control when the reference is slow. +3. **GPU split** (Gufo only). Native projections, fused feed-forward, fused + attention, convolution and idle share from a separate request-only profile. +4. **Concurrency.** Requests are queued; report C2 completion times and state + that no batching is implemented. + +A two-step smoke check is a development control, never the headline. Image +quality is measured in `EVALUATION.md`. + +## Video / audiovisual generation + +Model: `minimax-h3`. Reference: **audio.cpp** server (`minimax_h3` model +spec) if it executes on ROCm; verify first and fall back to a wrapper around +the official Diffusers pipeline, labelled. Full generation requires the user's +explicit approval and `--allow-full-generation`. + +1. **Loading.** Metadata readiness; first prompt-encoder layers time; peak + retained device memory. +2. **Presets.** Per preset (`exact 512x512`, `exact-1344x768`, `fast`, + `aggressive`): single-request wall, `first_preview_ms`, peak resident + memory, delivery-quality note. Columns: + `Preset | Gufo wall | wall | Gain | Gufo peak GiB | peak GiB | Gain`. +3. **Phase split** (Gufo only). Prompt encoding, AdaLN precompute, denoising, + VisualVAE, AudioVAE, mux, from `tools/gufo/h3_profile.py`. +4. **Component inventory.** Tensor bytes per component; never summed as if + resident together. +5. **Queue.** One worker; report C2 queue latency and say there is no + parallel execution. + +## Finish + +Render the tables, check that every Gufo/reference pair used the same +workload identity, update dates and the reproduction block, and list the +remaining `TODO` cells with the reason (tool missing, reference unsupported, +time budget). Update `EXPERIMENTS.md` only when a measurement changes a +retained decision. Summarize per table: Gufo, reference, best and worst gain, +and every cell where completion hashes or transcripts did not match. diff --git a/.claude/skills b/.claude/skills new file mode 120000 index 00000000..2b7a412b --- /dev/null +++ b/.claude/skills @@ -0,0 +1 @@ +../.agents/skills \ No newline at end of file diff --git a/.gitignore b/.gitignore index e96c61f7..a91e56d2 100644 --- a/.gitignore +++ b/.gitignore @@ -115,3 +115,6 @@ tests/quality/compsec.json # rocprofv3 counter dumps .rocprofv3/ + +# Claude Code local settings +.claude/settings.local.json diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index e6607975..03395c35 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -10,6 +10,15 @@ results and qualification gaps live in: - [Qwen3-ASR](models/qwen3-asr/BENCHMARKS.md) - [MiniMax H3](models/minimax-h3/BENCHMARKS.md) +## Model benchmark driver + +`tools/bench/model-bench.py` measures the tables of a model's `BENCHMARKS.md` +over HTTP for Gufo and for the reference project of its category (llama.cpp +for GGUF language models), writes per-table artifacts, and renders the tables +and SVG charts between the `` markers. Workloads and table +layouts are declared in `docs/models//artifacts/bench.json`; the +`benchmark-model` skill in `.agents/skills` describes the procedure. + ## Direct and serving measurements `gufo bench` measures the model path. Keep model artifact, prompt length, diff --git a/docs/models/deepseek-v4-flash/BENCHMARKS.md b/docs/models/deepseek-v4-flash/BENCHMARKS.md index b0e7c37e..4e9b7656 100644 --- a/docs/models/deepseek-v4-flash/BENCHMARKS.md +++ b/docs/models/deepseek-v4-flash/BENCHMARKS.md @@ -2,7 +2,8 @@ Linux x86-64, AMD `gfx1151`, 128 GB unified memory. Nix release binaries. `C` is simultaneous requests. Single-user measurements use **pp2048 / tg128**; -depth precedes the measured operation. Unknown current measurements are **TODO**. +depth is a cached prefix and precedes the measured operation. Unknown current +measurements are **TODO**. **Target parity remains open:** the optimized build misses the historical trajectory gate, and four post-prefill differential alerts remain unresolved. @@ -14,69 +15,104 @@ Target: `DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-073 `DeepSeek-V4-Flash-DSpark-support-0731.gguf`, revision `e7f04037032990db0346398d249baf9fb9df1ccc`. -Cold-file-cache launch to HTTP readiness: **36.82 s** on 2026-09-21, -including DSpark and two sessions at context capacity 4096. The integer -expert-routing tensor loads without conversion; serving replay remains exact -under the [state checks](EVALUATION.md). +Reference implementation: **llama.cpp** `llama-server` from this repository's +`flake.nix` (ROCm, gfx1151, same target GGUF), measured over HTTP with the +same prompts and timed scope. **Gain** is Gufo over llama.cpp, positive when +Gufo is faster; `Gain vs llama.cpp AR` compares DSpark against llama.cpp +without a draft. All llama.cpp cells are **TODO** until the first HTTP sweep. +Layout and method: [benchmark-model skill](../../../.agents/skills/benchmark-model/SKILL.md). + +## Loading + +Cold-file-cache launch to HTTP readiness on 2026-09-21, including DSpark and +two sessions at context capacity 4096. The integer expert-routing tensor +loads without conversion; serving replay remains exact under the +[state checks](EVALUATION.md). + + +| Target | Gufo ready | llama.cpp ready | Gain | +| --- | ---: | ---: | ---: | +| Flash 0731 | 36.82 s | TODO | TODO | + + +![Cold-file-cache launch to readiness](artifacts/charts/loading.svg) ## Single user, autoregressive -Latest depth control: **2026-09-19**, two release measurements at d0/d32K. -Values are mean ± standard deviation; initial kernel setup contributes to -depth-zero prefill variation. The final packing layout needs a d64K speed refresh. +Latest depth control: **2026-09-19**, two release measurements at d0/d32K +with `gufo bench`. Values are mean ± standard deviation; initial kernel +setup contributes to depth-zero prefill variation. The final packing layout +needs a d64K speed refresh. Gain uses means. -| Context depth | pp2048 tok/s | tg128 tok/s | -| ---: | ---: | ---: | -| 0 | 450.26 ± 33.18 | 17.74 ± 0.00 | -| 4,096 | TODO | TODO | -| 8,192 | TODO | TODO | -| 12,288 | TODO | TODO | -| 16,384 | TODO | TODO | -| 32,768 | 422.18 ± 3.61 | 14.86 ± 0.00 | -| 65,536 | TODO | TODO | + +| Depth | Gufo pp | llama.cpp pp | Gain | Gufo tg | llama.cpp tg | Gain | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 450.26 ± 33.18 | TODO | TODO | 17.74 ± 0.00 | TODO | TODO | +| 4,096 | TODO | TODO | TODO | TODO | TODO | TODO | +| 8,192 | TODO | TODO | TODO | TODO | TODO | TODO | +| 12,288 | TODO | TODO | TODO | TODO | TODO | TODO | +| 16,384 | TODO | TODO | TODO | TODO | TODO | TODO | +| 32,768 | 422.18 ± 3.61 | TODO | TODO | 14.86 ± 0.00 | TODO | TODO | +| 65,536 | TODO | TODO | TODO | TODO | TODO | TODO | + + +![Single user, autoregressive](artifacts/charts/single-ar.svg) ## Single user, DSpark Same workload and measurement date: two measurements at d0, one at d32K. -DSpark prefill includes support-state work. +DSpark prefill includes support-state work. llama.cpp has no DSpark +equivalent: the reference column is llama.cpp autoregressive generation with +the same target GGUF. -| Context depth | pp2048 tok/s | tg128 tok/s | -| ---: | ---: | ---: | -| 0 | 453.49 ± 25.96 | 17.45 ± 0.00 | -| 4,096 | TODO | TODO | -| 8,192 | TODO | TODO | -| 12,288 | TODO | TODO | -| 16,384 | TODO | TODO | -| 32,768 | 417.80 | 35.30 | -| 65,536 | TODO | TODO | + +| Depth | Gufo pp | Gufo tg | Acceptance | llama.cpp AR tg | Gain vs llama.cpp AR | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 453.49 ± 25.96 | 17.45 ± 0.00 | TODO | TODO | TODO | +| 4,096 | TODO | TODO | TODO | TODO | TODO | +| 8,192 | TODO | TODO | TODO | TODO | TODO | +| 12,288 | TODO | TODO | TODO | TODO | TODO | +| 16,384 | TODO | TODO | TODO | TODO | TODO | +| 32,768 | 417.80 | 35.30 | TODO | TODO | TODO | +| 65,536 | TODO | TODO | TODO | TODO | TODO | + + +![Single user, DSpark](artifacts/charts/single-dspark.svg) The CLI uses a repeating token sequence. Natural prompts can have substantially different acceptance and speed. Depth-zero generation starts from 16 tokens; it does not follow the measured 2048-token prefill. -## Multiple users, autoregressive +## Multiple users -Current pp2048/tg128 depth sweep: **TODO**. Report aggregate prompt throughput -and per-user generation throughput at each depth. +Aggregate delivered output tok/s (`aggregate.output_tokens_per_second.overall`). +Context capacity 4096, greedy, 128 output tokens, `cache_prompt=false`, fresh +server per point. Workloads come from the +[speculative corpus](../qwen3.8-27b/artifacts/speculative-corpus.json): +`repetition` runs `repetition_word`; `mixed` cycles through the distinct +corpus cases. `Exact` counts llama.cpp completions whose hash matches the +Gufo AR C1 reference. The current sweep is **TODO**, including acceptance and +output checks at every concurrency. -| Users | d0 / d4K / d8K / d12K / d16K / d32K | -| ---: | --- | -| 2 | TODO | -| 4 | TODO | -| 6 | TODO | -| 8 | TODO | + +| Users | Gufo AR | llama.cpp AR | Gain | Gufo DSpark | Gain vs llama.cpp AR | Exact | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | TODO | TODO | TODO | TODO | TODO | TODO | +| 2 | TODO | TODO | TODO | TODO | TODO | TODO | +| 4 | TODO | TODO | TODO | TODO | TODO | TODO | +| 6 | TODO | TODO | TODO | TODO | TODO | TODO | +| 8 | TODO | TODO | TODO | TODO | TODO | TODO | + -## Multiple users, DSpark - -Current pp2048/tg128 depth sweep: **TODO**, including acceptance and output -checks at every concurrency. - -| Users | d0 / d4K / d8K / d12K / d16K / d32K | -| ---: | --- | -| 2 | TODO | -| 4 | TODO | -| 6 | TODO | -| 8 | TODO | + +| Users | Gufo AR | llama.cpp AR | Gain | Gufo DSpark | Gain vs llama.cpp AR | Exact | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | TODO | TODO | TODO | TODO | TODO | TODO | +| 2 | TODO | TODO | TODO | TODO | TODO | TODO | +| 4 | TODO | TODO | TODO | TODO | TODO | TODO | +| 6 | TODO | TODO | TODO | TODO | TODO | TODO | +| 8 | TODO | TODO | TODO | TODO | TODO | TODO | + Latest fixed-prompt sampled HTTP control (2026-09-19, C2, temperature 1, top-p 0.95, seed 7): **17.40 tok/s per user** for an autumn explanation, @@ -95,19 +131,24 @@ never changes token decisions. ## Memory C1 with context capacity **262,144**. GPU-visible unified allocations exclude -separate CPU memory; capacity does not imply filling the window. +separate CPU memory; capacity does not imply filling the window. Gufo values +come from the earlier C1 capacity control with DSpark (the first row was +recorded as pp2048/4096 + tg128); a current refresh is TODO. llama.cpp resident device memory at the same `-c`: **TODO**. -| DSpark workload | GPU allocation | -| --- | ---: | -| pp2048/4096 + tg128 | 90.74 GiB | -| 16K prefix, pp4096 + tg128 | 91.46 GiB | + +| Workload | Gufo GiB | llama.cpp GiB | Gain | +| --- | ---: | ---: | ---: | +| pp2048 + tg128 | 90.74 | TODO | TODO | +| 16K prefix, pp4096 + tg128 | 91.46 | TODO | TODO | + -Measured in the earlier C1 capacity control; a current memory refresh is TODO. Target and -support weights, KV and working buffers all contribute. Compressed KV and score -scratch grow with actual use. Prefill reuses scratch across ordered stages; -indexer scoring borrows the idle attention-output range for temporary F16 keys. -Queries share that range and feed WMMA directly. Persistent KV precision is -unchanged. +![GPU-visible allocation](artifacts/charts/memory.svg) + +Target and support weights, KV and working buffers all contribute. Compressed +KV and score scratch grow with actual use. Prefill reuses scratch across +ordered stages; indexer scoring borrows the idle attention-output range for +temporary F16 keys. Queries share that range and feed WMMA directly. +Persistent KV precision is unchanged. ## Reproduce @@ -126,5 +167,7 @@ Use `-c 1,2,4,6,8` for concurrency and for pp4096. Sampled controls use the same `--temperature` and `--seed` in both modes. Context preparation and restoration are outside timing. Verbose output records hashes and draft counts; compare these alongside speed. Run model -jobs, builds and profiles sequentially. The HTTP benchmark is -`tools/serving/gufo-serving-bench.py`; distinguish cold requests from cache hits. +jobs, builds and profiles sequentially. `gufo bench` is the kernel-iteration +tool; the published comparison tables are measured over HTTP on both sides +and rendered with `tools/bench/model-bench.py` as described by the +`benchmark-model` skill. Distinguish cold requests from cache hits. diff --git a/docs/models/deepseek-v4-flash/artifacts/bench.json b/docs/models/deepseek-v4-flash/artifacts/bench.json new file mode 100644 index 00000000..8c3cddfa --- /dev/null +++ b/docs/models/deepseek-v4-flash/artifacts/bench.json @@ -0,0 +1,205 @@ +{ + "schema": "gufo-model-bench/1", + "category": "llm", + "model": { + "id": "DeepSeek-V4-Flash", + "docs": "deepseek-v4-flash" + }, + "files": { + "gguf": "target GGUF (IQ2XXS mixed, antirez/deepseek-v4-gguf)", + "dspark": "DSpark support GGUF" + }, + "variants": { + "default": { + "label": "Flash 0731", + "requires": [ + "gguf", + "dspark" + ] + } + }, + "sampling": { + "temperature": 0, + "seed": 1 + }, + "speculative": { + "mode": "dspark", + "label": "DSpark", + "gufo_args": [ + "--dspark-model", + "{dspark}" + ], + "reference": "ar" + }, + "tables": { + "loading": { + "title": "Cold-file-cache launch to readiness", + "per_variant": false, + "context": 4096, + "sessions": 2, + "repetitions": 1, + "metric": "seconds", + "better": "lower", + "rows": "variants" + }, + "single-ar": { + "title": "Single user, autoregressive", + "per_variant": false, + "prompt_tokens": 2048, + "output_tokens": 128, + "depths": [ + 0, + 4096, + 8192, + 12288, + 16384, + 32768, + 65536 + ], + "context": 67712, + "depth_tolerance": 0.005, + "prefix": { + "generator": "synthetic-paragraphs", + "seed": 1 + }, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "single-dspark": { + "title": "Single user, DSpark", + "per_variant": false, + "speculative": true, + "prompt_tokens": 2048, + "output_tokens": 128, + "depths": [ + 0, + 4096, + 8192, + 12288, + 16384, + 32768, + 65536 + ], + "context": 67712, + "depth_tolerance": 0.005, + "prefix": { + "generator": "synthetic-paragraphs", + "seed": 1 + }, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "multi-repetition": { + "title": "Multiple users, repetition", + "per_variant": false, + "modes": [ + "ar", + "dspark" + ], + "suite": "../../qwen3.8-27b/artifacts/speculative-corpus.json", + "cases": [ + "repetition_word" + ], + "corpus_layout": "homogeneous", + "concurrency": [ + 1, + 2, + 4, + 6, + 8 + ], + "context": 4096, + "output_tokens": 128, + "cache_prompt": false, + "warmup": 1, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "multi-mixed": { + "title": "Multiple users, mixed corpus", + "per_variant": false, + "modes": [ + "ar", + "dspark" + ], + "suite": "../../qwen3.8-27b/artifacts/speculative-corpus.json", + "cases": [ + "expository_pangram", + "cpp_ring_buffer", + "reasoning_train", + "summary_gpu", + "italian_explanation", + "chinese_explanation", + "json_schedule", + "creative_station", + "instruction_debug" + ], + "corpus_layout": "distinct", + "concurrency": [ + 1, + 2, + 4, + 6, + 8 + ], + "context": 4096, + "output_tokens": 128, + "cache_prompt": false, + "warmup": 1, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "memory": { + "title": "GPU-visible allocation", + "per_variant": false, + "context": 262144, + "workloads": [ + { + "id": "pp2048 + tg128", + "depth": 0, + "prompt_tokens": 2048, + "output_tokens": 128 + }, + { + "id": "16K prefix, pp4096 + tg128", + "depth": 16384, + "prompt_tokens": 4096, + "output_tokens": 128 + } + ], + "metric": "GiB", + "better": "lower" + } + }, + "gufo": { + "serve": [], + "llm": [ + "--think", + "off", + "--max-pending-per-client", + "8" + ], + "readiness": "/ready" + }, + "reference": { + "name": "llama.cpp", + "server": "llama-server", + "args": [ + "-ngl", + "999", + "-fa", + "on", + "--cache-reuse", + "0", + "--jinja", + "--reasoning", + "off" + ], + "endpoint_profile": "openai", + "readiness": "/health" + } +} diff --git a/docs/models/deepseek-v4-flash/artifacts/charts/loading.svg b/docs/models/deepseek-v4-flash/artifacts/charts/loading.svg new file mode 100644 index 00000000..7039a1e6 --- /dev/null +++ b/docs/models/deepseek-v4-flash/artifacts/charts/loading.svg @@ -0,0 +1,196 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + Flash 0731 + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 30 + + + + + + + + + + + + + 40 + + + + seconds to ready + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + llama.cpp + + + + + Cold-file-cache launch to readiness + + + + + + + + diff --git a/docs/models/deepseek-v4-flash/artifacts/charts/memory.svg b/docs/models/deepseek-v4-flash/artifacts/charts/memory.svg new file mode 100644 index 00000000..fe00c1da --- /dev/null +++ b/docs/models/deepseek-v4-flash/artifacts/charts/memory.svg @@ -0,0 +1,237 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + pp2048 + tg128 + + + + + + + + + + 16K prefix, pp4096 + tg128 + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 80 + + + + + + + + + + + + + 100 + + + + GiB + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + llama.cpp + + + + + GPU-visible allocation + + + + + + + + diff --git a/docs/models/deepseek-v4-flash/artifacts/charts/single-ar.svg b/docs/models/deepseek-v4-flash/artifacts/charts/single-ar.svg new file mode 100644 index 00000000..c56c9c2d --- /dev/null +++ b/docs/models/deepseek-v4-flash/artifacts/charts/single-ar.svg @@ -0,0 +1,544 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + + + + + + + 64K + + + + context depth (tokens) + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 100 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 300 + + + + + + + + + + + + + 400 + + + + + + + + + + + + + 500 + + + + prefill tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + + + + + + + llama.cpp + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + + + + + + + 64K + + + + context depth (tokens) + + + + + + + + + + + + + + 0.0 + + + + + + + + + + + + + 2.5 + + + + + + + + + + + + + 5.0 + + + + + + + + + + + + + 7.5 + + + + + + + + + + + + + 10.0 + + + + + + + + + + + + + 12.5 + + + + + + + + + + + + + 15.0 + + + + + + + + + + + + + 17.5 + + + + + + + + + + + + + 20.0 + + + + generation tok/s + + + + + + + + + + + + + + + + + + + Single user, autoregressive + + + + + + + + + + + diff --git a/docs/models/deepseek-v4-flash/artifacts/charts/single-dspark.svg b/docs/models/deepseek-v4-flash/artifacts/charts/single-dspark.svg new file mode 100644 index 00000000..67219b83 --- /dev/null +++ b/docs/models/deepseek-v4-flash/artifacts/charts/single-dspark.svg @@ -0,0 +1,535 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + + + + + + + 64K + + + + context depth (tokens) + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 5 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 15 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 25 + + + + + + + + + + + + + 30 + + + + + + + + + + + + + 35 + + + + + + + + + + + + + 40 + + + + generation tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo DSpark + + + + + + + + + + + + llama.cpp AR + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + + + + + + + 64K + + + + context depth (tokens) + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 80 + + + + + + + + + + + + + 100 + + + + acceptance % + + + + + + + + + + + + Single user, DSpark + + + + + + + + + + + diff --git a/docs/models/qwen3.8-27b/BENCHMARKS.md b/docs/models/qwen3.8-27b/BENCHMARKS.md index caf68b2d..4cca57e1 100644 --- a/docs/models/qwen3.8-27b/BENCHMARKS.md +++ b/docs/models/qwen3.8-27b/BENCHMARKS.md @@ -8,18 +8,29 @@ remain supported; a full comparison across context depths is **TODO**. PNG/JPEG image input uses the matching BF16 projector with AR or DFlash2. Native MTP is CLI-only. [Image usage and quality checks](README.md#images). +Reference implementation: **llama.cpp** `llama-server` from this repository's +`flake.nix` (ROCm, gfx1151, same GGUF files), measured over HTTP with the same +prompts and timed scope. **Gain** is Gufo over llama.cpp, positive when Gufo +is faster; `Gain vs llama.cpp AR` compares DFlash2 against llama.cpp without +a draft. Layout and method: [benchmark-model skill](../../../.agents/skills/benchmark-model/SKILL.md). + Each table identifies its workload and measurement date. Unmeasured points -are **TODO**; the concurrency sweep still needs a performance refresh. +are **TODO**. ## Loading and continuation 2026-09-21, one cold-file-cache launch to HTTP readiness, including Q4_K_M -DFlash2 and two sessions at context capacity 262144: +DFlash2 and two sessions at context capacity 262144. llama.cpp readiness +with the same GGUF: **TODO**. -| Target | Ready | -| --- | ---: | -| Q4 | 6.38 s | -| Q8 | 8.60 s | + +| Target | Gufo ready | llama.cpp ready | Gain | +| --- | ---: | ---: | ---: | +| Q4 | 6.38 s | TODO | TODO | +| Q8 | 8.60 s | TODO | TODO | + + +![Cold-file-cache launch to readiness](artifacts/charts/loading.svg) A 626-token Q4+DFlash2 prompt snapshot occupies **216 MiB**, independent of unused context capacity. Cancel/continue reused 661 tokens and prefilled @@ -28,34 +39,70 @@ These are bounded controls, not a long-conversation latency distribution. ## Single user, autoregressive -Standard sweep: **pp2048 / tg128**, C1. Cells are prefill / generation tok/s. -`TODO` means the current implementation needs a qualified measurement. +Standard sweep: **pp2048 / tg128**, C1, greedy. Cells are tok/s; depth is a +cached prefix, and the timed operation follows it. Gufo values measured +**2026-09-19** with `gufo bench`, one warmed release sample per point; Q4 d0 +is the mean of two controls (PP range 620.38–623.61 tok/s). The HTTP refresh +of both sides with identical prompts is **TODO**. -| Context depth | Q4 | Q8 | -| ---: | ---: | ---: | -| 0 | **622.00 / 11.74** | **445.86 / 7.11** | -| 4,096 | TODO | TODO | -| 8,192 | TODO | TODO | -| 12,288 | TODO | TODO | -| 16,384 | TODO | TODO | -| 32,768 | **467.63 / 10.30** | TODO | + +| Depth | Gufo pp | llama.cpp pp | Gain | Gufo tg | llama.cpp tg | Gain | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 622.00 | TODO | TODO | 11.74 | TODO | TODO | +| 4,096 | TODO | TODO | TODO | TODO | TODO | TODO | +| 8,192 | TODO | TODO | TODO | TODO | TODO | TODO | +| 12,288 | TODO | TODO | TODO | TODO | TODO | TODO | +| 16,384 | TODO | TODO | TODO | TODO | TODO | TODO | +| 32,768 | 467.63 | TODO | TODO | 10.30 | TODO | TODO | + + +![Single user, autoregressive](artifacts/charts/single-ar-q4.svg) + + +| Depth | Gufo pp | llama.cpp pp | Gain | Gufo tg | llama.cpp tg | Gain | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 445.86 | TODO | TODO | 7.11 | TODO | TODO | +| 4,096 | TODO | TODO | TODO | TODO | TODO | TODO | +| 8,192 | TODO | TODO | TODO | TODO | TODO | TODO | +| 12,288 | TODO | TODO | TODO | TODO | TODO | TODO | +| 16,384 | TODO | TODO | TODO | TODO | TODO | TODO | +| 32,768 | TODO | TODO | TODO | TODO | TODO | TODO | + + +![Single user, autoregressive](artifacts/charts/single-ar-q8.svg) -Measured **2026-09-19**, one warmed release sample per point; Q4 d0 is the -mean of two controls (PP range 620.38–623.61 tok/s). ## Single user, DFlash2 **pp2048 / tg128**, C1, greedy, Q4_K_M draft and adaptive controller. -Cells are prefill / generation tok/s. Depth zero refreshed **2026-09-21**; -32K measured **2026-09-19**. One warmed sample per point. +Depth zero refreshed **2026-09-21**; 32K measured **2026-09-19**. One warmed +sample per point. llama.cpp has no DFlash2 equivalent: the reference column +is llama.cpp autoregressive generation with the same target GGUF. -| Context depth | Q4 target | Q8 target | -| ---: | ---: | ---: | -| 0 | **635.75 / 27.86** | **464.32 / 25.03** | -| 4,096 | TODO | TODO | -| 8,192 | TODO | TODO | -| 12,288 | TODO | TODO | -| 16,384 | TODO | TODO | -| 32,768 | **442.86 / 31.01** | TODO | + +| Depth | Gufo pp | Gufo tg | Acceptance | llama.cpp AR tg | Gain vs llama.cpp AR | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 635.75 | 27.86 | TODO | TODO | TODO | +| 4,096 | TODO | TODO | TODO | TODO | TODO | +| 8,192 | TODO | TODO | TODO | TODO | TODO | +| 12,288 | TODO | TODO | TODO | TODO | TODO | +| 16,384 | TODO | TODO | TODO | TODO | TODO | +| 32,768 | 442.86 | 31.01 | 96.5% | TODO | TODO | + + +![Single user, DFlash2](artifacts/charts/single-dflash2-q4.svg) + + +| Depth | Gufo pp | Gufo tg | Acceptance | llama.cpp AR tg | Gain vs llama.cpp AR | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 464.32 | 25.03 | TODO | TODO | TODO | +| 4,096 | TODO | TODO | TODO | TODO | TODO | +| 8,192 | TODO | TODO | TODO | TODO | TODO | +| 12,288 | TODO | TODO | TODO | TODO | TODO | +| 16,384 | TODO | TODO | TODO | TODO | TODO | +| 32,768 | TODO | TODO | TODO | TODO | TODO | + + +![Single user, DFlash2](artifacts/charts/single-dflash2-q8.svg) The repetitive Q4 CLI input accepts 96.5% of proposals at d32768. Greedy outputs match AR. Natural prompts have different acceptance and speed. @@ -72,75 +119,140 @@ adaptive, greedy C1, one warmed release sample per draft. Generation tok/s: Full depth comparison across all three drafts and MTP performance: **TODO**. [Prompts, artifact identities and quality checks](EVALUATION.md). -## Multiple users, autoregressive +## Multiple users -Short generation controls, **2026-09-15/16**: `prose_tides`, context capacity 4096, -cached prompt, **tg64**, one warmup and one measured round. All concurrency -tables report **aggregate delivered tok/s**. C1 is the regression control. +Aggregate delivered output tok/s (`aggregate.output_tokens_per_second.overall`): +total output tokens divided by the sum of measured request-group spans. +Context capacity 4096, greedy, thinking off, **128 output tokens**, +`cache_prompt=false`, one warmup round. Workloads come from the +[speculative corpus](artifacts/speculative-corpus.json): `repetition` runs +`repetition_word`; `mixed` cycles through the nine distinct corpus cases. +DFlash2 uses the Q4_K_M draft with the adaptive controller. `Exact` counts +llama.cpp completions whose hash matches the Gufo AR C1 reference; low +counts mean the two servers produced different greedy text, so their +throughput is comparable but their outputs are not. -| Concurrency | Q4 | Q8 | -| ---: | ---: | ---: | -| 1 | 11.82 | 7.10 | -| 2 | 22.98 | 14.38 | -| 4 | 41.87 | 27.13 | -| 6 | 57.01 | 38.72 | -| 8 | 68.08 | 49.03 | +Measured **2026-09-16** at Gufo revision `61008879` against +`llama-server` 10063 (`7d56da7`, nixpkgs) with +`-ngl 999 -np 8 -c 32768 -fa on --cache-reuse 0 --jinja --reasoning off`, +one server process for each whole sweep. Retained artifacts: +`artifacts/{gufo-ar,gufo-dflash2,llama-server}-{mixed,repetition}.json`. +A refresh with the current binaries and a fresh server per point is **TODO**. -pp2048/tg128 across all five context depths at C2/4/6/8: **TODO**. + +| Users | Gufo AR | llama.cpp AR | Gain | Gufo DFlash2 | Gain vs llama.cpp AR | Exact | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 11.72 | 11.67 | +0.4% | 29.31 | +151.2% | 18/27 | +| 2 | 20.99 | 12.47 | +68.3% | 36.10 | +189.5% | 12/30 | +| 4 | 38.95 | 17.86 | +118.1% | 43.74 | +144.9% | 2/36 | +| 6 | 53.42 | 17.73 | +201.3% | 45.90 | +158.9% | 0/36 | +| 8 | 62.99 | 24.67 | +155.3% | 47.56 | +92.8% | 0/48 | + -## Multiple users, DFlash2 +![Multiple users, mixed corpus](artifacts/charts/multi-mixed-q4.svg) -Q4_K_M draft, adaptive controller, greedy **tg64**, cached prompts, context -capacity 4096. **2026-09-16**, three warmed rounds for C1/C4 repetition and Q8 -C6 repetition; one warmed pass for the remaining cells. -All three mixed-corpus prompts are warmed before measurement. -Aggregate delivered tok/s: + +| Users | Gufo AR | llama.cpp AR | Gain | Gufo DFlash2 | Gain vs llama.cpp AR | Exact | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 11.84 | 11.70 | +1.2% | 43.77 | +274.1% | 3/6 | +| 2 | 23.07 | 11.41 | +102.2% | 48.92 | +328.7% | 2/6 | +| 4 | 41.88 | 20.10 | +108.4% | 57.89 | +188.0% | 5/12 | +| 6 | 56.76 | 23.35 | +143.1% | 59.78 | +156.0% | 9/18 | +| 8 | 67.77 | 29.35 | +130.9% | 65.16 | +122.0% | 8/24 | + -| Concurrency | Q4 repetition | Q4 mixed corpus | Q8 repetition | Q8 mixed corpus | -| ---: | ---: | ---: | ---: | ---: | -| 1 | 62.08 | 36.46 | 48.85 | 24.77 | -| 2 | 84.07 | 41.85 | 78.64 | 33.00 | -| 4 | 93.62 | 46.48 | 89.60 | 35.66 | -| 6 | 96.06 | 55.02 | 93.48 | 42.34 | -| 8 | 100.37 | 56.73 | 96.81 | 43.92 | +![Multiple users, repetition](artifacts/charts/multi-repetition-q4.svg) -Repetition accepts **100%** of proposals. The mixed corpus contains 24 requests: -eight each of code, JSON and prose; acceptance is **61.21% for Q4 / 49.13% for -Q8**. Throughput is total output tokens divided by the sum of measured -request-group spans. C8 mixed latency (median / p95): **Q4 6.75 / 9.33 s; -Q8 8.36 / 12.37 s**. Physical widths, output hashes and acceptance counts are -checked at every concurrency; both C1 controls retain their performance. -The quality guide records each control's release. +DFlash2 acceptance is 49.6–54.2% on the mixed corpus and 71.1% on +repetition at every concurrency. Above C4 DFlash2 delivers less aggregate +throughput than Gufo AR: target quantized projections are the main scaling +limit. C1 DFlash2 already verifies seven or eight positions together; C2/C4 +increase that work to 14–16/28–32 positions, and larger batches bring limited +additional throughput. The **100 tok/s at C2 / 150 tok/s at C4** targets +remain unmet. -Target quantized projections are the main scaling limit. C1 DFlash2 already -verifies seven or eight positions together; C2/C4 increase that work to -14–16/28–32 positions. Larger batches currently bring limited additional -throughput. The **100 tok/s at C2 / 150 tok/s at C4** targets remain unmet. + +| Users | Gufo AR | llama.cpp AR | Gain | Gufo DFlash2 | Gain vs llama.cpp AR | Exact | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | TODO | TODO | TODO | TODO | TODO | TODO | +| 2 | TODO | TODO | TODO | TODO | TODO | TODO | +| 4 | TODO | TODO | TODO | TODO | TODO | TODO | +| 6 | TODO | TODO | TODO | TODO | TODO | TODO | +| 8 | TODO | TODO | TODO | TODO | TODO | TODO | + -Controller and verification details: [evaluation](EVALUATION.md). + +| Users | Gufo AR | llama.cpp AR | Gain | Gufo DFlash2 | Gain vs llama.cpp AR | Exact | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | TODO | TODO | TODO | TODO | TODO | TODO | +| 2 | TODO | TODO | TODO | TODO | TODO | TODO | +| 4 | TODO | TODO | TODO | TODO | TODO | TODO | +| 6 | TODO | TODO | TODO | TODO | TODO | TODO | +| 8 | TODO | TODO | TODO | TODO | TODO | TODO | + + +Earlier tg64 short-generation controls (2026-09-15/16, `prose_tides` and a +24-request mixed corpus) remain in Git history; they used a different +workload and are not comparable with the tables above. Physical widths, +output hashes and acceptance counts are checked at every concurrency; C1 +controls must retain their performance. Controller and verification details: +[evaluation](EVALUATION.md). + +## Memory + +GPU-visible unified allocation, C1, context capacity 262144, excluding +separate CPU memory. llama.cpp resident device memory at the same `-c`: **TODO**. + + +| Workload | Gufo GiB | llama.cpp GiB | Gain | +| --- | ---: | ---: | ---: | +| pp2048 + tg128 | TODO | TODO | TODO | +| 16K prefix, pp4096 + tg128 | TODO | TODO | TODO | + + + +| Workload | Gufo GiB | llama.cpp GiB | Gain | +| --- | ---: | ---: | ---: | +| pp2048 + tg128 | TODO | TODO | TODO | +| 16K prefix, pp4096 + tg128 | TODO | TODO | TODO | + + +## Image encoder + +Warm `mmproj-BF16.gguf` encoding, measured **2026-09-20**; excludes image +preprocessing, first weight upload and language-model prefill. Q4 and Q8 use +the same projector. Embeddings are byte-identical to the prior encoder; see +[experiments](EXPERIMENTS.md). llama.cpp `--mmproj` encode with the same +file: **TODO**. + + +| RGB image | Merged tokens | Gufo ms | llama.cpp ms | Gain | +| --- | ---: | ---: | ---: | ---: | +| 256×256 | 64 | TODO | TODO | TODO | +| 1024×1024 | 1024 | 1252 | TODO | TODO | + + +![Image encoder](artifacts/charts/image-encoder.svg) ## Reproduce and maintain quality -The shared image encoder, measured separately on 2026-09-20 with -`mmproj-BF16.gguf`, takes **1252 ms** for a 1024×1024 RGB image (1024 merged -tokens). This is a warm encode, excluding image preprocessing, first weight -upload and language-model prefill. Q4 and Q8 use the same projector. Embeddings -are byte-identical to the prior encoder; see [experiments](EXPERIMENTS.md). - ```sh nix build MODEL=/path/to/Qwen3.8-27B-UD-Q4_K_XL.gguf DRAFT=/path/to/Qwen3.8-27B-DFlash2-Q4_K_M.gguf ./result/bin/gufo bench --model "$MODEL" \ - -p 2048 -n 128 -d 0,4096,8192,12288,16384 -c 1 -r 2 -v + -p 2048 -n 128 -d 0,4096,8192,12288,16384,32768 -c 1 -r 2 -v ./result/bin/gufo bench --model "$MODEL" \ --speculative dflash2 --dflash-model "$DRAFT" \ - -p 2048 -n 128 -d 0,4096,8192,12288,16384 -c 1 -r 2 -v + -p 2048 -n 128 -d 0,4096,8192,12288,16384,32768 -c 1 -r 2 -v ``` -Use `-p 4096` for pp4096. Qwen `bench` is greedy and C1; use -[the HTTP benchmark](../../../tools/README.md) for sampled or concurrent requests. -DFlash2 prefill includes feature capture and draft context injection. +`gufo bench` is greedy and C1 and is the kernel-iteration tool. Published +comparison tables are measured over HTTP on both sides and rendered with +`tools/bench/model-bench.py` (`run --target gufo`, `run --target reference`, +`render`); the `benchmark-model` skill describes the workloads, artifacts and +table layout. DFlash2 prefill includes feature capture and draft context +injection. Start with `nix develop -c python3 tools/qwen27b/check.py fast`, then run the [affected quality checks](EVALUATION.md) before publishing speed. Greedy diff --git a/docs/models/qwen3.8-27b/artifacts/bench.json b/docs/models/qwen3.8-27b/artifacts/bench.json new file mode 100644 index 00000000..faeb355b --- /dev/null +++ b/docs/models/qwen3.8-27b/artifacts/bench.json @@ -0,0 +1,225 @@ +{ + "schema": "gufo-model-bench/1", + "category": "llm", + "model": { + "id": "Qwen3.8-27B", + "docs": "qwen3.8-27b" + }, + "files": { + "gguf": "target GGUF (UD-Q4_K_XL / UD-Q8_K_XL)", + "draft": "DFlash2 draft GGUF (Q4_K_M recommended)", + "mmproj": "BF16 vision projector, shared by both variants" + }, + "variants": { + "q4": { + "label": "Q4", + "requires": [ + "gguf", + "draft" + ] + }, + "q8": { + "label": "Q8", + "requires": [ + "gguf", + "draft" + ] + } + }, + "sampling": { + "temperature": 0, + "seed": 1 + }, + "speculative": { + "mode": "dflash2", + "label": "DFlash2", + "gufo_args": [ + "--speculative", + "dflash2", + "--dflash-model", + "{draft}" + ], + "reference": "ar" + }, + "tables": { + "loading": { + "title": "Cold-file-cache launch to readiness", + "per_variant": false, + "context": 262144, + "sessions": 2, + "repetitions": 1, + "metric": "seconds", + "better": "lower", + "rows": "variants" + }, + "single-ar": { + "title": "Single user, autoregressive", + "per_variant": true, + "prompt_tokens": 2048, + "output_tokens": 128, + "depths": [ + 0, + 4096, + 8192, + 12288, + 16384, + 32768 + ], + "context": 34944, + "depth_tolerance": 0.005, + "prefix": { + "generator": "synthetic-paragraphs", + "seed": 1 + }, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "single-dflash2": { + "title": "Single user, DFlash2", + "per_variant": true, + "speculative": true, + "prompt_tokens": 2048, + "output_tokens": 128, + "depths": [ + 0, + 4096, + 8192, + 12288, + 16384, + 32768 + ], + "context": 34944, + "depth_tolerance": 0.005, + "prefix": { + "generator": "synthetic-paragraphs", + "seed": 1 + }, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "multi-repetition": { + "title": "Multiple users, repetition", + "per_variant": true, + "modes": [ + "ar", + "dflash2" + ], + "suite": "../../qwen3.8-27b/artifacts/speculative-corpus.json", + "cases": [ + "repetition_word" + ], + "corpus_layout": "homogeneous", + "concurrency": [ + 1, + 2, + 4, + 6, + 8 + ], + "context": 4096, + "output_tokens": 128, + "cache_prompt": false, + "warmup": 1, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "multi-mixed": { + "title": "Multiple users, mixed corpus", + "per_variant": true, + "modes": [ + "ar", + "dflash2" + ], + "suite": "../../qwen3.8-27b/artifacts/speculative-corpus.json", + "cases": [ + "expository_pangram", + "cpp_ring_buffer", + "reasoning_train", + "summary_gpu", + "italian_explanation", + "chinese_explanation", + "json_schedule", + "creative_station", + "instruction_debug" + ], + "corpus_layout": "distinct", + "concurrency": [ + 1, + 2, + 4, + 6, + 8 + ], + "context": 4096, + "output_tokens": 128, + "cache_prompt": false, + "warmup": 1, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "memory": { + "title": "GPU-visible allocation", + "per_variant": true, + "context": 262144, + "workloads": [ + { + "id": "pp2048 + tg128", + "depth": 0, + "prompt_tokens": 2048, + "output_tokens": 128 + }, + { + "id": "16K prefix, pp4096 + tg128", + "depth": 16384, + "prompt_tokens": 4096, + "output_tokens": 128 + } + ], + "metric": "GiB", + "better": "lower" + }, + "image-encoder": { + "title": "Image encoder", + "per_variant": false, + "sizes": [ + 256, + 1024 + ], + "warmup": 1, + "repetitions": 3, + "metric": "ms", + "better": "lower" + } + }, + "gufo": { + "serve": [], + "llm": [ + "--think", + "off", + "--max-pending-per-client", + "8" + ], + "readiness": "/ready" + }, + "reference": { + "name": "llama.cpp", + "server": "llama-server", + "args": [ + "-ngl", + "999", + "-fa", + "on", + "--cache-reuse", + "0", + "--jinja", + "--reasoning", + "off" + ], + "endpoint_profile": "openai", + "readiness": "/health" + } +} diff --git a/docs/models/qwen3.8-27b/artifacts/charts/image-encoder.svg b/docs/models/qwen3.8-27b/artifacts/charts/image-encoder.svg new file mode 100644 index 00000000..6c1038bb --- /dev/null +++ b/docs/models/qwen3.8-27b/artifacts/charts/image-encoder.svg @@ -0,0 +1,267 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 256×256 + + + + + + + + + + 1024×1024 + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 400 + + + + + + + + + + + + + 600 + + + + + + + + + + + + + 800 + + + + + + + + + + + + + 1000 + + + + + + + + + + + + + 1200 + + + + + + + + + + + + + 1400 + + + + encode ms + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + llama.cpp + + + + + Image encoder + + + + + + + + diff --git a/docs/models/qwen3.8-27b/artifacts/charts/loading.svg b/docs/models/qwen3.8-27b/artifacts/charts/loading.svg new file mode 100644 index 00000000..d91c6848 --- /dev/null +++ b/docs/models/qwen3.8-27b/artifacts/charts/loading.svg @@ -0,0 +1,237 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + Q4 + + + + + + + + + + Q8 + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 2 + + + + + + + + + + + + + 4 + + + + + + + + + + + + + 6 + + + + + + + + + + + + + 8 + + + + + + + + + + + + + 10 + + + + seconds to ready + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + llama.cpp + + + + + Cold-file-cache launch to readiness + + + + + + + + diff --git a/docs/models/qwen3.8-27b/artifacts/charts/multi-mixed-q4.svg b/docs/models/qwen3.8-27b/artifacts/charts/multi-mixed-q4.svg new file mode 100644 index 00000000..1fcddee3 --- /dev/null +++ b/docs/models/qwen3.8-27b/artifacts/charts/multi-mixed-q4.svg @@ -0,0 +1,399 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + 2 + + + + + + + + + + 4 + + + + + + + + + + 6 + + + + + + + + + + 8 + + + + concurrent users + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 30 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 50 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 70 + + + + aggregate output tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo AR + + + + + + llama.cpp AR + + + + + + Gufo DFlash2 + + + + + Multiple users, mixed corpus (Q4) + + + + + + + + diff --git a/docs/models/qwen3.8-27b/artifacts/charts/multi-repetition-q4.svg b/docs/models/qwen3.8-27b/artifacts/charts/multi-repetition-q4.svg new file mode 100644 index 00000000..9dd108a6 --- /dev/null +++ b/docs/models/qwen3.8-27b/artifacts/charts/multi-repetition-q4.svg @@ -0,0 +1,354 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + 2 + + + + + + + + + + 4 + + + + + + + + + + 6 + + + + + + + + + + 8 + + + + concurrent users + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 80 + + + + aggregate output tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo AR + + + + + + llama.cpp AR + + + + + + Gufo DFlash2 + + + + + Multiple users, repetition (Q4) + + + + + + + + diff --git a/docs/models/qwen3.8-27b/artifacts/charts/single-ar-q4.svg b/docs/models/qwen3.8-27b/artifacts/charts/single-ar-q4.svg new file mode 100644 index 00000000..02853e19 --- /dev/null +++ b/docs/models/qwen3.8-27b/artifacts/charts/single-ar-q4.svg @@ -0,0 +1,539 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + context depth (tokens) + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 100 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 300 + + + + + + + + + + + + + 400 + + + + + + + + + + + + + 500 + + + + + + + + + + + + + 600 + + + + + + + + + + + + + 700 + + + + prefill tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + + + + + + + llama.cpp + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + context depth (tokens) + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 2 + + + + + + + + + + + + + 4 + + + + + + + + + + + + + 6 + + + + + + + + + + + + + 8 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 12 + + + + + + + + + + + + + 14 + + + + generation tok/s + + + + + + + + + + + + + + + + + + + Single user, autoregressive (Q4) + + + + + + + + + + + diff --git a/docs/models/qwen3.8-27b/artifacts/charts/single-ar-q8.svg b/docs/models/qwen3.8-27b/artifacts/charts/single-ar-q8.svg new file mode 100644 index 00000000..483a9fe3 --- /dev/null +++ b/docs/models/qwen3.8-27b/artifacts/charts/single-ar-q8.svg @@ -0,0 +1,520 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + context depth (tokens) + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 100 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 300 + + + + + + + + + + + + + 400 + + + + + + + + + + + + + 500 + + + + prefill tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + + + + + + + llama.cpp + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + context depth (tokens) + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 1 + + + + + + + + + + + + + 2 + + + + + + + + + + + + + 3 + + + + + + + + + + + + + 4 + + + + + + + + + + + + + 5 + + + + + + + + + + + + + 6 + + + + + + + + + + + + + 7 + + + + + + + + + + + + + 8 + + + + generation tok/s + + + + + + + + + + + + + + + + + + Single user, autoregressive (Q8) + + + + + + + + + + + diff --git a/docs/models/qwen3.8-27b/artifacts/charts/single-dflash2-q4.svg b/docs/models/qwen3.8-27b/artifacts/charts/single-dflash2-q4.svg new file mode 100644 index 00000000..a0c860ed --- /dev/null +++ b/docs/models/qwen3.8-27b/artifacts/charts/single-dflash2-q4.svg @@ -0,0 +1,506 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + context depth (tokens) + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 5 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 15 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 25 + + + + + + + + + + + + + 30 + + + + + + + + + + + + + 35 + + + + generation tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo DFlash2 + + + + + + + + + + + + llama.cpp AR + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + context depth (tokens) + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 80 + + + + + + + + + + + + + 100 + + + + acceptance % + + + + + + + + + + + + + + + + + Single user, DFlash2 (Q4) + + + + + + + + + + + diff --git a/docs/models/qwen3.8-27b/artifacts/charts/single-dflash2-q8.svg b/docs/models/qwen3.8-27b/artifacts/charts/single-dflash2-q8.svg new file mode 100644 index 00000000..02f1315a --- /dev/null +++ b/docs/models/qwen3.8-27b/artifacts/charts/single-dflash2-q8.svg @@ -0,0 +1,483 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + context depth (tokens) + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 5 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 15 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 25 + + + + + + + + + + + + + 30 + + + + generation tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo DFlash2 + + + + + + + + + + + + llama.cpp AR + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + context depth (tokens) + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 80 + + + + + + + + + + + + + 100 + + + + acceptance % + + + + + + + + + + + + Single user, DFlash2 (Q8) + + + + + + + + + + + diff --git a/docs/models/qwen3.8-27b/artifacts/gufo-ar-mixed.json b/docs/models/qwen3.8-27b/artifacts/multi-mixed-q4-gufo-ar.json similarity index 99% rename from docs/models/qwen3.8-27b/artifacts/gufo-ar-mixed.json rename to docs/models/qwen3.8-27b/artifacts/multi-mixed-q4-gufo-ar.json index ae6eab2b..2fe37604 100644 --- a/docs/models/qwen3.8-27b/artifacts/gufo-ar-mixed.json +++ b/docs/models/qwen3.8-27b/artifacts/multi-mixed-q4-gufo-ar.json @@ -38,11 +38,18 @@ "model": { "id": "Qwen3.8-27B" }, + "modelBench": { + "measuredOn": "2026-09-16", + "mode": "ar", + "table": "multi-mixed-q4", + "target": "gufo" + }, "notes": [ "gufo serve --sessions 8 llm --context 4096 --think off; one process for the whole sweep; Qwen3.8-27B-UD-Q4_K_XL (unsloth snapshot 4ca72078)", "speculative off", "Retained summary: raw request samples and rounds are omitted; aggregates and per-case completion hashes are preserved." ], + "rawReportSha256": "fede10185c63866fe67d1f272187226a176cedeb6d963a589711d41ce476cb92", "reference": { "source": "self-c1" }, @@ -3411,6 +3418,5 @@ "temperature": 0.0, "transport": "openai-chat-completions-sse", "warmupRounds": 1 - }, - "rawReportSha256": "fede10185c63866fe67d1f272187226a176cedeb6d963a589711d41ce476cb92" + } } diff --git a/docs/models/qwen3.8-27b/artifacts/gufo-dflash2-mixed.json b/docs/models/qwen3.8-27b/artifacts/multi-mixed-q4-gufo-dflash2.json similarity index 99% rename from docs/models/qwen3.8-27b/artifacts/gufo-dflash2-mixed.json rename to docs/models/qwen3.8-27b/artifacts/multi-mixed-q4-gufo-dflash2.json index e1aa95ce..356a703a 100644 --- a/docs/models/qwen3.8-27b/artifacts/gufo-dflash2-mixed.json +++ b/docs/models/qwen3.8-27b/artifacts/multi-mixed-q4-gufo-dflash2.json @@ -38,11 +38,18 @@ "model": { "id": "Qwen3.8-27B" }, + "modelBench": { + "measuredOn": "2026-09-16", + "mode": "dflash2", + "table": "multi-mixed-q4", + "target": "gufo" + }, "notes": [ "gufo serve --sessions 8 llm --context 4096 --think off; one process for the whole sweep; Qwen3.8-27B-UD-Q4_K_XL (unsloth snapshot 4ca72078)", "speculative dflash2, draft Qwen3.8-27B-DFlash2-Q4_K_M, draft-policy adaptive, draft-tokens 7", "Retained summary: raw request samples and rounds are omitted; aggregates and per-case completion hashes are preserved." ], + "rawReportSha256": "6df092eef5a228d2e5d8f62c1e8ffee1f66f1cbb8700dfe5452c771d100dec9a", "reference": { "modelId": "Qwen3.8-27B", "source": "report", @@ -3411,6 +3418,5 @@ "temperature": 0.0, "transport": "openai-chat-completions-sse", "warmupRounds": 1 - }, - "rawReportSha256": "6df092eef5a228d2e5d8f62c1e8ffee1f66f1cbb8700dfe5452c771d100dec9a" + } } diff --git a/docs/models/qwen3.8-27b/artifacts/llama-server-mixed.json b/docs/models/qwen3.8-27b/artifacts/multi-mixed-q4-reference.json similarity index 99% rename from docs/models/qwen3.8-27b/artifacts/llama-server-mixed.json rename to docs/models/qwen3.8-27b/artifacts/multi-mixed-q4-reference.json index 9ddfee29..4599d23a 100644 --- a/docs/models/qwen3.8-27b/artifacts/llama-server-mixed.json +++ b/docs/models/qwen3.8-27b/artifacts/multi-mixed-q4-reference.json @@ -38,10 +38,17 @@ "model": { "id": "Qwen3.8-27B" }, + "modelBench": { + "measuredOn": "2026-09-16", + "mode": null, + "table": "multi-mixed-q4", + "target": "reference" + }, "notes": [ "llama-server 10063 (7d56da7, nixpkgs llama-cpp-10063) -ngl 999 -np 8 -c 32768 -fa on --cache-reuse 0 --jinja --reasoning off; cache_prompt=false per request; one process for the whole sweep; same Q4 GGUF", "Retained summary: raw request samples and rounds are omitted; aggregates and per-case completion hashes are preserved." ], + "rawReportSha256": "c32ab4c82bbcf95fe3da182477cf7e25e19c7d2fac5217205e24dd7edcd5cfd9", "reference": { "modelId": "Qwen3.8-27B", "source": "report", @@ -3766,6 +3773,5 @@ "temperature": 0.0, "transport": "openai-chat-completions-sse", "warmupRounds": 1 - }, - "rawReportSha256": "c32ab4c82bbcf95fe3da182477cf7e25e19c7d2fac5217205e24dd7edcd5cfd9" + } } diff --git a/docs/models/qwen3.8-27b/artifacts/gufo-ar-repetition.json b/docs/models/qwen3.8-27b/artifacts/multi-repetition-q4-gufo-ar.json similarity index 99% rename from docs/models/qwen3.8-27b/artifacts/gufo-ar-repetition.json rename to docs/models/qwen3.8-27b/artifacts/multi-repetition-q4-gufo-ar.json index e11a7eb8..2b3f87b0 100644 --- a/docs/models/qwen3.8-27b/artifacts/gufo-ar-repetition.json +++ b/docs/models/qwen3.8-27b/artifacts/multi-repetition-q4-gufo-ar.json @@ -38,11 +38,18 @@ "model": { "id": "Qwen3.8-27B" }, + "modelBench": { + "measuredOn": "2026-09-16", + "mode": "ar", + "table": "multi-repetition-q4", + "target": "gufo" + }, "notes": [ "gufo serve --sessions 8 llm --context 4096 --think off; one process for the whole sweep; Qwen3.8-27B-UD-Q4_K_XL (unsloth snapshot 4ca72078)", "speculative off", "Retained summary: raw request samples and rounds are omitted; aggregates and per-case completion hashes are preserved." ], + "rawReportSha256": "20e56c1599a487ba711d9888c522709aba5588b31252dcaf442dacd1df51206a", "reference": { "source": "self-c1" }, @@ -1294,6 +1301,5 @@ "temperature": 0.0, "transport": "openai-chat-completions-sse", "warmupRounds": 1 - }, - "rawReportSha256": "20e56c1599a487ba711d9888c522709aba5588b31252dcaf442dacd1df51206a" + } } diff --git a/docs/models/qwen3.8-27b/artifacts/gufo-dflash2-repetition.json b/docs/models/qwen3.8-27b/artifacts/multi-repetition-q4-gufo-dflash2.json similarity index 99% rename from docs/models/qwen3.8-27b/artifacts/gufo-dflash2-repetition.json rename to docs/models/qwen3.8-27b/artifacts/multi-repetition-q4-gufo-dflash2.json index 6ed1fca0..bd6e1096 100644 --- a/docs/models/qwen3.8-27b/artifacts/gufo-dflash2-repetition.json +++ b/docs/models/qwen3.8-27b/artifacts/multi-repetition-q4-gufo-dflash2.json @@ -38,11 +38,18 @@ "model": { "id": "Qwen3.8-27B" }, + "modelBench": { + "measuredOn": "2026-09-16", + "mode": "dflash2", + "table": "multi-repetition-q4", + "target": "gufo" + }, "notes": [ "gufo serve --sessions 8 llm --context 4096 --think off; one process for the whole sweep; Qwen3.8-27B-UD-Q4_K_XL (unsloth snapshot 4ca72078)", "speculative dflash2, draft Qwen3.8-27B-DFlash2-Q4_K_M, draft-policy adaptive, draft-tokens 7", "Retained summary: raw request samples and rounds are omitted; aggregates and per-case completion hashes are preserved." ], + "rawReportSha256": "1a2ec688be053a6cf028eb683ad405a5236bf58d0225babdeb365543a2af36a5", "reference": { "modelId": "Qwen3.8-27B", "source": "report", @@ -1297,6 +1304,5 @@ "temperature": 0.0, "transport": "openai-chat-completions-sse", "warmupRounds": 1 - }, - "rawReportSha256": "1a2ec688be053a6cf028eb683ad405a5236bf58d0225babdeb365543a2af36a5" + } } diff --git a/docs/models/qwen3.8-27b/artifacts/llama-server-repetition.json b/docs/models/qwen3.8-27b/artifacts/multi-repetition-q4-reference.json similarity index 99% rename from docs/models/qwen3.8-27b/artifacts/llama-server-repetition.json rename to docs/models/qwen3.8-27b/artifacts/multi-repetition-q4-reference.json index d9d6d43e..46157d89 100644 --- a/docs/models/qwen3.8-27b/artifacts/llama-server-repetition.json +++ b/docs/models/qwen3.8-27b/artifacts/multi-repetition-q4-reference.json @@ -38,10 +38,17 @@ "model": { "id": "Qwen3.8-27B" }, + "modelBench": { + "measuredOn": "2026-09-16", + "mode": null, + "table": "multi-repetition-q4", + "target": "reference" + }, "notes": [ "llama-server 10063 (7d56da7, nixpkgs llama-cpp-10063) -ngl 999 -np 8 -c 32768 -fa on --cache-reuse 0 --jinja --reasoning off; cache_prompt=false per request; one process for the whole sweep; same Q4 GGUF", "Retained summary: raw request samples and rounds are omitted; aggregates and per-case completion hashes are preserved." ], + "rawReportSha256": "fde8c391cb16aa2f20591c6eacfa303a2ed0081275e4dfb04ecc43a933e046be", "reference": { "modelId": "Qwen3.8-27B", "source": "report", @@ -1344,6 +1351,5 @@ "temperature": 0.0, "transport": "openai-chat-completions-sse", "warmupRounds": 1 - }, - "rawReportSha256": "fde8c391cb16aa2f20591c6eacfa303a2ed0081275e4dfb04ecc43a933e046be" + } } diff --git a/docs/models/qwen3.8-flash-next/BENCHMARKS.md b/docs/models/qwen3.8-flash-next/BENCHMARKS.md index c32012b4..e5d79a4f 100644 --- a/docs/models/qwen3.8-flash-next/BENCHMARKS.md +++ b/docs/models/qwen3.8-flash-next/BENCHMARKS.md @@ -12,35 +12,80 @@ preserved. PNG/JPEG input works with AR/MTP and `mmproj-BF16.gguf`; see [image usage and validation](README.md#images). **Original unquantized-model and GGUF-conversion parity remain unqualified.** -Cold-file-cache launch to HTTP readiness: **14.91 s** on 2026-09-21, -including all four target shards, MTP and two sessions at context capacity -262144. A direct 4095-token snapshot occupies **221 MiB**; restore/replay +Reference implementation: **llama.cpp** `llama-server` from this repository's +`flake.nix` (ROCm, gfx1151, same sharded GGUF), measured over HTTP with the +same prompts and timed scope. **Gain** is Gufo over llama.cpp, positive when +Gufo is faster; `Gain vs llama.cpp AR` compares MTP against llama.cpp without +a draft. All llama.cpp cells are **TODO** until the first HTTP sweep. Layout +and method: [benchmark-model skill](../../../.agents/skills/benchmark-model/SKILL.md). + +## Loading + +Cold-file-cache launch to HTTP readiness on 2026-09-21, including all four +target shards, MTP and two sessions at context capacity 262144. + + +| Target | Gufo ready | llama.cpp ready | Gain | +| --- | ---: | ---: | ---: | +| UD-Q4_K_XL | 14.91 s | TODO | TODO | + + +![Cold-file-cache launch to readiness](artifacts/charts/loading.svg) + +A direct 4095-token snapshot occupies **221 MiB**; restore/replay qualification is in [evaluation](EVALUATION.md). -## Single user +## Single user, autoregressive -Tok/s, pp2048/tg128, greedy, seed 1, context capacity 133121. Depth precedes -the timed operation. The raw CLI uses a fixed repeated token pattern, warms -prefill and continues past EOS; d0 TG starts from 16 tokens. MTP PP includes -predictor catch-up through all known successor tokens. +Tok/s, pp2048/tg128, greedy, seed 1, context capacity 133121. Depth is a +cached prefix and precedes the timed operation. The raw CLI uses a fixed +repeated token pattern, warms prefill and continues past EOS; d0 TG starts +from 16 tokens. Gufo values from `gufo bench`, d0/d32K/d128K refreshed +2026-09-21, other depths 2026-09-19. -| Depth | AR pp2048 | MTP pp2048 | AR tg128 | MTP tg128 | MTP acceptance | + +| Depth | Gufo pp | llama.cpp pp | Gain | Gufo tg | llama.cpp tg | Gain | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 1537.5 | TODO | TODO | 26.43 | TODO | TODO | +| 4,096 | 1455.1 | TODO | TODO | 25.48 | TODO | TODO | +| 8,192 | 1432.0 | TODO | TODO | 25.25 | TODO | TODO | +| 12,288 | 1421.4 | TODO | TODO | 25.19 | TODO | TODO | +| 16,384 | 1412.3 | TODO | TODO | 24.91 | TODO | TODO | +| 32,768 | 1406.4 | TODO | TODO | 24.28 | TODO | TODO | +| 65,536 | 1342.7 | TODO | TODO | 23.08 | TODO | TODO | +| 131,072 | 1310.5 | TODO | TODO | 22.33 | TODO | TODO | + + +![Single user, autoregressive](artifacts/charts/single-ar.svg) + +AR prefill falls 14.8% from d0 to d128K. Single runs vary by about 1% within +a session and the machine drifted 2% across one evening; the d0 prefill +change was qualified by interleaved runs of both Nix binaries +(1559.6 → 1568.1 tok/s). The 1700 tok/s target and flat deep-context +throughput remain unmet. + +## Single user, MTP + +Same workload with `--speculative mtp`. MTP PP includes predictor catch-up +through all known successor tokens and adds about 2% to prefill time at the +three refreshed depths; greedy output matches AR at each depth. llama.cpp has +no MTP for this model: the reference column is llama.cpp autoregressive +generation with the same target GGUF. + + +| Depth | Gufo pp | Gufo tg | Acceptance | llama.cpp AR tg | Gain vs llama.cpp AR | | ---: | ---: | ---: | ---: | ---: | ---: | -| 0 | 1537.5 | 1503.6 | 26.43 | 43.95 | 69.2% | -| 4K | 1455.1 | TODO | 25.48 | 40.35 | 65.6% | -| 8K | 1432.0 | TODO | 25.25 | 35.68 | 58.3% | -| 12K | 1421.4 | TODO | 25.19 | 52.03 | 85.7% | -| 16K | 1412.3 | TODO | 24.91 | 56.31 | 88.5% | -| 32K | 1406.4 | 1380.3 | 24.28 | 65.65 | 100.0% | -| 64K | 1342.7 | TODO | 23.08 | 59.67 | 100.0% | -| 128K | 1310.5 | 1287.5 | 22.33 | 61.19 | 100.0% | +| 0 | 1503.6 | 43.95 | 69.2% | TODO | TODO | +| 4,096 | TODO | 40.35 | 65.6% | TODO | TODO | +| 8,192 | TODO | 35.68 | 58.3% | TODO | TODO | +| 12,288 | TODO | 52.03 | 85.7% | TODO | TODO | +| 16,384 | TODO | 56.31 | 88.5% | TODO | TODO | +| 32,768 | 1380.3 | 65.65 | 100.0% | TODO | TODO | +| 65,536 | TODO | 59.67 | 100.0% | TODO | TODO | +| 131,072 | 1287.5 | 61.19 | 100.0% | TODO | TODO | + -AR prefill falls 14.8% from d0 to d128K. MTP adds about 2% to prefill time -at the three refreshed depths, and its greedy output matches AR at each depth. -Single runs vary by about 1% within a session and the machine drifted 2% -across one evening; the d0 prefill change was qualified by interleaved runs of -both Nix binaries (1559.6 → 1568.1 tok/s). The 1700 tok/s target and flat -deep-context throughput remain unmet. +![Single user, MTP](artifacts/charts/single-mtp.svg) ```sh ./result/bin/gufo bench --model "$MODEL" -p 2048 -n 128 \ @@ -55,25 +100,47 @@ Sampled MTP, d0 raw prefix, seed 1, tg128, top-k/top-p/min-p disabled: | T=0.7 | 39.83 | 65.2% | | T=1.0 | 28.45 | 56.5% | -## Concurrent serving +## Multiple users -**Sum of individual request decode rates**, in tok/s. Each rate uses -`completion_tokens / decode_seconds` from server-reported decode time; -prefill and waiting between decode steps are excluded. Context 4096, greedy, -thinking off, 128 output tokens, one cold cohort on a fresh server per point; -no cache hits. Set `--sessions C --max-pending-per-client 8` when benchmarking -up to eight requests from one host. -The [corpus](../qwen3.8-27b/artifacts/speculative-corpus.json) -uses `repetition_word` for repetition and distinct requests cycling through -`expository_pangram`, `cpp_ring_buffer`, `reasoning_train` for mixed work. +Aggregate delivered output tok/s. Context 4096, greedy, thinking off, 128 +output tokens, `cache_prompt=false`, one cold cohort on a fresh server per +point. Workloads come from the +[corpus](../qwen3.8-27b/artifacts/speculative-corpus.json): `repetition` +runs `repetition_word`; `mixed` cycles distinct requests through +`expository_pangram`, `cpp_ring_buffer`, `reasoning_train`. `Exact` counts +llama.cpp completions whose hash matches the Gufo AR C1 reference. +Set `--sessions C --max-pending-per-client 8` when benchmarking up to eight +requests from one host. -| Users | AR repetitive | MTP repetitive | AR mixed | MTP mixed | -| ---: | ---: | ---: | ---: | ---: | -| 1 | 27.3 | 84.6 | 27.0 | 49.3 | -| 2 | 46.4 | 127.7 | 43.2 | 66.4 | -| 4 | 76.7 | 166.1 | 66.8 | 95.2 | -| 6 | 95.4 | 181.2 | 78.2 | 110.4 | -| 8 | 108.5 | **188.9** | 86.7 | 118.3 | +Gufo values measured 2026-09-20 as the **sum of individual request decode +rates** (`completion_tokens / decode_seconds` from server-reported decode +time, excluding prefill and waiting between decode steps). The HTTP refresh +that uses the corpus aggregate metric on both sides is **TODO**; until then +the Gufo and llama.cpp columns are not the same metric. + + +| Users | Gufo AR | llama.cpp AR | Gain | Gufo MTP | Gain vs llama.cpp AR | Exact | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 27.3 | TODO | TODO | 84.6 | TODO | TODO | +| 2 | 46.4 | TODO | TODO | 127.7 | TODO | TODO | +| 4 | 76.7 | TODO | TODO | 166.1 | TODO | TODO | +| 6 | 95.4 | TODO | TODO | 181.2 | TODO | TODO | +| 8 | 108.5 | TODO | TODO | **188.9** | TODO | TODO | + + +![Multiple users, repetition](artifacts/charts/multi-repetition.svg) + + +| Users | Gufo AR | llama.cpp AR | Gain | Gufo MTP | Gain vs llama.cpp AR | Exact | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 27.0 | TODO | TODO | 49.3 | TODO | TODO | +| 2 | 43.2 | TODO | TODO | 66.4 | TODO | TODO | +| 4 | 66.8 | TODO | TODO | 95.2 | TODO | TODO | +| 6 | 78.2 | TODO | TODO | 110.4 | TODO | TODO | +| 8 | 86.7 | TODO | TODO | 118.3 | TODO | TODO | + + +![Multiple users, mixed corpus](artifacts/charts/multi-mixed.svg) MTP acceptance is 100% on repetition and 74.2–84.9% on the mixed cohorts. Every completion matches AR; all cohorts report zero cache hits. @@ -83,18 +150,36 @@ drafts; KV caches and other session state are additional. **C1 is a single user.** The HTTP and raw CLI prompts above differ. Compare identical prompts and timing boundaries. -`gufo bench` is C1; use `tools/serving/gufo-serving-bench.py` for concurrency. +`gufo bench` is C1; the published tables come from `tools/bench/model-bench.py`. + +## Memory + +GPU-visible unified allocation, C1, context capacity 133121, excluding +separate CPU memory: **TODO**. llama.cpp resident device memory at the same +`-c`: **TODO**. + + +| Workload | Gufo GiB | llama.cpp GiB | Gain | +| --- | ---: | ---: | ---: | +| pp2048 + tg128 | TODO | TODO | TODO | +| 16K prefix, pp4096 + tg128 | TODO | TODO | TODO | + ## Image encoder Warm `mmproj-BF16.gguf` encoding; excludes preprocessing, first weight upload and language-model prefill. Complete embeddings are byte-identical to the -previous encoder and reproduce exactly across runs. +previous encoder and reproduce exactly across runs. llama.cpp `--mmproj` +encode with the same file: **TODO**. -| RGB image | Merged image tokens | Encode latency | -| --- | ---: | ---: | -| 256×256 | 64 | 21.3 ms | -| 1024×1024 | 1024 | 1249 ms | + +| RGB image | Merged tokens | Gufo ms | llama.cpp ms | Gain | +| --- | ---: | ---: | ---: | ---: | +| 256×256 | 64 | 21.3 | TODO | TODO | +| 1024×1024 | 1024 | 1249 | TODO | TODO | + + +![Image encoder](artifacts/charts/image-encoder.svg) The 4096-patch attention specialization saves 24 MiB of score/probability scratch. Other image shapes retain their original tile layout. diff --git a/docs/models/qwen3.8-flash-next/artifacts/bench.json b/docs/models/qwen3.8-flash-next/artifacts/bench.json new file mode 100644 index 00000000..926a69bc --- /dev/null +++ b/docs/models/qwen3.8-flash-next/artifacts/bench.json @@ -0,0 +1,216 @@ +{ + "schema": "gufo-model-bench/1", + "category": "llm", + "model": { + "id": "Qwen3.8-Flash-Next", + "docs": "qwen3.8-flash-next" + }, + "files": { + "gguf": "first shard of the sharded UD-Q4_K_XL GGUF", + "mtp": "mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf", + "mmproj": "BF16 vision projector" + }, + "variants": { + "default": { + "label": "UD-Q4_K_XL", + "requires": [ + "gguf", + "mtp" + ] + } + }, + "sampling": { + "temperature": 0, + "seed": 1 + }, + "speculative": { + "mode": "mtp", + "label": "MTP", + "gufo_args": [ + "--speculative", + "mtp", + "--mtp-model", + "{mtp}" + ], + "reference": "ar" + }, + "tables": { + "loading": { + "title": "Cold-file-cache launch to readiness", + "per_variant": false, + "context": 262144, + "sessions": 2, + "repetitions": 1, + "metric": "seconds", + "better": "lower", + "rows": "variants" + }, + "single-ar": { + "title": "Single user, autoregressive", + "per_variant": false, + "prompt_tokens": 2048, + "output_tokens": 128, + "depths": [ + 0, + 4096, + 8192, + 12288, + 16384, + 32768, + 65536, + 131072 + ], + "context": 133121, + "depth_tolerance": 0.005, + "prefix": { + "generator": "synthetic-paragraphs", + "seed": 1 + }, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "single-mtp": { + "title": "Single user, MTP", + "per_variant": false, + "speculative": true, + "prompt_tokens": 2048, + "output_tokens": 128, + "depths": [ + 0, + 4096, + 8192, + 12288, + 16384, + 32768, + 65536, + 131072 + ], + "context": 133121, + "depth_tolerance": 0.005, + "prefix": { + "generator": "synthetic-paragraphs", + "seed": 1 + }, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "multi-repetition": { + "title": "Multiple users, repetition", + "per_variant": false, + "modes": [ + "ar", + "mtp" + ], + "suite": "../../qwen3.8-27b/artifacts/speculative-corpus.json", + "cases": [ + "repetition_word" + ], + "corpus_layout": "homogeneous", + "concurrency": [ + 1, + 2, + 4, + 6, + 8 + ], + "context": 4096, + "output_tokens": 128, + "cache_prompt": false, + "warmup": 1, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "multi-mixed": { + "title": "Multiple users, mixed corpus", + "per_variant": false, + "modes": [ + "ar", + "mtp" + ], + "suite": "../../qwen3.8-27b/artifacts/speculative-corpus.json", + "cases": [ + "expository_pangram", + "cpp_ring_buffer", + "reasoning_train" + ], + "corpus_layout": "distinct", + "concurrency": [ + 1, + 2, + 4, + 6, + 8 + ], + "context": 4096, + "output_tokens": 128, + "cache_prompt": false, + "warmup": 1, + "repetitions": 1, + "metric": "tok/s", + "better": "higher" + }, + "memory": { + "title": "GPU-visible allocation", + "per_variant": false, + "context": 133121, + "workloads": [ + { + "id": "pp2048 + tg128", + "depth": 0, + "prompt_tokens": 2048, + "output_tokens": 128 + }, + { + "id": "16K prefix, pp4096 + tg128", + "depth": 16384, + "prompt_tokens": 4096, + "output_tokens": 128 + } + ], + "metric": "GiB", + "better": "lower" + }, + "image-encoder": { + "title": "Image encoder", + "per_variant": false, + "sizes": [ + 256, + 1024 + ], + "warmup": 1, + "repetitions": 3, + "metric": "ms", + "better": "lower" + } + }, + "gufo": { + "serve": [], + "llm": [ + "--think", + "off", + "--max-pending-per-client", + "8" + ], + "readiness": "/ready" + }, + "reference": { + "name": "llama.cpp", + "server": "llama-server", + "args": [ + "-ngl", + "999", + "-fa", + "on", + "--cache-reuse", + "0", + "--jinja", + "--reasoning", + "off" + ], + "endpoint_profile": "openai", + "readiness": "/health" + } +} diff --git a/docs/models/qwen3.8-flash-next/artifacts/charts/image-encoder.svg b/docs/models/qwen3.8-flash-next/artifacts/charts/image-encoder.svg new file mode 100644 index 00000000..845ce66a --- /dev/null +++ b/docs/models/qwen3.8-flash-next/artifacts/charts/image-encoder.svg @@ -0,0 +1,267 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 256×256 + + + + + + + + + + 1024×1024 + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 400 + + + + + + + + + + + + + 600 + + + + + + + + + + + + + 800 + + + + + + + + + + + + + 1000 + + + + + + + + + + + + + 1200 + + + + + + + + + + + + + 1400 + + + + encode ms + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + llama.cpp + + + + + Image encoder + + + + + + + + diff --git a/docs/models/qwen3.8-flash-next/artifacts/charts/loading.svg b/docs/models/qwen3.8-flash-next/artifacts/charts/loading.svg new file mode 100644 index 00000000..f91ea667 --- /dev/null +++ b/docs/models/qwen3.8-flash-next/artifacts/charts/loading.svg @@ -0,0 +1,241 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + UD-Q4_K_XL + + + + + + + + + + + + + + + + + + 0.0 + + + + + + + + + + + + + 2.5 + + + + + + + + + + + + + 5.0 + + + + + + + + + + + + + 7.5 + + + + + + + + + + + + + 10.0 + + + + + + + + + + + + + 12.5 + + + + + + + + + + + + + 15.0 + + + + + + + + + + + + + 17.5 + + + + seconds to ready + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + llama.cpp + + + + + Cold-file-cache launch to readiness + + + + + + + + diff --git a/docs/models/qwen3.8-flash-next/artifacts/charts/multi-mixed.svg b/docs/models/qwen3.8-flash-next/artifacts/charts/multi-mixed.svg new file mode 100644 index 00000000..b90606f2 --- /dev/null +++ b/docs/models/qwen3.8-flash-next/artifacts/charts/multi-mixed.svg @@ -0,0 +1,399 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + 2 + + + + + + + + + + 4 + + + + + + + + + + 6 + + + + + + + + + + 8 + + + + concurrent users + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 80 + + + + + + + + + + + + + 100 + + + + + + + + + + + + + 120 + + + + + + + + + + + + + 140 + + + + aggregate output tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo AR + + + + + + llama.cpp AR + + + + + + Gufo MTP + + + + + Multiple users, mixed corpus + + + + + + + + diff --git a/docs/models/qwen3.8-flash-next/artifacts/charts/multi-repetition.svg b/docs/models/qwen3.8-flash-next/artifacts/charts/multi-repetition.svg new file mode 100644 index 00000000..fed7382d --- /dev/null +++ b/docs/models/qwen3.8-flash-next/artifacts/charts/multi-repetition.svg @@ -0,0 +1,354 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + 2 + + + + + + + + + + 4 + + + + + + + + + + 6 + + + + + + + + + + 8 + + + + concurrent users + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 50 + + + + + + + + + + + + + 100 + + + + + + + + + + + + + 150 + + + + + + + + + + + + + 200 + + + + aggregate output tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo AR + + + + + + llama.cpp AR + + + + + + Gufo MTP + + + + + Multiple users, repetition + + + + + + + + diff --git a/docs/models/qwen3.8-flash-next/artifacts/charts/single-ar.svg b/docs/models/qwen3.8-flash-next/artifacts/charts/single-ar.svg new file mode 100644 index 00000000..aec6fd27 --- /dev/null +++ b/docs/models/qwen3.8-flash-next/artifacts/charts/single-ar.svg @@ -0,0 +1,588 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + + + + + + + 64K + + + + + + + + + + 128K + + + + context depth (tokens) + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 250 + + + + + + + + + + + + + 500 + + + + + + + + + + + + + 750 + + + + + + + + + + + + + 1000 + + + + + + + + + + + + + 1250 + + + + + + + + + + + + + 1500 + + + + + + + + + + + + + 1750 + + + + prefill tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo + + + + + + + + + + + + llama.cpp + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + + + + + + + 64K + + + + + + + + + + 128K + + + + context depth (tokens) + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 5 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 15 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 25 + + + + + + + + + + + + + 30 + + + + generation tok/s + + + + + + + + + + + + + + + + + + + + + + + + + Single user, autoregressive + + + + + + + + + + + diff --git a/docs/models/qwen3.8-flash-next/artifacts/charts/single-mtp.svg b/docs/models/qwen3.8-flash-next/artifacts/charts/single-mtp.svg new file mode 100644 index 00000000..088a40d4 --- /dev/null +++ b/docs/models/qwen3.8-flash-next/artifacts/charts/single-mtp.svg @@ -0,0 +1,572 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + + + + + + + 64K + + + + + + + + + + 128K + + + + context depth (tokens) + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 30 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 50 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 70 + + + + generation tok/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Gufo MTP + + + + + + + + + + + + llama.cpp AR + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 4K + + + + + + + + + + 8K + + + + + + + + + + 12K + + + + + + + + + + 16K + + + + + + + + + + 32K + + + + + + + + + + 64K + + + + + + + + + + 128K + + + + context depth (tokens) + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 80 + + + + + + + + + + + + + 100 + + + + acceptance % + + + + + + + + + + + + + + + + + + + + + + + + Single user, MTP + + + + + + + + + + + diff --git a/flake.lock b/flake.lock index ad559cd8..2309d820 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,25 @@ { "nodes": { + "audio-cpp": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1789940897, + "narHash": "sha256-+9tVcd+0TkDxg7JRU7M06Lvm20sXEbuerm62uhm6JKA=", + "owner": "0xShug0", + "repo": "audio.cpp", + "rev": "e3de8e3f3cbfac55ffa58df71426c41550a8598b", + "type": "github" + }, + "original": { + "owner": "0xShug0", + "repo": "audio.cpp", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1786384358, @@ -18,6 +38,7 @@ }, "root": { "inputs": { + "audio-cpp": "audio-cpp", "nixpkgs": "nixpkgs" } } diff --git a/flake.nix b/flake.nix index a70df340..062f5839 100644 --- a/flake.nix +++ b/flake.nix @@ -3,10 +3,16 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + # Reference ASR/TTS runtime for benchmark comparisons; exposes a + # ROCm gfx1151 package tuned for Strix Halo. + audio-cpp = { + url = "github:0xShug0/audio.cpp"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = - { self, nixpkgs }: + { self, nixpkgs, audio-cpp }: let # Strix Halo is a Linux x86-64-only target (see README non-goals). forAllSystems = nixpkgs.lib.genAttrs [ "x86_64-linux" ]; @@ -19,6 +25,25 @@ pkgs.${system}.callPackage ./.devops/nix/scope.nix { inherit version; } ); + # Reference LLM runtime for benchmark comparisons (llama-server, + # llama-bench), built for gfx1151 with unified memory like the Strix + # Halo toolbox. + llamaCpp = system: + (pkgs.${system}.llama-cpp.override { + rocmSupport = true; + rocmGpuTargets = [ "gfx1151" ]; + }).overrideAttrs + (oldAttrs: { + cmakeFlags = (oldAttrs.cmakeFlags or [ ]) ++ [ + "-DLLAMA_HIP_UMA=ON" # unified memory + ]; + # Pin the ROCm path explicitly and raise the local unroll + # threshold for gfx1151 kernels. + cmakeFlagsArray = (oldAttrs.cmakeFlagsArray or [ ]) ++ [ + "-DCMAKE_HIP_FLAGS=--rocm-path=${pkgs.${system}.rocmPackages.clr} -mllvm --amdgpu-unroll-threshold-local=600" + ]; + }); + alexnetWeights = system: pkgs.${system}.fetchurl { url = "https://download.pytorch.org/models/alexnet-owt-7be5be79.pth"; @@ -66,6 +91,7 @@ lpipsRocm ps.numpy ps.scipy + ps.matplotlib # benchmark charts (tools/bench/model-bench.py render) ] ); in @@ -120,6 +146,9 @@ pkgs.${system}.sox pkgs.${system}.rocmPackages.rocprofiler-sdk pkgs.${system}.sqlite + # Benchmark comparison baselines (see .agents/skills/benchmark-model). + (llamaCpp system) + audio-cpp.packages.${system}.rocm-gfx1151 ]; env = { TORCH_HOME = "${alexnetTorchHome system}"; diff --git a/tools/README.md b/tools/README.md index bb121fba..3cbc3a83 100644 --- a/tools/README.md +++ b/tools/README.md @@ -8,7 +8,7 @@ benchmark/tuning executables; Python tools run from this source tree. | Directory | Purpose | | --- | --- | -| `bench/` | Kernel microbenchmarks, hipBLASLt tuning, and direct speculative-corpus checks | +| `bench/` | Kernel microbenchmarks, hipBLASLt tuning, and direct speculative-corpus checks; `model-bench.py` drives per-model BENCHMARKS.md tables against llama.cpp | | `prof/` | rocprofv3 capture, stage summaries, and ISA inspection | | `serving/` | Concurrent HTTP serving benchmarks | | `ds4/` | DeepSeek-specific validation and experiments | diff --git a/tools/bench/model-bench.py b/tools/bench/model-bench.py new file mode 100755 index 00000000..5e5e9f14 --- /dev/null +++ b/tools/bench/model-bench.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from gufo.model_bench.cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/gufo/model_bench/__init__.py b/tools/gufo/model_bench/__init__.py new file mode 100644 index 00000000..e398701c --- /dev/null +++ b/tools/gufo/model_bench/__init__.py @@ -0,0 +1 @@ +"""Model benchmark driver: run BENCHMARKS.md tables for Gufo and a reference server.""" diff --git a/tools/gufo/model_bench/artifacts.py b/tools/gufo/model_bench/artifacts.py new file mode 100644 index 00000000..3ec9af21 --- /dev/null +++ b/tools/gufo/model_bench/artifacts.py @@ -0,0 +1,81 @@ +"""Retained per-table JSON artifacts under docs/models//artifacts.""" + +from __future__ import annotations + +import datetime as dt +import json +from pathlib import Path +from typing import Any + +from gufo.serving_bench import atomic_json + +from .config import BenchConfig, TableSpec + +ARTIFACT_TYPE = "model-bench-table" +SCHEMA_VERSION = 1 + + +def artifact_path( + config: BenchConfig, table: TableSpec, target: str, mode: str | None = None +) -> Path: + name = f"{table.id}-{target}" + (f"-{mode}" if mode else "") + ".json" + return config.artifacts_dir / name + + +def load_artifact(path: Path) -> dict[str, Any] | None: + if not path.exists(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + +def public_command(command: list[str]) -> list[str]: + """Server command with file paths reduced to their basenames.""" + return [Path(part).name if "/" in part else part for part in command] + + +def new_artifact( + config: BenchConfig, + table: TableSpec, + target: str, + *, + mode: str | None, + command: list[str], + source: dict[str, Any], + fingerprint: dict[str, Any] | None, + notes: list[str], +) -> dict[str, Any]: + return { + "artifactType": ARTIFACT_TYPE, + "schemaVersion": SCHEMA_VERSION, + "table": table.id, + "target": target, + "mode": mode, + "referenceName": config.reference_name, + "model": {"id": config.data["model"]["id"], "variant": table.variant}, + "measuredOn": dt.date.today().isoformat(), + "source": source, + "fingerprint": fingerprint, + "server": {"command": public_command(command)}, + "notes": notes, + "rows": {}, + } + + +def merge_rows(existing: dict[str, Any] | None, fresh: dict[str, Any]) -> dict[str, Any]: + """Keep rows from an earlier artifact that this run did not measure.""" + if existing is None: + return fresh + rows = dict(existing.get("rows", {})) + rows.update(fresh["rows"]) + merged = dict(fresh) + merged["rows"] = rows + previous = existing.get("measuredOn") + if previous and previous != fresh["measuredOn"]: + merged["notes"] = list(fresh["notes"]) + [ + f"rows not measured on {fresh['measuredOn']} retain values from {previous}" + ] + return merged + + +def save_artifact(path: Path, artifact: dict[str, Any]) -> None: + atomic_json(path, artifact) diff --git a/tools/gufo/model_bench/charts.py b/tools/gufo/model_bench/charts.py new file mode 100644 index 00000000..d74d4447 --- /dev/null +++ b/tools/gufo/model_bench/charts.py @@ -0,0 +1,176 @@ +"""SVG charts rendered from the tables in BENCHMARKS.md (an addition to the tables).""" + +from __future__ import annotations + +import math +import re +from pathlib import Path +from typing import Any + +from .config import BenchConfig, TableSpec +from .render import MARKER_RE, _number, layout_for, parse_table + +# Categorical slots from the validated default palette: Gufo, reference, Gufo speculative. +COLORS = {"gufo": "#2a78d6", "reference": "#eb6834", "spec": "#1baf7a"} +SURFACE = "#fcfcfb" +TEXT = "#0b0b0b" +TEXT_SECONDARY = "#52514e" +GRID = "#e6e5e1" +CHART_DIR = "artifacts/charts" +IMAGE_RE = r"\n\n!\[[^\]]*\]\(" + re.escape(CHART_DIR) + r"/{id}\.svg\)" + + +def _plt() -> Any: + try: + import matplotlib + except ImportError as exception: # pragma: no cover - environment dependent + raise SystemExit("charts need matplotlib; enter `nix develop`") from exception + matplotlib.use("svg") + import matplotlib.pyplot as plt + + plt.rcParams.update({ + "font.family": "sans-serif", "font.size": 9, "text.color": TEXT, + "axes.labelcolor": TEXT_SECONDARY, "axes.edgecolor": GRID, "axes.facecolor": SURFACE, + "figure.facecolor": SURFACE, "xtick.color": TEXT_SECONDARY, "ytick.color": TEXT_SECONDARY, + "axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.6, "axes.axisbelow": True, + "axes.spines.top": False, "axes.spines.right": False, "legend.frameon": False, + "svg.hashsalt": "gufo-model-bench", "svg.fonttype": "none", + }) + return plt + + +def _series(rows: dict[str, list[str]], labels: list[str], index: int) -> list[float]: + values = [] + for label in labels: + cells = rows.get(label, []) + value = _number(cells[index]) if index < len(cells) else None + values.append(math.nan if value is None else value) + return values + + +def _depth_ticks(labels: list[str]) -> list[str]: + """Compact tick labels: 0, 4K, 8K, ... for depth tables.""" + out = [] + for label in labels: + value = int(label.replace(",", "")) + out.append(f"{value // 1024}K" if value >= 1024 and value % 1024 == 0 else str(value)) + return out + + +def _has_data(*series: list[float]) -> bool: + return any(not math.isnan(v) for s in series for v in s) + + +def _finish_axes(ax: Any, labels: list[str], series: list[tuple[str, list[float], str]], ylabel: str, + ticks: list[str] | None = None) -> None: + ax.set_xticks(range(len(labels)), ticks or labels) + ax.set_ylabel(ylabel) + top = max((v for _, values, _ in series for v in values if not math.isnan(v)), default=1.0) + ax.set_ylim(0, top * 1.2) + ax.grid(axis="x", visible=False) + + +def _lines(ax: Any, labels: list[str], series: list[tuple[str, list[float], str]], ylabel: str, + ticks: list[str] | None = None) -> None: + for name, values, color in series: + points = [(i, v) for i, v in enumerate(values) if not math.isnan(v)] + ax.plot([p[0] for p in points], [p[1] for p in points], color=color, linewidth=2, marker="o", + markersize=6, markeredgecolor=SURFACE, markeredgewidth=1, label=name) + _finish_axes(ax, labels, series, ylabel, ticks) + + +def _bars(ax: Any, labels: list[str], series: list[tuple[str, list[float], str]], ylabel: str) -> None: + width = 0.8 / len(series) + for offset, (name, values, color) in enumerate(series): + x = [i - 0.4 + width * (offset + 0.5) for i in range(len(labels))] + ax.bar(x, [0 if math.isnan(v) else v for v in values], width=width - 0.04, color=color, + label=name, linewidth=0) + _finish_axes(ax, labels, series, ylabel) + + +def chart_for(config: BenchConfig, table: TableSpec, rows: dict[str, list[str]], path: Path) -> bool: + """Write one SVG for a rendered table; return False when there is nothing to draw.""" + layout = layout_for(config, table) + labels = layout.rows + ref = config.reference_name + spec_label = config.speculative["label"] + kind = table.kind + title = table.spec.get("title", table.id) + if table.variant: + title += f" ({config.variant_label(table.variant)})" + plt = _plt() + + if kind == "single" and not table.speculative: + gp, rp, gt, rt = (_series(rows, labels, i) for i in (0, 1, 3, 4)) + if not _has_data(gp, gt): + return False + fig, (a1, a2) = plt.subplots(1, 2, figsize=(8, 3.2)) + ticks = _depth_ticks(labels) + _lines(a1, labels, [("Gufo", gp, COLORS["gufo"]), (ref, rp, COLORS["reference"])], "prefill tok/s", ticks) + _lines(a2, labels, [("Gufo", gt, COLORS["gufo"]), (ref, rt, COLORS["reference"])], "generation tok/s", ticks) + a1.set_xlabel("context depth (tokens)") + a2.set_xlabel("context depth (tokens)") + a1.legend(loc="lower left") + elif kind == "single": + gt, acc, rt = (_series(rows, labels, i) for i in (1, 2, 3)) + if not _has_data(gt): + return False + fig, (a1, a2) = plt.subplots(1, 2, figsize=(8, 3.2)) + ticks = _depth_ticks(labels) + _lines(a1, labels, [(f"Gufo {spec_label}", gt, COLORS["spec"]), (f"{ref} AR", rt, COLORS["reference"])], + "generation tok/s", ticks) + _lines(a2, labels, [(f"{spec_label} acceptance", acc, COLORS["spec"])], "acceptance %", ticks) + a2.set_ylim(0, 105) + a1.set_xlabel("context depth (tokens)") + a2.set_xlabel("context depth (tokens)") + a1.legend(loc="lower left") + elif kind == "multi": + ga, ra, gs = (_series(rows, labels, i) for i in (0, 1, 3)) + if not _has_data(ga, gs): + return False + fig, ax = plt.subplots(figsize=(6, 3.2)) + _bars(ax, labels, [("Gufo AR", ga, COLORS["gufo"]), (f"{ref} AR", ra, COLORS["reference"]), + (f"Gufo {spec_label}", gs, COLORS["spec"])], "aggregate output tok/s") + ax.set_xlabel("concurrent users") + ax.legend(loc="upper left") + elif kind in ("loading", "memory", "image-encoder"): + first = 1 if kind == "image-encoder" else 0 + g, r = _series(rows, labels, first), _series(rows, labels, first + 1) + if not _has_data(g): + return False + unit = {"loading": "seconds to ready", "memory": "GiB", "image-encoder": "encode ms"}[kind] + fig, ax = plt.subplots(figsize=(max(4, 1.6 * len(labels) + 2), 3.0)) + _bars(ax, labels, [("Gufo", g, COLORS["gufo"]), (ref, r, COLORS["reference"])], unit) + ax.legend(loc="upper right") + else: + return False + + fig.suptitle(title, x=0.01, ha="left", fontsize=10, color=TEXT) + fig.tight_layout() + path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(path, format="svg", metadata={"Date": None, "Creator": None}) + plt.close(fig) + return True + + +def render_charts(config: BenchConfig, document: str, only: set[str] | None = None) -> tuple[str, list[str]]: + """Draw charts for rendered tables and place an image line after each table.""" + known = {t.id: t for t in config.tables()} + written: list[str] = [] + chart_dir = config.model_dir / CHART_DIR + + def replace(match: re.Match[str]) -> str: + table_id = match.group("id") + block = match.group(0) + if table_id not in known or (only and table_id not in only): + return block + path = chart_dir / f"{table_id}.svg" + if chart_for(config, known[table_id], parse_table(match.group("body")), path): + written.append(table_id) + return block + f"\n\n![{known[table_id].spec.get('title', table_id)}]({CHART_DIR}/{table_id}.svg)" + return block + + # Strip existing image lines so the pass is idempotent, then re-add for tables with data. + for table_id in known: + document = re.sub(IMAGE_RE.format(id=re.escape(table_id)), "", document) + return MARKER_RE.sub(replace, document), written diff --git a/tools/gufo/model_bench/cli.py b/tools/gufo/model_bench/cli.py new file mode 100644 index 00000000..9f8585ab --- /dev/null +++ b/tools/gufo/model_bench/cli.py @@ -0,0 +1,135 @@ +"""Command line: run tables for one target, or render BENCHMARKS.md.""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +from gufo.serving_bench import load_fingerprint, source_identity + +from .config import TARGETS, BenchConfig, load_config, parse_file_args + +FILE_ROLES = ("gguf", "draft", "mtp", "dspark", "mmproj") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="model-bench", + description="Measure docs/models//BENCHMARKS.md tables for Gufo and its reference server.", + ) + parser.add_argument("--model", required=True, help="docs/models/ directory name") + parser.add_argument("--gufo", type=Path, default=Path("result/bin/gufo"), help="Gufo binary") + for role in FILE_ROLES: + parser.add_argument(f"--{role}", action="append", default=[], metavar="[VARIANT=]PATH") + parser.add_argument("--log-dir", type=Path, default=Path("artifacts/model-bench"), + help="server logs (ignored directory)") + parser.add_argument("--config", type=Path, default=None, help="bench.json override (experiments)") + parser.add_argument("--artifacts-dir", type=Path, default=None, + help="write/read artifacts here instead of docs/models//artifacts") + sub = parser.add_subparsers(dest="command", required=True) + + run = sub.add_parser("run", help="measure tables for one target") + run.add_argument("--target", choices=TARGETS, required=True) + run.add_argument("--table", action="append", default=[], help="table id (repeatable, comma-separated)") + run.add_argument("--todo", action="store_true", help="measure only rows that are TODO in BENCHMARKS.md") + run.add_argument("--reference-binary", default=None, help="reference server executable (default from bench.json)") + run.add_argument("--drop-caches", default=None, + help="privileged shell command that drops the page cache before a loading launch") + + render = sub.add_parser("render", help="rewrite marked tables in BENCHMARKS.md from artifacts") + render.add_argument("--table", action="append", default=[]) + render.add_argument("--check", action="store_true", help="exit 1 when the document would change") + render.add_argument("--no-charts", action="store_true", help="skip SVG charts under artifacts/charts") + + sub.add_parser("tables", help="list table ids and their artifacts") + return parser + + +def _table_ids(values: list[str]) -> set[str]: + return {item.strip() for value in values for item in value.split(",") if item.strip()} + + +def _root() -> Path: + return Path(__file__).resolve().parents[3] + + +def cmd_tables(config: BenchConfig) -> int: + from .artifacts import artifact_path + + for table in config.tables(): + paths = [artifact_path(config, table, target) for target in TARGETS] + if table.kind == "multi": + paths = [artifact_path(config, table, "gufo", m) for m in table.spec.get("modes", ["ar"])] + paths.append(artifact_path(config, table, "reference")) + present = [p.name for p in paths if p.exists()] + print(f"{table.id:28} {table.kind:14} {', '.join(present) or '-'}") + return 0 + + +def cmd_render(config: BenchConfig, args: argparse.Namespace) -> int: + from .render import render_document + + document = config.benchmarks_path.read_text(encoding="utf-8") + updated, rendered = render_document(config, document, _table_ids(args.table) or None) + charts: list[str] = [] + if not args.no_charts and not args.check: + from .charts import render_charts + + updated, charts = render_charts(config, updated, _table_ids(args.table) or None) + if args.check: + if updated != document: + print(f"{config.benchmarks_path}: tables out of date") + return 1 + print("up to date") + return 0 + if updated != document: + config.benchmarks_path.write_text(updated, encoding="utf-8") + print(f"rendered {len(rendered)} tables and {len(charts)} charts in {config.benchmarks_path}") + return 0 + + +def cmd_run(config: BenchConfig, args: argparse.Namespace) -> int: + if config.category != "llm": + raise SystemExit(f"category {config.category!r} is not implemented yet") + from . import llm + + root = config.root + gufo_binary = args.gufo if args.gufo.is_absolute() else (root / args.gufo) + if not gufo_binary.exists(): + raise SystemExit(f"Gufo binary not found: {gufo_binary} (run `nix build`)") + reference_binary = args.reference_binary or config.data["reference"]["server"] + if args.target == "reference" and shutil.which(reference_binary) is None: + raise SystemExit(f"{reference_binary} not on PATH; enter `nix develop`") + revision, dirty = source_identity(root) + fingerprint = load_fingerprint(None, gufo_binary) + document = config.benchmarks_path.read_text(encoding="utf-8") if config.benchmarks_path.exists() else "" + session = llm.Session( + config, args.target, gufo_binary=gufo_binary, reference_binary=reference_binary, + source={"revision": revision, "dirty": dirty, "buildMode": "nix-release"}, + fingerprint=fingerprint, log_dir=(args.log_dir if args.log_dir.is_absolute() else root / args.log_dir), + document=document, todo_only=args.todo, drop_caches=args.drop_caches, + ) + wanted = _table_ids(args.table) + tables = [t for t in config.tables() if not wanted or t.id in wanted] + unknown = wanted - {t.id for t in config.tables()} + if unknown: + raise SystemExit(f"unknown tables: {', '.join(sorted(unknown))}") + for table in tables: + print(f"== {table.id} ({args.target})") + llm.run_table(session, table) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + config = load_config(_root(), args.model, args.config) + if args.artifacts_dir is not None: + config.artifacts_override = args.artifacts_dir.resolve() + parse_file_args(config, {role: getattr(args, role) for role in FILE_ROLES}) + if args.command == "tables": + return cmd_tables(config) + if args.command == "render": + return cmd_render(config, args) + return cmd_run(config, args) diff --git a/tools/gufo/model_bench/config.py b/tools/gufo/model_bench/config.py new file mode 100644 index 00000000..b6f8905a --- /dev/null +++ b/tools/gufo/model_bench/config.py @@ -0,0 +1,129 @@ +"""bench.json loading and table expansion.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +SCHEMA = "gufo-model-bench/1" +TARGETS = ("gufo", "reference") + + +@dataclass(frozen=True) +class TableSpec: + id: str + base: str + variant: str | None + spec: dict[str, Any] + + @property + def kind(self) -> str: + if self.base.startswith("single-"): + return "single" + if self.base.startswith("multi-"): + return "multi" + return self.base + + @property + def speculative(self) -> bool: + return bool(self.spec.get("speculative")) + + +@dataclass +class BenchConfig: + model: str + root: Path + data: dict[str, Any] + files: dict[str, dict[str, Path]] = field(default_factory=dict) + artifacts_override: Path | None = None + + @property + def model_dir(self) -> Path: + return self.root / "docs" / "models" / self.model + + @property + def artifacts_dir(self) -> Path: + return self.artifacts_override or (self.model_dir / "artifacts") + + @property + def benchmarks_path(self) -> Path: + return self.model_dir / "BENCHMARKS.md" + + @property + def category(self) -> str: + return self.data["category"] + + @property + def variants(self) -> dict[str, dict[str, Any]]: + return self.data["variants"] + + @property + def single_variant(self) -> bool: + return list(self.variants) == ["default"] + + @property + def reference_name(self) -> str: + return self.data["reference"]["name"] + + @property + def speculative(self) -> dict[str, Any]: + return self.data["speculative"] + + def tables(self) -> list[TableSpec]: + result: list[TableSpec] = [] + for base, spec in self.data["tables"].items(): + if spec.get("per_variant"): + for variant in self.variants: + result.append(TableSpec(f"{base}-{variant}", base, variant, spec)) + else: + result.append(TableSpec(base, base, None, spec)) + return result + + def table(self, table_id: str) -> TableSpec: + for table in self.tables(): + if table.id == table_id: + return table + raise KeyError(f"unknown table {table_id!r} for {self.model}") + + def variant_label(self, variant: str) -> str: + return self.variants[variant]["label"] + + def file(self, role: str, variant: str | None) -> Path: + variant = variant or "default" + by_variant = self.files.get(role, {}) + path = by_variant.get(variant) or by_variant.get("*") + if path is None: + description = self.data["files"].get(role, role) + raise SystemExit( + f"missing --{role} for variant {variant!r}: {description}" + ) + return path + + def require_files(self, variant: str | None) -> None: + for role in self.variants[variant or "default"].get("requires", []): + self.file(role, variant) + + +def load_config(root: Path, model: str, path: Path | None = None) -> BenchConfig: + path = path or root / "docs" / "models" / model / "artifacts" / "bench.json" + if not path.exists(): + raise SystemExit(f"no bench.json for {model}: {path}") + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("schema") != SCHEMA: + raise SystemExit(f"{path}: expected schema {SCHEMA}") + return BenchConfig(model=model, root=root, data=data) + + +def parse_file_args(config: BenchConfig, values: dict[str, list[str]]) -> None: + """Map `--gguf q4=/path` / `--gguf /path` arguments onto file roles.""" + for role, entries in values.items(): + for entry in entries: + variant, separator, raw = entry.partition("=") + if not separator: + variant, raw = ("default" if config.single_variant else "*"), entry + path = Path(raw).expanduser() + if not path.exists(): + raise SystemExit(f"--{role} {entry}: {path} does not exist") + config.files.setdefault(role, {})[variant] = path.absolute() diff --git a/tools/gufo/model_bench/llm.py b/tools/gufo/model_bench/llm.py new file mode 100644 index 00000000..6a86ddd8 --- /dev/null +++ b/tools/gufo/model_bench/llm.py @@ -0,0 +1,469 @@ +"""LLM category: loading, single-user depth sweeps, concurrency, memory.""" + +from __future__ import annotations + +import datetime as dt +import json +import random +import statistics +import subprocess +import threading +import time +import urllib.request +from pathlib import Path +from typing import Any + +from gufo.serving_bench import ( + RequestObservation, + load_prompt_suite, + load_reference_report, + run_corpus_benchmark, + run_request, +) + +from .artifacts import artifact_path, load_artifact, merge_rows, new_artifact, public_command, save_artifact +from .config import BenchConfig, TableSpec +from .servers import Server, drop_file_cache, rocm_used_gib, wait_process_exit + +MODEL_ALIAS = "bench" +CLIENT_ID = "model-bench" +REQUEST_TIMEOUT = 3600.0 + +WORDS = ( + "the river bends past a quiet town where old mills once ground wheat for " + "every family along the valley and children still climb the stone bridge " + "to watch boats carry timber salt and wool toward distant markets while " + "farmers mend fences count sheep and argue about rain clouds that gather " + "over the western hills each autumn evening before the harvest festival " + "brings music lanterns and long tables of bread cheese apples and cider " + "shared by neighbours who remember older winters when snow closed the " + "road for weeks and stories were traded by firelight instead of coins" +).split() + + +class Session: + """Shared run state: config, target, server factory and identities.""" + + def __init__( + self, + config: BenchConfig, + target: str, + *, + gufo_binary: Path, + reference_binary: str, + source: dict[str, Any], + fingerprint: dict[str, Any] | None, + log_dir: Path, + document: str, + todo_only: bool, + drop_caches: str | None = None, + ): + self.config = config + self.target = target + self.gufo_binary = gufo_binary + self.reference_binary = reference_binary + self.source = source + self.fingerprint = fingerprint + self.log_dir = log_dir + self.document = document + self.todo_only = todo_only + self.drop_caches = drop_caches + + @property + def profile(self) -> str: + return "gufo" if self.target == "gufo" else self.config.data["reference"]["endpoint_profile"] + + @property + def cache_prompt(self) -> bool | None: + return None if self.target == "gufo" else True + + # ----- server commands ------------------------------------------------- + + def gufo_command(self, table: TableSpec, *, mode: str | None, context: int, sessions: int, port: int) -> list[str]: + cfg = self.config + gguf = cfg.file("gguf", table.variant) + command = [str(self.gufo_binary), "serve", "--port", str(port), "--sessions", str(sessions), + *cfg.data["gufo"].get("serve", []), "llm", "--model", str(gguf), + "--context", str(context), "--served-model-name", MODEL_ALIAS, + *cfg.data["gufo"].get("llm", [])] + if "mmproj" in cfg.files: + command += ["--mmproj", str(cfg.file("mmproj", table.variant))] + if mode and mode != "ar": + for part in cfg.speculative["gufo_args"]: + if part.startswith("{") and part.endswith("}"): + part = str(cfg.file(part[1:-1], table.variant)) + command.append(part) + return command + + def reference_command(self, table: TableSpec, *, context: int, parallel: int, port: int) -> list[str]: + cfg = self.config + gguf = cfg.file("gguf", table.variant) + return [self.reference_binary, "-m", str(gguf), "-c", str(context), "-np", str(parallel), + "--port", str(port), "--host", "127.0.0.1", "--alias", MODEL_ALIAS, + *cfg.data["reference"]["args"]] + + def server(self, table: TableSpec, *, mode: str | None, context: int, sessions: int, tag: str) -> Server: + readiness = self.config.data["gufo"]["readiness"] if self.target == "gufo" else self.config.data["reference"]["readiness"] + log = self.log_dir / f"{table.id}-{self.target}-{tag}.log" + placeholder = Server([], readiness, log) + if self.target == "gufo": + command = self.gufo_command(table, mode=mode, context=context, sessions=sessions, port=placeholder.port) + else: + command = self.reference_command(table, context=context, parallel=sessions, port=placeholder.port) + placeholder.command = command + return placeholder + + def reference_version(self) -> str | None: + if self.target != "reference": + return None + if not hasattr(self, "_reference_version"): + completed = subprocess.run([self.reference_binary, "--version"], capture_output=True, text=True) + text = (completed.stdout or completed.stderr).strip().splitlines() + self._reference_version = text[0] if text else None + return self._reference_version + + def artifact(self, table: TableSpec, *, mode: str | None, command: list[str], notes: list[str]) -> dict[str, Any]: + version = self.reference_version() + if version: + notes = [f"{self.config.reference_name}: {version}", *notes] + return new_artifact(self.config, table, self.target, mode=mode, command=command, + source=self.source, fingerprint=self.fingerprint, notes=notes) + + def store(self, path: Path, artifact: dict[str, Any]) -> None: + save_artifact(path, merge_rows(load_artifact(path), artifact)) + print(f"artifact: {path}") + + def request(self, base_url: str, prompt: str, max_tokens: int, *, index: int = 0, + messages: list[dict[str, str]] | None = None) -> RequestObservation: + return run_request( + base_url=base_url, model=MODEL_ALIAS, prompt=prompt, max_tokens=max_tokens, + temperature=float(self.config.data["sampling"]["temperature"]), + timeout_seconds=REQUEST_TIMEOUT, client_id=CLIENT_ID, concurrency=1, + repetition=1, request_index=index, endpoint_profile=self.profile, + cache_prompt=self.cache_prompt, messages=messages, + ) + + def chat_text(self, base_url: str, messages: list[dict[str, str]], max_tokens: int) -> str: + """Non-streaming completion text, used to build a reusable conversation prefix.""" + payload = {"model": MODEL_ALIAS, "messages": messages, "max_tokens": max_tokens, + "temperature": float(self.config.data["sampling"]["temperature"]), "stream": False} + if self.cache_prompt is not None: + payload["cache_prompt"] = self.cache_prompt + body = json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + base_url.rstrip("/") + "/v1/chat/completions", data=body, + headers={"Content-Type": "application/json", "X-Client-ID": CLIENT_ID}, method="POST") + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response: + reply = json.loads(response.read().decode("utf-8")) + content = reply["choices"][0]["message"].get("content") + return content if isinstance(content, str) else "" + + +# ----- synthetic prompts --------------------------------------------------- + + +def synthetic_text(seed: int, words: int) -> str: + """Deterministic prose-like text with a fixed vocabulary.""" + rng = random.Random(seed) + out: list[str] = [] + count = 0 + while count < words: + sentence_len = min(rng.randint(6, 14), words - count) + sentence = [rng.choice(WORDS) for _ in range(sentence_len)] + sentence[0] = sentence[0].capitalize() + out.append(" ".join(sentence) + ".") + count += sentence_len + if rng.random() < 0.2: + out.append("\n\n") + return " ".join(out).replace(" \n\n ", "\n\n").strip() + + +class Tokenizer: + """Estimates token counts for synthetic text from server-reported usage.""" + + def __init__(self, session: Session, base_url: str): + self.session = session + self.base_url = base_url + overhead = session.request(base_url, "Hi", 1).prompt_tokens - 1 + probe_words = 3000 + probe = session.request(base_url, synthetic_text(7777, probe_words), 1).prompt_tokens + self.overhead = overhead + self.ratio = (probe - overhead) / probe_words + + def words_for(self, tokens: int) -> int: + return max(1, round(tokens / self.ratio)) + + +def _tolerance(target: int, fraction: float) -> int: + return max(32, int(target * fraction)) + + +def _mean_sd(values: list[float]) -> tuple[float, float | None]: + if len(values) < 2: + return values[0], None + return statistics.fmean(values), statistics.stdev(values) + + +# ----- tables ---------------------------------------------------------------- + + +def _selected_rows(session: Session, table: TableSpec, labels: dict[Any, str]) -> list[Any]: + """Row keys to measure, honouring --todo against the current document.""" + if not session.todo_only: + return list(labels) + from .render import todo_rows + + todo = todo_rows(session.config, session.document, table, session.target) + if todo is None: + return list(labels) + return [key for key, label in labels.items() if label in todo] + + +def run_loading(session: Session, table: TableSpec) -> None: + cfg = session.config + spec = table.spec + keys = _selected_rows(session, table, {v: cfg.variant_label(v) for v in cfg.variants}) + if not keys: + print(f"{table.id}: nothing to do") + return + rows: dict[str, Any] = {} + command: list[str] = [] + for variant in keys: + sub = TableSpec(table.id, table.base, variant, spec) + cfg.require_files(variant) + samples: list[float] = [] + for repetition in range(int(spec.get("repetitions", 1))): + drop_file_cache(session.drop_caches) + mode = cfg.speculative["mode"] if session.target == "gufo" else None + server = session.server(sub, mode=mode, context=int(spec["context"]), + sessions=int(spec.get("sessions", 2)), tag=f"{variant}-{repetition}") + command = server.command + with server: + assert server.ready_seconds is not None + samples.append(server.ready_seconds) + wait_process_exit(server) + mean, sd = _mean_sd(samples) + rows[variant] = {"ready_s": round(mean, 3), "ready_s_sd": None if sd is None else round(sd, 3), + "samples": len(samples)} + print(f"{table.id} {variant}: ready {mean:.2f} s") + artifact = session.artifact(table, mode=None, command=command, + notes=["cold file cache: `echo 3 > /proc/sys/vm/drop_caches` before each launch", + f"readiness = HTTP 200 on {cfg.data['gufo' if session.target == 'gufo' else 'reference']['readiness']}"]) + artifact["rows"] = rows + session.store(artifact_path(cfg, table, session.target), artifact) + + +def run_single(session: Session, table: TableSpec) -> None: + cfg = session.config + spec = table.spec + if table.speculative and session.target != "gufo": + print(f"{table.id}: reference uses the single-ar table ({cfg.speculative['reference']})") + return + cfg.require_files(table.variant) + depths = [int(d) for d in spec["depths"]] + keys = _selected_rows(session, table, {d: f"{d:,}" for d in depths}) + if not keys: + print(f"{table.id}: nothing to do") + return + prompt_tokens = int(spec["prompt_tokens"]) + output_tokens = int(spec["output_tokens"]) + fraction = float(spec.get("depth_tolerance", 0.005)) + repetitions = int(spec.get("repetitions", 1)) + base_seed = int(spec.get("prefix", {}).get("seed", 1)) + mode = cfg.speculative["mode"] if table.speculative else "ar" + + server = session.server(table, mode=mode, context=int(spec["context"]), sessions=1, tag="single") + rows: dict[str, Any] = {} + with server: + tokenizer = Tokenizer(session, server.base_url) + # Warm kernels and allocations with an untimed full-size request. + session.request(server.base_url, synthetic_text(8888, tokenizer.words_for(prompt_tokens)), 16) + for depth in keys: + pps: list[float] = [] + tgs: list[float] = [] + accepts: list[float] = [] + counts: list[dict[str, int]] = [] + for repetition in range(repetitions): + observation = _measure_depth(session, server.base_url, tokenizer, depth=depth, + prompt_tokens=prompt_tokens, output_tokens=output_tokens, + fraction=fraction, seed=base_seed, repetition=repetition) + if observation.prefill_tokens_per_second is not None: + pps.append(observation.prefill_tokens_per_second) + if observation.decode_tokens_per_second is not None: + tgs.append(observation.decode_tokens_per_second) + if observation.draft_acceptance is not None: + accepts.append(observation.draft_acceptance * 100.0) + counts.append({"cache_n": observation.cached_prompt_tokens, + "prompt_n": observation.prefill_tokens, + "predicted_n": observation.completion_tokens}) + row: dict[str, Any] = {"counts": counts, "samples": repetitions} + for name, values in (("pp", pps), ("tg", tgs), ("acceptance", accepts)): + if values: + mean, sd = _mean_sd(values) + row[name] = round(mean, 2) + row[f"{name}_sd"] = None if sd is None else round(sd, 2) + rows[str(depth)] = row + print(f"{table.id} d{depth}: pp {row.get('pp')} tg {row.get('tg')} acc {row.get('acceptance')}") + wait_process_exit(server) + artifact = session.artifact(table, mode=None, command=server.command, notes=[ + f"pp{prompt_tokens}/tg{output_tokens}; depth is a cached prefix of synthetic text; " + f"tolerance max(32, {fraction:.3%}) on cache_n and prompt_n; actual counts per sample in rows", + f"synthetic text: {tokenizer.ratio:.3f} tokens/word, template overhead {tokenizer.overhead} tokens", + ]) + artifact["rows"] = rows + session.store(artifact_path(cfg, table, session.target), artifact) + + +def _measure_depth(session: Session, base_url: str, tokenizer: Tokenizer, *, depth: int, prompt_tokens: int, + output_tokens: int, fraction: float, seed: int, repetition: int) -> RequestObservation: + """Time pp/tg after a cached conversation prefix of about `depth` tokens. + + Both servers reuse a prior turn's state: the prefix is sent as its own turn + (one generated token), and the measured request continues that conversation + with a new user turn of about `prompt_tokens` tokens. + """ + new_target = prompt_tokens - tokenizer.overhead + ratio = tokenizer.ratio + for attempt in range(4): + messages: list[dict[str, str]] = [] + if depth > 0: + prefix_target = depth - tokenizer.overhead - 1 + prefix = synthetic_text(seed + depth, max(1, round(prefix_target / ratio))) + reply = session.chat_text(base_url, [{"role": "user", "content": prefix}], 1) + messages = [{"role": "user", "content": prefix}, {"role": "assistant", "content": reply}] + new_text = synthetic_text(100_000 + depth * 10 + repetition * 100 + attempt, max(1, round(new_target / ratio))) + messages.append({"role": "user", "content": new_text}) + observation = session.request(base_url, new_text, output_tokens, index=attempt, messages=messages) + cache_ok = abs(observation.cached_prompt_tokens - depth) <= _tolerance(depth, fraction) + prefill_ok = abs(observation.prefill_tokens - prompt_tokens) <= _tolerance(prompt_tokens, fraction) + if cache_ok and prefill_ok: + return observation + print(f" d{depth} attempt {attempt}: cache_n {observation.cached_prompt_tokens} prompt_n " + f"{observation.prefill_tokens}; recalibrating") + measured = observation.prefill_tokens + observation.cached_prompt_tokens + expected_words = sum(len(m["content"].split()) for m in messages if m["role"] == "user") + ratio = (measured - tokenizer.overhead) / expected_words + raise RuntimeError(f"depth {depth}: cached prefix outside tolerance after 4 attempts") + + +def run_multi(session: Session, table: TableSpec) -> None: + cfg = session.config + spec = table.spec + cfg.require_files(table.variant) + levels = [int(c) for c in spec["concurrency"]] + keys = _selected_rows(session, table, {c: str(c) for c in levels}) + if not keys: + print(f"{table.id}: nothing to do") + return + suite = (cfg.model_dir / "artifacts" / spec["suite"]).resolve() + cases = load_prompt_suite(suite, selected=set(spec["cases"])) + modes = list(spec.get("modes", ["ar"])) if session.target == "gufo" else [None] + for mode in modes: + path = artifact_path(cfg, table, session.target, mode) + reference = None + ar_path = artifact_path(cfg, table, "gufo", "ar") + if path != ar_path and ar_path.exists(): + reference = load_reference_report(ar_path) + combined = load_artifact(path) + for users in keys: + context = int(spec["context"]) + server = session.server(table, mode=mode, context=context if session.target == "gufo" else context * users, + sessions=users, tag=f"{mode or 'ref'}-c{users}") + with server: + report = run_corpus_benchmark( + base_url=server.base_url, model=MODEL_ALIAS, cases=cases, + workload_id=f"{cfg.model}-{table.id}-{mode or session.target}", + max_tokens=int(spec["output_tokens"]), + temperature=float(cfg.data["sampling"]["temperature"]), + concurrency_levels=[users], warmup_rounds=int(spec.get("warmup", 1)), + repetitions=int(spec.get("repetitions", 1)), timeout_seconds=REQUEST_TIMEOUT, + fingerprint=session.fingerprint or {}, source_revision=session.source["revision"], + source_dirty=session.source["dirty"], suite_bytes=suite.read_bytes(), + corpus_layout=spec.get("corpus_layout", "distinct"), endpoint_profile=session.profile, + cache_prompt=False if not spec.get("cache_prompt", False) else None, + reference=reference, + notes=[note for note in (session.reference_version(), " ".join(public_command(server.command)), + "fresh server per concurrency level") if note], + ) + wait_process_exit(server) + if combined is None or combined.get("artifactType") != report.get("artifactType"): + combined = report + else: + combined["results"].update(report["results"]) + combined["notes"] = report.get("notes", combined.get("notes")) + combined["modelBench"] = {"table": table.id, "target": session.target, "mode": mode, + "measuredOn": dt.date.today().isoformat()} + save_artifact(path, combined) + rate = report["results"][f"c{users}"]["aggregate"]["output_tokens_per_second"]["overall"] + print(f"{table.id} {mode or session.target} C{users}: {rate:.2f} tok/s -> {path}") + + +class MemoryPoller: + def __init__(self) -> None: + self.peak: float | None = None + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def _run(self) -> None: + while not self._stop.is_set(): + used = rocm_used_gib() + if used is not None and (self.peak is None or used > self.peak): + self.peak = used + time.sleep(0.25) + + def __enter__(self) -> "MemoryPoller": + self._thread.start() + return self + + def __exit__(self, *exc: Any) -> None: + self._stop.set() + self._thread.join() + + +def run_memory(session: Session, table: TableSpec) -> None: + cfg = session.config + spec = table.spec + cfg.require_files(table.variant) + workloads = {w["id"]: w for w in spec["workloads"]} + keys = _selected_rows(session, table, {k: k for k in workloads}) + if not keys: + print(f"{table.id}: nothing to do") + return + if rocm_used_gib() is None: + raise SystemExit("memory table needs rocm-smi with --showmemuse --json") + mode = cfg.speculative["mode"] if session.target == "gufo" else "ar" + server = session.server(table, mode=mode, context=int(spec["context"]), sessions=1, tag="memory") + rows: dict[str, Any] = {} + with server: + tokenizer = Tokenizer(session, server.base_url) + for key in keys: + workload = workloads[key] + depth = int(workload["depth"]) + with MemoryPoller() as poller: + _measure_depth(session, server.base_url, tokenizer, depth=depth, + prompt_tokens=int(workload["prompt_tokens"]), + output_tokens=int(workload["output_tokens"]), fraction=0.02, seed=1, repetition=0) + rows[key] = {"gib": None if poller.peak is None else round(poller.peak, 2)} + print(f"{table.id} {key}: {rows[key]['gib']} GiB") + wait_process_exit(server) + artifact = session.artifact(table, mode=None, command=server.command, + notes=["peak device-wide VRAM use sampled with rocm-smi every 250 ms during the request"]) + artifact["rows"] = rows + session.store(artifact_path(cfg, table, session.target), artifact) + + +def run_image_encoder(session: Session, table: TableSpec) -> None: + raise SystemExit(f"{table.id}: image-encoder measurement is not implemented yet; leave the cells TODO") + + +RUNNERS = { + "loading": run_loading, + "single": run_single, + "multi": run_multi, + "memory": run_memory, + "image-encoder": run_image_encoder, +} + + +def run_table(session: Session, table: TableSpec) -> None: + RUNNERS[table.kind](session, table) diff --git a/tools/gufo/model_bench/render.py b/tools/gufo/model_bench/render.py new file mode 100644 index 00000000..e061d215 --- /dev/null +++ b/tools/gufo/model_bench/render.py @@ -0,0 +1,282 @@ +"""Render BENCHMARKS.md tables from artifacts, keeping hand-entered cells.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +from .artifacts import artifact_path, load_artifact +from .config import BenchConfig, TableSpec + +TODO = "TODO" +MARKER_RE = re.compile( + r"\n(?P.*?)", re.DOTALL +) +NUMBER_RE = re.compile(r"[-+]?\d[\d,]*\.?\d*") + + +@dataclass +class Column: + header: str + owner: str # gufo | reference | gain | exact | label + align: str = "---:" + + +@dataclass +class Layout: + columns: list[Column] + rows: list[str] # row labels in order + + +def _number(text: str | None) -> float | None: + if text is None: + return None + text = text.replace("**", "").strip() + if text == TODO or text in ("", "-", "n/a"): + return None + match = NUMBER_RE.search(text) + if match is None: + return None + try: + return float(match.group(0).replace(",", "")) + except ValueError: + return None + + +def _fmt(value: float | None, digits: int = 2, suffix: str = "") -> str: + if value is None: + return TODO + return f"{value:.{digits}f}{suffix}" + + +def _fmt_stat(row: dict[str, Any] | None, key: str, digits: int = 2, suffix: str = "") -> str | None: + if row is None or row.get(key) is None: + return None + text = _fmt(row[key], digits) + sd = row.get(f"{key}_sd") + if sd is not None: + text += f" ± {sd:.{digits}f}" + return text + suffix + + +def gain(gufo: float | None, reference: float | None, better: str) -> str: + if gufo is None or reference is None or gufo <= 0 or reference <= 0: + return TODO + ratio = gufo / reference if better == "higher" else reference / gufo + return f"{(ratio - 1) * 100:+.1f}%" + + +def _depth_label(depth: int) -> str: + return f"{depth:,}" + + +def _merged_tokens(size: int) -> int: + return (size // 32) ** 2 + + +def layout_for(config: BenchConfig, table: TableSpec) -> Layout: + ref = config.reference_name + spec_label = config.speculative["label"] + kind = table.kind + if kind == "loading": + return Layout( + [Column("Target", "label", "---"), Column("Gufo ready", "gufo"), + Column(f"{ref} ready", "reference"), Column("Gain", "gain")], + [config.variant_label(v) for v in config.variants], + ) + if kind == "single" and not table.speculative: + return Layout( + [Column("Depth", "label"), Column("Gufo pp", "gufo"), Column(f"{ref} pp", "reference"), + Column("Gain", "gain"), Column("Gufo tg", "gufo"), Column(f"{ref} tg", "reference"), + Column("Gain", "gain")], + [_depth_label(d) for d in table.spec["depths"]], + ) + if kind == "single": + return Layout( + [Column("Depth", "label"), Column("Gufo pp", "gufo"), Column("Gufo tg", "gufo"), + Column("Acceptance", "gufo"), Column(f"{ref} AR tg", "reference"), + Column(f"Gain vs {ref} AR", "gain")], + [_depth_label(d) for d in table.spec["depths"]], + ) + if kind == "multi": + return Layout( + [Column("Users", "label"), Column("Gufo AR", "gufo"), Column(f"{ref} AR", "reference"), + Column("Gain", "gain"), Column(f"Gufo {spec_label}", "gufo"), + Column(f"Gain vs {ref} AR", "gain"), Column("Exact", "exact")], + [str(c) for c in table.spec["concurrency"]], + ) + if kind == "memory": + return Layout( + [Column("Workload", "label", "---"), Column("Gufo GiB", "gufo"), + Column(f"{ref} GiB", "reference"), Column("Gain", "gain")], + [w["id"] for w in table.spec["workloads"]], + ) + if kind == "image-encoder": + return Layout( + [Column("RGB image", "label", "---"), Column("Merged tokens", "label"), + Column("Gufo ms", "gufo"), Column(f"{ref} ms", "reference"), Column("Gain", "gain")], + [f"{s}×{s}" for s in table.spec["sizes"]], + ) + raise SystemExit(f"no renderer for table {table.id}") + + +def parse_table(body: str) -> dict[str, list[str]]: + """Existing table cells keyed by first-column label.""" + lines = [line for line in body.strip().splitlines() if line.startswith("|")] + rows: dict[str, list[str]] = {} + for line in lines[2:]: + cells = [cell.strip() for cell in line.strip().strip("|").split("|")] + rows[cells[0].replace("**", "")] = cells[1:] + return rows + + +def existing_tables(document: str) -> dict[str, dict[str, list[str]]]: + return {m.group("id"): parse_table(m.group("body")) for m in MARKER_RE.finditer(document)} + + +def _row_values(config: BenchConfig, table: TableSpec) -> dict[str, dict[str, str | None]]: + """Fresh cell text per row label from artifacts; None means no artifact value.""" + values: dict[str, dict[str, str | None]] = {} + kind = table.kind + if kind == "multi": + gufo_ar = load_artifact(artifact_path(config, table, "gufo", "ar")) + gufo_spec = load_artifact(artifact_path(config, table, "gufo", config.speculative["mode"])) + reference = load_artifact(artifact_path(config, table, "reference")) + for users in table.spec["concurrency"]: + values[str(users)] = { + "gufo_ar": _fmt(_serving_rate(gufo_ar, users)) if gufo_ar else None, + "reference": _fmt(_serving_rate(reference, users)) if reference else None, + "gufo_spec": _fmt(_serving_rate(gufo_spec, users)) if gufo_spec else None, + "exact": _serving_exact(reference, users) if reference else None, + } + return values + + gufo = load_artifact(artifact_path(config, table, "gufo")) + if kind == "single" and table.speculative: + ar_table = config.table("single-ar" + (f"-{table.variant}" if table.variant else "")) + reference = load_artifact(artifact_path(config, ar_table, "reference")) + else: + reference = load_artifact(artifact_path(config, table, "reference")) + g_rows = (gufo or {}).get("rows", {}) + r_rows = (reference or {}).get("rows", {}) + + if kind == "loading": + for variant in config.variants: + values[config.variant_label(variant)] = { + "gufo": _fmt_stat(g_rows.get(variant), "ready_s", 2, " s"), + "reference": _fmt_stat(r_rows.get(variant), "ready_s", 2, " s"), + } + elif kind == "single": + for depth in table.spec["depths"]: + g, r = g_rows.get(str(depth)), r_rows.get(str(depth)) + values[_depth_label(depth)] = { + "gufo_pp": _fmt_stat(g, "pp"), "gufo_tg": _fmt_stat(g, "tg"), + "acceptance": _fmt_stat(g, "acceptance", 1, "%"), + "ref_pp": _fmt_stat(r, "pp"), "ref_tg": _fmt_stat(r, "tg"), + } + elif kind == "memory": + for workload in table.spec["workloads"]: + key = workload["id"] + values[key] = {"gufo": _fmt_stat(g_rows.get(key), "gib"), + "reference": _fmt_stat(r_rows.get(key), "gib")} + elif kind == "image-encoder": + for size in table.spec["sizes"]: + key = str(size) + values[f"{size}×{size}"] = {"gufo": _fmt_stat(g_rows.get(key), "ms", 1), + "reference": _fmt_stat(r_rows.get(key), "ms", 1)} + return values + + +def _serving_rate(report: dict[str, Any] | None, users: int) -> float | None: + if report is None: + return None + result = report.get("results", {}).get(f"c{users}") + if result is None: + return None + return result["aggregate"]["output_tokens_per_second"]["overall"] + + +def _serving_exact(report: dict[str, Any] | None, users: int) -> str | None: + if report is None: + return None + result = report.get("results", {}).get(f"c{users}") + exactness = (result or {}).get("completionExactness") + if not isinstance(exactness, dict): + return None + return f"{exactness['exactRequests']}/{exactness['comparedRequests']}" + + +def _pick(fresh: str | None, existing: str | None) -> str: + if fresh is not None: + return fresh + return existing if existing else TODO + + +def render_table(config: BenchConfig, table: TableSpec, existing: dict[str, list[str]] | None) -> str: + layout = layout_for(config, table) + values = _row_values(config, table) + better = table.spec.get("better", "higher") + kind = table.kind + lines = ["| " + " | ".join(c.header for c in layout.columns) + " |", + "| " + " | ".join(c.align for c in layout.columns) + " |"] + for label in layout.rows: + old = (existing or {}).get(label, []) + cell = lambda index: old[index] if index < len(old) else None # noqa: E731 + fresh = values.get(label, {}) + if kind == "loading" or kind == "memory": + g = _pick(fresh.get("gufo"), cell(0)) + r = _pick(fresh.get("reference"), cell(1)) + cells = [g, r, gain(_number(g), _number(r), better)] + elif kind == "single" and not table.speculative: + gp = _pick(fresh.get("gufo_pp"), cell(0)); rp = _pick(fresh.get("ref_pp"), cell(1)) + gt = _pick(fresh.get("gufo_tg"), cell(3)); rt = _pick(fresh.get("ref_tg"), cell(4)) + cells = [gp, rp, gain(_number(gp), _number(rp), better), + gt, rt, gain(_number(gt), _number(rt), better)] + elif kind == "single": + gp = _pick(fresh.get("gufo_pp"), cell(0)); gt = _pick(fresh.get("gufo_tg"), cell(1)) + acc = _pick(fresh.get("acceptance"), cell(2)); rt = _pick(fresh.get("ref_tg"), cell(3)) + cells = [gp, gt, acc, rt, gain(_number(gt), _number(rt), better)] + elif kind == "multi": + ga = _pick(fresh.get("gufo_ar"), cell(0)); ra = _pick(fresh.get("reference"), cell(1)) + gs = _pick(fresh.get("gufo_spec"), cell(3)); ex = _pick(fresh.get("exact"), cell(5)) + cells = [ga, ra, gain(_number(ga), _number(ra), better), gs, + gain(_number(gs), _number(ra), better), ex] + elif kind == "image-encoder": + size = int(label.split("×")[0]) + g = _pick(fresh.get("gufo"), cell(1)); r = _pick(fresh.get("reference"), cell(2)) + cells = [str(_merged_tokens(size)), g, r, gain(_number(g), _number(r), better)] + else: + raise SystemExit(f"no renderer for table {table.id}") + lines.append("| " + " | ".join([label, *cells]) + " |") + return "\n".join(lines) + "\n" + + +def render_document(config: BenchConfig, document: str, only: set[str] | None = None) -> tuple[str, list[str]]: + existing = existing_tables(document) + known = {t.id: t for t in config.tables()} + rendered: list[str] = [] + + def replace(match: re.Match[str]) -> str: + table_id = match.group("id") + if table_id not in known or (only and table_id not in only): + return match.group(0) + body = render_table(config, known[table_id], existing.get(table_id)) + rendered.append(table_id) + return f"\n{body}" + + return MARKER_RE.sub(replace, document), rendered + + +def todo_rows(config: BenchConfig, document: str, table: TableSpec, target: str) -> set[str] | None: + """Row labels whose target-owned cells are TODO; None when the table is absent.""" + tables = existing_tables(document) + if table.id not in tables: + return None + layout = layout_for(config, table) + owned = [i for i, c in enumerate(layout.columns[1:]) if c.owner == target or (target == "reference" and c.owner == "exact")] + todo: set[str] = set() + for label, cells in tables[table.id].items(): + if any(i >= len(cells) or cells[i] == TODO for i in owned): + todo.add(label) + return todo diff --git a/tools/gufo/model_bench/servers.py b/tools/gufo/model_bench/servers.py new file mode 100644 index 00000000..f4d84f98 --- /dev/null +++ b/tools/gufo/model_bench/servers.py @@ -0,0 +1,139 @@ +"""Server lifecycle for Gufo and reference servers.""" + +from __future__ import annotations + +import json +import os +import shutil +import signal +import socket +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +DEFAULT_READY_TIMEOUT = 3600.0 + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +DROP_CACHES = "sync; echo 3 > /proc/sys/vm/drop_caches" + + +def drop_file_cache(command: str | None = None) -> None: + """Drop the page cache so the next launch measures a cold file cache.""" + candidates = [command] if command else [ + DROP_CACHES, + f"sudo -n sh -c '{DROP_CACHES}'", + f"doas -n sh -c '{DROP_CACHES}'", + ] + for candidate in candidates: + completed = subprocess.run(candidate, shell=True, capture_output=True, text=True) + if completed.returncode == 0: + return + raise SystemExit( + "cold-file-cache loading needs privileges to run " + f"`{DROP_CACHES}`; pass --drop-caches with a command that does it " + "(for example `doas sh -c '...'`), or skip the loading table" + ) + + +def rocm_used_gib() -> float | None: + """Device-visible memory in use (VRAM + GTT) from rocm-smi, or None when unavailable.""" + if shutil.which("rocm-smi") is None: + return None + completed = subprocess.run( + ["rocm-smi", "--showmeminfo", "vram", "gtt", "--json"], capture_output=True, text=True + ) + if completed.returncode != 0: + return None + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError: + return None + total = 0.0 + found = False + for card in payload.values(): + if not isinstance(card, dict): + continue + for key in ("VRAM Total Used Memory (B)", "GTT Total Used Memory (B)"): + try: + total += float(card[key]) + found = True + except (KeyError, TypeError, ValueError): + continue + return total / (1024 ** 3) if found else None + + +class Server: + """A benchmark server process bound to a private loopback port.""" + + def __init__(self, command: list[str], readiness: str, log_path: Path, env: dict[str, str] | None = None): + self.command = command + self.readiness = readiness + self.log_path = log_path + self.env = env + self.port = free_port() + self.process: subprocess.Popen[bytes] | None = None + self.ready_seconds: float | None = None + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def start(self, timeout: float = DEFAULT_READY_TIMEOUT) -> "Server": + self.log_path.parent.mkdir(parents=True, exist_ok=True) + log = self.log_path.open("ab") + env = dict(os.environ) + if self.env: + env.update(self.env) + started = time.perf_counter() + self.process = subprocess.Popen( + self.command, stdout=log, stderr=subprocess.STDOUT, env=env, + start_new_session=True, + ) + deadline = started + timeout + url = self.base_url + self.readiness + while time.perf_counter() < deadline: + if self.process.poll() is not None: + raise RuntimeError( + f"server exited with {self.process.returncode} before readiness; see {self.log_path}" + ) + try: + with urllib.request.urlopen(url, timeout=5) as response: + if response.status == 200: + self.ready_seconds = time.perf_counter() - started + return self + except (urllib.error.URLError, urllib.error.HTTPError, ConnectionError, TimeoutError): + pass + time.sleep(0.05) + self.stop() + raise RuntimeError(f"server not ready within {timeout:.0f} s; see {self.log_path}") + + def stop(self) -> None: + if self.process is None or self.process.poll() is not None: + return + os.killpg(self.process.pid, signal.SIGTERM) + try: + self.process.wait(timeout=60) + except subprocess.TimeoutExpired: + os.killpg(self.process.pid, signal.SIGKILL) + self.process.wait() + + def __enter__(self) -> "Server": + return self.start() + + def __exit__(self, *exc: Any) -> None: + self.stop() + + +def wait_process_exit(server: Server) -> None: + server.stop() + # Give the driver time to release device memory before the next launch. + time.sleep(2.0) diff --git a/tools/gufo/serving_bench.py b/tools/gufo/serving_bench.py index b3a04105..84c4855f 100644 --- a/tools/gufo/serving_bench.py +++ b/tools/gufo/serving_bench.py @@ -327,12 +327,14 @@ def run_request( clock: Callable[[], float] = time.perf_counter, endpoint_profile: str = "gufo", cache_prompt: bool | None = None, + messages: list[dict[str, str]] | None = None, ) -> RequestObservation: if endpoint_profile not in ENDPOINT_PROFILES: raise ValueError("endpoint profile must be gufo or openai") payload: dict[str, Any] = { "model": model, - "messages": [{"role": "user", "content": prompt}], + # `messages` overrides the single-turn prompt for cached-prefix workloads. + "messages": messages or [{"role": "user", "content": prompt}], "max_tokens": max_tokens, "temperature": temperature, "stream": True, diff --git a/tools/serving/concurrency-table.py b/tools/serving/concurrency-table.py deleted file mode 100644 index d38f3ceb..00000000 --- a/tools/serving/concurrency-table.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -"""Render the README concurrency table from serving benchmark artifacts. - -Usage: - concurrency-table.py --dir docs/models/qwen3.8-27b/artifacts - -Reads `{gufo-ar,gufo-dflash2,llama-server}-{mixed,repetition}.json` and prints -a Markdown table with end-to-end aggregate output tokens per second per -concurrency level as `mixed / repetition` pairs, exact-match counts against the -Gufo AR C=1 reference, DFlash2 acceptance, DFlash2 over Gufo AR, and Gufo AR -over llama.cpp (the exact-match column says how comparable the outputs are). -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from typing import Any - -SERVERS = ("gufo-ar", "gufo-dflash2", "llama-server") -GROUPS = ("mixed", "repetition") - - -def _load(directory: Path, name: str, group: str) -> dict[str, Any] | None: - path = directory / f"{name}-{group}.json" - if not path.exists(): - return None - return json.loads(path.read_text(encoding="utf-8")) - - -def _result( - reports: dict[tuple[str, str], dict[str, Any] | None], - name: str, - group: str, - level: int, -) -> dict[str, Any] | None: - report = reports[(name, group)] - if report is None: - return None - return report["results"].get(f"c{level}") - - -def _rate(result: dict[str, Any] | None) -> float | None: - if result is None: - return None - return result["aggregate"]["output_tokens_per_second"]["overall"] - - -def _matched(result: dict[str, Any] | None) -> str: - if result is None: - return "-" - exactness = result.get("completionExactness") - if not isinstance(exactness, dict): - return "-" - return f"{exactness['exactRequests']}/{exactness['comparedRequests']}" - - -def _acceptance(result: dict[str, Any] | None) -> str: - if result is None: - return "-" - value = result["speculative"]["acceptance"] - return "-" if value is None else f"{value * 100:.1f}%" - - -def _fmt(value: float | None) -> str: - return "-" if value is None else f"{value:.2f}" - - -def _ratio(numerator: float | None, denominator: float | None) -> str: - if numerator is None or denominator is None or denominator <= 0.0: - return "-" - return f"{numerator / denominator:.2f}x" - - -def _pair(values: list[str]) -> str: - return " / ".join(values) - - -def render(directory: Path) -> str: - reports = { - (name, group): _load(directory, name, group) - for name in SERVERS - for group in GROUPS - } - levels: list[int] = [] - for report in reports.values(): - if report is not None: - for key in report["results"]: - level = int(key[1:]) - if level not in levels: - levels.append(level) - levels.sort() - - header = ( - "| Concurrency | Gufo AR | Gufo DFlash2 | DFlash2 acceptance | " - "DFlash2 / Gufo AR | llama.cpp AR | llama.cpp exact | " - "Gufo AR / llama.cpp |" - ) - rule = ( - "| ----------: | ------: | -----------: | -----------------: | " - "----------------: | -----------: | --------------: | " - "------------------: |" - ) - lines = [header, rule] - for level in levels: - ar = [_result(reports, "gufo-ar", group, level) for group in GROUPS] - dflash = [ - _result(reports, "gufo-dflash2", group, level) for group in GROUPS - ] - llama = [ - _result(reports, "llama-server", group, level) for group in GROUPS - ] - cells = [ - str(level), - _pair([_fmt(_rate(result)) for result in ar]), - _pair([_fmt(_rate(result)) for result in dflash]), - _pair([_acceptance(result) for result in dflash]), - _pair( - [ - _ratio(_rate(candidate), _rate(baseline)) - for candidate, baseline in zip(dflash, ar) - ] - ), - _pair([_fmt(_rate(result)) for result in llama]), - _pair([_matched(result) for result in llama]), - _pair( - [ - _ratio(_rate(gufo), _rate(candidate)) - for candidate, gufo in zip(llama, ar) - ] - ), - ] - lines.append("| " + " | ".join(cells) + " |") - return "\n".join(lines) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--dir", type=Path, required=True) - args = parser.parse_args(argv) - print(render(args.dir)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())