mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 14:21:06 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Niko1221
|
1e3bded2f7 | ||
|
Niko1221
|
ae3dc218ea | ||
|
Niko
|
3cba59ea31 | ||
|
Niko
|
ecba7dfc10 | ||
|
Niko1221
|
1314ebfc7f | ||
|
Niko1221
|
7664149081 | ||
|
Niko1221
|
6abee7f397 | ||
|
Niko1221
|
868dc724d6 | ||
|
Niko1221
|
f1b35531c3 | ||
|
Niko1221
|
20351c6030 | ||
|
Niko1221
|
bbba5bf576 | ||
|
pete
|
5f563ce3a3 |
No files matched your search
@@ -180,6 +180,7 @@ if(STRATA_ENABLE_CUDA)
|
||||
src/kernels/cuda/kv_stream.cu
|
||||
src/kernels/cuda/kv_q4.cu
|
||||
src/kernels/cuda/fused_gr.cu
|
||||
src/kernels/cuda/cvec.cu
|
||||
src/kernels/cuda/qsa_decode_attn.cu
|
||||
src/kernels/cuda/fused_gdn.cu
|
||||
src/kernels/cuda/dequant_bf16.cu
|
||||
@@ -369,6 +370,10 @@ if(STRATA_ENABLE_CUDA)
|
||||
target_include_directories(kv_q4_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
target_link_libraries(kv_q4_parity PRIVATE strata_kernels CUDA::cudart)
|
||||
add_test(NAME kv_q4_parity COMMAND kv_q4_parity)
|
||||
add_executable(cvec_parity src/kernels/cvec_parity.cpp)
|
||||
target_include_directories(cvec_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
target_link_libraries(cvec_parity PRIVATE strata_kernels CUDA::cudart)
|
||||
add_test(NAME cvec_parity COMMAND cvec_parity)
|
||||
|
||||
# Plan v0.3 P3: native MMVQ for 1..8 tokens (bitwise selftest + --bench timing); synthetic data, no model.
|
||||
if(STRATA_BUILD_TESTS)
|
||||
|
||||
@@ -3,54 +3,22 @@
|
||||
<p align="center"><b>Run a 125-billion-parameter AI model on a normal gaming PC</b><br>
|
||||
one NVIDIA card (12-24 GB) + 64 GB of RAM · Windows or Linux · one click to install</p>
|
||||
|
||||
<p align="center"><a href="https://github.com/Niko1221/Strata/releases/download/v0.1.10/Pagoda.mp4"><img src="docs/media/pagoda-preview.webp" width="720" alt="A voxel pagoda garden that Strata's model wrote, running in the browser"></a><br>
|
||||
<sub>A voxel pagoda garden, 1 shot prompt running on an RTX 5070 with Strata (IQ3_S, 128K context) ·
|
||||
<a href="https://github.com/Niko1221/Strata/releases/download/v0.1.10/Pagoda.mp4">full video (49 s)</a></sub></p>
|
||||
|
||||
Strata runs **[Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next)** - a large, smart AI model that
|
||||
normally needs a server - on your own PC. It writes its answers at **60-95 tokens per second** (a token is about ¾
|
||||
of a word): faster than you can read.
|
||||
|
||||
- **Private:** everything runs on your PC. Nothing is sent anywhere.
|
||||
- **Works with your apps:** chat apps, coding agents and scripts that speak the OpenAI or Anthropic API just work.
|
||||
- **Sees pictures** too, if you want (screenshots, photos, scanned pages).
|
||||
- **Free and open source.**
|
||||
|
||||
> **Jump to:** [Is my PC enough?](#is-my-pc-enough) · [Install](#install-3-steps) · [How fast?](#how-fast-is-it) ·
|
||||
> [Which model?](#which-model-should-i-pick) · [Using it](#using-it) · [Problems?](#something-went-wrong) ·
|
||||
> [How it works](#how-does-it-work) · [All the details](docs/DETAILS.md)
|
||||
> **Jump to:** [How fast?](#how-fast-is-it) · [Which model?](#which-model-should-i-pick) · [Install](#install) ·
|
||||
> [Using it](#using-it) · [Problems?](#something-went-wrong) · [How it works](#how-does-it-work) ·
|
||||
> [All the details](docs/DETAILS.md)
|
||||
|
||||
---
|
||||
|
||||
## Is my PC enough?
|
||||
|
||||
| You need | |
|
||||
| --- | --- |
|
||||
| **Graphics card** | NVIDIA RTX 30, 40 or 50 series with **12 GB of VRAM or more** |
|
||||
| **Memory (RAM)** | **64 GB** |
|
||||
| **Free disk space** | ~80 GB (an SSD makes the first start much faster) |
|
||||
| **System** | Windows 10/11, or Linux |
|
||||
|
||||
That's it. The only thing you install yourself is a current **NVIDIA driver**
|
||||
([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA App). Everything else - Python, the engine, the
|
||||
model - is set up for you.
|
||||
|
||||
## Install (3 steps)
|
||||
|
||||
**Windows**
|
||||
|
||||
1. [Download this project](https://github.com/Niko1221/Strata/archive/refs/heads/main.zip) and unzip it (or `git clone` it).
|
||||
2. Double-click **`START-HERE.bat`**.
|
||||
3. Answer 4 questions - or just press Enter each time for the recommended choice:
|
||||
- **Which model?** The original, or Swift 1.5 (a version that thinks shorter and answers sooner)
|
||||
- **Which size?** Q2_0, IQ2_XS, IQ3_XXS or IQ3_S - see [which model](#which-model-should-i-pick)
|
||||
- **How much context?** How much text it can keep in mind at once (it suggests one for your card)
|
||||
- **Images?** Whether it should also read pictures
|
||||
|
||||
Then it downloads everything (the model is ~70 GB, so the first time takes a while - you can stop and it picks up
|
||||
where it left off) and **starts the model**. Your browser opens the Strata app at `http://127.0.0.1:8080`.
|
||||
|
||||
**Next time**, just double-click `START-HERE.bat` again: it starts right away, nothing is downloaded twice. Close its
|
||||
window to stop the model.
|
||||
|
||||
**Linux:** run `./setup.sh` - same questions, same result.
|
||||
|
||||
## How fast is it?
|
||||
|
||||
Measured on an RTX 5070 (12 GB), a Ryzen 5 7600 and 64 GB of RAM:
|
||||
@@ -73,24 +41,58 @@ A card with more VRAM is faster, because more of the model fits on the GPU: an R
|
||||
|
||||
**The size** (the same model, compressed more or less):
|
||||
|
||||
| Size | Download | Speed | Quality | Pick it if... |
|
||||
| --- | ---: | --- | --- | --- |
|
||||
| **Q2_0** | 66 GB | fastest | good | you want speed |
|
||||
| **IQ2_XS** | 68 GB | fast | better | you want a good all-rounder (**recommended**) |
|
||||
| **IQ3_XXS** | 76 GB | slower | great | you want better answers (uses 43 GB of your 64 GB RAM) |
|
||||
| **IQ3_S** | 84 GB | slowest | best: matches the full model on the published tests | you want the very best answers (original model only; uses 50 GB of your 64 GB RAM, so close other big programs) |
|
||||
| Model | RAM+VRAM Requirements | Speed | Quality |
|
||||
| --- | ---: | --- | --- |
|
||||
| **Q2_0** | 37.6 GB | fastest | good |
|
||||
| **IQ2_XS** | 39.2 GB | fast | better (**recommended**) |
|
||||
| **IQ3_XXS** | 47.0 GB | slower | great |
|
||||
| **IQ3_S** | 54.8 GB | slowest | best: matches the full model on the published tests (original model only) |
|
||||
|
||||
**Will it fit?** Shard 1 is the part of the model that gets loaded when it starts: its experts go into your **RAM**,
|
||||
the rest onto your graphics card (the second shard, a 29 GB lookup table, stays on the SSD). So it fits when your
|
||||
**RAM is at least shard 1 + about 10 GB** for Windows and your other programs. With 64 GB of RAM every size fits
|
||||
(IQ3_S with little else open); with 48 GB, Q2_0 and IQ2_XS. A bigger graphics card makes it faster, but it doesn't
|
||||
lower the RAM needed.
|
||||
|
||||
**The version:**
|
||||
|
||||
- **Qwen3.8-Flash-Next** - the original.
|
||||
- **[Swift 1.5](https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF)** - a fine-tune by UkisAI
|
||||
that thinks much shorter before answering, so you get the answer sooner, with about the same quality. Same speed per
|
||||
token. Its own license applies (see its page).
|
||||
token, and about the same RAM as the same size of the original (no IQ3_S). Its own license applies (see its page).
|
||||
|
||||
Not sure? Take **IQ2_XS**. You can add another one later with `START-HERE.bat --setup`.
|
||||
|
||||
## Install
|
||||
|
||||
**You need:** an NVIDIA RTX 30, 40 or 50 card with 12 GB of VRAM or more, enough RAM for the size you pick (above),
|
||||
~80 GB of free disk space (an SSD makes the first start much faster), and Windows 10/11 or Linux. The only thing you
|
||||
install yourself is a current **NVIDIA driver** ([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA
|
||||
App). Everything else - Python, the engine, the model - is set up for you.
|
||||
|
||||
**Windows**
|
||||
|
||||
1. [Download this project](https://github.com/Niko1221/Strata/archive/refs/heads/main.zip) and unzip it (or `git clone` it).
|
||||
2. Double-click **`START-HERE.bat`**.
|
||||
3. Answer a few questions - or just press Enter each time for the recommended choice:
|
||||
- **Which model and size?** The original or Swift 1.5, and Q2_0, IQ2_XS, IQ3_XXS or IQ3_S - see [above](#which-model-should-i-pick)
|
||||
- **How much context?** How much text it can keep in mind at once (it suggests one for your card)
|
||||
- **Images?** Whether it should also read pictures
|
||||
- **Experimental speed projection?** Off unless you say yes - [read what it does](docs/DETAILS.md#experimental-speed-projection-experimental-off-by-default) first
|
||||
|
||||
Then it downloads everything (the model is ~70 GB, so the first time takes a while - you can stop and it picks up
|
||||
where it left off) and **starts the model**. Your browser opens the Strata app at `http://127.0.0.1:8080`.
|
||||
|
||||
**Next time**, just double-click `START-HERE.bat` again: it starts right away, nothing is downloaded twice. Close its
|
||||
window to stop the model.
|
||||
|
||||
**Linux:** run `./setup.sh` - same questions, same result.
|
||||
|
||||
## Using it
|
||||
|
||||
<p align="center"><img src="docs/media/runpagoda.png" width="900" alt="The Strata app's Monitor tab next to a coding agent"><br>
|
||||
<sub>The Strata app's <b>Monitor</b> (left) while a coding agent writes the pagoda garden from the video (right)</sub></p>
|
||||
|
||||
- **In the browser:** `http://127.0.0.1:8080` - the Strata app (it opens by itself when the model starts): **Chat**, a
|
||||
live **Monitor** of the model and your GPU/CPU/RAM, and **About** with the settings and addresses.
|
||||
- **Chat in the terminal:** `.venv\Scripts\python chat.py`
|
||||
@@ -99,24 +101,45 @@ Not sure? Take **IQ2_XS**. You can add another one later with `START-HERE.bat --
|
||||
- **Thinking:** the model thinks before it answers. Choose **off, low, medium or high** - in the chat page menu, with
|
||||
`/think low` in `chat.py`, or with your app's "reasoning effort" setting. Off is fastest; high is best for hard questions.
|
||||
- **Pictures:** in the chat page click **Picture**; in `chat.py` type `/image <path>`; in apps just attach them.
|
||||
- **From your phone or another PC:** see the [details](docs/DETAILS.md#using-it) (set an API key first).
|
||||
- **From your phone or another PC:** `START-HERE.bat --setup --host 0.0.0.0 --api-key <secret>`, then open the
|
||||
address the server window prints; see the [details](docs/DETAILS.md#using-it).
|
||||
- **Experimental speed projection (off by default):** an experimental control vector that setup can turn on; it
|
||||
changes how the model answers - read [what it does](docs/DETAILS.md#experimental-speed-projection-experimental-off-by-default) first.
|
||||
|
||||
**Good to know:** it answers one request at a time. The first message of a chat is read in full (about 1 minute per
|
||||
30,000 tokens); after that it keeps the conversation and reads only what is new, so follow-ups start in seconds.
|
||||
|
||||
## Something went wrong?
|
||||
|
||||
| What you see | What to do |
|
||||
| --- | --- |
|
||||
| `the NVIDIA driver is too old` | Update the driver (NVIDIA App or nvidia.com/drivers), restart the PC, run `START-HERE.bat` again. |
|
||||
| It stopped during download or setup | Run `START-HERE.bat` again - it continues where it stopped. |
|
||||
| `port 8080 is already in use` | Strata is already running - look for its window. |
|
||||
| The first start takes minutes | Normal: it loads 35-43 GB into RAM. The next start is faster. |
|
||||
| Slow, and the disk light is busy | Not enough free RAM: close other programs (browsers use a lot), or pick Q2_0 / IQ2_XS. |
|
||||
| "prompt exceeds the context" | The conversation is longer than the context you chose: run `START-HERE.bat --setup` and pick more. |
|
||||
**My PC froze, or got very slow, the first time Strata started.**
|
||||
That's normal the first time. Strata loads 35-55 GB into your RAM, locks part of it for the graphics card, and works
|
||||
out how much of the model fits on your GPU. The mouse can freeze for a few minutes. **Wait, and don't close the
|
||||
window.** The next starts are much faster. Still frozen after 10 minutes? Restart the PC, close other programs
|
||||
(browsers use a lot of RAM) and try again. If it keeps happening, pick a smaller size (Q2_0 or IQ2_XS).
|
||||
|
||||
More in the [full troubleshooting table](docs/DETAILS.md#troubleshooting). Still stuck? Open an issue and attach
|
||||
`strata-<model>.log` from this folder.
|
||||
**It stopped while downloading or installing.**
|
||||
Run `START-HERE.bat` again. It continues where it stopped.
|
||||
|
||||
**It says the NVIDIA driver is too old.**
|
||||
Update it (NVIDIA App or [nvidia.com/drivers](https://www.nvidia.com/drivers)), restart the PC, and run
|
||||
`START-HERE.bat` again.
|
||||
|
||||
**It says port 8080 is already in use.**
|
||||
Strata is already running. Look for its window.
|
||||
|
||||
**It's very slow and the disk light keeps blinking.**
|
||||
Your PC is out of free RAM. Close other programs, or pick a smaller size (Q2_0 or IQ2_XS).
|
||||
|
||||
**An answer stopped with "the engine stopped unexpectedly".**
|
||||
Usually not enough RAM (on Linux the system then stops the engine). Just send your message again: Strata starts the
|
||||
engine by itself. If it keeps happening, close other programs or pick a smaller size.
|
||||
|
||||
**It says the prompt exceeds the context.**
|
||||
The conversation is longer than the context you chose. Start a new chat, or run `START-HERE.bat --setup` and pick more
|
||||
context.
|
||||
|
||||
**Still stuck?** Look in the [full troubleshooting table](docs/DETAILS.md#troubleshooting), or open an issue and
|
||||
attach `strata-<model>.log` from the Strata folder.
|
||||
|
||||
## How does it work?
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# 2026-09-27: the experimental-speed-projection control vector
|
||||
|
||||
The package's `Qwen3.8-Flash-Next-experimental-speed-projection.gguf` (47 unit directions of 2,560, one per layer),
|
||||
applied as `--control-vector-scaled FILE:1.0 --control-vector-layer-range 4 44 --cvec-mode project --cvec-dir
|
||||
per-layer`: after layers 4-44, every hyper-connection stream `h -= (h . v) v`. The package describes it as a
|
||||
refusal-direction projection. Q2_0, RTX 5070, fixed experts (`--expert-cache 4000 --pcie-frac 0 --adapt-swaps 0`).
|
||||
|
||||
## Kernel (`cvec_parity`)
|
||||
|
||||
Project and add against a double-precision reference (max error 5e-7); with the switch off, the kernel's pending
|
||||
residual write is bitwise the fused hyper-connection read's own, so a loaded-but-off vector is the stock engine.
|
||||
|
||||
## Teacher-forced, stock vs projection (`esp_kl.py`, per-token path)
|
||||
|
||||
| Text | Tokens | Same top-1 | Mean KL | p99 KL | Perplexity stock | With the vector |
|
||||
|---|---:|---:|---:|---:|---:|---:|
|
||||
| Code | 511 | 92.8% | 0.098 | 1.69 | 2.594 | 2.982 |
|
||||
| Document | 1,023 | 89.8% | 0.058 | 0.84 | 30.299 | 30.984 |
|
||||
| Chat | 1,023 | 89.6% | 0.050 | 0.33 | 18.409 | 18.485 |
|
||||
| **All** | 2,557 | | **0.063** (max 4.10) | 1.00 | | |
|
||||
|
||||
The package reports a mean KL of 0.186 on its own neutral text with UD-Q2_K_XL in llama.cpp.
|
||||
|
||||
Cost on the same tokens: 44.34 -> 44.26, 46.74 -> 46.56, 47.02 -> 46.83 tok/s (0.2-0.4%).
|
||||
|
||||
## Serve (`esp_serve.py`)
|
||||
|
||||
- Loaded and switched off (`cvec=0`) vs an engine without it: 65 / 65 first tokens after prefixes (verify windows
|
||||
and the batched prompt path) and three 256-token generations **token-identical**.
|
||||
- On, off, on again: identical (the switch drops the conversation cache, so nothing computed one way is reused the
|
||||
other way).
|
||||
- The prompt paths steer like the per-token path: served-with-vector first tokens agree with the per-token path's
|
||||
teacher-forced argmax with the vector 62 / 65, and with the stock one 57 / 65 (stock served: 64 / 65 and 57 / 65).
|
||||
- Decode tok/s of 3 x 256-token generations: stock 66.4, on 76.7 / 81.2. **Not a like-for-like comparison:** the
|
||||
model writes different text with it on (on the code prompt it copied a long stretch of the source, which prompt
|
||||
lookup drafts at 91% acceptance, where stock stopped after 78 tokens). On the same tokens it is 0.2-0.4% slower.
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Teacher-forced logits, stock vs the experimental-speed-projection vector, on the same texts and fixed experts.
|
||||
|
||||
python esp_kl.py run # the dumps (per-token path)
|
||||
python esp_kl.py compare # same top-1, mean/median KL(stock || projection), perplexities
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
S = os.path.dirname(os.path.abspath(__file__))
|
||||
E = r"C:\Users\AI-Server\Desktop\Strata\Public\Engine"
|
||||
EXE = E + r"\build-dev\strata.exe"
|
||||
VEC = r"C:\Users\AI-Server\Desktop\Strata\experimental-speed-projection\Qwen3.8-Flash-Next-experimental-speed-projection.gguf"
|
||||
TEXTS = {"code": S + r"\p512.ids", "doc": S + r"\4k-02-long-doc-1k.ids", "chat": S + r"\4k-04-chat-1k.ids"}
|
||||
OUT = S + r"\esp-kl"
|
||||
ARMS = {"stock": [],
|
||||
"proj": ["--control-vector-scaled", VEC + ":1.0", "--control-vector-layer-range", "4", "44",
|
||||
"--cvec-mode", "project", "--cvec-dir", "per-layer"]}
|
||||
|
||||
|
||||
def base_args():
|
||||
c = json.load(open(S + r"\cfg-rel.json"))
|
||||
a = list(c["args"])
|
||||
i = a.index("--expert-cache")
|
||||
a[i + 1] = "4000"
|
||||
j = a.index("--prefill") # the prompt one token at a time: a logits row per position
|
||||
del a[j:j + 2]
|
||||
return a + ["--pcie-frac", "0", "--adapt-swaps", "0"], c
|
||||
|
||||
|
||||
def run():
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
args, c = base_args()
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = os.pathsep.join(c["lib_dirs"]) + os.pathsep + env["PATH"]
|
||||
for name, path in TEXTS.items():
|
||||
for arm, extra in ARMS.items():
|
||||
out = f"{OUT}\\{name}-{arm}.bin"
|
||||
if os.path.exists(out):
|
||||
continue
|
||||
cmd = [EXE, *args, *extra, "--tokens-file", path, "--max-new", "1", "--dump-logits", out]
|
||||
with open(f"{OUT}\\{name}-{arm}.log", "w") as log:
|
||||
r = subprocess.run(cmd, cwd=c["cwd"], stdout=log, stderr=subprocess.STDOUT, env=env)
|
||||
print(name, arm, "exit", r.returncode, flush=True)
|
||||
|
||||
|
||||
def load(p):
|
||||
h = np.fromfile(p, dtype=np.int32, count=2)
|
||||
return np.memmap(p, dtype=np.float32, mode="r", offset=8, shape=(int(h[1]), int(h[0])))
|
||||
|
||||
|
||||
def lsm(x):
|
||||
x = x.astype(np.float64)
|
||||
m = x.max(axis=-1, keepdims=True)
|
||||
return x - (m + np.log(np.exp(x - m).sum(axis=-1, keepdims=True)))
|
||||
|
||||
|
||||
def compare():
|
||||
print(f"{'text':<6}{'tokens':>7}{'same top-1':>11}{'mean KL':>9}{'median KL':>11}{'p99 KL':>8}{'ppl stock':>10}{'ppl proj':>9}")
|
||||
allkl = []
|
||||
for name, path in TEXTS.items():
|
||||
ids = [int(x) for x in open(path).read().replace(",", " ").split()]
|
||||
a, b = load(f"{OUT}\\{name}-stock.bin"), load(f"{OUT}\\{name}-proj.bin")
|
||||
n = min(a.shape[0], b.shape[0], len(ids) - 1)
|
||||
kl, same, la, lb = [], 0, [], []
|
||||
for r in range(0, n, 64):
|
||||
pa, pb = lsm(a[r:min(n, r + 64)]), lsm(b[r:min(n, r + 64)])
|
||||
kl += list((np.exp(pa) * (pa - pb)).sum(axis=1))
|
||||
same += int((pa.argmax(1) == pb.argmax(1)).sum())
|
||||
nxt = np.array(ids[r + 1:min(n, r + 64) + 1])
|
||||
la += list(pa[np.arange(len(nxt)), nxt])
|
||||
lb += list(pb[np.arange(len(nxt)), nxt])
|
||||
kl = np.array(kl)
|
||||
allkl += list(kl)
|
||||
print(f"{name:<6}{n:>7}{100 * same / n:>10.1f}%{kl.mean():>9.3f}{np.median(kl):>11.4f}{np.percentile(kl, 99):>8.2f}"
|
||||
f"{np.exp(-np.mean(la)):>10.3f}{np.exp(-np.mean(lb)):>9.3f}")
|
||||
allkl = np.array(allkl)
|
||||
print(f"all {len(allkl)} tokens: mean KL {allkl.mean():.3f}, max {allkl.max():.2f}, p99 {np.percentile(allkl, 99):.2f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
{"run": run, "compare": compare}[sys.argv[1]]()
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Serve-mode checks of the experimental-speed-projection vector (exact mode: fixed experts, no PCIe share).
|
||||
|
||||
python esp_serve.py run ARM # ARM = stock | vec; writes esp-serve/<arm>.json
|
||||
python esp_serve.py compare
|
||||
|
||||
Per arm: the first greedy token after many prefixes of three texts (short ones go through the verify windows, long
|
||||
ones through the batched prompt path; the conversation cache reuses the earlier prefix), and three 256-token greedy
|
||||
generations. The vec arm runs everything with cvec=1, then cvec=0, then cvec=1 again.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
|
||||
S = os.path.dirname(os.path.abspath(__file__))
|
||||
E = r"C:\Users\AI-Server\Desktop\Strata\Public\Engine"
|
||||
sys.path.insert(0, E)
|
||||
from serve.server import StrataEngine # noqa: E402
|
||||
|
||||
EXE = E + r"\build-dev\strata.exe"
|
||||
VEC = r"C:\Users\AI-Server\Desktop\Strata\experimental-speed-projection\Qwen3.8-Flash-Next-experimental-speed-projection.gguf"
|
||||
TEXTS = {"code": S + r"\p512.ids", "doc": S + r"\4k-02-long-doc-1k.ids", "chat": S + r"\4k-04-chat-1k.ids"}
|
||||
OUT = S + r"\esp-serve"
|
||||
VEC_ARGS = ["--control-vector-scaled", VEC + ":1.0", "--control-vector-layer-range", "4", "44",
|
||||
"--cvec-mode", "project", "--cvec-dir", "per-layer"]
|
||||
|
||||
|
||||
def ids_of(name):
|
||||
return [int(x) for x in open(TEXTS[name]).read().replace(",", " ").split()]
|
||||
|
||||
|
||||
def engine(extra):
|
||||
c = json.load(open(S + r"\cfg-rel.json"))
|
||||
a = list(c["args"])
|
||||
a[a.index("--expert-cache") + 1] = "4000"
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = os.pathsep.join(c["lib_dirs"]) + os.pathsep + env["PATH"]
|
||||
return StrataEngine(EXE, a + ["--pcie-frac", "0", "--adapt-swaps", "0"] + extra, cwd=c["cwd"],
|
||||
log=OUT + r"\engine.log", env=env)
|
||||
|
||||
|
||||
def gen(eng, ids, n, sampling):
|
||||
toks = [t for t in eng.generate(ids, n, sampling, threading.Event()) if t is not None]
|
||||
return toks, dict(eng.last)
|
||||
|
||||
|
||||
def suite(eng, sampling):
|
||||
out = {"first": {}, "long": {}}
|
||||
for name in TEXTS:
|
||||
ids = ids_of(name)
|
||||
# short prefixes growing by 4 (each continues the live session through the verify windows), then long ones
|
||||
# SHRINKING (no cached prefix, so each is read from 0 by the batched prompt path)
|
||||
lens = list(range(32, 72, 4)) + list(range(min(len(ids) - 1, 700), 95, -48))
|
||||
out["first"][name] = {L: gen(eng, ids[:L], 1, sampling)[0][0] for L in lens}
|
||||
for name in TEXTS:
|
||||
toks, last = gen(eng, ids_of(name)[:300], 256, sampling)
|
||||
out["long"][name] = {"tokens": toks, "last": last}
|
||||
return out
|
||||
|
||||
|
||||
def run(arm):
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
eng = engine(VEC_ARGS if arm == "vec" else [])
|
||||
res = {"info": eng.info}
|
||||
if arm == "stock":
|
||||
res["stock"] = suite(eng, {})
|
||||
else:
|
||||
res["on"] = suite(eng, {"experimental_speed_projection": True})
|
||||
res["off"] = suite(eng, {"experimental_speed_projection": False})
|
||||
res["on2"] = suite(eng, {"experimental_speed_projection": True})
|
||||
eng.close()
|
||||
json.dump(res, open(f"{OUT}\\{arm}.json", "w"))
|
||||
print(arm, "done", eng.info, flush=True)
|
||||
|
||||
|
||||
def compare():
|
||||
import numpy as np
|
||||
sys.path.insert(0, S)
|
||||
from esp_kl import load, OUT as KL
|
||||
st = json.load(open(OUT + r"\stock.json"))
|
||||
ve = json.load(open(OUT + r"\vec.json"))
|
||||
print("engine INFO (vec):", ve["info"])
|
||||
|
||||
def eq_first(a, b):
|
||||
tot = same = 0
|
||||
for name in a:
|
||||
for L, t in a[name].items():
|
||||
tot += 1
|
||||
same += int(b[name][L] == t)
|
||||
return same, tot
|
||||
|
||||
def eq_long(a, b):
|
||||
return all(a[n]["tokens"] == b[n]["tokens"] for n in a), \
|
||||
[(n, next((i for i, (x, y) in enumerate(zip(a[n]["tokens"], b[n]["tokens"])) if x != y), None)) for n in a]
|
||||
|
||||
print("1. vector loaded, switched off == no vector")
|
||||
print(" first tokens: %d / %d identical" % eq_first(st["stock"]["first"], ve["off"]["first"]))
|
||||
print(" 256-token generations identical:", *eq_long(st["stock"]["long"], ve["off"]["long"]))
|
||||
print("2. repeatable: on, off, on again")
|
||||
print(" first tokens: %d / %d identical" % eq_first(ve["on"]["first"], ve["on2"]["first"]))
|
||||
print(" 256-token generations identical:", *eq_long(ve["on"]["long"], ve["on2"]["long"]))
|
||||
print("3. the prompt paths against the per-token path's teacher-forced argmax (row L-1 predicts token L)")
|
||||
for arm, first in (("stock", st["stock"]["first"]), ("on", ve["on"]["first"])):
|
||||
for ref in ("stock", "proj"):
|
||||
agree = tot = 0
|
||||
for name in first:
|
||||
d = load(f"{KL}\\{name}-{ref}.bin")
|
||||
for L, t in first[name].items():
|
||||
L = int(L)
|
||||
if L - 1 < d.shape[0]:
|
||||
tot += 1
|
||||
agree += int(int(np.argmax(d[L - 1])) == t)
|
||||
print(f" served {arm:<5} vs per-token {ref:<5}: {agree} / {tot}")
|
||||
print("4. how different the text is: 256-token generations, on vs stock, first difference")
|
||||
print(" ", eq_long(st["stock"]["long"], ve["on"]["long"])[1])
|
||||
print("5. decode speed (exact mode: fixed experts, no PCIe share - not the normal setup)")
|
||||
for label, r in (("stock", st["stock"]), ("vec off", ve["off"]), ("vec on", ve["on"]), ("vec on 2", ve["on2"])):
|
||||
g = sum(v["last"]["generated"] for v in r["long"].values())
|
||||
ms = sum(v["last"]["decode_ms"] for v in r["long"].values())
|
||||
acc = sum(v["last"].get("drafts_accepted", 0) for v in r["long"].values())
|
||||
off = sum(v["last"].get("drafts_offered", 0) for v in r["long"].values())
|
||||
print(f" {label:<9} {g / ms * 1000:6.1f} tok/s drafts accepted {acc}/{off}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.argv[1] == "run":
|
||||
run(sys.argv[2])
|
||||
else:
|
||||
compare()
|
||||
@@ -0,0 +1,20 @@
|
||||
1. vector loaded, switched off == no vector
|
||||
first tokens: 65 / 65 identical
|
||||
256-token generations identical: True [('code', None), ('doc', None), ('chat', None)]
|
||||
2. repeatable: on, off, on again
|
||||
first tokens: 65 / 65 identical
|
||||
256-token generations identical: True [('code', None), ('doc', None), ('chat', None)]
|
||||
3. the prompt paths against the per-token path's teacher-forced argmax (row L-1 predicts token L)
|
||||
served stock vs per-token stock: 64 / 65
|
||||
served stock vs per-token proj : 57 / 65
|
||||
served on vs per-token stock: 57 / 65
|
||||
served on vs per-token proj : 62 / 65
|
||||
4. how different the text is: 256-token generations, on vs stock, first difference
|
||||
[('code', 11), ('doc', 6), ('chat', 3)]
|
||||
5. decode speed (exact mode: fixed experts, no PCIe share - not the normal setup)
|
||||
stock 66.4 tok/s drafts accepted 256/309
|
||||
vec off 65.7 tok/s drafts accepted 251/318
|
||||
vec on 76.7 tok/s drafts accepted 366/425
|
||||
vec on 2 81.2 tok/s drafts accepted 366/423
|
||||
|
||||
[exited with code 0]
|
||||
@@ -0,0 +1,6 @@
|
||||
text tokens same top-1 mean KL median KL p99 KL ppl stock ppl proj
|
||||
code 511 92.8% 0.098 0.0031 1.69 2.594 2.982
|
||||
doc 1023 89.8% 0.058 0.0271 0.84 30.299 30.984
|
||||
chat 1023 89.6% 0.050 0.0265 0.33 18.409 18.485
|
||||
all 2557 tokens: mean KL 0.063, max 4.10, p99 1.00
|
||||
|
||||
BIN
Binary file not shown.
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"mode": "project",
|
||||
"dir": "per-layer",
|
||||
"scale": 1.0,
|
||||
"layer_range": [
|
||||
4,
|
||||
44
|
||||
],
|
||||
"refusal_harmful_thinkoff": 2.0,
|
||||
"refusal_harmful_thinkon": 2.0,
|
||||
"refusal_harmless_thinkoff": 4.0,
|
||||
"refusal_harmless_thinkon": 6.0,
|
||||
"broken": 0,
|
||||
"kl_mean": 0.18598,
|
||||
"kl_max": 18.26549,
|
||||
"kl_p99": 3.143571,
|
||||
"decode_tps": 51.19,
|
||||
"vector_file": "Qwen3.8-Flash-Next-experimental-speed-projection.gguf",
|
||||
"base_model_gguf": "unsloth/Qwen3.8-Flash-Next-GGUF UD-Q2_K_XL",
|
||||
"base_model_sha256_shard1": "a4f3b21e77353999829f2f767e9ac21ce9c71d29a74f2cc9eda48c9bf23c8b86",
|
||||
"worktree_commit": "37e026e3b",
|
||||
"criterion_status": "PASS",
|
||||
"dirty_direction": false,
|
||||
"note": "projection ablation h -= s*(h.v)*v, v unit-normalized per layer. dir=per-layer",
|
||||
"branch": "cvec-hooks (github.com/Cudila/llama.cpp)"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# experimental-speed-projection (EXPERIMENTAL, off by default)
|
||||
|
||||
`Qwen3.8-Flash-Next-experimental-speed-projection.gguf` is a 480 KB control vector for Qwen3.8-Flash-Next: one unit
|
||||
direction of 2,560 values per layer (`direction.1` .. `direction.47`, GGUF `controlvector` format). The JSON beside it
|
||||
is the configuration its authors selected and their scores.
|
||||
|
||||
**What it does.** Turned on, Strata removes each direction from the residual stream after layers 4-44
|
||||
(`h -= (h . v) v` on every hyper-connection stream), the same operation as llama.cpp's `--cvec-mode project` in the
|
||||
package's patches. The package's own documentation describes the vector as a **refusal-direction projection**: the
|
||||
model declines far fewer requests (it reports 1 of 50 vs 50 of 50 on its test set), and removing refusals removes a
|
||||
safety behaviour - you are responsible for what the model writes with it on. It also shifts the model's predictions
|
||||
slightly on ordinary text (measured here: `bench/results/2026-09-27-esp/`). It is not an optimization in the engine:
|
||||
on the same text it costs 0.2-0.4% per token; a chat's speed with it on depends on the text the model writes.
|
||||
|
||||
**Turning it on.** `START-HERE.bat --setup` asks (default: off), or pass `--experimental-speed-projection on`. With it
|
||||
loaded, the web app's Sampling drawer and the API field `"experimental_speed_projection": false` switch it off per
|
||||
request. See `docs/DETAILS.md`, "Experimental speed projection".
|
||||
|
||||
**Origin and license.** A local package of the vector (the published original uses filenames containing "refusal";
|
||||
the bytes are the same), made from Qwen3.8-Flash-Next activations, so the Qwen Community License 1.0 of the model
|
||||
applies to it. Made for the original Qwen3.8-Flash-Next (not Swift 1.5).
|
||||
+55
-5
@@ -216,9 +216,18 @@ print(r.choices[0].message.content)
|
||||
15 s, and `GET /status` says what it is doing (`reading the prompt`, `answering`, tokens so far). Closing the
|
||||
connection or pressing stop in your app really stops the model, so the next request starts at once.
|
||||
- **Chat apps.** Any app with an "OpenAI-compatible" provider works: base URL `http://127.0.0.1:8080/v1`, any API key.
|
||||
- **Context.** Chosen in setup (8K-262K). Requests longer than that are refused, never silently cut.
|
||||
- **From other devices / the internet.** The server listens on your PC only (`127.0.0.1`). To reach it from elsewhere,
|
||||
put a tunnel in front of it, for example [cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/do-more-with-tunnels/trycloudflare/):
|
||||
- **Context.** Chosen in setup (8K-262K). Requests longer than that are refused, never silently cut. A request whose
|
||||
`max_tokens` would run past the context is refused too (400); agents that always ask for their full output cap
|
||||
can instead get it shortened to the room left: add `"fit_max_tokens": true` to `strata-<model>.json` (or pass
|
||||
`--fit-max-tokens` to `serve/server.py`). A prompt that leaves no room at all is still refused.
|
||||
- **From other devices on your network.** The server listens on your PC only (`127.0.0.1`) unless you say otherwise:
|
||||
run setup with `START-HERE.bat --setup --host 0.0.0.0 --api-key some-long-secret` (or add `"host": "0.0.0.0"` and
|
||||
`"api_key": "..."` to `strata-<model>.json`). The server window then prints this PC's addresses
|
||||
(`from other devices: http://192.168.x.x:8080/`); open that on the other device, or use `.../v1` as an API base URL.
|
||||
On Windows the firewall blocks it until you allow it: accept its prompt for Python (private networks), or run
|
||||
`New-NetFirewallRule -DisplayName "Strata 8080" -Direction Inbound -Protocol TCP -LocalPort 8080 -Action Allow -Profile Private`
|
||||
in an admin PowerShell, and make sure the network is set to Private.
|
||||
- **From the internet.** Put a tunnel in front of it, for example [cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/do-more-with-tunnels/trycloudflare/):
|
||||
`cloudflared tunnel --url http://127.0.0.1:8080`. **Set a key first**, or anyone with the link can use your PC:
|
||||
add `"api_key": "some-long-secret"` to `strata-<model>.json` (or set the `STRATA_API_KEY` environment variable);
|
||||
clients then send it as their API key.
|
||||
@@ -315,6 +324,45 @@ test images.
|
||||
|
||||
---
|
||||
|
||||
## Experimental speed projection (EXPERIMENTAL, off by default)
|
||||
|
||||
**This is an experiment, not a finished feature.** It ships with Strata but stays off unless you turn it on.
|
||||
|
||||
A 480 KB control vector for Qwen3.8-Flash-Next (`data/experimental-speed-projection/`, see its README). After each
|
||||
of layers 4-44 the engine removes one direction from every hyper-connection stream of the residual: `h -= (h . v) v`,
|
||||
one unit vector `v` per layer, exactly as llama.cpp does with the package's `--cvec-mode project` patches.
|
||||
|
||||
**What it changes.** The vector's own package describes it as a **refusal-direction projection**: with it the model
|
||||
declines far fewer requests (it reports 1 of 50 vs 50 of 50 on its test set), and removing refusals removes a safety
|
||||
behaviour - you are responsible for what the model writes with it on. It also shifts ordinary answers a little
|
||||
(measured below). It is not an optimization in the engine: on the same text it costs 0.2-0.4% per token. What a
|
||||
chat's tokens/s does with it on depends on the text the model writes (length, repetition, how well the drafts land),
|
||||
so measure it on your own prompts; the Monitor marks every request ESP or stock.
|
||||
|
||||
**Turning it on (at setup).** `START-HERE.bat --setup` asks "Turn on the experimental speed projection?" (default:
|
||||
no), or pass `--experimental-speed-projection on` (`off`, or a path to another vector GGUF). Only for the original
|
||||
Qwen3.8-Flash-Next, not Swift 1.5. It writes these engine flags (llama.cpp's) into `strata-<model>.json`:
|
||||
|
||||
```
|
||||
--control-vector-scaled <Strata>\data\experimental-speed-projection\Qwen3.8-Flash-Next-experimental-speed-projection.gguf:1.0
|
||||
--control-vector-layer-range 4 44 --cvec-mode project --cvec-dir per-layer
|
||||
```
|
||||
|
||||
The engine log then says `control vector mode = project, dir = per-layer, layers 4..44 (41 steered)`, and the web
|
||||
app's About tab lists it. (`--cvec-mode add` is llama.cpp's stock additive mode, for additive vectors.)
|
||||
|
||||
**Per request.** A loaded vector is on for every request unless it says otherwise: the web app's Sampling drawer has
|
||||
a switch, and the API takes `"experimental_speed_projection": false` in the request body (OpenAI and Anthropic; a
|
||||
config default goes in `"sampling": {"experimental_speed_projection": false}`). Switching drops the conversation
|
||||
cache once, since the model state was computed the other way. Switched off, the output is token-for-token the stock
|
||||
model's.
|
||||
|
||||
**Measured here** (Q2_0, fixed experts, 2,557 teacher-forced tokens of code, a document and a chat): the top-1 token
|
||||
changes at 10% of positions, mean KL from the stock model 0.063 nats (max 4.1), perplexity +15% on code, +2.3% on
|
||||
the document, +0.4% on the chat. Details: `bench/results/2026-09-27-esp/`.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | What to do |
|
||||
@@ -323,14 +371,16 @@ test images.
|
||||
| Python or the build tools could not be installed | Install what it names (links are printed), then run it again. Everything already done is kept. |
|
||||
| `port 8080 is already in use` | Strata is already running (look for its window), or another program uses the port: `START-HERE.bat --port 8081`. |
|
||||
| `cudaHostRegister ... out of memory` in the log | Normal on Windows: the engine pins the experts in per-layer slices instead. Only a problem if the load then fails. |
|
||||
| The first start takes minutes | It is reading 34-43 GB into RAM; the second start is faster while the files are in the OS cache. |
|
||||
| The first start takes minutes | It is reading 34-55 GB into RAM; the second start is faster while the files are in the OS cache. |
|
||||
| The PC freezes for a few minutes at the first start | Normal the first time: the engine loads the experts into RAM, pins part of it for the GPU and sizes the expert cache. Wait; don't close the window. Still frozen after 10 minutes: restart the PC, close other programs, try again, or pick a smaller size. |
|
||||
| `the engine stopped unexpectedly (exit code ...)` | The engine process ended mid-answer - usually out of RAM (Linux ends the biggest program: `sudo dmesg \| grep -i -E 'killed process\|out of memory'`). The next request starts it again by itself. If it repeats: close other programs or pick a smaller size. The server also warns at start when the model's experts leave less than ~6 GB of RAM for everything else. |
|
||||
| Slow output, disk light busy | Not enough free RAM: close other programs, or choose Q2_0 / IQ2_XS. |
|
||||
| `prompt ... exceeds the context` | The request is longer than the context you chose: run setup again with a bigger `--context`. |
|
||||
| Slower than the tables | The monitor plugged into the GPU and other GPU programs take VRAM from the expert cache; RAM running below its rated speed (enable EXPO/XMP in the BIOS) slows the CPU half. |
|
||||
| `this server was started without the vision encoder` | The model was set up for text only: run setup again with `--vision gpu`. |
|
||||
| A picture is refused or `cannot read the image` | The file is not a picture Pillow can open (JPEG, PNG, WebP, GIF, BMP, TIFF, AVIF work). |
|
||||
| Pictures are slow (10-30 s) | The encoder runs on the CPU: run setup again with `--vision gpu` (needs ~1.4 GB of VRAM). |
|
||||
| A request never finishes (older engines, mostly with pictures) | Run `START-HERE.bat` once to get engine 0.1.2 or newer. It keeps a real margin of VRAM free: the log says `... MiB of VRAM free with everything loaded` and warns when it is close to 0. |
|
||||
| A request never finishes: "reading the prompt", GPU "100%" at low power | The GPU ran out of VRAM (engines before 0.1.9 could end with ~30 MiB free at large contexts). Run `START-HERE.bat` once to get engine 0.1.9 or newer; the log then says `... MiB of VRAM free with everything loaded` (a few hundred) and names the `--vram-reserve-mib` to add if it is low. |
|
||||
| Anything else | The engine log is `strata-<model>.log` in this folder. |
|
||||
|
||||
---
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 9.2 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 375 KiB |
@@ -0,0 +1,56 @@
|
||||
// include/strata/kernels/cvec.hpp - a control vector on the residual stream: the engine's side of llama.cpp's
|
||||
// `llama_adapter_cvec` with the `--cvec-mode` / `--cvec-dir` extension of the `experimental-speed-projection`
|
||||
// package (its `02-cvec-projection-mode.patch`, and `01-qwen4exp-cvec-hooks.patch` for WHERE it applies).
|
||||
//
|
||||
// After layer l's FFN write - llama.cpp's `build_cvec(res_hc, il)` right after the layer's second
|
||||
// `build_hc_combine` - every one of the hc residual streams h of every token becomes
|
||||
//
|
||||
// project: h <- h - s_l (h . v_l) v_l v_l = d / |d|, s_l = |d|, d = the scaled direction
|
||||
// add: h <- h + d llama.cpp's stock additive control vector
|
||||
//
|
||||
// for l in [first, last]. `d` is the GGUF's `direction.<src>` times the file's scale (summed over files), with
|
||||
// src = l (per-layer) or one fixed layer (single:L, project mode only), exactly as llama.cpp's loader places it;
|
||||
// layer 0 has no direction. Off unless a vector is loaded (--control-vector-scaled); a loaded one is switched
|
||||
// per request through a device flag, so the captured graphs are the same either way. With the flag off the
|
||||
// result is bit-identical to the engine without a vector: the kernel then only applies the pending residual
|
||||
// write, with the fused read's own arithmetic.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace strata::kernels {
|
||||
|
||||
struct Cvec {
|
||||
const float* dir = nullptr; ///< device, n_layers * n_embd: project = the unit v_l, add = d_l; zero rows elsewhere
|
||||
const float* s = nullptr; ///< device, n_layers: project = s_l, add = 1; 0 = the layer is not steered
|
||||
const int* on = nullptr; ///< device int: 0 = this request runs the stock model
|
||||
int mode = 0; ///< 0 = project, 1 = add
|
||||
int first = 0, last = -1; ///< the steered layers, inclusive
|
||||
int64_t n_embd = 0, hc = 0;
|
||||
bool loaded() const { return dir != nullptr; }
|
||||
/// Layer l's FFN write is followed by the vector (with a real direction there). Decided at load time, never
|
||||
/// per request: it changes where the residual write happens, so the graphs depend on it.
|
||||
bool covers(int64_t l) const { return loaded() && l >= first && l <= last && l < (int64_t) steered.size() && steered[(size_t) l]; }
|
||||
std::vector<bool> steered; ///< host copy of s_l != 0
|
||||
};
|
||||
|
||||
/// The loaded vector (empty until `cvec_upload`).
|
||||
const Cvec& cvec();
|
||||
|
||||
/// Upload a vector built by the loader: `dir` is n_layers * n_embd, `s` n_layers (see `Cvec`). Starts ON.
|
||||
bool cvec_upload(const std::vector<float>& dir, const std::vector<float>& s, int mode, int first, int last,
|
||||
int64_t n_embd, int64_t hc, std::string& err);
|
||||
|
||||
/// The per-request switch. Synchronizes the device when it changes, so call it between requests.
|
||||
void cvec_set_enabled(bool on);
|
||||
bool cvec_enabled();
|
||||
|
||||
/// Layer `layer`'s vector on T tokens' residual stacks (`R + t * r_ld`, hc streams of n_embd). With `write`, the
|
||||
/// pending FFN write `R += bo * 2 sigmoid(inj / hc)` (the fused read's arithmetic) is applied first, for callers
|
||||
/// whose writes are folded into the next layer's read; without it, R must already hold the layer's output.
|
||||
void cvec_apply(float* R, int64_t layer, int64_t T, int64_t r_ld, const float* bo, int64_t bo_ld, const float* inj,
|
||||
int64_t inj_ld, bool write, void* stream);
|
||||
|
||||
} // namespace strata::kernels
|
||||
+269
-15
@@ -72,6 +72,10 @@ class MockEngine:
|
||||
yield t
|
||||
|
||||
|
||||
class EngineDied(RuntimeError):
|
||||
"""The engine process ended in the middle of a request (issue #27: on Linux, the out-of-memory killer)."""
|
||||
|
||||
|
||||
class StrataEngine:
|
||||
"""The resident engine: `strata --serve` reads `GEN <max_new> <ids>` lines and streams `T <id>` lines, then
|
||||
`DONE ...`. Requests are serialized by the service's FIFO, so one pipe is enough.
|
||||
@@ -83,6 +87,8 @@ class StrataEngine:
|
||||
|
||||
def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | None = None,
|
||||
env: dict | None = None):
|
||||
self.spawn = (exe, list(args), cwd, log, env) # to start it again after it died (issue #27)
|
||||
self.log_path = log
|
||||
self.log = open(log, "a", encoding="utf-8") if log else subprocess.DEVNULL
|
||||
self.proc = subprocess.Popen([exe, "--serve", *args], cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=self.log, text=True, encoding="utf-8", bufsize=1, env=env)
|
||||
@@ -114,8 +120,29 @@ class StrataEngine:
|
||||
def _pump(self):
|
||||
for line in self.proc.stdout:
|
||||
self.lines.put(line)
|
||||
self.ended = True # its output closed: it is gone, even before the OS says so
|
||||
self.lines.put(None)
|
||||
|
||||
def alive(self) -> bool:
|
||||
return not getattr(self, "ended", False) and self.proc.poll() is None
|
||||
|
||||
def exit_code(self):
|
||||
try:
|
||||
return self.proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
return None
|
||||
|
||||
def restart(self):
|
||||
"""Start the engine again (the same command) after it died; the new process has its own line queue."""
|
||||
try:
|
||||
self.proc.kill()
|
||||
except OSError:
|
||||
pass
|
||||
info = dict(self.info)
|
||||
self.ended = False
|
||||
self.__init__(*self.spawn)
|
||||
self.info = {**info, **self.info}
|
||||
|
||||
def _parse_done(self, line):
|
||||
f = line.split()
|
||||
self.last = {"generated": int(f[1]), "prompt_tokens": int(f[2]), "prompt_ms": float(f[3]),
|
||||
@@ -160,17 +187,27 @@ class StrataEngine:
|
||||
seed = sampling.get("seed")
|
||||
if isinstance(seed, int) and seed > 0:
|
||||
keys += f" seed={seed}"
|
||||
return keys
|
||||
return keys + StrataEngine.projection_key(sampling)
|
||||
|
||||
@staticmethod
|
||||
def projection_key(sampling: dict) -> str:
|
||||
"""`cvec=0|1`: the experimental-speed-projection control vector for this request, when the engine was
|
||||
started with one (--control-vector-scaled; an engine without one ignores the key). Absent = on."""
|
||||
on = sampling.get("experimental_speed_projection")
|
||||
return f" cvec={int(on)}" if isinstance(on, bool) else ""
|
||||
|
||||
def generate(self, ids, max_new, sampling, cancel, embeddings=None):
|
||||
"""Yields token ids, and None as a heartbeat every 10 s while the engine is quiet (reading a long prompt):
|
||||
the HTTP layer turns it into an SSE comment, which keeps clients' watchdogs calm and notices a client that
|
||||
has gone. A consumer that stops early (or `cancel`) makes the engine STOP, so it does not run to max_new."""
|
||||
self.progress = None
|
||||
head = f"GENI {int(max_new)} {embeddings}" if embeddings else \
|
||||
head = f"GENI {int(max_new)}{self.projection_key(sampling or {})} {embeddings}" if embeddings else \
|
||||
f"GEN {int(max_new)}{self.sampling_keys(sampling or {}) if not embeddings else ''}"
|
||||
self.proc.stdin.write(f"{head} {','.join(str(int(t)) for t in ids)}\n")
|
||||
self.proc.stdin.flush()
|
||||
try:
|
||||
self.proc.stdin.write(f"{head} {','.join(str(int(t)) for t in ids)}\n")
|
||||
self.proc.stdin.flush()
|
||||
except OSError: # the pipe is gone: the engine died (not the client)
|
||||
raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})") from None
|
||||
done = False
|
||||
try:
|
||||
while True:
|
||||
@@ -183,7 +220,7 @@ class StrataEngine:
|
||||
continue
|
||||
if line is None:
|
||||
done = True
|
||||
raise RuntimeError("the engine process ended")
|
||||
raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})")
|
||||
if line.startswith("T "):
|
||||
if cancel.is_set():
|
||||
return
|
||||
@@ -375,9 +412,13 @@ class Detokenizer:
|
||||
|
||||
class Service:
|
||||
def __init__(self, engine: Engine, tokenizer, template: ChatTemplate, model_name: str = "qwen3.8-flash-next",
|
||||
vision: Vision | None = None, sampling_defaults: dict | None = None):
|
||||
vision: Vision | None = None, sampling_defaults: dict | None = None,
|
||||
fit_max_tokens: bool = False):
|
||||
self.engine, self.tok, self.template, self.model, self.vision = engine, tokenizer, template, model_name, vision
|
||||
self.fit_max_tokens = fit_max_tokens # --fit-max-tokens: clamp the output cap instead of 400
|
||||
self.sampling_defaults = dict(sampling_defaults or {}) # the run config's `sampling` block
|
||||
self.shared = {} # the web app's Chat settings for every client (POST /settings)
|
||||
self.shared_path = None # where they are kept between starts (next to the config)
|
||||
self.fifo = threading.Lock()
|
||||
self.embeddings = threading.local() # the current request's image embeddings file (GENI)
|
||||
self.api_key = "" # when set, /v1/* needs it (Bearer or x-api-key)
|
||||
@@ -387,6 +428,41 @@ class Service:
|
||||
self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
|
||||
tokenizer.encode("<|endoftext|>", parse_special=True))
|
||||
|
||||
def set_shared(self, defaults) -> dict:
|
||||
"""The Chat settings every client gets for what it leaves out; {} / None = clients use their own again."""
|
||||
self.shared = clean_shared_defaults(defaults)
|
||||
if self.shared_path:
|
||||
try:
|
||||
if self.shared:
|
||||
Path(self.shared_path).write_text(json.dumps(self.shared, indent=1), encoding="utf-8")
|
||||
else:
|
||||
Path(self.shared_path).unlink(missing_ok=True)
|
||||
except OSError as e:
|
||||
print(f"[strata] could not save the shared settings: {e}", flush=True)
|
||||
return self.shared
|
||||
|
||||
def with_shared(self, req: dict, api: str) -> dict:
|
||||
"""The request with the shared thinking level and max tokens filled in where it has none of its own."""
|
||||
s = self.shared
|
||||
if not s:
|
||||
return req
|
||||
req = dict(req)
|
||||
if "max_tokens" in s and not req.get("max_tokens") and not req.get("max_completion_tokens"):
|
||||
req["max_tokens"] = s["max_tokens"]
|
||||
effort = s.get("reasoning_effort")
|
||||
if effort:
|
||||
if api == "openai":
|
||||
ctk = req.get("chat_template_kwargs") if isinstance(req.get("chat_template_kwargs"), dict) else {}
|
||||
if not req.get("reasoning_effort") and not req.get("reasoning") and \
|
||||
"enable_thinking" not in ctk and "reasoning_effort" not in ctk:
|
||||
req["reasoning_effort"] = effort
|
||||
elif not req.get("thinking") and not req.get("output_config"):
|
||||
if effort == "none":
|
||||
req["thinking"] = {"type": "disabled"}
|
||||
else:
|
||||
req["output_config"] = {"effort": effort}
|
||||
return req
|
||||
|
||||
def start_telemetry(self):
|
||||
"""The hardware sampler behind GET /metrics (serve/telemetry.py), recording this server's tok/s too."""
|
||||
if getattr(self, "telemetry", None) is None:
|
||||
@@ -462,14 +538,16 @@ class Service:
|
||||
f.write(path.read_bytes())
|
||||
self.embeddings.path = combined
|
||||
room = self.engine.max_context - CTX_SLACK - len(ids)
|
||||
if max_new is None or max_new <= 0:
|
||||
if max_new is None or max_new <= 0 or (self.fit_max_tokens and room < 1):
|
||||
if room < 1:
|
||||
raise ValueError(f"prompt ({len(ids)} tokens) leaves no room to answer in the context "
|
||||
f"({self.engine.max_context}); requests are never truncated")
|
||||
max_new = room
|
||||
elif max_new > room:
|
||||
raise ValueError(f"prompt ({len(ids)} tokens) + max tokens ({max_new}) exceeds the context "
|
||||
f"({self.engine.max_context}); requests are never truncated")
|
||||
if not self.fit_max_tokens:
|
||||
raise ValueError(f"prompt ({len(ids)} tokens) + max tokens ({max_new}) exceeds the context "
|
||||
f"({self.engine.max_context}); requests are never truncated")
|
||||
max_new = max(1, room) # --fit-max-tokens: a shorter completion beats a 400
|
||||
return ids, kwargs.get("enable_thinking", True) is not False, max_new
|
||||
|
||||
def _note(self, n, evs):
|
||||
@@ -507,9 +585,10 @@ class Service:
|
||||
|
||||
def run(self, ids, thinking, tools, max_new, sampling, cancel) -> Iterator[tuple[str, object]]:
|
||||
"""Yields ("event", Event) as text arrives, then ("done", {"finish": .., "completion_tokens": ..})."""
|
||||
if self.sampling_defaults: # config defaults; the request's own fields win (explicit 0 stays greedy)
|
||||
defaults = {**self.sampling_defaults, **self.shared} # the config's, then the Chat settings shared with apps
|
||||
if defaults: # the request's own fields win (explicit 0 stays greedy)
|
||||
req_values = {k: v for k, v in (sampling or {}).items() if v is not None}
|
||||
sampling = {**self.sampling_defaults, **req_values}
|
||||
sampling = {**defaults, **req_values}
|
||||
parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True)
|
||||
detok, n, finish = Detokenizer(self.tok), 0, "length"
|
||||
raw_ids = [] # every generated id (STRATA_DEBUG: dump raw model text)
|
||||
@@ -520,6 +599,14 @@ class Service:
|
||||
with self.fifo:
|
||||
with self.status_lock:
|
||||
self.status["queued"] -= 1
|
||||
if hasattr(self.engine, "alive") and not self.engine.alive():
|
||||
# issue #27: it died in an earlier request - start it again instead of failing every request
|
||||
code = self.engine.exit_code() if hasattr(self.engine, "exit_code") else None
|
||||
print(f"[strata] the engine had stopped (exit code {code}); starting it again "
|
||||
"(a minute or two) ...", flush=True)
|
||||
self.engine.restart()
|
||||
print("[strata] the engine is running again", flush=True)
|
||||
with self.status_lock:
|
||||
self.status.update(busy=True, phase="reading the prompt", prompt_tokens=len(ids), generated=0,
|
||||
started=time.time(), first_token=None, tool=None, tail="", max_tokens=max_new)
|
||||
last_print = time.time()
|
||||
@@ -544,6 +631,14 @@ class Service:
|
||||
yield "event", ev
|
||||
if cancel.is_set():
|
||||
finish = "cancel"
|
||||
except EngineDied as e:
|
||||
finish = "error"
|
||||
print(f"[strata] {e}. The usual cause is running out of RAM: Linux then ends the biggest program "
|
||||
"(check: sudo dmesg | grep -i -E 'killed process|out of memory'); Windows slows down instead. "
|
||||
"Close other programs or use a smaller model (Q2_0 / IQ2_XS). The next request starts the engine "
|
||||
f"again.{' Its log: ' + self.engine.log_path if getattr(self.engine, 'log_path', None) else ''}",
|
||||
flush=True)
|
||||
raise
|
||||
finally:
|
||||
gen.close() # STOP+drain to THIS request's DONE while still holding the
|
||||
# fifo, so a stop-token break can't leave the shared engine
|
||||
@@ -558,7 +653,10 @@ class Service:
|
||||
if self.status.get("busy"):
|
||||
last = dict(getattr(self.engine, "last", {}) or {})
|
||||
started = self.status.get("started", time.time())
|
||||
loaded = str((getattr(self.engine, "info", {}) or {}).get("cvec", 0)) not in ("0", "", "None")
|
||||
self.history.append({
|
||||
"projection": (sampling or {}).get("experimental_speed_projection") is not False
|
||||
if loaded else None,
|
||||
"time": started, "duration_s": round(time.time() - started, 1), "finish": finish,
|
||||
"prompt_tokens": len(ids), "reused": last.get("reused"), "output_tokens": n,
|
||||
"prompt_ms": last.get("prompt_ms"), "decode_ms": last.get("decode_ms"),
|
||||
@@ -817,6 +915,10 @@ def make_handler(svc: Service):
|
||||
if self._authorized():
|
||||
self._json(200, svc.metrics())
|
||||
return
|
||||
if path == "/settings":
|
||||
if self._authorized():
|
||||
self._json(200, {"shared": bool(svc.shared), "defaults": svc.shared})
|
||||
return
|
||||
if path == "":
|
||||
body = (ROOT / "serve" / "web" / "index.html").read_bytes()
|
||||
self.send_response(200)
|
||||
@@ -847,6 +949,9 @@ def make_handler(svc: Service):
|
||||
def do_POST(self):
|
||||
if not self._authorized():
|
||||
return
|
||||
if self.path.rstrip("/") == "/settings":
|
||||
self._settings()
|
||||
return
|
||||
try:
|
||||
req = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}")
|
||||
if self.path.rstrip("/") == "/v1/chat/completions":
|
||||
@@ -857,6 +962,30 @@ def make_handler(svc: Service):
|
||||
self._json(404, {"error": {"message": "not found"}})
|
||||
except ValueError as e:
|
||||
self._json(400, {"error": {"type": "invalid_request_error", "message": str(e)}})
|
||||
except EngineDied as e: # before the answer started (not streamed)
|
||||
self._json(503, {"error": {"type": "server_error", "message": f"{e}; the next request restarts it"}})
|
||||
|
||||
def _settings(self):
|
||||
# They change what every client gets, so only the app's own page may set them: JSON only (a form or a
|
||||
# "simple" cross-site request can't send it without a CORS preflight, which this server never grants),
|
||||
# and no foreign Origin
|
||||
body = self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
||||
if not self.headers.get("Content-Type", "").startswith("application/json"):
|
||||
self._json(415, {"error": {"message": "send application/json"}})
|
||||
return
|
||||
origin = self.headers.get("Origin")
|
||||
if origin and origin.split("://", 1)[-1] != self.headers.get("Host", ""):
|
||||
self._json(403, {"error": {"message": "settings can only be changed from Strata's own page"}})
|
||||
return
|
||||
try:
|
||||
req = json.loads(body or b"{}")
|
||||
shared = svc.set_shared(req.get("defaults") if isinstance(req, dict) else None)
|
||||
except ValueError as e:
|
||||
self._json(400, {"error": {"type": "invalid_request_error", "message": str(e)}})
|
||||
return
|
||||
print("[strata] other apps now use the Chat settings: " + ", ".join(f"{k}={v}" for k, v in shared.items())
|
||||
if shared else "[strata] other apps use their own settings again", flush=True)
|
||||
self._json(200, {"shared": bool(shared), "defaults": shared})
|
||||
|
||||
def _sse(self):
|
||||
self.send_response(200)
|
||||
@@ -865,6 +994,7 @@ def make_handler(svc: Service):
|
||||
self.end_headers()
|
||||
|
||||
def _openai(self, req):
|
||||
req = svc.with_shared(req, "openai")
|
||||
messages, tools, kw = openai_to_messages(req)
|
||||
max_new = int(req.get("max_completion_tokens") or req.get("max_tokens") or 0) # 0/-1: the rest of the context
|
||||
ids, thinking, max_new = svc.prepare(messages, tools, kw, max_new)
|
||||
@@ -885,8 +1015,12 @@ def make_handler(svc: Service):
|
||||
except OSError:
|
||||
cancel.set() # client went away: stop the engine
|
||||
chunks.close()
|
||||
except EngineDied as e: # mid-stream: say so, then end the stream properly
|
||||
err = {"error": {"type": "server_error", "message": f"{e}; the next request restarts it"}}
|
||||
self.wfile.write(b"data: " + json.dumps(err).encode() + b"\n\ndata: [DONE]\n\n")
|
||||
|
||||
def _anthropic(self, req):
|
||||
req = svc.with_shared(req, "anthropic")
|
||||
messages, tools, kw = anthropic_to_messages(req)
|
||||
max_new = int(req.get("max_tokens") or 0) # 0/-1: the rest of the context
|
||||
ids, thinking, max_new = svc.prepare(messages, tools, kw, max_new)
|
||||
@@ -908,6 +1042,9 @@ def make_handler(svc: Service):
|
||||
except OSError:
|
||||
cancel.set()
|
||||
events.close()
|
||||
except EngineDied as e: # mid-stream: Anthropic's error event
|
||||
err = {"type": "error", "error": {"type": "api_error", "message": f"{e}; the next request restarts it"}}
|
||||
self.wfile.write(b"event: error\ndata: " + json.dumps(err).encode() + b"\n\n")
|
||||
|
||||
return Handler
|
||||
|
||||
@@ -918,6 +1055,45 @@ class Server(ThreadingHTTPServer):
|
||||
allow_reuse_address = os.name != "nt"
|
||||
|
||||
|
||||
def warn_tight_ram(arena_mib) -> None:
|
||||
"""The model's experts live in RAM (INFO arena_mib, engine 0.1.10+). With less than ~6 GB left beside them for the
|
||||
system, the engine and this server, Linux ends the engine mid-answer when memory runs out (issue #27) and Windows
|
||||
pages to disk; say so at start instead of after a lost answer."""
|
||||
if not isinstance(arena_mib, int) or arena_mib <= 0:
|
||||
return
|
||||
try:
|
||||
import psutil
|
||||
total = psutil.virtual_memory().total
|
||||
except Exception: # noqa: BLE001 - psutil is optional here
|
||||
return
|
||||
left = total / 2**30 - arena_mib / 1024
|
||||
if left < 6:
|
||||
print(f"[strata] WARNING: RAM is tight - the model's experts take {arena_mib / 1024:.1f} GB of this PC's "
|
||||
f"{total / 2**30:.0f} GB, leaving {left:.1f} GB for everything else. "
|
||||
+ ("Linux may stop the engine in the middle of an answer. " if os.name != "nt" else
|
||||
"Windows will slow down (paging to disk). ")
|
||||
+ "Close other programs, or run START-HERE --setup and pick a smaller size (Q2_0 / IQ2_XS).", flush=True)
|
||||
|
||||
|
||||
def lan_addresses() -> list[str]:
|
||||
"""This PC's IPv4 addresses on its networks (what another device types in), without loopback/link-local."""
|
||||
import socket
|
||||
first, ips = None, set()
|
||||
try: # the address of the default route; sends nothing (UDP)
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
|
||||
s.connect(("10.255.255.255", 1))
|
||||
first = s.getsockname()[0]
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
for info in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET):
|
||||
ips.add(info[4][0])
|
||||
except OSError:
|
||||
pass
|
||||
ok = lambda ip: ip and not ip.startswith(("127.", "169.254.", "0."))
|
||||
return ([first] if ok(first) else []) + sorted(ip for ip in ips if ok(ip) and ip != first)
|
||||
|
||||
|
||||
def serve(svc: Service, host="127.0.0.1", port=8095) -> ThreadingHTTPServer:
|
||||
svc.start_telemetry()
|
||||
httpd = Server((host, port), make_handler(svc))
|
||||
@@ -925,6 +1101,46 @@ def serve(svc: Service, host="127.0.0.1", port=8095) -> ThreadingHTTPServer:
|
||||
return httpd
|
||||
|
||||
|
||||
SHARED_KEYS = ("reasoning_effort", "temperature", "top_p", "top_k", "seed", "max_tokens", "experimental_speed_projection")
|
||||
|
||||
|
||||
def clean_shared_defaults(d) -> dict:
|
||||
"""The Chat settings other apps get (POST /settings): only known keys, each checked; ValueError names a bad one."""
|
||||
if d is None:
|
||||
return {}
|
||||
if not isinstance(d, dict):
|
||||
raise ValueError("defaults must be an object")
|
||||
out = {}
|
||||
for key, value in d.items():
|
||||
if value is None or value == "":
|
||||
continue
|
||||
number = isinstance(value, (int, float)) and not isinstance(value, bool)
|
||||
if key == "reasoning_effort":
|
||||
if value not in ("none", "low", "medium", "high"):
|
||||
raise ValueError("reasoning_effort: none, low, medium or high")
|
||||
elif key == "temperature":
|
||||
if not number or not 0 <= value <= 2:
|
||||
raise ValueError("temperature: 0..2")
|
||||
elif key == "top_p":
|
||||
if not number or not 0 < value <= 1:
|
||||
raise ValueError("top_p: 0 < top_p <= 1")
|
||||
elif key == "top_k":
|
||||
if not number or value != int(value) or not 1 <= value <= 64:
|
||||
raise ValueError("top_k: an integer 1..64")
|
||||
value = int(value)
|
||||
elif key in ("seed", "max_tokens"):
|
||||
if not number or value != int(value) or value <= 0:
|
||||
raise ValueError(f"{key}: a positive integer")
|
||||
value = int(value)
|
||||
elif key == "experimental_speed_projection":
|
||||
if not isinstance(value, bool):
|
||||
raise ValueError("experimental_speed_projection: true or false")
|
||||
else:
|
||||
raise ValueError(f"unknown setting {key!r}")
|
||||
out[key] = float(value) if key in ("temperature", "top_p") else value
|
||||
return out
|
||||
|
||||
|
||||
def sampling_defaults_from_config(cfg: dict) -> dict:
|
||||
"""The run config's optional `sampling` block: defaults for the sampling fields a request leaves out, so
|
||||
a plain client gets configured sampling instead of greedy. Supported: temperature, top_p, top_k, min_p,
|
||||
@@ -973,6 +1189,11 @@ def sampling_defaults_from_config(cfg: dict) -> dict:
|
||||
if not number or value != int(value) or value <= 0:
|
||||
raise SystemExit(f"[strata] config sampling.seed={value!r}: expected a positive integer")
|
||||
out[key] = int(value)
|
||||
elif key == "experimental_speed_projection":
|
||||
if not isinstance(value, bool):
|
||||
raise SystemExit(f"[strata] config sampling.experimental_speed_projection={value!r}: expected true or "
|
||||
"false (the default for requests that leave it out, when the engine has the vector)")
|
||||
out[key] = value
|
||||
else:
|
||||
print(f"[strata] config sampling.{key}={value!r}: unknown key, ignored", flush=True)
|
||||
return out
|
||||
@@ -983,16 +1204,22 @@ def main() -> int:
|
||||
ap.add_argument("--engine", choices=["mock", "strata"], default="mock")
|
||||
ap.add_argument("--config", help="strata engine config (JSON: exe, args, cwd, tokenizer, model_name), "
|
||||
"written by setup.py")
|
||||
ap.add_argument("--host", default="127.0.0.1")
|
||||
ap.add_argument("--host", default=None,
|
||||
help="the address to listen on: 127.0.0.1 = this PC only (the default), 0.0.0.0 = also other devices "
|
||||
"on your network (set an API key); also \"host\" in the config")
|
||||
ap.add_argument("--script", default="Thinking about it.</think>\n\nHello from the mock engine.")
|
||||
ap.add_argument("--port", type=int, default=8095)
|
||||
ap.add_argument("--tokenizer", default=str(ROOT / "pack/full/tokenizer"),
|
||||
help="pack tokenizer directory (falls back to a byte tokenizer if absent)")
|
||||
ap.add_argument("--open", action="store_true", help="open the local page in the browser once the model is ready")
|
||||
ap.add_argument("--fit-max-tokens", action="store_true",
|
||||
help="clamp max_tokens to the remaining context instead of rejecting the request "
|
||||
"(default: reject with 400, like llama.cpp; also \"fit_max_tokens\": true in the config)")
|
||||
ap.add_argument("--api-key", default=os.environ.get("STRATA_API_KEY", ""),
|
||||
help="require this key on /v1/* (Authorization: Bearer ... or x-api-key); also $STRATA_API_KEY")
|
||||
a = ap.parse_args()
|
||||
cfg = json.loads(Path(a.config).read_text(encoding="utf-8-sig")) if a.config else {} # Notepad adds a BOM
|
||||
a.host = a.host or cfg.get("host") or "127.0.0.1" # issue #26: the run scripts pass no --host, the config can
|
||||
try: # before the minutes of loading: is the port free?
|
||||
Server((a.host, a.port), BaseHTTPRequestHandler).server_close()
|
||||
except OSError:
|
||||
@@ -1028,19 +1255,46 @@ def main() -> int:
|
||||
env=env)
|
||||
print("loading the model (the first start takes a minute or two) ...", flush=True)
|
||||
engine = StrataEngine(cfg["exe"], cfg["args"], cwd=cfg.get("cwd"), log=cfg.get("log"), env=env)
|
||||
warn_tight_ram(engine.info.get("arena_mib"))
|
||||
else:
|
||||
engine, vision, sampling_defaults = MockEngine(tok, a.script), None, {}
|
||||
# the model's own chat template (exported with its tokenizer), else the original model's
|
||||
tpl = tpath / "chat_template.jinja"
|
||||
svc = Service(engine, tok, ChatTemplate(tpl if tpl.exists() else ROOT / "serve/chat_template.jinja"),
|
||||
model_name=cfg.get("model_name", "qwen3.8-flash-next"), vision=vision,
|
||||
sampling_defaults=sampling_defaults)
|
||||
sampling_defaults=sampling_defaults,
|
||||
fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)
|
||||
svc.api_key = a.api_key or cfg.get("api_key", "")
|
||||
if a.config: # the Chat settings shared with other apps, from last time
|
||||
svc.shared_path = str(Path(a.config).with_suffix("")) + ".shared-settings.json"
|
||||
try:
|
||||
svc.shared = clean_shared_defaults(json.loads(Path(svc.shared_path).read_text(encoding="utf-8")))
|
||||
if svc.shared:
|
||||
print("[strata] other apps use the Chat settings: " +
|
||||
", ".join(f"{k}={v}" for k, v in svc.shared.items()), flush=True)
|
||||
except (OSError, ValueError):
|
||||
svc.shared = {}
|
||||
httpd = serve(svc, host=a.host, port=a.port)
|
||||
print(f"ready: http://{a.host}:{a.port}/v1 (OpenAI: /v1/chat/completions, Anthropic: /v1/messages, "
|
||||
here = "127.0.0.1" if a.host in ("0.0.0.0", "", "::") else a.host
|
||||
print(f"ready: http://{here}:{a.port}/v1 (OpenAI: /v1/chat/completions, Anthropic: /v1/messages, "
|
||||
f"context {engine.max_context} tokens{', images on' if vision else ''}"
|
||||
f"{', API key required' if svc.api_key else ''})", flush=True)
|
||||
print(f" open http://{a.host}:{a.port}/ in a browser to chat; close this window to stop the model", flush=True)
|
||||
print(f" open http://{here}:{a.port}/ in a browser to chat; close this window to stop the model", flush=True)
|
||||
if a.host not in ("127.0.0.1", "localhost", "::1"):
|
||||
# issue #26: reachable from other devices - say at which address, and what can still block it
|
||||
ips = lan_addresses()
|
||||
for ip in ips:
|
||||
print(f" from other devices: http://{ip}:{a.port}/ (API: http://{ip}:{a.port}/v1)", flush=True)
|
||||
if not ips:
|
||||
print(" from other devices: http://<this PC's IP address>:" + str(a.port) + "/", flush=True)
|
||||
if not svc.api_key:
|
||||
print(" WARNING: no API key - anyone on your network can use this model. Add \"api_key\": \"...\" "
|
||||
"to the config (clients send it as their API key; the web page asks for it)", flush=True)
|
||||
if os.name == "nt":
|
||||
print(" nothing arrives? Windows Firewall blocks it until allowed: accept its prompt for Python, or run "
|
||||
"in an admin PowerShell:\n New-NetFirewallRule -DisplayName \"Strata " + str(a.port) + "\" "
|
||||
"-Direction Inbound -Protocol TCP -LocalPort " + str(a.port) + " -Action Allow -Profile Private\n"
|
||||
" (and set this network to Private in Windows' network settings)", flush=True)
|
||||
if a.open:
|
||||
import webbrowser
|
||||
webbrowser.open(f"http://{'127.0.0.1' if a.host in ('0.0.0.0', '') else a.host}:{a.port}/")
|
||||
|
||||
+191
-1
@@ -14,7 +14,7 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from serve.frontend import ChatTemplate # noqa: E402
|
||||
from serve.server import CTX_SLACK, ByteTokenizer, MockEngine, Service, serve # noqa: E402
|
||||
from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, MockEngine, Service, serve # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CTX = 4096
|
||||
@@ -126,6 +126,196 @@ class MaxTokens(unittest.TestCase):
|
||||
del os.environ["STRATA_DEBUG"]
|
||||
|
||||
|
||||
class FitMaxTokens(unittest.TestCase):
|
||||
"""PR #24: --fit-max-tokens clamps an explicit budget that overshoots the context instead of a 400."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
tok = ByteTokenizer()
|
||||
cls.engine = RecordingEngine(tok, "</think>\n\n" + ANSWER, max_context=CTX)
|
||||
cls.svc = Service(cls.engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"), fit_max_tokens=True)
|
||||
cls.httpd = serve(cls.svc, port=0)
|
||||
cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}"
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.httpd.shutdown()
|
||||
cls.httpd.server_close()
|
||||
|
||||
post = MaxTokens.post
|
||||
call = MaxTokens.call
|
||||
|
||||
def test_overshoot_is_clamped_to_the_room(self):
|
||||
for api in ("openai", "anthropic"):
|
||||
with self.subTest(api=api):
|
||||
s, b, pt, ct = self.call(api, max_tokens=CTX)
|
||||
self.assertEqual(s, 200, b)
|
||||
self.assertEqual(self.engine.last_max_new, CTX - CTX_SLACK - pt)
|
||||
|
||||
def test_a_budget_that_fits_is_unchanged(self):
|
||||
s, b, _, ct = self.call("openai", max_tokens=50)
|
||||
self.assertEqual(s, 200, b)
|
||||
self.assertEqual(self.engine.last_max_new, 50)
|
||||
|
||||
def test_no_room_is_still_a_400(self):
|
||||
_, _, pt0, _ = self.call("openai", max_tokens=1)
|
||||
overhead = pt0 - len("hi")
|
||||
s, b, _, _ = self.call("openai", text="y" * (CTX - CTX_SLACK - overhead), max_tokens=100)
|
||||
self.assertEqual(s, 400, b)
|
||||
self.assertIn("no room to answer", b["error"]["message"])
|
||||
|
||||
|
||||
class DyingEngine(MockEngine):
|
||||
"""Issue #27: an engine that dies after a few tokens of its first answer, and comes back when restarted."""
|
||||
|
||||
def __init__(self, tok, script, max_context):
|
||||
super().__init__(tok, script, max_context=max_context)
|
||||
self.dead, self.restarts, self.die_after = False, 0, 5
|
||||
|
||||
def alive(self):
|
||||
return not self.dead
|
||||
|
||||
def restart(self):
|
||||
self.dead, self.die_after = False, None
|
||||
self.restarts += 1
|
||||
|
||||
def generate(self, ids, max_new, sampling, cancel, embeddings=None):
|
||||
for i, t in enumerate(super().generate(ids, max_new, sampling, cancel, embeddings)):
|
||||
if self.die_after is not None and i == self.die_after:
|
||||
self.dead = True
|
||||
raise EngineDied("the engine stopped unexpectedly (exit code -9)")
|
||||
yield t
|
||||
|
||||
|
||||
class EngineDeath(unittest.TestCase):
|
||||
"""Issue #27: a dead engine is an error (not "length"), and the next request starts it again."""
|
||||
|
||||
def test_error_then_restart(self):
|
||||
tok = ByteTokenizer()
|
||||
eng = DyingEngine(tok, "</think>\n\n" + ANSWER, max_context=CTX)
|
||||
svc = Service(eng, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
|
||||
httpd = serve(svc, port=0)
|
||||
base = f"http://127.0.0.1:{httpd.server_address[1]}"
|
||||
try:
|
||||
def post(body):
|
||||
req = urllib.request.Request(base + "/v1/chat/completions", data=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, r.read().decode()
|
||||
except urllib.error.HTTPError as e:
|
||||
with e:
|
||||
return e.code, e.read().decode()
|
||||
msgs = [{"role": "user", "content": "hi"}]
|
||||
code, text = post({"model": "m", "messages": msgs, "max_tokens": 50, "stream": True})
|
||||
self.assertEqual(code, 200)
|
||||
self.assertIn('"error"', text)
|
||||
self.assertIn("stopped unexpectedly", text)
|
||||
self.assertTrue(text.rstrip().endswith("data: [DONE]"))
|
||||
self.assertEqual(svc.metrics()["requests"][0]["finish"], "error")
|
||||
code, text = post({"model": "m", "messages": msgs, "max_tokens": 50})
|
||||
self.assertEqual(code, 200, text)
|
||||
self.assertEqual(eng.restarts, 1)
|
||||
self.assertEqual(json.loads(text)["usage"]["completion_tokens"], 50)
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
|
||||
|
||||
class SharedSettings(unittest.TestCase):
|
||||
"""The web app's "Use for other apps too": POST /settings makes its Chat settings every client's defaults."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
import tempfile
|
||||
|
||||
class Sampled(RecordingEngine):
|
||||
def generate(self, ids, max_new, sampling, cancel, embeddings=None):
|
||||
self.last_sampling = dict(sampling or {})
|
||||
yield from super().generate(ids, max_new, sampling, cancel, embeddings)
|
||||
|
||||
tok = ByteTokenizer()
|
||||
cls.engine = Sampled(tok, "</think>\n\nhello", max_context=CTX)
|
||||
cls.svc = Service(cls.engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
cls.svc.shared_path = os.path.join(cls.tmp.name, "strata-x.shared-settings.json")
|
||||
cls.httpd = serve(cls.svc, port=0)
|
||||
cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}"
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.httpd.shutdown()
|
||||
cls.httpd.server_close()
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def req(self, path, body, headers=None, raw=None):
|
||||
h = {"Content-Type": "application/json", **(headers or {})}
|
||||
r = urllib.request.Request(self.base + path, data=raw if raw is not None else json.dumps(body).encode(), headers=h)
|
||||
try:
|
||||
with urllib.request.urlopen(r, timeout=30) as resp:
|
||||
return resp.status, json.loads(resp.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
with e:
|
||||
return e.code, json.loads(e.read())
|
||||
|
||||
def chat(self, **extra):
|
||||
return self.req("/v1/chat/completions", {"model": "m", "messages": [{"role": "user", "content": "hi"}], **extra})
|
||||
|
||||
def tearDown(self):
|
||||
self.svc.set_shared(None)
|
||||
|
||||
def test_other_apps_get_the_chat_settings(self):
|
||||
d = {"temperature": 0.3, "top_p": 0.9, "top_k": 10, "seed": 7, "max_tokens": 77,
|
||||
"reasoning_effort": "low", "experimental_speed_projection": False}
|
||||
code, b = self.req("/settings", {"defaults": d})
|
||||
self.assertEqual(code, 200, b)
|
||||
self.assertTrue(b["shared"])
|
||||
self.assertTrue(os.path.exists(self.svc.shared_path))
|
||||
code, _ = self.chat() # a client that sets nothing
|
||||
self.assertEqual(code, 200)
|
||||
got = self.engine.last_sampling
|
||||
for k in ("temperature", "top_p", "top_k", "seed", "experimental_speed_projection"):
|
||||
self.assertEqual(got[k], d[k], k)
|
||||
self.assertEqual(self.engine.last_max_new, 77)
|
||||
code, _ = self.chat(temperature=0.9, max_tokens=5) # its own values win
|
||||
self.assertEqual(self.engine.last_sampling["temperature"], 0.9)
|
||||
self.assertEqual(self.engine.last_max_new, 5)
|
||||
r = self.svc.with_shared({"messages": []}, "openai")
|
||||
self.assertEqual(r["reasoning_effort"], "low")
|
||||
self.assertEqual(self.svc.with_shared({"reasoning_effort": "high"}, "openai")["reasoning_effort"], "high")
|
||||
self.assertEqual(self.svc.with_shared({}, "anthropic")["output_config"], {"effort": "low"})
|
||||
|
||||
def test_off_again(self):
|
||||
self.req("/settings", {"defaults": {"temperature": 0.3}})
|
||||
code, b = self.req("/settings", {"defaults": None})
|
||||
self.assertEqual((code, b["shared"]), (200, False))
|
||||
self.assertFalse(os.path.exists(self.svc.shared_path))
|
||||
self.chat()
|
||||
self.assertNotIn("temperature", self.engine.last_sampling)
|
||||
|
||||
def test_only_strata_s_own_page_may_set_them(self):
|
||||
code, _ = self.req("/settings", None, {"Content-Type": "text/plain"}, raw=b'{"defaults": {"temperature": 1}}')
|
||||
self.assertEqual(code, 415)
|
||||
code, _ = self.req("/settings", {"defaults": {"temperature": 1}}, {"Origin": "http://evil.example"})
|
||||
self.assertEqual(code, 403)
|
||||
code, b = self.req("/settings", {"defaults": {"temperature": 9}})
|
||||
self.assertEqual(code, 400)
|
||||
self.assertIn("temperature", b["error"]["message"])
|
||||
self.assertEqual(self.svc.shared, {})
|
||||
host = self.base.split("://", 1)[1]
|
||||
code, _ = self.req("/settings", {"defaults": {"temperature": 1}}, {"Origin": "http://" + host})
|
||||
self.assertEqual(code, 200)
|
||||
|
||||
def test_they_need_the_key_when_one_is_set(self):
|
||||
self.svc.api_key = "secret"
|
||||
try:
|
||||
self.assertEqual(self.req("/settings", {"defaults": {"temperature": 1}})[0], 401)
|
||||
self.assertEqual(self.req("/settings", {"defaults": {"temperature": 1}},
|
||||
{"Authorization": "Bearer secret"})[0], 200)
|
||||
finally:
|
||||
self.svc.api_key = ""
|
||||
|
||||
|
||||
class WebApp(unittest.TestCase):
|
||||
"""The web app (PR #22's dashboard idea, rebuilt): its page and files, and GET /metrics."""
|
||||
|
||||
|
||||
@@ -107,6 +107,7 @@ main { flex: 1; min-height: 0; display: flex; }
|
||||
.facts .st-btn--icon .st-icon { width: 16px; height: 16px; }
|
||||
.toggle-row { display: flex; align-items: center; justify-content: space-between; gap: var(--st-s-4); font-size: var(--st-fs-sm);
|
||||
font-weight: var(--st-fw-medium); color: var(--st-ink-soft); }
|
||||
.toggle-row .st-toggle { flex: none; } /* a long label must not squeeze the switch */
|
||||
|
||||
/* ---------------------------------------------------------------- drawer */
|
||||
.st-drawer { z-index: 40; }
|
||||
|
||||
+68
-6
@@ -270,7 +270,8 @@ function renderMonitor(live, hw, st, eng, h, last, requests) {
|
||||
body.innerHTML = requests.slice(0, 12).map((r) => {
|
||||
const [cls, text] = badge[r.finish] || ["", r.finish || "–"];
|
||||
const t = new Date(r.time * 1000).toLocaleTimeString([], {hour: "2-digit", minute: "2-digit", second: "2-digit"});
|
||||
return `<tr><td>${esc(t)}</td><td><span class="st-badge ${cls}">${esc(text)}</span></td><td class="num">${fmt(r.prompt_tokens)}</td>
|
||||
const proj = r.projection == null ? "" : ` <span class="st-badge${r.projection ? " st-badge--reading" : ""}" title="experimental speed projection ${r.projection ? "on" : "off"}">${r.projection ? "ESP" : "stock"}</span>`;
|
||||
return `<tr><td>${esc(t)}</td><td><span class="st-badge ${cls}">${esc(text)}</span>${proj}</td><td class="num">${fmt(r.prompt_tokens)}</td>
|
||||
<td class="num">${fmt(r.reused)}</td><td class="num">${fmt(r.output_tokens)}</td><td class="num">${fmt(r.decode_tok_s, 1)}</td>
|
||||
<td class="num">${fmt(r.duration_s, 1)} s</td></tr>`;
|
||||
}).join("");
|
||||
@@ -281,6 +282,15 @@ function facts(el, rows) {
|
||||
el.innerHTML = rows.filter((r) => r[1] != null && r[1] !== "").map(([k, v, copy]) =>
|
||||
`<dt>${esc(k)}</dt><dd>${copy ? `<code>${esc(v)}</code><button class="st-btn st-btn--icon" data-copy="${esc(v)}" aria-label="Copy">${icon("copy")}</button>` : esc(v)}</dd>`).join("");
|
||||
}
|
||||
// INFO cvec=project:4-44[:singleL] | add:A-B | 0
|
||||
function projectionText(c) {
|
||||
if (!c || c === "0" || c === 0) return null;
|
||||
const [mode, range, single] = String(c).split(":");
|
||||
const [a, b] = (range || "").split("-");
|
||||
return `${mode === "project" ? "Projection" : "Additive"} control vector on layers ${a}–${b}` +
|
||||
`${single ? ` (layer ${single.replace("single", "")}'s direction)` : ""}. Per chat in Sampling. Its package ` +
|
||||
"describes the vector as a refusal-direction projection; measure the speed yourself";
|
||||
}
|
||||
function renderAbout(eng, hw, st) {
|
||||
const kv = {int8: "8-bit", q4_0: "4-bit (Hadamard-rotated)", fp16: "16-bit"}[eng.kv] || eng.kv;
|
||||
facts($("facts-engine"), [
|
||||
@@ -291,6 +301,7 @@ function renderAbout(eng, hw, st) {
|
||||
["Experts in VRAM", eng.expert_slots ? `${fmt(eng.expert_slots)} (${gb((eng.expert_cache_mib || 0) * 1048576)} GB)` : null],
|
||||
["Speculation", eng.spec ? `MTP drafts up to ${Math.max(0, (eng.mtp_max || eng.spec) - 1)} tokens${eng.lookup ? ", prompt lookup on" : ""}` : null],
|
||||
["Images", eng.images ? "on" : "off"],
|
||||
["Experimental speed projection", projectionText(eng.cvec)],
|
||||
]);
|
||||
facts($("facts-hw"), [
|
||||
["GPU", st.gpu_name ? `${st.gpu_name}${hw.gpu_mem_total ? `, ${gb(hw.gpu_mem_total, 0)} GB` : ""}` : "not readable (NVML)"],
|
||||
@@ -376,7 +387,7 @@ function markdown(text) {
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ Chat
|
||||
const DEFAULTS = {thinking: "high", temperature: 0.6, top_p: 0.95, top_k: 20, max: "", seed: "", show: true};
|
||||
const DEFAULTS = {thinking: "high", temperature: 0.6, top_p: 0.95, top_k: 20, max: "", seed: "", show: true, esp: true};
|
||||
let settings = {...DEFAULTS, ...store.get("sampling", {})};
|
||||
let messages = store.get("chat", []);
|
||||
let attachments = []; // {name, url}
|
||||
@@ -515,6 +526,7 @@ async function send() {
|
||||
}
|
||||
if (settings.seed) body.seed = +settings.seed;
|
||||
if (settings.max) body.max_tokens = +settings.max;
|
||||
if (projectionLoaded()) body.experimental_speed_projection = !!settings.esp;
|
||||
|
||||
let firstAt = null, thinkStart = null, usage = null, frame = 0;
|
||||
const paint = () => { frame = 0; updateAssistant(el, m, true); scrollDown(); };
|
||||
@@ -562,7 +574,8 @@ async function send() {
|
||||
const n = usage ? usage.completion_tokens : null;
|
||||
if (n && firstAt) {
|
||||
const secs = (performance.now() - firstAt) / 1000;
|
||||
m.meta = `${fmt(n)} tokens${secs > 0.25 ? ` · ${fmt(n / secs, 1)} tok/s` : ""}${m.stopped ? " · stopped" : ""}`;
|
||||
m.meta = `${fmt(n)} tokens${secs > 0.25 ? ` · ${fmt(n / secs, 1)} tok/s` : ""}${m.stopped ? " · stopped" : ""}` +
|
||||
(projectionLoaded() ? (settings.esp ? " · projection on" : " · projection off") : "");
|
||||
} else if (m.stopped) {
|
||||
m.meta = "Stopped";
|
||||
}
|
||||
@@ -642,15 +655,50 @@ function openDrawer(open) {
|
||||
$("drawer").dataset.open = String(open);
|
||||
$("drawer").setAttribute("aria-hidden", String(!open));
|
||||
$("scrim").hidden = !open;
|
||||
if (open) loadDrawer();
|
||||
if (open) { loadDrawer(); loadShared(); }
|
||||
}
|
||||
function loadDrawer(s = settings) {
|
||||
for (const b of $("s-thinking").children) b.setAttribute("aria-checked", String(b.dataset.v === s.thinking));
|
||||
$("s-temp").value = s.temperature; $("s-topp").value = s.top_p; $("s-topk").value = s.top_k;
|
||||
$("s-max").value = s.max; $("s-seed").value = s.seed;
|
||||
$("s-show").setAttribute("aria-checked", String(!!s.show));
|
||||
$("s-esp").setAttribute("aria-checked", String(s.esp !== false));
|
||||
$("esp-row").hidden = !projectionLoaded();
|
||||
$("s-share").setAttribute("aria-checked", String(sharedOn));
|
||||
outputs();
|
||||
}
|
||||
// "Use for other apps too": the server keeps these settings as every client's defaults (GET/POST /settings)
|
||||
let sharedOn = false;
|
||||
async function loadShared() {
|
||||
try {
|
||||
const r = await fetch("/settings", {headers: headers()});
|
||||
if (r.ok) sharedOn = !!(await r.json()).shared;
|
||||
} catch (e) { /* an older server: the switch just stays off */ }
|
||||
$("s-share").setAttribute("aria-checked", String(sharedOn));
|
||||
}
|
||||
function sharedDefaults(s) {
|
||||
const d = {reasoning_effort: s.thinking, temperature: +s.temperature};
|
||||
if (+s.temperature > 0) Object.assign(d, {top_p: +s.top_p, top_k: +s.top_k});
|
||||
if (s.seed) d.seed = +s.seed;
|
||||
if (s.max) d.max_tokens = +s.max;
|
||||
if (projectionLoaded()) d.experimental_speed_projection = s.esp !== false;
|
||||
return d;
|
||||
}
|
||||
async function saveShared(on, s) {
|
||||
const r = await fetch("/settings", {method: "POST", headers: headers(true),
|
||||
body: JSON.stringify({defaults: on ? sharedDefaults(s) : null})});
|
||||
if (!r.ok) {
|
||||
let msg = `HTTP ${r.status}`;
|
||||
try { msg = (await r.json()).error.message || msg; } catch (e) { /* not json */ }
|
||||
throw new Error(msg);
|
||||
}
|
||||
sharedOn = !!(await r.json()).shared;
|
||||
}
|
||||
// the engine was started with the experimental-speed-projection control vector (INFO cvec=...)
|
||||
function projectionLoaded() {
|
||||
const c = lastMetrics && lastMetrics.engine ? lastMetrics.engine.cvec : 0;
|
||||
return !!c && c !== "0";
|
||||
}
|
||||
function outputs() {
|
||||
const t = +$("s-temp").value;
|
||||
$("o-temp").textContent = t === 0 ? "0 · greedy" : t.toFixed(2);
|
||||
@@ -663,14 +711,28 @@ function outputs() {
|
||||
for (const b of $("s-thinking").children) b.onclick = () => { for (const x of $("s-thinking").children) x.setAttribute("aria-checked", String(x === b)); outputs(); };
|
||||
for (const id of ["s-temp", "s-topp", "s-topk"]) $(id).oninput = outputs;
|
||||
$("s-show").onclick = () => $("s-show").setAttribute("aria-checked", String($("s-show").getAttribute("aria-checked") !== "true"));
|
||||
$("s-esp").onclick = () => $("s-esp").setAttribute("aria-checked", String($("s-esp").getAttribute("aria-checked") !== "true"));
|
||||
$("s-share").onclick = () => $("s-share").setAttribute("aria-checked", String($("s-share").getAttribute("aria-checked") !== "true"));
|
||||
$("s-reset").onclick = () => loadDrawer(DEFAULTS);
|
||||
$("s-apply").onclick = () => {
|
||||
$("s-apply").onclick = async () => {
|
||||
const sel = [...$("s-thinking").children].find((b) => b.getAttribute("aria-checked") === "true");
|
||||
settings = {thinking: sel ? sel.dataset.v : "high", temperature: +$("s-temp").value, top_p: +$("s-topp").value,
|
||||
top_k: +$("s-topk").value, max: $("s-max").value.trim(), seed: $("s-seed").value.trim(),
|
||||
show: $("s-show").getAttribute("aria-checked") === "true"};
|
||||
show: $("s-show").getAttribute("aria-checked") === "true",
|
||||
esp: $("s-esp").getAttribute("aria-checked") === "true"};
|
||||
store.set("sampling", settings);
|
||||
const share = $("s-share").getAttribute("aria-checked") === "true";
|
||||
openDrawer(false);
|
||||
if (share || sharedOn) {
|
||||
try {
|
||||
await saveShared(share, settings);
|
||||
toast("success", "Sampling saved", share ? "Other apps (omp, API clients) use these settings from their next request."
|
||||
: "Other apps use their own settings again.");
|
||||
} catch (e) {
|
||||
toast("error", "Saved here, but not for other apps", e.message, 6000);
|
||||
}
|
||||
return;
|
||||
}
|
||||
toast("success", "Sampling saved", settings.temperature === 0 ? "Greedy: the same question gives the same answer." : "");
|
||||
};
|
||||
$("sampling-btn").onclick = () => openDrawer(true);
|
||||
|
||||
@@ -192,6 +192,12 @@
|
||||
</div>
|
||||
<div class="toggle-row"><span>Show thinking<br><span class="muted small">expanded while it streams</span></span>
|
||||
<button class="st-toggle" id="s-show" role="switch" aria-checked="true" aria-label="Show thinking"></button></div>
|
||||
<div class="toggle-row" id="esp-row" hidden><span>Experimental speed projection<br><span class="muted small">the
|
||||
engine's control vector; off = the stock model. Switching reads the chat again once</span></span>
|
||||
<button class="st-toggle" id="s-esp" role="switch" aria-checked="true" aria-label="Experimental speed projection"></button></div>
|
||||
<div class="toggle-row"><span>Use for other apps too<br><span class="muted small">omp and other API clients get
|
||||
these settings for anything they don't set themselves</span></span>
|
||||
<button class="st-toggle" id="s-share" role="switch" aria-checked="false" aria-label="Use for other apps too"></button></div>
|
||||
<div class="drawer-actions">
|
||||
<button class="st-btn st-btn--secondary" id="s-reset" type="button">Reset</button>
|
||||
<button class="st-btn st-btn--primary" id="s-apply" type="button">Apply</button>
|
||||
|
||||
@@ -21,6 +21,8 @@ What the first run does (each step is skipped when it is already done):
|
||||
|
||||
Options: --family qwen|swift, --model Q2_0|IQ2_XS|IQ3_XXS|IQ3_S, --context 32768, --vision yes|no|gpu|cpu, --port 8080, --yes (recommended
|
||||
answers, no questions), --setup (install another model / change settings instead of starting), --no-start,
|
||||
--host 0.0.0.0 --api-key KEY (reach it from other devices on your network), --experimental-speed-projection on|off
|
||||
(EXPERIMENTAL, off by default),
|
||||
--models-dir DIR, --gguf-dir DIR (use GGUF files you already have), --build (compile instead of the ready-made
|
||||
engine), --check (only check this PC).
|
||||
"""
|
||||
@@ -54,7 +56,7 @@ PREBUILT_ASSET = "strata-windows-x64.zip" if WIN else "strata-linux-x64.zip"
|
||||
# the CUDA libraries the ready-made engine loads (the same CUDA 13.0 it is built with), from NVIDIA's pip packages
|
||||
CUDA_WHEELS = ["nvidia-cublas==13.0.2.14", "nvidia-cuda-runtime==13.0.96"]
|
||||
MIN_DRIVER = 580 # CUDA 13.0
|
||||
MIN_ENGINE = (0, 1, 8) # --kv q4_0 and the INFO line (the web app's Monitor), v0.1.8
|
||||
MIN_ENGINE = (0, 1, 10) # INFO arena_mib: the start-up RAM warning (issue #27), v0.1.10
|
||||
PY_PACKAGES = ["numpy", "jinja2", "regex", "pyyaml", "tqdm", "requests", "cmake", "ninja", "pillow", "psutil"]
|
||||
|
||||
MODELS = {
|
||||
@@ -86,6 +88,8 @@ FAMILIES = {
|
||||
"license": "Swift Open License 1.0: https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF"},
|
||||
}
|
||||
MMPROJ = "mmproj-Qwen3.8-Flash-Next-BF16.gguf"
|
||||
# EXPERIMENTAL, off by default (setup asks): a control vector shipped with the repository, see its README
|
||||
ESP_VECTOR = ROOT / "data" / "experimental-speed-projection" / "Qwen3.8-Flash-Next-experimental-speed-projection.gguf"
|
||||
# the image encoder on the GPU (~1.2 GB at 1024 image tokens) warms up before the engine starts, so the engine
|
||||
# sizes its expert slots around it and the default reserve (700 MiB) is enough; engines before 0.1.2 need more
|
||||
VISION = {"gpu": {"max_tokens": 1024, "reserve_mib": 700},
|
||||
@@ -652,7 +656,13 @@ def main() -> int:
|
||||
"precise)")
|
||||
ap.add_argument("--vision", choices=["yes", "no", "none", "gpu", "cpu"],
|
||||
help="let the model read images (yes = the encoder on the GPU)")
|
||||
ap.add_argument("--experimental-speed-projection", metavar="on|off|GGUF",
|
||||
help="EXPERIMENTAL, off by default: the control vector in data/experimental-speed-projection "
|
||||
"(or another GGUF) as a projection on layers 4-44; see docs/DETAILS.md")
|
||||
ap.add_argument("--port", type=int, default=8080)
|
||||
ap.add_argument("--host", help="where the server listens: 127.0.0.1 = this PC only (default), 0.0.0.0 = also other "
|
||||
"devices on your network (issue #26; set --api-key too)")
|
||||
ap.add_argument("--api-key", help="require this key from clients (recommended with --host 0.0.0.0)")
|
||||
ap.add_argument("--models-dir", default=str(ROOT / "models"), help="where the GGUF files go (~70 GB)")
|
||||
ap.add_argument("--gguf-dir", help="use GGUF files you already have (a folder with the two shards)")
|
||||
ap.add_argument("--yes", action="store_true", help="accept the recommended answers")
|
||||
@@ -781,6 +791,25 @@ def main() -> int:
|
||||
say(" download and keeps ~1.4 GB of VRAM free for the image encoder, so text is a few % slower.")
|
||||
vision = "gpu" if ask("Do you want images?", ["y", "n"], "n", a.yes) == "y" else "none"
|
||||
ok("images: " + {"none": "off", "gpu": "on", "cpu": "on (encoder on the CPU)"}[vision])
|
||||
# EXPERIMENTAL: the experimental-speed-projection control vector (data/experimental-speed-projection), off unless
|
||||
# chosen here; with it loaded, the web app and the API switch it off per request
|
||||
esp = None
|
||||
esp_choice = (a.experimental_speed_projection or "").strip()
|
||||
if family == "qwen":
|
||||
if not esp_choice:
|
||||
say()
|
||||
say(" EXPERIMENTAL - speed projection: a small control vector applied while the model runs (layers 4-44).")
|
||||
say(" It changes how the model answers: its package describes it as a refusal-direction projection (the")
|
||||
say(" model declines far fewer requests). Off unless you choose it; when on, the web app can switch it off")
|
||||
say(" per chat. Details: data/experimental-speed-projection/README.md")
|
||||
esp_choice = "on" if ask("Turn on the experimental speed projection?", ["y", "n"], "n", a.yes) == "y" else "off"
|
||||
if esp_choice.lower() not in ("off", "no", "n", "0"):
|
||||
esp = ESP_VECTOR if esp_choice.lower() in ("on", "yes", "y", "1") else Path(esp_choice).expanduser().resolve()
|
||||
if not esp.is_file():
|
||||
fail(f"the experimental speed projection's vector is missing: {esp}")
|
||||
ok("experimental speed projection: " + ("ON (experimental)" if esp else "off"))
|
||||
elif esp_choice.lower() not in ("", "off", "no", "n", "0"):
|
||||
warn("the experimental speed projection is made for the original Qwen3.8-Flash-Next, not Swift 1.5: left off")
|
||||
models_dir = Path(a.gguf_dir) if a.gguf_dir else Path(a.models_dir) / tag
|
||||
shards = [models_dir / fam["file"].format(q=model, i=i) for i in (1, 2)]
|
||||
have_model = all(s.exists() and (done(s) or a.gguf_dir) for s in shards)
|
||||
@@ -893,9 +922,17 @@ def main() -> int:
|
||||
ok(f"KV streaming on: the context's KV cache lives in RAM ({kv_ram_gb:.1f} GB), more experts fit in VRAM")
|
||||
if vision != "none":
|
||||
args += ["--vision", "--vram-reserve-mib", str(VISION[vision]["reserve_mib"])]
|
||||
if esp is not None:
|
||||
# the package's profile, with llama.cpp's flags (the engine takes the same ones)
|
||||
args += ["--control-vector-scaled", f"{esp}:1.0", "--control-vector-layer-range", "4", "44",
|
||||
"--cvec-mode", "project", "--cvec-dir", "per-layer"]
|
||||
cfg = {"exe": str(eng / EXE), "args": args, "cwd": str(ROOT), "tokenizer": str(pack / "tokenizer"),
|
||||
"model_name": f"{fam['name']}-{model.lower()}", "log": str(ROOT / f"strata-{tag.lower()}.log"),
|
||||
"lib_dirs": lib_dirs, "port": a.port}
|
||||
if a.host:
|
||||
cfg["host"] = a.host
|
||||
if a.api_key:
|
||||
cfg["api_key"] = a.api_key
|
||||
if vision != "none":
|
||||
cfg["vision"] = {"exe": str(eng / VEXE), "mmproj": str(mmproj), "model": str(shards[0]),
|
||||
"gpu": vision == "gpu", "max_tokens": VISION[vision]["max_tokens"]}
|
||||
@@ -910,6 +947,9 @@ def main() -> int:
|
||||
say("All set.")
|
||||
say(f" API (OpenAI): http://127.0.0.1:{a.port}/v1 (any API key; model name: anything)")
|
||||
say(f" API (Anthropic): http://127.0.0.1:{a.port}/v1/messages")
|
||||
if a.host and a.host not in ("127.0.0.1", "localhost"):
|
||||
say(f" Other devices: the server window prints this PC's address (http://<IP>:{a.port}/)"
|
||||
+ ("" if a.api_key else " - no API key set: anyone on your network can use it"))
|
||||
say(f" Next time: just run {'START-HERE.bat' if WIN else './setup.sh'} (or {script.name}) - it starts right away")
|
||||
if vision != "none":
|
||||
say(" Images: send them in the chat page, in chat.py (/image <path>) or over the API")
|
||||
|
||||
+20
-5
@@ -25,6 +25,7 @@
|
||||
#include "strata/kernels/native_rope.hpp"
|
||||
#include "strata/kernels/native_flash_attn.hpp"
|
||||
#include "strata/kernels/fused_gr.hpp"
|
||||
#include "strata/kernels/cvec.hpp"
|
||||
#include "strata/kernels/qsa_decode_attn.hpp"
|
||||
#include "strata/kernels/fused_gdn.hpp"
|
||||
#include "strata/kernels/qsa_select.hpp"
|
||||
@@ -1107,7 +1108,8 @@ bool block_layer_pre(const WeightTable& tables, const ModelGeometry& g, int64_t
|
||||
// the device half: `ple_block` reading `emb_dev`, and the history shift.
|
||||
const bool fused = g_fused_gr && stage_prefix == 0 && half == 0 &&
|
||||
strata::kernels::fused_gr_supported(g.n_embd, g.hc, g.hc_lr);
|
||||
bool pending_ffn = fused && layer > 0; // layer-1's FFN write has not been applied to R yet
|
||||
// layer-1's FFN write has not been applied to R yet - unless a control vector follows it, which needs R
|
||||
bool pending_ffn = fused && layer > 0 && !strata::kernels::cvec().covers(layer - 1);
|
||||
if (ple != nullptr && ple->ready() && layer == 1 && (half == 0 || half == 1)) {
|
||||
if (pending_ffn) {
|
||||
const strata::kernels::GrShapes gs0{g.n_embd, g.hc, g.hc_lr};
|
||||
@@ -1242,10 +1244,23 @@ st_begin(layer, 5, stream);
|
||||
if (g_shared_early ? !moe_combine_parts(g, layer, k, mb, parts, bb.block_out, stream, err)
|
||||
: !moe_finish(tables, g, layer, k, mb, bb.mixed, parts, bb.block_out, stream, err)) return false;
|
||||
st_end(layer, 5, stream); dump_half(bb, g, layer, bb.block_out, (uint64_t) g.n_embd, g.n_embd, stream); dump_half(bb, g, layer, bb.inject, (uint64_t) 2 * g.n_embd + g.hc, g.hc, stream); st_begin(layer, 6, stream);
|
||||
if (!(g_fused_gr && strata::kernels::fused_gr_supported(g.n_embd, g.hc, g.hc_lr)))
|
||||
gr_write(bb.R, bb.block_out, bb.inject, gs, bb.R, stream);
|
||||
else if (layer == g.n_layers - 1)
|
||||
gr_write(bb.R, bb.block_out, bb.inject2, gs, bb.R, stream); // materialise R for the head
|
||||
const bool steer = strata::kernels::cvec().covers(layer); // --control-vector-scaled: after this write
|
||||
try {
|
||||
if (!(g_fused_gr && strata::kernels::fused_gr_supported(g.n_embd, g.hc, g.hc_lr))) {
|
||||
gr_write(bb.R, bb.block_out, bb.inject, gs, bb.R, stream);
|
||||
if (steer) strata::kernels::cvec_apply(bb.R, layer, 1, g.hc * g.n_embd, nullptr, 0, nullptr, 0, false, stream);
|
||||
} else if (layer == g.n_layers - 1) {
|
||||
gr_write(bb.R, bb.block_out, bb.inject2, gs, bb.R, stream); // materialise R for the head
|
||||
if (steer) strata::kernels::cvec_apply(bb.R, layer, 1, g.hc * g.n_embd, nullptr, 0, nullptr, 0, false, stream);
|
||||
} else if (steer) {
|
||||
// the write the next layer's fused read would have folded, then the vector
|
||||
strata::kernels::cvec_apply(bb.R, layer, 1, g.hc * g.n_embd, bb.block_out, g.n_embd, bb.inject2, g.hc,
|
||||
true, stream);
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
err = std::string("block_layer_post: ") + e.what();
|
||||
return false;
|
||||
}
|
||||
st_end(layer, 6, stream); return true;}
|
||||
bool block_layer(const WeightTable& tables, const ModelGeometry& g, int64_t layer, int64_t pos, int32_t pos_base, const GdnBuffers& gb, const QsaState& qst, const QsaBuffers& qb, const MoEBuffers& mb, int64_t k, const BlockBuffers& bb, const float* parts, void* stream, std::string& err, const Doorbell* db, const PleRun* ple) {
|
||||
// The two halves back to back. `parts` must be THIS layer's experts and must be ready before the call -
|
||||
|
||||
+9
-2
@@ -11,6 +11,7 @@
|
||||
#include "strata/kernels/cpu/expert.hpp"
|
||||
#include "strata/kernels/elementwise.hpp"
|
||||
#include "strata/kernels/fused_gr.hpp"
|
||||
#include "strata/kernels/cvec.hpp"
|
||||
#include "strata/kernels/gr.hpp"
|
||||
#include "strata/kernels/kv_q4.hpp"
|
||||
#include "strata/kernels/kv_q8.hpp"
|
||||
@@ -331,7 +332,9 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
|
||||
wi[h] = need(v, (std::string(pfx[h]) + "inject.weight").c_str(), err);
|
||||
if (!wn[h] || !wd[h] || !wu[h] || !wi[h]) return false;
|
||||
}
|
||||
bool pending = l > 0; // the previous layer's FFN write, folded into this layer's first read
|
||||
// the previous layer's FFN write, folded into this layer's first read (a control vector after it has
|
||||
// already applied it)
|
||||
bool pending = l > 0 && !cvec().covers(l - 1);
|
||||
if (l == 1 && ple_on) {
|
||||
float* normalized = (float*) ((uint8_t*) ss.ple.scratch + ple_block_scratch_bytes());
|
||||
for (int t = tb; t < te; ++t) {
|
||||
@@ -569,8 +572,12 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
|
||||
mb.weights = w_ + t * K; mb.shared = shared_ + t * N;
|
||||
if (!moe_combine_parts(g, l, K, mb, parts_ + (size_t) t * K * N, bo_ + t * N, cs, err)) return false;
|
||||
}
|
||||
if (l == g.n_layers - 1)
|
||||
if (l == g.n_layers - 1) {
|
||||
for (int t = tb; t < te; ++t) gr_write(Rt(t), bo_ + t * N, inj2_ + t * HC, gs, Rt(t), cs);
|
||||
if (cvec().covers(l)) cvec_apply(Rt(tb), l, n, HC * N, nullptr, 0, nullptr, 0, false, cs);
|
||||
} else if (cvec().covers(l)) {
|
||||
cvec_apply(Rt(tb), l, n, HC * N, bo_ + tb * N, N, inj2_ + tb * HC, HC, true, cs);
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
// src/kernels/cuda/cvec.cu - see include/strata/kernels/cvec.hpp.
|
||||
#include "strata/kernels/cvec.hpp"
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
namespace strata::kernels {
|
||||
namespace {
|
||||
|
||||
constexpr int THREADS = 256;
|
||||
constexpr int MAXK = 16; // n_embd up to 4096, held in registers between the dot and the update
|
||||
|
||||
Cvec g_cvec;
|
||||
int* g_on = nullptr;
|
||||
bool g_on_host = false;
|
||||
|
||||
// the fused hyper-connection read's gate (fused_gr.cu), so a write done here is bitwise the one it would have folded
|
||||
__device__ __forceinline__ float sigmoidf_(float x) { return 1.0f / (1.0f + __expf(-x)); }
|
||||
|
||||
// one block per (stream, token): the pending write, then h . v over the stream, then the update
|
||||
__global__ void cvec_kernel(float* __restrict__ R, const float* __restrict__ dir, const float* __restrict__ s_l,
|
||||
const int* __restrict__ on, int mode, int64_t layer, int n, int hc, int64_t r_ld,
|
||||
const float* __restrict__ bo, int64_t bo_ld, const float* __restrict__ inj,
|
||||
int64_t inj_ld, int write) {
|
||||
const int c = blockIdx.x;
|
||||
const int64_t t = blockIdx.y;
|
||||
float* r = R + t * r_ld + (int64_t) c * n;
|
||||
const float s = s_l[layer];
|
||||
const bool steer = *on != 0 && s != 0.0f; // uniform over the block
|
||||
if (!steer && !write) return;
|
||||
const float* v = dir + layer * n;
|
||||
const float w = write ? 2.0f * sigmoidf_(inj[t * inj_ld + c] / (float) hc) : 0.0f;
|
||||
const float* b = write ? bo + t * bo_ld : nullptr;
|
||||
float x[MAXK];
|
||||
float dot = 0.0f;
|
||||
#pragma unroll
|
||||
for (int k = 0; k < MAXK; ++k) {
|
||||
const int d = threadIdx.x + k * THREADS;
|
||||
if (d < n) {
|
||||
float xv = r[d];
|
||||
if (write) xv = fmaf(b[d], w, xv);
|
||||
x[k] = xv;
|
||||
if (steer && mode == 0) dot = fmaf(xv, v[d], dot);
|
||||
}
|
||||
}
|
||||
if (steer && mode == 0) {
|
||||
__shared__ float part[THREADS / 32];
|
||||
#pragma unroll
|
||||
for (int o = 16; o > 0; o >>= 1) dot += __shfl_xor_sync(0xffffffffu, dot, o);
|
||||
if ((threadIdx.x & 31) == 0) part[threadIdx.x >> 5] = dot;
|
||||
__syncthreads();
|
||||
if (threadIdx.x < 32) {
|
||||
float p = threadIdx.x < THREADS / 32 ? part[threadIdx.x] : 0.0f;
|
||||
#pragma unroll
|
||||
for (int o = 16; o > 0; o >>= 1) p += __shfl_xor_sync(0xffffffffu, p, o);
|
||||
if (threadIdx.x == 0) part[0] = p;
|
||||
}
|
||||
__syncthreads();
|
||||
dot = part[0] * s; // s (h . v)
|
||||
}
|
||||
#pragma unroll
|
||||
for (int k = 0; k < MAXK; ++k) {
|
||||
const int d = threadIdx.x + k * THREADS;
|
||||
if (d < n) {
|
||||
float xv = x[k];
|
||||
if (steer) xv = mode == 0 ? fmaf(-dot, v[d], xv) : xv + v[d];
|
||||
r[d] = xv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
const Cvec& cvec() { return g_cvec; }
|
||||
|
||||
bool cvec_upload(const std::vector<float>& dir, const std::vector<float>& s, int mode, int first, int last,
|
||||
int64_t n_embd, int64_t hc, std::string& err) {
|
||||
if (n_embd < 1 || n_embd > (int64_t) THREADS * MAXK) { err = "control vector: unsupported n_embd"; return false; }
|
||||
if (s.empty() || dir.size() != s.size() * (size_t) n_embd) { err = "control vector: bad table sizes"; return false; }
|
||||
float* d_dir = nullptr;
|
||||
float* d_s = nullptr;
|
||||
int* d_on = nullptr;
|
||||
const int one = 1;
|
||||
if (cudaMalloc(&d_dir, dir.size() * sizeof(float)) != cudaSuccess ||
|
||||
cudaMalloc(&d_s, s.size() * sizeof(float)) != cudaSuccess || cudaMalloc(&d_on, sizeof(int)) != cudaSuccess ||
|
||||
cudaMemcpy(d_dir, dir.data(), dir.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
|
||||
cudaMemcpy(d_s, s.data(), s.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
|
||||
cudaMemcpy(d_on, &one, sizeof(int), cudaMemcpyHostToDevice) != cudaSuccess) {
|
||||
err = "control vector: device allocation failed";
|
||||
return false;
|
||||
}
|
||||
g_cvec.dir = d_dir;
|
||||
g_cvec.s = d_s;
|
||||
g_cvec.on = d_on;
|
||||
g_cvec.mode = mode;
|
||||
g_cvec.first = first;
|
||||
g_cvec.last = last;
|
||||
g_cvec.n_embd = n_embd;
|
||||
g_cvec.hc = hc;
|
||||
g_cvec.steered.assign(s.size(), false);
|
||||
for (size_t l = 0; l < s.size(); ++l) g_cvec.steered[l] = s[l] != 0.0f;
|
||||
g_on = d_on;
|
||||
g_on_host = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
void cvec_set_enabled(bool on) {
|
||||
if (g_on == nullptr || on == g_on_host) return;
|
||||
cudaDeviceSynchronize(); // nothing in flight may still read the flag
|
||||
const int v = on ? 1 : 0;
|
||||
cudaMemcpy(g_on, &v, sizeof(int), cudaMemcpyHostToDevice);
|
||||
g_on_host = on;
|
||||
}
|
||||
|
||||
bool cvec_enabled() { return g_on != nullptr && g_on_host; }
|
||||
|
||||
void cvec_apply(float* R, int64_t layer, int64_t T, int64_t r_ld, const float* bo, int64_t bo_ld, const float* inj,
|
||||
int64_t inj_ld, bool write, void* stream) {
|
||||
if (!g_cvec.loaded() || T < 1) return;
|
||||
const dim3 grid((unsigned) g_cvec.hc, (unsigned) T);
|
||||
cvec_kernel<<<grid, THREADS, 0, (cudaStream_t) stream>>>(R, g_cvec.dir, g_cvec.s, g_cvec.on, g_cvec.mode, layer,
|
||||
(int) g_cvec.n_embd, (int) g_cvec.hc, r_ld, bo, bo_ld,
|
||||
inj, inj_ld, write ? 1 : 0);
|
||||
if (cudaPeekAtLastError() != cudaSuccess) throw std::runtime_error("cvec_apply: launch failed");
|
||||
}
|
||||
|
||||
} // namespace strata::kernels
|
||||
@@ -0,0 +1,223 @@
|
||||
// src/kernels/cvec_parity.cpp - the control vector kernel (strata/kernels/cvec.hpp) against a host reference.
|
||||
// 1. project: h - s (h.v) v per stream and token, against double precision; the steered component is gone.
|
||||
// 2. add: h + d.
|
||||
// 3. switched off: R unchanged without a pending write; with one, R is BITWISE the write the fused read folds
|
||||
// (fused_gr_read with apply), which is what keeps a loaded-but-off vector identical to the stock engine.
|
||||
// 4. a layer without a direction is untouched.
|
||||
|
||||
#include "strata/kernels/cvec.hpp"
|
||||
#include "strata/kernels/fused_gr.hpp"
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
namespace k = strata::kernels;
|
||||
|
||||
namespace {
|
||||
|
||||
int g_fail = 0;
|
||||
|
||||
void ck(cudaError_t e, const char* w) {
|
||||
if (e != cudaSuccess) {
|
||||
std::fprintf(stderr, "CUDA error in %s: %s\n", w, cudaGetErrorString(e));
|
||||
std::exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T* dalloc(size_t n) {
|
||||
T* p = nullptr;
|
||||
ck(cudaMalloc(&p, n * sizeof(T)), "malloc");
|
||||
return p;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void up(T* d, const std::vector<T>& h) { ck(cudaMemcpy(d, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "up"); }
|
||||
template <typename T>
|
||||
std::vector<T> down(const T* d, size_t n) {
|
||||
std::vector<T> h(n);
|
||||
ck(cudaMemcpy(h.data(), d, n * sizeof(T), cudaMemcpyDeviceToHost), "down");
|
||||
return h;
|
||||
}
|
||||
|
||||
void check(bool ok, const char* what) {
|
||||
std::printf(" %-66s %s\n", what, ok ? "ok" : "FAIL");
|
||||
if (!ok) ++g_fail;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
constexpr int64_t L = 48, N = 2560, HC = 4, LR = 320, T = 5, D = HC * N;
|
||||
constexpr int64_t kLayer = 7, kOff = 3; // steered / not steered
|
||||
std::mt19937 rng(1234);
|
||||
std::normal_distribution<float> nd(0.0f, 1.0f);
|
||||
|
||||
// a unit direction at kLayer with a norm-2 scaled one's s = 2; nothing at kOff
|
||||
std::vector<float> dir((size_t) (L * N), 0.0f), s((size_t) L, 0.0f);
|
||||
{
|
||||
double nrm = 0.0;
|
||||
std::vector<float> v((size_t) N);
|
||||
for (auto& x : v) { x = nd(rng); nrm += (double) x * x; }
|
||||
nrm = std::sqrt(nrm);
|
||||
for (int64_t j = 0; j < N; ++j) dir[(size_t) (kLayer * N + j)] = (float) (v[(size_t) j] / nrm);
|
||||
s[(size_t) kLayer] = 2.0f;
|
||||
}
|
||||
std::string err;
|
||||
if (!k::cvec_upload(dir, s, /*project*/ 0, 4, 44, N, HC, err)) { std::fprintf(stderr, "%s\n", err.c_str()); return 2; }
|
||||
check(k::cvec().covers(kLayer) && !k::cvec().covers(kOff) && !k::cvec().covers(45), "covers(): steered layers only");
|
||||
|
||||
std::vector<float> R((size_t) (T * D)), bo((size_t) (T * N)), inj((size_t) (T * HC));
|
||||
for (auto& x : R) x = nd(rng) * 3.0f;
|
||||
for (auto& x : bo) x = nd(rng);
|
||||
for (auto& x : inj) x = nd(rng);
|
||||
float* dR = dalloc<float>(R.size());
|
||||
float* dbo = dalloc<float>(bo.size());
|
||||
float* dinj = dalloc<float>(inj.size());
|
||||
up(dbo, bo);
|
||||
up(dinj, inj);
|
||||
|
||||
// ---- 1. project, no pending write
|
||||
std::printf("project\n");
|
||||
up(dR, R);
|
||||
k::cvec_apply(dR, kLayer, T, D, nullptr, 0, nullptr, 0, false, nullptr);
|
||||
ck(cudaDeviceSynchronize(), "project");
|
||||
{
|
||||
const auto got = down(dR, R.size());
|
||||
double worst = 0.0, worst_dot = 0.0;
|
||||
for (int64_t t = 0; t < T; ++t)
|
||||
for (int64_t c = 0; c < HC; ++c) {
|
||||
const float* h = R.data() + t * D + c * N;
|
||||
const float* v = dir.data() + kLayer * N;
|
||||
double dot = 0.0;
|
||||
for (int64_t j = 0; j < N; ++j) dot += (double) h[j] * v[j];
|
||||
double after = 0.0;
|
||||
for (int64_t j = 0; j < N; ++j) {
|
||||
const double want = h[j] - 2.0 * dot * v[j];
|
||||
const double g = got[(size_t) (t * D + c * N + j)];
|
||||
worst = std::fmax(worst, std::fabs(g - want));
|
||||
after += g * v[j];
|
||||
}
|
||||
// s = 2 reflects the component: h.v -> -h.v
|
||||
worst_dot = std::fmax(worst_dot, std::fabs(after + dot));
|
||||
}
|
||||
std::printf(" max |err| %.3g, max |h'.v + h.v| %.3g\n", worst, worst_dot);
|
||||
check(worst < 1e-4, "project matches h - s (h.v) v (s = 2)");
|
||||
check(worst_dot < 1e-3, "the component along v is reflected (s = 2)");
|
||||
}
|
||||
// s = 1 removes it
|
||||
s[(size_t) kLayer] = 1.0f;
|
||||
if (!k::cvec_upload(dir, s, 0, 4, 44, N, HC, err)) return 2;
|
||||
up(dR, R);
|
||||
k::cvec_apply(dR, kLayer, T, D, nullptr, 0, nullptr, 0, false, nullptr);
|
||||
ck(cudaDeviceSynchronize(), "project s=1");
|
||||
{
|
||||
const auto got = down(dR, R.size());
|
||||
double worst = 0.0;
|
||||
for (int64_t t = 0; t < T; ++t)
|
||||
for (int64_t c = 0; c < HC; ++c) {
|
||||
double after = 0.0;
|
||||
for (int64_t j = 0; j < N; ++j) after += (double) got[(size_t) (t * D + c * N + j)] * dir[(size_t) (kLayer * N + j)];
|
||||
worst = std::fmax(worst, std::fabs(after));
|
||||
}
|
||||
std::printf(" max |h'.v| %.3g\n", worst);
|
||||
check(worst < 1e-3, "s = 1 projects the direction out");
|
||||
}
|
||||
|
||||
// ---- 4. a layer without a direction
|
||||
up(dR, R);
|
||||
k::cvec_apply(dR, kOff, T, D, nullptr, 0, nullptr, 0, false, nullptr);
|
||||
ck(cudaDeviceSynchronize(), "off layer");
|
||||
check(std::memcmp(down(dR, R.size()).data(), R.data(), R.size() * 4) == 0, "a layer without a direction is untouched");
|
||||
|
||||
// ---- 3. switched off
|
||||
std::printf("switched off\n");
|
||||
k::cvec_set_enabled(false);
|
||||
check(!k::cvec_enabled(), "cvec_enabled() follows the switch");
|
||||
up(dR, R);
|
||||
k::cvec_apply(dR, kLayer, T, D, nullptr, 0, nullptr, 0, false, nullptr);
|
||||
ck(cudaDeviceSynchronize(), "off");
|
||||
check(std::memcmp(down(dR, R.size()).data(), R.data(), R.size() * 4) == 0, "off, no pending write: R unchanged");
|
||||
{
|
||||
// the fused read's folded write, one token at a time, against the kernel's write-only pass
|
||||
std::vector<uint16_t> wd((size_t) (LR * D)), wu((size_t) (D * LR)), wi((size_t) (HC * D));
|
||||
std::vector<float> wn((size_t) D);
|
||||
auto bf = [&](float x) { uint32_t u; std::memcpy(&u, &x, 4); return (uint16_t) (u >> 16); };
|
||||
for (auto& x : wd) x = bf(nd(rng) * 0.02f);
|
||||
for (auto& x : wu) x = bf(nd(rng) * 0.02f);
|
||||
for (auto& x : wi) x = bf(nd(rng) * 0.02f);
|
||||
for (auto& x : wn) x = 1.0f + 0.1f * nd(rng);
|
||||
uint16_t* dwd = dalloc<uint16_t>(wd.size());
|
||||
uint16_t* dwu = dalloc<uint16_t>(wu.size());
|
||||
uint16_t* dwi = dalloc<uint16_t>(wi.size());
|
||||
float* dwn = dalloc<float>(wn.size());
|
||||
up(dwd, wd); up(dwu, wu); up(dwi, wi); up(dwn, wn);
|
||||
float* lo = dalloc<float>(LR);
|
||||
float* rs = dalloc<float>(HC);
|
||||
float* inj_out = dalloc<float>(HC);
|
||||
float* mixed = dalloc<float>(N);
|
||||
float* dF = dalloc<float>(R.size());
|
||||
up(dF, R);
|
||||
for (int64_t t = 0; t < T; ++t) {
|
||||
k::FusedGrArgs a;
|
||||
a.R = dF + t * D; a.R_out = dF + t * D; a.apply = true;
|
||||
a.bo_prev = dbo + t * N; a.inj_prev = dinj + t * HC;
|
||||
a.w_norm = dwn; a.w_down = dwd; a.w_up = dwu; a.w_inject = dwi;
|
||||
a.lo = lo; a.rs = rs; a.inject_out = inj_out; a.mixed = mixed;
|
||||
k::fused_gr_read(a, nullptr);
|
||||
}
|
||||
up(dR, R);
|
||||
k::cvec_apply(dR, kLayer, T, D, dbo, N, dinj, HC, true, nullptr);
|
||||
ck(cudaDeviceSynchronize(), "write only");
|
||||
const auto fused = down(dF, R.size()), mine = down(dR, R.size());
|
||||
check(std::memcmp(fused.data(), mine.data(), R.size() * 4) == 0,
|
||||
"off, pending write: bitwise the fused read's folded write");
|
||||
k::cvec_set_enabled(true);
|
||||
// on, with the write: the write, then the projection (reference from the fused result)
|
||||
up(dR, R);
|
||||
k::cvec_apply(dR, kLayer, T, D, dbo, N, dinj, HC, true, nullptr);
|
||||
ck(cudaDeviceSynchronize(), "write+project");
|
||||
const auto both = down(dR, R.size());
|
||||
double worst = 0.0;
|
||||
for (int64_t t = 0; t < T; ++t)
|
||||
for (int64_t c = 0; c < HC; ++c) {
|
||||
const float* h = fused.data() + t * D + c * N;
|
||||
const float* v = dir.data() + kLayer * N;
|
||||
double dot = 0.0;
|
||||
for (int64_t j = 0; j < N; ++j) dot += (double) h[j] * v[j];
|
||||
for (int64_t j = 0; j < N; ++j)
|
||||
worst = std::fmax(worst, std::fabs(both[(size_t) (t * D + c * N + j)] - (h[j] - dot * v[j])));
|
||||
}
|
||||
std::printf(" write + project: max |err| %.3g\n", worst);
|
||||
check(worst < 1e-4, "on, pending write: the write, then the projection");
|
||||
}
|
||||
|
||||
// ---- 2. add
|
||||
std::printf("add\n");
|
||||
for (int64_t j = 0; j < N; ++j) dir[(size_t) (kLayer * N + j)] *= 0.1f; // d = 0.1 v, s = 1
|
||||
s[(size_t) kLayer] = 1.0f;
|
||||
if (!k::cvec_upload(dir, s, /*add*/ 1, 4, 44, N, HC, err)) return 2;
|
||||
up(dR, R);
|
||||
k::cvec_apply(dR, kLayer, T, D, nullptr, 0, nullptr, 0, false, nullptr);
|
||||
ck(cudaDeviceSynchronize(), "add");
|
||||
{
|
||||
const auto got = down(dR, R.size());
|
||||
bool same = true;
|
||||
for (int64_t t = 0; t < T; ++t)
|
||||
for (int64_t c = 0; c < HC; ++c)
|
||||
for (int64_t j = 0; j < N; ++j) {
|
||||
const size_t i = (size_t) (t * D + c * N + j);
|
||||
same = same && got[i] == R[i] + dir[(size_t) (kLayer * N + j)];
|
||||
}
|
||||
check(same, "add is h + d in every stream (bitwise)");
|
||||
}
|
||||
|
||||
std::printf(g_fail ? "cvec_parity: %d FAILED\n" : "cvec_parity: all passed\n", g_fail);
|
||||
return g_fail ? 1 : 0;
|
||||
}
|
||||
@@ -10,6 +10,7 @@
|
||||
#include "strata/kernels/cpu/expert_layout.hpp"
|
||||
#include "strata/kernels/qsa.hpp"
|
||||
#include "strata/kernels/kv_stream.hpp"
|
||||
#include "strata/kernels/cvec.hpp"
|
||||
#include "strata/kernels/kv_q4.hpp"
|
||||
#include "strata/core/layer.hpp"
|
||||
#include "strata/kernels/qsa_decode_attn.hpp"
|
||||
@@ -651,6 +652,8 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
|
||||
}
|
||||
// ---- the hyper-connection write of this half
|
||||
gr_write(m.R, m.bo, m.inj, HC, T, m.cs);
|
||||
if (half == 1 && strata::kernels::cvec().covers(l)) // --control-vector-scaled
|
||||
strata::kernels::cvec_apply(m.R, l, T, D, nullptr, 0, nullptr, 0, false, m.cs);
|
||||
}
|
||||
}
|
||||
stats_.tokens += T;
|
||||
|
||||
+185
-26
@@ -44,6 +44,11 @@
|
||||
#include "strata/program/logits_selection.hpp"
|
||||
#include "strata/spec/draft_policy.hpp"
|
||||
#include "strata/spec/suffix_drafter.hpp"
|
||||
#include "strata/kernels/cvec.hpp"
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX // gguf_reader.hpp includes windows.h
|
||||
#endif
|
||||
#include "strata/artifact/gguf_reader.hpp"
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
@@ -259,6 +264,13 @@ struct Options {
|
||||
int suffix_draft = 3;
|
||||
/// The MTP's own window cap (0 = --spec): with --spec 6 --mtp-max-t 4 the long windows come from suffix matches.
|
||||
int mtp_max_t = 0;
|
||||
/// A control vector on the residual stream (strata/kernels/cvec.hpp), with llama.cpp's flags: the
|
||||
/// `experimental-speed-projection` profile passes `--control-vector-scaled FILE:1.0 --control-vector-layer-range
|
||||
/// 4 44 --cvec-mode project --cvec-dir per-layer`. None by default; --serve switches a loaded one per request.
|
||||
std::vector<std::pair<std::string, float>> cvec_files;
|
||||
int cvec_first = -1, cvec_last = -1; ///< llama.cpp's defaults: 1 .. the last layer
|
||||
int cvec_mode = 1; ///< 0 = project, 1 = add (llama.cpp's default)
|
||||
int cvec_single = -1; ///< --cvec-dir single:L (project mode): layer L's direction everywhere
|
||||
};
|
||||
|
||||
void usage() {
|
||||
@@ -330,6 +342,11 @@ void usage() {
|
||||
" --suffix-draft N prompt lookup: draft from an earlier repeat of the last N+ tokens of context\n"
|
||||
" when it pays (default 3; 0 = MTP only)\n"
|
||||
" --mtp-max-t M cap the MTP's windows at M tokens (0 = --spec; longer ones come from suffixes)\n"
|
||||
" --control-vector-scaled FILE:SCALE[,...] a control vector GGUF on the residual stream (llama.cpp's\n"
|
||||
" format; --control-vector FILE = scale 1). --serve: requests switch it (cvec=0|1)\n"
|
||||
" --control-vector-layer-range A B the layers it follows (inclusive; default 1 .. the last)\n"
|
||||
" --cvec-mode add|project h += s v (default) or h -= s (h.v) v with v unit\n"
|
||||
" --cvec-dir per-layer|single:L each layer's own direction (default) or layer L's everywhere (project)\n"
|
||||
" --no-token-graph A/B: two graphs per layer (the host launches each) instead of one per token\n"
|
||||
" --no-fused-gr A/B: the six-kernel hyper-connection read and a separate write (native)\n"
|
||||
" --prefill CHUNK batched prompt processing in chunks of CHUNK tokens (needs --native)\n"
|
||||
@@ -558,6 +575,80 @@ bool checkpoint_restore(const ConvCheckpoint& c, strata::core::SessionState& ss,
|
||||
return cudaDeviceSynchronize() == cudaSuccess;
|
||||
}
|
||||
|
||||
// --control-vector-scaled: llama.cpp's `common_control_vector_load` (every file's `direction.<l>` times its scale,
|
||||
// summed; layer 0 has none) and `llama_adapter_cvec::apply` with the projection-mode patch (project: the unit
|
||||
// direction and its norm as the scale), into the tables `cvec_upload` takes. `summary` is what INFO reports.
|
||||
bool load_control_vectors(const Options& o, const strata::core::ModelGeometry& g, std::string& summary, std::string& err) {
|
||||
const int64_t L = g.n_layers, N = g.n_embd;
|
||||
std::vector<float> data((size_t) (L * N), 0.0f);
|
||||
std::vector<bool> have((size_t) L, false);
|
||||
for (const auto& [path, scale] : o.cvec_files) {
|
||||
try {
|
||||
strata::GgufFile f(path);
|
||||
const strata::MetaValue* arch = f.get("general.architecture");
|
||||
if (arch == nullptr || arch->s != "controlvector") {
|
||||
err = path + ": not a control vector GGUF (general.architecture is not 'controlvector')";
|
||||
return false;
|
||||
}
|
||||
const strata::MetaValue* hint = f.get("controlvector.model_hint");
|
||||
if (hint != nullptr && hint->s != "qwen4exp")
|
||||
std::fprintf(stderr, "strata generate: %s was made for '%s', not qwen4exp\n", path.c_str(), hint->s.c_str());
|
||||
int found = 0;
|
||||
for (const strata::TensorInfo& t : f.tensors()) {
|
||||
if (t.name.rfind("direction.", 0) != 0) continue;
|
||||
const long l = std::strtol(t.name.c_str() + 10, nullptr, 10);
|
||||
if (l < 1 || l >= L) continue; // layer 0 has no vector; past the model is ignored, as in llama.cpp
|
||||
if (t.type != 0 || t.elements() != (uint64_t) N) {
|
||||
err = path + ": " + t.name + " must be " + std::to_string((long long) N) + " f32";
|
||||
return false;
|
||||
}
|
||||
const float* src = reinterpret_cast<const float*>(f.tensor_data(t));
|
||||
for (int64_t j = 0; j < N; ++j) data[(size_t) (l * N + j)] += scale * src[j];
|
||||
have[(size_t) l] = true;
|
||||
++found;
|
||||
}
|
||||
if (found == 0) { err = path + ": no direction.<layer> tensors"; return false; }
|
||||
} catch (const std::exception& e) {
|
||||
err = e.what();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const int first = o.cvec_first <= 0 ? 1 : o.cvec_first;
|
||||
const int last = (o.cvec_last <= 0 || o.cvec_last >= L) ? (int) L - 1 : o.cvec_last;
|
||||
const int single = o.cvec_mode == 0 ? o.cvec_single : -1;
|
||||
if (single >= 0 && (single >= L || !have[(size_t) single])) {
|
||||
err = "--cvec-dir single:" + std::to_string(single) + ": the vector has no direction for that layer";
|
||||
return false;
|
||||
}
|
||||
std::vector<float> dir((size_t) (L * N), 0.0f), s((size_t) L, 0.0f);
|
||||
int steered = 0;
|
||||
for (int64_t l = first; l <= last; ++l) {
|
||||
const int64_t src = single >= 0 ? single : l;
|
||||
if (!have[(size_t) src]) continue;
|
||||
const float* d = data.data() + (size_t) (src * N);
|
||||
if (o.cvec_mode == 0) {
|
||||
double nrm = 0.0;
|
||||
for (int64_t j = 0; j < N; ++j) nrm += (double) d[j] * d[j];
|
||||
nrm = std::sqrt(nrm);
|
||||
if (nrm <= 0.0) continue;
|
||||
s[(size_t) l] = (float) nrm;
|
||||
for (int64_t j = 0; j < N; ++j) dir[(size_t) (l * N + j)] = (float) (d[j] / nrm);
|
||||
} else {
|
||||
s[(size_t) l] = 1.0f;
|
||||
std::copy(d, d + N, dir.begin() + (size_t) (l * N));
|
||||
}
|
||||
++steered;
|
||||
}
|
||||
if (steered == 0) { err = "the control vector has no direction in layers " + std::to_string(first) + ".." + std::to_string(last); return false; }
|
||||
if (!strata::kernels::cvec_upload(dir, s, o.cvec_mode, first, last, N, g.hc, err)) return false;
|
||||
summary = std::string(o.cvec_mode == 0 ? "project" : "add") + ":" + std::to_string(first) + "-" + std::to_string(last) +
|
||||
(single >= 0 ? ":single" + std::to_string(single) : "");
|
||||
// the line llama.cpp's patched build prints, so a log shows the same thing
|
||||
std::fprintf(stderr, "strata generate: control vector mode = %s, dir = %s, layers %d..%d (%d steered)\n",
|
||||
o.cvec_mode == 0 ? "project" : "add", single >= 0 ? "single" : "per-layer", first, last, steered);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
@@ -680,6 +771,38 @@ int main(int argc, char** argv) {
|
||||
else if (a == "--short-read") o.short_read = std::max(0LL, std::atoll(next("--short-read")));
|
||||
else if (a == "--suffix-draft") o.suffix_draft = std::max(0, std::atoi(next("--suffix-draft")));
|
||||
else if (a == "--mtp-max-t") o.mtp_max_t = std::max(0, std::atoi(next("--mtp-max-t")));
|
||||
else if (a == "--control-vector") o.cvec_files.push_back({next("--control-vector"), 1.0f});
|
||||
else if (a == "--control-vector-scaled") {
|
||||
// FILE:SCALE, comma-separated; the LAST colon splits, so a Windows path (C:\...) keeps its drive
|
||||
std::stringstream list(next("--control-vector-scaled"));
|
||||
std::string item;
|
||||
while (std::getline(list, item, ',')) {
|
||||
const size_t colon = item.rfind(':');
|
||||
char* end = nullptr;
|
||||
const float sc = colon == std::string::npos ? 0.0f : std::strtof(item.c_str() + colon + 1, &end);
|
||||
if (colon == std::string::npos || colon == 0 || end == item.c_str() + colon + 1 || *end != '\0') {
|
||||
std::fprintf(stderr, "--control-vector-scaled: expected FILE:SCALE, got '%s'\n", item.c_str());
|
||||
return 2;
|
||||
}
|
||||
o.cvec_files.push_back({item.substr(0, colon), sc});
|
||||
}
|
||||
}
|
||||
else if (a == "--control-vector-layer-range") {
|
||||
o.cvec_first = std::atoi(next("--control-vector-layer-range"));
|
||||
o.cvec_last = std::atoi(next("--control-vector-layer-range"));
|
||||
}
|
||||
else if (a == "--cvec-mode") {
|
||||
const std::string m = next("--cvec-mode");
|
||||
if (m == "project") o.cvec_mode = 0;
|
||||
else if (m == "add") o.cvec_mode = 1;
|
||||
else { std::fprintf(stderr, "--cvec-mode: add or project, got '%s'\n", m.c_str()); return 2; }
|
||||
}
|
||||
else if (a == "--cvec-dir") {
|
||||
const std::string d = next("--cvec-dir");
|
||||
if (d == "per-layer") o.cvec_single = -1;
|
||||
else if (d.rfind("single:", 0) == 0) o.cvec_single = std::atoi(d.c_str() + 7);
|
||||
else { std::fprintf(stderr, "--cvec-dir: per-layer or single:L, got '%s'\n", d.c_str()); return 2; }
|
||||
}
|
||||
else if (a == "--no-spec-split") o.spec_split = false;
|
||||
else if (a == "--eos-ids") {
|
||||
std::string e;
|
||||
@@ -948,6 +1071,15 @@ int main(int argc, char** argv) {
|
||||
strata::kernels::ple_set_native_postops(o.native_ple_postops);
|
||||
const strata::core::ModelGeometry g;
|
||||
const int64_t K = 10;
|
||||
// before session_init: every graph captured from here on has the vector's kernels where it applies
|
||||
std::string cvec_summary = "0";
|
||||
if (!o.cvec_files.empty()) {
|
||||
std::string ce;
|
||||
if (!load_control_vectors(o, g, cvec_summary, ce)) {
|
||||
std::fprintf(stderr, "strata generate: control vector: %s\n", ce.c_str());
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
if (o.max_context < (int64_t) o.tokens.size() + o.max_new) {
|
||||
std::fprintf(stderr, "strata generate: --max-context %lld cannot hold %zu prompt + %lld new tokens\n",
|
||||
(long long) o.max_context, o.tokens.size(), (long long) o.max_new);
|
||||
@@ -1173,6 +1305,28 @@ int main(int argc, char** argv) {
|
||||
std::fprintf(stderr, "strata generate: profile %s: %zu ranked pairs, built for %lld slots\n",
|
||||
o.expert_profile.c_str(), profile.size(), (long long) pslots);
|
||||
}
|
||||
// THE HEAD BEFORE THE CACHE. The expert cache takes what is free minus the reserve, so everything allocated
|
||||
// after it comes out of the reserve. The native head (~0.5 GB with IQ3_S) was loaded after it and ate most of
|
||||
// the 700 MiB: 128K IQ3_S ended with 30 MiB free, the driver paged, and a request stalled for good at its first
|
||||
// verify window. Loaded first, the cache is sized around it.
|
||||
const strata::core::WeightRef* wo = wt.find("output.weight");
|
||||
if (wo == nullptr) { std::fprintf(stderr, "strata generate: output.weight is missing\n"); return 1; }
|
||||
const int64_t n_vocab = wo->ne1;
|
||||
strata::core::NativeHead native_head;
|
||||
if (!o.native_head_gguf.empty()) {
|
||||
if (!native_head.load(o.native_head_gguf, g.n_embd, n_vocab, err)) {
|
||||
std::fprintf(stderr, "strata generate: %s\n", err.c_str());
|
||||
return 1;
|
||||
}
|
||||
std::fprintf(stderr, "strata generate: experimental native Q5_K head, %llu bytes\n",
|
||||
(unsigned long long) native_head.weight_bytes());
|
||||
}
|
||||
std::vector<float> logits((size_t) n_vocab);
|
||||
float* d_logits = nullptr;
|
||||
if (cudaMalloc(&d_logits, (size_t) n_vocab * 4) != cudaSuccess) {
|
||||
std::fprintf(stderr, "strata generate: the logits buffer failed\n");
|
||||
return 1;
|
||||
}
|
||||
const bool auto_cache = o.expert_cache < 0;
|
||||
if (o.expert_cache < 0) {
|
||||
size_t free_b = 0, total_b = 0;
|
||||
@@ -1481,25 +1635,7 @@ int main(int argc, char** argv) {
|
||||
}
|
||||
|
||||
mem_mark("the expert cache and the graphs");
|
||||
std::fprintf(stderr, "strata generate: session is up; locating the head\n");
|
||||
const strata::core::WeightRef* wo = wt.find("output.weight");
|
||||
if (wo == nullptr) { std::fprintf(stderr, "strata generate: output.weight is missing\n"); return 1; }
|
||||
const int64_t n_vocab = wo->ne1;
|
||||
strata::core::NativeHead native_head;
|
||||
if (!o.native_head_gguf.empty()) {
|
||||
if (!native_head.load(o.native_head_gguf, g.n_embd, n_vocab, err)) {
|
||||
std::fprintf(stderr, "strata generate: %s\n", err.c_str());
|
||||
return 1;
|
||||
}
|
||||
std::fprintf(stderr, "strata generate: experimental native Q5_K head, %llu bytes\n",
|
||||
(unsigned long long) native_head.weight_bytes());
|
||||
}
|
||||
std::vector<float> logits((size_t) n_vocab);
|
||||
float* d_logits = nullptr;
|
||||
if (cudaMalloc(&d_logits, (size_t) n_vocab * 4) != cudaSuccess) {
|
||||
std::fprintf(stderr, "strata generate: the logits buffer failed\n");
|
||||
return 1;
|
||||
}
|
||||
std::fprintf(stderr, "strata generate: session is up\n");
|
||||
auto run_head = [&](void* stream) -> bool {
|
||||
if (!native_head.loaded())
|
||||
return strata::core::lm_head(wt, g, ss.block, d_logits, stream, err);
|
||||
@@ -2035,6 +2171,7 @@ int main(int argc, char** argv) {
|
||||
std::vector<ImgKey> live_imgs, req_imgs;
|
||||
bool live_ok = false;
|
||||
std::vector<ConvCheckpoint> checks;
|
||||
bool cvec_cached = true; // the control vector's state the live session and the checkpoints were read with
|
||||
int64_t pp_total = 0, pp_from = 0, pp_next_check = 0;
|
||||
Clock::time_point pp_t0 = Clock::now();
|
||||
auto imgs_below = [&](const std::vector<ImgKey>& all, int64_t L) {
|
||||
@@ -2174,21 +2311,29 @@ int main(int argc, char** argv) {
|
||||
// and a page-in while the verify graph spins on a host flag stalls the request for good
|
||||
size_t free_b = 0, total_b = 0;
|
||||
cudaMemGetInfo(&free_b, &total_b);
|
||||
std::fprintf(stderr, "strata serve: %lld MiB of VRAM free with everything loaded%s\n",
|
||||
(long long) (free_b >> 20), free_b < ((size_t) 128 << 20)
|
||||
? " - LOW: requests may stall; lower --max-context or raise --vram-reserve-mib" : "");
|
||||
// below ~256 MiB a later allocation (a first-used window's buffers, the desktop, another program) can make
|
||||
// the driver page GPU memory, and a verify graph spinning on a host flag then never finishes
|
||||
const int64_t free_mib = (int64_t) (free_b >> 20);
|
||||
if (free_mib >= 256) {
|
||||
std::fprintf(stderr, "strata serve: %lld MiB of VRAM free with everything loaded\n", (long long) free_mib);
|
||||
} else {
|
||||
std::fprintf(stderr, "strata serve: %lld MiB of VRAM free with everything loaded - LOW: requests may stall;"
|
||||
" add --vram-reserve-mib %lld to the config's args (or lower --max-context)\n",
|
||||
(long long) free_mib, (long long) (o.vram_reserve_mib + 512 - free_mib));
|
||||
}
|
||||
}
|
||||
// what the server's Monitor tab shows (servers before 0.1.8 skip unknown lines until READY)
|
||||
{
|
||||
size_t free_b = 0, total_b = 0;
|
||||
cudaMemGetInfo(&free_b, &total_b);
|
||||
std::printf("INFO context=%lld kv=%s kv_resident=%lld expert_slots=%lld expert_cache_mib=%lld spec=%d "
|
||||
"mtp_max=%d lookup=%d vram_free_mib=%lld\n",
|
||||
"mtp_max=%d lookup=%d vram_free_mib=%lld cvec=%s arena_mib=%lld\n",
|
||||
(long long) o.max_context, o.kv.c_str(),
|
||||
(long long) (g.n_qsa_layers() > 0 && ss.qsa_states[0].kv_mode == 1
|
||||
? ss.qsa_states[0].n_slots * 4 : 0),
|
||||
(long long) xcache.slots(), (long long) (xcache.bytes() >> 20), o.spec, o.mtp_max_t,
|
||||
o.suffix_draft, (long long) (free_b >> 20));
|
||||
o.suffix_draft, (long long) (free_b >> 20), cvec_summary.c_str(),
|
||||
(long long) (strata::kernels::cpu::expert_layout().total >> 20));
|
||||
}
|
||||
std::printf("READY %lld stop\n", (long long) o.max_context); // "stop": this engine honours STOP
|
||||
std::fflush(stdout);
|
||||
@@ -2225,7 +2370,8 @@ int main(int argc, char** argv) {
|
||||
unsigned long long req_seed = 0;
|
||||
float req_min_p = 0.0f, req_penalty_repeat = 1.0f, req_penalty_freq = 0.0f, req_penalty_present = 0.0f;
|
||||
int req_penalty_last_n = 0;
|
||||
if (!geni && endp != nullptr) {
|
||||
int req_cvec = 1; // cvec=0|1: a loaded control vector for this request (on when absent)
|
||||
if (endp != nullptr) { // GENI takes only cvec=; its file path is the first token without an =
|
||||
for (;;) {
|
||||
while (*endp == ' ') ++endp;
|
||||
const char* start = endp;
|
||||
@@ -2236,7 +2382,9 @@ int main(int argc, char** argv) {
|
||||
if (eq == std::string::npos) { endp = const_cast<char*>(start); break; }
|
||||
const std::string key = tok.substr(0, eq);
|
||||
const float fv = std::strtof(tok.c_str() + eq + 1, nullptr);
|
||||
if (key == "temperature") req_temperature = fv;
|
||||
if (key == "cvec") req_cvec = std::atoi(tok.c_str() + eq + 1);
|
||||
else if (geni) {} // image requests decode greedily
|
||||
else if (key == "temperature") req_temperature = fv;
|
||||
else if (key == "top_p") req_top_p = fv;
|
||||
else if (key == "top_k") req_top_k = std::atoi(tok.c_str() + eq + 1);
|
||||
else if (key == "min_p") req_min_p = fv;
|
||||
@@ -2362,6 +2510,17 @@ int main(int argc, char** argv) {
|
||||
if ((int32_t) ids[(size_t) i] != pre[(size_t) i]) return false;
|
||||
return imgs_below(req_imgs, L) == pre_imgs;
|
||||
};
|
||||
// the control vector for this request. The live session and the checkpoints were read one way, so a
|
||||
// switch reads the prompt again from the start
|
||||
if (strata::kernels::cvec().loaded()) {
|
||||
const bool want = req_cvec != 0;
|
||||
if (want != cvec_cached) {
|
||||
live_ok = false;
|
||||
checks.clear();
|
||||
cvec_cached = want;
|
||||
}
|
||||
strata::kernels::cvec_set_enabled(want);
|
||||
}
|
||||
int64_t resume = 0;
|
||||
bool from_live = false;
|
||||
if (o.prompt_cache > 0) {
|
||||
|
||||
Reference in new issue
Block a user