Niko1221/Strata
1
0 Fork 0
mirror of https://github.com/Niko1221/Strata.git synced 2026-11-10 17:33:53 -10:00
Code Issues Packages Projects Releases 1 Wiki Activity
Files
da170f8ae7c326a101ae7b50f43f5e46d131b376
Strata/serve/server.py
T

4374 lines
248 KiB
Python
Raw Blame History

"""serve/server.py - plan v0.3 P8: OpenAI and Anthropic endpoints over any engine that maps token ids to tokens.
python -m serve.server --engine mock --port 8095 (a scripted engine, for clients and tests)
python -m serve.server --engine strata --config strata.json --port 8080 (the real engine, resident)
Endpoints: POST /v1/chat/completions (OpenAI, stream and non-stream), POST /v1/messages (Anthropic, stream and
non-stream), GET /v1/models, GET /models, GET /props, GET /slots, GET /health, GET /mcp. One sequence at a time behind a FIFO (plan: one resident sequence).
Tools from MCP servers (serve/mcp.py, `"mcp_servers"` in the config or --mcp-config) are offered only to requests that
ask for them with `"strata_mcp": true` - the web app does; other clients see exactly the API they always saw.
Images (optional, when the config has a "vision" entry): OpenAI image_url parts and Anthropic image blocks (base64
data, http(s) URLs or local file paths) go through `strata-vision` (the model's mmproj file) and reach the engine as
embeddings (`GENI`). JPEG/PNG/BMP/GIF go straight in; WebP, TIFF, AVIF, ... (agents like omp send WebP) are
converted to PNG first with Pillow.
Requests whose prompt plus max tokens exceed the engine's context are REJECTED with 400, never truncated.
An unset (or 0, or -1) max tokens means "unlimited": whatever the prompt leaves of the context.
The engine boundary is `Engine.generate(prompt_ids, max_new, sampling, cancel) -> iterator of token ids`.
`StrataEngine` keeps one `strata --serve` process resident (weights, expert arena and VRAM tier load once) and
talks to it over stdin/stdout; `MockEngine` is a scripted stand-in that makes every API path testable without a GPU.
"""
from __future__ import annotations
import argparse
import contextlib
import collections
import base64
import hashlib
import hmac
import codecs
import ctypes
import json
import os
import queue
import re
import select
import signal
import socket
import struct
import subprocess
import sys
import tempfile
import threading
import time
import urllib.request
import uuid
from http.client import HTTPException
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Iterator, Protocol
from urllib.parse import parse_qs, urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tools"))
sys.path.insert(0, str(ROOT)) # run as a script (run-<model>.bat) as well as a module
from serve.frontend import (ChatTemplate, Event, OutputParser, anthropic_to_messages, # noqa: E402
images_of, mark_think_literals, openai_to_messages, unmark_think_literals)
from serve.mcp import McpCancelled, hub_from_config # noqa: E402
from serve import runconfig # noqa: E402
from serve.winjob import contain # noqa: E402
from serve.structured import StructuredOutputError, prepare_format, validated_json # noqa: E402
from serve import responses as responses_api # noqa: E402
from serve.responses import ResponsesError, error_body as responses_error_body # noqa: E402
IM_END = "<|im_end|>"
IMAGE_PAD = "<|image_pad|>"
# #458: the trailing effort turn ("effort_position": "end"); low is the template's own sentence, medium (which the
# template says nothing for: no xhigh sentence is medium there) says it, since the xhigh one stays at the top
EFFORT_TURN = "<|im_start|>system\n{}<|im_end|>\n"
EFFORT_TEXT = {"low": "Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the "
"conclusion without unnecessary elaboration.",
"medium": "Reasoning effort is set to medium. Think as much as the task needs, without unnecessary "
"elaboration."}
VISION_START = "<|vision_start|>"
# #606: a reply that repeats one token this many times in a row is ended there ("length"): a model in a loop, or a
# broken state that answers one token forever (an issue saw 36,689 tokens of "!"). The config's "repeat_stop_tokens"
# sets it; 0 turns it off.
REPEAT_STOP_TOKENS = 256
# #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)
REASONING_WRAP_UP = "\n\nI have thought about this long enough; time to give my answer.\n</think>\n\n"
LOOPBACK_NAMES = ("localhost", "127.0.0.1", "::1")
CTX_SLACK = 8 # `strata --serve` rejects prompt + max_new + 8 > context: keep the same margin here
# The live tok/s is a rate over a window, not a mean since the first token: a mean reads ~1/elapsed at the first
# token (the Monitor showed five-digit numbers) and then undershoots for the first second of every answer.
RATE_WINDOW_S = 2.0
RATE_MIN_SPAN_S = 0.25 # younger than this there is no rate yet: the mean so far, with the span floored here
# #481: a running request whose engine prints nothing (no T, PP or any other line) for this long has lost step with the
# server (the engine's main thread waits, untimed, for its next command): the engine is ended and the request fails;
# the next request starts it again. The config's "engine_silence_s" sets it (0: wait forever, as before).
ENGINE_SILENCE_S = 300.0
# ... except while a prompt is read: a PP line comes once per chunk (up to 32768 tokens with --prefill auto, issue
# #282), and the slowest PCs read ~100 tok/s, so a first chunk can take minutes before the first line. Until the first
# PP the wait adds the chunk's tokens at PP_FLOOR_TOK_S; after one, a chunk may take PP_SLACK x the last one's time.
PP_CHUNK_MAX = 32768
PP_FLOOR_TOK_S = 50.0
PP_SLACK = 3.0
# ------------------------------------------------------------------------------------------------ engines
class Engine(Protocol):
max_context: int
def generate(self, ids: list[int], max_new: int, sampling: dict, cancel: threading.Event) -> Iterator[int]: ...
class MockEngine:
"""Replays a scripted completion (text) as token ids, one per step, then the end-of-turn token. Given a list of
scripts, each request gets the next one and the last one repeats (a tool call, then the answer after it)."""
def __init__(self, tokenizer, script: str | list[str], max_context: int = 32768, delay_s: float = 0.0):
self.tok, self.max_context, self.delay = tokenizer, max_context, delay_s
end = tokenizer.encode(IM_END, parse_special=True)
self.scripts = [tokenizer.encode(x, parse_special=True) + end for x in ([script] if isinstance(script, str)
else script)]
self.script, self.turns = self.scripts[0], 0
self.last_prompt: list[int] = []
def generate(self, ids, max_new, sampling, cancel, embeddings=None):
self.last_prompt = list(ids)
self.last_embeddings = embeddings
if len(self.scripts) > 1:
self.script = self.scripts[min(self.turns, len(self.scripts) - 1)]
self.turns += 1
for t in self.script[:max_new]:
if cancel.is_set():
return
if self.delay:
time.sleep(self.delay)
yield t
class EngineDied(RuntimeError):
"""The engine process ended in the middle of a request (issue #27: on Linux, the out-of-memory killer)."""
class EngineStarting(RuntimeError):
"""The engine is (re)starting and has not said READY yet (#344): no context size to plan a request with - a 503,
not a 400 about the prompt."""
class ModelBusy(RuntimeError):
"""Explicit model controls must not interrupt active or queued requests."""
class EngineSilent(EngineDied):
"""#481: the engine said nothing for too long during a request (or never acknowledged a STOP): the two sides lost
step - the engine waiting for its next command, the server for this request's end - and the server ended it. An
EngineDied, so the request ends with an error and the next one starts the engine again."""
class EngineStuck(RuntimeError):
"""The engine process did not end after QUIT, terminate and kill: the server keeps it (and says so) rather than
reporting its GPU and RAM as given back."""
class GpuBusy(RuntimeError):
"""The model is unloaded and the GPU has less free VRAM than min_free_vram_mib: something else (a game, another
model server) is using it, so the engine is not started into the little that is left."""
ENGINE_REQUEST = re.compile(
# "+ 12288 of 98179 read": a request cancelled while its prompt was read (#471)
r"prompt (?P<prompt>\d+) tokens = (?P<reused>\d+) reused \+ \d+(?: of \d+)? read in (?P<read>[\d.]+) ms "
r"\((?P<pp>[\d.]+) tok/s\), "
r"(?P<gen>\d+) generated in (?P<gen_ms>[\d.]+) ms \((?P<tg>[\d.]+) tok/s\)")
_echoing: set[str] = set() # the logs echo_requests already follows (restart() runs StrataEngine.__init__ again)
DRAFT_HEAD_FAIL = "the draft head does not fit"
DRAFT_HEAD_HINT = ("a smaller draft vocabulary needs less VRAM: --draft-vocab cyrillic (English, code and the Cyrillic "
"script) or --draft-vocab en (English and code, ~215 MiB less than the default). Start once with "
"it - START-HERE.bat --draft-vocab en (Windows) or ./setup.sh --draft-vocab en - and the model "
"keeps it; or a smaller --context in setup.")
def start_failure_hint(log: str | None, offset: int) -> str:
"""#474: what to change when the engine stopped at the start because the MTP draft head did not fit the VRAM
left: the engine's own `strata mtp:` lines after that failure (0.1.36+: what it needs, what is free, the smaller
subsets), else the same advice in words for an older engine. "" for any other failure: the log says why."""
if not log:
return ""
try:
with open(log, "rb") as f:
f.seek(offset)
text = f.read().decode("utf-8", "replace")
except (OSError, ValueError):
return ""
if DRAFT_HEAD_FAIL not in text:
return ""
said = [x.strip()[len("strata mtp: "):] for x in text.splitlines()
if x.strip().startswith("strata mtp: ") and ("draft head over" in x or "hint:" in x)]
return ". mtp: " + DRAFT_HEAD_FAIL + ". " + (" ".join(said) if said else "Hint: " + DRAFT_HEAD_HINT)
def start_log_tail(log: str | None, offset: int, n: int = 20) -> str:
"""#496: the last n lines this start wrote to the engine log, for the error when the engine ended before READY -
whatever the failure, the engine's own reason is in them (people posted the traceback without the log). "" when
there is no log or nothing in it from this start."""
if not log:
return ""
try:
with open(log, "rb") as f:
f.seek(0, 2)
start = max(offset, f.tell() - 64 * 1024) # enough for 20 lines, never an earlier start's
f.seek(start)
text = f.read().decode("utf-8", "replace")
except (OSError, ValueError):
return ""
lines = text.splitlines()[1 if start > offset else 0:] # not a line cut in half
lines = [x.rstrip() for x in lines if x.strip()][-n:]
if not lines:
return ""
return "\nthe engine log's last lines:\n" + "\n".join(" " + x for x in lines)
def echo_requests(log_path: str, offset: int) -> None:
"""STRATA_REQUEST_LINES=1: one stdout line per finished request, from the engine's own summary in its log.
The engine's stderr goes to the log file (the start narrator reads it), so a supervisor that only sees this
process's output - a tray, llama-swap - has no per-request numbers. This re-states the engine's line with the
total the two times make: `request prompt P cached C output O prompt_read R ms total S ms prefill X tok/s decode Y
tok/s` (prompt_read: the time the engine spent reading the prompt's new tokens, not a time to first token).
"""
with open(log_path, "r", encoding="utf-8", errors="replace") as f:
f.seek(offset)
while True:
line = f.readline()
if not line:
time.sleep(0.2)
continue
m = ENGINE_REQUEST.search(line)
if m:
read_ms, gen_ms = float(m["read"]), float(m["gen_ms"])
print("[strata] request prompt %s cached %s output %s prompt_read %.0f ms total %.0f ms prefill %s "
"tok/s decode %s tok/s" % (m["prompt"], m["reused"], m["gen"], read_ms, read_ms + gen_ms, m["pp"],
m["tg"]), flush=True)
def experts_loading_words(args: list, size: str) -> str:
"""#505: what the start does with the experts, by the engine's flags (generate.cpp's option parsing): a RAM budget
copies the hottest N GiB into RAM (--resident-budget-gib), the resident low-RAM mode the ones the GPU does not hold
(--resident-experts), plain --mmap-experts reads them from the model files through the OS file cache (nothing is
loaded into RAM up front); otherwise all of them go into RAM."""
if "--resident-budget-gib" in args:
try:
n = f"up to {float(args[args.index('--resident-budget-gib') + 1]):g} GiB"
except (IndexError, ValueError):
n = "a RAM budget"
return (f"loading the most-used experts into RAM ({n}; the rest are read from the model files as needed) "
"and locking part of them for the GPU.")
if "--resident-experts" in args:
return (f"loading the experts the GPU does not hold into RAM (of {size}) and locking part of them for the "
"GPU.")
if "--mmap-experts" in args:
return (f"mapping the experts from the model files ({size}, --mmap-experts): they are not loaded into RAM - "
"the OS file cache reads them as the GPU's expert cache fills and as requests need them.")
return f"loading the experts into RAM ({size}) and locking part of them for the GPU."
def narrate_start(log_path: str, offset: int, args: list, done: threading.Event, heartbeat=20.0) -> None:
"""While the engine starts, say in the server window what it is doing, from its log: the start reads tens of GB
into RAM and locks part of it for the GPU, and on many PCs everything is slow or frozen for a minute or more -
people closed the window thinking it had hung. The warning comes at that step, not after it."""
gb = 0.0
if "--native" in args: # about the size of the experts it will read
try:
gb = os.path.getsize(args[args.index("--native") + 1]) / 1e9
except (OSError, IndexError):
pass
size = f"about {gb:.0f} GB" if gb >= 1 else "tens of GB"
loading = experts_loading_words(args, size)
t0 = last = time.time()
said = set()
def say(key, text):
nonlocal last
if key not in said:
said.add(key)
last = time.time()
print(text, flush=True)
say("weights", "[strata] starting the engine: reading the model's weights ...")
pos = offset
while not done.wait(0.5):
try:
with open(log_path, "rb") as f:
f.seek(pos)
chunk = f.read()
except OSError:
chunk = b""
if chunk.count(b"\n"):
cut = chunk.rfind(b"\n") + 1
pos += cut
for line in chunk[:cut].decode("utf-8", "replace").splitlines():
if "PLE on" in line or "expert arena:" in line or "experts via mmap" in line: # #505: mapped
say("arena", f"[strata] {loading}\n"
" YOUR PC CAN BE SLOW OR STOP RESPONDING FOR 1-3 MINUTES NOW - this is normal.\n"
" Please wait and don't close this window; the browser opens when it is ready.")
elif " loaded " in line and "GiB at" in line:
say("loaded", "[strata] experts loaded: " + line.split(" loaded ", 1)[1].strip() +
f" ({time.time() - t0:.0f} s so far)")
elif "expert cache " in line and " slots, " in line and "auto" not in line:
n = line.split("expert cache ", 1)[1].split(";")[0].replace(" slots,", " experts,").strip()
say("cache", f"[strata] filling the GPU's expert cache ({n}) ...")
elif "session is up" in line:
say("up", "[strata] almost ready ...")
if time.time() - last > heartbeat:
last = time.time()
print(f"[strata] still starting ({time.time() - t0:.0f} s) - please wait ...", flush=True)
class ConvCacheLog:
"""#596: the engine's conversation cache as its log tells it (the engine writes "strata serve: conversation
cache: parked N tokens ...; parked=P bytes=B evictions=E" and "restored N tokens ...; parked=P bytes=B" to stderr,
which is the log): read on from where it was last read, from the start of the engine's current run."""
EVENT = re.compile(r"conversation cache: (parked|skipped|restored) (\d+) tokens.*?parked=(\d+) bytes=(\d+)"
r"(?: evictions=(\d+))?")
DROPPED = re.compile(r"conversation cache: dropped \d+ superseded .*?parked=(\d+)")
READ_MAX = 1 << 20 # at most the last MiB of new lines per read
def __init__(self):
self.key, self.pos = None, 0
self.reset()
def reset(self):
self.state = {"parked": 0, "bytes": 0, "evictions": 0, "parks": 0, "restores": 0, "last_event": None,
"last_tokens": None, "last_at": None}
def poll(self, path, start) -> dict:
"""The state after the log's new lines; `start` is where the engine's current run began in it."""
if not path or start is None:
return dict(self.state)
if self.key != (path, start): # another start of the engine: its cache starts empty
self.key, self.pos = (path, start), start
self.reset()
try:
size = os.path.getsize(path)
if size < self.pos: # the log was emptied or replaced
self.pos = 0
if size - self.pos > self.READ_MAX:
self.pos = size - self.READ_MAX
with open(path, "rb") as f:
f.seek(self.pos)
data = f.read(size - self.pos)
except OSError:
return dict(self.state)
end = data.rfind(b"\n") + 1 # whole lines only: the rest is read next time
self.pos += end
now = time.time()
for line in data[:end].decode("utf-8", "replace").splitlines():
if "conversation cache:" not in line:
continue
m = self.EVENT.search(line)
if m:
st = self.state
st["parked"], st["bytes"] = int(m.group(3)), int(m.group(4))
if m.group(5) is not None:
st["evictions"] = int(m.group(5))
if m.group(1) != "skipped":
st["parks" if m.group(1) == "parked" else "restores"] += 1
st["last_event"], st["last_tokens"], st["last_at"] = m.group(1), int(m.group(2)), now
continue
m = self.DROPPED.search(line)
if m:
self.state["parked"] = int(m.group(1))
return dict(self.state)
def conversation_cache_view(info: dict, hist: list, totals: dict, parked: dict) -> dict:
"""#596: the Monitor's Conversation cache card: the parked conversations (the engine's opt-in
--conversation-cache-mib: budget, slots, what its log says) and how much of the prompts the cache gave back."""
mib = info.get("conversation_cache_mib")
last = hist[-1] if hist else None
return {"enabled": isinstance(mib, int) and mib > 0, "budget_mib": mib if isinstance(mib, int) else None,
"slots": info.get("conversation_cache_slots"), **parked,
"requests": len(hist), "requests_reused": sum(1 for r in hist if (r.get("reused") or 0) > 0),
"reused_tokens": totals.get("reused", 0), "prompt_tokens": totals.get("prompt_tokens", 0),
"last_reused": last.get("reused") if last else None,
"last_prompt": last.get("prompt_tokens") if last else None}
_BTRACE = bool(os.environ.get("STRATA_BATCH_TRACE"))
def btrace(*a):
if _BTRACE:
print("[batch-trace]", threading.get_ident() % 10000, *a, file=sys.stderr, flush=True)
EOS_IDS = {248044, 248046} # <|endoftext|>, <|im_end|>: the engine's default --eos-ids
# The engine's own words for a verifier that cannot go on (its GPU waits were released, or an earlier window never
# finished, #267): the process may stay alive and even print DONE, but it must not take another request.
FATAL_PREFIXES = ("ERR verify: timed out at layer ", "ERR verify batch: timed out at layer ",
"ERR verify: an earlier window never finished", "ERR verify batch: an earlier window never finished")
class StrataEngine:
"""The resident engine: `strata --serve` reads `GEN <max_new> <ids>` lines and streams `T <id>` lines, then
`DONE ...`. Requests are serialized by the service's FIFO, so one pipe is enough.
Per-request sampling rides the same line as engine-side keys between max_new and the ids
(`temperature=F top_p=F top_k=N seed=N`, the engine's own spelling). An absent temperature keeps the
engine's default, which is greedy; `temperature=0` means the same thing, so it is not forwarded.
"""
silence_s = ENGINE_SILENCE_S # #481: main() sets the config's engine_silence_s (survives restart())
silent_note = None # #481: why the server ended a silent engine (death_note says it)
batch = 0 # --batch: the engine's batch slots (0: one request at a time)
# `last`: the figures of the engine's last DONE line. With batch slots several requests run at once, each on its
# own thread, so each one reads its own request's figures (Service.run records them); one at a time, the one dict.
@property
def last(self):
if self.batch:
tl = self.__dict__.setdefault("_tl", threading.local())
if not hasattr(tl, "last"):
tl.last = {}
return tl.last
return self.__dict__.get("_last", {})
@last.setter
def last(self, value):
self.__dict__["_last"] = value
if self.batch:
tl = self.__dict__.setdefault("_tl", threading.local())
tl.last = value
def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | None = None,
env: dict | None = None, lazy: bool = False):
self.spawn = (exe, list(args), cwd, log, env) # to start it again after it died (issue #27)
paths = {k: v for k, v in zip(args, args[1:]) if k in ("--native", "--pack")}
self.model_path = paths.get("--native") or paths.get("--pack", "pack/full")
self.log_path = log
self.proc, self.pump, self.log = None, None, None
self.ended, self.unloaded = True, True
self.max_context = int(args[args.index("--max-context") + 1]) if "--max-context" in args else 4096
self.can_stop = False # the engine honours a STOP line mid-request (READY <ctx> stop)
self.last = {}
self.info = {} # INFO key=value facts (engine 0.1.8+): kv, expert slots, ... (Monitor tab)
self.prefill_tok_s_mean = None
self.progress = None # (read, total) prompt tokens while a prompt is read, from PP lines
self.silent_note = None
try: # a ready-made engine's BUILD.json says its version
self.info["version"] = json.loads((Path(exe).parent / "BUILD.json").read_text()).get("version")
except (OSError, ValueError):
self.info["version"] = None
if lazy:
return
self.unloaded = False # `ended` stays True until READY (below): not alive while starting (#344)
self.log = open(log, "a", encoding="utf-8") if log else subprocess.DEVNULL
loading = threading.Event() # set once READY: the narrator below stops
log_start = os.path.getsize(log) if log else 0 # where this start's lines begin (start_failure_hint)
self.log_start = log_start # #596: the Monitor's conversation cache reads from here
if log:
threading.Thread(target=narrate_start, args=(log, os.path.getsize(log), args, loading),
daemon=True).start()
# once per log: the follower keeps reading the same (appended) log across restarts and reloads
if os.environ.get("STRATA_REQUEST_LINES") and os.path.abspath(log) not in _echoing:
_echoing.add(os.path.abspath(log))
threading.Thread(target=echo_requests, args=(log, os.path.getsize(log)), daemon=True).start()
self.proc = subprocess.Popen([exe, "--serve", *args], cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=self.log, text=True, encoding="utf-8", bufsize=1, env=env)
contain(self.proc) # ends with the server, however it ends (Windows)
self.max_context = 0
for line in self.proc.stdout:
if line.startswith("INFO "):
for kv in line.split()[1:]:
k, _, v = kv.partition("=")
self.info[k] = int(v) if v.lstrip("-").isdigit() else v
if line.startswith("READY"):
f = line.split()
self.max_context = int(f[1])
self.can_stop = "stop" in f[2:]
break
loading.set()
if self.max_context <= 0:
try: # its pipes and our handle on its log (the log stays)
self.proc.wait(timeout=5)
self.proc.stdin.close()
self.proc.stdout.close()
if log:
self.log.close()
except (OSError, subprocess.TimeoutExpired):
pass
raise RuntimeError("the engine exited before it was ready" + (f" (see {log})" if log else "") +
start_failure_hint(log, log_start) + start_log_tail(log, log_start))
self.known_ctx = self.max_context # survives a failed restart: requests keep their limit and restart it
# (from PR #41, midhatn) a locally built engine can sit next to another release's BUILD.json: engines that
# report their own version (INFO engine=, 0.1.8+) win, the manifest stays the fallback for older ones
if self.info.get("engine"):
self.info["version"] = str(self.info["engine"])
self.ended = False # READY: alive from here (restart() set it True, #344)
# the engine's stdout on a thread, so a request can wait with a timeout (heartbeats, cancel checks)
self.lines: queue.Queue = queue.Queue()
# --batch: concurrent requests in the engine's batch slots (see generate_batched). The engine says how many
# it runs (INFO batch_slots=N: it may have fewer than asked, or none, when they do not fit)
asked = next((int(args[args.index(k) + 1]) for k in ("--batch", "--slots") if k in args), 0)
self.batch = int(self.info.get("batch_slots") or 0)
if asked and self.batch != asked:
print(f"[strata] parallel requests: {asked} asked, the engine runs {self.batch or 'one at a time'} "
"(its log says why)", flush=True)
groups = int(args[args.index("--batch-groups") + 1]) if "--batch-groups" in args else 1
groups = groups if self.batch and groups > 0 and self.batch % groups == 0 else 1
gs = self.batch // groups if self.batch else 0
# slots in the order that spreads requests over the pipeline's groups first: 0, gs, 2gs, .., 1, gs+1, ..
self.slot_order = [g * gs + t for t in range(gs) for g in range(groups)]
self.slot_q = [queue.Queue() for _ in range(self.batch)]
self.slot_busy = [False] * self.batch
# what each slot's sessions hold (prompt + every token a window fed), so a conversation's next turn goes to
# the slot that has its start (the engine checks it again); when the slot was last used
self.slot_held: list[list[int]] = [[] for _ in range(self.batch)]
self.slot_used = [0.0] * self.batch
self.slot_live: list[dict | None] = [None] * self.batch # /metrics: the request in each slot
self.slot_cv = threading.Condition()
self.waiting = 0 # requests waiting for the control lines (ctl)
self.wait_lens: list[list[int]] = [] # ... their prompt lengths (a long read gives way to short ones)
self.ctl_epoch = 0 # how often the control lines were taken
self._yielded = None # (slot, tokens read): the last request on them gave way
self.ctl = threading.Lock() # one admission or solo request on the control lines at a time
self.wlock = threading.Lock() # stdin writes from several request threads
self.pump = threading.Thread(target=self._pump, daemon=True)
self.pump.start()
def _pump(self):
proc, lines = self.proc, self.lines # this process's: a restart replaces both (#344)
slot_q = self.slot_q
for line in proc.stdout:
# checked before batch routing: a fatal line is never a slot's own
if line.startswith(FATAL_PREFIXES):
# release_gpu_waits invalidates the verifier, even if the native
# process stays alive and prints DONE afterwards.
if self.proc is proc:
self.silent_note = "Unrecoverable native verification failure: " + line[4:].strip()
self.ended = True
try:
proc.terminate()
except OSError:
pass
break # publish EOF, never the trailing DONE
if line.startswith(("BT ", "BDONE ")) and slot_q: # --batch: a batch slot's own lines
try:
slot_q[int(line.split()[1])].put(line)
continue
except (IndexError, ValueError):
pass
lines.put(line)
if self.proc is proc: # a killed engine's pump must not mark its successor dead
self.ended = True # its output closed: it is gone, even before the OS says so
lines.put(None)
for q in slot_q:
q.put(None)
def death_note(self) -> str:
"""Why the engine most likely ended, from the end of its log: its own watchdog (issue #29), else RAM."""
if getattr(self, "silent_note", None): # #481: the server ended it, not the OS or the engine itself
return self.silent_note
tail = ""
try:
with open(self.log_path, "rb") as f:
f.seek(0, 2)
f.seek(max(0, f.tell() - 4096))
tail = f.read().decode("utf-8", "replace")
except (OSError, TypeError):
pass
for line in reversed(tail.splitlines()):
if "issue #29" in line:
return ("The engine stopped itself because it had stopped making progress - a hang it caught. Its log "
"line: " + line.strip() + " - please report it at github.com/Niko1221/Strata/issues.")
rc = self.proc.poll()
last = next((x.strip() for x in reversed(tail.splitlines()) if x.strip().startswith(("strata", "ERR"))), "")
if rc is not None and rc >= 0 and last: # it ended by itself: its own last words say why (#215)
return (f"The engine exited (code {rc}). Its last log line: {last} - if that does not explain it, please "
"report it at github.com/Niko1221/Strata/issues with the log.")
return ("The usual cause is running out of RAM: Linux then ends the biggest program (check: sudo dmesg | "
"grep -i -E 'killed process|out of memory'); Windows slows down instead. Close other programs or use a "
"smaller model (Q2_0 / IQ2_XS).")
def alive(self) -> bool:
return self.proc is not None and not getattr(self, "ended", False) and self.proc.poll() is None
def exit_code(self):
if self.proc is None:
return None
try:
return self.proc.wait(timeout=5)
except subprocess.TimeoutExpired:
return None
def unload(self):
"""Release GPU/RAM between requests, retaining the spawn config for automatic reloading."""
self.close()
self.unloaded = True
RESTART_RETRY_S = 15.0 # between the tries of restart(): a dying engine's VRAM may take a while to come back
def restart(self, tries: int = 3):
"""Start the engine again (the same command) after it died; the new process has its own line queue.
close() ends and waits for the old process first (EngineStuck when it cannot be ended). A start that still
exits before READY - a dead engine's VRAM can take a while to come back, notably on ROCm - is retried
(PR #637)."""
self.close()
info = dict(self.info)
self.starting = True # prepare() answers 503 "starting" meanwhile (#344)
try:
for i in range(tries):
# #344: not alive until READY - __init__ sets max_context to 0 and blocks until the engine says
# READY, and a request that saw alive() in that window skipped load() and failed with "context
# (0)". __init__ clears `ended` itself once READY (before its pump thread can set it again).
self.ended = True
try:
self.__init__(*self.spawn)
break
except RuntimeError:
try:
self.proc.wait(timeout=60)
except (subprocess.TimeoutExpired, OSError):
pass
if i == tries - 1:
raise
print(f"[strata] the engine did not start (try {i + 1} of {tries}); again in "
f"{self.RESTART_RETRY_S:g} s", flush=True)
time.sleep(self.RESTART_RETRY_S)
finally:
self.starting = False
self.info = {**info, **self.info}
def _parse_done(self, line):
f = line.split()
self.last = {"generated": int(f[1]), "prompt_tokens": int(f[2]), "prompt_ms": float(f[3]),
"decode_ms": float(f[4]), "finish": f[5]}
if len(f) >= 9: # the conversation cache's fields (engine 0.1.3+)
self.last.update(drafts_accepted=int(f[6]), drafts_offered=int(f[7]), reused=int(f[8]))
if len(f) >= 11: # decode hit rate fields
self.last.update(hits=int(f[9]), lookups=int(f[10]))
if len(f) >= 14: # the expert tiers (engine 0.1.31+): RAM / file blobs, file MB
self.last.update(ram_blobs=int(f[11]), file_blobs=int(f[12]), file_mb=float(f[13]))
if len(f) >= 15: # #471 (engine 0.1.36+): the prompt tokens actually read
self.last.update(prompt_read=int(f[14]))
if len(f) >= 16: # #588 (engine 0.1.39+): routed experts read over PCIe
self.last.update(offloaded=int(f[15]))
def vram(self, reserve_mib: int | None, timeout: float = 120.0) -> dict:
"""#533: `VRAM <reserve_mib>` between requests (the caller holds the service's FIFO): the engine shrinks its
expert cache until that much VRAM is free, or grows it back when more is free; None: back to the reserve it
started with. -> the engine's figures (expert_slots, expert_cache_mib, vram_free_mib, ...); raises ValueError
with the engine's reason (e.g. it was not started with --vram-elastic), EngineDied when it ended."""
if not self.alive():
raise EngineDied("the engine is not running")
self.proc.stdin.write("VRAM" + ("" if reserve_mib is None else f" {int(reserve_mib)}") + "\n")
self.proc.stdin.flush()
deadline = time.time() + timeout
while True:
try:
line = self.lines.get(timeout=max(0.1, deadline - time.time()))
except queue.Empty:
raise EngineDied("the engine did not answer the VRAM command") from None
if line is None:
raise EngineDied("the engine ended")
line = line.strip()
if line.startswith("ERR"):
raise ValueError(line[4:].strip() or "the engine refused the VRAM command")
if line.startswith("VRAM "):
out = {}
for kv in line.split()[1:]:
k, _, v = kv.partition("=")
out[k] = int(v) if v.lstrip("-").isdigit() else v
self.info.update({k: out[k] for k in ("expert_slots", "vram_free_mib") if k in out})
self.info["vram"] = out
return out
if time.time() > deadline:
raise EngineDied("the engine did not answer the VRAM command")
@staticmethod
def sampling_keys(sampling: dict) -> str:
keys = ""
t = sampling.get("temperature")
if isinstance(t, (int, float)) and float(t) > 0.0:
keys += f" temperature={float(t)!r}"
tp = sampling.get("top_p")
if isinstance(tp, (int, float)) and float(tp) < 1.0:
keys += f" top_p={float(tp)!r}"
tk = sampling.get("top_k")
if isinstance(tk, int) and not isinstance(tk, bool) and tk >= 0:
# the engine's sampled path keeps at most 64 candidates: 0 ("off") and wider lists get all 64
keys += f" top_k={tk if 1 <= tk <= 64 else 64}"
mp = sampling.get("min_p")
if isinstance(mp, (int, float)) and 0.0 < float(mp) <= 1.0:
keys += f" min_p={float(mp)!r}"
rp = sampling.get("repetition_penalty")
rp_on = isinstance(rp, (int, float)) and float(rp) != 1.0
pf = sampling.get("frequency_penalty")
pf_on = isinstance(pf, (int, float)) and float(pf) != 0.0
pp = sampling.get("presence_penalty")
pp_on = isinstance(pp, (int, float)) and float(pp) != 0.0
if rp_on:
keys += f" penalty_repeat={float(rp)!r}"
if pf_on:
keys += f" penalty_freq={float(pf)!r}"
if pp_on:
keys += f" penalty_present={float(pp)!r}"
if rp_on or pf_on or pp_on:
# a penalty without a window counts over nothing: the engine's default is the last 64 tokens
pln = sampling.get("penalty_last_n")
if isinstance(pln, int) and not isinstance(pln, bool) and pln > 0:
keys += f" penalty_last_n={pln}"
else:
keys += " penalty_last_n=64"
seed = sampling.get("seed")
if isinstance(seed, int) and seed > 0:
keys += f" seed={seed}"
# setup's calibration (tools/calibrate.py): engine settings for this request only, measured without a restart
tune = sampling.get("strata_tune")
if isinstance(tune, dict):
for k in ("pcie_frac", "spec_min_p"):
v = tune.get(k)
if isinstance(v, (int, float)) and not isinstance(v, bool) and 0.0 <= float(v) <= 1.0:
keys += f" {k}={float(v)!r}"
return keys + StrataEngine.projection_key(sampling)
@staticmethod
def projection_key(sampling: dict) -> str:
"""`cvec=0|1`: the experimental-speed-projection control vector for this request, when the engine was
started with one (--control-vector-scaled; an engine without one ignores the key). Absent = on."""
on = sampling.get("experimental_speed_projection")
return f" cvec={int(on)}" if isinstance(on, bool) else ""
def _send(self, text: str):
btrace("send>", text[:60])
try:
with self.wlock:
self.proc.stdin.write(text + "\n")
self.proc.stdin.flush()
except OSError: # the pipe is gone: the engine died (not the client)
raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})") from None
def _control(self, cancel, on_token, stop_when=None):
"""Reads the control lines of the request on them (GEN / BGEN), yielding None heartbeats. Calls on_token(id)
for each `T`; returns ("done", None) at DONE, or ("badm", continues) at BADM (after DONE). `stop_when()`
true sends STOP once (the request is then read to its DONE)."""
stopped = False
while True:
try:
line = self.lines.get(timeout=10.0)
except queue.Empty:
if cancel.is_set() and not stopped:
self._send("STOP")
stopped = True
yield None
continue
if line is None:
raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})")
if not line.startswith(("PP ", "INFO")):
btrace("ctl<", self._ctl_mode, line.strip()[:60])
if line.startswith("T "):
on_token(int(line[2:]))
if not stopped and (cancel.is_set() or (stop_when is not None and stop_when())):
self._send("STOP")
stopped = True
yield False # a token is pending (not a heartbeat)
elif line.startswith("PP "):
f = line.split()
if len(f) >= 3 and f[1].isdigit() and f[2].isdigit():
self.progress = (int(f[1]), int(f[2]))
self.prefill_tok_s_mean = float(f[4]) if len(f) >= 5 else None
if not stopped and cancel.is_set():
# #879: a Stop during the prompt read went out only after the first token (or after 10 s of
# silence, which a stream of PP lines never allows): the engine read the whole prompt first
self._send("STOP")
stopped = True
yield None
elif line.startswith("DONE"):
self._parse_done(line)
self._last_done = line
elif line.startswith("BADM "):
f = line.split()
self._ctl_result = ("badm", len(f) >= 3 and f[2] == "1")
return
elif line.startswith("YIELDED "): # the read gave way (BYIELD): <slot> <tokens read>
f = line.split()
if len(f) >= 3 and f[1].lstrip("-").isdigit() and f[2].isdigit():
self._yielded = (int(f[1]), int(f[2]))
elif line.startswith("ERR"):
raise ValueError(line[4:].strip())
if line.startswith("DONE") and self._ctl_mode == "solo":
self._ctl_result = ("done", None)
return
def _drain_control(self, until: str, timeout: float = 300.0):
"""After a consumer left early: read the control lines up to the next `until` line (DONE or BADM) so the next
request does not read this one's leftovers. Returns that line (None: the engine ended or never answered)."""
end = time.monotonic() + timeout
while time.monotonic() < end:
try:
line = self.lines.get(timeout=max(0.1, end - time.monotonic()))
except queue.Empty:
break
if line is None:
return None
if line.startswith("DONE"):
self._parse_done(line)
if line.startswith(until) or line.startswith("ERR"):
return line
return None
def _release_slot_when_done(self, slot: int, stream: list[int] | None = None):
"""A slot whose consumer left (a stop token, a stop string, a disconnect): BSTOP it and free it once the engine
says BDONE (in the background). `stream`: the prompt and every token of it so far - with the tokens still to
come before BDONE, all but the last are what the slot holds then (the next turn of its conversation)."""
try:
self._send(f"BSTOP {slot}")
except EngineDied:
pass
def wait():
end = time.monotonic() + 600.0
tail = list(stream or [])
while time.monotonic() < end:
try:
line = self.slot_q[slot].get(timeout=5.0)
except queue.Empty:
continue
if line is None:
tail = []
break
if line.startswith("BT "):
try:
tail.append(int(line.split()[2]))
except (IndexError, ValueError):
tail = []
if line.startswith("BDONE "):
break
self.slot_held[slot] = tail[:-1] if stream and tail else []
with self.slot_cv:
self.slot_busy[slot] = False
self.slot_cv.notify_all()
threading.Thread(target=wait, daemon=True).start()
YIELDS_MAX = 2 # #656: how often one request's prompt read gives way to a shorter waiting one
def _take_control(self, cancel, plen: int, after_epoch: int | None = None):
"""Waits for the control lines (one prompt read at a time), yielding None heartbeats; False when cancelled.
`after_epoch`: a request whose read gave way lets the requests waiting then go first."""
entry = [plen]
with self.slot_cv:
self.waiting += 1
self.wait_lens.append(entry)
beat = time.monotonic()
try:
while True:
with self.slot_cv:
turn = after_epoch is None or self.ctl_epoch > after_epoch or self.waiting <= 1
if turn and self.ctl.acquire(timeout=0.5):
break
if not turn:
with self.slot_cv:
self.slot_cv.wait(timeout=0.5)
if cancel.is_set():
return False
if time.monotonic() - beat >= 10.0: # a heartbeat every 10 s, as the engine's own wait
beat = time.monotonic()
yield None
finally:
with self.slot_cv:
self.waiting -= 1
self.wait_lens.remove(entry)
with self.slot_cv:
self.ctl_epoch += 1
self.slot_cv.notify_all()
return True
SOLO_AGAIN_MAX = 2 # how often a request left alone in a slot goes back to the solo path
SOLO_AGAIN_MIN_LEFT = 32 # ... only with at least this many tokens still allowed (max_tokens)
def _may_go_solo(self, left: int, times: int, embeddings) -> bool:
"""A request decoding in a slot that is alone now (no other slot busy, nobody waiting) goes back to the solo
path with its MTP drafts: the engine continues it from the slot's sessions (its slot cache; INFO
slot_cache=1). STRATA_PARALLEL_SOLO=0 keeps it in the slot."""
if (embeddings or times >= self.SOLO_AGAIN_MAX or left < self.SOLO_AGAIN_MIN_LEFT or
not (self.info or {}).get("slot_cache") or os.environ.get("STRATA_PARALLEL_SOLO") == "0"):
return False
with self.slot_cv:
return sum(1 for b in self.slot_busy if b) == 1 and self.waiting == 0
def _shorter_waiting(self, plen: int) -> bool:
"""#656: someone waits for the control lines with a prompt under half this one's: worth giving way to."""
with self.slot_cv:
return any(e[0] * 2 <= plen for e in self.wait_lens)
def generate_batched(self, ids, max_new, sampling, cancel, embeddings=None):
"""--batch. Alone (no slot busy, nobody waiting): the solo path (GEN, with drafts: the fastest stream)
- and when another request arrives meanwhile, this one is STOPped and continues in a batch slot (BGEN with
its prompt + what it generated: the engine reuses that prefix). Otherwise: BGEN into a free slot, then the
slot's own BT lines until BDONE. Several requests run at once; the control lines (prompt reading, admission)
are taken one request at a time - and a long prompt read gives way at a chunk boundary to a waiting request
with a much shorter prompt (#656: `BYIELD <slot>`; the part read waits in a slot and the read goes on after).
A consumer that stops early leaves the engine in step: the solo request is STOPped and read to its DONE, an
admission to its BADM, a slot is BSTOPped and freed at its BDONE."""
if not self.alive():
raise EngineDied("the engine is unavailable; this request was not sent")
self.progress = None
keys = self.sampling_keys(sampling or {})
out: list[int] = []
pending: list[int] = []
prompt, left = list(ids), int(max_new)
ok = yield from self._take_control(cancel, len(prompt))
if not ok:
return
holding = True
btrace("ctl acquired")
slot, gen0, reserved = None, None, None
phase = "none" # solo -> (admit -> slot) ; "done" once the engine has finished with this request
stop_sent = False
yields, solo_again = 0, 0
try:
while True: # a request in a slot that is left alone goes back to the solo path
if not holding:
ok = yield from self._take_control(cancel, len(prompt))
if not ok:
return
holding = True
with self.slot_cv:
alone = not any(self.slot_busy) and self.waiting == 0
if alone and left > 1:
head = f"GENI {left}{keys} {embeddings}" if embeddings else f"GEN {left}{keys}"
self._send(f"{head} {','.join(str(int(t)) for t in prompt)}")
phase = "solo"
self._ctl_mode, self._ctl_result, self._yielded = "solo", None, None
def others():
with self.slot_cv:
return self.waiting > 0
for x in self._control(cancel, pending.append, stop_when=others):
while pending:
t = pending.pop(0)
out.append(t)
yield t
if x is None: # a heartbeat (False: a token, flushed above)
if (reserved is None and not out and not embeddings and yields < self.YIELDS_MAX and
self._shorter_waiting(len(prompt))):
with self.slot_cv: # the slot the part read will wait in
reserved = self.pick_slot(prompt)
if reserved is not None:
self.slot_busy[reserved] = True
if reserved is not None:
self._send(f"BYIELD {reserved}")
yield None
phase = "none" # its DONE is read
while pending:
t = pending.pop(0)
out.append(t)
yield t
if self._yielded is not None and reserved is not None and self._yielded[0] == reserved:
slot, reserved = reserved, None # gave way: the read goes on in that slot (below)
elif reserved is not None: # it did not give way: the slot is free again
with self.slot_cv:
self.slot_busy[reserved] = False
self.slot_cv.notify_all()
reserved = None
if slot is None:
finish = (self.last or {}).get("finish") if isinstance(self.last, dict) else None
left = int(max_new) - len(out)
if cancel.is_set() or left <= 0 or finish in ("stop", "length") or (out and out[-1] in EOS_IDS):
return
prompt = list(ids) + out # promoted: it continues in a batch slot from here
while True:
if slot is None:
# a free slot (they free themselves at BDONE, which needs no control lines): the one that holds
# the start of this prompt (its conversation's last turn), else the one used longest ago
with self.slot_cv:
while True:
slot = self.pick_slot(prompt)
if slot is not None:
self.slot_busy[slot] = True
break
self.slot_cv.wait(timeout=10.0)
if cancel.is_set():
return
if self._yielded is not None: # it gave way: the others waiting then go first
self.slot_held[slot] = list(prompt[:self._yielded[1]])
self._yielded = None
yields += 1
self.ctl.release()
holding = False
with self.slot_cv:
epoch = self.ctl_epoch
ok = yield from self._take_control(cancel, len(prompt), after_epoch=epoch)
if not ok:
return
holding = True
while not self.slot_q[slot].empty():
self.slot_q[slot].get_nowait()
head = f"BGENI {slot} {left}{keys} {embeddings}" if embeddings else f"BGEN {slot} {left}{keys}"
live = {"slot": slot, "state": "reading", "prompt_tokens": len(prompt), "generated": 0,
"started": time.time(), "first_token": None}
self.slot_live[slot] = live
self._send(f"{head} {','.join(str(int(t)) for t in prompt)}")
self.slot_held[slot] = [] # the admission overwrites what the slot held
phase = "admit"
self._ctl_mode, self._ctl_result, self._yielded = "batch", None, None
asked = False
for x in self._control(cancel, pending.append):
while pending:
t = pending.pop(0)
out.append(t)
yield t
if x is None:
if (not asked and not embeddings and yields < self.YIELDS_MAX and
self._shorter_waiting(len(prompt))):
self._send(f"BYIELD {slot}")
asked = True
yield None
cont = bool(self._ctl_result and self._ctl_result[1])
phase = "slot" if cont else "none"
while pending:
t = pending.pop(0)
out.append(t)
yield t
if not cont and self._yielded is not None and self._yielded[0] == slot and not cancel.is_set():
continue # gave way: again once the shorter request is in
break
self.ctl.release()
holding = False
if not cont:
return
live.update(state="decoding", first_token=time.time(), generated=len(out))
gen0 = len(out) - 1 # the admission's own token: the slot feeds it first
going_solo = False
while True:
try:
line = self.slot_q[slot].get(timeout=10.0)
except queue.Empty:
if cancel.is_set() and not stop_sent:
self._send(f"BSTOP {slot}")
stop_sent = True
yield None
continue
if line is None:
phase = "none"
raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})")
if line.startswith("BT "):
t = int(line.split()[2])
out.append(t)
live["generated"] = len(out)
if cancel.is_set():
if not stop_sent:
self._send(f"BSTOP {slot}")
stop_sent = True
continue
yield t
if not going_solo and not stop_sent and self._may_go_solo(int(max_new) - len(out), solo_again,
embeddings):
self._send(f"BSTOP {slot}") # alone now: on with the drafts (below)
going_solo = True
elif line.startswith("BDONE "):
phase = "none"
f = line.split()
if len(f) >= 5 and isinstance(self.last, dict):
self.last = {**self.last, "finish": f[3], "decode_ms": float(f[4]),
"generated": int(f[2]) if f[2].isdigit() else self.last.get("generated")}
# what the slot's sessions hold now: the prompt and every token fed (all but the last one)
self.slot_held[slot] = list(prompt) + out[gen0:-1] if len(out) > gen0 else []
if (going_solo and f[3:4] == ["cancel"] and not cancel.is_set() and len(out) < int(max_new)
and not (out and out[-1] in EOS_IDS)):
# the solo path continues it: the engine copies the slot's sessions back (all but the
# last token are in them) and decodes with MTP drafts again
self.slot_live[slot] = None
self.slot_used[slot] = time.time()
with self.slot_cv:
self.slot_busy[slot] = False
self.slot_cv.notify_all()
slot, gen0 = None, None
prompt, left = list(ids) + out, int(max_new) - len(out)
solo_again += 1
btrace("back to the solo path")
break
return
finally:
# a consumer that left early (or an error): keep the engine and this server in step
btrace("finally phase", phase, "slot", slot, "holding", holding)
try:
if phase == "solo":
self._send("STOP")
self._drain_control("DONE")
elif phase == "admit":
line = self._drain_control("BADM")
if line and line.startswith("BADM ") and line.split()[2:3] == ["1"]:
phase = "slot"
except EngineDied:
pass
if holding:
self.ctl.release()
if reserved is not None:
with self.slot_cv:
self.slot_busy[reserved] = False
self.slot_cv.notify_all()
if slot is not None:
self.slot_live[slot] = None
self.slot_used[slot] = time.time()
if phase == "slot":
self.slot_held[slot] = []
stream = list(prompt) + out[gen0:] if gen0 is not None and len(out) > gen0 else None
self._release_slot_when_done(slot, stream) # freed at its BDONE
else:
with self.slot_cv:
self.slot_busy[slot] = False
self.slot_cv.notify_all()
def pick_slot(self, prompt: list[int]) -> int | None:
"""A free slot for `prompt` (the caller holds slot_cv): the one whose held tokens are the longest start of the
prompt (the engine then reads only the rest), else an empty one, else the one used longest ago - so the
conversations other slots hold stay for their next turns. None: every slot is busy."""
free = [b for b in self.slot_order if not self.slot_busy[b]]
if not free:
return None
def held_prefix(b):
h = self.slot_held[b]
return len(h) if h and len(h) < len(prompt) and prompt[:len(h)] == h else 0
best = max(free, key=held_prefix)
if held_prefix(best) > 0:
return best
return min(free, key=lambda b: (bool(self.slot_held[b]), self.slot_used[b]))
def slots_view(self) -> list[dict]:
"""/metrics: each batch slot - idle (with the tokens it holds for a next turn), reading or decoding."""
now, view = time.time(), []
for b in range(self.batch):
r = self.slot_live[b]
if r is None:
view.append({"slot": b, "state": "idle", "held_tokens": len(self.slot_held[b])})
continue
ft = r.get("first_token")
view.append({"slot": b, "state": r["state"], "prompt_tokens": r["prompt_tokens"],
"generated": r["generated"], "elapsed_s": round(now - r["started"], 1),
"tok_s": round(r["generated"] / max(1e-6, now - ft), 1) if ft else None})
return view
def generate(self, ids, max_new, sampling, cancel, embeddings=None):
"""Yields token ids, and None as a heartbeat every 10 s while the engine is quiet (reading a long prompt):
the HTTP layer turns it into an SSE comment, which keeps clients' watchdogs calm and notices a client that
has gone. A consumer that stops early (or `cancel`) makes the engine STOP, so it does not run to max_new."""
if not self.alive():
raise EngineDied("the engine is unavailable; this request was not sent")
if getattr(self, "batch", 0):
yield from self.generate_batched(ids, max_new, sampling, cancel, embeddings)
return
self.progress = None
self.prefill_tok_s_mean = None
# an image request takes the same sampling keys as text (#75: it used to decode greedily whatever was asked)
head = f"GENI {int(max_new)}{self.sampling_keys(sampling or {})} {embeddings}" if embeddings else \
f"GEN {int(max_new)}{self.sampling_keys(sampling or {})}"
try:
self.proc.stdin.write(f"{head} {','.join(str(int(t)) for t in ids)}\n")
self.proc.stdin.flush()
except OSError: # the pipe is gone: the engine died (not the client)
raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})") from None
done = False
# #481: how long the engine may stay silent from here. Until the first line: the request's first prompt
# chunk at the slowest prompt reading on top of silence_s; a PP line resets it to its own chunk's time.
silence = float(self.silence_s or 0)
allow = silence + min(len(ids), PP_CHUNK_MAX) / PP_FLOOR_TOK_S if silence > 0 else 0.0
heard, read_to = time.monotonic(), 0
try:
while True:
wait = 10.0
if allow > 0:
wait = min(wait, allow - (time.monotonic() - heard))
if wait <= 0:
done = True # no STOP and no drain: nothing is listening
raise self._silent(f"the engine said nothing for {time.monotonic() - heard:.0f} s during "
"the request")
try:
line = self.lines.get(timeout=wait)
except queue.Empty:
if cancel.is_set():
return
if wait >= 10.0:
yield None # the 10 s heartbeat (the deadline's short waits are not)
continue
if line is None:
done = True
raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})")
heard = time.monotonic() # any line is output: T, PP, RESUME, INFO ...
if line.startswith("T "):
allow = silence
if cancel.is_set():
return
yield int(line[2:])
elif line.startswith("PP "):
f = line.split()
if len(f) >= 3 and f[1].isdigit() and f[2].isdigit():
self.progress = (int(f[1]), int(f[2])) # prompt progress, one per chunk: also a heartbeat (the
self.prefill_tok_s_mean = float(f[4]) if len(f) >= 5 else None
rate, chunk = self.prefill_tok_s_mean or 0.0, int(f[1]) - read_to
read_to = int(f[1])
if silence > 0 and rate > 0 and chunk > 0: # #481: the next chunk, as long as this one
allow = max(silence, PP_SLACK * chunk / rate)
if cancel.is_set(): # lines reset the 10 s wait, so without this a long prompt
return # would send no keep-alives at all)
yield None
elif line.startswith("RESUME "): # the reused tokens: the first chunk starts after them
try:
read_to = int(line.split()[1])
except (IndexError, ValueError):
pass
elif line.startswith("DONE"):
self._parse_done(line)
done = True
return
elif line.startswith("ERR"):
done = True
raise ValueError(line[4:].strip())
finally:
if not done: # the consumer stopped early: stop the engine, drain to DONE
if self.can_stop:
try:
self.proc.stdin.write("STOP\n")
self.proc.stdin.flush()
except OSError:
pass
# #481: never an untimed wait here - it holds the request FIFO, and an engine that lost step never
# answers. An engine that honours STOP gets the current allowance in all (a STOP during a prompt
# chunk is seen after it); an older one runs on to max_new, so each line only has to come in time.
heard = time.monotonic()
while True:
left = allow - (time.monotonic() - heard) if allow > 0 else None
try:
if left is not None and left <= 0:
raise queue.Empty
line = self.lines.get(timeout=left)
except queue.Empty:
raise self._silent("the engine did not finish the request after it was stopped (STOP) "
f"within {allow:.0f} s") from None
if line is None or line.startswith("ERR"):
break
if line.startswith("DONE"):
self._parse_done(line)
break
if not self.can_stop:
heard = time.monotonic()
def _silent(self, what: str) -> EngineSilent:
"""#481: end an engine that lost step with the server (its main thread waits for a command the server never
sends), so the next request starts it again: killed now, its GPU and RAM go back with the process."""
self.silent_note = ("The engine and the server lost step (issue #481; a very slow PC can raise "
"\"engine_silence_s\" in the config, 0 = wait forever). If it happens again, please add "
"the end of the engine log to github.com/Niko1221/Strata/issues/481.")
self.ended = True # not alive from now: the next request restarts it
proc = self.proc
try:
proc.kill()
proc.wait(timeout=20) # restart() -> close() handles one that is still exiting
except (OSError, AttributeError, subprocess.TimeoutExpired):
pass
return EngineSilent(f"{what}; the server ended the engine")
def close(self):
"""End the engine process: QUIT first (the engine frees its memory itself - unpinning tens of GB can take
a while), then terminate, then kill, each given 20 s. Raises EngineStuck when it still runs after all three."""
if self.proc is None:
return
try:
if self.proc.poll() is None:
try:
self.proc.stdin.write("QUIT\n")
self.proc.stdin.flush()
self.proc.stdin.close() # Windows' detached stdin reader must see EOF before shutdown
self.proc.wait(timeout=20)
except (OSError, ValueError, subprocess.TimeoutExpired):
self.proc.terminate()
self.proc.wait(timeout=20)
except subprocess.TimeoutExpired:
self.proc.kill()
try:
self.proc.wait(timeout=20)
except subprocess.TimeoutExpired:
raise EngineStuck("Strata is still releasing GPU/RAM; retry unloading after it exits") from None
except OSError:
pass
finally:
if self.proc.poll() is not None:
if self.pump is not None:
self.pump.join(timeout=2)
self.proc.stdin.close()
self.proc.stdout.close()
if self.log not in (None, subprocess.DEVNULL):
self.log.close()
self.proc = None
self.ended = True
self.progress, self.last = None, {}
IMAGE_URL_MAX = 32 << 20 # an image URL is read up to this (the web app attaches pictures of up to 20 MB)
def network_path(path: str) -> bool:
r"""A path to another computer: \\host\share or //host/share (either slash), \\?\UNC\host\..., \\.\UNC\host\...
or the NT form \??\UNC\host\... Windows connects to that host at the first look at the path (os.path.isfile is
enough) and signs in with the user's NTLM credentials. \\?\ and \??\ name local files too; no client needs them."""
p = path.replace("/", "\\")
return p.startswith("\\\\") or p.startswith("\\??\\")
class Vision:
"""The resident image encoder: `strata-vision` (llama.cpp mtmd + the mmproj file) reads `ENC <image> <out>`
lines and writes each image's embeddings; results are cached by the image's hash, so a conversation that
sends the same picture again (every turn, with most clients) encodes it once."""
def __init__(self, cfg: dict, log=None, env: dict | None = None):
args = [cfg["exe"], "--mmproj", cfg["mmproj"], "--model", cfg["model"]]
if cfg.get("gpu"):
args.append("--gpu")
if cfg.get("threads"):
args += ["--threads", str(cfg["threads"])]
if cfg.get("max_tokens"):
args += ["--max-tokens", str(cfg["max_tokens"])]
self.dir = Path(tempfile.mkdtemp(prefix="strata-vision-"))
self.spawn = (args, log, env) # to start it again after an unload
self.stopped = False
self._start()
self.lock = threading.Lock()
self.cache: dict[str, tuple[Path, int]] = {}
def _start(self):
args, log, env = self.spawn
self.proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=log or subprocess.DEVNULL,
text=True, encoding="utf-8", bufsize=1, env=env)
contain(self.proc)
line = self.proc.stdout.readline()
if not line.startswith("READY"):
raise RuntimeError("the vision encoder did not start: " + line.strip())
self.stopped = False
def alive(self) -> bool:
return not self.stopped and self.proc.poll() is None
def unload(self):
"""Stop the encoder process (its VRAM or RAM goes back); the encoded images stay cached on disk."""
self.close()
self.stopped = True
def restart(self):
"""Start the encoder again after an unload (or if it died); the cache of encoded images is kept."""
try:
self.proc.kill()
except OSError:
pass
self._start()
@staticmethod
def load(source: str) -> bytes:
if source.startswith("data:"):
try:
return base64.b64decode(source.split(",", 1)[1])
except (IndexError, ValueError) as e: # no comma, or not base64
raise ValueError(f"the image's data: URL could not be read ({e})") from None
if source.startswith(("http://", "https://")):
return Vision.download(source)
path = source[7:] if source.startswith("file://") else source
# refused before anything looks at the path: file://host/share/x.png is \\host\share\x.png ("file://C:/x.png",
# a drive, is a client's spelling of file:///C:/x.png)
host = re.split(r"[/\\]", path, maxsplit=1)[0] if source.startswith("file://") else ""
if network_path(path) or host and not re.fullmatch(r"[A-Za-z]:|localhost", host, re.IGNORECASE):
raise ValueError("an image file must be on this computer: network paths (\\\\host\\share, //host/share, "
"file://host/...) are not read")
if path and os.path.isfile(path):
try:
return Path(path).read_bytes()
except OSError as e:
raise ValueError(f"the image could not be read from {path} ({e})") from None
raise ValueError("an image must be a data: URL, an http(s) URL or a local file path")
@staticmethod
def download(url: str) -> bytes:
"""An image URL's bytes, at most IMAGE_URL_MAX of them: the whole response was read, so a huge or endless one
filled the memory. One that cannot be read is a ValueError (a 400 that says so), not a dropped connection."""
too_big = f"the image URL's file is over {IMAGE_URL_MAX >> 20} MiB"
try:
with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "strata"}), timeout=60) as r:
size = r.headers.get("Content-Length") or ""
if size.isdigit() and int(size) > IMAGE_URL_MAX:
raise ValueError(too_big)
data = r.read(IMAGE_URL_MAX + 1)
except (OSError, HTTPException) as e:
raise ValueError(f"the image URL could not be read: {e}") from None
if len(data) > IMAGE_URL_MAX:
raise ValueError(too_big)
return data
@staticmethod
def normalize(data: bytes) -> bytes:
"""The formats strata-vision's decoder (stb_image) reads pass through; anything else is converted to PNG."""
if data[:3] == b"\xff\xd8\xff" or data[:8] == b"\x89PNG\r\n\x1a\n" or data[:2] == b"BM" or \
data[:6] in (b"GIF87a", b"GIF89a"):
return data
try:
import io
from PIL import Image
except ImportError:
raise ValueError("this image format needs Pillow (python -m pip install pillow); JPEG, PNG, BMP and "
"GIF work without it") from None
try:
im = Image.open(io.BytesIO(data))
im.load()
except Exception as e:
raise ValueError(f"the image could not be read ({e})") from None
if im.mode in ("RGBA", "LA", "P") and "transparency" in im.info or im.mode in ("RGBA", "LA"):
im = im.convert("RGBA")
bg = Image.new("RGB", im.size, (255, 255, 255)) # transparent areas become white, not black
bg.paste(im, mask=im.split()[-1])
im = bg
elif im.mode != "RGB":
im = im.convert("RGB")
out = io.BytesIO()
im.save(out, format="PNG")
return out.getvalue()
def encode(self, source: str | bytes) -> tuple[Path, int]:
"""-> (embeddings file, number of image tokens). `source`: what load() reads, or the image's bytes."""
data = self.normalize(source if isinstance(source, bytes) else self.load(source))
key = hashlib.sha256(data).hexdigest()[:32]
with self.lock:
if key in self.cache:
return self.cache[key]
img, out = self.dir / f"{key}.img", self.dir / f"{key}.sve"
img.write_bytes(data)
try:
self.proc.stdin.write(f"ENC {img} {out}\n")
self.proc.stdin.flush()
line = self.proc.stdout.readline().strip()
finally: # #352: also when the encoder's pipe is gone
img.unlink(missing_ok=True)
if not line.startswith("OK"):
raise ValueError("the image could not be read: " + (line[4:] if line.startswith("ERR") else
"the vision encoder stopped"))
self.cache[key] = (out, int(line.split()[1]))
if len(self.cache) > 64: # oldest first
old = next(iter(self.cache))
self.cache.pop(old)[0].unlink(missing_ok=True)
return self.cache[key]
def close(self):
try:
self.proc.stdin.write("QUIT\n")
self.proc.stdin.flush()
self.proc.wait(timeout=10)
except Exception:
self.proc.kill()
def gpu_list(cfg: dict) -> list[int]:
"""The config's "gpu": one card (2), or several for a layer split ([0, 2] or "0,2"), numbered as nvidia-smi
numbers them; [] when it names none."""
g = cfg.get("gpu")
if g is None or g == "":
return []
items = g if isinstance(g, (list, tuple)) else str(g).split(",")
return [int(str(x).strip()) for x in items if str(x).strip() != ""]
def effort_end_args(cfg: dict, exe: str, tok) -> list[str] | None:
"""#458 (opt-in): the engine arguments for "effort_position": "end" - the id of "system" as --tail-role-token, so
the engine checkpoints in front of the trailing effort turn - or None when the config leaves it at the top (the
default) or the engine is too old for it (said once; the prompt stays the default one). ValueError for a value
other than "start" / "end"."""
pos = cfg.get("effort_position", "start")
if pos not in ("start", "end"):
raise ValueError(f'"effort_position" must be "start" (the default) or "end", not {pos!r}')
if pos == "start":
return None
try:
with open(exe, "rb") as f:
known = b"--tail-role-token" in f.read()
except OSError:
known = False
role = tok.encode("system", parse_special=True)
if not known or len(role) != 1:
print("[strata] effort_position \"end\" needs engine 0.1.39 or newer (--tail-role-token): the reasoning "
"effort stays at the top of the prompt", flush=True)
return None
print("[strata] effort_position end: a request's non-default reasoning effort goes right before the answer, so "
"changing it keeps the cached conversation", flush=True)
return ["--tail-role-token", str(role[0])]
def engine_silence_s(cfg: dict) -> float:
"""#481: the config's "engine_silence_s" - seconds an engine may print nothing during a request before the server
ends it (default ENGINE_SILENCE_S; 0 = wait forever). ValueError for anything but a number >= 0."""
v = cfg.get("engine_silence_s")
if v is None:
return ENGINE_SILENCE_S
if isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0:
raise ValueError(f'"engine_silence_s" must be a number of seconds >= 0 (0 = no limit), not {v!r}')
return float(v)
def layer_split_value(cfg: dict) -> str:
"""#644: the config's "layer_split" as the engine's --layer-split value: "auto" (the default), or the FIRST LAYER of
each later GPU's share - one rising number per GPU after the first, e.g. "24,36,42" for 4 GPUs (layers 0-23 on the
first card, 24-35, 36-41, 42 to the end) - as a string or a JSON list ([24, 36, 42]). ValueError with the format
and an example for anything else, such as per-card layer counts ("24,16,12,12")."""
v = cfg.get("layer_split")
if v is None or (isinstance(v, str) and v.strip().lower() in ("", "auto")):
return "auto"
n = len(gpu_list(cfg))
items = v if isinstance(v, (list, tuple)) else str(v).split(",") if isinstance(v, (str, int)) else None
vals = []
for x in items or []:
try:
if isinstance(x, bool) or isinstance(x, float):
raise ValueError
vals.append(int(str(x).strip()))
except ValueError:
vals = None
break
want = max(n - 1, 1)
example = ",".join(str(round(48 * (i + 1) / (want + 1))) for i in range(want))
hint = (f'"layer_split" is the first layer of each later GPU, one rising number per GPU after the first '
f'({want} for {n} GPUs), not a count of layers per card - e.g. "{example}" (or [{example.replace(",", ", ")}])'
f' for an even share of 48 layers, or "auto" (the default) to place them by each card\'s free VRAM')
if not vals:
raise ValueError(f"{hint}; got {v!r}")
if n > 1 and len(vals) != n - 1:
raise ValueError(f"{hint}; got {len(vals)} number(s) ({v!r}) for {n} GPUs")
if vals[0] < 2 or any(b <= a for a, b in zip(vals, vals[1:])):
raise ValueError(f"{hint}; got {v!r}, which does not rise from 2 or more")
return ",".join(str(x) for x in vals)
def engine_args(cfg: dict) -> list[str]:
"""The engine's arguments: the config's, and with several GPUs the layer split across them ("layer_split" in the
config: "auto" by default, or the first layer of each later GPU's share, e.g. "18" or "16,32"; see
layer_split_value)."""
args = list(cfg["args"])
if len(gpu_list(cfg)) > 1 and "--layer-split" not in args:
args += ["--layer-split", layer_split_value(cfg)]
# opt-in: an auto split runs on the first card alone when it holds every profiled expert and the KV
if len(gpu_list(cfg)) > 1 and cfg.get("split_skip_if_fits") and "--split-skip-if-fits" not in args:
args.append("--split-skip-if-fits")
# #533 (opt-in): "vram_elastic": true - the expert cache in segments, so POST /v1/vram can give VRAM back to other
# programs and take it back; "vram_segment_mib" sets the segment size (the engine's default: 512)
if cfg.get("vram_elastic") is True and "--vram-elastic" not in args:
args.append("--vram-elastic")
seg = cfg.get("vram_segment_mib")
if isinstance(seg, int) and not isinstance(seg, bool) and seg > 0 and "--vram-segment-mib" not in args:
args += ["--vram-segment-mib", str(seg)]
args += parallel_args(cfg, args)
return learned_profile_args(cfg, args)
PARALLEL_MAX = 8 # the engine's batch window holds at most 8 rows (kVerifyMaxT)
def parallel_args(cfg: dict, args: list[str]) -> list[str]:
"""#465 (opt-in): "parallel": N in the config - up to N requests decode together in the engine's batch slots
(--batch N); more wait their turn. 1 or absent: one request at a time, as before. A value the engine cannot use
is said and passed on: the engine warns and adjusts (recommend, never force)."""
if "--batch" in args or "--slots" in args:
return []
n = cfg.get("parallel")
if n is None or n is False:
return []
if isinstance(n, bool) or not isinstance(n, int):
print(f'[strata] "parallel" must be a whole number of requests (2..{PARALLEL_MAX}), not {n!r}: ignored',
flush=True)
return []
if n <= 1:
return []
if n > PARALLEL_MAX:
print(f'[strata] "parallel": {n} - the engine runs at most {PARALLEL_MAX} requests together; it will use '
f"{PARALLEL_MAX}", flush=True)
return ["--batch", str(n)]
def profile_shape(path: str) -> tuple[int, int] | None:
"""An expert profile's (layers, experts per layer), from its header (tools/make_profile.py's format), or None
when the file is missing, is not one or is shorter than the pairs its header promises."""
try:
with open(path, "rb") as f:
head = f.read(24)
size = os.fstat(f.fileno()).st_size
except OSError:
return None
if len(head) < 24 or head[:4] != b"STRP":
return None
_, nl, ne, _, n = struct.unpack("<5I", head[4:])
return (nl, ne) if size >= 24 + 4 * n else None
def learned_profile_args(cfg: dict, args: list[str]) -> list[str]:
"""#477 (opt-in): "expert_profile_save": "<path>" in the config has the engine (0.1.36+) save what its adaptive
tier learned there - on QUIT and every "expert_profile_save_every" minutes (10 by default, 0 = at QUIT only) -
and the next start begins from it instead of the config's --expert-profile, when it is a profile of the same
model (its header's layers and experts match); otherwise from the config's own, as before. A relative path is
the engine's (the config's "cwd"). Without the key, the arguments are the config's, unchanged."""
save = cfg.get("expert_profile_save")
if not isinstance(save, str) or not save.strip() or "--expert-profile-save" in args:
return args
args = args + ["--expert-profile-save", save]
every = cfg.get("expert_profile_save_every")
if isinstance(every, (int, float)) and not isinstance(every, bool) and every >= 0:
args += ["--expert-profile-save-every", str(every)]
if "--expert-profile" in args[:-1]:
i = args.index("--expert-profile") + 1
here = cfg.get("cwd") or "."
learned = profile_shape(save if os.path.isabs(save) else os.path.join(here, save))
base = profile_shape(args[i] if os.path.isabs(args[i]) else os.path.join(here, args[i]))
if learned is not None and learned == base:
args[i] = save
return args
def hip_visible(cfg: dict) -> list[int]:
"""AMD: the devices the engine should see, as the HIP runtime numbers them (HIP_VISIBLE_DEVICES).
On Linux setup's KFD order is HIP's order, so the config's "gpu" is it. On Windows setup finds the cards in the
display-adapter order, and an integrated Radeon that HIP also enumerates takes ordinal 0 and pushes the discrete
card to 1 (#325): setup records the ordinal `strata-device --list-devices` gave the card as "hip_ordinal", which
wins for a one-card config. Without it (a config from before), the config's "gpu"."""
ordinal = cfg.get("hip_ordinal")
if ordinal is not None and str(ordinal).strip() != "" and len(gpu_list(cfg)) <= 1:
try:
return [int(str(ordinal).strip())]
except ValueError:
pass
return gpu_list(cfg)
def child_env(cfg: dict) -> dict:
"""The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
compiled it) first on the library search path."""
env = dict(os.environ)
if hip_visible(cfg) and cfg.get("backend") == "hip": # AMD: numbered as HIP numbers them (hip_visible)
env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in hip_visible(cfg))
elif gpu_list(cfg): # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # own default order (fastest first) can number the cards otherwise
env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
for k, v in (cfg.get("env") or {}).items(): # engine settings the config carries (AMD: the GEMM tuning table)
env[str(k)] = str(v)
dirs = [d for d in cfg.get("lib_dirs") or [] if Path(d).is_dir()]
if dirs:
var = "PATH" if os.name == "nt" else "LD_LIBRARY_PATH"
env[var] = os.pathsep.join(dirs + ([env[var]] if env.get(var) else []))
return env
def vision_env(cfg: dict, env: dict) -> dict:
"""The image encoder's environment: the engine's, unless the config's vision section names its own "cuda_device"
(numbered like nvidia-smi) - then the encoder runs on that card alone, so a spare GPU can hold it while the engine
keeps all of its own cards' VRAM (#408, Efs-O). Without it, nothing changes."""
dev = (cfg.get("vision") or {}).get("cuda_device")
if dev is None:
return env
env = dict(env)
if cfg.get("backend") == "hip":
env["HIP_VISIBLE_DEVICES"] = str(dev)
else:
env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
env["CUDA_VISIBLE_DEVICES"] = str(dev)
return env
class ByteTokenizer:
"""Tiny stand-in tokenizer for tests without the pack: one id per UTF-8 byte, specials as ids >= 256."""
SPECIALS = ["<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|vision_start|>", "<|image_pad|>", "<|vision_end|>"]
ALWAYS = () # specials matched without parse_special (type 4, as <think>)
def encode(self, text, parse_special=False, plain=()):
out, i = [], 0
while i < len(text):
for k, s in enumerate(self.SPECIALS):
if (parse_special or s in self.ALWAYS) and text.startswith(s, i) and not any(
a <= i < b for a, b in plain):
out.append(256 + k)
i += len(s)
break
else:
out.extend(text[i].encode("utf-8"))
i += 1
return out
def decode(self, ids, errors="replace"):
raw = bytearray()
for t in ids:
raw += self.SPECIALS[t - 256].encode() if t >= 256 else bytes([t])
return raw.decode("utf-8", errors=errors)
# ------------------------------------------------------------------------------------------------ core
class Detokenizer:
"""Incremental decode: each token's bytes go through an incremental UTF-8 decoder, which emits the complete
characters and holds a multi-byte character split across tokens until it is complete (invalid bytes become
U+FFFD, as a whole decode with errors="replace" makes them). Constant time per token - the old re-decode of
every generated id cost 2 ms per token after 8K tokens and 4 ms after 16K (perf-review F-1). A tokenizer
without `token_bytes` (the tests' byte tokenizer) keeps the re-decode."""
def __init__(self, tok):
self.tok, self.ids, self.sent = tok, [], 0
self.inc = codecs.getincrementaldecoder("utf-8")(errors="replace") if hasattr(tok, "token_bytes") else None
def pending(self) -> bool:
"""A character is split across the tokens so far: its first bytes are held."""
if self.inc is not None:
return bool(self.inc.getstate()[0])
return self.tok.decode(self.ids).endswith("\ufffd")
def push(self, t: int) -> str:
if self.inc is not None:
return self.inc.decode(self.tok.token_bytes(t))
self.ids.append(t)
text = self.tok.decode(self.ids)
if text.endswith("�"):
return ""
delta, self.sent = text[self.sent:], len(text)
return delta
OPENAI_MAX_STOP = 4 # OpenAI's limit on `stop`
def stop_strings(req: dict) -> list[str]:
"""The request's stop strings: OpenAI's `stop` (a string or a list of up to 4) or Anthropic's `stop_sequences`
(a list). Empty strings are left out. A bad value is a ValueError, so a 400 before anything is generated."""
req = req or {}
if req.get("stop") is not None:
stop = req["stop"]
stop = [stop] if isinstance(stop, str) else stop
if not isinstance(stop, list) or not all(isinstance(x, str) for x in stop):
raise ValueError("stop must be a string or a list of strings")
if len(stop) > OPENAI_MAX_STOP:
raise ValueError(f"stop takes at most {OPENAI_MAX_STOP} strings")
elif req.get("stop_sequences") is not None:
stop = req["stop_sequences"]
if not isinstance(stop, list) or not all(isinstance(x, str) for x in stop):
raise ValueError("stop_sequences must be a list of strings")
else:
return []
return [x for x in stop if x]
class StopMatcher:
"""Cuts the answer's text at the first stop string. Text that could still be the start of a stop string is held
back (only that tail, never more), so a stop string split across tokens or chunks is still found and never sent."""
def __init__(self, stops: list[str]):
self.stops, self.held, self.hit = stops, "", None
def push(self, text: str) -> str:
"""-> the text that is safe to send now; after a match, self.hit is the stop string and nothing more comes."""
if self.hit is not None:
return ""
buf = self.held + text
first = None
for s in self.stops:
i = buf.find(s)
if i >= 0 and (first is None or i < first[0]):
first = (i, s)
if first is not None:
self.held, self.hit = "", first[1]
return buf[:first[0]]
keep = 0 # the longest end of buf that a stop string starts with
for s in self.stops:
for k in range(min(len(s) - 1, len(buf)), keep, -1):
if buf.endswith(s[:k]):
keep = k
break
self.held = buf[len(buf) - keep:] if keep else ""
return buf[:len(buf) - keep]
def flush(self) -> str:
held, self.held = self.held, ""
return held
class Service:
def __init__(self, engine: Engine, tokenizer, template: ChatTemplate, model_name: str = "qwen3.8-flash-next",
vision: Vision | None = None, sampling_defaults: dict | None = None,
fit_max_tokens: bool = False):
self.engine, self.tok, self.template, self.model, self.vision = engine, tokenizer, template, model_name, vision
self.fit_max_tokens = fit_max_tokens # --fit-max-tokens: clamp the output cap instead of 400
self.aliases: list[str] = [] # #297: other names of the model (the config's `aliases`)
self.sampling_defaults = dict(sampling_defaults or {}) # the run config's `sampling` block
self.shared = {} # the web app's Chat settings for every client (POST /settings)
self.shared_path = None # where they are kept between starts (next to the config)
self.fifo = threading.Lock()
self.embeddings = threading.local() # the current request's image embeddings file (GENI)
self.api_key = "" # when set, /v1/* needs it (Bearer or x-api-key)
# #458 (opt-in, the config's "effort_position": "end"): a non-default reasoning effort goes in a short system
# turn before the answer instead of the top of the prompt, so switching it keeps the cached conversation
self.effort_end = False
self.conv_log = ConvCacheLog() # #596: the parked conversations, from the engine's log
self.config_path = None # #564: the run config the web page's Settings view edits
self.config_lock = threading.Lock()
# #321: browser pages of these origins may call /v1/* (CORS; "*" = any page - only with an api_key that
# matters); empty = no CORS headers at all, as before
self.cors_origins: list[str] = []
# #321: origins that count as Strata's own page for /settings and MCP tools, e.g. the web app reached through a
# reverse proxy or tunnel whose Host differs ("https://strata.example.com"); never a wildcard
self.trusted_origins: list[str] = []
# DNS rebinding: extra Host names this server answers to (the config's allowed_hosts, $STRATA_ALLOWED_HOSTS;
# "*" = any), and every name it answers to, which serve() works out from the address it listens on
self.allowed_hosts: list[str] = []
self.host_names: set[str] = set(LOOPBACK_NAMES)
self.status = {"busy": False, "queued": 0} # GET /status: what the model is doing right now
self.rate = collections.deque(maxlen=32) # (time, generated) samples for the live tok/s window
# "parallel" (the engine's batch slots): every running request's own status and rate window; self.status
# then says busy while any runs and shows the newest one
self.live_reqs: dict[int, tuple[dict, collections.deque]] = {}
# #332: the API request monitor (/api-monitor) keeps the last 100 requests' prompts and answers in memory,
# so it is off unless the config's "api_monitor" (or --api-monitor) turns it on
self.api_monitor = False
self.api_requests = collections.deque(maxlen=100) # bounded I/O in memory; no headers or API keys
self.request_trace = threading.local()
self.history = collections.deque(maxlen=500) # the last finished requests, newest last (GET /metrics)
# since the server started (the Monitor's totals, issue #35)
self.totals = {"since": time.time(), "requests": 0, "prompt_tokens": 0, "reused": 0, "output_tokens": 0,
"prompt_ms": 0.0, "decode_ms": 0.0,
"drafts_offered": 0, "drafts_accepted": 0} # #457: the MTP drafts, summed where reported
self.last_timings = None # the last finished request's, llama.cpp's names (/v1/status)
self.last_request_at = None # when a request last started or finished
self.started_at = time.time()
self.status_lock = threading.Lock()
self.mcp = None # serve/mcp.py's McpHub when MCP servers are configured
# sharing the GPU with other programs (all off by default): unload the engine after this many idle seconds,
# only start it again when this much VRAM is free, and run this command first (e.g. to unload another
# server's model); the next request after an unload starts the engine again
self.idle_unload_s = 0
self.min_free_vram_mib = 0
self.before_load = None
self.vram_reserve = None # #533: the last POST /v1/vram reserve (None: the start's)
self.reasoning_budget_tokens = 0 # #123: the config's default thinking budget (0: none)
self.repeat_stop_tokens = REPEAT_STOP_TOKENS # #606: one token this many times in a row ends a reply (0: off)
self.anthropic_think_unasked = True # #278: "anthropic_thinking": "on_request" -> False
self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
tokenizer.encode("<|endoftext|>", parse_special=True))
def loaded(self) -> bool:
return not hasattr(self.engine, "alive") or self.engine.alive()
def set_aliases(self, aliases) -> None:
"""#297: the config's `aliases` - a list of names (or one comma-separated string), like llama-server's --alias.
ValueError for anything else."""
if aliases is None:
aliases = []
if isinstance(aliases, str):
aliases = aliases.split(",")
if not isinstance(aliases, list) or not all(isinstance(x, str) for x in aliases):
raise ValueError(f"aliases={aliases!r}: expected a list of model names")
names = []
for x in (x.strip() for x in aliases):
if x and x != self.model and x not in names:
names.append(x)
self.aliases = names
def model_names(self) -> list[str]:
return [self.model, *self.aliases]
def model_for(self, req) -> str:
"""The name to answer with: the request's own when it is the model's name or an alias (#297), else the model's.
Other names are still served, as before."""
asked = req.get("model") if isinstance(req, dict) else None
return asked if isinstance(asked, str) and asked in self.aliases else self.model
def reasoning_budget(self, req) -> int | None:
"""#123: the most tokens this request may think, or None: the request's `reasoning_budget_tokens`, else the
config's. 0 (or less) means no budget, so a request can turn a configured one off. ValueError (a 400) for
anything that is not a whole number."""
value = (req or {}).get("reasoning_budget_tokens") if isinstance(req, dict) else None
if value is None:
value = self.reasoning_budget_tokens
if isinstance(value, float) and value.is_integer():
value = int(value)
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError(f"reasoning_budget_tokens={value!r}: expected a whole number of tokens (0: no budget)")
return value if value > 0 else None
def _vision_down(self) -> bool:
return self.vision is not None and hasattr(self.vision, "alive") and not self.vision.alive()
def free_vram_mib(self) -> int | None:
"""Free VRAM on the engine's (first) GPU, from NVML (AMD backend: amdgpu's sysfs files, #301); None when it can't
be read (then nothing is refused)."""
try:
if getattr(self, "backend", None) == "hip":
from serve.telemetry import free_vram_mib
return free_vram_mib(int(getattr(self, "gpu_index", 0) or 0), amd=True)
from serve.telemetry import _Nvml
nv = _Nvml(int(getattr(self, "gpu_index", 0) or 0))
if not nv.ok():
return None
m = nv.Mem()
if nv.lib.nvmlDeviceGetMemoryInfo(nv.dev, ctypes.byref(m)) != 0:
return None
return int(m.free >> 20)
except Exception:
return None
def ensure_loaded(self):
"""Start the engine if it is not running (unloaded, or it died - issue #27), after the before_load hook and
the free-VRAM check. The caller holds self.fifo."""
if self.loaded() and not self._vision_down():
return
if self.before_load:
cmd = self.before_load
print(f"[strata] before loading: {cmd if isinstance(cmd, str) else ' '.join(map(str, cmd))}", flush=True)
try:
subprocess.run(cmd, shell=isinstance(cmd, str), timeout=120, stdin=subprocess.DEVNULL)
except (OSError, subprocess.SubprocessError) as e:
print(f"[strata] the before_load command failed ({e}); loading anyway", flush=True)
if self.min_free_vram_mib:
free = self.free_vram_mib()
deadline = time.time() + 15 # memory another process just gave back can take a moment
while free is not None and free < self.min_free_vram_mib and time.time() < deadline:
time.sleep(0.5)
free = self.free_vram_mib()
if free is not None and free < self.min_free_vram_mib:
raise GpuBusy(f"the GPU is in use by another program: {free} MiB of VRAM free, the model needs "
f"{self.min_free_vram_mib} (min_free_vram_mib) - it stays unloaded until that is free")
if self._vision_down(): # first, as at a start: a GPU encoder takes its VRAM before
print("[strata] starting the vision encoder again ...", flush=True) # the engine sizes its cache
self.vision.restart()
if self.loaded():
return
if getattr(self.engine, "unloaded", False):
print("[strata] loading the model again (it was unloaded) ...", flush=True)
else:
code = self.engine.exit_code() if hasattr(self.engine, "exit_code") else None
print(f"[strata] the engine had stopped (exit code {code}); starting it again "
"(a minute or two) ...", flush=True)
self.engine.restart()
print("[strata] the engine is running again", flush=True)
if self.vram_reserve is not None and hasattr(self.engine, "vram"): # #533: the reserve asked for last
try:
self.engine.vram(self.vram_reserve)
except (ValueError, EngineDied) as e:
print(f"[strata] the VRAM reserve ({self.vram_reserve} MiB) was not applied: {e}", flush=True)
vram_wait_s = 300.0 # #533: how long POST /v1/vram waits for a running request
def vram(self, reserve_mib: int | None) -> dict:
"""#533, POST /v1/vram: keep `reserve_mib` of VRAM free for other programs (None: the reserve the engine
started with), applied between requests - a request that is running finishes first (up to wait_s). Only an
engine started with --vram-elastic (the config's "vram_elastic": true) can do it; it never resizes on its own.
An unloaded engine applies it when it loads. -> {"status": ..., and the engine's figures}."""
if not hasattr(self.engine, "vram"):
raise ValueError("this engine cannot resize its VRAM use")
if not self.fifo.acquire(timeout=self.vram_wait_s):
raise ModelBusy("a request is still running; try again when it has finished")
# parallel requests do not hold the fifo: the engine's control lines (a prompt being admitted) are taken too,
# and the engine refuses a resize while a slot decodes
ctl = getattr(self.engine, "ctl", None) if getattr(self.engine, "batch", 0) else None
if ctl is not None and not ctl.acquire(timeout=self.vram_wait_s):
self.fifo.release()
raise ModelBusy("a request is still running; try again when it has finished")
try:
if not self.loaded():
self.vram_reserve = reserve_mib
return {"status": "not loaded", "reserve_mib": reserve_mib,
"note": "applied when the model loads"}
out = self.engine.vram(reserve_mib)
self.vram_reserve = reserve_mib
print(f"[strata] VRAM: {out.get('vram_free_mib')} MiB free, expert cache {out.get('expert_cache_mib')} of "
f"{out.get('expert_cache_full_mib')} MiB ({out.get('expert_slots')} experts)", flush=True)
return {"status": "ok", **out}
finally:
if ctl is not None:
ctl.release()
self.fifo.release()
def _say_died(self, e: Exception) -> None:
"""The server window's line for an engine that died (or was ended, #481) in the middle of a request."""
note = self.engine.death_note() if hasattr(self.engine, "death_note") else ""
log = getattr(self.engine, "log_path", None)
print(f"[strata] {e}. {note} The next request starts the engine again."
f"{' Its log: ' + log if log else ''}", flush=True)
def load(self):
"""POST /load and every generation request: start the engine now if it is unloaded (raises GpuBusy)."""
# a request is on its way: the idle thread must not unload between this and the request's own start
self.last_request_at = time.time()
if self.loaded() and not self._vision_down():
return
trace = getattr(self.request_trace, "record", None)
waiting = time.perf_counter()
with self.fifo:
loading = time.perf_counter()
if trace is not None:
with self.status_lock:
trace["queue_s"] += round(loading - waiting, 3)
trace["state"] = "loading"
try:
self.ensure_loaded()
finally:
if trace is not None:
with self.status_lock:
trace["load_s"] += round(time.perf_counter() - loading, 3)
trace["state"] = "queued"
def unload(self, idle_for: float | None = None) -> str:
"""Stop the engine between requests: "unloaded", "not loaded", "busy" (a request is running or waiting, or
with idle_for: one ran more recently than that) or "unsupported"."""
if not hasattr(self.engine, "unload"):
return "unsupported"
if not self.fifo.acquire(blocking=False):
return "busy"
try:
if not self.engine.alive():
return "not loaded"
with self.status_lock:
if self.status.get("busy") or self.status.get("queued"):
return "busy"
if idle_for is not None and time.time() - (self.last_request_at or self.started_at) < idle_for:
return "busy"
self.engine.unload()
if self.vision is not None and hasattr(self.vision, "unload"):
self.vision.unload()
print(f"[strata] model unloaded{f' after {idle_for:.0f} s idle' if idle_for else ''}; "
"the next request loads it again", flush=True)
return "unloaded"
finally:
self.fifo.release()
def start_idle_unload(self):
if not self.idle_unload_s or not hasattr(self.engine, "unload"):
return
print(f"[strata] the model unloads after {self.idle_unload_s} s without requests", flush=True)
def loop():
while True:
time.sleep(max(1.0, min(30.0, self.idle_unload_s / 4)))
try:
self.unload(idle_for=self.idle_unload_s)
except EngineStuck as e: # tried again at the next turn; the thread keeps running
print(f"[strata] idle unload: {e}", flush=True)
threading.Thread(target=loop, daemon=True).start()
def set_shared(self, defaults) -> dict:
"""The Chat settings every client gets for what it leaves out; {} / None = clients use their own again."""
self.shared = clean_shared_defaults(defaults)
if self.shared_path:
try:
if self.shared:
Path(self.shared_path).write_text(json.dumps(self.shared, indent=1), encoding="utf-8")
else:
Path(self.shared_path).unlink(missing_ok=True)
except OSError as e:
print(f"[strata] could not save the shared settings: {e}", flush=True)
return self.shared
def with_shared(self, req: dict, api: str) -> dict:
"""The request with the shared thinking level and max tokens filled in where it has none of its own."""
s = self.shared
if not s:
return req
req = dict(req)
if "max_tokens" in s and not req.get("max_tokens") and not req.get("max_completion_tokens"):
req["max_tokens"] = s["max_tokens"]
effort = s.get("reasoning_effort")
if effort:
if api == "openai":
ctk = req.get("chat_template_kwargs") if isinstance(req.get("chat_template_kwargs"), dict) else {}
if not req.get("reasoning_effort") and not req.get("reasoning") and \
"enable_thinking" not in ctk and "reasoning_effort" not in ctk:
req["reasoning_effort"] = effort
elif not req.get("thinking") and not req.get("output_config"):
if effort == "none":
req["thinking"] = {"type": "disabled"}
else:
req["output_config"] = {"effort": effort}
return req
def start_telemetry(self):
"""The hardware sampler behind GET /metrics (serve/telemetry.py), recording this server's tok/s too."""
if getattr(self, "telemetry", None) is None:
from serve.telemetry import Telemetry
self.telemetry = Telemetry(extra=lambda: {"tok_s": self._tok_s(), "tok_s_mean": self._tok_s_mean(),
"prefill_tok_s_mean": self._prefill_tok_s_mean()},
gpu_index=int(getattr(self, "gpu_index", 0) or 0),
gpu_indices=getattr(self, "gpu_indices", None),
amd=getattr(self, "backend", None) == "hip")
def _tok_s(self):
"""tok/s over the last RATE_WINDOW_S seconds. Returns 0.0 while nothing is generating. With parallel requests:
the sum of theirs."""
with self.status_lock:
many = [(dict(st), list(rt)) for st, rt in self.live_reqs.values()]
if many:
return sum(self._rate_of(st, rt) for st, rt in many)
with self.status_lock:
s = dict(self.status)
rate = list(self.rate)
return self._rate_of(s, rate)
@staticmethod
def _rate_of(s, rate):
if not s.get("busy") or not s.get("first_token"):
return 0.0
now = time.time()
newest = rate[-1] if rate else None
oldest = next(((t, g) for t, g in rate if now - t <= RATE_WINDOW_S), None)
if newest and oldest and newest[0] - oldest[0] >= RATE_MIN_SPAN_S:
return max(0.0, (newest[1] - oldest[1]) / (newest[0] - oldest[0]))
return s["generated"] / max(RATE_MIN_SPAN_S, now - s["first_token"])
def _tok_s_mean(self):
"""The whole-request mean since the first token (the old formula), kept so the two can be compared."""
with self.status_lock:
many = [dict(st) for st, _ in self.live_reqs.values()] or [dict(self.status)]
return sum(s["generated"] / max(1e-6, time.time() - s["first_token"]) for s in many
if s.get("busy") and s.get("first_token"))
def _prefill_tok_s_mean(self):
"""Engine-reported mean over newly read tokens, excluding the cached prefix."""
with self.status_lock:
reading = self.status.get("busy") and self.status.get("first_token") is None
return getattr(self.engine, "prefill_tok_s_mean", None) if reading else 0.0
def begin_request(self, path, req):
"""#332: a monitor record for this request, or None when the monitor is off (nothing is kept then)."""
if not self.api_monitor:
return None
raw = json.dumps(req, ensure_ascii=False, indent=2)
record = {"id": uuid.uuid4().hex[:12], "path": path, "model": req.get("model") or self.model,
"started_at": time.time(), "state": "queued", "stream": bool(req.get("stream")),
"response_format": (req.get("response_format") or {}).get("type")
if isinstance(req.get("response_format"), dict) else None,
"input": raw[:262144], "input_truncated": len(raw) > 262144,
"output": "", "reasoning": "", "output_truncated": False, "reasoning_truncated": False,
"queue_s": 0.0, "load_s": 0.0, "first_token_s": None,
"_clock": time.perf_counter()}
with self.status_lock:
self.api_requests.append(record)
self.request_trace.record = record
return record
def request_records(self, request_id=None):
with self.status_lock:
records = list(self.api_requests)
if request_id:
record = next((r for r in records if r["id"] == request_id), None)
if record is None:
return None
return {k: v for k, v in record.items() if not k.startswith("_")}
return [{k: v for k, v in r.items()
if k not in ("input", "output", "reasoning", "response") and not k.startswith("_")}
| {"wallclock_s": r.get("wallclock_s", round(time.perf_counter() - r["_clock"], 3))}
for r in reversed(records)]
def metrics(self, all_requests=False) -> dict:
"""GET /metrics: what the Monitor tab shows - the engine's facts, what it is doing, the last requests, and
the hardware (with a minute of history per series)."""
with self.status_lock:
s = dict(self.status)
if self.live_reqs: # parallel requests: the newest one's status
s = {**s, **dict(list(self.live_reqs.values())[-1][0]), "queued": s.get("queued", 0)}
hist = list(self.history)
totals = dict(self.totals)
now = time.time()
progress = getattr(self.engine, "progress", None)
if s.get("busy") and s.get("first_token") is None:
state = "reading"
elif s.get("busy"):
state = "generating"
elif not self.loaded():
state = "unloaded"
else:
state = "idle"
live = {"state": state, "queued": s.get("queued", 0), "phase": s.get("phase") if s.get("busy") else None,
"prompt_tokens": s.get("prompt_tokens") if s.get("busy") else None,
"prompt_read": None, "prompt_total": None, "generated": s.get("generated") if s.get("busy") else None,
"max_tokens": s.get("max_tokens") if s.get("busy") else None,
"elapsed_s": round(now - s["started"], 1) if s.get("busy") and s.get("started") else None,
"tok_s": round(self._tok_s(), 1) if state == "generating" else None,
"tok_s_mean": round(self._tok_s_mean(), 1) if state == "generating" else None,
"prefill_tok_s_mean": getattr(self.engine, "prefill_tok_s_mean", None) if s.get("busy") else None,
"tok_s_window_s": RATE_WINDOW_S if state == "generating" else None}
if state == "reading" and progress:
live["prompt_read"], live["prompt_total"] = progress
par = int(getattr(self.engine, "batch", 0) or 0)
if par: # #465: the requests running together, slot by slot
with self.status_lock:
running = len(self.live_reqs)
slots = self.engine.slots_view() if hasattr(self.engine, "slots_view") else []
# the requests in flight that are not in a slot: the one alone on the solo path, or waiting their turn
in_slots = sum(1 for x in slots if x["state"] != "idle")
live.update(parallel=par, running=running, slots=slots, outside_slots=max(0, running - in_slots),
waiting=int(getattr(self.engine, "waiting", 0) or 0))
if running and state == "idle":
live["state"] = "generating"
engine = {"model": self.model, "max_context": self.engine.max_context, "images": self.vision is not None,
**dict(getattr(self.engine, "info", {}) or {})}
tel = self.telemetry.snapshot() if getattr(self, "telemetry", None) else {"now": {}, "history": {}, "static": {}}
parked = self.conv_log.poll(getattr(self.engine, "log_path", None), getattr(self.engine, "log_start", None))
return {"engine": engine, "live": live, "requests": hist[::-1][:None if all_requests else 12],
"conversation_cache": conversation_cache_view(engine, hist, totals, parked),
"requests_kept": len(hist), "totals": totals, "hardware": tel["now"],
"hardware_static":
tel["static"], "history": tel["history"], "time": now}
def v1_status(self) -> dict:
"""GET /v1/status: what this server is and does, for a client that would rather ask than guess (a front-end
that polls its OpenAI-compatible server's status, collabosm's for one): the model and its window, images,
the APIs, what is running, and the last request's timings in llama.cpp's names. /metrics has the rest."""
with self.status_lock:
s, totals = dict(self.status), dict(self.totals)
last_t, last_at = (dict(self.last_timings) if self.last_timings else None), self.last_request_at
tel = self.telemetry.snapshot() if getattr(self, "telemetry", None) else {"now": {}, "static": {}}
hw, static = tel.get("now") or {}, tel.get("static") or {}
def scaled(v, unit, digits=0):
return round(v / unit, digits) if isinstance(v, (int, float)) else None
busy, ctx = bool(s.get("busy")), self.engine.max_context
images = self.vision is not None
return {
"service": "strata", "model": self.model,
"loaded": self.loaded(), "auto_load": hasattr(self.engine, "restart"),
"structured_output": {"formats": ["json_object", "json_schema"], "method": "prompt_and_validate",
"constrained_decoding": False, "stream_buffered": True},
"engine": (getattr(self.engine, "info", {}) or {}).get("version"),
"started": int(self.started_at), "uptime_s": int(time.time() - self.started_at),
"cache_max_tokens": ctx,
"context": {"native": ctx, "max_positions": ctx},
# one request at a time (more wait their turn), or "parallel": N batch slots (#465)
"concurrency": {"serving": max(1, int(getattr(self.engine, "batch", 0) or 0)),
"requested": max(1, int(getattr(self.engine, "batch", 0) or 0))},
"dialects": ["/v1/chat/completions", "/v1/messages", "/v1/responses"],
"vision": {"enabled": images, "available": images, "error": None},
"activity": {"requests": totals["requests"] + int(busy), "in_flight": int(busy) + int(s.get("queued") or 0),
"last_request_at": int(last_at) if last_at else None},
"last_timings": last_t,
"vram": dict((getattr(self.engine, "info", {}) or {}).get("vram") or {},
elastic=bool((getattr(self.engine, "info", {}) or {}).get("vram_elastic"))),
"machine": {
"at": int(time.time()),
"gpu": {"name": static.get("gpu_name"), "used_mib": scaled(hw.get("gpu_mem_used"), 2 ** 20),
"total_mib": scaled(hw.get("gpu_mem_total"), 2 ** 20), "util_pct": hw.get("gpu_util"),
"temp_c": hw.get("gpu_temp"), "power_w": hw.get("gpu_power")} if static.get("gpu_name") else None,
"ram": {"used_gib": scaled(hw.get("ram_used"), 2 ** 30, 1),
"total_gib": scaled(hw.get("ram_total"), 2 ** 30, 1)} if hw.get("ram_total") else None}}
def render_prompt(self, messages, tools, kwargs) -> str:
"""The template rendered. #458: with `effort_end`, a request with a non-default effort (low, medium or no
thinking) is rendered as a default one up to the answer - the same prompt start, so the conversation cache
keeps it - and its effort follows in a short system turn right before the answer (thinking off: the template's
empty thinking block). The engine (--tail-role-token) checkpoints in front of that turn."""
effort = kwargs.get("reasoning_effort")
off = kwargs.get("enable_thinking") is False
if not self.effort_end or (not off and effort in (None, "", "xhigh", "high")):
return self.template.render(messages, tools=tools, **kwargs)
base = {k: v for k, v in kwargs.items() if k not in ("reasoning_effort", "enable_thinking")}
history = self.template.render(messages, tools=tools, add_generation_prompt=False, **base)
full = self.template.render(messages, tools=tools, add_generation_prompt=True, **kwargs)
mine = self.template.render(messages, tools=tools, add_generation_prompt=False, **kwargs)
if not full.startswith(mine): # a template this cannot take apart: rendered as asked
return full
tail = "" if off else EFFORT_TURN.format(EFFORT_TEXT.get(effort, EFFORT_TEXT["medium"]))
return history + tail + full[len(mine):]
def encode_prompt(self, messages, tools, kwargs) -> list[int]:
"""The request's prompt: the template rendered and tokenized. #537: a <think> / </think> written inside a
message's text is encoded as the text it is, not as the model's reasoning markers (the template's own are)."""
marked, marked_tools, changed = mark_think_literals(messages, tools)
prompt = self.render_prompt(marked, marked_tools, kwargs)
if not changed:
return self.tok.encode(prompt, parse_special=True)
prompt, plain = unmark_think_literals(prompt)
return self.tok.encode(prompt, parse_special=True, plain=plain)
def prepare(self, messages, tools, kwargs, max_new=None):
"""-> (ids, thinking, max_new). An unset or non-positive max_new (some clients send -1) means "unlimited":
the rest of the context."""
ids = self.encode_prompt(messages, tools, kwargs)
self.embeddings.path = None
images = images_of(messages)
if images:
if self.vision is None:
raise ValueError("this server was started without the vision encoder (run setup again and choose "
"'vision'), so it cannot read images")
pad = self.tok.encode(IMAGE_PAD, parse_special=True)[0]
start = self.tok.encode(VISION_START, parse_special=True)[0]
# An image URL is downloaded first, outside the FIFO: under it, a slow server held every other request
# for as long as urlopen waited.
images = [Vision.download(src) if src.startswith(("http://", "https://")) else src for src in images]
# Encode only while the engine is idle: the engine and the image encoder (a separate process) must not
# run on the GPU at the same time - an encode during a running request left that request stuck at
# "reading the prompt" with CPU and GPU busy, for good (reproduced). So encoding takes its turn in the
# same FIFO as the requests.
with self.fifo:
encoded = [self.vision.encode(src) for src in images]
# one <|image_pad|> per image -> one per image token. Only the markers the template writes for an image
# (right after <|vision_start|>) are images: the same text inside a message (an agent reading these docs,
# #150) is kept as plain text, or it took an image's place and the counts no longer matched.
literal = self.tok.encode(IMAGE_PAD, parse_special=False)
out, k = [], 0
for j, t in enumerate(ids):
if t == pad and j > 0 and ids[j - 1] == start and k < len(encoded):
out += [pad] * encoded[k][1]
k += 1
elif t == pad:
out += literal
else:
out.append(t)
if k != len(encoded):
raise ValueError("the prompt and its images do not match")
ids = out
combined = self.vision.dir / f"req-{uuid.uuid4().hex[:12]}.sve"
with open(combined, "wb") as f:
for path, _ in encoded:
f.write(path.read_bytes())
self.embeddings.path = combined
ctx = self.engine.max_context
if ctx <= 0:
if getattr(self.engine, "starting", False): # #344: (re)starting, not a prompt that is too long
raise EngineStarting("the engine is starting (a minute or two); try again shortly")
# a failed restart leaves max_context 0: check against the last known context, so the request reaches
# run() - which starts the engine again - instead of failing with "exceeds the context (0)" forever
ctx = getattr(self.engine, "known_ctx", 0)
if ctx <= 0:
raise EngineStarting("the engine is starting (a minute or two); try again shortly")
room = ctx - CTX_SLACK - len(ids)
if max_new is None or max_new <= 0 or (self.fit_max_tokens and room < 1):
if room < 1:
raise ValueError(f"prompt ({len(ids)} tokens) leaves no room to answer in the context "
f"({ctx}); requests are never truncated")
max_new = room
elif max_new > room:
if not self.fit_max_tokens:
raise ValueError(f"prompt ({len(ids)} tokens) + max tokens ({max_new}) exceeds the context "
f"({ctx}); requests are never truncated. Send a smaller "
f"max_tokens (at most {max(0, room)} here), or add \"fit_max_tokens\": true to the "
"model's strata-<model>.json to shorten it to the room left (#545)")
max_new = max(1, room) # --fit-max-tokens: a shorter completion beats a 400
return ids, kwargs.get("enable_thinking", True) is not False, max_new
def _note(self, n, evs, st=None, rate=None):
with self.status_lock:
s = self.status if st is None else st
s["generated"] = n
if s.get("first_token") is None:
s["first_token"] = time.time()
(self.rate if rate is None else rate).append((time.time(), n)) # the live rate's window (RATE_WINDOW_S)
for ev in evs:
if ev.kind == "reasoning":
s["phase"] = "thinking"
elif ev.kind == "content":
s["phase"] = "answering"
elif ev.kind == "tool_start":
s["phase"], s["tool"] = f"writing a tool call: {ev.call.name}", ev.call.name
elif ev.kind == "tool_call":
s["phase"] = "tool call complete"
s["tail"] = ((s.get("tail") or "") + (ev.text or ""))[-600:]
def _progress(self, last_print, every=1.0, st=None):
"""A progress line in the server window every `every` seconds while a request runs."""
now = time.time()
if now - last_print < every:
return last_print
with self.status_lock:
s = dict(self.status if st is None else st)
el = now - s.get("started", now)
if s.get("first_token") is None:
pr = getattr(self.engine, "progress", None) # (position reached, prompt tokens): a reused prefix counts
done = f"{pr[0]:,} of {pr[1]:,}" if pr and pr[1] else f"{s.get('prompt_tokens', 0):,}" # as read (#29)
print(f"[strata] reading the prompt: {done} tokens, {el:.0f} s so far", flush=True)
else:
rate = s["generated"] / max(1e-6, now - s["first_token"])
print(f"[strata] {s['phase']}: {s['generated']} of max {s.get('max_tokens')} tokens, {rate:.1f} tok/s, "
f"{el:.0f} s", flush=True)
return now
def run(self, ids, thinking, tools, max_new, sampling, cancel) -> Iterator[tuple[str, object]]:
"""Yields ("event", Event) as text arrives, then ("done", {"finish": .., "completion_tokens": ..})."""
budget = self.reasoning_budget(sampling) if thinking else None # #123: opt-in, off by default
defaults = {**self.sampling_defaults, **self.shared} # the config's, then the Chat settings shared with apps
if defaults: # the request's own fields win (explicit 0 stays greedy)
req_values = {k: v for k, v in (sampling or {}).items() if v is not None}
sampling = {**defaults, **req_values}
parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True)
detok, n, finish = Detokenizer(self.tok), 0, "length"
run_tok, run_len, repeated = None, 0, False # #606: the current run of one repeated token
thinking_n = 0 # tokens written while thinking (Responses' reasoning_tokens)
stop_list = stop_strings(sampling)
stops = StopMatcher(stop_list) if stop_list else None
def cut(evs):
"""The events with the answer's text cut at a stop string (OpenAI stop, Anthropic stop_sequences)."""
if stops is None:
return evs
out = []
for ev in evs:
if stops.hit is not None:
break # nothing after the stop string is sent
if ev.kind == "content":
text = stops.push(ev.text or "")
if text:
out.append(Event("content", text))
continue
held = stops.flush() # a stop string does not run across a tool call or thinking
if held:
out.append(Event("content", held))
out.append(ev)
return out
timings, before = None, None # this request's timings; the engine's `last` before it
raw_ids = [] # every generated id (STRATA_DEBUG: dump raw model text)
emb = getattr(self.embeddings, "path", None)
# Identity token: only a DONE line replaces engine.last, so a request that died, errored or was
# disconnected must not have the PREVIOUS request's decode figures recorded as its own.
engine_last0 = getattr(self.engine, "last", None)
trace = getattr(self.request_trace, "record", None)
waiting = time.perf_counter()
# #465: with "parallel" (the engine's batch slots) requests run at once: each keeps its own status and rate
# window (self.status says busy while any runs); one at a time they are self.status / self.rate, as before
par = bool(getattr(self.engine, "batch", 0))
st = {} if par else self.status
rate = collections.deque(maxlen=32) if par else self.rate
with self.status_lock:
self.status["queued"] += 1
try:
# --batch: the engine runs several requests at once (StrataEngine.generate_batched orders them)
with (contextlib.nullcontext() if getattr(self.engine, "batch", 0) else self.fifo):
try:
with self.status_lock:
if trace is not None:
trace["queue_s"] += round(time.perf_counter() - waiting, 3)
trace["state"] = "generating"
self.status["queued"] -= 1
# issue #27: it died in an earlier request (or was unloaded) - start it again instead of failing
# (parallel requests do not hold the fifo: one of them starts it, the others wait for that)
with (self.fifo if par else contextlib.nullcontext()):
self.ensure_loaded()
with self.status_lock:
st.update(busy=True, phase="reading the prompt", prompt_tokens=len(ids),
generated=0, started=time.time(), first_token=None, tool=None, tail="",
max_tokens=max_new)
if par:
self.live_reqs[id(st)] = (st, rate)
self.status.update(st)
self.last_request_at = time.time()
rate.clear() # the previous request's samples must not leak into this one
before = getattr(self.engine, "last", None)
last_print = time.time()
prompt, thought = ids, 0 # thought: the reasoning tokens so far (the budget's count)
while True:
gen = self.engine.generate(prompt, max_new - n, sampling, cancel, embeddings=emb) if emb \
else self.engine.generate(prompt, max_new - n, sampling, cancel)
seg, wrap, leaving = [], False, False # this pass's tokens; the budget is reached; closed
try:
for t in gen:
if t is None: # heartbeat while the engine is quiet
last_print = self._progress(last_print, st=st)
yield "ping", None
continue
n += 1
if trace is not None and trace["first_token_s"] is None:
with self.status_lock:
trace["first_token_s"] = round(time.perf_counter() - trace["_clock"], 3)
if t in self.stop_ids:
finish = "stop"
raw_ids.append(t)
break
raw_ids.append(t)
seg.append(t)
thinking_n += parser.state in ("reasoning", "rcall")
run_len = run_len + 1 if t == run_tok else 1
run_tok = t
if self.repeat_stop_tokens and run_len >= self.repeat_stop_tokens:
repeated = True # #606: a degenerate output, not an answer: end it here
break
evs = cut(parser.feed(detok.push(t)))
self._note(n, evs, st, rate)
last_print = self._progress(last_print, st=st)
for ev in evs:
yield "event", ev
if stops is not None and stops.hit is not None:
finish = "stop" # gen.close() below STOPs the engine, as for a stop token
break
if budget and parser.state == "reasoning":
thought += 1
# at a clean point: no tag held back, no character split across tokens
if thought >= budget and not parser.buf and not detok.pending():
wrap = True
break
except EngineDied as e:
finish = "error"
self._say_died(e)
raise
except ValueError as e: # the engine's ERR line (it may have ended after it)
finish = "error"
print(f"[strata] the engine reported an error: {e}", flush=True)
raise
except GeneratorExit: # the client went away: an engine that never acknowledges
leaving = True # the STOP below is ended, but no error replaces this
raise
finally:
try:
gen.close() # STOP+drain to THIS request's DONE while still holding the
# fifo, so a stop-token break can't leave the shared engine
# queue mid-drain for the next request to read as its own DONE
except EngineSilent as e: # #481: the STOP was never acknowledged: the engine is ended
finish = "error"
self._say_died(e)
if not leaving and not cancel.is_set():
raise
if not wrap or cancel.is_set():
break
# #123: the thinking reached reasoning_budget_tokens. Close it the way the model would (a
# short wrap-up and </think>) and let it answer: the next pass's prompt is this one plus what
# was generated plus the wrap-up, so the engine continues from the prefix it already holds.
budget = None
extra = self.tok.encode(REASONING_WRAP_UP, parse_special=True)
if max_new - n - len(extra) < 1:
break # no room left to answer: "length", as without a budget
print(f"[strata] thinking budget reached ({thought} tokens): wrapping up the thinking",
flush=True)
for t in extra:
n += 1
raw_ids.append(t)
thinking_n += parser.state in ("reasoning", "rcall")
evs = cut(parser.feed(detok.push(t)))
self._note(n, evs, st, rate)
for ev in evs:
yield "event", ev
if stops is not None and stops.hit is not None:
finish = "stop"
break
prompt = prompt + seg + extra
if cancel.is_set():
finish = "cancel"
elif repeated:
print(f"[strata] the reply repeated one token ({self.tok.decode([run_tok])!r}) "
f"{run_len} times in a row: ended as \"length\" (repeat_stop_tokens in "
"strata-<model>.json; 0 turns this off). If a new request with a short prompt does the "
"same, restart the server and report it (#606)", flush=True)
except GeneratorExit: # the client disconnected mid-stream
finish = "disconnect"
raise
finally:
# #266: settle this request's status, history and totals while still holding the fifo: once
# it is released the next request sets its own status, which this must not record or clear
with self.status_lock:
if st.get("busy"):
# only this request's DONE counts: same object means no DONE arrived (death, error,
# disconnect)
last = dict(getattr(self.engine, "last", {}) or {}) \
if getattr(self.engine, "last", None) is not engine_last0 else {}
started = st.get("started", time.time())
cvec = (getattr(self.engine, "info", {}) or {}).get("cvec", 0)
loaded = str(cvec) not in ("0", "", "None")
hit_rate = round(last["hits"] / last["lookups"], 3) if last.get("lookups") else None
# #588: the share the GPU read over PCIe (--pcie-frac) or another GPU computed - not in
# the hit rate's lookups, which is the VRAM share of the rest (None: an older engine)
routed = (last.get("lookups") or 0) + (last.get("offloaded") or 0)
pcie_share = round(last["offloaded"] / routed, 3) \
if last.get("offloaded") is not None and routed else None
seen = prompt_tokens_seen(len(ids), last) # #471: < len(ids) when cancelled mid-read
self.history.append({
"projection": (sampling or {}).get("experimental_speed_projection") is not False
if loaded else None,
"time": started, "duration_s": round(time.time() - started, 1), "finish": finish,
"prompt_tokens": seen, "reused": last.get("reused"), "output_tokens": n,
# the request's whole prompt, and the tokens read of it (None: an older engine)
"prompt_total": len(ids), "prompt_read": last.get("prompt_read"),
"engine_generated": last.get("generated"),
"prompt_ms": last.get("prompt_ms"), "decode_ms": last.get("decode_ms"),
"decode_tok_s": round(last["generated"] / (last["decode_ms"] / 1000), 1)
if n and last.get("generated") and last.get("decode_ms") else None,
"hit_rate": hit_rate, "pcie_share": pcie_share, "ram_blobs": last.get("ram_blobs"),
"file_blobs": last.get("file_blobs"), "file_mb": last.get("file_mb"),
# #457: the speculative drafts from the DONE line (None: the engine did not say)
"drafts_offered": last.get("drafts_offered"),
"drafts_accepted": last.get("drafts_accepted")})
t = self.totals
t["requests"] += 1
t["prompt_tokens"] += seen
t["reused"] += last.get("reused") or 0
t["output_tokens"] += n
t["prompt_ms"] += last.get("prompt_ms") or 0.0
t["decode_ms"] += last.get("decode_ms") or 0.0
t["drafts_offered"] += last.get("drafts_offered") or 0
t["drafts_accepted"] += last.get("drafts_accepted") or 0
fresh = getattr(self.engine, "last", None)
if fresh is not None and fresh is not before: # the engine's clock for THIS request
timings = request_timings(seen, n, last)
self.last_timings = dict(timings, at=int(time.time())) if timings else None
self.last_request_at = time.time()
now = time.time()
el = now - st.get("started", now)
ft = st.get("first_token")
rate = n / max(1e-6, now - ft) if ft else 0.0
hit_msg = f", expert cache {hit_rate*100:.1f}% hit" if hit_rate is not None else ""
if hit_msg and pcie_share:
hit_msg += f" (+{pcie_share*100:.1f}% of the routed experts over PCIe)"
print(f"[strata] done: {n} tokens in {el:.0f} s ({rate:.1f} tok/s) "
f"({finish}, cancel={cancel.is_set()}){hit_msg}", flush=True)
if finish == "length" and parser.state in ("reasoning", "rcall"): # #530
print("[strata] the reply reached max tokens while still thinking, so it has no "
"answer: a thinking budget (reasoning_budget_tokens, in the request or in "
"strata-<model>.json for every request) leaves room to answer", flush=True)
if os.environ.get("STRATA_DEBUG") and raw_ids:
print(f"[strata] raw: {self.tok.decode(raw_ids)!r}", flush=True)
st["busy"] = False
st.pop("tail", None) # #212: the answer's end is not kept once it is done
st.pop("tool", None)
if par:
self.live_reqs.pop(id(st), None)
if self.live_reqs: # the newest request still running
self.status.update(list(self.live_reqs.values())[-1][0])
else:
self.status.update(busy=False)
self.status.pop("tail", None)
self.status.pop("tool", None)
finally:
if emb:
Path(emb).unlink(missing_ok=True)
for ev in cut(parser.finish()):
yield "event", ev
if stops is not None and stops.hit is None and stops.held:
yield "event", Event("content", stops.flush()) # the held tail was not a stop string after all
done = {"finish": finish, "completion_tokens": n, "reused": (timings or {}).get("cache_n", 0),
"timings": timings, "reasoning_tokens": thinking_n}
if stops is not None and stops.hit is not None:
done["stop_sequence"] = stops.hit
yield "done", done
def prompt_tokens_seen(prompt_tokens: int, last: dict) -> int:
"""#471: the prompt tokens a request got through - all of them, unless the engine's DONE line says a cancel stopped
its prompt read part-way (then the reused ones plus those read). /metrics' history and totals count these, so a
cancelled read is neither recorded as the whole prompt nor given a rate from tokens it never read. An engine
before 0.1.36 does not say (no `prompt_read`): the whole prompt, as before."""
read = last.get("prompt_read")
if read is None or last.get("finish") != "cancel":
return prompt_tokens
return min(prompt_tokens, int(last.get("reused") or 0) + int(read))
def request_timings(prompt_tokens: int, generated: int, last: dict) -> dict | None:
"""One request's `timings` in llama.cpp's names (what its clients show as speed), from the engine's own clock
(StrataEngine.last): prompt_n is what was read, cache_n what the conversation cache already held. None when the
engine keeps no clock (MockEngine)."""
if last.get("prompt_ms") is None:
return None
cache_n = int(last.get("reused") or 0)
prompt_n, prompt_ms, decode_ms = max(0, prompt_tokens - cache_n), float(last["prompt_ms"]), float(last.get("decode_ms") or 0)
decoded = int(last.get("generated") or generated) # the engine's count gives its rate, as /metrics does
return {"cache_n": cache_n, "prompt_n": prompt_n, "prompt_ms": round(prompt_ms, 1),
"prompt_per_token_ms": round(prompt_ms / prompt_n, 3) if prompt_n else None,
"prompt_per_second": round(prompt_n / (prompt_ms / 1000), 1) if prompt_n and prompt_ms > 0 else None,
"predicted_n": generated, "predicted_ms": round(decode_ms, 1),
"predicted_per_token_ms": round(decode_ms / decoded, 3) if decoded else None,
"predicted_per_second": round(decoded / (decode_ms / 1000), 1) if decoded and decode_ms > 0 else None,
# the speculative drafts, as llama.cpp names them (from PR #83, @mikicvi): only when the engine reported them
**({"draft_n": int(last["drafts_offered"]), "draft_n_accepted": int(last["drafts_accepted"])}
if last.get("drafts_offered") is not None else {})}
def _debug_req(api, req, messages, tools, max_new, thinking, prompt_tokens):
"""One compact line per request while diagnosing blank/empty turns. Set STRATA_DEBUG=1 to enable."""
if not os.environ.get("STRATA_DEBUG"):
return
last = messages[-1] if messages else {}
body = last.get("content")
if isinstance(body, list):
body = " ".join(p.get("text", "") for p in body if isinstance(p, dict))
preview = (str(body or "")[:80]).replace("\n", " ")
print(f"[strata] req {api}: msgs={len(messages)} tools={len(tools or [])} "
f"max_tokens_raw={req.get('max_tokens')!r}/{req.get('max_completion_tokens')!r} "
f"max_new={max_new} thinking={thinking} stream={bool(req.get('stream'))} "
f"prompt_tokens={prompt_tokens} last={last.get('role')!r}:{preview!r}", flush=True)
# ------------------------------------------------------------------------------------------------ MCP tool loop
def run_with_mcp(svc: Service, hub, messages, tools, kw, ids, thinking, max_new, max_req, sampling, cancel,
mcp_names):
"""Service.run with the MCP tools executed here: the model writes a call to an MCP tool, the server runs it, adds
the call and its result to the conversation and lets the model continue - up to `max_rounds` times. Yields what
Service.run yields (text, thinking, the request's own tool calls) plus ("mcp", {...}) for the tool activity, and
one ("done", ...) at the very end with the output tokens of every round.
`mcp_names`: the MCP tools this request offered; any other call is one of the request's own tools and ends the
turn as always (the client answers it). MCP calls written in the same answer are then not run (their results
could not reach the model before the client's)."""
max_rounds = int(hub.settings["max_rounds"])
total, rounds, done = 0, 0, None
messages = list(messages)
while True:
text, reasoning, calls, own_calls = [], [], [], 0
for kind, x in svc.run(ids, thinking, tools, max_new, sampling, cancel):
if kind == "done":
done = x
continue
if kind == "event":
ev: Event = x
if ev.call is not None and ev.call.name in mcp_names:
if ev.kind == "tool_start": # the model has started writing a call: say so at once
yield "mcp", {"event": "start", "id": ev.call.id, "name": ev.call.name}
elif ev.kind == "tool_call":
calls.append(ev.call)
continue # its argument pieces are not streamed to the client
if ev.kind == "tool_call":
own_calls += 1
elif ev.kind == "content":
text.append(ev.text)
elif ev.kind == "reasoning":
reasoning.append(ev.text)
yield kind, x
total += done["completion_tokens"]
run_them = calls and not own_calls and done["finish"] == "stop" and not cancel.is_set()
if run_them and rounds >= max_rounds:
yield "mcp", {"event": "limit", "max_rounds": max_rounds}
run_them = False
if not run_them:
for c in calls: # announced, never run: close them in the client's view
yield "mcp", {"event": "result", "id": c.id, "ok": False, "skipped": True, "text": "not run",
"chars": 0, "truncated": False, "ms": 0}
break
rounds += 1
results = []
for c in calls:
s, tool = hub.routes().get(c.name, (None, c.name))
yield "mcp", {"event": "call", "id": c.id, "name": c.name, "server": s.name if s else None,
"tool": tool, "arguments": c.arguments, "round": rounds}
# The call runs on a thread while this generator keeps yielding heartbeats: they reach the client as
# keep-alives, which is how a client that went away (the web app's Stop) is noticed during a slow tool.
box = {}
def work(c=c, box=box):
try:
box["r"] = hub.call(c.name, c.arguments, cancel)
except McpCancelled:
box["cancelled"] = True
worker = threading.Thread(target=work, daemon=True)
worker.start()
try:
while worker.is_alive():
worker.join(1.0)
if worker.is_alive():
yield "ping", None
except GeneratorExit:
cancel.set() # the client is gone: stop the tool too
raise
if "r" not in box:
break
r = box["r"]
print(f"[strata] tool {c.name}: {'ok' if r['ok'] else 'error'}, {r['chars']:,} characters in "
f"{r['ms'] / 1000:.1f} s{' (truncated for the model)' if r['truncated'] else ''}", flush=True)
results.append(r["text"])
yield "mcp", {"event": "result", "id": c.id, **{k: r[k] for k in ("ok", "text", "chars", "truncated", "ms")}}
if cancel.is_set() or len(results) < len(calls):
done = {**done, "finish": "cancel"}
break
messages.append({"role": "assistant", "content": "".join(text).strip(),
**({"reasoning_content": "".join(reasoning).strip()} if reasoning else {}),
"tool_calls": [{"function": {"name": c.name, "arguments": c.arguments}} for c in calls]})
messages += [{"role": "tool", "content": r} for r in results]
ids, thinking, max_new = svc.prepare(messages, tools, kw, max_req)
yield "done", {**done, "completion_tokens": total, "prompt_tokens": len(ids)}
# ------------------------------------------------------------------------------------------------ OpenAI
def openai_chunks(svc: Service, req: dict, ids, thinking, tools, max_new, cancel, run=None):
"""`run`: the events to send instead of Service.run's (run_with_mcp); its ("mcp", {...}) items become chunks with
an empty delta and a `strata_mcp` field, which only the web app reads."""
cid, created = "chatcmpl-" + uuid.uuid4().hex[:24], int(time.time())
model = svc.model_for(req)
def chunk(delta, finish=None):
return {"id": cid, "object": "chat.completion.chunk", "created": created, "model": model,
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}]}
yield chunk({"role": "assistant", "content": ""})
calls = 0
streamed = {} # tool call id -> index, for calls sent piece by piece
finished = set() # ... and the ones whose final tool_call came (#211)
for kind, x in run if run is not None else svc.run(ids, thinking, tools, max_new, req, cancel):
if kind == "ping":
yield None
elif kind == "mcp":
c = chunk({})
c["strata_mcp"] = x
yield c
elif kind == "event":
ev: Event = x
if ev.kind == "reasoning" and ev.text:
yield chunk({"reasoning_content": ev.text})
elif ev.kind == "content" and ev.text:
yield chunk({"content": ev.text})
elif ev.kind == "tool_start":
streamed[ev.call.id] = calls
calls += 1
yield chunk({"tool_calls": [{"index": streamed[ev.call.id], "id": ev.call.id, "type": "function",
"function": {"name": ev.call.name, "arguments": ""}}]})
elif ev.kind == "tool_args":
yield chunk({"tool_calls": [{"index": streamed[ev.call.id], "function": {"arguments": ev.text}}]})
elif ev.kind == "tool_call" and ev.call.id in streamed:
finished.add(ev.call.id)
elif ev.kind == "tool_call":
yield chunk({"tool_calls": [{"index": calls, "id": ev.call.id, "type": "function",
"function": {"name": ev.call.name,
"arguments": json.dumps(ev.call.arguments, ensure_ascii=False)}}]})
calls += 1
else:
# a streamed call without its final tool_call is one the output ended inside: not "tool_calls" (#211)
whole = calls and streamed.keys() <= finished
finish = "tool_calls" if whole and x["finish"] == "stop" else {"cancel": "stop"}.get(x["finish"], x["finish"])
last = chunk({}, finish)
pt = x.get("prompt_tokens", len(ids)) # after MCP rounds: the last round's prompt
last["usage"] = {"prompt_tokens": pt, "completion_tokens": x["completion_tokens"],
"total_tokens": pt + x["completion_tokens"],
# the part of the prompt the conversation cache already held (OpenAI's field)
"prompt_tokens_details": {"cached_tokens": x.get("reused") or 0}}
if x.get("timings"):
last["timings"] = x["timings"] # llama.cpp's field: the speed its clients show
yield last
def _is_json(text: str) -> bool:
try:
json.loads(text)
return True
except ValueError:
return False
def openai_collect(chunks) -> dict:
content, reasoning, by_index, last, mcp = [], [], {}, None, []
for c in chunks:
if c is None: # a heartbeat
continue
if c.get("strata_mcp"):
mcp.append(c["strata_mcp"])
d = c["choices"][0]["delta"]
content.append(d.get("content") or "")
reasoning.append(d.get("reasoning_content") or "")
for tc in d.get("tool_calls") or []: # streamed calls arrive in pieces: merge them by index
cur = by_index.setdefault(tc.get("index", len(by_index)), {"id": None, "type": "function",
"function": {"name": "", "arguments": ""}})
cur["id"] = tc.get("id") or cur["id"]
fn = tc.get("function") or {}
cur["function"]["name"] += fn.get("name") or ""
cur["function"]["arguments"] += fn.get("arguments") or ""
last = c
calls = [by_index[i] for i in sorted(by_index)]
if last["choices"][0]["finish_reason"] != "tool_calls":
calls = [c for c in calls if _is_json(c["function"]["arguments"])] # a call the output ended inside (#211)
msg = {"role": "assistant", "content": "".join(content) or None}
if "".join(reasoning):
msg["reasoning_content"] = "".join(reasoning)
if calls:
msg["tool_calls"] = calls
if mcp:
msg["strata_mcp"] = mcp
out = {"id": last["id"], "object": "chat.completion", "created": last["created"], "model": last["model"],
"choices": [{"index": 0, "message": msg, "finish_reason": last["choices"][0]["finish_reason"]}],
"usage": last["usage"]}
if last.get("timings"):
out["timings"] = last["timings"]
return out
def structured_chunks(chunks, validator):
"""Buffer structured streams so a client never receives unvalidated content."""
buffered = []
heartbeat = time.monotonic()
try:
for chunk in chunks:
if chunk is not None:
buffered.append(chunk)
if chunk is None or time.monotonic() - heartbeat >= 1:
heartbeat = time.monotonic()
yield None
result = openai_collect(buffered)
choice = result["choices"][0]
content = validated_json(choice["message"]["content"], validator, choice["finish_reason"])
yield buffered[0]
delta = {"content": content}
if choice["message"].get("reasoning_content"):
delta["reasoning_content"] = choice["message"]["reasoning_content"]
yield {**buffered[0], "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}
yield buffered[-1]
finally:
chunks.close()
# ------------------------------------------------------------------------------------------------ Anthropic
def anthropic_events(svc: Service, req: dict, ids, thinking, tools, max_new, cancel):
mid = "msg_" + uuid.uuid4().hex[:24]
yield "message_start", {"type": "message_start", "message": {
"id": mid, "type": "message", "role": "assistant", "model": svc.model_for(req), "content": [],
"stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": len(ids), "output_tokens": 0}}}
index, open_kind, used_tool = -1, None, False
def close():
return ("content_block_stop", {"type": "content_block_stop", "index": index})
streamed, finished = set(), set() # calls sent piece by piece; those whose final tool_call came (#211)
for kind, x in svc.run(ids, thinking, tools, max_new, req, cancel):
if kind == "ping":
yield None
continue
if kind == "event":
ev: Event = x
if ev.kind == "tool_args":
yield "content_block_delta", {"type": "content_block_delta", "index": index,
"delta": {"type": "input_json_delta", "partial_json": ev.text}}
continue
if ev.kind == "tool_call" and ev.call.id in streamed:
finished.add(ev.call.id)
continue
want = {"reasoning": "thinking", "content": "text", "tool_call": "tool_use", "tool_start": "tool_use"}[ev.kind]
if ev.kind not in ("tool_call", "tool_start") and not ev.text:
continue
if ev.kind == "tool_start":
streamed.add(ev.call.id)
used_tool = True
if open_kind != want or want == "tool_use":
if open_kind is not None:
yield close()
index += 1
open_kind = want
block = {"thinking": {"type": "thinking", "thinking": "", "signature": ""},
"text": {"type": "text", "text": ""},
"tool_use": {"type": "tool_use", "id": ev.call.id if ev.call else "", "name":
ev.call.name if ev.call else "", "input": {}}}[want]
yield "content_block_start", {"type": "content_block_start", "index": index, "content_block": block}
if want == "thinking":
yield "content_block_delta", {"type": "content_block_delta", "index": index,
"delta": {"type": "thinking_delta", "thinking": ev.text}}
elif want == "text":
yield "content_block_delta", {"type": "content_block_delta", "index": index,
"delta": {"type": "text_delta", "text": ev.text}}
elif ev.kind == "tool_start":
pass # its input follows as tool_args pieces
else:
used_tool = True
yield "content_block_delta", {"type": "content_block_delta", "index": index, "delta": {
"type": "input_json_delta", "partial_json": json.dumps(ev.call.arguments, ensure_ascii=False)}}
else:
if open_kind is not None:
yield close()
stop = "stop_sequence" if x.get("stop_sequence") is not None else \
"tool_use" if used_tool and streamed <= finished and x["finish"] == "stop" else \
{"stop": "end_turn", "length": "max_tokens", "cancel": "end_turn"}[x["finish"]]
# the final counts, Anthropic's way: input_tokens leaves out what the conversation cache already held,
# which is cache_read_input_tokens (message_start could only say the whole prompt)
reused = min(x.get("reused") or 0, len(ids))
yield "message_delta", {"type": "message_delta", "delta": {"stop_reason": stop,
"stop_sequence": x.get("stop_sequence")},
"usage": {"input_tokens": len(ids) - reused, "cache_read_input_tokens": reused,
"output_tokens": x["completion_tokens"]}}
yield "message_stop", {"type": "message_stop"}
def anthropic_collect(events) -> dict:
msg, blocks = None, []
for item in events:
if item is None: # a heartbeat
continue
name, e = item
if name == "message_start":
msg = e["message"]
elif name == "content_block_start":
blocks.append(dict(e["content_block"]))
elif name == "content_block_delta":
d, b = e["delta"], blocks[-1]
if d["type"] == "text_delta":
b["text"] += d["text"]
elif d["type"] == "thinking_delta":
b["thinking"] += d["thinking"]
else: # input_json_delta pieces: parsed when complete
b["_json"] = b.get("_json", "") + d["partial_json"]
elif name == "content_block_stop" and blocks and "_json" in blocks[-1]:
b = blocks[-1]
try:
b["input"] = json.loads(b.pop("_json") or "{}")
except ValueError: # a call the output ended inside (#211): it has no input to give
blocks.pop()
elif name == "message_delta":
msg["stop_reason"] = e["delta"]["stop_reason"]
msg["stop_sequence"] = e["delta"].get("stop_sequence")
msg["usage"].update(e["usage"])
msg["content"] = blocks
return msg
# ------------------------------------------------------------------------------------------------ HTTP
def make_handler(svc: Service):
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.0" # SSE ends by closing the connection
record = None # #332: this request's monitor record, if kept
watch_done = None # #430 #431: stops this request's disconnect watcher
body_read = False # whether a handler took this request's body
DRAIN_SECONDS = 5 # the longest an unread body is read and dropped
def log_message(self, fmt, *args):
pass
def handle_one_request(self):
super().handle_one_request()
self._drain_body()
def _body(self) -> bytes:
self.body_read = True
return self.rfile.read(int(self.headers.get("Content-Length", 0)))
def _drain_body(self):
"""An answer sent before the body was read (a 401, a 403, /load, a method with no handler) must not close
the connection on unread bytes: the close then sends a reset, and a client that sends its body after the
headers (http.client, urllib, requests) gets a connection error instead of the answer. So the body is
read and dropped here, once, after an answer, in pieces so that its size is never held in memory. What
has not arrived DRAIN_SECONDS later is left unread: the limit is time, so that a conversation of many
megabytes, at any speed the client has, still gets its answer."""
headers = getattr(self, "headers", None)
if self.body_read or headers is None:
return
try:
left = int(headers.get("Content-Length", 0))
except ValueError:
return
if left <= 0:
return
deadline = time.monotonic() + self.DRAIN_SECONDS
try:
while left > 0 and (wait := deadline - time.monotonic()) > 0:
self.connection.settimeout(wait) # a client that never sends what it announced
piece = self.rfile.read1(min(left, 1 << 20)) # one socket read: read() would wait for it all
if not piece:
break
left -= len(piece)
except OSError:
pass
def parse_request(self):
"""Without an API key, every request (any method) first passes the Host check: DNS rebinding protection
(host_allowed). With a key a rebinding page cannot authenticate, so the check is skipped: tunnels and
proxies that pass their own name on keep working."""
if not super().parse_request():
return False
host = self.headers.get("Host")
if svc.api_key or host_allowed(host, svc.host_names, "*" in svc.allowed_hosts):
return True
print(f"[strata] refused a request for Host {host!r} from {self.client_address[0]}: not a name this server "
f"answers to (add it to \"allowed_hosts\" in the config or STRATA_ALLOWED_HOSTS, or set an API key)",
flush=True)
self._json(403, {"error": {"type": "forbidden", "message":
f"Host {host!r} is not allowed (DNS rebinding protection). Reaching Strata under this "
f"name on purpose? Add it to \"allowed_hosts\" in the config (strata-<model>.json) or to "
f"the STRATA_ALLOWED_HOSTS environment variable, or set an API key (\"api_key\"), which "
f"turns this check off"}})
return False
def _foreign_page(self) -> bool:
"""Without an API key, a /v1 POST from a browser page of another site (any site can POST text/plain
there without a CORS preflight) would burn GPU time: an Origin header must name an allowed page, and
then the body must be JSON. No Origin (curl, the SDKs, other servers): any content type, as before."""
origin = self.headers.get("Origin")
if svc.api_key or not origin:
return False
if not origin_allowed(origin, self.headers.get("Host"), svc.host_names,
[*svc.trusted_origins, *svc.cors_origins]):
print(f"[strata] refused an API request from the web page {origin!r} (no API key; add its host to "
f"\"allowed_hosts\" or its origin to \"cors_origins\" in the config)", flush=True)
self._json(403, {"error": {"type": "forbidden", "message":
f"web pages of {origin} may not use this server without an API key; set \"api_key\", "
f"or add the page's host to \"allowed_hosts\" (or its origin to \"cors_origins\") "
f"in the config"}})
return True
if not self.headers.get("Content-Type", "").startswith("application/json"):
self._json(415, {"error": {"message": "send application/json"}})
return True
return False
def _watch_client(self, cancel: threading.Event) -> None:
"""#430 #431: cancel the request as soon as its client hangs up. A non-streamed request writes nothing
until it ends, and a streamed one only a keep-alive per prompt chunk (and the first write after a hang-up
usually still succeeds), so a dropped request kept the engine busy until its answer or the whole prompt
was done. Every 0.5 s: the socket readable with nothing to read (EOF) means the client closed it. A
request is HTTP/1.0 and fully read here, so no later bytes are expected - data is not a hang-up."""
done = self.watch_done = threading.Event()
sock = self.connection
def watch():
while not done.wait(0.5) and not cancel.is_set():
try:
readable, _, _ = select.select([sock], [], [], 0)
gone = bool(readable) and sock.recv(1, socket.MSG_PEEK) == b""
except (ConnectionError, TimeoutError):
gone = True
except (OSError, ValueError): # the socket was closed here: the request has ended
return
if gone:
self._note(outcome="disconnected")
cancel.set()
return
threading.Thread(target=watch, daemon=True, name="strata-client-watch").start()
def _note(self, **values):
"""#332: what the monitor shows about this request (nothing when the monitor is off)."""
if self.record is not None:
with svc.status_lock:
self.record.update(values)
def _cors(self):
"""#321: CORS headers for an API path (/v1/*) and an origin the config lists in cors_origins - nothing
otherwise, so a browser keeps every other page away from the API, /settings, /unload and the MCP tools."""
if not svc.cors_origins or not self.path.split("?")[0].startswith("/v1/"):
return
origin = (self.headers.get("Origin") or "").rstrip("/")
if "*" in svc.cors_origins:
allow = "*"
elif origin and origin in svc.cors_origins:
allow = origin
self.send_header("Vary", "Origin")
else:
return
self.send_header("Access-Control-Allow-Origin", allow)
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
# the headers the preflight asks for (SDKs add their own; "*" does not cover Authorization)
asked = self.headers.get("Access-Control-Request-Headers")
self.send_header("Access-Control-Allow-Headers",
asked or "Authorization, Content-Type, x-api-key, anthropic-version, anthropic-beta")
self.send_header("Access-Control-Max-Age", "600")
def do_OPTIONS(self):
# a CORS preflight: no credentials come with it, so no API key; the headers only for cors_origins
self.send_response(204)
self._cors()
self.send_header("Content-Length", "0")
self.end_headers()
def _json(self, code, obj):
if self.record is not None:
with svc.status_lock:
self.record["http_status"] = code
raw = json.dumps(obj, ensure_ascii=False, indent=2)
self.record.update(response=raw[:262144], response_truncated=len(raw) > 262144)
for key in ("error", "usage", "timings"):
if key in obj:
self.record[key] = obj[key]
body = json.dumps(obj, ensure_ascii=False).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json")
self._cors()
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _authorized(self) -> bool:
if not svc.api_key:
return True
auth = self.headers.get("Authorization", "")
given = auth[7:].strip() if auth.lower().startswith("bearer ") else self.headers.get("x-api-key", "")
if hmac.compare_digest(given.encode(), svc.api_key.encode()): # #213: constant-time
return True
self._json(401, {"error": {"type": "authentication_error", "message": "missing or wrong API key"}})
return False
def do_GET(self):
path = self.path.split("?")[0].rstrip("/")
if path.startswith("/fonts/"):
# the web app's font (Outfit, OFL: serve/web/fonts); the page falls back to the system font
name = path[len("/fonts/"):]
f = ROOT / "serve" / "web" / "fonts" / name
if "/" in name or "\\" in name or not name.endswith(".woff2") or not f.is_file():
self._json(404, {"error": {"message": "not found"}})
return
body = f.read_bytes()
self.send_response(200)
self.send_header("Content-Type", "font/woff2")
self.send_header("Cache-Control", "max-age=86400")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
if path.startswith("/web/"):
# the web app's own files (serve/web): styles, script, icon sprite - same origin, no CDN
name = path[len("/web/"):]
types = {".css": "text/css; charset=utf-8", ".js": "text/javascript; charset=utf-8",
".svg": "image/svg+xml"}
f = ROOT / "serve" / "web" / name
ext = os.path.splitext(name)[1]
if "/" in name or "\\" in name or ext not in types or not f.is_file():
self._json(404, {"error": {"message": "not found"}})
return
body = f.read_bytes()
self.send_response(200)
self.send_header("Content-Type", types[ext])
self.send_header("Cache-Control", "no-cache")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
if path == "/metrics":
if self._authorized():
# the last 12 requests; `?requests=all` every one kept (the Monitor's "Show all", issue #35)
self._json(200, svc.metrics(all_requests="requests=all" in self.path))
return
if path == "/api/requests" and svc.api_monitor:
if self._authorized():
request_id = parse_qs(urlsplit(self.path).query).get("id", [None])[0]
records = svc.request_records(request_id)
self._json(404 if records is None else 200,
{"error": {"message": "request no longer retained"}} if records is None else
records if request_id else {"requests": records, "retention": 100, "persistent": False,
"loaded": svc.loaded(), "auto_load": hasattr(svc.engine, "restart")})
return
if path == "/settings":
if self._authorized():
self._json(200, {"shared": bool(svc.shared), "defaults": svc.shared})
return
if path == "/config": # #564: the Settings view's keys of the run config
if self._authorized():
self._config_get()
return
if path == "/mcp":
# the MCP servers, their state and tools (the web app's switch and Monitor card)
if self._authorized():
self._json(200, svc.mcp.status() if svc.mcp else {"servers": [], "tools": 0})
return
if path == "" or (path == "/api-monitor" and svc.api_monitor):
body = (ROOT / "serve" / "web" / ("monitor.html" if path else "index.html")).read_bytes()
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
elif path in ("/health", "/api/health"):
self._json(200, {"status": "ok", "max_context": svc.engine.max_context, "model": svc.model,
"images": svc.vision is not None, "api_key": bool(svc.api_key),
"loaded": svc.loaded(), "service": "strata"})
elif path == "/status":
if not self._authorized(): # #212: it shows the end of the last answer
return
with svc.status_lock:
s = dict(svc.status)
now = time.time()
if s.get("busy"):
s["elapsed_s"] = round(now - s["started"], 1)
if s.get("first_token"):
s["tokens_per_s"] = round(svc._tok_s(), 1)
s["tokens_per_s_mean"] = round(svc._tok_s_mean(), 1)
for k in ("started", "first_token"):
s.pop(k, None)
self._json(200, s)
elif path in ("/v1/models", "/models"):
if self._authorized():
loaded = svc.loaded()
model = {"id": svc.model, "object": "model", "status": {"value": "loaded"},
"meta": {"n_ctx": svc.engine.max_context},
"architecture": {"input_modalities": ["text", "image"] if svc.vision is not None else ["text"],
"output_modalities": ["text"]}}
if not loaded and (svc.idle_unload_s or getattr(svc.engine, "unloaded", False)):
model["status"] = {"value": "unloaded"} # like llama-server's router: listed, loads on use
loaded = True
if svc.aliases: # #297: the aliases, and each one listed under its own id
model["aliases"] = list(svc.aliases)
data = [model, *({**model, "id": x, "alias_of": svc.model} for x in svc.aliases)]
self._json(200, {"object": "list", "data": data if loaded else []})
elif path == "/props":
if self._authorized():
self._props()
elif path == "/slots":
if self._authorized():
loaded = not hasattr(svc.engine, "alive") or svc.engine.alive()
with svc.status_lock:
busy = bool(svc.status.get("busy"))
slot = {"id": 0, "n_ctx": svc.engine.max_context, "is_processing": busy}
self._json(200, [slot] if loaded else [])
elif path == "/v1/status":
if self._authorized():
self._json(200, svc.v1_status())
else:
self._json(404, {"error": {"message": "not found"}})
def do_POST(self):
if not self._authorized():
return
path = self.path.split("?")[0].rstrip("/") # issue #55: Claude Code posts /v1/messages?beta=true
if path.startswith("/v1/") and self._foreign_page():
return
if path == "/settings":
self._settings()
return
if path == "/config":
self._config_post()
return
if path in ("/unload", "/load") and not self._control_body():
return
# JSON from Strata's own page only, as /settings: else a plain form POST from any site unloads the model
if path in ("/unload", "/load") and not self._own_page("the model can be loaded or unloaded"):
return
if path == "/unload": # give the GPU back now (between requests)
try:
r = svc.unload()
except EngineStuck as e:
self._json(503, {"error": {"type": "server_error", "message": str(e)}})
return
self._json(409 if r == "busy" else 200, {"status": r})
return
if path == "/load": # load now, e.g. ahead of a request
try:
svc.load()
self._json(200, {"status": "loaded"})
except GpuBusy as e:
self._json(503, {"error": {"type": "server_error", "message": str(e)}})
return
try:
req = json.loads(self._body() or b"{}")
if not isinstance(req, dict):
raise ValueError("send a JSON object")
if path.startswith("/v1/responses/"): # retrieve/delete/cancel/compact: nothing is stored
self._json(404, responses_error_body("this server keeps no responses (stateless); send the "
"whole conversation to POST /v1/responses", code="not_found"))
return
if path in ("/v1/load", "/v1/unload"):
if not self._own_page("the model can be loaded or unloaded"):
return
if req.get("model") not in (None, svc.model):
self._json(404, {"error": {"message": "model not found"}})
return
if path == "/v1/unload":
result = svc.unload()
if result == "busy":
raise ModelBusy("a request is running or queued")
else:
if not svc.fifo.acquire(blocking=False):
raise ModelBusy("a request is running or queued")
try:
with svc.status_lock:
if svc.status.get("busy") or svc.status.get("queued"):
raise ModelBusy("a request is running or queued")
svc.ensure_loaded()
svc.last_request_at = time.time()
finally:
svc.fifo.release()
result = "loaded"
self._json(200, {"status": result, **svc.v1_status()})
return
if path == "/v1/vram": # #533: give VRAM back to other programs, or take it
if not self._own_page("the VRAM reserve can be changed"):
return
r = req.get("reserve_mib")
if r is not None and (isinstance(r, bool) or not isinstance(r, int) or r < 0):
raise ValueError("reserve_mib: a whole number of MiB (0 or more), or null for the start's")
try:
self._json(200, svc.vram(r))
except EngineDied as e:
self._json(503, {"error": {"type": "server_error", "message": str(e)}})
return
if path in ("/v1/chat/completions", "/v1/messages", "/v1/responses"):
self.record = svc.begin_request(path, req)
if path == "/v1/responses":
self._responses(req)
elif path == "/v1/chat/completions":
self._openai(req)
elif path == "/v1/messages":
self._anthropic(req)
elif path == "/v1/messages/count_tokens":
self._count_tokens(req)
else:
self._json(404, {"error": {"message": "not found"}})
except ValueError as e:
print(f"[strata] 400 invalid request: {e}", flush=True)
if path == "/v1/responses":
self._json(400, responses_error_body(str(e)))
else:
self._json(400, {"error": {"type": "invalid_request_error", "message": str(e)}})
except ModelBusy as e:
self._json(409, {"error": {"type": "model_busy", "message": str(e)}})
except StructuredOutputError as e:
self._json(502, {"error": {"type": "structured_output_failed", "code": "structured_output_failed",
"message": str(e)}})
except (GpuBusy, EngineStarting) as e:
self._json(503, {"error": {"type": "server_error", "message": str(e)}})
except EngineDied as e: # before the answer started (not streamed)
self._json(503, {"error": {"type": "server_error", "message": f"{e}; the next request restarts it"}})
except EngineStuck as e: # an unload or restart that could not end the engine
self._json(503, {"error": {"type": "server_error", "message": str(e)}})
except OSError:
self._note(outcome="disconnected")
raise # as before #332: the server's own handling
finally:
if self.watch_done is not None:
self.watch_done.set()
record = self.record
if record is not None:
with svc.status_lock:
record["wallclock_s"] = round(time.perf_counter() - record["_clock"], 3)
record["finished_at"] = time.time()
record["state"] = "error" if record.get("error") else record.get("outcome", "completed")
svc.request_trace.record = None
def _props(self):
model = parse_qs(urlsplit(self.path).query).get("model", [svc.model])[0]
if model not in svc.model_names():
self._json(404, {"error": {"message": "model not found"}})
return
if not svc.loaded() and not getattr(svc.engine, "unloaded", False):
self._json(503, {"error": {"message": "the engine is not running"}})
return
defaults = {**svc.sampling_defaults, **svc.shared}
names = {"repetition_penalty": "repeat_penalty", "penalty_last_n": "repeat_last_n"}
params = {names.get(k, k): v for k, v in defaults.items()
if k in ("temperature", "top_p", "top_k", "min_p", "seed", "repetition_penalty",
"presence_penalty", "frequency_penalty", "penalty_last_n")}
params["n_predict"] = svc.shared.get("max_tokens", -1)
props = {"default_generation_settings": {"n_ctx": svc.engine.max_context, "params": params},
"total_slots": 1, "model_alias": svc.model, "chat_template": svc.template.source,
"modalities": {"vision": svc.vision is not None}, "models_autoload": hasattr(svc.engine, "restart"),
"is_sleeping": not svc.loaded()}
if getattr(svc.engine, "model_path", None):
props["model_path"] = svc.engine.model_path
version = getattr(svc.engine, "info", {}).get("version")
if version:
props["build_info"] = "Strata " + str(version)
self._json(200, props)
def _control_body(self) -> bool:
"""Consume the unused control body before replying/closing (Windows otherwise sends a TCP reset)."""
try:
length = int(self.headers.get("Content-Length", "0"))
except ValueError:
self._json(400, {"error": {"message": "invalid Content-Length"}})
return False
if not 0 <= length <= 65536:
self._json(413, {"error": {"message": "control request body is limited to 64 KiB"}})
return False
timeout = self.connection.gettimeout()
try:
self.connection.settimeout(2.0)
complete = len(self.rfile.read(length)) == length
except OSError:
complete = False
finally:
self.connection.settimeout(timeout)
self.body_read = True # nothing left for the drain to wait for (#594)
if not complete:
self._json(400, {"error": {"message": "incomplete control request body"}})
return complete
def _own_page(self, what) -> bool:
"""Only JSON (a form or a "simple" cross-site request can't send it without a CORS preflight, which this
server never grants) and no foreign Origin: a web page elsewhere must not change settings or run tools."""
if not self.headers.get("Content-Type", "").startswith("application/json"):
self._json(415, {"error": {"message": "send application/json"}})
return False
if self._foreign_origin():
self._json(403, {"error": {"message": f"{what} only from Strata's own page (or an origin in the "
f"config's trusted_origins)"}})
return False
return True
def _config_get(self):
if not svc.config_path:
self._json(404, {"error": {"message": "this server was started without a run config"}})
return
try:
cfg = runconfig.load(svc.config_path)
except (OSError, ValueError) as e:
self._json(500, {"error": {"type": "server_error", "message": f"the run config cannot be read: {e}"}})
return
self._json(200, runconfig.view(cfg, svc.config_path))
def _config_post(self):
"""#564: change a few documented keys of the run config - JSON from Strata's own page only, as
/settings (the key is checked before); every other key of the file stays as it is."""
body = self._body()
if not self._own_page("the run config can be changed"):
return
if not svc.config_path:
self._json(404, {"error": {"message": "this server was started without a run config"}})
return
try:
req = json.loads(body or b"{}")
with svc.config_lock:
cfg = runconfig.load(svc.config_path)
new, changed = runconfig.apply(cfg, req.get("set") if isinstance(req, dict) else None)
if changed:
bak = runconfig.save(svc.config_path, new)
except ValueError as e:
self._json(400, {"error": {"type": "invalid_request_error", "message": str(e)}})
return
except OSError as e:
self._json(500, {"error": {"type": "server_error", "message": f"the run config cannot be written: {e}"}})
return
if changed:
print(f"[strata] the Settings view changed {', '.join(changed)} in {Path(svc.config_path).name} "
f"(the earlier file: {bak.name}); used from the next start", flush=True)
self._json(200, {**runconfig.view(new, svc.config_path), "changed": changed})
def _foreign_origin(self) -> bool:
"""A web page of another origin sent this. The Origin must be this server's own address (host and port),
or an origin the config trusts (trusted_origins: the web app behind a reverse proxy or tunnel); no Origin
is a client that is no browser. Headers a proxy adds (X-Forwarded-*, CF-Ray, CF-Connecting-IP) prove
nothing about the page that sent the request, so they open nothing."""
origin = (self.headers.get("Origin") or "").rstrip("/")
return bool(origin) and origin.split("://", 1)[-1] != self.headers.get("Host", "") and \
origin not in svc.trusted_origins
def _no_local_images(self, messages):
"""A web page of another origin must not have a file on this computer read as an image. Without an
api_key, _foreign_page refuses such a page unless cors_origins lets it in - and a page cors_origins lets
in (["*"]: every page) could name any file and, CORS allowing, read what the model says about it. data:
and http(s) images are the page's own to send; files come from Strata's own page, a trusted origin or a
client that is no browser. ValueError (a 400)."""
if self._foreign_origin() and any(not src.startswith(("data:", "http://", "https://"))
for src in images_of(messages)):
raise ValueError("a web page of another origin cannot have a file on this computer read as an "
"image: send it as a data: URL (or add the page's origin to the config's "
"trusted_origins)")
def _settings(self):
# They change what every client gets, so only the app's own page may set them
body = self._body()
if not self._own_page("settings can be changed"):
return
try:
req = json.loads(body or b"{}")
shared = svc.set_shared(req.get("defaults") if isinstance(req, dict) else None)
except ValueError as e:
self._json(400, {"error": {"type": "invalid_request_error", "message": str(e)}})
return
print("[strata] other apps now use the Chat settings: " + ", ".join(f"{k}={v}" for k, v in shared.items())
if shared else "[strata] other apps use their own settings again", flush=True)
self._json(200, {"shared": bool(shared), "defaults": shared})
def _sse(self):
self._note(http_status=200)
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("X-Accel-Buffering", "no") # nginx and similar proxies pass each event at once
self._cors()
self.end_headers()
def _capture(self, items, api):
"""Retain bounded input/output for the monitor without changing the API response (the items as they
are when the monitor is off)."""
if self.record is None:
return items
return self._captured(items, api)
def _captured(self, items, api):
try:
for item in items:
if item is not None:
if api == "openai":
delta = item["choices"][0]["delta"]
content, reasoning = delta.get("content", ""), delta.get("reasoning_content", "")
usage, timings = item.get("usage"), item.get("timings")
elif api == "responses":
kind = item["type"]
content = item["delta"] if kind == "response.output_text.delta" else ""
reasoning = item["delta"] if kind == "response.reasoning_text.delta" else ""
usage, timings = (item.get("response") or {}).get("usage"), None
else:
_, event = item
delta = event.get("delta", {})
content, reasoning = delta.get("text", ""), delta.get("thinking", "")
usage, timings = event.get("usage"), None
with svc.status_lock:
for name, value in (("output", content), ("reasoning", reasoning)):
if value:
combined = self.record[name] + value
self.record[name] = combined[:262144]
self.record[name + "_truncated"] |= len(combined) > 262144
if usage:
self.record["usage"] = usage
if timings:
self.record["timings"] = timings
yield item
finally:
items.close()
def _openai(self, req):
req = svc.with_shared(req, "openai")
messages, tools, kw = openai_to_messages(req)
self._no_local_images(messages)
messages, validator = prepare_format(req.get("response_format"), messages)
if validator is not None and (tools or req.get("strata_mcp")):
raise ValueError("structured response_format with tools/MCP is not supported")
svc.load()
max_req = max_new = int(req.get("max_completion_tokens") or req.get("max_tokens") or 0) # 0/-1: the rest
use_mcp = req.get("strata_mcp") is True and svc.mcp is not None # the web app's opt-in (serve/mcp.py)
own = {t.get("name") for t in tools or [] if isinstance(t, dict)} # #592: a second line of defence
if use_mcp:
if not self._own_page("MCP tools can be used"): # tools run with the user's rights on this PC
return
svc.mcp.wait(10) # servers still starting (only right after start)
extra = svc.mcp.template_tools(exclude=own) # the request's own tools win a name clash
use_mcp = bool(extra)
tools = (tools or []) + extra or None
svc.reasoning_budget(req) # a bad value is a 400 before anything is sent
stop_strings(req) # ... and so is a bad stop / stop_sequences
ids, thinking, max_new = svc.prepare(messages, tools, kw, max_new)
_debug_req("openai", req, messages, tools, max_new, thinking, len(ids))
cancel = threading.Event()
self._watch_client(cancel) # #430 #431
run = run_with_mcp(svc, svc.mcp, messages, tools, kw, ids, thinking, max_new, max_req, req, cancel,
{t["name"] for t in extra}) if use_mcp else None
chunks = openai_chunks(svc, req, ids, thinking, tools, max_new, cancel, run=run)
if validator is not None:
chunks = structured_chunks(chunks, validator)
chunks = self._capture(chunks, "openai")
if not req.get("stream"):
return self._json(200, openai_collect(chunks))
self._sse()
try:
for c in chunks:
if c is None:
self.wfile.write(b": keep-alive\n\n") # an SSE comment: clients ignore it
else:
self.wfile.write(b"data: " + json.dumps(c, ensure_ascii=False).encode() + b"\n\n")
self.wfile.flush()
self.wfile.write(b"data: [DONE]\n\n")
except OSError:
self._note(outcome="disconnected")
cancel.set() # client went away: stop the engine
chunks.close()
except EngineDied as e: # mid-stream: say so, then end the stream properly
err = {"error": {"type": "server_error", "message": f"{e}; the next request restarts it"}}
self._note(error=err["error"])
self.wfile.write(b"data: " + json.dumps(err).encode() + b"\n\ndata: [DONE]\n\n")
except StructuredOutputError as e:
err = {"error": {"type": "structured_output_failed", "code": "structured_output_failed", "message": str(e)}}
self._note(error=err["error"])
self.wfile.write(b"data: " + json.dumps(err).encode() + b"\n\ndata: [DONE]\n\n")
except ValueError as e: # the engine's ERR after the stream started: the
err = {"error": {"type": "server_error", "message": str(e)}} # headers are sent, so no 400 now
self._note(error=err["error"])
self.wfile.write(b"data: " + json.dumps(err).encode() + b"\n\ndata: [DONE]\n\n")
def _responses(self, req):
"""#451: POST /v1/responses - OpenAI's Responses API, stateless (serve/responses.py), on the chat path.
Errors use the Responses format; once the stream has started they arrive as a response.failed event."""
try:
ids, thinking, tools, max_new, asm, validator, req = self._responses_prepare(req)
except ResponsesError as e:
return self._json(e.status, e.body())
except ModelBusy as e:
return self._json(409, responses_error_body(str(e), "server_error", code="model_busy"))
except (GpuBusy, EngineStarting, EngineStuck, EngineDied) as e:
return self._json(503, responses_error_body(str(e), "server_error", code="server_error"))
cancel = threading.Event()
self._watch_client(cancel) # #430 #431
def check(text, finish):
return validated_json(text, validator, finish)
def events():
yield from asm.start()
done = None
for kind, x in svc.run(ids, thinking, tools, max_new, req, cancel):
if kind == "ping":
yield None
elif kind == "event":
yield from asm.feed(x)
elif kind == "done":
done = x
if done is not None and done["finish"] != "cancel" and not cancel.is_set():
yield from asm.finish(done, check)
items = self._capture(events(), "responses")
if not req.get("stream"):
try:
result = responses_api.collect(items)
except StructuredOutputError as e:
return self._json(502, responses_error_body(str(e), "server_error",
code="structured_output_failed"))
except EngineDied as e:
return self._json(503, responses_error_body(f"{e}; the next request restarts it", "server_error",
code="server_error"))
except ValueError as e: # the engine's ERR line
return self._json(500, responses_error_body(str(e), "server_error", code="server_error"))
return self._json(200, result)
self._sse()
last = time.monotonic()
def send(e):
self.wfile.write(f"event: {e['type']}\n".encode() + b"data: " +
json.dumps(e, ensure_ascii=False).encode() + b"\n\n")
try:
for e in items:
if e is None:
# Codex's idle timeout (5 minutes by default) counts events, not SSE comments: while a long
# prompt is read, a response.in_progress every 15 s tells it the server is still working
if time.monotonic() - last >= 15:
send(asm.in_progress())
last = time.monotonic()
else:
self.wfile.write(b": keep-alive\n\n")
else:
send(e)
last = time.monotonic()
self.wfile.flush()
except OSError:
self._note(outcome="disconnected")
cancel.set() # client went away: stop the engine
items.close()
except EngineDied as e: # mid-stream: response.failed, then the stream ends
self._responses_failed(asm, f"{e}; the next request restarts it", "server_error", send)
except StructuredOutputError as e:
self._responses_failed(asm, str(e), "structured_output_failed", send)
except ValueError as e: # the engine's ERR after the stream started
self._responses_failed(asm, str(e), "server_error", send)
def _responses_failed(self, asm, message, code, send):
e = asm.failed(message, code)
self._note(error=e["response"]["error"])
try:
send(e)
self.wfile.flush()
except OSError:
pass
def _responses_prepare(self, req):
responses_api.check_request(req)
messages = responses_api.input_messages(req)
tools, names, skipped = responses_api.request_tools(req)
kw = responses_api.template_kwargs(req, svc.shared)
try:
self._no_local_images(messages) # as on the chat route: no file read for a foreign page
except ValueError as e:
raise ResponsesError(str(e), "input") from None
try:
messages, validator = prepare_format(responses_api.text_format(req), messages)
except ValueError as e:
raise ResponsesError(str(e).replace("response_format", "text.format"), "text.format") from None
if validator is not None and tools:
raise ResponsesError("a JSON text.format with tools is not supported", "text.format",
"unsupported_parameter")
noted = svc.__dict__.setdefault("hosted_tools_noted", set()) # said once per tool, not per request
if set(skipped) - noted:
print(f"[strata] /v1/responses: left out the hosted tools {', '.join(sorted(set(skipped) - noted))} "
f"(this server cannot run them)", flush=True)
noted.update(skipped)
if req.get("max_output_tokens") is None and "max_tokens" in svc.shared: # the shared Chat settings
req = {**req, "max_output_tokens": svc.shared["max_tokens"]}
try:
svc.reasoning_budget(req) # a bad value is a 400 before anything is sent
except ValueError as e:
raise ResponsesError(str(e), "reasoning_budget_tokens") from None
try:
stop_strings(req)
except ValueError as e:
raise ResponsesError(str(e), "stop") from None
svc.load()
try:
ids, thinking, max_new = svc.prepare(messages, tools, kw, req.get("max_output_tokens") or 0)
except ResponsesError:
raise
except ValueError as e: # too long for the context, an image without vision
raise ResponsesError(str(e), "input", "context_length_exceeded" if "context" in str(e) else None) \
from None
_debug_req("responses", req, messages, tools, max_new, thinking, len(ids))
include = req.get("include") if isinstance(req.get("include"), list) else []
asm = responses_api.Assembler(req, svc.model_for(req), len(ids), names,
"reasoning.encrypted_content" in include, validator is not None)
# the request is also the sampling dict, as on the chat path (temperature, top_p, top_k, seed, ...)
return ids, thinking, tools, max_new, asm, validator, req
def _count_tokens(self, req):
"""Anthropic's token count, which Claude Code asks for its context figures: the prompt this server would
read for the same request, rendered and tokenized - the model does not run."""
req = svc.with_shared(req, "anthropic")
messages, tools, kw = anthropic_to_messages(req, svc.anthropic_think_unasked)
stop_strings(req) # the same 400 as the request itself would get
self._json(200, {"input_tokens": len(svc.encode_prompt(messages, tools, kw))})
def _anthropic(self, req):
svc.load()
req = svc.with_shared(req, "anthropic")
messages, tools, kw = anthropic_to_messages(req, svc.anthropic_think_unasked)
self._no_local_images(messages)
max_new = int(req.get("max_tokens") or 0) # 0/-1: the rest of the context
svc.reasoning_budget(req) # a bad value is a 400 before anything is sent
stop_strings(req) # ... and so is a bad stop / stop_sequences
ids, thinking, max_new = svc.prepare(messages, tools, kw, max_new)
_debug_req("anthropic", req, messages, tools, max_new, thinking, len(ids))
cancel = threading.Event()
self._watch_client(cancel) # #430 #431
events = anthropic_events(svc, req, ids, thinking, tools, max_new, cancel)
events = self._capture(events, "anthropic")
if not req.get("stream"):
return self._json(200, anthropic_collect(events))
self._sse()
try:
for item in events:
if item is None:
self.wfile.write(b": keep-alive\n\n")
else:
name, e = item
self.wfile.write(f"event: {name}\n".encode() + b"data: " +
json.dumps(e, ensure_ascii=False).encode() + b"\n\n")
self.wfile.flush()
except OSError:
self._note(outcome="disconnected")
cancel.set()
events.close()
except EngineDied as e: # mid-stream: Anthropic's error event
err = {"type": "error", "error": {"type": "api_error", "message": f"{e}; the next request restarts it"}}
self._note(error=err["error"])
self.wfile.write(b"event: error\ndata: " + json.dumps(err).encode() + b"\n\n")
except ValueError as e: # the engine's ERR after the stream started
err = {"type": "error", "error": {"type": "api_error", "message": str(e)}}
self._note(error=err["error"])
self.wfile.write(b"event: error\ndata: " + json.dumps(err).encode() + b"\n\n")
return Handler
class Server(ThreadingHTTPServer):
# On Windows SO_REUSEADDR lets a second server bind a port that is already serving, and requests then land on
# either one (a forgotten second start of run-<model>.bat). Without it the second start fails loudly instead.
allow_reuse_address = os.name != "nt"
def handle_error(self, request, client_address):
if not isinstance(sys.exc_info()[1], ConnectionError): # a client that hangs up needs no stack trace
super().handle_error(request, client_address)
def warn_tight_ram(arena_mib) -> None:
"""The model's experts live in RAM (INFO arena_mib, engine 0.1.10+). With less than ~6 GB left beside them for the
system, the engine and this server, Linux ends the engine mid-answer when memory runs out (issue #27) and Windows
pages to disk; say so at start instead of after a lost answer."""
if not isinstance(arena_mib, int) or arena_mib <= 0:
return
try:
import psutil
total = psutil.virtual_memory().total
except Exception: # noqa: BLE001 - psutil is optional here
return
left = total / 2**30 - arena_mib / 1024
if left < 6:
print(f"[strata] WARNING: RAM is tight - the model's experts take {arena_mib / 1024:.1f} GB of this PC's "
f"{total / 2**30:.0f} GB, leaving {left:.1f} GB for everything else. "
+ ("Linux may stop the engine in the middle of an answer. " if os.name != "nt" else
"Windows will slow down (paging to disk). ")
+ "Close other programs, or run START-HERE --setup and pick a smaller size (Q2_0 / IQ2_XS).", flush=True)
DESKTOP_FREE_MIB = 2048 # #560 #516: below this, an AMD card that also drives a Linux desktop can run out
DESKTOP_RESERVE_MIB = 3072 # what kept KDE/Wayland alive beside a full expert cache in both reports
def linux_desktop(env=None) -> bool:
"""A graphical session on Linux (Wayland or X): its compositor, browser and apps take VRAM after the model has."""
env = os.environ if env is None else env
return os.name != "nt" and sys.platform != "darwin" and bool(env.get("WAYLAND_DISPLAY") or env.get("DISPLAY"))
def desktop_vram_note(backend, vram_free_mib, args: list, desktop: bool) -> str:
"""#560 #516: on Linux, when the desktop needs VRAM the AMD card does not have, amdgpu moves GPU memory (the expert
cache, ~24 GB) to system RAM, which the experts already fill - the OOM killer then ends the compositor. The
default reserve (700 MiB) is sized for a card without a desktop. A recommendation, nothing changes: "" when it
does not apply."""
if backend != "hip" or not desktop or not isinstance(vram_free_mib, int) or vram_free_mib >= DESKTOP_FREE_MIB:
return ""
try:
reserve = int(args[args.index("--vram-reserve-mib") + 1]) if "--vram-reserve-mib" in args else 700
except (ValueError, IndexError):
reserve = 700
if reserve >= DESKTOP_RESERVE_MIB:
return ""
return (f"[strata] note: {vram_free_mib} MiB of VRAM free with the model loaded. If this AMD card also drives your "
"desktop and the desktop or apps crash after the start (the driver moves the expert cache to RAM and "
"the OOM killer ends the session), keep more VRAM free: ./setup.sh --vram-reserve-mib "
f"{DESKTOP_RESERVE_MIB} (remembered; the expert cache gets "
f"{(DESKTOP_RESERVE_MIB - reserve) / 1024:.1f} GB less, a few % of speed)")
def lan_addresses() -> list[str]:
"""This PC's IPv4 addresses on its networks (what another device types in), without loopback/link-local."""
import socket
first, ips = None, set()
try: # the address of the default route; sends nothing (UDP)
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
s.connect(("10.255.255.255", 1))
first = s.getsockname()[0]
except OSError:
pass
try:
for info in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET):
ips.add(info[4][0])
except OSError:
pass
ok = lambda ip: ip and not ip.startswith(("127.", "169.254.", "0."))
return ([first] if ok(first) else []) + sorted(ip for ip in ips if ok(ip) and ip != first)
def host_name(value) -> str:
"""The name in a Host header (or an origin's host[:port]): "Example.com:8080" -> "example.com",
"[::1]:8095" -> "::1"; "" when it is malformed."""
v = (value or "").strip().lower()
if v.startswith("["):
name, sep, rest = v[1:].partition("]")
return name if sep and (not rest or (rest[:1] == ":" and rest[1:].isdigit())) else ""
if v.count(":") == 1:
v, port = v.split(":")
if not port.isdigit():
return ""
elif ":" in v: # a bare IPv6 address (a config entry)
return v
v = v.rstrip(".")
return v if v and all(c.isalnum() or c in "-._" for c in v) else ""
def _is_ip(name: str) -> bool:
import ipaddress
try:
ipaddress.ip_address(name)
return True
except ValueError:
return False
def _name_in(name: str, names) -> bool:
"""`name` is one of `names`, or below an entry that starts with a dot (".example.com")."""
return name in names or any(n.startswith(".") and (name.endswith(n) or name == n[1:]) for n in names)
def allowed_hosts_of(value, env: str = "") -> list[str]:
"""The config's allowed_hosts (a name or a list) plus $STRATA_ALLOWED_HOSTS (comma-separated): host names, "*" or
".example.com" (it and every name below it). A scheme, port or path is dropped ("https://a.example.com:8443/"
-> "a.example.com"); a wrong entry stops the start (ValueError)."""
items = [] if value in (None, "") else [value] if isinstance(value, str) else value
if not isinstance(items, list) or not all(isinstance(x, str) for x in items):
raise ValueError("allowed_hosts: expected a host name or a list of them")
out = []
for raw in items + [x for x in env.split(",") if x.strip()]:
x = raw.strip().lower()
if x == "*":
out.append(x)
continue
x = x.split("://", 1)[-1].split("/", 1)[0]
dot = x.startswith(".")
name = host_name(x[1:] if dot else x)
if not name:
raise ValueError(f"allowed_hosts: {raw!r} is not a host name like strata.example.com")
out.append("." + name if dot else name)
return list(dict.fromkeys(out))
def host_names_for(bind_host: str, allowed_hosts=(), trusted_origins=()) -> set[str]:
"""Every name this server answers to besides an IP address: localhost, the address it listens on, the config's
allowed_hosts and trusted_origins' hosts, and - listening beyond this PC (0.0.0.0 or a LAN address) - this PC's
name and LAN addresses (the LAN addresses matter for the Origin check, which takes no IP on trust)."""
names = set(LOOPBACK_NAMES)
bind = host_name(bind_host)
if bind and bind not in ("0.0.0.0", "::"):
names.add(bind)
if bind not in LOOPBACK_NAMES:
try:
pc = socket.gethostname().lower()
names.update((pc, pc + ".local"))
except OSError:
pass
names.update(("host.docker.internal", *lan_addresses()))
names.update(x for x in allowed_hosts if x != "*")
names.update(filter(None, (host_name(o.split("://", 1)[-1]) for o in trusted_origins)))
return names
def host_allowed(host, names, any_host=False) -> bool:
"""DNS rebinding: a web page of another site whose name its DNS points at 127.0.0.1 reaches this server as the
same origin, so the browser lets it read every answer. Its requests carry that site's name in Host, so only
the names this server answers to pass. An IP address passes (a page served from an IP is that IP's own page;
rebinding needs a name), as does "*.localhost" (browsers never ask DNS for it) and a request without a Host
header (HTTP/1.0 clients; browsers always send one)."""
if any_host or not (host or "").strip():
return True
name = host_name(host)
return bool(name) and (_is_ip(name) or name == "localhost" or name.endswith(".localhost")
or _name_in(name, names))
def origin_allowed(origin: str, host, names, origins=()) -> bool:
"""A browser page's Origin that may use the model without an API key: this server's own page (the Origin is the
request's own Host), a page on one of the names this server answers to (any port), or an origin the config lists
(trusted_origins, cors_origins). Unlike the Host check no IP passes on trust: a page served from any other IP is
another site. "null" (a sandboxed frame, a file:// page) does not pass: any site can send it. An origin of another
scheme (chrome-extension://, moz-extension://, an Electron app's app://) does: no web site can send one."""
origin = (origin or "").strip().rstrip("/")
if origin in origins or "*" in origins: # cors_origins ["*"]: the config lets every page in
return True
scheme, sep, rest = origin.lower().partition("://")
if not sep or not scheme:
return False
if scheme not in ("http", "https"):
return True
if host and rest == host.strip().lower():
return True
name = host_name(rest)
return bool(name) and (name in LOOPBACK_NAMES or name.endswith(".localhost") or _name_in(name, names))
def serve(svc: Service, host="127.0.0.1", port=8095) -> ThreadingHTTPServer:
svc.host_names = host_names_for(host, svc.allowed_hosts, svc.trusted_origins)
svc.start_telemetry()
httpd = Server((host, port), make_handler(svc))
threading.Thread(target=httpd.serve_forever, daemon=True).start()
return httpd
def origins_of(value, key: str, wildcard: bool) -> list[str]:
"""#321: a config's origin list ("https://chat.example.com" or a list of them) - scheme://host[:port], no path;
"*" only where `wildcard` allows it. A wrong entry stops the start rather than opening less or more than meant."""
if value is None or value == "" or value == []:
return []
items = [value] if isinstance(value, str) else value
if not isinstance(items, list) or not all(isinstance(x, str) for x in items):
raise SystemExit(f"[strata] {key}: expected an origin or a list of origins")
out = []
for x in items:
x = x.strip().rstrip("/")
if x == "*" and wildcard:
out.append(x)
continue
scheme, sep, rest = x.partition("://")
if scheme not in ("http", "https") or not sep or not rest or "/" in rest or "*" in rest:
raise SystemExit(f"[strata] {key}: {x!r} is not an origin like https://chat.example.com"
+ ("" if wildcard else " (no wildcards here)"))
out.append(x)
return out
SHARED_KEYS = ("reasoning_effort", "temperature", "top_p", "top_k", "seed", "max_tokens", "experimental_speed_projection")
def clean_shared_defaults(d) -> dict:
"""The Chat settings other apps get (POST /settings): only known keys, each checked; ValueError names a bad one."""
if d is None:
return {}
if not isinstance(d, dict):
raise ValueError("defaults must be an object")
out = {}
for key, value in d.items():
if value is None or value == "":
continue
number = isinstance(value, (int, float)) and not isinstance(value, bool)
if key == "reasoning_effort":
if value not in ("none", "low", "medium", "high"):
raise ValueError("reasoning_effort: none, low, medium or high")
elif key == "temperature":
if not number or not 0 <= value <= 2:
raise ValueError("temperature: 0..2")
elif key == "top_p":
if not number or not 0 < value <= 1:
raise ValueError("top_p: 0 < top_p <= 1")
elif key == "top_k":
if not number or value != int(value) or not 1 <= value <= 64:
raise ValueError("top_k: an integer 1..64")
value = int(value)
elif key in ("seed", "max_tokens"):
if not number or value != int(value) or value <= 0:
raise ValueError(f"{key}: a positive integer")
value = int(value)
elif key == "experimental_speed_projection":
if not isinstance(value, bool):
raise ValueError("experimental_speed_projection: true or false")
else:
raise ValueError(f"unknown setting {key!r}")
out[key] = float(value) if key in ("temperature", "top_p") else value
return out
def sampling_defaults_from_config(cfg: dict) -> dict:
"""The run config's optional `sampling` block: defaults for the sampling fields a request leaves out, so
a plain client gets configured sampling instead of greedy. Supported: temperature, top_p, top_k, min_p,
presence_penalty, repetition_penalty, frequency_penalty, penalty_last_n, seed. The request's own fields
always win - an explicit temperature=0 still means greedy, a field set to null falls back to the default.
A bad value refuses to start the server (a typo'd config should not quietly change sampling); unknown keys
are named at startup and ignored."""
out = {}
for key, value in (cfg.get("sampling") or {}).items():
if value is None:
continue
number = isinstance(value, (int, float)) and not isinstance(value, bool)
if key == "temperature":
if not number or value < 0:
raise SystemExit(f"[strata] config sampling.temperature={value!r}: expected a number >= 0 (0 = greedy)")
out[key] = float(value)
elif key == "top_p":
if not number or not 0 < value <= 1:
raise SystemExit(f"[strata] config sampling.top_p={value!r}: expected 0 < top_p <= 1")
out[key] = float(value)
elif key == "min_p":
if not number or not 0 <= value <= 1:
raise SystemExit(f"[strata] config sampling.min_p={value!r}: expected 0 <= min_p <= 1")
out[key] = float(value)
elif key == "top_k":
if not number or value != int(value) or not 1 <= value <= 64:
raise SystemExit(f"[strata] config sampling.top_k={value!r}: the sampled path takes an integer 1..64")
out[key] = int(value)
elif key == "presence_penalty":
if not number or value < 0:
raise SystemExit(f"[strata] config sampling.presence_penalty={value!r}: expected a number >= 0")
out[key] = float(value)
elif key == "frequency_penalty":
if not number or value < 0:
raise SystemExit(f"[strata] config sampling.frequency_penalty={value!r}: expected a number >= 0")
out[key] = float(value)
elif key == "repetition_penalty":
if not number or value <= 0:
raise SystemExit(f"[strata] config sampling.repetition_penalty={value!r}: expected a number > 0 (1 = off)")
out[key] = float(value)
elif key == "penalty_last_n":
if not number or value != int(value) or value < 0:
raise SystemExit(f"[strata] config sampling.penalty_last_n={value!r}: expected a non-negative integer")
out[key] = int(value)
elif key == "seed":
if not number or value != int(value) or value <= 0:
raise SystemExit(f"[strata] config sampling.seed={value!r}: expected a positive integer")
out[key] = int(value)
elif key == "experimental_speed_projection":
if not isinstance(value, bool):
raise SystemExit(f"[strata] config sampling.experimental_speed_projection={value!r}: expected true or "
"false (the default for requests that leave it out, when the engine has the vector)")
out[key] = value
else:
print(f"[strata] config sampling.{key}={value!r}: unknown key, ignored", flush=True)
return out
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--engine", choices=["mock", "strata"], default="mock")
ap.add_argument("--config", help="strata engine config (JSON: exe, args, cwd, tokenizer, model_name), "
"written by setup.py")
ap.add_argument("--host", default=None,
help="the address to listen on: 127.0.0.1 = this PC only (the default), 0.0.0.0 = also other devices "
"on your network (set an API key); also \"host\" in the config")
ap.add_argument("--script", action="append",
help="the mock engine's answer (default: a short greeting); given more than once, requests get "
"them in turn and the last one repeats")
ap.add_argument("--port", type=int, default=8095)
ap.add_argument("--gpu", help="the GPU to run on, as nvidia-smi numbers them, or several for a layer split "
"(\"0,2\"; also \"gpu\" in the config)")
ap.add_argument("--tokenizer", default=str(ROOT / "pack/full/tokenizer"),
help="pack tokenizer directory (falls back to a byte tokenizer if absent)")
ap.add_argument("--open", action="store_true", help="open the local page in the browser once the model is ready")
ap.add_argument("--fit-max-tokens", action="store_true",
help="clamp max_tokens to the remaining context instead of rejecting the request "
"(default: reject with 400, like llama.cpp; also \"fit_max_tokens\": true in the config)")
ap.add_argument("--api-key", default=os.environ.get("STRATA_API_KEY", ""),
help="require this key on /v1/* (Authorization: Bearer ... or x-api-key); also $STRATA_API_KEY")
ap.add_argument("--mcp-config", help="a JSON file with MCP servers in Claude Desktop's format ({\"mcpServers\": "
"{...}}); the web app's chat can use their tools (also \"mcp_servers\" in "
"the config)")
ap.add_argument("--lazy", action="store_true", help="start the text-only API unloaded; load on first request")
ap.add_argument("--api-monitor", action="store_true",
help="the API request monitor at /api-monitor: keeps the last 100 requests' prompts and answers in "
"memory (also \"api_monitor\": true in the config; off by default)")
ap.add_argument("--idle-unload", type=float, default=None, metavar="SECONDS",
help="unload the model after this many seconds without requests, so other programs (games, other "
"model servers) can use the VRAM; the next request loads it again (also \"idle_unload_s\" "
"in the config; default: never)")
ap.add_argument("--min-free-vram-mib", type=int, default=None,
help="load an unloaded model only when this much VRAM is free, else answer 503 (also "
"\"min_free_vram_mib\" in the config; default: always load)")
ap.add_argument("--before-load", help="a command run before the model is loaded again (e.g. to unload another "
"server's model; also \"before_load\" in the config, a string or a list)")
a = ap.parse_args()
cfg = json.loads(Path(a.config).read_text(encoding="utf-8-sig")) if a.config else {} # Notepad adds a BOM
if a.gpu is not None:
cfg["gpu"] = int(a.gpu) if a.gpu.strip().isdigit() else a.gpu
a.host = a.host or cfg.get("host") or "127.0.0.1" # issue #26: the run scripts pass no --host, the config can
try: # before the minutes of loading: is the port free?
Server((a.host, a.port), BaseHTTPRequestHandler).server_close()
except OSError:
ap.error(f"port {a.port} is already in use - is Strata (or another server) already running? "
f"Close it, or start this one with a different --port")
if cfg.get("tokenizer"):
a.tokenizer = cfg["tokenizer"]
tok = ByteTokenizer()
tpath = Path(a.tokenizer)
if a.engine == "strata" and not (tpath / "vocab.json").exists():
ap.error(f"the model's tokenizer is missing ({tpath / 'vocab.json'}); run setup again")
if (tpath / "vocab.json").exists():
import strata_tokenizer as ST
vocab = json.loads((tpath / "vocab.json").read_text(encoding="utf-8"))
tokens = [None] * len(vocab)
for t, i in vocab.items():
tokens[i] = t
merges = (tpath / "merges.txt").read_text(encoding="utf-8").split("\n")
types = json.loads((tpath / "token_type.json").read_text())
tok = ST.Tokenizer(tokens, merges, types)
hub = hub_from_config(cfg, a.mcp_config) # before the minutes of loading: a bad entry stops here
if a.engine == "strata":
if not cfg:
ap.error("--engine strata needs --config")
vision = None
env = child_env(cfg)
sampling_defaults = sampling_defaults_from_config(cfg)
if sampling_defaults:
pretty = ", ".join(f"{k}={v}" for k, v in sampling_defaults.items())
print(f"[strata] sampling defaults from the config: {pretty}", flush=True)
lazy = a.lazy or cfg.get("lazy_load") is True
if lazy and cfg.get("vision"):
ap.error("lazy loading is text-only; disable vision in the config")
if cfg.get("vision"):
print("loading the vision encoder ...", flush=True)
# relative paths are the config's cwd's, as for the engine below
vcfg = {k: (os.path.abspath(os.path.join(cfg.get("cwd") or ".", v))
if k in ("exe", "mmproj", "model") and isinstance(v, str) and not os.path.isabs(v) else v)
for k, v in cfg["vision"].items()}
vision = Vision(vcfg, log=open(cfg["log"], "a", encoding="utf-8") if cfg.get("log") else None,
env=vision_env(cfg, env))
print("model unloaded; the first request loads it ..." if lazy else
"loading the model (the first start takes a minute or two) ...", flush=True)
if len(gpu_list(cfg)) > 1:
try:
split = layer_split_value(cfg) # #644: before the (minutes-long) start
except ValueError as e:
raise SystemExit(f"[strata] config {e}")
print(f"[strata] layer split across GPUs {gpu_list(cfg)} ({split})", flush=True)
# a relative "exe" is the config's cwd's: Windows' CreateProcess resolves "engine/strata.exe" against nothing
# it is told about (WinError 2), so it is made absolute here
exe = cfg["exe"] if os.path.isabs(cfg["exe"]) else os.path.abspath(os.path.join(cfg.get("cwd") or ".", cfg["exe"]))
try:
silence = engine_silence_s(cfg) # #481: checked before the (minutes-long) start
except ValueError as e:
raise SystemExit(f"[strata] config {e}")
try:
effort_end = effort_end_args(cfg, exe, tok) # #458
except ValueError as e:
raise SystemExit(f"[strata] config {e}")
engine = StrataEngine(exe, engine_args(cfg) + (effort_end or []), cwd=cfg.get("cwd"), log=cfg.get("log"),
env=env, lazy=lazy)
engine.silence_s = silence # an attribute of its own: restart() keeps it
warn_tight_ram(engine.info.get("arena_mib"))
note = desktop_vram_note(cfg.get("backend"), engine.info.get("vram_free_mib"), engine.spawn[1],
linux_desktop())
if note: # #560 #516: before --open starts a browser on that card
print(note, flush=True)
else:
effort_end = None
engine, vision, sampling_defaults = MockEngine(tok, a.script or [
"Thinking about it.</think>\n\nHello from the mock engine."]), None, {}
# the model's own chat template (exported with its tokenizer), else the original model's
tpl = tpath / "chat_template.jinja"
svc = Service(engine, tok, ChatTemplate(tpl if tpl.exists() else ROOT / "serve/chat_template.jinja"),
model_name=cfg.get("model_name", "qwen3.8-flash-next"), vision=vision,
sampling_defaults=sampling_defaults,
fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)
try:
svc.set_aliases(cfg.get("aliases")) # #297: other names the model answers to
except ValueError as e:
raise SystemExit(f"[strata] config {e}")
if svc.aliases:
print(f"[strata] model aliases: {', '.join(svc.aliases)}", flush=True)
if ("STRATA_API_KEY" in os.environ and not os.environ["STRATA_API_KEY"].strip()) or any(x == "--api-key" and i + 1 < len(sys.argv) and not sys.argv[i + 1].strip() or x.strip() == "--api-key="
for i, x in enumerate(sys.argv)):
# #213: an empty key would switch authentication off without a word
print("[strata] an API key was given but it is empty: set a key, or leave --api-key / STRATA_API_KEY out",
file=sys.stderr)
return 2
svc.api_key = a.api_key or cfg.get("api_key", "")
svc.cors_origins = origins_of(cfg.get("cors_origins"), "cors_origins", wildcard=True)
svc.trusted_origins = origins_of(cfg.get("trusted_origins"), "trusted_origins", wildcard=False)
try:
svc.allowed_hosts = allowed_hosts_of(cfg.get("allowed_hosts"), os.environ.get("STRATA_ALLOWED_HOSTS", ""))
except ValueError as e:
raise SystemExit(f"[strata] config {e}")
if svc.allowed_hosts:
print("[strata] Host check off: any name reaches this server (allowed_hosts \"*\")" if "*" in svc.allowed_hosts
else f"[strata] also answers to the host names {', '.join(svc.allowed_hosts)} (allowed_hosts)", flush=True)
svc.api_monitor = a.api_monitor or cfg.get("api_monitor") is True
if svc.api_monitor:
print("[strata] API request monitor on (/api-monitor): the last 100 requests' prompts and answers are kept in "
"memory" + ("" if svc.api_key else "; anyone who can reach this server can read them (no API key)"),
flush=True)
if svc.cors_origins:
print(f"[strata] CORS on /v1/* for {', '.join(svc.cors_origins)}"
+ ("" if svc.api_key or "*" not in svc.cors_origins else
" - WARNING: any web page may use the model (no API key)"), flush=True)
svc.idle_unload_s = a.idle_unload if a.idle_unload is not None else float(cfg.get("idle_unload_s") or 0)
svc.min_free_vram_mib = a.min_free_vram_mib if a.min_free_vram_mib is not None else \
int(cfg.get("min_free_vram_mib") or 0)
svc.before_load = a.before_load or cfg.get("before_load") or None
mode = str(cfg.get("anthropic_thinking") or "model") # #278: "on_request" = only when the request asks
if mode not in ("model", "on_request"):
raise SystemExit(f"[strata] config anthropic_thinking must be \"model\" or \"on_request\", not {mode!r}")
svc.anthropic_think_unasked = mode == "model"
rs = cfg.get("repeat_stop_tokens", REPEAT_STOP_TOKENS) # #606: opt-out with 0
if isinstance(rs, bool) or not isinstance(rs, int) or rs < 0:
raise SystemExit(f"[strata] config \"repeat_stop_tokens\" must be a whole number >= 0 (0 = off), not {rs!r}")
svc.repeat_stop_tokens = rs
svc.effort_end = bool(effort_end) # #458: "effort_position": "end" with an engine that has it
if cfg.get("reasoning_budget_tokens") is not None: # #123: a default thinking budget for every request
try:
svc.reasoning_budget_tokens = cfg["reasoning_budget_tokens"]
budget = svc.reasoning_budget({})
except ValueError as e:
raise SystemExit(f"[strata] config {e}")
if budget:
print(f"[strata] thinking budget: {budget} tokens (reasoning_budget_tokens; a request can set its own)",
flush=True)
svc.gpu_index = (gpu_list(cfg) or [0])[0] # the Monitor reads the card the engine runs on (issue #51)
svc.gpu_indices = gpu_list(cfg) # ... or every card of a layer split (issue #112)
svc.backend = cfg.get("backend") # "hip": the AMD cards' readings come from sysfs (#301)
if a.config:
svc.config_path = a.config # #564: the web page's Settings view
if a.config: # the Chat settings shared with other apps, from last time
svc.shared_path = str(Path(a.config).with_suffix("")) + ".shared-settings.json"
try:
svc.shared = clean_shared_defaults(json.loads(Path(svc.shared_path).read_text(encoding="utf-8")))
if svc.shared:
print("[strata] other apps use the Chat settings: " +
", ".join(f"{k}={v}" for k, v in svc.shared.items()), flush=True)
except (OSError, ValueError):
svc.shared = {}
if hub is not None:
import atexit
svc.mcp = hub
print(f"[strata] starting {len(hub.servers)} MCP server{'s' * (len(hub.servers) != 1)} for the web app's "
f"chat: {', '.join(hub.servers)}", flush=True)
hub.start()
atexit.register(hub.close) # the servers Strata started end with it
httpd = serve(svc, host=a.host, port=a.port)
svc.start_idle_unload()
here = "127.0.0.1" if a.host in ("0.0.0.0", "", "::") else a.host
print(f"ready: http://{here}:{a.port}/v1 (OpenAI: /v1/chat/completions, Anthropic: /v1/messages, "
f"context {engine.max_context} tokens{', images on' if vision else ''}"
f"{', API key required' if svc.api_key else ''})", flush=True)
print(f" open http://{here}:{a.port}/ in a browser to chat; close this window to stop the model", flush=True)
if a.host not in ("127.0.0.1", "localhost", "::1"):
# issue #26: reachable from other devices - say at which address, and what can still block it
ips = lan_addresses()
for ip in ips:
print(f" from other devices: http://{ip}:{a.port}/ (API: http://{ip}:{a.port}/v1)", flush=True)
if not ips:
print(" from other devices: http://<this PC's IP address>:" + str(a.port) + "/", flush=True)
if not svc.api_key:
print(" WARNING: no API key - anyone on your network can use this model. Add \"api_key\": \"...\" "
"to the config (clients send it as their API key; the web page asks for it)", flush=True)
if os.name == "nt":
print(" nothing arrives? Windows Firewall blocks it until allowed: accept its prompt for Python, or run "
"in an admin PowerShell:\n New-NetFirewallRule -DisplayName \"Strata " + str(a.port) + "\" "
"-Direction Inbound -Protocol TCP -LocalPort " + str(a.port) + " -Action Allow -Profile Private\n"
" (and set this network to Private in Windows' network settings)", flush=True)
if a.open and cfg.get("open_browser") is not False: # #609: the config's "open_browser": false wins (an older
import webbrowser # run-<model>.bat still passes --open)
webbrowser.open(f"http://{'127.0.0.1' if a.host in ('0.0.0.0', '') else a.host}:{a.port}/")
# #96: docker stop sends SIGTERM, which Python ignores by default, so the container's PID 1 would be killed after
# the grace period with the engine still running. SIGTERM takes Ctrl+C's path below (QUIT to the engine).
# SIGINT keeps Python's own handler, so Ctrl+C and a second Ctrl+C work as before.
def on_sigterm(signum, frame):
raise KeyboardInterrupt
try:
signal.signal(signal.SIGTERM, on_sigterm)
except (ValueError, OSError, AttributeError): # not the main thread
pass
try:
while True:
time.sleep(1) # Windows never delivers Ctrl+C to an untimed Event.wait()
except KeyboardInterrupt:
print("\n[strata] stopping (Ctrl+C again to end the engine at once) ...", flush=True)
closers = [httpd.shutdown, getattr(engine, "close", None), vision.close if vision else None,
hub.close if hub is not None else None]
for close in filter(None, closers):
try:
close()
except KeyboardInterrupt: # a second Ctrl+C: don't wait for the engine to free its memory
if getattr(engine, "proc", None):
engine.proc.kill()
print("[strata] stopped", flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())
Reference in new issue View Git Blame Copy Permalink