Merge f-1441-perf (Responses/Codex honours reasoning_effort; setup keeps the installed size on a re-run) into f-1441

This commit is contained in:
Niko1221 committed 2026-10-14 01:51:12 +02:00
commit 296b59d2f7
6 files changed
+314 -10

No files matched your search

+1 -1
View File
@@ -1300,7 +1300,7 @@ are refused with an error that says so.
| `instructions` | The system message (with leading `developer` messages; later ones become user messages, as on the chat path) |
| `tools` | `function` tools, `namespace` tools (the model sees `namespace.name`; calls come back with `namespace` and `name`), `custom` tools (one free-form `input` string). Hosted tools (`web_search`, `file_search`, ...) are left out: the model cannot run them |
| `tool_choice` | `"none"` hides the tools; anything else lets the model choose (it cannot be forced) |
| `reasoning.effort` | `none`/`minimal`, `low`, `medium`, `high`/`xhigh`; without it the model's default (high) |
| `reasoning.effort` | `none`/`minimal`, `low`, `medium`, `high`/`xhigh`; without it the config's `reasoning_effort` (0.1.44.1; 0.1.44 ignored it here), else the model's default (high) |
| `max_output_tokens` | The output cap (thinking included). Running out ends the response `incomplete` (`max_output_tokens`) |
| `text.format` | `json_schema` and `json_object` use the [JSON response formats](#json-response-formats) (checked, not constrained) |
| `thread_source` `thread_title` in `client_metadata["x-codex-turn-metadata"]` | Codex's thread-title request (opt-in: `"codex_thread_titles": true` in `strata-<model>.json`, off by default), sent beside each user turn from another session with `tools: []`: answered without the engine as `{"title": "<the user's line, cut to 36 characters>"}`, so the conversation keeps the prompt cache. The title is the user's line, not one the model wrote |
+3 -1
View File
@@ -546,7 +546,9 @@ def openai_to_messages(req: dict) -> tuple[list[dict], list[dict] | None, dict]:
kwargs = {}
# OpenAI Chat Completions: "reasoning_effort"; Responses style: "reasoning": {"effort": ...}
reasoning = req.get("reasoning") if isinstance(req.get("reasoning"), dict) else {}
kwargs.update(effort_kwargs(req.get("reasoning_effort") or reasoning.get("effort")))
given = req.get("reasoning_effort")
# `false` is a level (thinking off, effort_kwargs): an `or` between the two fields dropped it
kwargs.update(effort_kwargs(given if given is not None and given != "" else reasoning.get("effort")))
# the vLLM / llama.cpp convention: {"chat_template_kwargs": {"enable_thinking": false, "reasoning_effort": "low"}}
for k, v in (req.get("chat_template_kwargs") or {}).items():
if k == "enable_thinking" and not v:
+7 -2
View File
@@ -3535,10 +3535,15 @@ class Service:
print(f"[strata] could not save the shared settings: {e}", flush=True)
return self.shared
def effort_default(self) -> str | None:
"""The thinking level of a request that names none: the Chat settings shared with apps first, then the config's
"reasoning_effort" (#1641). Every API reads it here (the Responses API did not: 0.1.44.1)."""
return self.shared.get("reasoning_effort") or self.config_effort
def with_shared(self, req: dict, api: str) -> dict:
"""The request with the shared thinking level and max tokens filled in where it has none of its own."""
s = self.shared
effort = s.get("reasoning_effort") or self.config_effort # the Chat settings first, then the config's
effort = self.effort_default()
if not s and not effort:
return req
req = dict(req)
@@ -5977,7 +5982,7 @@ def make_handler(svc: Service):
responses_api.check_request(req)
messages = responses_api.input_messages(req)
tools, names, skipped = responses_api.request_tools(req)
kw = responses_api.template_kwargs(req, svc.shared)
kw = responses_api.template_kwargs(req, {"reasoning_effort": svc.effort_default()})
try:
self._no_local_images(messages) # as on the chat route: no file read for a foreign page
except ValueError as e:
+172
View File
@@ -0,0 +1,172 @@
"""serve/test_effort_paths.py - every way a client sets the thinking level reaches the model's prompt (mock engine, no GPU).
The level is nothing but a sentence in the rendered prompt (the template: xhigh / low sentences, none = the empty think
block, medium = no sentence), so "the effort does not work" is always a request form whose level never reached the template.
This pins the prompt each form produces, over the three APIs, the Chat settings shared with apps and the config's
"reasoning_effort" (#1641). 0.1.44.1: the Responses API (/v1/responses, Codex) ignored the config's level.
python -m unittest serve.test_effort_paths -v
"""
from __future__ import annotations
import json
import sys
import unittest
import urllib.error
import urllib.request
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from serve.frontend import ChatTemplate # noqa: E402
from serve.server import ByteTokenizer, MockEngine, Service, serve # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
MSG = [{"role": "user", "content": "hello"}]
def level_of(prompt: str) -> str:
"""What the template made of the level: xhigh and low have their sentence, none ends in the empty think block, and
medium is the prompt with no sentence at all (the model's own wording)."""
if prompt.rstrip().endswith("</think>"):
return "none"
if "Reasoning effort is set to low." in prompt:
return "low"
if "Reasoning effort is set to xhigh." in prompt:
return "xhigh"
return "medium"
class EffortPaths(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tok = ByteTokenizer()
cls.engine = MockEngine(cls.tok, "<think>\nhm\n</think>\n\nok", max_context=8192)
cls.svc = Service(cls.engine, cls.tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
cls.httpd = serve(cls.svc, port=0)
cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}"
@classmethod
def tearDownClass(cls):
cls.httpd.shutdown()
cls.httpd.server_close()
def setUp(self):
self.svc.config_effort = None
self.svc.shared = {}
def post(self, path, body):
req = urllib.request.Request(self.base + path, data=json.dumps(body).encode(),
headers={"Content-Type": "application/json"})
self.engine.last_prompt = []
try:
with urllib.request.urlopen(req, timeout=30) as r:
status = r.status
r.read()
except urllib.error.HTTPError as e:
with e:
status = e.code
return status
def level(self, path, body):
status = self.post(path, body)
self.assertEqual(status, 200, (path, body))
return level_of(self.tok.decode(self.engine.last_prompt))
# ---------------------------------------------------------------- the request's own level, every spelling
def test_openai_reasoning_effort(self):
for given, want in ((None, "xhigh"), ("none", "none"), ("off", "none"), ("minimal", "none"), ("low", "low"),
("medium", "medium"), ("high", "xhigh"), ("xhigh", "xhigh"), ("max", "xhigh"),
("LOW", "low"), (False, "none")):
body = {"messages": MSG, **({} if given is None else {"reasoning_effort": given})}
with self.subTest(given=given):
self.assertEqual(self.level("/v1/chat/completions", body), want)
def test_openai_reasoning_object_and_template_kwargs(self):
for given, want in (("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("none", "none")):
with self.subTest(given=given):
self.assertEqual(self.level("/v1/chat/completions", {"messages": MSG, "reasoning": {"effort": given}}), want)
self.assertEqual(self.level("/v1/chat/completions",
{"messages": MSG, "chat_template_kwargs": {"reasoning_effort": given}}), want)
self.assertEqual(self.level("/v1/chat/completions",
{"messages": MSG, "chat_template_kwargs": {"enable_thinking": False}}), "none")
def test_an_unknown_level_is_a_400_not_the_default(self):
self.assertEqual(self.post("/v1/chat/completions", {"messages": MSG, "reasoning_effort": "huge"}), 400)
def test_anthropic_output_config_and_thinking(self):
for given, want in (("low", "low"), ("medium", "medium"), ("high", "xhigh")):
with self.subTest(effort=given):
self.assertEqual(self.level("/v1/messages", {"messages": MSG, "max_tokens": 50,
"output_config": {"effort": given}}), want)
for budget, want in ((500, "low"), (2047, "low"), (2048, "medium"), (8191, "medium"), (8192, "xhigh"), (31999, "xhigh")):
with self.subTest(budget=budget):
self.assertEqual(self.level("/v1/messages", {"messages": MSG, "max_tokens": 50,
"thinking": {"type": "enabled", "budget_tokens": budget}}), want)
self.assertEqual(self.level("/v1/messages", {"messages": MSG, "max_tokens": 50, "thinking": {"type": "disabled"}}),
"none")
self.assertEqual(self.level("/v1/messages", {"messages": MSG, "max_tokens": 50}), "xhigh") # nothing asked: it thinks
def test_responses_reasoning_effort(self):
for given, want in (("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("none", "none"), ("minimal", "none")):
with self.subTest(given=given):
self.assertEqual(self.level("/v1/responses", {"input": "hello", "reasoning": {"effort": given}}), want)
self.assertEqual(self.level("/v1/responses", {"input": "hello"}), "xhigh")
# ---------------------------------------------------------------- the level of a request that names none
def test_config_level_reaches_every_api(self):
for cfg, want in (("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("none", "none")):
self.svc.config_effort = cfg
with self.subTest(config=cfg):
self.assertEqual(self.level("/v1/chat/completions", {"messages": MSG}), want)
self.assertEqual(self.level("/v1/messages", {"messages": MSG, "max_tokens": 50}), want)
self.assertEqual(self.level("/v1/responses", {"input": "hello"}), want) # 0.1.44.1: it ignored the config
def test_the_requests_own_level_beats_the_config(self):
self.svc.config_effort = "low"
self.assertEqual(self.level("/v1/chat/completions", {"messages": MSG, "reasoning_effort": "high"}), "xhigh")
self.assertEqual(self.level("/v1/messages", {"messages": MSG, "max_tokens": 50, "output_config": {"effort": "high"}}),
"xhigh")
self.assertEqual(self.level("/v1/messages", {"messages": MSG, "max_tokens": 50,
"thinking": {"type": "enabled", "budget_tokens": 20000}}), "xhigh")
self.assertEqual(self.level("/v1/responses", {"input": "hello", "reasoning": {"effort": "high"}}), "xhigh")
self.assertEqual(self.level("/v1/chat/completions", {"messages": MSG, "reasoning_effort": "none"}), "none")
def test_the_chat_settings_shared_with_apps_beat_the_config(self):
self.svc.config_effort = "low"
self.svc.shared = {"reasoning_effort": "high"}
for path, body in (("/v1/chat/completions", {"messages": MSG}), ("/v1/messages", {"messages": MSG, "max_tokens": 50}),
("/v1/responses", {"input": "hello"})):
with self.subTest(path=path):
self.assertEqual(self.level(path, body), "xhigh")
self.svc.shared = {"reasoning_effort": "none"}
for path, body in (("/v1/chat/completions", {"messages": MSG}), ("/v1/messages", {"messages": MSG, "max_tokens": 50}),
("/v1/responses", {"input": "hello"})):
with self.subTest(path=path, shared="none"):
self.assertEqual(self.level(path, body), "none")
def test_the_web_apps_request_shape(self):
# serve/web/app.js: {model, messages, stream, reasoning_effort: settings.thinking} with none / low / medium / high
for given, want in (("none", "none"), ("low", "low"), ("medium", "medium"), ("high", "xhigh")):
with self.subTest(given=given):
self.assertEqual(self.level("/v1/chat/completions", {"model": "m", "messages": MSG, "stream": True,
"temperature": 0.6, "top_p": 0.95, "top_k": 20,
"reasoning_effort": given}), want)
def test_tools_and_a_system_message_keep_the_level(self):
tools = [{"type": "function", "function": {"name": "f", "description": "d",
"parameters": {"type": "object", "properties": {}}}}]
body = {"messages": [{"role": "system", "content": "be brief"}] + MSG, "tools": tools}
self.assertEqual(self.level("/v1/chat/completions", {**body, "reasoning_effort": "low"}), "low")
self.assertEqual(self.level("/v1/chat/completions", {**body, "reasoning_effort": "high"}), "xhigh")
self.assertEqual(self.level("/v1/chat/completions", {**body, "reasoning_effort": "none"}), "none")
def test_the_level_follows_a_conversation(self):
history = MSG + [{"role": "assistant", "content": "hi", "reasoning_content": "greet"},
{"role": "user", "content": "and now?"}]
for given, want in (("low", "low"), ("medium", "medium"), ("none", "none")):
with self.subTest(given=given):
self.assertEqual(self.level("/v1/chat/completions", {"messages": history, "reasoning_effort": given}), want)
if __name__ == "__main__":
unittest.main()
+27 -6
View File
@@ -2250,8 +2250,23 @@ def recommended_size(family: str, names: list, ram: float) -> str:
return "UD-IQ4_XS" if family == "unsloth" and "UD-IQ4_XS" in names else names[0]
def size_tag(family: str, m: str) -> str:
return " (recommended)" if m in RECOMMENDED_SIZES.get(family, ()) else ""
def size_tag(family: str, m: str, installed: str | None = None) -> str:
rec, have = m in RECOMMENDED_SIZES.get(family, ()), m == installed
return " (recommended, installed)" if rec and have else " (recommended)" if rec else " (installed)" if have else ""
def installed_size(family: str, names: list, have: list) -> str | None:
"""0.1.44.1: the size of this family that is already installed (the most recently used config first), or None. Setup
run again (SETUP.bat, --setup) pre-selects it: the 0.1.44 recommendation (IQ3_S, about 15% slower than IQ3_XXS and
7 GB more RAM) is for a first install, and Enter must not move an installed model to another size."""
for c in have:
try:
ch = choices_from_config(c)
except (OSError, ValueError, KeyError, TypeError, IndexError):
continue
if ch.get("family") == family and ch.get("model") in names:
return ch["model"]
return None
def unsloth_default_env(family: str, model: str, hip: bool, in_budget: bool) -> dict:
@@ -4401,6 +4416,10 @@ SETUP_KEYS = frozenset({"exe", "args", "cwd", "tokenizer", "model_name", "log",
"gpu", "gpus_asked", "layer_split", "host", "api_key", "draft_vocab", "vision"})
SETUP_ENV = frozenset({"STRATA_HIPBLASLT_TUNING", "STRATA_RESIDENT_PIN", "STRATA_NO_ARENA_THP", "STRATA_MMQ_KQ", "STRATA_HC_Q8",
"STRATA_ROUTE_TAIL_SKIP"}) # the "env" entries setup writes
# 0.1.44 added these three, written for Unsloth's files only (unsloth_default_env). In any other model's config they are the
# user's own (STRATA_ROUTE_TAIL_SKIP=0 to turn the tail skip off, STRATA_MMQ_KQ=1 ...): a setup run again keeps them there
# (0.1.44 dropped them with the rest of SETUP_ENV).
SETUP_ENV_UNSLOTH = frozenset({"STRATA_MMQ_KQ", "STRATA_HC_Q8", "STRATA_ROUTE_TAIL_SKIP"})
SETUP_VISION = frozenset({"exe", "mmproj", "model", "gpu", "max_tokens", "threads"})
@@ -4415,7 +4434,8 @@ def carry_over(old: dict, cfg: dict) -> list[str]:
if k not in SETUP_KEYS and k not in cfg:
cfg[k] = v
kept.append(k)
env = {k: v for k, v in (old.get("env") or {}).items() if k not in SETUP_ENV and k not in (cfg.get("env") or {})} \
mine = SETUP_ENV if "-unsloth-" in str(cfg.get("model_name") or "") else SETUP_ENV - SETUP_ENV_UNSLOTH
env = {k: v for k, v in (old.get("env") or {}).items() if k not in mine and k not in (cfg.get("env") or {})} \
if isinstance(old.get("env"), dict) else {}
if env:
cfg["env"] = {**(cfg.get("env") or {}), **env}
@@ -6140,20 +6160,21 @@ def main() -> int:
+ (f" (or {a.model}: " + ", ".join(f"--family {f} --model {a.model}" for f in elsewhere_fams) + ")"
if elsewhere_fams else "")
+ (f".\n {SUPPORTED_GGUFS}" if a.gguf_dir else ""))
installed = installed_size(family, names, have)
for i, m in enumerate(names, 1):
d = MODELS[m]
fit = "" if ram >= d["ram_gb"] else f" <- needs {d['ram_gb']} GB RAM, you have {ram:.0f}"
if d.get("budget"):
keeps = (f"~{resident_budget_gib(m, ram)} GB" if unsloth_budget(m, ram, gpu, a.resident_budget_gib)
else "all")
say(f" {i}) {m}{size_tag(family, m)} {d['about']}; download {d['download_gb']:.0f} GB, keeps {keeps} of its "
say(f" {i}) {m}{size_tag(family, m, installed)} {d['about']}; download {d['download_gb']:.0f} GB, keeps {keeps} of its "
f"{d['arena_gb']:.0f} GB of experts in RAM{fit}")
continue
if low_ram_needed(m, ram) and low_ram_fits(m, ram, low_ram_vram(gpu)) and a.low_ram != "off":
fit = (f" <- fits in the low-RAM mode (the GPU holds ~{100 * low_ram_gpu_share(m, low_ram_vram(gpu)):.0f}%, "
+ ("the rest in RAM)" if low_ram_resident(m, ram, low_ram_vram(gpu)) else "the rest from the SSD)"))
say(f" {i}) {m:8s}{size_tag(family, m)} {d['about']}; download {d['download_gb']:.0f} GB, uses ~{d['arena_gb']:.0f} GB of RAM{fit}")
rec = str(names.index(recommended_size(family, names, ram)) + 1)
say(f" {i}) {m:8s}{size_tag(family, m, installed)} {d['about']}; download {d['download_gb']:.0f} GB, uses ~{d['arena_gb']:.0f} GB of RAM{fit}")
rec = str(names.index(installed or recommended_size(family, names, ram)) + 1)
model = a.model or names[int(ask("Which size?", [str(i) for i in range(1, len(names) + 1)], rec, a.yes)) - 1]
budget, q4_split = None, False
in_budget = bool(MODELS[model].get("budget")) and unsloth_budget(model, ram, gpu, a.resident_budget_gib)
+104
View File
@@ -0,0 +1,104 @@
"""0.1.44.1: setup run again over an installed model keeps its size.
0.1.44 made IQ3_S the recommended Qwen size on a PC with 64 GB or more (IQ3_S: about 15% slower decode than IQ3_XXS on an
RTX 5070, docs/MODELS.md, and about 7 GB more RAM). A first install takes the recommendation; SETUP.bat / --setup on a
folder that already has a model pre-selects the installed size (and `--yes` takes it), so Enter never moves a working
install to a slower size. The recommended size stays marked, every size stays a choice.
python -m unittest tools.test_setup_rerun_size
"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "tools"))
import setup # noqa: E402
import test_setup_golden as g # noqa: E402
PC64 = (63.7, [g.card(0, "NVIDIA GeForce RTX 5070", 11.9, "120")])
def earlier(model: str, family_tag: str = "") -> tuple:
tag = f"{family_tag}{model.lower()}"
return (f"strata-{tag}.json", {"exe": "x", "args": ["--pack", f"p/{tag}", "--max-context", "32768"], "cwd": ".",
"tokenizer": "t", "model_name": f"qwen3.8-flash-next-{tag}", "log": "l", "port": 8080})
class RerunKeepsSize(unittest.TestCase):
def run_setup(self, argv, configs, answers=""):
ram, found = PC64
code, out, cfg, asked = g.install(ram, found, argv, answers=answers, configs=configs)
self.assertEqual(code, 0, out[-2000:])
return out, cfg, asked
def test_first_install_takes_the_recommendation(self):
out, cfg, _ = self.run_setup(["--family", "qwen", "--no-start"], [])
self.assertEqual(cfg["model_name"], "qwen3.8-flash-next-iq3_s")
self.assertNotIn("(installed)", out)
def test_enter_over_an_installed_iq3_xxs_keeps_it(self):
out, cfg, asked = self.run_setup(["--family", "qwen", "--setup", "--no-start"], [earlier("IQ3_XXS")])
self.assertEqual(cfg["model_name"], "qwen3.8-flash-next-iq3_xxs", out[-1500:])
line_s = next(x for x in out.splitlines() if " IQ3_S " in x or " IQ3_S\t" in x or "IQ3_S " in x)
self.assertIn("(recommended)", line_s) # still marked, still a choice
line_xxs = next(x for x in out.splitlines() if "IQ3_XXS" in x and "download" in x)
self.assertIn("(installed)", line_xxs)
self.assertTrue(any("Which size?" in a for a in asked))
def test_yes_over_an_installed_size_keeps_it_too(self):
_, cfg, asked = self.run_setup(["--family", "qwen", "--setup", "--no-start"], [earlier("Q2_0")], answers=None)
self.assertEqual(cfg["model_name"], "qwen3.8-flash-next-q2_0")
self.assertEqual(asked, [])
def test_a_named_size_still_wins(self):
_, cfg, _ = self.run_setup(["--family", "qwen", "--model", "IQ3_S", "--setup", "--no-start"], [earlier("IQ3_XXS")])
self.assertEqual(cfg["model_name"], "qwen3.8-flash-next-iq3_s")
def test_another_family_is_not_touched(self):
# a Qwen IQ3_XXS install does not pre-select anything for the Unsloth family
_, cfg, _ = self.run_setup(["--family", "unsloth", "--setup", "--no-start"], [earlier("IQ3_XXS")])
self.assertIn("unsloth", cfg["model_name"])
def test_the_helper_prefers_the_most_recent_config_of_the_family(self):
import json
import tempfile
with tempfile.TemporaryDirectory() as tmp:
t = Path(tmp)
paths = []
for name, cfg in (earlier("IQ3_S"), earlier("Q2_0")):
p = t / name
p.write_text(json.dumps(cfg))
paths.append(p)
names = [m for m in setup.MODELS if "qwen" in setup.MODELS[m].get("families", ("qwen", "swift"))]
self.assertEqual(setup.installed_size("qwen", names, paths), "IQ3_S") # first = most recently used
self.assertEqual(setup.installed_size("qwen", names, paths[1:]), "Q2_0")
self.assertIsNone(setup.installed_size("qwen", names, []))
class RerunKeepsOwnEnv(unittest.TestCase):
"""0.1.44 put STRATA_MMQ_KQ / STRATA_HC_Q8 / STRATA_ROUTE_TAIL_SKIP into SETUP_ENV (the entries setup writes, which a
setup run again does not carry over): a GSQ-RCO config of the user's own STRATA_ROUTE_TAIL_SKIP=0 was dropped with them.
Those three are setup's only in an Unsloth config."""
OLD = {"env": {"STRATA_ROUTE_TAIL_SKIP": "0", "STRATA_MMQ_KQ": "1", "STRATA_HC_Q8": "0", "STRATA_RESIDENT_PIN": "0",
"MY_VAR": "1"}}
def test_a_gsq_rco_config_keeps_the_users_values(self):
new = {"model_name": "qwen3.8-flash-next-iq3_xxs"}
kept = setup.carry_over(dict(self.OLD), new)
self.assertEqual(new["env"], {"STRATA_ROUTE_TAIL_SKIP": "0", "STRATA_MMQ_KQ": "1", "STRATA_HC_Q8": "0", "MY_VAR": "1"})
self.assertIn("env STRATA_ROUTE_TAIL_SKIP", kept)
self.assertNotIn("env STRATA_RESIDENT_PIN", kept) # the older entries stay setup's own
def test_an_unsloth_config_gets_setups_values_not_the_old_ones(self):
new = {"model_name": "qwen3.8-flash-next-unsloth-ud-q4_k_xl", "env": {"STRATA_HC_Q8": "1"}}
setup.carry_over(dict(self.OLD), new)
self.assertEqual(new["env"], {"STRATA_HC_Q8": "1", "MY_VAR": "1"})
if __name__ == "__main__":
unittest.main()