Merge branch 'f-138-fixB' into f-138-prs

A rebinding page cannot send the key, so with one configured the check only
broke tunnels and proxies that pass their own name on (cloudflared does).
Keyless setups keep it, with allowed_hosts; the 403 now also says that
setting an API key turns the check off.

.github/FUNDING.yml

@@ -0,0 +1 @@
+buy_me_a_coffee: strataengine

.gitignore

@@ -37,6 +37,7 @@ td.log
 /strata-*.json
 /strata-*.log
 /build-strata.bat
+/.strata-mcp/
 
 # image encoder (setup.py --vision): build output and the generated build script
 build-vision*/
@@ -49,3 +50,4 @@ build-vision.bat
 *.done
 /third_party/*.zip
 
+/.rocm-win/

AGENTS.md

@@ -0,0 +1,23 @@
+# AGENTS.md
+
+Strata runs the Qwen3.8-Flash-Next mixture-of-experts model (and its Coder, Swift 1.5 and Unsloth variants) on a
+normal PC: one NVIDIA or AMD graphics card plus system RAM, on Windows or Linux. It has a C++/CUDA/HIP engine
+(`src/`, `include/`), a Python server with an OpenAI- and Anthropic-compatible API and a web app (`serve/`), and a
+one-click installer (`setup.py`, started by `START-HERE.bat` / `setup.sh`).
+
+## Installing Strata for a user
+
+Follow **[docs/AI_SETUP.md](docs/AI_SETUP.md)**: check the PC, pick the model by RAM, run setup non-interactively,
+start and verify the server, and connect the user's apps. Never expose the server beyond `127.0.0.1` without
+`--api-key`. As an alternative to shell commands, Strata's MCP server ([docs/MCP_SERVER.md](docs/MCP_SERVER.md))
+offers the same steps as tools.
+
+## Working on the code
+
+- How the engine works, every measured number, the API and all settings: [docs/DETAILS.md](docs/DETAILS.md) and
+  the [paper](docs/paper/Strata-Paper.pdf).
+- AMD (HIP) build and validation: [docs/AMD_HIP.md](docs/AMD_HIP.md); multi-GPU: [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
+- Setup's own tests run without a GPU or downloads: `python tools/test_setup_<name>.py` (for example
+  `tools/test_setup_amd.py`, `tools/test_setup_choices.py`).
+- Keep the docs' style: plain words, measured numbers with what they were measured on, no claims without a
+  measurement.

CMakeLists.txt

@@ -8,7 +8,7 @@
 # below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
 # and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.32 LANGUAGES CXX)
+project(strata VERSION 0.1.38 LANGUAGES CXX)
 add_compile_definitions(STRATA_VERSION="${PROJECT_VERSION}")
 # issue #31: release builds keep a PDB beside the exe (not shipped in it), so a minidump from a stalled engine can
 # be read back to the exact source line
@@ -274,6 +274,15 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     target_include_directories(qsa_prompt_attn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
     target_link_libraries(qsa_prompt_attn_parity PRIVATE strata_kernels CUDA::cudart)
   endif()
+  # #512: the top-k's active-block bound (Turing's dispatch, forced on any card by STRATA_TOPK_ACTIVE_ANY) against
+  # the capacity rule, selected ids bitwise
+  if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA)
+    add_executable(qsa_topk_active_parity src/kernels/qsa_topk_active_parity.cpp)
+    target_include_directories(qsa_topk_active_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+    target_link_libraries(qsa_topk_active_parity PRIVATE strata_kernels CUDA::cudart)
+    add_test(NAME qsa_topk_active_parity COMMAND qsa_topk_active_parity)
+    set_tests_properties(qsa_topk_active_parity PROPERTIES SKIP_RETURN_CODE 77)
+  endif()
   # the QSA selection (block scores on the warp kernel and the tensor-core one, top-k): a GPU bench with a
   # scorer-vs-scorer comparison and an FP64 accuracy gate (synthetic, no model); the ctest runs a 32K context under a
   # 262144-cell capacity (the top-k dispatch follows the capacity) and exits non-zero on a failed gate
@@ -340,7 +349,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   # about arithmetic, and this one knows the ORDER the arithmetic happens in.
   add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
     src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp
-    src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp)
+    src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/peer_experts.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
   target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
   if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/file_expert_source_test.cpp")
@@ -350,6 +359,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     add_test(NAME file_expert_source_test COMMAND file_expert_source_test)
     set_tests_properties(file_expert_source_test PROPERTIES TIMEOUT 60)
   endif()
+  # #477: the learned expert profile (--expert-profile-save): its order and make_profile.py's file format (CPU only)
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/expert_profile_save_test.cpp")
+    enable_testing()
+    add_executable(expert_profile_save_test tests/core/expert_profile_save_test.cpp)
+    target_link_libraries(expert_profile_save_test PRIVATE strata_engine)
+    add_test(NAME expert_profile_save_test COMMAND expert_profile_save_test)
+    set_tests_properties(expert_profile_save_test PROPERTIES TIMEOUT 60)
+  endif()
   if(STRATA_BUILD_TESTS AND STRATA_NATIVE_EXPERTS)
     # native_experts.txt v2-v4 (the real packs' copies and the per-role shard column) and the arena filled from
     # synthetic GGUF shards at the real expert dimensions
@@ -362,6 +379,11 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     target_link_libraries(native_dense_ple_key_test PRIVATE strata_engine)
     add_test(NAME native_dense_ple_key_test COMMAND native_dense_ple_key_test)
     set_tests_properties(native_dense_ple_key_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60)
+    # #369: per-layer admission on a sized (native pack) cache starts each layer in its own range
+    add_executable(expert_cache_per_layer_test tests/core/expert_cache_per_layer_test.cpp)
+    target_link_libraries(expert_cache_per_layer_test PRIVATE strata_engine)
+    add_test(NAME expert_cache_per_layer_test COMMAND expert_cache_per_layer_test)
+    set_tests_properties(expert_cache_per_layer_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60)
   endif()
   if(STRATA_BUILD_CONVERSATION_TESTS)
     add_executable(conversation_snapshot_test src/core/conversation_snapshot_test.cpp)
@@ -421,6 +443,15 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
       add_test(NAME hip_handoff COMMAND hip_handoff)
       list(APPEND _strata_hip_tests hip_handoff)
     endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/mapped_alias.cpp")
+      # #325: reports what a hipHostGetDevicePointer alias is on this stack (on Windows: the host pointer, and a
+      # D2D copy into it does not land - hip_handoff's timeout there). A diagnosis: exits 0 whenever it ran.
+      add_executable(hip_mapped_alias tests/hip/mapped_alias.cpp)
+      target_link_libraries(hip_mapped_alias PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_mapped_alias COMMAND hip_mapped_alias)
+      set_tests_properties(hip_mapped_alias PROPERTIES SKIP_RETURN_CODE 77)
+      list(APPEND _strata_hip_tests hip_mapped_alias)
+    endif()
     if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/native_qsa_score.cpp")
       add_executable(hip_native_qsa_score tests/hip/native_qsa_score.cpp)
       target_link_libraries(hip_native_qsa_score PRIVATE strata_kernels strata_hip_runtime)
@@ -536,6 +567,19 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   target_include_directories(iq_multi_parity PRIVATE ${_strata_gpu_include_directories})
   target_link_libraries(iq_multi_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME iq_multi_parity COMMAND iq_multi_parity)
+  # the prompt path's DeltaNet recurrence: gdn_rec_kh_kernel against gdn_rec_cols_pipe_kernel, bitwise (GPU,
+  # synthetic, no model; --bench adds timings)
+  if(STRATA_ENABLE_CUDA)
+    add_executable(gdn_rec_parity src/prefill/gdn_rec_parity.cu)
+    target_link_libraries(gdn_rec_parity PRIVATE ${_strata_gpu_runtime_target})
+    add_test(NAME gdn_rec_parity COMMAND gdn_rec_parity)
+  endif()
+  # native_expert_grouped's group stride and fused SwiGLU + q8_1, bitwise vs the launches before them (GPU, synthetic,
+  # no model; --bench adds v1/new timings of a window's 48 calls in a graph)
+  add_executable(native_grouped_parity src/kernels/native_grouped_parity.cpp)
+  target_include_directories(native_grouped_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(native_grouped_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  add_test(NAME native_grouped_parity COMMAND native_grouped_parity)
   add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)
   target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})
   target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
@@ -566,6 +610,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   set_tests_properties(sampler_parity_one_block PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=0;STRATA_SAMPLER_ONE_BLOCK=1")
   add_test(NAME sampler_parity_old COMMAND sampler_parity --selftest)
   set_tests_properties(sampler_parity_old PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=1;STRATA_SAMPLER_ONE_BLOCK=0")
+  # S19: the decode QSA top-k and the greedy argmax on thread-block clusters (sm_90+) vs the one-CTA kernels, bitwise,
+  # captured in a graph too (GPU, synthetic, no model; skips below sm_90); --bench: per-call times
+  if(STRATA_ENABLE_CUDA)
+    add_executable(decode_cluster_parity src/kernels/decode_cluster_parity.cpp)
+    target_include_directories(decode_cluster_parity PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(decode_cluster_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+    add_test(NAME decode_cluster_parity COMMAND decode_cluster_parity --selftest)
+  endif()
 
   add_executable(rope_parity src/kernels/rope_parity.cpp)
   target_include_directories(rope_parity PRIVATE ${_strata_gpu_include_directories})
@@ -854,7 +906,10 @@ if(STRATA_NATIVE_EXPERTS)
       set(_strata_gg ${strata_llamacpp_SOURCE_DIR}/ggml)
     endif()
     set(_strata_mmq ${_strata_gg}/src/ggml-cuda)
-    add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
+    # #136: the Q2_0 pack's prompt experts on Strata's own int8 kernels (moe_fused.cu, opt-in STRATA_PF_FUSED=1; CUDA
+    # only: the HIP branch below keeps MMQ, and an sm_75 device or an sm_75-only build reports them unavailable)
+    add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/moe_fused.cu src/prefill/moe_fused_iq.cu
+                src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
     # q8_0: the drafter's matrices (E-9); STRATA_ORCA_Q4KS_MMQ adds q5_0 (#296, in _strata_mmq_types), and
     # STRATA_MMQ_KQUANTS q4_k q5_k q5_1, the experts of Unsloth's UD-Q4_K_XL (docs/UNSLOTH_Q4.md; the dispatch
     # follows eddoursul/Strata 7706c1b)
@@ -879,7 +934,19 @@ if(STRATA_NATIVE_EXPERTS)
       target_compile_definitions(strata_mmq PRIVATE STRATA_ORCA_Q4KS_MMQ=1)
     endif()
     target_link_libraries(strata_prefill PUBLIC strata_mmq)
-    target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+    target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1 STRATA_PREFILL_FUSED=1)
+    if(STRATA_BUILD_TESTS)
+      # #136: the fused int8 experts against a double-precision reference and the MMQ path (exit 77 below sm_80)
+      add_executable(prefill_fused_moe_test tests/cuda/prefill_fused_moe_test.cpp)
+      target_link_libraries(prefill_fused_moe_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
+      add_test(NAME prefill_fused_moe_test COMMAND prefill_fused_moe_test)
+      set_tests_properties(prefill_fused_moe_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 300)
+      # #136: the same for the native packs' formats (IQ2_XXS IQ2_XS IQ2_S IQ3_XXS IQ3_S IQ4_XS, down Q2_0 IQ4_NL)
+      add_executable(prefill_fused_iq_test tests/cuda/prefill_fused_iq_test.cpp)
+      target_link_libraries(prefill_fused_iq_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
+      add_test(NAME prefill_fused_iq_test COMMAND prefill_fused_iq_test)
+      set_tests_properties(prefill_fused_iq_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600)
+    endif()
     if(STRATA_BUILD_TESTS AND STRATA_MMQ_KQUANTS)
       # the UD-Q4_K_XL expert formats through MMQ against ggml's dequantized weights (synthetic, a few seconds)
       add_executable(prefill_mmq_kquant_test tests/cuda/prefill_mmq_kquant_test.cpp)
@@ -938,8 +1005,13 @@ if(STRATA_NATIVE_EXPERTS)
     target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact ${_strata_gpu_runtime_target})
     if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA)
       # Unsloth UD-Q4_K_XL's three expert pairs (and Q8_0 alone, its embedding) on synthetic ggml-quantized weights
-      # at 2560/640: GPU and CPU experts against the float reference, GPU dequantizers against to_float
-      foreach(_pair q4_K/q5_1 q4_K/q8_0 q5_K/q8_0 q8_0/q8_0)
+      # at 2560/640: GPU and CPU experts against the float reference, GPU dequantizers against to_float.
+      # q4_K/q5_0 and q5_0/q8_0: a plain llama-quantize "_M" tier GGUF (not Unsloth's dynamic method) commonly
+      # puts Q5_0 on down-projections (sometimes gate/up too) where this model's native_experts.txt had no GPU
+      # kernel for it before; q5_0/q5_0 (both roles Q5_0) is left out of this list since stacking the same
+      # lower-precision format on both roles pushes the GPU path over this test's 3% threshold (quantization
+      # noise, not a kernel bug -- see the PR).
+      foreach(_pair q4_K/q5_1 q4_K/q8_0 q5_K/q8_0 q8_0/q8_0 q4_K/q5_0 q5_0/q8_0)
         string(REPLACE "/" "_" _name "${_pair}")
         add_test(NAME native_expert_parity_${_name} COMMAND native_expert_parity --synthetic ${_pair})
         set_tests_properties(native_expert_parity_${_name} PROPERTIES TIMEOUT 120)

README.md

@@ -1,279 +1,185 @@
 <h1 align="center">Strata</h1>
 
-<p align="center"><b>Run a 125-billion-parameter AI model on a normal gaming PC</b><br>
-one NVIDIA card (12-24 GB) + 64 GB of RAM · Windows or Linux · one click to install</p>
+<p align="center"><b>Run a 125-billion-parameter AI model on your own gaming PC</b><br>
+NVIDIA or AMD graphics card (12 GB or more) · Windows or Linux · free and open source</p>
 
 <p align="center"><a href="https://github.com/Niko1221/Strata/releases/download/v0.1.10/Pagoda.mp4"><img src="docs/media/pagoda-preview.webp" width="720" alt="A voxel pagoda garden that Strata's model wrote, running in the browser"></a><br>
 <sub>A voxel pagoda garden, 1 shot prompt running on an RTX 5070 with Strata (IQ3_S, 128K context) ·
 <a href="https://github.com/Niko1221/Strata/releases/download/v0.1.10/Pagoda.mp4">full video (49 s)</a></sub></p>
 
 Strata runs **[Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next)** - a large, smart AI model that
-normally needs a server - on your own PC. It writes its answers at **60-95 tokens per second** (a token is about ¾
-of a word): faster than you can read.
-
-- **Free and open source.**
-
-> **Jump to:** [How fast?](#how-fast-is-it) · [Which model?](#which-model-should-i-pick) · [Install](#install) ·
-> [Using it](#using-it) · [Problems?](#something-went-wrong) · [How it works](#how-does-it-work) ·
-> [All the details](docs/DETAILS.md)
-
----
+normally needs a server - on a normal PC. It chats, writes code, reads pictures and works with your apps and coding
+agents, and nothing leaves your PC.
 
 ## How fast is it?
 
-Measured on an RTX 5070 (12 GB), a Ryzen 5 7600 and 64 GB of RAM:
+Measured on two ordinary gaming PCs. "Writes answers" is how fast the reply appears in a short chat; "reads your
+prompt" is how fast it takes in what you send (a 32K-token document, code or chat history). A token is about ¾ of a
+word, so 60 tokens per second is faster than you can read.
 
-| Size | Writes answers (short chat) | Writes answers (128K context) | Reads your prompt |
-| --- | ---: | ---: | ---: |
-| **Q2_0** | 93 tokens/s | 74 tokens/s | 2,170 tokens/s |
-| **IQ2_XS** | 79 tokens/s | 63 tokens/s | 2,090 tokens/s |
-| **IQ3_XXS** | 62 tokens/s | 49 tokens/s | 1,750 tokens/s |
-| **IQ3_S** | 53 tokens/s | 46 tokens/s | 1,620 tokens/s |
-| **Coder** (IQ1_M) | 55 tokens/s | 43 tokens/s | 2,180 tokens/s |
+<table>
+<tr><th>NVIDIA: RTX 5070 (12 GB), Ryzen 5 7600, 64 GB RAM</th><th>AMD: RX 9070 XT (16 GB), Ryzen 9 3900X, 47 GB RAM</th></tr>
+<tr><td>
 
-- **Writes answers** = how fast the reply appears (tokens per second).
-- **Reads your prompt** = how fast it takes in what you send (long documents, code, chat history), measured on a
-  32K-token prompt; a 4K prompt reads at 910-1,580 tokens/s. A 32K prompt takes about 15 seconds with Q2_0.
+| Size | Writes answers | Reads your prompt |
+| --- | ---: | ---: |
+| **Q2_0** | 94 tokens/s | 2,650 tokens/s |
+| **IQ2_XS** | 79 tokens/s | 2,090 tokens/s |
+| **IQ3_XXS** | 62 tokens/s | 1,750 tokens/s |
+| **IQ3_S** | 53 tokens/s | 1,620 tokens/s |
+| **Coder** | 55 tokens/s | 2,180 tokens/s |
 
-A card with more VRAM is faster, because more of the model fits on the GPU: an RTX 3090 (24 GB) should do roughly
-100-140 tokens per second. All measurements, long-context numbers and estimates for other cards are in the
-[details](docs/DETAILS.md#speed-measured).
+</td><td>
 
-Every PC is different: `START-HERE.bat --calibrate` measures a few engine settings on yours and keeps the fastest
-(about 5-10 minutes; on the PC above it made the Coder 7% faster).
+| Size | Writes answers | Reads your prompt |
+| --- | ---: | ---: |
+| **Q2_0** | 60 tokens/s | 1,160 tokens/s |
+| **IQ2_XS** | 52 tokens/s | 1,110 tokens/s |
+| **Coder** | 44 tokens/s | 1,420 tokens/s |
 
-Measured Strata on your own PC? See [Community benchmark results](docs/COMMUNITY_BENCHMARKS.md)
-for a report template and how to share your results in a pull request.
+</td></tr>
+</table>
 
-**Two or three NVIDIA cards?** Just run `START-HERE.bat`: it lists your cards, says which ones Strata can use, and
-asks whether to share the model across them (recommended when two can). An install made on one card asks once at
-its next start. Or choose yourself: `START-HERE.bat --gpus 0,2` (both, remembered), `--gpus all`, or `--gpu 0` (one
-card, this start only). Each card keeps the experts of its own layers, and prompts flow through the cards in a
-pipeline: on an RTX 5080 + RTX 3090 prompts were read 18-20% faster than on the 5080 alone, decoding on par.
-Every card must be an RTX 20 series or newer with 8 GB or more. See [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
+A card with more VRAM is faster: an RTX 3090 (24 GB) should write roughly 100-140 tokens per second. Long chats,
+other cards: [speed of each model](docs/MODELS.md#how-fast-is-each-size), [community results](docs/COMMUNITY_BENCHMARKS.md).
 
-## Which model should I pick?
+<p align="center"><a href="https://buymeacoffee.com/strataengine"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" height="50"></a><br>
+<sub>Strata is free. If it runs well on your PC, a coffee keeps the work on it going.</sub></p>
 
-**The size** (the same model, compressed more or less):
+## What you need
 
-| Model | RAM+VRAM Requirements | Speed | Quality |
-| --- | ---: | --- | --- |
-| **Q2_0** | 37.6 GB | fastest | good |
-| **IQ2_XS** | 39.2 GB | fast | better (**recommended**) |
-| **IQ3_XXS** | 47.0 GB | slower | great |
-| **IQ3_S** | 54.8 GB | slowest | best: matches the full model on the published tests (original model only) |
+| | |
+| --- | --- |
+| **Graphics card** | **NVIDIA** GeForce RTX 20, 30, 40 or 50 series, or **AMD** Radeon RX 7900 XT / XTX, RX 7800 XT / 7700 XT, RX 9060 XT, RX 9070 / 9070 XT, Radeon AI PRO R9700 or RX 6800 / 6900 series - with **12 GB of VRAM or more** |
+| **RAM** | 32 GB or more - how much decides [which model](#which-model-should-i-pick) fits; 64 GB runs every size |
+| **Disk** | about 80 GB free, on an SSD if you can (the first start is much faster) |
+| **System** | Windows 10 / 11 or Linux, and a current graphics driver from NVIDIA or AMD |
 
-**Will it fit?** Shard 1 is the part of the model that gets loaded when it starts: its experts go into your **RAM**,
-the rest onto your graphics card (the second shard, a 29 GB lookup table, stays on the SSD). So it fits when your
-**RAM is at least shard 1 + about 10 GB** for Windows and your other programs. With 64 GB of RAM every size fits
-(IQ3_S with little else open); with 48 GB, Q2_0 and IQ2_XS. A bigger graphics card makes it faster, but it doesn't
-lower the RAM needed.
-
-**The version:**
-
-- **Qwen3.8-Flash-Next** - the original.
-- **[Coder](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF)** - ISTA-DASLab's coding
-  version: half of the experts removed, keeping the ones that code, tool use and images need (91% of the full model's
-  SWE-bench Verified score, 99% of LiveCodeBench, by its authors). One size (IQ1_M: its experts stored like IQ3_S):
-  shard 1 is **29.6 GB**, so it fits a PC with **32 GB of RAM**, runs 262K context on 64 GB, and reads long prompts
-  the fastest of all. Weaker outside coding.
-- **[Swift 1.5](https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF)** - a fine-tune by UkisAI
-  that thinks much shorter before answering, so you get the answer sooner, with about the same quality. Same speed per
-  token, and about the same RAM as the same size of the original (no IQ3_S). Its own license applies (see its page).
-
-Not sure? Take **IQ2_XS** - or the **Coder** if you mainly write code, or have 32-48 GB of RAM. You can add another
-one later with `SETUP.bat` (the same as `START-HERE.bat --setup`; on Linux `./setup.sh --setup`).
-
-For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](docs/ORCA.md).
-It needs an explicit packing conversion and is not an installer menu option.
-
-**Unsloth's 4-bit UD-Q4_K_XL** (experimental) is the fourth version in setup's menu (`--family unsloth`): the closest
-to the full model, but a 111 GB download whose 77 GB of experts do not fit in RAM. Strata keeps your RAM minus 24 GB
-of them in RAM and reads the rest from the SSD while it answers: 7-8.5 tokens/s on a 64 GB PC with a 12 GB GPU, several
-times slower than the sizes above, and long prompts are slow. It needs 48 GB of RAM or more, an NVMe SSD and one
-NVIDIA GPU (no images yet). Details and measurements: [UD-Q4_K_XL](docs/UNSLOTH_Q4.md).
-
-An **AMD Radeon RX 7900 XT / XTX, RX 9070 / 9070 XT or Radeon AI PRO R9700 on Linux** works too (experimental; the
-RX 7800 XT / 7700 XT and RX 9060 XT were validated by their owners; the RX 6800 / 6900 series, gfx1030, is community-reported):
-`./setup.sh --backend hip`, chosen by itself on a PC with no NVIDIA card Strata can use. It installs ROCm without sudo
-and compiles the engine (no images yet; several cards with `--gpus`). Details: [AMD HIP](docs/AMD_HIP.md).
+Everything else is installed for you. Two or three cards can share the model ([multi-GPU](docs/MULTI_GPU.md)).
+The full list: [docs/INSTALL.md](docs/INSTALL.md#what-you-need).
 
 ## Install
 
-**You need:** an NVIDIA RTX 20, 30, 40 or 50 card with 12 GB of VRAM or more (RTX 20 since 0.1.27), enough RAM for the size you pick (above;
-a big GPU makes up for less RAM - the [low-RAM mode](docs/DETAILS.md)),
-~80 GB of free disk space (an SSD makes the first start much faster), and Windows 10/11 or Linux. The only thing you
-install yourself is a current **NVIDIA driver** ([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA
-App). Everything else - Python, the engine, the model - is set up for you.
+### Let your AI set it up
 
-**Windows**
+Use an AI coding assistant (Claude Code, Cursor, Codex, GitHub Copilot, ...)? Paste this into it:
 
-1. [Download this project](https://github.com/Niko1221/Strata/archive/refs/heads/main.zip) and unzip it (or `git clone` it).
-2. Double-click **`START-HERE.bat`**.
-3. Answer a few questions - or just press Enter each time for the recommended choice:
-   - **Which model and size?** The original or Swift 1.5, and Q2_0, IQ2_XS, IQ3_XXS or IQ3_S - see [above](#which-model-should-i-pick)
-   - **How much context?** How much text it can keep in mind at once (it suggests one for your card). 384K and
-     512K (experimental) extend the model past its trained 262K by rope scaling - the setup turns it on itself (yarn and a
-     covering factor; `--rope-scaling`/`--rope-scale` override) ([details](docs/DETAILS.md))
-   - **Images?** Whether it should also read pictures
-   - **Experimental speed projection?** Off unless you say yes - [read what it does](docs/DETAILS.md#experimental-speed-projection-experimental-off-by-default) first
+```text
+Set up Strata on this PC for me: https://github.com/Niko1221/Strata - follow docs/AI_SETUP.md in that repository.
+```
 
-Then it downloads everything (the model is ~70 GB, so the first time takes a while - you can stop and it picks up
-where it left off) and **starts the model**. Your browser opens the Strata app at `http://127.0.0.1:8080`.
+It checks your graphics card, RAM and disk, picks the model that fits, installs it, starts it and tells you how to
+connect your apps. AI tools can also install, start and stop Strata themselves through its
+[MCP server](docs/MCP_SERVER.md).
+
+### Or do it yourself
+
+[Download Strata](https://github.com/Niko1221/Strata/archive/refs/heads/main.zip) and unzip it (or `git clone` it).
+**Windows:** double-click **`START-HERE.bat`**. **Linux:** run **`./setup.sh`** in the Strata folder.
+
+The same steps for NVIDIA and AMD: the installer finds your card and sets up the right engine for it. It asks which
+model, which size, how much context (how much text it keeps in mind) and whether it should read pictures - press
+Enter each time for the recommended answer. Then it downloads the model (~70 GB; you can stop and it continues where
+it left off) and starts it. Your browser opens the Strata app at `http://127.0.0.1:8080`.
 
 > **While the model starts, your PC can be slow or stop responding for 1-3 minutes** (longest the first time): Strata
 > loads 35-55 GB into your RAM and locks part of it for the graphics card. That's normal - wait, and don't close the
 > window. The window tells you what it is doing.
 
-**Next time**, just double-click `START-HERE.bat` again: it starts right away, nothing is downloaded twice. Close its
-window to stop the model.
+**Next time**, run `START-HERE.bat` (or `./setup.sh`) again: it starts right away, nothing is downloaded twice. Close
+its window to stop the model. `UPDATE.bat` (`./update.sh`) updates Strata without starting it. Updating, Docker,
+several cards, where the files go and every option:
+[docs/INSTALL.md](docs/INSTALL.md).
 
-**Updating:** download the new version and unzip it anywhere (or `git pull`), then run `START-HERE.bat` in it. The
-model files are kept in a `Strata-data` folder next to your Strata folder, so a new copy finds them and sets itself up
-the same way - nothing big is downloaded again.
+## Which model should I pick?
 
-**Linux:** run `./setup.sh` - same questions, same result.
+The installer recommends one for your RAM. The same model comes in sizes that are compressed more or less: smaller
+is faster, larger is a bit smarter.
 
-**Docker (Linux):** the same idea, in a container.
+| Your RAM | Take | Why |
+| --- | --- | --- |
+| **32 GB** | **Coder** | it fits 32 GB, and it is made for code (with a 24 GB card, Q2_0 and IQ2_XS run too) |
+| **48 GB** | **IQ2_XS** (or Q2_0, the fastest) | the larger sizes do not fit |
+| **64 GB** | **IQ2_XS** (recommended), or IQ3_XXS / IQ3_S | every size fits; IQ3_S is the best, and the slowest |
+| **96 GB or more** | **IQ3_S**, or Unsloth's 4-bit (experimental) | room for the largest sizes with everything else open |
 
-1. Host: Docker with the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
-   and a driver **580 or newer** (CUDA 13.0).
-2. Build (this compiles the engine into the image, so the container never compiles):
-   `docker build -t strata .`
-   `docker build -t strata --build-arg CUDA_ARCHITECTURES=89 .` builds for one card only (faster).
-   The default covers RTX 30 (86), RTX 40 (89), RTX 50 (120) and A-series (80); a card outside that
-   set needs a rebuild with its own arch. Add `--build-arg BUILD_VISION=0` to skip the image encoder.
-3. Run (the first start downloads the ~70 GB model, then starts; later starts go straight to serving):
-   `docker run --rm --gpus all -p 8080:8080 --ulimit memlock=-1 -v strata-data:/data strata`
+- **[Coder](docs/MODELS.md#coder)** - a coding version with half of the experts removed: 91% of the full model's
+  SWE-bench Verified score (by its authors), fits 32 GB of RAM. Weaker outside code, including Chinese and other
+  CJK text (#438): for those, take Q2_0, IQ2_XS or IQ3_S, which keep every expert.
+- **[Swift 1.5](docs/MODELS.md#swift-15)** - a fine-tune that thinks much shorter before it answers, so you get the
+  answer sooner, at about the same quality.
+- **[Unsloth UD-Q4_K_XL](docs/MODELS.md#unsloth-ud-q4_k_xl-experimental)** (experimental) - the closest to the full
+  model, but most of it is read from the SSD while it answers: 7-8.5 tokens/s on a 64 GB PC.
+- **[OrcaRouter's Uncensored IQ3_XXS](docs/MODELS.md#orcarouter-uncensored-iq3_xxs)** - a manual setup, not in the
+  installer's menu.
 
-   The setup choices are env vars: `-e MODEL=IQ2_XS -e FAMILY=qwen -e CONTEXT=32768 -e VISION=no`
-   (or `MODEL=Q2_0|IQ3_XXS|IQ3_S`, `FAMILY=swift|coder`; the defaults above are the recommended ones).
-   `-e VISION=cpu` keeps the image encoder on the CPU. `-e KV=int8|q4_0|k8v4` picks the KV cache
-   precision; `k8v4` is INT8 K with 4-bit V and keeps its KV in VRAM from 64K up.
-   Only the model files, the prepared pack, the MTP layer and the install config live in the
-   `strata-data` volume; the engine is part of the image. Switching between models already on the
-   volume needs no setup pass: `-e MODEL=Q2_0 -e FAMILY=coder` picks that model's config. Add
-   `-e REINSTALL=1` only to change settings for a model already set up (context, vision, KV, host,
-   api_key, LOW_RAM), since those are recorded in its config.
-   Strata loads 32-62 GB into RAM. `--gpus all` on a host with two usable cards takes both: the
-   layer split is setup's recommended default ([docs/MULTI_GPU.md](docs/MULTI_GPU.md)), and a volume
-   set up for one card switches to the pair on its first start there. Pin one card with `-e GPU=0`,
-   or name them with `-e GPUS=0,2` and where the later card's layers start with `-e LAYER_SPLIT=18`.
-   A memory limit needs `-e LOW_RAM=on`, which maps the model's experts from the pack instead of
-   keeping them in RAM: setup.py measures the host's RAM, not the container's limit, so it cannot
-   see a cap. LOW_RAM runs on one card.
-   The server listens on `0.0.0.0:8080` by default; set `-e API_KEY=<secret>` before exposing the port
-   to a network. The image has a `HEALTHCHECK` on `/health`, so `docker ps` shows the container
-   healthy once the model is loaded, and `GET /v1/status` says what it is running.
+Sizes, downloads and what fits where: [docs/MODELS.md](docs/MODELS.md). You can add another model later with
+`SETUP.bat` (Linux: `./setup.sh --setup`).
 
 ## Using it
 
 <p align="center"><img src="docs/media/runpagoda.png" width="900" alt="The Strata app's Monitor tab next to a coding agent"><br>
 <sub>The Strata app's <b>Monitor</b> (left) while a coding agent writes the pagoda garden from the video (right)</sub></p>
 
-- **In the browser:** `http://127.0.0.1:8080` - the Strata app (it opens by itself when the model starts): **Chat**, a
-  live **Monitor** of the model and your GPU/CPU/RAM, and **About** with the settings and addresses.
-- **Chat in the terminal:** `.venv\Scripts\python chat.py`
-- **Your apps and coding agents:** add it as an "OpenAI-compatible" provider with base URL
-  **`http://127.0.0.1:8080/v1`**, any API key and any model name. Apps that use Anthropic's API: `http://127.0.0.1:8080/v1/messages`.
-- **Thinking:** the model thinks before it answers. Choose **off, low, medium or high** - in the chat page menu, with
-  `/think low` in `chat.py`, or with your app's "reasoning effort" setting. Off is fastest; high is best for hard questions.
-- **Pictures:** in the chat page click **Picture**; in `chat.py` type `/image <path>`; in apps just attach them.
-- **From your phone or another PC:** `START-HERE.bat --setup --host 0.0.0.0 --api-key <secret>`, then open the
-  address the server window prints; see the [details](docs/DETAILS.md#using-it).
-- **Experimental speed projection (off by default):** an experimental control vector that setup can turn on; it
-  changes how the model answers - read [what it does](docs/DETAILS.md#experimental-speed-projection-experimental-off-by-default) first.
+- **In the browser:** `http://127.0.0.1:8080` - **Chat**, a live **Monitor** of the model and your GPU/CPU/RAM, and
+  **About** with the settings and addresses.
+- **Your apps and coding agents:** add an "OpenAI-compatible" provider with base URL **`http://127.0.0.1:8080/v1`**,
+  any API key and any model name. Apps that use Anthropic's API: `http://127.0.0.1:8080/v1/messages` (Claude Code:
+  `ANTHROPIC_BASE_URL=http://127.0.0.1:8080`).
+- **Thinking:** choose **off, low, medium or high** in the chat menu or your app's "reasoning effort". Off is
+  fastest; high is best for hard questions.
+- **Pictures:** say yes to "Images?" in setup, then click **Picture** in the chat, or attach them in your app
+  (AMD cards: on Linux through the processor, not on Windows yet).
+- **From your phone or another PC:** `START-HERE.bat --setup --host 0.0.0.0 --api-key <secret>` - always with a key.
+- **Good to know:** it answers one request at a time. The first message of a chat is read in full (about 1 minute
+  per 30,000 tokens); follow-ups start in seconds.
 
-**Good to know:** it answers one request at a time. The first message of a chat is read in full (about 1 minute per
-30,000 tokens); after that it keeps the conversation and reads only what is new, so follow-ups start in seconds.
-
-### Where things are stored
-
-- **Your chats: only in your browser.** The Chat tab keeps the conversation, its settings and the API key you typed
-  in the browser's local storage (`strata.*` keys) - not on the server and not in the Strata folder. Pictures are not
-  kept, only their names. Another browser or a private window starts empty; clearing the site's data deletes them.
-- **How the model starts:** `strata-<model>.json` in the Strata folder (context, GPUs, host, API key, ...), written
-  by setup; next to it `run-<model>.bat` / `.sh`, the log `strata-<model>.log` and, when you use "Use for other
-  apps too", `strata-<model>.shared-settings.json`.
-- **The model files** (`models/`, `packs/`, `mtp/`, 70-120 GB): in **`Strata-data` next to the Strata folder**, or
-  wherever `--data-dir` put them.
-- **Where that data folder is:** `%APPDATA%\Strata\settings.json` on Windows, `~/.config/strata/settings.json` on
-  Linux ([details](docs/DETAILS.md)).
+More: [where your chats are stored](docs/INSTALL.md#where-things-are-stored), [the API](docs/DETAILS.md#using-it).
 
 ## Something went wrong?
 
-**My PC froze, or got very slow, the first time Strata started.**
-That's normal while it starts, most of all the first time. Strata loads 35-55 GB into your RAM, locks part of it for
-the graphics card, and works out how much of the model fits on your GPU. The mouse can freeze for a few minutes. **Wait, and don't close the
-window.** The next starts are much faster. Still frozen after 10 minutes? Restart the PC, close other programs
-(browsers use a lot of RAM) and try again. If it keeps happening, pick a smaller size (Q2_0 or IQ2_XS).
+- **My PC froze the first time Strata started.** Normal while it loads the model: wait, don't close the window.
+  Still frozen after 10 minutes? Restart the PC, close other programs and try again, or pick a smaller size.
+- **It stopped while downloading or installing.** Run `START-HERE.bat` (or `./setup.sh`) again: it continues where
+  it stopped.
+- **It's very slow and the disk light keeps blinking, or "the engine stopped unexpectedly".** Not enough free RAM:
+  close other programs (browsers use a lot), or pick a smaller size (Q2_0 or IQ2_XS).
+- **It says port 8080 is already in use.** Strata is already running - look for its window.
 
-**It stopped while downloading or installing.**
-Run `START-HERE.bat` again. It continues where it stopped.
-
-**It says the NVIDIA driver is too old.**
-Update it (NVIDIA App or [nvidia.com/drivers](https://www.nvidia.com/drivers)), restart the PC, and run
-`START-HERE.bat` again.
-
-**It says port 8080 is already in use.**
-Strata is already running. Look for its window.
-
-**It's very slow and the disk light keeps blinking.**
-Your PC is out of free RAM. Close other programs, or pick a smaller size (Q2_0 or IQ2_XS).
-
-**An answer stopped with "the engine stopped unexpectedly".**
-Usually not enough RAM (on Linux the system then stops the engine). Just send your message again: Strata starts the
-engine by itself. If it keeps happening, close other programs or pick a smaller size.
-
-**It says the prompt exceeds the context.**
-The conversation is longer than the context you chose. Start a new chat, or run `SETUP.bat` and pick more
-context.
-
-**Still stuck?** Look in the [full troubleshooting table](docs/DETAILS.md#troubleshooting), or open an issue and
-attach `strata-<model>.log` from the Strata folder.
+More problems and their fixes: [docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md). Still stuck? Open an
+[issue](https://github.com/Niko1221/Strata/issues) and attach `strata-<model>.log` from the Strata folder.
 
 ## How does it work?
 
 Models like this one normally run on servers with hundreds of gigabytes of graphics memory. Your graphics card has
-12-24 GB. Strata makes it fit by **sharing the work across your whole PC** - the same idea as a kitchen, where the
-things you use all the time stay on the counter and the rest waits in the pantry.
+12-24 GB. Strata makes it fit by **sharing the work across your whole PC** - like a kitchen, where the things you use
+all the time stay on the counter and the rest waits in the pantry.
 
 <p align="center"><img src="docs/media/how-it-works.svg" width="860" alt="The model's 24,576 experts: the busiest on the graphics card, all of them in RAM, a lookup table on the SSD"></p>
 
-- **The model is a team of 24,576 small specialists ("experts"),** and each word it writes needs only 10 of them.
-  So it doesn't have to have all of them on the graphics card at once.
-- **Your graphics card** does the part of the work needed for every word, and keeps the few thousand experts that
-  are asked most often. It keeps learning which ones those are while you use it.
-- **Your RAM** holds every expert. When a word needs one the card doesn't have, **your processor** works on it -
-  at the same time as the graphics card, so neither waits for the other.
-- **Your SSD** holds a big lookup table; the model only reads a few small rows of it per word.
+- **The model is a team of 24,576 small specialists ("experts"),** and each word needs only 10 of them.
+- **Your graphics card** keeps the few thousand experts that are asked most often; **your RAM** holds all of them,
+  and **your processor** works on the rest at the same time. **Your SSD** holds a big lookup table.
 
 <p align="center"><img src="docs/media/guess-and-check.svg" width="860" alt="A small helper guesses the next words; the big model checks them all at once and keeps the right ones"></p>
 
-- **Guess, then check.** A small, fast helper built into the model guesses the next few words, and the big model
-  checks all the guesses in one go. It keeps the ones it agrees with and writes the next word itself - so one step
-  often produces several words. The helper only guesses - the big model decides every word - so you get the same
-  quality answer, 1.6-1.8x sooner.
-- **Long texts are read in big pieces** (up to 8,192 tokens - pieces of words - at a time), which is why a long
-  document or code base is read at over 1,000 tokens per second.
+- **Guess, then check:** a small helper guesses the next few words and the big model checks them all at once, so
+  you get the same answer, 1.6-1.8x sooner.
+- **Long texts are read in big pieces** (up to 8,192 tokens at a time): over 1,000 tokens per second.
 
-Want the full picture? The [details](docs/DETAILS.md#how-it-works) explain every part and its numbers, and the
-[paper](docs/paper/Strata-Paper.pdf) tells the whole story, with the measurements behind it.
+The longer explanation: [docs/HOW_IT_WORKS.md](docs/HOW_IT_WORKS.md). Every part and its numbers: [the
+details](docs/DETAILS.md#how-it-works) and the [paper](docs/paper/Strata-Paper.pdf).
 
-## Credits
+## Credits and license
 
-- Model: [Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next) by the Qwen team; compressed versions by
-  [ISTA-DASLab](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF);
-  [Swift 1.5](https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF) by UkisAI; the experimental
-  [UD-Q4_K_XL](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF) by Unsloth (its support follows
-  [eddoursul/Strata](https://github.com/eddoursul/Strata)). Their licenses apply to the model files.
-- Built with parts of [llama.cpp / ggml](https://github.com/ggml-org/llama.cpp) (MIT). Ideas from
-  [Splash](https://github.com/incoai/splash), [ninfer](https://github.com/Neroued/ninfer) and
-  [HyperQwen](https://github.com/syv-ai/HyperQwen). More in the [details](docs/DETAILS.md#credits-and-licenses).
+The model is [Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next) by the Qwen team, compressed by
+[ISTA-DASLab](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF), UkisAI (Swift 1.5) and Unsloth;
+Strata is built with parts of [llama.cpp / ggml](https://github.com/ggml-org/llama.cpp). All credits:
+[docs/HOW_IT_WORKS.md](docs/HOW_IT_WORKS.md#credits). Strata is open source under the [MIT License](LICENSE); a few
+parts and every model carry their own licenses ([which ones](docs/HOW_IT_WORKS.md#license)).
 
-## License
+## Support Strata
 
-Strata is open source under the [MIT License](LICENSE). A few parts carry their own licenses: `third_party/ggml`
-(MIT, llama.cpp / ggml), the web app's font (SIL Open Font License 1.1) and the experimental speed projection's
-vector in `data/experimental-speed-projection` (Qwen Community License 1.0, from the model's activations). The
-models are not part of this repository; each model's own license applies to its files.
+Strata is free and open source. If it is useful to you, you can support its development:
+
+<p align="center"><a href="https://buymeacoffee.com/strataengine"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" height="50"></a></p>

START-HERE.bat

@@ -1,6 +1,6 @@
 @echo off
 rem Strata for Windows: the first run installs everything and starts the model; later runs just start it.
-rem Needs only an NVIDIA driver. Python is installed for your user account if it is missing (no admin needed).
+rem Needs only an NVIDIA or AMD graphics driver. Python is installed for your user account if it is missing (no admin needed).
 setlocal
 title Strata
 cd /d "%~dp0"

UPDATE.bat

@@ -0,0 +1,36 @@
+@echo off
+rem Update Strata without starting the model (#475): the newest code (git pull, when this folder is a git clone), then
+rem what START-HERE.bat does before a start - the engine (a new one when this version needs it), the Python packages,
+rem each installed model's settings and draft subset. The model files are not touched. Start the model later with
+rem START-HERE.bat. Options are passed on to setup.py.
+setlocal
+title Strata - update
+cd /d "%~dp0"
+rem All of it in one block: cmd reads a .bat file while it runs it, and the git pull can change this very file.
+(
+  if exist ".git" (
+    where git >nul 2>nul || (
+      echo  This folder is a git clone, but git is not on PATH: install Git for Windows ^(winget install Git.Git^)
+      echo  or run "git pull" here yourself, then run UPDATE.bat again.
+      pause
+      exit /b 1
+    )
+    echo  Getting the newest Strata ^(git pull^) ...
+    git pull --ff-only || (
+      echo.
+      echo  git pull did not succeed ^(the reason is above^): nothing was updated. Files you changed here can stop it:
+      echo  "git status" lists them.
+      pause
+      exit /b 1
+    )
+  ) else (
+    echo  This copy of Strata was not made with git, so it cannot fetch new files itself. Download the newest one:
+    echo    https://github.com/Niko1221/Strata/archive/refs/heads/main.zip
+    echo  unzip it anywhere and run START-HERE.bat ^(or UPDATE.bat^) in it: it finds the model files in Strata-data
+    echo  and sets itself up the same way - nothing big is downloaded again.
+    echo  Checking this copy's engine and settings meanwhile ...
+  )
+  call "%~dp0START-HERE.bat" --update %*
+  if not errorlevel 1 pause
+  exit /b
+)

cmake/hip_backend.cmake

@@ -12,8 +12,14 @@ endif()
 set(_strata_hip_validated gfx1100 gfx1201)
 set(_strata_hip_community gfx1101 gfx1200)
 set(_strata_hip_unvalidated gfx1102 gfx1030)
+# CMake hands HIP a ';' list, but a -DCMAKE_HIP_ARCHITECTURES typed by hand (or ROCm's own Windows tooling) may use
+# spaces, which foreach(IN LISTS) would otherwise treat as one element.
+string(REPLACE " " ";" _strata_hip_norm "${CMAKE_HIP_ARCHITECTURES}")
 set(STRATA_HIP_ARCH_LIST "")
-foreach(_arch IN LISTS CMAKE_HIP_ARCHITECTURES)
+foreach(_arch IN LISTS _strata_hip_norm)
+  if(_arch STREQUAL "")
+    continue()
+  endif()
   string(REGEX REPLACE ":.*$" "" _base "${_arch}")      # gfx1100:xnack- -> gfx1100
   if(_base IN_LIST _strata_hip_validated)
   elseif(_base IN_LIST _strata_hip_community)
@@ -62,11 +68,19 @@ target_include_directories(strata_hip_runtime BEFORE INTERFACE
   "${STRATA_HIP_COMPAT_INCLUDE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/include")
 target_compile_definitions(strata_hip_runtime INTERFACE STRATA_USE_HIP=1 "STRATA_HIP_ARCHS=\"${STRATA_HIP_ARCHS}\"")
 target_link_libraries(strata_hip_runtime INTERFACE hip::host)
-foreach(_language IN ITEMS CXX HIP)
+# The shim renames the CUDA runtime to HIP, force-included into every host and device source. On Windows the host
+# compiler is ROCm's clang++ too (tools/hip/build_windows.bat: CMake refuses to mix cl.exe with Clang HIP), which takes
+# -include like it does on Linux; an MSVC-style front end (cl / clang-cl) takes /FI instead. Forward slashes, so /FI
+# does not read the path's backslashes as escapes.
+file(TO_CMAKE_PATH "${STRATA_HIP_COMPAT_INCLUDE_DIR}/cuda_runtime.h" _strata_hip_force)
+if(MSVC)
+  target_compile_options(strata_hip_runtime INTERFACE "$<$<COMPILE_LANGUAGE:CXX>:/FI${_strata_hip_force}>")
+else()
   target_compile_options(strata_hip_runtime INTERFACE
-    "$<$<COMPILE_LANGUAGE:${_language}>:-include>"
-    "$<$<COMPILE_LANGUAGE:${_language}>:${STRATA_HIP_COMPAT_INCLUDE_DIR}/cuda_runtime.h>")
-endforeach()
+    "$<$<COMPILE_LANGUAGE:CXX>:-include>" "$<$<COMPILE_LANGUAGE:CXX>:${_strata_hip_force}>")
+endif()
+target_compile_options(strata_hip_runtime INTERFACE
+  "$<$<COMPILE_LANGUAGE:HIP>:-include>" "$<$<COMPILE_LANGUAGE:HIP>:${_strata_hip_force}>")
 
 # CMake does not infer HIP from Strata's existing CUDA-shaped .cu suffixes.
 file(GLOB_RECURSE _strata_hip_sources CONFIGURE_DEPENDS

docs/AI_SETUP.md

@@ -0,0 +1,196 @@
+# Installing Strata for a user (instructions for AI agents)
+
+This page is written for an AI coding assistant (Claude Code, Cursor, Codex, GitHub Copilot, ...) that a user asked to
+set up Strata on their PC. Strata runs the Qwen3.8-Flash-Next model locally on one NVIDIA or AMD graphics card plus
+system RAM, and serves an OpenAI- and Anthropic-compatible API on `http://127.0.0.1:8080`. The human-oriented
+version of everything below is [INSTALL.md](INSTALL.md).
+
+Work through the steps in order. Tell the user what you are doing in plain words; they may not be technical.
+
+## 0. Ground rules
+
+- **Never expose the server without an API key.** Keep the default `--host 127.0.0.1`. Only if the user asks for
+  access from other devices, use `--host 0.0.0.0` **together with** `--api-key <a long random secret>`, and give the
+  user the key. Never put a tunnel or port forward in front of a server without a key.
+- Do not change the user's system beyond what setup does (setup installs Python for the user account if needed,
+  and everything else inside the Strata folder and its `Strata-data` folder). Ask before installing drivers.
+- The model download is ~70 GB (up to 111 GB for Unsloth's 4-bit). Confirm the user is fine with that before you
+  start, especially on a metered connection.
+- Setup is long-running (an hour or more on a slow connection). Run it in the background or with a long timeout and
+  poll its output; do not kill it because it is quiet for a while. It is resumable: running the same command again
+  continues where it stopped.
+
+## 1. Check the PC
+
+The quickest way is setup's own check, which prints the GPU(s), driver, RAM, CPU, free disk and whether this PC can run
+Strata, and installs nothing except Python and the `.venv` it needs to run (step 2 gets the repository first):
+
+```
+Windows:  START-HERE.bat --check
+Linux:    ./setup.sh --check
+```
+
+To check by hand:
+
+| What | Windows (PowerShell) | Linux |
+| --- | --- | --- |
+| OS | `[Environment]::OSVersion` | `cat /etc/os-release` |
+| NVIDIA GPU, VRAM, driver | `nvidia-smi --query-gpu=index,name,memory.total,driver_version --format=csv` | the same |
+| AMD GPU | `Get-CimInstance Win32_VideoController \| Select-Object Name, AdapterRAM` (AdapterRAM is capped at 4 GB; trust the model name) | `lspci \| grep -i -E 'vga\|display'`; VRAM: `cat /sys/class/drm/card*/device/mem_info_vram_total` (bytes); `rocm-smi` only if ROCm is installed |
+| RAM | `(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB` | `free -g` |
+| Free disk | `Get-PSDrive -PSProvider FileSystem` | `df -h .` |
+
+Requirements (details: [INSTALL.md](INSTALL.md#what-you-need)):
+
+- **GPU:** NVIDIA RTX 20, 30, 40 or 50 series, or AMD Radeon RX 7900 XT / XTX, RX 7800 XT / 7700 XT, RX 9060 XT,
+  RX 9070 / 9070 XT, Radeon AI PRO R9700, RX 6800 / 6900 series; 12 GB of VRAM or more (an NVIDIA card with 8 GB runs,
+  slowly). GTX 10 series and older, and integrated GPUs, are not supported.
+- **Driver:** NVIDIA 580 or newer. AMD on Linux: the kernel's amdgpu driver; on Windows: a current AMD Adrenalin
+  driver. If the driver is missing or too old, tell the user to update it (NVIDIA App / nvidia.com/drivers, or AMD
+  Software) and restart; do not install drivers yourself unless they ask.
+- **RAM:** 32 GB or more (see step 3). **Disk:** ~80 GB free, ideally on an NVMe SSD. **CPU:** x86-64 with AVX2.
+- **OS:** Windows 10/11 or Linux (Ubuntu 22.04/24.04 are fully automatic).
+
+If the PC does not meet them, say which part is missing and stop.
+
+## 2. Get Strata
+
+```
+git clone https://github.com/Niko1221/Strata.git
+```
+
+Without git: download https://github.com/Niko1221/Strata/archive/refs/heads/main.zip and unzip it. Pick a drive with
+enough free space: the model files go to a `Strata-data` folder **next to** the Strata folder (or `--data-dir <path>`
+on another drive).
+
+## 3. Pick the model
+
+By the PC's RAM (ask the user whether they mainly want it for code - then the Coder is a good pick at any RAM size):
+
+| RAM | `--family` / `--model` | Notes |
+| --- | --- | --- |
+| 32 GB | `--family coder` (size IQ1_M) | for code; with a 24 GB card Q2_0 and IQ2_XS also run (setup picks the low-RAM mode) |
+| 48 GB | `--family qwen --model IQ2_XS` | or `Q2_0` (fastest) |
+| 64 GB | `--family qwen --model IQ2_XS` (recommended) | `IQ3_XXS` / `IQ3_S` are slower and a bit better |
+| 96 GB+ | `--family qwen --model IQ3_S` | `--family unsloth --model UD-Q4_K_XL` is experimental: NVIDIA only, NVMe SSD, 7-8.5 tokens/s on 64 GB |
+
+`--family swift` (Swift 1.5, a fine-tune that thinks shorter; sizes Q2_0, IQ2_XS, IQ3_XXS) is the alternative to
+`qwen`. With `--yes` and no `--model`, setup picks the recommended size for the RAM itself. More: [MODELS.md](MODELS.md).
+
+## 4. Run setup without questions
+
+From the Strata folder (Windows: in `cmd`, or `cmd /c START-HERE.bat ...` from PowerShell):
+
+```
+Windows:  START-HERE.bat --yes --family qwen --model IQ2_XS --no-start
+Linux:    ./setup.sh --yes --family qwen --model IQ2_XS --no-start
+```
+
+The flags (all of them: `START-HERE.bat --help`):
+
+| Flag | Meaning |
+| --- | --- |
+| `--yes` | take the recommended answer to every question (no prompts) |
+| `--family qwen\|swift\|coder\|unsloth` | the model version |
+| `--model Q2_0\|IQ2_XS\|IQ3_XXS\|IQ3_S\|IQ1_M\|UD-Q4_K_XL` | the size (the Coder is IQ1_M, Unsloth UD-Q4_K_XL) |
+| `--context N` | context in tokens; default by VRAM: 32768 under 14 GB, 65536 under 20 GB, else 131072 |
+| `--vision yes\|no\|gpu\|cpu` | read pictures; `--yes` leaves images off. AMD cards: `cpu` |
+| `--gpu N` / `--gpus 0,1` / `--gpus all` | one card, or several sharing the model (default: the card with the most VRAM) |
+| `--backend cuda\|hip` | NVIDIA or AMD engine; chosen by itself on a PC with only one kind of card |
+| `--data-dir PATH` | where the 70-120 GB of model files go |
+| `--port N` | the server port (default 8080) |
+| `--host H --api-key K` | listen beyond this PC; **only together with a key** |
+| `--no-start` | install only, do not start the server |
+| `--setup` | install another model or change settings of an installed one |
+| `--check` | only check the PC |
+
+`--no-start` is recommended for agents: the server runs in the foreground until its window is closed, which would
+block your shell. Start it separately in step 6.
+
+Notes:
+- **Linux:** setup uses `sudo apt` (or dnf/pacman) to install Python with venv if it is missing, and on AMD may need
+  `build-essential` and `git`. You cannot type the user's password: if a `sudo` step is needed, ask the user to run it
+  (e.g. `sudo apt install python3-venv build-essential git`) and then rerun setup.
+- **AMD on Linux:** setup installs ROCm into `.venv` (~10 GB, no sudo) unless a system ROCm 7 exists, and compiles the
+  engine for the card (10-20 minutes, once).
+- **AMD on Windows (new in 0.1.34):** setup downloads the ready-made AMD engine (~550 MB, ROCm included) - nothing
+  is compiled and only the AMD driver is needed. Before the model download it runs `engine\strata-device.exe
+  --list-devices`; if that does not list the card, the driver is the problem (tell the user to update AMD Software).
+  One card per model and no images on Windows for now. It is new: ask the user to report how it runs (docs/AMD_HIP.md).
+- **NVIDIA with no ready-made engine for the card:** setup offers to install build tools and compile (20-40 minutes);
+  `--yes` accepts.
+
+## 5. While it downloads, tell the user
+
+- It downloads about 70 GB from Hugging Face; how long depends on their connection (at 100 Mbit/s roughly 1.5-2
+  hours). It can be stopped and continues where it left off.
+- Then it prepares the model for their PC (a few minutes) and, on AMD on Linux, compiles the engine.
+- **When the model starts, the PC can be slow or stop responding for 1-3 minutes** (longest the first time): Strata
+  loads 35-55 GB into RAM and locks part of it for the graphics card. That is normal; they should not close the window.
+- Nothing leaves their PC: the model runs locally.
+
+## 6. Start the server
+
+Setup prints the start script it wrote (`start script: run-<model>.bat`). The name is the family tag plus the size,
+lower case: `run-iq2_xs`, `run-swift-iq2_xs`, `run-coder-iq1_m`, `run-unsloth-ud-q4_k_xl`.
+
+```
+Windows (PowerShell):  Start-Process -FilePath ".\run-iq2_xs.bat"            (opens its own window)
+Linux:                 nohup ./run-iq2_xs.sh > strata-server.out 2>&1 &
+```
+
+Or `START-HERE.bat` / `./setup.sh` without flags, which starts the installed model (it asks which one when several are
+installed; add `--yes` to take the first). The server opens the user's browser on `http://127.0.0.1:8080` when it is
+ready. Closing its window (or stopping the process) stops the model.
+
+## 7. Verify
+
+The HTTP server answers once the model is loaded (30-90 s on later starts, a few minutes the first time). Poll:
+
+```
+curl http://127.0.0.1:8080/health
+```
+
+It returns `{"status": "ok", "model": ..., "max_context": ..., "images": ..., "api_key": ..., "loaded": true, ...}`.
+Then:
+
+```
+curl http://127.0.0.1:8080/v1/models
+curl http://127.0.0.1:8080/v1/chat/completions -H "Content-Type: application/json" -d "{\"model\": \"strata\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in five words.\"}], \"max_tokens\": 64, \"reasoning_effort\": \"none\"}"
+```
+
+With an API key, add `-H "Authorization: Bearer <key>"`. The server window prints a `ready: http://127.0.0.1:8080/v1`
+line when it is ready; the engine log is `strata-<model>.log` in the Strata folder.
+
+## 8. Connect the user's apps
+
+- **Browser:** `http://127.0.0.1:8080` - Chat, a live Monitor, and About (settings and addresses).
+- **Any OpenAI-compatible app or agent:** base URL `http://127.0.0.1:8080/v1`, any API key (or the configured one),
+  any model name.
+- **Anthropic-compatible apps:** `http://127.0.0.1:8080/v1/messages`.
+- **Claude Code:** `ANTHROPIC_BASE_URL=http://127.0.0.1:8080`, `ANTHROPIC_MODEL` set to a Claude model name it knows
+  (Strata ignores the name), and any `ANTHROPIC_AUTH_TOKEN` (or the configured key).
+- **Thinking level:** `"reasoning_effort": "none" | "low" | "medium" | "high"` (default high).
+- Strata answers one request at a time. API details: [DETAILS.md](DETAILS.md#using-it).
+
+## 9. When something fails
+
+| Symptom | What to do |
+| --- | --- |
+| `the NVIDIA driver is too old` | The user updates the driver (NVIDIA App or nvidia.com/drivers) and restarts; rerun setup. |
+| Download or install stopped | Rerun the same setup command; it continues. |
+| `port 8080 is already in use` | Strata is already running (check `/health`), or use `--port 8081`. |
+| Very slow, disk busy, or "the engine stopped unexpectedly" | Not enough free RAM: close programs, or set up a smaller size (`--setup --model Q2_0` or IQ2_XS). |
+| `ExpertCache: cudaMalloc(...) failed: out of memory` with free VRAM (Windows) | The page file is off or tiny: set it to "System managed" and restart. |
+| `prompt ... exceeds the context` | Rerun setup with `--setup --context <bigger>`. |
+| No AMD GPU found (Linux) | The amdgpu driver lists no GPU; check the card and driver. Integrated GPUs are not supported. |
+| Python or build tools could not be installed | Install what the message names (links are printed), then rerun. |
+
+More: [TROUBLESHOOTING.md](TROUBLESHOOTING.md) and the [full table](DETAILS.md#troubleshooting). If it still fails,
+collect `strata-<model>.log` and the setup output, and suggest an issue at https://github.com/Niko1221/Strata/issues.
+
+## Alternative: the MCP server
+
+Strata also has an MCP server, so an AI tool can check, install, start and stop Strata through tool calls
+(`strata_status`, `strata_models`, `strata_install`, `strata_start`, `strata_stop`, `strata_logs`) instead of the
+shell commands above. See [MCP_SERVER.md](MCP_SERVER.md).

docs/AMD_HIP.md

@@ -1,10 +1,12 @@
-# Experimental AMD HIP backend (gfx1100, gfx1101, gfx1200, gfx1201, gfx1030)
+# AMD Radeon: the HIP backend (gfx1100, gfx1101, gfx1200, gfx1201, gfx1030)
 
-This is a Linux source build for the RX 7900 XT / XTX (RDNA3, gfx1100) and the
+Strata runs on AMD Radeon cards through its HIP backend, the same engine as on NVIDIA compiled for AMD. This page
+covers the build on Linux (on Windows a ready-made engine, see [Windows](#windows)) for the RX 7900 XT / XTX (RDNA3, gfx1100) and the
 RX 9070 / 9070 XT / Radeon AI PRO R9700 (RDNA4, gfx1201; see [RDNA4](#rdna4-gfx1201)). The RX 7800 XT / 7700 XT
 (gfx1101) and the RX 9060 XT (gfx1200) were validated by their owners (see [Community-validated
-cards](#community-validated-cards)); the RX 6800 / 6900 series (RDNA2, gfx1030) builds and runs too, reported by a community machine and not yet validated by the maintainers (see [RDNA2](#rdna2-gfx1030)). It is opt-in; the NVIDIA installer and CUDA build remain the default. Other AMD
-architectures, wave64, Windows HIP, and mixed AMD/NVIDIA execution are outside this contribution.
+cards](#community-validated-cards)); the RX 6800 / 6900 series (RDNA2, gfx1030) builds and runs too, reported by a community machine and not yet validated by the maintainers (see [RDNA2](#rdna2-gfx1030)). Setup chooses it by itself on a PC with no NVIDIA card Strata can use (`--backend hip` on a PC with both); the
+install steps for users are in [INSTALL.md](INSTALL.md#amd-cards). Other AMD architectures, wave64, and mixed
+AMD/NVIDIA execution in one run are not supported.
 
 The backend maps the CUDA-shaped runtime and BLAS calls to HIP/hipBLAS, uses
 RDNA2/RDNA3/RDNA4's signed integer dot instruction for quantized kernels, and supplies
@@ -49,6 +51,71 @@ the kernel's amdgpu driver (no ROCm install needed):
 
 The rest of setup is the same as on NVIDIA: the model download, the start script, the server.
 
+## Windows
+
+Since 0.1.34 an AMD card on Windows is set up like an NVIDIA one: download Strata, double-click `START-HERE.bat`.
+On a PC with no NVIDIA card Strata can use, the AMD card is chosen by itself; with both, setup asks
+(`START-HERE.bat --backend hip` picks AMD directly).
+
+- **You need:** Windows 10 or 11 (64-bit), one of the cards above, and a current AMD driver ([AMD Software:
+  Adrenalin Edition](https://www.amd.com/en/support/download/drivers.html)). Nothing else: no ROCm or HIP SDK
+  install, no compiler, no admin rights.
+- **Detection:** setup reads the display adapters Windows lists (their PCI ids; the VRAM size from the display
+  driver's registry entry). An integrated Radeon is listed as not supported.
+- **Engine:** the ready-made `strata-windows-x64-hip.zip` from the release (built by `tools\hip\build_windows.bat`
+  for gfx1100, gfx1101, gfx1102, gfx1200, gfx1201 and gfx1030) goes into `engine\`. It carries the ROCm libraries the
+  engine loads (`engine\rocm\bin`: the HIP runtime, hipBLAS / rocBLAS / hipBLASLt with their kernels for these cards,
+  amd_comgr and the Microsoft C++ runtime; ROCm 10.2.0a20260930 from AMD's TheRock builds, licenses in
+  `engine\rocm\licenses`). The HIP runtime works through the AMD driver's own components, so the driver is the one
+  thing it needs from the PC.
+- **The HIP runtime next to `strata.exe` (0.1.35, #468 #461):** `amdhip64_7.dll` and `amd_comgr.dll` are also put in
+  `engine\` (setup copies them there on every start). Windows looks in the program's folder before System32, where
+  some AMD drivers install their own `amdhip64_7.dll`; with that one, the bundled libraries crashed on the first
+  prompt (an access violation, or `hipErrorInvalidDeviceFunction`). The engine's log names the runtime it loaded
+  (`strata generate: HIP runtime ...`).
+- **Before the ~60 GB model download** setup runs `engine\strata-device.exe --list-devices` (with `engine\rocm\bin` on
+  the PATH): if the HIP runtime does not see the card, setup stops there and points to the driver. It also gives the
+  card's HIP number: with an integrated Radeon that is device 1, not 0 (#325). From then on setup lists the AMD cards
+  as HIP numbers them, so `--gpu N` and the config's `"gpu"` are HIP numbers.
+- **Differences from Windows-on-NVIDIA and Linux-on-AMD:** no images yet (the CPU image encoder is Linux-only for
+  now), one card per model (`--gpus` is Linux-only for now), no calibration.
+- Two Windows-only engine details (#247, #325): hipBLAS can return success and still leave `hipErrorInvalidValue`
+  set after some BF16/FP16 GEMMs (seen on gfx1201); the engine clears that one stale error after a GEMM that
+  succeeded, on Windows only. `hipHostGetDevicePointer` returns the host pointer itself on Windows: kernels read
+  mapped memory through it correctly, but a device-to-device copy into it does not land, which is why
+  `tests/hip/handoff` times out there (the engine does not use that copy; `tests/hip/mapped_alias` reports it).
+- Two more (#380, #377, by BlueKingMuch, measured on an RX 6800 that drives the desktop): `hipMemGetInfo` on Windows
+  does not subtract what the desktop and other programs hold on the card, so `--expert-cache auto` filled the card
+  past what Windows keeps in VRAM and decode fell from 41 to 30 tok/s. The engine now lowers that free figure by
+  what Windows' video memory budget for the process withholds (logged once: `strata: Windows budgets N of this
+  card's M MiB ...`; `STRATA_WDDM_BUDGET=0` turns it off). And the PCIe probe times its copies on the host clock
+  there, since HIP's events read impossible speeds (3,300-26,000 GB/s), so a slow link now gets a smaller
+  `pcie_frac` as on NVIDIA.
+
+**What is validated (0.1.34):** #325's author ran the engine of this port on an RX 9070 XT (Windows 11, ROCm
+10.2.0a20260930 in `.venv`, compiled on the PC): Coder IQ1_M at 32K, 29.2 tok/s decode, ~181 tok/s prefill,
+correct answers; ctest 42 of 46. The maintainers have no Windows AMD card: the release zip was built on an NVIDIA PC,
+and checked there on the Ryzen CPU's integrated Radeon (gfx1036, a test build of the same tree): with only the zip's
+libraries on the PATH, `strata-device` lists the card and a hipBLAS BF16 GEMM matches the CPU; the HIP ctest passes
+52 of 56, the 4 failures the same as on the RX 9070 XT (`hip_handoff`, and three tests that need a pack fixture).
+The ready-made zip itself has not run a model on a discrete card yet - please report.
+
+**Reporting a Windows AMD run** (an issue, or on #325): your card and driver version (AMD Software > System), then
+
+```bat
+engine\strata-device.exe --list-devices
+engine\strata-device.exe --selftest
+```
+
+(from the Strata folder, after `set PATH=%CD%\engine\rocm\bin;%PATH%`), the end of `strata-<model>.log`, and the
+speed lines the server window prints for a first answer.
+
+**Building it yourself:** `tools\hip\build_windows.bat` (Visual Studio 2022 Build Tools with the C++ workload, Python,
+git; no admin, no AMD GPU) installs ROCm from AMD's TheRock wheels into `.rocm-win`, builds, and packages
+`dist\strata-windows-x64-hip.zip`; `START-HERE.bat --backend hip --prebuilt dist\` installs that one.
+`tools\hip\build_windows.bat tests` also builds the HIP tests (`ctest` in `build-hip-win`, with
+`.rocm-win\Lib\site-packages\_rocm_sdk_devel\bin` on the PATH). `STRATA_HIP_ARCHS` picks other architectures.
+
 ## Build
 
 Requirements: a working ROCm driver/runtime, HIP development headers and
@@ -123,7 +190,12 @@ The worker count above was used on a 16-core CPU; measure it for your CPU.
 The 4K context is a smoke-test starting point, not a model limit. The expert cache
 sizes itself automatically and leaves 1 GiB of VRAM headroom.
 
-The installer supports this backend (see "Install with setup" above). The vision helper is NVIDIA-only for now.
+**The card also drives a Linux desktop (#560 #516):** keep more VRAM free than the default 700 MiB, e.g.
+`./setup.sh --vram-reserve-mib 3072`. With the cache filling the card, the desktop's next VRAM need makes amdgpu move
+GPU memory to system RAM (GTT), the OOM killer then ends KWin/plasmashell or `systemd-oomd` ends apps, or the
+compositor fails with "Failed to pin framebuffer with error -12".
+
+The installer supports this backend (see "Install with setup" above). Images run through the CPU encoder for now (`--vision cpu`).
 Setup installs one AMD card, or several with `--gpus` (the engine's layer split; see RDNA4 below).
 
 ## RDNA4 (gfx1201)
@@ -162,6 +234,10 @@ an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 39
   so none was shipped for it: there the plain hipBLAS path is already close. For hipBLASLt 1.5.0 (ROCm
   10.2.0a nightly) `tools/hip/gfx1201-hipblaslt-100500.txt` is shipped (see "Tuning table" below); on one R9700 it
   measured +3.9% prompt speed on 4,210-token prompts (1,590 vs 1,531 tok/s), a modest gain.
+  With the hipBLASLt 1.2.2 of a system ROCm 7.2.4 the plain path is far off, and `tools/hip/gfx1201-hipblaslt-100202.txt`
+  is shipped for it (24 of the gfx1100 table's 26 shapes; setup uses it only with that exact version): on an R9700
+  with the full IQ3_XXS, `--kv int8 --kv-resident 32768`, a fresh 32K prompt read at 638 -> 1,177 tok/s and a 7K one
+  at 656 -> 1,164 tok/s with it, decode unchanged.  `hip_prefill_hipblaslt_gemm` passes with it.
 - **Both cards in one run (layer split, engine 0.1.30):** the config's `"backend": "hip", "gpu": [1, 0]` (R9700
   first) runs through `serve/server.py` (setup writes it with `--gpus 1,0` since 0.1.31). Auto split put layers 0-27 on the R9700 and
   28-47 on the 9070 XT. With every expert on the GPUs the split gives exactly the tokens of the R9700 alone (4K and
@@ -183,13 +259,15 @@ an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 39
   (split: ~51), the tokens of the R9700 alone (docs/MULTI_GPU.md).
 - **Speed switches (engine 0.1.32, measured on the R9700 / 9070 XT with the Coder IQ1_M pack):**
   - the MoE router (`router_top10`) runs a HIP kernel without its serial FP64 sum and block barriers by default: the
-    same ids and weights bit for bit (`hip_router_fast` checks 65,536 rows), 39 -> 9-12 us per call, decode
-    62.4 -> 70.0 tok/s on the R9700 and +4% on the 9070 XT, the same greedy tokens (5 + 5 starts).
+    same ids and weights bit for bit (`hip_router_fast` checks 65,536 rows), 39 -> 9-12 us per call, the same greedy
+    tokens (5 + 5 starts). Decode: 62.4 -> 70.0 tok/s on the R9700 and +4% on the 9070 XT in one A/B here (ROCm 10.2
+    nightly); a user's repeated A/B/A/B with setup's TheRock 7.10 wheels and an i5-12600K measured +1-4%, inside a
+    12-20% run-to-run spread (#432) - how much it gains depends on how much of decode the router is on that setup.
     `STRATA_HIP_ROUTER_OLD=1` runs the portable kernel.
   - `STRATA_HIP_WMMA=1` (opt-in, gfx12 only, int8 KV): the prompt path's QSA attention on RDNA4 matrix cores
     (`v_wmma_f32_16x16x16_f16`, FP16 hi + lo halves like the CUDA tensor-core kernel; `hip_prompt_attn_wmma` bounds it
     against the FP32 kernel and FP64). 7.2-7.5x the portable kernel; R9700 prompts 4K 1,784 -> 2,427 tok/s, 16K
-    1,797 -> 2,700. Not bitwise: greedy text differs from token ~50 on, as with the CUDA tensor-core attention. With it
+    1,797 -> 2,700 (a user with TheRock 7.10: +29-34%, #432). Not bitwise: greedy text differs from token ~50 on, as with the CUDA tensor-core attention. With it
     the prompt path's expert ring is 96 slots (as STRATA_PREFILL_RING=96): with the default 384 the 9070 XT's 4K
     prompts fell to 718 tok/s; with 96 they gain (1,017 -> 1,211; 16K 1,518 -> 2,032). PR #329
     (bsorensen110) contributed an equivalent gfx12 WMMA kernel of the same speed (within 1%); this one also masks KV
@@ -205,8 +283,9 @@ an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 39
 
 ## Community-validated cards
 
-Run by their owners, not on the maintainers' machines; setup accepts them like gfx1100 / gfx1201. No hipBLASLt table
-is shipped for them (make one with [Tuning table](#tuning-table) and compare the prompt speed with and without it).
+Run by their owners, not on the maintainers' machines; setup accepts them like gfx1100 / gfx1201. setup ships a
+hipBLASLt table for gfx1200 (the numbers below); gfx1101 has none (make one with [Tuning table](#tuning-table)
+and compare the prompt speed with and without it).
 
 - **gfx1101, RX 7800 XT 16 GB** (jhohertz, #254; engine 0.1.29, Ryzen 9 5950X, 121 GiB RAM, system ROCm with
   hipBLASLt 1.4.1): `./setup.sh --backend hip` detected the card and compiled the engine; `strata-device --selftest`
@@ -226,6 +305,17 @@ is shipped for them (make one with [Tuning table](#tuning-table) and compare the
   Greedy output was the same across runs. For comparison, llama.cpp's HIP build measured 20 tok/s decode and
   450 tok/s prompt on that card.
 
+  Since 0.1.38 setup ships that table (`tools/hip/gfx1200-hipblaslt-100202.txt`). On engine 0.1.31, one binary (ctest
+  37/37, without `ple_parity` and `platform_memory_test`), prompts of repeated code blocks, prefill 2560 for
+  the short prompt and 16384 with `--kv int8 --kv-resident 65536` otherwise, two runs each - prompt speed
+  with the table vs plain hipBLAS (no table):
+
+  | prompt | with the table | plain hipBLAS | gain |
+  |---|---|---|---|
+  | 2,374 tokens | 597-599 tok/s | 396-398 tok/s | 1.50x |
+  | 65,045 tokens | 754-757 tok/s | 380 tok/s | 1.99x |
+  | 130,091 tokens (the same flags) | 719 tok/s | 363-370 tok/s | 1.98x |
+
 ## RDNA2 (gfx1030)
 
 The RX 6800 / 6800 XT / 6900 XT / 6950 XT run the same kernels: wave32 and 64 KiB of LDS per workgroup. The one

docs/DETAILS.md

@@ -1,11 +1,13 @@
 # Strata - the details
 
 The technical side of Strata: every measured number, the API, images, all settings and how the engine works.
-New here? Start with the [README](../README.md) - it has everything you need to install and use it.
+New here? Start with the [README](../README.md); installing step by step is in [INSTALL.md](INSTALL.md), the models in
+[MODELS.md](MODELS.md), common problems in [TROUBLESHOOTING.md](TROUBLESHOOTING.md).
 
 > **On this page:** [Speed](#speed-measured) · [Other GPUs](#other-gpus-estimated) · [Which model?](#which-model) ·
 > [Requirements](#before-you-start) · [Windows](#windows) · [Linux](#linux) · [API](#using-it) ·
-> [MCP tools](#tools-from-mcp-servers) · [Images](#images-vision) ·
+> [MCP tools](#tools-from-mcp-servers) · [MCP server](#manage-strata-from-your-ai-assistant-mcp-server) ·
+> [Images](#images-vision) ·
 > [Troubleshooting](#troubleshooting) · [How it works](#how-it-works)
 
 ---
@@ -17,6 +19,15 @@ RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1
 MTP speculative decoding on. "262K" is the model's full context window (a 259,943-token prompt). The IQ2_XS row was
 measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
 
+**Engine 0.1.36 (#136), the same PC:** Q2_0's prompt experts run on fused int8 tensor-core kernels (RTX 30 and newer):
+4K 1,294 -> 1,570, 32K 2,170 -> 2,653, 128K 2,123 -> 2,468 tokens/s (+16-22%), as close to an FP16 reference as the
+previous kernels (closer at 32K: teacher-forced KL 0.009 vs 0.012). The decode path's block selection and greedy
+argmax run on thread-block clusters (RTX 50, sm_90+; other cards keep the previous kernels; the same tokens): Q2_0 output at 4K 89 -> 93.5, at 128K
+64.5 -> 76.4 tokens/s. `STRATA_PF_FUSED=0` keeps the previous prompt kernels (byte-identical answers to 0.1.35);
+`STRATA_PF_FUSED=1` also runs the native IQ packs' fused kernels (opt-in: IQ2_XS prompts +12% at 4K, +3% at 32K, the
+IQ3 packs about even); `STRATA_QSA_CLUSTER=0` / `STRATA_ARGMAX_MULTI=0` turn the decode kernels off. The tables
+below are 0.1.26's.
+
 ### Prompt processing (tokens/s)
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
@@ -50,14 +61,17 @@ accepted, so a different answer to the same prompt moves it by several percent.
 [`bench/results/2026-09-28-speed-0114`](../bench/results/2026-09-28-speed-0114/README.md).
 
 IQ3_XXS and IQ3_S at 262K are not measured: with their 43 / 50 GB of experts, a 260K-token context brings a 64 GB PC
-to its memory limit. Use up to 128K with them on 64 GB (setup caps it). IQ3_S (engine 0.1.4 or newer) is only published
-for the original model, not for Swift 1.5.
+to its memory limit by setup's estimate (the experts + the context's KV cache + 24 GB), so setup recommends up to 128K
+with them on 64 GB. A longer context you choose (`--context 262144`, or a pick in its list) is kept, with a note: users
+ran IQ3_S at 256K on 64 GB with RAM to spare (#406). In the low-RAM mode the KV cache stays in VRAM and the context
+does not count against RAM. IQ3_S (engine 0.1.4 or newer) is only published for the original model, not for Swift 1.5.
 
 **KV streaming (engine 0.1.5):** at 64K and more, setup keeps the context's KV cache in RAM and only the part the
 attention reads in VRAM (`--kv-resident 32768`), so more experts fit on the GPU. Q2_0 at 262K: 50.9 -> 62.6 tokens/s
 (1,589 -> 3,872 experts in VRAM); at 128K about +6%. The attention reads exactly the same values (only where the KV lives changes); it
 costs ~13.7 KB of RAM per context token (1.7 GB at 128K). Existing installs: run `START-HERE.bat --setup` once to turn
-it on.
+it on. Setup turns it on when the RAM has room for it; `--kv-streaming on|off` overrides that (on past the RAM test with a
+note; never with `--kv k8v4` or under WSL, which cannot stream).
 
 **4-bit KV cache (engine 0.1.8, optional):** `START-HERE.bat --setup` asks above 8K context (or pass `--kv q4_0`). It
 halves the KV cache's memory with a Hadamard rotation before 4-bit rounding (PR #21), about 4% faster at 128K, but it
@@ -76,7 +90,14 @@ slightly differently. How many tokens share an expert depends on the drafts in a
 at temperature 0 can end in a different (equally good) answer when the drafting, the cache state or a resumed
 conversation differ (issue #152). `STRATA_IQ_MT_MIN=1` (in the config's `env`) uses the multi-token kernels for
 every group: the answer then no longer depends on the drafting. Measured on a Ryzen 7600 (AVX-512): IQ3_S decode
--1..-3%, the other models the same; the default stays the fastest rule.
+-1..-3%, the other models the same; the default stays the fastest rule. Through the server, two more things carry
+over from one request to the next (#410): the adaptive tier moves experts between RAM and VRAM (the GPU and the CPU
+round an expert differently), and the prompt cache resumes a repeated prompt and reads only its tail through the
+decode path. For byte-identical repeats add `--prompt-cache 0 --adapt-swaps 0 --pcie-frac 0` to the engine's args
+as well (#410): the PCIe share of the missed experts (computed on the GPU instead of the CPU) still made the first
+answer after a start differ from the next ones. Measured here (IQ3_XXS, a 3.6K-token prompt, 4 repeats): with all
+three switches 1 answer of 4, without `--pcie-frac 0` 2 of 4 (the first one differs), with the defaults 2 of 4.
+`--pcie-frac 0` costs decode speed (the missed experts all run on the CPU), so keep it for A/B runs.
 
 **The draft layer's tokens (0.1.27, `--draft-vocab`):** the MTP draft layer can only propose tokens from a subset
 of the vocabulary (`mtp/rt/draft_vocab.bin`). Since 0.1.27 the subset includes every Chinese, Japanese and Korean
@@ -86,7 +107,10 @@ English/code subset from before (40,525 ids, ~110 MiB less VRAM, English answers
 almost no drafts). `--draft-vocab cyrillic` takes the English/code subset plus the whole Cyrillic script (58,963
 ids): the shipped subsets hold 142 of the vocabulary's 18,580 Cyrillic tokens, so Ukrainian or Russian answers got
 1.4 tokens a round; with it 2.1, and 83 -> 109 tokens/s (RTX 5090, the NVFP4 fork), English unchanged.
-`tools/draft_vocab.py` builds and inspects subsets.
+`tools/draft_vocab.py` builds and inspects subsets. When the start stops with "the draft head does not fit" (a
+12 GB card with a long context, #474), the engine says how much the head needs, how much VRAM is free and which
+smaller subset fits, and the server's start error repeats it; setup suggests `--draft-vocab en` on cards under
+14 GB (only a suggestion: nothing changes unless you pass it).
 
 **Low-RAM mode (engine 0.1.26, chosen by setup):** normally all of a model's experts are copied into RAM (23-50 GB,
 pinned) and the GPU holds a copy of the most-used ones. On a PC whose RAM cannot hold them beside the system (the
@@ -114,6 +138,12 @@ other ~18 GB), a 32 GB PC with a 12-16 GB GPU the Coder; IQ3_XXS on a 32 GB PC s
   not hold do not fit, it says so and runs the plain mapped mode. The server log shows, per request, how many expert
   reads went to the file (`resident RAM: ... blob reads from the file`: 0 in steady use).
 - `--low-ram resident|mmap` forces one variant (also on a PC with enough RAM, e.g. to try it).
+- Several GPUs (#364, #384): setup recommends one GPU in the low-RAM mode (the resident variant has no layer split
+  yet), and asks; `--gpus 0,1` (or answering 2) shares the model across them with the mapped variant
+  (`--mmap-experts`): the cards together hold more of the experts, and two users measured it 1.3-1.6x faster than
+  one card, but the OS file cache can fill the RAM to 0 free during long prompts. `--yes` keeps one GPU. A config
+  with `--resident-experts` started with `--gpus` switches to `--mmap-experts` with a note, and the engine runs that
+  pair as `--mmap-experts` with a warning instead of refusing it.
 
 **Low-RAM mode without `experts.bin` (engine 0.1.31):** for the native packs (IQ2_XS, IQ3_XXS, IQ3_S, the Coder, Swift,
 Q2_0 packed by `tools/iq_pack.py`; not the canonical Q2_0 pack setup makes for AVX-512 CPUs) the mapped mode no longer
@@ -128,7 +158,9 @@ engine fetches a layer's missing experts on 8 threads (`STRATA_FETCH_THREADS`) w
 **A RAM budget (engine 0.1.31, `--resident-budget-gib N`):** the resident variant for a model whose experts do not all
 fit: the N GiB of experts the GPU cache does not hold that the expert profile ranks hottest are copied into RAM at
 start (locked; page-locked when the driver allows the whole budget), and the rest are read from the files through the
-OS file cache. It implies `--mmap-experts` and leaves 4 GB of free RAM (a larger N is clamped, with a message). With
+OS file cache. It implies `--mmap-experts` and leaves 4 GB of free RAM (a larger N is clamped to that less 256 MiB,
+with a message; #403: a clamped budget no longer fails the safety check that follows, and a budget that cannot be
+kept at all is a warning, with every expert read from the files). Setup sets N with `--resident-budget-gib N`. With
 the GGUF read in place it also warms the next layer's likely experts: while the CPU works on a layer, a thread applies
 the next layer's router to this layer's input and asks the OS for the pages of the predicted experts that neither the
 GPU nor the RAM budget holds (only pages - the experts computed are the same; `STRATA_LOOKAHEAD=0` turns it off). This
@@ -139,7 +171,8 @@ RTX 5070, against ~3 tokens/s before these changes.
 RAM copy, blobs and MB from the files, the time spent reading them; `routing prefetch`: how many of the file reads had
 been warmed). The server log has the same per request (`expert tiers: GPU ... hits ...; RAM ... blobs, files ...
 blobs ... MB read`), and `GET /metrics` lists `ram_blobs`, `file_blobs` and `file_mb` for each recent request (with
-engine 0.1.31 or newer).
+engine 0.1.31 or newer). It also lists each request's speculative drafts, `drafts_offered` and `drafts_accepted`
+(`null` when the engine did not report them), and their sums since the server started in `totals` (#457).
 
 Time to first token is prompt length / prompt speed: with Q2_0 about 4 s at 4K, 25 s at 32K, under 2 minutes at 128K
 and 4.5 minutes at 262K (engine 0.1.13 made long prompts about twice as fast, below).
@@ -235,12 +268,13 @@ the positions of short greedy answers, 90-91% after a 16K prompt, differing most
 
 ## Before you start
 
-You need **only an NVIDIA driver** (version 580 or newer; update it with the NVIDIA App or from
-[nvidia.com/drivers](https://www.nvidia.com/drivers)). Everything else is installed for you the first time.
+You need **only a graphics driver**: NVIDIA 580 or newer (update it with the NVIDIA App or from
+[nvidia.com/drivers](https://www.nvidia.com/drivers)), or for AMD the one in [INSTALL.md](INSTALL.md#what-you-need).
+Everything else is installed for you the first time.
 
 | | |
 | --- | --- |
-| GPU | NVIDIA **RTX 20, 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070 and an RTX 3090; RTX 20 (Turing, since 0.1.27) was tested by a contributor on an RTX 2070. |
+| GPU | NVIDIA **RTX 20, 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070 and an RTX 3090; RTX 20 (Turing, since 0.1.27) was tested by a contributor on an RTX 2070. Or AMD **Radeon RX 7900 XT / XTX, RX 7800 XT / 7700 XT, RX 9060 XT, RX 9070 / 9070 XT, Radeon AI PRO R9700, RX 6800 / 6900 series**: [AMD_HIP.md](AMD_HIP.md). |
 | RAM | **64 GB** recommended (see the table above). |
 | CPU | x86-64 with AVX2 (any Intel/AMD desktop CPU from the last ~8 years). AVX-512 (Ryzen 7000/9000) is a bit faster. |
 | Disk | ~70-80 GB free for the model, ~6 GB for the MTP layer (+1 GB with images). **Q2_0 on an AVX-512 CPU** also writes a one-time ~40 GB copy of its experts for the fast CPU kernel. An NVMe SSD is strongly recommended. |
@@ -371,7 +405,8 @@ server, three server options (all off by default; also as keys in `strata-<model
 | `--min-free-vram-mib 11000` | `"min_free_vram_mib": 11000` | load an unloaded model only when that much VRAM is free (it waits up to 15 s for memory being given back), else answer **503** "the GPU is in use by another program" instead of starting into what a game left (with several GPUs it checks the first one) |
 | `--before-load "cmd"` | `"before_load": "cmd"` or `["cmd", "arg"]` | a command run before the model is loaded again, e.g. one that unloads another server's model |
 
-`POST /unload` unloads it now (`409` while a request is running) and `POST /load` loads it ahead of a request;
+`POST /unload` unloads it now (`409` while a request is running) and `POST /load` loads it ahead of a request (both
+with `Content-Type: application/json`, e.g. `curl -X POST -H "Content-Type: application/json" localhost:8080/unload`);
 `/health` says `"loaded"`, `/v1/models` lists it as `unloaded` (like llama.cpp's router), `/props` sets
 `is_sleeping` and the Monitor shows the state. Unloading ends the engine process - and the image encoder, when images
 are on; it is started again first, as at a start - so their VRAM and RAM go straight back. The model files stay in
@@ -379,6 +414,17 @@ the OS file cache, so loading again takes seconds while that RAM is not needed e
 16 GB with Q2_0 in the low-RAM mode: unloading takes ~0.3 s, and a request to an unloaded model answered after
 4.6 s (text) or 14.7 s (a picture, image encoder on the CPU).
 
+**Keep what the expert cache learned across restarts (opt-in, engine 0.1.36, #477):** a start fills the GPU's expert
+cache from the shipped profile, and the adaptive tier (`--adapt-every`) then moves in the experts your requests use.
+With `"expert_profile_save": "expert-profile-learned.bin"` in `strata-<model>.json` the engine saves that as a
+profile - the experts in VRAM first, then the routing it counted since the start, then the shipped order - on a
+clean exit and every 10 minutes between requests (`"expert_profile_save_every": 5` for another interval, `0` for
+exit only), written to a temporary file and renamed, so a crash never leaves half a file. The next start begins from
+it instead of the config's `--expert-profile` when it is a profile of the same model (else from the config's, as
+before). A relative path is in the Strata folder; one file per model, and a profile per project works the same way
+(point the key at another file). The file is a fingerprint of what you used the model for: it stays on your PC.
+Without the key nothing is counted or written. Setup rewrites the config when run again: add the key again then.
+
 ---
 
 ## Using it
@@ -437,6 +483,40 @@ print(r.choices[0].message.content)
   15 s, and `GET /status` says what it is doing (`reading the prompt`, `answering`, tokens so far). Closing the
   connection or pressing stop in your app really stops the model, so the next request starts at once.
 - **Chat apps.** Any app with an "OpenAI-compatible" provider works: base URL `http://127.0.0.1:8080/v1`, any API key.
+- **OpenCode** (#543). A starting point for `opencode.jsonc` (in your project, or `~/.config/opencode/`); the field
+  names are OpenCode's, so check its config docs if your version differs:
+
+  ```jsonc
+  {
+    "$schema": "https://opencode.ai/config.json",
+    "provider": {
+      "strata": {
+        "npm": "@ai-sdk/openai-compatible",
+        "name": "Strata (local)",
+        "options": { "baseURL": "http://127.0.0.1:8080/v1", "apiKey": "none" },  // or your api_key
+        "models": {
+          "strata": {
+            "name": "Qwen3.8-Flash-Next (Strata)",
+            // context: what you chose in setup; output: what one reply may use (prompt + output must fit)
+            "limit": { "context": 262144, "output": 32768 },
+            "options": { "reasoningEffort": "high" },                  // sent as reasoning_effort
+            "variants": {                                              // switch between them in OpenCode
+              "low": { "reasoningEffort": "low" },
+              "medium": { "reasoningEffort": "medium" },
+              "none": { "reasoningEffort": "none" }
+            }
+          }
+        }
+      }
+    },
+    "model": "strata/strata"
+  }
+  ```
+
+  Set `limit.context` to the context you chose in setup: OpenCode compacts the conversation before it gets there.
+  Keep `limit.output` well under it: a request whose prompt plus `max_tokens` runs past the context is refused (see
+  **Context** below), or add `"fit_max_tokens": true` to `strata-<model>.json`. For a hard cap on the thinking, add
+  `"reasoning_budget_tokens": N` to `strata-<model>.json` (see above).
 - **Claude Code** (Strata 0.1.17 or newer): set `ANTHROPIC_BASE_URL=http://127.0.0.1:8080` and
   `ANTHROPIC_MODEL` to a Claude model name it knows (it refuses names it doesn't; Strata ignores the name), plus any
   `ANTHROPIC_AUTH_TOKEN` (or your `api_key`, if you set one).
@@ -460,9 +540,30 @@ print(r.choices[0].message.content)
   clients then send it as their API key. Streamed answers carry `X-Accel-Buffering: no`, so nginx-style proxies pass
   each token on at once. The web app's settings and MCP tools only answer Strata's own page: when you open it through
   a proxy or tunnel whose address differs, add that address, e.g. `"trusted_origins": ["https://strata.example.com"]`.
+  With the key set, any `Host` name reaches the server (see Host names below).
 - **From web apps in a browser (CORS).** Off by default. `"cors_origins": ["https://chat.example.com"]` lets pages of
   those origins call `/v1/*` from the browser (Open WebUI's direct connections, browser extensions); `["*"]` lets any
   page do it - only sensible with an API key. It never opens `/settings`, `/unload` or the MCP tools.
+- **Host names (DNS rebinding).** A web page of another site can point its own name at `127.0.0.1` and then reach
+  this server as if it were its own, so without an API key the server answers only requests whose `Host` is a name
+  it knows (with a key the check is off: such a page cannot send the key, and tunnels and proxies that pass their
+  own name on keep working):
+  `localhost` (and `*.localhost`), any IP address (`127.0.0.1`, `[::1]`, `192.168.x.x`, ...), the address it
+  listens on and, when it listens beyond this PC (`0.0.0.0` or a LAN address), this PC's name (`mypc`, `mypc.local`)
+  and `host.docker.internal`; any port. Others get **403** naming the setting, and the server window prints one line
+  for each. Reach it under another name (a reverse proxy that keeps the name, a tunnel, a DNS name on your network,
+  another container's name for it)? Add the name: `"allowed_hosts": ["strata.example.com"]` in
+  `strata-<model>.json` or `STRATA_ALLOWED_HOSTS=strata.example.com` (comma-separated); `".example.com"` allows that
+  name and every name below it, and `["*"]` turns the check off (so does setting `api_key`). The hosts of
+  `trusted_origins` count as allowed. Requests without a `Host` header (HTTP/1.0 clients) pass.
+- **Web pages without an API key.** Without `api_key`, a `POST` to `/v1/*` that carries an `Origin` header (a
+  browser page sent it) is answered only for Strata's own page, pages on `localhost` or an allowed host name (any
+  port), the origins in `trusted_origins` or `cors_origins`, and browser extensions and desktop apps
+  (`chrome-extension://`, `moz-extension://`, `app://`: no web site can send those), and only with a JSON body; any
+  other page, and `Origin: null`, gets **403**. Clients that send no `Origin` (curl, the OpenAI and Anthropic SDKs,
+  other servers) are not affected. With
+  an API key, the key decides. `POST /unload` and `POST /load` take `Content-Type: application/json` from Strata's
+  own page (or no `Origin`), like `/settings`.
 
 **Conversation cache.** A request that continues a chat reads only the part after what the engine already holds: the
 live session, or one of the checkpoints it keeps in RAM (up to 6, ~118 MB each, taken at the start of each new
@@ -609,6 +710,29 @@ unmeasured: all the runs above are text.
 
 ---
 
+## Manage Strata from your AI assistant (MCP server)
+
+`tools/strata_mcp.py` is an MCP server for Claude Code, Claude Desktop, Cursor, VS Code, Codex and other assistants.
+Once it is added, you can ask your assistant "install Strata for this PC", "start Strata" or "is Strata running?".
+In Claude Code, add it with:
+
+```bash
+claude mcp add strata -- python C:\Users\you\Strata\tools\strata_mcp.py
+```
+
+It has eight tools: status (the running model, what is installed, the hardware, a recommended size), the model
+list, install, start, stop, logs, a speed test, and connection settings for other apps.
+
+Install runs `setup.py` with `--yes` in the background. Before it downloads anything, it shows the plan and waits
+for your OK. Start and stop work like the run scripts and the server's own unload. The MCP server only ends
+processes it started itself. It uses only Python's standard library, so it works before `.venv` exists.
+
+The config snippets for every client, the tool arguments and the safety rules are in
+[docs/MCP_SERVER.md](MCP_SERVER.md). This is the opposite direction from
+[Tools from MCP servers](#tools-from-mcp-servers) above, where the Strata model calls *your* MCP tools.
+
+---
+
 ## Images (vision)
 
 The model has a vision encoder: [`mmproj-Qwen3.8-Flash-Next-BF16.gguf`](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF)
@@ -624,6 +748,11 @@ helper (`strata-vision`, from llama.cpp's `mtmd` library) and adds it to your st
 A picture becomes up to 1,024 tokens of the context (a 640x480 photo: 300). The same picture sent again, as chat apps
 do on every turn, is encoded only once.
 
+**A spare GPU for the encoder (0.1.33, #408):** with a card the engine doesn't use, add `"cuda_device": 2` (numbered
+like `nvidia-smi`) to the `"vision"` section of `strata-<model>.json`: the encoder then runs on that card alone. Lower
+`--vram-reserve-mib` in `"args"` to 700 as well, so the engine's cards keep that VRAM for the expert cache. The
+encoder's card needs code in the ready-made encoder (RTX 20/30/40/50).
+
 ### Sending a picture
 
 **Terminal chat:** type `/image <path to a picture>`, press Enter, then type your question.
@@ -743,6 +872,7 @@ the document, +0.4% on the chat. Details: `bench/results/2026-09-27-esp/`.
 | Pictures are slow (10-30 s) | The encoder runs on the CPU: run setup again with `--vision gpu` (needs ~1.4 GB of VRAM). |
 | A request never finishes: "reading the prompt", GPU "100%" at low power | The GPU ran out of VRAM (engines before 0.1.9 could end with ~30 MiB free at large contexts). Run `START-HERE.bat` once to get engine 0.1.9 or newer; the log then says `... MiB of VRAM free with everything loaded` (a few hundred) and names the `--vram-reserve-mib` to add if it is low. |
 | Generation stops mid-answer, GPU "100%", one CPU core busy | Fixed in engine 0.1.12 (issue #29, a race in the CPU expert pool on big-VRAM cards). Since then a request that stops moving ends with an error instead of hanging (after 2 minutes; 1 minute from 0.1.13): the log says `no progress for ... s ... (issue #29)` with where it stopped, and the next request starts the engine again. If you see that line, please open an issue with it. Engine 0.1.13 adds a stall report under it (what every expert-pool thread and the GPU handshake were doing, memory and page faults) and, on Windows, a `strata-stall-<pid>.dmp` file with every thread's stack: attach both. (`STRATA_WATCHDOG_S` sets the time in seconds; 0 turns it off.) Engine 0.1.14 fixes the stall those reports found (issue #31: with the IQ packs the host could wait forever inside the NVIDIA driver while copying experts in a verify window; the experts are now copied by a GPU kernel, `--pcie-mode dma` restores the old way). |
+| `the engine said nothing for ... s during the request` or `... did not finish the request after it was stopped (STOP)` | Issue #481: the engine and the server lost step (the engine waits for its next command, the server for the request's end; GPU at 0 %, nothing in the log). The server ends the engine after 300 s without a line from it during a request (while a prompt is read: each chunk may take three times the previous one's time, the first one up to its tokens at 50 tok/s more), the request ends with an error and the next request starts the engine again. `"engine_silence_s": 600` in `strata-<model>.json` sets the time (0 = wait forever, as before). If you see it, please add the end of the engine log to #481. |
 | `out of memory: cudaFuncSetAttribute` in the log (IQ3_XXS, long prompt) | Fixed in engine 0.1.15: CUDA loaded a kernel's code when it was first needed, and mid-prompt there was no VRAM left for it. Run `START-HERE.bat` (Windows) or `./setup.sh` (Linux) once to update. |
 | Anything else | The engine log is `strata-<model>.log` in this folder. |
 

docs/HOW_IT_WORKS.md

@@ -0,0 +1,77 @@
+# How does Strata work?
+
+Models like Qwen3.8-Flash-Next normally run on servers with hundreds of gigabytes of graphics memory. Your graphics
+card has 12-24 GB. Strata makes it fit by **sharing the work across your whole PC** - the same idea as a kitchen, where
+the things you use all the time stay on the counter and the rest waits in the pantry. Back to the
+[README](../README.md#how-does-it-work).
+
+## Who does what
+
+<p align="center"><img src="media/how-it-works.svg" width="860" alt="The model's 24,576 experts: the busiest on the graphics card, all of them in RAM, a lookup table on the SSD"></p>
+
+- **The model is a team of 24,576 small specialists ("experts"),** and each word it writes needs only 10 of them.
+  So it doesn't have to have all of them on the graphics card at once.
+- **Your graphics card** does the part of the work needed for every word, and keeps the few thousand experts that
+  are asked most often. It keeps learning which ones those are while you use it.
+- **Your RAM** holds every expert. When a word needs one the card doesn't have, **your processor** works on it -
+  at the same time as the graphics card, so neither waits for the other.
+- **Your SSD** holds a big lookup table; the model only reads a few small rows of it per word.
+
+The same in technical terms:
+
+- **GPU (VRAM):** attention and DeltaNet mixers, the gated-residual weights, routers, shared experts, output head, the
+  MTP draft layer, the KV cache (from 64K: only its most-read part, the rest streams from RAM), and an **expert
+  cache** that fills the rest of VRAM with the most-used experts (it adapts to the conversation while you chat).
+  More VRAM matters more than a faster GPU: every extra GB holds ~700 more experts, and every expert on the GPU is one
+  the CPU does not have to compute.
+- **RAM:** all 24,576 experts, pinned. The CPU computes the experts that are not on the GPU **in place**, at the same
+  time as the GPU works on the cached ones (AVX-512 / AVX2 kernels, ggml's for the i-quants). On a PC with little RAM
+  and a big card, the [low-RAM mode](MODELS.md#a-big-graphics-card-and-little-ram) keeps in RAM only what the GPU does
+  not hold.
+- **SSD:** the 28.8 GB n-gram table, read a few rows per token through the OS cache.
+
+This works the same on NVIDIA (CUDA) and AMD (HIP) cards: the same engine, compiled for each.
+
+## Guess, then check
+
+<p align="center"><img src="media/guess-and-check.svg" width="860" alt="A small helper guesses the next words; the big model checks them all at once and keeps the right ones"></p>
+
+- **Guess, then check.** A small, fast helper built into the model guesses the next few words, and the big model
+  checks all the guesses in one go. It keeps the ones it agrees with and writes the next word itself - so one step
+  often produces several words. The helper only guesses - the big model decides every word - so you get the same
+  quality answer, 1.6-1.8x sooner.
+- In technical terms: the model's own MTP layer drafts up to 3 tokens; one pass over all 48 layers checks them, 2.4-3.2
+  tokens per pass on average. When the reply repeats the context (code edits, quoted text), **prompt lookup** drafts
+  up to 5 tokens from the earlier copy, but only where its measured acceptance and cost say it pays: code edits 6-11%
+  faster, other text unchanged. The drafts are checked like the MTP's, so the output is the same.
+
+## Reading long texts
+
+**Long texts are read in big pieces** (up to 8,192 tokens - pieces of words - at a time), which is why a long
+document or code base is read at over 1,000 tokens per second. The experts of the next layer stream to the GPU over
+PCIe while the current layer's attention runs. After the first message, Strata keeps the conversation and reads only
+what is new, so follow-ups start in seconds.
+
+## More
+
+The [details](DETAILS.md#how-it-works) explain every part and its numbers, and the
+[paper](paper/Strata-Paper.pdf) tells the whole story, with the measurements behind it.
+
+## Credits
+
+- Model: [Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next) by the Qwen team; compressed versions by
+  [ISTA-DASLab](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF);
+  [Swift 1.5](https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF) by UkisAI; the experimental
+  [UD-Q4_K_XL](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF) by Unsloth (its support follows
+  [eddoursul/Strata](https://github.com/eddoursul/Strata)). Their licenses apply to the model files.
+- Built with parts of [llama.cpp / ggml](https://github.com/ggml-org/llama.cpp) (MIT). Ideas from
+  [Splash](https://github.com/incoai/splash), [ninfer](https://github.com/Neroued/ninfer) and
+  [HyperQwen](https://github.com/syv-ai/HyperQwen). More in the [details](DETAILS.md#credits-and-licenses).
+- The AMD (HIP) backend: [AMD_HIP.md](AMD_HIP.md) names the contributors and the machines it was validated on.
+
+## License
+
+Strata is open source under the [MIT License](../LICENSE). A few parts carry their own licenses: `third_party/ggml`
+(MIT, llama.cpp / ggml), the web app's font (SIL Open Font License 1.1) and the experimental speed projection's
+vector in `data/experimental-speed-projection` (Qwen Community License 1.0, from the model's activations). The
+models are not part of this repository; each model's own license applies to its files.

docs/INSTALL.md

@@ -0,0 +1,222 @@
+# Installing Strata
+
+Everything about installing, starting, updating and configuring Strata, on Windows and Linux, with an NVIDIA or an
+AMD graphics card. The short version is in the [README](../README.md#install); an AI coding assistant can do all of
+this for you with [AI_SETUP.md](AI_SETUP.md).
+
+> **On this page:** [What you need](#what-you-need) · [Windows](#windows) · [Linux](#linux) ·
+> [AMD cards](#amd-cards) · [Several cards](#two-or-three-cards) · [Docker](#docker-linux) · [Updating](#updating) ·
+> [Where things are stored](#where-things-are-stored) · [Setup's questions](#setups-questions) ·
+> [Tuning](#tuning-for-your-pc) · [All options](#options-without-questions)
+
+## What you need
+
+| | |
+| --- | --- |
+| **GPU** | **NVIDIA** RTX 20, 30, 40 or 50 series, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070 and an RTX 3090; RTX 20 (Turing, since 0.1.27) was tested by a contributor on an RTX 2070. **AMD** Radeon RX 7900 XT / XTX, RX 9070 / 9070 XT and Radeon AI PRO R9700 (validated), RX 7800 XT / 7700 XT and RX 9060 XT (validated by their owners), RX 6800 / 6900 series (community-reported), with 12 GB of VRAM or more. See [AMD cards](#amd-cards). |
+| **RAM** | Enough for the size you pick ([which model](MODELS.md#pick-by-ram)); **64 GB** runs every size. A big GPU makes up for less RAM - the [low-RAM mode](MODELS.md#a-big-graphics-card-and-little-ram). |
+| **CPU** | x86-64 with AVX2 (any Intel/AMD desktop CPU from the last ~8 years). AVX-512 (Ryzen 7000/9000) is a bit faster. |
+| **Disk** | ~70-80 GB free for the model, ~6 GB for the MTP layer (+1 GB with images). **Q2_0 on an AVX-512 CPU** also writes a one-time ~40 GB copy of its experts for the fast CPU kernel. On Linux with an AMD card, ROCm takes ~10 GB more when setup installs it. An NVMe SSD is strongly recommended: it makes the first start much faster. |
+| **OS** | Windows 10/11, or Linux (Ubuntu 22.04/24.04 get everything installed automatically). |
+| **Driver** | **NVIDIA:** a current driver, version 580 or newer ([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA App). **AMD:** on Linux the kernel's amdgpu driver (no ROCm install needed); on Windows a current AMD Software: Adrenalin Edition driver ([amd.com/support](https://www.amd.com/en/support)). |
+
+The driver is the only thing you install yourself. Everything else - Python, the engine, the model - is set up for
+you the first time: Python 3.12 if you have none (for your user account, no admin), a private Python environment in
+`.venv/`, the Strata engine, the model and the MTP draft layer. On NVIDIA it uses the ready-made engine for RTX
+20/30/40/50 and NVIDIA's CUDA libraries from pip (~0.4 GB); if no ready-made engine fits your PC, it offers to install
+the build tools (Visual Studio Build Tools + CUDA Toolkit on Windows, `build-essential` + CUDA on Ubuntu) and compiles
+the engine for your GPU (asks first; 20-40 minutes once). More: [details](DETAILS.md#before-you-start).
+
+## Windows
+
+1. [Download this project](https://github.com/Niko1221/Strata/archive/refs/heads/main.zip) and unzip it (or
+   `git clone` it).
+2. Double-click **`START-HERE.bat`**.
+3. Answer a few questions - or just press Enter each time for the recommended choice ([the questions](#setups-questions)).
+
+Then it downloads everything (the model is ~70 GB, so the first time takes a while - you can stop and it picks up
+where it left off) and **starts the model**. Your browser opens the Strata app at `http://127.0.0.1:8080`.
+
+> **While the model starts, your PC can be slow or stop responding for 1-3 minutes** (longest the first time): Strata
+> loads 35-55 GB into your RAM and locks part of it for the graphics card. That's normal - wait, and don't close the
+> window. The window tells you what it is doing.
+
+**Next time**, just double-click `START-HERE.bat` again: it starts right away (30-90 s to load the model), nothing is
+downloaded twice. Close its window to stop the model. `SETUP.bat` (the same as `START-HERE.bat --setup`) installs
+another model or changes the settings. Starting Strata from Task Scheduler at logon needs two task settings, or the
+start is 24x slower: [Running it at startup](DETAILS.md#running-it-at-startup-task-scheduler).
+
+## Linux
+
+```bash
+./setup.sh
+```
+
+The same questions, the same automatic install (it uses `sudo apt` for Python and, only if it has to compile, for
+the build tools), and the same start: `http://127.0.0.1:8080`. Later runs of `./setup.sh` (or `./run-<model>.sh`)
+start the model directly; `./setup.sh --setup` installs another model or changes the settings. Other distributions,
+WSL and compiling: [details](DETAILS.md#linux).
+
+## AMD cards
+
+The steps are the same as with NVIDIA: `START-HERE.bat` on Windows, `./setup.sh` on Linux. Setup finds the Radeon
+card and chooses the AMD (HIP) engine by itself on a PC with no NVIDIA card Strata can use; `--backend hip` chooses
+it on a PC that has both. Supported cards: RX 7900 XT / XTX (gfx1100), RX 7800 XT / 7700 XT (gfx1101), RX 9060 XT
+(gfx1200), RX 9070 / 9070 XT and Radeon AI PRO R9700 (gfx1201), and the RX 6800 / 6900 series (gfx1030). Integrated
+Radeon GPUs are listed as not supported.
+
+On Linux setup uses a system ROCm 7 when there is one, or installs ROCm into `.venv` from AMD's wheels (~10 GB, no
+sudo), and compiles the engine on your PC for the card (10-20 minutes, once; it needs a C++ compiler and git:
+`sudo apt install build-essential git`). Several AMD cards share the model with `--gpus`, as on NVIDIA.
+
+What differs from NVIDIA for now: pictures are read by the image encoder on the CPU (`--vision cpu`, 10-30 s per
+picture), `--calibrate` is NVIDIA-only, and Unsloth's 4-bit model needs an NVIDIA card. Measurements per card, the
+build by hand and the tuning tables: [AMD_HIP.md](AMD_HIP.md).
+
+## Two or three cards
+
+**Two or three NVIDIA cards?** Just run `START-HERE.bat`: it lists your cards, says which ones Strata can use, and
+asks whether to share the model across them (recommended when two can). An install made on one card asks once at
+its next start. Or choose yourself: `START-HERE.bat --gpus 0,2` (both, remembered), `--gpus all`, or `--gpu 0` (one
+card, this start only). Each card keeps the experts of its own layers, and prompts flow through the cards in a
+pipeline: on an RTX 5080 + RTX 3090 prompts were read 18-20% faster than on the 5080 alone, decoding on par.
+Every card must be an RTX 20 series or newer with 8 GB or more. See [MULTI_GPU.md](MULTI_GPU.md).
+
+**Several AMD cards:** setup takes one card (the one with the most VRAM, or `--gpu N`) unless you name more:
+`--gpus 1,0` splits the model's layers across them, the first one the main card (numbers as setup lists them). A
+split pays only when no single card holds the model's experts ([AMD_HIP.md](AMD_HIP.md#rdna4-gfx1201)).
+
+## Docker (Linux)
+
+The same idea, in a container (NVIDIA cards).
+
+1. Host: Docker with the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
+   and a driver **580 or newer** (CUDA 13.0).
+2. Build (this compiles the engine into the image, so the container never compiles):
+   `docker build -t strata .`
+   `docker build -t strata --build-arg CUDA_ARCHITECTURES=89 .` builds for one card only (faster).
+   The default covers RTX 30 (86), RTX 40 (89), RTX 50 (120) and A-series (80); a card outside that
+   set needs a rebuild with its own arch. Add `--build-arg BUILD_VISION=0` to skip the image encoder.
+3. Run (the first start downloads the ~70 GB model, then starts; later starts go straight to serving):
+   `docker run --rm --gpus all -p 8080:8080 --ulimit memlock=-1 -v strata-data:/data strata`
+
+   The setup choices are env vars: `-e MODEL=IQ2_XS -e FAMILY=qwen -e CONTEXT=32768 -e VISION=no`
+   (or `MODEL=Q2_0|IQ3_XXS|IQ3_S`, `FAMILY=swift|coder`; the defaults above are the recommended ones).
+   `-e VISION=cpu` keeps the image encoder on the CPU. `-e KV=int8|q4_0|k8v4` picks the KV cache
+   precision; `k8v4` is INT8 K with 4-bit V and keeps its KV in VRAM from 64K up.
+   Only the model files, the prepared pack, the MTP layer and the install config live in the
+   `strata-data` volume; the engine is part of the image. Switching between models already on the
+   volume needs no setup pass: `-e MODEL=Q2_0 -e FAMILY=coder` picks that model's config. Add
+   `-e REINSTALL=1` only to change settings for a model already set up (context, vision, KV, host,
+   api_key, LOW_RAM), since those are recorded in its config.
+   Strata loads 32-62 GB into RAM. `--gpus all` on a host with two usable cards takes both: the
+   layer split is setup's recommended default ([MULTI_GPU.md](MULTI_GPU.md)), and a volume
+   set up for one card switches to the pair on its first start there. Pin one card with `-e GPU=0`,
+   or name them with `-e GPUS=0,2` and where the later card's layers start with `-e LAYER_SPLIT=18`.
+   A memory limit needs `-e LOW_RAM=on`, which maps the model's experts from the pack instead of
+   keeping them in RAM: setup.py measures the host's RAM, not the container's limit, so it cannot
+   see a cap. LOW_RAM runs on one card unless `GPUS` names several (then the experts the cards do not hold are
+   read through the OS file cache, which can fill the RAM during long prompts).
+   The server listens on `0.0.0.0:8080` by default; set `-e API_KEY=<secret>` before exposing the port
+   to a network. The image has a `HEALTHCHECK` on `/health`, so `docker ps` shows the container
+   healthy once the model is loaded, and `GET /v1/status` says what it is running.
+
+## Updating
+
+**`UPDATE.bat`** (Linux: `./update.sh`) updates Strata without starting the model - for when the GPU is busy with
+something else, or you just want the new version ready. In a `git clone` it runs `git pull`, then does what
+`START-HERE.bat` does before a start: the engine (a new ready-made one when the new version needs it; on Linux a
+compiled engine is compiled again when its source changed), the Python packages, and each installed model's settings
+and draft subset. The model files are not touched (at most a new engine is downloaded) and no question is asked.
+Close the model's window first (a running engine cannot be replaced); start the model later with `START-HERE.bat` as
+usual. In a copy that was downloaded as a zip it says to download the new zip (below): it cannot fetch new files
+itself.
+
+Or by hand: download the new version and unzip it anywhere (or `git pull`), then run `START-HERE.bat` (Linux:
+`./setup.sh`) in it. The model files are kept in a `Strata-data` folder next to your Strata folder, so a new copy
+finds them and sets itself up the same way - nothing big is downloaded again. On Linux after a `git pull`, setup
+compiles the engine again when its source changed (a minute or two for the changed files); if that compile fails, it
+says so and starts the engine you had.
+
+## Where things are stored
+
+- **Your chats: only in your browser.** The Chat tab keeps the conversation, its settings and the API key you typed
+  in the browser's local storage (`strata.*` keys) - not on the server and not in the Strata folder. Pictures are not
+  kept, only their names. Another browser or a private window starts empty; clearing the site's data deletes them.
+- **How the model starts:** `strata-<model>.json` in the Strata folder (context, GPUs, host, API key, ...), written
+  by setup; next to it `run-<model>.bat` / `.sh`, the log `strata-<model>.log` and, when you use "Use for other
+  apps too", `strata-<model>.shared-settings.json`.
+- **The model files** (`models/`, `packs/`, `mtp/`, 70-120 GB): in **`Strata-data` next to the Strata folder**, or
+  wherever `--data-dir` put them.
+- **Where that data folder is:** `%APPDATA%\Strata\settings.json` on Windows, `~/.config/strata/settings.json` on
+  Linux ([details](DETAILS.md#before-you-start)).
+- **The program itself:** `.venv/`, `engine/` and `third_party/` in the Strata folder.
+
+## Setup's questions
+
+- **Which model and size?** The original, Swift 1.5, the Coder or Unsloth's 4-bit, and Q2_0, IQ2_XS, IQ3_XXS or
+  IQ3_S - see [which model](MODELS.md).
+- **How much context?** How much text it can keep in mind at once (it suggests one for your card). 384K and
+  512K (experimental) extend the model past its trained 262K by rope scaling - the setup turns it on itself (yarn and a
+  covering factor; `--rope-scaling`/`--rope-scale` override) ([details](DETAILS.md): "Context extension past
+  262K").
+- **Images?** Whether it should also read pictures. The image encoder (0.9 GB) runs on the GPU (0.1-0.5 s per
+  picture, ~1.4 GB of VRAM kept free for it) or on the CPU (10-30 s per picture, nothing on the GPU); with AMD cards
+  on the CPU for now. [Details](DETAILS.md#images-vision).
+- **Experimental speed projection?** Off unless you say yes - [read what it does](DETAILS.md#experimental-speed-projection-experimental-off-by-default)
+  first. It changes how the model answers, and only the original model offers it.
+
+With `--yes` setup takes the recommended answer to every question.
+
+## Tuning for your PC
+
+Every PC is different: `START-HERE.bat --calibrate` (Linux: `./setup.sh --calibrate`) measures a few engine settings
+on yours and keeps the fastest (about 5-10 minutes; on an RTX 5070 with a Ryzen 5 7600 it made the Coder 7% faster).
+It keeps a setting only when it is more than 3% faster, and the result is remembered per PC and model, so updates
+keep it. NVIDIA cards for now. [What it measures](DETAILS.md#double-click-start-herebat).
+
+## Options without questions
+
+```
+START-HERE.bat --setup                          install another model, or change context / images
+SETUP.bat                                       the same (double-click it)
+START-HERE.bat --model IQ2_XS --context 32768 --vision yes --yes     no questions
+START-HERE.bat --gguf-dir D:\models\IQ2_XS       use GGUF files you already have
+START-HERE.bat --data-dir E:\Strata-data         keep the model files somewhere else
+START-HERE.bat --port 8081                      another port
+START-HERE.bat --gpu 1                          another GPU (setup picks the one with the most VRAM)
+START-HERE.bat --gpus 0,2                       several GPUs sharing the model
+START-HERE.bat --vram-reserve-mib 2048          leave 2 GB of VRAM free for other programs (remembered)
+START-HERE.bat --setup --backend hip            the AMD engine on a PC that also has an NVIDIA card
+START-HERE.bat --setup --host 0.0.0.0 --api-key <secret>     reachable from other devices, with a key
+START-HERE.bat --calibrate                      tune the engine for this PC (about 5-10 minutes), then start
+START-HERE.bat --check                          only check this PC
+```
+
+**Leaving VRAM for other programs (#493):** Strata fills the graphics card's free VRAM with experts (the expert cache)
+and leaves `--vram-reserve-mib` MiB free: 700 by default. For a game, a 3D program or another model beside it, leave
+more: `START-HERE.bat --vram-reserve-mib 2048` (Linux: `./setup.sh --vram-reserve-mib 2048`) writes it into the
+model's `strata-<model>.json` and starts it; at setup (`--setup --vram-reserve-mib 2048`) it goes into the new config.
+By hand: add `"--vram-reserve-mib", "2048"` to the config's `"args"` list and restart. The expert cache is then that
+much smaller, so answers can be a little slower. The engine sizes its cache from the VRAM free when it starts: what
+another program already holds then is left alone anyway; the reserve is room for what it takes later.
+
+**An AMD card that also drives a Linux desktop (#560 #516):** with the default reserve the expert cache fills the
+card, and when the desktop (the compositor, a browser, a new app) needs more VRAM, the amdgpu driver moves GPU memory
+to system RAM, which the model's experts already fill: the OOM killer then ends the desktop session (KWin, plasmashell)
+or the compositor fails ("Failed to pin framebuffer"). `./setup.sh --vram-reserve-mib 3072` fixed it in both reports
+(about 2.3 GB fewer experts in VRAM, a few % of speed). Setup and the server window say so on such a PC.
+
+**A card under 8 GB (#496):** when the default reserve leaves the expert cache too little room, the engine lowers the
+reserve (down to 300 MiB) until it fits, and warns if the card then ends nearly full. If the start stops with "no VRAM
+is left for the expert cache", that log line says how much is short; an 8K context and `--draft-vocab en` at setup
+free the most.
+
+**Model files downloaded by hand, or from a mirror (#495):** setup's step 5 prints the folder it expects them in
+(`Strata-data\models\<SIZE>\`, e.g. `Strata-data\models\IQ3_XXS\`): put them there with their original names, or
+point setup at them with `--gguf-dir`. To let setup download from a Hugging Face mirror itself, set `HF_ENDPOINT`
+first (Windows: `set HF_ENDPOINT=https://hf-mirror.com`, Linux: `export HF_ENDPOINT=https://hf-mirror.com`): the same
+pinned revisions and checks apply, and the MTP draft layer comes from there too.
+
+On Linux the same options go to `./setup.sh`. `START-HERE.bat --help` lists them all. The server's own settings
+(sharing the GPU with games, MCP tools, CORS, API keys, the API itself) are in the [details](DETAILS.md#using-it).

docs/MCP_SERVER.md

@@ -0,0 +1,205 @@
+# Strata's MCP server: let your AI assistant install and run Strata
+
+`tools/strata_mcp.py` is an [MCP](https://modelcontextprotocol.io) server for the AI assistant you already use
+(Claude Code, Claude Desktop, Cursor, VS Code, Codex ...). Add it once, then ask in plain words:
+
+- "Install Strata for this PC."
+- "Start Strata." / "Stop Strata."
+- "Is Strata running? How much VRAM does it use?"
+- "Which Strata model fits my PC?"
+- "Strata didn't start - what does the log say?"
+- "How fast is Strata on this PC?" (a short speed test)
+- "How do I connect Open WebUI / the OpenAI SDK / Claude Code to Strata?"
+
+The assistant calls the server's tools. The tools run Strata's own setup and start the model the way the start
+scripts do. It is one Python file that uses only the standard library, so it works before setup has created
+`.venv`. It needs Python 3.10 or newer, 64-bit on Windows.
+
+It is not the same thing as [Tools from MCP servers](DETAILS.md#tools-from-mcp-servers). That feature
+(`serve/mcp.py`) lets the *Strata model* call tools from your MCP servers in its chat page. This page is the other
+direction: *your assistant* manages Strata.
+
+## Add it
+
+You need a copy of Strata: the zip from GitHub, or `git clone https://github.com/Niko1221/Strata`. Below,
+`C:\Users\you\Strata` (Windows) and `/home/you/Strata` (Linux) stand for that folder. Use `python3` on Linux. On
+Windows, use `py` if `python` is not on your PATH.
+
+### Claude Code
+
+```bash
+claude mcp add strata -- python C:\Users\you\Strata\tools\strata_mcp.py         # Windows
+claude mcp add strata -- python3 /home/you/Strata/tools/strata_mcp.py           # Linux
+```
+
+Add `--scope user` to have it in every project. `/mcp` in Claude Code shows that it is connected.
+
+### Claude Desktop
+
+Open Settings > Developer > Edit Config (`claude_desktop_config.json`) and add:
+
+```json
+{
+  "mcpServers": {
+    "strata": {
+      "command": "python",
+      "args": ["C:\\Users\\you\\Strata\\tools\\strata_mcp.py"]
+    }
+  }
+}
+```
+
+On Linux, use `"command": "python3"` and `"args": ["/home/you/Strata/tools/strata_mcp.py"]`. Restart Claude
+Desktop after the change.
+
+### Cursor
+
+Use `~/.cursor/mcp.json` for every project, or `.cursor/mcp.json` in one project. The format is the same as
+Claude Desktop's:
+
+```json
+{
+  "mcpServers": {
+    "strata": {
+      "command": "python",
+      "args": ["C:\\Users\\you\\Strata\\tools\\strata_mcp.py"]
+    }
+  }
+}
+```
+
+### VS Code (GitHub Copilot agent mode)
+
+Use `.vscode/mcp.json` in a workspace, or run "MCP: Open User Configuration" from the command palette:
+
+```json
+{
+  "servers": {
+    "strata": {
+      "type": "stdio",
+      "command": "python",
+      "args": ["C:\\Users\\you\\Strata\\tools\\strata_mcp.py"]
+    }
+  }
+}
+```
+
+On Linux, use `"command": "python3"` and `"args": ["/home/you/Strata/tools/strata_mcp.py"]`.
+
+### Codex CLI
+
+In `~/.codex/config.toml`:
+
+```toml
+[mcp_servers.strata]
+command = "python"
+args = ['C:\Users\you\Strata\tools\strata_mcp.py']    # Linux: command = "python3", args = ["/home/you/Strata/tools/strata_mcp.py"]
+tool_timeout_sec = 180                                # strata_stop / strata_benchmark can take longer than 60 s
+```
+
+### Any other MCP client
+
+The command is `python <Strata folder>/tools/strata_mcp.py`. The transport is stdio. It speaks MCP revision
+2025-06-18, and also 2025-03-26 and 2024-11-05.
+
+`--root <folder>` manages a Strata folder other than the one the file is in. For example:
+`python D:\tools\strata_mcp.py --root D:\Strata`.
+
+## The tools
+
+| Tool | What it does | Arguments (all optional) |
+| --- | --- | --- |
+| `strata_status` | Whether a model is running: the model, engine version, context, busy or idle, VRAM and RAM in use. What is installed: models, start scripts, engine version, data folder, downloads. An install in progress. This PC's hardware: OS, CPU, RAM, GPUs, page file, free disk. The model and context setup recommends for this PC. | `port`, `hardware` (true/false) |
+| `strata_models` | Every family and size setup offers, with download size, RAM needed, whether it fits this PC, and whether it is installed. Also the context lengths and the recommendation. | - |
+| `strata_install` | Runs setup without questions, in the background (see below). Without `confirm: true` it only returns the plan. A second call shows the progress. `cancel: true` stops the install. | `family` (qwen, swift, coder, unsloth), `model` (Q2_0, IQ2_XS, IQ3_XXS, IQ3_S, IQ1_M, UD-Q4_K_XL), `context` (8192 ... 524288), `vision` (no, yes, gpu, cpu), `kv` (int8, q4_0, k8v4), `gpu` (a number) or `gpus` ("0,2", "all"), `backend` (auto, cuda, hip), `low_ram`, `port`, `data_dir`, `confirm`, `cancel` |
+| `strata_start` | Starts an installed model in the background and waits until it answers. If it is not ready after `wait_seconds` (default 50), it returns "loading". | `model` (an id from `strata_status`, e.g. `q2_0`, `coder-iq1_m`; default: the most recently used), `port`, `gpu`, `wait_seconds` (0-600) |
+| `strata_stop` | Unloads the model through the server's own API, then ends the server it started. | `port`, `force` |
+| `strata_logs` | The last lines of the server log, the engine log or the setup log. | `source` (server, engine, setup), `model`, `lines` (1-500) |
+| `strata_benchmark` | Sends a short fixed request (greedy, thinking off) and reports output and prompt tokens/s. | `port`, `max_tokens` (16-512) |
+| `strata_connect_info` | The OpenAI and Anthropic base URLs, the model id, whether a key is needed, and settings for Claude Code, Codex, the OpenAI SDK, curl, Cursor, Continue and Open WebUI. | `port` |
+
+Every result is JSON with a one-line `summary`. It is sent both as text and as MCP structured content.
+
+## How it works
+
+**Install.** `strata_install` uses setup's own logic and tables (it imports `setup.py`'s `MODELS`, `FAMILIES`,
+`CONTEXTS` and RAM checks) to fill in what you left out:
+
+- the size setup itself would pick: IQ3_XXS from 60 GB of RAM, Q2_0 from about 44 GB, the Coder from about 28 GB,
+  and the Coder in the low-RAM mode below that when the GPU makes up for the RAM;
+- the context setup recommends for the GPU's VRAM;
+- images off.
+
+The first call returns the plan: the download size (58-111 GB), the free disk space, the RAM check, and the exact
+setup command. The assistant should show you the plan and call again with `confirm: true` once you agree.
+
+It then runs the same steps `START-HERE.bat` / `setup.sh` run:
+
+1. It creates `.venv` with the Python it runs on, if `.venv` is missing.
+2. It runs `setup.py --yes --no-start --family ... --model ... --context ... --vision ...` as a background process.
+   The output goes to `.strata-mcp/setup.log` in the Strata folder.
+
+The tool returns at once. Call it again to see the step (1-7), the download percentage, or setup's error message.
+Everything setup has finished is skipped on the next run, and downloads resume. If the AI app closes, the install
+keeps running. The install is refused while a model this server started is running, because setup may replace the
+engine files that model uses.
+
+**Start.** `strata_start` runs the same command as the `run-<model>` script, `serve/server.py --engine strata
+--config strata-<model>.json --port N`, with the venv's Python, without `--open` (no browser) and without a console
+window. The server's output goes to `.strata-mcp/server.log`. The tool then polls `GET /health` until the model
+answers. Loading takes 1-3 minutes. If the server ends while loading, the tool returns the end of its log. The model
+keeps running after the AI app closes, until `strata_stop`. If the AI app runs its tools inside a Windows job that
+does not allow processes to leave it, the model ends together with the app.
+
+Two cases are refused:
+
+- a model is already running: the tool returns it and starts nothing;
+- another program uses the port: the tool says so.
+
+**Stop.** `strata_stop` first sends `POST /unload`. The server sends the engine `QUIT`, and the engine frees its
+VRAM and pinned RAM itself, as when you close the window. Then the server ends:
+
+- Linux: SIGTERM, which runs the server's own Ctrl+C path.
+- Windows: the process ends. Its job object ends the vision encoder and the MCP servers it started.
+
+While a request is running, the tool refuses unless you pass `force: true`.
+
+**Status.** The tool reads `GET /health`, `/v1/status`, `/status` and, on older servers, `/metrics` on 127.0.0.1.
+If a config of the folder sets an `api_key`, the tool sends that key with its own requests. It never shows the key.
+
+## Safety
+
+- **No shell, no free paths.** Every argument is checked against a fixed schema and against setup's own choices. An
+  unknown argument, a model or family setup does not offer, or a `gpus` value other than digits and commas is
+  refused before anything runs. Commands are passed as argument lists; nothing goes through a shell.
+- **Paths.** The only path argument is `data_dir`. It must be one of:
+  - the current Strata data folder;
+  - a folder inside the Strata folder;
+  - a new absolute folder whose name starts with `Strata` (e.g. `D:\Strata-data`), whose parent exists, and which is
+    not in a system folder.
+
+  `..`, relative paths and network (UNC) paths are refused. Logs are read only from `.strata-mcp/` and from the
+  engine log a config names, if that log is inside the Strata folder or the data folder.
+- **Only its own processes.** The server records the process id *and start time* of every process it starts, in
+  `.strata-mcp/*.json`. It only ends a process when both match, so a process id the OS has given to another program
+  is never touched. A Strata started some other way (its window, a run script, a service) is only asked over its
+  API to unload the model. To end that server, close its window.
+- **Big downloads need a yes.** `strata_install` does nothing until it is called with `confirm: true`. The plan
+  shows the download size and the free space first, and the install is refused if the disk is too small.
+- **Local only.** The server checks `127.0.0.1` and never changes the network settings: it does not install with
+  `--host 0.0.0.0` or an API key. To share Strata on your network, run setup yourself (see
+  [Using it](DETAILS.md#using-it)). Experimental options are left off too: the speed projection, rope scaling past
+  262K and `--build`. Run setup by hand for those.
+
+## Troubleshooting
+
+- **The assistant does not see the tools.** Run the command by hand:
+  `python C:\Users\you\Strata\tools\strata_mcp.py`. It should print `[strata-mcp] ready (Strata folder ...)` to
+  stderr and wait for input (Ctrl+C ends it). If it reports `not a Strata folder`, the path is wrong or `--root` is
+  needed. Python older than 3.10 is refused.
+- **The install failed.** Ask the assistant for the setup log (`strata_logs` with `source: setup`), or open
+  `.strata-mcp/setup.log`. Setup's `[X]` line says what to fix. Then install again: every finished step is skipped.
+- **The model does not start.** Get the server log with `strata_logs`, and the engine log with `source: engine`.
+  [Troubleshooting](DETAILS.md#troubleshooting) has the usual causes.
+- **Tests.** `python -m unittest tools/test_strata_mcp.py` runs the tests with a fake setup and server. They do not
+  download anything or use the GPU.

docs/MODELS.md

@@ -0,0 +1,141 @@
+# Which model? Sizes, versions and what fits
+
+Strata runs one model, [Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next), in several **sizes**
+(the same model, compressed more or less) and several **versions** (the original, a coding version, a fine-tune).
+The installer recommends one for your PC; this page explains the choice. Back to the [README](../README.md).
+
+> **On this page:** [Pick by RAM](#pick-by-ram) · [Speed](#how-fast-is-each-size) · [The sizes](#the-sizes) ·
+> [Will it fit?](#will-it-fit) · [The versions](#the-versions) · [Adding another model](#adding-or-switching-models)
+
+## Pick by RAM
+
+| Your RAM | Take | Why |
+| --- | --- | --- |
+| **32 GB** | **Coder** | it fits 32 GB; best at code, weaker at everything else ([why](#coder)). With a 24 GB card, Q2_0 and IQ2_XS run too ([low-RAM mode](#a-big-graphics-card-and-little-ram)) and are the better pick for general use |
+| **48 GB** | **IQ2_XS** (or Q2_0, the fastest) | the larger sizes do not fit |
+| **64 GB** | **IQ2_XS** (recommended), or IQ3_XXS / IQ3_S | every size fits (IQ3_S with little else open) |
+| **96 GB or more** | **IQ3_S**, or [Unsloth's 4-bit](#unsloth-ud-q4_k_xl-experimental) (experimental) | room for the largest sizes |
+
+Not sure? Take **IQ2_XS**. The **Coder** is the one that fits a 32 GB PC, but it keeps only 256 of the 512 experts,
+chosen on code data, so it is weaker outside code and in languages other than English. For general use, or whenever
+your RAM allows, take a full model (Q2_0, IQ2_XS or IQ3_S).
+
+## How fast is each size
+
+Measured on an RTX 5070 (12 GB), a Ryzen 5 7600 and 64 GB of RAM:
+
+| Size | Writes answers (short chat) | Writes answers (128K context) | Reads your prompt |
+| --- | ---: | ---: | ---: |
+| **Q2_0** | 94 tokens/s | 76 tokens/s | 2,650 tokens/s |
+| **IQ2_XS** | 79 tokens/s | 63 tokens/s | 2,090 tokens/s |
+| **IQ3_XXS** | 62 tokens/s | 49 tokens/s | 1,750 tokens/s |
+| **IQ3_S** | 53 tokens/s | 46 tokens/s | 1,620 tokens/s |
+| **Coder** (IQ1_M) | 55 tokens/s | 43 tokens/s | 2,180 tokens/s |
+
+Measured on an AMD RX 9070 XT (16 GB), a Ryzen 9 3900X and 47 GB of RAM (Linux, setup's own install; Q2_0 is
+above setup's RAM estimate for 47 GB and was installed with `--model Q2_0 --yes`):
+
+| Size | Writes answers (short chat) | Writes answers (128K context) | Reads your prompt |
+| --- | ---: | ---: | ---: |
+| **Q2_0** | 60 tokens/s | 48 tokens/s | 1,160 tokens/s |
+| **IQ2_XS** | 52 tokens/s | 36 tokens/s | 1,110 tokens/s |
+| **Coder** (IQ1_M) | 44 tokens/s | 33 tokens/s | 1,420 tokens/s |
+
+- **Writes answers** = how fast the reply appears (tokens per second; a token is about ¾ of a word).
+- **Reads your prompt** = how fast it takes in what you send (long documents, code, chat history), measured on a
+  32K-token prompt; a 4K prompt reads at 910-1,580 tokens/s. A 32K prompt takes about 15 seconds with Q2_0.
+
+A card with more VRAM is faster, because more of the model fits on the GPU: an RTX 3090 (24 GB) should do roughly
+100-140 tokens per second. All measurements, long-context numbers and estimates for other cards are in the
+[details](DETAILS.md#speed-measured). The AMD measurements per card (RX 9070 XT, Radeon AI PRO R9700, RX 7800 XT,
+RX 9060 XT, RX 6900 XT) are in [AMD_HIP.md](AMD_HIP.md#rdna4-gfx1201).
+
+Every PC is different: `START-HERE.bat --calibrate` measures a few engine settings on yours and keeps the fastest
+(about 5-10 minutes; on the PC above it made the Coder 7% faster; NVIDIA cards for now). Measured Strata on your own
+PC? See [Community benchmark results](COMMUNITY_BENCHMARKS.md) for a report template and how to share your results
+in a pull request.
+
+## The sizes
+
+| Model | RAM+VRAM Requirements | Speed | Quality |
+| --- | ---: | --- | --- |
+| **Q2_0** | 37.6 GB | fastest | good |
+| **IQ2_XS** | 39.2 GB | fast | better (**recommended**) |
+| **IQ3_XXS** | 47.0 GB | slower | great |
+| **IQ3_S** | 54.8 GB | slowest | best: matches the full model on the published tests (original model only) |
+
+The download is 66-76 GB for the three smaller sizes ([details](DETAILS.md#which-model)); the first start also
+fetches the MTP draft layer (~6 GB, +1 GB with images).
+
+## Will it fit?
+
+Shard 1 is the part of the model that gets loaded when it starts: its experts go into your **RAM**, the rest onto
+your graphics card (the second shard, a 29 GB lookup table, stays on the SSD). So it fits when your **RAM is at least
+shard 1 + about 10 GB** for Windows and your other programs. With 64 GB of RAM every size fits (IQ3_S with little
+else open); with 48 GB, Q2_0 and IQ2_XS. A bigger graphics card makes it faster, but it doesn't lower the RAM needed
+- except in the low-RAM mode below.
+
+### A big graphics card and little RAM
+
+On a PC whose RAM cannot hold the experts beside the system, setup maps them from the model's files instead and keeps
+in RAM only what the graphics card does not hold (the [low-RAM mode](DETAILS.md#speed-measured), chosen by setup). For
+example, a 32 GB PC with a 24 GB GPU runs Q2_0, IQ2_XS and the Coder this way, and a 32 GB PC with a 12-16 GB GPU the
+Coder. With a small card most experts then come from the SSD and it is much slower (setup says so).
+`START-HERE.bat --setup --low-ram on|off` overrides the choice.
+
+## The versions
+
+### Qwen3.8-Flash-Next
+
+The original, in all four sizes. With images, and with the
+[experimental speed projection](DETAILS.md#experimental-speed-projection-experimental-off-by-default) as an option.
+
+### Coder
+
+**[Coder](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF)** - ISTA-DASLab's coding version:
+it keeps 256 of each layer's 512 experts, chosen on code (with agentic and image) data, and drops the rest (91% of
+the full model's SWE-bench Verified score, 99% of LiveCodeBench, by its authors). One size (IQ1_M: its experts stored
+like IQ3_S): shard 1 is **29.6 GB**, so it fits a PC with **32 GB of RAM**, runs 262K context on 64 GB, and reads long
+prompts the fastest of all. With half the experts gone it is weaker outside code and in languages other than
+English, Chinese and other CJK text included (#438: Chinese answers came out wrong or looping where English was
+fine). For general use, or whenever your RAM allows, take a size that keeps every expert: Q2_0, IQ2_XS or IQ3_S.
+More: [details](DETAILS.md#or-the-coder-half-the-experts-for-code).
+
+```
+START-HERE.bat --setup --family coder
+```
+
+### Swift 1.5
+
+**[Swift 1.5](https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF)** - a fine-tune by UkisAI that
+thinks much shorter before answering, so you get the answer sooner, with about the same quality. Same speed per
+token, and about the same RAM as the same size of the original (no IQ3_S). Its own license applies (see its page).
+More: [details](DETAILS.md#or-swift-15-a-fine-tune-that-thinks-shorter).
+
+```
+START-HERE.bat --setup --family swift --model IQ2_XS
+```
+
+### Unsloth UD-Q4_K_XL (experimental)
+
+**Unsloth's 4-bit UD-Q4_K_XL** (experimental) is the fourth version in setup's menu (`--family unsloth`): the closest
+to the full model, but a 111 GB download whose 77 GB of experts do not fit in RAM. Strata keeps your RAM minus 24 GB
+of them in RAM and reads the rest from the SSD while it answers: 7-8.5 tokens/s on a 64 GB PC with a 12 GB GPU, several
+times slower than the sizes above, and long prompts are slow. It needs 48 GB of RAM or more, an NVMe SSD and one
+NVIDIA GPU (no images yet). Details and measurements: [UD-Q4_K_XL](UNSLOTH_Q4.md).
+
+```
+START-HERE.bat --setup --family unsloth --model UD-Q4_K_XL
+```
+
+### OrcaRouter Uncensored IQ3_XXS
+
+For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](ORCA.md). It needs an
+explicit packing conversion and is not an installer menu option.
+
+## Adding or switching models
+
+You can add another model any time with `SETUP.bat` (the same as `START-HERE.bat --setup`; on Linux
+`./setup.sh --setup`). Files another model shares are not downloaded again (the Coder uses the original's shard 2 and
+vision encoder). With more than one model installed, `START-HERE.bat` asks which one to start; `run-<model>.bat`
+(Linux: `run-<model>.sh`) starts one directly.

docs/MULTI_GPU.md

@@ -8,7 +8,8 @@ speed comes from (decode then barely touches the CPU pool).
 
 This is pipeline (layer) parallelism, not tensor parallelism: a token crosses from one card to the next once per
 verify window (a few hundred KB through pinned RAM), not twice per layer. No NVLink or peer-to-peer access is
-needed; cards on x4 or x1 slots work, and the PCIe share of each card is probed on its own link.
+needed; cards on x4 or x1 slots work, and the PCIe share of each card is probed on its own link. A `--pcie-frac` you give
+is every card's share and skips those probes; there is no per-card setting yet.
 
 ## Using it
 
@@ -43,7 +44,8 @@ now on; the answer is kept.
 **Not supported** (setup says so and names the cards that can be used instead):
 - a card older than the RTX 20 series (compute capability below 7.5: GTX 10 and older);
 - a card with less than 8 GB of VRAM, together with others (each card holds a copy of the dense weights and its
-  own prompt buffers);
+  own prompt buffers) - unless you name it with `--gpus`: then setup says the risk and asks (`--yes` with the named
+  cards goes ahead);
 - Intel GPUs, and a mix of NVIDIA and AMD cards. (AMD cards share a model among themselves: `./setup.sh --backend
   hip --gpus 1,0`, see [AMD_HIP.md](AMD_HIP.md).)
 

docs/SECOND_GPU.md

@@ -83,3 +83,42 @@ short decode requests.
 The existing warning about the CUDA0 expert-cache GPU hit path still applies:
 its outputs diverge from cache-off runs. Treat performance as experimental
 until the generated tokens have been validated.
+
+## Peer tier (`--peer-device`)
+
+`--peer-device N` puts a second adaptive expert cache on CUDA device N. It
+takes the ranked pairs CUDA0's cache does not hold, as many as fit. The peer
+computes the rows of its own experts, for decode windows and for prompt
+chunks; the activations and the results cross NVLink or another P2P path. The
+tier adapts while the server runs, like the primary cache. It is an
+alternative to the CUDA1-3 caches above, not a third tier beside them.
+
+- `--peer-device N` (default off): enable the tier on CUDA device N (N >= 1).
+- `--peer-reserve-mib M` (default 600): leave M MiB free on the peer card; the
+  cache takes what remains. The prompt-path buffers need this headroom.
+- `--peer-slots N` (default 0): cap the tier at N experts; 0 = as many as fit.
+- `--peer-adapt-swaps N` (default -1): swaps per adaptive round on the peer;
+  -1 uses the primary's `--adapt-swaps`.
+- `--peer-prefill-rows N` (default -1): the share of each prompt chunk's rows
+  the peer computes; -1 is half of chunk x top-k, 0 keeps prompt rows on the
+  primary.
+
+`--peer-device` requires `--expert-profile` and an enabled expert cache, and
+the device must be visible; it refuses otherwise. It also refuses
+`--layer-split` (a different second-GPU mode: use one or the other) and
+`--expert-cache-device1..3` (the peer tier already caches experts on that card):
+
+    strata generate: --peer-device cannot be combined with --layer-split (use one or the other)
+    strata generate: --peer-device cannot be combined with --expert-cache-device1..3 (the peer tier already caches experts there)
+
+Without `--peer-device` the binary is unchanged; its output is byte-identical
+to the release. With `--peer-device` and the same expert set split across the
+two cards, the generated tokens are byte-identical to the single-GPU run under
+the exactness gate.
+
+With a peer the prompt path keeps the MMQ path; the fused int8 prompt path is
+not yet combined with the peer's rows. Mapped host buffers gain
+`cudaHostAllocPortable` only with a peer, since only then does a second
+context write them. The tier size is manual for now (`--peer-reserve-mib`,
+`--peer-slots`); automatic sizing on small cards wants the buffer lending of
+#216 and is a follow-up.

docs/TROUBLESHOOTING.md

@@ -0,0 +1,70 @@
+# Something went wrong?
+
+The common problems and what to do. Back to the [README](../README.md#something-went-wrong). The full table of
+error messages, with the older engine fixes, is in the [details](DETAILS.md#troubleshooting).
+
+## While installing or starting
+
+**My PC froze, or got very slow, the first time Strata started.**
+That's normal while it starts, most of all the first time. Strata loads 35-55 GB into your RAM, locks part of it for
+the graphics card, and works out how much of the model fits on your GPU. The mouse can freeze for a few minutes.
+**Wait, and don't close the window.** The next starts are much faster. Still frozen after 10 minutes? Restart the
+PC, close other programs (browsers use a lot of RAM) and try again. If it keeps happening, pick a smaller size (Q2_0
+or IQ2_XS).
+
+**It stopped while downloading or installing.**
+Run `START-HERE.bat` (Linux: `./setup.sh`) again. It continues where it stopped.
+
+**It says the NVIDIA driver is too old.**
+Update it (NVIDIA App or [nvidia.com/drivers](https://www.nvidia.com/drivers)), restart the PC, and run
+`START-HERE.bat` again.
+
+**It says port 8080 is already in use.**
+Strata is already running. Look for its window. Or another program uses the port: `START-HERE.bat --port 8081`.
+
+**Python or the build tools could not be installed.**
+Install what it names (links are printed), then run it again. Everything already done is kept.
+
+**The first start takes minutes.**
+It is reading 34-55 GB into RAM; the second start is faster while the files are in the OS cache. Started from Task
+Scheduler, it can be 24x slower: see [Running it at startup](DETAILS.md#running-it-at-startup-task-scheduler).
+
+## While it answers
+
+**It's very slow and the disk light keeps blinking.**
+Your PC is out of free RAM. Close other programs, or pick a smaller size (Q2_0 or IQ2_XS).
+
+**An answer stopped with "the engine stopped unexpectedly".**
+Usually not enough RAM (on Linux the system then stops the engine). Just send your message again: Strata starts the
+engine by itself. If it keeps happening, close other programs or pick a smaller size.
+
+**It says the prompt exceeds the context.**
+The conversation is longer than the context you chose. Start a new chat, or run `SETUP.bat` and pick more
+context.
+
+**It is slower than the tables.**
+The monitor plugged into the graphics card and other GPU programs take VRAM from the expert cache; RAM running below
+its rated speed (enable EXPO/XMP in the BIOS) slows the CPU half.
+
+**Pictures are refused, or slow.**
+"this server was started without the vision encoder": the model was set up for text only - run setup again with
+`--vision gpu` (or `--vision cpu`). Pictures that take 10-30 s are read by the encoder on the CPU; `--vision gpu`
+(NVIDIA, ~1.4 GB of VRAM) makes it 0.1-0.5 s.
+
+## AMD cards
+
+**"No AMD GPU found (the amdgpu driver's KFD topology is empty)" on Linux.**
+The kernel's amdgpu driver is not loaded for the card. Integrated Radeon GPUs are listed as not supported; setup
+lists every card it found and whether Strata can use it.
+
+**The engine stops at start with the card's name, its architecture and the build's list.**
+The engine was compiled for another card (for example after moving the Strata folder to another PC). Run
+`./setup.sh --setup --backend hip`: it compiles the engine for this card's architecture.
+
+**Large pinned host allocations fail on ROCm although RAM is free.**
+See [AMD_HIP.md](AMD_HIP.md#model-and-serving-configuration): the mapped expert mode avoids the full pinned arena.
+
+## Still stuck?
+
+Look in the [full troubleshooting table](DETAILS.md#troubleshooting), or open an
+[issue](https://github.com/Niko1221/Strata/issues) and attach `strata-<model>.log` from the Strata folder.

docs/UNSLOTH_Q4.md

@@ -22,15 +22,26 @@ START-HERE.bat --setup --family unsloth --model UD-Q4_K_XL
 
 What setup does differently for this model:
 
-- It needs 48 GB of RAM or more and engine 0.1.32 or newer (checked before anything is downloaded), and an NVIDIA
-  GPU. One GPU only: with `--gpus` it uses the first one and says so. No images (the vision encoder is not wired to
-  this file yet) and no experimental speed projection (not tested with it).
+- It needs 48 GB of RAM or more (with less it asks, default no; `--model UD-Q4_K_XL --yes` installs it anyway) and
+  engine 0.1.32 or newer (checked before anything is downloaded), and an NVIDIA
+  GPU: it has not been run on AMD cards (its prompt kernels for the Q4_K / Q5_K experts are NVIDIA-only), so with
+  `--backend hip` setup says so and asks before the download (#429; `--model UD-Q4_K_XL --yes` tries it). One GPU
+  by default: the RAM budget below has no layer split (the engine refuses `--resident-budget-gib` with one). No
+  images (the vision encoder is not wired to this file yet) and no experimental speed projection (not tested with it).
+- Several GPUs (#498): when the RAM holds the GGUF files and 24 GB more (~135 GB of RAM) and two or more cards can
+  share it, setup asks (one GPU stays the default; `--gpus 0,1` takes the split). The split runs **without** the RAM budget: all 77 GB of experts are loaded into RAM from the GGUFs at
+  start, the files pass through the OS file cache while they load, and the config gets `"gpu": [0, 1]` and
+  `"layer_split": "auto"`. Measured on 2x RTX 3090 with 165 GiB (#498): decode 31 tok/s on one card with the budget,
+  64-78 tok/s split (55 tok/s at a 128K prompt), with `MemAvailable` never under 68 GiB. With less RAM, `--gpus`
+  keeps one GPU and says so, and `START-HERE.bat --gpus 0,1` on an installed UD-Q4_K_XL stops with the reason
+  (it used to keep the budget, and the engine exited with code 2).
 - It downloads the four shards below from the pinned revision `38bb39e` (resumable, like the other models), then
   checks each one's size and SHA-256 against the table below; the check takes a few minutes once and is remembered
   in the file's finish mark. A file with the wrong hash is deleted, so the next run downloads it again.
 - It packs with `--compat-bf16` and never writes `experts.bin` (`--low-ram` does not apply).
 - The RAM budget is the PC's RAM less 24 GB: `--resident-budget-gib 40` on 64 GB, at most all 71 GiB of experts on
-  96 GB or more. From a 64K context up, where the KV cache moves to RAM, its size comes out of the budget.
+  96 GB or more. From a 64K context up, where the KV cache moves to RAM, its size comes out of the budget. Setup's
+  `--resident-budget-gib N` sets another one (a bigger one is kept, with a note on what it risks).
 - It recommends an 8K context on a GPU under 14 GB (every GB of KV cache is a GB less of cached experts).
 
 ## The files
@@ -130,7 +141,7 @@ budget take first.
 
 **Choosing the RAM budget N:** your RAM minus 20-24 GB (the OS, the engine itself, the 126 MB router copy, and room for
 the OS file cache that serves the rest). On 64 GB, 40. The engine clamps a budget larger than the RAM it finds free
-(minus 4 GB) and says so. Everything above N comes from the SSD for every token, so N is the setting that matters
+(minus 4 GB and a 256 MiB margin) and says so; before #403's fix such a clamped budget could then fail the start. Everything above N comes from the SSD for every token, so N is the setting that matters
 most; a bigger budget was faster in every measurement (24 / 32 / 40 GiB). When the driver page-locks the whole budget
 (24 GiB did on this PC, 32 and 40 did not), the GPU also computes a share of the misses over PCIe, as in the resident
 low-RAM mode; `STRATA_RESIDENT_PIN=0` keeps the budget locked only (the CPU then computes every miss).

include/strata/core/device.hpp

@@ -38,6 +38,16 @@ std::string gpu_arch_problem(int ordinal);
 // The GPU architectures this binary was compiled for ("gfx1100,gfx1201"); "" on CUDA builds.
 const char* compiled_gpu_archs();
 
+// How many devices the runtime enumerates, numbered as HIP_VISIBLE_DEVICES / CUDA_VISIBLE_DEVICES number them; 0 when
+// there is none (or no usable runtime).  `device_info` throws on a card this binary cannot run, so a caller that LISTS
+// every card - the ones it has no code for included (`strata-device --list-devices`) - counts them with this first.
+int device_count();
+
+// One line per device for `strata-device --list-devices`, without the arch check that `device_info` applies:
+// "arch gfx1201, 15.9 GiB" (HIP) or "compute capability 12.0, 11.9 GiB" (CUDA), plus the device's name.  false when
+// the runtime cannot describe it.
+bool device_summary(int ordinal, std::string& name, std::string& detail);
+
 // Throws when there is no CUDA device.  The engine targets sm_120 specifically and must say so rather than
 // run slowly on something else: `CMakeLists.txt` already refuses to COMPILE for another architecture, and
 // this is the matching check at run time (a binary can be carried to a different machine).

include/strata/core/expert_cache.hpp

@@ -55,6 +55,24 @@ inline constexpr int32_t kNotResident = -1;
 bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_expert,
                          std::vector<std::pair<int32_t, int32_t>>& ranked, int64_t& slots, std::string& err);
 
+/// #477 (--expert-profile-save): what the adaptive tier learned, as a profile ranking EVERY (layer, expert) pair -
+/// the experts resident in VRAM now first (where the swaps left the cache), then the rest; within each, by `heat`
+/// (the routing the adaptive tier counted since the start, descending), then by `prior` (the profile the engine
+/// started from: an expert this run never routed keeps its old place), then by index.  A start from it begins
+/// where this one ended; one with more slots adds the hottest of the rest, one with fewer keeps the hottest.
+/// `resident` and `heat`: n_layers x n_expert entries (`heat` may be empty: no counts, the prior decides).
+std::vector<std::pair<int32_t, int32_t>> rank_learned_profile(int64_t n_layers, int64_t n_expert,
+                                                              const std::vector<uint8_t>& resident,
+                                                              const std::vector<double>& heat,
+                                                              const std::vector<std::pair<int32_t, int32_t>>& prior);
+
+/// #477: writes `ranked` in tools/make_profile.py's format, byte for byte: `STRP`, `1, n_layers, n_expert,
+/// n_ranked, n_ranked` (uint32), the pairs (uint16 layer, uint16 expert), then the n_layers x n_expert int32 table
+/// of each pair's rank (-1: not ranked).  Atomically: a `<path>.tmp` beside it, renamed over `path` once complete,
+/// so a reader (the next start) never sees half a file.  False with `err` when it could not be written.
+bool write_expert_profile(const std::string& path, int64_t n_layers, int64_t n_expert,
+                          const std::vector<std::pair<int32_t, int32_t>>& ranked, std::string& err);
+
 class ExpertCache {
 public:
     ExpertCache() = default;

include/strata/core/expert_source.hpp

@@ -43,6 +43,7 @@ struct ExpertLayout;
 
 namespace strata::core {
 
+class PeerExperts;   // multi-GPU: the second GPU's expert tier (peer_experts.hpp)
 class RemoteExperts;
 struct LoadStats;
 
@@ -336,6 +337,9 @@ struct ExpertDispatch {
     std::vector<float> usage;
     int64_t multi_misses = 0;      ///< distinct (layer, expert) pairs the CPU computed in verify windows
     int64_t multi_entries = 0;     ///< routed (token, expert) entries the CPU served in verify windows
+    /// Multi-GPU: the second GPU's tier.  Its experts are computed there instead of on the CPU (kind 2).
+    PeerExperts* peer = nullptr;
+    int64_t peer_entries = 0;      ///< routed entries the peer served in verify windows
     /// Set when `dispatch` could not produce an answer.  The loop itself has no error channel, so this is
     /// where a source failure surfaces: the driver checks it after `session_loop` returns rather than the
     /// engine computing from a half-filled `parts`.
@@ -421,6 +425,10 @@ public:
     /// CS-T, `budget_bytes` > 0 (`--resident-budget-gib`): only as many of those experts as fit `budget_bytes`, taken
     /// in `rank` order (the expert profile: the hottest after the GPU cache's), are copied; the rest stay on the
     /// mapped files (the SSD tier).  No lend region then (a lent slot's expert is read from the files).
+    /// #467: `budget_bytes` = `kResidentWhatFits` is that path sized by the RAM alone (available minus the headroom
+    /// and the #403 margin) - the soft --resident-experts mode's second try when the whole complement does not fit;
+    /// false when not even one expert fits.  On Windows the mapped experts leave the working set before any reading.
+    static constexpr uint64_t kResidentWhatFits = ~0ull;
     bool pin_cache_complement(
         const ExpertCache& cache, std::string& err, bool pin = true,
         const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs = {}, int64_t lend_from_slot = -1,
@@ -467,6 +475,16 @@ public:
     double file_ms() const { return (double) file_us_.load(std::memory_order_relaxed) / 1000.0; }
     /// Threads `prefetch` reads the GGUF with (STRATA_FETCH_THREADS, default 8).
     void set_fetch_threads(int n) { fetch_threads_ = n < 1 ? 1 : n; }
+    /// #286 (Windows): read the experts straight from the drive (FILE_FLAG_NO_BUFFERING, overlapped) instead of
+    /// through the mapped files - the GGUF in place, or a pack's experts.bin - when the file cache could not keep them
+    /// beside `ram_bytes` (the RAM budget), cached now or not (their mapped pages would land in the working set); see
+    /// experts_unbuffered.  The mapped reads' page faults are one small request each, and the pages they bring in
+    /// take the RAM the budget was sized for.  `why` says what decided.
+    bool set_unbuffered(uint64_t ram_bytes, std::string& why);
+    bool unbuffered() const { return !direct_.empty(); }
+    /// #286, unbuffered: assembles the blobs of these pairs ahead of the `blob` calls that will ask for them (the
+    /// GPU cache's fill from the profile) - one batch of reads instead of one blob at a time.  At most 64 pairs.
+    void prefetch_pairs(const std::pair<int32_t, int32_t>* pairs, int64_t n);
 
     const uint8_t* blob(int64_t layer, int64_t expert) override;
     bool pinned(int64_t layer, int64_t expert) const override;
@@ -481,7 +499,8 @@ public:
     /// CS-T: the GGUF in place asks the OS for the predicted experts' pages (PrefetchVirtualMemory on Windows,
     /// madvise(WILLNEED) elsewhere), skipping the RAM copy's.
     void warm(int64_t layer, const int64_t* experts, int64_t n) override;
-    bool warms() const override { return !role_ptr_.empty(); }
+    /// Not when the reads are unbuffered: the warmed pages would be read through the file cache, a second time.
+    bool warms() const override { return !role_ptr_.empty() && direct_.empty(); }
     /// Of the blobs the file tier read for the decode, how many had been warmed for their layer beforehand.
     int64_t warmed_hits() const { return warm_hits_.load(std::memory_order_relaxed); }
     int64_t warmed() const { return warm_count_.load(std::memory_order_relaxed); }
@@ -498,6 +517,18 @@ private:
     const uint8_t* staged_blob(int64_t layer, int64_t expert);
     bool claim_stage(int64_t key, size_t& v, bool& fill);
     bool fill_stage(size_t v, int64_t layer, int64_t expert, uint8_t* dst);
+    void publish_stage(size_t v, int64_t layer, bool ok, double us);
+    struct Fill { size_t v; int64_t layer, e; uint8_t* dst; };
+    /// The claimed buffers' blobs: one overlapped batch when unbuffered, else the fetch threads' mapped copies.
+    void fill_many(const std::vector<Fill>& todo);
+    /// #286: the blobs from the drive, unbuffered: every role's 4 KiB-aligned window is read at once (overlapped)
+    /// into this thread's aligned buffer, then copied into place.  False when a read fails.
+    bool read_direct(const Fill* fills, size_t n) const;
+    std::vector<std::string> paths_;          ///< the mapped files, as maps_
+    std::vector<void*> direct_;               ///< #286: per file, an unbuffered overlapped handle (Windows)
+    std::vector<int> role_file_;              ///< 3 x n_layers: index into maps_ / direct_
+    /// blobs assembled in the stage buffers (`blob` hands those out): the GGUF in place, or any unbuffered source
+    bool staged() const { return !role_ptr_.empty() || !direct_.empty(); }
     static constexpr uint64_t kNoComplement = detail::kNoCacheComplement;
     // ---- CS-T: the GGUF shards in place
     std::string gguf_;
@@ -644,7 +675,8 @@ private:
 /// layout's type and dimensions, and inside the file.  `native` is the --native shard (see set_gguf).
 bool check_experts_gguf(const std::string& native, const strata::kernels::cpu::ExpertLayout& lay, std::string& err);
 /// Fills `dst` (lay.total bytes, the experts.bin layout) from the GGUF files, one role at a time.
+/// `unbuffered`: each chunk read past the file cache (Windows).
 LoadStats load_experts_gguf(const std::string& native, uint8_t* dst, const strata::kernels::cpu::ExpertLayout& lay,
-                            int threads);
+                            int threads, bool unbuffered = false);
 
 }  // namespace strata::core

include/strata/core/peer_experts.hpp

@@ -0,0 +1,106 @@
+// include/strata/core/peer_experts.hpp - multi-GPU: a second expert tier on another GPU.
+//
+// The primary GPU (device 0) keeps everything it had: the dense weights, the KV cache, the MTP layer, its own
+// adaptive expert tier and the captured verify graph.  This class adds an expert tier on a SECOND card.  From
+// the verify graph's point of view nothing changes: for each layer the pool thread decides which routed experts
+// the primary computes (its VRAM tier, the PCIe share), and every remaining expert is a "CPU row" the graph reads
+// from mapped host memory after the pool raises its flag.  The peer takes the rows whose experts it holds: the
+// pool thread launches them on the peer right after publishing the primary's plan, the CPU computes the rest at
+// the same time, and the peer's rows are written into the same mapped rows before the pool returns.
+//
+// The peer computes with the SAME kernels as the primary's hit path (q8_1 activations + native_expert_grouped),
+// from the same activation values, so an expert gives bit-identical rows on either card.
+//
+// The tier follows the conversation like the primary's: `adapt` swaps the most-routed experts that neither card
+// holds into the peer's least-used slots (per layer, so a slot keeps its layer's blob size).
+#pragma once
+
+#include "strata/core/expert_cache.hpp"
+#include "strata/core/expert_source.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace strata::core {
+
+/// multi-GPU: set once at startup when a peer device is configured (--peer-device N).  The mapped host buffers
+/// the peer card writes into are then allocated with cudaHostAllocPortable; without a peer the flag stays off.
+void set_peer_portable(bool on);
+bool peer_portable();
+
+class PeerExperts {
+public:
+    PeerExperts() = default;
+    ~PeerExperts();
+    PeerExperts(const PeerExperts&) = delete;
+    PeerExperts& operator=(const PeerExperts&) = delete;
+
+    /// Opens the tier on `device`: fills it with the first pairs of `ranked` that `primary` does not hold, as many
+    /// as fit in the card's free memory minus `reserve_mib` (`max_slots` > 0 caps the count).
+    bool open(int device, const std::vector<std::pair<int32_t, int32_t>>& ranked, const ExpertCache& primary,
+              ExpertSource& src, int64_t n_layers, int64_t n_expert, int reserve_mib, int64_t max_slots,
+              std::string& err);
+    void close();
+    bool valid() const { return device_ >= 0; }
+    int device() const { return device_; }
+
+    /// Whether the peer holds (layer, expert) right now.
+    bool has(int64_t layer, int64_t expert) const { return res_[(size_t) (layer * n_expert_ + expert)] >= 0; }
+
+    /// The device address of (layer, expert)'s blob on the peer, or null when it is not resident.
+    const uint8_t* slot_ptr(int64_t layer, int64_t expert) {
+        const int32_t sl = res_[(size_t) (layer * n_expert_ + expert)];
+        return sl >= 0 ? cache_.device_slot(sl) : nullptr;
+    }
+    /// Whether the primary (device 0) and the peer can read each other's memory directly (NVLink / PCIe P2P).
+    bool p2p() const { return p2p_; }
+
+    /// Starts the peer's share of one layer: the entries i with kind[i] == 2 (x: n_tok rows of n_embd floats).
+    /// multi-GPU: with `out` (pinned, portable host rows) the peer writes its rows straight into
+    /// out[entry] with a scatter kernel (no D2H copy + host memcpy in finish); STRATA_PEER_DIRECT=0: the old way.
+    bool launch(int64_t layer, const float* x, const int32_t* ids, int64_t n_tok, int64_t k, const int32_t* kind,
+                std::string& err, float* out = nullptr);
+    /// Waits for the launched share and writes its rows into `out` (row i = entry i, n_embd floats).
+    bool finish(float* out, std::string& err);
+
+    /// Adaptive tier: swap up to `max_swaps` experts routed at least twice that neither the primary (`res0`, slot
+    /// or -1) nor the peer holds into the peer's least-used slots.  Copies run on the peer's refill stream; a
+    /// swapped-in expert becomes resident at `apply_pending`.  Call only between verify windows.
+    bool adapt(const float* usage, const int32_t* res0, int max_swaps, std::string& err);
+    void apply_pending(bool wait);
+
+    int64_t resident() const { return resident_; }
+    /// the last launch writes its rows into the host rows itself (the caller must not touch them)
+    bool launched_direct() const { return launched_direct_; }
+    double gib() const { return cache_.gib(); }
+    int64_t entries() const { return entries_; }      ///< routed entries the peer computed
+    int64_t experts() const { return experts_; }      ///< distinct (layer, expert) the peer computed
+    int64_t swaps() const { return swaps_; }
+    double ms_wait = 0;                               ///< host time spent in finish() waiting for the peer
+
+private:
+    int device_ = -1;
+    bool p2p_ = false;
+    int64_t n_layers_ = 0, n_expert_ = 0;
+    ExpertCache cache_;
+    ExpertSource* src_ = nullptr;
+    std::vector<int32_t> res_;                        ///< [n_layers * n_expert] -> peer slot or -1
+    std::vector<std::pair<int32_t, int32_t>> pending_;   ///< (residency index, slot) once the copies land
+    int64_t resident_ = 0, entries_ = 0, experts_ = 0, swaps_ = 0;
+    cudaStream_t stream_ = nullptr, refill_ = nullptr;
+    cudaEvent_t refill_ev_ = nullptr;
+    float* h_x_ = nullptr;   float* d_x_ = nullptr;   // n_tok x n_embd
+    float* h_out_ = nullptr; float* d_out_ = nullptr; // compact rows
+    void* h_meta_ = nullptr; void* d_meta_ = nullptr; // groups: ptr | start | dst | tok | count
+    uint8_t* d_q8_ = nullptr;
+    void* d_scratch_ = nullptr;
+    std::vector<int32_t> row_of_;                     ///< compact row -> original entry
+    int64_t launched_rows_ = 0;
+    bool launched_direct_ = false;
+};
+
+}  // namespace strata::core

include/strata/core/pinned.hpp

@@ -104,6 +104,22 @@ LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_byte
 LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
                               const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk);
 
+/// The same ranges read UNBUFFERED straight into `dst` (Windows): no staging buffer and no file-cache copy - the
+/// drive's DMA lands where the experts live. Every range, `dst` and `chunk` must be 4 KiB aligned; returns ok =
+/// false with an empty `error` when they are not (or off Windows), and the caller falls back to load_experts_ranges.
+LoadStats load_experts_direct(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
+                              const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk);
+
+/// Whether the expert files are better read unbuffered (Windows; false elsewhere): a timed probe of random 64 KiB
+/// reads says they are not in the OS file cache (a cached read takes ~10 us, the drive ~80), and the RAM left
+/// beside the arena (`arena_bytes`) could not keep them cached for the next start either - so a warm restart after
+/// an idle unload never gets slower, and only a start that reads the drive anyway skips the cache's copy.
+/// STRATA_UNBUFFERED_LOAD=1 / 0 forces it. `why` says what decided.
+/// `cache_counts` false (the file tier with a RAM budget): only whether the files could be kept decides - their mapped
+/// pages land in the process's working set, so a partly cached file would still take the RAM the budget was sized for.
+bool experts_unbuffered(const std::vector<std::string>& files, uint64_t arena_bytes, std::string& why,
+                        bool cache_counts = true);
+
 // FNV-1a 64.  Per layer, so a corrupt or short read names WHICH layer rather than just failing a whole-file
 // comparison - the same reason the Phase 1 tools report the first differing element.
 uint64_t fnv1a64(const uint8_t* p, uint64_t n, uint64_t seed = 1469598103934665603ull);

include/strata/core/verify.hpp

@@ -254,6 +254,7 @@ private:
     int32_t* plan_ = nullptr;                                     // device copy of the plan block
     uint8_t* staging_ = nullptr;                                  // VRAM slots for the PCIe share of the misses
     static constexpr int64_t kStagingBlobs = 16;
+    static constexpr int64_t kPcieGroupRows = 4;                  // the PCIe call's groups side by side (of <= 16)
     uint8_t* hit_xq_ = nullptr;
     uint8_t* nat_xq_ = nullptr;   // plan v0.3 P6: q8_1 activations for a native pack's grouped experts
     float* hit_xs_ = nullptr;

include/strata/hip_compat/cuda_runtime.h

@@ -1,5 +1,14 @@
 #pragma once
 // Included only by STRATA_ENABLE_HIP builds. CUDA builds use NVIDIA headers.
+#if defined(_WIN32)
+// hip_runtime.h pulls in <windows.h> on Windows: keep its min/max macros and the rarely used APIs out of the engine.
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#endif
 #include <hip/hip_runtime.h>
 // Do not let HIP's legacy macro corrupt libstdc++ attribute names.
 #ifdef __noinline__
@@ -9,11 +18,15 @@
 #define cudaDevAttrMultiProcessorCount hipDeviceAttributeMultiprocessorCount
 #define cudaDevAttrClockRate hipDeviceAttributeClockRate
 #define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor
+#define cudaDevAttrComputeCapabilityMinor hipDeviceAttributeComputeCapabilityMinor
 #define cudaDeviceGetAttribute hipDeviceGetAttribute
+#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer
+#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess
 #define cudaDeviceProp hipDeviceProp_t
 #define cudaDeviceSynchronize hipDeviceSynchronize
 #define cudaDriverGetVersion hipDriverGetVersion
 #define cudaErrorNotReady hipErrorNotReady
+#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled
 #define cudaErrorStreamCaptureUnsupported hipErrorStreamCaptureUnsupported
 #define cudaError_t hipError_t
 #define cudaEventCreate hipEventCreate
@@ -52,10 +65,22 @@
 #define cudaLaunchHostFunc hipLaunchHostFunc
 #define cudaMalloc hipMalloc
 #define cudaMallocHost(...) (::strata::hip_compat::malloc_host(__VA_ARGS__))
+#if defined(_WIN32)
+// Windows: hipMemGetInfo counts this process's own allocations only, so the desktop's and other programs' share of
+// the card is invisible to it, and VRAM sized by it overfills the card (WDDM then moves memory out to system RAM and
+// the GPU slows down).  Its free figure is lowered by what Windows' video memory budget for this process withholds
+// (src/core/device.cu; STRATA_WDDM_BUDGET=0: hipMemGetInfo as it is).
+namespace strata::hip_compat {
+hipError_t mem_get_info(size_t* free_bytes, size_t* total_bytes);
+}
+#define cudaMemGetInfo ::strata::hip_compat::mem_get_info
+#else
 #define cudaMemGetInfo hipMemGetInfo
+#endif
 #define cudaMemcpy hipMemcpy
 #define cudaMemcpy2DAsync hipMemcpy2DAsync
 #define cudaMemcpyAsync hipMemcpyAsync
+#define cudaMemcpyPeerAsync hipMemcpyPeerAsync
 #define cudaMemcpyDefault hipMemcpyDefault
 #define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice
 #define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost

include/strata/kernels/cpu/iq_avx2.hpp

@@ -1,5 +1,5 @@
 // include/strata/kernels/cpu/iq_avx2.hpp - AVX-2 multi-token dot products for the i-quant expert
-// formats (IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S) against Q8_K activations (ggml's block_q8_K).
+// formats (IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS) against Q8_K activations (ggml's block_q8_K).
 //
 // The AVX-512 kernel's scheme on the CPUs without it (AMD Zen 2/3, Intel Core 12th-14th gen): the weights of a
 // 32-value chunk are decoded once per verify window and every token applies them with five instructions.

include/strata/kernels/elementwise.hpp

@@ -104,6 +104,8 @@ void doorbell_wait(const uint32_t* d_flag, const uint32_t* d_seq, void* stream);
 void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream);
 /// `rows` rows of `width` floats from mapped memory, except the rows listed in hit_rows[0, *count) (device),
 /// which are written +0.0 instead (a verify window's GPU-computed entries: the pool leaves zeros there).
+/// multi-GPU: dst[rows[r] * width ..] = src[r * width ..] for r < n (float4; dst may be mapped host memory).
+void scatter_rows_f32(const float* src, float* dst, const int32_t* rows, int64_t n, int64_t width, void* stream);
 void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
                            const int32_t* count, void* stream);
 

include/strata/kernels/iq_kernels.hpp

@@ -56,9 +56,17 @@ size_t native_expert_scratch_bytes(int64_t cap_entries, int64_t n_ff);
 /// Grouped experts in the native format: group g's blob at device address grp_ptr[g]; its entries
 /// [grp_start[g], grp_start[g+1]) read token ent_tok[e]'s q8_1 activation (n_embd/32 blocks per token in x_q8_1)
 /// and write row ent_dst[e] of `out` (n_embd floats).  Counts are read on the device.
+/// `grid_groups` (1 .. cap_groups; 0 = cap_groups) groups run side by side, a block row each striding over the rest:
+/// a call that usually has few groups or none (the verify window's PCIe share) launches less for the ones it does
+/// not have.  The results do not depend on it.
 void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long* grp_ptr, const int32_t* grp_start,
                            const int32_t* n_groups, const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups,
-                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream);
+                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream,
+                           int64_t grid_groups = 0);
+/// true: `native_expert_grouped`'s launches before the group stride (STRATA_GROUPED_V1=1 at startup) - a block row
+/// per possible group, SwiGLU and the q8_1 quantization as two kernels over all cap_entries.  Bitwise the same results
+/// (native_grouped_parity checks it); kept for A/B timing.  Set before graph capture; captured graphs keep theirs.
+void native_grouped_set_v1(bool v1);
 
 /// `iq_mmvq` and `native_expert_grouped` decode each weight part once and apply it to every column / entry;
 /// true selects the older kernels that decode it again per column (STRATA_OLD_IQ_MMVQ=1 at startup).  Both give

include/strata/kernels/ngram.hpp

@@ -114,7 +114,7 @@ enum class PleIo { Direct, Mmap };
 
 struct PleIoOptions {
     PleIo mode = PleIo::Direct;
-    uint32_t max_inflight = 64;      ///< outstanding SSD reads (decode needs 16; prefill chunks use more)
+    uint32_t max_inflight = 256;     ///< outstanding SSD reads (decode needs 16; a prompt chunk saturates the SSD at 256)
     uint64_t cache_rows = 1u << 20;  ///< bounded row cache: 1,048,576 rows x 90 B ~ 95 MB; 0 disables
     bool io_thread = true;           ///< reads submitted by a worker thread, not the caller
     /// Mmap mode only (`--ple-io ram`): lock the whole mapped table in RAM at open, so no SSD read ever sits on

include/strata/kernels/qsa_prompt_attn.hpp

@@ -14,7 +14,8 @@
 //   * q and p are split into FP16 hi + lo parts (two MMAs each), so they keep ~22 bits: the result differs from
 //     the FP32 kernel by summation order and the exp2 rounding, not by an FP16 cast.
 // Not bitwise equal to `qsa_decode_attn_batch`; `qsa_prompt_attn_parity` bounds the difference and the prompt
-// quality gate (needles, teacher-forced top-1) checks it end to end. Q4_0 KV is not handled (returns false).
+// quality gate (needles, teacher-forced top-1) checks it end to end. Q4_0 KV (mode 4): each block's codes enter
+// exactly as int8 and its scale (one per 32 values) in FP32, as int8 KV's do (STRATA_PROMPT_ATTN_Q4=0: the old kernel).
 #pragma once
 
 #include "strata/kernels/qsa_decode_attn.hpp"
@@ -24,7 +25,7 @@
 namespace strata::kernels {
 
 /// Same arguments and output as `qsa_decode_attn_batch` minus the scratch. Returns false (nothing launched) when the
-/// pools are Q4_0 or the geometry is not 24 heads / 2 KV heads / 256: the caller then uses the old kernel.
+/// geometry is not 24 heads / 2 KV heads / 256: the caller then uses the old kernel.
 bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
                            int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream);
 

include/strata/kernels/qsa_select.hpp

@@ -11,7 +11,8 @@
 //                      a chunk it may already hold a block completed later, so it is not read).
 //   qsa_block_topk   : one block per query; a 4-pass radix select over the query's n_bid + 1 blocks, each
 //                      weighted by its cell count, then the cells emitted in ascending order with ties to the
-//                      lowest index - the same selection as topk_kernel, over a quarter of the elements.
+//                      lowest index - the same selection as topk_kernel, over a quarter of the elements.  On sm_90+
+//                      a call of up to 16 queries runs it on a cluster of 8 CTAs per query: the same ids.
 //
 // Queries carry their own step record (pos, n_kv, n_bid, width) as everywhere else in QSA.
 #pragma once
@@ -40,7 +41,14 @@ void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64
                     int64_t active_blocks = -1);   ///< > 0: no query of the call has more than this many blocks (n_bid + 1;
                                                    ///< the same contract as qsa_block_scores's). The register kernel is
                                                    ///< chosen by this, not by the capacity max_blocks (a long
-                                                   ///< --max-context otherwise sends every short prompt to the slow one)
+                                                   ///< --max-context otherwise sends every short prompt to the slow one).
+                                                   ///< CUDA uses the bound only on sm_75; HIP keeps its existing policy.
+                                                   ///< Omit it for captured graphs whose context can grow after capture.
+/// The same ids on a thread-block cluster of 8 CTAs per query (sm_90+, CUDA; S19). qsa_block_topk takes it for calls
+/// of up to 16 queries unless STRATA_QSA_CLUSTER=0. False (nothing launched) where it cannot run: HIP, a card or a
+/// build below sm_90, or a capacity whose keys do not fit one cluster's shared memory. Capturable.
+bool qsa_block_topk_cluster(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                            const QsaShapes& s, int32_t* ids, void* stream);
 /// The original kernel (keys read from memory on every radix pass), for tests: the same ids.
 void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
                         const QsaShapes& s, int32_t* ids, void* stream);

include/strata/kernels/sampler.hpp

@@ -42,6 +42,11 @@ struct SamplerParams {
 void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
                    const SamplerParams& p, int* out, void* stream);
 
+// The greedy pick (no penalties) on a thread-block cluster of 8 CTAs per row (sm_90+, CUDA; S19): the same token as
+// sample_tokens' one-block argmax, which takes it unless STRATA_ARGMAX_MULTI=0.  False (nothing launched) where it
+// cannot run: HIP, a card or a build below sm_90.  Device pointers; capturable.
+bool sample_greedy_cluster(const float* logits, int n_tokens, int n_vocab, int* out, void* stream);
+
 // ---- COUPLED DRAFT SAMPLING (include/strata/core/coupled_draft.hpp, STRATA_SPEC_COUPLED=1).  Device pointers
 // throughout; every per-request / per-round value comes from device memory, so the calls can be captured.
 //

include/strata/prefill/kernels.hpp

@@ -43,8 +43,8 @@ void gdn_gates(const float* ab, const float* dt, const float* ssm_a, float* gate
 /// The 4-tap causal conv + SiLU over the chunk (history [C][3] in, updated to the chunk's last three inputs), then
 /// the L2 norm of the q and k heads of every token.  h: [T, C].
 void gdn_conv(float* history, const float* qkv, const float* conv_w, float* h, int64_t T, float eps, void* stream);
-/// The recurrence over the chunk, block per value head, state in registers; y[t] = rmsnorm(o) * gamma * sigmoid(z)
-/// (FP32 and FP16 bits: the out projection is quantized).
+/// The recurrence over the chunk, state in registers; y16[t] = rmsnorm(o) * gamma * sigmoid(z) in FP16 (what the
+/// out projection reads); y is FP32 scratch.
 void gdn_recurrence(float* state, const float* h, const float* gate, const float* beta, const float* z,
                     const float* gamma, float eps, float* y, uint16_t* y16, int64_t T, void* stream);
 

include/strata/prefill/moe_fused.hpp

@@ -0,0 +1,64 @@
+// include/strata/prefill/moe_fused.hpp - #136 (prompt speed): the routed experts of a Q2_0 pack in the prompt path on
+// Strata's own int8 tensor-core kernels, opt-in with STRATA_PF_FUSED=1 (NVIDIA sm_80 and newer; sm_75 and HIP: MMQ).
+//
+// The MMQ path (moe_mmq.hpp) per MoE layer: a host sync to group the (token, k) pairs by expert, a gather of each
+// expert into a 16-expert group buffer, then per group the q8_1 activations of every routed slot, gate/up, SwiGLU,
+// the q8_1 of H and down - five passes over per-slot buffers.  Here:
+//   - the activations are rounded to int8 once per TOKEN (not per slot), with a scale per 32 values;
+//   - the grouping (counts, offsets, the rows of each expert, a list of 64-row tiles) runs on the GPU from the
+//     router's ids, so the host never waits for the routing;
+//   - one launch per batch of up to kMaxBatch experts reads each expert where it already is (a resident cache slot
+//     or a slot of the streamed ring: a pointer per expert, no gather) - gate/up with SwiGLU and the int8 rounding
+//     of H in its epilogue, then down into the per-slot rows the existing combine reads.
+// The 2-bit codes are unpacked in registers and multiplied with mma.sync m16n8k32 (s8 x s8 -> s32); Q2_0's offset
+// (w = d * (q - 1)) is folded into the activation blocks' sums.  The numbers differ from MMQ's (another int8
+// rounding of the activations and of H) by about as much as MMQ differs from FP32 (tests/cuda/prefill_fused_moe_test).
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace strata::prefill::fused {
+
+/// Routed rows (token, k pairs of one expert) per tile.
+constexpr int kTileRows = 64;
+/// Experts per launch: their blob pointers travel in the launch's parameters (1 KB).
+constexpr int kMaxBatch = 128;
+
+/// This build has the kernels (a CUDA build; HIP and builds without MMQ do not).
+bool built();
+/// built(), and the current device is sm_80 or newer (mma.sync with s8 operands at m16n8k32).
+bool available();
+/// available(), unless STRATA_PF_FUSED=0 (read once): the Q2_0 pack's fused experts, on by default since 0.1.36.
+bool enabled();
+/// STRATA_PF_FUSED=1 given explicitly, and available(): the native IQ packs' fused kernels (opt-in).
+bool requested();
+
+/// Bytes of `rows` activation rows of `cols` values (a multiple of 64) in the kernels' int8 form: per 64 values 64
+/// codes and {d, c} of each half (c = -(1.5 * 2^23 + the half's code sum), what the epilogue adds back).
+size_t act_bytes(int64_t rows, int64_t cols);
+/// Bytes of the grouping tables for `n` routed rows over `n_expert` experts.
+size_t group_bytes(int64_t n, int n_expert);
+
+/// x [rows][cols] FP32 -> `xa` (act_bytes(rows, cols)).
+void quantize_act(const float* x, int64_t rows, int64_t cols, void* xa, void* stream);
+
+/// The routing `ids` [n = tokens * k] -> per expert its rows (counts, offsets, the 64-row tile list, in `scratch`,
+/// group_bytes(n, n_expert)), `slot[i]` = the row of pair i (the row of the per-slot outputs the combine reads) and
+/// `src[row]` = its token (i / k).  The order of the rows within an expert is not fixed; no output depends on it.
+void group(const int32_t* ids, int64_t n, int k, int n_expert, void* scratch, int32_t* slot, int32_t* src,
+           void* stream);
+
+/// Experts [e0, e1) of one layer (e1 - e0 <= kMaxBatch); blob[e - e0]: the Strata Q2_0 blob of expert e on the
+/// device (16-byte aligned), read only for an expert with rows.
+struct Batch {
+    int e0 = 0, e1 = 0;
+    const uint8_t* blob[kMaxBatch] = {};
+};
+/// The batch's products: gate/up from `xa` (quantize_act of the layer's input, [tokens][2560]) through the rows
+/// `src` of `group`, SwiGLU, H as int8 into `ha` (act_bytes(n, 640)), down into `dm` [n][2560] FP32 at the rows of
+/// `group`.  `n`: the layer's routed rows (tokens * k), which bounds the launch grid.
+void experts(const Batch& b, int n_expert, int64_t n, const void* scratch, const void* xa, const int32_t* src, void* ha,
+             float* dm, void* stream);
+
+}  // namespace strata::prefill::fused

include/strata/prefill/moe_fused_iq.hpp

@@ -0,0 +1,41 @@
+// include/strata/prefill/moe_fused_iq.hpp - #136: the fused int8 prompt experts (moe_fused.hpp) for the native GGUF
+// packs (tools/iq_pack.py: an expert is its raw GGUF slices [gate rows | up rows | down rows], formats per layer in
+// expert_layout().fmt), opt-in with the same STRATA_PF_FUSED=1 (NVIDIA sm_80 and newer; sm_75 and HIP: MMQ).
+//
+// The grouping on the GPU (fused::group), the per-expert pointers (fused::Batch: a cache slot or a ring slot, no
+// gather) and the launch shape are the Q2_0 path's.  What differs is the load stage: the i-quant blocks are decoded to
+// int8 in shared memory before the mma (codebook and sign lookups, as llama.cpp's MMQ load_tiles does), with the
+// block's scale per 32 (or per 16: IQ2_XS, IQ2_S, multiplied at m16n8k16) beside them.  The activations and H are
+// int8 per 32 values in their natural order here (quantize_act_native), not the Q2_0 path's permuted one.
+//
+// Covered: gate/up IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS; down Q2_0, IQ4_NL - every layer of the IQ2_XS,
+// IQ3_XXS and IQ3_S packs but the IQ1_M ones (which MMQ does not cover either).  Other layers keep MMQ.
+#pragma once
+
+#include "strata/prefill/moe_fused.hpp"
+
+#include <cstddef>
+#include <cstdint>
+
+namespace strata::prefill::fused {
+
+/// One layer's native expert geometry (cpu::NativeFmt's fields the kernels read; n_embd 2560, n_ff 640).
+struct NativeGeom {
+    int gu_type = -1, d_type = -1;    ///< ggml types
+    size_t gu_row = 0, d_row = 0;     ///< bytes per gate/up row and per down row
+    size_t up_off = 0, down_off = 0;  ///< inside the blob
+};
+
+/// enabled() (STRATA_PF_FUSED=1 on sm_80+), and the kernels cover this gate/up and down pair on this device.
+bool native_supported(int gu_type, int d_type);
+
+/// x [rows][cols] FP32 -> `xa` (act_bytes(rows, cols)): int8 per 32 values in natural order, the native kernels' form.
+void quantize_act_native(const float* x, int64_t rows, int64_t cols, void* xa, void* stream);
+
+/// experts() for native blobs: b.blob[e - e0] is the native blob of expert e (gate at 0, up at g.up_off, down at
+/// g.down_off; 2-byte aligned).  `xa` from quantize_act_native; H in `ha` (act_bytes(n, 640)); down into `dm` at the
+/// rows of `group`.
+void experts_native(const Batch& b, const NativeGeom& g, int n_expert, int64_t n, const void* scratch, const void* xa,
+                    const int32_t* src, void* ha, float* dm, void* stream);
+
+}  // namespace strata::prefill::fused

include/strata/prefill/moe_mmq.hpp

@@ -15,6 +15,10 @@ bool built();
 /// MMQ covers this ggml type (the i-quants and Q2_0 the packs use, Q8_0, and in a CUDA build with STRATA_MMQ_KQUANTS
 /// the Q4_K / Q5_K / Q5_1 of Unsloth's UD-Q4_K_XL; IQ1_M is not covered).
 bool supported(int ggml_type);
+/// #420: `supported`, and on every visible GPU llama.cpp's MMQ has a tile for this type and a weight matrix of
+/// `w_rows` rows that fits the card's shared memory - the same test its tile choice makes, which aborts the process
+/// ("J_best=0") when nothing fits.  false (said once per type) keeps that product on the non-MMQ path.
+bool fits(int ggml_type, int64_t w_rows);
 /// Bytes of one expert's gate+up ([2*n_ff, n_embd]) or down ([n_embd, n_ff]) weights in `ggml_type`.
 size_t matrix_bytes(int ggml_type, int64_t rows, int64_t cols);
 /// Bytes of `rows` activation rows of `cols` values quantized for MMQ (the row padded to 512 values).
@@ -60,6 +64,16 @@ private:
 /// slot: gate rows then up rows at `gu_dst`, down at `d_dst`.
 void gather_native(const void* gate, const void* up, size_t gu_half_bytes, const void* down, size_t d_bytes,
                    void* gu_dst, void* d_dst, void* stream);
+/// gather_native for an MMQ group's experts [first, n) in ONE launch: expert q's blob (`blob[q]`; gate at +0, up at
+/// +up_off, down at +down_off) to gu_dst + q * gu_stride and d_dst + q * d_stride - the same bytes as one gather_native
+/// each.  Every pointer, offset and size 16-byte aligned (false otherwise: nothing launched, gather one at a time).
+constexpr int kGatherGroupMax = 16;
+struct GatherGroup {
+    const uint8_t* blob[kGatherGroupMax] = {};
+    int first = 0, n = 0;
+};
+bool gather_native_group(const GatherGroup& g, size_t up_off, size_t gu_half_bytes, size_t down_off, size_t d_bytes,
+                         void* gu_dst, size_t gu_stride, void* d_dst, size_t d_stride, void* stream);
 /// A Strata-pack Q2_0 expert blob (codes and fp16 scales in separate planes, gate/up rows interleaved) into GGUF
 /// Q2_0 blocks: gate/up [1280, 2560] at `gu_dst` (rows stay interleaved), down [2560, 640] at `d_dst`.  Same values.
 void gather_strata_q2(const uint8_t* blob, void* gu_dst, void* d_dst, void* stream);

include/strata/prefill/prefill.hpp

@@ -21,6 +21,8 @@
 #include <memory>
 #include <string>
 
+namespace strata::core { class PeerExperts; }
+
 namespace strata::prefill {
 
 struct PrefillStats {
@@ -86,6 +88,11 @@ public:
 
     const PrefillStats& stats() const { return stats_; }
 
+    /// multi-GPU: the experts the peer GPU holds are computed THERE for every prompt chunk (up to `cap_rows` routed
+    /// rows per layer; the rest of the peer's experts are read by this GPU over P2P).  Allocates the peer's buffers for
+    /// chunks of up to init's `chunk` tokens.  Needs P2P between the two cards.
+    bool set_peer(core::PeerExperts* peer, int64_t cap_rows, std::string& err);
+
     /// Plan v0.3 P6: called after every chunk with the chunk's final multi-stream residual rows (device,
     /// T x hc*n_embd, valid until the next chunk) and the chunk's first position; the MTP draft layer builds its
     /// K/V from them.  The prefill stream is synchronized before the call.

serve/frontend.py

@@ -25,12 +25,17 @@ from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 
 # ------------------------------------------------------------------------------------------------ template
+class TemplateRequestError(jinja2.exceptions.TemplateError, ValueError):
+    """The template refused the request's messages (e.g. "No user query found in messages."): a ValueError, so the
+    client gets a 400 with the template's message instead of a dropped connection (#365)."""
+
+
 class ChatTemplate:
     """The model's chat template, rendered with the same Jinja settings as transformers' apply_chat_template."""
 
     def __init__(self, path: str | Path):
         def raise_exception(message):
-            raise jinja2.exceptions.TemplateError(message)
+            raise TemplateRequestError(message)
 
         def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False):
             return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators, sort_keys=sort_keys)
@@ -138,10 +143,27 @@ def _late_system_to_user(messages: list[dict]) -> list[dict]:
     return [dict(m, role="user") if m.get("role") == "system" and i > 0 else m for i, m in enumerate(messages)]
 
 
+def _object_list(value, name: str) -> list[dict]:
+    """#460: a request's "messages" (or a message's "tool_calls") as a list of objects.  Some clients send the array
+    double-encoded, as a JSON string, which used to be iterated character by character and crashed on m.get: such a
+    string is decoded.  Anything that is still not a list of objects is a ValueError, which the server answers with
+    a 400 naming the field.  None (or no field) is an empty list, as a missing field always was."""
+    if value is None:
+        return []
+    if isinstance(value, str):
+        try:
+            value = json.loads(value)
+        except ValueError:
+            raise ValueError(f"{name} must be a list of objects (a string was sent that is not JSON)") from None
+    if not isinstance(value, list) or not all(isinstance(m, dict) for m in value):
+        raise ValueError(f"{name} must be a list of objects")
+    return value
+
+
 def openai_to_messages(req: dict) -> tuple[list[dict], list[dict] | None, dict]:
     """OpenAI Chat Completions -> (template messages, template tools, template kwargs)."""
     messages = []
-    for m in req.get("messages", []):
+    for m in _object_list(req.get("messages"), "messages"):
         role = m.get("role")
         if role == "developer":
             role = "system"
@@ -150,8 +172,10 @@ def openai_to_messages(req: dict) -> tuple[list[dict], list[dict] | None, dict]:
             out["reasoning_content"] = m["reasoning_content"]
         if m.get("tool_calls"):
             calls = []
-            for c in m["tool_calls"]:
+            for c in _object_list(m["tool_calls"], "tool_calls"):
                 fn = c.get("function", c)
+                if not isinstance(fn, dict):
+                    raise ValueError("tool_calls must be a list of objects (each with a \"function\" object)")
                 args = fn.get("arguments")
                 if isinstance(args, str):               # the template requires a mapping, not a JSON string
                     args = json.loads(args) if args.strip() else {}
@@ -181,7 +205,7 @@ def anthropic_to_messages(req: dict, think_unasked: bool = True) -> tuple[list[d
     system = req.get("system")
     if system:
         messages.append({"role": "system", "content": _text_of(system)})
-    for m in req.get("messages", []):
+    for m in _object_list(req.get("messages"), "messages"):
         content = m.get("content")
         if isinstance(content, str):
             messages.append({"role": m["role"], "content": content})

serve/server.py

@@ -31,7 +31,10 @@ import json
 import os
 import queue
 import re
+import select
 import signal
+import socket
+import struct
 import subprocess
 import sys
 import tempfile
@@ -58,11 +61,22 @@ IMAGE_PAD = "<|image_pad|>"
 VISION_START = "<|vision_start|>"
 # #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)
 REASONING_WRAP_UP = "\n\nI have thought about this long enough; time to give my answer.\n</think>\n\n"
+LOOPBACK_NAMES = ("localhost", "127.0.0.1", "::1")
 CTX_SLACK = 8               # `strata --serve` rejects prompt + max_new + 8 > context: keep the same margin here
 # The live tok/s is a rate over a window, not a mean since the first token: a mean reads ~1/elapsed at the first
 # token (the Monitor showed five-digit numbers) and then undershoots for the first second of every answer.
 RATE_WINDOW_S = 2.0
 RATE_MIN_SPAN_S = 0.25      # younger than this there is no rate yet: the mean so far, with the span floored here
+# #481: a running request whose engine prints nothing (no T, PP or any other line) for this long has lost step with the
+# server (the engine's main thread waits, untimed, for its next command): the engine is ended and the request fails;
+# the next request starts it again.  The config's "engine_silence_s" sets it (0: wait forever, as before).
+ENGINE_SILENCE_S = 300.0
+# ... except while a prompt is read: a PP line comes once per chunk (up to 32768 tokens with --prefill auto, issue
+# #282), and the slowest PCs read ~100 tok/s, so a first chunk can take minutes before the first line.  Until the first
+# PP the wait adds the chunk's tokens at PP_FLOOR_TOK_S; after one, a chunk may take PP_SLACK x the last one's time.
+PP_CHUNK_MAX = 32768
+PP_FLOOR_TOK_S = 50.0
+PP_SLACK = 3.0
 
 
 # ------------------------------------------------------------------------------------------------ engines
@@ -110,6 +124,12 @@ class ModelBusy(RuntimeError):
     """Explicit model controls must not interrupt active or queued requests."""
 
 
+class EngineSilent(EngineDied):
+    """#481: the engine said nothing for too long during a request (or never acknowledged a STOP): the two sides lost
+    step - the engine waiting for its next command, the server for this request's end - and the server ended it.  An
+    EngineDied, so the request ends with an error and the next one starts the engine again."""
+
+
 class EngineStuck(RuntimeError):
     """The engine process did not end after QUIT, terminate and kill: the server keeps it (and says so) rather than
     reporting its GPU and RAM as given back."""
@@ -121,12 +141,60 @@ class GpuBusy(RuntimeError):
 
 
 ENGINE_REQUEST = re.compile(
-    r"prompt (?P<prompt>\d+) tokens = (?P<reused>\d+) reused \+ \d+ read in (?P<read>[\d.]+) ms \((?P<pp>[\d.]+) tok/s\), "
+    # "+ 12288 of 98179 read": a request cancelled while its prompt was read (#471)
+    r"prompt (?P<prompt>\d+) tokens = (?P<reused>\d+) reused \+ \d+(?: of \d+)? read in (?P<read>[\d.]+) ms "
+    r"\((?P<pp>[\d.]+) tok/s\), "
     r"(?P<gen>\d+) generated in (?P<gen_ms>[\d.]+) ms \((?P<tg>[\d.]+) tok/s\)")
 
 
 _echoing: set[str] = set()      # the logs echo_requests already follows (restart() runs StrataEngine.__init__ again)
 
+DRAFT_HEAD_FAIL = "the draft head does not fit"
+DRAFT_HEAD_HINT = ("a smaller draft vocabulary needs less VRAM: --draft-vocab cyrillic (English, code and the Cyrillic "
+                   "script) or --draft-vocab en (English and code, ~215 MiB less than the default). Start once with "
+                   "it - START-HERE.bat --draft-vocab en (Windows) or ./setup.sh --draft-vocab en - and the model "
+                   "keeps it; or a smaller --context in setup.")
+
+
+def start_failure_hint(log: str | None, offset: int) -> str:
+    """#474: what to change when the engine stopped at the start because the MTP draft head did not fit the VRAM
+    left: the engine's own `strata mtp:` lines after that failure (0.1.36+: what it needs, what is free, the smaller
+    subsets), else the same advice in words for an older engine.  "" for any other failure: the log says why."""
+    if not log:
+        return ""
+    try:
+        with open(log, "rb") as f:
+            f.seek(offset)
+            text = f.read().decode("utf-8", "replace")
+    except (OSError, ValueError):
+        return ""
+    if DRAFT_HEAD_FAIL not in text:
+        return ""
+    said = [x.strip()[len("strata mtp: "):] for x in text.splitlines()
+            if x.strip().startswith("strata mtp: ") and ("draft head over" in x or "hint:" in x)]
+    return ". mtp: " + DRAFT_HEAD_FAIL + ". " + (" ".join(said) if said else "Hint: " + DRAFT_HEAD_HINT)
+
+
+def start_log_tail(log: str | None, offset: int, n: int = 20) -> str:
+    """#496: the last n lines this start wrote to the engine log, for the error when the engine ended before READY -
+    whatever the failure, the engine's own reason is in them (people posted the traceback without the log).  "" when
+    there is no log or nothing in it from this start."""
+    if not log:
+        return ""
+    try:
+        with open(log, "rb") as f:
+            f.seek(0, 2)
+            start = max(offset, f.tell() - 64 * 1024)   # enough for 20 lines, never an earlier start's
+            f.seek(start)
+            text = f.read().decode("utf-8", "replace")
+    except (OSError, ValueError):
+        return ""
+    lines = text.splitlines()[1 if start > offset else 0:]   # not a line cut in half
+    lines = [x.rstrip() for x in lines if x.strip()][-n:]
+    if not lines:
+        return ""
+    return "\nthe engine log's last lines:\n" + "\n".join("  " + x for x in lines)
+
 
 def echo_requests(log_path: str, offset: int) -> None:
     """STRATA_REQUEST_LINES=1: one stdout line per finished request, from the engine's own summary in its log.
@@ -151,6 +219,27 @@ def echo_requests(log_path: str, offset: int) -> None:
                                                  m["tg"]), flush=True)
 
 
+def experts_loading_words(args: list, size: str) -> str:
+    """#505: what the start does with the experts, by the engine's flags (generate.cpp's option parsing): a RAM budget
+    copies the hottest N GiB into RAM (--resident-budget-gib), the resident low-RAM mode the ones the GPU does not hold
+    (--resident-experts), plain --mmap-experts reads them from the model files through the OS file cache (nothing is
+    loaded into RAM up front); otherwise all of them go into RAM."""
+    if "--resident-budget-gib" in args:
+        try:
+            n = f"up to {float(args[args.index('--resident-budget-gib') + 1]):g} GiB"
+        except (IndexError, ValueError):
+            n = "a RAM budget"
+        return (f"loading the most-used experts into RAM ({n}; the rest are read from the model files as needed) "
+                "and locking part of them for the GPU.")
+    if "--resident-experts" in args:
+        return (f"loading the experts the GPU does not hold into RAM (of {size}) and locking part of them for the "
+                "GPU.")
+    if "--mmap-experts" in args:
+        return (f"mapping the experts from the model files ({size}, --mmap-experts): they are not loaded into RAM - "
+                "the OS file cache reads them as the GPU's expert cache fills and as requests need them.")
+    return f"loading the experts into RAM ({size}) and locking part of them for the GPU."
+
+
 def narrate_start(log_path: str, offset: int, args: list, done: threading.Event, heartbeat=20.0) -> None:
     """While the engine starts, say in the server window what it is doing, from its log: the start reads tens of GB
     into RAM and locks part of it for the GPU, and on many PCs everything is slow or frozen for a minute or more -
@@ -162,6 +251,7 @@ def narrate_start(log_path: str, offset: int, args: list, done: threading.Event,
         except (OSError, IndexError):
             pass
     size = f"about {gb:.0f} GB" if gb >= 1 else "tens of GB"
+    loading = experts_loading_words(args, size)
     t0 = last = time.time()
     said = set()
 
@@ -185,8 +275,8 @@ def narrate_start(log_path: str, offset: int, args: list, done: threading.Event,
             cut = chunk.rfind(b"\n") + 1
             pos += cut
             for line in chunk[:cut].decode("utf-8", "replace").splitlines():
-                if "PLE on" in line or "expert arena:" in line:
-                    say("arena", f"[strata] loading the experts into RAM ({size}) and locking part of them for the GPU.\n"
+                if "PLE on" in line or "expert arena:" in line or "experts via mmap" in line:   # #505: mapped
+                    say("arena", f"[strata] {loading}\n"
                                  "         YOUR PC CAN BE SLOW OR STOP RESPONDING FOR 1-3 MINUTES NOW - this is normal.\n"
                                  "         Please wait and don't close this window; the browser opens when it is ready.")
                 elif " loaded " in line and "GiB at" in line:
@@ -210,6 +300,8 @@ class StrataEngine:
     (`temperature=F top_p=F top_k=N seed=N`, the engine's own spelling).  An absent temperature keeps the
     engine's default, which is greedy; `temperature=0` means the same thing, so it is not forwarded.
     """
+    silence_s = ENGINE_SILENCE_S         # #481: main() sets the config's engine_silence_s (survives restart())
+    silent_note = None                   # #481: why the server ended a silent engine (death_note says it)
 
     def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | None = None,
                  env: dict | None = None, lazy: bool = False):
@@ -225,7 +317,8 @@ class StrataEngine:
         self.info = {}                   # INFO key=value facts (engine 0.1.8+): kv, expert slots, ... (Monitor tab)
         self.prefill_tok_s_mean = None
         self.progress = None             # (read, total) prompt tokens while a prompt is read, from PP lines
-        try:                             # a ready-made engine's BUILD.json says its version
+        self.silent_note = None
+        try:                            # a ready-made engine's BUILD.json says its version
             self.info["version"] = json.loads((Path(exe).parent / "BUILD.json").read_text()).get("version")
         except (OSError, ValueError):
             self.info["version"] = None
@@ -234,6 +327,7 @@ class StrataEngine:
         self.unloaded = False            # `ended` stays True until READY (below): not alive while starting (#344)
         self.log = open(log, "a", encoding="utf-8") if log else subprocess.DEVNULL
         loading = threading.Event()                     # set once READY: the narrator below stops
+        log_start = os.path.getsize(log) if log else 0  # where this start's lines begin (start_failure_hint)
         if log:
             threading.Thread(target=narrate_start, args=(log, os.path.getsize(log), args, loading),
                              daemon=True).start()
@@ -257,7 +351,16 @@ class StrataEngine:
                 break
         loading.set()
         if self.max_context <= 0:
-            raise RuntimeError("the engine exited before it was ready" + (f" (see {log})" if log else ""))
+            try:                                        # its pipes and our handle on its log (the log stays)
+                self.proc.wait(timeout=5)
+                self.proc.stdin.close()
+                self.proc.stdout.close()
+                if log:
+                    self.log.close()
+            except (OSError, subprocess.TimeoutExpired):
+                pass
+            raise RuntimeError("the engine exited before it was ready" + (f" (see {log})" if log else "") +
+                               start_failure_hint(log, log_start) + start_log_tail(log, log_start))
         # (from PR #41, midhatn) a locally built engine can sit next to another release's BUILD.json: engines that
         # report their own version (INFO engine=, 0.1.8+) win, the manifest stays the fallback for older ones
         if self.info.get("engine"):
@@ -278,6 +381,8 @@ class StrataEngine:
 
     def death_note(self) -> str:
         """Why the engine most likely ended, from the end of its log: its own watchdog (issue #29), else RAM."""
+        if getattr(self, "silent_note", None):          # #481: the server ended it, not the OS or the engine itself
+            return self.silent_note
         tail = ""
         try:
             with open(self.log_path, "rb") as f:
@@ -336,6 +441,8 @@ class StrataEngine:
             self.last.update(hits=int(f[9]), lookups=int(f[10]))
         if len(f) >= 14:                                  # the expert tiers (engine 0.1.31+): RAM / file blobs, file MB
             self.last.update(ram_blobs=int(f[11]), file_blobs=int(f[12]), file_mb=float(f[13]))
+        if len(f) >= 15:                                  # #471 (engine 0.1.36+): the prompt tokens actually read
+            self.last.update(prompt_read=int(f[14]))
 
     @staticmethod
     def sampling_keys(sampling: dict) -> str:
@@ -406,19 +513,34 @@ class StrataEngine:
         except OSError:                                  # the pipe is gone: the engine died (not the client)
             raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})") from None
         done = False
+        # #481: how long the engine may stay silent from here.  Until the first line: the request's first prompt
+        # chunk at the slowest prompt reading on top of silence_s; a PP line resets it to its own chunk's time.
+        silence = float(self.silence_s or 0)
+        allow = silence + min(len(ids), PP_CHUNK_MAX) / PP_FLOOR_TOK_S if silence > 0 else 0.0
+        heard, read_to = time.monotonic(), 0
         try:
             while True:
+                wait = 10.0
+                if allow > 0:
+                    wait = min(wait, allow - (time.monotonic() - heard))
+                    if wait <= 0:
+                        done = True                       # no STOP and no drain: nothing is listening
+                        raise self._silent(f"the engine said nothing for {time.monotonic() - heard:.0f} s during "
+                                           "the request")
                 try:
-                    line = self.lines.get(timeout=10)
+                    line = self.lines.get(timeout=wait)
                 except queue.Empty:
                     if cancel.is_set():
                         return
-                    yield None
+                    if wait >= 10.0:
+                        yield None                        # the 10 s heartbeat (the deadline's short waits are not)
                     continue
                 if line is None:
                     done = True
                     raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})")
+                heard = time.monotonic()                  # any line is output: T, PP, RESUME, INFO ...
                 if line.startswith("T "):
+                    allow = silence
                     if cancel.is_set():
                         return
                     yield int(line[2:])
@@ -427,9 +549,18 @@ class StrataEngine:
                     if len(f) >= 3 and f[1].isdigit() and f[2].isdigit():
                         self.progress = (int(f[1]), int(f[2]))             # prompt progress, one per chunk: also a heartbeat (the
                         self.prefill_tok_s_mean = float(f[4]) if len(f) >= 5 else None
+                        rate, chunk = self.prefill_tok_s_mean or 0.0, int(f[1]) - read_to
+                        read_to = int(f[1])
+                        if silence > 0 and rate > 0 and chunk > 0:   # #481: the next chunk, as long as this one
+                            allow = max(silence, PP_SLACK * chunk / rate)
                     if cancel.is_set():                   # lines reset the 10 s wait, so without this a long prompt
                         return                            # would send no keep-alives at all)
                     yield None
+                elif line.startswith("RESUME "):          # the reused tokens: the first chunk starts after them
+                    try:
+                        read_to = int(line.split()[1])
+                    except (IndexError, ValueError):
+                        pass
                 elif line.startswith("DONE"):
                     self._parse_done(line)
                     done = True
@@ -445,13 +576,41 @@ class StrataEngine:
                         self.proc.stdin.flush()
                     except OSError:
                         pass
+                # #481: never an untimed wait here - it holds the request FIFO, and an engine that lost step never
+                # answers.  An engine that honours STOP gets the current allowance in all (a STOP during a prompt
+                # chunk is seen after it); an older one runs on to max_new, so each line only has to come in time.
+                heard = time.monotonic()
                 while True:
-                    line = self.lines.get()
+                    left = allow - (time.monotonic() - heard) if allow > 0 else None
+                    try:
+                        if left is not None and left <= 0:
+                            raise queue.Empty
+                        line = self.lines.get(timeout=left)
+                    except queue.Empty:
+                        raise self._silent("the engine did not finish the request after it was stopped (STOP) "
+                                           f"within {allow:.0f} s") from None
                     if line is None or line.startswith("ERR"):
                         break
                     if line.startswith("DONE"):
                         self._parse_done(line)
                         break
+                    if not self.can_stop:
+                        heard = time.monotonic()
+
+    def _silent(self, what: str) -> EngineSilent:
+        """#481: end an engine that lost step with the server (its main thread waits for a command the server never
+        sends), so the next request starts it again: killed now, its GPU and RAM go back with the process."""
+        self.silent_note = ("The engine and the server lost step (issue #481; a very slow PC can raise "
+                            "\"engine_silence_s\" in the config, 0 = wait forever). If it happens again, please add "
+                            "the end of the engine log to github.com/Niko1221/Strata/issues/481.")
+        self.ended = True                               # not alive from now: the next request restarts it
+        proc = self.proc
+        try:
+            proc.kill()
+            proc.wait(timeout=20)                       # restart() -> close() handles one that is still exiting
+        except (OSError, AttributeError, subprocess.TimeoutExpired):
+            pass
+        return EngineSilent(f"{what}; the server ended the engine")
 
     def close(self):
         """End the engine process: QUIT first (the engine frees its memory itself - unpinning tens of GB can take
@@ -585,10 +744,12 @@ class Vision:
                 return self.cache[key]
             img, out = self.dir / f"{key}.img", self.dir / f"{key}.sve"
             img.write_bytes(data)
-            self.proc.stdin.write(f"ENC {img} {out}\n")
-            self.proc.stdin.flush()
-            line = self.proc.stdout.readline().strip()
-            img.unlink(missing_ok=True)
+            try:
+                self.proc.stdin.write(f"ENC {img} {out}\n")
+                self.proc.stdin.flush()
+                line = self.proc.stdout.readline().strip()
+            finally:                                                   # #352: also when the encoder's pipe is gone
+                img.unlink(missing_ok=True)
             if not line.startswith("OK"):
                 raise ValueError("the image could not be read: " + (line[4:] if line.startswith("ERR") else
                                                                      "the vision encoder stopped"))
@@ -617,6 +778,17 @@ def gpu_list(cfg: dict) -> list[int]:
     return [int(str(x).strip()) for x in items if str(x).strip() != ""]
 
 
+def engine_silence_s(cfg: dict) -> float:
+    """#481: the config's "engine_silence_s" - seconds an engine may print nothing during a request before the server
+    ends it (default ENGINE_SILENCE_S; 0 = wait forever).  ValueError for anything but a number >= 0."""
+    v = cfg.get("engine_silence_s")
+    if v is None:
+        return ENGINE_SILENCE_S
+    if isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0:
+        raise ValueError(f'"engine_silence_s" must be a number of seconds >= 0 (0 = no limit), not {v!r}')
+    return float(v)
+
+
 def engine_args(cfg: dict) -> list[str]:
     """The engine's arguments: the config's, and with several GPUs the layer split across them ("layer_split" in the
     config: "auto" by default, or the first layer of each later GPU's share, e.g. "18" or "16,32")."""
@@ -626,15 +798,69 @@ def engine_args(cfg: dict) -> list[str]:
     # opt-in: an auto split runs on the first card alone when it holds every profiled expert and the KV
     if len(gpu_list(cfg)) > 1 and cfg.get("split_skip_if_fits") and "--split-skip-if-fits" not in args:
         args.append("--split-skip-if-fits")
+    return learned_profile_args(cfg, args)
+
+
+def profile_shape(path: str) -> tuple[int, int] | None:
+    """An expert profile's (layers, experts per layer), from its header (tools/make_profile.py's format), or None
+    when the file is missing, is not one or is shorter than the pairs its header promises."""
+    try:
+        with open(path, "rb") as f:
+            head = f.read(24)
+            size = os.fstat(f.fileno()).st_size
+    except OSError:
+        return None
+    if len(head) < 24 or head[:4] != b"STRP":
+        return None
+    _, nl, ne, _, n = struct.unpack("<5I", head[4:])
+    return (nl, ne) if size >= 24 + 4 * n else None
+
+
+def learned_profile_args(cfg: dict, args: list[str]) -> list[str]:
+    """#477 (opt-in): "expert_profile_save": "<path>" in the config has the engine (0.1.36+) save what its adaptive
+    tier learned there - on QUIT and every "expert_profile_save_every" minutes (10 by default, 0 = at QUIT only) -
+    and the next start begins from it instead of the config's --expert-profile, when it is a profile of the same
+    model (its header's layers and experts match); otherwise from the config's own, as before.  A relative path is
+    the engine's (the config's "cwd").  Without the key, the arguments are the config's, unchanged."""
+    save = cfg.get("expert_profile_save")
+    if not isinstance(save, str) or not save.strip() or "--expert-profile-save" in args:
+        return args
+    args = args + ["--expert-profile-save", save]
+    every = cfg.get("expert_profile_save_every")
+    if isinstance(every, (int, float)) and not isinstance(every, bool) and every >= 0:
+        args += ["--expert-profile-save-every", str(every)]
+    if "--expert-profile" in args[:-1]:
+        i = args.index("--expert-profile") + 1
+        here = cfg.get("cwd") or "."
+        learned = profile_shape(save if os.path.isabs(save) else os.path.join(here, save))
+        base = profile_shape(args[i] if os.path.isabs(args[i]) else os.path.join(here, args[i]))
+        if learned is not None and learned == base:
+            args[i] = save
     return args
 
 
+def hip_visible(cfg: dict) -> list[int]:
+    """AMD: the devices the engine should see, as the HIP runtime numbers them (HIP_VISIBLE_DEVICES).
+
+    On Linux setup's KFD order is HIP's order, so the config's "gpu" is it.  On Windows setup finds the cards in the
+    display-adapter order, and an integrated Radeon that HIP also enumerates takes ordinal 0 and pushes the discrete
+    card to 1 (#325): setup records the ordinal `strata-device --list-devices` gave the card as "hip_ordinal", which
+    wins for a one-card config.  Without it (a config from before), the config's "gpu"."""
+    ordinal = cfg.get("hip_ordinal")
+    if ordinal is not None and str(ordinal).strip() != "" and len(gpu_list(cfg)) <= 1:
+        try:
+            return [int(str(ordinal).strip())]
+        except ValueError:
+            pass
+    return gpu_list(cfg)
+
+
 def child_env(cfg: dict) -> dict:
     """The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
     compiled it) first on the library search path."""
     env = dict(os.environ)
-    if gpu_list(cfg) and cfg.get("backend") == "hip":   # AMD: numbered as HIP numbers them (setup's KFD order)
-        env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
+    if hip_visible(cfg) and cfg.get("backend") == "hip":   # AMD: numbered as HIP numbers them (hip_visible)
+        env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in hip_visible(cfg))
     elif gpu_list(cfg):                              # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
         env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"      # own default order (fastest first) can number the cards otherwise
         env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
@@ -647,6 +873,22 @@ def child_env(cfg: dict) -> dict:
     return env
 
 
+def vision_env(cfg: dict, env: dict) -> dict:
+    """The image encoder's environment: the engine's, unless the config's vision section names its own "cuda_device"
+    (numbered like nvidia-smi) - then the encoder runs on that card alone, so a spare GPU can hold it while the engine
+    keeps all of its own cards' VRAM (#408, Efs-O).  Without it, nothing changes."""
+    dev = (cfg.get("vision") or {}).get("cuda_device")
+    if dev is None:
+        return env
+    env = dict(env)
+    if cfg.get("backend") == "hip":
+        env["HIP_VISIBLE_DEVICES"] = str(dev)
+    else:
+        env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+        env["CUDA_VISIBLE_DEVICES"] = str(dev)
+    return env
+
+
 class ByteTokenizer:
     """Tiny stand-in tokenizer for tests without the pack: one id per UTF-8 byte, specials as ids >= 256."""
     SPECIALS = ["<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|vision_start|>", "<|image_pad|>", "<|vision_end|>"]
@@ -719,6 +961,10 @@ class Service:
         # #321: origins that count as Strata's own page for /settings and MCP tools, e.g. the web app reached through a
         # reverse proxy or tunnel whose Host differs ("https://strata.example.com"); never a wildcard
         self.trusted_origins: list[str] = []
+        # DNS rebinding: extra Host names this server answers to (the config's allowed_hosts, $STRATA_ALLOWED_HOSTS;
+        # "*" = any), and every name it answers to, which serve() works out from the address it listens on
+        self.allowed_hosts: list[str] = []
+        self.host_names: set[str] = set(LOOPBACK_NAMES)
         self.status = {"busy": False, "queued": 0}      # GET /status: what the model is doing right now
         self.rate = collections.deque(maxlen=32)        # (time, generated) samples for the live tok/s window
         # #332: the API request monitor (/api-monitor) keeps the last 100 requests' prompts and answers in memory,
@@ -729,7 +975,8 @@ class Service:
         self.history = collections.deque(maxlen=500)    # the last finished requests, newest last (GET /metrics)
         # since the server started (the Monitor's totals, issue #35)
         self.totals = {"since": time.time(), "requests": 0, "prompt_tokens": 0, "reused": 0, "output_tokens": 0,
-                       "prompt_ms": 0.0, "decode_ms": 0.0}
+                       "prompt_ms": 0.0, "decode_ms": 0.0,
+                       "drafts_offered": 0, "drafts_accepted": 0}   # #457: the MTP drafts, summed where reported
         self.last_timings = None                         # the last finished request's, llama.cpp's names (/v1/status)
         self.last_request_at = None                      # when a request last started or finished
         self.started_at = time.time()
@@ -842,6 +1089,13 @@ class Service:
         self.engine.restart()
         print("[strata] the engine is running again", flush=True)
 
+    def _say_died(self, e: Exception) -> None:
+        """The server window's line for an engine that died (or was ended, #481) in the middle of a request."""
+        note = self.engine.death_note() if hasattr(self.engine, "death_note") else ""
+        log = getattr(self.engine, "log_path", None)
+        print(f"[strata] {e}. {note} The next request starts the engine again."
+              f"{' Its log: ' + log if log else ''}", flush=True)
+
     def load(self):
         """POST /load and every generation request: start the engine now if it is unloaded (raises GpuBusy)."""
         # a request is on its way: the idle thread must not unload between this and the request's own start
@@ -1131,7 +1385,9 @@ class Service:
         elif max_new > room:
             if not self.fit_max_tokens:
                 raise ValueError(f"prompt ({len(ids)} tokens) + max tokens ({max_new}) exceeds the context "
-                                 f"({self.engine.max_context}); requests are never truncated")
+                                 f"({self.engine.max_context}); requests are never truncated. Send a smaller "
+                                 f"max_tokens (at most {max(0, room)} here), or add \"fit_max_tokens\": true to the "
+                                 "model's strata-<model>.json to shorten it to the room left (#545)")
             max_new = max(1, room)          # --fit-max-tokens: a shorter completion beats a 400
         return ids, kwargs.get("enable_thinking", True) is not False, max_new
 
@@ -1212,7 +1468,7 @@ class Service:
                     while True:
                         gen = self.engine.generate(prompt, max_new - n, sampling, cancel, embeddings=emb) if emb \
                             else self.engine.generate(prompt, max_new - n, sampling, cancel)
-                        seg, wrap = [], False           # this pass's tokens; the budget is reached
+                        seg, wrap, leaving = [], False, False   # this pass's tokens; the budget is reached; closed
                         try:
                             for t in gen:
                                 if t is None:               # heartbeat while the engine is quiet
@@ -1242,19 +1498,25 @@ class Service:
                                         break
                         except EngineDied as e:
                             finish = "error"
-                            note = self.engine.death_note() if hasattr(self.engine, "death_note") else ""
-                            log = getattr(self.engine, "log_path", None)
-                            print(f"[strata] {e}. {note} The next request starts the engine again."
-                                  f"{' Its log: ' + log if log else ''}", flush=True)
+                            self._say_died(e)
                             raise
                         except ValueError as e:             # the engine's ERR line (it may have ended after it)
                             finish = "error"
                             print(f"[strata] the engine reported an error: {e}", flush=True)
                             raise
+                        except GeneratorExit:               # the client went away: an engine that never acknowledges
+                            leaving = True                  # the STOP below is ended, but no error replaces this
+                            raise
                         finally:
-                            gen.close()                 # STOP+drain to THIS request's DONE while still holding the
-                            #                             fifo, so a stop-token break can't leave the shared engine
-                            #                             queue mid-drain for the next request to read as its own DONE
+                            try:
+                                gen.close()             # STOP+drain to THIS request's DONE while still holding the
+                                #                         fifo, so a stop-token break can't leave the shared engine
+                                #                         queue mid-drain for the next request to read as its own DONE
+                            except EngineSilent as e:   # #481: the STOP was never acknowledged: the engine is ended
+                                finish = "error"
+                                self._say_died(e)
+                                if not leaving and not cancel.is_set():
+                                    raise
                         if not wrap or cancel.is_set():
                             break
                         # #123: the thinking reached reasoning_budget_tokens.  Close it the way the model would (a
@@ -1292,27 +1554,35 @@ class Service:
                             cvec = (getattr(self.engine, "info", {}) or {}).get("cvec", 0)
                             loaded = str(cvec) not in ("0", "", "None")
                             hit_rate = round(last["hits"] / last["lookups"], 3) if last.get("lookups") else None
+                            seen = prompt_tokens_seen(len(ids), last)   # #471: < len(ids) when cancelled mid-read
                             self.history.append({
                                 "projection": (sampling or {}).get("experimental_speed_projection") is not False
                                 if loaded else None,
                                 "time": started, "duration_s": round(time.time() - started, 1), "finish": finish,
-                                "prompt_tokens": len(ids), "reused": last.get("reused"), "output_tokens": n,
+                                "prompt_tokens": seen, "reused": last.get("reused"), "output_tokens": n,
+                                # the request's whole prompt, and the tokens read of it (None: an older engine)
+                                "prompt_total": len(ids), "prompt_read": last.get("prompt_read"),
                                 "engine_generated": last.get("generated"),
                                 "prompt_ms": last.get("prompt_ms"), "decode_ms": last.get("decode_ms"),
                                 "decode_tok_s": round(last["generated"] / (last["decode_ms"] / 1000), 1)
                                 if n and last.get("generated") and last.get("decode_ms") else None,
                                 "hit_rate": hit_rate, "ram_blobs": last.get("ram_blobs"),
-                                "file_blobs": last.get("file_blobs"), "file_mb": last.get("file_mb")})
+                                "file_blobs": last.get("file_blobs"), "file_mb": last.get("file_mb"),
+                                # #457: the speculative drafts from the DONE line (None: the engine did not say)
+                                "drafts_offered": last.get("drafts_offered"),
+                                "drafts_accepted": last.get("drafts_accepted")})
                             t = self.totals
                             t["requests"] += 1
-                            t["prompt_tokens"] += len(ids)
+                            t["prompt_tokens"] += seen
                             t["reused"] += last.get("reused") or 0
                             t["output_tokens"] += n
                             t["prompt_ms"] += last.get("prompt_ms") or 0.0
                             t["decode_ms"] += last.get("decode_ms") or 0.0
+                            t["drafts_offered"] += last.get("drafts_offered") or 0
+                            t["drafts_accepted"] += last.get("drafts_accepted") or 0
                             fresh = getattr(self.engine, "last", None)
                             if fresh is not None and fresh is not before:      # the engine's clock for THIS request
-                                timings = request_timings(len(ids), n, last)
+                                timings = request_timings(seen, n, last)
                                 self.last_timings = dict(timings, at=int(time.time())) if timings else None
                             self.last_request_at = time.time()
                             now = time.time()
@@ -1322,6 +1592,10 @@ class Service:
                             hit_msg = f", expert cache {hit_rate*100:.1f}% hit" if hit_rate is not None else ""
                             print(f"[strata] done: {n} tokens in {el:.0f} s ({rate:.1f} tok/s) "
                                   f"({finish}, cancel={cancel.is_set()}){hit_msg}", flush=True)
+                            if finish == "length" and parser.state == "reasoning":   # #530
+                                print("[strata] the reply reached max tokens while still thinking, so it has no "
+                                      "answer: a thinking budget (reasoning_budget_tokens, in the request or in "
+                                      "strata-<model>.json for every request) leaves room to answer", flush=True)
                             if os.environ.get("STRATA_DEBUG") and raw_ids:
                                 print(f"[strata] raw: {self.tok.decode(raw_ids)!r}", flush=True)
                         self.status["busy"] = False
@@ -1336,6 +1610,17 @@ class Service:
                        "timings": timings}
 
 
+def prompt_tokens_seen(prompt_tokens: int, last: dict) -> int:
+    """#471: the prompt tokens a request got through - all of them, unless the engine's DONE line says a cancel stopped
+    its prompt read part-way (then the reused ones plus those read).  /metrics' history and totals count these, so a
+    cancelled read is neither recorded as the whole prompt nor given a rate from tokens it never read.  An engine
+    before 0.1.36 does not say (no `prompt_read`): the whole prompt, as before."""
+    read = last.get("prompt_read")
+    if read is None or last.get("finish") != "cancel":
+        return prompt_tokens
+    return min(prompt_tokens, int(last.get("reused") or 0) + int(read))
+
+
 def request_timings(prompt_tokens: int, generated: int, last: dict) -> dict | None:
     """One request's `timings` in llama.cpp's names (what its clients show as speed), from the engine's own clock
     (StrataEngine.last): prompt_n is what was read, cache_n what the conversation cache already held.  None when the
@@ -1688,10 +1973,76 @@ def make_handler(svc: Service):
         protocol_version = "HTTP/1.0"                       # SSE ends by closing the connection
 
         record = None                                       # #332: this request's monitor record, if kept
+        watch_done = None                                   # #430 #431: stops this request's disconnect watcher
 
         def log_message(self, fmt, *args):
             pass
 
+        def parse_request(self):
+            """Without an API key, every request (any method) first passes the Host check: DNS rebinding protection
+            (host_allowed).  With a key a rebinding page cannot authenticate, so the check is skipped: tunnels and
+            proxies that pass their own name on keep working."""
+            if not super().parse_request():
+                return False
+            host = self.headers.get("Host")
+            if svc.api_key or host_allowed(host, svc.host_names, "*" in svc.allowed_hosts):
+                return True
+            print(f"[strata] refused a request for Host {host!r} from {self.client_address[0]}: not a name this server "
+                  f"answers to (add it to \"allowed_hosts\" in the config or STRATA_ALLOWED_HOSTS, or set an API key)",
+                  flush=True)
+            self._json(403, {"error": {"type": "forbidden", "message":
+                             f"Host {host!r} is not allowed (DNS rebinding protection). Reaching Strata under this "
+                             f"name on purpose? Add it to \"allowed_hosts\" in the config (strata-<model>.json) or to "
+                             f"the STRATA_ALLOWED_HOSTS environment variable, or set an API key (\"api_key\"), which "
+                             f"turns this check off"}})
+            return False
+
+        def _foreign_page(self) -> bool:
+            """Without an API key, a /v1 POST from a browser page of another site (any site can POST text/plain
+            there without a CORS preflight) would burn GPU time: an Origin header must name an allowed page, and
+            then the body must be JSON.  No Origin (curl, the SDKs, other servers): any content type, as before."""
+            origin = self.headers.get("Origin")
+            if svc.api_key or not origin:
+                return False
+            if not origin_allowed(origin, self.headers.get("Host"), svc.host_names,
+                                  [*svc.trusted_origins, *svc.cors_origins]):
+                print(f"[strata] refused an API request from the web page {origin!r} (no API key; add its host to "
+                      f"\"allowed_hosts\" or its origin to \"cors_origins\" in the config)", flush=True)
+                self._json(403, {"error": {"type": "forbidden", "message":
+                                 f"web pages of {origin} may not use this server without an API key; set \"api_key\", "
+                                 f"or add the page's host to \"allowed_hosts\" (or its origin to \"cors_origins\") "
+                                 f"in the config"}})
+                return True
+            if not self.headers.get("Content-Type", "").startswith("application/json"):
+                self._json(415, {"error": {"message": "send application/json"}})
+                return True
+            return False
+
+        def _watch_client(self, cancel: threading.Event) -> None:
+            """#430 #431: cancel the request as soon as its client hangs up.  A non-streamed request writes nothing
+            until it ends, and a streamed one only a keep-alive per prompt chunk (and the first write after a hang-up
+            usually still succeeds), so a dropped request kept the engine busy until its answer or the whole prompt
+            was done.  Every 0.5 s: the socket readable with nothing to read (EOF) means the client closed it.  A
+            request is HTTP/1.0 and fully read here, so no later bytes are expected - data is not a hang-up."""
+            done = self.watch_done = threading.Event()
+            sock = self.connection
+
+            def watch():
+                while not done.wait(0.5) and not cancel.is_set():
+                    try:
+                        readable, _, _ = select.select([sock], [], [], 0)
+                        gone = bool(readable) and sock.recv(1, socket.MSG_PEEK) == b""
+                    except (ConnectionError, TimeoutError):
+                        gone = True
+                    except (OSError, ValueError):            # the socket was closed here: the request has ended
+                        return
+                    if gone:
+                        self._note(outcome="disconnected")
+                        cancel.set()
+                        return
+
+            threading.Thread(target=watch, daemon=True, name="strata-client-watch").start()
+
         def _note(self, **values):
             """#332: what the monitor shows about this request (nothing when the monitor is off)."""
             if self.record is not None:
@@ -1870,9 +2221,14 @@ def make_handler(svc: Service):
             if not self._authorized():
                 return
             path = self.path.split("?")[0].rstrip("/")   # issue #55: Claude Code posts /v1/messages?beta=true
+            if path.startswith("/v1/") and self._foreign_page():
+                return
             if path == "/settings":
                 self._settings()
                 return
+            # JSON from Strata's own page only, as /settings: else a plain form POST from any site unloads the model
+            if path in ("/unload", "/load") and not self._own_page("the model can be loaded or unloaded"):
+                return
             if path == "/unload":                            # give the GPU back now (between requests)
                 try:
                     r = svc.unload()
@@ -1944,6 +2300,8 @@ def make_handler(svc: Service):
                 self._note(outcome="disconnected")
                 raise                                        # as before #332: the server's own handling
             finally:
+                if self.watch_done is not None:
+                    self.watch_done.set()
                 record = self.record
                 if record is not None:
                     with svc.status_lock:
@@ -2073,6 +2431,7 @@ def make_handler(svc: Service):
             ids, thinking, max_new = svc.prepare(messages, tools, kw, max_new)
             _debug_req("openai", req, messages, tools, max_new, thinking, len(ids))
             cancel = threading.Event()
+            self._watch_client(cancel)                       # #430 #431
             run = run_with_mcp(svc, svc.mcp, messages, tools, kw, ids, thinking, max_new, max_req, req, cancel,
                                {t["name"] for t in extra}) if use_mcp else None
             chunks = openai_chunks(svc, req, ids, thinking, tools, max_new, cancel, run=run)
@@ -2124,6 +2483,7 @@ def make_handler(svc: Service):
             ids, thinking, max_new = svc.prepare(messages, tools, kw, max_new)
             _debug_req("anthropic", req, messages, tools, max_new, thinking, len(ids))
             cancel = threading.Event()
+            self._watch_client(cancel)                       # #430 #431
             events = anthropic_events(svc, req, ids, thinking, tools, max_new, cancel)
             events = self._capture(events, "anthropic")
             if not req.get("stream"):
@@ -2184,6 +2544,36 @@ def warn_tight_ram(arena_mib) -> None:
               + "Close other programs, or run START-HERE --setup and pick a smaller size (Q2_0 / IQ2_XS).", flush=True)
 
 
+DESKTOP_FREE_MIB = 2048          # #560 #516: below this, an AMD card that also drives a Linux desktop can run out
+DESKTOP_RESERVE_MIB = 3072       # what kept KDE/Wayland alive beside a full expert cache in both reports
+
+
+def linux_desktop(env=None) -> bool:
+    """A graphical session on Linux (Wayland or X): its compositor, browser and apps take VRAM after the model has."""
+    env = os.environ if env is None else env
+    return os.name != "nt" and sys.platform != "darwin" and bool(env.get("WAYLAND_DISPLAY") or env.get("DISPLAY"))
+
+
+def desktop_vram_note(backend, vram_free_mib, args: list, desktop: bool) -> str:
+    """#560 #516: on Linux, when the desktop needs VRAM the AMD card does not have, amdgpu moves GPU memory (the expert
+    cache, ~24 GB) to system RAM, which the experts already fill - the OOM killer then ends the compositor.  The
+    default reserve (700 MiB) is sized for a card without a desktop.  A recommendation, nothing changes: "" when it
+    does not apply."""
+    if backend != "hip" or not desktop or not isinstance(vram_free_mib, int) or vram_free_mib >= DESKTOP_FREE_MIB:
+        return ""
+    try:
+        reserve = int(args[args.index("--vram-reserve-mib") + 1]) if "--vram-reserve-mib" in args else 700
+    except (ValueError, IndexError):
+        reserve = 700
+    if reserve >= DESKTOP_RESERVE_MIB:
+        return ""
+    return (f"[strata] note: {vram_free_mib} MiB of VRAM free with the model loaded. If this AMD card also drives your "
+            "desktop and the desktop or apps crash after the start (the driver moves the expert cache to RAM and "
+            "the OOM killer ends the session), keep more VRAM free: ./setup.sh --vram-reserve-mib "
+            f"{DESKTOP_RESERVE_MIB} (remembered; the expert cache gets "
+            f"{(DESKTOP_RESERVE_MIB - reserve) / 1024:.1f} GB less, a few % of speed)")
+
+
 def lan_addresses() -> list[str]:
     """This PC's IPv4 addresses on its networks (what another device types in), without loopback/link-local."""
     import socket
@@ -2203,7 +2593,114 @@ def lan_addresses() -> list[str]:
     return ([first] if ok(first) else []) + sorted(ip for ip in ips if ok(ip) and ip != first)
 
 
+def host_name(value) -> str:
+    """The name in a Host header (or an origin's host[:port]): "Example.com:8080" -> "example.com",
+    "[::1]:8095" -> "::1"; "" when it is malformed."""
+    v = (value or "").strip().lower()
+    if v.startswith("["):
+        name, sep, rest = v[1:].partition("]")
+        return name if sep and (not rest or (rest[:1] == ":" and rest[1:].isdigit())) else ""
+    if v.count(":") == 1:
+        v, port = v.split(":")
+        if not port.isdigit():
+            return ""
+    elif ":" in v:                                       # a bare IPv6 address (a config entry)
+        return v
+    v = v.rstrip(".")
+    return v if v and all(c.isalnum() or c in "-._" for c in v) else ""
+
+
+def _is_ip(name: str) -> bool:
+    import ipaddress
+    try:
+        ipaddress.ip_address(name)
+        return True
+    except ValueError:
+        return False
+
+
+def _name_in(name: str, names) -> bool:
+    """`name` is one of `names`, or below an entry that starts with a dot (".example.com")."""
+    return name in names or any(n.startswith(".") and (name.endswith(n) or name == n[1:]) for n in names)
+
+
+def allowed_hosts_of(value, env: str = "") -> list[str]:
+    """The config's allowed_hosts (a name or a list) plus $STRATA_ALLOWED_HOSTS (comma-separated): host names, "*" or
+    ".example.com" (it and every name below it).  A scheme, port or path is dropped ("https://a.example.com:8443/"
+    -> "a.example.com"); a wrong entry stops the start (ValueError)."""
+    items = [] if value in (None, "") else [value] if isinstance(value, str) else value
+    if not isinstance(items, list) or not all(isinstance(x, str) for x in items):
+        raise ValueError("allowed_hosts: expected a host name or a list of them")
+    out = []
+    for raw in items + [x for x in env.split(",") if x.strip()]:
+        x = raw.strip().lower()
+        if x == "*":
+            out.append(x)
+            continue
+        x = x.split("://", 1)[-1].split("/", 1)[0]
+        dot = x.startswith(".")
+        name = host_name(x[1:] if dot else x)
+        if not name:
+            raise ValueError(f"allowed_hosts: {raw!r} is not a host name like strata.example.com")
+        out.append("." + name if dot else name)
+    return list(dict.fromkeys(out))
+
+
+def host_names_for(bind_host: str, allowed_hosts=(), trusted_origins=()) -> set[str]:
+    """Every name this server answers to besides an IP address: localhost, the address it listens on, the config's
+    allowed_hosts and trusted_origins' hosts, and - listening beyond this PC (0.0.0.0 or a LAN address) - this PC's
+    name and LAN addresses (the LAN addresses matter for the Origin check, which takes no IP on trust)."""
+    names = set(LOOPBACK_NAMES)
+    bind = host_name(bind_host)
+    if bind and bind not in ("0.0.0.0", "::"):
+        names.add(bind)
+    if bind not in LOOPBACK_NAMES:
+        try:
+            pc = socket.gethostname().lower()
+            names.update((pc, pc + ".local"))
+        except OSError:
+            pass
+        names.update(("host.docker.internal", *lan_addresses()))
+    names.update(x for x in allowed_hosts if x != "*")
+    names.update(filter(None, (host_name(o.split("://", 1)[-1]) for o in trusted_origins)))
+    return names
+
+
+def host_allowed(host, names, any_host=False) -> bool:
+    """DNS rebinding: a web page of another site whose name its DNS points at 127.0.0.1 reaches this server as the
+    same origin, so the browser lets it read every answer.  Its requests carry that site's name in Host, so only
+    the names this server answers to pass.  An IP address passes (a page served from an IP is that IP's own page;
+    rebinding needs a name), as does "*.localhost" (browsers never ask DNS for it) and a request without a Host
+    header (HTTP/1.0 clients; browsers always send one)."""
+    if any_host or not (host or "").strip():
+        return True
+    name = host_name(host)
+    return bool(name) and (_is_ip(name) or name == "localhost" or name.endswith(".localhost")
+                           or _name_in(name, names))
+
+
+def origin_allowed(origin: str, host, names, origins=()) -> bool:
+    """A browser page's Origin that may use the model without an API key: this server's own page (the Origin is the
+    request's own Host), a page on one of the names this server answers to (any port), or an origin the config lists
+    (trusted_origins, cors_origins).  Unlike the Host check no IP passes on trust: a page served from any other IP is
+    another site.  "null" (a sandboxed frame, a file:// page) does not pass: any site can send it.  An origin of another
+    scheme (chrome-extension://, moz-extension://, an Electron app's app://) does: no web site can send one."""
+    origin = (origin or "").strip().rstrip("/")
+    if origin in origins or "*" in origins:             # cors_origins ["*"]: the config lets every page in
+        return True
+    scheme, sep, rest = origin.lower().partition("://")
+    if not sep or not scheme:
+        return False
+    if scheme not in ("http", "https"):
+        return True
+    if host and rest == host.strip().lower():
+        return True
+    name = host_name(rest)
+    return bool(name) and (name in LOOPBACK_NAMES or name.endswith(".localhost") or _name_in(name, names))
+
+
 def serve(svc: Service, host="127.0.0.1", port=8095) -> ThreadingHTTPServer:
+    svc.host_names = host_names_for(host, svc.allowed_hosts, svc.trusted_origins)
     svc.start_telemetry()
     httpd = Server((host, port), make_handler(svc))
     threading.Thread(target=httpd.serve_forever, daemon=True).start()
@@ -2412,7 +2909,8 @@ def main() -> int:
             vcfg = {k: (os.path.abspath(os.path.join(cfg.get("cwd") or ".", v))
                         if k in ("exe", "mmproj", "model") and isinstance(v, str) and not os.path.isabs(v) else v)
                     for k, v in cfg["vision"].items()}
-            vision = Vision(vcfg, log=open(cfg["log"], "a", encoding="utf-8") if cfg.get("log") else None, env=env)
+            vision = Vision(vcfg, log=open(cfg["log"], "a", encoding="utf-8") if cfg.get("log") else None,
+                            env=vision_env(cfg, env))
         print("model unloaded; the first request loads it ..." if lazy else
               "loading the model (the first start takes a minute or two) ...", flush=True)
         if len(gpu_list(cfg)) > 1:
@@ -2420,8 +2918,17 @@ def main() -> int:
         # a relative "exe" is the config's cwd's: Windows' CreateProcess resolves "engine/strata.exe" against nothing
         # it is told about (WinError 2), so it is made absolute here
         exe = cfg["exe"] if os.path.isabs(cfg["exe"]) else os.path.abspath(os.path.join(cfg.get("cwd") or ".", cfg["exe"]))
+        try:
+            silence = engine_silence_s(cfg)             # #481: checked before the (minutes-long) start
+        except ValueError as e:
+            raise SystemExit(f"[strata] config {e}")
         engine = StrataEngine(exe, engine_args(cfg), cwd=cfg.get("cwd"), log=cfg.get("log"), env=env, lazy=lazy)
+        engine.silence_s = silence                      # an attribute of its own: restart() keeps it
         warn_tight_ram(engine.info.get("arena_mib"))
+        note = desktop_vram_note(cfg.get("backend"), engine.info.get("vram_free_mib"), engine.spawn[1],
+                                 linux_desktop())
+        if note:                                        # #560 #516: before --open starts a browser on that card
+            print(note, flush=True)
     else:
         engine, vision, sampling_defaults = MockEngine(tok, a.script or [
             "Thinking about it.</think>\n\nHello from the mock engine."]), None, {}
@@ -2446,6 +2953,13 @@ def main() -> int:
     svc.api_key = a.api_key or cfg.get("api_key", "")
     svc.cors_origins = origins_of(cfg.get("cors_origins"), "cors_origins", wildcard=True)
     svc.trusted_origins = origins_of(cfg.get("trusted_origins"), "trusted_origins", wildcard=False)
+    try:
+        svc.allowed_hosts = allowed_hosts_of(cfg.get("allowed_hosts"), os.environ.get("STRATA_ALLOWED_HOSTS", ""))
+    except ValueError as e:
+        raise SystemExit(f"[strata] config {e}")
+    if svc.allowed_hosts:
+        print("[strata] Host check off: any name reaches this server (allowed_hosts \"*\")" if "*" in svc.allowed_hosts
+              else f"[strata] also answers to the host names {', '.join(svc.allowed_hosts)} (allowed_hosts)", flush=True)
     svc.api_monitor = a.api_monitor or cfg.get("api_monitor") is True
     if svc.api_monitor:
         print("[strata] API request monitor on (/api-monitor): the last 100 requests' prompts and answers are kept in "

serve/test_security.py

@@ -0,0 +1,255 @@
+"""serve/test_security.py - the Host check (DNS rebinding) and the Origin check for requests without an API key,
+against the mock engine (no GPU, no pack).
+
+    python -m unittest serve.test_security -v
+"""
+from __future__ import annotations
+
+import contextlib
+import http.client
+import io
+import json
+import socket
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from serve.frontend import ChatTemplate  # noqa: E402
+from serve.server import (ByteTokenizer, MockEngine, Service, allowed_hosts_of, host_allowed,  # noqa: E402
+                          host_name, host_names_for, origin_allowed, serve)
+
+ROOT = Path(__file__).resolve().parents[1]
+LOCAL = {"localhost", "127.0.0.1", "::1"}
+
+
+class HostNames(unittest.TestCase):
+    def test_host_name(self):
+        self.assertEqual(host_name("Example.COM:8080"), "example.com")
+        self.assertEqual(host_name("localhost"), "localhost")
+        self.assertEqual(host_name("[::1]:8095"), "::1")
+        self.assertEqual(host_name("[::1]"), "::1")
+        self.assertEqual(host_name("strata.example.com."), "strata.example.com")
+        for bad in ("", "a:b", "[::1", "[::1]x", "a b", "evil.com/x"):
+            self.assertEqual(host_name(bad), "", bad)
+
+    def test_allowed_hosts_of(self):
+        self.assertEqual(allowed_hosts_of(None), [])
+        self.assertEqual(allowed_hosts_of("Strata.Example.com"), ["strata.example.com"])
+        self.assertEqual(allowed_hosts_of(["https://a.example.com:8443/x", ".example.org", "*"], "box, nas.lan "),
+                         ["a.example.com", ".example.org", "*", "box", "nas.lan"])
+        for bad in (["bad name"], 5, [3], "http://"):
+            with self.assertRaises(ValueError):
+                allowed_hosts_of(bad)
+
+    def test_names_from_the_bind_address(self):
+        with mock.patch("serve.server.lan_addresses", return_value=["192.168.1.20"]):
+            self.assertEqual(host_names_for("127.0.0.1"), LOCAL)                 # this PC only: nothing else
+            self.assertEqual(host_names_for("localhost"), LOCAL)
+            names = host_names_for("0.0.0.0")                                    # other devices too
+            self.assertIn(socket.gethostname().lower(), names)
+            self.assertIn(socket.gethostname().lower() + ".local", names)
+            self.assertIn("192.168.1.20", names)
+            self.assertIn("10.0.0.7", host_names_for("10.0.0.7"))
+            names = host_names_for("127.0.0.1", ["strata.example.com", "*"], ["https://chat.example.net"])
+            self.assertTrue({"strata.example.com", "chat.example.net"} <= names)
+            self.assertNotIn("*", names)
+
+
+class HostCheck(unittest.TestCase):
+    def test_accepted(self):
+        for host in ("localhost:8095", "LOCALHOST", "127.0.0.1:1", "[::1]:8095", "192.168.1.20:8095", "[fe80::1]",
+                     "app.localhost:8095", None, "", "box:8095", "a.example.org", "example.org:443"):
+            self.assertTrue(host_allowed(host, LOCAL | {"box", ".example.org"}), host)
+
+    def test_refused(self):
+        # a rebinding page sends its own name; lookalikes of allowed names stay out
+        for host in ("evil.example.com", "evil.example.com:8095", "localhost.evil.com", "127.0.0.1.nip.io",
+                     "box.evil.com", "xexample.org", "bad host", "a:b:c:d:e:f:g:h:i"):
+            self.assertFalse(host_allowed(host, LOCAL | {"box", ".example.org"}), host)
+
+    def test_any_host(self):
+        self.assertTrue(host_allowed("evil.example.com", LOCAL, any_host=True))
+
+
+class OriginCheck(unittest.TestCase):
+    def test_accepted(self):
+        names = LOCAL | {"box"}
+        self.assertTrue(origin_allowed("http://127.0.0.1:8095", "127.0.0.1:8095", names))      # Strata's own page
+        self.assertTrue(origin_allowed("http://192.168.1.20:8095", "192.168.1.20:8095", names))  # ... by LAN IP
+        self.assertTrue(origin_allowed("http://localhost:3000", "127.0.0.1:8095", names))      # a local app
+        self.assertTrue(origin_allowed("http://[::1]:3000/", "127.0.0.1:8095", names))
+        self.assertTrue(origin_allowed("https://box", "127.0.0.1:8095", names))
+        self.assertTrue(origin_allowed("https://chat.example.com", "x", names, ["https://chat.example.com"]))
+        self.assertTrue(origin_allowed("https://any.example.com", "x", names, ["*"]))           # cors_origins ["*"]
+        # browser extensions and desktop apps: no web site can send another scheme than http(s) (or "null")
+        for origin in ("chrome-extension://abcdef", "moz-extension://1234-5678", "app://."):
+            self.assertTrue(origin_allowed(origin, "127.0.0.1:8095", names), origin)
+
+    def test_refused(self):
+        names = LOCAL | {"box"}
+        for origin in ("http://evil.example.com", "http://1.2.3.4", "http://192.168.1.99:8095", "null", "",
+                       "http://localhost.evil.com", "https://box.evil.com", "://x"):
+            self.assertFalse(origin_allowed(origin, "127.0.0.1:8095", names, ["https://chat.example.com"]), origin)
+
+
+class OverHttp(unittest.TestCase):
+    """The checks in the running server, as a browser and as curl/the SDKs send their requests."""
+
+    def setUp(self):
+        tok = ByteTokenizer()
+        self.svc = Service(MockEngine(tok, "</think>\n\nok", max_context=4096), tok,
+                           ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        self.httpd = None
+
+    def start(self, **attrs):
+        for k, v in attrs.items():
+            setattr(self.svc, k, v)
+        self.httpd = serve(self.svc, port=0)
+        self.port = self.httpd.server_address[1]
+
+    def tearDown(self):
+        if self.httpd:
+            self.httpd.shutdown()
+            self.httpd.server_close()
+
+    def req(self, method, path, body=None, headers=None, host=None):
+        """(status, JSON, what the server printed); `host` replaces the Host header http.client sends."""
+        c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
+        h = dict(headers or {})
+        if host is not None:
+            h["Host"] = host
+        data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode()
+        out = io.StringIO()
+        try:
+            with contextlib.redirect_stdout(out):
+                c.request(method, path, body=data, headers=h)
+                r = c.getresponse()
+                raw = r.read()
+            return r.status, json.loads(raw) if raw[:1] in (b"{", b"[") else raw, out.getvalue()
+        finally:
+            c.close()
+
+    def chat_body(self):
+        return {"model": "m", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 64}
+
+    # --- Host
+    def test_rebinding_name_is_refused_everywhere(self):
+        self.start()
+        for method, path in (("GET", "/status"), ("GET", "/metrics"), ("GET", "/"), ("GET", "/health"),
+                             ("POST", "/settings"), ("POST", "/v1/chat/completions"), ("OPTIONS", "/v1/models")):
+            code, body, log = self.req(method, path, {} if method == "POST" else None,
+                                       {"Content-Type": "application/json"}, host=f"evil.example.com:{self.port}")
+            self.assertEqual(code, 403, path)
+            if method != "OPTIONS":
+                self.assertIn("allowed_hosts", body["error"]["message"])
+            self.assertEqual(log.count("\n"), 1, path)                          # one log line each
+            self.assertIn("evil.example.com", log)
+
+    def test_local_names_and_ips_pass(self):
+        self.start()
+        for host in (f"127.0.0.1:{self.port}", f"localhost:{self.port}", f"[::1]:{self.port}",
+                     f"192.168.1.20:{self.port}"):
+            self.assertEqual(self.req("GET", "/status", host=host)[0], 200, host)
+
+    def test_no_host_header_passes(self):
+        self.start()
+        with socket.create_connection(("127.0.0.1", self.port), timeout=10) as s:      # an HTTP/1.0 client
+            s.sendall(b"GET /health HTTP/1.0\r\n\r\n")
+            raw = b""
+            while chunk := s.recv(65536):
+                raw += chunk
+        self.assertTrue(raw.startswith(b"HTTP/1.0 200"), raw[:40])
+
+    def test_allowed_hosts_and_wildcard(self):
+        self.start(allowed_hosts=["strata.example.com", ".home.arpa"])
+        self.assertEqual(self.req("GET", "/status", host="strata.example.com")[0], 200)
+        self.assertEqual(self.req("GET", "/status", host="nas.home.arpa:8095")[0], 200)
+        self.assertEqual(self.req("GET", "/status", host="evil.example.com")[0], 403)
+        self.svc.allowed_hosts = ["*"]
+        self.assertEqual(self.req("GET", "/status", host="evil.example.com")[0], 200)
+
+    def test_refusal_names_both_ways_out(self):
+        self.start()
+        message = self.req("GET", "/status", host="evil.example.com")[1]["error"]["message"]
+        self.assertIn("allowed_hosts", message)
+        self.assertIn("api_key", message)
+
+    def test_with_a_key_the_host_check_is_off(self):
+        # a rebinding page cannot authenticate; tunnels and proxies that pass their own name on keep working
+        self.start(api_key="s3cret")
+        tunnel = "random-words.trycloudflare.com"
+        self.assertEqual(self.req("GET", "/status", host=tunnel)[0], 401)
+        self.assertEqual(self.req("GET", "/status", headers={"Authorization": "Bearer s3cret"}, host=tunnel)[0], 200)
+        self.assertEqual(self.req("GET", "/health", host=tunnel)[0], 200)
+        code, _, log = self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                {"Content-Type": "application/json", "Authorization": "Bearer s3cret"}, host=tunnel)
+        self.assertEqual(code, 200)
+        self.assertNotIn("refused", log)
+
+    def test_a_trusted_origin_s_host_is_allowed(self):
+        self.start(trusted_origins=["https://strata.example.com"])
+        self.assertEqual(self.req("GET", "/status", host="strata.example.com")[0], 200)
+
+    # --- Origin on /v1 without an API key
+    def test_curl_and_sdks_without_origin(self):
+        self.start()
+        for ctype in ("application/json", "text/plain", None):              # curl -d sends a form type
+            code, body, _ = self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                     {"Content-Type": ctype} if ctype else {})
+            self.assertEqual(code, 200, ctype)
+            self.assertEqual(body["choices"][0]["message"]["content"], "ok")
+
+    def test_cross_site_page_is_refused(self):
+        self.start()
+        for path in ("/v1/chat/completions", "/v1/messages", "/v1/messages/count_tokens"):
+            for ctype in ("text/plain", "application/json"):
+                code, body, log = self.req("POST", path, self.chat_body(),
+                                           {"Content-Type": ctype, "Origin": "http://evil.example.com"})
+                self.assertEqual(code, 403, (path, ctype))
+                self.assertIn("api_key", body["error"]["message"])
+                self.assertIn("evil.example.com", log)
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                  {"Content-Type": "text/plain", "Origin": "null"})[0], 403)
+
+    def test_own_and_local_pages_pass(self):
+        self.start()
+        json_type = {"Content-Type": "application/json"}
+        for origin in (f"http://127.0.0.1:{self.port}", "http://localhost:3000", "chrome-extension://abcdef"):
+            self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                      {**json_type, "Origin": origin})[0], 200, origin)
+        # a browser page sends JSON; text/plain from a page is the cross-site "simple request" shape
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                  {"Content-Type": "text/plain", "Origin": f"http://127.0.0.1:{self.port}"})[0], 415)
+
+    def test_configured_origins_pass(self):
+        self.start(cors_origins=["https://chat.example.com"], trusted_origins=["https://strata.example.com"],
+                   allowed_hosts=["webui.lan"])
+        for origin in ("https://chat.example.com", "https://strata.example.com", "http://webui.lan:3000"):
+            self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                      {"Content-Type": "application/json", "Origin": origin})[0], 200, origin)
+
+    def test_with_a_key_the_key_decides(self):
+        self.start(api_key="s3cret")
+        headers = {"Content-Type": "text/plain", "Origin": "http://evil.example.com"}
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(), headers)[0], 401)
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                  {**headers, "Authorization": "Bearer s3cret"})[0], 200)
+
+    # --- /unload and /load
+    def test_unload_and_load_need_json_from_the_own_page(self):
+        self.start()
+        form = {"Content-Type": "application/x-www-form-urlencoded"}
+        for path in ("/unload", "/load"):
+            self.assertEqual(self.req("POST", path, b"a=1", form)[0], 415, path)
+            self.assertEqual(self.req("POST", path, b"", {})[0], 415, path)
+            self.assertEqual(self.req("POST", path, {}, {"Content-Type": "application/json",
+                                                         "Origin": "http://evil.example.com"})[0], 403, path)
+        self.assertEqual(self.req("POST", "/unload", {}, {"Content-Type": "application/json"})[0], 200)
+        self.assertEqual(self.req("POST", "/load", {}, {"Content-Type": "application/json",
+                                                        "Origin": f"http://127.0.0.1:{self.port}"})[0], 200)
+
+
+if __name__ == "__main__":
+    unittest.main()

serve/test_server.py

@@ -4,6 +4,8 @@
 """
 from __future__ import annotations
 
+import contextlib
+import io
 import json
 import os
 import sys
@@ -18,7 +20,9 @@ from unittest import mock
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 from serve.frontend import ChatTemplate  # noqa: E402
-from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, GpuBusy, MockEngine, Service, StrataEngine, request_timings, serve  # noqa: E402
+from serve.server import (CTX_SLACK, ByteTokenizer, EngineDied, GpuBusy, MockEngine, Service, StrataEngine,  # noqa: E402
+                          engine_args, prompt_tokens_seen, request_timings, serve, start_failure_hint)
+from types import SimpleNamespace  # noqa: E402
 
 ROOT = Path(__file__).resolve().parents[1]
 CTX = 4096
@@ -95,6 +99,13 @@ class MaxTokens(unittest.TestCase):
         m = ENGINE_REQUEST.search(line)
         self.assertIsNotNone(m)
         self.assertEqual((m["prompt"], m["reused"], m["gen"], m["tg"]), ("1200", "1000", "30", "100.0"))
+        # #471: a request cancelled while its prompt was read says how far it got
+        m = ENGINE_REQUEST.search("strata serve: prompt 98179 tokens = 0 reused + 12288 of 98179 read in 17565 ms "
+                                  "(699.6 tok/s), 0 generated in 0 ms (0.0 tok/s), drafts accepted 0 of 0, "
+                                  "0 checkpoints (cancelled)")
+        self.assertIsNotNone(m)
+        self.assertEqual((m["prompt"], m["reused"], m["read"], m["pp"], m["gen"]),
+                         ("98179", "0", "17565", "699.6", "0"))
 
     def test_unset_budget_is_the_rest_of_the_context(self):
         cases = {"openai": [{"max_tokens": -1}, {"max_tokens": 0}, {}, {"max_tokens": None},
@@ -126,6 +137,8 @@ class MaxTokens(unittest.TestCase):
                 s, b, _, _ = self.call(api, max_tokens=CTX)
                 self.assertEqual(s, 400)
                 self.assertIn("exceeds the context", b["error"]["message"])
+                self.assertIn("\"fit_max_tokens\": true", b["error"]["message"])     # #545: says how to get past it
+                self.assertRegex(b["error"]["message"], r"at most \d+ here")
 
     def test_unset_budget_with_a_near_full_prompt(self):
         _, _, pt0, _ = self.call("openai", max_tokens=1)
@@ -455,6 +468,55 @@ class ClientShapes(unittest.TestCase):
         for part in ("Be brief.", "Now use digits.", "Also this.", "1+1?"):
             self.assertIn(part, text)
 
+    def test_no_user_turn_is_a_400(self):
+        # #365: the template's own refusal (Qwen's "No user query found in messages." when no turn is a user's query)
+        # answers 400, not a dropped connection
+        with tempfile.TemporaryDirectory() as d:
+            tpl = Path(d) / "chat_template.jinja"
+            tpl.write_text("{% if messages[-1].role != 'user' %}{{ raise_exception('No user query found in messages.') }}"
+                           "{% endif %}{{ messages[-1].content }}", encoding="utf-8")
+            template, self.svc.template = self.svc.template, ChatTemplate(tpl)
+            try:
+                status, b = self.post("/v1/chat/completions", {
+                    "model": "x", "max_tokens": 20, "messages": [{"role": "system", "content": "Only a system."}]})
+                self.assertEqual(status, 400, b)
+                self.assertIn("No user query found", b["error"]["message"])
+            finally:
+                self.svc.template = template
+        status, b = self.post("/v1/chat/completions", {"model": "x", "max_tokens": 20,
+                                                       "messages": [{"role": "user", "content": "hi"}]})
+        self.assertEqual(status, 200, b)                            # the server goes on
+
+    def test_messages_as_a_json_string_over_http(self):
+        # #460: a double-encoded "messages" is answered; one that is not a list of objects is a 400, not a 500
+        encoded = json.dumps([{"role": "user", "content": "1+1?"}])
+        for path in ("/v1/chat/completions", "/v1/messages"):
+            with self.subTest(path=path):
+                status, b = self.post(path, {"model": "x", "max_tokens": 20, "messages": encoded})
+                self.assertEqual(status, 200, b)
+                self.assertIn("1+1?", self.prompt_text())
+                status, b = self.post(path, {"model": "x", "max_tokens": 20, "messages": ["hi"]})
+                self.assertEqual(status, 400, b)
+                self.assertIn("messages must be a list of objects", b["error"]["message"])
+
+    def test_vision_temp_image_removed_when_the_pipe_fails(self):
+        # #352: the temporary image goes even when the encoder's pipe raises
+        from serve.server import Vision
+
+        class Gone:
+            def write(self, _):
+                raise BrokenPipeError("the encoder is gone")
+
+        v = Vision.__new__(Vision)
+        v.dir, v.lock, v.cache = Path(tempfile.mkdtemp(prefix="strata-vision-test-")), threading.Lock(), {}
+        v.proc = mock.Mock(stdin=Gone())
+        with mock.patch.object(Vision, "load", return_value=b""), mock.patch.object(Vision, "normalize",
+                                                                                   return_value=b"png"):
+            with self.assertRaises(BrokenPipeError):
+                v.encode("x")
+        self.assertEqual(list(v.dir.iterdir()), [])
+        v.dir.rmdir()
+
     def test_leading_system_unchanged(self):
         from serve.frontend import anthropic_to_messages, openai_to_messages
         msgs, _, _ = openai_to_messages({"messages": [{"role": "developer", "content": "D"}, {"role": "user", "content": "u"}]})
@@ -462,6 +524,32 @@ class ClientShapes(unittest.TestCase):
         msgs, _, _ = anthropic_to_messages({"system": "S", "messages": [{"role": "user", "content": "u"}]})
         self.assertEqual([m["role"] for m in msgs], ["system", "user"])
 
+    def test_messages_sent_as_a_json_string(self):
+        # #460: a client that double-encodes "messages" (and "tool_calls") as a JSON string gets them decoded; what is
+        # still not a list of objects is a ValueError (the server's 400), not an AttributeError on m.get
+        from serve.frontend import anthropic_to_messages, openai_to_messages
+        call = [{"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{\"x\": 1}"}}]
+        listed = [{"role": "user", "content": "u"}, {"role": "assistant", "content": "", "tool_calls": call}]
+        encoded = [listed[0], dict(listed[1], tool_calls=json.dumps(call))]
+        want = openai_to_messages({"messages": listed})[0]
+        self.assertEqual(want[1]["tool_calls"], [{"function": {"name": "f", "arguments": {"x": 1}}}])
+        self.assertEqual(openai_to_messages({"messages": json.dumps(listed)})[0], want)
+        self.assertEqual(openai_to_messages({"messages": json.dumps(encoded)})[0], want)
+        anth = [{"role": "user", "content": "u"}]
+        self.assertEqual(anthropic_to_messages({"messages": json.dumps(anth)})[0],
+                         anthropic_to_messages({"messages": anth})[0])
+        self.assertEqual(openai_to_messages({})[0], [])                     # no field: nothing, as before
+        self.assertEqual(openai_to_messages({"messages": None})[0], [])
+        for bad in ("not json", "\"a string\"", json.dumps({"role": "user"}), ["hi"], [{"role": "user"}, 3], 5,
+                    {"role": "user", "content": "u"}):
+            for fn in (openai_to_messages, anthropic_to_messages):
+                with self.subTest(bad=bad, fn=fn.__name__):
+                    with self.assertRaisesRegex(ValueError, "messages must be a list of objects"):
+                        fn({"messages": bad})
+        for calls in (["f"], "[1]", [{"function": "f"}], "{"):
+            with self.subTest(calls=calls), self.assertRaisesRegex(ValueError, "tool_calls must be a list of objects"):
+                openai_to_messages({"messages": [{"role": "assistant", "content": "", "tool_calls": calls}]})
+
 
 class SamplingKeys(unittest.TestCase):
     """The GEN line's sampling keys: top_k 0 ("off") or wider than the engine's 64 get the widest list, 64 (they used
@@ -503,6 +591,30 @@ class GpuChoice(unittest.TestCase):
         self.assertEqual(plain.get("CUDA_VISIBLE_DEVICES"), os.environ.get("CUDA_VISIBLE_DEVICES"))
         self.assertEqual(plain.get("CUDA_DEVICE_ORDER"), os.environ.get("CUDA_DEVICE_ORDER"))
 
+    def test_vision_device(self):
+        # #408: the image encoder on its own card; the engine's environment stays as it was
+        from serve.server import child_env, vision_env
+        cfg = {"gpu": [0, 1], "vision": {"exe": "v", "cuda_device": 2}}
+        env = child_env(cfg)
+        venv = vision_env(cfg, env)
+        self.assertEqual(venv["CUDA_VISIBLE_DEVICES"], "2")
+        self.assertEqual(venv["CUDA_DEVICE_ORDER"], "PCI_BUS_ID")
+        self.assertEqual(env["CUDA_VISIBLE_DEVICES"], "0,1")
+        plain = {"gpu": [0, 1], "vision": {"exe": "v"}}
+        self.assertIs(vision_env(plain, env), env)          # no cuda_device: the engine's environment, unchanged
+
+    def test_hip_ordinal(self):
+        """#325: on Windows the HIP ordinal setup resolved wins over the config's "gpu" (an iGPU takes HIP's 0)."""
+        from serve.server import child_env
+        self.assertEqual(child_env({"backend": "hip", "gpu": 1})["HIP_VISIBLE_DEVICES"], "1")       # Linux: KFD order
+        self.assertEqual(child_env({"backend": "hip", "hip_ordinal": 1})["HIP_VISIBLE_DEVICES"], "1")
+        self.assertEqual(child_env({"backend": "hip", "gpu": 0, "hip_ordinal": 1})["HIP_VISIBLE_DEVICES"], "1")
+        self.assertEqual(child_env({"backend": "hip", "gpu": [1, 0], "hip_ordinal": 2})["HIP_VISIBLE_DEVICES"],
+                         "1,0")                                          # a layer split keeps its list
+        self.assertEqual(child_env({"backend": "hip", "gpu": 0, "hip_ordinal": "x"})["HIP_VISIBLE_DEVICES"], "0")
+        plain = child_env({"backend": "hip"})
+        self.assertEqual(plain.get("HIP_VISIBLE_DEVICES"), os.environ.get("HIP_VISIBLE_DEVICES"))
+
 
 class RecordingPrompt(MockEngine):
     def generate(self, ids, max_new, sampling, cancel, embeddings=None):
@@ -532,6 +644,59 @@ class DyingEngine(MockEngine):
             yield t
 
 
+class SlowPromptEngine(MockEngine):
+    """Reads a "long prompt" for up to 20 s without a token (the engine sends nothing then), stopping on cancel."""
+
+    def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+        self.cancelled_after = None
+        t0 = time.monotonic()
+        while time.monotonic() - t0 < 20:
+            if cancel.is_set():
+                self.cancelled_after = time.monotonic() - t0
+                return
+            time.sleep(0.05)
+        yield from super().generate(ids, max_new, sampling, cancel, embeddings)
+
+
+class ClientHangUp(unittest.TestCase):
+    """#430 #431: a client that hangs up during a long prompt read cancels the request within about a second -
+    non-streamed (which writes nothing until the end) and streamed (one keep-alive per prompt chunk) alike."""
+
+    @classmethod
+    def setUpClass(cls):
+        tok = ByteTokenizer()
+        cls.engine = SlowPromptEngine(tok, "</think>\n\nOK", max_context=CTX)
+        cls.svc = Service(cls.engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        cls.httpd = serve(cls.svc, port=0)
+        cls.port = cls.httpd.server_address[1]
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.httpd.shutdown()
+        cls.httpd.server_close()
+
+    def hang_up(self, stream):
+        import socket as so
+        body = json.dumps({"model": "x", "max_tokens": 20, "stream": stream,
+                           "messages": [{"role": "user", "content": "a long prompt"}]}).encode()
+        c = so.create_connection(("127.0.0.1", self.port))
+        c.sendall(b"POST /v1/chat/completions HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: application/json\r\n"
+                  b"Content-Length: " + str(len(body)).encode() + b"\r\n\r\n" + body)
+        time.sleep(1.0)
+        c.close()
+        t0 = time.monotonic()
+        while self.engine.cancelled_after is None and time.monotonic() - t0 < 10:
+            time.sleep(0.05)
+        self.assertIsNotNone(self.engine.cancelled_after, "the request was not cancelled")
+        self.assertLess(self.engine.cancelled_after, 3.0)
+
+    def test_non_streamed(self):
+        self.hang_up(False)
+
+    def test_streamed(self):
+        self.hang_up(True)
+
+
 class EngineDeath(unittest.TestCase):
     """Issue #27: a dead engine is an error (not "length"), and the next request starts it again."""
 
@@ -595,6 +760,284 @@ class EngineDeath(unittest.TestCase):
             httpd.server_close()
 
 
+class DoneLineEngine(MockEngine):
+    """The mock engine whose `last` comes from a DONE line, parsed as StrataEngine parses it."""
+
+    def __init__(self, *a, done_lines=(), **kw):
+        super().__init__(*a, **kw)
+        self.done_lines = list(done_lines)
+
+    def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+        try:
+            yield from super().generate(ids, max_new, sampling, cancel, embeddings)
+        finally:
+            StrataEngine._parse_done(self, self.done_lines.pop(0))
+
+
+class DraftCounts(unittest.TestCase):
+    """#457: GET /metrics gives each request's speculative draft counts (offered / accepted, from the engine's DONE
+    line; None when the line has no such fields) and their running sums in the totals."""
+
+    def test_drafts_in_history_and_totals(self):
+        tok = ByteTokenizer()
+        engine = DoneLineEngine(tok, "</think>\n\nok", max_context=CTX, done_lines=[
+            "DONE 4 20 40.0 30.0 stop 7 12 0",                  # 7 of 12 drafts accepted
+            "DONE 4 20 40.0 30.0 stop",                          # an engine that reports no drafts
+            "DONE 4 20 40.0 30.0 stop 3 5 0 9 10"])
+        svc = Service(engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        self.assertEqual((svc.totals["drafts_offered"], svc.totals["drafts_accepted"]), (0, 0))
+        httpd = serve(svc, port=0)
+        base = f"http://127.0.0.1:{httpd.server_address[1]}"
+        try:
+            for _ in range(3):
+                body = json.dumps({"model": "m", "max_tokens": 10, "messages": [{"role": "user", "content": "hi"}]})
+                with urllib.request.urlopen(urllib.request.Request(base + "/v1/chat/completions", data=body.encode(),
+                                                                   headers={"Content-Type": "application/json"}),
+                                            timeout=30) as r:
+                    self.assertEqual(r.status, 200)
+            with urllib.request.urlopen(base + "/metrics", timeout=10) as r:
+                m = json.loads(r.read())
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+        rows = m["requests"]                                      # newest first
+        self.assertEqual([(r["drafts_offered"], r["drafts_accepted"]) for r in rows], [(5, 3), (None, None), (12, 7)])
+        self.assertEqual((m["totals"]["drafts_offered"], m["totals"]["drafts_accepted"]), (17, 10))
+
+
+class LearnedProfile(unittest.TestCase):
+    """#477: "expert_profile_save" in the config: the engine saves its learned profile there, and the next start
+    begins from it when it is a profile of the same model; without the key the arguments are unchanged."""
+
+    def write(self, path, nl=48, ne=512, n=4):
+        sys.path.insert(0, str(ROOT / "tools"))
+        import make_profile
+        old = make_profile.N_LAYER
+        make_profile.N_LAYER = nl
+        try:
+            make_profile.write_profile(path, [(i % nl, i // nl) for i in range(n)], n_expert=ne)
+        finally:
+            make_profile.N_LAYER = old
+
+    def test_without_the_key_nothing_changes(self):
+        args = ["--native", "x", "--expert-profile", "data/expert-profile.bin", "--adapt-every", "4"]
+        self.assertEqual(engine_args({"args": list(args)}), args)
+        self.assertEqual(engine_args({"args": list(args), "expert_profile_save": ""}), args)
+
+    def test_save_and_start_from_it(self):
+        d = Path(tempfile.mkdtemp())
+        self.write(d / "base.bin")
+        cfg = {"args": ["--expert-profile", "base.bin"], "cwd": str(d), "expert_profile_save": "learned.bin",
+               "expert_profile_save_every": 5}
+        # nothing saved yet: the config's profile, and the engine is told where to save
+        self.assertEqual(engine_args(cfg), ["--expert-profile", "base.bin", "--expert-profile-save", "learned.bin",
+                                            "--expert-profile-save-every", "5"])
+        self.write(d / "learned.bin")
+        self.assertEqual(engine_args(cfg)[:2], ["--expert-profile", "learned.bin"])
+        self.write(d / "learned.bin", ne=256)                    # another model's: not used
+        self.assertEqual(engine_args(cfg)[:2], ["--expert-profile", "base.bin"])
+        (d / "learned.bin").write_bytes(b"STRP" + bytes(20))     # not a whole profile
+        self.assertEqual(engine_args(cfg)[:2], ["--expert-profile", "base.bin"])
+        self.write(d / "learned.bin")
+        (d / "learned.bin").write_bytes((d / "learned.bin").read_bytes()[:30])   # truncated
+        self.assertEqual(engine_args(cfg)[:2], ["--expert-profile", "base.bin"])
+
+    def test_no_profile_in_the_args(self):
+        cfg = {"args": ["--native", "x"], "expert_profile_save": "learned.bin"}
+        self.assertEqual(engine_args(cfg), ["--native", "x", "--expert-profile-save", "learned.bin"])
+
+
+class DraftHeadHint(unittest.TestCase):
+    """#474: a start that stopped at "the draft head does not fit" says what to change, from this start's log lines."""
+
+    def log(self, text, before=""):
+        d = tempfile.mkdtemp()
+        p = Path(d) / "engine.log"
+        p.write_text(before + text, encoding="utf-8")
+        return str(p), len(before.encode())
+
+    def test_the_engines_hint_is_relayed(self):
+        p, off = self.log("strata mtp: the draft head over 106299 tokens needs 348 MiB of VRAM and 120 MiB is free.\n"
+                          "strata mtp: hint: a smaller draft vocabulary needs less VRAM: --draft-vocab en (...)\n"
+                          "strata serve: mtp: the draft head does not fit\n")
+        h = start_failure_hint(p, off)
+        self.assertIn("the draft head does not fit", h)
+        self.assertIn("348 MiB", h)
+        self.assertIn("--draft-vocab en", h)
+
+    def test_an_older_engine_gets_the_advice_in_words(self):
+        p, off = self.log("strata serve: mtp: the draft head does not fit\n")
+        self.assertIn("--draft-vocab en", start_failure_hint(p, off))
+
+    def test_other_failures_and_earlier_starts_add_nothing(self):
+        p, off = self.log("strata serve: cannot open the pack\n")
+        self.assertEqual(start_failure_hint(p, off), "")
+        # an earlier start's failure (before this start's offset) is not this one's
+        p, off = self.log("strata serve: cannot open the pack\n",
+                          before="strata serve: mtp: the draft head does not fit\n")
+        self.assertEqual(start_failure_hint(p, off), "")
+        self.assertEqual(start_failure_hint(None, 0), "")
+        self.assertEqual(start_failure_hint(str(Path(tempfile.mkdtemp()) / "missing.log"), 0), "")
+
+
+class DesktopVramNote(unittest.TestCase):
+    """#560 #516: an AMD card on a Linux desktop with little VRAM left after the start gets a recommended reserve."""
+
+    def test_when_it_applies(self):
+        from serve.server import desktop_vram_note
+        note = desktop_vram_note("hip", 624, ["--kv", "int8"], True)
+        self.assertIn("624 MiB of VRAM free", note)
+        self.assertIn("--vram-reserve-mib 3072", note)
+        self.assertIn("2.3 GB less", note)
+
+    def test_when_it_does_not(self):
+        from serve.server import desktop_vram_note
+        self.assertEqual(desktop_vram_note(None, 624, [], True), "")               # NVIDIA
+        self.assertEqual(desktop_vram_note("hip", 624, [], False), "")             # no desktop session
+        self.assertEqual(desktop_vram_note("hip", 2994, [], True), "")             # room left
+        self.assertEqual(desktop_vram_note("hip", None, [], True), "")             # lazy start: no INFO yet
+        self.assertEqual(desktop_vram_note("hip", 900, ["--vram-reserve-mib", "4000"], True), "")   # already raised
+
+    def test_desktop_detection(self):
+        from serve import server
+        with mock.patch.object(server.os, "name", "posix"), mock.patch.object(server.sys, "platform", "linux"):
+            self.assertTrue(server.linux_desktop({"WAYLAND_DISPLAY": "wayland-0"}))
+            self.assertFalse(server.linux_desktop({}))
+
+
+class StartFailureLog(unittest.TestCase):
+    """#496: whatever stopped the engine before READY, the error carries this start's last log lines."""
+
+    def test_the_last_lines_of_this_start(self):
+        from serve.server import start_log_tail
+        d = tempfile.mkdtemp()
+        p = Path(d) / "engine.log"
+        before = "strata serve: an earlier start's line\n"
+        p.write_text(before + "".join(f"strata serve: line {i}\n" for i in range(30)) + "\n"
+                     "strata serve: cannot open the pack\n", encoding="utf-8")
+        tail = start_log_tail(str(p), len(before.encode()))
+        self.assertIn("the engine log's last lines:", tail)
+        self.assertIn("cannot open the pack", tail)
+        self.assertIn("line 29", tail)
+        self.assertNotIn("line 10\n", tail + "\n")                 # 20 lines: 11..29 and the last one
+        self.assertIn("line 11", tail)
+        self.assertNotIn("earlier start", tail)
+        short = start_log_tail(str(p), len(before.encode()), n=3)
+        self.assertEqual(short.count("\n  "), 3)
+        self.assertEqual(start_log_tail(str(p), p.stat().st_size), "")   # nothing from this start
+        self.assertEqual(start_log_tail(None, 0), "")
+        self.assertEqual(start_log_tail(str(Path(d) / "missing.log"), 0), "")
+
+    def test_the_start_error_has_them(self):
+        import serve.server as server
+        fake = "import sys\nsys.stderr.write('strata serve: cannot open the pack packs/x\\n')\nsys.exit(2)\n"
+        with tempfile.TemporaryDirectory() as d:
+            script, log = Path(d) / "fake_strata.py", Path(d) / "strata.log"
+            script.write_text(fake, encoding="utf-8")
+            log.write_text("an earlier start\n", encoding="utf-8")
+            real = server.subprocess.Popen
+            with mock.patch.object(server.subprocess, "Popen",
+                                   lambda cmd, **kw: real([sys.executable, str(script), *cmd[1:]], **kw)), \
+                    mock.patch.object(server, "narrate_start", lambda *a, **k: None):
+                with self.assertRaises(RuntimeError) as cm:
+                    StrataEngine("strata", [], log=str(log))
+            text = str(cm.exception)
+            self.assertIn("exited before it was ready", text)
+            self.assertIn("cannot open the pack packs/x", text)
+            self.assertNotIn("an earlier start", text)
+
+
+class StartNarrator(unittest.TestCase):
+    """#505: the start's words say what happens to the experts - --mmap-experts loads nothing into RAM up front."""
+
+    def test_the_words_follow_the_flags(self):
+        from serve.server import experts_loading_words
+        self.assertIn("loading the experts into RAM (about 38 GB)", experts_loading_words([], "about 38 GB"))
+        mapped = experts_loading_words(["--mmap-experts"], "about 47 GB")
+        self.assertIn("mapping the experts from the model files (about 47 GB", mapped)
+        self.assertIn("not loaded into RAM", mapped)
+        self.assertIn("the GPU does not hold", experts_loading_words(["--resident-experts"], "about 47 GB"))
+        budget = experts_loading_words(["--mmap-experts", "--resident-budget-gib", "71"], "about 50 GB")
+        self.assertIn("up to 71 GiB", budget)
+        self.assertNotIn("about 50 GB", budget)
+
+    def narrate(self, args, lines):
+        import contextlib
+        import io
+        from serve.server import narrate_start
+        d = tempfile.mkdtemp()
+        log = Path(d) / "engine.log"
+        log.write_text("".join(x + "\n" for x in lines), encoding="utf-8")
+        done, out = threading.Event(), io.StringIO()
+        with contextlib.redirect_stdout(out):
+            t = threading.Thread(target=narrate_start, args=(str(log), 0, args, done))
+            t.start()
+            time.sleep(0.8)
+            done.set()
+            t.join(5)
+        return out.getvalue()
+
+    def test_a_mapped_start(self):
+        said = self.narrate(["--mmap-experts"], ["strata generate: experts via mmap (--mmap-experts; the GGUF shards "
+                                                 "in place, no experts.bin)"])
+        self.assertIn("mapping the experts from the model files", said)
+        self.assertNotIn("loading the experts into RAM", said)
+
+    def test_an_arena_start(self):
+        said = self.narrate([], ["strata generate: expert arena: resident, 31.64 GiB",
+                                 "strata generate: loaded 31.64 GiB at 3.17 GiB/s"])
+        self.assertIn("loading the experts into RAM (tens of GB)", said)
+        self.assertIn("experts loaded: 31.64 GiB at 3.17 GiB/s", said)
+
+
+class CancelledRead(unittest.TestCase):
+    """#471: a request cancelled while its prompt was read is recorded with the tokens the engine read (the DONE
+    line's 15th field), not the whole prompt; an older engine's line (no such field) keeps the whole prompt."""
+
+    def test_parse_done_read_field(self):
+        e = SimpleNamespace()
+        StrataEngine._parse_done(e, "DONE 0 98179 17565.0 0.0 cancel 0 0 0 0 0 0 0 0.0 12288")
+        self.assertEqual((e.last["prompt_tokens"], e.last["prompt_read"], e.last["finish"]), (98179, 12288, "cancel"))
+        StrataEngine._parse_done(e, "DONE 0 98179 17565.0 0.0 cancel 0 0 0 0 0 0 0 0.0")
+        self.assertNotIn("prompt_read", e.last)
+
+    def test_prompt_tokens_seen(self):
+        last = {"finish": "cancel", "reused": 1000, "prompt_read": 2000}
+        self.assertEqual(prompt_tokens_seen(98179, last), 3000)
+        self.assertEqual(prompt_tokens_seen(98179, {**last, "finish": "stop"}), 98179)   # read in full: all of it
+        self.assertEqual(prompt_tokens_seen(98179, {"finish": "cancel", "reused": 0}), 98179)   # an older engine
+        self.assertEqual(prompt_tokens_seen(98179, {}), 98179)                          # no DONE at all
+        self.assertEqual(prompt_tokens_seen(10, {**last, "prompt_read": 50}), 10)       # never past the prompt
+
+    def test_history_and_totals(self):
+        tok = ByteTokenizer()
+        engine = DoneLineEngine(tok, "</think>\n\nok", max_context=CTX, done_lines=[
+            "DONE 0 20 400.0 0.0 cancel 0 0 0 0 0 0 0 0.0 8",      # stopped after 8 prompt tokens
+            "DONE 4 20 40.0 30.0 stop 0 0 0 0 0 0 0 0.0 20",
+            "DONE 0 20 400.0 0.0 cancel 0 0 0"])                   # an older engine: no read count
+        svc = Service(engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        httpd = serve(svc, port=0)
+        base = f"http://127.0.0.1:{httpd.server_address[1]}"
+        try:
+            for _ in range(3):
+                body = json.dumps({"model": "m", "max_tokens": 10, "messages": [{"role": "user", "content": "hi"}]})
+                with urllib.request.urlopen(urllib.request.Request(base + "/v1/chat/completions", data=body.encode(),
+                                                                   headers={"Content-Type": "application/json"}),
+                                            timeout=30) as r:
+                    self.assertEqual(r.status, 200)
+            with urllib.request.urlopen(base + "/metrics", timeout=10) as r:
+                m = json.loads(r.read())
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+        rows = m["requests"][::-1]                                # oldest first
+        total = rows[0]["prompt_total"]
+        self.assertGreater(total, 8)
+        self.assertEqual([r["prompt_total"] for r in rows], [total] * 3)
+        self.assertEqual([(r["prompt_tokens"], r["prompt_read"]) for r in rows], [(8, 8), (total, 20), (total, None)])
+        self.assertEqual(m["totals"]["prompt_tokens"], 8 + 2 * total)
+
+
 class LiveRate(unittest.TestCase):
     """The Monitor's Speed readout: live.tok_s is a rate, and a request that never got a DONE keeps no counters.
 
@@ -1397,6 +1840,20 @@ class ThinkingBudget(unittest.TestCase):
         self.assertEqual(len(self.engine.prompts), 1)
         self.assertEqual(b["choices"][0]["message"]["reasoning_content"], ThinkingEngine.THOUGHT[:20])
 
+    def test_a_reply_cut_while_thinking_is_named_in_the_log(self):
+        """#530: max tokens reached inside the thinking gives an empty answer; the server log says what helps."""
+        hint = "reached max tokens while still thinking"
+        for extra, said in (({"max_tokens": 10}, True), ({}, False)):
+            with self.subTest(extra=extra):
+                out = io.StringIO()
+                with contextlib.redirect_stdout(out):
+                    code, b = self.openai(**extra)
+                self.assertEqual(code, 200, b)
+                self.assertEqual(b["choices"][0]["finish_reason"], "length" if said else "stop")
+                self.assertEqual(hint in out.getvalue(), said, out.getvalue())
+                if said:
+                    self.assertIn("reasoning_budget_tokens", out.getvalue())
+
     def test_a_bad_value_is_a_400(self):
         for bad in ("lots", 2.5, True, [1]):
             with self.subTest(value=bad):
@@ -1714,5 +2171,163 @@ class AmdTelemetry(unittest.TestCase):
                 self.assertEqual(svc.free_vram_mib(), 26 << 10)
 
 
+class SilentEngine(unittest.TestCase):
+    """#481: an engine that prints nothing for engine_silence_s during a request (or never acknowledges a STOP) has
+    lost step with the server: it is ended and the request fails with EngineDied, instead of waiting forever."""
+
+    def bare(self, silence, can_stop=False):
+        import io
+        import queue
+        engine = StrataEngine.__new__(StrataEngine)
+        engine.proc = mock.Mock()
+        engine.proc.stdin = io.StringIO()
+        engine.proc.poll.return_value = None
+        engine.lines, engine.can_stop, engine.max_context = queue.Queue(), can_stop, 4096
+        engine.silence_s, engine.log_path = silence, None
+        return engine
+
+    def later(self, engine, delay, *lines):
+        def put():
+            time.sleep(delay)
+            for x in lines:
+                engine.lines.put(x)
+        threading.Thread(target=put, daemon=True).start()
+
+    def test_silence_mid_answer_ends_the_engine(self):
+        from serve.server import EngineSilent
+        engine = self.bare(0.3)
+        engine.lines.put("T 5")
+        gen = engine.generate([1], 10, {}, threading.Event())
+        self.assertEqual(next(gen), 5)
+        t0 = time.monotonic()
+        with self.assertRaises(EngineSilent) as cm:
+            next(gen)
+        self.assertIsInstance(cm.exception, EngineDied)          # every EngineDied path handles it
+        self.assertLess(time.monotonic() - t0, 5)
+        engine.proc.kill.assert_called_once()
+        self.assertFalse(engine.alive())                          # the next request restarts it
+        self.assertIn("#481", engine.death_note())
+        self.assertNotIn("STOP", engine.proc.stdin.getvalue())    # nothing is listening: no STOP, no drain
+
+    def test_prompt_chunks_set_the_wait(self):
+        # a PP line every second, at 100 tok/s: far over a 0.3 s silence, but each chunk is on time for its size
+        engine = self.bare(0.3)
+        engine.lines.put("RESUME 0")
+        engine.lines.put("PP 100 300 1000 100.0")
+        self.later(engine, 1.0, "PP 200 300 2000 100.0", "T 7", "DONE 1 300 2000 1 length")
+        self.assertEqual(list(engine.generate([1], 10, {}, threading.Event())), [None, None, 7])
+        engine.proc.kill.assert_not_called()
+
+    def test_a_long_first_chunk_is_allowed(self):
+        # 100 prompt tokens at the slowest prompt reading (50 tok/s): 2 s on top of the silence before the first PP
+        engine = self.bare(0.3)
+        self.later(engine, 1.0, "PP 100 100 1000 100.0", "DONE 0 100 1000 0 length")
+        self.assertEqual(list(engine.generate([1] * 100, 10, {}, threading.Event())), [None])
+        engine.proc.kill.assert_not_called()
+
+    def test_a_stop_never_acknowledged(self):
+        from serve.server import EngineSilent
+        engine = self.bare(0.3, can_stop=True)
+        engine.lines.put("T 5")
+        gen = engine.generate([1], 10, {}, threading.Event())
+        self.assertEqual(next(gen), 5)
+        with self.assertRaises(EngineSilent):
+            gen.close()                                           # the consumer stopped: STOP, then the drain
+        self.assertIn("STOP", engine.proc.stdin.getvalue())
+        engine.proc.kill.assert_called_once()
+        self.assertFalse(engine.alive())
+
+    def test_zero_waits_as_before(self):
+        engine = self.bare(0)
+        self.later(engine, 0.5, "T 5", "DONE 1 1 1 1 length")
+        self.assertEqual(list(engine.generate([1], 10, {}, threading.Event())), [5])
+
+    def test_config(self):
+        from serve.server import ENGINE_SILENCE_S, engine_silence_s
+        self.assertEqual(engine_silence_s({}), ENGINE_SILENCE_S)
+        self.assertEqual(engine_silence_s({"engine_silence_s": 0}), 0.0)
+        self.assertEqual(engine_silence_s({"engine_silence_s": 900}), 900.0)
+        for bad in (-1, "300", True):
+            with self.assertRaises(ValueError):
+                engine_silence_s({"engine_silence_s": bad})
+
+
+FAKE_LOST_STEP = '''import pathlib, sys, time
+mark = pathlib.Path(sys.argv[sys.argv.index("--mark") + 1])
+mode = sys.argv[sys.argv.index("--mode") + 1]
+print("READY 4096 stop", flush=True)
+for line in sys.stdin:
+    if line.startswith("QUIT"):
+        break
+    if line.startswith("GEN"):
+        if not mark.exists():                    # the first engine loses step (#481): it never says DONE
+            mark.touch()
+            print("T 104", flush=True)
+            if mode == "stop":
+                print("T 257", flush=True)       # <|im_end|>: the server STOPs and drains, and the engine is silent
+            time.sleep(3600)
+        print("T 111", flush=True)
+        print("T 107", flush=True)
+        print("DONE 2 5 1.0 1.0 length", flush=True)
+'''
+
+
+class LostStep(unittest.TestCase):
+    """#481 over HTTP, with a real process: the request with the silent engine ends with an error, and the next one
+    starts the engine again and is answered (the server used to wait forever, holding the request FIFO)."""
+
+    def run_mode(self, mode, stream):
+        import serve.server as server
+        with tempfile.TemporaryDirectory() as d:
+            script, mark = Path(d) / "fake_strata.py", Path(d) / "lost"
+            script.write_text(FAKE_LOST_STEP, encoding="utf-8")
+            real = server.subprocess.Popen
+            with mock.patch.object(server.subprocess, "Popen",
+                                   lambda cmd, **kw: real([sys.executable, str(script), *cmd[1:]], **kw)):
+                eng = StrataEngine("strata", ["--mark", str(mark), "--mode", mode])
+                eng.silence_s = 0.5
+                tok = ByteTokenizer()
+                svc = Service(eng, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+                httpd = serve(svc, port=0)
+                base = f"http://127.0.0.1:{httpd.server_address[1]}"
+                body = {"model": "m", "max_tokens": 2, "reasoning_effort": "none", "stream": stream,
+                        "messages": [{"role": "user", "content": "hi"}]}
+                try:
+                    first = eng.proc
+                    req = urllib.request.Request(base + "/v1/chat/completions", data=json.dumps(body).encode(),
+                                                 headers={"Content-Type": "application/json"})
+                    try:
+                        with urllib.request.urlopen(req, timeout=60) as r:
+                            text = r.read().decode()
+                    except urllib.error.HTTPError as e:
+                        self.assertEqual(e.code, 503)
+                        text = e.read().decode()
+                    self.assertIn("the next request restarts it", text)
+                    self.assertIsNotNone(first.poll(), "the silent engine still runs")
+                    self.assertEqual(svc.history[-1]["finish"], "error")
+                    with urllib.request.urlopen(req, timeout=60) as r:
+                        text = r.read().decode()
+                    self.assertIsNot(eng.proc, first)
+                    if stream:
+                        answer = "".join(json.loads(x[6:])["choices"][0]["delta"].get("content") or ""
+                                         for x in text.splitlines() if x.startswith("data: {") and "choices" in x)
+                    else:
+                        answer = json.loads(text)["choices"][0]["message"]["content"]
+                    self.assertEqual(answer, "ok")
+                finally:
+                    httpd.shutdown()
+                    httpd.server_close()
+                    eng.unload()
+
+    def test_silent_mid_answer(self):
+        self.run_mode("silent", stream=False)
+
+    def test_silent_mid_stream(self):
+        self.run_mode("silent", stream=True)
+
+    def test_stop_never_acknowledged(self):
+        self.run_mode("stop", stream=False)
+
+
 if __name__ == "__main__":
     unittest.main()

setup.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 # Strata for Linux: the first run installs everything and starts the model; later runs just start it.
-# Needs only an NVIDIA driver (or, experimental: an AMD RX 7900 XT/XTX with the amdgpu driver, see docs/AMD_HIP.md).
+# Needs only an NVIDIA driver (or, for an AMD Radeon card, the kernel's amdgpu driver: see docs/AMD_HIP.md).
 # Python (with venv) is installed through apt/dnf if it is missing (asks for sudo).
 cd "$(dirname "$0")" || exit 1
 # Python 3.10+ that can make a venv WITH pip: Debian/Ubuntu ship `venv` without `ensurepip` (that is the separate

src/core/device.cu

@@ -73,6 +73,125 @@ const char* compiled_gpu_archs() {
 #endif
 }
 
+int device_count() {
+    int count = 0;
+    if (cudaGetDeviceCount(&count) != cudaSuccess) {   // HIP without a usable device reports an error, not 0
+        cudaGetLastError();
+        return 0;
+    }
+    return count < 0 ? 0 : count;
+}
+
+bool device_summary(int ordinal, std::string& name, std::string& detail) {
+    cudaDeviceProp p{};
+    if (ordinal < 0 || ordinal >= device_count() || cudaGetDeviceProperties(&p, ordinal) != cudaSuccess) {
+        cudaGetLastError();
+        return false;
+    }
+    char buf[160];
+#if defined(STRATA_USE_HIP)
+    std::snprintf(buf, sizeof(buf), "arch %s, %.1f GiB, wave%d", base_arch(p.gcnArchName).c_str(),
+                  (double) p.totalGlobalMem / (1024.0 * 1024 * 1024), p.warpSize);
+#else
+    std::snprintf(buf, sizeof(buf), "compute capability %d.%d, %.1f GiB", p.major, p.minor,
+                  (double) p.totalGlobalMem / (1024.0 * 1024 * 1024));
+#endif
+    name = p.name;
+    detail = buf;
+    return true;
+}
+
+}  // namespace strata::core
+
+#if defined(STRATA_USE_HIP) && defined(_WIN32)
+// The free VRAM figure on Windows HIP (include/strata/hip_compat/cuda_runtime.h maps cudaMemGetInfo here).
+//
+// hipMemGetInfo (ROCclr's PAL backend, Device::globalFreeMemory) is the card's size minus this process's own
+// allocations: it asks Windows for this process's usage only, never for what the desktop and other programs hold.
+// So on a card that also drives the desktop the engine counted ~930 MiB that was not there, the expert cache filled
+// the card, and WDDM moved memory out to system RAM: decode at 30 tok/s instead of 41 (RX 6800, HIP SDK 7.2).
+// Windows itself gives each process a video memory budget (DXGI QueryVideoMemoryInfo) that does account for the
+// others.  Measured on that card: the budget sits 0.8 GiB below the card's size while this process is small and
+// 1.8 GiB below once it holds 11 GiB; allocations past it still succeed, and are what Windows moves out.  The
+// free figure here is hipMemGetInfo's lowered by what the budget withholds (the card's size minus the budget).
+// Subtracting from HIP's own figure, rather than taking the budget minus DXGI's usage, keeps memory the HIP runtime
+// has freed and holds in its cache counted as free, as hipMemGetInfo counts it.  dxgi.dll is loaded at run time, so
+// nothing new is linked; when it or the card's adapter cannot be found, hipMemGetInfo's figure stands.
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <dxgi1_4.h>
+#include <atomic>
+#include <cstdlib>
+#include <mutex>
+
+namespace strata::hip_compat {
+namespace {
+IDXGIAdapter3* budget_adapter(int device) {
+    constexpr int kMaxDevices = 16;
+    static std::mutex mu;
+    static IDXGIAdapter3* adapters[kMaxDevices] = {};
+    static bool tried[kMaxDevices] = {};
+    if (device < 0 || device >= kMaxDevices) return nullptr;
+    std::lock_guard<std::mutex> lock(mu);
+    if (tried[device]) return adapters[device];
+    tried[device] = true;
+    hipDeviceProp_t p{};
+    if (hipGetDeviceProperties(&p, device) != hipSuccess) {
+        (void) hipGetLastError();
+        return nullptr;
+    }
+    LUID luid{};
+    std::memcpy(&luid, p.luid, sizeof luid);
+    using CreateFactory = HRESULT(WINAPI*)(REFIID, void**);
+    static HMODULE dxgi = LoadLibraryW(L"dxgi.dll");   // kept for the process' lifetime, like the adapters
+    const auto create = dxgi ? (CreateFactory) (void*) GetProcAddress(dxgi, "CreateDXGIFactory1") : nullptr;
+    IDXGIFactory4* factory = nullptr;
+    if (create == nullptr || FAILED(create(__uuidof(IDXGIFactory4), (void**) &factory))) return nullptr;
+    IDXGIAdapter3* adapter = nullptr;
+    if (FAILED(factory->EnumAdapterByLuid(luid, __uuidof(IDXGIAdapter3), (void**) &adapter))) adapter = nullptr;
+    factory->Release();
+    adapters[device] = adapter;
+    return adapter;
+}
+}  // namespace
+
+hipError_t mem_get_info(size_t* free_bytes, size_t* total_bytes) {
+    const hipError_t e = hipMemGetInfo(free_bytes, total_bytes);
+    static const bool off = [] {
+        const char* v = std::getenv("STRATA_WDDM_BUDGET");
+        return v != nullptr && std::atoi(v) == 0;
+    }();
+    if (e != hipSuccess || off || free_bytes == nullptr || total_bytes == nullptr) return e;
+    int device = 0;
+    if (hipGetDevice(&device) != hipSuccess) {
+        (void) hipGetLastError();
+        return e;
+    }
+    IDXGIAdapter3* adapter = budget_adapter(device);
+    DXGI_QUERY_VIDEO_MEMORY_INFO local{};
+    if (adapter == nullptr || FAILED(adapter->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_LOCAL, &local)) ||
+        local.Budget == 0 || local.Budget >= *total_bytes)
+        return e;
+    const size_t withheld = *total_bytes - (size_t) local.Budget;   // the desktop's and other programs' share
+    static std::atomic<bool> said{false};
+    if (!said.exchange(true)) {
+        std::fprintf(stderr, "strata: Windows budgets %llu of this card's %llu MiB for this process; free VRAM is "
+                             "counted within that (STRATA_WDDM_BUDGET=0: off)\n",
+                     (unsigned long long) (local.Budget >> 20), (unsigned long long) (*total_bytes >> 20));
+    }
+    *free_bytes = *free_bytes > withheld ? *free_bytes - withheld : 0;
+    return e;
+}
+}  // namespace strata::hip_compat
+#endif
+
+namespace strata::core {
+
 std::string gpu_arch_problem(int ordinal) {
 #if defined(STRATA_USE_HIP)
     int count = 0;

src/core/device_main.cpp

@@ -9,6 +9,12 @@
 #include <cstdio>
 #include <cstring>
 #include <string>
+#if defined(STRATA_USE_HIP) && defined(_WIN32)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
 
 static std::string human(uint64_t b) {
     char buf[64];
@@ -19,10 +25,14 @@ static std::string human(uint64_t b) {
 
 int main(int argc, char** argv) {
     bool selftest = false;
+    bool list_devices = false;
     for (int i = 1; i < argc; ++i) {
         if (std::strcmp(argv[i], "--selftest") == 0) selftest = true;
+        else if (std::strcmp(argv[i], "--list-devices") == 0) list_devices = true;
         else if (std::strcmp(argv[i], "--help") == 0 || std::strcmp(argv[i], "-h") == 0) {
-            std::printf("usage: strata-device [--selftest]\n");
+            std::printf("usage: strata-device [--selftest] [--list-devices]\n"
+                        "  --list-devices  every GPU the runtime enumerates, numbered as HIP_VISIBLE_DEVICES /\n"
+                        "                  CUDA_VISIBLE_DEVICES number them, and whether this binary can run it\n");
             return 0;
         } else {
             std::fprintf(stderr, "unknown argument: %s\n", argv[i]);
@@ -30,6 +40,28 @@ int main(int argc, char** argv) {
         }
     }
 
+    // The runtime's numbering, which setup needs on Windows: there an integrated Radeon is HIP device 0 and pushes the
+    // discrete card to 1, while setup finds the cards in the display-adapter order (#325).  No arch check here - the
+    // cards this binary has no code for are part of the answer.  Format (setup.py's hip_devices parses it):
+    //   device N: <name>
+    //     arch gfx1201, 15.9 GiB, wave32          (CUDA: compute capability 12.0, 11.9 GiB)
+    //     cannot run: <why>                       (only for a card this binary cannot run)
+    if (list_devices) {
+        const int count = strata::core::device_count();
+        if (count == 0) std::printf("(no GPU device)\n");
+        for (int ordinal = 0; ordinal < count; ++ordinal) {
+            std::string name, detail;
+            if (!strata::core::device_summary(ordinal, name, detail)) {
+                std::printf("device %d: (the runtime cannot describe it)\n", ordinal);
+                continue;
+            }
+            std::printf("device %d: %s\n  %s\n", ordinal, name.c_str(), detail.c_str());
+            if (const std::string why = strata::core::gpu_arch_problem(ordinal); !why.empty())
+                std::printf("  cannot run: %s\n", why.c_str());
+        }
+        return 0;
+    }
+
     try {
         const strata::core::DeviceInfo d = strata::core::device_info(0);
         std::printf("device %d: %s\n", d.ordinal, d.name.c_str());
@@ -42,6 +74,13 @@ int main(int argc, char** argv) {
         std::printf("  multiprocessors     %d\n", d.multi_processor_count);
         std::printf("  VRAM total / free   %s / %s\n", human(d.total_bytes).c_str(), human(d.free_bytes).c_str());
         std::printf("  driver / runtime    %d / %d\n", d.driver_version, d.runtime_version);
+#if defined(STRATA_USE_HIP) && defined(_WIN32)
+        // #468 #461: which HIP runtime this process loaded - the one beside the exe, or an AMD driver's System32 copy
+        if (HMODULE h = GetModuleHandleA("amdhip64_7.dll")) {
+            char path[MAX_PATH] = {};
+            if (GetModuleFileNameA(h, path, MAX_PATH) > 0) std::printf("  HIP runtime         %s\n", path);
+        }
+#endif
 
         // The planner's view against the card's.  A plan that does not fit in what is actually FREE is the
         // failure this print exists to make visible at startup rather than at token 4000.

src/core/expert_cache.cpp

@@ -3,7 +3,9 @@
 
 #include <cuda_runtime.h>
 
+#include <algorithm>
 #include <cstdio>
+#include <filesystem>
 #include <utility>
 #include <cstring>
 
@@ -67,6 +69,77 @@ bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_ex
     return true;
 }
 
+std::vector<std::pair<int32_t, int32_t>> rank_learned_profile(int64_t n_layers, int64_t n_expert,
+                                                              const std::vector<uint8_t>& resident,
+                                                              const std::vector<double>& heat,
+                                                              const std::vector<std::pair<int32_t, int32_t>>& prior) {
+    const size_t n = (size_t) (n_layers * n_expert);
+    std::vector<int64_t> prior_rank(n, INT64_MAX);
+    for (size_t r = 0; r < prior.size(); ++r) {
+        const auto [l, e] = prior[r];
+        if (l >= 0 && l < n_layers && e >= 0 && e < n_expert) {
+            int64_t& pr = prior_rank[(size_t) (l * n_expert + e)];
+            if (pr == INT64_MAX) pr = (int64_t) r;
+        }
+    }
+    std::vector<int64_t> order(n);
+    for (size_t i = 0; i < n; ++i) order[i] = (int64_t) i;
+    auto res = [&](int64_t i) { return (size_t) i < resident.size() && resident[(size_t) i] != 0; };
+    auto ht = [&](int64_t i) { return (size_t) i < heat.size() ? heat[(size_t) i] : 0.0; };
+    std::stable_sort(order.begin(), order.end(), [&](int64_t a, int64_t b) {
+        if (res(a) != res(b)) return res(a);
+        if (ht(a) != ht(b)) return ht(a) > ht(b);
+        if (prior_rank[(size_t) a] != prior_rank[(size_t) b]) return prior_rank[(size_t) a] < prior_rank[(size_t) b];
+        return a < b;
+    });
+    std::vector<std::pair<int32_t, int32_t>> ranked(n);
+    for (size_t r = 0; r < n; ++r)
+        ranked[r] = {(int32_t) (order[r] / n_expert), (int32_t) (order[r] % n_expert)};
+    return ranked;
+}
+
+bool write_expert_profile(const std::string& path, int64_t n_layers, int64_t n_expert,
+                          const std::vector<std::pair<int32_t, int32_t>>& ranked, std::string& err) {
+    if (n_layers <= 0 || n_expert <= 0 || n_layers > 65535 || n_expert > 65535) {
+        err = "write_expert_profile: the model's layout does not fit the format";
+        return false;
+    }
+    std::vector<int32_t> table((size_t) (n_layers * n_expert), -1);
+    std::vector<uint16_t> pairs;
+    pairs.reserve(ranked.size() * 2);
+    for (size_t r = 0; r < ranked.size(); ++r) {
+        const auto [l, e] = ranked[r];
+        if (l < 0 || l >= n_layers || e < 0 || e >= n_expert) {
+            err = "write_expert_profile: a ranked pair is out of range";
+            return false;
+        }
+        table[(size_t) (l * n_expert + e)] = (int32_t) r;
+        pairs.push_back((uint16_t) l);
+        pairs.push_back((uint16_t) e);
+    }
+    const uint32_t hdr[5] = {1u, (uint32_t) n_layers, (uint32_t) n_expert, (uint32_t) ranked.size(),
+                             (uint32_t) ranked.size()};
+    const std::string tmp = path + ".tmp";
+    std::FILE* f = std::fopen(tmp.c_str(), "wb");
+    if (f == nullptr) {
+        err = "write_expert_profile: cannot create " + tmp;
+        return false;
+    }
+    // the format is little-endian (make_profile.py's "<"): so is every machine this engine runs on
+    bool ok = std::fwrite("STRP", 1, 4, f) == 4 && std::fwrite(hdr, 4, 5, f) == 5 &&
+              (pairs.empty() || std::fwrite(pairs.data(), 2, pairs.size(), f) == pairs.size()) &&
+              std::fwrite(table.data(), 4, table.size(), f) == table.size();
+    ok = (std::fclose(f) == 0) && ok;
+    std::error_code ec;
+    if (ok) std::filesystem::rename(tmp, path, ec);   // replaces an existing file (MoveFileEx / rename(2))
+    if (!ok || ec) {
+        std::filesystem::remove(tmp, ec);
+        err = "write_expert_profile: cannot write " + path;
+        return false;
+    }
+    return true;
+}
+
 ExpertCache::~ExpertCache() { close(); }
 
 #if defined(STRATA_USE_HIP)
@@ -182,7 +255,14 @@ bool ExpertCache::open_sized(const std::vector<int64_t>& slot_bytes, int64_t n_l
         return false;
     }
 #endif
+    // #369: each layer's cursor at the bottom of its own range, as open() seeds it - open() above ran on byte-sized
+    // "slots", so its seeds are not slot indices
     layer_next_.assign((size_t) (n_layers > 0 ? n_layers : 0), 0);
+    for (int64_t l = 0; l < n_layers; ++l) {
+        int64_t lo = 0, hi = 0;
+        layer_slot_range(l, lo, hi);
+        layer_next_[(size_t) l] = (int32_t) lo;
+    }
     return true;
 }
 

src/core/expert_source.cpp

@@ -1,6 +1,7 @@
 // src/core/expert_source.cpp - the adapter.  See the header for the three clauses of the contract.
 #include "strata/core/expert_source.hpp"
 #include "strata/core/remote_experts.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 #include "strata/artifact/gguf_reader.hpp"
 
@@ -426,6 +427,7 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     file_ = f;
     mapping_ = m;
     base_ = (const uint8_t*) view;
+    paths_.assign(1, path);
 #else
     const int fd = ::open(path.c_str(), O_RDONLY);
     if (fd < 0) { err = "FileExpertSource: cannot open " + path; return false; }
@@ -506,6 +508,12 @@ void FileExpertSource::close() {
     }
     role_ptr_.clear();
     role_bytes_.clear();
+    role_file_.clear();
+    paths_.clear();
+#if defined(_WIN32)
+    for (void* h : direct_) CloseHandle((HANDLE) h);
+#endif
+    direct_.clear();
     {
         std::lock_guard<std::mutex> lk(stage_mu_);
         stage_buf_.clear();
@@ -571,6 +579,7 @@ bool FileExpertSource::open_gguf(std::string& err) {
     std::map<std::string, size_t> index;
     role_ptr_.assign((size_t) (3 * n_layers_), nullptr);
     role_bytes_.assign((size_t) (3 * n_layers_), 0);
+    role_file_.assign((size_t) (3 * n_layers_), 0);
     for (int64_t l = 0; l < n_layers_; ++l) {
         const auto& fm = lay.fmt[(size_t) l];
         const uint64_t per[3] = {fm.up_off, fm.up_off, lay.bytes[(size_t) l] - fm.down_off};
@@ -620,9 +629,11 @@ bool FileExpertSource::open_gguf(std::string& err) {
                 m.base = (const uint8_t*) view;
 #endif
                 maps_.push_back(m);
+                paths_.push_back(path);
                 it = index.emplace(path, maps_.size() - 1).first;
             }
             const Map& m = maps_[it->second];
+            role_file_[i] = (int) it->second;
             const uint64_t at = lay.gguf_off[i], bytes = per[r] * (uint64_t) n_expert_;
             if (at > m.bytes || bytes > m.bytes - at) {   // check_experts_gguf proved it; the mapping must agree
                 err = "FileExpertSource: an expert span runs past the end of " + path;
@@ -640,6 +651,10 @@ bool FileExpertSource::open_gguf(std::string& err) {
 
 bool FileExpertSource::copy_from_files(int64_t layer, int64_t expert, uint8_t* dst) const {
     if (dst == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
+    if (!direct_.empty()) {   // #286: from the drive; a failed read falls back to the mapping below
+        const Fill f{0, layer, expert, dst};
+        if (read_direct(&f, 1)) return true;
+    }
     if (!role_ptr_.empty()) {
         uint64_t at = 0;
         for (int r = 0; r < 3; ++r) {
@@ -710,7 +725,11 @@ bool FileExpertSource::claim_stage(int64_t key, size_t& v, bool& fill) {
 bool FileExpertSource::fill_stage(size_t v, int64_t layer, int64_t expert, uint8_t* dst) {
     const auto t0 = std::chrono::steady_clock::now();
     const bool ok = copy_from_files(layer, expert, dst);
-    const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+    publish_stage(v, layer, ok, std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count());
+    return ok;
+}
+
+void FileExpertSource::publish_stage(size_t v, int64_t layer, bool ok, double us) {
     file_us_.fetch_add((uint64_t) us, std::memory_order_relaxed);
     if (ok) {
         file_read_bytes_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);
@@ -725,7 +744,6 @@ bool FileExpertSource::fill_stage(size_t v, int64_t layer, int64_t expert, uint8
         }
     }
     stage_cv_.notify_all();
-    return ok;
 }
 
 const uint8_t* FileExpertSource::staged_blob(int64_t layer, int64_t expert) {
@@ -749,8 +767,7 @@ const uint8_t* FileExpertSource::staged_blob(int64_t layer, int64_t expert) {
 }
 
 void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n) {
-    if (role_ptr_.empty() || n <= 0 || layer < 0 || layer >= n_layers_) return;
-    struct Fill { size_t v; int64_t e; uint8_t* dst; };
+    if (!staged() || n <= 0 || layer < 0 || layer >= n_layers_) return;
     std::vector<Fill> todo;
     {
         std::lock_guard<std::mutex> lk(stage_mu_);
@@ -764,7 +781,7 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
             size_t v = 0;
             bool fill = false;
             if (!claim_stage(layer * n_expert_ + e, v, fill) && fill) {
-                todo.push_back({v, e, stage_buf_[v].get()});
+                todo.push_back({v, layer, e, stage_buf_[v].get()});
                 if (warm_stamp_) {
                     const uint32_t s = warm_stamp_[index].load(std::memory_order_relaxed);
                     if (s != 0 && (uint64_t) s + 3 >= epoch_ + 1) warm_hits_.fetch_add(1, std::memory_order_relaxed);
@@ -772,7 +789,52 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
             }
         }
     }
+    fill_many(todo);
+}
+
+void FileExpertSource::prefetch_pairs(const std::pair<int32_t, int32_t>* pairs, int64_t n) {
+    if (direct_.empty() || pairs == nullptr || n <= 0) return;   // unbuffered only: the mapped fill stays as it was
+    std::vector<Fill> todo;
+    {
+        std::lock_guard<std::mutex> lk(stage_mu_);
+        for (int64_t i = 0; i < std::min<int64_t>(n, 64); ++i) {
+            const int64_t l = pairs[i].first, e = pairs[i].second;
+            if (l < 0 || e < 0 || l >= n_layers_ || e >= n_expert_) continue;
+            size_t v = 0;
+            bool fill = false;
+            if (!claim_stage(l * n_expert_ + e, v, fill) && fill) todo.push_back({v, l, e, stage_buf_[v].get()});
+        }
+    }
+    fill_many(todo);
+}
+
+void FileExpertSource::fill_many(const std::vector<Fill>& todo) {
     if (todo.empty()) return;
+    if (!direct_.empty()) {
+        // #286: overlapped batches - every role window of 16 blobs in flight at once; a big batch (the profile
+        // fill) on up to 4 threads, a decode layer's few misses on this one
+        std::atomic<size_t> next{0};
+        auto work = [&] {
+            for (size_t at; (at = next.fetch_add(16)) < todo.size();) {
+                const size_t k = std::min<size_t>(16, todo.size() - at);
+                const auto t0 = std::chrono::steady_clock::now();
+                const bool ok = read_direct(todo.data() + at, k);
+                const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+                for (size_t i = at; i < at + k; ++i) {
+                    const Fill& f = todo[i];
+                    // a failed batch: each blob again on its own (copy_from_files falls back to the mapping)
+                    if (ok) publish_stage(f.v, f.layer, true, us / (double) k);
+                    else (void) fill_stage(f.v, f.layer, f.e, f.dst);
+                }
+            }
+        };
+        const size_t nt = std::min<size_t>(4, (todo.size() + 15) / 16);
+        std::vector<std::thread> th;
+        for (size_t t = 1; t < nt; ++t) th.emplace_back(work);
+        work();
+        for (auto& t : th) t.join();
+        return;
+    }
 #if defined(_WIN32)
     // One PrefetchVirtualMemory call for every slice about to be copied: the memory manager reads them in large
     // requests, all queued at once, where the copies' page faults would read a few clusters each.  The copies below
@@ -789,7 +851,7 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
             ranges.reserve(todo.size() * 3);
             for (const Fill& f : todo)
                 for (int r = 0; r < 3; ++r) {
-                    const size_t i = (size_t) (3 * layer + r);
+                    const size_t i = (size_t) (3 * f.layer + r);
                     ranges.push_back({(PVOID) (role_ptr_[i] + (size_t) ((uint64_t) f.e * role_bytes_[i])),
                                       (SIZE_T) role_bytes_[i]});
                 }
@@ -801,7 +863,7 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
     std::atomic<size_t> next{0};
     auto work = [&] {
         for (size_t i; (i = next.fetch_add(1)) < todo.size();)
-            (void) fill_stage(todo[i].v, layer, todo[i].e, todo[i].dst);
+            (void) fill_stage(todo[i].v, todo[i].layer, todo[i].e, todo[i].dst);
     };
     const size_t nt = std::min<size_t>(todo.size(), (size_t) fetch_threads_);
     std::vector<std::thread> th;
@@ -810,6 +872,159 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
     for (auto& t : th) t.join();
 }
 
+bool FileExpertSource::set_unbuffered(uint64_t ram_bytes, std::string& why) {
+#if defined(_WIN32)
+    if (base_ == nullptr || paths_.empty() || !direct_.empty()) {
+        why = !direct_.empty() ? "already unbuffered" : "no expert files open";
+        return !direct_.empty();
+    }
+    if (!experts_unbuffered(paths_, ram_bytes, why, /*cache_counts=*/false)) return false;
+    for (const std::string& path : paths_) {
+        const int wide = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, nullptr, 0);
+        std::vector<wchar_t> w((size_t) (wide > 0 ? wide : 1), L'\0');
+        if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, w.data(), wide);
+        HANDLE h = CreateFileW(w.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                               FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);
+        if (h == INVALID_HANDLE_VALUE) {
+            why += "; cannot open " + path + " unbuffered (error " + std::to_string((unsigned long long) GetLastError()) +
+                   "), read through the file cache";
+            for (void* d : direct_) CloseHandle((HANDLE) d);
+            direct_.clear();
+            return false;
+        }
+        direct_.push_back(h);
+    }
+    if (role_ptr_.empty()) {   // experts.bin: blob() now assembles into the stage buffers, sized for the largest blob
+        std::lock_guard<std::mutex> lk(stage_mu_);
+        for (uint64_t b : layer_blob_bytes_) stage_blob_ = std::max(stage_blob_, b);
+    }
+    return true;
+#else
+    (void) ram_bytes;
+    why = "through the file cache (not Windows)";
+    return false;
+#endif
+}
+
+bool FileExpertSource::read_direct(const Fill* fills, size_t n) const {
+#if defined(_WIN32)
+    // NTFS runs the unbuffered reads of a file one at a time while the file is mapped or cached anywhere (the engine
+    // maps the GGUF shards for the token embedding and the PLE table): measured on a PCIe 5 drive, 512 KiB reads at
+    // queue depth 48 make 10.4 GB/s on a file nobody maps and 3.4 GB/s on a mapped one, 8 MiB reads 8.6 GB/s.  So
+    // windows less than kGap apart (the same role of nearby experts) are merged into requests of up to kMerge bytes.
+    constexpr uint64_t kSector = 4096, kGap = 1ull << 20, kMerge = 32ull << 20;
+    struct Window { int file; uint64_t a0, size, skip, n, at; uint8_t* dst; size_t req; uint64_t in_req; };
+    struct Req { HANDLE h; uint64_t a0, size, pos; };
+    // this thread's aligned buffer and events, kept for its next batch
+    struct Scratch {
+        uint8_t* buf = nullptr;
+        size_t cap = 0;
+        std::vector<HANDLE> ev;
+        std::vector<OVERLAPPED> ov;
+        std::vector<Window> win;
+        std::vector<size_t> order;
+        std::vector<Req> req;
+        std::vector<DWORD> got;
+        ~Scratch() {
+            if (buf != nullptr) VirtualFree(buf, 0, MEM_RELEASE);
+            for (HANDLE e : ev) CloseHandle(e);
+        }
+    };
+    thread_local Scratch sc;
+    sc.win.clear();
+    for (size_t k = 0; k < n; ++k) {
+        const Fill& f = fills[k];
+        if (f.dst == nullptr || f.layer < 0 || f.e < 0 || f.layer >= n_layers_ || f.e >= n_expert_) return false;
+        if (role_ptr_.empty()) {   // experts.bin: the blob is one contiguous range
+            const uint64_t per = layer_blob_bytes_[(size_t) f.layer];
+            const uint64_t off = layer_offsets_[(size_t) f.layer] + (uint64_t) f.e * per;
+            const uint64_t a0 = off / kSector * kSector, a1 = (off + per + kSector - 1) / kSector * kSector;
+            sc.win.push_back({0, a0, a1 - a0, off - a0, per, 0, f.dst, 0, 0});
+            continue;
+        }
+        uint64_t at = 0;
+        for (int r = 0; r < 3; ++r) {
+            const size_t i = (size_t) (3 * f.layer + r);
+            const uint64_t per = role_bytes_[i];
+            const Map& m = maps_[(size_t) role_file_[i]];
+            const uint64_t off = (uint64_t) (role_ptr_[i] - m.base) + (uint64_t) f.e * per;
+            const uint64_t a0 = off / kSector * kSector, a1 = (off + per + kSector - 1) / kSector * kSector;
+            sc.win.push_back({role_file_[i], a0, a1 - a0, off - a0, per, at, f.dst, 0, 0});
+            at += per;
+        }
+    }
+    sc.order.resize(sc.win.size());
+    for (size_t w = 0; w < sc.win.size(); ++w) sc.order[w] = w;
+    std::sort(sc.order.begin(), sc.order.end(), [&](size_t a, size_t b) {
+        return sc.win[a].file != sc.win[b].file ? sc.win[a].file < sc.win[b].file : sc.win[a].a0 < sc.win[b].a0;
+    });
+    sc.req.clear();
+    uint64_t total = 0;
+    int last_file = -1;
+    for (size_t w : sc.order) {
+        Window& x = sc.win[w];
+        if (!sc.req.empty() && x.file == last_file) {
+            Req& q = sc.req.back();
+            const uint64_t end = q.a0 + q.size, xend = x.a0 + x.size;
+            if (x.a0 <= end + kGap && std::max(end, xend) - q.a0 <= kMerge) {
+                if (xend > end) {
+                    total += xend - end;
+                    q.size = xend - q.a0;
+                }
+                x.req = sc.req.size() - 1;
+                x.in_req = x.a0 - q.a0;
+                continue;
+            }
+        }
+        sc.req.push_back({(HANDLE) direct_[(size_t) x.file], x.a0, x.size, total});
+        total += x.size;
+        last_file = x.file;
+        x.req = sc.req.size() - 1;
+        x.in_req = 0;
+    }
+    if (total > sc.cap) {
+        if (sc.buf != nullptr) VirtualFree(sc.buf, 0, MEM_RELEASE);
+        sc.cap = (size_t) ((total + (1u << 20) - 1) >> 20 << 20);
+        sc.buf = (uint8_t*) VirtualAlloc(nullptr, sc.cap, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+        if (sc.buf == nullptr) { sc.cap = 0; return false; }
+    }
+    while (sc.ev.size() < sc.req.size()) {
+        HANDLE e = CreateEventW(nullptr, TRUE, FALSE, nullptr);
+        if (e == nullptr) return false;
+        sc.ev.push_back(e);
+    }
+    sc.ov.assign(sc.req.size(), OVERLAPPED{});
+    // every request in flight before the first wait: the drive sees the whole batch as one queue
+    size_t issued = 0;
+    bool ok = true;
+    for (size_t q = 0; q < sc.req.size(); ++q) {
+        const Req& r = sc.req[q];
+        OVERLAPPED& o = sc.ov[q];
+        o.Offset = (DWORD) r.a0;
+        o.OffsetHigh = (DWORD) (r.a0 >> 32);
+        o.hEvent = sc.ev[q];
+        if (!ReadFile(r.h, sc.buf + r.pos, (DWORD) r.size, nullptr, &o) && GetLastError() != ERROR_IO_PENDING) {
+            ok = false;
+            break;
+        }
+        ++issued;
+    }
+    std::vector<DWORD>& got = sc.got;
+    got.assign(sc.req.size(), 0);
+    for (size_t q = 0; q < issued; ++q)
+        if (!GetOverlappedResult(sc.req[q].h, &sc.ov[q], &got[q], TRUE)) ok = false;
+    if (!ok || issued < sc.req.size()) return false;
+    for (const Window& x : sc.win) {
+        // a request may run past the end of the file: only the role's own bytes have to arrive
+        if ((uint64_t) got[x.req] < x.in_req + x.skip + x.n) return false;
+        std::memcpy(x.dst + x.at, sc.buf + sc.req[x.req].pos + x.in_req + x.skip, (size_t) x.n);
+    }
+    return true;
+#else
+    (void) fills; (void) n;
+    return false;
+#endif
+}
 
 void FileExpertSource::warm(int64_t layer, const int64_t* experts, int64_t n) {
     if (role_ptr_.empty() || n <= 0 || layer < 0 || layer >= n_layers_) return;
@@ -932,7 +1147,7 @@ void RouterLookahead::run() {
 void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k) {
     (void) ids;
     (void) k;
-    if (role_ptr_.empty()) return;
+    if (!staged()) return;
     std::lock_guard<std::mutex> lk(stage_mu_);
     if (layer != last_layer_) {
         ++epoch_;
@@ -941,7 +1156,7 @@ void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k)
 }
 
 bool FileExpertSource::transient(int64_t layer, int64_t expert) const {
-    if (role_ptr_.empty() || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
+    if (!staged() || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
     const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
     if (complement_ready_ && index < complement_offsets_.size() && complement_offsets_[index] != kNoComplement)
         return false;
@@ -1024,6 +1239,23 @@ bool FileExpertSource::pin_cache_complement(
     uint64_t bytes = 0;
     if (!detail::make_cache_complement_plan(n_layers_, n_expert_, layer_blob_bytes_, primary_gpu_pairs,
                                             additional_gpu_pairs, offsets, bytes, err)) return false;
+#if defined(_WIN32)
+    // #467: the GPU cache's pre-fill touched its experts through the mapping (~19 GiB on a 24 GB card), and Windows
+    // counts those file pages in this process's working set, not as available: a 32 GB PC read 0.44 GiB here
+    // (20.7 GiB before the start).  Trimmed, they move to the standby list (still cached, counted as available).
+    // Resident mode only: nothing else calls this function.  Locked/pinned pages stay; the rest fault back softly.
+    {
+        uint64_t before = 0, after = 0;
+        const bool read_before = available_memory_bytes(before);
+        (void) SetProcessWorkingSetSize(GetCurrentProcess(), (SIZE_T) -1, (SIZE_T) -1);
+        if (read_before && available_memory_bytes(after))
+            std::fprintf(stderr, "FileExpertSource: available RAM %.2f GiB, %.2f GiB after the mapped experts left the "
+                                 "process working set (#467)\n",
+                         (double) before / 1073741824.0, (double) after / 1073741824.0);
+    }
+#endif
+    const bool what_fits = budget_bytes == kResidentWhatFits;   // #467: the soft mode's second try
+    uint64_t budget_physical = 0;   // #403: the RAM reading a budget was sized from (0: no budget)
     if (budget_bytes > 0) {
         // CS-T: a RAM budget.  The complement's experts in `rank` order (the expert profile, hottest first) while
         // they fit, the rest left on the mapped files; clamped to what the RAM has room for.
@@ -1032,13 +1264,25 @@ bool FileExpertSource::pin_cache_complement(
             err = "FileExpertSource: cannot determine available RAM for --resident-budget-gib";
             return false;
         }
+        budget_physical = physical;
         const uint64_t room = physical > headroom_bytes ? physical - headroom_bytes : 0;
         if (budget_bytes > room) {
-            std::fprintf(stderr, "FileExpertSource: --resident-budget-gib %.2f is more than the RAM has room for "
-                                 "(%.2f GiB available minus %.0f GiB headroom): %.2f GiB\n",
-                         (double) budget_bytes / 1073741824.0, (double) physical / 1073741824.0,
-                         (double) headroom_bytes / 1073741824.0, (double) room / 1073741824.0);
-            budget_bytes = room;
+            // #403: 256 MiB under the room, so the engine's own allocations after this reading still leave the
+            // headroom (a budget clamped to exactly the room failed the safety check below on a reading a few MB
+            // lower).  An unclamped budget is unchanged.
+            const uint64_t margin = 256ull << 20;
+            const uint64_t clamped = room > margin ? room - margin : 0;
+            if (what_fits)
+                std::fprintf(stderr, "FileExpertSource: RAM room for the complement: %.2f GiB (%.2f GiB available "
+                                     "minus %.0f GiB headroom and a 0.25 GiB margin)\n",
+                             (double) clamped / 1073741824.0, (double) physical / 1073741824.0,
+                             (double) headroom_bytes / 1073741824.0);
+            else
+                std::fprintf(stderr, "FileExpertSource: --resident-budget-gib %.2f is more than the RAM has room for "
+                                     "(%.2f GiB available minus %.0f GiB headroom and a 0.25 GiB margin): %.2f GiB\n",
+                             (double) budget_bytes / 1073741824.0, (double) physical / 1073741824.0,
+                             (double) headroom_bytes / 1073741824.0, (double) clamped / 1073741824.0);
+            budget_bytes = clamped;
         }
         std::vector<uint64_t> ranked(offsets.size(), kNoComplement);
         uint64_t at = 0;
@@ -1054,6 +1298,10 @@ bool FileExpertSource::pin_cache_complement(
                 at += b;
                 ++held;
             }
+        if (what_fits && held == 0) {   // #467: nothing to keep - the caller's plain mmap fallback, not an empty copy
+            err = "FileExpertSource: the RAM has no room for any expert of the complement";
+            return false;
+        }
         std::fprintf(stderr, "FileExpertSource: RAM budget %.2f GiB: %lld of the %.2f GiB of experts the GPU cache does "
                              "not hold, by profile rank; the rest are read from the files\n",
                      (double) budget_bytes / 1073741824.0, (long long) held, (double) bytes / 1073741824.0);
@@ -1065,8 +1313,10 @@ bool FileExpertSource::pin_cache_complement(
     const bool lend = lend_from_slot >= 0 && lend_from_slot < n_slots && additional_gpu_pairs.empty();
     uint64_t budget = std::numeric_limits<uint64_t>::max();
     if (bytes > 0 || lend) {
-        uint64_t physical = 0;
-        if (!available_memory_bytes(physical)) {
+        // #403: with a budget, the reading it was sized from - a second reading a few MB lower (the engine's own
+        // allocations, the file cache) failed a budget the first one had clamped.  (A budget turns `lend` off.)
+        uint64_t physical = budget_physical;
+        if (physical == 0 && !available_memory_bytes(physical)) {
             err = "FileExpertSource: cannot determine available RAM for the resident-memory safety check";
             return false;
         }
@@ -1236,17 +1486,43 @@ bool FileExpertSource::pin_cache_complement(
             const int64_t layer = next_layer.fetch_add(1);
             if (layer >= n_layers_ || failed.load()) return;
             const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer];
+            std::vector<Fill> batch;   // #286, unbuffered: 32 blobs' reads in flight at once, merged where near
+            auto flush = [&]() {
+                if (batch.empty()) return true;
+                bool ok = read_direct(batch.data(), batch.size());
+                for (size_t i = 0; !ok && i < batch.size(); ++i)
+                    if (!copy_from_files(batch[i].layer, batch[i].e, batch[i].dst)) return false;
+                copied.fetch_add(blob_bytes * (uint64_t) batch.size());
+                batch.clear();
+                return true;
+            };
             for (int64_t expert = 0; expert < n_expert_; ++expert) {
                 const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
                 const uint64_t offset = offsets[index];
                 if (offset == kNoComplement) continue;
-                if (offset > bytes || blob_bytes > bytes - offset ||
-                    !copy_from_files(layer, expert, (uint8_t*) host + (size_t) offset)) {
+                if (offset > bytes || blob_bytes > bytes - offset) {
+                    fail("FileExpertSource: invalid blob bounds while building the cache complement");
+                    return;
+                }
+                uint8_t* dst = (uint8_t*) host + (size_t) offset;
+                if (!direct_.empty()) {
+                    batch.push_back({0, layer, expert, dst});
+                    if (batch.size() == 32 && !flush()) {
+                        fail("FileExpertSource: an expert could not be read while building the cache complement");
+                        return;
+                    }
+                    continue;
+                }
+                if (!copy_from_files(layer, expert, dst)) {
                     fail("FileExpertSource: invalid blob bounds while building the cache complement");
                     return;
                 }
                 copied.fetch_add(blob_bytes);
             }
+            if (!flush()) {
+                fail("FileExpertSource: an expert could not be read while building the cache complement");
+                return;
+            }
 #if !defined(_WIN32)
             if (role_ptr_.empty()) {   // experts.bin; the GGUF in place leaves its pages to the OS
             const uint64_t layer_offset = layer_offsets_[(size_t) layer];
@@ -1409,7 +1685,7 @@ const uint8_t* FileExpertSource::blob(int64_t layer, int64_t expert) {
         }
     }
     if (from_files) {
-        if (!role_ptr_.empty()) {
+        if (staged()) {
             result = staged_blob(layer, expert);          // counts its bytes
         } else {
             result = mapped_blob(layer, expert);
@@ -1624,7 +1900,8 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
             if (first_of[i] == i) {
                 distinct[nd++] = i;
                 const int32_t e = ids[i];
-                if (e >= 0 && e < d.n_expert && d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] < 0) ++nmiss;
+                if (e >= 0 && e < d.n_expert && d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] < 0 &&
+                    !(d.peer != nullptr && d.peer->has(d.layers, e))) ++nmiss;
             }
         }
         const bool pcie_ok = d.pcie_num > 0 && d.src->pcie_layer(d.layers);
@@ -1644,6 +1921,8 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                     kd = 0;
                     ptr = (unsigned long long) (d.cache_base + (d.cache_slot_off ? (size_t) d.cache_slot_off[slot]
                                                                                  : (size_t) slot * (size_t) d.cache_blob));
+                } else if (d.peer != nullptr && d.peer->has(d.layers, e)) {
+                    kd = 2;                        // multi-GPU: the second GPU computes it
                 } else {
                     if (miss_rank >= nmiss - m && fetches < P.staging_cap && fetches < 64) {
                         const uint8_t* src = d.src->pinned(d.layers, e) ? d.src->blob(d.layers, e) : nullptr;
@@ -1698,7 +1977,18 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
         for (int64_t i = 0; i < n; ++i) {
             const int32_t e = ids[i];
             kind[i] = (e >= 0 && e < d.n_expert && d.host_res != nullptr &&
-                       d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] >= 0) ? 0 : -1;
+                       d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] >= 0) ? 0
+                    : (e >= 0 && e < d.n_expert && d.peer != nullptr && d.peer->has(d.layers, e)) ? 2 : -1;
+        }
+    }
+    if (d.peer != nullptr) {                 // multi-GPU: start the second GPU's share before the CPU's own work
+        std::string perr;
+        if (!d.peer->launch(d.layers, x_f, ids, n_tok, k, kind, perr, out)) {
+            std::fprintf(stderr, "strata: %s (layer %lld)\n", perr.c_str(), (long long) d.layers);
+            d.failed = true;
+            d.fail = "the peer GPU's experts could not be launched";
+            d.fail_layer = d.layers;
+            return;
         }
     }
     if (d.remote_count > 0) {
@@ -1741,9 +2031,12 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                 d.fail_expert = e;
                 return;
             }
-            if (kind[i] >= 0) {             // CUDA0, PCIe, or a remote result staged into this row below
+            if (kind[i] >= 0) {             // CUDA0, PCIe, or a remote/peer result staged into this row below
                 if (kind[i] == 0) ++d.cache_hits;
-                std::memset(row, 0, (size_t) H * sizeof(float));
+                else if (kind[i] == 2 && d.peer != nullptr) ++d.peer_entries;
+                // multi-GPU: a direct peer launch is writing this row right now - zeroing it would race it
+                if (!(kind[i] == 2 && d.peer != nullptr && d.peer->launched_direct()))
+                    std::memset(row, 0, (size_t) H * sizeof(float));
                 continue;
             }
             ++d.cache_refused;
@@ -1781,6 +2074,16 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                 d.failed = true; d.fail = remote_error.c_str(); d.fail_layer = d.layers; return;
             }
     }
+    if (d.peer != nullptr) {                 // multi-GPU: the second GPU's rows, into the same mapped rows
+        std::string perr;
+        if (!d.peer->finish(out, perr)) {
+            std::fprintf(stderr, "strata: %s (layer %lld)\n", perr.c_str(), (long long) d.layers);
+            d.failed = true;
+            d.fail = "the peer GPU's experts failed";
+            d.fail_layer = d.layers;
+            return;
+        }
+    }
     const auto c4 = std::chrono::steady_clock::now();
     pt("ran");
     auto ms = [](auto a, auto b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
@@ -2080,13 +2383,104 @@ bool check_experts_gguf(const std::string& gguf, const strata::kernels::cpu::Exp
 // [gate rows | up rows | down rows] - the layout tools/iq_pack.py would have written to experts.bin.  Each role
 // is read from its own file (native_experts.txt v4: a shard boundary can fall inside a layer; per role as in
 // #255, gopinath87607).  The caller checks the spans first (check_experts_gguf).
+// `unbuffered` (Windows, experts_unbuffered): each chunk's 4 KiB-aligned window is read with FILE_FLAG_NO_BUFFERING into
+// an aligned buffer and scattered into the blobs - no copy through the file cache when the drive is read anyway.
 LoadStats load_experts_gguf(const std::string& gguf, uint8_t* dst, const strata::kernels::cpu::ExpertLayout& lay,
-                            int threads) {
+                            int threads, bool unbuffered) {
     LoadStats st;
     st.layers = (uint64_t) lay.n_layers;
     const auto t0 = std::chrono::steady_clock::now();
     std::atomic<int64_t> next{0};
     std::atomic<bool> bad{false};
+#if defined(_WIN32)
+    if (unbuffered) {
+        uint64_t max_chunk = 0;
+        for (int64_t l = 0; l < lay.n_layers; ++l) {
+            const auto& fm = lay.fmt[(size_t) l];
+            max_chunk = std::max<uint64_t>(max_chunk, std::max<uint64_t>(fm.up_off, lay.bytes[(size_t) l] - fm.down_off) * 16);
+        }
+        std::mutex err_mu;
+        std::string err;
+        auto worker = [&]() {
+            constexpr uint64_t kSector = 4096;
+            const uint64_t cap = (max_chunk + 2 * kSector + kSector - 1) / kSector * kSector;
+            uint8_t* buf = (uint8_t*) VirtualAlloc(nullptr, (size_t) cap, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+            HANDLE h = INVALID_HANDLE_VALUE;
+            std::string open_name;
+            auto fail = [&](const std::string& what) {
+                std::lock_guard<std::mutex> g(err_mu);
+                if (err.empty()) err = what;
+                bad = true;
+            };
+            if (buf == nullptr) fail("cannot allocate a read buffer");
+            for (;;) {
+                const int64_t l = next.fetch_add(1);
+                if (l >= lay.n_layers || bad) break;
+                const auto& fm = lay.fmt[(size_t) l];
+                const uint64_t blob = lay.bytes[(size_t) l];
+                const uint64_t per[3] = {fm.up_off, fm.up_off, blob - fm.down_off};
+                const uint64_t at[3] = {0, fm.up_off, fm.down_off};
+                for (int r = 0; r < 3 && !bad; ++r) {
+                    // the handle is kept while consecutive roles share a file (every layer of a v3 pack)
+                    const std::string name = expert_gguf_file(gguf, lay, l, r);
+                    if (name != open_name) {
+                        if (h != INVALID_HANDLE_VALUE) CloseHandle(h);
+                        const int wide = MultiByteToWideChar(CP_UTF8, 0, name.c_str(), -1, nullptr, 0);
+                        std::vector<wchar_t> w((size_t) std::max(wide, 1), L'\0');
+                        if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, name.c_str(), -1, w.data(), wide);
+                        h = CreateFileW(w.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                                        FILE_FLAG_NO_BUFFERING | FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
+                        if (h == INVALID_HANDLE_VALUE) {
+                            fail("cannot open " + name + " (error " + std::to_string((unsigned long long) GetLastError()) + ")");
+                            break;
+                        }
+                        open_name = name;
+                    }
+                    const uint64_t src = lay.gguf_off[(size_t) (3 * l + r)];
+                    const uint64_t total = per[r] * (uint64_t) lay.n_expert;
+                    const uint64_t chunk = per[r] * 16;           // 16 experts per read
+                    for (uint64_t done = 0; done < total; done += chunk) {
+                        const uint64_t n = std::min<uint64_t>(chunk, total - done);
+                        const uint64_t a0 = (src + done) / kSector * kSector;
+                        const uint64_t a1 = (src + done + n + kSector - 1) / kSector * kSector;
+                        OVERLAPPED ov{};
+                        ov.Offset = (DWORD) a0;
+                        ov.OffsetHigh = (DWORD) (a0 >> 32);
+                        DWORD got = 0;
+                        // the window may run past the end of the file: only the tensor's own bytes have to arrive
+                        if (!ReadFile(h, buf, (DWORD) (a1 - a0), &got, &ov) || (uint64_t) got < src + done - a0 + n) {
+                            fail("short unbuffered read of layer " + std::to_string(l) + " in " + name + " (error " +
+                                 std::to_string((unsigned long long) GetLastError()) + ")");
+                            break;
+                        }
+                        const uint8_t* q = buf + (src + done - a0);
+                        for (uint64_t k = 0; k < n / per[r]; ++k) {
+                            const uint64_t e = done / per[r] + k;
+                            std::memcpy(dst + lay.blob_offset(l, (int64_t) e) + at[r], q + k * per[r], (size_t) per[r]);
+                        }
+                    }
+                }
+            }
+            if (h != INVALID_HANDLE_VALUE) CloseHandle(h);
+            if (buf != nullptr) VirtualFree(buf, 0, MEM_RELEASE);
+        };
+        std::vector<std::thread> pool;
+        for (int i = 1; i < threads; ++i) pool.emplace_back(worker);
+        worker();
+        for (auto& t : pool) t.join();
+        if (bad) {
+            st.seconds = -1.0;
+            st.ok = false;
+            st.error = err.empty() ? "unreadable shard while reading the experts from the GGUF" : err;
+            return st;
+        }
+        st.bytes = lay.total;
+        st.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+        return st;
+    }
+#else
+    (void) unbuffered;
+#endif
     auto worker = [&]() {
         // #230: `fread` on a `FILE*`, as load_experts_ranges (#89): MSVC's `std::ifstream::read` splits a request
         // into 4095-byte freads, which took this path to 0.02 GiB/s on a Windows install without experts.bin.
@@ -2210,8 +2604,31 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
               (why.empty() ? std::string{} : ": " + why);
         return false;
     }
-    const LoadStats st = from_gguf ? load_experts_gguf(gguf_, a->data(), lay, threads)
-                                   : load_experts_ranges(path, a->data(), loff, lbytes, threads, /*chunk=*/8u << 20);
+    // #285: unbuffered when the drive is read anyway and the file cache could not keep the experts for the next
+    // start either (a 64 GB PC); otherwise the buffered readers, which a warm restart serves from the cache
+    std::vector<std::string> files;
+    if (from_gguf) {
+        for (int64_t l = 0; l < n_layers; ++l)
+            for (int r = 0; r < 3; ++r) {
+                const std::string f = expert_gguf_file(gguf_, lay, l, r);
+                if (std::find(files.begin(), files.end(), f) == files.end()) files.push_back(f);
+            }
+    } else {
+        files.push_back(path);
+    }
+    std::string why;
+    const bool unbuffered = experts_unbuffered(files, want + (uint64_t) blob, why);
+    const int readers = unbuffered ? std::max(threads, 16) : threads;   // 16 keep a PCIe 5 drive's queue full
+    LoadStats st;
+    if (from_gguf) {
+        st = load_experts_gguf(gguf_, a->data(), lay, readers, unbuffered);
+    } else {
+        if (unbuffered) st = load_experts_direct(path, a->data(), loff, lbytes, readers, /*chunk=*/8u << 20);
+        if (!unbuffered || (!st.ok && st.error.empty()))   // unaligned ranges: the buffered reader
+            st = load_experts_ranges(path, a->data(), loff, lbytes, threads, /*chunk=*/8u << 20);
+    }
+    std::fprintf(stderr, "strata generate: expert arena read %s (%s)\n", unbuffered ? "unbuffered" : "through the file cache",
+                 why.c_str());
     if (!st.ok) {
         delete a;
         err = "ArenaExpertSource: the expert load was refused: " + (st.error.empty() ? std::string("unknown") : st.error);

src/core/mtp.cpp

@@ -4,6 +4,7 @@
 #include "strata/core/on_device.hpp"
 
 #include "strata/core/native_head.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
 #include "strata/kernels/cpu/expert.hpp"
 #include "strata/kernels/elementwise.hpp"
@@ -30,6 +31,7 @@
 #include <atomic>
 #include <chrono>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <exception>
 #include <fstream>
@@ -55,7 +57,7 @@ struct Bump {
 };
 
 bool mapped(size_t bytes, void** h, void** d) {
-    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped) != cudaSuccess) return false;
+    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false;
     std::memset(*h, 0, bytes);
     return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;
 }
@@ -372,6 +374,45 @@ void MtpDrafter::set_draft_history(const int32_t* tail, int64_t n_tail, int32_t
     coupled_hist_base(tail, n_tail, next, h, h_chist_ + (kCoupledHistCap - h));
 }
 
+namespace {
+// #474: what to change when the draft head's token subset does not fit the VRAM left - a smaller subset that setup
+// ships (data/draft_vocab_*.bin; their token counts below), and how to pick it.  The start used to stop at "the draft
+// head does not fit" with no hint.  Text only, after the failure: nothing changes for a start that fits.
+void draft_head_hint(int64_t n_tokens, int64_t row_bytes) {
+    size_t free_b = 0, total_b = 0;
+    const bool have_free = cudaMemGetInfo(&free_b, &total_b) == cudaSuccess;
+    auto mib = [&](int64_t n) { return (double) (n * row_bytes) / 1048576.0; };
+    std::string free_s;
+    if (have_free) {
+        char b[64];
+        std::snprintf(b, sizeof(b), " and %.0f MiB is free", (double) free_b / 1048576.0);
+        free_s = b;
+    }
+    std::fprintf(stderr, "strata mtp: the draft head over %lld tokens needs %.0f MiB of VRAM%s.\n",
+                 (long long) n_tokens, mib(n_tokens), free_s.c_str());
+    struct Subset { const char* name; int64_t tokens; const char* what; };
+    static const Subset smaller[] = {{"cyrillic", 58963, "English, code and the Cyrillic script"},
+                                     {"en", 40525, "English and code"}};
+    std::string opts;
+    for (const Subset& s : smaller)
+        if (s.tokens < n_tokens) {
+            char b[160];
+            std::snprintf(b, sizeof(b), "%s--draft-vocab %s (%s, ~%.0f MiB)", opts.empty() ? "" : " or ", s.name,
+                          s.what, mib(s.tokens));
+            opts += b;
+        }
+    if (!opts.empty())
+        std::fprintf(stderr, "strata mtp: hint: a smaller draft vocabulary needs less VRAM: %s. Start once with it - "
+                             "START-HERE.bat --draft-vocab en (Windows) or ./setup.sh --draft-vocab en - and the model "
+                             "keeps it (\"draft_vocab\" in its strata-*.json config); or a smaller --context in "
+                             "setup.\n",
+                     opts.c_str());
+    else
+        std::fprintf(stderr, "strata mtp: hint: this is already the smallest shipped draft vocabulary: a smaller "
+                             "--context (or closing what else uses the GPU) leaves it room.\n");
+}
+}  // namespace
+
 bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err) {
     const OnDevice on_device(device_);
     wt_ = &wt;
@@ -395,6 +436,7 @@ bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float
             if (cudaMalloc((void**) &dvocab_, raw.size()) != cudaSuccess ||
                 cudaMalloc((void**) &dhead_, (size_t) (n_dvocab_ * row_bytes)) != cudaSuccess) {
                 err = "mtp: the draft head does not fit";
+                draft_head_hint(n_dvocab_, row_bytes);
                 return false;
             }
             cudaMemcpy(dvocab_, raw.data(), raw.size(), cudaMemcpyHostToDevice);
@@ -688,8 +730,21 @@ bool MtpDrafter::prefill(const float* R_rows, const int32_t* next_tokens, int64_
     // E-4: every group's token / step / position records uploaded at once; each group is then device copies and a
     // graph on the one stream, with a single sync at the end (a group of <= max_t rows used to be staged in mapped
     // memory and synced before the next: ~5,500 host round trips on a 32K prompt).  The same work in the same
-    // order.  STRATA_MTP_PREFILL_SYNC=1 keeps the old loop.
-    static const bool per_group_sync = std::getenv("STRATA_MTP_PREFILL_SYNC") != nullptr;
+    // order.  STRATA_MTP_PREFILL_SYNC=1 keeps the old loop, =0 forces E-4.
+    // HIP defaults to the old loop.  This pass runs whenever E-9 (Prefill::draft_kv) declines, which it does for the
+    // drafter's ring (KV streaming, --kv-resident), and on gfx1201 / ROCm 7.2.4 the E-4 queue (a graph launch and four
+    // device copies per group, ~2,000 groups per 8192-token chunk, no sync) sometimes never completes: the prompt hangs
+    // in hipGraphLaunch or the final sync until the watchdog ends the engine.  R9700, full IQ3_XXS, --kv-resident
+    // 32768, mixed load (chats, 32K and 7K prompts, deep follow-ups): E-4 hung in the first round on 2 of 2 tries,
+    // the old loop ran 30 of 30 rounds clean, prompt speed unchanged.
+    static const bool per_group_sync = [] {
+        if (const char* v = std::getenv("STRATA_MTP_PREFILL_SYNC")) return std::atoi(v) != 0;
+#if defined(STRATA_USE_HIP)
+        return true;
+#else
+        return false;
+#endif
+    }();
     const int64_t NHp = g_->n_head, per_row = 1 + 4 + NHp;
     if (!per_group_sync && n > 0) {
         if (pf_cap_ < n * per_row) {

src/core/native_head.cpp

@@ -150,10 +150,31 @@ bool NativeEmbed::load(const std::vector<std::string>& shards, int64_t n_embd, i
         } else {
             std::memcpy(host_, gguf.tensor_data(*t), bytes_);
             void* d = nullptr;
+#if defined(STRATA_USE_HIP)
+            // #325: the Windows HIP stack can refuse the device alias of a mapped allocation (and, when it gives
+            // one, it is the host address itself - unified addressing; kernels read it correctly there, a
+            // device-to-device copy into it does not land: tests/hip/mapped_alias). The table is only gathered from,
+            // so without an alias it goes into VRAM like the unpinnable case above, instead of failing the start.
+            if (cudaHostGetDevicePointer(&d, host_, 0) != cudaSuccess || d == nullptr) {
+                cudaGetLastError();
+                d = nullptr;
+                if (cudaMalloc(&d, bytes_) != cudaSuccess ||
+                    cudaMemcpy(d, gguf.tensor_data(*t), bytes_, cudaMemcpyHostToDevice) != cudaSuccess) {
+                    if (d) cudaFree(d);
+                    cudaGetLastError();
+                    err = "native embedding: no device alias for the mapped table, and no VRAM to copy it into";
+                    return false;
+                }
+                cudaFreeHost(host_);   // the destructor frees dev_ when host_ is null
+                host_ = nullptr;
+                std::fprintf(stderr, "strata: native embedding: no device alias for the mapped table, kept in VRAM\n");
+            }
+#else
             if (cudaHostGetDevicePointer(&d, host_, 0) != cudaSuccess) {
                 err = "native embedding: no device alias for the mapped table";
                 return false;
             }
+#endif
             dev_ = d;
         }
         type_ = (int) t->type;

src/core/peer_experts.cpp

@@ -0,0 +1,325 @@
+// src/core/peer_experts.cpp - see include/strata/core/peer_experts.hpp.
+#include "strata/core/peer_experts.hpp"
+
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/elementwise.hpp"
+#include "strata/kernels/iq_kernels.hpp"
+#include "strata/kernels/quantize_act.hpp"
+#include "strata/kernels/s2_expert_grouped.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdio>
+#include <cstring>
+#include <cstdlib>
+#include <stdexcept>
+
+namespace strata::core {
+
+static bool g_peer_portable = false;
+void set_peer_portable(bool on) { g_peer_portable = on; }
+bool peer_portable() { return g_peer_portable; }
+
+namespace {
+
+constexpr int64_t H = strata::kernels::cpu::H;
+constexpr int64_t CAP = strata::kernels::cpu::MAXT * 10;   // entries per layer: MAXT tokens x top-10
+
+struct Meta {                           // one block, uploaded with one copy per layer
+    unsigned long long ptr[CAP];
+    int32_t start[CAP + 1];
+    int32_t dst[CAP];
+    int32_t tok[CAP];
+    int32_t count[4];                   // [0] groups, [1] entries
+    int32_t out_row[CAP];               // multi-GPU: compact row -> entry row in the host `out`
+};
+
+/// Switches the calling thread to `dev` and back.  The engine's own thread lives on device 0.
+struct On {
+    int prev = 0;
+    explicit On(int dev) { cudaGetDevice(&prev); if (prev != dev) cudaSetDevice(dev); }
+    ~On() { int cur = 0; cudaGetDevice(&cur); if (cur != prev) cudaSetDevice(prev); }
+};
+
+bool ck(cudaError_t e, const char* what, std::string& err) {
+    if (e == cudaSuccess) return true;
+    err = std::string("peer experts: ") + what + ": " + cudaGetErrorString(e);
+    return false;
+}
+
+}  // namespace
+
+PeerExperts::~PeerExperts() { close(); }
+
+void PeerExperts::close() {
+    if (device_ < 0) return;
+    {
+        On on(device_);
+        if (stream_) cudaStreamSynchronize(stream_);
+        if (refill_) cudaStreamSynchronize(refill_);
+        cache_.close();
+        if (d_x_) cudaFree(d_x_);
+        if (d_out_) cudaFree(d_out_);
+        if (d_meta_) cudaFree(d_meta_);
+        if (d_q8_) cudaFree(d_q8_);
+        if (d_scratch_) cudaFree(d_scratch_);
+        if (h_x_) cudaFreeHost(h_x_);
+        if (h_out_) cudaFreeHost(h_out_);
+        if (h_meta_) cudaFreeHost(h_meta_);
+        if (refill_ev_) cudaEventDestroy(refill_ev_);
+        if (stream_) cudaStreamDestroy(stream_);
+        if (refill_) cudaStreamDestroy(refill_);
+    }
+    d_x_ = d_out_ = h_x_ = h_out_ = nullptr;
+    d_meta_ = h_meta_ = d_scratch_ = nullptr;
+    d_q8_ = nullptr;
+    stream_ = refill_ = nullptr;
+    refill_ev_ = nullptr;
+    device_ = -1;
+}
+
+bool PeerExperts::open(int device, const std::vector<std::pair<int32_t, int32_t>>& ranked, const ExpertCache& primary,
+                       ExpertSource& src, int64_t n_layers, int64_t n_expert, int reserve_mib, int64_t max_slots,
+                       std::string& err) {
+    close();
+    int count = 0;
+    if (!ck(cudaGetDeviceCount(&count), "cudaGetDeviceCount", err)) return false;
+    if (device < 1 || device >= count) {
+        err = "peer experts: CUDA device " + std::to_string(device) + " is not visible (" + std::to_string(count) +
+              " devices; set CUDA_VISIBLE_DEVICES)";
+        return false;
+    }
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    n_layers_ = n_layers;
+    n_expert_ = n_expert;
+    src_ = &src;
+    res_.assign((size_t) (n_layers * n_expert), kNotResident);
+    {   // direct access both ways (NVLink or another P2P path): the prompt path copies activations and results over it
+        int a = 0, b = 0;
+        cudaDeviceCanAccessPeer(&a, 0, device);
+        cudaDeviceCanAccessPeer(&b, device, 0);
+        p2p_ = a && b;
+        if (p2p_) {
+            On on0(0);
+            cudaError_t e0 = cudaDeviceEnablePeerAccess(device, 0);
+            if (e0 == cudaErrorPeerAccessAlreadyEnabled) { cudaGetLastError(); e0 = cudaSuccess; }
+            On on1(device);
+            cudaError_t e1 = cudaDeviceEnablePeerAccess(0, 0);
+            if (e1 == cudaErrorPeerAccessAlreadyEnabled) { cudaGetLastError(); e1 = cudaSuccess; }
+            p2p_ = e0 == cudaSuccess && e1 == cudaSuccess;
+        }
+    }
+    On on(device);
+    device_ = device;
+    // scratch first, so the slots take what is really left
+    int64_t ff = strata::kernels::cpu::FF;
+    if (lay.native)
+        for (const auto& f : lay.fmt) ff = std::max<int64_t>(ff, f.n_ff);
+    const size_t scratch = std::max<size_t>((size_t) strata::kernels::moe_hit_grouped_scratch_bytes(CAP, H, ff),
+                                            strata::kernels::native_expert_scratch_bytes(CAP, ff));
+    const bool alloc_ok =
+        ck(cudaStreamCreateWithFlags(&stream_, cudaStreamNonBlocking), "stream", err) &&
+        ck(cudaStreamCreateWithFlags(&refill_, cudaStreamNonBlocking), "refill stream", err) &&
+        ck(cudaEventCreateWithFlags(&refill_ev_, cudaEventDisableTiming), "event", err) &&
+        ck(cudaHostAlloc((void**) &h_x_, (size_t) CAP * H * sizeof(float), cudaHostAllocPortable | cudaHostAllocMapped), "input staging", err) &&
+        ck(cudaHostAlloc((void**) &h_out_, (size_t) CAP * H * sizeof(float), cudaHostAllocPortable | cudaHostAllocMapped), "result staging", err) &&
+        ck(cudaHostAlloc(&h_meta_, sizeof(Meta), cudaHostAllocPortable | cudaHostAllocMapped), "plan staging", err) &&
+        ck(cudaMalloc((void**) &d_x_, (size_t) CAP * H * sizeof(float)), "input", err) &&
+        ck(cudaMalloc((void**) &d_out_, (size_t) CAP * H * sizeof(float)), "result", err) &&
+        ck(cudaMalloc(&d_meta_, sizeof(Meta)), "plan", err) &&
+        ck(cudaMalloc((void**) &d_q8_, (size_t) CAP * (H / 32) * 36), "activations", err) &&
+        ck(cudaMalloc(&d_scratch_, scratch), "scratch", err);
+    if (!alloc_ok) { close(); return false; }
+
+    // the pairs the primary does not hold, in rank order, as many as fit
+    size_t free_b = 0, total_b = 0;
+    if (!ck(cudaMemGetInfo(&free_b, &total_b), "cudaMemGetInfo", err)) { close(); return false; }
+    const uint64_t reserve = (uint64_t) std::max(reserve_mib, 128) << 20;
+    const uint64_t budget = free_b > reserve ? free_b - reserve : 0;
+    std::vector<std::pair<int32_t, int32_t>> pick;
+    std::vector<int64_t> sizes;
+    uint64_t used = 0;
+    for (const auto& pr : ranked) {
+        if (primary.slot_of(pr.first, pr.second) >= 0) continue;
+        const uint64_t b = lay.blob_bytes(pr.first);
+        const uint64_t b256 = lay.native ? (b + 255) / 256 * 256 : lay.max_blob;
+        if (used + b256 > budget) break;
+        if (max_slots > 0 && (int64_t) pick.size() >= max_slots) break;
+        used += b256;
+        pick.push_back(pr);
+        sizes.push_back((int64_t) b);
+    }
+    if (pick.empty()) { err = "peer experts: no room or no expert left for the peer"; close(); return false; }
+    const bool opened = lay.native ? cache_.open_sized(sizes, n_layers, n_expert, err)
+                                   : cache_.open((int64_t) pick.size(), n_layers, n_expert, (int64_t) lay.max_blob, err);
+    if (!opened) { err = "peer experts: " + err; close(); return false; }
+    // open() zeroes the arena with cudaMemset on the legacy stream, which is not ordered against the non-blocking
+    // refill stream: wait for it, or the zeroing can land on top of the fills
+    if (!ck(cudaDeviceSynchronize(), "arena zeroing", err)) { close(); return false; }
+    for (const auto& pr : pick) {
+        const int32_t slot = cache_.admit(pr.first, pr.second);
+        const uint8_t* b = src.blob(pr.first, pr.second);
+        if (slot < 0 || b == nullptr || !cache_.fill_slot(slot, b, refill_, err, (int64_t) lay.blob_bytes(pr.first))) {
+            err = "peer experts: fill failed: " + err;
+            close();
+            return false;
+        }
+        res_[(size_t) (pr.first * n_expert + pr.second)] = slot;
+    }
+    if (!ck(cudaStreamSynchronize(refill_), "fill", err)) { close(); return false; }
+    const auto& f0 = pick.front();
+    if (!cache_.verify_slot(res_[(size_t) (f0.first * n_expert + f0.second)], src.blob(f0.first, f0.second), err,
+                            (int64_t) lay.blob_bytes(f0.first))) {
+        err = "peer experts: " + err;
+        close();
+        return false;
+    }
+    resident_ = (int64_t) pick.size();
+    return true;
+}
+
+bool PeerExperts::launch(int64_t layer, const float* x, const int32_t* ids, int64_t n_tok, int64_t k,
+                         const int32_t* kind, std::string& err, float* out) {
+    static const bool direct_env = [] { const char* v = std::getenv("STRATA_PEER_DIRECT"); return v == nullptr || std::atoi(v) != 0; }();
+    const bool direct = direct_env && out != nullptr;
+    launched_direct_ = false;
+    launched_rows_ = 0;
+    row_of_.clear();
+    const int64_t n = n_tok * k;
+    if (n > CAP) { err = "peer experts: window too large"; return false; }
+    Meta& m = *(Meta*) h_meta_;
+    int groups = 0, rows = 0;
+    // distinct experts in routing order; each one's entries become compact rows
+    for (int64_t i = 0; i < n; ++i) {
+        if (kind[i] != 2) continue;
+        bool seen = false;
+        for (int64_t j = 0; j < i; ++j)
+            if (kind[j] == 2 && ids[j] == ids[i]) { seen = true; break; }
+        if (seen) continue;
+        const int32_t slot = res_[(size_t) (layer * n_expert_ + ids[i])];
+        if (slot < 0) { err = "peer experts: an entry was planned for the peer but is not resident"; return false; }
+        m.ptr[groups] = (unsigned long long) cache_.device_slot(slot);
+        m.start[groups] = rows;
+        for (int64_t j = i; j < n; ++j)
+            if (kind[j] == 2 && ids[j] == ids[i]) {
+                m.dst[rows] = rows;
+                m.tok[rows] = (int32_t) (j / k);
+                m.out_row[rows] = (int32_t) j;
+                row_of_.push_back((int32_t) j);
+                ++rows;
+            }
+        ++groups;
+        ++experts_;
+    }
+    if (groups == 0) return true;
+    m.start[groups] = rows;
+    m.count[0] = groups;
+    m.count[1] = rows;
+    entries_ += rows;
+    launched_rows_ = rows;
+    // direct: x is the verifier's pinned (portable) doorbell row block - copied from where it is
+    if (!direct) std::memcpy(h_x_, x, (size_t) (n_tok * H) * sizeof(float));
+    On on(device_);
+    const cudaStream_t s = stream_;
+    if (!ck(cudaMemcpyAsync(d_x_, direct ? x : h_x_, (size_t) (n_tok * H) * sizeof(float), cudaMemcpyHostToDevice, s), "input", err) ||
+        !ck(cudaMemcpyAsync(d_meta_, h_meta_, sizeof(Meta), cudaMemcpyHostToDevice, s), "plan", err))
+        return false;
+    Meta* dm = (Meta*) d_meta_;
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    if (lay.native) {
+        strata::kernels::quantize_q8_1_rows(d_x_, n_tok, H, d_q8_, s);
+        const auto& f = lay.fmt[(size_t) layer];
+        const auto L = strata::kernels::native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);
+        strata::kernels::native_expert_grouped(L, dm->ptr, dm->start, dm->count, dm->dst, dm->tok, groups, rows, d_q8_,
+                                               d_scratch_, d_out_, s);
+    } else {
+        err = "peer experts: only native packs are supported";
+        return false;
+    }
+    if (direct) {   // the rows straight into the host `out` (zero-copy, coalesced float4 writes)
+        try {
+            strata::kernels::scatter_rows_f32(d_out_, out, dm->out_row, rows, H, s);
+        } catch (const std::exception& e) { err = std::string("peer experts: scatter: ") + e.what(); return false; }
+        launched_direct_ = true;
+        return true;
+    }
+    return ck(cudaMemcpyAsync(h_out_, d_out_, (size_t) rows * H * sizeof(float), cudaMemcpyDeviceToHost, s),
+              "results", err);
+}
+
+bool PeerExperts::finish(float* out, std::string& err) {
+    if (launched_rows_ == 0) return true;
+    const auto t0 = std::chrono::steady_clock::now();
+    {
+        On on(device_);
+        // spin on the stream (a blocking sync would sleep the pool thread and wake it late)
+        cudaError_t e;
+        while ((e = cudaStreamQuery(stream_)) == cudaErrorNotReady) {
+        }
+        if (!ck(e, "finish", err)) return false;
+    }
+    ms_wait += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
+    if (launched_direct_) { launched_rows_ = 0; return true; }
+    for (size_t r = 0; r < row_of_.size(); ++r)
+        std::memcpy(out + (size_t) row_of_[r] * H, h_out_ + r * H, (size_t) H * sizeof(float));
+    launched_rows_ = 0;
+    return true;
+}
+
+bool PeerExperts::adapt(const float* usage, const int32_t* res0, int max_swaps, std::string& err) {
+    if (!pending_.empty() || max_swaps <= 0) return true;
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    struct Swap { float gain; int32_t layer, in, out; };
+    std::vector<Swap> swaps;
+    std::vector<std::pair<float, int32_t>> cand, vict;
+    for (int64_t l = 0; l < n_layers_; ++l) {
+        cand.clear();
+        vict.clear();
+        const float* u = usage + l * n_expert_;
+        const int32_t* r0 = res0 + l * n_expert_;
+        const int32_t* r1 = res_.data() + l * n_expert_;
+        for (int32_t e = 0; e < (int32_t) n_expert_; ++e) {
+            if (r1[e] >= 0) vict.emplace_back(u[e], e);
+            else if (r0[e] < 0 && u[e] >= 2.0f) cand.emplace_back(u[e], e);
+        }
+        if (cand.empty() || vict.empty()) continue;
+        std::sort(cand.begin(), cand.end(), [](auto& a, auto& b) { return a.first > b.first; });
+        const size_t nc = std::min(cand.size(), vict.size());
+        std::partial_sort(vict.begin(), vict.begin() + (ptrdiff_t) nc, vict.end(),
+                          [](auto& a, auto& b) { return a.first < b.first; });
+        for (size_t i = 0; i < nc; ++i) {
+            if (cand[i].first < vict[i].first + 1.5f) break;
+            swaps.push_back({cand[i].first - vict[i].first, (int32_t) l, cand[i].second, vict[i].second});
+        }
+    }
+    if (swaps.empty()) return true;
+    std::sort(swaps.begin(), swaps.end(), [](const Swap& a, const Swap& b) { return a.gain > b.gain; });
+    if ((int) swaps.size() > max_swaps) swaps.resize((size_t) max_swaps);
+    On on(device_);
+    for (const Swap& s : swaps) {
+        const size_t in = (size_t) (s.layer * n_expert_ + s.in), out = (size_t) (s.layer * n_expert_ + s.out);
+        const int32_t slot = res_[out];
+        const uint8_t* b = src_->blob(s.layer, s.in);
+        if (slot < 0 || b == nullptr) continue;
+        if (!ck(cudaMemcpyAsync(cache_.device_slot(slot), b, (size_t) lay.blob_bytes(s.layer), cudaMemcpyHostToDevice,
+                                refill_), "refill", err))
+            return false;
+        res_[out] = kNotResident;              // evicted now: the CPU computes it meanwhile
+        pending_.emplace_back((int32_t) in, slot);
+        ++swaps_;
+    }
+    return ck(cudaEventRecord(refill_ev_, refill_), "refill event", err);
+}
+
+void PeerExperts::apply_pending(bool wait) {
+    if (pending_.empty()) return;
+    {
+        On on(device_);
+        if (wait) cudaEventSynchronize(refill_ev_);
+        else if (cudaEventQuery(refill_ev_) != cudaSuccess) return;
+    }
+    for (const auto& [i, slot] : pending_) res_[(size_t) i] = slot;
+    pending_.clear();
+}
+
+}  // namespace strata::core

src/core/pinned.cu

@@ -199,15 +199,32 @@ void* reserve(uint64_t bytes, PageBacking& got, std::string& note, const std::st
         return (uint8_t*) map + kSharedArenaHeaderBytes;
     }
 
-    void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE,
-                   MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, -1, 0);
-    if (p != MAP_FAILED) {
-        got = PageBacking::LargePages;
-        note = "hugetlb 2 MB pages";
-        return p;
+    // STRATA_NO_LARGEPAGES=1 is the same-run A/B switch the Windows branch documents; honor it
+    // here too, so the large-page path can be compared without changing the pool or rebooting.
+    if (std::getenv("STRATA_NO_LARGEPAGES") != nullptr) {
+        note = "large pages skipped (STRATA_NO_LARGEPAGES); using 4 KB pages";
+    } else {
+        void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE,
+                       MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, -1, 0);
+        if (p != MAP_FAILED) {
+            got = PageBacking::LargePages;
+            note = "hugetlb 2 MB pages";
+            return p;
+        }
+        // MAP_HUGETLB is all-or-nothing: a pool smaller than the mapping fails exactly like an absent
+        // one, and the old message guessed "no hugetlb pool configured" either way. Name the shortfall:
+        // how many 2 MiB pages the mapping needs against what vm.nr_hugepages actually holds.
+        const unsigned long long need = ((unsigned long long) bytes + (1ull << 21) - 1) / (1ull << 21);
+        unsigned long long pool = 0;
+        bool have_pool = false;
+        if (std::FILE* f = std::fopen("/proc/sys/vm/nr_hugepages", "r")) {
+            have_pool = std::fscanf(f, "%llu", &pool) == 1;
+            std::fclose(f);
+        }
+        note = "MAP_HUGETLB unavailable (needed " + std::to_string(need) + " 2 MiB pages, vm.nr_hugepages=" +
+               (have_pool ? std::to_string(pool) : std::string("?")) + "); using 4 KB pages";
     }
-    note = "MAP_HUGETLB unavailable (no hugetlb pool configured?); using 4 KB pages";
-    p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
     got = PageBacking::NormalPages;
     return p == MAP_FAILED ? nullptr : p;
 #endif
@@ -413,6 +430,141 @@ LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_byte
     return load_experts_ranges(path, dst, off, n, threads, chunk);
 }
 
+LoadStats load_experts_direct(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
+                              const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk) {
+    LoadStats st;
+    st.ok = false;
+#ifdef _WIN32
+    constexpr uint64_t kAlign = 4096;
+    if (((uintptr_t) dst % kAlign) != 0 || chunk == 0 || chunk % kAlign != 0) return st;
+    struct Piece { uint64_t off, n; };
+    std::vector<Piece> pieces;
+    uint64_t bytes = 0;
+    for (size_t L = 0; L < layer_off.size(); ++L) {
+        if (layer_off[L] % kAlign != 0 || layer_bytes[L] % kAlign != 0) return st;
+        for (uint64_t p = 0; p < layer_bytes[L]; p += chunk)
+            pieces.push_back({layer_off[L] + p, std::min<uint64_t>(chunk, layer_bytes[L] - p)});
+        bytes += layer_bytes[L];
+    }
+    if (threads < 1) threads = 1;
+    const auto t0 = std::chrono::steady_clock::now();
+    std::atomic<size_t> next{0};
+    std::mutex err_mu;
+    std::string err;
+    const int wide = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, nullptr, 0);
+    std::vector<wchar_t> wpath((size_t) std::max(wide, 1), L'\0');
+    if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, wpath.data(), wide);
+    auto worker = [&]() {
+        HANDLE h = CreateFileW(wpath.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                               FILE_FLAG_NO_BUFFERING | FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
+        if (h == INVALID_HANDLE_VALUE) {
+            std::lock_guard<std::mutex> g(err_mu);
+            if (err.empty())
+                err = "cannot open " + path + " unbuffered (error " + std::to_string((unsigned long long) GetLastError()) + ")";
+            next = pieces.size();
+            return;
+        }
+        for (;;) {
+            const size_t i = next.fetch_add(1);
+            if (i >= pieces.size()) break;
+            OVERLAPPED ov{};
+            ov.Offset = (DWORD) pieces[i].off;
+            ov.OffsetHigh = (DWORD) (pieces[i].off >> 32);
+            DWORD got = 0;
+            if (!ReadFile(h, dst + pieces[i].off, (DWORD) pieces[i].n, &got, &ov) || got != pieces[i].n) {
+                std::lock_guard<std::mutex> g(err_mu);
+                if (err.empty())
+                    err = "short unbuffered read at offset " + std::to_string(pieces[i].off) + ": got " + std::to_string(got) +
+                          " of " + std::to_string(pieces[i].n) + " B (error " +
+                          std::to_string((unsigned long long) GetLastError()) + ")";
+                next = pieces.size();
+                break;
+            }
+        }
+        CloseHandle(h);
+    };
+    std::vector<std::thread> pool;
+    for (int i = 1; i < threads; ++i) pool.emplace_back(worker);
+    worker();
+    for (auto& t : pool) t.join();
+    st.layers = layer_off.size();
+    if (!err.empty()) {
+        st.seconds = -1.0;
+        st.error = err;
+        return st;
+    }
+    st.ok = true;
+    st.bytes = bytes;
+    st.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+#else
+    (void) path; (void) dst; (void) layer_off; (void) layer_bytes; (void) threads; (void) chunk;
+#endif
+    return st;
+}
+
+bool experts_unbuffered(const std::vector<std::string>& files, uint64_t arena_bytes, std::string& why,
+                        bool cache_counts) {
+    const char* env = std::getenv("STRATA_UNBUFFERED_LOAD");
+    if (env != nullptr && env[0] != '\0') {
+        why = std::string("STRATA_UNBUFFERED_LOAD=") + env;
+        return env[0] != '0';
+    }
+#ifdef _WIN32
+    LARGE_INTEGER freq{}, a{}, b{};
+    QueryPerformanceFrequency(&freq);
+    constexpr DWORD kRead = 64 << 10;
+    constexpr int kSamples = 16;
+    std::vector<uint8_t> buf(kRead);
+    uint64_t total_bytes = 0;
+    int fast = 0, n = 0;
+    uint64_t seed = (uint64_t) GetTickCount64() * 6364136223846793005ull + 1442695040888963407ull;
+    for (const std::string& f : files) {
+        const int wide = MultiByteToWideChar(CP_UTF8, 0, f.c_str(), -1, nullptr, 0);
+        std::vector<wchar_t> w((size_t) std::max(wide, 1), L'\0');
+        if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, f.c_str(), -1, w.data(), wide);
+        HANDLE h = CreateFileW(w.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS,
+                               nullptr);
+        if (h == INVALID_HANDLE_VALUE) continue;
+        LARGE_INTEGER size{};
+        GetFileSizeEx(h, &size);
+        total_bytes += (uint64_t) size.QuadPart;
+        // random offsets: a probe must not find the blocks an earlier probe put into the cache
+        for (int i = 0; i < kSamples && (uint64_t) size.QuadPart > 2ull * kRead; ++i) {
+            seed = seed * 6364136223846793005ull + 1442695040888963407ull;
+            const uint64_t off = ((seed >> 17) % ((uint64_t) size.QuadPart - kRead)) / kRead * kRead;
+            OVERLAPPED ov{};
+            ov.Offset = (DWORD) off;
+            ov.OffsetHigh = (DWORD) (off >> 32);
+            DWORD got = 0;
+            QueryPerformanceCounter(&a);
+            const BOOL ok = ReadFile(h, buf.data(), kRead, &got, &ov);
+            QueryPerformanceCounter(&b);
+            if (!ok) continue;
+            ++n;
+            fast += (double) (b.QuadPart - a.QuadPart) * 1e6 / (double) freq.QuadPart < 30.0;
+        }
+        CloseHandle(h);
+    }
+    const bool cached = n > 0 && fast * 4 >= n * 3;
+    MEMORYSTATUSEX ms{};
+    ms.dwLength = sizeof ms;
+    GlobalMemoryStatusEx(&ms);
+    const uint64_t avail = ms.ullAvailPhys;
+    // what the cache could keep beside the arena (~4 GiB for everything else)
+    const uint64_t room = avail > arena_bytes + (4ull << 30) ? avail - arena_bytes - (4ull << 30) : 0;
+    const bool keepable = room >= total_bytes;
+    char msg[200];
+    std::snprintf(msg, sizeof msg, "%d of %d probe reads from the file cache; %.1f GiB available, %.1f GiB of files",
+                  fast, n, (double) avail / (1ull << 30), (double) total_bytes / (1ull << 30));
+    why = msg;
+    return (!cached || !cache_counts) && !keepable;
+#else
+    (void) files; (void) arena_bytes; (void) cache_counts;
+    why = "buffered (not Windows)";
+    return false;
+#endif
+}
+
 LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
                               const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk) {
     LoadStats st;

src/core/verify.cpp

@@ -6,6 +6,7 @@
 
 #include "strata/core/native_head.hpp"
 #include "strata/core/on_device.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/kernels/iq_kernels.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
@@ -69,7 +70,7 @@ struct Bump {
 };
 
 bool mapped(size_t bytes, void** h, void** d) {
-    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped) != cudaSuccess) return false;
+    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false;   // multi-GPU: portable only with a peer - the peer card writes its rows into them
     std::memset(*h, 0, bytes);
     return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;
 }
@@ -210,7 +211,9 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
     }
     if (hits.d_res == nullptr || hits.cache_base == nullptr || hits.blob <= 0) {
         err = "verify: needs the profile-filled VRAM expert tier (--expert-profile and --expert-cache); with "
-              "--expert-cache auto, no VRAM was left for it: lower --max-context, use --kv k8v4 or images on the CPU";
+              "--expert-cache auto, no VRAM was left for it - the 'no VRAM is left for the expert cache' line above "
+              "says how much is short and what makes room (a smaller --max-context, --kv q4_0, setup --draft-vocab en, "
+              "images on the CPU)";
         return false;
     }
     std::string why;
@@ -745,20 +748,21 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
         const int32_t* p_start2 = pl + ptr_off + 4 * capx;
         float* hit_out = hit_out_ + (size_t) tb * K * N;
         const auto& lay = strata::kernels::cpu::expert_layout();
-        // plan v0.3 P6: the VRAM groups now; the PCIe groups once the copy engine has landed them in staging
-        auto grouped = [&](const unsigned long long* gp, const int32_t* gs, const int32_t* gn) {
+        // plan v0.3 P6: the VRAM groups now; the PCIe groups once the copy engine has landed them in staging.
+        // `gy`: the native launch's groups side by side (0: cap, one block row per possible group).
+        auto grouped = [&](const unsigned long long* gp, const int32_t* gs, const int32_t* gn, int64_t gy) {
             if (lay.native) {
                 // the layer's GGUF formats (i-quant gate/up, Q2_0 / IQ4_NL down)
                 const auto& f = lay.fmt[(size_t) l];
                 const NativeExpertLayout L = native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);
                 native_expert_grouped(L, gp, gs, gn, p_dst, p_tok, cap, cap,
-                                      nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, hit_out, cs);
+                                      nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, hit_out, cs, gy);
             } else {
                 moe_grouped_s2(gp, gs, gn, p_dst, p_tok, cap, cap, hit_xq_ + (size_t) tb * (N / 32) * 34,
                                hit_xs_ + (size_t) tb * (N / 32), hit_scratch_, hit_out, cs);
             }
         };
-        grouped(p_ptr, p_start, p_counts);
+        grouped(p_ptr, p_start, p_counts, 0);
         stamp(l, 20, grp);
         if (device_plan_) wait_flag_ge_or(m_flagB_, ring, skip_ + grp, cs);
         else wait_flag_ge(m_flagB_, ring, cs);                 // the PCIe share is in staging (DMA) or mapped
@@ -769,7 +773,9 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             rebase_ptrs((unsigned long long*) p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), cs);
         }
         stamp(l, 21, grp);
-        grouped(p_ptr2, p_start2, p_counts + 2);
+        // the PCIe share is pcie_frac of the misses: a few groups when the cache is cold, usually none (always none at
+        // pcie_frac 0), so its launch is kPcieGroupRows block rows striding over the groups, not cap of them
+        grouped(p_ptr2, p_start2, p_counts + 2, kPcieGroupRows);
         stamp(l, 22, grp);
         if (device_plan_) {   // no CPU share when the device planned the group: its rows are zeros
             wait_flag_ge_or(m_flag_, ring, skip_ + grp, cs);

src/kernels/cpu/iq_avx2.cpp

@@ -9,7 +9,7 @@
 // with one, `vpsignb`.  The arithmetic is ggml's (ggml-cpu/quants.c, the `_generic` references) - only the
 // order of the float additions differs.
 //
-// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22).  IQ1_M stays on ggml-cpu.
+// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22), IQ4_XS (23).  IQ1_M stays on ggml-cpu.
 #include "strata/kernels/cpu/iq_avx2.hpp"
 
 #define GGML_COMMON_DECL_CPP
@@ -177,6 +177,28 @@ template <> struct Fmt32<21> {   // IQ3_S: d, qs[64], qh[8], signs[32], scales[4
     }
 };
 
+template <> struct Fmt32<23> {   // IQ4_XS: d, scales_h, scales_l[4], qs[128] - 136 B, 8 signed sub-scales
+    // ggml's ggml_vec_dot_iq4_xs_q8_K: the same 16-value codebook as IQ4_NL, but each of the eight 32-value
+    // sub-blocks carries its own 6-bit scale, read as two nibbles of scales_l[p] plus two bits of scales_h, and
+    // used SIGNED as (ls - 32).  The scale therefore folds into the int16 operand of madd_epi16 instead of
+    // becoming a float multiply per sub-block, and the codebook sign is carried the way iq4nl_rows does it.
+    static constexpr int bytes = 136;
+    static constexpr float K = 1.0f;
+    static inline void decode(const uint8_t* b, int j, int half, __m256i& g, __m256i& sgn, __m256i& sc) {
+        const int H = 2 * j + half;   // the eight 32-value sub-blocks, in the order the activation bytes come
+        const __m128i values = _mm_loadu_si128((const __m128i*) kvalues_iq4nl);
+        const __m128i bits = _mm_loadu_si128((const __m128i*) (b + 8 + 16 * H));
+        const __m128i m4 = _mm_set1_epi8(0x0f);
+        const __m256i q4 = _mm256_inserti128_si256(
+            _mm256_castsi128_si256(_mm_shuffle_epi8(values, _mm_and_si128(bits, m4))),
+            _mm_shuffle_epi8(values, _mm_and_si128(_mm_srli_epi16(bits, 4), m4)), 1);
+        g   = _mm256_sign_epi8(q4, q4);                       // |w|, the unsigned operand of maddubs
+        sgn = _mm256_sign_epi8(_mm256_set1_epi8(1), q4);      // w's sign, applied to the activation
+        const int ls = ((b[4 + (H >> 1)] >> (4 * (H & 1))) & 0xf) | (((u16(b + 2) >> (2 * H)) & 3) << 4);
+        sc = _mm256_set1_epi16((short) (ls - 32));
+    }
+};
+
 template <int TY, int NT>
 inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res) {
     __m256 accf[NT];
@@ -357,7 +379,7 @@ void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void*
 }  // namespace
 
 bool iq256_supported(int type) noexcept {
-    return type == 16 || type == 17 || type == 18 || type == 21 || type == 22;
+    return type == 16 || type == 17 || type == 18 || type == 21 || type == 22 || type == 23;
 }
 
 void iq256_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt,
@@ -368,6 +390,7 @@ void iq256_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off,
         case 18: gu_rows_nt<18>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         case 21: gu_rows_nt<21>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         case 22: gu_rows_nt<22>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 23: gu_rows_nt<23>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         default: break;
     }
 }
@@ -380,6 +403,7 @@ void iq256_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void*
         case 18: dot_rows_nt<18>(nt, w, row_bytes, n, act, out, r0, r1); break;
         case 21: dot_rows_nt<21>(nt, w, row_bytes, n, act, out, r0, r1); break;
         case 22: dot_rows_nt<22>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 23: dot_rows_nt<23>(nt, w, row_bytes, n, act, out, r0, r1); break;
         default: break;
     }
 }

src/kernels/cpu/native_expert.cpp

@@ -98,12 +98,16 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*
         kq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
         return;
     }
-    if (nt >= mt_min && iq512_supported(f.gu_type)) {
-        if (avx512) {
+    // A format with only an AVX-2 kernel (IQ4_XS, #415) takes it on AVX-2 CPUs only: an AVX-512 CPU keeps ggml-cpu for
+    // it, as before (its rows would round differently).  Each kernel only for the formats it implements: falling
+    // through an empty switch would leave ff unwritten instead of falling back to ggml-cpu.
+    static const bool cpu512 = cpu_avx512_ok();
+    if (nt >= mt_min && (iq512_supported(f.gu_type) || (!cpu512 && iq256_supported(f.gu_type)))) {
+        if (avx512 && iq512_supported(f.gu_type)) {
             iq512_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
             return;
         }
-        if (avx2) {
+        if (avx2 && iq256_supported(f.gu_type)) {
             iq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
             return;
         }

src/kernels/cuda/elementwise.cu

@@ -251,6 +251,23 @@ __global__ void copy_rows_from_mapped_kernel(float4* __restrict__ dst, const vol
         for (int64_t i = threadIdx.x; i < row4; i += blockDim.x) d[i] = const_cast<const float4*>(sr)[i];
     }
 }
+namespace {
+__global__ void scatter_rows_kernel(const float4* __restrict__ src, float4* dst, const int32_t* __restrict__ rows,
+                                    int64_t w4) {
+    const int64_t r = blockIdx.x;
+    const float4* s = src + r * w4;
+    float4* d = dst + (int64_t) rows[r] * w4;
+    for (int64_t i = threadIdx.x; i < w4; i += blockDim.x) d[i] = s[i];
+}
+}  // namespace
+void scatter_rows_f32(const float* src, float* dst, const int32_t* rows, int64_t n, int64_t width, void* stream) {
+    if (n <= 0) return;
+    if ((width & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {
+        std::fprintf(stderr, "scatter_rows_f32: width must be a multiple of 4 and both pointers 16-byte aligned\n");
+        std::exit(1);
+    }
+    scatter_rows_kernel<<<(unsigned) n, 128, 0, (cudaStream_t) stream>>>((const float4*) src, (float4*) dst, rows, width / 4);
+}
 void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
                            const int32_t* count, void* stream) {
     if (rows <= 0) return;

src/kernels/cuda/fused_gr.cu

@@ -196,7 +196,7 @@ __global__ void __launch_bounds__(THREADS) gr_norm_multi_kernel(GrMulti m) {
 // which fits all eight tokens in one 40 KiB launch and stages 160 chunks of 8 (5 per lane, half the registers held
 // for the weight prefetch); smaller tiles raise how many blocks share an SM (the 41-block grid), e.g. three blocks
 // of four tokens instead of one on sm_75.
-template <int TILEV>
+template <int TILEV, int MAX_T = kFusedGrMaxT>
 __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) {
     constexpr int TQ = TILEV / 8 / 32;      // uint4 weight chunks per lane per tile
     extern __shared__ __align__(16) float tile[];   // [T][TILEV]
@@ -207,9 +207,9 @@ __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) {
     const bool active = !(inject_block && (m.a[0].w_inject == nullptr || warp >= HC));
     const uint16_t* wrow = (inject_block ? m.a[0].w_inject : m.a[0].w_down) + (size_t) (active ? row : 0) * D;
     const uint4* w4 = reinterpret_cast<const uint4*>(wrow);
-    float acc[kFusedGrMaxT];
+    float acc[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) acc[k] = 0.0f;
+    for (int k = 0; k < MAX_T; ++k) acc[k] = 0.0f;
     for (int base = 0; base < D; base += TILEV) {
         uint4 wv[TQ];
         if (active) {
@@ -229,17 +229,17 @@ __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) {
         for (int q = 0; q < TQ; ++q) {
             const int j = lane + 32 * q;
 #pragma unroll
-            for (int k = 0; k < kFusedGrMaxT; ++k)
+            for (int k = 0; k < MAX_T; ++k)
                 if (k < T) acc[k] += dot8(wv[q], tile + k * TILEV + j * 8);
         }
     }
     if (!active) return;
-    float s[kFusedGrMaxT];
+    float s[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
+    for (int k = 0; k < MAX_T; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
     // lane k writes token k (every lane holds every sum after the xor reduction)
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) {
+    for (int k = 0; k < MAX_T; ++k) {
         if (k >= T || lane != k) continue;
         if (inject_block) {
             m.a[k].inject_out[row] = s[k];
@@ -616,6 +616,7 @@ __device__ __forceinline__ void stage_htile(const GrMulti& m, int T, int h, floa
     }
 }
 
+template <int MAX_T = kFusedGrMaxT>
 __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
     extern __shared__ __align__(16) float4 hbuf[];      // 2 buffers x [T][2][160] float4
     const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
@@ -626,9 +627,9 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
     const uint16_t* wrow = (inject_block ? m.a[0].w_inject : m.a[0].w_down) + (size_t) (active ? row : 0) * D;
     const uint4* w4 = reinterpret_cast<const uint4*>(wrow);
     const size_t buf_f4 = (size_t) T * (H_TILE / 4);    // float4 per buffer (the second one follows the first)
-    float acc[kFusedGrMaxT];
+    float acc[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) acc[k] = 0.0f;
+    for (int k = 0; k < MAX_T; ++k) acc[k] = 0.0f;
     uint4 wv[HQ], wnext[HQ];
     if (active) {
 #pragma unroll
@@ -653,7 +654,7 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
             for (int q = 0; q < HQ; ++q) {
                 const int j = lane + 32 * q;
 #pragma unroll
-                for (int k = 0; k < kFusedGrMaxT; ++k) {
+                for (int k = 0; k < MAX_T; ++k) {
                     if (k < T) {
                         const float4* pk = cur + (size_t) k * (H_TILE / 4);
                         acc[k] += dot8v(wv[q], pk[j], pk[H_TILE / 8 + j]);
@@ -670,12 +671,12 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
         }
     }
     if (!active) return;
-    float s[kFusedGrMaxT];
+    float s[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
+    for (int k = 0; k < MAX_T; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
     // lane k writes token k (every lane holds every sum after the xor reduction)
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) {
+    for (int k = 0; k < MAX_T; ++k) {
         if (k >= T || lane != k) continue;
         if (inject_block) {
             m.a[k].inject_out[row] = s[k];
@@ -729,7 +730,7 @@ int down_chunk(bool staged, int* tile_out) {
         }
         int want_staged = (int) (2 * kFusedGrMaxT * H_TILE * sizeof(float));
         if (optin > 0 && want_staged > optin) want_staged = optin;
-        cudaFuncSetAttribute(gr_down_staged_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, want_staged);
+        cudaFuncSetAttribute(gr_down_staged_kernel<>, cudaFuncAttributeMaxDynamicSharedMemorySize, want_staged);
         cudaGetLastError();      // drop any error the attempt left behind
         // The opt-in is a promise a pre-Volta card does not keep: an sm_60 answers 65536 and accepts the
         // cudaFuncSetAttribute for 61440 B, then fails the LAUNCH with "invalid argument".  What such a card
@@ -778,9 +779,21 @@ void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long
             for (int k = 0; k < ct; ++k) c.a[k] = m.a[c0 + k];
         }
         const size_t smem = (size_t) ct * per_tok;
-        if (staged) gr_down_staged_kernel<<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
-        else if (tv == 1280) gr_down_multi_kernel<1280><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
-        else gr_down_multi_kernel<2560><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        // #443, opt-in STRATA_GR_DOWN_MAX4=1: launches of up to 4 tokens hold 4 tokens' sums per thread instead of
+        // kFusedGrMaxT - the same tile, block size, accumulation order and plain/split/staged path, so the same bits
+        // (read once: this runs per layer when decode is not captured)
+        static const bool max4_on = [] { const char* v = std::getenv("STRATA_GR_DOWN_MAX4"); return v && std::atoi(v) != 0; }();
+        const bool max4 = max4_on && ct <= 4;
+        if (staged) {
+            if (max4) gr_down_staged_kernel<4><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+            else gr_down_staged_kernel<><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        } else if (tv == 1280) {
+            if (max4) gr_down_multi_kernel<1280, 4><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+            else gr_down_multi_kernel<1280><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        } else {
+            if (max4) gr_down_multi_kernel<2560, 4><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+            else gr_down_multi_kernel<2560><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        }
     }
     if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0 + 1, (void*) st);
     gr_up_multi_kernel<<<UPM_BLOCKS, THREADS, 0, st>>>(m);
@@ -835,9 +848,11 @@ void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi
             if (need1 <= limit &&
                 cudaFuncSetAttribute(gr_down_v3_kernel<1>, cudaFuncAttributeMaxDynamicSharedMemorySize, need1) == cudaSuccess)
                 split = 1;
-            else if (need2 <= limit &&
-                     cudaFuncSetAttribute(gr_down_v3_kernel<2>, cudaFuncAttributeMaxDynamicSharedMemorySize, need2) == cudaSuccess)
-                split = 2;
+            else if (need2 <= limit)
+                // #375 (kenh0u): the S = 2 split (a 64 KB opt-in card: Turing) disagrees with itself in gr_parity
+                // (graph replay vs direct call) - such a card keeps the default read until that split is fixed
+                std::fprintf(stderr, "strata: STRATA_GR_V3=1 needs the two-half split on this card, which fails its "
+                                     "checks (#375): the default read is used\n");
             cudaGetLastError();   // drop any error the attempts left behind
             split3[dev3] = split;
         }

src/kernels/cuda/iq_kernels.cu

@@ -295,7 +295,7 @@ __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(const void* __restrict__ vb
 // Q4_K / Q5_K gate/up and Q5_1 / Q8_0 down: llama.cpp's vec_dot_*_q8_1 (vecdotq.cuh, VDR 2 each), transcribed; the
 // Q5_1 min term is the one departure (below).  Via eddoursul/Strata 8029fa9 (iq_dot.cuh) and #255 (Q8_0,
 // gopinath87607), which agree with llama.cpp and with each other.
-constexpr int VDR_Q4_K = 2, VDR_Q5_K = 2, VDR_Q5_1 = 2, VDR_Q8_0 = 2;
+constexpr int VDR_Q4_K = 2, VDR_Q5_K = 2, VDR_Q5_1 = 2, VDR_Q5_0 = 2, VDR_Q8_0 = 2;
 
 __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(const int* __restrict__ v, const int* __restrict__ u,
                                                              const uint8_t* __restrict__ sc, const uint8_t* __restrict__ m,
@@ -405,6 +405,36 @@ __device__ __forceinline__ float vec_dot_q5_K_q8_1(const void* __restrict__ vbq,
 // of the ORIGINAL activations.  That is ggml-cpu's convention (its q8_1 `s` is d * sum(q)) and the one the K-quant
 // mins above use; the scaled and the min term then see the same activation (eddoursul/Strata measured 1.1-1.2%
 // against 1.9% relative error per expert).  Result: sumi * (d5 * d8) + sumu * (m5 * d8).
+// Q5_0: llama.cpp's integer chain verbatim (vecdotq.cuh vec_dot_q5_0_q8_1_impl) -- symmetric (one delta, no
+// learned min), so unlike Q5_1 there is no min-term rounding choice to make: the constant -16 per-weight offset
+// is accounted for via the q8_1 block's `ds.y` (sum of the ORIGINAL activations), exactly as upstream.
+__device__ __forceinline__ float vec_dot_q5_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,
+                                                   const int& kbx, const int& iqs) {
+    const block_q5_0* bq5_0 = (const block_q5_0*) vbq + kbx;
+    int sumi = 0;
+#pragma unroll
+    for (int i = 0; i < VDR_Q5_0; ++i) {
+        const int vl = get_int_b2(bq5_0->qs, iqs + i);
+        const int vh = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i));
+        const int u0 = get_int_b4(bq8_1->qs, iqs + i), u1 = get_int_b4(bq8_1->qs, iqs + i + QI5_0);
+        int vi0 = (vl >> 0) & 0x0F0F0F0F;
+        vi0 |= (vh << 4) & 0x00000010;
+        vi0 |= (vh << 11) & 0x00001000;
+        vi0 |= (vh << 18) & 0x00100000;
+        vi0 |= (vh << 25) & 0x10000000;
+        sumi = ggml_cuda_dp4a(vi0, u0, sumi);
+        int vi1 = (vl >> 4) & 0x0F0F0F0F;
+        vi1 |= (vh >> 12) & 0x00000010;
+        vi1 |= (vh >> 5) & 0x00001000;
+        vi1 |= (vh << 2) & 0x00100000;
+        vi1 |= (vh << 9) & 0x10000000;
+        sumi = ggml_cuda_dp4a(vi1, u1, sumi);
+    }
+    const float d5 = __half2float(bq5_0->d);
+    const float d8 = __low2float(bq8_1->ds);
+    const float s8 = __high2float(bq8_1->ds);
+    return d5 * (sumi * d8 - (16.0f * VDR_Q5_0 / QI5_0) * s8);
+}
 __device__ __forceinline__ float vec_dot_q5_1_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,
                                                    const int& kbx, const int& iqs) {
     const block_q5_1* bq5_1 = (const block_q5_1*) vbq + kbx;
@@ -470,14 +500,16 @@ template<> struct Fmt<13> { static constexpr int qk = 256, ipb = QI5_K / VDR_Q5_
     __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_K_q8_1(v, y, kbx, iqs); } };
 template<> struct Fmt<7> { static constexpr int qk = 32, ipb = QI5_1 / VDR_Q5_1, step = VDR_Q5_1;
     __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_1_q8_1(v, y, kbx, iqs); } };
+template<> struct Fmt<6> { static constexpr int qk = 32, ipb = QI5_0 / VDR_Q5_0, step = VDR_Q5_0;
+    __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_0_q8_1(v, y, kbx, iqs); } };
 template<> struct Fmt<8> { static constexpr int qk = 32, ipb = QI8_0 / VDR_Q8_0, step = VDR_Q8_0;
     __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q8_0_q8_1(v, y, kbx, iqs); } };
 
 // The formats of each role, one list each so a type cannot be in one switch and missing from another.  Every
 // entry is a kernel template for each CUDA architecture of the build, hence two lists rather than one.
-#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(8)
-#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(8)
-#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(8)
+#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(6) X(8)
+#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(8)
+#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(6) X(8)
 
 __device__ __forceinline__ float warp_sum(float v) {
 #pragma unroll
@@ -854,6 +886,12 @@ __global__ void __launch_bounds__(128) mmvq_multi_kernel(const uint8_t* __restri
 // ---------------------------------------------------------------- grouped native experts
 constexpr int GU_ROWS = 8;     // rows per block (one warp each)
 
+// Group g is computed by block row g % gridDim.y: rows y, y + gridDim.y, ... < *n_groups.  The callers size the
+// launch for the most groups a call can have (cap: every entry its own expert), but the count is only known on the
+// device, and a verify window's PCIe call usually has no group at all: with one block row per possible group every
+// unused one was cap x (2 n_ff / 8 + n_embd / 8) blocks that started only to return.  A smaller gridDim.y strides
+// instead.  Each group's rows are computed by the same warp code in the same order either way, so the results do
+// not depend on gridDim.y (gridDim.y = cap_groups is the one-row-per-group launch).
 template<int TG>
 __global__ void __launch_bounds__(256) native_gu_kernel(const unsigned long long* __restrict__ grp_ptr,
                                                         const int32_t* __restrict__ grp_start,
@@ -861,20 +899,21 @@ __global__ void __launch_bounds__(256) native_gu_kernel(const unsigned long long
                                                         const int32_t* __restrict__ ent_tok,
                                                         const block_q8_1* __restrict__ xq, NativeExpertLayout L,
                                                         float* __restrict__ gate, float* __restrict__ up) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int row = blockIdx.x * GU_ROWS + warp;             // 0 .. 2*n_ff
     if (row >= 2 * L.n_ff) return;
     const bool is_up = row >= L.n_ff;
     const int r = is_up ? row - (int) L.n_ff : row;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
+    const size_t off = (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
     const int nb = (int) (L.n_embd / Fmt<TG>::qk), xb = (int) (L.n_embd / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
-    for (int e = e0; e < e1; ++e) {
-        const float s = row_dot<TG>(wr, xq + (size_t) ent_tok[e] * xb, nb, lane);
-        if (lane == 0) (is_up ? up : gate)[(size_t) e * L.n_ff + r] = s;
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; ++e) {
+            const float s = row_dot<TG>(wr, xq + (size_t) ent_tok[e] * xb, nb, lane);
+            if (lane == 0) (is_up ? up : gate)[(size_t) e * L.n_ff + r] = s;
+        }
     }
 }
 
@@ -890,28 +929,29 @@ __global__ void __launch_bounds__(256) native_gu_multi_kernel(const unsigned lon
                                                               const int32_t* __restrict__ ent_tok,
                                                               const block_q8_1* __restrict__ xq, NativeExpertLayout L,
                                                               float* __restrict__ gate, float* __restrict__ up) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int row = blockIdx.x * GU_ROWS + warp;             // 0 .. 2*n_ff
     if (row >= 2 * L.n_ff) return;
     const bool is_up = row >= L.n_ff;
     const int r = is_up ? row - (int) L.n_ff : row;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
+    const size_t w_off = (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
     const int nb = (int) (L.n_embd / Fmt<TG>::qk), xb = (int) (L.n_embd / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
     float* dst = is_up ? up : gate;
-    for (int e = e0; e < e1; e += GRP_NC) {
-        const int n = min(GRP_NC, e1 - e);
-        int off[GRP_NC];
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {   // the group stride, as native_gu_kernel
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + w_off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; e += GRP_NC) {
+            const int n = min(GRP_NC, e1 - e);
+            int off[GRP_NC];
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c) off[c] = ent_tok[e + min(c, n - 1)] * xb;
-        float s[GRP_NC];
-        row_dot_multi<TG, GRP_NC>(wr, xq, off, n, nb, lane, s);
+            for (int c = 0; c < GRP_NC; ++c) off[c] = ent_tok[e + min(c, n - 1)] * xb;
+            float s[GRP_NC];
+            row_dot_multi<TG, GRP_NC>(wr, xq, off, n, nb, lane, s);
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c)
-            if (c < n && lane == c) dst[(size_t) (e + c) * L.n_ff + r] = s[c];
+            for (int c = 0; c < GRP_NC; ++c)
+                if (c < n && lane == c) dst[(size_t) (e + c) * L.n_ff + r] = s[c];
+        }
     }
 }
 
@@ -930,18 +970,19 @@ __global__ void __launch_bounds__(256) native_down_kernel(const unsigned long lo
                                                           const int32_t* __restrict__ ent_dst,
                                                           const block_q8_1* __restrict__ hq, NativeExpertLayout L,
                                                           float* __restrict__ out) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int r = blockIdx.x * 8 + warp;
     if (r >= L.n_embd) return;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + L.down_off + (size_t) r * L.d_row;
+    const size_t off = L.down_off + (size_t) r * L.d_row;
     const int nb = (int) (L.n_ff / Fmt<TD>::qk), hb = (int) (L.n_ff / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
-    for (int e = e0; e < e1; ++e) {
-        const float s = row_dot<TD>(wr, hq + (size_t) e * hb, nb, lane);
-        if (lane == 0) out[(size_t) ent_dst[e] * L.n_embd + r] = s;
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {   // as native_gu_kernel
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; ++e) {
+            const float s = row_dot<TD>(wr, hq + (size_t) e * hb, nb, lane);
+            if (lane == 0) out[(size_t) ent_dst[e] * L.n_embd + r] = s;
+        }
     }
 }
 
@@ -953,33 +994,32 @@ __global__ void __launch_bounds__(256) native_down_multi_kernel(const unsigned l
                                                                 const int32_t* __restrict__ ent_dst,
                                                                 const block_q8_1* __restrict__ hq, NativeExpertLayout L,
                                                                 float* __restrict__ out) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int r = blockIdx.x * 8 + warp;
     if (r >= L.n_embd) return;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + L.down_off + (size_t) r * L.d_row;
+    const size_t w_off = L.down_off + (size_t) r * L.d_row;
     const int nb = (int) (L.n_ff / Fmt<TD>::qk), hb = (int) (L.n_ff / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
-    for (int e = e0; e < e1; e += GRP_NC) {
-        const int n = min(GRP_NC, e1 - e);
-        int off[GRP_NC];
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {   // the group stride, as native_gu_kernel
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + w_off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; e += GRP_NC) {
+            const int n = min(GRP_NC, e1 - e);
+            int off[GRP_NC];
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c) off[c] = (e + min(c, n - 1)) * hb;
-        float s[GRP_NC];
-        row_dot_multi<TD, GRP_NC>(wr, hq, off, n, nb, lane, s);
+            for (int c = 0; c < GRP_NC; ++c) off[c] = (e + min(c, n - 1)) * hb;
+            float s[GRP_NC];
+            row_dot_multi<TD, GRP_NC>(wr, hq, off, n, nb, lane, s);
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c)
-            if (c < n && lane == c) out[(size_t) ent_dst[e + c] * L.n_embd + r] = s[c];
+            for (int c = 0; c < GRP_NC; ++c)
+                if (c < n && lane == c) out[(size_t) ent_dst[e + c] * L.n_embd + r] = s[c];
+        }
     }
 }
 
 // ---------------------------------------------------------------- q8_1 (quantize.cu)
-__global__ void quantize_q8_1_kernel(const float* __restrict__ x, block_q8_1* __restrict__ y, long long n) {
-    const long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x;
-    if (i >= n) return;
-    const float xi = x[i];
+// value i of a q8_1 row set (the warp holds block i / 32, lane = i % 32)
+__device__ __forceinline__ void q8_1_store(const float xi, block_q8_1* __restrict__ y, const long long i) {
     float amax = fabsf(xi), sum = xi;
 #pragma unroll
     for (int o = 16; o > 0; o >>= 1) {
@@ -993,6 +1033,40 @@ __global__ void quantize_q8_1_kernel(const float* __restrict__ x, block_q8_1* __
     if (iqs == 0) y[ib].ds = make_half2(d, sum);
 }
 
+__global__ void quantize_q8_1_kernel(const float* __restrict__ x, block_q8_1* __restrict__ y, long long n) {
+    const long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= n) return;
+    q8_1_store(x[i], y, i);
+}
+
+// swiglu_entries_kernel and quantize_q8_1_kernel in one pass, over the call's own entries only:
+// [grp_start[0], grp_start[*n_groups]) (a call's entries are contiguous; the verify window's PCIe call starts after
+// its VRAM call) instead of all cap_entries twice - nothing reads the others: the down kernel reads its groups'.
+// n_ff is a multiple of 32, so a warp is one q8_1 block and the bounds are warp-uniform.  The values are the two
+// kernels': the SwiGLU product is rounded on its own - it must not contract into the first add of q8_1_store's block
+// sum, as it could not when it went through memory (CUDA: __fmul_rn; HIP's __fmul_rn is a plain product, so there
+// the product is written here under contract(off)) - then q8_1_store unchanged: the same blocks, bit for bit.
+__global__ void __launch_bounds__(256) swiglu_q8_1_entries_kernel(const float* __restrict__ gate,
+                                                                  const float* __restrict__ up,
+                                                                  const int32_t* __restrict__ grp_start,
+                                                                  const int32_t* __restrict__ n_groups, int n_ff,
+                                                                  block_q8_1* __restrict__ hq) {
+#if defined(__HIPCC__)
+#pragma clang fp contract(off)
+#endif
+    const long long lo = (long long) grp_start[0] * n_ff, hi = (long long) grp_start[*n_groups] * n_ff;
+    for (long long i = lo + (long long) blockIdx.x * blockDim.x + threadIdx.x; i < hi;
+         i += (long long) gridDim.x * blockDim.x) {
+        const float g = gate[i];
+#if defined(__HIPCC__)
+        const float h = (g / (1.0f + __expf(-g))) * up[i];
+#else
+        const float h = __fmul_rn(g / (1.0f + __expf(-g)), up[i]);
+#endif
+        q8_1_store(h, hq, i);
+    }
+}
+
 // ---------------------------------------------------------------- dequant (dequantize.cuh)
 template<typename dst_t> __device__ __forceinline__ dst_t cvt(float v);
 template<> __device__ __forceinline__ float cvt<float>(float v) { return v; }
@@ -1194,6 +1268,23 @@ __device__ void dq_q5_k(const void* vx, int64_t ibs, dst_t* yy, int tid) {
     }
 }
 template<typename dst_t>
+__device__ void dq_q5_0(const void* vx, int64_t ibs, dst_t* yy, int tid) {
+    // symmetric: one delta (no min), the high bit folded in then centered by the format's fixed -16 offset
+    const block_q5_0* x = (const block_q5_0*) vx + ibs * (QK_K / QK5_0);
+    const int ib = tid % 8, il = tid / 8;
+    const float d = __half2float(x[ib].d);
+    uint32_t qh;
+    memcpy(&qh, x[ib].qh, sizeof(qh));
+    dst_t* y = yy + 32 * ib;
+    for (int j = 0; j < 4; ++j) {
+        const int iqs = 4 * il + j;
+        const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
+        const int xh_1 = ((qh >> (iqs + 12))) & 0x10;
+        y[iqs] = cvt<dst_t>(((float) ((x[ib].qs[iqs] & 0xf) | xh_0) - 16.0f) * d);
+        y[iqs + 16] = cvt<dst_t>(((float) ((x[ib].qs[iqs] >> 4) | xh_1) - 16.0f) * d);
+    }
+}
+template<typename dst_t>
 __device__ void dq_q5_1(const void* vx, int64_t ibs, dst_t* yy, int tid) {
     const block_q5_1* x = (const block_q5_1*) vx + ibs * (QK_K / QK5_1);
     const int ib = tid % 8, il = tid / 8;
@@ -1243,6 +1334,7 @@ __device__ __forceinline__ void dq_dispatch(int ty, const void* vx, int64_t ibs,
         case 12: dq_q4_k(vx, ibs, y, tid); break;
         case 13: dq_q5_k(vx, ibs, y, tid); break;
         case 7: dq_q5_1(vx, ibs, y, tid); break;
+        case 6: dq_q5_0(vx, ibs, y, tid); break;
         case 8: dq_q8_0(vx, ibs, y, tid); break;
         case 30: dq_bf16(vx, ibs, y, tid); break;
         default: break;
@@ -1267,7 +1359,7 @@ __global__ void dequant_gu_kernel(int ty, const void* __restrict__ gate, const v
 // the types dq_dispatch dequantizes
 bool is_iq(int t) {
     return t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||
-           t == 12 || t == 13 || t == 7 || t == 8;
+           t == 12 || t == 13 || t == 7 || t == 6 || t == 8;
 }
 // values per block of the types the grouped expert kernels take (0 = none)
 int gu_qk(int t) {
@@ -1347,6 +1439,7 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {
         case 12: return (size_t) (n / 256) * sizeof(block_q4_K);
         case 13: return (size_t) (n / 256) * sizeof(block_q5_K);
         case 7: return (size_t) (n / 32) * sizeof(block_q5_1);
+        case 6: return (size_t) (n / 32) * sizeof(block_q5_0);
         case 8: return (size_t) (n / 32) * sizeof(block_q8_0);
         case 30: return (size_t) n * 2;   // BF16: the token embedding only (iq_embed_rows, iq_dequant_f32)
         default: return 0;
@@ -1439,10 +1532,18 @@ size_t native_expert_scratch_bytes(int64_t cap, int64_t n_ff) {
     return 3 * ((f + 255) & ~(size_t) 255) + (((size_t) cap * (size_t) (n_ff / 32) * sizeof(block_q8_1) + 255) & ~(size_t) 255);
 }
 
+namespace {
+bool g_grouped_v1 = env_on("STRATA_GROUPED_V1");
+}  // namespace
+
+void native_grouped_set_v1(bool v1) { g_grouped_v1 = v1; }
+
 void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long* grp_ptr, const int32_t* grp_start,
                            const int32_t* n_groups, const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups,
-                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream) {
+                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream,
+                           int64_t grid_groups) {
     if (cap_groups <= 0 || cap_entries <= 0) return;
+    if (L.n_ff % 32 != 0) { std::fprintf(stderr, "native_expert_grouped: n_ff %lld\n", (long long) L.n_ff); std::exit(1); }
     cudaStream_t s = (cudaStream_t) stream;
     const size_t f = (size_t) cap_entries * (size_t) L.n_ff * sizeof(float), fa = (f + 255) & ~(size_t) 255;
     float* gate = (float*) scratch;
@@ -1450,7 +1551,9 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
     float* h = (float*) ((uint8_t*) scratch + 2 * fa);
     block_q8_1* hq = (block_q8_1*) ((uint8_t*) scratch + 3 * fa);
     const auto* X = (const block_q8_1*) x_q8_1;
-    const dim3 ggu((unsigned) ((2 * L.n_ff + GU_ROWS - 1) / GU_ROWS), (unsigned) cap_groups);
+    const bool v1 = g_grouped_v1;
+    const int64_t gy = (v1 || grid_groups <= 0 || grid_groups > cap_groups) ? cap_groups : grid_groups;
+    const dim3 ggu((unsigned) ((2 * L.n_ff + GU_ROWS - 1) / GU_ROWS), (unsigned) gy);
     switch (L.gu_type) {
 #define STRATA_GU(T) case T: launch_gu<T>(ggu, s, grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up); break;
         STRATA_GU_FMTS(STRATA_GU)
@@ -1459,9 +1562,15 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
     }
     check("native_expert_grouped/gu");
     const long long nh = (long long) cap_entries * L.n_ff;
-    swiglu_entries_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(gate, up, h, nh);
-    quantize_q8_1_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(h, hq, nh);
-    const dim3 gd((unsigned) ((L.n_embd + 7) / 8), (unsigned) cap_groups);
+    if (v1) {
+        swiglu_entries_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(gate, up, h, nh);
+        quantize_q8_1_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(h, hq, nh);
+    } else {
+        swiglu_q8_1_entries_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(gate, up, grp_start, n_groups,
+                                                                                (int) L.n_ff, hq);
+    }
+    check("native_expert_grouped/swiglu");
+    const dim3 gd((unsigned) ((L.n_embd + 7) / 8), (unsigned) gy);
     switch (L.d_type) {
 #define STRATA_DOWN(T) case T: launch_down<T>(gd, s, grp_ptr, grp_start, n_groups, ent_dst, hq, L, out); break;
         STRATA_D_FMTS(STRATA_DOWN)

src/kernels/cuda/qsa_prompt_attn.cu

@@ -78,18 +78,22 @@ __device__ __forceinline__ uint32_t pack_h2(float lo_k, float hi_k) {   // eleme
 // KV_MODE 1: int8 codes + fp16 scale per 64 values. KV_MODE 0: fp16 values (scales 1).
 // KV_MODE 3 (hybrid K8V4): K as mode 1, V as mode 0 - the row's q4_0 blocks are dequantized to fp16 at
 // gather, so everything downstream of the load is the mode-0 V path; the caller un-rotates the output.
+// KV_MODE 4 (Q4_0 K and V, `--kv q4_0`): mode 1 with a scale per 32 values - each q4_0 block's codes enter as exact
+// int8 (code - 8) and its fp16 scale is applied in FP32, as mode 1's are; the caller rotates q and un-rotates the
+// output (kv_q4.hpp).
 template <int KV_MODE>
 struct Smem {
     using KElem = typename std::conditional<KV_MODE == 0, __half, int8_t>::type;
-    using VElem = typename std::conditional<KV_MODE == 1, int8_t, __half>::type;
+    using VElem = typename std::conditional<KV_MODE == 1 || KV_MODE == 4, int8_t, __half>::type;
     static constexpr int KROW = KV_MODE == 0 ? HD + 8 : HD + 16;   // elements; 16-byte aligned rows, banks spread
-    static constexpr int VROW = KV_MODE == 1 ? HD + 16 : HD + 8;
+    static constexpr int VROW = (KV_MODE == 1 || KV_MODE == 4) ? HD + 16 : HD + 8;
+    static constexpr int NG = KV_MODE == 4 ? HD / QK4_0 : 4;      // scale groups per row (q4_0: 8 of 32, else 4 of 64)
     __half qh[16][QS];
     __half ql[16][QS];
     KElem k[CH][KROW];
     VElem v[CH][VROW];
-    float ks[CH][4];
-    float vs[CH][4];
+    float ks[CH][NG];
+    float vs[CH][NG];
     float s[16][CH + 1];
     float qmax[THREADS / 32];
     float alpha[16];
@@ -154,7 +158,39 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
         __syncthreads();   // rows ready; the previous chunk's p.v is done with k, v, s
         // gather the chunk's K and V rows (16-byte pieces; K8V4's V as q4_0 blocks dequantized to fp16)
         // and their scales
-        {
+        if constexpr (KV_MODE == 4) {
+            // q4_0 K and V: one (K or V, cell, block) per item - the block's 16 code bytes (2-byte aligned: eight
+            // 16-bit loads) become 32 exact int8 codes (element j in the low nibble of byte j, j + 16 in the high
+            // one), its fp16 scale goes to ks/vs
+            constexpr int BLKS = HD / QK4_0;
+            constexpr int BYTES = BLKS * (int) sizeof(block_q4_0);
+            for (int i = t; i < 2 * CH * BLKS; i += THREADS) {
+                const int kv = i / (CH * BLKS), rem = i % (CH * BLKS), c = rem / BLKS, b = rem % BLKS;
+                const long long r = S.row[c];
+                uint4 lo = make_uint4(0, 0, 0, 0), hi = make_uint4(0, 0, 0, 0);
+                float d = 0.0f;
+                if (r >= 0) {
+                    const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((kv == 0 ? p.k_q4 : p.v_q4) + r * BYTES) + b;
+                    d = __half2float(__ushort_as_half(__ldg(&blk->d)));
+                    const uint16_t* q16 = reinterpret_cast<const uint16_t*>(blk->qs);
+                    uint32_t w[4];
+#pragma unroll
+                    for (int j = 0; j < 4; ++j)
+                        w[j] = (uint32_t) __ldg(q16 + 2 * j) | ((uint32_t) __ldg(q16 + 2 * j + 1) << 16);
+                    // per byte: the low nibble minus 8 is element j, the high one minus 8 element j + 16
+                    lo = make_uint4(__vsub4(w[0] & 0x0F0F0F0Fu, 0x08080808u), __vsub4(w[1] & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4(w[2] & 0x0F0F0F0Fu, 0x08080808u), __vsub4(w[3] & 0x0F0F0F0Fu, 0x08080808u));
+                    hi = make_uint4(__vsub4((w[0] >> 4) & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4((w[1] >> 4) & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4((w[2] >> 4) & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4((w[3] >> 4) & 0x0F0F0F0Fu, 0x08080808u));
+                }
+                int8_t* dst = kv == 0 ? &S.k[c][b * QK4_0] : &S.v[c][b * QK4_0];
+                reinterpret_cast<uint4*>(dst)[0] = lo;
+                reinterpret_cast<uint4*>(dst)[1] = hi;
+                (kv == 0 ? S.ks : S.vs)[c][b] = d;
+            }
+        } else {
             constexpr int KPIECES = HD * (int) sizeof(typename Smem<KV_MODE>::KElem) / 16;   // per K row
             for (int i = t; i < CH * KPIECES; i += THREADS) {
                 const int c = i / KPIECES, pc = i % KPIECES;
@@ -222,17 +258,18 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             }
         }
         __syncthreads();
-        // scores: warp w takes cells 8w..8w+7 (one n-tile) over all 256 dims, per 64-dim scale group
+        // scores: warp w takes cells 8w..8w+7 (one n-tile) over all 256 dims, per scale group (64 dims; q4_0's 32)
+        constexpr int NG = Smem<KV_MODE>::NG, KPG = HD / 16 / NG;   // groups per row, 16-dim MMA steps per group
 #pragma unroll
         for (int nt = 0; nt < CH / 32; ++nt) {
             const int cb = (warp + 4 * nt) * 8;
             float sc[4] = {0.f, 0.f, 0.f, 0.f};
 #pragma unroll
-            for (int g = 0; g < 4; ++g) {
+            for (int g = 0; g < NG; ++g) {
                 float tg[4] = {0.f, 0.f, 0.f, 0.f};
 #pragma unroll
-                for (int kk = 0; kk < 4; ++kk) {
-                    const int k0 = (g * 4 + kk) * 16;
+                for (int kk = 0; kk < KPG; ++kk) {
+                    const int k0 = (g * KPG + kk) * 16;
                     uint32_t ah[4], al[4], b[2];
                     ah[0] = *reinterpret_cast<const uint32_t*>(&S.qh[gid][k0 + 2 * tig]);
                     ah[1] = *reinterpret_cast<const uint32_t*>(&S.qh[gid + 8][k0 + 2 * tig]);
@@ -296,24 +333,31 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             }
         }
         __syncthreads();
-        // p.v: warp w owns dims [64w, 64w+64), which is int8 scale group w. The scale is folded into p relative to
-        // the chunk's largest, times 2^14 (p' <= 2^14: its lo half stays out of FP16's subnormal range); the chunk's
-        // sum is then added to the running one in FP32 with the factor taken back out
+        // p.v: warp w owns dims [64w, 64w+64), which is int8 scale group w (q4_0: groups 2w and 2w+1, four n-tiles
+        // each). A group's scale is folded into p relative to the chunk's largest magnitude (q4_0's scales are signed:
+        // ggml's d = max / -8), times 2^14 (|p'| <= 2^14: inside FP16's range, its lo half out of the subnormals); the
+        // chunk's sum is then added to the running one in FP32 with the factor taken back out
         {
-            float vmax = 0.0f;
-#pragma unroll
-            for (int c = lane; c < CH; c += 32) vmax = fmaxf(vmax, S.vs[c][warp]);
-#pragma unroll
-            for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));
-            const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f, vdown = vmax * (1.0f / 16384.0f);
+            constexpr int GPW = NG / 4, JPG = 8 / GPW;   // scale groups per warp, n-tiles per group
+            float vdown_g[GPW];
             float tmp[8][4];
 #pragma unroll
             for (int j = 0; j < 8; ++j) tmp[j][0] = tmp[j][1] = tmp[j][2] = tmp[j][3] = 0.0f;
+#pragma unroll
+            for (int gi = 0; gi < GPW; ++gi) {
+            const int vg = warp * GPW + gi;
+            float vmax = 0.0f;
+#pragma unroll
+            for (int c = lane; c < CH; c += 32) vmax = fmaxf(vmax, fabsf(S.vs[c][vg]));
+#pragma unroll
+            for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));
+            const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f;
+            vdown_g[gi] = vmax * (1.0f / 16384.0f);
 #pragma unroll
             for (int ks = 0; ks < CH / 16; ++ks) {
                 const int cA = ks * 16 + 2 * tig, cB = cA + 8;
-                const float w0 = S.vs[cA][warp] * vup, w1 = S.vs[cA + 1][warp] * vup, w2 = S.vs[cB][warp] * vup,
-                            w3 = S.vs[cB + 1][warp] * vup;
+                const float w0 = S.vs[cA][vg] * vup, w1 = S.vs[cA + 1][vg] * vup, w2 = S.vs[cB][vg] * vup,
+                            w3 = S.vs[cB + 1][vg] * vup;
                 const float p00 = S.s[gid][cA] * w0, p01 = S.s[gid][cA + 1] * w1;
                 const float p10 = S.s[gid + 8][cA] * w0, p11 = S.s[gid + 8][cA + 1] * w1;
                 const float p02 = S.s[gid][cB] * w2, p03 = S.s[gid][cB + 1] * w3;
@@ -332,10 +376,11 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
                     f = __half22float2(h[3]); al[3] = pack_h2(p12 - f.x, p13 - f.y);
                 }
 #pragma unroll
-                for (int j = 0; j < 8; ++j) {
+                for (int jj = 0; jj < JPG; ++jj) {
+                    const int j = gi * JPG + jj;
                     const int d = warp * 64 + j * 8 + gid;
                     uint32_t b[2];
-                    if constexpr (KV_MODE == 1) {
+                    if constexpr (KV_MODE == 1 || KV_MODE == 4) {
                         const uint32_t x0 = (uint8_t) S.v[cA][d] | ((uint32_t) (uint8_t) S.v[cA + 1][d] << 8);
                         const uint32_t x1 = (uint8_t) S.v[cB][d] | ((uint32_t) (uint8_t) S.v[cB + 1][d] << 8);
                         b[0] = i8x2_to_h2(x0);
@@ -352,9 +397,11 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
 #endif
                 }
             }
+            }
             const float a0 = S.alpha[gid], a1 = S.alpha[gid + 8];
 #pragma unroll
             for (int j = 0; j < 8; ++j) {
+                const float vdown = vdown_g[j / JPG];
                 acc[j][0] = fmaf(acc[j][0], a0, tmp[j][0] * vdown);
                 acc[j][1] = fmaf(acc[j][1], a0, tmp[j][1] * vdown);
                 acc[j][2] = fmaf(acc[j][2], a1, tmp[j][2] * vdown);
@@ -979,21 +1026,24 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3
     {   // sm_75 or newer: the MMA above compiles for both.  sm_80+ runs the cp.async kernel (launch_i8); Turing has
         // no cp.async, so it runs the v1 kernel (launch<1>, same accuracy, another summation order).  An older card
         // keeps the old kernel.
-        static int cc_major[64] = {};
+        // #371: the compute capability with its minor - sm_70 (V100) has no m16n8k8 (the kernels trap below sm_75)
+        static int cc[64] = {};
         int dev = 0;
         if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
-        if (cc_major[dev] == 0) {
-            int major = 0;
-            if (cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess) {
+        if (cc[dev] == 0) {
+            int major = 0, minor = 0;
+            if (cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess ||
+                cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev) != cudaSuccess) {
                 cudaGetLastError();
                 return false;
             }
             // STRATA_QSA_WARP=1|attn (an A/B arm): the pre-sm_80 kernels on any card, as RTX 20 runs them
             const char* w = std::getenv("STRATA_QSA_WARP");
-            cc_major[dev] = w && (!std::strcmp(w, "1") || !std::strcmp(w, "attn")) ? 7 : strata::cc_major_of(major);
+            cc[dev] = w && (!std::strcmp(w, "1") || !std::strcmp(w, "attn")) ? 75
+                      : 10 * strata::cc_major_of(major) + strata::cc_minor_of(minor);
         }
-        if (cc_major[dev] < 7) return false;
-        turing = cc_major[dev] < 8;
+        if (cc[dev] < 75) return false;
+        turing = cc[dev] < 80;
     }
 #if defined(__HIPCC__)
     // the tensor-core kernels are compiled out on AMD (its major version is not a CUDA sm); RDNA4 has its own int8-KV
@@ -1005,10 +1055,18 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3
         return launch_wmma(q, pools, ids, steps, cap, s, attn, n_q, (cudaStream_t) stream);
     return false;
 #endif
-    if (pools.k_q4 != nullptr || s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !ids ||
-        !steps || !pools.page_table)
+    if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !ids || !steps || !pools.page_table)
         return false;
     cudaStream_t st = (cudaStream_t) stream;
+    if (pools.k_q4 != nullptr) {   // Q4_0 K and V (--kv q4_0): mode 4.  STRATA_PROMPT_ATTN_Q4=0: the old kernel (A/B)
+        static const bool q4_off = [] {
+            const char* v = std::getenv("STRATA_PROMPT_ATTN_Q4");
+            return v != nullptr && v[0] == '0';
+        }();
+        // sm_80+ only: on Turing mode 4 would run as pairs of m16n8k8 MMAs, which no parity run has checked yet
+        if (q4_off || turing || pools.v_q4 == nullptr) return false;
+        return launch<4>(q, pools, ids, steps, cap, s, attn, n_q, st);
+    }
     if (pools.k_q != nullptr && pools.v_q4 != nullptr) {   // hybrid K8V4: int8 K + dequantized-q4 V
         if (!pools.k_scale) return false;
         return launch<3>(q, pools, ids, steps, cap, s, attn, n_q, st);

src/kernels/cuda/qsa_select.cu

@@ -644,6 +644,229 @@ __global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(co
         }
     }
 }
+
+#if !defined(__HIPCC__)
+// ---- the decode top-k on a thread-block cluster (sm_90+; S19).  One CTA per query (block_topk_reg_kernel, or
+// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to
+// 65,538 blocks on ONE SM while the rest of the GPU idles - a decode window has 1-5 queries.  Here a cluster of CL_N
+// CTAs shares a query: CTA r holds the keys of blocks [r * per, (r + 1) * per) in shared memory, builds the digit
+// histogram of its keys, and PUSHES it into slot r of every CTA's `hin` (distributed shared memory); after one cluster
+// barrier each CTA sums the CL_N slots itself, so all of them take the same digit.  The emit pass needs each CTA's
+// cells above / at the threshold: pushed the same way, then each CTA offsets its own by the ranks before it.
+// RTX 5070, per call at capacity = context (decode_cluster_parity --bench): 21.9 -> 15.6 us at 32K, 58 -> 18 at 128K,
+// 200 -> 22 at 262K (the one-CTA kernels' digit search was serial, too: one thread over 256 bins per pass).
+//
+// IDENTICAL IDS, by construction rather than by luck: the threshold thr (the width-th largest key, cells counted with
+// their weights) and `above` (cells with a larger key) are pure functions of the multiset of (key, weight) - integer
+// histograms, summed in any order - and the digit rule is block_topk_kernel's, written as a scan (the largest digit d
+// whose cells at or above it reach `need`; 0 when none). The cells emitted are then fixed: every cell of a block with
+// key > thr, and the first eq_budget = width - above cells at thr in ascending order (a block at thr may be cut),
+// written in ascending cell order. Position of a thread's first cell = (cells above thr before it) + min(cells at thr
+// before it, eq_budget): the telescoped sum of the reference's per-thread clamp. NaN keys are 0 (order_key) as there.
+//
+// Barriers: barrier.cluster arrive / wait (each CTA's threads all take part; it also orders the CTA's own shared
+// memory, so it doubles as __syncthreads). Phase 0 is a relaxed arrive at entry, waited before the first remote store
+// (a CTA's shared memory exists once it runs). Pushes go out before the arrive that releases them, and nothing reads
+// another CTA's memory after the last wait - so a CTA may exit without a closing barrier (a wait counts the threads
+// that have not exited). `hin` alternates by pass parity: CTA x reads slot buffer p&1 in pass p before arriving at
+// pass p+1's barrier, and nobody writes buffer p&1 again (pass p+2) before waiting on that barrier.
+constexpr int CL_N = 8;        // CTAs per query (the portable cluster size)
+constexpr int CL_T = 1024;     // threads per CTA
+constexpr int CL_MAXQ = 16;    // larger calls (prefill sub-batches) fill the GPU with one CTA per query already
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
+#define STRATA_SEL_CLUSTER 1
+#else
+#define STRATA_SEL_CLUSTER 0   // an older target's code is a trap; qsa_block_topk_cluster never launches it there
+#endif
+
+#if STRATA_SEL_CLUSTER
+__device__ __forceinline__ void cl_arrive_relaxed() {
+    asm volatile("barrier.cluster.arrive.relaxed.aligned;\n" ::: "memory");
+}
+__device__ __forceinline__ void cl_arrive() { asm volatile("barrier.cluster.arrive.release.aligned;\n" ::: "memory"); }
+__device__ __forceinline__ void cl_wait() { asm volatile("barrier.cluster.wait.acquire.aligned;\n" ::: "memory"); }
+__device__ __forceinline__ unsigned cl_rank() {
+    unsigned r;
+    asm volatile("mov.u32 %0, %%cluster_ctarank;\n" : "=r"(r));
+    return r;
+}
+template <typename T> __device__ __forceinline__ T* cl_map(T* p, unsigned rank) {   // p in CTA `rank`'s shared memory
+    uint64_t o;
+    asm volatile("mapa.u64 %0, %1, %2;\n" : "=l"(o) : "l"((uint64_t) p), "r"(rank));
+    return reinterpret_cast<T*>(o);
+}
+// exclusive block scan of a 64-bit value over CL_T threads; `total` = the CTA's sum
+__device__ __forceinline__ unsigned long long cl_excl_scan64(unsigned long long v, unsigned long long* s_warp,
+                                                             unsigned long long& total) {
+    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+    unsigned long long x = v;
+#pragma unroll
+    for (int o = 1; o < 32; o <<= 1) {
+        const unsigned long long y = __shfl_up_sync(0xffffffffu, x, o);
+        if (lane >= o) x += y;
+    }
+    if (lane == 31) s_warp[warp] = x;
+    __syncthreads();
+    if (warp == 0) {
+        const unsigned long long w = s_warp[lane];
+        unsigned long long z = w;
+#pragma unroll
+        for (int o = 1; o < 32; o <<= 1) {
+            const unsigned long long y = __shfl_up_sync(0xffffffffu, z, o);
+            if (lane >= o) z += y;
+        }
+        s_warp[lane] = z - w;
+        if (lane == 31) s_warp[32] = z;
+    }
+    __syncthreads();
+    total = s_warp[32];
+    return s_warp[warp] + x - v;
+}
+#endif
+
+// grid (CL_N, nq), cluster (CL_N, 1, 1), CL_T threads, dynamic shared memory: ceil(max_blocks / CL_N) keys
+__global__ void __launch_bounds__(CL_T) block_topk_cluster_kernel(const float* __restrict__ scores,
+                                                                  const int32_t* __restrict__ steps, int64_t max_blocks,
+                                                                  int64_t cap, int32_t* __restrict__ ids) {
+#if STRATA_SEL_CLUSTER
+    extern __shared__ uint32_t keys[];                  // this CTA's blocks' keys
+    __shared__ __align__(16) int hin[2][CL_N][256];     // the cluster's histograms, slot = the pushing rank
+    __shared__ __align__(16) int hloc[2][256];          // this CTA's histogram
+    __shared__ unsigned long long cnt[CL_N];            // per rank: its cells above thr (low 32 bits) and at it (high)
+    __shared__ unsigned long long s_warp[33];
+    __shared__ uint32_t s_prefix;
+    __shared__ int s_above;
+    const unsigned rank = cl_rank();
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+    cl_arrive_relaxed();                                // phase 0: this CTA runs
+    const int64_t qi = blockIdx.y;
+    const int32_t* st = steps + qi * kStepCount;
+    const int64_t n_kv = st[kStepNKv], n_bid = st[kStepNBid], width = st[kStepWidth];
+    int32_t* out = ids + qi * cap;
+    if (n_kv <= width) {                                // the identity, split over the cluster; no remote access
+        for (int64_t j = (int64_t) rank * CL_T + t; j < n_kv; j += (int64_t) CL_N * CL_T) out[j] = (int32_t) j;
+        return;
+    }
+    // the tail block n_bid weighs its cells (0..3); with none it is not a candidate (the reference skips w == 0)
+    const int wtail = (int) (n_kv - n_bid * R);
+    const int64_t nbe = n_bid + (wtail > 0 ? 1 : 0);
+    const int64_t per = (nbe + CL_N - 1) / CL_N;       // <= ceil(max_blocks / CL_N), the keys the host sized
+    const int64_t lo = (int64_t) rank * per < nbe ? (int64_t) rank * per : nbe;
+    const int64_t hi = lo + per < nbe ? lo + per : nbe;
+    const int n = (int) (hi - lo);
+    const int tail_i = (wtail > 0 && n_bid >= lo && n_bid < hi) ? (int) (n_bid - lo) : -1;
+    const float* sc = scores + qi * max_blocks + lo;
+    for (int i = t; i < n; i += CL_T) keys[i] = order_key(sc[i]);
+    for (int i = t; i < 2 * 256; i += CL_T) (&hloc[0][0])[i] = 0;
+    __syncthreads();
+    uint32_t prefix = 0;
+    int above = 0;                                      // cells strictly above the digits fixed so far
+    for (int pass = 0; pass < 4; ++pass) {
+        const int shift = 24 - 8 * pass;
+        const uint32_t hmask = pass == 0 ? 0u : (0xffffffffu << (shift + 8));
+        int* hl = hloc[pass & 1];
+        // warp-aggregated: the scores share their top bits, so plain atomics would queue on a few bins
+        for (int i0 = 0; i0 < n; i0 += CL_T) {
+            const int i = i0 + t;
+            int bin = -1;
+            if (i < n) {
+                const uint32_t k = keys[i];
+                if ((k & hmask) == prefix) bin = (int) ((k >> shift) & 255);
+            }
+            const unsigned same = __match_any_sync(0xffffffffu, bin);
+            if (bin >= 0 && lane == __ffs(same) - 1) atomicAdd(&hl[bin], __popc(same) * R);
+            if (bin >= 0 && i == tail_i) atomicAdd(&hl[bin], wtail - R);
+        }
+        __syncthreads();
+        if (pass == 0) cl_wait();                       // phase 0 done: every CTA's shared memory is there
+        for (int i = t; i < CL_N * 64; i += CL_T) {     // the histogram into slot `rank` of every CTA, as int4
+            const int dst = i >> 6, c = i & 63;
+            *cl_map(reinterpret_cast<int4*>(&hin[pass & 1][rank][0]) + c, (unsigned) dst) =
+                reinterpret_cast<const int4*>(hl)[c];
+        }
+        for (int i = t; i < 256; i += CL_T) hloc[(pass + 1) & 1][i] = 0;
+        cl_arrive();
+        cl_wait();
+        if (warp == 0) {
+            // lane L owns digits 255 - 8L down to 248 - 8L; the digit holding the need-th cell from the top
+            const int need = (int) width - above;
+            int tot[8], part = 0;
+#pragma unroll
+            for (int k = 0; k < 8; ++k) {
+                const int d = 255 - 8 * lane - k;
+                int v = 0;
+#pragma unroll
+                for (int r = 0; r < CL_N; ++r) v += hin[pass & 1][r][d];
+                tot[k] = v;
+                part += v;
+            }
+            int incl = part;
+#pragma unroll
+            for (int o = 1; o < 32; o <<= 1) {
+                const int y = __shfl_up_sync(0xffffffffu, incl, o);
+                if (lane >= o) incl += y;
+            }
+            const int excl = incl - part;
+            const unsigned hit = __ballot_sync(0xffffffffu, excl < need && incl >= need);
+            // no hit cannot happen (the cells total n_kv > width); the reference would then take digit 0
+            const int who = hit ? __ffs(hit) - 1 : 31;
+            if (lane == who) {
+                int acc = excl, k = 0;
+                for (; k < 7; ++k) {
+                    if (acc + tot[k] >= need) break;
+                    acc += tot[k];
+                }
+                s_prefix = prefix | ((uint32_t) (255 - 8 * lane - k) << shift);
+                s_above = above + acc;
+            }
+        }
+        __syncthreads();
+        prefix = s_prefix;
+        above = s_above;
+    }
+    const uint32_t thr = prefix;
+    const int64_t eq_budget = width - above;           // cells at thr that fit, lowest index first
+    // each thread a contiguous run of the CTA's keys; an odd run length keeps the shared reads conflict-free
+    int seg = (n + CL_T - 1) / CL_T;
+    if ((seg & 1) == 0 && seg > 0) ++seg;
+    const int s0 = t * seg < n ? t * seg : n, s1 = s0 + seg < n ? s0 + seg : n;
+    uint32_t gt = 0, eq = 0;
+    for (int i = s0; i < s1; ++i) {
+        const uint32_t k = keys[i], w = i == tail_i ? (uint32_t) wtail : (uint32_t) R;
+        if (k > thr) gt += w;
+        else if (k == thr) eq += w;
+    }
+    unsigned long long cta;
+    const unsigned long long before =
+        cl_excl_scan64((unsigned long long) gt | ((unsigned long long) eq << 32), s_warp, cta);
+    if (t < CL_N) *cl_map(&cnt[rank], (unsigned) t) = cta;
+    cl_arrive();
+    cl_wait();
+    int64_t gt_off = 0, eq_off = 0;
+    for (unsigned r = 0; r < rank; ++r) {
+        gt_off += (int64_t) (cnt[r] & 0xffffffffu);
+        eq_off += (int64_t) (cnt[r] >> 32);
+    }
+    const int64_t eq_before = eq_off + (int64_t) (before >> 32);
+    int64_t wpos = gt_off + (int64_t) (before & 0xffffffffu) + (eq_before < eq_budget ? eq_before : eq_budget);
+    int64_t eq_left = eq_budget - eq_before;
+    if (eq_left < 0) eq_left = 0;
+    for (int i = s0; i < s1; ++i) {
+        const uint32_t k = keys[i];
+        const int w = i == tail_i ? wtail : R;
+        const int64_t b = lo + i;
+        if (k > thr) {
+            for (int c = 0; c < w; ++c) out[wpos++] = (int32_t) (b * R + c);
+        } else if (k == thr) {
+            for (int c = 0; c < w && eq_left > 0; ++c, --eq_left) out[wpos++] = (int32_t) (b * R + c);
+        }
+    }
+#else
+    (void) scores; (void) steps; (void) max_blocks; (void) cap; (void) ids;
+    __trap();
+#endif
+}
+#endif  // !__HIPCC__
 }  // namespace
 
 void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
@@ -745,21 +968,121 @@ void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, i
     if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
 }
 
+#if !defined(__HIPCC__)
+// Only Turing has a retained model measurement for this CUDA dispatch. Other CUDA devices keep the capacity rule
+// (the RTX 5070 regression below). Cache the properties per calling thread; layer-split device switches are checked.
+static bool topk_active_turing_device() {
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess) return false;
+    static thread_local int cached_device = -1;
+    static thread_local bool turing = false;
+    if (dev != cached_device) {
+        cudaDeviceProp prop{};
+        if (cudaGetDeviceProperties(&prop, dev) != cudaSuccess) return false;
+        turing = prop.major == 7 && prop.minor == 5;
+        cached_device = dev;
+    }
+    return turing;
+}
+#endif
+
+bool qsa_block_topk_cluster(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                            const QsaShapes& s, int32_t* ids, void* stream) {
+#if defined(__HIPCC__)
+    (void) scores; (void) steps; (void) nq; (void) max_blocks; (void) cap; (void) s; (void) ids; (void) stream;
+    return false;
+#else
+    if (nq <= 0) return true;
+    if (s.idx_block != R || cap < qsa_selection_width(kTopkMaxCells, s) || nq > 65535 || max_blocks <= 0) return false;
+    const size_t smem = (size_t) ((max_blocks + CL_N - 1) / CL_N) * sizeof(uint32_t);
+    // Per device (a layer split runs on several): 1 the cluster kernel runs here, 2 it does not. It needs sm_90+ (the
+    // card's, or STRATA_EMULATE_CC's) AND code built for it: a build with only older code JIT-compiles their PTX, whose
+    // copy of this kernel is a trap - the function's PTX version says which. `opt`: the dynamic shared memory opted in.
+    static int ok[64] = {};
+    static size_t opt[64] = {};
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+    if (ok[dev] == 0) {
+        int major = 0;
+        cudaFuncAttributes fa{};
+        const bool code = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+                          strata::cc_major_of(major) >= 9 &&
+                          cudaFuncGetAttributes(&fa, block_topk_cluster_kernel) == cudaSuccess && fa.ptxVersion >= 90 &&
+                          fa.binaryVersion >= 90;
+        cudaGetLastError();
+        ok[dev] = code ? 1 : 2;
+    }
+    if (ok[dev] != 1) return false;
+    if (smem > opt[dev]) {   // a larger capacity: opt in, and check that a cluster of CL_N such CTAs can be resident
+        int clusters = 0;
+        cudaLaunchConfig_t q{};
+        cudaLaunchAttribute qa[1];
+        qa[0].id = cudaLaunchAttributeClusterDimension;
+        qa[0].val.clusterDim.x = CL_N;
+        qa[0].val.clusterDim.y = 1;
+        qa[0].val.clusterDim.z = 1;
+        q.gridDim = dim3(CL_N, 1, 1);
+        q.blockDim = dim3(CL_T, 1, 1);
+        q.dynamicSmemBytes = smem;
+        q.attrs = qa;
+        q.numAttrs = 1;
+        if (cudaFuncSetAttribute(block_topk_cluster_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) smem) !=
+                cudaSuccess ||
+            cudaOccupancyMaxActiveClusters(&clusters, block_topk_cluster_kernel, &q) != cudaSuccess || clusters < 1) {
+            cudaGetLastError();
+            return false;    // this capacity takes the one-CTA kernels; a smaller one may still fit
+        }
+        opt[dev] = smem;
+    }
+    cudaLaunchConfig_t cfg{};
+    cudaLaunchAttribute at[1];
+    at[0].id = cudaLaunchAttributeClusterDimension;
+    at[0].val.clusterDim.x = CL_N;
+    at[0].val.clusterDim.y = 1;
+    at[0].val.clusterDim.z = 1;
+    cfg.gridDim = dim3(CL_N, (unsigned) nq, 1);
+    cfg.blockDim = dim3(CL_T, 1, 1);
+    cfg.dynamicSmemBytes = smem;
+    cfg.stream = (cudaStream_t) stream;
+    cfg.attrs = at;
+    cfg.numAttrs = 1;
+    const cudaError_t e = cudaLaunchKernelEx(&cfg, block_topk_cluster_kernel, scores, steps, max_blocks, cap, ids);
+    if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk cluster: %s\n", cudaGetErrorString(e)); std::exit(1); }
+    return true;
+#endif
+}
+
 void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
                     const QsaShapes& s, int32_t* ids, void* stream, int64_t active_blocks) {
     // keys in registers when every query's blocks fit (contexts up to ~135K cells); the same ids. STRATA_TOPK_OLD=1:
     // the kernel that reads them from memory on every pass
     static const bool old = std::getenv("STRATA_TOPK_OLD") != nullptr;
     if (nq <= 0) return;
+#if !defined(__HIPCC__)
+    // sm_90+: a cluster of CL_N CTAs per query for the calls of a few queries (decode windows); the same ids.
+    // STRATA_QSA_CLUSTER=0: the one-CTA kernels below
+    static const bool cluster = [] {
+        const char* v = std::getenv("STRATA_QSA_CLUSTER");
+        return !v || std::atoi(v) != 0;
+    }();
+    if (cluster && !old && nq <= CL_MAXQ && qsa_block_topk_cluster(scores, steps, nq, max_blocks, cap, s, ids, stream))
+        return;
+#endif
     // the blocks a query can have: the call's active count when the caller knows it (the prompt path), else the capacity.
     // Decode (no count) keeps the capacity rule and the original register width: nothing changes there.
 #if defined(__HIPCC__)
     const bool counted = active_blocks > 0;
 #else
-    // CUDA keeps 0.1.32's capacity rule: #337's dispatch was measured on RDNA4 only, and on the RTX 5070 the 64K
-    // prompts read 1-3% slower with it
-    const bool counted = false;
-    (void) active_blocks;
+    // Turing: --max-context 262144 makes the stride 65538, even while a 131K prompt's active blocks fit in
+    // TK_T * TK_PER registers. Use the prefill bound on sm_75, keeping max_blocks as the score-row stride.
+    // Other CUDA devices keep 0.1.32's capacity rule: #337 was measured on RDNA4, and RTX 5070 64K prompts were
+    // 1-3% slower. Decode/captured graphs omit the bound and never query the device here.
+    static const bool capacity_guard = std::getenv("STRATA_TOPK_CAPACITY_GUARD") != nullptr;
+    // STRATA_TOPK_ACTIVE_ANY=1 (tests): the Turing dispatch on any CUDA card, so qsa_topk_active_parity checks it
+    // on whatever card runs the tests (the kernels are the same on every architecture)
+    static const bool any_card = [] { const char* v = std::getenv("STRATA_TOPK_ACTIVE_ANY"); return v && v[0] == '1'; }();
+    const bool counted = !capacity_guard && active_blocks > 0 && active_blocks <= max_blocks &&
+                         (any_card || topk_active_turing_device());
 #endif
     const int64_t reach = counted && active_blocks < max_blocks ? active_blocks : max_blocks;
     const int64_t fit = (int64_t) TK_T * (counted ? TK_PER_MAX : TK_PER);

src/kernels/cuda/sampler.cu

@@ -10,7 +10,8 @@
 // produces a valid token, so only a comparison at the distribution level can tell them apart; the parity test
 // does that against an independently computed distribution.
 //
-// `sampler_greedy_kernel` is the plain argmax, one block per token over the vocabulary.  The sampled chain has
+// `sampler_greedy_kernel` is the plain argmax, one block per token over the vocabulary (on sm_90+ without penalties,
+// `sampler_greedy_cluster_kernel`: the same token from a cluster of 8 CTAs per row).  The sampled chain has
 // three implementations that pick the same token, bit for bit:
 //   - the SPLIT top_k (default): `sampler_split_part_kernel` cuts each row into 4,096-logit blocks over the whole
 //     GPU, each keeps its own top_k, and `sampler_split_merge_kernel` merges those lists and runs the tail;
@@ -20,6 +21,7 @@
 // The two new ones share `sampled_tail_warp` (top_p / min_p / temperature / draw on one warp).
 #include "strata/kernels/sampler.hpp"
 #include "strata/core/coupled_draft.hpp"
+#include "strata/core/emulate.hpp"
 
 #include <cuda_runtime.h>
 
@@ -166,6 +168,96 @@ __global__ void sampler_greedy_kernel(const float* __restrict__ logits, int n_vo
     }
 }
 
+#if !defined(__HIPCC__)
+/// **THE SAME ARGMAX ON A THREAD-BLOCK CLUSTER (sm_90+, S19), FOR THE CALLS WITHOUT PENALTIES.**  One block per
+/// token reads its 1 MB row (248,320 logits) on one SM with one load in flight per thread: latency-bound, 40 us per
+/// call on an RTX 5070, for every verify window and every draft step.  Here a cluster of `kAmCtas` CTAs shares the
+/// row, each thread keeps four loads in flight, and the CTAs' results meet in CTA 0's shared memory (distributed
+/// shared memory): 6 us (decode_cluster_parity --bench).
+///
+/// The answer is a function of the row alone, so it is the one-block kernel's bit for bit: the LOWEST index whose value
+/// is the largest non-NaN value above -inf (each thread walks its elements in ascending order with a strict `>`, and
+/// every merge takes the larger value or, on equality, the smaller index - an order-free rule), and 0 when there is
+/// none (all -inf / NaN), as there.  NaN never wins a `>`.
+///
+/// Barriers: a relaxed cluster arrive at entry, waited before the remote store (CTA 0 must be running); each CTA's
+/// result goes to slot `rank` of CTA 0, released by the next arrive; CTAs 1.. then exit (a cluster wait counts the
+/// threads that have not exited) and CTA 0 waits, reads its own slots, and writes the token.
+constexpr int kAmCtas = 8;      // CTAs per token (the portable cluster size)
+constexpr int kAmThreads = 1024;
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
+#define STRATA_AM_CLUSTER 1
+#else
+#define STRATA_AM_CLUSTER 0     // older targets: a trap, never launched (sample_greedy_cluster checks)
+#endif
+#if STRATA_AM_CLUSTER
+__device__ __forceinline__ void am_take(float s, int v, float& bv, int& best) {
+    if (s > bv) { bv = s; best = v; }
+}
+__device__ __forceinline__ void am_merge(float ov, int oi, float& bv, int& best) {
+    if (ov > bv || (ov == bv && oi < best)) { bv = ov; best = oi; }
+}
+#endif
+// grid (kAmCtas, n_tokens), cluster (kAmCtas, 1, 1), kAmThreads threads
+__global__ void __launch_bounds__(kAmThreads) sampler_greedy_cluster_kernel(const float* __restrict__ logits,
+                                                                            int n_vocab, int* __restrict__ out) {
+#if STRATA_AM_CLUSTER
+    __shared__ float sv[32];
+    __shared__ int si[32];
+    __shared__ float cv[kAmCtas];    // on CTA 0: each CTA's result
+    __shared__ int ci[kAmCtas];
+    unsigned rank;
+    asm volatile("mov.u32 %0, %%cluster_ctarank;\n" : "=r"(rank));
+    asm volatile("barrier.cluster.arrive.relaxed.aligned;\n" ::: "memory");
+    const float* l = logits + (size_t) blockIdx.y * n_vocab;
+    float bv = __int_as_float(0xff800000);   // -inf; `n_vocab` is "no candidate", as in sampler_greedy_kernel
+    int best = n_vocab;
+    constexpr int S = kAmCtas * kAmThreads;
+    int v = (int) rank * kAmThreads + (int) threadIdx.x;
+    for (; v + 3 * S < n_vocab; v += 4 * S) {   // four independent loads, then taken in ascending order
+        const float x0 = l[v], x1 = l[v + S], x2 = l[v + 2 * S], x3 = l[v + 3 * S];
+        am_take(x0, v, bv, best);
+        am_take(x1, v + S, bv, best);
+        am_take(x2, v + 2 * S, bv, best);
+        am_take(x3, v + 3 * S, bv, best);
+    }
+    for (; v < n_vocab; v += S) am_take(l[v], v, bv, best);
+    for (int off = 16; off > 0; off >>= 1)
+        am_merge(__shfl_down_sync(0xFFFFFFFFu, bv, off), __shfl_down_sync(0xFFFFFFFFu, best, off), bv, best);
+    const int warp = (int) (threadIdx.x >> 5), lane = (int) (threadIdx.x & 31);
+    if (lane == 0) { sv[warp] = bv; si[warp] = best; }
+    __syncthreads();
+    if (warp == 0) {
+        bv = sv[lane];
+        best = si[lane];
+        for (int off = 16; off > 0; off >>= 1)
+            am_merge(__shfl_down_sync(0xFFFFFFFFu, bv, off), __shfl_down_sync(0xFFFFFFFFu, best, off), bv, best);
+    }
+    asm volatile("barrier.cluster.wait.acquire.aligned;\n" ::: "memory");   // CTA 0 runs
+    if (threadIdx.x == 0) {
+        uint64_t a;
+        asm volatile("mapa.u64 %0, %1, %2;\n" : "=l"(a) : "l"((uint64_t) &cv[rank]), "r"(0u));
+        *reinterpret_cast<float*>(a) = bv;
+        asm volatile("mapa.u64 %0, %1, %2;\n" : "=l"(a) : "l"((uint64_t) &ci[rank]), "r"(0u));
+        *reinterpret_cast<int*>(a) = best;
+    }
+    asm volatile("barrier.cluster.arrive.release.aligned;\n" ::: "memory");
+    if (rank != 0) return;
+    asm volatile("barrier.cluster.wait.acquire.aligned;\n" ::: "memory");
+    if (warp == 0) {
+        bv = lane < kAmCtas ? cv[lane] : __int_as_float(0xff800000);
+        best = lane < kAmCtas ? ci[lane] : n_vocab;
+        for (int off = 16; off > 0; off >>= 1)
+            am_merge(__shfl_down_sync(0xFFFFFFFFu, bv, off), __shfl_down_sync(0xFFFFFFFFu, best, off), bv, best);
+        if (lane == 0) out[blockIdx.y] = (best < n_vocab) ? best : 0;
+    }
+#else
+    (void) logits; (void) n_vocab; (void) out;
+    __trap();
+#endif
+}
+#endif  // !__HIPCC__
+
 /// **THE SAMPLED PATH, ONE BLOCK PER TOKEN.**  The kernel below replaced a version that ran the whole chain
 /// in ONE THREAD per token (`<<<ceil(T/64), 64>>>`, so a 4-token window fielded four threads): `top_k` alone
 /// was `k` sequential scans of the vocabulary with an inner sweep over the already-taken list - 20 x 248,320
@@ -833,6 +925,55 @@ int2* split_scratch(void* stream, size_t entries) {
 
 }  // namespace
 
+bool sample_greedy_cluster(const float* logits, int n_tokens, int n_vocab, int* out, void* stream) {
+#if defined(__HIPCC__)
+    (void) logits; (void) n_tokens; (void) n_vocab; (void) out; (void) stream;
+    return false;
+#else
+    if (n_tokens <= 0 || n_vocab <= 0) return true;
+    if (n_tokens > 65535) return false;
+    // Per device (a layer split runs on several): 1 the cluster kernel runs here, 2 it does not - sm_90+ (the card's,
+    // or STRATA_EMULATE_CC's), code built for it (an older build's PTX holds a trap: the PTX version says which), and
+    // room for one cluster of kAmCtas CTAs.
+    static int ok[64] = {};
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+    cudaLaunchConfig_t cfg{};
+    cudaLaunchAttribute at[1];
+    at[0].id = cudaLaunchAttributeClusterDimension;
+    at[0].val.clusterDim.x = kAmCtas;
+    at[0].val.clusterDim.y = 1;
+    at[0].val.clusterDim.z = 1;
+    cfg.gridDim = dim3(kAmCtas, 1, 1);
+    cfg.blockDim = dim3(kAmThreads, 1, 1);
+    cfg.dynamicSmemBytes = 0;
+    cfg.stream = (cudaStream_t) stream;
+    cfg.attrs = at;
+    cfg.numAttrs = 1;
+    if (ok[dev] == 0) {
+        int major = 0, clusters = 0;
+        cudaFuncAttributes fa{};
+        const bool runs = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+                          strata::cc_major_of(major) >= 9 &&
+                          cudaFuncGetAttributes(&fa, sampler_greedy_cluster_kernel) == cudaSuccess &&
+                          fa.ptxVersion >= 90 && fa.binaryVersion >= 90 &&
+                          cudaOccupancyMaxActiveClusters(&clusters, sampler_greedy_cluster_kernel, &cfg) ==
+                              cudaSuccess &&
+                          clusters >= 1;
+        cudaGetLastError();
+        ok[dev] = runs ? 1 : 2;
+    }
+    if (ok[dev] != 1) return false;
+    cfg.gridDim = dim3(kAmCtas, (unsigned) n_tokens, 1);
+    const cudaError_t e = cudaLaunchKernelEx(&cfg, sampler_greedy_cluster_kernel, logits, n_vocab, out);
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "sample_tokens cluster launch: %s\n", cudaGetErrorString(e));
+        std::exit(1);
+    }
+    return true;
+#endif
+}
+
 void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
                    const SamplerParams& p, int* out, void* stream) {
     if (n_tokens <= 0 || n_vocab <= 0) return;
@@ -845,10 +986,17 @@ void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* hi
                                ? (unsigned) ((n_vocab + 31) / 32) * sizeof(unsigned)   // the penalty bitmap
                                : 0;
     if (p.greedy || p.temperature <= 0.0f) {
+        // Without penalties (shmem == 0: no window) on sm_90+, a cluster of CTAs per token - the same token; see
+        // `sampler_greedy_cluster_kernel`.  STRATA_ARGMAX_MULTI=0: always the one-block kernel.
+        static const bool multi = [] {
+            const char* v = std::getenv("STRATA_ARGMAX_MULTI");
+            return !v || std::atoi(v) != 0;
+        }();
         // One block per token, 1,024 threads over the vocabulary.  See `sampler_greedy_kernel`.
         const int gthreads = 1024;
-        sampler_greedy_kernel<<<(unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream>>>(
-            logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);
+        if (!(multi && shmem == 0 && sample_greedy_cluster(logits, n_tokens, n_vocab, out, stream)))
+            sampler_greedy_kernel<<<(unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream>>>(
+                logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);
     } else if (sampled_path() == SampledPath::Old) {
         // The same block-per-token shape: the selection's k argmax rounds reduce inside the block.  See
         // `sampler_kernel`'s header for what the old one-thread-per-token launch cost.

src/kernels/decode_cluster_parity.cpp

@@ -0,0 +1,424 @@
+// src/kernels/decode_cluster_parity.cpp - S19: the decode kernels on thread-block clusters (sm_90+) against the ones
+// they replace, bit for bit (GPU, synthetic, no model).
+//
+//   QSA top-k : qsa_block_topk_cluster vs qsa_block_topk's one-CTA dispatch (the register kernel up to 33,792 blocks,
+//               block_topk_kernel above) and vs qsa_block_topk_ref - every id of every row of the `cap`-wide buffer,
+//               including the entries past the width that neither may write.
+//   argmax    : sample_greedy_cluster vs sample_tokens' one-block greedy kernel.
+//
+// Inputs: random scores like a real indexer's, and adversarial ones - all equal, a few distinct values (ties at the
+// threshold across the CTAs of a cluster), NaN, +-inf, -0.0 against +0.0, denormals, monotone rows, all NaN; contexts
+// from below the selection width (the identity) to 262,144 cells, every tail weight (n_kv mod 4), 1-16 queries; and
+// one CUDA graph that captures both cluster launches, replayed with new inputs.
+//
+//   decode_cluster_parity --selftest     parity (exit 0 = identical everywhere); skipped (exit 0) below sm_90
+//   decode_cluster_parity --bench        per-call times, old vs new, at 32K / 128K / 262K
+//
+// The old paths are reached through the public dispatchers with STRATA_QSA_CLUSTER=0 / STRATA_ARGMAX_MULTI=0, set
+// here before their first call (they read the variables once).
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_select.hpp"
+#include "strata/kernels/sampler.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+
+void set_env(const char* name, const char* value) {
+#if defined(_WIN32)
+    _putenv_s(name, value);
+#else
+    setenv(name, value, 1);
+#endif
+}
+
+template <typename T> struct Dev {
+    T* p = nullptr;
+    size_t n = 0;
+    explicit Dev(size_t count) : n(count) { ck(cudaMalloc(&p, count * sizeof(T) + 64), "cudaMalloc"); }
+    ~Dev() { cudaFree(p); }
+    void put(const std::vector<T>& h) {
+        ck(cudaMemcpy(p, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "up");
+        // #548 #536: a pageable cudaMemcpy can return before its DMA lands, and the graph replays and the timing
+        // cases run on a cudaStreamNonBlocking stream that does not wait for the legacy stream - so wait here
+        ck(cudaDeviceSynchronize(), "up landed");
+    }
+    std::vector<T> get() const {
+        std::vector<T> h(n);
+        ck(cudaMemcpy(h.data(), p, n * sizeof(T), cudaMemcpyDeviceToHost), "down");
+        return h;
+    }
+};
+
+const float kNaN = std::numeric_limits<float>::quiet_NaN(), kInf = std::numeric_limits<float>::infinity();
+
+// ---- QSA top-k fixtures
+enum Pattern { kRandom, kAllEqual, kFewValues, kNaNs, kInfs, kSignedZero, kDenormal, kAscending, kDescending, kAllNaN,
+               kQuantized, kPatterns };
+const char* pattern_name(int p) {
+    static const char* n[] = {"random", "all-equal", "few-values", "nan", "inf", "signed-zero", "denormal", "ascending",
+                              "descending", "all-nan", "quantized"};
+    return n[p];
+}
+
+// one query's row of block scores (blocks 0..n_bid); the tail block gets the +1e9 the score kernel gives it
+void fill_row(float* row, int64_t n_kv, int64_t n_bid, int pattern, std::mt19937& rng) {
+    std::normal_distribution<float> nd(0.f, 1.f);
+    std::uniform_int_distribution<int> pick(0, 99);
+    for (int64_t b = 0; b <= n_bid; ++b) {
+        float v = 0.f;
+        const float g = nd(rng);
+        switch (pattern) {
+            case kRandom: v = std::fmax(0.f, g) * 3.f + std::fmax(0.f, nd(rng)); break;   // relu sums: exact zeros too
+            case kAllEqual: v = 1.f; break;
+            case kFewValues: v = (float) (pick(rng) % 4) * 0.5f; break;
+            case kNaNs: v = pick(rng) < 15 ? kNaN : std::fabs(g); break;
+            case kInfs: {
+                const int r = pick(rng);
+                v = r < 5 ? kInf : r < 15 ? -kInf : r < 30 ? -std::fabs(g) : g;
+                break;
+            }
+            case kSignedZero: v = pick(rng) < 50 ? -0.0f : (pick(rng) < 50 ? 0.0f : -std::fabs(g) * 1e-3f); break;
+            case kDenormal: v = (float) (pick(rng) % 7) * 1e-41f; break;
+            case kAscending: v = (float) b; break;
+            case kDescending: v = (float) (n_bid - b); break;
+            case kAllNaN: v = kNaN; break;
+            case kQuantized: v = std::round(std::fabs(g) * 8.f) / 8.f; break;
+        }
+        row[b] = v;
+    }
+    if (n_kv % 4 != 0 && pattern != kAllNaN && pattern != kAllEqual) row[n_bid] += 1e9f;
+}
+
+struct TopkCase {
+    int64_t ctx;         // the last query's n_kv
+    int64_t capacity;    // --max-context: max_blocks = capacity / 4 + 2 (the engine's rule)
+    int nq;
+    int pattern;
+};
+
+int run_topk_case(const TopkCase& c, std::mt19937& rng, bool verbose) {
+    const k::QsaShapes s = k::qsa_real_shapes();
+    const int64_t max_blocks = c.capacity / 4 + 2, cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    std::vector<float> sc((size_t) (c.nq * max_blocks), 0.f);
+    std::vector<int32_t> steps((size_t) c.nq * k::kStepCount);
+    for (int i = 0; i < c.nq; ++i) {
+        const int64_t n_kv = c.ctx - (c.nq - 1 - i);   // the window's queries, one cell apart, the last at ctx
+        int32_t* st = steps.data() + (size_t) i * k::kStepCount;
+        st[k::kStepPos] = (int32_t) (n_kv - 1);
+        st[k::kStepNKv] = (int32_t) n_kv;
+        st[k::kStepNBid] = (int32_t) (n_kv / s.idx_block);
+        st[k::kStepWidth] = (int32_t) k::qsa_selection_width(n_kv, s);
+        fill_row(sc.data() + (size_t) i * max_blocks, n_kv, n_kv / s.idx_block, c.pattern, rng);
+    }
+    Dev<float> d_sc(sc.size());
+    const size_t n_ids = (size_t) (c.nq * cap);
+    Dev<int32_t> d_st(steps.size()), d_new(n_ids), d_old(n_ids), d_ref(n_ids);
+    d_sc.put(sc);
+    d_st.put(steps);
+    ck(cudaMemset(d_new.p, 0xA5, d_new.n * 4), "memset");
+    ck(cudaMemset(d_old.p, 0xA5, d_old.n * 4), "memset");
+    ck(cudaMemset(d_ref.p, 0xA5, d_ref.n * 4), "memset");
+    if (!k::qsa_block_topk_cluster(d_sc.p, d_st.p, c.nq, max_blocks, cap, s, d_new.p, nullptr)) {
+        std::printf("FAIL topk ctx %lld cap %lld nq %d: the cluster kernel did not run\n", (long long) c.ctx,
+                    (long long) c.capacity, c.nq);
+        return 1;
+    }
+    k::qsa_block_topk(d_sc.p, d_st.p, c.nq, max_blocks, cap, s, d_old.p, nullptr);
+    k::qsa_block_topk_ref(d_sc.p, d_st.p, c.nq, max_blocks, cap, s, d_ref.p, nullptr);
+    ck(cudaDeviceSynchronize(), "topk sync");
+    const std::vector<int32_t> a = d_new.get(), b = d_old.get(), r = d_ref.get();
+    int64_t diff_old = 0, diff_ref = 0, first = -1;
+    for (size_t i = 0; i < a.size(); ++i) {
+        if (a[i] != b[i]) { ++diff_old; if (first < 0) first = (int64_t) i; }
+        if (a[i] != r[i]) ++diff_ref;
+    }
+    const bool ok = diff_old == 0 && diff_ref == 0;
+    if (!ok || verbose)
+        std::printf("%s topk ctx %7lld cap %7lld nq %2d %-11s: %lld ids differ from the one-CTA dispatch, %lld from "
+                    "the reference%s\n", ok ? "ok  " : "FAIL", (long long) c.ctx, (long long) c.capacity, c.nq,
+                    pattern_name(c.pattern), (long long) diff_old, (long long) diff_ref,
+                    first >= 0 ? (" (first at " + std::to_string(first) + ")").c_str() : "");
+    return ok ? 0 : 1;
+}
+
+// ---- argmax fixtures
+enum AmPattern { aRandom, aTies, aAllEqual, aAllNegInf, aAllNaN, aNaNMax, aPosInf, aSignedZero, aLast, aFirst,
+                 aDenormal, aTiesAcross, aPatterns };
+const char* am_name(int p) {
+    static const char* n[] = {"random", "ties", "all-equal", "all--inf", "all-nan", "nan+max", "+inf", "signed-zero",
+                              "max-last", "max-first", "denormal", "ties-across"};
+    return n[p];
+}
+
+void fill_logits(float* l, int nv, int pattern, std::mt19937& rng) {
+    std::normal_distribution<float> nd(0.f, 3.f);
+    std::uniform_int_distribution<int> at(0, nv - 1);
+    for (int v = 0; v < nv; ++v) l[v] = nd(rng);
+    switch (pattern) {
+        case aRandom: break;
+        case aTies: { const int n = 1 + at(rng) % 9; for (int i = 0; i < n; ++i) l[at(rng)] = 100.f; break; }
+        case aAllEqual: for (int v = 0; v < nv; ++v) l[v] = 0.25f; break;
+        case aAllNegInf: for (int v = 0; v < nv; ++v) l[v] = -kInf; break;
+        case aAllNaN: for (int v = 0; v < nv; ++v) l[v] = kNaN; break;
+        case aNaNMax:
+            for (int v = 0; v < nv; ++v) if (at(rng) % 3 == 0) l[v] = kNaN;
+            l[at(rng)] = 50.f;
+            l[at(rng)] = 50.f;
+            break;
+        case aPosInf: for (int i = 0; i < 4; ++i) l[at(rng)] = kInf; break;
+        case aSignedZero:
+            for (int v = 0; v < nv; ++v) l[v] = -std::fabs(l[v]) - 1.f;
+            for (int i = 0; i < 6; ++i) l[at(rng)] = (i & 1) ? 0.0f : -0.0f;
+            break;
+        case aLast: l[nv - 1] = 1e30f; break;
+        case aFirst: l[0] = 1e30f; if (nv > 1) l[nv - 1] = 1e30f; break;
+        case aDenormal: for (int v = 0; v < nv; ++v) l[v] = (float) (at(rng) % 5) * 1e-42f - 1e-30f * (v & 1); break;
+        case aTiesAcross:   // the same maximum in several CTAs' and threads' ranges (8 x 1024 stride), plus -inf, NaN
+            for (int v = 0; v < nv; v += 1 + at(rng) % 9000) l[v] = 7.f;
+            for (int v = 3; v < nv; v += 11) l[v] = kNaN;
+            for (int v = 5; v < nv; v += 13) l[v] = -kInf;
+            break;
+    }
+}
+
+int run_argmax_case(int nv, int T, int pattern, std::mt19937& rng, bool verbose) {
+    std::vector<float> l((size_t) nv * T);
+    for (int t = 0; t < T; ++t) fill_logits(l.data() + (size_t) t * nv, nv, t % 2 == 0 ? pattern : aRandom, rng);
+    Dev<float> d_l(l.size());
+    Dev<int> d_new((size_t) T), d_old((size_t) T);
+    d_l.put(l);
+    ck(cudaMemset(d_new.p, 0xA5, (size_t) T * 4), "memset");
+    ck(cudaMemset(d_old.p, 0x5A, (size_t) T * 4), "memset");
+    k::SamplerParams sp;
+    sp.greedy = true;
+    sp.temperature = 0.f;
+    if (!k::sample_greedy_cluster(d_l.p, T, nv, d_new.p, nullptr)) {
+        std::printf("FAIL argmax nv %d T %d: the cluster kernel did not run\n", nv, T);
+        return 1;
+    }
+    k::sample_tokens(d_l.p, T, nv, nullptr, 0, sp, d_old.p, nullptr);
+    ck(cudaDeviceSynchronize(), "argmax sync");
+    const std::vector<int> a = d_new.get(), b = d_old.get();
+    // and an independent host answer: the lowest index of the largest non-NaN value above -inf, else 0
+    int bad = 0;
+    for (int t = 0; t < T; ++t) {
+        const float* r = l.data() + (size_t) t * nv;
+        int hb = -1;
+        for (int v = 0; v < nv; ++v)
+            if (r[v] > -kInf && (hb < 0 || r[v] > r[hb])) hb = v;
+        if (hb < 0) hb = 0;
+        if (a[(size_t) t] != b[(size_t) t] || a[(size_t) t] != hb) ++bad;
+    }
+    if (bad || verbose)
+        std::printf("%s argmax nv %6d T %d %-11s: %d rows differ (new %d, old %d)\n", bad ? "FAIL" : "ok  ", nv, T,
+                    am_name(pattern), bad, a[0], b[0]);
+    return bad ? 1 : 0;
+}
+
+// both cluster launches captured in one graph, replayed over new inputs written into the same buffers
+int run_graph_case(std::mt19937& rng) {
+    const k::QsaShapes s = k::qsa_real_shapes();
+    const int nq = 5, nv = 248320;
+    const int64_t capacity = 131072 + 272, max_blocks = capacity / 4 + 2;
+    const int64_t cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    Dev<float> d_sc((size_t) (nq * max_blocks)), d_l((size_t) nv * nq);
+    Dev<int32_t> d_st((size_t) nq * k::kStepCount), d_new((size_t) (nq * cap)), d_old((size_t) (nq * cap));
+    Dev<int> a_new((size_t) nq), a_old((size_t) nq);
+    ck(cudaMemset(d_new.p, 0, d_new.n * 4), "memset");   // the entries past a row's width: written by neither
+    ck(cudaMemset(d_old.p, 0, d_old.n * 4), "memset");
+    cudaStream_t cs;
+    ck(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking), "stream");
+    cudaGraph_t g = nullptr;
+    cudaGraphExec_t ge = nullptr;
+    ck(cudaStreamBeginCapture(cs, cudaStreamCaptureModeThreadLocal), "begin capture");
+    const bool l1 = k::qsa_block_topk_cluster(d_sc.p, d_st.p, nq, max_blocks, cap, s, d_new.p, cs);
+    const bool l2 = k::sample_greedy_cluster(d_l.p, nq, nv, a_new.p, cs);
+    ck(cudaStreamEndCapture(cs, &g), "end capture");
+    if (!l1 && !l2) {   // no cluster code here (a card or a build below sm_90): nothing to compare
+        if (g) cudaGraphDestroy(g);
+        cudaStreamDestroy(cs);
+        return -1;
+    }
+    if (!l1 || !l2) { std::printf("FAIL graph: one cluster kernel did not run\n"); return 1; }
+    ck(cudaGraphInstantiate(&ge, g, 0), "instantiate");
+    int fails = 0;
+    for (int rep = 0; rep < 6; ++rep) {
+        const int64_t ctx = 100000 + rep * 4001 + rep % 4;
+        std::vector<float> sc((size_t) (nq * max_blocks), 0.f), l((size_t) nv * nq);
+        std::vector<int32_t> steps((size_t) nq * k::kStepCount);
+        for (int i = 0; i < nq; ++i) {
+            const int64_t n_kv = ctx - (nq - 1 - i);
+            int32_t* st = steps.data() + (size_t) i * k::kStepCount;
+            st[k::kStepPos] = (int32_t) (n_kv - 1);
+            st[k::kStepNKv] = (int32_t) n_kv;
+            st[k::kStepNBid] = (int32_t) (n_kv / 4);
+            st[k::kStepWidth] = (int32_t) k::qsa_selection_width(n_kv, s);
+            fill_row(sc.data() + (size_t) i * max_blocks, n_kv, n_kv / 4, rep % 2 ? kFewValues : kRandom, rng);
+            fill_logits(l.data() + (size_t) i * nv, nv, rep % 2 ? aTiesAcross : aRandom, rng);
+        }
+        d_sc.put(sc);
+        d_st.put(steps);
+        d_l.put(l);
+        ck(cudaGraphLaunch(ge, cs), "graph launch");
+        k::qsa_block_topk(d_sc.p, d_st.p, nq, max_blocks, cap, s, d_old.p, cs);
+        k::SamplerParams sp;
+        sp.greedy = true;
+        k::sample_tokens(d_l.p, nq, nv, nullptr, 0, sp, a_old.p, cs);
+        ck(cudaStreamSynchronize(cs), "graph sync");
+        const auto x = d_new.get(), y = d_old.get();
+        const auto u = a_new.get(), v = a_old.get();
+        int64_t d1 = 0, d2 = 0;
+        for (size_t i = 0; i < x.size(); ++i) d1 += x[i] != y[i];
+        for (size_t i = 0; i < u.size(); ++i) d2 += u[i] != v[i];
+        if (d1 || d2) ++fails;
+        std::printf("%s graph replay %d (ctx %lld): %lld ids, %lld tokens differ\n", d1 || d2 ? "FAIL" : "ok  ", rep,
+                    (long long) ctx, (long long) d1, (long long) d2);
+    }
+    cudaGraphExecDestroy(ge);
+    cudaGraphDestroy(g);
+    cudaStreamDestroy(cs);
+    return fails;
+}
+
+// back-to-back launches on `cs` (the engine's are too: one stream, inside a graph)
+template <typename F> float time_us(cudaStream_t cs, F&& f, int reps = 300) {
+    cudaEvent_t a, b;
+    cudaEventCreate(&a);
+    cudaEventCreate(&b);
+    for (int i = 0; i < 5; ++i) f();
+    cudaEventRecord(a, cs);
+    for (int i = 0; i < reps; ++i) f();
+    cudaEventRecord(b, cs);
+    ck(cudaEventSynchronize(b), "time");
+    float ms = 0.f;
+    cudaEventElapsedTime(&ms, a, b);
+    cudaEventDestroy(a);
+    cudaEventDestroy(b);
+    return ms * 1000.f / (float) reps;
+}
+
+int bench() {
+    std::mt19937 rng(11);
+    const k::QsaShapes s = k::qsa_real_shapes();
+    const int64_t cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    cudaStream_t cs;
+    ck(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking), "stream");
+    std::printf("QSA top-k, per call (us); capacity = context + 272 as the speed runs set --max-context\n");
+    for (const int64_t ctx : {32768LL, 131072LL, 262144LL - 272}) {
+        for (const int nq : {1, 5}) {
+            const int64_t capacity = ctx + 272, max_blocks = capacity / 4 + 2;
+            std::vector<float> sc((size_t) (nq * max_blocks), 0.f);
+            std::vector<int32_t> steps((size_t) nq * k::kStepCount);
+            for (int i = 0; i < nq; ++i) {
+                const int64_t n_kv = ctx - (nq - 1 - i);
+                int32_t* st = steps.data() + (size_t) i * k::kStepCount;
+                st[k::kStepPos] = (int32_t) (n_kv - 1);
+                st[k::kStepNKv] = (int32_t) n_kv;
+                st[k::kStepNBid] = (int32_t) (n_kv / 4);
+                st[k::kStepWidth] = (int32_t) k::qsa_selection_width(n_kv, s);
+                fill_row(sc.data() + (size_t) i * max_blocks, n_kv, n_kv / 4, kRandom, rng);
+            }
+            Dev<float> d_sc(sc.size());
+            Dev<int32_t> d_st(steps.size()), d_ids((size_t) (nq * cap));
+            d_sc.put(sc);
+            d_st.put(steps);
+            const float* sp = d_sc.p;
+            const int32_t* st = d_st.p;
+            int32_t* ids = d_ids.p;
+            const float t_old = time_us(cs, [&] { k::qsa_block_topk(sp, st, nq, max_blocks, cap, s, ids, cs); });
+            const float t_ref = time_us(cs, [&] { k::qsa_block_topk_ref(sp, st, nq, max_blocks, cap, s, ids, cs); });
+            const float t_new = time_us(cs, [&] { k::qsa_block_topk_cluster(sp, st, nq, max_blocks, cap, s, ids, cs); });
+            std::printf("  ctx %6lld nq %d: one-CTA dispatch %7.1f  (ref %7.1f)  cluster %6.1f  -> %.1fx\n",
+                        (long long) ctx, nq, t_old, t_ref, t_new, t_old / t_new);
+        }
+    }
+    std::printf("argmax, per call (us)\n");
+    for (const int nv : {248320, 32768}) {
+        for (const int T : {1, 5}) {
+            std::vector<float> l((size_t) nv * T);
+            for (int t = 0; t < T; ++t) fill_logits(l.data() + (size_t) t * nv, nv, aRandom, rng);
+            Dev<float> d_l(l.size());
+            Dev<int> d_o((size_t) T);
+            d_l.put(l);
+            k::SamplerParams sp;
+            sp.greedy = true;
+            const float t_old = time_us(cs, [&] { k::sample_tokens(d_l.p, T, nv, nullptr, 0, sp, d_o.p, cs); });
+            const float t_new = time_us(cs, [&] { k::sample_greedy_cluster(d_l.p, T, nv, d_o.p, cs); });
+            std::printf("  n_vocab %6d T %d: one block %6.1f  cluster %5.1f  -> %.1fx\n", nv, T, t_old, t_new,
+                        t_old / t_new);
+        }
+    }
+    cudaStreamDestroy(cs);
+    return 0;
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    set_env("STRATA_QSA_CLUSTER", "0");     // the dispatchers: the one-CTA kernels (the cluster ones called directly)
+    set_env("STRATA_ARGMAX_MULTI", "0");
+    const bool do_bench = argc > 1 && std::strcmp(argv[1], "--bench") == 0;
+    const bool verbose = argc > 2 && std::strcmp(argv[2], "-v") == 0;
+    int dev = 0, major = 0;
+    ck(cudaGetDevice(&dev), "device");
+    ck(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev), "cc");
+    std::mt19937 rng(20261002);
+    // first, so that both kernels' first calls (their per-device checks and opt-ins) happen inside a stream capture,
+    // as in the engine's decode graphs
+    const int gfails = run_graph_case(rng);
+    if (gfails < 0) {
+        std::printf("SKIP: no thread-block clusters on this card / build (compute capability %d)\n", major);
+        return 0;
+    }
+    if (do_bench) return bench();
+    int fails = 0, cases = 0;
+    // contexts: below the width (identity), at it, just past it, odd tails, the register kernel's range, past it
+    // (block_topk_kernel), up to 262,144; capacities: tight (the speed runs) and 262,144 (a long --max-context)
+    const int64_t ctxs[] = {7, 2050, 2051, 2052, 2053, 2054, 2055, 3001, 9000, 32768, 32769, 65538, 100003,
+                            131072, 135170, 135171, 200001, 262142, 262143, 262144};
+    for (const int64_t ctx : ctxs) {
+        for (const int64_t capacity : {ctx + 272, (int64_t) 262144 + 272}) {
+            if (capacity < ctx) continue;
+            for (int p = 0; p < kPatterns; ++p) {
+                const int nq = (p % 3 == 0) ? 1 : (p % 3 == 1 ? 5 : 16);
+                if (nq > ctx) continue;
+                fails += run_topk_case({ctx, capacity, nq, p}, rng, verbose);
+                ++cases;
+            }
+        }
+    }
+    std::printf("QSA top-k: %d cases, %d failed\n", cases, fails);
+    int afails = 0, acases = 0;
+    for (const int nv : {248320, 248321, 248319, 32771, 24576, 8193, 1000, 33, 1}) {
+        for (int p = 0; p < aPatterns; ++p) {
+            for (const int T : {1, 5}) {
+                afails += run_argmax_case(nv, T, p, rng, verbose);
+                ++acases;
+            }
+        }
+    }
+    std::printf("argmax: %d cases, %d failed\n", acases, afails);
+    const int total = fails + afails + gfails;
+    std::printf("%s\n", total == 0 ? "PASS: the cluster kernels are bitwise identical" : "FAIL");
+    return total == 0 ? 0 : 1;
+}

src/kernels/gr_parity.cpp

@@ -26,6 +26,7 @@
 
 #include <cmath>
 #include <cstdio>
+#include <algorithm>
 #include <cstdlib>
 #include <cstring>
 #include <random>
@@ -331,7 +332,22 @@ int fused_multi_lds_parity(const float* d_norm, const uint16_t* d_down, const ui
         check(cudaMemcpy(s.mixed.data(), d_mixed, s.mixed.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read mixed");
         return s;
     };
-    auto same = [](const Snapshot& a, const Snapshot& b) {
+    // the split read (STRATA_GR_V3=1) sums in another order than the single-token kernel: equal within float
+    // rounding, not to the bit, so it is compared with a relative tolerance; the default kernels bit for bit
+    static const bool v3 = [] { const char* v = std::getenv("STRATA_GR_V3"); return v != nullptr && std::atoi(v) != 0; }();
+    auto close = [](const std::vector<float>& x, const std::vector<float>& y) {
+        double worst = 0.0, mag = 1e-30;
+        for (size_t i = 0; i < x.size(); ++i) {
+            worst = std::max(worst, (double) std::fabs(x[i] - y[i]));
+            mag = std::max(mag, (double) std::fabs(y[i]));
+        }
+        if (worst > 2e-6 * mag)
+            std::printf("    tolerance: worst %.3e of max |ref| %.3e (rel %.3e), n %zu\n", worst, mag, worst / mag, x.size());
+        return worst <= 2e-6 * mag;
+    };
+    auto same = [&](const Snapshot& a, const Snapshot& b) {
+        if (v3)   // `lo` is the default kernels' workspace between down and up; the split read keeps it in shared memory
+            return close(a.r_out, b.r_out) && close(a.rs, b.rs) && close(a.inject, b.inject) && close(a.mixed, b.mixed);
         return std::memcmp(a.r_out.data(), b.r_out.data(), a.r_out.size() * sizeof(float)) == 0 &&
                std::memcmp(a.lo.data(), b.lo.data(), a.lo.size() * sizeof(float)) == 0 &&
                std::memcmp(a.rs.data(), b.rs.data(), a.rs.size() * sizeof(float)) == 0 &&

src/kernels/native_expert_parity.cpp

@@ -94,7 +94,9 @@ int check_blob(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int se
             ffp[k] = ff[k].data();
         }
         cpu::native_gu_rows(f, blob.data(), a, NT, ffp, 0, (int) FF);
-        if (cpu::iq512_supported(f.gu_type)) {
+        // either multi-token kernel: IQ4_XS has an AVX-2 one and no AVX-512 one, so the gate cannot be
+        // iq512_supported alone - that would leave the format untested on every CPU.
+        if (cpu::iq512_supported(f.gu_type) || cpu::iq256_supported(f.gu_type)) {
             // ggml's own vec_dot, same Q8_K activations: the reference for both multi-token kernels
             // (float-order differences only)
             const auto* tc = ggml_get_type_traits_cpu((ggml_type) f.gu_type);
@@ -143,8 +145,9 @@ int check_blob(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int se
                             tag, us1, 2.0 * f.up_off / us1 / 1e3, usg, 2.0 * f.up_off / usg / 1e3);
                 std::printf("          gate+up %d tokens one thread: %s %.0f us vs ggml %d x %.0f us\n", NT, tag, usn, NT, usg);
             };
-            if (cpu::cpu_avx512_ok()) check("avx512", true);   // guarded: the binary runs on AVX-2 CPUs too
-            check("avx2", false);
+            // guarded: the binary runs on AVX-2 CPUs too, and IQ4_XS has no AVX-512 kernel (an empty switch)
+            if (cpu::cpu_avx512_ok() && cpu::iq512_supported(f.gu_type)) check("avx512", true);
+            if (cpu::iq256_supported(f.gu_type)) check("avx2", false);
         }
         for (int k = 0; k < NT; ++k) {
             cpu::native_quant_h(f, ff[k].data(), hq[k].data());

src/kernels/native_grouped_parity.cpp

@@ -0,0 +1,351 @@
+// src/kernels/native_grouped_parity.cpp - native_expert_grouped's group stride (grid_groups) and its fused SwiGLU +
+// q8_1 pass, against the launches before them (native_grouped_set_v1: a block row per possible group, SwiGLU and the
+// q8_1 quantization as two kernels over all cap_entries).  Synthetic blobs, no model.
+//
+//     build/native_grouped_parity           the checks (a ctest; needs the GPU)
+//     build/native_grouped_parity --bench   the checks, then v1/new timings of 48 calls in a graph (a window's layers)
+//
+// Every gate/up format with every down format; calls of 0..cap groups of 1..T entries each (an empty call is what the
+// verify window's PCIe call usually is), entries from 0 and from past 0 (the PCIe call's follow the VRAM call's),
+// scattered destinations, stale scratch, grid_groups 0 (= cap), 1, 2, 3, 4 and above cap: the output must be BITWISE
+// equal to v1's, the rows the call must not write included (both start from the same NaN pattern).
+//
+// Random bytes are valid codes for every format here (every grid index is in range); only the fp16 block scales are
+// set, small enough that the SwiGLU outputs keep a finite fp16 q8_1 scale.
+#include "strata/kernels/iq_kernels.hpp"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <numeric>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+
+int g_fail = 0;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "CUDA error in %s: %s\n", what, cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+
+template<typename T>
+T* dalloc(size_t n) {
+    T* p = nullptr;
+    ck(cudaMalloc(&p, n * sizeof(T) + 256), "malloc");
+    ck(cudaMemset(p, 0, n * sizeof(T) + 256), "memset");
+    return p;
+}
+
+const char* name_of(int t) {
+    switch (t) {
+        case 16: return "IQ2_XXS";
+        case 17: return "IQ2_XS";
+        case 18: return "IQ3_XXS";
+        case 20: return "IQ4_NL";
+        case 21: return "IQ3_S";
+        case 22: return "IQ2_S";
+        case 23: return "IQ4_XS";
+        case 29: return "IQ1_M";
+        case 42: return "Q2_0";
+        case 12: return "Q4_K";
+        case 13: return "Q5_K";
+        case 7: return "Q5_1";
+        case 8: return "Q8_0";
+        default: return "?";
+    }
+}
+int block_values(int t) { return t == 20 || t == 7 || t == 8 ? 32 : t == 42 ? 64 : 256; }
+
+// `rows` rows of `n` values of format t: random bytes, then a finite fp16 scale in every block
+std::vector<uint8_t> random_rows(int t, int64_t rows, int64_t n, std::mt19937& rng) {
+    const size_t rb = k::iq_row_bytes(t, n), bs = k::iq_row_bytes(t, block_values(t));
+    std::vector<uint8_t> w((size_t) rows * rb);
+    std::uniform_int_distribution<int> byte(0, 255), ex(2, 8), man(0, 1023), sgn(0, 1);
+    for (auto& b : w) b = (uint8_t) byte(rng);
+    for (size_t o = 0; o < w.size(); o += bs) {
+        if (t == 29) {
+            // IQ1_M: the fp16 scale is the top nibbles of its four scale words (bytes 48..55), the sign and the
+            // exponent's top bits in the last: 0x1 / 0x2 there (0x9 / 0xA negative) keeps it in 2^-11 .. 2^-3
+            const uint8_t nib = (uint8_t) ((sgn(rng) ? 0x8 : 0x0) | (1 + (byte(rng) & 1)));
+            w[o + 55] = (uint8_t) ((w[o + 55] & 0x0F) | (nib << 4));
+        } else {   // the other formats start their block with the fp16 scale: 2^-13 .. 2^-6
+            // (Q4_K / Q5_K: 2^-14 .. 2^-11, their 6-bit sub-scales multiply it by up to 63)
+            const int e = (t == 12 || t == 13) ? 1 + ex(rng) % 4 : ex(rng);
+            const uint16_t h = (uint16_t) ((sgn(rng) ? 0x8000 : 0) | (e << 10) | man(rng));
+            std::memcpy(&w[o], &h, 2);
+            if (t == 12 || t == 13 || t == 7) {   // Q4_K / Q5_K: dmin, Q5_1: m - the second fp16, as small
+                const int em = (t == 7) ? ex(rng) : 1 + ex(rng) % 4;
+                const uint16_t m = (uint16_t) ((sgn(rng) ? 0x8000 : 0) | (em << 10) | man(rng));
+                std::memcpy(&w[o + 2], &m, 2);
+            }
+        }
+    }
+    return w;
+}
+
+// one window's worth of expert calls on one format pair: blobs, T tokens' q8_1 activations, cap = T * K entries
+struct Setup {
+    k::NativeExpertLayout L;
+    int T = 0, K = 0, cap = 0, n_blobs = 0;
+    size_t slot = 0;
+    uint8_t* blobs = nullptr;
+    uint8_t* xq = nullptr;
+    uint8_t* scr = nullptr;
+    size_t scr_bytes = 0;
+    float* out = nullptr;
+    size_t out_floats = 0;
+    unsigned long long* ptr = nullptr;
+    int32_t *start = nullptr, *n = nullptr, *dst = nullptr, *tok = nullptr;
+
+    Setup(int gu, int dt, int64_t H, int64_t FF, int tokens, int k_per_token, int blobs_n, std::mt19937& rng,
+          cudaStream_t s) {
+        L = k::native_expert_layout(gu, dt, H, FF);
+        T = tokens; K = k_per_token; cap = T * K; n_blobs = blobs_n;
+        slot = (L.bytes + 255) & ~(size_t) 255;
+        blobs = dalloc<uint8_t>(slot * (size_t) n_blobs);
+        for (int b = 0; b < n_blobs; ++b) {
+            // gate rows | up rows (format gu, H values each) | down rows (format dt, FF values each)
+            auto blob = random_rows(gu, 2 * FF, H, rng);
+            const auto down = random_rows(dt, H, FF, rng);
+            blob.insert(blob.end(), down.begin(), down.end());
+            if (blob.size() != L.bytes) { std::fprintf(stderr, "blob %zu != %zu bytes\n", blob.size(), L.bytes); std::exit(2); }
+            ck(cudaMemcpy(blobs + slot * (size_t) b, blob.data(), blob.size(), cudaMemcpyHostToDevice), "blob");
+        }
+        std::normal_distribution<float> nd(0.f, 1.f);
+        std::vector<float> x((size_t) T * H);
+        for (auto& v : x) v = nd(rng);
+        float* dx = dalloc<float>(x.size());
+        ck(cudaMemcpy(dx, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+        xq = dalloc<uint8_t>((size_t) T * (H / 32) * 36);
+        k::quantize_q8_1_rows(dx, T, H, xq, s);
+        ck(cudaStreamSynchronize(s), "quantize");
+        cudaFree(dx);
+        scr_bytes = k::native_expert_scratch_bytes(cap, FF);
+        scr = dalloc<uint8_t>(scr_bytes);
+        out_floats = (size_t) cap * H;
+        out = dalloc<float>(out_floats);
+        ptr = dalloc<unsigned long long>(cap);
+        start = dalloc<int32_t>(cap + 1);
+        n = dalloc<int32_t>(1);
+        dst = dalloc<int32_t>(cap);
+        tok = dalloc<int32_t>(cap);
+    }
+    ~Setup() {
+        cudaFree(blobs); cudaFree(xq); cudaFree(scr); cudaFree(out);
+        cudaFree(ptr); cudaFree(start); cudaFree(n); cudaFree(dst); cudaFree(tok);
+    }
+    // a plan: `groups` groups of 1..T entries from entry `base` on (as many as fit in cap)
+    int plan(int groups, int base, std::mt19937& rng) {
+        std::uniform_int_distribution<int> size(1, T);
+        std::vector<int> sizes;
+        int used = base;
+        for (int g = 0; g < groups; ++g) {
+            const int m = std::min(size(rng), cap - used);
+            if (m <= 0) break;
+            sizes.push_back(m);
+            used += m;
+        }
+        return plan_sizes(sizes, base, rng);
+    }
+    // groups of these sizes from entry `base` on, each its own blob (as long as there are enough: a window's groups
+    // are different experts); scattered destinations, random tokens
+    int plan_sizes(const std::vector<int>& sizes, int base, std::mt19937& rng) {
+        std::uniform_int_distribution<int> tk(0, T - 1);
+        std::vector<int> order(n_blobs);
+        std::iota(order.begin(), order.end(), 0);
+        std::shuffle(order.begin(), order.end(), rng);
+        std::vector<unsigned long long> p;
+        std::vector<int32_t> st(1, base), tk_v, ds(cap);
+        for (const int m : sizes) {
+            p.push_back((unsigned long long) (blobs + slot * (size_t) order[p.size() % order.size()]));
+            st.push_back(st.back() + m);
+        }
+        if (st.back() > cap) { std::fprintf(stderr, "plan: %d entries > cap %d\n", st.back(), cap); std::exit(2); }
+        const int ng = (int) p.size();
+        std::iota(ds.begin(), ds.end(), 0);
+        std::shuffle(ds.begin(), ds.end(), rng);                  // scattered rows of `out`
+        for (int e = 0; e < cap; ++e) tk_v.push_back(tk(rng));
+        if (ng) ck(cudaMemcpy(ptr, p.data(), p.size() * 8, cudaMemcpyHostToDevice), "ptr");
+        ck(cudaMemcpy(start, st.data(), st.size() * 4, cudaMemcpyHostToDevice), "start");
+        ck(cudaMemcpy(n, &ng, 4, cudaMemcpyHostToDevice), "n");
+        ck(cudaMemcpy(dst, ds.data(), ds.size() * 4, cudaMemcpyHostToDevice), "dst");
+        ck(cudaMemcpy(tok, tk_v.data(), tk_v.size() * 4, cudaMemcpyHostToDevice), "tok");
+        return ng;
+    }
+    void call(int64_t grid_groups, cudaStream_t s) {
+        k::native_expert_grouped(L, ptr, start, n, dst, tok, cap, cap, xq, scr, out, s, grid_groups);
+    }
+    // `out` after one call from the NaN pattern, the scratch holding `junk` (what earlier calls left there)
+    std::vector<uint32_t> result(bool v1, int64_t grid_groups, int junk, cudaStream_t s) {
+        ck(cudaMemset(out, 0xFF, out_floats * 4), "memset");
+        ck(cudaMemset(scr, junk, scr_bytes), "memset");
+        k::native_grouped_set_v1(v1);
+        call(grid_groups, s);
+        k::native_grouped_set_v1(false);
+        ck(cudaStreamSynchronize(s), "sync");
+        std::vector<uint32_t> o(out_floats);
+        ck(cudaMemcpy(o.data(), out, o.size() * 4, cudaMemcpyDeviceToHost), "out");
+        return o;
+    }
+};
+
+void check(int gu, int dt, int64_t H, int64_t FF, cudaStream_t s, std::mt19937& rng) {
+    Setup S(gu, dt, H, FF, 6, 4, 9, rng, s);   // 6 tokens x 4 experts: cap 24 entries
+    int calls = 0, bad = 0;
+    size_t written = 0;
+    for (const int base : {0, 7}) {
+        for (const int groups : {0, 1, 2, 3, 5, 8, 24}) {
+            const int ng = S.plan(groups, base, rng);
+            const auto ref = S.result(true, 0, 0x00, s);
+            size_t w = 0;
+            for (const uint32_t v : ref) w += v != 0xFFFFFFFFu;
+            written += w;
+            for (const int64_t gy : {(int64_t) 0, (int64_t) 1, (int64_t) 2, (int64_t) 3, (int64_t) 4, (int64_t) S.cap + 3}) {
+                const auto got = S.result(false, gy, (calls & 1) ? 0x7F : 0x00, s);
+                ++calls;
+                size_t diff = 0;
+                for (size_t i = 0; i < ref.size(); ++i) diff += ref[i] != got[i];
+                if (diff) {
+                    std::printf("  %-8s/%-7s base %d, %d groups, grid_groups %lld: %zu of %zu floats differ  FAIL\n",
+                                name_of(gu), name_of(dt), base, ng, (long long) gy, diff, ref.size());
+                    ++bad;
+                }
+            }
+        }
+    }
+    std::printf("%-8s/%-7s %5lld x %4lld  %d calls: %s (%zu output floats written in all)\n", name_of(gu), name_of(dt),
+                (long long) H, (long long) FF, calls, bad ? "FAIL" : "bitwise equal to v1", written);
+    if (written == 0) { std::printf("  nothing was written: the check checked nothing  FAIL\n"); ++bad; }
+    g_fail += bad;
+}
+
+// ------------------------------------------------------------------------------------------------ --bench
+// A window's 48 calls, one per layer, each with ITS OWN experts as the layers have: copies of the Setup's blobs in one
+// arena, so the weights stream from DRAM as in the engine.  (48 calls re-reading the same 28 experts - 73 MB, about an
+// Ada L2 - would read part of them from L2; the engine's layers share none.)
+struct Layers {
+    uint8_t* arena = nullptr;
+    std::vector<unsigned long long*> ptr;           // per layer: its groups' blobs (device)
+    Layers(Setup& S, int ng) {
+        const int per = std::max(ng, 1);
+        arena = dalloc<uint8_t>(S.slot * (size_t) (48 * per));
+        for (int l = 0; l < 48; ++l) {
+            std::vector<unsigned long long> p;
+            for (int j = 0; j < per; ++j) {
+                uint8_t* d = arena + S.slot * (size_t) (l * per + j);
+                ck(cudaMemcpy(d, S.blobs + S.slot * (size_t) ((l * per + j) % S.n_blobs), S.L.bytes,
+                              cudaMemcpyDeviceToDevice), "blob copy");
+                p.push_back((unsigned long long) d);
+            }
+            unsigned long long* dp = dalloc<unsigned long long>(p.size());
+            ck(cudaMemcpy(dp, p.data(), p.size() * 8, cudaMemcpyHostToDevice), "ptr");
+            ptr.push_back(dp);
+        }
+    }
+    Layers(const Layers&) = delete;
+    Layers& operator=(const Layers&) = delete;
+    ~Layers() {
+        for (auto* p : ptr) cudaFree(p);
+        cudaFree(arena);
+    }
+};
+
+// the 48 calls of one variant captured in one graph
+cudaGraphExec_t make_graph(cudaStream_t s, Setup& S, const Layers& Ls, bool v1, int64_t gy) {
+    k::native_grouped_set_v1(v1);
+    cudaGraph_t g;
+    cudaGraphExec_t ge;
+    ck(cudaStreamBeginCapture(s, cudaStreamCaptureModeThreadLocal), "capture");
+    for (int l = 0; l < 48; ++l)
+        k::native_expert_grouped(S.L, Ls.ptr[l], S.start, S.n, S.dst, S.tok, S.cap, S.cap, S.xq, S.scr, S.out, s, gy);
+    ck(cudaStreamEndCapture(s, &g), "capture");
+    ck(cudaGraphInstantiate(&ge, g, 0), "instantiate");
+    ck(cudaGraphDestroy(g), "graph");
+    k::native_grouped_set_v1(false);
+    return ge;
+}
+
+// one launch of the graph: microseconds per call
+float launch_us(cudaStream_t s, cudaGraphExec_t ge, cudaEvent_t e0, cudaEvent_t e1) {
+    ck(cudaEventRecord(e0, s), "record");
+    ck(cudaGraphLaunch(ge, s), "launch");
+    ck(cudaEventRecord(e1, s), "record");
+    ck(cudaEventSynchronize(e1), "bench");
+    float ms = 0;
+    ck(cudaEventElapsedTime(&ms, e0, e1), "elapsed");
+    return 1e3f * ms / 48.0f;
+}
+
+float median(std::vector<float> v) {
+    std::sort(v.begin(), v.end());
+    return v[v.size() / 2];
+}
+
+// The two variants' launches ALTERNATE (v1-new, new-v1, ...) after a warm-up, and each reports its median: a card
+// under sustained load steps its clock down after a fraction of a second (an RTX 4080 SUPER went from 80 to 89 us
+// per gate/up call within one run), and whichever variant ran before the step would look faster.
+constexpr int kWarmPairs = 20, kPairs = 40;
+
+void bench(cudaStream_t s, std::mt19937& rng) {
+    std::printf("\n--bench: microseconds per call, a window's 48 calls in a graph, each layer with its own experts (the\n"
+                "weights stream from DRAM as in the engine); v1 / new (grid_groups): medians of %d launches each,\n"
+                "alternating, after %d warm-up pairs - idle GPU assumed\n", kPairs, kWarmPairs);
+    // a verify window's shape: 4 tokens x 10 experts, 2560 x 768 IQ3_S / IQ4_XS experts
+    Setup S(21, 23, 2560, 768, 4, 10, 32, rng, s);
+    std::vector<int> vram(16, 1);
+    vram.insert(vram.end(), 12, 2);                                   // 28 groups, 40 entries
+    const struct { const char* what; std::vector<int> sizes; int64_t gy; } rows[] = {
+        {"no group (the PCIe call at pcie_frac 0)", {}, 4},
+        {"1 group of 2 entries (a PCIe call)", {2}, 4},
+        {"28 groups, 40 entries (a VRAM call)", vram, 0},
+    };
+    cudaEvent_t e0, e1;
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    for (const auto& r : rows) {
+        const int ng = S.plan_sizes(r.sizes, 0, rng);
+        const Layers Ls(S, ng);                                       // 48 x 28 experts: 3.7 GB for the VRAM call
+        cudaGraphExec_t ga = make_graph(s, S, Ls, true, 0), gb = make_graph(s, S, Ls, false, r.gy);
+        std::vector<float> ta, tb;
+        for (int i = 0; i < kWarmPairs + kPairs; ++i) {
+            const bool a_first = i % 2 == 0;
+            const float x = launch_us(s, a_first ? ga : gb, e0, e1), y = launch_us(s, a_first ? gb : ga, e0, e1);
+            if (i < kWarmPairs) continue;
+            ta.push_back(a_first ? x : y);
+            tb.push_back(a_first ? y : x);
+        }
+        ck(cudaGraphExecDestroy(ga), "graph");
+        ck(cudaGraphExecDestroy(gb), "graph");
+        std::printf("  %-42s %8.2f / %8.2f us (grid_groups %lld)\n", r.what, median(ta), median(tb), (long long) r.gy);
+    }
+    cudaEventDestroy(e0);
+    cudaEventDestroy(e1);
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    const bool do_bench = argc > 1 && std::string(argv[1]) == "--bench";
+    cudaStream_t s;
+    ck(cudaStreamCreate(&s), "stream");
+    std::mt19937 rng(316);
+    for (int gu : {16, 17, 18, 21, 22, 23, 29, 42, 12, 13, 8})        // STRATA_GU_FMTS
+        for (int dt : {20, 23, 42, 7, 8}) check(gu, dt, 512, 256, s, rng);   // STRATA_D_FMTS; IQ4_XS: n_ff % 256
+    check(21, 20, 2560, 640, s, rng);                                   // a model's shapes
+    check(21, 23, 2560, 768, s, rng);
+    if (do_bench) bench(s, rng);
+    std::printf("native_grouped_parity: %d failures\n", g_fail);
+    cudaStreamDestroy(s);
+    return g_fail ? 1 : 0;
+}

src/kernels/ngram.cpp

@@ -209,13 +209,9 @@ bool PleTable::open(const std::string& gguf_path, std::string& err, const PleIoO
         close();
         return false;
     }
-    // The direct reader has fixed 90-byte rows. Keep it for IQ4_NL; Q5_0's
-    // 110-byte rows use the mapped path until that reader supports variable rows.
-    if (impl_->q5_0 && io.mode == PleIo::Direct) {
-        err = "Q5_0 PLE requires --ple-io mmap";
-        close();
-        return false;
-    }
+    // PleReader's row_bytes has been a runtime parameter since the FP8 table (160 B rows) needed it; Q5_0's
+    // 110 B rows go through the exact same generic path (ple_reader_test --selftest covers both 90 and 110 B
+    // rows: straddling, caching, in-flight tickets, keep-alive). This refusal was stale.
     impl_->n_rows = t->shape[1];
     impl_->data = impl_->file->tensor_data(*t);
 

src/kernels/qsa_prompt_attn_parity.cpp

@@ -1,7 +1,7 @@
 // src/kernels/qsa_prompt_attn_parity.cpp - perf-review D-1: the tensor-core prompt attention (qsa_prompt_attn.hpp)
 // against the FP32 kernel it replaces (`qsa_decode_attn_batch`) and an FP64 host reference (GPU, synthetic, no model).
 //
-// Int8 and FP16 pools with random codes, scales and queries; selections shaped like the prompt path's (the 2,051
+// Int8, FP16 and Q4_0 pools with random codes, scales and queries; selections shaped like the prompt path's (the 2,051
 // widest, a recent window plus older cells that drift slowly from one query to the next, so neighbours share most
 // of them as they do in a real prompt; short contexts take every cell). Checks:
 //   1. against FP64, the new kernel's error is no larger than a small multiple of the old kernel's (both FP32 math);
@@ -12,6 +12,7 @@
 #include "strata/kernels/qsa.hpp"
 #include "strata/kernels/qsa_decode_attn.hpp"
 #include "strata/kernels/qsa_prompt_attn.hpp"
+#include "strata/kernels/kv_q4.hpp"
 
 #include <cuda_fp16.h>
 #include <cuda_runtime.h>
@@ -39,7 +40,7 @@ template <typename T> T* up(const std::vector<T>& h) {
 float h2f(uint16_t b) { __half h; *reinterpret_cast<uint16_t*>(&h) = b; return __half2float(h); }
 uint16_t f2h(float f) { __half h = __float2half(f); return *reinterpret_cast<uint16_t*>(&h); }
 
-int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
+int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16, 2 q4_0
     const k::QsaShapes s = k::qsa_real_shapes();
     const int64_t HD = s.head_dim, NKV = s.n_head_kv, NH = s.n_head, PS = s.page_size;
     const int64_t pages = (ctx + PS - 1) / PS, rows = pages * NKV * PS;
@@ -50,7 +51,22 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
     // pools; page table a shuffled permutation (the readers must follow it)
     std::vector<int8_t> kq, vq;
     std::vector<uint16_t> ks, vs, kh, vh;
-    if (fmt == 1) {
+    std::vector<uint8_t> k4, v4;   // q4_0: 8 blocks of {fp16 d, 16 code bytes} per row (kv_q4.hpp)
+    constexpr int64_t B4 = 18, ROW4 = 8 * B4;
+    if (fmt == 2) {
+        std::uniform_int_distribution<int> byte(0, 255);
+        // the scales as a q4_0 pool holds them: signed (ggml's d = max / -8) and spread over two decades, so a chunk's
+        // cells mix signs and magnitudes (a positive-only bound on them overflowed FP16 on real K/V)
+        std::uniform_real_distribution<float> lg(std::log(0.01f), std::log(1.0f));
+        auto sc4 = [&](std::mt19937& r) { return std::exp(lg(r)) * (byte(r) & 1 ? -1.0f : 1.0f); };
+        k4.resize(rows * ROW4); v4.resize(rows * ROW4);
+        for (auto* pool : {&k4, &v4})
+            for (int64_t b = 0; b < rows * 8; ++b) {
+                const uint16_t d = f2h(sc4(rng));
+                std::memcpy(pool->data() + b * B4, &d, 2);
+                for (int j = 0; j < 16; ++j) (*pool)[b * B4 + 2 + j] = (uint8_t) byte(rng);
+            }
+    } else if (fmt == 1) {
         kq.resize(rows * HD); vq.resize(rows * HD); ks.resize(rows * 4); vs.resize(rows * 4);
         for (auto& x : kq) x = (int8_t) code(rng);
         for (auto& x : vq) x = (int8_t) code(rng);
@@ -98,8 +114,19 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
         std::copy(v.begin(), v.end(), sel);
     }
     k::QsaAttnPools pl;
-    if (fmt == 1) { pl.k_q = up(kq); pl.v_q = up(vq); pl.k_scale = up(ks); pl.v_scale = up(vs); }
+    if (fmt == 2) { pl.k_q4 = up(k4); pl.v_q4 = up(v4); }
+    else if (fmt == 1) { pl.k_q = up(kq); pl.v_q = up(vq); pl.k_scale = up(ks); pl.v_scale = up(vs); }
     else { pl.k_pool = up(kh); pl.v_pool = up(vh); }
+    // a q4_0 value: block d / 32 of the row, element j = d % 32 in the low nibble of byte j (j < 16), else the high
+    // nibble of byte j - 16, minus 8, times the block's scale
+    auto q4v = [&](const std::vector<uint8_t>& pool, int64_t row, int64_t d) {
+        const uint8_t* blk = pool.data() + row * ROW4 + (d / 32) * B4;
+        uint16_t sc;
+        std::memcpy(&sc, blk, 2);
+        const int j = (int) (d % 32);
+        const int code = j < 16 ? (blk[2 + j] & 0x0F) : (blk[2 + j - 16] >> 4);
+        return (double) (code - 8) * h2f(sc);
+    };
     pl.page_table = up(table);
     const int32_t* d_ids = up(ids);
     const int32_t* d_steps = up(steps);
@@ -139,7 +166,8 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
                 const int64_t cell = sel[c], row = ((int64_t) table[cell / PS] * NKV + kvh) * PS + cell % PS;
                 double a = 0;
                 for (int64_t d = 0; d < HD; ++d) {
-                    const double kv = fmt == 1 ? (double) kq[row * HD + d] * h2f(ks[row * 4 + d / 64]) : h2f(kh[row * HD + d]);
+                    const double kv = fmt == 2 ? q4v(k4, row, d)
+                                      : fmt == 1 ? (double) kq[row * HD + d] * h2f(ks[row * 4 + d / 64]) : h2f(kh[row * HD + d]);
                     a += (double) q[(i * NH + h) * HD + d] * kv;
                 }
                 sco[c] = a / 16.0;
@@ -151,7 +179,8 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
                 double a = 0;
                 for (int64_t c = 0; c < w; ++c) {
                     const int64_t cell = sel[c], row = ((int64_t) table[cell / PS] * NKV + kvh) * PS + cell % PS;
-                    const double vv = fmt == 1 ? (double) vq[row * HD + d] * h2f(vs[row * 4 + d / 64]) : h2f(vh[row * HD + d]);
+                    const double vv = fmt == 2 ? q4v(v4, row, d)
+                                      : fmt == 1 ? (double) vq[row * HD + d] * h2f(vs[row * 4 + d / 64]) : h2f(vh[row * HD + d]);
                     a += sco[c] * vv;
                 }
                 const double r = a / l;
@@ -187,12 +216,13 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
     const bool ok2 = diff <= 1e-4 * scale;
     std::printf("%s %s ctx %lld, %lld queries: vs FP64 old %.3g new %.3g (output scale %.3g); new vs old %.3g (%.2g of "
                 "scale); %.3f -> %.3f ms per chunk (%.2fx)\n",
-                ok1 && ok2 ? "PASS" : "FAIL", fmt == 1 ? "int8" : "fp16", (long long) ctx, (long long) nq, err_old,
+                ok1 && ok2 ? "PASS" : "FAIL", fmt == 2 ? "q4_0" : fmt == 1 ? "int8" : "fp16", (long long) ctx, (long long) nq, err_old,
                 err_new, ref_scale, diff, diff / scale, ms_old / reps, ms_new / reps, ms_old / ms_new);
     cudaFree((void*) d_ids); cudaFree((void*) d_steps); cudaFree((void*) d_q); cudaFree(d_old); cudaFree(d_new);
     cudaFree(scratch);
     cudaFree((void*) pl.k_q); cudaFree((void*) pl.v_q); cudaFree((void*) pl.k_scale); cudaFree((void*) pl.v_scale);
     cudaFree((void*) pl.k_pool); cudaFree((void*) pl.v_pool); cudaFree((void*) pl.page_table);
+    cudaFree((void*) pl.k_q4); cudaFree((void*) pl.v_q4);
     return ok1 && ok2 ? 0 : 1;
 }
 }  // namespace
@@ -222,6 +252,8 @@ int main(int argc, char** argv) {
     fails += run(1, ctx, nq, reps);
 #if !defined(__HIP_PLATFORM_AMD__)
     fails += run(0, ctx, nq, reps);   // FP16 KV: the RDNA4 kernel takes int8 KV only
+    fails += run(2, ctx, nq, reps);   // Q4_0 KV (mode 4)
+    fails += run(2, 1500, std::min<int64_t>(nq, 1500), reps);
 #endif
     fails += run(1, 1500, std::min<int64_t>(nq, 1500), reps);   // short context: the selection is every cell
     fails += run(1, 2100, std::min<int64_t>(nq, 256), reps);    // the identity-to-sparse edge

src/kernels/qsa_topk_active_parity.cpp

@@ -0,0 +1,95 @@
+// Capacity 262144, live prefill up to the register boundary, and the legacy fallbacks.
+// Compare selected IDs only: unused output padding is not part of the API contract.
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_select.hpp"
+#include <cuda_runtime.h>
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+namespace {
+void ck(cudaError_t e) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "CUDA: %s\n", cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+struct Case { int64_t ctx, queries; int mode; };
+bool check(Case c) {
+    constexpr int64_t stride = 262144 / 4 + 2;
+    const auto s = k::qsa_real_shapes();
+    const int64_t cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    std::vector<int32_t> steps((size_t)(c.queries * k::kStepCount));
+    std::vector<float> scores((size_t)(c.queries * stride), -999.f);
+    std::mt19937 rng(742913);
+    std::uniform_real_distribution<float> random(0.f, 200.f);
+    for (int64_t i = 0; i < c.queries; ++i) {
+        auto* st = steps.data() + i * k::kStepCount;
+        const int64_t pos = c.ctx - c.queries + i;
+        st[k::kStepPos] = (int32_t)pos;
+        st[k::kStepNKv] = (int32_t)(pos + 1);
+        st[k::kStepNBid] = (int32_t)((pos + 1) / s.idx_block);
+        st[k::kStepWidth] = (int32_t)k::qsa_selection_width(pos + 1, s);
+        for (int64_t j = 0; j <= st[k::kStepNBid]; ++j)
+            scores[(size_t)(i * stride + j)] = c.mode == 1 ? 0.f : c.mode == 2 ? float(j % 7) : random(rng);
+        if ((pos + 1) % s.idx_block)
+            scores[(size_t)(i * stride + st[k::kStepNBid])] += 1e9f;
+    }
+    int64_t active = steps[(size_t)((c.queries - 1) * k::kStepCount + k::kStepNBid)] + 1;
+    if (c.mode == 3) active = 0;
+    if (c.mode == 4) active = stride + 1;
+    float* ds = nullptr;
+    int32_t *dt = nullptr, *da = nullptr, *db = nullptr;
+    ck(cudaMalloc(&ds, scores.size() * sizeof(float)));
+    ck(cudaMalloc(&dt, steps.size() * sizeof(int32_t)));
+    ck(cudaMalloc(&da, (size_t)(c.queries * cap) * sizeof(int32_t)));
+    ck(cudaMalloc(&db, (size_t)(c.queries * cap) * sizeof(int32_t)));
+    ck(cudaMemcpy(ds, scores.data(), scores.size() * sizeof(float), cudaMemcpyHostToDevice));
+    ck(cudaMemcpy(dt, steps.data(), steps.size() * sizeof(int32_t), cudaMemcpyHostToDevice));
+    // No bound: the exact capacity-based dispatch used by captured decode.
+    k::qsa_block_topk(ds, dt, c.queries, stride, cap, s, da, nullptr);
+    k::qsa_block_topk(ds, dt, c.queries, stride, cap, s, db, nullptr, active);
+    ck(cudaDeviceSynchronize());
+    std::vector<int32_t> a((size_t)(c.queries * cap)), b(a.size());
+    ck(cudaMemcpy(a.data(), da, a.size() * sizeof(int32_t), cudaMemcpyDeviceToHost));
+    ck(cudaMemcpy(b.data(), db, b.size() * sizeof(int32_t), cudaMemcpyDeviceToHost));
+    bool ok = true;
+    for (int64_t i = 0; i < c.queries; ++i) {
+        const int64_t w = steps[(size_t)(i * k::kStepCount + k::kStepWidth)];
+        if (!std::equal(a.begin() + i * cap, a.begin() + i * cap + w, b.begin() + i * cap)) {
+            std::fprintf(stderr, "FAIL ctx=%lld queries=%lld mode=%d row=%lld\n",
+                         (long long)c.ctx, (long long)c.queries, c.mode, (long long)i);
+            ok = false;
+            break;
+        }
+    }
+    ck(cudaFree(ds)); ck(cudaFree(dt)); ck(cudaFree(da)); ck(cudaFree(db));
+    return ok;
+}
+}
+
+int main() {
+    int devices = 0;
+    if (cudaGetDeviceCount(&devices) != cudaSuccess || devices == 0) {
+        std::puts("SKIP: no CUDA device");
+        return 77;
+    }
+    // the active-bound dispatch is Turing's in the engine; forced here so every CUDA card checks it
+#if defined(_WIN32)
+    _putenv_s("STRATA_TOPK_ACTIVE_ANY", "1");
+#else
+    setenv("STRATA_TOPK_ACTIVE_ANY", "1", 1);
+#endif
+    const Case cases[] = {
+        {1111, 256, 0}, {8192, 256, 0}, {32768, 256, 0}, {131072, 256, 0},
+        {131075, 255, 0}, {135164, 256, 0}, {135168, 256, 0}, {250022, 256, 0},
+        {262144, 256, 0}, {131072, 1, 0}, {131072, 8, 0}, {131072, 257, 0},
+        {131072, 256, 1}, {131072, 256, 2}, {131072, 256, 3}, {131072, 256, 4},
+    };
+    for (const auto c : cases) if (!check(c)) return 1;
+    std::puts("PASS: 16 top-k active-bound cases, selected IDs bitwise identical");
+    return 0;
+}

src/ngram/ple_reader_test.cpp

@@ -41,33 +41,33 @@ int g_fail = 0;
 
 constexpr uint64_t HEADER = 192;   // the real shard's data offset, so rows are misaligned the same way
 
-void expected_row(uint32_t row, uint8_t* out) {
-    for (uint32_t b = 0; b < ng::ROW_BYTES; ++b) out[b] = (uint8_t) ((row * 2654435761u + b * 97u) >> 7);
+void expected_row(uint32_t row, uint32_t rb, uint8_t* out) {
+    for (uint32_t b = 0; b < rb; ++b) out[b] = (uint8_t) ((row * 2654435761u + b * 97u) >> 7);
     std::memcpy(out, &row, 4);
 }
 
-bool make_table(const std::string& path, uint32_t rows) {
+bool make_table(const std::string& path, uint32_t rows, uint32_t rb) {
     std::ofstream f(path, std::ios::binary);
     std::vector<uint8_t> head(HEADER, 0xAB);
     f.write((const char*) head.data(), (std::streamsize) head.size());
-    uint8_t r[ng::ROW_BYTES];
+    std::vector<uint8_t> r(rb);
     for (uint32_t i = 0; i < rows; ++i) {
-        expected_row(i, r);
-        f.write((const char*) r, ng::ROW_BYTES);
+        expected_row(i, rb, r.data());
+        f.write((const char*) r.data(), rb);
     }
     return (bool) f;
 }
 
-bool check_rows(ng::PleReader& rd, const std::vector<uint32_t>& rows, uint32_t n_rows, const char* what) {
-    std::vector<uint8_t> out(rows.size() * ng::ROW_BYTES, 0xCC);
+bool check_rows(ng::PleReader& rd, const std::vector<uint32_t>& rows, uint32_t n_rows, uint32_t rb, const char* what) {
+    std::vector<uint8_t> out(rows.size() * rb, 0xCC);
     std::string err;
     const auto t = rd.issue(rows.data(), rows.size(), out.data());
     if (!rd.collect(t, err)) { CHECK(false, "%s: collect failed: %s", what, err.c_str()); return false; }
-    uint8_t want[ng::ROW_BYTES];
+    std::vector<uint8_t> want(rb);
     for (size_t i = 0; i < rows.size(); ++i) {
-        if (rows[i] >= n_rows) std::memset(want, 0, sizeof want);
-        else expected_row(rows[i], want);
-        if (std::memcmp(want, &out[i * ng::ROW_BYTES], ng::ROW_BYTES) != 0) {
+        if (rows[i] >= n_rows) std::memset(want.data(), 0, rb);
+        else expected_row(rows[i], rb, want.data());
+        if (std::memcmp(want.data(), &out[i * rb], rb) != 0) {
             CHECK(false, "%s: row %u (index %zu) differs", what, rows[i], i);
             return false;
         }
@@ -75,50 +75,53 @@ bool check_rows(ng::PleReader& rd, const std::vector<uint32_t>& rows, uint32_t n
     return true;
 }
 
-int selftest(const std::string& dir) {
+// row_bytes: ng::ROW_BYTES (90, IQ4_NL) is the production default; 110 (#296, OrcaRouter's Q5_0 PLE rows) is
+// run too, through the exact same generic row_bytes path -- nothing here is IQ4_NL-specific, so a second row
+// size run here is the correctness evidence for lifting ngram.cpp's "Q5_0 PLE requires --ple-io mmap" refusal.
+int selftest(const std::string& dir, uint32_t rb) {
     const uint32_t N = 500000;                          // 45 MB: large enough for thousands of distinct pages
-    const std::string path = dir + "/ple_reader_selftest.bin";
-    if (!make_table(path, N)) { std::fprintf(stderr, "cannot write %s\n", path.c_str()); return 2; }
+    const std::string path = dir + "/ple_reader_selftest_" + std::to_string(rb) + ".bin";
+    if (!make_table(path, N, rb)) { std::fprintf(stderr, "cannot write %s\n", path.c_str()); return 2; }
     std::mt19937 rng(7);
     for (bool thr : {false, true})
     for (uint64_t cache : {0ull, 4096ull}) {
         for (uint32_t inflight : {1u, 8u, 64u}) {
             ng::PleReader rd;
             std::string err;
-            CHECK(rd.open(path, HEADER, N, inflight, cache, err, thr), "open: %s", err.c_str());
+            CHECK(rd.open(path, HEADER, N, inflight, cache, err, thr, rb), "open: %s", err.c_str());
             // decode-shaped tickets: 16 random rows
             for (int t = 0; t < 200; ++t) {
                 std::vector<uint32_t> rows(16);
                 for (auto& r : rows) r = rng() % N;
-                check_rows(rd, rows, N, "decode");
+                check_rows(rd, rows, N, rb, "decode");
             }
-            // rows that straddle a 4 KiB boundary: byte offset of row r is HEADER + 90 r
+            // rows that straddle a 4 KiB boundary: byte offset of row r is HEADER + rb * r
             std::vector<uint32_t> straddle;
             for (uint32_t r = 0; r < N && straddle.size() < 64; ++r) {
-                const uint64_t a = HEADER + (uint64_t) r * ng::ROW_BYTES;
-                if (a / 4096 != (a + ng::ROW_BYTES - 1) / 4096) straddle.push_back(r);
+                const uint64_t a = HEADER + (uint64_t) r * rb;
+                if (a / 4096 != (a + rb - 1) / 4096) straddle.push_back(r);
             }
-            check_rows(rd, straddle, N, "straddle");
+            check_rows(rd, straddle, N, rb, "straddle");
             // duplicates, neighbours on one page, the first and last rows, and out-of-range rows
-            check_rows(rd, {5, 5, 6, 7, 5, 0, N - 1, N, 0xFFFFFFFFu, 44, 45}, N, "dedup/edges");
+            check_rows(rd, {5, 5, 6, 7, 5, 0, N - 1, N, 0xFFFFFFFFu, 44, 45}, N, rb, "dedup/edges");
             // a prefill-shaped ticket much larger than the in-flight window
             std::vector<uint32_t> bulk(20000);
             for (auto& r : bulk) r = rng() % N;
-            check_rows(rd, bulk, N, "bulk");
+            check_rows(rd, bulk, N, rb, "bulk");
             // two tickets in flight at once, collected in reverse order
             std::vector<uint32_t> a(16), b(16);
             for (auto& r : a) r = rng() % N;
             for (auto& r : b) r = rng() % N;
-            std::vector<uint8_t> oa(16 * ng::ROW_BYTES), ob(16 * ng::ROW_BYTES);
+            std::vector<uint8_t> oa(16 * rb), ob(16 * rb);
             const auto ta = rd.issue(a.data(), 16, oa.data());
             const auto tb = rd.issue(b.data(), 16, ob.data());
             CHECK(rd.collect(tb, err) && rd.collect(ta, err), "two tickets: %s", err.c_str());
-            uint8_t want[ng::ROW_BYTES];
+            std::vector<uint8_t> want(rb);
             for (int i = 0; i < 16; ++i) {
-                expected_row(a[i], want);
-                CHECK(!std::memcmp(want, &oa[i * ng::ROW_BYTES], ng::ROW_BYTES), "ticket a row %d", i);
-                expected_row(b[i], want);
-                CHECK(!std::memcmp(want, &ob[i * ng::ROW_BYTES], ng::ROW_BYTES), "ticket b row %d", i);
+                expected_row(a[i], rb, want.data());
+                CHECK(!std::memcmp(want.data(), &oa[i * rb], rb), "ticket a row %d", i);
+                expected_row(b[i], rb, want.data());
+                CHECK(!std::memcmp(want.data(), &ob[i * rb], rb), "ticket b row %d", i);
             }
             if (cache > 0) CHECK(rd.stats().cache_hits > 0, "the row cache never hit");
             CHECK(rd.cache_size() <= rd.cache_capacity(), "row cache exceeded its bound");
@@ -128,12 +131,12 @@ int selftest(const std::string& dir) {
     for (bool thr : {false, true}) {
         ng::PleReader rd;
         std::string err;
-        CHECK(rd.open(path, HEADER, N, 16, 0, err, thr), "open: %s", err.c_str());
+        CHECK(rd.open(path, HEADER, N, 16, 0, err, thr, rb), "open: %s", err.c_str());
         rd.set_injected_delay_us(3000);
         std::vector<uint32_t> rows(16);
         for (auto& r : rows) r = rng() % N;
         const double t0 = now_us();
-        check_rows(rd, rows, N, "delayed");
+        check_rows(rd, rows, N, rb, "delayed");
         CHECK(now_us() - t0 >= 3000, "injected delay not observed (%.0f us)", now_us() - t0);
         CHECK(rd.stats().late_injected > 0, "no read was held back");
     }
@@ -143,11 +146,11 @@ int selftest(const std::string& dir) {
     {
         ng::PleReader rd;
         std::string err;
-        CHECK(rd.open(path, HEADER, N, 16, 4096, err, true), "open: %s", err.c_str());
+        CHECK(rd.open(path, HEADER, N, 16, 4096, err, true, rb), "open: %s", err.c_str());
         rd.set_keepalive(20.0, 0.5);
         std::vector<uint32_t> rows(16);
         for (auto& r : rows) r = rng() % N;
-        check_rows(rd, rows, N, "keep-alive, first ticket");
+        check_rows(rd, rows, N, rb, "keep-alive, first ticket");
         const uint64_t reads0 = rd.snapshot().reads;
         const auto nap = [](int ms) { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); };
         nap(300);
@@ -161,14 +164,14 @@ int selftest(const std::string& dir) {
         CHECK(k3 == k2, "keep-alive went on after the window (%llu -> %llu)", (unsigned long long) k2,
               (unsigned long long) k3);
         CHECK(k2 <= k1 + 20, "keep-alive: %llu reads by the end of a 0.5 s window", (unsigned long long) k2);
-        check_rows(rd, rows, N, "keep-alive, cached ticket");   // the row cache serves all 16: no row read
+        check_rows(rd, rows, N, rb, "keep-alive, cached ticket");   // the row cache serves all 16: no row read
         CHECK(rd.snapshot().reads == reads0, "cached rows were read again");
         nap(200);
         CHECK(rd.snapshot().keepalive_reads >= k3 + 5, "the keep-alive did not start again (%llu -> %llu)",
               (unsigned long long) k3, (unsigned long long) rd.snapshot().keepalive_reads);
         std::vector<uint32_t> more(2000);
         for (auto& r : more) r = rng() % N;
-        check_rows(rd, more, N, "keep-alive, rows afterwards");
+        check_rows(rd, more, N, rb, "keep-alive, rows afterwards");
         rd.set_keepalive(0, 0.5);                           // off
         const uint64_t k4 = rd.snapshot().keepalive_reads;
         nap(200);
@@ -179,16 +182,16 @@ int selftest(const std::string& dir) {
     {   // the caller's thread does the reads: no worker, so no keep-alive
         ng::PleReader rd;
         std::string err;
-        CHECK(rd.open(path, HEADER, N, 16, 0, err, false), "open: %s", err.c_str());
+        CHECK(rd.open(path, HEADER, N, 16, 0, err, false, rb), "open: %s", err.c_str());
         rd.set_keepalive(20.0, 1.0);
         std::vector<uint32_t> rows(16);
         for (auto& r : rows) r = rng() % N;
-        check_rows(rd, rows, N, "keep-alive, caller thread");
+        check_rows(rd, rows, N, rb, "keep-alive, caller thread");
         std::this_thread::sleep_for(std::chrono::milliseconds(150));
         CHECK(rd.snapshot().keepalive_reads == 0, "keep-alive without the worker thread");
     }
     std::filesystem::remove(path);
-    std::printf("ple_reader selftest: %s\n", g_fail ? "FAILED" : "OK");
+    std::printf("ple_reader selftest (row_bytes=%u): %s\n", rb, g_fail ? "FAILED" : "OK");
     return g_fail ? 1 : 0;
 }
 
@@ -282,7 +285,13 @@ int main(int argc, char** argv) {
         else if (a == "--sync") sync_submit = true;
         else { std::fprintf(stderr, "usage: ple_reader_test --selftest [--dir D] | --gguf SHARD2 [--rows N] [--tokens F]\n"); return 2; }
     }
-    if (self) return selftest(dir);
+    if (self) {
+        // ng::ROW_BYTES (90, IQ4_NL, production default) and 110 (#296, OrcaRouter's Q5_0 PLE rows) through the
+        // same generic row_bytes path -- see the comment on selftest().
+        const int r90 = selftest(dir, ng::ROW_BYTES);
+        const int r110 = selftest(dir, 110);
+        return r90 != 0 ? r90 : r110;
+    }
     if (!gguf.empty()) return real(gguf, rows, tokens, inflight, direct_first, direct_only, sync_submit);
     std::fprintf(stderr, "nothing to do\n");
     return 2;

src/prefill/gdn_rec_parity.cu

@@ -0,0 +1,886 @@
+// src/prefill/gdn_rec_parity.cu - the prompt path's DeltaNet recurrence, bit for bit and timed.
+//
+// src/prefill/kernels.cu runs gdn_rec_kh_kernel (the three value heads that share a key head in one thread, the
+// inputs staged in blocks of 8 tokens) on CUDA sm_80+ cards that hold all 64 of its blocks at once, and
+// gdn_rec_cols_pipe_kernel elsewhere and with STRATA_GDN_KEYHEAD=0.  Both are copied below verbatim, with the output
+// norm (gdn_out_norm_kernel) and the norm as it was before, which also stored the normalized value in FP32 although
+// nothing reads it.  The two recurrences do the same arithmetic in the same order per value head and column, so the
+// output, the state after it and the FP16 output must be the same bits.
+//
+// Synthetic inputs of the model's shape (q/k L2-normalized per key head as gdn_l2_kernel leaves them), no model.
+//   gdn_rec_parity            the bit checks (T from 1 to 4099, a non-zero state as after an earlier chunk)
+//   gdn_rec_parity --bench    and the timings at T = 8192 and 32768 (one layer, all 48 value heads); the two
+//                             variants' launches alternate, since a card under load steps its clock down after a
+//                             fraction of a second (see native_grouped_parity); then both at 1 to 6 blocks per
+//                             SM, as on cards with more or fewer SMs (scale), and both with the engine's grids on
+//                             32 to all of this card's SMs, the others held by a sleeping kernel (fewer_sms)
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+#include <vector>
+
+namespace {
+
+constexpr int S = 128, HK = 16, HV = 48, C = 10240;   // as kernels.cu: head size, key heads, value heads, q|k|v width
+constexpr int RG = 4, RPG = S / RG;
+constexpr int CB = 32, NCB = S / CB;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+
+__device__ __forceinline__ float warp_sum(float v) {
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
+    return v;
+}
+__device__ __forceinline__ float sigm(float x) { return 1.0f / (1.0f + __expf(-x)); }
+__device__ __forceinline__ uint16_t hf(float f) { return __half_as_ushort(__float2half_rn(f)); }
+
+// ------------------------------------------------------------------ the engine's kernels (src/prefill/kernels.cu)
+// gdn_rec_cols_kernel with the next token's inputs (q/k rows, v, gate, beta) loaded into registers while this token
+// computes (software pipelining).  The same arithmetic in the same order: the same bits, and the same CB-column split.
+// STRATA_GDN_PIPELINE=0: gdn_rec_cols_kernel.
+__global__ void __launch_bounds__(CB * RG) gdn_rec_cols_pipe_kernel(float* __restrict__ state, const float* __restrict__ h,
+                                                                      const float* __restrict__ gate,
+                                                                      const float* __restrict__ beta,
+                                                                      float* __restrict__ oc_out, int64_t T) {
+    constexpr int NT = CB * RG, LPT = S / NT;   // threads, q/k rows loaded per thread
+    __shared__ float sk[S], sq[S], red[RG][CB];
+    const int head = blockIdx.x / NCB, cb = blockIdx.x % NCB;
+    const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
+    const int qh = head % HK;
+    float s[RPG];
+    float* base = state + ((size_t) (rg * RPG) * HV + head) * S + col;
+    const size_t rs = (size_t) HV * S;
+#pragma unroll
+    for (int r = 0; r < RPG; ++r) s[r] = base[r * rs];
+    float nq[LPT], nk[LPT], nv = 0.0f, ng = 0.0f, nb = 0.0f;
+    auto fetch = [&](int64_t t) {
+        const float* ht = h + t * C;
+#pragma unroll
+        for (int u = 0; u < LPT; ++u) { nq[u] = ht[qh * S + tid + u * NT]; nk[u] = ht[HK * S + qh * S + tid + u * NT]; }
+        nv = ht[2 * HK * S + head * S + col];
+        ng = gate[t * HV + head];
+        nb = beta[t * HV + head];
+    };
+    if (T > 0) fetch(0);
+    for (int64_t t = 0; t < T; ++t) {
+        float cq[LPT], ck[LPT];
+#pragma unroll
+        for (int u = 0; u < LPT; ++u) { cq[u] = nq[u]; ck[u] = nk[u]; }
+        const float cv = nv, cg = ng, cbt = nb;
+        __syncthreads();
+#pragma unroll
+        for (int u = 0; u < LPT; ++u) { sq[tid + u * NT] = cq[u]; sk[tid + u * NT] = ck[u]; }
+        __syncthreads();
+        if (t + 1 < T) fetch(t + 1);
+        const float g = __expf(cg);
+        float kv = 0.0f;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) kv = fmaf(s[r], sk[rg * RPG + r], kv);
+        red[rg][c] = kv;
+        __syncthreads();
+        const float kv_col = red[0][c] + red[1][c] + red[2][c] + red[3][c];
+        const float delta = (cv - g * kv_col) * cbt;
+        float o = 0.0f;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) {
+            s[r] = fmaf(g, s[r], sk[rg * RPG + r] * delta);
+            o = fmaf(s[r], sq[rg * RPG + r], o);
+        }
+        __syncthreads();
+        red[rg][c] = o;
+        __syncthreads();
+        if (rg == 0) oc_out[t * HV * S + head * S + col] = (red[0][c] + red[1][c] + red[2][c] + red[3][c]) * rsqrtf((float) S);
+    }
+#pragma unroll
+    for (int r = 0; r < RPG; ++r) base[r * rs] = s[r];
+}
+// The recurrence with one thread for the three value heads that share a key head (head % HK): column c of heads
+// qh, qh + 16 and qh + 32, row group rg.  gdn_rec_cols_pipe_kernel spends its time in shared memory, not in
+// arithmetic: every thread of a warp needs the same 32 q and k values per token (the k twice), and a warp receives one
+// such broadcast value per clock however wide the load.  Here every q/k value a thread loads feeds three heads, and a
+// token's k row goes into registers once for both of its uses.  The inputs come in blocks of GDN_TB tokens, copied to
+// shared memory by cp.async while the block before computes (one token ahead is shorter than a load from L2 takes),
+// and the two cross-row-group sums have their own arrays, so a token needs 2 __syncthreads instead of 5: the second
+// one of a token orders every read of rkv before the next token's writes, the next token's first one every read of ro
+// before the writes after it.  64 blocks instead of 192.  Per value head and column the same arithmetic in the same
+// order: the same bits (src/prefill/gdn_rec_parity.cu checks them and times the variants: 1.41x on a 4080 Super).
+// sm_80+ (gdn_keyhead_ok); STRATA_GDN_KEYHEAD=0: gdn_rec_cols_pipe_kernel.
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
+#define STRATA_GDN_CP_ASYNC 0   // Turing builds: plain copies (never launched there, see gdn_keyhead_ok)
+#else
+#define STRATA_GDN_CP_ASYNC 1
+#endif
+__device__ __forceinline__ void gdn_cp4(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *smem = *gmem;
+#endif
+}
+__device__ __forceinline__ void gdn_cp16(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *reinterpret_cast<float4*>(smem) = *reinterpret_cast<const float4*>(gmem);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_commit() {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.commit_group;\n" ::);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_wait_prev() {   // every group but the newest has landed
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.wait_group 1;\n" ::);
+#endif
+}
+constexpr int GDN_TB = 8, VPK = HV / HK;   // tokens per staged block, value heads per key head
+__global__ void __launch_bounds__(CB * RG) gdn_rec_kh_kernel(float* __restrict__ state, const float* __restrict__ h,
+                                                               const float* __restrict__ gate,
+                                                               const float* __restrict__ beta,
+                                                               float* __restrict__ oc_out, int64_t T) {
+    constexpr int TB = GDN_TB, NT = CB * RG, QKP = S / 4, VP = CB / 4;   // threads, 16-byte pieces of a q/k row, of v
+    __shared__ __align__(16) float sq[2][TB][S];
+    __shared__ __align__(16) float sk[2][TB][S];
+    __shared__ __align__(16) float sv[2][TB][VPK][CB];
+    __shared__ float sg[2][TB][VPK], sb[2][TB][VPK], rkv[VPK][RG][CB], ro[VPK][RG][CB];
+    const int qh = blockIdx.x / NCB, cb = blockIdx.x % NCB;
+    const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
+    float s[VPK][RPG];
+    const size_t rs = (size_t) HV * S;
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        const float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) s[j][r] = base[r * rs];
+    }
+    const int64_t nblk = (T + TB - 1) / TB;
+    auto stage = [&](int64_t k) {   // tokens [k * TB, k * TB + TB) into buffer k & 1
+        const int bb = (int) (k & 1);
+        const int64_t t0 = k * TB;
+        for (int p = tid; p < TB * 2 * QKP; p += NT) {
+            const int i = p / (2 * QKP), w = p % (2 * QKP), isk = w / QKP, jj = (w % QKP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(isk ? &sk[bb][i][jj] : &sq[bb][i][jj], h + (t0 + i) * C + (isk ? HK * S : 0) + qh * S + jj);
+        }
+        for (int p = tid; p < TB * VPK * VP; p += NT) {
+            const int i = p / (VPK * VP), w = p % (VPK * VP), j = w / VP, jj = (w % VP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(&sv[bb][i][j][jj], h + (t0 + i) * C + 2 * HK * S + (qh + j * HK) * S + cb * CB + jj);
+        }
+        for (int p = tid; p < 2 * TB * VPK; p += NT) {
+            const int isb = p / (TB * VPK), w = p % (TB * VPK), i = w / VPK, j = w % VPK;
+            if (t0 + i < T)
+                gdn_cp4(isb ? &sb[bb][i][j] : &sg[bb][i][j], (isb ? beta : gate) + (t0 + i) * HV + qh + j * HK);
+        }
+    };
+    if (nblk > 0) stage(0);
+    gdn_cp_commit();
+    for (int64_t k = 0; k < nblk; ++k) {
+        // buffer (k + 1) & 1 was read by block k - 1, whose last token's second __syncthreads every thread has passed
+        if (k + 1 < nblk) stage(k + 1);
+        gdn_cp_commit();
+        gdn_cp_wait_prev();
+        __syncthreads();
+        const int bb = (int) (k & 1);
+        const int n = (int) ((T - k * TB) < TB ? (T - k * TB) : TB);
+        for (int i = 0; i < n; ++i) {
+            const int64_t t = k * TB + i;
+            float kc[RPG];
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) kc[r] = sk[bb][i][rg * RPG + r];
+            float g[VPK], kv[VPK], delta[VPK], o[VPK];
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) { g[j] = __expf(sg[bb][i][j]); kv[j] = 0.0f; o[j] = 0.0f; }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r)
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) kv[j] = fmaf(s[j][r], kc[r], kv[j]);
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) rkv[j][rg][c] = kv[j];
+            __syncthreads();
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) {
+                const float kv_col = rkv[j][0][c] + rkv[j][1][c] + rkv[j][2][c] + rkv[j][3][c];
+                delta[j] = (sv[bb][i][j][c] - g[j] * kv_col) * sb[bb][i][j];
+            }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) {
+                const float qr = sq[bb][i][rg * RPG + r];
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) {
+                    s[j][r] = fmaf(g[j], s[j][r], kc[r] * delta[j]);
+                    o[j] = fmaf(s[j][r], qr, o[j]);
+                }
+            }
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) ro[j][rg][c] = o[j];
+            __syncthreads();
+            if (rg < VPK)   // row group j writes head j's output
+                oc_out[t * HV * S + (qh + rg * HK) * S + col] =
+                    (ro[rg][0][c] + ro[rg][1][c] + ro[rg][2][c] + ro[rg][3][c]) * rsqrtf((float) S);
+        }
+    }
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) base[r * rs] = s[j][r];
+    }
+}
+// the output norm over a head's 128 columns, into the FP16 copy the out projection reads (the FP32 output before the
+// norm stays in its scratch buffer)
+__global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
+                                                         float eps, const float* __restrict__ y,
+                                                         uint16_t* __restrict__ y16) {
+    __shared__ float wsum[4];
+    const int64_t t = blockIdx.x;
+    const int head = blockIdx.y, col = threadIdx.x;
+    const size_t at = (size_t) t * HV * S + (size_t) head * S + col;
+    const float oc = y[at];
+    float sp = warp_sum(oc * oc);
+    if ((col & 31) == 0) wsum[col >> 5] = sp;
+    __syncthreads();
+    const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
+    const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
+    y16[at] = hf(v);
+}
+
+// ------------------------------------------------------------------ the output norm before (0.1.31)
+__global__ void __launch_bounds__(S) gdn_out_norm_old_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
+                                                             float eps, float* __restrict__ y,
+                                                             uint16_t* __restrict__ y16) {
+    __shared__ float wsum[4];
+    const int64_t t = blockIdx.x;
+    const int head = blockIdx.y, col = threadIdx.x;
+    const size_t at = (size_t) t * HV * S + (size_t) head * S + col;
+    const float oc = y[at];
+    float sp = warp_sum(oc * oc);
+    if ((col & 31) == 0) wsum[col >> 5] = sp;
+    __syncthreads();
+    const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
+    const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
+    y[at] = v;
+    y16[at] = hf(v);
+}
+
+// ------------------------------------------------------------------ synthetic inputs
+__device__ __forceinline__ uint32_t mix(uint64_t x) {
+    x ^= x >> 33; x *= 0xff51afd7ed558ccdULL; x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53ULL; x ^= x >> 33;
+    return (uint32_t) x;
+}
+__device__ __forceinline__ float uni(uint64_t key) { return ((mix(key) >> 8) + 0.5f) * (1.0f / 16777216.0f); }
+__device__ __forceinline__ float gauss(uint64_t key) {
+    const float u1 = uni(key * 2 + 1), u2 = uni(key * 2 + 2);
+    return sqrtf(-2.0f * logf(u1)) * cosf(6.2831853f * u2);
+}
+__global__ void fill_gauss(float* x, int64_t n, float scale, uint64_t seed, float offset = 0.0f) {
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
+        x[i] = offset + scale * gauss(seed * 0x9E3779B97F4A7C15ULL + (uint64_t) i);
+}
+// the q and k rows L2-normalized per key head, as gdn_l2_kernel leaves them for the recurrence
+__global__ void l2_rows(float* h, float eps) {
+    float* x = h + (int64_t) blockIdx.y * C + (int64_t) blockIdx.x * S;
+    const float v = x[threadIdx.x];
+    __shared__ float part[4];
+    const float sq = warp_sum(v * v);
+    if ((threadIdx.x & 31) == 0) part[threadIdx.x >> 5] = sq;
+    __syncthreads();
+    x[threadIdx.x] = v * rsqrtf(part[0] + part[1] + part[2] + part[3] + eps);
+}
+// gate = log of the decay (mostly near 1, some strong), beta in (0, 1)
+__global__ void fill_gates(float* gate, float* beta, int64_t n, uint64_t seed) {
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) {
+        const float u = uni(seed * 0x9E3779B97F4A7C15ULL + 7 * (uint64_t) i);
+        gate[i] = -0.02f - 0.8f * u * u * u;
+        beta[i] = 1.0f / (1.0f + expf(-gauss(seed * 0xD1B54A32D192ED03ULL + (uint64_t) i)));
+    }
+}
+__global__ void count_diff(const uint32_t* a, const uint32_t* b, int64_t n, unsigned long long* out) {
+    unsigned long long d = 0;
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
+        d += a[i] != b[i];
+    if (d) atomicAdd(out, d);
+}
+
+// ------------------------------------------------------------------ holding SMs (fewer_sms)
+__device__ __forceinline__ uint64_t gtime() {
+    uint64_t t;
+    asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
+    return t;
+}
+__device__ __forceinline__ int sm_id() {
+    unsigned id;
+    asm volatile("mov.u32 %0, %%smid;" : "=r"(id));
+    return (int) id;
+}
+__device__ __forceinline__ void nap(unsigned ns) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+    __nanosleep(ns);
+#else
+    (void) ns;
+#endif
+}
+// limits that a run that works never reaches (a launch here takes milliseconds); they keep a run that does not work
+// from waiting forever, and the host drops its time
+constexpr uint64_t kHoldLimitNs = 1000000000ull, kWaitLimitNs = 500000000ull;
+// One block per SM: it asks for all the shared memory a block may have (launched with that much dynamic shared
+// memory), so no block of another kernel fits beside it.  It notes its SM and sleeps until release says gen.
+__global__ void hold_sm(unsigned long long* sitting, const volatile unsigned* release, unsigned gen, int* where,
+                        unsigned* late) {
+    where[blockIdx.x] = sm_id();
+    __threadfence();
+    atomicAdd(sitting, 1ull);
+    const uint64_t t0 = gtime();
+    while (*release != gen) {
+        if (gtime() - t0 > kHoldLimitNs) { atomicAdd(late, 1u); return; }
+        nap(20000);
+    }
+}
+// on the timed launch's stream, before it: returns once all holding blocks so far sit on their SMs
+__global__ void wait_sitting(const volatile unsigned long long* sitting, unsigned long long all, unsigned* late) {
+    const uint64_t t0 = gtime();
+    while (*sitting < all) {
+        if (gtime() - t0 > kWaitLimitNs) { atomicAdd(late, 1u); return; }
+        nap(1000);
+    }
+}
+__global__ void release_sms(volatile unsigned* release, unsigned gen) { *release = gen; }
+// Where the blocks of a grid land: launched like the recurrence it stands for (the same grid and block size, and as
+// much dynamic shared memory as lets as many blocks share an SM), every block notes its SM and stays until all of them
+// have landed, so they all sit at once as the recurrence's blocks do for the whole chunk.
+__global__ void land(int* where, unsigned* landed, unsigned* apart) {
+    extern __shared__ unsigned char land_smem[];   // only its size matters
+    if (threadIdx.x != 0 || threadIdx.y != 0) return;
+    where[blockIdx.x] = sm_id();
+    atomicAdd(landed, 1u);
+    const uint64_t t0 = gtime();
+    while (*(volatile unsigned*) landed < gridDim.x) {
+        if (gtime() - t0 > kWaitLimitNs) { atomicAdd(apart, 1u); return; }   // they did not all fit at once
+        nap(1000);
+    }
+    (void) land_smem;
+}
+
+template <typename T> T* dalloc(size_t n) {
+    T* p = nullptr;
+    ck(cudaMalloc((void**) &p, n * sizeof(T) + 16), "cudaMalloc");
+    return p;
+}
+unsigned long long diff_words(const void* a, const void* b, size_t bytes) {
+    static unsigned long long* d = dalloc<unsigned long long>(1);
+    ck(cudaMemset(d, 0, sizeof(*d)), "memset");
+    count_diff<<<1024, 256>>>((const uint32_t*) a, (const uint32_t*) b, (int64_t) (bytes / 4), d);
+    unsigned long long h = 0;
+    ck(cudaMemcpy(&h, d, sizeof(h), cudaMemcpyDeviceToHost), "diff");
+    return h;
+}
+
+struct Inputs {
+    int64_t T = 0;
+    float *h = nullptr, *gate = nullptr, *beta = nullptr, *z = nullptr, *gamma = nullptr, *state0 = nullptr;
+    explicit Inputs(int64_t t, uint64_t seed) : T(t) {
+        h = dalloc<float>((size_t) T * C);
+        gate = dalloc<float>((size_t) T * HV);
+        beta = dalloc<float>((size_t) T * HV);
+        z = dalloc<float>((size_t) T * HV * S);
+        gamma = dalloc<float>(S);
+        state0 = dalloc<float>((size_t) S * HV * S);
+        fill_gauss<<<4096, 256>>>(h, T * C, 1.0f, seed);
+        l2_rows<<<dim3(2 * HK, (unsigned) T), S>>>(h, 1e-6f);
+        fill_gates<<<1024, 256>>>(gate, beta, T * HV, seed + 1);
+        fill_gauss<<<4096, 256>>>(z, T * HV * S, 1.0f, seed + 2);
+        fill_gauss<<<1, 128>>>(gamma, S, 0.1f, seed + 3, 1.0f);
+        fill_gauss<<<1024, 256>>>(state0, (int64_t) S * HV * S, 0.05f, seed + 4);
+        ck(cudaDeviceSynchronize(), "inputs");
+    }
+    Inputs(const Inputs&) = delete;
+    Inputs& operator=(const Inputs&) = delete;
+    ~Inputs() { cudaFree(h); cudaFree(gate); cudaFree(beta); cudaFree(z); cudaFree(gamma); cudaFree(state0); }
+};
+
+enum Rec { kBefore, kKeyHead, kRecCount };
+void run_rec(Rec v, float* state, const Inputs& in, float* oc, cudaStream_t s = 0) {
+    if (v == kBefore)
+        gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, s>>>(state, in.h, in.gate, in.beta, oc, in.T);
+    else
+        gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG), 0, s>>>(state, in.h, in.gate, in.beta, oc, in.T);
+}
+
+int check_bits(int64_t T, uint64_t seed) {
+    Inputs in(T, seed);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state[kRecCount];
+    float* oc[kRecCount];
+    for (int v = 0; v < kRecCount; ++v) {
+        state[v] = dalloc<float>(st);
+        oc[v] = dalloc<float>(oc_n);
+        ck(cudaMemcpy(state[v], in.state0, st * 4, cudaMemcpyDeviceToDevice), "state copy");
+        run_rec((Rec) v, state[v], in, oc[v]);
+    }
+    ck(cudaDeviceSynchronize(), "recurrence");
+    int fails = 0;
+    const unsigned long long d_oc = diff_words(oc[kBefore], oc[kKeyHead], oc_n * 4),
+                             d_st = diff_words(state[kBefore], state[kKeyHead], st * 4);
+    if (d_oc || d_st) {
+        std::printf("FAIL T=%lld recurrence: %llu of %zu outputs and %llu of %zu state values differ\n", (long long) T,
+                    d_oc, oc_n, d_st, st);
+        ++fails;
+    }
+    // the output norm: before (in place, with the FP32 store) against now (the FP16 store only), on the same output
+    uint16_t* y16a = dalloc<uint16_t>(oc_n);
+    uint16_t* y16b = dalloc<uint16_t>(oc_n);
+    float* yb = dalloc<float>(oc_n);
+    ck(cudaMemcpy(yb, oc[kBefore], oc_n * 4, cudaMemcpyDeviceToDevice), "y copy");
+    gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, yb, y16b);
+    gdn_out_norm_old_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc[kBefore], y16a);
+    ck(cudaDeviceSynchronize(), "norm");
+    const unsigned long long d16 = diff_words(y16a, y16b, oc_n * 2);
+    if (d16) {
+        std::printf("FAIL T=%lld output norm: %llu of %zu FP16 pairs differ\n", (long long) T, d16, oc_n / 2);
+        ++fails;
+    }
+    std::printf("T=%-6lld %s\n", (long long) T, fails ? "DIFFERENT" : "the same bits (output, state, FP16 output)");
+    for (int v = 0; v < kRecCount; ++v) { cudaFree(state[v]); cudaFree(oc[v]); }
+    cudaFree(y16a); cudaFree(y16b); cudaFree(yb);
+    return fails;
+}
+
+float median(std::vector<float> v) {
+    std::sort(v.begin(), v.end());
+    return v.empty() ? 0.0f : v[v.size() / 2];
+}
+
+void bench(int64_t T) {
+    Inputs in(T, 0xB3);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state = dalloc<float>(st);
+    float* oc = dalloc<float>(oc_n);
+    uint16_t* y16 = dalloc<uint16_t>(oc_n);
+    cudaEvent_t e0, e1;
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    auto timed = [&](auto&& launch) {
+        ck(cudaMemcpy(state, in.state0, st * 4, cudaMemcpyDeviceToDevice), "state");
+        cudaEventRecord(e0);
+        launch();
+        cudaEventRecord(e1);
+        ck(cudaEventSynchronize(e1), "timed");
+        float ms = 0.0f;
+        cudaEventElapsedTime(&ms, e0, e1);
+        return ms;
+    };
+    const int warm = 3, rounds = T >= 16384 ? 10 : 20;
+    std::vector<float> t_rec[kRecCount], t_norm[2];
+    for (int i = 0; i < warm + rounds; ++i) {
+        for (int j = 0; j < kRecCount; ++j) {
+            const Rec v = (Rec) ((i + j) % kRecCount);   // a different variant first in each round
+            const float ms = timed([&] { run_rec(v, state, in, oc); });
+            if (i >= warm) t_rec[v].push_back(ms);
+        }
+    }
+    for (int i = 0; i < warm + rounds; ++i) {
+        for (int j = 0; j < 2; ++j) {
+            const int v = (i + j) % 2;
+            const float ms = timed([&] {
+                if (v == 0) gdn_out_norm_old_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc, y16);
+                else gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc, y16);
+            });
+            if (i >= warm) t_norm[v].push_back(ms);
+        }
+    }
+    const float r0 = median(t_rec[kBefore]), r1 = median(t_rec[kKeyHead]), n0 = median(t_norm[0]), n1 = median(t_norm[1]);
+    std::printf("  T = %lld tokens, one layer (48 value heads), medians of %d alternating runs:\n", (long long) T, rounds);
+    std::printf("    recurrence, before (gdn_rec_cols_pipe_kernel)  %8.2f ms  (%.3f us per token)\n", r0,
+                1000.0f * r0 / (float) T);
+    std::printf("    recurrence, gdn_rec_kh_kernel                  %8.2f ms  (%.3f us per token)  %.2fx\n", r1,
+                1000.0f * r1 / (float) T, r0 / r1);
+    std::printf("    output norm, before (FP32 and FP16 store)      %8.2f ms\n", n0);
+    std::printf("    output norm, FP16 store only                   %8.2f ms  %.2fx\n", n1, n0 / n1);
+    std::printf("    36 DeltaNet layers per %lld-token block: recurrence + norm %.0f ms -> %.0f ms (-%.0f ms)\n",
+                (long long) T, 36.0f * (r0 + n0), 36.0f * (r1 + n1), 36.0f * (r0 + n0 - r1 - n1));
+    cudaEventDestroy(e0);
+    cudaEventDestroy(e1);
+    cudaFree(state); cudaFree(oc); cudaFree(y16);
+}
+
+// The two kernels at a given number of blocks per SM, as on cards with more or fewer SMs than this one: the kernel
+// before puts ceil(192 / SMs) of its blocks on an SM, gdn_rec_kh_kernel ceil(64 / SMs), and the busiest SM sets the
+// pace (every block walks the whole chunk).  b blocks per SM here: b rounds of one block per SM, each round one launch
+// of SMs blocks (the first heads; split in launches of at most the engine's grid), every launch on its own stream with
+// its own state and output, all started together, so the block scheduler puts one block of each round on every SM.
+// The kernels are the ones above, unchanged.
+void scale(int64_t T) {
+    int dev = 0, sms = 0, per_sm[kRecCount] = {};
+    ck(cudaGetDevice(&dev), "device");
+    ck(cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev), "SMs");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kBefore], gdn_rec_cols_pipe_kernel, CB * RG, 0), "occupancy");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kKeyHead], gdn_rec_kh_kernel, CB * RG, 0), "occupancy");
+    constexpr int kMaxB = 6, kMaxLaunches = 6;
+    const int full[kRecCount] = {HV * NCB, HK * NCB};   // the grids as the engine launches them
+    auto parts = [&](int v) { return (sms + full[v] - 1) / full[v]; };   // launches per round
+    Inputs in(T, 0xC7);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state[kMaxLaunches];
+    float* oc[kMaxLaunches];
+    cudaStream_t stream[kMaxLaunches];
+    cudaEvent_t e0, e1, done[kMaxLaunches];
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    for (int c = 0; c < kMaxLaunches; ++c) {
+        state[c] = dalloc<float>(st);
+        oc[c] = dalloc<float>(oc_n);
+        ck(cudaStreamCreateWithFlags(&stream[c], cudaStreamNonBlocking), "stream");
+        ck(cudaEventCreate(&done[c]), "event");
+    }
+    struct Case { int v, b; };
+    std::vector<Case> cases;
+    for (int v = 0; v < kRecCount; ++v)
+        for (int b = 1; b <= std::min(kMaxB, per_sm[v]); ++b)
+            if (b * parts(v) <= kMaxLaunches) cases.push_back({v, b});
+    auto run = [&](const Case& cs) {
+        for (int c = 0; c < kMaxLaunches; ++c) ck(cudaMemcpy(state[c], in.state0, st * 4, cudaMemcpyDeviceToDevice), "state");
+        ck(cudaDeviceSynchronize(), "sync");
+        const int np = parts(cs.v), n = cs.b * np;
+        cudaEventRecord(e0, stream[0]);
+        for (int c = 0; c < n; ++c) {
+            const int k = c % np, grid = sms / np + (k < sms % np ? 1 : 0);   // round c / np, part k
+            if (c > 0) cudaStreamWaitEvent(stream[c], e0, 0);
+            if (cs.v == kBefore)
+                gdn_rec_cols_pipe_kernel<<<grid, dim3(CB, RG), 0, stream[c]>>>(state[c], in.h, in.gate, in.beta, oc[c], T);
+            else
+                gdn_rec_kh_kernel<<<grid, dim3(CB, RG), 0, stream[c]>>>(state[c], in.h, in.gate, in.beta, oc[c], T);
+            cudaEventRecord(done[c], stream[c]);
+        }
+        for (int c = 1; c < n; ++c) cudaStreamWaitEvent(stream[0], done[c], 0);
+        cudaEventRecord(e1, stream[0]);
+        ck(cudaEventSynchronize(e1), "scale");
+        float ms = 0.0f;
+        cudaEventElapsedTime(&ms, e0, e1);
+        return ms;
+    };
+    const int warm = 2, rounds = 9;
+    std::vector<std::vector<float>> t(cases.size());
+    for (int i = 0; i < warm + rounds; ++i)
+        for (size_t j = 0; j < cases.size(); ++j) {
+            const size_t k = (i + j) % cases.size();   // a different case first in each round
+            const float ms = run(cases[k]);
+            if (i >= warm) t[k].push_back(ms);
+        }
+    auto us = [&](int v, int b) {   // per token, or < 0 if not measured
+        for (size_t k = 0; k < cases.size(); ++k)
+            if (cases[k].v == v && cases[k].b == b) return 1000.0f * median(t[k]) / (float) T;
+        return -1.0f;
+    };
+    std::printf("  per token at a given number of blocks per SM (%lld tokens, medians of %d alternating runs; this card has "
+                "%d SMs):\n", (long long) T, rounds, sms);
+    std::printf("    blocks per SM   before      gdn_rec_kh_kernel\n");
+    for (int b = 1; b <= kMaxB; ++b) {
+        const float u0 = us(kBefore, b), u1 = us(kKeyHead, b);
+        if (u0 < 0 && u1 < 0) continue;
+        char a0[32] = "-", a1[32] = "-";
+        if (u0 >= 0) std::snprintf(a0, sizeof a0, "%.3f us", u0);
+        if (u1 >= 0) std::snprintf(a1, sizeof a1, "%.3f us", u1);
+        std::printf("    %8d        %-11s %s\n", b, a0, a1);
+    }
+    std::printf("  so on a card with N SMs (the kernel before puts ceil(192 / N) blocks on an SM, gdn_rec_kh_kernel "
+                "ceil(64 / N), at most %d):\n", per_sm[kKeyHead]);
+    const int bands[][2] = {{192, 0}, {96, 191}, {64, 95}, {48, 63}, {39, 47}, {32, 38}};
+    for (const auto& bd : bands) {
+        const int nsm = bd[0], b0 = (192 + nsm - 1) / nsm, b1 = (64 + nsm - 1) / nsm;
+        const float u0 = us(kBefore, b0), u1 = us(kKeyHead, b1);
+        char range[32];
+        if (bd[1]) std::snprintf(range, sizeof range, "%d-%d SMs", bd[0], bd[1]);
+        else std::snprintf(range, sizeof range, "%d+ SMs", bd[0]);
+        if (u0 < 0 || u1 < 0)
+            std::printf("    %-12s before %d / new %d per SM: not measured on this card\n", range, b0, b1);
+        else
+            std::printf("    %-12s before %d / new %d per SM: %.3f -> %.3f us per token, %.2fx\n", range, b0, b1, u0, u1,
+                        u0 / u1);
+    }
+    for (int c = 0; c < kMaxLaunches; ++c) {
+        cudaFree(state[c]);
+        cudaFree(oc[c]);
+        cudaStreamDestroy(stream[c]);
+        cudaEventDestroy(done[c]);
+    }
+    cudaEventDestroy(e0);
+    cudaEventDestroy(e1);
+}
+
+// The engine's grids on N of this card's SMs.  scale() puts as many blocks on every SM as the busiest SM of a card with
+// N SMs gets; on that card only some SMs get that many (192 blocks on 80 SMs: 3 on 32 SMs, 2 on the other 48), the
+// rest of the card has less to do, and the estimate came out 4-9 % above the real launch on two cards.  Here the
+// recurrences run as the engine launches them, while hold_sm keeps all but N SMs: its blocks go out first, a small
+// kernel on the timed launch's stream waits until they all sit, and one after the timed launch lets them go.  land()
+// shows for every N where the two grids land: none on a held SM, and how many blocks share the busiest SM.  The
+// recurrences are the ones above, unchanged; L2, memory and clock stay this card's.
+void fewer_sms(int64_t T) {
+    int dev = 0, sms = 0, optin = 0, smem_sm = 0, reserved = 0, hold_per_sm = 0, per_sm[kRecCount] = {};
+    ck(cudaGetDevice(&dev), "device");
+    ck(cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev), "SMs");
+    ck(cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev), "shared memory");
+    ck(cudaDeviceGetAttribute(&smem_sm, cudaDevAttrMaxSharedMemoryPerMultiprocessor, dev), "shared memory");
+    ck(cudaDeviceGetAttribute(&reserved, cudaDevAttrReservedSharedMemoryPerBlock, dev), "shared memory");
+    for (const void* f : {(const void*) hold_sm, (const void*) land}) {
+        ck(cudaFuncSetAttribute(f, cudaFuncAttributeMaxDynamicSharedMemorySize, optin), "shared memory");
+        ck(cudaFuncSetAttribute(f, cudaFuncAttributePreferredSharedMemoryCarveout, (int) cudaSharedmemCarveoutMaxShared),
+           "carveout");
+    }
+    // Loaded now rather than at a first launch: with lazy loading (CUDA 12's default) a kernel's first launch may wait
+    // until the device is idle, and so for hold_sm, which waits for release_sms.
+    cudaFuncAttributes fa[kRecCount], fx;
+    ck(cudaFuncGetAttributes(&fa[kBefore], gdn_rec_cols_pipe_kernel), "attributes");
+    ck(cudaFuncGetAttributes(&fa[kKeyHead], gdn_rec_kh_kernel), "attributes");
+    for (const void* f : {(const void*) hold_sm, (const void*) wait_sitting, (const void*) release_sms, (const void*) land})
+        ck(cudaFuncGetAttributes(&fx, f), "attributes");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&hold_per_sm, hold_sm, 1, optin), "occupancy");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kBefore], gdn_rec_cols_pipe_kernel, CB * RG, 0), "occupancy");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kKeyHead], gdn_rec_kh_kernel, CB * RG, 0), "occupancy");
+    const int grid[kRecCount] = {HV * NCB, HK * NCB};   // as the engine launches them
+    std::printf("  the engine's grids on N of this card's %d SMs (%lld tokens; the other SMs held by a sleeping kernel, see "
+                "fewer_sms):\n", sms, (long long) T);
+    const int left = smem_sm - optin - reserved;   // shared memory a holding block leaves on its SM
+    if (hold_per_sm != 1 || (int) fa[kBefore].sharedSizeBytes + reserved <= left ||
+        (int) fa[kKeyHead].sharedSizeBytes + reserved <= left) {
+        std::printf("    not on this card: a block of a recurrence would fit beside a holding block (%d bytes left)\n",
+                    left);
+        return;
+    }
+    // land(): as much dynamic shared memory as still lets as many of its blocks share an SM as of the recurrence
+    int land_smem[kRecCount];
+    for (int v = 0; v < kRecCount; ++v) {
+        int lo = 0, hi = optin;
+        while (lo < hi) {
+            const int mid = (lo + hi + 1) / 2;
+            int nb = 0;
+            ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&nb, land, CB * RG, mid), "occupancy");
+            if (nb >= per_sm[v]) lo = mid;
+            else hi = mid - 1;
+        }
+        land_smem[v] = lo;
+    }
+    // N: all SMs, and below that the SM counts of cards and the band edges, as long as both grids fit at once
+    std::vector<int> ns = {sms};
+    for (const int n : {170, 144, 132, 128, 112, 108, 96, 95, 84, 82, 80, 76, 72, 70, 66, 64, 63, 60, 56, 52, 48, 47, 46, 44,
+                        40, 39, 38, 36, 34, 32})
+        if (n < sms && (int64_t) n * per_sm[kBefore] >= grid[kBefore] && (int64_t) n * per_sm[kKeyHead] >= grid[kKeyHead])
+            ns.push_back(n);
+
+    Inputs in(T, 0xD5);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state = dalloc<float>(st);
+    float* oc = dalloc<float>(oc_n);
+    unsigned long long* sitting = dalloc<unsigned long long>(1);
+    unsigned* flags = dalloc<unsigned>(4);
+    unsigned *release = flags, *late = flags + 1, *landed = flags + 2, *apart = flags + 3;
+    int* where = dalloc<int>((size_t) sms + grid[kBefore]);   // the holding blocks' SMs, then land()'s
+    ck(cudaMemset(sitting, 0, sizeof(*sitting)), "memset");
+    ck(cudaMemset(flags, 0, 4 * sizeof(unsigned)), "memset");
+    cudaStream_t s_hold, s_run;
+    ck(cudaStreamCreateWithFlags(&s_hold, cudaStreamNonBlocking), "stream");
+    ck(cudaStreamCreateWithFlags(&s_run, cudaStreamNonBlocking), "stream");
+    cudaEvent_t e0, e1;
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    unsigned long long all = 0;   // holding blocks launched so far: sitting counts every one that sat down
+    unsigned gen = 0;
+    auto hold = [&](int n) {   // all but n SMs, before the next launch on s_run
+        if (n >= sms) return;
+        ++gen;
+        all += (unsigned long long) (sms - n);
+        hold_sm<<<sms - n, 1, optin, s_hold>>>(sitting, release, gen, where, late);
+        ck(cudaGetLastError(), "hold_sm");
+        const cudaError_t q = cudaStreamQuery(s_hold);   // hands the launch to the GPU now (Windows collects launches)
+        if (q != cudaErrorNotReady) ck(q, "hold_sm");
+        (void) cudaGetLastError();   // not ready is no error
+        wait_sitting<<<1, 1, 0, s_run>>>(sitting, all, late);
+    };
+    auto let_go = [&](int n) {
+        if (n < sms) release_sms<<<1, 1, 0, s_run>>>(release, gen);
+    };
+    auto settle = [&](const char* what) {
+        ck(cudaStreamSynchronize(s_run), what);
+        ck(cudaStreamSynchronize(s_hold), what);
+    };
+    auto count = [&](const unsigned* c) {   // after settle()
+        unsigned h = 0;
+        ck(cudaMemcpy(&h, c, sizeof(h), cudaMemcpyDeviceToHost), "count");
+        return h;
+    };
+    auto done = [&] {
+        cudaEventDestroy(e0);
+        cudaEventDestroy(e1);
+        cudaStreamDestroy(s_hold);
+        cudaStreamDestroy(s_run);
+        cudaFree(state); cudaFree(oc); cudaFree(sitting); cudaFree(flags); cudaFree(where);
+    };
+
+    // where the grids land, once per N
+    struct Landing { bool held = true; int busiest[kRecCount] = {}, on[kRecCount] = {}; };
+    std::vector<Landing> lands(ns.size());
+    for (size_t i = 0; i < ns.size(); ++i) {
+        const int n = ns[i];
+        for (int v = 0; v < kRecCount; ++v) {
+            bool sat = false;
+            unsigned apart0 = 0;
+            for (int attempt = 0; attempt < 2 && !sat; ++attempt) {
+                const unsigned late0 = count(late);
+                apart0 = count(apart);
+                ck(cudaMemsetAsync(landed, 0, sizeof(unsigned), s_run), "memset");
+                hold(n);
+                land<<<grid[v], dim3(CB, RG), land_smem[v], s_run>>>(where + sms, landed, apart);
+                ck(cudaGetLastError(), "land");
+                let_go(n);
+                settle("land");
+                sat = count(late) == late0;
+            }
+            if (!sat) {   // twice: the holding kernel and the launches beside it do not run at the same time here
+                std::printf("    not on this card: the other SMs could not be held (the kernel holding them and the "
+                            "launches beside it did not run at the same time)\n");
+                done();
+                return;
+            }
+            std::vector<int> w((size_t) sms + grid[v]);
+            ck(cudaMemcpy(w.data(), where, w.size() * sizeof(int), cudaMemcpyDeviceToHost), "where");
+            std::map<int, int> held, blocks;
+            for (int b = 0; b < (n < sms ? sms - n : 0); ++b) ++held[w[b]];
+            for (int b = 0; b < grid[v]; ++b) ++blocks[w[sms + b]];
+            bool ok = count(apart) == apart0 && (int) held.size() == (n < sms ? sms - n : 0) && (int) blocks.size() <= n;
+            int most = 0, on = 0;
+            for (const auto& sb : blocks) {
+                if (held.count(sb.first)) ok = false;
+                if (sb.second > most) { most = sb.second; on = 0; }
+                if (sb.second == most) ++on;
+            }
+            lands[i].held = lands[i].held && ok;
+            lands[i].busiest[v] = most;
+            lands[i].on[v] = on;
+        }
+    }
+
+    // the timings: every N and recurrence once per round, a different one first in each round
+    struct Case { size_t i; Rec v; };
+    std::vector<Case> cases;
+    for (size_t i = 0; i < ns.size(); ++i)
+        if (lands[i].held)
+            for (int v = 0; v < kRecCount; ++v) cases.push_back({i, (Rec) v});
+    auto run = [&](const Case& cs) {   // ms, or < 0 if the SMs were not held for the whole launch
+        const int n = ns[cs.i];
+        const unsigned late0 = count(late);
+        ck(cudaMemcpyAsync(state, in.state0, st * 4, cudaMemcpyDeviceToDevice, s_run), "state");
+        hold(n);
+        ck(cudaEventRecord(e0, s_run), "event");
+        run_rec(cs.v, state, in, oc, s_run);
+        ck(cudaEventRecord(e1, s_run), "event");
+        let_go(n);
+        settle("fewer SMs");
+        if (count(late) != late0) return -1.0f;
+        float ms = 0.0f;
+        ck(cudaEventElapsedTime(&ms, e0, e1), "event");
+        return ms;
+    };
+    const int warm = 2, rounds = 15, max_dropped = 5;
+    std::vector<std::vector<float>> t(cases.size());
+    int dropped = 0;
+    for (int r = 0; r < warm + rounds && dropped <= max_dropped; ++r)
+        for (size_t j = 0; j < cases.size() && dropped <= max_dropped; ++j) {
+            const size_t k = (r + j) % cases.size();
+            const float ms = run(cases[k]);
+            if (ms < 0.0f) ++dropped;
+            else if (r >= warm) t[k].push_back(ms);
+        }
+    if (dropped > max_dropped) {   // each such run waits for a limit of a second or so
+        std::printf("    stopped: in %d runs the other SMs were not held for the whole launch\n", dropped);
+        done();
+        return;
+    }
+    auto us = [&](size_t i, int v) {   // per token, or < 0 if not measured
+        for (size_t k = 0; k < cases.size(); ++k)
+            if (cases[k].i == i && cases[k].v == v && !t[k].empty()) return 1000.0f * median(t[k]) / (float) T;
+        return -1.0f;
+    };
+    auto blocks_on = [](char* out, size_t size, int most, int on) {
+        std::snprintf(out, size, "%d block%s on %d SM%s", most, most == 1 ? "" : "s", on, on == 1 ? "" : "s");
+    };
+    std::printf("  (medians of %d alternating runs; L2, memory and clock stay this card's)\n", rounds);
+    std::printf("      N   before: busiest SM   new: busiest SM      us per token, before -> new\n");
+    for (size_t i = 0; i < ns.size(); ++i) {
+        if (!lands[i].held) {
+            std::printf("    %3d   the other SMs were not held (a block landed on one, or a wait ran out)\n", ns[i]);
+            continue;
+        }
+        char b0[40], b1[40];
+        blocks_on(b0, sizeof b0, lands[i].busiest[kBefore], lands[i].on[kBefore]);
+        blocks_on(b1, sizeof b1, lands[i].busiest[kKeyHead], lands[i].on[kKeyHead]);
+        const float u0 = us(i, kBefore), u1 = us(i, kKeyHead);
+        if (u0 < 0.0f || u1 < 0.0f) std::printf("    %3d   %-20s %-20s not measured\n", ns[i], b0, b1);
+        else std::printf("    %3d   %-20s %-20s %.3f -> %.3f   %.2fx\n", ns[i], b0, b1, u0, u1, u0 / u1);
+    }
+    if (dropped) std::printf("    (%d runs dropped: the other SMs were not held for the whole launch)\n", dropped);
+    std::printf("  so on a card with N SMs, as measured here (the kernel before puts ceil(192 / N) blocks on the busiest SM, "
+                "gdn_rec_kh_kernel ceil(64 / N)):\n");
+    const int bands[][2] = {{96, 191}, {64, 95}, {48, 63}, {39, 47}, {32, 38}};
+    for (const auto& bd : bands) {
+        const int b0 = (192 + bd[0] - 1) / bd[0], b1 = (64 + bd[0] - 1) / bd[0];
+        float lo = 0.0f, hi = 0.0f;
+        int cnt = 0, nmin = 0, nmax = 0;
+        for (size_t i = 0; i < ns.size(); ++i) {
+            const float u0 = us(i, kBefore), u1 = us(i, kKeyHead);
+            if (ns[i] < bd[0] || ns[i] > bd[1] || u0 < 0.0f || u1 < 0.0f) continue;
+            const float x = u0 / u1;
+            if (cnt == 0 || x < lo) lo = x;
+            if (cnt == 0 || x > hi) hi = x;
+            if (cnt == 0 || ns[i] < nmin) nmin = ns[i];
+            if (cnt == 0 || ns[i] > nmax) nmax = ns[i];
+            ++cnt;
+        }
+        char range[32];
+        std::snprintf(range, sizeof range, "%d-%d SMs", bd[0], bd[1]);
+        if (cnt > 0)
+            std::printf("    %-12s before %d / new %d per SM: %.2fx to %.2fx (%d to %d SMs)\n", range, b0, b1, lo, hi, nmin,
+                        nmax);
+        else if (bd[0] > sms)
+            std::printf("    %-12s before %d / new %d per SM: more SMs than this card has (see the estimate above)\n", range,
+                        b0, b1);
+        else
+            std::printf("    %-12s before %d / new %d per SM: not measured\n", range, b0, b1);
+    }
+    done();
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    bool do_bench = false;
+    for (int i = 1; i < argc; ++i)
+        if (std::strcmp(argv[i], "--bench") == 0) do_bench = true;
+    cudaDeviceProp p{};
+    ck(cudaGetDeviceProperties(&p, 0), "device");
+    std::printf("gdn_rec_parity on %s (sm_%d%d)\n", p.name, p.major, p.minor);
+    int fails = 0;
+    const int64_t Ts[] = {1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 33, 257, 1000, 4099};
+    uint64_t seed = 11;
+    for (const int64_t T : Ts) fails += check_bits(T, seed++);
+    if (do_bench) {
+        std::printf("--bench: the prompt path's DeltaNet recurrence\n");
+        bench(8192);
+        bench(32768);
+        scale(8192);
+        fewer_sms(8192);
+    }
+    std::printf("gdn_rec_parity: %d failures\n", fails);
+    return fails ? 1 : 0;
+}

src/prefill/gemm.cu

@@ -42,6 +42,23 @@ void ck(cublasStatus_t s, const char* what) {
     }
 }
 
+// #247/#325: on Windows (seen on gfx1201), hipBLAS can return success with the correct BF16/FP16 product for some
+// shapes (hc up once T >= 96, the router) and still leave hipErrorInvalidValue set, which the next kernel's error
+// check turns into an exit. The multiply has finished, so that one stale error is cleared after a GEMM that succeeded;
+// any other error still stops the engine. Windows only: on Linux a stale hipErrorInvalidValue is a real error from
+// an earlier call and keeps being reported. A no-op everywhere else (CUDA compiles none of it).
+#if defined(__HIPCC__) && defined(_WIN32)
+void absorb_hipblas_sticky(const char* what) {
+    const hipError_t sticky = hipGetLastError();
+    if (sticky == hipSuccess || sticky == hipErrorInvalidValue) return;
+    std::fprintf(stderr, "prefill gemm: %s left %s\n", what, hipGetErrorString(sticky));
+    std::exit(1);
+}
+#define STRATA_ABSORB_HIPBLAS_STICKY(what) absorb_hipblas_sticky(what)
+#else
+#define STRATA_ABSORB_HIPBLAS_STICKY(what) ((void) 0)
+#endif
+
 // A setup call whose failure the engine survives (the handle keeps its defaults), as before #240 - but said.
 void note(cublasStatus_t s, const char* what) {
     if (s != CUBLAS_STATUS_SUCCESS) std::fprintf(stderr, "prefill gemm: %s: cuBLAS status %d (continuing)\n", what, (int) s);
@@ -363,6 +380,7 @@ void Gemm::bf16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64
 #if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
     if (try_hipblaslt(hipblaslt_state_, strata::prefill::hipblaslt::InputType::bf16, X, W, Y, T, N, K, ldy,
                       beta, stream_)) {
+        STRATA_ABSORB_HIPBLAS_STICKY("hipBLASLt bf16");
         return;
     }
 #endif
@@ -371,6 +389,7 @@ void Gemm::bf16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64
                     CUDA_R_16BF, (int) K, X, CUDA_R_16BF, (int) K, &beta, Y, CUDA_R_32F, (int) ldy,
                     CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT),
        "cublasGemmEx");
+    STRATA_ABSORB_HIPBLAS_STICKY("cublasGemmEx");
 }
 
 void Gemm::f16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy,
@@ -381,6 +400,7 @@ void Gemm::f16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_
 #if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
     if (try_hipblaslt(hipblaslt_state_, strata::prefill::hipblaslt::InputType::f16, X, W, Y, T, N, K, ldy,
                       beta, stream_)) {
+        STRATA_ABSORB_HIPBLAS_STICKY("hipBLASLt f16");
         return;
     }
 #endif
@@ -388,6 +408,7 @@ void Gemm::f16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_
                     CUDA_R_16F, (int) K, X, CUDA_R_16F, (int) K, &beta, Y, CUDA_R_32F, (int) ldy,
                     CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT),
        "cublasGemmEx f16");
+    STRATA_ABSORB_HIPBLAS_STICKY("cublasGemmEx f16");
 }
 
 void Gemm::native(const uint16_t* X, int ggml_type, const void* W_blocks, float* Y, int64_t T, int64_t N, int64_t K,

src/prefill/ggml_cuda_host.cu

@@ -80,17 +80,30 @@ const ggml_cuda_device_info & ggml_cuda_info() {
             d.smpbo = prop.sharedMemPerBlock;
             d.integrated = false;
             d.supports_cooperative_launch = false;
-#else
-            d.cc = 100 * strata::cc_major_of(prop.major) + 10 * strata::cc_minor_of(prop.minor);   // STRATA_EMULATE_CC
-            d.smpbo = strata::smem_optin_of((int) prop.sharedMemPerBlockOptin);
-            d.integrated = prop.integrated != 0;
-            d.supports_cooperative_launch = prop.cooperativeLaunch != 0;
-#endif
             d.nsm = prop.multiProcessorCount;
             d.smpb = prop.sharedMemPerBlock;
+            d.warp_size = prop.warpSize;
+#else
+            // #542: the fields MMQ's choices read come from cudaDeviceGetAttribute, which is ABI-stable: a
+            // cudaDeviceProp filled by an older libcudart than the headers (a CUDA 13 build that links a CUDA 12
+            // libcudart.so) is shifted - sharedMemPerBlockOptin read 1 there, and every i-quant lost MMQ (#420's
+            // fits()).  In a matched build both give the same values.
+            auto attr = [id](cudaDeviceAttr a) {
+                int v = 0;
+                CUDA_CHECK(cudaDeviceGetAttribute(&v, a, id));
+                return v;
+            };
+            d.cc = 100 * strata::cc_major_of(attr(cudaDevAttrComputeCapabilityMajor)) +
+                   10 * strata::cc_minor_of(attr(cudaDevAttrComputeCapabilityMinor));   // STRATA_EMULATE_CC
+            d.smpbo = strata::smem_optin_of(attr(cudaDevAttrMaxSharedMemoryPerBlockOptin));
+            d.integrated = attr(cudaDevAttrIntegrated) != 0;
+            d.supports_cooperative_launch = attr(cudaDevAttrCooperativeLaunch) != 0;
+            d.nsm = attr(cudaDevAttrMultiProcessorCount);
+            d.smpb = (size_t) attr(cudaDevAttrMaxSharedMemoryPerBlock);
+            d.warp_size = attr(cudaDevAttrWarpSize);
+#endif
             d.vmm = false;
             d.total_vram = prop.totalGlobalMem;
-            d.warp_size = prop.warpSize;
             d.physical_device = id;
             d.physical_share_count = 1;
             d.virtual_index = 0;

src/prefill/kernels.cu

@@ -433,8 +433,172 @@ __global__ void __launch_bounds__(CB * RG) gdn_rec_cols_pipe_kernel(float* __res
 #pragma unroll
     for (int r = 0; r < RPG; ++r) base[r * rs] = s[r];
 }
+#if !defined(__HIPCC__)
+// The recurrence with one thread for the three value heads that share a key head (head % HK): column c of heads
+// qh, qh + 16 and qh + 32, row group rg.  gdn_rec_cols_pipe_kernel spends its time in shared memory, not in
+// arithmetic: every thread of a warp needs the same 32 q and k values per token (the k twice), and a warp receives one
+// such broadcast value per clock however wide the load.  Here every q/k value a thread loads feeds three heads, and a
+// token's k row goes into registers once for both of its uses.  The inputs come in blocks of GDN_TB tokens, copied to
+// shared memory by cp.async while the block before computes (one token ahead is shorter than a load from L2 takes),
+// and the two cross-row-group sums have their own arrays, so a token needs 2 __syncthreads instead of 5: the second
+// one of a token orders every read of rkv before the next token's writes, the next token's first one every read of ro
+// before the writes after it.  64 blocks instead of 192.  Per value head and column the same arithmetic in the same
+// order: the same bits (src/prefill/gdn_rec_parity.cu checks them and times the variants: 1.41x on a 4080 Super).
+// sm_80+ cards that hold its 64 blocks at once (gdn_keyhead_ok); STRATA_GDN_KEYHEAD=0: gdn_rec_cols_pipe_kernel.
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
+#define STRATA_GDN_CP_ASYNC 0   // Turing builds: plain copies (never launched there, see gdn_keyhead_ok)
+#else
+#define STRATA_GDN_CP_ASYNC 1
+#endif
+__device__ __forceinline__ void gdn_cp4(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *smem = *gmem;
+#endif
+}
+__device__ __forceinline__ void gdn_cp16(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *reinterpret_cast<float4*>(smem) = *reinterpret_cast<const float4*>(gmem);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_commit() {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.commit_group;\n" ::);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_wait_prev() {   // every group but the newest has landed
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.wait_group 1;\n" ::);
+#endif
+}
+constexpr int GDN_TB = 8, VPK = HV / HK;   // tokens per staged block, value heads per key head
+__global__ void __launch_bounds__(CB * RG) gdn_rec_kh_kernel(float* __restrict__ state, const float* __restrict__ h,
+                                                               const float* __restrict__ gate,
+                                                               const float* __restrict__ beta,
+                                                               float* __restrict__ oc_out, int64_t T) {
+    constexpr int TB = GDN_TB, NT = CB * RG, QKP = S / 4, VP = CB / 4;   // threads, 16-byte pieces of a q/k row, of v
+    __shared__ __align__(16) float sq[2][TB][S];
+    __shared__ __align__(16) float sk[2][TB][S];
+    __shared__ __align__(16) float sv[2][TB][VPK][CB];
+    __shared__ float sg[2][TB][VPK], sb[2][TB][VPK], rkv[VPK][RG][CB], ro[VPK][RG][CB];
+    const int qh = blockIdx.x / NCB, cb = blockIdx.x % NCB;
+    const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
+    float s[VPK][RPG];
+    const size_t rs = (size_t) HV * S;
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        const float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) s[j][r] = base[r * rs];
+    }
+    const int64_t nblk = (T + TB - 1) / TB;
+    auto stage = [&](int64_t k) {   // tokens [k * TB, k * TB + TB) into buffer k & 1
+        const int bb = (int) (k & 1);
+        const int64_t t0 = k * TB;
+        for (int p = tid; p < TB * 2 * QKP; p += NT) {
+            const int i = p / (2 * QKP), w = p % (2 * QKP), isk = w / QKP, jj = (w % QKP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(isk ? &sk[bb][i][jj] : &sq[bb][i][jj], h + (t0 + i) * C + (isk ? HK * S : 0) + qh * S + jj);
+        }
+        for (int p = tid; p < TB * VPK * VP; p += NT) {
+            const int i = p / (VPK * VP), w = p % (VPK * VP), j = w / VP, jj = (w % VP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(&sv[bb][i][j][jj], h + (t0 + i) * C + 2 * HK * S + (qh + j * HK) * S + cb * CB + jj);
+        }
+        for (int p = tid; p < 2 * TB * VPK; p += NT) {
+            const int isb = p / (TB * VPK), w = p % (TB * VPK), i = w / VPK, j = w % VPK;
+            if (t0 + i < T)
+                gdn_cp4(isb ? &sb[bb][i][j] : &sg[bb][i][j], (isb ? beta : gate) + (t0 + i) * HV + qh + j * HK);
+        }
+    };
+    if (nblk > 0) stage(0);
+    gdn_cp_commit();
+    for (int64_t k = 0; k < nblk; ++k) {
+        // buffer (k + 1) & 1 was read by block k - 1, whose last token's second __syncthreads every thread has passed
+        if (k + 1 < nblk) stage(k + 1);
+        gdn_cp_commit();
+        gdn_cp_wait_prev();
+        __syncthreads();
+        const int bb = (int) (k & 1);
+        const int n = (int) ((T - k * TB) < TB ? (T - k * TB) : TB);
+        for (int i = 0; i < n; ++i) {
+            const int64_t t = k * TB + i;
+            float kc[RPG];
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) kc[r] = sk[bb][i][rg * RPG + r];
+            float g[VPK], kv[VPK], delta[VPK], o[VPK];
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) { g[j] = __expf(sg[bb][i][j]); kv[j] = 0.0f; o[j] = 0.0f; }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r)
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) kv[j] = fmaf(s[j][r], kc[r], kv[j]);
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) rkv[j][rg][c] = kv[j];
+            __syncthreads();
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) {
+                const float kv_col = rkv[j][0][c] + rkv[j][1][c] + rkv[j][2][c] + rkv[j][3][c];
+                delta[j] = (sv[bb][i][j][c] - g[j] * kv_col) * sb[bb][i][j];
+            }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) {
+                const float qr = sq[bb][i][rg * RPG + r];
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) {
+                    s[j][r] = fmaf(g[j], s[j][r], kc[r] * delta[j]);
+                    o[j] = fmaf(s[j][r], qr, o[j]);
+                }
+            }
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) ro[j][rg][c] = o[j];
+            __syncthreads();
+            if (rg < VPK)   // row group j writes head j's output
+                oc_out[t * HV * S + (qh + rg * HK) * S + col] =
+                    (ro[rg][0][c] + ro[rg][1][c] + ro[rg][2][c] + ro[rg][3][c]) * rsqrtf((float) S);
+        }
+    }
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) base[r * rs] = s[j][r];
+    }
+}
+// gdn_rec_kh_kernel where it pays: a CUDA card with cp.async (sm_80+) that holds all 64 of its blocks at once (each
+// walks the whole chunk, so blocks left for a second wave would double the time).  The busiest SM sets the pace: from
+// 64 SMs up this kernel has one block per SM, below that two on some SMs, while the kernel before has ceil(192 / SMs).
+// With the engine's grids on fewer SMs (gdn_rec_parity --bench), on a 4080 SUPER / a 3090: 1.40-1.42x / 1.28-1.29x at
+// 64 SMs and more, 1.02-1.04x / 1.03-1.06x at 48 to 63 (two blocks against four: a draw, slower on neither),
+// 1.28-1.31x / 1.28-1.30x at 39 to 47, 1.53-1.57x / 1.54-1.55x at 32 to 38.  Per call, from the current device (a
+// layer split can mix cards).
+bool gdn_keyhead_ok() {
+    static const bool off = [] { const char* v = std::getenv("STRATA_GDN_KEYHEAD"); return v != nullptr && std::atoi(v) == 0; }();
+    if (off) return false;
+    static int known[64] = {};   // per device: 0 not asked yet, 1 yes, 2 no
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+    if (known[dev] == 0) {
+        int major = 0, sms = 0, per_sm = 0;
+        const bool yes = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+                         cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev) == cudaSuccess && major >= 8 &&
+                         cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm, gdn_rec_kh_kernel, CB * RG, 0) ==
+                             cudaSuccess &&
+                         (int64_t) per_sm * sms >= (int64_t) HK * NCB;
+        if (!yes) cudaGetLastError();
+        known[dev] = yes ? 1 : 2;
+    }
+    return known[dev] == 1;
+}
+#endif
+// the output norm over a head's 128 columns, into the FP16 copy the out projection reads (the FP32 output before the
+// norm stays in its scratch buffer)
 __global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
-                                                         float eps, float* __restrict__ y, uint16_t* __restrict__ y16) {
+                                                         float eps, const float* __restrict__ y,
+                                                         uint16_t* __restrict__ y16) {
     __shared__ float wsum[4];
     const int64_t t = blockIdx.x;
     const int head = blockIdx.y, col = threadIdx.x;
@@ -445,7 +609,6 @@ __global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict
     __syncthreads();
     const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
     const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
-    y[at] = v;
     y16[at] = hf(v);
 }
 
@@ -748,6 +911,11 @@ void gdn_recurrence(float* state, const float* h, const float* gate, const float
         gdn_rec_kernel<<<HV, dim3(S, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, z, gamma, eps, y, y16, T);
     } else {
         static const bool pipe = [] { const char* v = std::getenv("STRATA_GDN_PIPELINE"); return v == nullptr || std::atoi(v) != 0; }();
+#if !defined(__HIPCC__)
+        if (pipe && gdn_keyhead_ok())   // the value heads of a key head in one thread (same bits)
+            gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);
+        else
+#endif
         if (pipe)   // the software-pipelined loads (same bits)
             gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);
         else

src/prefill/moe_fused.cu

@@ -0,0 +1,459 @@
+// src/prefill/moe_fused.cu - see include/strata/prefill/moe_fused.hpp (#136: the Q2_0 pack's prompt experts on
+// Strata's own int8 tensor-core kernels, with the grouping on the GPU).
+//
+// The arithmetic.  A Q2_0 block is 64 weights w = d_w * (q - 1), q in 0..3, four codes per byte, weight j of a block
+// in byte j / 4 at bits 2 * (j % 4).  An activation block of 32 values is x = d_x * a with a = round(x / d_x) in
+// -127..127.  Over one 32-value half-block
+//     sum w x = d_w * d_x * (sum q a - sum a),
+// so the int8 product takes the codes as they are (0..3 is a valid s8) and the offset is the activations' own sum,
+// stored with the block.  The dot product (|sum q a| <= 32 * 3 * 127 < 2^22) becomes a float exactly with one
+// integer add: as_float(0x4B400000 + dot) = 1.5 * 2^23 + dot, and the block stores c = -(1.5 * 2^23 + sum a), so
+// as_float(0x4B400000 + dot) + c = dot - sum a with no rounding.  Per output and 64-block that is
+//     acc += d_w * (d_x0 * t0 + d_x1 * t1)        (t = dot - sum a of each half)
+// - two integer adds, two float adds and three multiply-adds.
+//
+// The K order.  mma.sync m16n8k32 takes four consecutive k of a row (A) or column (B) per register.  Four consecutive
+// weights are one code byte; four codes of one register instead come from the same 2-bit field of four consecutive
+// bytes - (word >> 2t) & 0x03030303 holds weights t, t+4, t+8, t+12 of a 16-weight word.  So the kernels use that order
+// for the k of a half-block: mma k index 16h + 4t + j stands for weight (and activation) 16h + t + 4j.  The
+// activations are stored in it (perm32), and a lane's two B registers (h = 0 and 1) are adjacent: one 8-byte load.
+#include "strata/prefill/moe_fused.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <mutex>
+
+namespace strata::prefill::fused {
+namespace {
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "prefill fused experts: %s: %s\n", what, cudaGetErrorString(e));
+        std::exit(1);
+    }
+}
+
+constexpr int AB = 80;                  // activation bytes per 64 values: 64 codes, then {d0, c0, d1, c1}
+constexpr int MAGIC = 0x4B400000;       // the bits of 1.5 * 2^23
+constexpr float MAGICF = 12582912.0f;
+// The Strata Q2_0 blob (moe_mmq.cu strata_q2_kernel, kernels.cu blob_dequant_kernel): gate/up codes [1280][640 B]
+// (rows interleaved: gate of feature f at row 2f, its up at 2f + 1), down codes [2560][160 B], gate/up scales
+// [1280][40] fp16, down scales [2560][10] fp16.
+constexpr size_t O_GU_CODES = 0, O_D_CODES = (size_t) 1280 * 640, O_GU_SC = O_D_CODES + (size_t) 2560 * 160,
+                 O_D_SC = O_GU_SC + (size_t) 1280 * 40 * 2;
+constexpr int THREADS = 512;            // 16 warps: 4 along the weight rows (64 each) x 4 along the tile rows (16 each)
+constexpr int WROWS = 256;              // weight rows per work item: 128 features of gate/up, or 256 outputs of down
+constexpr int STAGES = 4;               // cp.async pipeline depth, one 64-weight block of K per stage
+constexpr int STAGE_BYTES = WROWS * 16 + kTileRows * AB;
+constexpr size_t smem_bytes(bool gu) {
+    return (size_t) WROWS * (gu ? 40 : 10) * 2 + STAGES * STAGE_BYTES + kTileRows * 4;
+}
+
+// the stored position of value k of a 32-value half-block (see "The K order")
+__host__ __device__ __forceinline__ int perm32(int k) { return 8 * (k & 3) + 4 * (k >> 4) + ((k & 15) >> 2); }
+
+struct Tables {
+    int32_t *cnt, *off, *fill, *ts;      // per expert: rows, first row (E + 1), placement cursor, first tile (E + 1)
+    int2* tiles;                         // {expert, first row}, in expert order
+};
+size_t tiles_at(int n_expert) { return ((size_t) (4 * n_expert + 2) * 4 + 15) / 16 * 16; }
+Tables tables(void* scratch, int n_expert) {
+    int32_t* p = (int32_t*) scratch;
+    return {p, p + n_expert, p + 2 * n_expert + 1, p + 3 * n_expert + 1,
+            (int2*) ((uint8_t*) scratch + tiles_at(n_expert))};
+}
+
+// ---- activations: one warp per 64 values
+__global__ void quant_act_kernel(const float* __restrict__ x, int64_t nblk, uint8_t* __restrict__ xa) {
+    const int64_t w = (int64_t) blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32;
+    const int lane = threadIdx.x & 31;
+    if (w >= nblk) return;
+    const float v0 = x[w * 64 + lane], v1 = x[w * 64 + 32 + lane];
+    float a0 = fabsf(v0), a1 = fabsf(v1);
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) {
+        a0 = fmaxf(a0, __shfl_xor_sync(0xffffffffu, a0, o));
+        a1 = fmaxf(a1, __shfl_xor_sync(0xffffffffu, a1, o));
+    }
+    const int q0 = a0 > 0.0f ? __float2int_rn(v0 * (127.0f / a0)) : 0;
+    const int q1 = a1 > 0.0f ? __float2int_rn(v1 * (127.0f / a1)) : 0;
+    int s0 = q0, s1 = q1;
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) {
+        s0 += __shfl_xor_sync(0xffffffffu, s0, o);
+        s1 += __shfl_xor_sync(0xffffffffu, s1, o);
+    }
+    uint8_t* out = xa + w * AB;
+    out[perm32(lane)] = (uint8_t) (int8_t) q0;
+    out[32 + perm32(lane)] = (uint8_t) (int8_t) q1;
+    if (lane == 0)
+        *(float4*) (out + 64) = make_float4(a0 / 127.0f, -(MAGICF + (float) s0), a1 / 127.0f, -(MAGICF + (float) s1));
+}
+
+// ---- grouping: counts, offsets and tiles, placement
+__global__ void count_kernel(const int32_t* __restrict__ ids, int64_t n, int E, int32_t* __restrict__ cnt) {
+    extern __shared__ int32_t hist[];
+    for (int e = threadIdx.x; e < E; e += blockDim.x) hist[e] = 0;
+    __syncthreads();
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) {
+        const int e = ids[i];
+        if ((unsigned) e < (unsigned) E) atomicAdd(&hist[e], 1);
+    }
+    __syncthreads();
+    for (int e = threadIdx.x; e < E; e += blockDim.x)
+        if (hist[e]) atomicAdd(&cnt[e], hist[e]);
+}
+
+// exclusive prefix sum over the block (blockDim a multiple of 32); *total = the block's sum
+__device__ int block_excl_sum(int v, int* total, int* sh) {
+    const int lane = threadIdx.x & 31, w = threadIdx.x >> 5, nw = blockDim.x >> 5;
+    int x = v;
+#pragma unroll
+    for (int o = 1; o < 32; o <<= 1) {
+        const int y = __shfl_up_sync(0xffffffffu, x, o);
+        if (lane >= o) x += y;
+    }
+    if (lane == 31) sh[w] = x;
+    __syncthreads();
+    if (w == 0) {
+        int s = lane < nw ? sh[lane] : 0;
+#pragma unroll
+        for (int o = 1; o < 32; o <<= 1) {
+            const int y = __shfl_up_sync(0xffffffffu, s, o);
+            if (lane >= o) s += y;
+        }
+        sh[lane] = s;
+    }
+    __syncthreads();
+    const int r = x - v + (w > 0 ? sh[w - 1] : 0);
+    *total = sh[nw - 1];
+    __syncthreads();   // sh is reused by the next call
+    return r;
+}
+
+// one block, a thread per expert (E <= 1024)
+__global__ void scan_kernel(Tables tb, int E) {
+    __shared__ int sh[32];
+    const int e = threadIdx.x;
+    const int c = e < E ? tb.cnt[e] : 0, nt = (c + kTileRows - 1) / kTileRows;
+    int rows = 0, tiles = 0;
+    const int o = block_excl_sum(c, &rows, sh), to = block_excl_sum(nt, &tiles, sh);
+    if (e < E) {
+        tb.off[e] = o;
+        tb.fill[e] = o;
+        tb.ts[e] = to;
+        for (int i = 0; i < nt; ++i) tb.tiles[to + i] = make_int2(e, o + i * kTileRows);
+    }
+    if (e == 0) {
+        tb.off[E] = rows;
+        tb.ts[E] = tiles;
+    }
+}
+
+__global__ void place_kernel(const int32_t* __restrict__ ids, int64_t n, int k, int E, Tables tb,
+                             int32_t* __restrict__ slot, int32_t* __restrict__ src) {
+    const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= n) return;
+    const int e = ids[i];
+    if ((unsigned) e >= (unsigned) E) { slot[i] = 0; return; }   // (the router never writes one)
+    const int p = atomicAdd(&tb.fill[e], 1);
+    slot[i] = p;
+    src[p] = (int32_t) (i / k);
+}
+
+// ---- the expert products
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+__device__ __forceinline__ void cp16(void* dst, const void* src) {
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(dst)),
+                 "l"(src));
+}
+__device__ __forceinline__ void cp4(void* dst, const void* src) {
+    asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"((unsigned) __cvta_generic_to_shared(dst)),
+                 "l"(src));
+}
+__device__ __forceinline__ void cp_commit() { asm volatile("cp.async.commit_group;\n" ::); }
+template <int N> __device__ __forceinline__ void cp_wait() { asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); }
+// d = A (16 x 32 s8, row) * B (32 x 8 s8, col), int32, from zero
+__device__ __forceinline__ void mma_s8(int (&d)[4], const uint32_t (&a)[4], uint32_t b0, uint32_t b1) {
+    asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%10,%10,%10};\n"
+        : "=r"(d[0]), "=r"(d[1]), "=r"(d[2]), "=r"(d[3])
+        : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1), "r"(0));
+}
+#endif
+
+// One work item = (tile of up to 64 routed rows of expert e, a block of 256 weight rows); a persistent grid walks
+// the batch's items (their count is on the device).  GU: gate/up rows of 128 features against the rows' tokens'
+// activations (`act` per token, 2560 values), SwiGLU, H to int8 per 32 features into `out` (per row, 640 values).
+// !GU: down rows (256 outputs) against H (`act` per row), FP32 into `dm` per row.
+// Warp (wf, wt): weight rows 64 wf.. as four m16 tiles (GU: a tile = 8 features, its gate rows as mma rows 0-7 and
+// its up rows as 8-15, so a lane holds gate and up of one feature: SwiGLU in registers), tile rows 16 wt.. as two n8.
+template <bool GU>
+__global__ void __launch_bounds__(THREADS, 1)
+expert_kernel(const Batch b, const Tables tb, const uint8_t* __restrict__ act, const int32_t* __restrict__ src,
+              uint8_t* __restrict__ out, float* __restrict__ dm) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+    constexpr int NKB = GU ? 40 : 10;                 // 64-weight blocks along K
+    constexpr int NFB = GU ? 5 : 10;                  // weight-row blocks per tile
+    constexpr int CODE_LD = GU ? 640 : 160;           // code bytes per weight row
+    constexpr size_t O_CODES = GU ? O_GU_CODES : O_D_CODES, O_SC = GU ? O_GU_SC : O_D_SC;
+    constexpr int ACT_LD = NKB * AB;                  // bytes per activation row: 3200 (a token), 800 (a row's H)
+    constexpr unsigned M2 = 0x03030303u;
+    extern __shared__ __align__(16) uint8_t smem[];
+    const __half* wsc = (const __half*) smem;                     // [256][NKB] the work item's weight scales
+    uint8_t* stages = smem + WROWS * NKB * 2;                     // [STAGES] x {codes [256][16], act [64][AB]}
+    int* srow = (int*) (stages + STAGES * STAGE_BYTES);           // [64] the activation row of each tile row
+
+    const int tid = threadIdx.x, lane = tid & 31, warp = tid >> 5, g = lane >> 2, tig = lane & 3;
+    const int wf = warp & 3, nb0 = 16 * (warp >> 2);
+    int ra[4], rb[4];                                             // the local weight rows of mma rows g and g + 8
+#pragma unroll
+    for (int i = 0; i < 4; ++i) {
+        if (GU) { ra[i] = 2 * (32 * wf + 8 * i + g); rb[i] = ra[i] + 1; }
+        else { ra[i] = 64 * wf + 16 * i + g; rb[i] = ra[i] + 8; }
+    }
+    const int t0 = tb.ts[b.e0], nwork = (tb.ts[b.e1] - t0) * NFB;
+    for (int w = blockIdx.x; w < nwork; w += gridDim.x) {
+        const int2 tl = tb.tiles[t0 + w / NFB];
+        const int fb = w % NFB, e = tl.x, row0 = tl.y, nrows = min(kTileRows, tb.off[e + 1] - row0);
+        const uint8_t* blob = b.blob[e - b.e0];
+        const int rbase = fb * WROWS;
+        __syncthreads();                                          // the previous item is done with the buffers
+        if (tid < kTileRows) {
+            const int r = row0 + min(tid, nrows - 1);             // rows past the tile's end repeat its last one
+            srow[tid] = GU ? src[r] : r;
+        }
+        for (int c = tid; c < WROWS * 5; c += THREADS) {          // the scales: 5 x 16 B (GU) or 5 x 4 B a row
+            const int r = c / 5, q = c % 5;
+            if (GU) cp16(smem + r * 80 + q * 16, blob + O_SC + (size_t) (rbase + r) * 80 + q * 16);
+            else cp4(smem + r * 20 + q * 4, blob + O_SC + (size_t) (rbase + r) * 20 + q * 4);
+        }
+        __syncthreads();                                          // srow
+        auto load = [&](int kb) {
+            uint8_t* st = stages + (kb % STAGES) * STAGE_BYTES;
+            if (tid < WROWS) cp16(st + tid * 16, blob + O_CODES + (size_t) (rbase + tid) * CODE_LD + kb * 16);
+            const int a = tid - (THREADS - kTileRows * 5);        // the last 320 threads: 64 rows x 5 x 16 B
+            if (a >= 0) {
+                const int r = a / 5, q = a % 5;
+                cp16(st + WROWS * 16 + r * AB + q * 16, act + (size_t) srow[r] * ACT_LD + kb * AB + q * 16);
+            }
+        };
+#pragma unroll
+        for (int s = 0; s < STAGES - 1; ++s) {
+            if (s < NKB) load(s);
+            cp_commit();
+        }
+        // a warp whose rows are all past the tile's end only takes part in the loads
+        const bool on0 = nb0 < nrows, on1 = nb0 + 8 < nrows;
+        float acc[4][2][4];
+#pragma unroll
+        for (int i = 0; i < 4; ++i)
+#pragma unroll
+            for (int n = 0; n < 2; ++n)
+#pragma unroll
+                for (int q = 0; q < 4; ++q) acc[i][n][q] = 0.0f;
+        for (int kb = 0; kb < NKB; ++kb) {
+            cp_wait<STAGES - 2>();
+            __syncthreads();
+            if (kb + STAGES - 1 < NKB) load(kb + STAGES - 1);
+            cp_commit();
+            if (!on0) continue;
+            const uint8_t* st = stages + (kb % STAGES) * STAGE_BYTES;
+            const uint8_t* sa = st + WROWS * 16;
+            uint2 bq[2][2];                                       // [n8 tile][half]: the B registers
+            float4 sc[2][2];                                      // [n8 tile][column 2 tig + c]: {d0, c0, d1, c1}
+#pragma unroll
+            for (int n = 0; n < 2; ++n) {
+                if (n == 1 && !on1) break;
+                const uint8_t* r = sa + (nb0 + 8 * n + g) * AB + 8 * tig;
+                bq[n][0] = *(const uint2*) r;
+                bq[n][1] = *(const uint2*) (r + 32);
+                sc[n][0] = *(const float4*) (sa + (nb0 + 8 * n + 2 * tig) * AB + 64);
+                sc[n][1] = *(const float4*) (sa + (nb0 + 8 * n + 2 * tig + 1) * AB + 64);
+            }
+#pragma unroll
+            for (int i = 0; i < 4; ++i) {
+                const uint4 ca = *(const uint4*) (st + ra[i] * 16), cb = *(const uint4*) (st + rb[i] * 16);
+                const float da = __half2float(wsc[ra[i] * NKB + kb]), db = __half2float(wsc[rb[i] * NKB + kb]);
+                const int s = 2 * tig;
+                const uint32_t a0[4] = {(ca.x >> s) & M2, (cb.x >> s) & M2, (ca.y >> s) & M2, (cb.y >> s) & M2};
+                const uint32_t a1[4] = {(ca.z >> s) & M2, (cb.z >> s) & M2, (ca.w >> s) & M2, (cb.w >> s) & M2};
+#pragma unroll
+                for (int n = 0; n < 2; ++n) {
+                    if (n == 1 && !on1) break;
+                    int d0[4], d1[4];
+                    mma_s8(d0, a0, bq[n][0].x, bq[n][0].y);
+                    mma_s8(d1, a1, bq[n][1].x, bq[n][1].y);
+#pragma unroll
+                    for (int q = 0; q < 4; ++q) {
+                        const float4 z = sc[n][q & 1];
+                        const float u0 = __int_as_float(MAGIC + d0[q]) + z.y, u1 = __int_as_float(MAGIC + d1[q]) + z.w;
+                        acc[i][n][q] = fmaf(q < 2 ? da : db, fmaf(z.z, u1, z.x * u0), acc[i][n][q]);
+                    }
+                }
+            }
+        }
+        if (!on0) continue;
+        if (GU) {
+            // SwiGLU of feature 32 wf + 8 i + g (lane g of m tile i), tile rows nb0 + 8 n + 2 tig + c; H to int8 per
+            // row over the warp's 32 features (= one 32-value half-block of the down product's K)
+            const int blk = 4 * fb + wf;
+#pragma unroll
+            for (int n = 0; n < 2; ++n) {
+                if (n == 1 && !on1) break;
+#pragma unroll
+                for (int c = 0; c < 2; ++c) {
+                    float h[4], am = 0.0f;
+#pragma unroll
+                    for (int i = 0; i < 4; ++i) {
+                        const float gt = acc[i][n][c], up = acc[i][n][2 + c];
+                        h[i] = gt / (1.0f + __expf(-gt)) * up;
+                        am = fmaxf(am, fabsf(h[i]));
+                    }
+#pragma unroll
+                    for (int o = 4; o < 32; o <<= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
+                    const float inv = am > 0.0f ? 127.0f / am : 0.0f;
+                    int qv[4], sum = 0;
+#pragma unroll
+                    for (int i = 0; i < 4; ++i) { qv[i] = __float2int_rn(h[i] * inv); sum += qv[i]; }
+#pragma unroll
+                    for (int o = 4; o < 32; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
+                    const int r = nb0 + 8 * n + 2 * tig + c;
+                    if (r < nrows) {
+                        uint8_t* o = out + (size_t) (row0 + r) * (10 * AB) + (blk >> 1) * AB;
+                        const int hh = blk & 1;
+#pragma unroll
+                        for (int i = 0; i < 4; ++i) o[32 * hh + perm32(8 * i + g)] = (uint8_t) (int8_t) qv[i];
+                        if (g == 0) *(float2*) (o + 64 + 8 * hh) = make_float2(am / 127.0f, -(MAGICF + (float) sum));
+                    }
+                }
+            }
+        } else {
+#pragma unroll
+            for (int i = 0; i < 4; ++i)
+#pragma unroll
+                for (int n = 0; n < 2; ++n) {
+                    if (n == 1 && !on1) break;
+#pragma unroll
+                    for (int q = 0; q < 4; ++q) {
+                        const int r = nb0 + 8 * n + 2 * tig + (q & 1);
+                        if (r < nrows) dm[(size_t) (row0 + r) * 2560 + rbase + (q < 2 ? ra[i] : rb[i])] = acc[i][n][q];
+                    }
+                }
+        }
+    }
+#endif
+}
+
+unsigned blocks(int64_t n, int per) { return (unsigned) ((n + per - 1) / per); }
+
+struct DevInfo {
+    bool done = false;
+    int cc = 0, sms = 0, occ_gu = 0, occ_d = 0;
+};
+std::mutex g_mu;
+DevInfo g_dev[32];
+
+const DevInfo& dev_info() {
+    int dev = 0;
+    cudaGetDevice(&dev);
+    std::lock_guard<std::mutex> lk(g_mu);
+    DevInfo& d = g_dev[dev & 31];
+    if (d.done) return d;
+    d.done = true;
+    int major = 0, minor = 0;
+    cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+    cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);
+    cudaDeviceGetAttribute(&d.sms, cudaDevAttrMultiProcessorCount, dev);
+    d.cc = 10 * major + minor;
+    if (d.cc < 80) return d;
+    // the device code of this card's arch must have the kernels (a build without an sm_80+ target has an empty body)
+    cudaFuncAttributes fa{};
+    if (cudaFuncGetAttributes(&fa, expert_kernel<true>) != cudaSuccess || fa.ptxVersion < 80) {
+        cudaGetLastError();
+        d.cc = 0;
+        return d;
+    }
+    ck(cudaFuncSetAttribute(expert_kernel<true>, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) smem_bytes(true)),
+       "smem gu");
+    ck(cudaFuncSetAttribute(expert_kernel<false>, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) smem_bytes(false)),
+       "smem down");
+    cudaOccupancyMaxActiveBlocksPerMultiprocessor(&d.occ_gu, expert_kernel<true>, THREADS, smem_bytes(true));
+    cudaOccupancyMaxActiveBlocksPerMultiprocessor(&d.occ_d, expert_kernel<false>, THREADS, smem_bytes(false));
+    if (d.occ_gu < 1 || d.occ_d < 1) d.cc = 0;                   // does not fit this card: the MMQ path
+    cudaGetLastError();
+    return d;
+}
+
+}  // namespace
+
+bool built() { return true; }
+bool available() { return dev_info().cc >= 80; }
+// 0.1.36: on by default for the Q2_0 pack (+13-23% prompts on an RTX 5070; teacher-forced against the FP16 path as
+// close as MMQ at 8K and closer at 32K: phaseA-tests s18-tf); STRATA_PF_FUSED=0 keeps MMQ.  The native packs' kernels
+// (moe_fused_iq) stay opt-in: STRATA_PF_FUSED=1 (requested()).
+bool enabled() {
+    static const bool env = [] {
+        const char* v = std::getenv("STRATA_PF_FUSED");
+        return v == nullptr || v[0] != '0';
+    }();
+    return env && available();
+}
+bool requested() {
+    static const bool env = [] {
+        const char* v = std::getenv("STRATA_PF_FUSED");
+        return v != nullptr && v[0] == '1';
+    }();
+    return env && available();
+}
+
+size_t act_bytes(int64_t rows, int64_t cols) { return (size_t) rows * (size_t) (cols / 64) * AB; }
+size_t group_bytes(int64_t n, int n_expert) {
+    return tiles_at(n_expert) + ((size_t) (n + kTileRows - 1) / kTileRows + (size_t) n_expert) * sizeof(int2);
+}
+
+void quantize_act(const float* x, int64_t rows, int64_t cols, void* xa, void* stream) {
+    if (rows <= 0) return;
+    const int64_t nblk = rows * (cols / 64);
+    quant_act_kernel<<<blocks(nblk, 8), 256, 0, (cudaStream_t) stream>>>(x, nblk, (uint8_t*) xa);
+    ck(cudaGetLastError(), "quantize_act");
+}
+
+void group(const int32_t* ids, int64_t n, int k, int n_expert, void* scratch, int32_t* slot, int32_t* src,
+           void* stream) {
+    if (n_expert < 1 || n_expert > 1024) {
+        std::fprintf(stderr, "prefill fused experts: %d experts (the grouping takes 1 to 1024)\n", n_expert);
+        std::exit(1);
+    }
+    const cudaStream_t s = (cudaStream_t) stream;
+    const Tables tb = tables(scratch, n_expert);
+    ck(cudaMemsetAsync(tb.cnt, 0, (size_t) n_expert * 4, s), "group memset");
+    if (n > 0) {
+        const unsigned nb = std::min<unsigned>(blocks(n, 1024), 4u * (unsigned) std::max(dev_info().sms, 1));
+        count_kernel<<<nb, 1024, (size_t) n_expert * 4, s>>>(ids, n, n_expert, tb.cnt);
+    }
+    scan_kernel<<<1, (unsigned) ((n_expert + 31) / 32 * 32), 0, s>>>(tb, n_expert);
+    if (n > 0) place_kernel<<<blocks(n, 256), 256, 0, s>>>(ids, n, k, n_expert, tb, slot, src);
+    ck(cudaGetLastError(), "group");
+}
+
+void experts(const Batch& b, int n_expert, int64_t n, const void* scratch, const void* xa, const int32_t* src,
+             void* ha, float* dm, void* stream) {
+    if (b.e1 <= b.e0 || n <= 0) return;
+    const DevInfo& d = dev_info();
+    const cudaStream_t s = (cudaStream_t) stream;
+    const Tables tb = tables(const_cast<void*>(scratch), n_expert);
+    // the most tiles the batch can have: every row in it, plus a partial tile per expert
+    const int64_t tiles = (n + kTileRows - 1) / kTileRows + (b.e1 - b.e0);
+    const unsigned g_gu = (unsigned) std::min<int64_t>(tiles * 5, (int64_t) d.sms * d.occ_gu);
+    const unsigned g_d = (unsigned) std::min<int64_t>(tiles * 10, (int64_t) d.sms * d.occ_d);
+    expert_kernel<true><<<g_gu, THREADS, smem_bytes(true), s>>>(b, tb, (const uint8_t*) xa, src, (uint8_t*) ha,
+                                                                nullptr);
+    expert_kernel<false><<<g_d, THREADS, smem_bytes(false), s>>>(b, tb, (const uint8_t*) ha, src, nullptr, dm);
+    ck(cudaGetLastError(), "experts");
+}
+
+}  // namespace strata::prefill::fused

src/prefill/moe_fused_iq.cu

@@ -0,0 +1,630 @@
+// src/prefill/moe_fused_iq.cu - see include/strata/prefill/moe_fused_iq.hpp (#136: the native packs' prompt experts on
+// the fused int8 kernels of moe_fused.cu).
+//
+// The arithmetic.  An activation block of 32 values is x = d_x * a, a = round(x / d_x) in -127..127, stored in natural
+// order (per 64 values: 64 codes, then {d_0, d_1} and 8 unused bytes - 80 bytes, the Q2_0 path's size).  A weight
+// block of 32 (or 16) values is w = d_w * q with q an int8: the i-quants' codebook entries with their signs applied
+// (IQ2_XXS / IQ2_XS / IQ2_S: 0..43, IQ3_XXS: 0..62, IQ3_S: 0..15, IQ4_XS / IQ4_NL: -127..113), Q2_0's code - 1.  So
+//     sum w x = d_w * d_x * sum q a,     |sum q a| <= 32 * 127 * 127 < 2^22,
+// and the int32 dot becomes a float exactly with one float add: the mma starts from C = 0x4B400000 (the bits of
+// 1.5 * 2^23), so as_float(D) - 1.5 * 2^23 is the dot.  Per output and 32 values: that add, d_w * d_x, one fma (the
+// formats with a scale per 16 values: two dots, each times its d_w, then d_x).  d_w is llama.cpp's own: IQ2_XXS
+// d (2s + 1) / 8, IQ2_XS / IQ2_S d (2s + 1) / 8 per 16, IQ3_XXS d (2s + 1) / 4, IQ3_S d (2s + 1), IQ4_XS d (s - 32),
+// IQ4_NL and Q2_0 d (dequantize_row_* in ggml-quants.c, mmq-load-tiles.cuh).
+//
+// The load stage.  Per 64 values of K (a stage), a thread decodes one 32-value sub-block of one of the work item's
+// weight rows: its block bytes are read from the blob with 16-bit loads (the GGUF blocks are 2-byte aligned: 66 / 74 /
+// 82 / 98 / 110 / 136 / 18 bytes) one stage ahead into registers (and two 256-value blocks ahead into L2), then turned
+// into 32 int8 and the sub-block's scales in shared memory, double-buffered.  The codebooks live in shared memory (up
+// to 8 KB, IQ2_S).  The activations arrive by cp.async, four stages deep, as in moe_fused.cu; the fragments come by
+// ldmatrix.  The products are mma.sync m16n8k32 (m16n8k16 for the formats with a scale per 16 values); the epilogues
+// (SwiGLU, H to int8 per 32 features; down into the per-slot rows) are moe_fused.cu's.
+#include "strata/prefill/moe_fused_iq.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include "ggml.h"
+
+#define GGML_COMMON_DECL_CUDA
+#define GGML_COMMON_IMPL_CUDA
+#include "ggml-common.h"
+
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <mutex>
+
+namespace strata::prefill::fused {
+namespace {
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "prefill fused experts (native): %s: %s\n", what, cudaGetErrorString(e));
+        std::exit(1);
+    }
+}
+
+constexpr int AB = 80;                  // activation bytes per 64 values: 64 codes, {d0, d1}, 8 unused
+constexpr int MAGIC = 0x4B400000;       // the bits of 1.5 * 2^23
+constexpr float MAGICF = 12582912.0f;
+// 16 warps, each 64 weight rows x 16 routed rows; WW of them along the weight rows and 16 / WW along the routed rows: a
+// work item is (WR = 64 WW weight rows) x (TR = 256 / WW routed rows of one expert) - 256 x 64 or 128 x 128, chosen
+// per launch by the rows per expert (pick_ww)
+constexpr int THREADS = 512;
+constexpr int WLD = 80;                 // bytes per decoded weight row of a stage: 64 int8 + 16 (no bank conflicts)
+__host__ __device__ constexpr int tile_rows(int ww) { return 256 / ww; }
+__host__ __device__ constexpr int weight_rows(int ww) { return 64 * ww; }
+constexpr int ASTAGES = 4;              // cp.async depth of the activations (64 values of K a stage)
+constexpr int GU_ROWS_K = 2560, D_ROWS_K = 640;   // K of gate/up (n_embd) and of down (n_ff)
+
+// the formats (ggml type ids)
+constexpr int T_IQ2_XXS = GGML_TYPE_IQ2_XXS, T_IQ2_XS = GGML_TYPE_IQ2_XS, T_IQ2_S = GGML_TYPE_IQ2_S,
+              T_IQ3_XXS = GGML_TYPE_IQ3_XXS, T_IQ3_S = GGML_TYPE_IQ3_S, T_IQ4_XS = GGML_TYPE_IQ4_XS,
+              T_IQ4_NL = GGML_TYPE_IQ4_NL, T_Q2_0 = GGML_TYPE_Q2_0;
+static_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_xs) == 74 && sizeof(block_iq2_s) == 82 &&
+              sizeof(block_iq3_xxs) == 98 && sizeof(block_iq3_s) == 110 && sizeof(block_iq4_xs) == 136 &&
+              sizeof(block_iq4_nl) == 18 && sizeof(block_q2_0) == 18, "the block layouts this file decodes");
+
+// block bytes, scale per 16 values, codebook bytes in shared memory
+__host__ __device__ constexpr int block_bytes(int t) {
+    return t == T_IQ2_XXS ? 66 : t == T_IQ2_XS ? 74 : t == T_IQ2_S ? 82 : t == T_IQ3_XXS ? 98 : t == T_IQ3_S ? 110
+         : t == T_IQ4_XS ? 136 : 18;
+}
+__host__ __device__ constexpr bool per16(int t) { return t == T_IQ2_XS || t == T_IQ2_S; }
+__host__ __device__ constexpr int grid_bytes(int t) {
+    return t == T_IQ2_XXS ? 256 * 8 : t == T_IQ2_XS ? 512 * 8 : t == T_IQ2_S ? 1024 * 8 : t == T_IQ3_XXS ? 256 * 4
+         : t == T_IQ3_S ? 512 * 4 : 0;
+}
+__host__ __device__ constexpr size_t smem_bytes(int t, int ww) {
+    return (size_t) 2 * weight_rows(ww) * (WLD + 16) + (size_t) ASTAGES * tile_rows(ww) * AB +
+           (size_t) tile_rows(ww) * 4 + grid_bytes(t);
+}
+
+// moe_fused.cu's grouping tables (the same layout: fused::group writes them)
+struct Tables {
+    int32_t *cnt, *off, *fill, *ts;
+    int2* tiles;
+};
+size_t tiles_at(int n_expert) { return ((size_t) (4 * n_expert + 2) * 4 + 15) / 16 * 16; }
+Tables tables(void* scratch, int n_expert) {
+    int32_t* p = (int32_t*) scratch;
+    return {p, p + n_expert, p + 2 * n_expert + 1, p + 3 * n_expert + 1,
+            (int2*) ((uint8_t*) scratch + tiles_at(n_expert))};
+}
+
+// ---- activations in natural order: one warp per 64 values
+__global__ void quant_act_nat_kernel(const float* __restrict__ x, int64_t nblk, uint8_t* __restrict__ xa) {
+    const int64_t w = (int64_t) blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32;
+    const int lane = threadIdx.x & 31;
+    if (w >= nblk) return;
+    const float v0 = x[w * 64 + lane], v1 = x[w * 64 + 32 + lane];
+    float a0 = fabsf(v0), a1 = fabsf(v1);
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) {
+        a0 = fmaxf(a0, __shfl_xor_sync(0xffffffffu, a0, o));
+        a1 = fmaxf(a1, __shfl_xor_sync(0xffffffffu, a1, o));
+    }
+    uint8_t* out = xa + w * AB;
+    out[lane] = (uint8_t) (int8_t) (a0 > 0.0f ? __float2int_rn(v0 * (127.0f / a0)) : 0);
+    out[32 + lane] = (uint8_t) (int8_t) (a1 > 0.0f ? __float2int_rn(v1 * (127.0f / a1)) : 0);
+    if (lane == 0) *(float4*) (out + 64) = make_float4(a0 / 127.0f, a1 / 127.0f, 0.0f, 0.0f);
+}
+
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+__device__ __forceinline__ void cp16(void* dst, const void* src) {
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(dst)),
+                 "l"(src));
+}
+__device__ __forceinline__ void cp_commit() { asm volatile("cp.async.commit_group;\n" ::); }
+__device__ __forceinline__ void pf_l2(const void* p) { asm volatile("prefetch.global.L2 [%0];\n" ::"l"(p)); }
+template <int N> __device__ __forceinline__ void cp_wait() { asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); }
+// d = MAGIC + A (16 x 32 s8, row) * B (32 x 8 s8, col): the int32 dot with the magic bias already added (the C
+// operand), so as_float(d) - 1.5 * 2^23 is the dot as a float
+__device__ __forceinline__ void mma32(int (&d)[4], const uint32_t (&a)[4], uint32_t b0, uint32_t b1) {
+    asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%10,%10,%10};\n"
+        : "=r"(d[0]), "=r"(d[1]), "=r"(d[2]), "=r"(d[3])
+        : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1), "r"(MAGIC));
+}
+// the same at m16n8k16 (A 16 x 16, B 16 x 8)
+__device__ __forceinline__ void mma16(int (&d)[4], uint32_t a0, uint32_t a1, uint32_t b0) {
+    asm("mma.sync.aligned.m16n8k16.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5}, {%6}, {%7,%7,%7,%7};\n"
+        : "=r"(d[0]), "=r"(d[1]), "=r"(d[2]), "=r"(d[3])
+        : "r"(a0), "r"(a1), "r"(b0), "r"(MAGIC));
+}
+__device__ __forceinline__ float dotf(int d) { return __int_as_float(d) - MAGICF; }
+// four 8 x 16-byte matrices, row addresses from lanes 0-7, 8-15, 16-23, 24-31: lane (g, t) gets bytes 4t..4t+3 of row g
+// of each - an mma fragment
+__device__ __forceinline__ void ldsm4(uint32_t (&r)[4], const void* p) {
+    asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
+                 : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3])
+                 : "r"((unsigned) __cvta_generic_to_shared(p))
+                 : "memory");
+}
+
+// ---- the load stage: a 32-value sub-block's bytes into registers, then int8 and scales
+__device__ __forceinline__ uint32_t ld16(const uint8_t* p) { return *(const uint16_t*) p; }
+__device__ __forceinline__ uint32_t ld32(const uint8_t* p) { return ld16(p) | (ld16(p + 2) << 16); }
+__device__ __forceinline__ float half_at(uint32_t w) { return __half2float(__ushort_as_half((unsigned short) w)); }
+// llama.cpp's sign unpacking: 7 bits of signs, the 8th their parity (bit 7 of v may be anything)
+__device__ __forceinline__ uint32_t unpack_ksigns(uint32_t v) {
+    v &= 0xFF;
+    const uint32_t p = __popc(v) & 1;
+    return (v ^ p << 7) * 0x01010101u;
+}
+// 8 bytes of a codebook entry with the sign byte `s` (broadcast) applied: bits 0-3 to .x, 4-7 to .y
+__device__ __forceinline__ void signed8(uint32_t gx, uint32_t gy, uint32_t s, uint32_t& qx, uint32_t& qy) {
+    const uint32_t m0 = __vcmpne4(s & 0x08040201u, 0), m1 = __vcmpne4(s & 0x80402010u, 0);
+    qx = __vsub4(gx ^ m0, m0);
+    qy = __vsub4(gy ^ m1, m1);
+}
+// 8 nibbles of q4 through a 16-entry int8 table (4 words): the low nibbles' values in .x, the high ones' in .y
+__device__ __forceinline__ void table16(uint32_t q4, const uint32_t (&t)[4], uint32_t& lo, uint32_t& hi) {
+    uint32_t tmp[2];
+    const uint32_t sel = 0x32103210u | ((q4 & 0x88888888u) >> 1);
+#pragma unroll
+    for (int i = 0; i < 2; ++i) {
+        const uint32_t sh = 16 * i;
+        const uint32_t l = __byte_perm(t[0], t[1], q4 >> sh), h = __byte_perm(t[2], t[3], q4 >> sh);
+        tmp[i] = __byte_perm(l, h, sel >> sh);
+    }
+    lo = __byte_perm(tmp[0], tmp[1], 0x6420);
+    hi = __byte_perm(tmp[0], tmp[1], 0x7531);
+}
+
+// Raw bytes of sub-block `ib` of the block at `bp` (IQ: the 256-value super-block, ib 0..7; Q2_0: the 64-value block,
+// ib 0..1; IQ4_NL: the 32-value block).
+template <int T> __device__ __forceinline__ void load_unit(const uint8_t* bp, int ib, uint32_t (&w)[5]) {
+    if constexpr (T == T_IQ2_XXS) {
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld16(bp);
+    } else if constexpr (T == T_IQ2_XS) {
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld16(bp) | ((uint32_t) bp[66 + ib] << 16);
+    } else if constexpr (T == T_IQ2_S) {
+        w[0] = ld32(bp + 2 + 4 * ib); w[1] = ld32(bp + 34 + 4 * ib);
+        w[2] = ld16(bp) | ((uint32_t) bp[66 + ib] << 16) | ((uint32_t) bp[74 + ib] << 24);
+    } else if constexpr (T == T_IQ3_XXS) {
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld32(bp + 66 + 4 * ib); w[3] = ld16(bp);
+    } else if constexpr (T == T_IQ3_S) {
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld32(bp + 74 + 4 * ib);
+        w[3] = ld16(bp) | ((uint32_t) bp[66 + ib] << 16) | ((uint32_t) ((bp[106 + ib / 2] >> (4 * (ib & 1))) & 15) << 24);
+    } else if constexpr (T == T_IQ4_XS) {
+#pragma unroll
+        for (int k = 0; k < 4; ++k) w[k] = ld32(bp + 8 + 16 * ib + 4 * k);
+        const uint32_t ls = ((bp[4 + ib / 2] >> (4 * (ib & 1))) & 15) | (((ld16(bp + 2) >> (2 * ib)) & 3) << 4);
+        w[4] = ld16(bp) | (ls << 16);
+    } else if constexpr (T == T_IQ4_NL) {
+#pragma unroll
+        for (int k = 0; k < 4; ++k) w[k] = ld32(bp + 2 + 4 * k);
+        w[4] = ld16(bp);
+    } else {   // Q2_0
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld16(bp);
+    }
+}
+
+// The sub-block as 32 int8 (q[0..7], natural order) and its scales (s0: values 0-15, s1: 16-31).
+template <int T>
+__device__ __forceinline__ void convert(const uint32_t (&w)[5], const uint8_t* grid, const uint32_t (&kv)[4],
+                                        uint32_t (&q)[8], float& s0, float& s1) {
+    if constexpr (T == T_IQ2_XXS) {
+        const uint2* g = (const uint2*) grid;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint2 e = g[(w[0] >> (8 * l)) & 255];
+            signed8(e.x, e.y, unpack_ksigns(w[1] >> (7 * l)), q[2 * l], q[2 * l + 1]);
+        }
+        s0 = s1 = half_at(w[2]) * (float) ((w[1] >> 27) | 1) * 0.125f;
+    } else if constexpr (T == T_IQ2_XS) {
+        const uint2* g = (const uint2*) grid;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint32_t c = (w[l >> 1] >> (16 * (l & 1))) & 0xFFFF;
+            const uint2 e = g[c & 511];
+            signed8(e.x, e.y, unpack_ksigns(c >> 9), q[2 * l], q[2 * l + 1]);
+        }
+        const float d = half_at(w[2]);
+        const uint32_t sc = w[2] >> 16;
+        s0 = d * (float) (2 * (sc & 15) + 1) * 0.125f;
+        s1 = d * (float) (2 * ((sc >> 4) & 15) + 1) * 0.125f;
+    } else if constexpr (T == T_IQ2_S) {
+        const uint2* g = (const uint2*) grid;
+        const uint32_t qh = (w[2] >> 16) & 255;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint2 e = g[((w[0] >> (8 * l)) & 255) | ((qh << (8 - 2 * l)) & 0x300)];
+            signed8(e.x, e.y, ((w[1] >> (8 * l)) & 255) * 0x01010101u, q[2 * l], q[2 * l + 1]);
+        }
+        const float d = half_at(w[2]);
+        const uint32_t sc = w[2] >> 24;
+        s0 = d * (float) (2 * (sc & 15) + 1) * 0.125f;
+        s1 = d * (float) (2 * (sc >> 4) + 1) * 0.125f;
+    } else if constexpr (T == T_IQ3_XXS) {
+        const uint32_t* g = (const uint32_t*) grid;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint32_t i0 = (w[l >> 1] >> (16 * (l & 1))) & 255, i1 = (w[l >> 1] >> (16 * (l & 1) + 8)) & 255;
+            signed8(g[i0], g[i1], unpack_ksigns(w[2] >> (7 * l)), q[2 * l], q[2 * l + 1]);
+        }
+        s0 = s1 = half_at(w[3]) * (float) (2 * (w[2] >> 28) + 1) * 0.25f;
+    } else if constexpr (T == T_IQ3_S) {
+        const uint32_t* g = (const uint32_t*) grid;
+        const uint32_t qh = (w[3] >> 16) & 255;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint32_t i0 = (w[l >> 1] >> (16 * (l & 1))) & 255, i1 = (w[l >> 1] >> (16 * (l & 1) + 8)) & 255;
+            signed8(g[i0 | ((qh << (8 - 2 * l)) & 256)], g[i1 | ((qh << (7 - 2 * l)) & 256)],
+                    ((w[2] >> (8 * l)) & 255) * 0x01010101u, q[2 * l], q[2 * l + 1]);
+        }
+        s0 = s1 = half_at(w[3]) * (float) (1 + 2 * (w[3] >> 24));
+    } else if constexpr (T == T_IQ4_XS || T == T_IQ4_NL) {
+#pragma unroll
+        for (int k = 0; k < 4; ++k) table16(w[k], kv, q[k], q[4 + k]);
+        s0 = s1 = T == T_IQ4_XS ? half_at(w[4]) * (float) ((int) (w[4] >> 16) - 32) : half_at(w[4]);
+    } else {   // Q2_0: code - 1 via a byte table {-1, 0, 1, 2}
+#pragma unroll
+        for (int h = 0; h < 4; ++h) {
+            const uint32_t c = (w[h >> 1] >> (16 * (h & 1))) & 0xFFFF;
+            const uint32_t qe = __byte_perm(0x020100FFu, 0x020100FFu, c & 0x7777);
+            const uint32_t qo = __byte_perm(0x020100FFu, 0x020100FFu, (c >> 2) & 0x7777);
+            q[2 * h] = __byte_perm(qe, qo, 0x5140);
+            q[2 * h + 1] = __byte_perm(qe, qo, 0x7362);
+        }
+        s0 = s1 = half_at(w[2]);
+    }
+}
+
+template <int T> __device__ __forceinline__ const void* grid_src() {
+    if constexpr (T == T_IQ2_XXS) return iq2xxs_grid;
+    else if constexpr (T == T_IQ2_XS) return iq2xs_grid;
+    else if constexpr (T == T_IQ2_S) return iq2s_grid;
+    else if constexpr (T == T_IQ3_XXS) return iq3xxs_grid;
+    else if constexpr (T == T_IQ3_S) return iq3s_grid;
+    else return nullptr;
+}
+#endif
+
+// One work item = (TR routed rows of expert e - one or two of group()'s 64-row tiles -, a block of WR weight rows);
+// a persistent grid walks the batch's items.  GU: gate/up rows of WR / 2 features (format WT) against the rows'
+// tokens' activations, SwiGLU, H to int8 per 32 features into `out`.  !GU: down rows (format WT) against H, FP32 into
+// `dm`.  Warp (wf, wt): weight rows 64 wf.. as four m16 tiles (GU: a tile = 8 features, gate rows as mma rows 0-7 and
+// up rows as 8-15, so a lane holds gate and up of one feature), routed rows 16 wt.. as two n8.
+template <int WT, bool GU, int WW>
+__global__ void __launch_bounds__(THREADS, 1)
+native_kernel(const Batch b, const NativeGeom geo, const Tables tb, const uint8_t* __restrict__ act,
+              const int32_t* __restrict__ src, uint8_t* __restrict__ out, float* __restrict__ dm) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+    constexpr int TR = tile_rows(WW), WR = weight_rows(WW);
+    constexpr int WT_BYTES = WR * WLD, WS_FLOATS = WR * 4, ACT_STAGE = TR * AB;
+    constexpr int NS = (GU ? GU_ROWS_K : D_ROWS_K) / 64;   // 64-value stages along K
+    constexpr int NFB = (GU ? 1280 : 2560) / WR;          // weight-row blocks per tile
+    constexpr int ACT_LD = NS * AB;                       // bytes per activation row: 3200 (a token), 800 (a row's H)
+    constexpr int BS = block_bytes(WT);
+    constexpr bool K16 = per16(WT);
+    extern __shared__ __align__(16) uint8_t smem[];
+    uint8_t* wt = smem;                                               // [2][WR][WLD] decoded int8 weights
+    float* ws = (float*) (smem + 2 * WT_BYTES);                       // [2][WR][4] their scales per 16 values
+    uint8_t* stages = (uint8_t*) (ws + 2 * WS_FLOATS);                // [ASTAGES][TR][AB] activations
+    int* srow = (int*) (stages + ASTAGES * ACT_STAGE);                // [TR] the activation row of each tile row
+    uint8_t* sgrid = (uint8_t*) (srow + TR);                          // the codebook
+
+    const int tid = threadIdx.x, lane = tid & 31, warp = tid >> 5, g = lane >> 2, tig = lane & 3;
+    const int wf = warp % WW, nb0 = 16 * (warp / WW);
+    if constexpr (grid_bytes(WT) > 0) {
+        const uint32_t* gs = (const uint32_t*) grid_src<WT>();
+        for (int i = tid; i < grid_bytes(WT) / 4; i += THREADS) ((uint32_t*) sgrid)[i] = gs[i];
+    }
+    uint32_t kv[4] = {0, 0, 0, 0};
+    if constexpr (WT == T_IQ4_XS || WT == T_IQ4_NL) {
+#pragma unroll
+        for (int k = 0; k < 16; ++k) kv[k >> 2] |= (uint32_t) (uint8_t) kvalues_iq4nl[k] << (8 * (k & 3));
+    }
+    // Local weight rows: m tile i of warp wf is rows 64 wf + 16 i.. (mma rows g and g + 8: rows +g and +8+g).  For
+    // gate/up a tile is 8 features, its gate rows then its up rows, so a lane holds gate and up of one feature.  The
+    // fragments come by ldmatrix: lane l gives the address of row (l & 7) of matrix l >> 3 - A: {rows 0-7, rows 8-15}
+    // x {bytes 0-15, 16-31} of a 32-value chunk; B: {tokens nb0 + 0-7, + 8-15} x {bytes 0-15, 16-31}.
+    const int lm = lane >> 3, l8 = lane & 7;
+    const int a_off = (64 * wf + 8 * (lm & 1) + l8) * WLD + 16 * (lm >> 1);
+    const int b_off = (nb0 + 8 * (lm >> 1) + l8) * AB + 16 * (lm & 1);
+    // this thread's decode unit (the first 2 WR threads): local weight row ur, sub-block uj of each stage
+    const bool dec = tid < 2 * WR;
+    const int ur = dec ? tid >> 1 : 0, uj = tid & 1;
+    const int t0 = tb.ts[b.e0], nwork = (tb.ts[b.e1] - t0) * NFB;
+    for (int w = blockIdx.x; w < nwork; w += gridDim.x) {
+        const int2 tl = tb.tiles[t0 + w / NFB];
+        const int fb = w % NFB, e = tl.x, row0 = tl.y;
+        if ((row0 - tb.off[e]) % TR != 0) continue;              // a 64-row tile inside an earlier item's TR rows
+        const int nrows = min(TR, tb.off[e + 1] - row0);
+        const uint8_t* blob = b.blob[e - b.e0];
+        const int rbase = fb * WR;
+        // local row ur: gate/up - feature (WR / 2) fb + 8 (ur >> 4) + (ur & 7), its up row when bit 3 is set
+        const uint8_t* wrow = GU ? blob + ((ur & 8) ? geo.up_off : 0) +
+                                       (size_t) (fb * (WR / 2) + 8 * (ur >> 4) + (ur & 7)) * geo.gu_row
+                                 : blob + geo.down_off + (size_t) (rbase + ur) * geo.d_row;
+        auto unit = [&](int s) -> const uint8_t* {                  // the block of stage s's sub-block
+            if (GU) return wrow + (s >> 2) * BS;
+            return WT == T_IQ4_NL ? wrow + (2 * s + uj) * BS : wrow + s * BS;
+        };
+        auto sub = [&](int s) { return GU ? 2 * (s & 3) + uj : uj; };
+        // the weight bytes into L2 ahead of the register loads (one stage ahead only - less than a DRAM round trip):
+        // gate/up PF super-blocks ahead (the row's two threads take a block's first and last line), down the whole row
+        // slice at once (180 or 360 bytes a row)
+        constexpr int PF = 2;
+        auto prefetch_sb = [&](int sb) {
+            if (dec && sb < GU_ROWS_K / 256) pf_l2(wrow + sb * BS + (uj ? BS - 1 : 0));
+        };
+        if (GU) {
+#pragma unroll
+            for (int sb = 0; sb < PF; ++sb) prefetch_sb(sb);
+        } else if (dec) {
+            for (size_t o = 128 * (size_t) uj; o < geo.d_row + 127; o += 256) pf_l2(wrow + min(o, geo.d_row - 1));
+        }
+        __syncthreads();                                          // the previous item is done with the buffers
+        if (tid < TR) srow[tid] = tid < nrows ? (GU ? src[row0 + tid] : row0 + tid) : -1;
+        __syncthreads();                                          // srow
+        // the rows past the item's end are not loaded: a warp entirely past it skips the products, and the columns
+        // of a partial one are never written (no reduction mixes columns)
+        auto load_act = [&](int s) {
+            uint8_t* st = stages + (s % ASTAGES) * ACT_STAGE;
+            for (int c = tid; c < TR * 5; c += THREADS) {
+                const int r = c / 5, q = c % 5;
+                if (r < nrows) cp16(st + r * AB + q * 16, act + (size_t) srow[r] * ACT_LD + s * AB + q * 16);
+            }
+        };
+        auto put = [&](const uint32_t (&raw)[5], int buf) {
+            if (!dec) return;
+            uint32_t q[8];
+            float s0, s1;
+            convert<WT>(raw, sgrid, kv, q, s0, s1);
+            uint4* d = (uint4*) (wt + buf * WT_BYTES + ur * WLD + 32 * uj);
+            d[0] = make_uint4(q[0], q[1], q[2], q[3]);
+            d[1] = make_uint4(q[4], q[5], q[6], q[7]);
+            *(float2*) (ws + buf * WS_FLOATS + ur * 4 + 2 * uj) = make_float2(s0, s1);
+        };
+#pragma unroll
+        for (int s = 0; s < ASTAGES - 1; ++s) {
+            if (s < NS) load_act(s);
+            cp_commit();
+        }
+        uint32_t raw[5] = {0, 0, 0, 0, 0};
+        if (dec) load_unit<WT>(unit(0), sub(0), raw);
+        put(raw, 0);
+        if (dec) load_unit<WT>(unit(1), sub(1), raw);
+        // a warp whose rows are all past the tile's end only takes part in the loads
+        const bool on0 = nb0 < nrows, on1 = nb0 + 8 < nrows;
+        float acc[4][2][4];
+#pragma unroll
+        for (int i = 0; i < 4; ++i)
+#pragma unroll
+            for (int n = 0; n < 2; ++n)
+#pragma unroll
+                for (int q = 0; q < 4; ++q) acc[i][n][q] = 0.0f;
+        for (int s = 0; s < NS; ++s) {
+            cp_wait<ASTAGES - 2>();
+            __syncthreads();
+            if (s + ASTAGES - 1 < NS) load_act(s + ASTAGES - 1);
+            cp_commit();
+            if (GU && (s & 3) == 0) prefetch_sb((s >> 2) + PF);
+            if (on0) {
+                const uint8_t* W = wt + (s & 1) * WT_BYTES;
+                const float* S = ws + (s & 1) * WS_FLOATS;
+                const uint8_t* sa = stages + (s % ASTAGES) * ACT_STAGE;
+                // the stage's scales: per weight row 4 (per 16 values), per routed row 2 (per 32)
+                float2 dx[2][2];                                  // [n8 tile][column 2 tig + cc]
+#pragma unroll
+                for (int n = 0; n < 2; ++n)
+#pragma unroll
+                    for (int cc = 0; cc < 2; ++cc) dx[n][cc] = *(const float2*) (sa + (nb0 + 8 * n + 2 * tig + cc) * AB + 64);
+#pragma unroll
+                for (int h = 0; h < 2; ++h) {                     // the stage's two 32-value halves
+                    uint32_t bq[4];                               // {n0 bytes 0-15, n0 16-31, n1 0-15, n1 16-31}
+                    ldsm4(bq, sa + b_off + 32 * h);
+#pragma unroll
+                    for (int i = 0; i < 4; ++i) {
+                        uint32_t a[4];                            // {rows g, g + 8} x {bytes 0-15, 16-31}
+                        ldsm4(a, W + a_off + 16 * i * WLD + 32 * h);
+                        const float2 swa = *(const float2*) (S + (64 * wf + 16 * i + g) * 4 + 2 * h);
+                        const float2 swb = *(const float2*) (S + (64 * wf + 16 * i + 8 + g) * 4 + 2 * h);
+                        const float wa0 = swa.x, wa1 = swa.y, wb0 = swb.x, wb1 = swb.y;
+#pragma unroll
+                        for (int n = 0; n < 2; ++n) {
+                            if (n == 1 && !on1) break;
+                            if constexpr (K16) {
+                                // a scale per 16 weights: the two halves' dots, each times its scale, then d_x
+                                int d0[4], d1[4];
+                                mma16(d0, a[0], a[1], bq[2 * n]);
+                                mma16(d1, a[2], a[3], bq[2 * n + 1]);
+#pragma unroll
+                                for (int q = 0; q < 4; ++q) {
+                                    const float v = fmaf(q < 2 ? wa1 : wb1, dotf(d1[q]), (q < 2 ? wa0 : wb0) * dotf(d0[q]));
+                                    acc[i][n][q] = fmaf(h ? dx[n][q & 1].y : dx[n][q & 1].x, v, acc[i][n][q]);
+                                }
+                            } else {
+                                int d[4];
+                                mma32(d, a, bq[2 * n], bq[2 * n + 1]);
+#pragma unroll
+                                for (int q = 0; q < 4; ++q) {
+                                    const float p = (q < 2 ? wa0 : wb0) * (h ? dx[n][q & 1].y : dx[n][q & 1].x);
+                                    acc[i][n][q] = fmaf(p, dotf(d[q]), acc[i][n][q]);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            // the next stage's weights (into the other buffer, read after the next barrier).  (Decoding them before
+            // this stage's products instead was slower on the RTX 5070: a layer of 2048 / 3584 / 8192 tokens, IQ2_S
+            // 5.2 / 9.5 / 17.5 ms -> 6.0 / 10.2 / 18.3.)
+            if (s + 1 < NS) {
+                put(raw, (s + 1) & 1);
+                if (dec && s + 2 < NS) load_unit<WT>(unit(s + 2), sub(s + 2), raw);
+            }
+        }
+        if (!on0) continue;
+        if (GU) {
+            // SwiGLU of feature (WR / 2) fb + 32 wf + 8 i + g (lane g of m tile i: rows 64 wf + 16 i + g and + 8), tile
+            // rows nb0 + 8 n + 2 tig + c; H to int8 per row over the warp's 32 features (one 32-value half-block of the
+            // down product's K)
+            const int blk = WW * fb + wf;
+#pragma unroll
+            for (int n = 0; n < 2; ++n) {
+                if (n == 1 && !on1) break;
+#pragma unroll
+                for (int c = 0; c < 2; ++c) {
+                    float h[4], am = 0.0f;
+#pragma unroll
+                    for (int i = 0; i < 4; ++i) {
+                        const float gt = acc[i][n][c], up = acc[i][n][2 + c];
+                        h[i] = gt / (1.0f + __expf(-gt)) * up;
+                        am = fmaxf(am, fabsf(h[i]));
+                    }
+#pragma unroll
+                    for (int o = 4; o < 32; o <<= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
+                    const float inv = am > 0.0f ? 127.0f / am : 0.0f;
+                    const int r = nb0 + 8 * n + 2 * tig + c;
+                    if (r < nrows) {
+                        uint8_t* o = out + (size_t) (row0 + r) * (10 * AB) + (blk >> 1) * AB;
+                        const int hh = blk & 1;
+#pragma unroll
+                        for (int i = 0; i < 4; ++i) o[32 * hh + 8 * i + g] = (uint8_t) (int8_t) __float2int_rn(h[i] * inv);
+                        if (g == 0) *(float*) (o + 64 + 4 * hh) = am / 127.0f;
+                    }
+                }
+            }
+        } else {
+#pragma unroll
+            for (int i = 0; i < 4; ++i)
+#pragma unroll
+                for (int n = 0; n < 2; ++n) {
+                    if (n == 1 && !on1) break;
+#pragma unroll
+                    for (int q = 0; q < 4; ++q) {
+                        const int r = nb0 + 8 * n + 2 * tig + (q & 1);
+                        if (r < nrows) dm[(size_t) (row0 + r) * 2560 + rbase + 64 * wf + 16 * i + 8 * (q >> 1) + g] = acc[i][n][q];
+                    }
+                }
+        }
+    }
+#endif
+}
+
+unsigned blocks(int64_t n, int per) { return (unsigned) ((n + per - 1) / per); }
+
+// per device: whether every kernel here runs (sm_80+, device code in this build, fits), and their occupancy
+struct DevInfo {
+    bool done = false, ok = false;
+    int sms = 0, occ = 1;
+};
+std::mutex g_mu;
+DevInfo g_dev[32];
+
+template <int T, bool GU, int WW> bool setup_ww(int& occ) {
+    cudaFuncAttributes fa{};
+    if (cudaFuncGetAttributes(&fa, native_kernel<T, GU, WW>) != cudaSuccess || fa.ptxVersion < 80) return false;
+    if (cudaFuncSetAttribute(native_kernel<T, GU, WW>, cudaFuncAttributeMaxDynamicSharedMemorySize,
+                             (int) smem_bytes(T, WW)) != cudaSuccess)
+        return false;
+    int o = 0;
+    if (cudaOccupancyMaxActiveBlocksPerMultiprocessor(&o, native_kernel<T, GU, WW>, THREADS, smem_bytes(T, WW)) !=
+            cudaSuccess || o < 1)
+        return false;
+    occ = std::min(occ, o);
+    return true;
+}
+template <int T, bool GU> bool setup_one(int& occ) { return setup_ww<T, GU, 4>(occ) && setup_ww<T, GU, 2>(occ); }
+
+const DevInfo& dev_info() {
+    int dev = 0;
+    cudaGetDevice(&dev);
+    std::lock_guard<std::mutex> lk(g_mu);
+    DevInfo& d = g_dev[dev & 31];
+    if (d.done) return d;
+    d.done = true;
+    int major = 0;
+    cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+    cudaDeviceGetAttribute(&d.sms, cudaDevAttrMultiProcessorCount, dev);
+    if (major < 8) return d;
+    int occ = 1 << 20;
+    d.ok = setup_one<T_IQ2_XXS, true>(occ) && setup_one<T_IQ2_XS, true>(occ) && setup_one<T_IQ2_S, true>(occ) &&
+           setup_one<T_IQ3_XXS, true>(occ) && setup_one<T_IQ3_S, true>(occ) && setup_one<T_IQ4_XS, true>(occ) &&
+           setup_one<T_Q2_0, false>(occ) && setup_one<T_IQ4_NL, false>(occ);
+    d.occ = d.ok ? occ : 1;
+    cudaGetLastError();
+    return d;
+}
+
+bool gu_covered(int t) {
+    return t == T_IQ2_XXS || t == T_IQ2_XS || t == T_IQ2_S || t == T_IQ3_XXS || t == T_IQ3_S || t == T_IQ4_XS;
+}
+bool d_covered(int t) { return t == T_Q2_0 || t == T_IQ4_NL; }
+
+template <int T, bool GU>
+void launch(int ww, unsigned grid, const Batch& b, const NativeGeom& g, const Tables& tb, const void* act,
+            const int32_t* src, void* out, float* dm, cudaStream_t s) {
+    const uint8_t* a = (const uint8_t*) act;
+    uint8_t* o = (uint8_t*) out;
+    if (ww == 4) native_kernel<T, GU, 4><<<grid, THREADS, smem_bytes(T, 4), s>>>(b, g, tb, a, src, o, dm);
+    else native_kernel<T, GU, 2><<<grid, THREADS, smem_bytes(T, 2), s>>>(b, g, tb, a, src, o, dm);
+}
+
+// The work item's shape for a layer of `n` routed rows over `n_expert` experts: 128 routed rows when an expert has
+// about one such tile (56 to 112 rows on average), else 64.  Measured on the RTX 5070 (tests/cuda/prefill_fused_iq_test,
+// a layer of 2048 / 3584 / 8192 tokens, ms, 64 -> 128): IQ2_S 5.2 -> 6.7, 9.5 -> 9.3, 17.5 -> 19.5; IQ2_XXS 4.2 ->
+// 5.8, 7.4 -> 7.6, 13.6 -> 16.5; IQ3_S 5.6 -> 6.6, 10.2 -> 8.8, 18.4 -> 19.0; IQ3_XXS 5.1 -> 6.3, 9.1 -> 8.3, 16.5 ->
+// 17.5 (256 rows was slower still).  STRATA_PF_FUSED_TILE=64|128 forces one.
+int pick_ww(int64_t n, int n_expert) {
+    static const int forced = [] {
+        const char* v = std::getenv("STRATA_PF_FUSED_TILE");
+        const int t = v ? std::atoi(v) : 0;
+        return t == 64 ? 4 : t == 128 ? 2 : 0;
+    }();
+    if (forced) return forced;
+    const double avg = (double) n / std::max(n_expert, 1);
+    return avg > 56.0 && avg <= 112.0 ? 2 : 4;
+}
+
+}  // namespace
+
+bool native_supported(int gu_type, int d_type) {
+    static const bool off = [] {   // STRATA_PF_FUSED_NATIVE=0: the native packs keep MMQ under STRATA_PF_FUSED=1 (A/B)
+        const char* v = std::getenv("STRATA_PF_FUSED_NATIVE");
+        return v != nullptr && v[0] == '0';
+    }();
+    return !off && requested() && gu_covered(gu_type) && d_covered(d_type) && dev_info().ok;   // opt-in (=1)
+}
+
+void quantize_act_native(const float* x, int64_t rows, int64_t cols, void* xa, void* stream) {
+    if (rows <= 0) return;
+    const int64_t nblk = rows * (cols / 64);
+    quant_act_nat_kernel<<<blocks(nblk, 8), 256, 0, (cudaStream_t) stream>>>(x, nblk, (uint8_t*) xa);
+    ck(cudaGetLastError(), "quantize_act_native");
+}
+
+void experts_native(const Batch& b, const NativeGeom& g, int n_expert, int64_t n, const void* scratch, const void* xa,
+                    const int32_t* src, void* ha, float* dm, void* stream) {
+    if (b.e1 <= b.e0 || n <= 0) return;
+    const DevInfo& d = dev_info();
+    if (!d.ok || !gu_covered(g.gu_type) || !d_covered(g.d_type)) {
+        std::fprintf(stderr, "prefill fused experts (native): types %d / %d are not covered here\n", g.gu_type, g.d_type);
+        std::exit(1);
+    }
+    const cudaStream_t s = (cudaStream_t) stream;
+    const Tables tb = tables(const_cast<void*>(scratch), n_expert);
+    const int ww = pick_ww(n, n_expert);
+    // the most 64-row tiles the batch can have (every row in it, plus a partial tile per expert) - the items of the
+    // ones inside a larger item end at once
+    const int64_t tiles = (n + kTileRows - 1) / kTileRows + (b.e1 - b.e0);
+    const unsigned g_gu = (unsigned) std::min<int64_t>(tiles * (1280 / weight_rows(ww)), (int64_t) d.sms * d.occ);
+    const unsigned g_d = (unsigned) std::min<int64_t>(tiles * (2560 / weight_rows(ww)), (int64_t) d.sms * d.occ);
+    switch (g.gu_type) {
+        case T_IQ2_XXS: launch<T_IQ2_XXS, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        case T_IQ2_XS: launch<T_IQ2_XS, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        case T_IQ2_S: launch<T_IQ2_S, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        case T_IQ3_XXS: launch<T_IQ3_XXS, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        case T_IQ3_S: launch<T_IQ3_S, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        default: launch<T_IQ4_XS, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+    }
+    if (g.d_type == T_Q2_0) launch<T_Q2_0, false>(ww, g_d, b, g, tb, ha, src, nullptr, dm, s);
+    else launch<T_IQ4_NL, false>(ww, g_d, b, g, tb, ha, src, nullptr, dm, s);
+    ck(cudaGetLastError(), "experts_native");
+}
+
+}  // namespace strata::prefill::fused

src/prefill/moe_mmq.cu

@@ -29,6 +29,22 @@ __global__ void copy16_kernel(const uint4* __restrict__ a, int64_t na, const uin
     else if (i < na + nb) ab_dst[i] = b[i - na];
     else if (i < na + nb + nc) c_dst[i - na - nb] = c[i - na - nb];
 }
+// copy16_kernel for an MMQ group: blockIdx.y is the expert (first + y)
+struct GroupArgs {
+    const uint8_t* blob[kGatherGroupMax];
+    int64_t up_off, down_off, gu_stride, d_stride;   // in uint4
+};
+__global__ void copy16_group_kernel(GroupArgs ga, int first, int64_t na, int64_t nc, uint4* __restrict__ gu_dst,
+                                    uint4* __restrict__ d_dst) {
+    const int q = first + (int) blockIdx.y;
+    const uint4* src = (const uint4*) ga.blob[q];
+    uint4* ab = gu_dst + (int64_t) q * ga.gu_stride;
+    uint4* cd = d_dst + (int64_t) q * ga.d_stride;
+    const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < na) ab[i] = src[i];
+    else if (i < 2 * na) ab[i] = src[ga.up_off + (i - na)];
+    else if (i < 2 * na + nc) cd[i - 2 * na] = src[ga.down_off + (i - 2 * na)];
+}
 __global__ void copy1_kernel(const uint8_t* __restrict__ a, int64_t na, const uint8_t* __restrict__ b, int64_t nb,
                              uint8_t* __restrict__ ab_dst, const uint8_t* __restrict__ c, int64_t nc, uint8_t* __restrict__ c_dst) {
     const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
@@ -107,6 +123,34 @@ bool supported(int t) {
     }
 }
 
+bool fits(int t, int64_t w_rows) {
+    if (!supported(t)) return false;
+    // mul_mat_q_case's choice: the "fallback" configs when the rows are not a multiple of 128; then
+    // mul_mat_q_switch_J's loop - a tile size whose config exists for this card and fits its shared memory
+    const bool fallback = w_rows % 128 != 0;
+    const ggml_cuda_device_info& info = ggml_cuda_info();
+    for (int id = 0; id < info.device_count; ++id) {
+        const int cc = info.devices[id].cc;
+        const size_t smpbo = info.devices[id].smpbo;
+        bool any = false;
+        for (int J = 8; J <= 128 && !any; J += 8) {
+            const ggml_cuda_mmq_config c = ggml_cuda_mmq_get_config((ggml_type) t, J, fallback, cc);
+            any = c.type != GGML_TYPE_COUNT && mmq_get_nbytes_shared(c, cc) <= smpbo;
+        }
+        if (!any) {
+            static bool said[GGML_TYPE_COUNT] = {};
+            if (t >= 0 && t < GGML_TYPE_COUNT && !said[t]) {
+                said[t] = true;
+                std::fprintf(stderr, "strata: prompt kernels: llama.cpp's MMQ has no tile for %s (%lld rows) on GPU %d "
+                                     "(cc %d, %zu bytes of shared memory per block): that product takes the non-MMQ path "
+                                     "(#420)\n", ggml_type_name((ggml_type) t), (long long) w_rows, id, cc, smpbo);
+            }
+            return false;
+        }
+    }
+    return true;
+}
+
 size_t matrix_bytes(int t, int64_t rows, int64_t cols) {
     return (size_t) rows * (size_t) (cols / ggml_blck_size((ggml_type) t)) * ggml_type_size((ggml_type) t);
 }
@@ -182,6 +226,25 @@ void gather_native(const void* gate, const void* up, size_t gu_half_bytes, const
     ck(cudaGetLastError(), "gather_native");
 }
 
+bool gather_native_group(const GatherGroup& g, size_t up_off, size_t gu_half_bytes, size_t down_off, size_t d_bytes,
+                         void* gu_dst, size_t gu_stride, void* d_dst, size_t d_stride, void* stream) {
+    if (g.first < 0 || g.n <= g.first || g.n > kGatherGroupMax) return false;
+    uintptr_t a = (uintptr_t) gu_dst | (uintptr_t) d_dst | up_off | gu_half_bytes | down_off | d_bytes | gu_stride | d_stride;
+    for (int q = g.first; q < g.n; ++q) a |= (uintptr_t) g.blob[q];
+    if (a % 16 != 0) return false;
+    GroupArgs ga{};
+    for (int q = g.first; q < g.n; ++q) ga.blob[q] = g.blob[q];
+    ga.up_off = (int64_t) up_off / 16;
+    ga.down_off = (int64_t) down_off / 16;
+    ga.gu_stride = (int64_t) gu_stride / 16;
+    ga.d_stride = (int64_t) d_stride / 16;
+    const int64_t na = (int64_t) gu_half_bytes / 16, nc = (int64_t) d_bytes / 16;
+    copy16_group_kernel<<<dim3(blocks(2 * na + nc), (unsigned) (g.n - g.first)), 256, 0, (cudaStream_t) stream>>>(
+        ga, g.first, na, nc, (uint4*) gu_dst, (uint4*) d_dst);
+    ck(cudaGetLastError(), "gather_native_group");
+    return true;
+}
+
 void gather_strata_q2(const uint8_t* blob, void* gu_dst, void* d_dst, void* stream) {
     strata_q2_kernel<<<blocks(1280LL * 40 + 2560LL * 10), 256, 0, (cudaStream_t) stream>>>(blob, (uint16_t*) gu_dst,
                                                                                          (uint16_t*) d_dst);

tests/core/expert_cache_per_layer_test.cpp

@@ -0,0 +1,44 @@
+// #369: per-layer admission on a sized cache (open_sized, every native pack) - each layer admits into its own slot
+// range, as on the uniform cache, so no two layers write the same slot.  Needs a CUDA device (exits 77 without one).
+#include "strata/core/expert_cache.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdio>
+#include <string>
+#include <vector>
+
+int main() {
+    int n = 0;
+    if (cudaGetDeviceCount(&n) != cudaSuccess || n == 0) {
+        std::puts("no CUDA device: skipped");
+        return 77;
+    }
+    std::string err;
+    for (int sized = 0; sized < 2; ++sized) {
+        strata::core::ExpertCache cache;
+        const bool ok = sized ? cache.open_sized(std::vector<int64_t>(6, 1024), 3, 4, err)
+                              : cache.open(6, 3, 4, 1024, err);
+        if (!ok) {
+            std::fprintf(stderr, "open: %s\n", err.c_str());
+            return 1;
+        }
+        cache.set_per_layer_admission(true);
+        for (int64_t layer = 0; layer < 3; ++layer) {
+            int64_t lo = 0, hi = 0;
+            cache.layer_slot_range(layer, lo, hi);
+            for (int64_t e = 0; e < 3; ++e) {
+                const int32_t slot = cache.admit(layer, e);
+                const bool want_room = lo + e < hi;
+                if (want_room ? slot != (int32_t) (lo + e) : slot != strata::core::kNotResident) {
+                    std::fprintf(stderr, "%s cache: layer %lld expert %lld got slot %d (range %lld..%lld)\n",
+                                 sized ? "sized" : "uniform", (long long) layer, (long long) e, slot, (long long) lo,
+                                 (long long) hi - 1);
+                    return 1;
+                }
+            }
+        }
+    }
+    std::puts("expert cache per-layer admission: uniform and sized caches passed");
+    return 0;
+}

tests/core/expert_profile_save_test.cpp

@@ -0,0 +1,80 @@
+// #477: --expert-profile-save's profile - rank_learned_profile's order (resident first, then the counted routing, then
+// the profile the engine started from) and write_expert_profile's file, byte for byte tools/make_profile.py's format,
+// read back by read_expert_profile.  CPU only: no device is touched.
+#include "strata/core/expert_cache.hpp"
+
+#include <cstdio>
+#include <cstring>
+#include <filesystem>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+int fails = 0;
+void check(bool ok, const char* what) {
+    if (!ok) {
+        std::fprintf(stderr, "FAIL: %s\n", what);
+        ++fails;
+    }
+}
+}  // namespace
+
+int main() {
+    using Pair = std::pair<int32_t, int32_t>;
+    const int64_t L = 3, E = 4;   // 12 pairs, index = layer * 4 + expert
+    std::vector<uint8_t> resident(12, 0);
+    resident[5] = 1;              // (1, 1)
+    resident[2] = 1;              // (0, 2)
+    std::vector<double> heat(12, 0.0);
+    heat[2] = 1.0;
+    heat[5] = 7.0;
+    heat[11] = 9.0;               // (2, 3): the hottest, but not resident
+    heat[0] = 3.0;
+    const std::vector<Pair> prior = {{2, 0}, {1, 3}, {0, 1}};   // a profile that ranks three pairs
+    const std::vector<Pair> r = strata::core::rank_learned_profile(L, E, resident, heat, prior);
+    check(r.size() == 12, "every pair is ranked");
+    const std::vector<Pair> head = {{1, 1}, {0, 2}, {2, 3}, {0, 0}, {2, 0}, {1, 3}, {0, 1}, {0, 3}};
+    for (size_t i = 0; i < head.size() && i < r.size(); ++i)
+        check(r[i] == head[i], "resident by heat, then the rest by heat, then the prior, then the index");
+    // no counts at all: the resident ones, then the prior's order
+    const std::vector<Pair> r0 = strata::core::rank_learned_profile(L, E, resident, {}, prior);
+    check(r0.size() == 12 && r0[0] == Pair(0, 2) && r0[1] == Pair(1, 1) && r0[2] == Pair(2, 0) &&
+          r0[3] == Pair(1, 3) && r0[4] == Pair(0, 1) && r0[5] == Pair(0, 0), "without heat: resident, then prior");
+
+    const std::filesystem::path dir = std::filesystem::temp_directory_path() / "strata_profile_save_test";
+    std::filesystem::create_directories(dir);
+    const std::string path = (dir / "learned.bin").string();
+    std::string err;
+    check(strata::core::write_expert_profile(path, L, E, r, err), "written");
+    check(!std::filesystem::exists(path + ".tmp"), "no temporary file left");
+    // the bytes make_profile.write_profile would write for the same ranking
+    std::vector<uint8_t> want(4);
+    std::memcpy(want.data(), "STRP", 4);
+    auto u32 = [&](uint32_t v) { for (int b = 0; b < 4; ++b) want.push_back((uint8_t) (v >> (8 * b))); };
+    auto u16 = [&](uint32_t v) { want.push_back((uint8_t) v); want.push_back((uint8_t) (v >> 8)); };
+    for (uint32_t v : {1u, 3u, 4u, 12u, 12u}) u32(v);
+    for (const Pair& p : r) { u16((uint32_t) p.first); u16((uint32_t) p.second); }
+    std::vector<int32_t> table(12, -1);
+    for (size_t i = 0; i < r.size(); ++i) table[(size_t) (r[i].first * E + r[i].second)] = (int32_t) i;
+    for (int32_t v : table) u32((uint32_t) v);
+    std::vector<uint8_t> got(want.size() + 1);
+    if (std::FILE* f = std::fopen(path.c_str(), "rb")) {
+        got.resize(std::fread(got.data(), 1, got.size(), f));
+        std::fclose(f);
+    }
+    check(got == want, "the file is make_profile.py's format, byte for byte");
+    std::vector<Pair> back;
+    int64_t slots = 0;
+    check(strata::core::read_expert_profile(path, L, E, back, slots, err) && back == r && slots == 12,
+          "read back by the loader");
+    // a second save replaces the first (the rename over an existing file)
+    check(strata::core::write_expert_profile(path, L, E, r0, err), "written again");
+    check(strata::core::read_expert_profile(path, L, E, back, slots, err) && back == r0, "replaced");
+    check(!strata::core::write_expert_profile(path, L, E, {{3, 0}}, err), "a pair out of range is refused");
+    check(strata::core::read_expert_profile(path, L, E, back, slots, err) && back == r0,
+          "a refused save keeps the file");
+    std::filesystem::remove_all(dir);
+    if (fails == 0) std::puts("expert_profile_save_test: OK");
+    return fails == 0 ? 0 : 1;
+}

tests/cuda/prefill_fused_iq_test.cpp

@@ -0,0 +1,633 @@
+// prefill_fused_iq_test - #136: the fused int8 expert kernels for the native packs (moe_fused_iq.hpp, STRATA_PF_FUSED=1)
+// on random native expert blobs ([gate rows | up rows | down rows], GGUF blocks) and random routing, per format pair,
+// against
+//   - a double-precision reference: the weights dequantized by ggml's own to_float, times the FP32 activations
+//     (gate/up, SwiGLU, down), and
+//   - the MMQ path in prefill.cpp's sequence (gather_native into 16-expert groups, q8_1 of the slots' activations,
+//     gate/up, SwiGLU, q8_1 of H, down).
+// Both round the activations and H to int8 per 32 values, differently, so this is not a bitwise test: the fused
+// path's error against the reference has to be comparable to MMQ's own (at most 1.5x its RMS and 2x its worst row).
+// The pairs cover every format the kernels take: gate/up IQ2_XXS IQ2_XS IQ2_S IQ3_XXS IQ3_S IQ4_XS, down Q2_0 IQ4_NL.
+// Part 1 (reference), per pair: 256 tokens over 64 experts, skewed routing - experts with 0 rows, with one, with
+// several 64-row tiles - an all-zero token, per-expert blobs at unrelated (2-byte aligned) addresses, three launches of
+// different expert ranges.  Part 2 (timing): one layer at the engine's chunks (2048, 3584, 8192 tokens; 512 experts,
+// top 10) on both paths and on MMQ's products alone (no gathers), for the IQ2_XS, IQ3_XXS and IQ3_S packs' most
+// common layers, with the grouping checked and a sample of rows against the reference.  --no-ref, --no-timing,
+// --only=NAME, --chunks=A,B.  Exit 77 without a CUDA device of sm_80 or newer.
+#include "strata/prefill/moe_fused_iq.hpp"
+#include "strata/prefill/moe_mmq.hpp"
+
+#include "ggml.h"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <random>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace {
+namespace mmq = strata::prefill::mmq;
+namespace fused = strata::prefill::fused;
+
+constexpr int N = 2560, FF = 640, K = 10, GROUP = 16;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) throw std::runtime_error(std::string(what) + ": " + cudaGetErrorString(e));
+}
+
+struct Dev {
+    void* p = nullptr;
+    explicit Dev(size_t n) { ck(cudaMalloc(&p, n), "cudaMalloc"); }
+    ~Dev() { cudaFree(p); }
+    Dev(const Dev&) = delete;
+    Dev& operator=(const Dev&) = delete;
+    template <typename T> T* as() const { return (T*) p; }
+};
+
+struct Pair {
+    ggml_type gu, d;
+    const char* name;
+};
+
+struct Geo {
+    size_t gu_row, d_row, up_off, down_off, bytes;
+    explicit Geo(const Pair& p) {
+        gu_row = ggml_row_size(p.gu, N);
+        d_row = ggml_row_size(p.d, FF);
+        up_off = gu_row * FF;
+        down_off = 2 * up_off;
+        bytes = down_off + d_row * N;
+    }
+    fused::NativeGeom native(const Pair& p) const {
+        fused::NativeGeom g;
+        g.gu_type = (int) p.gu; g.d_type = (int) p.d; g.gu_row = gu_row; g.d_row = d_row; g.up_off = up_off;
+        g.down_off = down_off;
+        return g;
+    }
+};
+
+// random blocks with a sane fp16 scale `d` at each block's start (every format here begins with it): scales chosen so
+// that the weights' typical size is that of a 2-3 bit expert (a few hundredths)
+void fill_matrix(uint8_t* m, ggml_type t, int64_t rows, int64_t cols, std::mt19937& rng) {
+    const size_t bs = ggml_type_size(t), nb = (size_t) rows * (size_t) (cols / ggml_blck_size(t));
+    std::uniform_int_distribution<int> byte(0, 255);
+    const float lo = t == GGML_TYPE_IQ3_S ? 0.0004f : t == GGML_TYPE_IQ4_XS ? 0.00005f
+                   : t == GGML_TYPE_IQ4_NL ? 0.0002f : t == GGML_TYPE_Q2_0 ? 0.004f : 0.0008f;
+    std::uniform_real_distribution<float> sc(lo, 6.0f * lo);
+    for (size_t i = 0; i < nb * bs; ++i) m[i] = (uint8_t) byte(rng);
+    for (size_t b = 0; b < nb; ++b) {
+        const ggml_fp16_t h = ggml_fp32_to_fp16(sc(rng));
+        std::memcpy(m + b * bs, &h, 2);
+    }
+}
+std::vector<uint8_t> make_blob(const Pair& p, const Geo& g, std::mt19937& rng) {
+    std::vector<uint8_t> b(g.bytes);
+    fill_matrix(b.data(), p.gu, 2 * FF, N, rng);
+    fill_matrix(b.data() + g.down_off, p.d, N, FF, rng);
+    return b;
+}
+
+// ggml's dequantized weights: gate [640][2560], up [640][2560], down [2560][640]
+void dequant(const Pair& p, const Geo& g, const std::vector<uint8_t>& b, std::vector<float>& gu, std::vector<float>& dn) {
+    const auto* tg = ggml_get_type_traits(p.gu);
+    const auto* td = ggml_get_type_traits(p.d);
+    gu.resize((size_t) 2 * FF * N);
+    dn.resize((size_t) N * FF);
+    for (int r = 0; r < 2 * FF; ++r) tg->to_float(b.data() + (size_t) r * g.gu_row, gu.data() + (size_t) r * N, N);
+    for (int r = 0; r < N; ++r) td->to_float(b.data() + g.down_off + (size_t) r * g.d_row, dn.data() + (size_t) r * FF, FF);
+}
+
+// activations: N(0, 1) with a few large values (the hidden state has outliers); token `zero` all zero
+std::vector<float> make_x(int T, std::mt19937& rng, int zero) {
+    std::vector<float> x((size_t) T * N);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    std::uniform_int_distribution<int> pick(0, 199);
+    for (float& v : x) v = nd(rng) * (pick(rng) == 0 ? 12.0f : 1.0f);
+    if (zero >= 0) std::fill(x.begin() + (size_t) zero * N, x.begin() + (size_t) (zero + 1) * N, 0.0f);
+    return x;
+}
+
+// K distinct experts per token; `hot` > 0 skews the first choices towards the lowest ids
+std::vector<int32_t> make_ids(int T, int E, std::mt19937& rng, int hot, int unused) {
+    std::vector<int32_t> ids((size_t) T * K);
+    std::uniform_int_distribution<int> any(0, E - 1 - unused);
+    for (int t = 0; t < T; ++t)
+        for (int k = 0; k < K; ++k) {
+            int e;
+            bool dup;
+            do {
+                e = (hot > 0 && k < 3 && (int) (rng() % 4) != 0) ? (int) (rng() % (unsigned) hot) : any(rng);
+                dup = false;
+                for (int j = 0; j < k; ++j) dup |= ids[(size_t) t * K + j] == e;
+            } while (dup);
+            ids[(size_t) t * K + k] = e;
+        }
+    return ids;
+}
+
+struct Routing {
+    std::vector<int32_t> cnt, off, src;   // per expert; sorted row -> token (rows in pair order within an expert)
+    std::vector<int32_t> row_of;          // pair -> sorted row
+};
+Routing sort_rows(const std::vector<int32_t>& ids, int E) {
+    Routing r;
+    r.cnt.assign((size_t) E, 0);
+    for (int32_t e : ids) ++r.cnt[(size_t) e];
+    r.off.assign((size_t) E + 1, 0);
+    for (int e = 0; e < E; ++e) r.off[(size_t) e + 1] = r.off[(size_t) e] + r.cnt[(size_t) e];
+    std::vector<int32_t> fill(r.off.begin(), r.off.end() - 1);
+    r.src.resize(ids.size());
+    r.row_of.resize(ids.size());
+    for (size_t i = 0; i < ids.size(); ++i) {
+        const int32_t p = fill[(size_t) ids[i]]++;
+        r.row_of[i] = p;
+        r.src[(size_t) p] = (int32_t) (i / K);
+    }
+    return r;
+}
+
+// MMQ, as prefill.cpp runs a native layer: per group of 16 routed experts a gather, gate/up, SwiGLU (gate rows, then
+// up rows), q8_1 of H, down.  Returns Dm [rows][N] in sorted-row order.
+struct MmqBufs {
+    Dev xq, gu, h, hq, dm, ident, bounds, grp_gu, grp_d;
+    MmqBufs(const Pair& p, int64_t rows, int E)
+        : xq(mmq::q8_bytes(rows, N)), gu((size_t) rows * 1280 * 4), h((size_t) rows * FF * 4),
+          hq(mmq::q8_bytes(rows, FF)), dm((size_t) rows * N * 4), ident((size_t) rows * 4),
+          bounds((size_t) (2 * (E + E / GROUP + 2)) * 4), grp_gu(GROUP * mmq::matrix_bytes(p.gu, 1280, N) + 4096),
+          grp_d(GROUP * mmq::matrix_bytes(p.d, N, FF) + 4096) {}
+};
+// `direct`: no gathers - the products read the blobs where they are (they must lie at a fixed stride, every expert
+// with rows: the timing part's layout), the cost of MMQ's own kernels alone
+void run_mmq(const Pair& p, const Geo& geo, mmq::Context& ctx, MmqBufs& b, const Routing& r, const float* x_dev,
+             const int32_t* src_dev, const std::vector<const uint8_t*>& blob, int64_t rows, cudaStream_t s,
+             bool direct = false, size_t stride = 0) {
+    const size_t gub = mmq::matrix_bytes(p.gu, 1280, N), db = mmq::matrix_bytes(p.d, N, FF);
+    const int E = (int) r.cnt.size();
+    std::vector<int32_t> order;
+    for (int e = 0; e < E; ++e) if (r.cnt[(size_t) e] > 0) order.push_back(e);
+    const size_t n = order.size(), ng = (n + GROUP - 1) / GROUP;
+    std::vector<int32_t> bh(n + 1 + ng * (GROUP + 1));
+    for (size_t j = 0; j < n; ++j) bh[j] = r.off[(size_t) order[j]];
+    bh[n] = (int32_t) rows;
+    for (size_t g = 0; g < ng; ++g)
+        for (size_t i = 0; i <= GROUP; ++i)
+            bh[n + 1 + g * (GROUP + 1) + i] = bh[std::min(n, g * GROUP + i)] - bh[g * GROUP];
+    ck(cudaMemcpyAsync(b.bounds.p, bh.data(), bh.size() * 4, cudaMemcpyHostToDevice, s), "bounds");
+    mmq::iota(b.ident.as<int32_t>(), rows, s);
+    mmq::quantize(x_dev, src_dev, b.xq.p, (int) p.gu, N, N, rows, s);
+    for (size_t j = 0; j < n; ++j) {
+        const size_t q = j % GROUP;
+        const uint8_t* bl = blob[(size_t) order[j]];
+        if (!direct)
+            mmq::gather_native(bl, bl + geo.up_off, gub / 2, bl + geo.down_off, db, b.grp_gu.as<uint8_t>() + q * gub,
+                               b.grp_d.as<uint8_t>() + q * db, s);
+        if (q + 1 < GROUP && j + 1 < n) continue;
+        const size_t j0 = j - q, g = j0 / GROUP;
+        const int ngx = (int) (q + 1);
+        const int64_t r0 = bh[j0], nr = bh[j + 1] - r0;
+        int64_t maxr = 0;
+        for (size_t i = j0; i <= j; ++i) maxr = std::max<int64_t>(maxr, r.cnt[(size_t) order[i]]);
+        if (!direct) {
+            ck(cudaMemsetAsync(b.grp_gu.as<uint8_t>() + (size_t) ngx * gub, 0, 4096, s), "tail");
+            ck(cudaMemsetAsync(b.grp_d.as<uint8_t>() + (size_t) ngx * db, 0, 4096, s), "tail");
+        }
+        const uint8_t* b0 = blob[(size_t) order[j0]];
+        mmq::Product gu;
+        gu.w = direct ? (const void*) b0 : b.grp_gu.p; gu.type = (int) p.gu; gu.w_rows = 1280; gu.w_cols = N;
+        gu.expert_bytes = direct ? stride : gub;
+        gu.n = ngx; gu.xq = b.xq.p; gu.bounds = b.bounds.as<int32_t>() + j0; gu.ids = b.ident.as<int32_t>();
+        gu.total_rows = rows; gu.max_rows = maxr; gu.dst = b.gu.as<float>(); gu.ld_dst = 1280;
+        ctx.run(gu, s);
+        mmq::swiglu(b.gu.as<float>() + r0 * 1280, b.h.as<float>() + r0 * FF, nr, FF, false, s);
+        mmq::quantize(b.h.as<float>() + r0 * FF, nullptr, b.hq.p, (int) p.d, FF, FF, nr, s);
+        mmq::Product dn;
+        dn.w = direct ? (const void*) (b0 + geo.down_off) : b.grp_d.p; dn.type = (int) p.d; dn.w_rows = N; dn.w_cols = FF;
+        dn.expert_bytes = direct ? stride : db;
+        dn.n = ngx; dn.xq = b.hq.p; dn.bounds = b.bounds.as<int32_t>() + n + 1 + g * (GROUP + 1);
+        dn.ids = b.ident.as<int32_t>(); dn.total_rows = nr; dn.max_rows = maxr; dn.dst = b.dm.as<float>() + r0 * N;
+        dn.ld_dst = N;
+        ctx.run(dn, s);
+    }
+}
+
+// the fused path: quantize, group on the device, then launches over expert ranges `cuts` (e.g. {0, 20, 64})
+struct FusedBufs {
+    Dev xa, ha, dm, scratch, slot, src, ids;
+    FusedBufs(int T, int64_t rows, int E)
+        : xa(fused::act_bytes(T, N)), ha(fused::act_bytes(rows, FF)), dm((size_t) rows * N * 4),
+          scratch(fused::group_bytes(rows, E)), slot((size_t) rows * 4), src((size_t) rows * 4),
+          ids((size_t) rows * 4) {}
+};
+void run_fused(const Pair& p, const Geo& geo, FusedBufs& b, const float* x_dev, int T, int E,
+               const std::vector<int>& cuts, const std::vector<const uint8_t*>& blob, cudaStream_t s) {
+    const int64_t rows = (int64_t) T * K;
+    const fused::NativeGeom ng = geo.native(p);
+    fused::quantize_act_native(x_dev, T, N, b.xa.p, s);
+    fused::group(b.ids.as<int32_t>(), rows, K, E, b.scratch.p, b.slot.as<int32_t>(), b.src.as<int32_t>(), s);
+    for (size_t c = 0; c + 1 < cuts.size(); ++c) {
+        for (int e0 = cuts[c]; e0 < cuts[c + 1]; e0 += fused::kMaxBatch) {
+            fused::Batch bt;
+            bt.e0 = e0;
+            bt.e1 = std::min(cuts[c + 1], e0 + fused::kMaxBatch);
+            for (int e = bt.e0; e < bt.e1; ++e) bt.blob[e - bt.e0] = blob[(size_t) e];
+            fused::experts_native(bt, ng, E, rows, b.scratch.p, b.xa.p, b.src.as<int32_t>(), b.ha.p, b.dm.as<float>(), s);
+        }
+    }
+}
+
+struct Err {
+    double rms = 0, worst = 0;   // RMS error / RMS of the reference; the worst row's max |error| / its max |ref|
+};
+// y_a[row_a(i)] against y_b[row_b(i)] for every pair i (rows of N values)
+template <typename RA, typename RB>
+Err compare(const std::vector<float>& a, RA row_a, const std::vector<float>& b, RB row_b, size_t pairs) {
+    double e2 = 0, r2 = 0;
+    Err out;
+    for (size_t i = 0; i < pairs; ++i) {
+        const float* ya = a.data() + (size_t) row_a(i) * N;
+        const float* yb = b.data() + (size_t) row_b(i) * N;
+        double me = 0, mr = 0;
+        for (int o = 0; o < N; ++o) {
+            if (!std::isfinite(ya[o])) throw std::runtime_error("a non-finite output");
+            const double d = (double) ya[o] - yb[o];
+            e2 += d * d;
+            r2 += (double) yb[o] * yb[o];
+            me = std::max(me, std::fabs(d));
+            mr = std::max(mr, (double) std::fabs(yb[o]));
+        }
+        if (mr > 0) out.worst = std::max(out.worst, me / mr);
+        else if (me > 0) out.worst = 1e30;   // a zero row must stay exactly zero
+    }
+    out.rms = r2 > 0 ? std::sqrt(e2 / r2) : 0;
+    return out;
+}
+
+std::vector<float> download(const Dev& d, size_t n) {
+    std::vector<float> h(n);
+    ck(cudaMemcpy(h.data(), d.p, n * 4, cudaMemcpyDeviceToHost), "download");
+    return h;
+}
+std::vector<int32_t> download_i(const Dev& d, size_t n) {
+    std::vector<int32_t> h(n);
+    ck(cudaMemcpy(h.data(), d.p, n * 4, cudaMemcpyDeviceToHost), "download");
+    return h;
+}
+
+// part 1: against the double-precision reference
+void reference_part(const Pair& p, cudaStream_t s) {
+    constexpr int T = 256, E = 64, ZERO = 77;
+    const Geo geo(p);
+    std::mt19937 rng(136 + (unsigned) p.gu * 7 + (unsigned) p.d);
+    std::vector<std::vector<uint8_t>> host((size_t) E);
+    std::vector<std::unique_ptr<Dev>> dev;
+    std::vector<const uint8_t*> blob((size_t) E);
+    for (int e = 0; e < E; ++e) {
+        host[(size_t) e] = make_blob(p, geo, rng);
+        dev.push_back(std::make_unique<Dev>(geo.bytes + 4096 * (size_t) (e % 3) + 64));   // unrelated addresses
+        uint8_t* q = dev.back()->as<uint8_t>() + 256 * (size_t) (e % 3) + 2 * (size_t) (e % 2);   // 2-byte aligned
+        ck(cudaMemcpy(q, host[(size_t) e].data(), geo.bytes, cudaMemcpyHostToDevice), "blob");
+        blob[(size_t) e] = q;
+    }
+    const std::vector<float> x = make_x(T, rng, ZERO);
+    const std::vector<int32_t> ids = make_ids(T, E, rng, 2, 4);   // experts 0, 1 hot (several tiles), 60-63 unused
+    const int64_t rows = (int64_t) T * K;
+    const Routing r = sort_rows(ids, E);
+    std::printf("%s - reference part: %d tokens, %d experts, rows per expert: max %d, experts without rows %d\n", p.name,
+                T, E, *std::max_element(r.cnt.begin(), r.cnt.end()), (int) std::count(r.cnt.begin(), r.cnt.end(), 0));
+
+    Dev x_dev(x.size() * 4), src_dev((size_t) rows * 4);
+    ck(cudaMemcpy(x_dev.p, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+    ck(cudaMemcpy(src_dev.p, r.src.data(), r.src.size() * 4, cudaMemcpyHostToDevice), "src");
+    // (the uploads above are on the legacy stream and the work below on a non-blocking one: a cudaMemcpy from pageable
+    // memory may return before its DMA lands - on Windows it can wait in the queue until the next synchronous call)
+    ck(cudaDeviceSynchronize(), "uploads");
+    mmq::Context ctx;
+    MmqBufs mb(p, rows, E);
+    run_mmq(p, geo, ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s);
+    FusedBufs fb(T, rows, E);
+    ck(cudaMemcpy(fb.ids.p, ids.data(), ids.size() * 4, cudaMemcpyHostToDevice), "ids");
+    ck(cudaMemset(fb.dm.p, 0xff, (size_t) rows * N * 4), "sentinel");   // an unwritten row is NaN
+    ck(cudaDeviceSynchronize(), "uploads");
+    run_fused(p, geo, fb, x_dev.as<float>(), T, E, {0, 1, 20, E}, blob, s);
+    ck(cudaStreamSynchronize(s), "sync");
+    const std::vector<float> y_mmq = download(mb.dm, (size_t) rows * N), y_f = download(fb.dm, (size_t) rows * N);
+    const std::vector<int32_t> slot = download_i(fb.slot, (size_t) rows), fsrc = download_i(fb.src, (size_t) rows);
+    for (int64_t i = 0; i < rows; ++i)
+        if (fsrc[(size_t) slot[(size_t) i]] != (int32_t) (i / K)) throw std::runtime_error("group: slot/src disagree");
+
+    // the reference, an expert at a time on all cores
+    std::vector<float> ref((size_t) rows * N);
+    std::vector<std::thread> th;
+    const int nth = std::max(1, std::min(16, (int) std::thread::hardware_concurrency()));
+    for (int w = 0; w < nth; ++w)
+        th.emplace_back([&, w] {
+            std::vector<float> gu, dn, h(FF);
+            for (int e = w; e < E; e += nth) {
+                if (r.cnt[(size_t) e] == 0) continue;
+                dequant(p, geo, host[(size_t) e], gu, dn);
+                for (int64_t i = 0; i < rows; ++i) {
+                    if (ids[(size_t) i] != e) continue;
+                    const float* xr = x.data() + (size_t) (i / K) * N;
+                    for (int f = 0; f < FF; ++f) {
+                        double gt = 0, up = 0;
+                        const float *wg = gu.data() + (size_t) f * N, *wu = gu.data() + (size_t) (FF + f) * N;
+                        for (int k = 0; k < N; ++k) { gt += (double) wg[k] * xr[k]; up += (double) wu[k] * xr[k]; }
+                        h[(size_t) f] = (float) (gt / (1.0 + std::exp(-gt)) * up);
+                    }
+                    for (int o = 0; o < N; ++o) {
+                        double a = 0;
+                        const float* wd = dn.data() + (size_t) o * FF;
+                        for (int f = 0; f < FF; ++f) a += (double) wd[f] * h[(size_t) f];
+                        ref[(size_t) i * N + o] = (float) a;
+                    }
+                }
+            }
+        });
+    for (auto& t : th) t.join();
+
+    auto pair = [](size_t i) { return (int64_t) i; };
+    auto mrow = [&](size_t i) { return (int64_t) r.row_of[i]; };
+    auto frow = [&](size_t i) { return (int64_t) slot[i]; };
+    const Err em = compare(y_mmq, mrow, ref, pair, (size_t) rows);
+    const Err ef = compare(y_f, frow, ref, pair, (size_t) rows);
+    const Err efm = compare(y_f, frow, y_mmq, mrow, (size_t) rows);
+    std::printf("  MMQ   vs FP32 reference: rel RMS %.3e  worst row max rel %.3e\n", em.rms, em.worst);
+    std::printf("  fused vs FP32 reference: rel RMS %.3e  worst row max rel %.3e\n", ef.rms, ef.worst);
+    std::printf("  fused vs MMQ           : rel RMS %.3e  worst row max rel %.3e\n", efm.rms, efm.worst);
+    double zmax = 0;
+    for (int k = 0; k < K; ++k)
+        for (int o = 0; o < N; ++o)
+            zmax = std::max(zmax, (double) std::fabs(y_f[(size_t) slot[(size_t) (ZERO * K + k)] * N + o]));
+    if (zmax != 0) throw std::runtime_error(std::string(p.name) + ": the all-zero token's outputs are not zero");
+    if (ef.rms > 1.5 * em.rms || ef.worst > 2.0 * em.worst)
+        throw std::runtime_error(std::string(p.name) + ": the fused path's error is not comparable to MMQ's");
+}
+
+// part 2: one layer at a real chunk, timed
+void timing_part(const Pair& p, int T, cudaStream_t s) {
+    constexpr int E = 512;
+    const Geo geo(p);
+    std::mt19937 rng(1360);
+    const int64_t rows = (int64_t) T * K;
+    // 512 distinct blobs - the products read each once per tile, as the engine does
+    // at a stride of whole blocks of both formats (MMQ's expert stride counts blocks), + MMQ's read past the end
+    size_t unit = ggml_type_size(p.gu);
+    while (unit % ggml_type_size(p.d) != 0 || unit % 16 != 0) unit += ggml_type_size(p.gu);
+    const size_t stride = (geo.bytes + unit - 1) / unit * unit;
+    Dev blobs((size_t) E * stride + 4096);
+    ck(cudaMemset(blobs.p, 0, (size_t) E * stride + 4096), "zero");
+    {
+        std::vector<uint8_t> b = make_blob(p, geo, rng);
+        for (int e = 0; e < E; ++e) {
+            b[4 + (size_t) (e % 16)] ^= (uint8_t) e;   // not all equal (inside the first block, not its scale)
+            ck(cudaMemcpy(blobs.as<uint8_t>() + (size_t) e * stride, b.data(), geo.bytes, cudaMemcpyHostToDevice),
+               "blob");
+        }
+    }
+    std::vector<const uint8_t*> blob((size_t) E);
+    for (int e = 0; e < E; ++e) blob[(size_t) e] = blobs.as<uint8_t>() + (size_t) e * stride;
+    const std::vector<float> x = make_x(T, rng, -1);
+    const std::vector<int32_t> ids = make_ids(T, E, rng, 0, 0);
+    const Routing r = sort_rows(ids, E);
+    Dev x_dev(x.size() * 4), src_dev((size_t) rows * 4);
+    ck(cudaMemcpy(x_dev.p, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+    ck(cudaMemcpy(src_dev.p, r.src.data(), r.src.size() * 4, cudaMemcpyHostToDevice), "src");
+    mmq::Context ctx;
+    MmqBufs mb(p, rows, E);
+    FusedBufs fb(T, rows, E);
+    ck(cudaMemcpy(fb.ids.p, ids.data(), ids.size() * 4, cudaMemcpyHostToDevice), "ids");
+    ck(cudaDeviceSynchronize(), "uploads");   // (see reference_part)
+    cudaEvent_t a, b;
+    ck(cudaEventCreate(&a), "event");
+    ck(cudaEventCreate(&b), "event");
+    auto once = [&](auto&& f) {
+        ck(cudaEventRecord(a, s), "record");
+        f();
+        ck(cudaEventRecord(b, s), "record");
+        ck(cudaEventSynchronize(b), "sync");
+        float t = 0;
+        ck(cudaEventElapsedTime(&t, a, b), "elapsed");
+        return t;
+    };
+    auto mmq_run = [&] { run_mmq(p, geo, ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s); };
+    auto fused_run = [&] { run_fused(p, geo, fb, x_dev.as<float>(), T, E, {0, E}, blob, s); };
+    const bool all_routed = std::count(r.cnt.begin(), r.cnt.end(), 0) == 0;
+    auto direct_run = [&] {
+        run_mmq(p, geo, ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s, true, stride);
+    };
+    if (std::getenv("S20_IDSDIAG")) {   // which step overwrites the routing ids on the device
+        auto check = [&](const char* after) {
+            ck(cudaStreamSynchronize(s), "sync");
+            const std::vector<int32_t> d = download_i(fb.ids, (size_t) rows);
+            int64_t bad = 0;
+            for (size_t i = 0; i < (size_t) rows; ++i) bad += d[i] != ids[i];
+            std::printf("    ids on the device after %s: %lld of %lld differ (fb.ids %p, mb.dm %p..%p, fb.dm %p..%p)\n",
+                        after, (long long) bad, (long long) rows, fb.ids.p, mb.dm.p,
+                        (void*) (mb.dm.as<uint8_t>() + (size_t) rows * N * 4), fb.dm.p,
+                        (void*) (fb.dm.as<uint8_t>() + (size_t) rows * N * 4));
+        };
+        check("the upload");
+        mmq_run();
+        check("an MMQ run");
+        fused_run();
+        check("a fused run");
+        if (all_routed) {
+            direct_run();
+            check("an MMQ run without gathers");
+        }
+    }
+    // ~1 s of both first (the clocks ramp up), then alternating rounds; the medians
+    const bool no_mmq = std::getenv("S20_NOMMQ") != nullptr;   // (debug: the fused runs alone)
+    for (float spent = 0; spent < 1000.0f;) spent += (no_mmq ? 0.0f : once(mmq_run)) + once(fused_run);
+    std::vector<float> tm, tf, td;
+    for (int rep = 0; rep < 8; ++rep) {
+        tm.push_back(no_mmq ? 1.0f : once(mmq_run));
+        tf.push_back(once(fused_run));
+        if (all_routed && !no_mmq) td.push_back(once(direct_run));
+    }
+    if (td.empty()) td.push_back(0.0f);
+    std::sort(tm.begin(), tm.end());
+    std::sort(tf.begin(), tf.end());
+    std::sort(td.begin(), td.end());
+    const float t_mmq = tm[tm.size() / 2], t_f = tf[tf.size() / 2];
+    if (all_routed)
+        std::printf("  (MMQ's products alone, no gathers: %.3f ms - fused %.2fx)\n", td[td.size() / 2],
+                    td[td.size() / 2] / t_f);
+    std::vector<float> tf2;   // back to back
+    for (int rep = 0; rep < 8; ++rep) tf2.push_back(once(fused_run));
+    std::sort(tf2.begin(), tf2.end());
+    std::printf("  (fused back to back: median %.3f ms, min %.3f; alternating min %.3f; MMQ min %.3f)\n", tf2[4],
+                tf2[0], tf[0], tm[0]);
+    const std::vector<float> y_mmq = download(mb.dm, (size_t) rows * N), y_f = download(fb.dm, (size_t) rows * N);
+    const std::vector<int32_t> slot = download_i(fb.slot, (size_t) rows);
+    const Err efm = compare(y_f, [&](size_t i) { return (int64_t) slot[i]; }, y_mmq,
+                            [&](size_t i) { return (int64_t) r.row_of[i]; }, (size_t) rows);
+    std::printf("%s - timing part: %d tokens x top %d over %d experts, one layer: MMQ path %.3f ms, fused %.3f ms "
+                "(%.2fx); fused vs MMQ rel RMS %.3e worst row %.3e\n",
+                p.name, T, K, E, t_mmq, t_f, t_mmq / t_f, efm.rms, efm.worst);
+    {   // the grouping: every pair in its expert's rows (the near-identical blobs here would hide a wrong expert)
+        const std::vector<int32_t> fsrc = download_i(fb.src, (size_t) rows);
+        int64_t bad = 0, bad_src = 0;
+        for (size_t i = 0; i < (size_t) rows; ++i) {
+            const int e = ids[i];
+            bad += slot[i] < r.off[(size_t) e] || slot[i] >= r.off[(size_t) e + 1];
+            bad_src += fsrc[(size_t) slot[i]] != (int32_t) (i / K);
+        }
+        if (bad || bad_src) {
+            std::vector<int32_t> tbl((size_t) 4 * E + 2);
+            ck(cudaMemcpy(tbl.data(), fb.scratch.p, tbl.size() * 4, cudaMemcpyDeviceToHost), "tables");
+            int64_t bad_cnt = 0, bad_off = 0, bad_ids = 0;
+            const std::vector<int32_t> d_ids = download_i(fb.ids, (size_t) rows);
+            for (size_t i = 0; i < (size_t) rows; ++i) bad_ids += d_ids[i] != ids[i];
+            std::printf("  the routing ids on the device: %lld of %lld differ (first %d %d %d)\n", (long long) bad_ids,
+                        (long long) rows, d_ids[0], d_ids[1], d_ids[2]);
+            const int32_t smin = *std::min_element(slot.begin(), slot.end()), smax = *std::max_element(slot.begin(), slot.end());
+            std::printf("  slots %d..%d; off[0..3] %d %d %d %d off[E] %d; fill[0..3] %d %d %d %d; ts[0..3] %d %d %d %d "
+                        "ts[E] %d\n", smin, smax, tbl[E], tbl[E + 1], tbl[E + 2], tbl[E + 3], tbl[2 * E], tbl[2 * E + 1],
+                        tbl[2 * E + 2], tbl[2 * E + 3], tbl[2 * E + 4], tbl[3 * E + 1], tbl[3 * E + 2], tbl[3 * E + 3],
+                        tbl[3 * E + 4], tbl[4 * E + 1]);
+            // the group again on these ids, synchronously
+            fused::group(fb.ids.as<int32_t>(), rows, K, E, fb.scratch.p, fb.slot.as<int32_t>(), fb.src.as<int32_t>(), s);
+            ck(cudaStreamSynchronize(s), "sync");
+            std::vector<int32_t> t2((size_t) 4 * E + 2);
+            ck(cudaMemcpy(t2.data(), fb.scratch.p, t2.size() * 4, cudaMemcpyDeviceToHost), "tables");
+            std::printf("  group() again, synchronized: cnt[0..3] %d %d %d %d off[E] %d\n", t2[0], t2[1], t2[2], t2[3],
+                        t2[(size_t) 2 * E]);
+            for (int e = 0; e < E; ++e) {
+                bad_cnt += tbl[(size_t) e] != r.cnt[(size_t) e];
+                bad_off += tbl[(size_t) E + e] != r.off[(size_t) e];
+            }
+            std::printf("  GROUPING WRONG: %lld pairs outside their expert's rows, %lld src mismatches; tables: %lld counts "
+                        "and %lld offsets differ from the host's (cnt[0..3] %d %d %d %d vs %d %d %d %d)\n",
+                        (long long) bad, (long long) bad_src, (long long) bad_cnt, (long long) bad_off, tbl[0], tbl[1],
+                        tbl[2], tbl[3], r.cnt[0], r.cnt[1], r.cnt[2], r.cnt[3]);
+            throw std::runtime_error(std::string(p.name) + ": the grouping is wrong");
+        }
+    }
+    if (std::getenv("S20_ROWDIAG")) {   // the pairs whose rows differ most: expert, its rows, the row's place
+        const std::vector<int32_t> fsrc = download_i(fb.src, (size_t) rows);
+        std::vector<std::pair<double, size_t>> worst;
+        for (size_t i = 0; i < (size_t) rows; ++i) {
+            const float* a = y_f.data() + (size_t) slot[i] * N;
+            const float* m = y_mmq.data() + (size_t) r.row_of[i] * N;
+            double me = 0, mr = 0;
+            for (int o = 0; o < N; ++o) { me = std::max(me, (double) std::fabs(a[o] - m[o])); mr = std::max(mr, (double) std::fabs(m[o])); }
+            worst.push_back({mr > 0 ? me / mr : me, i});
+        }
+        std::sort(worst.rbegin(), worst.rend());
+        for (int k = 0; k < 12; ++k) {
+            const size_t i = worst[(size_t) k].second;
+            const int e = ids[i];
+            // the expert's rows start at its first fused row: the smallest slot of its pairs
+            int32_t first = INT32_MAX;
+            for (size_t j = 0; j < (size_t) rows; ++j) if (ids[j] == e) first = std::min(first, slot[j]);
+            std::printf("    pair %zu token %zu expert %d (%d rows) fused row %d (+%d in the expert) max rel %.3e\n", i,
+                        i / K, e, r.cnt[(size_t) e], slot[i], slot[i] - first, worst[(size_t) k].first);
+        }
+        (void) fsrc;
+    }
+    // both against the double-precision reference on a sample of 96 pairs
+    {
+        std::vector<size_t> pick;
+        for (size_t i = 0; i < 96; ++i) pick.push_back((i * 2654435761u) % (size_t) rows);
+        std::vector<float> ref(pick.size() * N), ym(pick.size() * N), yf(pick.size() * N);
+        std::vector<uint8_t> hb(geo.bytes);
+        std::vector<float> gu, dn, h(FF);
+        for (size_t j = 0; j < pick.size(); ++j) {
+            const size_t i = pick[j];
+            const int e = ids[i];
+            ck(cudaMemcpy(hb.data(), blob[(size_t) e], geo.bytes, cudaMemcpyDeviceToHost), "blob back");
+            dequant(p, geo, hb, gu, dn);
+            const float* xr = x.data() + (i / K) * N;
+            for (int f = 0; f < FF; ++f) {
+                double gt = 0, up = 0;
+                const float *wg = gu.data() + (size_t) f * N, *wu = gu.data() + (size_t) (FF + f) * N;
+                for (int k = 0; k < N; ++k) { gt += (double) wg[k] * xr[k]; up += (double) wu[k] * xr[k]; }
+                h[(size_t) f] = (float) (gt / (1.0 + std::exp(-gt)) * up);
+            }
+            for (int o = 0; o < N; ++o) {
+                double a = 0;
+                const float* wd = dn.data() + (size_t) o * FF;
+                for (int f = 0; f < FF; ++f) a += (double) wd[f] * h[(size_t) f];
+                ref[j * N + o] = (float) a;
+            }
+            std::copy_n(y_mmq.begin() + (ptrdiff_t) ((size_t) r.row_of[i] * N), N, ym.begin() + (ptrdiff_t) (j * N));
+            std::copy_n(y_f.begin() + (ptrdiff_t) ((size_t) slot[i] * N), N, yf.begin() + (ptrdiff_t) (j * N));
+        }
+        auto id = [](size_t j) { return (int64_t) j; };
+        const Err em = compare(ym, id, ref, id, pick.size()), ef = compare(yf, id, ref, id, pick.size());
+        std::printf("  sample of %zu pairs vs FP32 reference: MMQ rel RMS %.3e worst %.3e, fused rel RMS %.3e worst "
+                    "%.3e\n", pick.size(), em.rms, em.worst, ef.rms, ef.worst);
+        if (ef.rms > 1.5 * em.rms || ef.worst > 2.0 * em.worst)
+            throw std::runtime_error(std::string(p.name) + ": the fused path's error is not comparable to MMQ's (timing)");
+    }
+    cudaEventDestroy(a);
+    cudaEventDestroy(b);
+    if (efm.rms > 0.05) throw std::runtime_error(std::string(p.name) + ": the paths disagree at the real shape");
+}
+}  // namespace
+
+int main(int argc, char** argv) {
+    try {
+#ifdef _WIN32
+        _putenv_s("STRATA_PF_FUSED", "1");
+#else
+        setenv("STRATA_PF_FUSED", "1", 1);
+#endif
+        int n = 0;
+        if (cudaGetDeviceCount(&n) != cudaSuccess || n == 0) { std::printf("no CUDA device: skipped\n"); return 77; }
+        if (!fused::available()) { std::printf("the fused kernels need sm_80 or newer: skipped\n"); return 77; }
+        if (!mmq::built()) { std::printf("no MMQ in this build\n"); return 1; }
+        const std::vector<Pair> pairs = {
+            {GGML_TYPE_IQ2_S, GGML_TYPE_Q2_0, "IQ2_S / Q2_0"},       {GGML_TYPE_IQ2_XXS, GGML_TYPE_Q2_0, "IQ2_XXS / Q2_0"},
+            {GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_NL, "IQ2_XS / IQ4_NL"}, {GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ4_NL, "IQ3_XXS / IQ4_NL"},
+            {GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_NL, "IQ3_S / IQ4_NL"},   {GGML_TYPE_IQ4_XS, GGML_TYPE_Q2_0, "IQ4_XS / Q2_0"}};
+        const char* only = nullptr;   // --only=NAME: the pairs whose name contains NAME
+        for (int i = 1; i < argc; ++i)
+            if (std::strncmp(argv[i], "--only=", 7) == 0) only = argv[i] + 7;
+        bool timing = true, ref = true;   // --no-timing, --no-ref: skip a part
+        std::vector<int> chunks = {2048, 3584, 8192};   // --chunks=A,B: the timing part's chunk sizes
+        for (int i = 1; i < argc; ++i) {
+            if (std::strcmp(argv[i], "--no-timing") == 0) timing = false;
+            if (std::strcmp(argv[i], "--no-ref") == 0) ref = false;
+            if (std::strncmp(argv[i], "--chunks=", 9) == 0) {
+                chunks.clear();
+                for (const char* c = argv[i] + 9; *c;) {
+                    chunks.push_back(std::atoi(c));
+                    while (*c && *c != ',') ++c;
+                    if (*c == ',') ++c;
+                }
+            }
+        }
+        cudaStream_t s = nullptr;
+        ck(cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking), "stream");
+        for (const Pair& p : pairs) {
+            if (only && std::string(p.name).find(only) == std::string::npos) continue;
+            if (!fused::native_supported((int) p.gu, (int) p.d))
+                throw std::runtime_error(std::string(p.name) + ": not covered by the native kernels");
+            if (ref) reference_part(p, s);
+        }
+        if (timing) {
+            // the IQ2_XS pack's layers (34 of 48 IQ2_S / Q2_0, 11 IQ2_XXS / Q2_0), the IQ3_S pack's most common, IQ3_XXS's
+            for (const Pair& p : {pairs[0], pairs[1], pairs[4], pairs[3]}) {
+                if (only && std::string(p.name).find(only) == std::string::npos) continue;
+                for (int T : chunks) timing_part(p, T, s);
+            }
+        }
+        ck(cudaStreamDestroy(s), "destroy");
+        std::printf("prefill fused MoE (native formats) parity passed\n");
+        return 0;
+    } catch (const std::exception& e) {
+        std::fprintf(stderr, "prefill fused MoE (native formats) parity failed: %s\n", e.what());
+        return 1;
+    }
+}

tests/cuda/prefill_fused_moe_test.cpp

@@ -0,0 +1,420 @@
+// prefill_fused_moe_test - #136: the fused int8 expert kernels of the prompt path (moe_fused.hpp, STRATA_PF_FUSED=1)
+// on random Strata Q2_0 expert blobs and random routing, against
+//   - a double-precision reference: the dequantized weights times the FP32 activations (gate/up, SwiGLU, down), and
+//   - the MMQ path in prefill.cpp's sequence (gather_strata_q2 into 16-expert groups, q8_1 of the slots' activations,
+//     gate/up, SwiGLU, q8_1 of H, down).
+// Both round the activations and H to int8 per 32 values, differently, so this is not a bitwise test: the fused
+// path's error against the reference has to be comparable to MMQ's own (at most 1.5x its RMS and 2x its worst row).
+// Part 1 (reference): 256 tokens over 64 experts, skewed routing - experts with 0 rows, with one, with several 64-row
+// tiles - an all-zero token, per-expert blobs at unrelated addresses, three launches of different expert ranges.
+// Part 2 (timing): one layer at a real chunk (2048 tokens, 512 experts, top 10) on both paths, with their agreement.
+// Exit 77 without a CUDA device of sm_80 or newer.
+#include "strata/prefill/moe_fused.hpp"
+#include "strata/prefill/moe_mmq.hpp"
+
+#include "ggml.h"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <memory>
+#include <random>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace {
+namespace mmq = strata::prefill::mmq;
+namespace fused = strata::prefill::fused;
+
+constexpr int N = 2560, FF = 640, K = 10, GROUP = 16, Q2 = 42;   // GGML_TYPE_Q2_0
+constexpr size_t BLOB = 1382400, O_D_CODES = (size_t) 1280 * 640, O_GU_SC = O_D_CODES + (size_t) 2560 * 160,
+                 O_D_SC = O_GU_SC + (size_t) 1280 * 40 * 2;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) throw std::runtime_error(std::string(what) + ": " + cudaGetErrorString(e));
+}
+
+struct Dev {
+    void* p = nullptr;
+    explicit Dev(size_t n) { ck(cudaMalloc(&p, n), "cudaMalloc"); }
+    ~Dev() { cudaFree(p); }
+    Dev(const Dev&) = delete;
+    Dev& operator=(const Dev&) = delete;
+    template <typename T> T* as() const { return (T*) p; }
+};
+
+// a Strata Q2_0 blob: random codes, scales in [0.004, 0.03] (a 2-bit expert's range)
+std::vector<uint8_t> make_blob(std::mt19937& rng) {
+    std::vector<uint8_t> b(BLOB);
+    std::uniform_int_distribution<int> byte(0, 255);
+    std::uniform_real_distribution<float> sc(0.004f, 0.03f);
+    for (size_t i = 0; i < O_GU_SC; ++i) b[i] = (uint8_t) byte(rng);
+    for (size_t i = O_GU_SC; i < BLOB; i += 2) {
+        const ggml_fp16_t h = ggml_fp32_to_fp16(sc(rng));
+        std::memcpy(&b[i], &h, 2);
+    }
+    return b;
+}
+
+// the blob's weights as floats: gate/up [1280][2560] (interleaved rows), down [2560][640]
+void dequant(const std::vector<uint8_t>& b, std::vector<float>& gu, std::vector<float>& dn) {
+    auto row = [&](size_t codes, size_t ld, size_t scales, int nb, int r, float* out) {
+        for (int k = 0; k < nb * 64; ++k) {
+            ggml_fp16_t h;
+            std::memcpy(&h, &b[scales + ((size_t) r * nb + k / 64) * 2], 2);
+            const int q = (b[codes + (size_t) r * ld + k / 4] >> (2 * (k % 4))) & 3;
+            out[k] = ggml_fp16_to_fp32(h) * (float) (q - 1);
+        }
+    };
+    gu.resize((size_t) 1280 * N);
+    dn.resize((size_t) N * FF);
+    for (int r = 0; r < 1280; ++r) row(0, 640, O_GU_SC, 40, r, gu.data() + (size_t) r * N);
+    for (int r = 0; r < N; ++r) row(O_D_CODES, 160, O_D_SC, 10, r, dn.data() + (size_t) r * FF);
+}
+
+// activations: N(0, 1) with a few large values (the hidden state has outliers); token `zero` all zero
+std::vector<float> make_x(int T, std::mt19937& rng, int zero) {
+    std::vector<float> x((size_t) T * N);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    std::uniform_int_distribution<int> pick(0, 199);
+    for (float& v : x) v = nd(rng) * (pick(rng) == 0 ? 12.0f : 1.0f);
+    if (zero >= 0) std::fill(x.begin() + (size_t) zero * N, x.begin() + (size_t) (zero + 1) * N, 0.0f);
+    return x;
+}
+
+// K distinct experts per token; `hot` > 0 skews the first choices towards the lowest ids
+std::vector<int32_t> make_ids(int T, int E, std::mt19937& rng, int hot, int unused) {
+    std::vector<int32_t> ids((size_t) T * K);
+    std::uniform_int_distribution<int> any(0, E - 1 - unused);
+    for (int t = 0; t < T; ++t)
+        for (int k = 0; k < K; ++k) {
+            int e;
+            bool dup;
+            do {
+                e = (hot > 0 && k < 3 && (int) (rng() % 4) != 0) ? (int) (rng() % (unsigned) hot) : any(rng);
+                dup = false;
+                for (int j = 0; j < k; ++j) dup |= ids[(size_t) t * K + j] == e;
+            } while (dup);
+            ids[(size_t) t * K + k] = e;
+        }
+    return ids;
+}
+
+struct Routing {
+    std::vector<int32_t> cnt, off, src;   // per expert; sorted row -> token (rows in pair order within an expert)
+    std::vector<int32_t> row_of;          // pair -> sorted row
+};
+Routing sort_rows(const std::vector<int32_t>& ids, int E) {
+    Routing r;
+    r.cnt.assign((size_t) E, 0);
+    for (int32_t e : ids) ++r.cnt[(size_t) e];
+    r.off.assign((size_t) E + 1, 0);
+    for (int e = 0; e < E; ++e) r.off[(size_t) e + 1] = r.off[(size_t) e] + r.cnt[(size_t) e];
+    std::vector<int32_t> fill(r.off.begin(), r.off.end() - 1);
+    r.src.resize(ids.size());
+    r.row_of.resize(ids.size());
+    for (size_t i = 0; i < ids.size(); ++i) {
+        const int32_t p = fill[(size_t) ids[i]]++;
+        r.row_of[i] = p;
+        r.src[(size_t) p] = (int32_t) (i / K);
+    }
+    return r;
+}
+
+// MMQ, as prefill.cpp runs it: per group of 16 routed experts a gather, gate/up, SwiGLU, q8_1 of H, down.  Returns
+// Dm [rows][N] in sorted-row order.
+struct MmqBufs {
+    Dev xq, gu, h, hq, dm, ident, bounds, grp_gu, grp_d;
+    MmqBufs(int64_t rows, int E)
+        : xq(mmq::q8_bytes(rows, N)), gu((size_t) rows * 1280 * 4), h((size_t) rows * FF * 4),
+          hq(mmq::q8_bytes(rows, FF)),
+          dm((size_t) rows * N * 4), ident((size_t) rows * 4), bounds((size_t) (2 * (E + E / GROUP + 2)) * 4),
+          grp_gu(GROUP * mmq::matrix_bytes(Q2, 1280, N) + 4096), grp_d(GROUP * mmq::matrix_bytes(Q2, N, FF) + 4096) {}
+};
+void run_mmq(mmq::Context& ctx, MmqBufs& b, const Routing& r, const float* x_dev, const int32_t* src_dev,
+             const std::vector<const uint8_t*>& blob, int64_t rows, cudaStream_t s) {
+    const size_t gub = mmq::matrix_bytes(Q2, 1280, N), db = mmq::matrix_bytes(Q2, N, FF);
+    const int E = (int) r.cnt.size();
+    std::vector<int32_t> order;
+    for (int e = 0; e < E; ++e) if (r.cnt[(size_t) e] > 0) order.push_back(e);
+    const size_t n = order.size(), ng = (n + GROUP - 1) / GROUP;
+    std::vector<int32_t> bh(n + 1 + ng * (GROUP + 1));
+    for (size_t j = 0; j < n; ++j) bh[j] = r.off[(size_t) order[j]];
+    bh[n] = (int32_t) rows;
+    for (size_t g = 0; g < ng; ++g)
+        for (size_t i = 0; i <= GROUP; ++i)
+            bh[n + 1 + g * (GROUP + 1) + i] = bh[std::min(n, g * GROUP + i)] - bh[g * GROUP];
+    ck(cudaMemcpyAsync(b.bounds.p, bh.data(), bh.size() * 4, cudaMemcpyHostToDevice, s), "bounds");
+    mmq::iota(b.ident.as<int32_t>(), rows, s);
+    mmq::quantize(x_dev, src_dev, b.xq.p, Q2, N, N, rows, s);
+    for (size_t j = 0; j < n; ++j) {
+        const size_t q = j % GROUP;
+        mmq::gather_strata_q2(blob[(size_t) order[j]], b.grp_gu.as<uint8_t>() + q * gub, b.grp_d.as<uint8_t>() + q * db,
+                              s);
+        if (q + 1 < GROUP && j + 1 < n) continue;
+        const size_t j0 = j - q, g = j0 / GROUP;
+        const int ngx = (int) (q + 1);
+        const int64_t r0 = bh[j0], nr = bh[j + 1] - r0;
+        int64_t maxr = 0;
+        for (size_t i = j0; i <= j; ++i) maxr = std::max<int64_t>(maxr, r.cnt[(size_t) order[i]]);
+        ck(cudaMemsetAsync(b.grp_gu.as<uint8_t>() + (size_t) ngx * gub, 0, 4096, s), "tail");
+        ck(cudaMemsetAsync(b.grp_d.as<uint8_t>() + (size_t) ngx * db, 0, 4096, s), "tail");
+        mmq::Product gu;
+        gu.w = b.grp_gu.p; gu.type = Q2; gu.w_rows = 1280; gu.w_cols = N; gu.expert_bytes = gub;
+        gu.n = ngx; gu.xq = b.xq.p; gu.bounds = b.bounds.as<int32_t>() + j0; gu.ids = b.ident.as<int32_t>();
+        gu.total_rows = rows; gu.max_rows = maxr; gu.dst = b.gu.as<float>(); gu.ld_dst = 1280;
+        ctx.run(gu, s);
+        mmq::swiglu(b.gu.as<float>() + r0 * 1280, b.h.as<float>() + r0 * FF, nr, FF, true, s);
+        mmq::quantize(b.h.as<float>() + r0 * FF, nullptr, b.hq.p, Q2, FF, FF, nr, s);
+        mmq::Product dn;
+        dn.w = b.grp_d.p; dn.type = Q2; dn.w_rows = N; dn.w_cols = FF; dn.expert_bytes = db;
+        dn.n = ngx; dn.xq = b.hq.p; dn.bounds = b.bounds.as<int32_t>() + n + 1 + g * (GROUP + 1);
+        dn.ids = b.ident.as<int32_t>(); dn.total_rows = nr; dn.max_rows = maxr; dn.dst = b.dm.as<float>() + r0 * N;
+        dn.ld_dst = N;
+        ctx.run(dn, s);
+    }
+}
+
+// the fused path: quantize, group on the device, then launches over expert ranges `cuts` (e.g. {0, 20, 64})
+struct FusedBufs {
+    Dev xa, ha, dm, scratch, slot, src, ids;
+    FusedBufs(int T, int64_t rows, int E)
+        : xa(fused::act_bytes(T, N)), ha(fused::act_bytes(rows, FF)), dm((size_t) rows * N * 4),
+          scratch(fused::group_bytes(rows, E)), slot((size_t) rows * 4), src((size_t) rows * 4),
+          ids((size_t) rows * 4) {}
+};
+void run_fused(FusedBufs& b, const float* x_dev, int T, int E, const std::vector<int>& cuts,
+               const std::vector<const uint8_t*>& blob, cudaStream_t s) {
+    const int64_t rows = (int64_t) T * K;
+    fused::quantize_act(x_dev, T, N, b.xa.p, s);
+    fused::group(b.ids.as<int32_t>(), rows, K, E, b.scratch.p, b.slot.as<int32_t>(), b.src.as<int32_t>(), s);
+    for (size_t c = 0; c + 1 < cuts.size(); ++c) {
+        for (int e0 = cuts[c]; e0 < cuts[c + 1]; e0 += fused::kMaxBatch) {
+            fused::Batch bt;
+            bt.e0 = e0;
+            bt.e1 = std::min(cuts[c + 1], e0 + fused::kMaxBatch);
+            for (int e = bt.e0; e < bt.e1; ++e) bt.blob[e - bt.e0] = blob[(size_t) e];
+            fused::experts(bt, E, rows, b.scratch.p, b.xa.p, b.src.as<int32_t>(), b.ha.p, b.dm.as<float>(), s);
+        }
+    }
+}
+
+struct Err {
+    double rms = 0, worst = 0;   // RMS error / RMS of the reference; the worst row's max |error| / its max |ref|
+};
+// y_a[row_a(i)] against y_b[row_b(i)] for every pair i (rows of N values)
+template <typename RA, typename RB>
+Err compare(const std::vector<float>& a, RA row_a, const std::vector<float>& b, RB row_b, size_t pairs) {
+    double e2 = 0, r2 = 0;
+    Err out;
+    for (size_t i = 0; i < pairs; ++i) {
+        const float* ya = a.data() + (size_t) row_a(i) * N;
+        const float* yb = b.data() + (size_t) row_b(i) * N;
+        double me = 0, mr = 0;
+        for (int o = 0; o < N; ++o) {
+            if (!std::isfinite(ya[o])) throw std::runtime_error("a non-finite output");
+            const double d = (double) ya[o] - yb[o];
+            e2 += d * d;
+            r2 += (double) yb[o] * yb[o];
+            me = std::max(me, std::fabs(d));
+            mr = std::max(mr, (double) std::fabs(yb[o]));
+        }
+        if (mr > 0) out.worst = std::max(out.worst, me / mr);
+        else if (me > 0) out.worst = 1e30;   // a zero row must stay exactly zero
+    }
+    out.rms = r2 > 0 ? std::sqrt(e2 / r2) : 0;
+    return out;
+}
+
+std::vector<float> download(const Dev& d, size_t n) {
+    std::vector<float> h(n);
+    ck(cudaMemcpy(h.data(), d.p, n * 4, cudaMemcpyDeviceToHost), "download");
+    return h;
+}
+std::vector<int32_t> download_i(const Dev& d, size_t n) {
+    std::vector<int32_t> h(n);
+    ck(cudaMemcpy(h.data(), d.p, n * 4, cudaMemcpyDeviceToHost), "download");
+    return h;
+}
+
+// part 1: against the double-precision reference
+void reference_part(cudaStream_t s) {
+    constexpr int T = 256, E = 64, ZERO = 77;
+    std::mt19937 rng(136);
+    std::vector<std::vector<uint8_t>> host((size_t) E);
+    std::vector<std::unique_ptr<Dev>> dev;
+    std::vector<const uint8_t*> blob((size_t) E);
+    for (int e = 0; e < E; ++e) {
+        host[(size_t) e] = make_blob(rng);
+        dev.push_back(std::make_unique<Dev>(BLOB + 4096 * (size_t) (e % 3)));   // unrelated addresses
+        uint8_t* p = dev.back()->as<uint8_t>() + 256 * (size_t) (e % 3);       // 16-byte aligned, not 512
+        ck(cudaMemcpy(p, host[(size_t) e].data(), BLOB, cudaMemcpyHostToDevice), "blob");
+        blob[(size_t) e] = p;
+    }
+    const std::vector<float> x = make_x(T, rng, ZERO);
+    const std::vector<int32_t> ids = make_ids(T, E, rng, 2, 4);   // experts 0, 1 hot (several tiles), 60-63 unused
+    const int64_t rows = (int64_t) T * K;
+    const Routing r = sort_rows(ids, E);
+    std::printf("reference part: %d tokens, %d experts, rows per expert: max %d, experts without rows %d\n", T, E,
+                *std::max_element(r.cnt.begin(), r.cnt.end()), (int) std::count(r.cnt.begin(), r.cnt.end(), 0));
+
+    Dev x_dev(x.size() * 4), src_dev((size_t) rows * 4);
+    ck(cudaMemcpy(x_dev.p, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+    ck(cudaMemcpy(src_dev.p, r.src.data(), r.src.size() * 4, cudaMemcpyHostToDevice), "src");
+    mmq::Context ctx;
+    MmqBufs mb(rows, E);
+    // the uploads (legacy stream) must land before the non-blocking stream reads them (S20 found the kernels could run
+    // on all-zero routing otherwise - every pair on one expert: fast and wrong)
+    ck(cudaDeviceSynchronize(), "uploads");
+    run_mmq(ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s);
+    FusedBufs fb(T, rows, E);
+    ck(cudaMemcpy(fb.ids.p, ids.data(), ids.size() * 4, cudaMemcpyHostToDevice), "ids");
+    ck(cudaMemset(fb.dm.p, 0xff, (size_t) rows * N * 4), "sentinel");   // an unwritten row is NaN
+    ck(cudaDeviceSynchronize(), "uploads");
+    run_fused(fb, x_dev.as<float>(), T, E, {0, 1, 20, E}, blob, s);
+    ck(cudaStreamSynchronize(s), "sync");
+    const std::vector<float> y_mmq = download(mb.dm, (size_t) rows * N), y_f = download(fb.dm, (size_t) rows * N);
+    const std::vector<int32_t> slot = download_i(fb.slot, (size_t) rows), fsrc = download_i(fb.src, (size_t) rows);
+    for (int64_t i = 0; i < rows; ++i)
+        if (fsrc[(size_t) slot[(size_t) i]] != (int32_t) (i / K)) throw std::runtime_error("group: slot/src disagree");
+
+    // the reference, an expert at a time on all cores
+    std::vector<float> ref((size_t) rows * N);
+    std::vector<std::thread> th;
+    const int nth = std::max(1, std::min(16, (int) std::thread::hardware_concurrency()));
+    for (int w = 0; w < nth; ++w)
+        th.emplace_back([&, w] {
+            std::vector<float> gu, dn, h(FF);
+            for (int e = w; e < E; e += nth) {
+                if (r.cnt[(size_t) e] == 0) continue;
+                dequant(host[(size_t) e], gu, dn);
+                for (int64_t i = 0; i < rows; ++i) {
+                    if (ids[(size_t) i] != e) continue;
+                    const float* xr = x.data() + (size_t) (i / K) * N;
+                    for (int f = 0; f < FF; ++f) {
+                        double gt = 0, up = 0;
+                        const float *wg = gu.data() + (size_t) (2 * f) * N, *wu = wg + N;
+                        for (int k = 0; k < N; ++k) { gt += (double) wg[k] * xr[k]; up += (double) wu[k] * xr[k]; }
+                        h[(size_t) f] = (float) (gt / (1.0 + std::exp(-gt)) * up);
+                    }
+                    for (int o = 0; o < N; ++o) {
+                        double a = 0;
+                        const float* wd = dn.data() + (size_t) o * FF;
+                        for (int f = 0; f < FF; ++f) a += (double) wd[f] * h[(size_t) f];
+                        ref[(size_t) i * N + o] = (float) a;
+                    }
+                }
+            }
+        });
+    for (auto& t : th) t.join();
+
+    auto pair = [](size_t i) { return (int64_t) i; };
+    auto mrow = [&](size_t i) { return (int64_t) r.row_of[i]; };
+    auto frow = [&](size_t i) { return (int64_t) slot[i]; };
+    const Err em = compare(y_mmq, mrow, ref, pair, (size_t) rows);
+    const Err ef = compare(y_f, frow, ref, pair, (size_t) rows);
+    const Err efm = compare(y_f, frow, y_mmq, mrow, (size_t) rows);
+    std::printf("  MMQ   vs FP32 reference: rel RMS %.3e  worst row max rel %.3e\n", em.rms, em.worst);
+    std::printf("  fused vs FP32 reference: rel RMS %.3e  worst row max rel %.3e\n", ef.rms, ef.worst);
+    std::printf("  fused vs MMQ           : rel RMS %.3e  worst row max rel %.3e\n", efm.rms, efm.worst);
+    double zmax = 0;
+    for (int k = 0; k < K; ++k)
+        for (int o = 0; o < N; ++o)
+            zmax = std::max(zmax, (double) std::fabs(y_f[(size_t) slot[(size_t) (ZERO * K + k)] * N + o]));
+    if (zmax != 0) throw std::runtime_error("the all-zero token's outputs are not zero");
+    if (ef.rms > 1.5 * em.rms || ef.worst > 2.0 * em.worst)
+        throw std::runtime_error("the fused path's error is not comparable to MMQ's");
+}
+
+// part 2: one layer at a real chunk, timed
+void timing_part(cudaStream_t s) {
+    constexpr int T = 2048, E = 512;
+    std::mt19937 rng(1360);
+    const int64_t rows = (int64_t) T * K;
+    // 512 distinct blobs (708 MB) - the products read each once per tile, as the engine does
+    Dev blobs((size_t) E * BLOB);
+    {
+        std::vector<uint8_t> b = make_blob(rng);
+        for (int e = 0; e < E; ++e) {
+            b[(size_t) e * 7919 % O_GU_SC] ^= (uint8_t) e;   // not all equal
+            ck(cudaMemcpy(blobs.as<uint8_t>() + (size_t) e * BLOB, b.data(), BLOB, cudaMemcpyHostToDevice), "blob");
+        }
+    }
+    std::vector<const uint8_t*> blob((size_t) E);
+    for (int e = 0; e < E; ++e) blob[(size_t) e] = blobs.as<uint8_t>() + (size_t) e * BLOB;
+    const std::vector<float> x = make_x(T, rng, -1);
+    const std::vector<int32_t> ids = make_ids(T, E, rng, 0, 0);
+    const Routing r = sort_rows(ids, E);
+    Dev x_dev(x.size() * 4), src_dev((size_t) rows * 4);
+    ck(cudaMemcpy(x_dev.p, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+    ck(cudaMemcpy(src_dev.p, r.src.data(), r.src.size() * 4, cudaMemcpyHostToDevice), "src");
+    mmq::Context ctx;
+    MmqBufs mb(rows, E);
+    FusedBufs fb(T, rows, E);
+    ck(cudaMemcpy(fb.ids.p, ids.data(), ids.size() * 4, cudaMemcpyHostToDevice), "ids");
+    ck(cudaDeviceSynchronize(), "uploads");   // the legacy-stream uploads before the timed non-blocking stream
+    cudaEvent_t a, b;
+    ck(cudaEventCreate(&a), "event");
+    ck(cudaEventCreate(&b), "event");
+    auto time = [&](auto&& f) {
+        std::vector<float> ms;
+        for (int rep = 0; rep < 6; ++rep) {
+            ck(cudaEventRecord(a, s), "record");
+            f();
+            ck(cudaEventRecord(b, s), "record");
+            ck(cudaEventSynchronize(b), "sync");
+            float t = 0;
+            ck(cudaEventElapsedTime(&t, a, b), "elapsed");
+            if (rep > 0) ms.push_back(t);   // the first is the warm-up
+        }
+        std::sort(ms.begin(), ms.end());
+        return ms[ms.size() / 2];
+    };
+    // the engine's batches: up to 128 experts per launch (the streamed ring decides; 4 launches here)
+    const float t_mmq = time([&] { run_mmq(ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s); });
+    const float t_f = time([&] { run_fused(fb, x_dev.as<float>(), T, E, {0, E}, blob, s); });
+    const float t_f1 = time([&] {   // one launch per 16 experts, MMQ's group size
+        std::vector<int> cuts;
+        for (int e = 0; e <= E; e += 16) cuts.push_back(e);
+        run_fused(fb, x_dev.as<float>(), T, E, cuts, blob, s);
+    });
+    const std::vector<float> y_mmq = download(mb.dm, (size_t) rows * N), y_f = download(fb.dm, (size_t) rows * N);
+    const std::vector<int32_t> slot = download_i(fb.slot, (size_t) rows);
+    const Err efm = compare(y_f, [&](size_t i) { return (int64_t) slot[i]; }, y_mmq,
+                            [&](size_t i) { return (int64_t) r.row_of[i]; }, (size_t) rows);
+    std::printf("timing part: %d tokens x top %d over %d experts (%.1f rows per expert), one layer:\n", T, K, E,
+                (double) rows / E);
+    std::printf("  MMQ path (gathers, q8_1, products, SwiGLU, 32 groups): %.3f ms\n", t_mmq);
+    std::printf("  fused path (quantize, group, 4 launches of 128):       %.3f ms  (%.2fx)\n", t_f, t_mmq / t_f);
+    std::printf("  fused path, launches of 16 experts:                    %.3f ms\n", t_f1);
+    std::printf("  fused vs MMQ: rel RMS %.3e  worst row max rel %.3e\n", efm.rms, efm.worst);
+    cudaEventDestroy(a);
+    cudaEventDestroy(b);
+    if (efm.rms > 0.05) throw std::runtime_error("the paths disagree at the real shape");
+}
+}  // namespace
+
+int main(int argc, char** argv) {
+    try {
+        int n = 0;
+        if (cudaGetDeviceCount(&n) != cudaSuccess || n == 0) { std::printf("no CUDA device: skipped\n"); return 77; }
+        if (!fused::available()) { std::printf("the fused kernels need sm_80 or newer: skipped\n"); return 77; }
+        if (!mmq::built()) { std::printf("no MMQ in this build\n"); return 1; }
+        cudaStream_t s = nullptr;
+        ck(cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking), "stream");
+        reference_part(s);
+        if (!(argc > 1 && std::strcmp(argv[1], "--no-timing") == 0)) timing_part(s);
+        ck(cudaStreamDestroy(s), "destroy");
+        std::printf("prefill fused MoE parity passed\n");
+        return 0;
+    } catch (const std::exception& e) {
+        std::fprintf(stderr, "prefill fused MoE parity failed: %s\n", e.what());
+        return 1;
+    }
+}

tests/hip/mapped_alias.cpp

@@ -0,0 +1,108 @@
+// Is a hipHostGetDevicePointer alias usable on this stack? (#325)
+//
+// tests/hip/handoff.cpp fails on gfx1201/Windows with "separate copy/ring timeout": it does a
+// hipMemcpyAsync(..., hipMemcpyDeviceToDevice) into a hipHostGetDevicePointer alias, then waits for
+// a doorbell that copy should have published. This test pins down the properties that decide it.
+//
+// On Linux the alias is a device address of the pinned host memory, a D2D copy lands, the host sees
+// it, and a kernel reads it. On Windows (#325, RX 9070 XT) the runtime hands back the HOST pointer
+// itself (unified addressing; this test prints both addresses), and a D2D copy into it does not
+// publish - hip_handoff's timeout there. The engine itself runs on Windows (the reporter's serving
+// run gave correct answers), so kernels reading mapped memory through that address work; only the
+// copy-engine path does not. src/core/native_head.cpp keeps the token embedding in VRAM when the
+// alias is refused outright.
+//
+// Reporting, not gating: a Windows host is EXPECTED to fail some checks below, so this test exits 0
+// whenever it ran at all, and 77 (skipped) without a device. When the alias IS the host pointer the
+// D2D copy is not attempted (it may fault rather than report); the kernel read is, since that is
+// what the engine relies on.
+
+#include "strata/kernels/elementwise.hpp"
+
+#include <hip/hip_runtime.h>
+
+#include <cstdio>
+#include <vector>
+
+namespace {
+
+int not_held = 0;
+
+void report(const char* what, bool holds) {
+    std::printf("  %-44s %s\n", what, holds ? "holds" : "DOES NOT HOLD");
+    if (!holds) ++not_held;
+}
+
+// A kernel reads `n` floats through `alias` into VRAM; the result is copied back and compared.
+bool kernel_reads(const float* alias, const std::vector<float>& want) {
+    const int64_t n = (int64_t) want.size();
+    float* out = nullptr;
+    if (hipMalloc(reinterpret_cast<void**>(&out), want.size() * sizeof(float)) != hipSuccess) return false;
+    strata::kernels::copy_from_mapped(out, alias, n, nullptr);
+    std::vector<float> got(want.size(), -1.0f);
+    const bool ran = hipDeviceSynchronize() == hipSuccess &&
+                     hipMemcpy(got.data(), out, want.size() * sizeof(float), hipMemcpyDeviceToHost) == hipSuccess;
+    hipFree(out);
+    hipGetLastError();
+    return ran && got == want;
+}
+
+}  // namespace
+
+int main() {
+    int runtime = 0;
+    if (hipRuntimeGetVersion(&runtime) != hipSuccess) return 1;
+    int count = 0;
+    if (hipGetDeviceCount(&count) != hipSuccess || count < 1) {
+        std::printf("no HIP device\n");
+        return 77;  // ctest "skipped"
+    }
+    std::printf("HIP runtime %d, %d device(s)\n", runtime, count);
+
+    const int64_t n = 4096;                       // enough elements for copy_from_mapped's float4 path
+    const size_t bytes = size_t(n) * sizeof(float);
+    std::vector<float> pattern(size_t(n), 0.0f);
+    for (int64_t i = 0; i < n; ++i) pattern[size_t(i)] = float(i + 1);
+
+    float* host = nullptr;
+    if (hipHostMalloc(reinterpret_cast<void**>(&host), bytes, hipHostMallocMapped) != hipSuccess) return 1;
+    void* alias = nullptr;
+    if (hipHostGetDevicePointer(&alias, host, 0) != hipSuccess || alias == nullptr) {
+        std::printf("hipHostGetDevicePointer failed: native_head.cpp keeps the embedding in VRAM here\n");
+        hipGetLastError();
+        hipHostFree(host);
+        return 0;
+    }
+    const bool distinct = alias != reinterpret_cast<void*>(host);
+    std::printf("\n  host  %p\n  alias %p\n  -> %s\n\n", static_cast<void*>(host), alias,
+                distinct ? "a distinct device address" : "the host pointer itself (unified addressing)");
+    report("the alias is a distinct device address", distinct);
+
+    // What the engine relies on: a kernel reading mapped host memory through the alias.
+    for (int64_t i = 0; i < n; ++i) host[i] = pattern[size_t(i)];
+    report("a kernel reads the alias correctly", kernel_reads(reinterpret_cast<const float*>(alias), pattern));
+
+    // What tests/hip/handoff.cpp attempts: a DeviceToDevice copy INTO the alias, seen by the host.
+    if (distinct) {
+        float* device = nullptr;
+        std::vector<float> twice(pattern);
+        for (float& v : twice) v *= 2.0f;
+        if (hipMalloc(reinterpret_cast<void**>(&device), bytes) == hipSuccess &&
+            hipMemcpy(device, twice.data(), bytes, hipMemcpyHostToDevice) == hipSuccess) {
+            const hipError_t copy = hipMemcpy(alias, device, bytes, hipMemcpyDeviceToDevice);
+            report("a D2D copy into the alias succeeds", copy == hipSuccess);
+            bool seen = hipDeviceSynchronize() == hipSuccess;
+            for (int64_t i = 0; seen && i < n; ++i) seen = host[i] == twice[size_t(i)];
+            report("the host sees a D2D copy into the alias", seen);
+        }
+        if (device) hipFree(device);
+        hipGetLastError();
+    } else {
+        std::printf("  the D2D copy into the alias is not attempted: with alias == host it may fault rather than\n"
+                    "  report (tests/hip/handoff's timeout on Windows)\n");
+    }
+
+    std::printf("\n%d check(s) did not hold (a Windows host is expected to fail some)\n", not_held);
+    hipHostFree(host);
+    return 0;
+}

tools/calibrate.py

@@ -119,10 +119,30 @@ def run(cfg: dict, say=print, start_engine=None) -> dict:
     tok = ST.Tokenizer(toks, (tpath / "merges.txt").read_text(encoding="utf-8").split("\n"),
                        json.loads((tpath / "token_type.json").read_text()))
     ids_list = [chat_ids(tok, p) for p in PROMPTS]
-    args = list(cfg["args"])
-    if isinstance(cfg.get("gpu"), list) and "--layer-split" not in args:   # several cards: measured as it runs
-        args += ["--layer-split", str(cfg.get("layer_split") or "auto")]
-    return measure(args, ids_list, start_engine, say)
+    return measure(engine_args(cfg), ids_list, start_engine, say)
+
+
+def engine_args(cfg: dict) -> list[str]:
+    """The arguments the server starts this config's engine with (serve.server.engine_args), so the tuning measures
+    the engine as it runs.  #447: this used to read any "gpu" list as a layer split and add --layer-split auto, which
+    broke a one-card config with a helper card for the expert tier ("gpu": [0] + --expert-cache-device1: the engine
+    refuses a split that leaves no card without a stage), missed the "0,2" spelling, and ignored split_skip_if_fits."""
+    from serve.server import engine_args as server_args
+    return server_args(cfg)
+
+
+def engine_error(log: str | None, since: int = 0) -> str | None:
+    """#447: the engine's own reason for a failed start - the last line of its log written after byte `since` that is
+    its own ("strata ..." or "ERR ..."), else the last line there; None without a log or a new line."""
+    if not log:
+        return None
+    try:
+        with open(log, "rb") as f:
+            f.seek(since)
+            lines = [x.strip() for x in f.read()[-16384:].decode("utf-8", "replace").splitlines() if x.strip()]
+    except OSError:
+        return None
+    return next((x for x in reversed(lines) if x.startswith(("strata", "ERR"))), lines[-1] if lines else None)
 
 
 def measure(base_args: list[str], ids_list, start_engine, say=print) -> dict:

tools/hip/build_windows.bat

@@ -0,0 +1,86 @@
+@echo off
+rem Strata's HIP (AMD) engine for Windows, and the ready-made zip setup.py downloads (strata-windows-x64-hip.zip).
+rem
+rem   tools\hip\build_windows.bat              build strata.exe + strata-device.exe and package dist\strata-windows-x64-hip.zip
+rem   tools\hip\build_windows.bat tests        also build the HIP unit tests (ctest in build-hip-win; they need an AMD GPU)
+rem
+rem Needs: Visual Studio 2022 (or 2019) Build Tools with the C++ workload (the linker, the CRT and the Windows SDK),
+rem Python 3.10+ and git.  No admin rights and no AMD GPU: ROCm comes from AMD's TheRock Python wheels (the same
+rem ROCm 10.2 nightly #325 ran on an RX 9070 XT), installed into a private venv (ROCM_VENV, ~7 GB).  CMake refuses to
+rem mix cl.exe with Clang for HIP, so ROCm's clang compiles the host code too (targeting the MSVC ABI, with MSVC's
+rem headers and linker from vcvars); cmake/hip_backend.cmake force-includes the CUDA->HIP shim.
+rem
+rem Settings (environment variables, all optional):
+rem   STRATA_HIP_ARCHS     gfx1100;gfx1101;gfx1102;gfx1200;gfx1201;gfx1030 (the cards setup supports, + gfx1102)
+rem   STRATA_ROCM_VERSION  10.2.0a20260930        STRATA_ROCM_INDEX  https://nightly.repo.amd.com/rocm/whl-next/
+rem   ROCM_VENV            <repo>\.rocm-win       BUILD_DIR          <repo>\build-hip-win     DIST_DIR  <repo>\dist
+rem   STRATA_GGML_DIR      a llama.cpp checkout at the pinned commit (default: CMake fetches it)
+setlocal EnableDelayedExpansion
+for %%I in ("%~dp0..\..") do set "SRC=%%~fI"
+if not defined STRATA_HIP_ARCHS set "STRATA_HIP_ARCHS=gfx1100;gfx1101;gfx1102;gfx1200;gfx1201;gfx1030"
+if not defined STRATA_ROCM_VERSION set "STRATA_ROCM_VERSION=10.2.0a20260930"
+if not defined STRATA_ROCM_INDEX set "STRATA_ROCM_INDEX=https://nightly.repo.amd.com/rocm/whl-next/"
+if not defined ROCM_VENV set "ROCM_VENV=%SRC%\.rocm-win"
+if not defined BUILD_DIR set "BUILD_DIR=%SRC%\build-hip-win"
+if not defined DIST_DIR set "DIST_DIR=%SRC%\dist"
+set "TESTS=OFF"
+if /i "%~1"=="tests" set "TESTS=ON"
+
+rem ---- 1. ROCm (TheRock wheels: the compiler, the HIP runtime, hipBLAS/hipBLASLt/rocBLAS, a device package per arch)
+set "PY="
+py -3 -c "import sys" >nul 2>nul && set "PY=py -3"
+if not defined PY python -c "import sys" >nul 2>nul && set "PY=python"
+if not defined PY (echo Python 3.10+ is needed & exit /b 1)
+if not exist "%ROCM_VENV%\Scripts\python.exe" %PY% -m venv "%ROCM_VENV%" || exit /b 1
+set "EXTRAS=libraries,devel"
+for %%A in (%STRATA_HIP_ARCHS:;= %) do set "EXTRAS=!EXTRAS!,device-%%A"
+set "STAMP=%ROCM_VENV%\strata-rocm.txt"
+set "WANT=%STRATA_ROCM_VERSION% %EXTRAS%"
+set "HAVE="
+if exist "%STAMP%" set /p HAVE=<"%STAMP%"
+if not "!HAVE!"=="!WANT!" (
+  echo Installing ROCm %STRATA_ROCM_VERSION% [%EXTRAS%] into %ROCM_VENV% ...
+  "%ROCM_VENV%\Scripts\python.exe" -m pip install --disable-pip-version-check --index-url "%STRATA_ROCM_INDEX%" "rocm[%EXTRAS%]==%STRATA_ROCM_VERSION%" || exit /b 1
+  "%ROCM_VENV%\Scripts\rocm-sdk.exe" init || exit /b 1
+  >"%STAMP%" echo !WANT!
+)
+for /f "delims=" %%R in ('"%ROCM_VENV%\Scripts\rocm-sdk.exe" path --root') do set "ROCM=%%R"
+if not exist "%ROCM%\lib\llvm\bin\clang++.exe" (echo ROCm has no compiler in "%ROCM%" & exit /b 1)
+set "ROCM_F=%ROCM:\=/%"
+set "BITCODE=%ROCM_F%/lib/llvm/amdgcn/bitcode"
+
+rem ---- 2. MSVC: the linker, the C runtime and the Windows SDK (ROCm's clang uses them)
+set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
+set "VS="
+if exist "%VSWHERE%" for /f "delims=" %%V in ('"%VSWHERE%" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath') do set "VS=%%V"
+if not defined VS (echo Visual Studio Build Tools with the C++ workload are needed & exit /b 1)
+call "%VS%\VC\Auxiliary\Build\vcvars64.bat" >nul || exit /b 1
+set "HIP_PLATFORM=amd"
+set "HIP_PATH=%ROCM%"
+set "ROCM_PATH=%ROCM%"
+set "PATH=%ROCM%\bin;%ROCM%\lib\llvm\bin;%PATH%"
+
+rem ---- 3. configure + build (STRATA_PORTABLE: the CPU kernels for an AVX2 baseline, as in the NVIDIA zip)
+set "GGML="
+if defined STRATA_GGML_DIR set "GGML=-DSTRATA_GGML_DIR=%STRATA_GGML_DIR:\=/%"
+if not exist "%BUILD_DIR%\build.ninja" (
+  cmake -G Ninja -S "%SRC%" -B "%BUILD_DIR%" -DCMAKE_BUILD_TYPE=Release ^
+    -DSTRATA_ENABLE_HIP=ON -DSTRATA_ENABLE_CUDA=OFF -DSTRATA_BUILD_TESTS=%TESTS% -DSTRATA_PREFILL_MMQ=ON ^
+    -DSTRATA_NATIVE_EXPERTS=ON -DSTRATA_PORTABLE=ON "-DCMAKE_HIP_ARCHITECTURES=%STRATA_HIP_ARCHS%" ^
+    "-DCMAKE_C_COMPILER=%ROCM_F%/lib/llvm/bin/clang.exe" "-DCMAKE_CXX_COMPILER=%ROCM_F%/lib/llvm/bin/clang++.exe" ^
+    "-DCMAKE_HIP_COMPILER=%ROCM_F%/lib/llvm/bin/clang++.exe" "-DCMAKE_HIP_COMPILER_ROCM_ROOT=%ROCM_F%" ^
+    "-DCMAKE_PREFIX_PATH=%ROCM_F%" "-DCMAKE_HIP_FLAGS=--rocm-path=%ROCM_F% --rocm-device-lib-path=%BITCODE%" ^
+    %GGML% || exit /b 1
+)
+if "%TESTS%"=="ON" (
+  cmake --build "%BUILD_DIR%" || exit /b 1
+) else (
+  cmake --build "%BUILD_DIR%" --target strata strata-device || exit /b 1
+)
+
+rem ---- 4. the zip: the two programs, the ROCm DLLs they load (+ rocBLAS/hipBLASLt kernels for these archs), licenses
+"%ROCM_VENV%\Scripts\python.exe" "%SRC%\tools\hip\package_windows.py" --build "%BUILD_DIR%" --rocm "%ROCM%" ^
+  --archs "%STRATA_HIP_ARCHS%" --rocm-version "%STRATA_ROCM_VERSION%" --out "%DIST_DIR%" || exit /b 1
+echo.
+echo Done: %DIST_DIR%\strata-windows-x64-hip.zip
+endlocal

tools/hip/gfx1200-hipblaslt-100202.txt

@@ -0,0 +1,69 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1200 100202
+bf16 10240 2560 10240 2560 30831
+bf16 10240 2560 10240 256 31013
+bf16 10240 2560 10240 512 31013
+bf16 10240 320 10240 2560 30902
+bf16 10240 320 10240 256 30831
+bf16 10240 320 10240 512 30779
+bf16 12288 2560 12288 2560 30969
+bf16 12288 2560 12288 256 31013
+bf16 2560 10240 10240 256 30654
+bf16 2560 10240 10240 512 30998
+bf16 2560 12288 2560 256 30654
+bf16 2560 2560 2560 2560 30963
+bf16 2560 2560 2560 256 30789
+bf16 2560 2560 2560 512 30844
+bf16 2560 512 4096 512 30654
+bf16 2560 6144 2560 2560 30654
+bf16 2560 640 2560 2560 30726
+bf16 2560 640 2560 256 30654
+bf16 2560 640 2560 512 30654
+bf16 2560 96 4096 37 30681
+bf16 2560 96 4096 48 30664
+bf16 320 10240 320 2560 31046
+bf16 320 10240 320 256 30668
+bf16 48 2560 96 4096 30904
+bf16 6144 2560 6144 2560 30831
+bf16 6144 2560 6144 256 30969
+bf16 6144 2560 6144 512 31013
+bf16 640 2560 640 2560 31171
+bf16 640 2560 640 256 30912
+bf16 640 2560 640 512 30686
+f16 10240 2560 10240 2560 55549
+f16 10240 2560 10240 256 55715
+f16 10240 2560 10240 512 55549
+f16 10240 320 10240 2560 55604
+f16 10240 320 10240 256 55430
+f16 10240 320 10240 512 55481
+f16 12288 2560 12288 2560 55671
+f16 12288 2560 12288 256 55715
+f16 12288 2560 12288 512 55533
+f16 2560 10240 10240 2560 55731
+f16 2560 10240 10240 256 55356
+f16 2560 10240 10240 512 55730
+f16 2560 12288 2560 2560 55729
+f16 2560 12288 2560 256 55356
+f16 2560 12288 2560 512 55693
+f16 2560 2560 2560 2560 55665
+f16 2560 2560 2560 256 55491
+f16 2560 2560 2560 512 55581
+f16 2560 512 4096 37 55381
+f16 2560 512 4096 512 55356
+f16 2560 6144 2560 2560 55356
+f16 2560 6144 2560 256 55356
+f16 2560 6144 2560 512 55356
+f16 2560 640 2560 2560 55554
+f16 2560 640 2560 256 55356
+f16 2560 640 2560 512 55356
+f16 2560 96 4096 48 55383
+f16 320 10240 320 2560 55545
+f16 320 10240 320 256 55517
+f16 320 10240 320 512 55606
+f16 512 2560 512 4096 55575
+f16 6144 2560 6144 2560 55533
+f16 6144 2560 6144 256 55671
+f16 6144 2560 6144 512 55715
+f16 640 2560 640 2560 55873
+f16 640 2560 640 256 55614
+f16 640 2560 640 512 55826

tools/hip/gfx1201-hipblaslt-100202.txt

@@ -0,0 +1,26 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1201 100202
+bf16 1 2560 1 4096 74196
+bf16 48 2560 96 4096 73754
+bf16 48 2560 96 8192 73754
+bf16 128 2560 128 4096 73754
+bf16 128 2560 128 8192 73715
+bf16 320 10240 320 4096 73838
+bf16 320 10240 320 8192 73838
+bf16 512 2560 512 8192 73803
+bf16 2560 2560 2560 8192 73778
+bf16 10240 320 10240 8192 73761
+f16 512 2560 512 4096 88193
+f16 512 2560 512 8192 88307
+f16 640 2560 640 4096 88251
+f16 640 2560 640 8192 88192
+f16 2560 640 2560 4096 88216
+f16 2560 640 2560 8192 88307
+f16 2560 6144 2560 4096 88281
+f16 2560 6144 2560 8192 88281
+f16 6144 2560 6144 4096 88255
+f16 6144 2560 6144 8192 88307
+f16 10240 2560 10240 4096 88255
+f16 10240 2560 10240 8192 88255
+f16 12288 2560 12288 4096 88255
+f16 12288 2560 12288 8192 88255

tools/hip/package_windows.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Package the Windows HIP engine as the ready-made zip setup.py downloads (strata-windows-x64-hip.zip).
+
+    python tools/hip/package_windows.py --build build-hip-win --rocm <rocm-sdk path --root> \\
+        --archs "gfx1100;gfx1201" --rocm-version 10.2.0a20260930 --out dist
+
+tools/hip/build_windows.bat runs it after the build.  The zip holds:
+
+    strata.exe, strata-device.exe, BUILD.json      (backend "hip", the archs, the ROCm and hipBLASLt versions)
+    amdhip64_7.dll, amd_comgr.dll                 the HIP runtime beside the exes too (#468 #461: before System32)
+    rocm/bin/*.dll                                 the ROCm DLLs the two programs load (their import tables, followed
+                                                   through the ROCm DLLs, + amd_comgr.dll, which the HIP runtime loads
+                                                   by name) and the Microsoft C++ runtime they import
+    rocm/bin/rocblas/library, rocm/bin/hipblaslt/library   the GEMM kernels, for these archs only
+    rocm/.kpack/                                   rocBLAS's own device code for these archs (rocblas.dll finds it at
+                                                   ../.kpack from its folder)
+    rocm/licenses/                                 the shipped components' licenses, and NOTICE.txt
+
+setup.py puts rocm/bin first on the engine's PATH (the config's lib_dirs).  The AMD display driver (AMD Software:
+Adrenalin Edition) is the only thing the user installs: TheRock's HIP runtime runs on top of it.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+ASSET = "strata-windows-x64-hip.zip"
+PROGRAMS = ("strata.exe", "strata-device.exe")
+DYNAMIC = ("amd_comgr.dll",)                     # LoadLibrary'd by amdhip64_7.dll: not in any import table
+CRT = ("msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "vcruntime140.dll", "vcruntime140_1.dll")
+KPACKS = ("blas_lib",)       # rocblas.dll's own device code; rocSOLVER's (27 MB per arch) is never launched by Strata
+LICENSES = ("rocblas", "rocsolver", "hipblas", "hipblas-common", "hipblaslt", "amd_comgr", "rocm-core")
+NOTICE = """Strata's Windows HIP engine ships these AMD ROCm components unmodified, from AMD's TheRock Python wheels
+(ROCm {rocm}, https://nightly.repo.amd.com/rocm/whl-next/, built from https://github.com/ROCm/TheRock):
+
+{dlls}
+
+Licenses: rocBLAS, hipBLAS, hipBLAS-common, hipBLASLt (and its TensileLite host and origami libraries), rocm-core: MIT;
+rocSOLVER: BSD 2-clause; amd_comgr: Apache 2.0 with LLVM exceptions (files in this folder).  The HIP runtime
+(amdhip64_7.dll, https://github.com/ROCm/rocm-systems/tree/develop/projects/clr) and rocm_kpack.dll (TheRock) are MIT
+licensed; their wheels carry no license file.  The Microsoft Visual C++ runtime DLLs (msvcp140*.dll,
+vcruntime140*.dll) are redistributable under the Visual Studio license (Distributable Code).
+"""
+
+
+def imports(objdump: Path, dll: Path) -> list[str]:
+    text = subprocess.run([str(objdump), "-p", str(dll)], capture_output=True, text=True, check=True).stdout
+    return [m.group(1) for m in re.finditer(r"DLL Name:\s*(\S+)", text)]
+
+
+def crt_dir() -> Path | None:
+    """The MSVC redist folder with the C++ runtime DLLs (vcvars sets VCToolsRedistDir)."""
+    base = os.environ.get("VCToolsRedistDir")
+    if not base:
+        return None
+    hits = sorted(Path(base, "x64").glob("Microsoft.VC*.CRT"))
+    return hits[-1] if hits else None
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--build", required=True, type=Path)
+    ap.add_argument("--rocm", required=True, type=Path, help="the ROCm root (rocm-sdk path --root)")
+    ap.add_argument("--archs", required=True, help="the archs the engine was compiled for, ';'-separated")
+    ap.add_argument("--rocm-version", required=True)
+    ap.add_argument("--out", required=True, type=Path)
+    a = ap.parse_args()
+    archs = [x for x in re.split(r"[;, ]+", a.archs) if x]
+    rbin = a.rocm / "bin"
+    objdump = a.rocm / "lib" / "llvm" / "bin" / "llvm-objdump.exe"
+    stage = a.out / "strata-windows-x64-hip"
+    shutil.rmtree(stage, ignore_errors=True)
+    (stage / "rocm" / "bin").mkdir(parents=True)
+
+    for p in PROGRAMS:
+        shutil.copy2(a.build / p, stage / p)
+
+    # the ROCm DLLs: the programs' imports, followed through the ROCm DLLs themselves
+    rocm_dlls = {p.name.lower(): p for p in rbin.glob("*.dll")}
+    need, todo, crt = [], [stage / p for p in PROGRAMS] + [rbin / d for d in DYNAMIC], set()
+    while todo:
+        f = todo.pop()
+        for name in imports(objdump, f):
+            low = name.lower()
+            if low in CRT:
+                crt.add(low)
+            elif low in rocm_dlls and low not in need:
+                need.append(low)
+                todo.append(rocm_dlls[low])
+    for d in DYNAMIC:
+        if d not in need:
+            need.append(d)
+    for d in sorted(need):
+        shutil.copy2(rocm_dlls[d], stage / "rocm" / "bin" / rocm_dlls[d].name)
+    cdir = crt_dir()
+    if cdir is None:
+        sys.exit("VCToolsRedistDir is not set: run this from a Visual Studio x64 developer prompt (vcvars64.bat)")
+    for d in sorted(crt):
+        shutil.copy2(cdir / d, stage / "rocm" / "bin" / d)
+    # #468 #461: the HIP runtime (and the compiler it loads by name) next to the exes as well - Windows searches the
+    # exe's folder before System32, where an AMD driver may install its own amdhip64_7.dll (found before PATH's
+    # rocm/bin); rocBLAS/hipBLASLt stay in rocm/bin, which they resolve their kernel libraries and ../.kpack from
+    for pat in ("amdhip64_*.dll", "amd_comgr*.dll"):
+        for f in (stage / "rocm" / "bin").glob(pat):
+            shutil.copy2(f, stage / f.name)
+
+    # the GEMM kernels and the libraries' own device code, for these archs
+    lib = stage / "rocm" / "bin" / "rocblas" / "library"
+    lib.mkdir(parents=True)
+    for f in (rbin / "rocblas" / "library").iterdir():
+        m = re.search(r"gfx[0-9a-f]+", f.name)
+        if m is None or m.group() in archs:
+            shutil.copy2(f, lib / f.name)
+    for arch in archs:
+        src = rbin / "hipblaslt" / "library" / arch
+        if src.is_dir():                                  # hipBLASLt has no RDNA2 (gfx1030) kernels
+            shutil.copytree(src, stage / "rocm" / "bin" / "hipblaslt" / "library" / arch)
+    kp = stage / "rocm" / ".kpack"
+    kp.mkdir()
+    kpack_src = a.rocm / ".kpack"
+    for arch in archs:
+        for k in KPACKS:
+            f = kpack_src / f"{k}_{arch}.kpack"
+            if not f.exists():
+                sys.exit(f"missing {f}: install the rocm-sdk-device-{arch} wheel and run rocm-sdk init")
+            shutil.copy2(f, kp / f.name)
+
+    lic = stage / "rocm" / "licenses"
+    lic.mkdir()
+    for comp in LICENSES:
+        for f in (a.rocm / "share" / "doc" / comp).glob("LICENSE*"):
+            shutil.copy2(f, lic / f"{comp}-{f.name}")
+    shipped = sorted(p.name for p in (stage / "rocm" / "bin").glob("*.dll"))
+    (lic / "NOTICE.txt").write_text(NOTICE.format(rocm=a.rocm_version, dlls="\n".join("  " + d for d in shipped)),
+                                    encoding="utf-8")
+
+    version = re.search(r"project\(strata VERSION ([\d.]+)", (ROOT / "CMakeLists.txt").read_text()).group(1)
+    hl = (a.rocm / "include" / "hipblaslt" / "hipblaslt-version.h").read_text()
+    hlv = [int(re.search(rf"#define\s+HIPBLASLT_VERSION_{k}\s+(\d+)", hl).group(1)) for k in ("MAJOR", "MINOR", "PATCH")]
+    meta = {"source": "prebuilt", "backend": "hip", "platform": "windows-x64", "version": version, "archs": archs,
+            "rocm": a.rocm_version, "hipblaslt_version": hlv[0] * 100000 + hlv[1] * 100 + hlv[2], "vision": "none",
+            "lib_dirs": ["rocm/bin"]}
+    (stage / "BUILD.json").write_text(json.dumps(meta, indent=1), encoding="utf-8")
+
+    z = a.out / ASSET
+    z.unlink(missing_ok=True)
+    with zipfile.ZipFile(z, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as f:
+        for p in sorted(stage.rglob("*")):
+            if p.is_file():
+                f.write(p, p.relative_to(stage).as_posix())
+    size = sum(p.stat().st_size for p in stage.rglob("*") if p.is_file())
+    print(f"{z}: {z.stat().st_size / 2**20:.0f} MiB ({size / 2**20:.0f} MiB unpacked), engine {version}, "
+          f"{', '.join(archs)}, ROCm {a.rocm_version}, DLLs: {', '.join(shipped)}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/mtp_fetch.py

@@ -31,8 +31,10 @@ import urllib.request
 # repository no longer has it, the current files are read instead, with a message (resolve_repo).
 PINNED_REVISION = "de4b8e4d43b917e7706784d8bb445c9af86a3540"
 REVISION = os.environ.get("STRATA_MTP_REVISION") or PINNED_REVISION
-REPO = "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/resolve/%s/" % REVISION
-PINNED = "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/resolve/%s/" % PINNED_REVISION   # SHA256's revision
+# #495: HF_ENDPOINT (a mirror, e.g. https://hf-mirror.com) serves the same revision; the SHA256 checks below still apply
+HF_ENDPOINT = (os.environ.get("HF_ENDPOINT") or "").strip().rstrip("/") or "https://huggingface.co"
+REPO = HF_ENDPOINT + "/Qwen/Qwen3.8-Flash-Next/resolve/%s/" % REVISION
+PINNED = HF_ENDPOINT + "/Qwen/Qwen3.8-Flash-Next/resolve/%s/" % PINNED_REVISION   # SHA256's revision
 DTYPE_BYTES = {"BF16": 2, "F16": 2, "F32": 4, "F8_E4M3": 1, "I64": 8, "I32": 4}
 BAD = 3                                             # `verify`'s exit code: a tensor is missing or corrupt
 

tools/test_calibrate.py

@@ -11,6 +11,7 @@ import sys
 import tempfile
 import unittest
 from pathlib import Path
+from unittest import mock
 
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT / "tools"))
@@ -91,6 +92,53 @@ class Calibrate(unittest.TestCase):
         self.assertIsNone(CAL.arg_value(b, "--pool-workers"))
         self.assertEqual(b.count("--spec-min-p"), 1)
 
+    def test_engine_args_are_the_servers(self):
+        # #447: a "gpu" list is a layer split only with two or more cards; "0,2" is one too; split_skip_if_fits counts
+        helper = {"args": BASE + ["--expert-cache-device1", "1800"], "gpu": [0], "env": {"CUDA_VISIBLE_DEVICES": "0,1"}}
+        self.assertEqual(CAL.engine_args(helper), helper["args"])                 # one stage + a helper card: no split
+        self.assertEqual(CAL.engine_args({"args": BASE, "gpu": 1}), BASE)
+        self.assertEqual(CAL.engine_args({"args": BASE, "gpu": [0, 2]}), BASE + ["--layer-split", "auto"])
+        self.assertEqual(CAL.engine_args({"args": BASE, "gpu": "0,2", "layer_split": "18"}),
+                         BASE + ["--layer-split", "18"])
+        self.assertEqual(CAL.engine_args({"args": BASE, "gpu": [0, 1], "split_skip_if_fits": True}),
+                         BASE + ["--layer-split", "auto", "--split-skip-if-fits"])
+        own = BASE + ["--layer-split", ""]                                       # the config's own value wins
+        self.assertEqual(CAL.engine_args({"args": own, "gpu": [0, 1]}), own)
+
+    def test_run_measures_with_the_servers_args(self):
+        with tempfile.TemporaryDirectory() as d:
+            tok = Path(d)
+            (tok / "vocab.json").write_text(json.dumps({"a": 0, "b": 1}))
+            (tok / "merges.txt").write_text("")
+            (tok / "token_type.json").write_text(json.dumps([1, 1]))
+            seen = []
+            saved = CAL.measure
+            CAL.measure = lambda args, ids_list, start_engine, say=print: seen.append(args) or {}
+            fake = type("ST", (), {"Tokenizer": lambda *a: type("T", (), {"encode": lambda s, t, **k: [0]})()})
+            try:
+                with mock.patch.dict(sys.modules, {"strata_tokenizer": fake}):
+                    CAL.run({"tokenizer": d, "args": list(BASE), "gpu": [0]}, start_engine=lambda a: None)
+            finally:
+                CAL.measure = saved
+        self.assertEqual(seen, [BASE])
+
+    def test_engine_error(self):
+        with tempfile.TemporaryDirectory() as d:
+            log = Path(d) / "strata-x.log"
+            log.write_text("strata generate: an old error\n", encoding="utf-8")
+            since = log.stat().st_size
+            with open(log, "a", encoding="utf-8") as f:
+                f.write("loading ...\nstrata generate: --expert-cache-remote with a layer split needs a GPU that runs "
+                        "no stage (2 visible, 2 used by the split)\n\n")
+            self.assertEqual(CAL.engine_error(str(log), since), "strata generate: --expert-cache-remote with a layer "
+                             "split needs a GPU that runs no stage (2 visible, 2 used by the split)")
+            with open(log, "a", encoding="utf-8") as f:
+                f.write("Segmentation fault\n")
+            self.assertIn("--expert-cache-remote", CAL.engine_error(str(log), since))   # the engine's own line first
+            self.assertEqual(CAL.engine_error(str(log), log.stat().st_size), None)   # nothing new since
+            self.assertIsNone(CAL.engine_error(None))
+            self.assertIsNone(CAL.engine_error(str(Path(d) / "missing.log")))
+
     def test_worker_candidates(self):
         self.assertEqual(CAL.worker_candidates(6), [6, 4, 3])
         self.assertEqual(CAL.worker_candidates(23), [23, 15, 12])
@@ -152,6 +200,35 @@ class SetupIntegration(unittest.TestCase):
         self.assertEqual(json.loads(cfg_path.read_text())["args"], BASE)
         self.assertIsNone(self.S.saved_calibration({"args": BASE, "model_name": "m"}))
 
+    def test_failed_calibration_says_the_engines_reason(self):
+        # #447: the engine's last error line is printed, and the --calibrate run repeats the failure before it starts
+        import contextlib
+        import io
+        log = Path(self.tmp.name) / "strata-q2_0.log"
+        log.write_text("strata generate: an error of an earlier start\n", encoding="utf-8")
+        cfg_path = Path(self.tmp.name) / "strata-q2_0.json"
+        cfg_path.write_text(json.dumps({"exe": "x", "args": list(BASE), "model_name": "m", "log": str(log)}))
+
+        def boom(*a, **k):
+            with open(log, "a", encoding="utf-8") as f:
+                f.write("strata generate: --expert-cache-remote with a layer split needs a GPU that runs no stage\n")
+            raise RuntimeError(f"the engine exited before it was ready (see {log})")
+        CAL.run = boom
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            self.assertFalse(self.S.calibrate_config(cfg_path))
+        self.assertIn("the engine said: strata generate: --expert-cache-remote with a layer split", out.getvalue())
+        self.assertNotIn("an earlier start", out.getvalue())
+        out = io.StringIO()
+        with mock.patch.object(self.S, "ROOT", Path(self.tmp.name)), \
+                mock.patch.object(self.S, "data_folder", lambda d: (Path(self.tmp.name) / "data", [])), \
+                mock.patch.object(self.S, "update_installed_engine", lambda *a: None), \
+                mock.patch.object(sys, "argv", ["setup.py", "--calibrate", "--no-start"]), \
+                contextlib.redirect_stdout(out):
+            self.assertEqual(self.S.main(), 0)
+        self.assertIn("this PC is NOT tuned: the tuning failed (the reason is above); the model keeps the default "
+                      "settings", out.getvalue())
+
 
 if __name__ == "__main__":
     unittest.main()

tools/test_setup_amd.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 import sys
 import tempfile
 import unittest
+from unittest import mock
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -116,6 +117,40 @@ class GpuLists(unittest.TestCase):
                 else:
                     setup.os.environ["ROCM_PATH"] = old
 
+    def test_runtime_only_system_rocm_falls_back_to_the_wheels(self):
+        """#446: a system ROCm 7 with hipcc and libhipblas but no HIP development files (no hip-lang CMake package, no
+        hip_runtime.h) is not used for the build: a warning says what is missing and the wheels path follows (here
+        it stops at the two-family check, which only the wheels path makes); with the files it is used as before."""
+        said = []
+        setup.say = lambda msg="": said.append(msg)          # tearDown puts the real one back
+        with tempfile.TemporaryDirectory() as d:
+            sysroot = Path(d)
+            for rel, text in (("bin/hipcc", ""), ("lib/libhipblas.so.3", ""),
+                              ("include/rocm-core/rocm_version.h",
+                               "#define ROCM_VERSION_MAJOR 7\n#define ROCM_VERSION_MINOR 14\n")):
+                (sysroot / rel).parent.mkdir(parents=True, exist_ok=True)
+                (sysroot / rel).write_text(text)
+            with mock.patch.dict(setup.os.environ, {"ROCM_PATH": d}):
+                with self.assertRaises(SystemExit):
+                    setup.rocm_root(["gfx1100", "gfx1201"])
+                self.assertIn(f"the ROCm in {sysroot} has no HIP development files (lib/cmake/hip-lang/hip-lang-"
+                              "config.cmake, include/hip/hip_runtime.h): using AMD's wheels", "\n".join(said))
+                self.assertIn("two GPU families", "\n".join(said))
+                for lib in ("lib64", "lib"):                 # either place CMake looks
+                    with self.subTest(lib=lib):
+                        cfg = sysroot / lib / "cmake/hip-lang/hip-lang-config.cmake"
+                        cfg.parent.mkdir(parents=True, exist_ok=True)
+                        cfg.write_text("")
+                        said.clear()
+                        with self.assertRaises(SystemExit):
+                            setup.rocm_root(["gfx1100", "gfx1201"])
+                        self.assertIn("(include/hip/hip_runtime.h)", "\n".join(said))
+                        (sysroot / "include/hip").mkdir(parents=True, exist_ok=True)
+                        (sysroot / "include/hip/hip_runtime.h").write_text("")
+                        self.assertEqual(setup.rocm_root(["gfx1100", "gfx1201"]), (sysroot, [str(sysroot / "lib")]))
+                        cfg.unlink()
+                        (sysroot / "include/hip/hip_runtime.h").unlink()
+
     def test_build_for_every_arch(self):
         """build_engine_hip compiles for the set of the chosen cards' archs and records it in BUILD.json."""
         calls = {}
@@ -152,5 +187,190 @@ class GpuLists(unittest.TestCase):
                     setattr(setup, k, v)
 
 
+class WindowsDetection(unittest.TestCase):
+    """Windows: the AMD cards from the display adapters (mocked Win32_VideoController rows and display-class registry
+    values), the HIP runtime's numbering from `strata-device --list-devices`, and the card setup chose matched to its
+    HIP ordinal (#325: an integrated Radeon is HIP's device 0)."""
+    ADAPTERS = [  # Win32_VideoController: name, PNPDeviceID, AdapterRAM (32-bit: at most 4 GB)
+        {"name": "NVIDIA GeForce RTX 5070", "pnp": r"PCI\VEN_10DE&DEV_2F04&SUBSYS_1234&REV_A1\4&1", "ram": 4293918720},
+        {"name": "AMD Radeon RX 9070 XT", "pnp": r"PCI\VEN_1002&DEV_7550&SUBSYS_0E3B1002&REV_C0\6&2", "ram": 4293918720},
+        {"name": "AMD Radeon(TM) Graphics", "pnp": r"PCI\VEN_1002&DEV_164E&SUBSYS_88771043&REV_C1\4&3", "ram": 536870912},
+        {"name": "AMD Radeon RX 7800 XT", "pnp": r"PCI\VEN_1002&DEV_7499&SUBSYS_0000&REV_C8\6&4", "ram": 4293918720},
+    ]
+    REGISTRY = [  # the display class's driver instances: 64-bit VRAM size (as int, or as bytes)
+        {"DriverDesc": "NVIDIA GeForce RTX 5070", "MatchingDeviceId": r"PCI\VEN_10DE&DEV_2F04",
+         "HardwareInformation.qwMemorySize": 12 << 30},
+        {"DriverDesc": "AMD Radeon RX 6600", "MatchingDeviceId": r"PCI\VEN_1002&DEV_73FF",     # a removed card
+         "HardwareInformation.qwMemorySize": 8 << 30},
+        {"DriverDesc": "AMD Radeon RX 9070 XT", "MatchingDeviceId": r"PCI\VEN_1002&DEV_7550&REV_C0",
+         "HardwareInformation.qwMemorySize": (16 << 30).to_bytes(8, "little"), "DriverVersion": "32.0.21013.1000"},
+        {"DriverDesc": "AMD Radeon(TM) Graphics", "MatchingDeviceId": r"PCI\VEN_1002&DEV_164E&REV_C1",
+         "HardwareInformation.qwMemorySize": 512 << 20},
+        {"DriverDesc": "AMD Radeon RX 7800 XT", "MatchingDeviceId": r"PCI\VEN_1002&DEV_7499",
+         "HardwareInformation.qwMemorySize": 16 << 30},
+    ]
+    LIST = ("device 0: AMD Radeon(TM) Graphics\n  arch gfx1036, 28.1 GiB, wave32\n"
+            "  cannot run: GPU 0 (AMD Radeon(TM) Graphics, gfx1036) is not an architecture this Strata engine was "
+            "compiled for (gfx1100,gfx1101,gfx1102,gfx1200,gfx1201,gfx1030); ...\n"
+            "device 1: AMD Radeon RX 9070 XT\n  arch gfx1201, 15.9 GiB, wave32\n"
+            "device 2: AMD Radeon RX 7800 XT\n  arch gfx1101, 16.0 GiB, wave32\n")
+
+    def test_adapters(self):
+        g = setup.amd_gpus_windows(self.ADAPTERS, self.REGISTRY)
+        self.assertEqual([x["name"] for x in g], ["AMD Radeon RX 9070 XT", "AMD Radeon(TM) Graphics",
+                                                  "AMD Radeon RX 7800 XT"])     # AMD only, present only, in order
+        self.assertEqual([x["index"] for x in g], [0, 1, 2])
+        self.assertEqual(g[0]["arch"], "gfx1201")                               # by PCI device id
+        self.assertEqual(g[2]["arch"], "gfx1101")                               # an unlisted id: by its name
+        self.assertTrue(g[1]["arch"].startswith("unknown"))                     # the iGPU: listed, not supported
+        self.assertAlmostEqual(g[0]["vram_gb"], 16.0)                           # the registry's 64-bit size
+        self.assertEqual(g[0]["driver"], "32.0.21013.1000")
+        self.assertEqual([setup.amd_problem(x) is None for x in g], [True, False, True])
+
+    def test_registry_alone(self):
+        """No WMI answer: the registry's own list (which can hold a removed card)."""
+        g = setup.amd_gpus_windows([], self.REGISTRY)
+        self.assertEqual([x["arch"] for x in g if not x["arch"].startswith("unknown")], ["gfx1201", "gfx1101"])
+
+    def test_arch_names(self):
+        for name, arch in (("AMD Radeon RX 9070 GRE", "gfx1201"), ("AMD Radeon AI PRO R9700", "gfx1201"),
+                           ("AMD Radeon RX 9060 XT", "gfx1200"), ("AMD Radeon RX 7900 GRE", "gfx1100"),
+                           ("AMD Radeon PRO W7800", "gfx1100"), ("AMD Radeon RX 7700 XT", "gfx1101"),
+                           ("AMD Radeon RX 7600", "gfx1102"), ("AMD Radeon RX 6950 XT", "gfx1030"),
+                           ("AMD Radeon RX 6800M", ""), ("AMD Radeon 780M Graphics", ""), ("AMD Radeon RX 7700S", "")):
+            self.assertEqual(setup.win_amd_arch(None, name), arch, name)
+        self.assertEqual(setup.win_amd_arch(0x744C, "whatever"), "gfx1100")
+
+    def test_list_devices(self):
+        h = setup.hip_devices(text=self.LIST)
+        self.assertEqual([(x["index"], x["arch"]) for x in h], [(0, "gfx1036"), (1, "gfx1201"), (2, "gfx1101")])
+        self.assertAlmostEqual(h[1]["vram_gb"], 15.9)
+        self.assertIn("not an architecture", h[0]["cannot_run"])
+        self.assertIsNotNone(setup.amd_problem(h[0]))
+        self.assertIsNone(setup.amd_problem(h[1]))
+        cannot = setup.hip_devices(text="device 0: AMD Radeon RX 9070 XT\n  arch gfx1201, 15.9 GiB, wave32\n"
+                                        "  cannot run: GPU 0 runs wave64\n")
+        self.assertEqual(setup.amd_problem(cannot[0]), "GPU 0 runs wave64")   # the engine's own check counts
+        self.assertEqual(setup.hip_devices(text="(no GPU device)\n"), [])
+
+    def test_the_card_by_its_hip_ordinal(self):
+        """#325: setup's GPU 0 (the 9070 XT, display order) is HIP's device 1 behind the iGPU."""
+        listed = setup.amd_gpus_windows(self.ADAPTERS, self.REGISTRY)
+        hip = setup.hip_devices(text=self.LIST)
+        self.assertEqual(setup.hip_match(listed[0], listed, hip)["index"], 1)
+        self.assertEqual(setup.hip_match(listed[2], listed, hip)["index"], 2)
+        two = [{"index": 0, "arch": "gfx1201"}, {"index": 1, "arch": "gfx1201"}]   # two of a kind: by rank
+        self.assertEqual(setup.hip_match(two[1], two, [{"index": 0, "arch": "gfx1036"}, {"index": 1, "arch": "gfx1201"},
+                                                       {"index": 2, "arch": "gfx1201"}])["index"], 2)
+        self.assertIsNone(setup.hip_match({"index": 0, "arch": "gfx1100"}, [{"index": 0, "arch": "gfx1100"}], hip))
+
+    def test_hip_card(self):
+        listed = setup.amd_gpus_windows(self.ADAPTERS, self.REGISTRY)
+        with mock.patch.object(setup, "hip_devices", lambda probe=None, text=None: setup_hip(self.LIST)), \
+                mock.patch.object(setup, "ok", lambda *a: None):
+            g = setup.hip_card(Path("engine"), listed[0], listed)
+        self.assertEqual((g["index"], g["count"], g["arch"], g["vram_gb"]), (1, 3, "gfx1201", 15.9))
+        with mock.patch.object(setup, "hip_devices", lambda probe=None, text=None: []), \
+                mock.patch.object(setup, "say", lambda *a, **k: None), self.assertRaises(SystemExit):
+            setup.hip_card(Path("engine"), listed[0], listed)          # no AMD driver: stops before the download
+        with mock.patch.object(setup, "hip_devices", lambda probe=None, text=None: setup_hip(self.LIST)[:1]), \
+                mock.patch.object(setup, "say", lambda *a, **k: None), self.assertRaises(SystemExit):
+            setup.hip_card(Path("engine"), listed[0], listed)          # HIP does not list the card
+
+    def test_windows_dispatch(self):
+        """amd_gpus() on Windows: HIP's numbering once the HIP engine answers, the display adapters before."""
+        with mock.patch.object(setup, "WIN", True), \
+                mock.patch.object(setup, "hip_devices", lambda probe=None, text=None: None), \
+                mock.patch.object(setup, "amd_gpus_windows", lambda adapters=None, registry=None: ["display"]):
+            self.assertEqual(setup.amd_gpus(), ["display"])
+        with mock.patch.object(setup, "WIN", True), \
+                mock.patch.object(setup, "hip_devices", lambda probe=None, text=None: ["hip"]), \
+                mock.patch.object(setup, "amd_gpus_windows", lambda adapters=None, registry=None: ["display"]):
+            self.assertEqual(setup.amd_gpus(), ["hip"])
+
+    def test_no_hip_engine_no_probe(self):
+        with tempfile.TemporaryDirectory() as d:
+            eng = Path(d)
+            (eng / "strata-device.exe").write_text("x")
+            (eng / "BUILD.json").write_text('{"backend": "cuda"}')     # an NVIDIA engine is never asked
+            self.assertIsNone(setup.hip_devices(eng / "strata-device.exe"))
+
+    def test_prebuilt_hip_zip(self):
+        """get_prebuilt_hip: a published zip (here a local folder) is unpacked into engine/; one without code for the
+        card, or older than the first Windows HIP release, is refused."""
+        import json
+        import zipfile
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            pub = root / "pub"
+            pub.mkdir()
+
+            def publish(meta):
+                with zipfile.ZipFile(pub / setup.WIN_HIP_ASSET, "w") as z:
+                    z.writestr("strata.exe", "engine")
+                    z.writestr("strata-device.exe", "probe")
+                    z.writestr("rocm/bin/amdhip64_7.dll", "dll")
+                    z.writestr("BUILD.json", json.dumps(meta))
+            ver = ".".join(map(str, setup.WIN_HIP_MIN_ENGINE))
+            good = {"source": "prebuilt", "backend": "hip", "version": ver, "archs": ["gfx1100", "gfx1201"],
+                    "lib_dirs": ["rocm/bin"]}
+            with mock.patch.object(setup, "ROOT", root), mock.patch.object(setup, "say", lambda *a, **k: None), \
+                    mock.patch.object(setup, "ok", lambda *a: None), mock.patch.object(setup, "warn", lambda *a: None):
+                publish({**good, "archs": ["gfx1100"]})
+                self.assertIsNone(setup.get_prebuilt_hip(str(pub) + "/", {"arch": "gfx1201"}))
+                publish({**good, "version": "0.1.30"})
+                self.assertIsNone(setup.get_prebuilt_hip(str(pub) + "/", {"arch": "gfx1201"}))
+                publish(good)
+                eng = setup.get_prebuilt_hip(str(pub) + "/", {"arch": "gfx1201"})
+                self.assertEqual(eng, root / "engine")
+                self.assertTrue((eng / "rocm" / "bin" / "amdhip64_7.dll").exists())
+                self.assertEqual(setup.hip_lib_dirs(eng), [eng / "rocm" / "bin"])
+                (pub / setup.WIN_HIP_ASSET).unlink()           # installed: kept, nothing downloaded again
+                self.assertEqual(setup.get_prebuilt_hip(str(pub) + "/", {"arch": "gfx1201"}), eng)
+
+
+_HIP_DEVICES = setup.hip_devices                      # the real parser, for the tests that mock setup.hip_devices
+
+
+def setup_hip(text):
+    return _HIP_DEVICES(text=text)
+
+
+class WindowsHipVision(unittest.TestCase):
+    def test_no_cpu_encoder_on_windows(self):
+        with mock.patch.object(setup, "WIN", True), mock.patch.object(setup, "warn", lambda *a: None):
+            self.assertEqual(setup.hip_vision("cpu"), "none")
+            self.assertEqual(setup.hip_vision("yes"), "none")
+        with mock.patch.object(setup, "WIN", False), mock.patch.object(setup, "warn", lambda *a: None):
+            self.assertEqual(setup.hip_vision("cpu"), "cpu")
+
+
+class HipRuntimeBesideExe(unittest.TestCase):
+    """#468 #461: the bundled HIP runtime (and amd_comgr) goes next to strata.exe, so an AMD driver's System32 copy is
+    not found first; rocBLAS and the rest stay in rocm/bin."""
+
+    def test_copies_only_the_runtime(self):
+        import json
+        with tempfile.TemporaryDirectory() as d:
+            eng = Path(d)
+            rb = eng / "rocm" / "bin"
+            rb.mkdir(parents=True)
+            (eng / "BUILD.json").write_text(json.dumps({"backend": "hip", "lib_dirs": ["rocm/bin"]}))
+            for n, data in (("amdhip64_7.dll", b"hip-3686"), ("amd_comgr.dll", b"comgr"), ("rocblas.dll", b"blas")):
+                (rb / n).write_bytes(data)
+            setup.hip_runtime_beside_exe(eng)
+            self.assertEqual((eng / "amdhip64_7.dll").read_bytes(), b"hip-3686")
+            self.assertEqual((eng / "amd_comgr.dll").read_bytes(), b"comgr")
+            self.assertFalse((eng / "rocblas.dll").exists())          # finds its kernels relative to rocm/bin
+            (rb / "amdhip64_7.dll").write_bytes(b"hip-3690-newer")    # a newer zip: replaced on the next start
+            setup.hip_runtime_beside_exe(eng)
+            self.assertEqual((eng / "amdhip64_7.dll").read_bytes(), b"hip-3690-newer")
+
+    def test_no_rocm_bin_is_a_no_op(self):
+        with tempfile.TemporaryDirectory() as d:
+            setup.hip_runtime_beside_exe(Path(d))                   # no BUILD.json (a CUDA or Linux engine)
+            self.assertEqual(list(Path(d).iterdir()), [])
+
+
 if __name__ == "__main__":
     unittest.main()

tools/test_setup_choices.py

@@ -92,6 +92,64 @@ class GgufDirShards(unittest.TestCase):
         self.assertEqual(self.shards(["a-00001-of-00002.gguf", "b-00001-of-00002.gguf"]), want)   # ambiguous
 
 
+class GgufDirUnsupported(unittest.TestCase):
+    """#444: --gguf-dir with GGUFs Strata cannot run (Unsloth's UD-IQ3_XXS, UD-Q2_K_XL) says so, names the files it
+    runs (ISTA-DASLab's GSQ-RCO, the Coder's, Unsloth's UD-Q4_K_XL) and the --family/--model of the usable ones."""
+    UD = ["Qwen3.8-Flash-Next-UD-IQ3_XXS-%05d-of-00003.gguf" % i for i in range(1, 4)] + \
+         ["Qwen3.8-Flash-Next-UD-Q2_K_XL-%05d-of-00003.gguf" % i for i in range(1, 4)]
+
+    def problem(self, names, family="qwen", model="IQ3_XXS"):
+        with tempfile.TemporaryDirectory() as d:
+            for n in names:
+                (Path(d) / n).write_bytes(b"")
+            fam = setup.FAMILIES[family]
+            first = setup.gguf_dir_shards(Path(d), fam, model)[0]
+            p = setup.gguf_dir_problem(Path(d), first, fam, model)
+            return p and (p[0].replace(d, "<D>"), p[1])
+
+    def test_quant_names(self):
+        for name, want in (("Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf", "UD-IQ3_XXS"),
+                           ("Qwen3.8-Flash-Next-UD-Q2_K_XL-00001-of-00003.gguf", "UD-Q2_K_XL"),
+                           ("model-Q4_K_M.gguf", "Q4_K_M"),
+                           ("Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf", None),
+                           ("Qwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf", None),
+                           ("my-IQ3_XXS-00001-of-00003.gguf", None), ("mmproj-F16.gguf", None),
+                           ("a-00001-of-00002.gguf", None)):
+            self.assertEqual(setup.gguf_unsupported(name), want, name)
+
+    def test_an_unsloth_ud_file_is_not_taken_for_a_gsq_rco_size(self):
+        msg, hint = self.problem(self.UD)                   # IQ3_XXS: UD-IQ3_XXS has the size in its name
+        self.assertEqual(msg, "Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf is UD-IQ3_XXS, a GGUF Strata "
+                              "cannot run")
+        self.assertIn("ISTA-DASLab's GSQ-RCO files", hint)
+        self.assertIn("Unsloth's UD-Q4_K_XL only", hint)
+
+    def test_a_folder_without_the_choice_names_what_is_there(self):
+        gsq = ["Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-%05d-of-00002.gguf" % i for i in (1, 2)]
+        msg, hint = self.problem(self.UD + gsq, "unsloth", "UD-Q4_K_XL")
+        self.assertEqual(msg, "<D> has no Qwen3.8-Flash-Next (Unsloth) UD-Q4_K_XL file")
+        self.assertIn("Not usable here: Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf, "
+                      "Qwen3.8-Flash-Next-UD-Q2_K_XL-00001-of-00003.gguf", hint)
+        self.assertIn("Usable here: --family coder --model IQ1_M", hint)
+
+    def test_usable_folders_are_left_alone(self):
+        good = ["Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-%05d-of-00002.gguf" % i for i in (1, 2)]
+        self.assertIsNone(self.problem(good + self.UD))
+        self.assertIsNone(self.problem(["my-IQ3_XXS-%05d-of-00003.gguf" % i for i in (1, 2, 3)]))
+        self.assertIsNone(self.problem([]))                 # nothing there: check_shards' "missing", as before
+
+    def test_the_size_of_another_family(self):
+        from test_setup_golden import PROFILES, install
+        ram, found = PROFILES["64GB-1x32GB"]
+        with tempfile.TemporaryDirectory() as d:
+            code, out, _, _ = install(ram, found, ["--gguf-dir", d, "--family", "unsloth", "--model", "IQ3_XXS"])
+        self.assertEqual(code, 1)
+        self.assertIn("has no IQ3_XXS model file", out)
+        self.assertIn("choose one of: UD-Q4_K_XL (or IQ3_XXS: --family qwen --model IQ3_XXS, --family swift --model "
+                      "IQ3_XXS)", out)
+        self.assertIn("Strata runs ISTA-DASLab's GSQ-RCO files", out)
+
+
 class ExperimentalSm60(unittest.TestCase):
     """#295: Pascal (6.x) and Volta (7.0) only with STRATA_EXPERIMENTAL_SM60=1, built with -DSTRATA_EXPERIMENTAL_SM60=ON
     and a CUDA 12.x toolkit; nothing changes without the variable."""
@@ -128,7 +186,7 @@ class ExperimentalSm60(unittest.TestCase):
                         str(old): "Cuda compilation tools, release 12.9, V12.9.86"}
             with mock.patch.object(setup, "WIN", False), mock.patch.object(setup.shutil, "which", lambda n: str(new)), \
                     mock.patch.dict(os.environ, {"CUDA_PATH": str(Path(d) / "12")}), \
-                    mock.patch.object(setup, "out", lambda cmd: versions[cmd[0]]):
+                    mock.patch.object(setup, "out", lambda cmd: versions.get(cmd[0], "")):  # #414
                 self.assertEqual(setup.find_nvcc(), (str(new), (13, 0)))
                 self.assertEqual(setup.find_nvcc(below=(13, 0)), (str(old), (12, 9)))
 
@@ -160,6 +218,7 @@ class ExperimentalSm60(unittest.TestCase):
 class HipVision(unittest.TestCase):
     """#304: --vision cpu with --backend hip builds the CPU image encoder beside the HIP engine."""
 
+    @mock.patch.object(setup, "WIN", False)            # Linux: compiled here (Windows has no AMD image encoder yet)
     def test_the_choice(self):
         self.assertEqual(quiet(setup.hip_vision, "cpu"), ("cpu", ""))
         for asked in (None, "no", "none"):
@@ -208,5 +267,124 @@ class HipVision(unittest.TestCase):
         self.assertNotIn("vision_src", meta)
 
 
+class VramReserve(unittest.TestCase):
+    """#493: --vram-reserve-mib N - VRAM the engine leaves free for other programs - goes into the config only when
+    given (the default config is tools/test_setup_golden.py's, unchanged), at setup and on a start."""
+
+    def setUp(self):
+        from test_setup_golden import PROFILES
+        self.ram, self.found = PROFILES["64GB-1x32GB"]
+
+    def install(self, *extra):
+        from test_setup_golden import install
+        return install(self.ram, self.found, ["--family", "qwen", "--model", "Q2_0", "--no-start", *extra])
+
+    def test_given_at_setup(self):
+        code, out, cfg, _ = self.install("--vram-reserve-mib", "2048")
+        self.assertEqual(code, 0, out)
+        a = cfg["args"]
+        self.assertEqual(a.count("--vram-reserve-mib"), 1)
+        self.assertEqual(a[a.index("--vram-reserve-mib") + 1], "2048")
+        self.assertIn("VRAM kept free for other programs: 2048 MiB", out)
+        code, out, cfg, _ = self.install()
+        self.assertNotIn("--vram-reserve-mib", cfg["args"])          # not given: the engine's default, as before
+
+    def test_with_images_one_value(self):
+        code, out, cfg, _ = self.install("--vision", "cpu", "--vram-reserve-mib", "1500")
+        self.assertEqual(code, 0, out)
+        a = cfg["args"]
+        self.assertEqual(a.count("--vram-reserve-mib"), 1)
+        self.assertEqual(a[a.index("--vram-reserve-mib") + 1], "1500")
+
+    def test_a_negative_value_is_refused(self):
+        with contextlib.redirect_stderr(io.StringIO()) as err:
+            code, out, cfg, _ = self.install("--vram-reserve-mib", "-1")
+        self.assertEqual(code, 2)
+        self.assertIn("--vram-reserve-mib takes a number of MiB", err.getvalue())
+
+    def test_kept_from_an_earlier_install(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = Path(d) / "strata-q2_0.json"
+            p.write_text(json.dumps({"args": ["--max-context", "8192", "--vram-reserve-mib", "2048"]}))
+            self.assertEqual(setup.choices_from_config(p)["vram_reserve_mib"], 2048)
+            p.write_text(json.dumps({"args": ["--vision", "--vram-reserve-mib", "700"], "vision": {"gpu": True}}))
+            self.assertIsNone(setup.choices_from_config(p)["vram_reserve_mib"])   # the images' own reserve
+            p.write_text(json.dumps({"args": ["--max-context", "8192"]}))
+            self.assertIsNone(setup.choices_from_config(p)["vram_reserve_mib"])
+
+    def test_given_on_a_start(self):
+        with tempfile.TemporaryDirectory() as d:
+            exe = Path(d) / "strata.exe"
+            exe.write_bytes(b"")
+            p = Path(d) / "strata-q2_0.json"
+            p.write_text(json.dumps({"exe": str(exe), "args": ["--kv", "int8"], "gpu": 0, "gpus_asked": True}))
+            call = mock.Mock(return_value=0)
+            with mock.patch.object(setup, "gpus", lambda: self.found), \
+                    mock.patch.object(setup, "is_wsl", lambda: False), \
+                    mock.patch.object(setup, "ensure_engine_for", lambda cards, path, cfg, yes: cfg), \
+                    mock.patch.object(setup.subprocess, "call", call):
+                for n in ("2048", "1024"):
+                    _, out = quiet(setup.start, p, None, None, False, True, None, {"vram_reserve_mib": int(n)})
+                    self.assertEqual(json.loads(p.read_text())["args"], ["--kv", "int8", "--vram-reserve-mib", n])
+        self.assertIn("1024 MiB of VRAM kept free", out)
+        self.assertTrue(call.called)
+
+
+class SmallCardTip(unittest.TestCase):
+    """#496: a card under 8 GB (a 6 GB laptop RTX 3060) gets a tip for when the start has no room for the expert cache;
+    the engine chooses its own reserve there, so the config is the one any card gets.  An 8 GB card: no tip."""
+
+    def install(self, vram, *extra):
+        from test_setup_golden import card, install
+        return install(63.7, [card(0, "NVIDIA GeForce RTX 3060 Laptop GPU", vram, "86")],
+                       ["--family", "qwen", "--model", "Q2_0", "--no-start", *extra])
+
+    def test_a_6gb_card(self):
+        code, out, cfg, _ = self.install(6.0)
+        self.assertEqual(code, 0, out)
+        self.assertNotIn("--vram-reserve-mib", cfg["args"])          # the engine decides (no fixed reserve)
+        self.assertIn("no VRAM is left for the expert cache", out)
+        self.assertIn("--draft-vocab en", out)
+        self.assertIn("--mtp", cfg["args"])                          # the draft layer stays: the server needs it
+
+    def test_an_8gb_card_has_no_tip(self):
+        for vram in (8188 / 1024, 8.0):                               # nvidia-smi lists an 8 GB card as 8188 MiB
+            code, out, cfg, _ = self.install(vram)
+            self.assertEqual(code, 0, out)
+            self.assertNotIn("no VRAM is left for the expert cache", out)
+            self.assertNotIn("--vram-reserve-mib", cfg["args"])
+
+    def test_a_given_reserve_is_kept(self):
+        code, out, cfg, _ = self.install(6.0, "--vram-reserve-mib", "500")
+        self.assertEqual(code, 0, out)
+        a = cfg["args"]
+        self.assertEqual(a[a.index("--vram-reserve-mib") + 1], "500")
+
+    def test_the_tip(self):
+        self.assertIn("an 8K context", " ".join(setup.small_card_note(32768, None)))
+        self.assertIn("--draft-vocab en", " ".join(setup.small_card_note(32768, "cjk")))
+        tip = " ".join(setup.small_card_note(8192, "en"))
+        self.assertNotIn("--draft-vocab", tip)
+        self.assertIn("close other programs", tip)
+
+
+class DesktopReserveTip(unittest.TestCase):
+    """#560 #516: an AMD card on a Linux desktop gets a recommended reserve (3072 MiB) - a tip, the config is not
+    changed."""
+
+    def test_desktop_detection(self):
+        with mock.patch.object(setup.sys, "platform", "linux"):
+            self.assertTrue(setup.linux_desktop({"WAYLAND_DISPLAY": "wayland-0"}))
+            self.assertTrue(setup.linux_desktop({"DISPLAY": ":0"}))
+            self.assertFalse(setup.linux_desktop({}))                 # a headless box / ssh
+        with mock.patch.object(setup.sys, "platform", "win32"):
+            self.assertFalse(setup.linux_desktop({"DISPLAY": ":0"}))  # Windows counts the desktop itself (#497)
+
+    def test_the_tip(self):
+        tip = " ".join(setup.desktop_reserve_note())
+        self.assertIn("--vram-reserve-mib 3072", tip)
+        self.assertIn("desktop", tip)
+
+
 if __name__ == "__main__":
     unittest.main()

tools/test_setup_draft_vocab.py

@@ -49,5 +49,29 @@ class Refresh(unittest.TestCase):
             self.assertEqual(hashlib.sha256((rt / "draft_vocab.bin").read_bytes()).hexdigest(), want)
 
 
+class SmallCardNote(unittest.TestCase):
+    """#474: a card under 14 GB is told about a smaller draft subset - a note only, and not when one was chosen."""
+
+    def test_note_on_a_small_card_only(self):
+        note = setup.draft_vocab_note(12.0, None)
+        self.assertTrue(note)
+        self.assertIn("--draft-vocab en", " ".join(note))
+        self.assertIn("the draft head does not fit", " ".join(note))
+        for vram, chosen in ((16.0, None), (24.0, None), (12.0, "en"), (12.0, "cyrillic"), (12.0, "cjk"), (0.0, None)):
+            self.assertEqual(setup.draft_vocab_note(vram, chosen), [], (vram, chosen))
+
+    def test_sizes_follow_the_shipped_subsets(self):
+        # the MiB the note gives scale with the subsets' token counts (4 bytes per token id in data/)
+        sizes = {}
+        for choice, name in setup.DRAFT_VOCABS.items():
+            p = ROOT / "data" / name
+            if not p.exists():
+                self.skipTest(f"data/{name} is not in this checkout")
+            sizes[choice] = p.stat().st_size // 4
+        for choice in ("en", "cyrillic"):
+            want = setup.DRAFT_VOCAB_MIB["cjk"] * sizes[choice] / sizes["cjk"]
+            self.assertAlmostEqual(setup.DRAFT_VOCAB_MIB[choice], want, delta=3)
+
+
 if __name__ == "__main__":
     unittest.main()

tools/test_setup_golden.json

@@ -0,0 +1,966 @@
+{
+ "128GB-1x24GB qwen IQ3_S": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_s",
+    "--native",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_s.log",
+   "model_name": "qwen3.8-flash-next-iq3_s",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_s/tokenizer"
+  }
+ },
+ "128GB-1x24GB qwen IQ3_XXS": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_xxs",
+    "--native",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_xxs.log",
+   "model_name": "qwen3.8-flash-next-iq3_xxs",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_xxs/tokenizer"
+  }
+ },
+ "128GB-1x24GB qwen Q2_0": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/q2_0",
+    "--native",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-q2_0.log",
+   "model_name": "qwen3.8-flash-next-q2_0",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/q2_0/tokenizer"
+  }
+ },
+ "128GB-1x24GB qwen recommended": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_xxs",
+    "--native",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_xxs.log",
+   "model_name": "qwen3.8-flash-next-iq3_xxs",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_xxs/tokenizer"
+  }
+ },
+ "128GB-1x24GB unsloth UD-Q4_K_XL": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/unsloth-ud-q4_k_xl",
+    "--native",
+    "<T>/models/unsloth-UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "32768",
+    "--kv",
+    "int8",
+    "--resident-budget-gib",
+    "71"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-unsloth-ud-q4_k_xl.log",
+   "model_name": "qwen3.8-flash-next-unsloth-ud-q4_k_xl",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/unsloth-ud-q4_k_xl/tokenizer"
+  }
+ },
+ "32GB-2x24GB qwen IQ3_S": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_s",
+    "--native",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--mmap-experts"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_s.log",
+   "model_name": "qwen3.8-flash-next-iq3_s",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_s/tokenizer"
+  }
+ },
+ "32GB-2x24GB qwen IQ3_XXS": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_xxs",
+    "--native",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--mmap-experts"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_xxs.log",
+   "model_name": "qwen3.8-flash-next-iq3_xxs",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_xxs/tokenizer"
+  }
+ },
+ "32GB-2x24GB qwen Q2_0": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/q2_0",
+    "--native",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--resident-experts"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-q2_0.log",
+   "model_name": "qwen3.8-flash-next-q2_0",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/q2_0/tokenizer"
+  }
+ },
+ "32GB-2x24GB qwen recommended": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/q2_0",
+    "--native",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--resident-experts"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-q2_0.log",
+   "model_name": "qwen3.8-flash-next-q2_0",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/q2_0/tokenizer"
+  }
+ },
+ "32GB-2x24GB unsloth UD-Q4_K_XL": {
+  "code": 1,
+  "config": null
+ },
+ "47GB-2x16GB qwen IQ3_S": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_s",
+    "--native",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "65536",
+    "--kv",
+    "int8",
+    "--mmap-experts"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_s.log",
+   "model_name": "qwen3.8-flash-next-iq3_s",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_s/tokenizer"
+  }
+ },
+ "47GB-2x16GB qwen IQ3_XXS": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_xxs",
+    "--native",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "65536",
+    "--kv",
+    "int8",
+    "--resident-experts"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_xxs.log",
+   "model_name": "qwen3.8-flash-next-iq3_xxs",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_xxs/tokenizer"
+  }
+ },
+ "47GB-2x16GB qwen Q2_0": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/q2_0",
+    "--native",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "65536",
+    "--kv",
+    "int8"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": [
+    0,
+    1
+   ],
+   "gpus_asked": true,
+   "layer_split": "auto",
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-q2_0.log",
+   "model_name": "qwen3.8-flash-next-q2_0",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/q2_0/tokenizer"
+  }
+ },
+ "47GB-2x16GB qwen recommended": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/q2_0",
+    "--native",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "65536",
+    "--kv",
+    "int8"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": [
+    0,
+    1
+   ],
+   "gpus_asked": true,
+   "layer_split": "auto",
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-q2_0.log",
+   "model_name": "qwen3.8-flash-next-q2_0",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/q2_0/tokenizer"
+  }
+ },
+ "47GB-2x16GB unsloth UD-Q4_K_XL": {
+  "code": 1,
+  "config": null
+ },
+ "64GB-1x32GB qwen IQ3_S": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_s",
+    "--native",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_s.log",
+   "model_name": "qwen3.8-flash-next-iq3_s",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_s/tokenizer"
+  }
+ },
+ "64GB-1x32GB qwen IQ3_XXS": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_xxs",
+    "--native",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_xxs.log",
+   "model_name": "qwen3.8-flash-next-iq3_xxs",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_xxs/tokenizer"
+  }
+ },
+ "64GB-1x32GB qwen Q2_0": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/q2_0",
+    "--native",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-q2_0.log",
+   "model_name": "qwen3.8-flash-next-q2_0",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/q2_0/tokenizer"
+  }
+ },
+ "64GB-1x32GB qwen recommended": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_xxs",
+    "--native",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "131072",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_xxs.log",
+   "model_name": "qwen3.8-flash-next-iq3_xxs",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_xxs/tokenizer"
+  }
+ },
+ "64GB-1x32GB unsloth UD-Q4_K_XL": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/unsloth-ud-q4_k_xl",
+    "--native",
+    "<T>/models/unsloth-UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "32768",
+    "--kv",
+    "int8",
+    "--resident-budget-gib",
+    "40"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-unsloth-ud-q4_k_xl.log",
+   "model_name": "qwen3.8-flash-next-unsloth-ud-q4_k_xl",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/unsloth-ud-q4_k_xl/tokenizer"
+  }
+ },
+ "96GB-1x16GB qwen IQ3_S": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_s",
+    "--native",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "65536",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_s.log",
+   "model_name": "qwen3.8-flash-next-iq3_s",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_s/tokenizer"
+  }
+ },
+ "96GB-1x16GB qwen IQ3_XXS": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_xxs",
+    "--native",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "65536",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_xxs.log",
+   "model_name": "qwen3.8-flash-next-iq3_xxs",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_xxs/tokenizer"
+  }
+ },
+ "96GB-1x16GB qwen Q2_0": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/q2_0",
+    "--native",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "65536",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-q2_0.log",
+   "model_name": "qwen3.8-flash-next-q2_0",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/q2_0/tokenizer"
+  }
+ },
+ "96GB-1x16GB qwen recommended": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/iq3_xxs",
+    "--native",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+    "--ple-gguf",
+    "<T>/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "65536",
+    "--kv",
+    "int8",
+    "--kv-resident",
+    "32768"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-iq3_xxs.log",
+   "model_name": "qwen3.8-flash-next-iq3_xxs",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/iq3_xxs/tokenizer"
+  }
+ },
+ "96GB-1x16GB unsloth UD-Q4_K_XL": {
+  "code": 0,
+  "config": {
+   "args": [
+    "--pack",
+    "<T>/data/packs/unsloth-ud-q4_k_xl",
+    "--native",
+    "<T>/models/unsloth-UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf",
+    "--expert-profile",
+    "<T>/data/expert-profile.bin",
+    "--expert-cache",
+    "auto",
+    "--prefill",
+    "auto",
+    "--spec",
+    "4",
+    "--spec-min-p",
+    "0.5",
+    "--mtp",
+    "<T>/data/mtp/rt",
+    "--max-context",
+    "32768",
+    "--kv",
+    "int8",
+    "--resident-budget-gib",
+    "71"
+   ],
+   "cwd": "<T>",
+   "exe": "<T>/engine/<EXE>",
+   "gpu": 0,
+   "gpus_asked": true,
+   "lib_dirs": [
+    "<cuda>"
+   ],
+   "log": "<T>/strata-unsloth-ud-q4_k_xl.log",
+   "model_name": "qwen3.8-flash-next-unsloth-ud-q4_k_xl",
+   "port": 8080,
+   "tokenizer": "<T>/data/packs/unsloth-ud-q4_k_xl/tokenizer"
+  }
+ }
+}

tools/test_setup_golden.py

@@ -0,0 +1,208 @@
+"""The owner's rule for setup (#403 #406 #364 #384): setup RECOMMENDS, it never forces - and the recommended answers
+stay exactly what they were.  setup.main() with --yes (and with Enter for every question) on a few PCs writes the same
+run config, byte for byte, as 0.1.32 plus the fixes on top of it did (tools/test_setup_golden.json, written by that
+setup).  Every outside effect is mocked: no GPU, no downloads, nothing written outside a temp folder.
+
+    python -m unittest tools.test_setup_golden
+    python tools/test_setup_golden.py --write      (only to record a new baseline on purpose)
+"""
+from __future__ import annotations
+
+import contextlib
+import io
+import json
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+import setup  # noqa: E402
+
+GOLDEN = Path(__file__).with_name("test_setup_golden.json")
+
+
+def card(i, name, vram, arch):
+    return {"index": i, "name": name, "vram_gb": vram, "arch": arch, "driver": "580.97"}
+
+
+# the PCs: total RAM as ram_gb() reads it (a little under the label), the cards as nvidia-smi lists them
+PROFILES = {
+    "64GB-1x32GB": (63.7, [card(0, "NVIDIA GeForce RTX 5090", 31.8, "120")]),                  # #406
+    "32GB-2x24GB": (31.9, [card(i, "NVIDIA GeForce RTX 3090", 24.0, "86") for i in range(2)]),  # #364
+    "47GB-2x16GB": (47.0, [card(i, "NVIDIA GeForce RTX 5060 Ti", 15.9, "120") for i in range(2)]),   # #384
+    "96GB-1x16GB": (95.8, [card(0, "NVIDIA GeForce RTX 5080", 15.9, "120")]),
+    "128GB-1x24GB": (127.8, [card(0, "NVIDIA GeForce RTX 4090", 24.0, "89")]),
+}
+# (family, model): None = the recommended size
+MODELS = [("qwen", None), ("qwen", "Q2_0"), ("qwen", "IQ3_XXS"), ("qwen", "IQ3_S"), ("unsloth", "UD-Q4_K_XL")]
+
+
+class FakeGGUF:
+    """gguf_reader.GGUFFile: shard 2 holds the PLE table."""
+    def __init__(self, path):
+        names = ["per_layer_token_embd.weight"] if "00002-of" in str(path) else ["blk.0.ffn_up_exps.weight"]
+        self.tensors = [types.SimpleNamespace(name=n) for n in names]
+
+
+def normalize(v, t: Path):
+    """The config with the temp folder and the platform's names taken out (it compares on Windows and Linux)."""
+    if isinstance(v, dict):
+        return {k: normalize(x, t) for k, x in v.items()}
+    if isinstance(v, list):
+        return [normalize(x, t) for x in v]
+    if isinstance(v, str):
+        return v.replace(str(t), "<T>").replace("\\", "/").replace(setup.EXE, "<EXE>")
+    return v
+
+
+def install(ram, found, argv, answers=None, extra=(), avx512=False, configs=()):
+    """setup.main() on a mocked PC -> (exit code, printed text, the written config or None, the questions asked).
+    answers: None = --yes (a question fails the run), "" = Enter for every question, a list of answers in order, or
+    {words of a question: its answer} (Enter for the others).
+    extra: more patches (mock.patch objects).  configs: (name, dict) run configs already in the Strata folder."""
+    with tempfile.TemporaryDirectory() as tmp:
+        t = Path(tmp)
+        (t / "data" / "mtp" / "rt").mkdir(parents=True)
+        (t / "data" / "mtp" / "rt" / "experts.bin").write_bytes(b"")
+        eng = t / "engine"
+        eng.mkdir()
+        (eng / "BUILD.json").write_text(json.dumps({"version": "0.1.32", "source": "local", "cuda_dirs": ["<cuda>"]}))
+        for name, c in configs:
+            (t / name).write_text(json.dumps(c))
+        asked = []
+
+        def fake_input(prompt=""):
+            asked.append(prompt)
+            if answers is None:
+                raise AssertionError(f"asked {prompt!r} with --yes")
+            if isinstance(answers, list):
+                return answers.pop(0) if answers else ""
+            if isinstance(answers, dict):                  # {words of the question: answer}, else Enter
+                return next((v for k, v in answers.items() if k in prompt), "")
+            return answers
+
+        def fake_download(url, dst, what=None):
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            dst.write_bytes(b"")
+            setup.mark(dst)
+
+        patches = [
+            mock.patch.object(setup, "ROOT", t),
+            mock.patch.object(setup, "GPU_PICK", None),
+            mock.patch.object(setup, "data_folder", lambda d: (t / "data", [])),
+            mock.patch.object(setup, "load_settings", lambda: {}),
+            mock.patch.object(setup, "save_settings", lambda s: None),
+            mock.patch.object(setup, "gpus", lambda: found),
+            mock.patch.object(setup, "amd_gpus", lambda: []),
+            mock.patch.object(setup, "ram_gb", lambda: ram),
+            mock.patch.object(setup, "cpu_info", lambda: ("Test CPU", True, avx512)),
+            mock.patch.object(setup, "page_file_gb", lambda: 16.0),
+            mock.patch.object(setup, "is_wsl", lambda: False),
+            mock.patch.object(setup, "free_gb", lambda p: 900.0),
+            mock.patch.object(setup, "pip_install", lambda *a, **k: None),
+            mock.patch.object(setup, "get_llama_cpp", lambda: t / "llama.cpp"),
+            mock.patch.object(setup, "get_prebuilt", lambda *a, **k: eng),
+            mock.patch.object(setup, "update_installed_engine", lambda *a, **k: None),
+            mock.patch.object(setup, "download", fake_download),
+            mock.patch.object(setup, "check_shards", lambda shards: None),
+            mock.patch.object(setup, "verify_sha256", lambda *a: None),
+            mock.patch.object(setup, "run", lambda *a, **k: None),
+            mock.patch.object(setup, "mtp_corrupt", lambda *a, **k: False),
+            mock.patch.object(setup, "refresh_draft_vocab", lambda *a, **k: None),
+            mock.patch.object(setup, "write_run_script", lambda tag, cfg, port: t / f"run-{tag}.bat"),
+            mock.patch.object(setup, "saved_calibration", lambda cfg: None),
+            mock.patch.object(setup, "calibrate_config", mock.Mock(side_effect=AssertionError("calibrated"))),
+            mock.patch.dict(sys.modules, {"gguf_reader": types.SimpleNamespace(GGUFFile=FakeGGUF)}),
+            mock.patch.object(sys, "argv", ["setup.py", "--models-dir", str(t / "models"), *argv]
+                              + (["--yes"] if answers is None else [])),
+            mock.patch("builtins.input", fake_input),
+            *extra,
+        ]
+        out = io.StringIO()
+        code = None
+        with contextlib.ExitStack() as st:
+            for p in patches:
+                st.enter_context(p)
+            with contextlib.redirect_stdout(out):
+                try:
+                    code = setup.main()
+                except SystemExit as e:
+                    code = e.code
+        written = sorted(t.glob("strata-*.json"), key=lambda p: p.stat().st_mtime)
+        cfg = None
+        if written and code == 0:
+            cfg = normalize(json.loads(written[-1].read_text(encoding="utf-8")), t)
+        return code, out.getvalue(), cfg, asked
+
+
+def argv_for(family, model):
+    return ["--family", family, "--no-start"] + (["--model", model] if model else [])
+
+
+def record() -> dict:
+    """Every profile x model with --yes: the exit code and the config (the baseline this test compares against)."""
+    res = {}
+    for prof, (ram, found) in PROFILES.items():
+        for family, model in MODELS:
+            code, out, cfg, _ = install(ram, found, argv_for(family, model))
+            res[f"{prof} {family} {model or 'recommended'}"] = {"code": code, "config": cfg}
+    return res
+
+
+class Golden(unittest.TestCase):
+    """--yes and Enter: the same config as the baseline wherever the baseline wrote one."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.golden = json.loads(GOLDEN.read_text(encoding="utf-8"))
+
+    def cases(self):
+        for prof, (ram, found) in PROFILES.items():
+            for family, model in MODELS:
+                key = f"{prof} {family} {model or 'recommended'}"
+                if self.golden[key]["code"] == 0:
+                    yield key, ram, found, family, model
+
+    def test_the_baseline_covers_every_kind_of_pc(self):
+        written = [k for k, v in self.golden.items() if v["code"] == 0]
+        self.assertGreaterEqual(len(written), 20)
+        lowram_multi = self.golden["32GB-2x24GB qwen IQ3_XXS"]["config"]   # #364: low-RAM, two cards -> one
+        self.assertIn("--resident-experts" if "--resident-experts" in lowram_multi["args"] else "--mmap-experts",
+                      lowram_multi["args"])
+        self.assertNotIsInstance(lowram_multi.get("gpu"), list)
+
+    def test_yes(self):
+        for key, ram, found, family, model in self.cases():
+            with self.subTest(key):
+                code, out, cfg, asked = install(ram, found, argv_for(family, model))
+                self.assertEqual(code, 0, out[-3000:])
+                self.assertEqual(cfg, self.golden[key]["config"], out[-3000:])
+                self.assertEqual(asked, [])
+
+    def test_enter_for_every_question(self):
+        for key, ram, found, family, model in self.cases():
+            with self.subTest(key):
+                code, out, cfg, asked = install(ram, found, argv_for(family, model), answers="")
+                self.assertEqual(code, 0, out[-3000:])
+                self.assertEqual(cfg, self.golden[key]["config"], out[-3000:])
+
+    def test_the_refusals_with_yes_alone_stay(self):
+        """A size the baseline refused with --yes and no explicit --model still stops (--yes alone is no consent)."""
+        for prof, (ram, found) in PROFILES.items():
+            for family, model in MODELS:
+                key = f"{prof} {family} {model or 'recommended'}"
+                if self.golden[key]["code"] != 0 and model is None:
+                    with self.subTest(key):
+                        self.assertNotEqual(install(ram, found, argv_for(family, model))[0], 0)
+
+
+if __name__ == "__main__":
+    if "--write" in sys.argv:
+        GOLDEN.write_text(json.dumps(record(), indent=1, sort_keys=True) + "\n", encoding="utf-8")
+        print(f"wrote {GOLDEN}")
+    else:
+        unittest.main()

tools/test_setup_lowram.py

@@ -46,8 +46,10 @@ class OneGpuWhy(unittest.TestCase):
         need = arena + setup.LOW_RAM_HEADROOM_GB
         self.assertIn(f"{arena:.0f} + {setup.LOW_RAM_HEADROOM_GB} = {need:.0f} GB", text)
         self.assertIn("this PC has 46 GB", text)
-        self.assertIn("layer split", text)                            # why one GPU
-        self.assertIn("--low-ram off", text)                          # and the way to keep them all
+        self.assertIn("layer split", text)                            # why one GPU is recommended
+        self.assertIn("--gpus 0,1", text)                             # #364 #384: and the way to use them all
+        self.assertIn("OS file cache", text)
+        self.assertNotIn("--low-ram off", text)                       # (that pages the experts: not the way)
 
     def test_an_explicit_choice_says_so(self):
         text = "\n".join(setup.low_ram_one_gpu_why("Q2_0", 64, "resident"))

tools/test_setup_pins.py

@@ -98,6 +98,48 @@ class HuggingFacePins(unittest.TestCase):
         finally:
             mtp_fetch.REPO = pinned
 
+    def test_hf_endpoint(self):
+        # #495: HF_ENDPOINT (a mirror) serves the same pinned revision; a trailing slash and blanks are dropped
+        repo = "ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF"
+        with mock.patch.dict(setup.os.environ, {}, clear=False):
+            setup.os.environ.pop("HF_ENDPOINT", None)
+            self.assertEqual(setup.hf(repo), f"https://huggingface.co/{repo}/resolve/{setup.HF_REVISIONS[repo]}/")
+            for value in ("https://hf-mirror.com", "https://hf-mirror.com/", " https://hf-mirror.com/ "):
+                setup.os.environ["HF_ENDPOINT"] = value
+                url = setup.hf(repo)
+                self.assertEqual(url, f"https://hf-mirror.com/{repo}/resolve/{setup.HF_REVISIONS[repo]}/")
+                self.assertRegex(url, SHA)
+                self.assertEqual(setup.hf_unpinned(url + "x.gguf"), f"https://hf-mirror.com/{repo}/resolve/main/x.gguf")
+            setup.os.environ["HF_ENDPOINT"] = ""
+            self.assertEqual(setup.hf_endpoint(), "https://huggingface.co")
+
+    def test_mtp_fetch_honours_hf_endpoint(self):
+        import importlib
+        import mtp_fetch
+        try:
+            with mock.patch.dict(mtp_fetch.os.environ, {"HF_ENDPOINT": "https://hf-mirror.com/"}):
+                importlib.reload(mtp_fetch)
+                self.assertTrue(mtp_fetch.REPO.startswith("https://hf-mirror.com/Qwen/Qwen3.8-Flash-Next/resolve/"))
+                self.assertRegex(mtp_fetch.REPO, SHA)
+                self.assertTrue(mtp_fetch.pinned())                 # the tensors' SHA-256 checks still apply
+        finally:
+            importlib.reload(mtp_fetch)
+        self.assertTrue(mtp_fetch.REPO.startswith("https://huggingface.co/"))
+
+    def test_step5_names_the_folder(self):
+        # #495: where setup expects the model files, and how to give it files downloaded by hand
+        from test_setup_golden import PROFILES, install
+        ram, found = PROFILES["96GB-1x16GB"]
+        with mock.patch.dict(setup.os.environ, {"HF_ENDPOINT": "https://hf-mirror.com"}):
+            code, out, cfg, _ = install(ram, found, ["--family", "qwen", "--model", "IQ3_XXS", "--no-start"])
+        self.assertEqual(code, 0)
+        text = out.replace("\\", "/")
+        self.assertRegex(text, r"The model files go in .*/models/IQ3_XXS")
+        self.assertIn("put them here with their original names (Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+                      text)
+        self.assertIn("--gguf-dir", text)
+        self.assertIn("Downloading from https://hf-mirror.com (HF_ENDPOINT)", text)
+
 
 class Engine(unittest.TestCase):
     def setUp(self):

tools/test_setup_prompts.py

@@ -0,0 +1,89 @@
+"""Setup prompts require input or --yes; EOF must not authorize system installs (#393).
+No GPU, downloads or installers are used.
+
+    python -m unittest tools.test_setup_prompts
+"""
+from __future__ import annotations
+
+import contextlib
+import io
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+import setup  # noqa: E402
+
+
+class Prompts(unittest.TestCase):
+    def test_eof_stops_with_guidance(self):
+        out = io.StringIO()
+        with mock.patch("sys.stdin", io.StringIO()), contextlib.redirect_stdout(out):
+            with self.assertRaises(SystemExit) as cm:
+                setup.ask("Install them now?", ["y", "n"], "y", False)
+        self.assertEqual(cm.exception.code, 1)
+        self.assertIn("Install them now?", out.getvalue())
+        self.assertIn("--yes", out.getvalue())
+        self.assertIn("Setup stopped", out.getvalue())
+
+    def test_yes_accepts_the_default_without_reading_input(self):
+        with mock.patch("builtins.input", side_effect=AssertionError("unexpected prompt")):
+            for default in ("y", "n"):
+                with self.subTest(default=default):
+                    self.assertEqual(setup.ask("Install?", ["y", "n"], default, True), default)
+
+    def test_enter_and_explicit_answers(self):
+        for answer, expected in (("\n", "y"), ("  \n", "y"), ("y\n", "y"), (" N \n", "n")):
+            with self.subTest(answer=answer), mock.patch("sys.stdin", io.StringIO(answer)), \
+                    contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(setup.ask("Install?", ["y", "n"], "y", False), expected)
+
+    def test_invalid_answer_reprompts_and_preserves_choice_case(self):
+        out = io.StringIO()
+        with mock.patch("sys.stdin", io.StringIO("invalid\nyArN\n")), contextlib.redirect_stdout(out):
+            self.assertEqual(setup.ask("Method?", ["Yarn", "Linear"], "Linear", False), "Yarn")
+        self.assertIn("please answer one of: Yarn, Linear", out.getvalue())
+
+    def test_eof_after_invalid_answer_stops(self):
+        with mock.patch("sys.stdin", io.StringIO("invalid\n")), contextlib.redirect_stdout(io.StringIO()):
+            with self.assertRaises(SystemExit) as cm:
+                setup.ask("Install?", ["y", "n"], "y", False)
+        self.assertEqual(cm.exception.code, 1)
+
+
+class BuildTools(unittest.TestCase):
+    def test_eof_never_starts_an_installer(self):
+        for win in (True, False):
+            with self.subTest(win=win), mock.patch.object(setup, "WIN", win), \
+                    mock.patch.object(setup, "find_nvcc", return_value=(None, (0, 0))), \
+                    mock.patch.object(setup, "find_vcvars", return_value=None), \
+                    mock.patch.object(setup.shutil, "which", side_effect=lambda name: None if name == "g++" else name), \
+                    mock.patch.object(setup, "run", side_effect=AssertionError("installer started")) as run, \
+                    mock.patch.object(setup, "download", side_effect=AssertionError("download started")) as download, \
+                    mock.patch("sys.stdin", io.StringIO()), contextlib.redirect_stdout(io.StringIO()):
+                with self.assertRaises(SystemExit) as cm:
+                    setup.install_build_tools({"arch": "120"}, False)
+                self.assertEqual(cm.exception.code, 1)
+                run.assert_not_called()
+                download.assert_not_called()
+
+    def test_yes_still_installs_missing_windows_tools(self):
+        nvcc, vcvars = Path("nvcc"), Path("vcvars64.bat")
+        with mock.patch.object(setup, "WIN", True), \
+                mock.patch.object(setup, "find_nvcc", side_effect=[(None, (0, 0)), (nvcc, (13, 0))]), \
+                mock.patch.object(setup, "find_vcvars", side_effect=[None, vcvars, vcvars, vcvars]), \
+                mock.patch.object(setup.shutil, "which", return_value="winget"), \
+                mock.patch.object(setup, "run") as run, \
+                mock.patch("builtins.input", side_effect=AssertionError("unexpected prompt")), \
+                contextlib.redirect_stdout(io.StringIO()):
+            self.assertEqual(setup.install_build_tools({"arch": "120"}, True), (nvcc, vcvars))
+        commands = [call.args[0] for call in run.call_args_list]
+        self.assertEqual(len(commands), 2)
+        self.assertIn("Microsoft.VisualStudio.2022.BuildTools", commands[0])
+        self.assertIn("Nvidia.CUDA", commands[1])
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/test_setup_risk.py

@@ -0,0 +1,414 @@
+"""Tests for the owner's rule in setup.py - it recommends, it never forces: a choice bigger than the recommendation is
+kept and its risk said (#406's 256K on 64 GB, #364/#384's two GPUs in the low-RAM mode, #403's RAM budget), and the
+stops that stay are the ones a user can confirm (confirm_risk: --yes with an explicit flag, or y).  The defaults are
+tools/test_setup_golden.py's.  Mocked - no GPU, no downloads, nothing written outside a temp folder.
+
+    python -m unittest tools.test_setup_risk
+"""
+from __future__ import annotations
+
+import contextlib
+import io
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "tools"))
+import setup  # noqa: E402
+from test_setup_golden import PROFILES, card, install  # noqa: E402
+
+
+def arg(cfg, flag):
+    a = cfg["args"]
+    return a[a.index(flag) + 1] if flag in a else None
+
+
+def run(fn, *args, stdin=None):
+    """-> (exit code or None, printed text, questions asked); stdin answers every input() (None: never asked)."""
+    out, asked, code = io.StringIO(), [], None
+
+    def fake_input(prompt=""):
+        asked.append(prompt)
+        if stdin is None:
+            raise AssertionError(f"asked {prompt!r} with --yes")
+        return stdin
+
+    with contextlib.redirect_stdout(out), mock.patch("builtins.input", fake_input):
+        try:
+            fn(*args)
+        except SystemExit as e:
+            code = e.code
+    return code, out.getvalue(), asked
+
+
+class ConfirmRisk(unittest.TestCase):
+    """S6: one helper for the stops a user may take on."""
+
+    def go(self, explicit, yes, stdin=None):
+        return run(setup.confirm_risk, "it may not fit", explicit, yes, "it does not fit", "a smaller size",
+                   stdin=stdin)
+
+    def test_yes_with_an_explicit_choice_is_consent(self):
+        code, out, asked = self.go(True, True)
+        self.assertIsNone(code)
+        self.assertIn("it may not fit", out)
+        self.assertEqual(asked, [])
+
+    def test_yes_alone_keeps_the_stop(self):
+        code, out, asked = self.go(False, True)
+        self.assertEqual(code, 1)
+        self.assertIn("it does not fit", out)
+        self.assertIn("a smaller size", out)
+
+    def test_interactive_default_is_no(self):
+        for explicit in (True, False):
+            code, out, asked = self.go(explicit, False, stdin="")
+            self.assertEqual(code, 1)
+            self.assertIn("[n]", asked[0])
+            self.assertIsNone(self.go(explicit, False, stdin="y")[0])
+
+    def test_paging_with_an_explicit_model(self):
+        code, out, asked = run(setup.confirm_paging, "IQ3_XXS", 54, "auto", True, True)
+        self.assertIsNone(code, out)
+        self.assertIn("installing IQ3_XXS with 54 GB of RAM, as you chose (--model)", out)
+
+
+class Context(unittest.TestCase):
+    """S1 (#406 #364): the RAM rule recommends, an explicit or picked context is kept with a note."""
+    RAM64, GPU32 = PROFILES["64GB-1x32GB"]
+
+    def test_the_ram_rule(self):
+        self.assertEqual(setup.ram_ctx("IQ3_S", 63.7), 131072)
+        self.assertEqual(setup.ram_ctx("IQ3_XXS", 63.7), 131072)
+        self.assertEqual(setup.ram_ctx("IQ3_XXS", 95.8), 524288)      # 43 + 7.2 + 24 GB fits 96
+        self.assertEqual(setup.ram_ctx("IQ3_S", 76.5), 131072)
+        self.assertEqual(setup.ram_ctx("IQ3_S", 78.5), 262144)
+        self.assertEqual(setup.ram_ctx("Q2_0", 31.9), 524288)          # other sizes: no RAM rule
+        self.assertEqual(setup.ram_ctx("IQ3_XXS", 31.9, low_ram=True), 524288)   # the KV cache is in VRAM there
+        self.assertIsNone(setup.ctx_ram_need("IQ3_XXS", 262144, low_ram=True))
+        self.assertAlmostEqual(setup.ctx_ram_need("IQ3_S", 262144), 77.9, places=1)
+
+    def test_an_explicit_256k_is_kept(self):
+        code, out, cfg, _ = install(self.RAM64, self.GPU32, ["--family", "qwen", "--model", "IQ3_S", "--no-start",
+                                                             "--context", "262144"])
+        self.assertEqual(code, 0, out)
+        self.assertEqual(arg(cfg, "--max-context"), "262144")
+        self.assertNotIn("--rope-scaling", cfg["args"])
+        self.assertIn("256K with IQ3_S needs ~78 GB of RAM by setup's estimate", out)
+        self.assertIn("this PC has 64. Kept as you chose: it may be slower or run out of RAM under load. 128K is the "
+                      "recommended size.", out)
+
+    def test_a_picked_256k_is_kept_and_the_menu_says_the_risk(self):
+        code, out, cfg, asked = install(self.RAM64, self.GPU32, ["--family", "qwen", "--model", "IQ3_S", "--no-start"],
+                                        answers={"Context?": "5"})
+        self.assertEqual(code, 0, out)
+        self.assertEqual(arg(cfg, "--max-context"), "262144")
+        self.assertIn("256K tokens   (needs ~78 GB RAM, this PC has 64: may run out of memory)", out)
+        self.assertIn("128K tokens   (recommended for your GPU)\n", out)
+        self.assertIn("Kept as you chose", out)
+
+    def test_low_ram_mode_keeps_262k_without_a_ram_note(self):
+        ram, found = PROFILES["32GB-2x24GB"]                              # #364
+        code, out, cfg, _ = install(ram, found, ["--family", "qwen", "--model", "IQ3_XXS", "--no-start",
+                                                 "--context", "262144"])
+        self.assertEqual(code, 0, out)
+        self.assertEqual(arg(cfg, "--max-context"), "262144")
+        self.assertNotIn("Kept as you chose: it may be slower", out)
+
+    def test_the_earlier_install_keeps_its_context(self):
+        """An update (a new copy of Strata) set up like the earlier install keeps its 256K instead of capping it."""
+        with tempfile.TemporaryDirectory() as d:
+            prev = Path(d) / "strata-iq3_s.json"
+            prev.write_text(json.dumps({"args": ["--max-context", "262144", "--kv", "int8"], "port": 8080}))
+            started = mock.Mock(return_value=0)
+            code, out, cfg, _ = install(self.RAM64, self.GPU32, [], extra=[
+                mock.patch.object(setup, "previous_config", lambda *a: prev),
+                mock.patch.object(setup, "start", started)])
+        self.assertEqual(code, 0, out)
+        self.assertEqual(arg(cfg, "--max-context"), "262144")
+        self.assertIn("Kept as you chose", out)
+        self.assertTrue(started.called)
+
+
+class BrokenEarlierConfig(unittest.TestCase):
+    """#459: a new copy of Strata set up like an earlier install skips an earlier config that does not parse (an
+    empty strata-*.json crashed START-HERE with JSONDecodeError) and goes on as a fresh install; configs are written
+    whole (a temporary file moved over the old one)."""
+    RAM64, GPU32 = PROFILES["64GB-1x32GB"]
+
+    def setup_with(self, files):
+        with tempfile.TemporaryDirectory() as d:
+            for i, (name, text) in enumerate(files):                  # in order, oldest first
+                (Path(d) / name).write_text(text, encoding="utf-8")
+                os.utime(Path(d) / name, (1_700_000_000 + i, 1_700_000_000 + i))
+            return install(self.RAM64, self.GPU32, [], extra=[
+                mock.patch.object(setup, "other_installs", lambda settings: [Path(d)]),
+                mock.patch.object(setup, "start", mock.Mock(return_value=0))])
+
+    def test_an_empty_config_is_skipped(self):
+        for text, why in (("", "the file is empty"), ("{\"args\": [", "not valid JSON"), ("[1]", "not a JSON object")):
+            with self.subTest(text=text):
+                code, out, cfg, _ = self.setup_with([("strata-iq3_s.json", text)])
+                self.assertEqual(code, 0, out)
+                self.assertIn("skipped the earlier config ", out)
+                self.assertIn(f"strata-iq3_s.json ({why}): setting this copy up without it", out)
+                self.assertNotIn("Found your earlier install", out)
+                self.assertIsNotNone(cfg)                                # the fresh install's config
+
+    def test_the_newest_readable_config_is_used(self):
+        good = json.dumps({"args": ["--max-context", "262144", "--kv", "int8"], "port": 8080})
+        code, out, cfg, _ = self.setup_with([("strata-iq3_s.json", good), ("strata-q2_0.json", "")])
+        self.assertEqual(code, 0, out)
+        self.assertIn("strata-q2_0.json (the file is empty)", out)
+        self.assertIn("Found your earlier install", out)
+        self.assertIn("(iq3_s)", out)
+        self.assertEqual(arg(cfg, "--max-context"), "262144")
+
+    def test_write_config_leaves_no_partial_file(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = Path(d) / "strata-iq3_s.json"
+            p.write_text("old", encoding="utf-8")
+            setup.write_config(p, {"args": ["--kv", "int8"]})
+            self.assertEqual(p.read_text(encoding="utf-8"), json.dumps({"args": ["--kv", "int8"]}, indent=1))
+            self.assertEqual([f.name for f in Path(d).iterdir()], ["strata-iq3_s.json"])   # no .tmp left
+            with mock.patch.object(Path, "write_text", side_effect=OSError(28, "No space left on device")):
+                with self.assertRaises(OSError):
+                    setup.write_config(p, {"args": []})
+            self.assertEqual(json.loads(p.read_text(encoding="utf-8")), {"args": ["--kv", "int8"]})   # kept whole
+
+
+class LowRamGpus(unittest.TestCase):
+    """S2/S3 (#364 #384): one GPU recommended in the low-RAM mode, all of them when asked for."""
+
+    def test_explicit_gpus_are_kept_with_the_mapped_variant(self):
+        for prof, model in (("32GB-2x24GB", "IQ3_XXS"), ("32GB-2x24GB", "Q2_0"), ("47GB-2x16GB", "IQ3_XXS")):
+            with self.subTest(prof=prof, model=model):
+                ram, found = PROFILES[prof]
+                code, out, cfg, _ = install(ram, found, ["--family", "qwen", "--model", model, "--no-start",
+                                                         "--gpus", "0,1"])
+                self.assertEqual(code, 0, out)
+                self.assertEqual(cfg["gpu"], [0, 1])
+                self.assertEqual(cfg["layer_split"], "auto")
+                self.assertIn("--mmap-experts", cfg["args"])
+                self.assertNotIn("--resident-experts", cfg["args"])
+                self.assertIn("as you chose (--gpus)", out)
+                self.assertIn("RAM can fill up to 0 free", out)
+                self.assertIn("low-RAM mode on 2 GPUs", out)
+
+    def test_the_share_counts_every_card(self):
+        ram, found = PROFILES["47GB-2x16GB"]                              # #384: 2 x 16 GB, 47 GB, IQ3_XXS at 64K
+        code, out, cfg, _ = install(ram, found, ["--family", "qwen", "--model", "IQ3_XXS", "--no-start",
+                                                 "--gpus", "0,1"])
+        one = setup.low_ram_gpu_gb("IQ3_XXS", 15.9, 65536, "int8")
+        self.assertIn(f"the GPUs hold ~{100 * 2 * one / setup.MODELS['IQ3_XXS']['arena_gb']:.0f}% of them", out)
+
+    def test_asked_and_answered_two(self):
+        ram, found = PROFILES["32GB-2x24GB"]
+        code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
+                                        answers={"Low-RAM mode: which GPUs?": "2"})
+        self.assertEqual(code, 0, out)
+        self.assertTrue(any("Low-RAM mode: which GPUs? [1]" in q for q in asked), asked)
+        self.assertEqual(cfg["gpu"], [0, 1])
+        self.assertIn("--mmap-experts", cfg["args"])
+        self.assertIn("1) GPU 0 (NVIDIA GeForce RTX 3090, 24 GB) only", out)
+
+    def test_yes_and_enter_keep_one_gpu(self):
+        ram, found = PROFILES["32GB-2x24GB"]
+        for answers in (None, ""):
+            code, out, cfg, _ = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
+                                        answers=answers)
+            self.assertEqual(code, 0, out)
+            self.assertEqual(cfg["gpu"], 0)
+            self.assertIn("--resident-experts", cfg["args"])
+
+    def test_low_ram_resident_with_gpus(self):
+        ram, found = PROFILES["32GB-2x24GB"]
+        code, out, cfg, _ = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start", "--gpus", "0,1",
+                                                 "--low-ram", "resident"])
+        self.assertEqual(code, 0, out)
+        self.assertEqual(cfg["gpu"], [0, 1])
+        self.assertIn("--mmap-experts", cfg["args"])
+        self.assertIn("--low-ram resident has no layer split yet", out)
+
+    def test_low_ram_resident_alone_keeps_one_gpu_without_asking(self):
+        ram, found = PROFILES["32GB-2x24GB"]
+        code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start",
+                                                     "--low-ram", "resident"], answers="")
+        self.assertEqual(code, 0, out)
+        self.assertEqual(cfg["gpu"], 0)
+        self.assertFalse(any("Low-RAM mode" in q for q in asked), asked)
+
+
+class StartOnSeveralGpus(unittest.TestCase):
+    """A resident low-RAM config started on several GPUs reads the experts through the file cache (#364 #384)."""
+
+    def test_split_mmap(self):
+        cfg = {"args": ["--pack", "p", "--resident-experts", "--kv", "int8"]}
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            self.assertTrue(setup.split_mmap(cfg))
+            self.assertFalse(setup.split_mmap(cfg))
+            self.assertFalse(setup.split_mmap({"args": ["--mmap-experts"]}))
+        self.assertEqual(cfg["args"], ["--pack", "p", "--mmap-experts", "--kv", "int8"])
+        self.assertIn("no layer split yet", out.getvalue())
+
+    def offer(self, args, stdin):
+        found = PROFILES["32GB-2x24GB"][1]
+        with tempfile.TemporaryDirectory() as d:
+            p = Path(d) / "strata-q2_0.json"
+            cfg = {"args": list(args)}
+            with mock.patch.object(setup, "gpus", lambda: found), \
+                    mock.patch.object(setup, "engine_runs_on", lambda g: True):
+                code, out, asked = run(setup.offer_together, p, cfg, stdin is None, stdin=stdin)
+            return code, out, asked, json.loads(p.read_text())
+
+    def test_offer_together_recommends_one_gpu_for_a_resident_config(self):
+        code, out, asked, cfg = self.offer(["--resident-experts"], None)       # --yes: the recommendation
+        self.assertIsNone(code, out)
+        self.assertNotIn("gpu", cfg)
+        self.assertEqual(cfg["args"], ["--resident-experts"])
+        code, out, asked, cfg = self.offer(["--resident-experts"], "y")
+        self.assertIn("[n]", asked[0])
+        self.assertEqual(cfg["gpu"], [0, 1])
+        self.assertEqual(cfg["args"], ["--mmap-experts"])
+        code, out, asked, cfg = self.offer(["--mmap-experts"], None)           # other configs: as before
+        self.assertEqual(cfg["gpu"], [0, 1])
+
+    def test_start_with_gpus(self):
+        found = PROFILES["32GB-2x24GB"][1]
+        with tempfile.TemporaryDirectory() as d:
+            exe = Path(d) / "strata.exe"
+            exe.write_bytes(b"")
+            p = Path(d) / "strata-q2_0.json"
+            p.write_text(json.dumps({"exe": str(exe), "args": ["--resident-experts"], "gpu": 0, "gpus_asked": True}))
+            call = mock.Mock(return_value=0)
+            with mock.patch.object(setup, "gpus", lambda: found), \
+                    mock.patch.object(setup, "ensure_engine_for", lambda cards, path, cfg, yes: cfg), \
+                    mock.patch.object(setup, "is_wsl", lambda: False), \
+                    mock.patch.object(setup.subprocess, "call", call):
+                code, out, _ = run(setup.start, p, None, [0, 1], False, True)
+            cfg = json.loads(p.read_text())
+        self.assertIsNone(code, out)
+        self.assertEqual(cfg["gpu"], [0, 1])
+        self.assertEqual(cfg["args"], ["--mmap-experts"])
+        self.assertIn("no layer split yet", out)
+        self.assertTrue(call.called)
+
+
+class SmallCard(unittest.TestCase):
+    """S9: a card under 8 GB named with --gpus is a risk to confirm, not a stop."""
+    FOUND = [card(0, "NVIDIA GeForce RTX 3090", 24.0, "86"), card(1, "NVIDIA GeForce RTX 2060", 6.0, "75")]
+
+    def test_named_with_yes(self):
+        code, out, cfg, _ = install(63.7, self.FOUND, ["--family", "qwen", "--model", "Q2_0", "--no-start",
+                                                       "--gpus", "0,1"])
+        self.assertEqual(code, 0, out)
+        self.assertEqual(cfg["gpu"], [0, 1])
+        self.assertIn("GPU 1 (NVIDIA GeForce RTX 2060) has 6 GB of VRAM", out)
+        self.assertIn("is used together with the others, as you chose", out)
+
+    def test_named_and_enter_stops(self):
+        code, out, cfg, asked = install(63.7, self.FOUND, ["--family", "qwen", "--model", "Q2_0", "--no-start",
+                                                           "--gpus", "0,1"], answers="")
+        self.assertEqual(code, 1)
+        self.assertTrue(any("Use it anyway?" in q for q in asked), asked)
+
+    def test_not_named_is_not_offered(self):
+        code, out, cfg, _ = install(63.7, self.FOUND, ["--family", "qwen", "--model", "Q2_0", "--no-start"])
+        self.assertEqual(code, 0, out)
+        self.assertEqual(cfg["gpu"], 0)                                   # one card: as before
+
+
+class SplitShortCard(unittest.TestCase):
+    """#448: a second card too small to lend a split's prompt chunk: the first card alone recommended, the pair kept
+    when named."""
+    FOUND = [card(0, "NVIDIA RTX PRO 4500 Blackwell", 31.8, "120"), card(1, "NVIDIA GeForce RTX 3080", 10.0, "86")]
+
+    def test_rule(self):
+        self.assertEqual([g["index"] for g in setup.split_short(self.FOUND)], [1])
+        self.assertEqual(setup.split_short(PROFILES["32GB-2x24GB"][1]), [])
+        self.assertEqual(setup.split_short(PROFILES["47GB-2x16GB"][1]), [])
+        eights = [card(i, "NVIDIA GeForce RTX 3070", 8.0, "86") for i in range(2)]
+        self.assertEqual(setup.split_short(eights), [])                   # neither card alone reads more
+
+    def test_yes_takes_the_first_card(self):
+        code, out, cfg, _ = install(127.8, self.FOUND, ["--family", "qwen", "--model", "Q2_0", "--no-start"])
+        self.assertEqual(code, 0, out)
+        self.assertEqual(cfg["gpu"], 0)
+        self.assertIn("too small to lend a split its prompt buffers", out)
+
+    def test_named_pair_is_kept(self):
+        code, out, cfg, _ = install(127.8, self.FOUND, ["--family", "qwen", "--model", "Q2_0", "--no-start",
+                                                        "--gpus", "0,1"])
+        self.assertEqual(code, 0, out)
+        self.assertEqual(cfg["gpu"], [0, 1])
+
+    def test_offer_together_defaults_to_one(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = Path(d) / "strata-q2_0.json"
+            with mock.patch.object(setup, "gpus", lambda: self.FOUND), \
+                    mock.patch.object(setup, "engine_runs_on", lambda g: True):
+                code, out, asked = run(setup.offer_together, p, {"args": ["--mmap-experts"]}, True)
+            cfg = json.loads(p.read_text())
+        self.assertIsNone(code, out)
+        self.assertNotIn("gpu", cfg)
+        self.assertIn("#448", out)
+
+
+class RamFloor(unittest.TestCase):
+    """S7: less RAM than the smallest model needs: a stop with --yes alone, a risk with --model."""
+    FOUND = [card(0, "NVIDIA GeForce RTX 3060", 8.0, "86")]
+
+    def test_yes_alone_stops(self):
+        code, out, cfg, _ = install(19.9, self.FOUND, ["--family", "qwen", "--no-start"])
+        self.assertEqual(code, 1)
+        self.assertIn("--model NAME --yes", out)
+
+    def test_an_explicit_model_goes_on(self):
+        code, out, cfg, _ = install(19.9, self.FOUND, ["--family", "coder", "--model", "IQ1_M", "--no-start"])
+        self.assertEqual(code, 0, out)
+        self.assertIn("going on with 20 GB of RAM, as you chose", out)
+
+
+class KvStreaming(unittest.TestCase):
+    """8: --kv-streaming on|off|auto (auto: the RAM test, as before)."""
+    RAM, FOUND = PROFILES["64GB-1x32GB"]
+
+    def go(self, model, *flags):
+        return install(self.RAM, self.FOUND, ["--family", "qwen", "--model", model, "--no-start", *flags])
+
+    def test_off(self):
+        code, out, cfg, _ = self.go("IQ3_XXS", "--kv-streaming", "off")
+        self.assertEqual(code, 0, out)
+        self.assertNotIn("--kv-resident", cfg["args"])
+        self.assertIn("KV streaming off, as you chose", out)
+
+    def test_on_past_the_ram_test(self):
+        code, out, cfg, _ = self.go("IQ3_S", "--kv-streaming", "on")         # auto leaves it off on 64 GB
+        self.assertEqual(code, 0, out)
+        self.assertEqual(arg(cfg, "--kv-resident"), "32768")
+        self.assertIn("Kept as you chose (--kv-streaming on)", out)
+
+    def test_on_where_it_cannot(self):
+        code, out, cfg, _ = self.go("Q2_0", "--kv-streaming", "on", "--kv", "k8v4")
+        self.assertNotIn("--kv-resident", cfg["args"])
+        self.assertIn("it refuses the pair", out)
+        code, out, cfg, _ = self.go("Q2_0", "--kv-streaming", "on", "--context", "32768")
+        self.assertNotIn("--kv-resident", cfg["args"])
+        self.assertIn("a context under 64K is not streamed", out)
+
+    def test_budget_flag_on_another_size(self):
+        code, out, cfg, _ = self.go("Q2_0", "--resident-budget-gib", "30")
+        self.assertEqual(code, 0, out)
+        self.assertNotIn("--resident-budget-gib", cfg["args"])
+        self.assertIn("--resident-budget-gib is for UD-Q4_K_XL", out)
+
+
+if __name__ == "__main__":
+    unittest.main()