Merge branch 'f-138-fixB' into f-138-prs

A rebinding page cannot send the key, so with one configured the check only
broke tunnels and proxies that pass their own name on (cloudflared does).
Keyless setups keep it, with allowed_hosts; the 403 now also says that
setting an API key turns the check off.

CMakeLists.txt

@@ -8,7 +8,7 @@
 # below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
 # and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.35 LANGUAGES CXX)
+project(strata VERSION 0.1.38 LANGUAGES CXX)
 add_compile_definitions(STRATA_VERSION="${PROJECT_VERSION}")
 # issue #31: release builds keep a PDB beside the exe (not shipped in it), so a minidump from a stalled engine can
 # be read back to the exact source line
@@ -274,6 +274,15 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     target_include_directories(qsa_prompt_attn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
     target_link_libraries(qsa_prompt_attn_parity PRIVATE strata_kernels CUDA::cudart)
   endif()
+  # #512: the top-k's active-block bound (Turing's dispatch, forced on any card by STRATA_TOPK_ACTIVE_ANY) against
+  # the capacity rule, selected ids bitwise
+  if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA)
+    add_executable(qsa_topk_active_parity src/kernels/qsa_topk_active_parity.cpp)
+    target_include_directories(qsa_topk_active_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+    target_link_libraries(qsa_topk_active_parity PRIVATE strata_kernels CUDA::cudart)
+    add_test(NAME qsa_topk_active_parity COMMAND qsa_topk_active_parity)
+    set_tests_properties(qsa_topk_active_parity PROPERTIES SKIP_RETURN_CODE 77)
+  endif()
   # the QSA selection (block scores on the warp kernel and the tensor-core one, top-k): a GPU bench with a
   # scorer-vs-scorer comparison and an FP64 accuracy gate (synthetic, no model); the ctest runs a 32K context under a
   # 262144-cell capacity (the top-k dispatch follows the capacity) and exits non-zero on a failed gate
@@ -340,7 +349,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   # about arithmetic, and this one knows the ORDER the arithmetic happens in.
   add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
     src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp
-    src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp)
+    src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/peer_experts.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
   target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
   if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/file_expert_source_test.cpp")
@@ -350,6 +359,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     add_test(NAME file_expert_source_test COMMAND file_expert_source_test)
     set_tests_properties(file_expert_source_test PROPERTIES TIMEOUT 60)
   endif()
+  # #477: the learned expert profile (--expert-profile-save): its order and make_profile.py's file format (CPU only)
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/expert_profile_save_test.cpp")
+    enable_testing()
+    add_executable(expert_profile_save_test tests/core/expert_profile_save_test.cpp)
+    target_link_libraries(expert_profile_save_test PRIVATE strata_engine)
+    add_test(NAME expert_profile_save_test COMMAND expert_profile_save_test)
+    set_tests_properties(expert_profile_save_test PROPERTIES TIMEOUT 60)
+  endif()
   if(STRATA_BUILD_TESTS AND STRATA_NATIVE_EXPERTS)
     # native_experts.txt v2-v4 (the real packs' copies and the per-role shard column) and the arena filled from
     # synthetic GGUF shards at the real expert dimensions
@@ -550,6 +567,19 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   target_include_directories(iq_multi_parity PRIVATE ${_strata_gpu_include_directories})
   target_link_libraries(iq_multi_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME iq_multi_parity COMMAND iq_multi_parity)
+  # the prompt path's DeltaNet recurrence: gdn_rec_kh_kernel against gdn_rec_cols_pipe_kernel, bitwise (GPU,
+  # synthetic, no model; --bench adds timings)
+  if(STRATA_ENABLE_CUDA)
+    add_executable(gdn_rec_parity src/prefill/gdn_rec_parity.cu)
+    target_link_libraries(gdn_rec_parity PRIVATE ${_strata_gpu_runtime_target})
+    add_test(NAME gdn_rec_parity COMMAND gdn_rec_parity)
+  endif()
+  # native_expert_grouped's group stride and fused SwiGLU + q8_1, bitwise vs the launches before them (GPU, synthetic,
+  # no model; --bench adds v1/new timings of a window's 48 calls in a graph)
+  add_executable(native_grouped_parity src/kernels/native_grouped_parity.cpp)
+  target_include_directories(native_grouped_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(native_grouped_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  add_test(NAME native_grouped_parity COMMAND native_grouped_parity)
   add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)
   target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})
   target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
@@ -580,6 +610,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   set_tests_properties(sampler_parity_one_block PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=0;STRATA_SAMPLER_ONE_BLOCK=1")
   add_test(NAME sampler_parity_old COMMAND sampler_parity --selftest)
   set_tests_properties(sampler_parity_old PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=1;STRATA_SAMPLER_ONE_BLOCK=0")
+  # S19: the decode QSA top-k and the greedy argmax on thread-block clusters (sm_90+) vs the one-CTA kernels, bitwise,
+  # captured in a graph too (GPU, synthetic, no model; skips below sm_90); --bench: per-call times
+  if(STRATA_ENABLE_CUDA)
+    add_executable(decode_cluster_parity src/kernels/decode_cluster_parity.cpp)
+    target_include_directories(decode_cluster_parity PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(decode_cluster_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+    add_test(NAME decode_cluster_parity COMMAND decode_cluster_parity --selftest)
+  endif()
 
   add_executable(rope_parity src/kernels/rope_parity.cpp)
   target_include_directories(rope_parity PRIVATE ${_strata_gpu_include_directories})
@@ -868,7 +906,10 @@ if(STRATA_NATIVE_EXPERTS)
       set(_strata_gg ${strata_llamacpp_SOURCE_DIR}/ggml)
     endif()
     set(_strata_mmq ${_strata_gg}/src/ggml-cuda)
-    add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
+    # #136: the Q2_0 pack's prompt experts on Strata's own int8 kernels (moe_fused.cu, opt-in STRATA_PF_FUSED=1; CUDA
+    # only: the HIP branch below keeps MMQ, and an sm_75 device or an sm_75-only build reports them unavailable)
+    add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/moe_fused.cu src/prefill/moe_fused_iq.cu
+                src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
     # q8_0: the drafter's matrices (E-9); STRATA_ORCA_Q4KS_MMQ adds q5_0 (#296, in _strata_mmq_types), and
     # STRATA_MMQ_KQUANTS q4_k q5_k q5_1, the experts of Unsloth's UD-Q4_K_XL (docs/UNSLOTH_Q4.md; the dispatch
     # follows eddoursul/Strata 7706c1b)
@@ -893,7 +934,19 @@ if(STRATA_NATIVE_EXPERTS)
       target_compile_definitions(strata_mmq PRIVATE STRATA_ORCA_Q4KS_MMQ=1)
     endif()
     target_link_libraries(strata_prefill PUBLIC strata_mmq)
-    target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+    target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1 STRATA_PREFILL_FUSED=1)
+    if(STRATA_BUILD_TESTS)
+      # #136: the fused int8 experts against a double-precision reference and the MMQ path (exit 77 below sm_80)
+      add_executable(prefill_fused_moe_test tests/cuda/prefill_fused_moe_test.cpp)
+      target_link_libraries(prefill_fused_moe_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
+      add_test(NAME prefill_fused_moe_test COMMAND prefill_fused_moe_test)
+      set_tests_properties(prefill_fused_moe_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 300)
+      # #136: the same for the native packs' formats (IQ2_XXS IQ2_XS IQ2_S IQ3_XXS IQ3_S IQ4_XS, down Q2_0 IQ4_NL)
+      add_executable(prefill_fused_iq_test tests/cuda/prefill_fused_iq_test.cpp)
+      target_link_libraries(prefill_fused_iq_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
+      add_test(NAME prefill_fused_iq_test COMMAND prefill_fused_iq_test)
+      set_tests_properties(prefill_fused_iq_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600)
+    endif()
     if(STRATA_BUILD_TESTS AND STRATA_MMQ_KQUANTS)
       # the UD-Q4_K_XL expert formats through MMQ against ggml's dequantized weights (synthetic, a few seconds)
       add_executable(prefill_mmq_kquant_test tests/cuda/prefill_mmq_kquant_test.cpp)
@@ -952,8 +1005,13 @@ if(STRATA_NATIVE_EXPERTS)
     target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact ${_strata_gpu_runtime_target})
     if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA)
       # Unsloth UD-Q4_K_XL's three expert pairs (and Q8_0 alone, its embedding) on synthetic ggml-quantized weights
-      # at 2560/640: GPU and CPU experts against the float reference, GPU dequantizers against to_float
-      foreach(_pair q4_K/q5_1 q4_K/q8_0 q5_K/q8_0 q8_0/q8_0)
+      # at 2560/640: GPU and CPU experts against the float reference, GPU dequantizers against to_float.
+      # q4_K/q5_0 and q5_0/q8_0: a plain llama-quantize "_M" tier GGUF (not Unsloth's dynamic method) commonly
+      # puts Q5_0 on down-projections (sometimes gate/up too) where this model's native_experts.txt had no GPU
+      # kernel for it before; q5_0/q5_0 (both roles Q5_0) is left out of this list since stacking the same
+      # lower-precision format on both roles pushes the GPU path over this test's 3% threshold (quantization
+      # noise, not a kernel bug -- see the PR).
+      foreach(_pair q4_K/q5_1 q4_K/q8_0 q5_K/q8_0 q8_0/q8_0 q4_K/q5_0 q5_0/q8_0)
         string(REPLACE "/" "_" _name "${_pair}")
         add_test(NAME native_expert_parity_${_name} COMMAND native_expert_parity --synthetic ${_pair})
         set_tests_properties(native_expert_parity_${_name} PROPERTIES TIMEOUT 120)

README.md

@@ -23,7 +23,7 @@ word, so 60 tokens per second is faster than you can read.
 
 | Size | Writes answers | Reads your prompt |
 | --- | ---: | ---: |
-| **Q2_0** | 93 tokens/s | 2,170 tokens/s |
+| **Q2_0** | 94 tokens/s | 2,650 tokens/s |
 | **IQ2_XS** | 79 tokens/s | 2,090 tokens/s |
 | **IQ3_XXS** | 62 tokens/s | 1,750 tokens/s |
 | **IQ3_S** | 53 tokens/s | 1,620 tokens/s |
@@ -87,7 +87,8 @@ it left off) and starts it. Your browser opens the Strata app at `http://127.0.0
 > window. The window tells you what it is doing.
 
 **Next time**, run `START-HERE.bat` (or `./setup.sh`) again: it starts right away, nothing is downloaded twice. Close
-its window to stop the model. Updating, Docker, several cards, where the files go and every option:
+its window to stop the model. `UPDATE.bat` (`./update.sh`) updates Strata without starting it. Updating, Docker,
+several cards, where the files go and every option:
 [docs/INSTALL.md](docs/INSTALL.md).
 
 ## Which model should I pick?

UPDATE.bat

@@ -0,0 +1,36 @@
+@echo off
+rem Update Strata without starting the model (#475): the newest code (git pull, when this folder is a git clone), then
+rem what START-HERE.bat does before a start - the engine (a new one when this version needs it), the Python packages,
+rem each installed model's settings and draft subset. The model files are not touched. Start the model later with
+rem START-HERE.bat. Options are passed on to setup.py.
+setlocal
+title Strata - update
+cd /d "%~dp0"
+rem All of it in one block: cmd reads a .bat file while it runs it, and the git pull can change this very file.
+(
+  if exist ".git" (
+    where git >nul 2>nul || (
+      echo  This folder is a git clone, but git is not on PATH: install Git for Windows ^(winget install Git.Git^)
+      echo  or run "git pull" here yourself, then run UPDATE.bat again.
+      pause
+      exit /b 1
+    )
+    echo  Getting the newest Strata ^(git pull^) ...
+    git pull --ff-only || (
+      echo.
+      echo  git pull did not succeed ^(the reason is above^): nothing was updated. Files you changed here can stop it:
+      echo  "git status" lists them.
+      pause
+      exit /b 1
+    )
+  ) else (
+    echo  This copy of Strata was not made with git, so it cannot fetch new files itself. Download the newest one:
+    echo    https://github.com/Niko1221/Strata/archive/refs/heads/main.zip
+    echo  unzip it anywhere and run START-HERE.bat ^(or UPDATE.bat^) in it: it finds the model files in Strata-data
+    echo  and sets itself up the same way - nothing big is downloaded again.
+    echo  Checking this copy's engine and settings meanwhile ...
+  )
+  call "%~dp0START-HERE.bat" --update %*
+  if not errorlevel 1 pause
+  exit /b
+)

docs/AMD_HIP.md

@@ -84,6 +84,13 @@ On a PC with no NVIDIA card Strata can use, the AMD card is chosen by itself; wi
   succeeded, on Windows only. `hipHostGetDevicePointer` returns the host pointer itself on Windows: kernels read
   mapped memory through it correctly, but a device-to-device copy into it does not land, which is why
   `tests/hip/handoff` times out there (the engine does not use that copy; `tests/hip/mapped_alias` reports it).
+- Two more (#380, #377, by BlueKingMuch, measured on an RX 6800 that drives the desktop): `hipMemGetInfo` on Windows
+  does not subtract what the desktop and other programs hold on the card, so `--expert-cache auto` filled the card
+  past what Windows keeps in VRAM and decode fell from 41 to 30 tok/s. The engine now lowers that free figure by
+  what Windows' video memory budget for the process withholds (logged once: `strata: Windows budgets N of this
+  card's M MiB ...`; `STRATA_WDDM_BUDGET=0` turns it off). And the PCIe probe times its copies on the host clock
+  there, since HIP's events read impossible speeds (3,300-26,000 GB/s), so a slow link now gets a smaller
+  `pcie_frac` as on NVIDIA.
 
 **What is validated (0.1.34):** #325's author ran the engine of this port on an RX 9070 XT (Windows 11, ROCm
 10.2.0a20260930 in `.venv`, compiled on the PC): Coder IQ1_M at 32K, 29.2 tok/s decode, ~181 tok/s prefill,
@@ -183,6 +190,11 @@ The worker count above was used on a 16-core CPU; measure it for your CPU.
 The 4K context is a smoke-test starting point, not a model limit. The expert cache
 sizes itself automatically and leaves 1 GiB of VRAM headroom.
 
+**The card also drives a Linux desktop (#560 #516):** keep more VRAM free than the default 700 MiB, e.g.
+`./setup.sh --vram-reserve-mib 3072`. With the cache filling the card, the desktop's next VRAM need makes amdgpu move
+GPU memory to system RAM (GTT), the OOM killer then ends KWin/plasmashell or `systemd-oomd` ends apps, or the
+compositor fails with "Failed to pin framebuffer with error -12".
+
 The installer supports this backend (see "Install with setup" above). Images run through the CPU encoder for now (`--vision cpu`).
 Setup installs one AMD card, or several with `--gpus` (the engine's layer split; see RDNA4 below).
 
@@ -222,6 +234,10 @@ an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 39
   so none was shipped for it: there the plain hipBLAS path is already close. For hipBLASLt 1.5.0 (ROCm
   10.2.0a nightly) `tools/hip/gfx1201-hipblaslt-100500.txt` is shipped (see "Tuning table" below); on one R9700 it
   measured +3.9% prompt speed on 4,210-token prompts (1,590 vs 1,531 tok/s), a modest gain.
+  With the hipBLASLt 1.2.2 of a system ROCm 7.2.4 the plain path is far off, and `tools/hip/gfx1201-hipblaslt-100202.txt`
+  is shipped for it (24 of the gfx1100 table's 26 shapes; setup uses it only with that exact version): on an R9700
+  with the full IQ3_XXS, `--kv int8 --kv-resident 32768`, a fresh 32K prompt read at 638 -> 1,177 tok/s and a 7K one
+  at 656 -> 1,164 tok/s with it, decode unchanged.  `hip_prefill_hipblaslt_gemm` passes with it.
 - **Both cards in one run (layer split, engine 0.1.30):** the config's `"backend": "hip", "gpu": [1, 0]` (R9700
   first) runs through `serve/server.py` (setup writes it with `--gpus 1,0` since 0.1.31). Auto split put layers 0-27 on the R9700 and
   28-47 on the 9070 XT. With every expert on the GPUs the split gives exactly the tokens of the R9700 alone (4K and
@@ -267,8 +283,9 @@ an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 39
 
 ## Community-validated cards
 
-Run by their owners, not on the maintainers' machines; setup accepts them like gfx1100 / gfx1201. No hipBLASLt table
-is shipped for them (make one with [Tuning table](#tuning-table) and compare the prompt speed with and without it).
+Run by their owners, not on the maintainers' machines; setup accepts them like gfx1100 / gfx1201. setup ships a
+hipBLASLt table for gfx1200 (the numbers below); gfx1101 has none (make one with [Tuning table](#tuning-table)
+and compare the prompt speed with and without it).
 
 - **gfx1101, RX 7800 XT 16 GB** (jhohertz, #254; engine 0.1.29, Ryzen 9 5950X, 121 GiB RAM, system ROCm with
   hipBLASLt 1.4.1): `./setup.sh --backend hip` detected the card and compiled the engine; `strata-device --selftest`
@@ -288,6 +305,17 @@ is shipped for them (make one with [Tuning table](#tuning-table) and compare the
   Greedy output was the same across runs. For comparison, llama.cpp's HIP build measured 20 tok/s decode and
   450 tok/s prompt on that card.
 
+  Since 0.1.38 setup ships that table (`tools/hip/gfx1200-hipblaslt-100202.txt`). On engine 0.1.31, one binary (ctest
+  37/37, without `ple_parity` and `platform_memory_test`), prompts of repeated code blocks, prefill 2560 for
+  the short prompt and 16384 with `--kv int8 --kv-resident 65536` otherwise, two runs each - prompt speed
+  with the table vs plain hipBLAS (no table):
+
+  | prompt | with the table | plain hipBLAS | gain |
+  |---|---|---|---|
+  | 2,374 tokens | 597-599 tok/s | 396-398 tok/s | 1.50x |
+  | 65,045 tokens | 754-757 tok/s | 380 tok/s | 1.99x |
+  | 130,091 tokens (the same flags) | 719 tok/s | 363-370 tok/s | 1.98x |
+
 ## RDNA2 (gfx1030)
 
 The RX 6800 / 6800 XT / 6900 XT / 6950 XT run the same kernels: wave32 and 64 KiB of LDS per workgroup. The one

docs/DETAILS.md

@@ -19,6 +19,15 @@ RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1
 MTP speculative decoding on. "262K" is the model's full context window (a 259,943-token prompt). The IQ2_XS row was
 measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
 
+**Engine 0.1.36 (#136), the same PC:** Q2_0's prompt experts run on fused int8 tensor-core kernels (RTX 30 and newer):
+4K 1,294 -> 1,570, 32K 2,170 -> 2,653, 128K 2,123 -> 2,468 tokens/s (+16-22%), as close to an FP16 reference as the
+previous kernels (closer at 32K: teacher-forced KL 0.009 vs 0.012). The decode path's block selection and greedy
+argmax run on thread-block clusters (RTX 50, sm_90+; other cards keep the previous kernels; the same tokens): Q2_0 output at 4K 89 -> 93.5, at 128K
+64.5 -> 76.4 tokens/s. `STRATA_PF_FUSED=0` keeps the previous prompt kernels (byte-identical answers to 0.1.35);
+`STRATA_PF_FUSED=1` also runs the native IQ packs' fused kernels (opt-in: IQ2_XS prompts +12% at 4K, +3% at 32K, the
+IQ3 packs about even); `STRATA_QSA_CLUSTER=0` / `STRATA_ARGMAX_MULTI=0` turn the decode kernels off. The tables
+below are 0.1.26's.
+
 ### Prompt processing (tokens/s)
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
@@ -98,7 +107,10 @@ English/code subset from before (40,525 ids, ~110 MiB less VRAM, English answers
 almost no drafts). `--draft-vocab cyrillic` takes the English/code subset plus the whole Cyrillic script (58,963
 ids): the shipped subsets hold 142 of the vocabulary's 18,580 Cyrillic tokens, so Ukrainian or Russian answers got
 1.4 tokens a round; with it 2.1, and 83 -> 109 tokens/s (RTX 5090, the NVFP4 fork), English unchanged.
-`tools/draft_vocab.py` builds and inspects subsets.
+`tools/draft_vocab.py` builds and inspects subsets. When the start stops with "the draft head does not fit" (a
+12 GB card with a long context, #474), the engine says how much the head needs, how much VRAM is free and which
+smaller subset fits, and the server's start error repeats it; setup suggests `--draft-vocab en` on cards under
+14 GB (only a suggestion: nothing changes unless you pass it).
 
 **Low-RAM mode (engine 0.1.26, chosen by setup):** normally all of a model's experts are copied into RAM (23-50 GB,
 pinned) and the GPU holds a copy of the most-used ones. On a PC whose RAM cannot hold them beside the system (the
@@ -393,7 +405,8 @@ server, three server options (all off by default; also as keys in `strata-<model
 | `--min-free-vram-mib 11000` | `"min_free_vram_mib": 11000` | load an unloaded model only when that much VRAM is free (it waits up to 15 s for memory being given back), else answer **503** "the GPU is in use by another program" instead of starting into what a game left (with several GPUs it checks the first one) |
 | `--before-load "cmd"` | `"before_load": "cmd"` or `["cmd", "arg"]` | a command run before the model is loaded again, e.g. one that unloads another server's model |
 
-`POST /unload` unloads it now (`409` while a request is running) and `POST /load` loads it ahead of a request;
+`POST /unload` unloads it now (`409` while a request is running) and `POST /load` loads it ahead of a request (both
+with `Content-Type: application/json`, e.g. `curl -X POST -H "Content-Type: application/json" localhost:8080/unload`);
 `/health` says `"loaded"`, `/v1/models` lists it as `unloaded` (like llama.cpp's router), `/props` sets
 `is_sleeping` and the Monitor shows the state. Unloading ends the engine process - and the image encoder, when images
 are on; it is started again first, as at a start - so their VRAM and RAM go straight back. The model files stay in
@@ -401,6 +414,17 @@ the OS file cache, so loading again takes seconds while that RAM is not needed e
 16 GB with Q2_0 in the low-RAM mode: unloading takes ~0.3 s, and a request to an unloaded model answered after
 4.6 s (text) or 14.7 s (a picture, image encoder on the CPU).
 
+**Keep what the expert cache learned across restarts (opt-in, engine 0.1.36, #477):** a start fills the GPU's expert
+cache from the shipped profile, and the adaptive tier (`--adapt-every`) then moves in the experts your requests use.
+With `"expert_profile_save": "expert-profile-learned.bin"` in `strata-<model>.json` the engine saves that as a
+profile - the experts in VRAM first, then the routing it counted since the start, then the shipped order - on a
+clean exit and every 10 minutes between requests (`"expert_profile_save_every": 5` for another interval, `0` for
+exit only), written to a temporary file and renamed, so a crash never leaves half a file. The next start begins from
+it instead of the config's `--expert-profile` when it is a profile of the same model (else from the config's, as
+before). A relative path is in the Strata folder; one file per model, and a profile per project works the same way
+(point the key at another file). The file is a fingerprint of what you used the model for: it stays on your PC.
+Without the key nothing is counted or written. Setup rewrites the config when run again: add the key again then.
+
 ---
 
 ## Using it
@@ -459,6 +483,40 @@ print(r.choices[0].message.content)
   15 s, and `GET /status` says what it is doing (`reading the prompt`, `answering`, tokens so far). Closing the
   connection or pressing stop in your app really stops the model, so the next request starts at once.
 - **Chat apps.** Any app with an "OpenAI-compatible" provider works: base URL `http://127.0.0.1:8080/v1`, any API key.
+- **OpenCode** (#543). A starting point for `opencode.jsonc` (in your project, or `~/.config/opencode/`); the field
+  names are OpenCode's, so check its config docs if your version differs:
+
+  ```jsonc
+  {
+    "$schema": "https://opencode.ai/config.json",
+    "provider": {
+      "strata": {
+        "npm": "@ai-sdk/openai-compatible",
+        "name": "Strata (local)",
+        "options": { "baseURL": "http://127.0.0.1:8080/v1", "apiKey": "none" },  // or your api_key
+        "models": {
+          "strata": {
+            "name": "Qwen3.8-Flash-Next (Strata)",
+            // context: what you chose in setup; output: what one reply may use (prompt + output must fit)
+            "limit": { "context": 262144, "output": 32768 },
+            "options": { "reasoningEffort": "high" },                  // sent as reasoning_effort
+            "variants": {                                              // switch between them in OpenCode
+              "low": { "reasoningEffort": "low" },
+              "medium": { "reasoningEffort": "medium" },
+              "none": { "reasoningEffort": "none" }
+            }
+          }
+        }
+      }
+    },
+    "model": "strata/strata"
+  }
+  ```
+
+  Set `limit.context` to the context you chose in setup: OpenCode compacts the conversation before it gets there.
+  Keep `limit.output` well under it: a request whose prompt plus `max_tokens` runs past the context is refused (see
+  **Context** below), or add `"fit_max_tokens": true` to `strata-<model>.json`. For a hard cap on the thinking, add
+  `"reasoning_budget_tokens": N` to `strata-<model>.json` (see above).
 - **Claude Code** (Strata 0.1.17 or newer): set `ANTHROPIC_BASE_URL=http://127.0.0.1:8080` and
   `ANTHROPIC_MODEL` to a Claude model name it knows (it refuses names it doesn't; Strata ignores the name), plus any
   `ANTHROPIC_AUTH_TOKEN` (or your `api_key`, if you set one).
@@ -482,9 +540,30 @@ print(r.choices[0].message.content)
   clients then send it as their API key. Streamed answers carry `X-Accel-Buffering: no`, so nginx-style proxies pass
   each token on at once. The web app's settings and MCP tools only answer Strata's own page: when you open it through
   a proxy or tunnel whose address differs, add that address, e.g. `"trusted_origins": ["https://strata.example.com"]`.
+  With the key set, any `Host` name reaches the server (see Host names below).
 - **From web apps in a browser (CORS).** Off by default. `"cors_origins": ["https://chat.example.com"]` lets pages of
   those origins call `/v1/*` from the browser (Open WebUI's direct connections, browser extensions); `["*"]` lets any
   page do it - only sensible with an API key. It never opens `/settings`, `/unload` or the MCP tools.
+- **Host names (DNS rebinding).** A web page of another site can point its own name at `127.0.0.1` and then reach
+  this server as if it were its own, so without an API key the server answers only requests whose `Host` is a name
+  it knows (with a key the check is off: such a page cannot send the key, and tunnels and proxies that pass their
+  own name on keep working):
+  `localhost` (and `*.localhost`), any IP address (`127.0.0.1`, `[::1]`, `192.168.x.x`, ...), the address it
+  listens on and, when it listens beyond this PC (`0.0.0.0` or a LAN address), this PC's name (`mypc`, `mypc.local`)
+  and `host.docker.internal`; any port. Others get **403** naming the setting, and the server window prints one line
+  for each. Reach it under another name (a reverse proxy that keeps the name, a tunnel, a DNS name on your network,
+  another container's name for it)? Add the name: `"allowed_hosts": ["strata.example.com"]` in
+  `strata-<model>.json` or `STRATA_ALLOWED_HOSTS=strata.example.com` (comma-separated); `".example.com"` allows that
+  name and every name below it, and `["*"]` turns the check off (so does setting `api_key`). The hosts of
+  `trusted_origins` count as allowed. Requests without a `Host` header (HTTP/1.0 clients) pass.
+- **Web pages without an API key.** Without `api_key`, a `POST` to `/v1/*` that carries an `Origin` header (a
+  browser page sent it) is answered only for Strata's own page, pages on `localhost` or an allowed host name (any
+  port), the origins in `trusted_origins` or `cors_origins`, and browser extensions and desktop apps
+  (`chrome-extension://`, `moz-extension://`, `app://`: no web site can send those), and only with a JSON body; any
+  other page, and `Origin: null`, gets **403**. Clients that send no `Origin` (curl, the OpenAI and Anthropic SDKs,
+  other servers) are not affected. With
+  an API key, the key decides. `POST /unload` and `POST /load` take `Content-Type: application/json` from Strata's
+  own page (or no `Origin`), like `/settings`.
 
 **Conversation cache.** A request that continues a chat reads only the part after what the engine already holds: the
 live session, or one of the checkpoints it keeps in RAM (up to 6, ~118 MB each, taken at the start of each new
@@ -793,6 +872,7 @@ the document, +0.4% on the chat. Details: `bench/results/2026-09-27-esp/`.
 | Pictures are slow (10-30 s) | The encoder runs on the CPU: run setup again with `--vision gpu` (needs ~1.4 GB of VRAM). |
 | A request never finishes: "reading the prompt", GPU "100%" at low power | The GPU ran out of VRAM (engines before 0.1.9 could end with ~30 MiB free at large contexts). Run `START-HERE.bat` once to get engine 0.1.9 or newer; the log then says `... MiB of VRAM free with everything loaded` (a few hundred) and names the `--vram-reserve-mib` to add if it is low. |
 | Generation stops mid-answer, GPU "100%", one CPU core busy | Fixed in engine 0.1.12 (issue #29, a race in the CPU expert pool on big-VRAM cards). Since then a request that stops moving ends with an error instead of hanging (after 2 minutes; 1 minute from 0.1.13): the log says `no progress for ... s ... (issue #29)` with where it stopped, and the next request starts the engine again. If you see that line, please open an issue with it. Engine 0.1.13 adds a stall report under it (what every expert-pool thread and the GPU handshake were doing, memory and page faults) and, on Windows, a `strata-stall-<pid>.dmp` file with every thread's stack: attach both. (`STRATA_WATCHDOG_S` sets the time in seconds; 0 turns it off.) Engine 0.1.14 fixes the stall those reports found (issue #31: with the IQ packs the host could wait forever inside the NVIDIA driver while copying experts in a verify window; the experts are now copied by a GPU kernel, `--pcie-mode dma` restores the old way). |
+| `the engine said nothing for ... s during the request` or `... did not finish the request after it was stopped (STOP)` | Issue #481: the engine and the server lost step (the engine waits for its next command, the server for the request's end; GPU at 0 %, nothing in the log). The server ends the engine after 300 s without a line from it during a request (while a prompt is read: each chunk may take three times the previous one's time, the first one up to its tokens at 50 tok/s more), the request ends with an error and the next request starts the engine again. `"engine_silence_s": 600` in `strata-<model>.json` sets the time (0 = wait forever, as before). If you see it, please add the end of the engine log to #481. |
 | `out of memory: cudaFuncSetAttribute` in the log (IQ3_XXS, long prompt) | Fixed in engine 0.1.15: CUDA loaded a kernel's code when it was first needed, and mid-prompt there was no VRAM left for it. Run `START-HERE.bat` (Windows) or `./setup.sh` (Linux) once to update. |
 | Anything else | The engine log is `strata-<model>.log` in this folder. |
 

docs/INSTALL.md

@@ -123,11 +123,20 @@ The same idea, in a container (NVIDIA cards).
 
 ## Updating
 
-Download the new version and unzip it anywhere (or `git pull`), then run `START-HERE.bat` (Linux: `./setup.sh`) in
-it. The model files are kept in a `Strata-data` folder next to your Strata folder, so a new copy finds them and sets
-itself up the same way - nothing big is downloaded again. On Linux after a `git pull`, setup compiles the engine
-again when its source changed (a minute or two for the changed files); if that compile fails, it says so and starts
-the engine you had.
+**`UPDATE.bat`** (Linux: `./update.sh`) updates Strata without starting the model - for when the GPU is busy with
+something else, or you just want the new version ready. In a `git clone` it runs `git pull`, then does what
+`START-HERE.bat` does before a start: the engine (a new ready-made one when the new version needs it; on Linux a
+compiled engine is compiled again when its source changed), the Python packages, and each installed model's settings
+and draft subset. The model files are not touched (at most a new engine is downloaded) and no question is asked.
+Close the model's window first (a running engine cannot be replaced); start the model later with `START-HERE.bat` as
+usual. In a copy that was downloaded as a zip it says to download the new zip (below): it cannot fetch new files
+itself.
+
+Or by hand: download the new version and unzip it anywhere (or `git pull`), then run `START-HERE.bat` (Linux:
+`./setup.sh`) in it. The model files are kept in a `Strata-data` folder next to your Strata folder, so a new copy
+finds them and sets itself up the same way - nothing big is downloaded again. On Linux after a `git pull`, setup
+compiles the engine again when its source changed (a minute or two for the changed files); if that compile fails, it
+says so and starts the engine you had.
 
 ## Where things are stored
 
@@ -177,11 +186,37 @@ START-HERE.bat --data-dir E:\Strata-data         keep the model files somewhere
 START-HERE.bat --port 8081                      another port
 START-HERE.bat --gpu 1                          another GPU (setup picks the one with the most VRAM)
 START-HERE.bat --gpus 0,2                       several GPUs sharing the model
+START-HERE.bat --vram-reserve-mib 2048          leave 2 GB of VRAM free for other programs (remembered)
 START-HERE.bat --setup --backend hip            the AMD engine on a PC that also has an NVIDIA card
 START-HERE.bat --setup --host 0.0.0.0 --api-key <secret>     reachable from other devices, with a key
 START-HERE.bat --calibrate                      tune the engine for this PC (about 5-10 minutes), then start
 START-HERE.bat --check                          only check this PC
 ```
 
+**Leaving VRAM for other programs (#493):** Strata fills the graphics card's free VRAM with experts (the expert cache)
+and leaves `--vram-reserve-mib` MiB free: 700 by default. For a game, a 3D program or another model beside it, leave
+more: `START-HERE.bat --vram-reserve-mib 2048` (Linux: `./setup.sh --vram-reserve-mib 2048`) writes it into the
+model's `strata-<model>.json` and starts it; at setup (`--setup --vram-reserve-mib 2048`) it goes into the new config.
+By hand: add `"--vram-reserve-mib", "2048"` to the config's `"args"` list and restart. The expert cache is then that
+much smaller, so answers can be a little slower. The engine sizes its cache from the VRAM free when it starts: what
+another program already holds then is left alone anyway; the reserve is room for what it takes later.
+
+**An AMD card that also drives a Linux desktop (#560 #516):** with the default reserve the expert cache fills the
+card, and when the desktop (the compositor, a browser, a new app) needs more VRAM, the amdgpu driver moves GPU memory
+to system RAM, which the model's experts already fill: the OOM killer then ends the desktop session (KWin, plasmashell)
+or the compositor fails ("Failed to pin framebuffer"). `./setup.sh --vram-reserve-mib 3072` fixed it in both reports
+(about 2.3 GB fewer experts in VRAM, a few % of speed). Setup and the server window say so on such a PC.
+
+**A card under 8 GB (#496):** when the default reserve leaves the expert cache too little room, the engine lowers the
+reserve (down to 300 MiB) until it fits, and warns if the card then ends nearly full. If the start stops with "no VRAM
+is left for the expert cache", that log line says how much is short; an 8K context and `--draft-vocab en` at setup
+free the most.
+
+**Model files downloaded by hand, or from a mirror (#495):** setup's step 5 prints the folder it expects them in
+(`Strata-data\models\<SIZE>\`, e.g. `Strata-data\models\IQ3_XXS\`): put them there with their original names, or
+point setup at them with `--gguf-dir`. To let setup download from a Hugging Face mirror itself, set `HF_ENDPOINT`
+first (Windows: `set HF_ENDPOINT=https://hf-mirror.com`, Linux: `export HF_ENDPOINT=https://hf-mirror.com`): the same
+pinned revisions and checks apply, and the MTP draft layer comes from there too.
+
 On Linux the same options go to `./setup.sh`. `START-HERE.bat --help` lists them all. The server's own settings
 (sharing the GPU with games, MCP tools, CORS, API keys, the API itself) are in the [details](DETAILS.md#using-it).

docs/MODELS.md

@@ -11,12 +11,14 @@ The installer recommends one for your PC; this page explains the choice. Back to
 
 | Your RAM | Take | Why |
 | --- | --- | --- |
-| **32 GB** | **Coder** | it fits 32 GB, and it is made for code (with a 24 GB card, Q2_0 and IQ2_XS run too: [low-RAM mode](#a-big-graphics-card-and-little-ram)) |
+| **32 GB** | **Coder** | it fits 32 GB; best at code, weaker at everything else ([why](#coder)). With a 24 GB card, Q2_0 and IQ2_XS run too ([low-RAM mode](#a-big-graphics-card-and-little-ram)) and are the better pick for general use |
 | **48 GB** | **IQ2_XS** (or Q2_0, the fastest) | the larger sizes do not fit |
 | **64 GB** | **IQ2_XS** (recommended), or IQ3_XXS / IQ3_S | every size fits (IQ3_S with little else open) |
 | **96 GB or more** | **IQ3_S**, or [Unsloth's 4-bit](#unsloth-ud-q4_k_xl-experimental) (experimental) | room for the largest sizes |
 
-Not sure? Take **IQ2_XS** - or the **Coder** if you mainly write code, or have 32-48 GB of RAM.
+Not sure? Take **IQ2_XS**. The **Coder** is the one that fits a 32 GB PC, but it keeps only 256 of the 512 experts,
+chosen on code data, so it is weaker outside code and in languages other than English. For general use, or whenever
+your RAM allows, take a full model (Q2_0, IQ2_XS or IQ3_S).
 
 ## How fast is each size
 
@@ -24,7 +26,7 @@ Measured on an RTX 5070 (12 GB), a Ryzen 5 7600 and 64 GB of RAM:
 
 | Size | Writes answers (short chat) | Writes answers (128K context) | Reads your prompt |
 | --- | ---: | ---: | ---: |
-| **Q2_0** | 93 tokens/s | 74 tokens/s | 2,170 tokens/s |
+| **Q2_0** | 94 tokens/s | 76 tokens/s | 2,650 tokens/s |
 | **IQ2_XS** | 79 tokens/s | 63 tokens/s | 2,090 tokens/s |
 | **IQ3_XXS** | 62 tokens/s | 49 tokens/s | 1,750 tokens/s |
 | **IQ3_S** | 53 tokens/s | 46 tokens/s | 1,620 tokens/s |
@@ -91,12 +93,12 @@ The original, in all four sizes. With images, and with the
 ### Coder
 
 **[Coder](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF)** - ISTA-DASLab's coding version:
-half of the experts removed, keeping the ones that code, tool use and images need (91% of the full model's SWE-bench
-Verified score, 99% of LiveCodeBench, by its authors). One size (IQ1_M: its experts stored like IQ3_S): shard 1 is
-**29.6 GB**, so it fits a PC with **32 GB of RAM**, runs 262K context on 64 GB, and reads long prompts the fastest of
-all. It is weaker outside code, and that includes Chinese and other CJK text (#438: Chinese answers came out wrong
-or looping where English was fine). For general chat or CJK text, take a size that keeps every expert: Q2_0,
-IQ2_XS or IQ3_S.
+it keeps 256 of each layer's 512 experts, chosen on code (with agentic and image) data, and drops the rest (91% of
+the full model's SWE-bench Verified score, 99% of LiveCodeBench, by its authors). One size (IQ1_M: its experts stored
+like IQ3_S): shard 1 is **29.6 GB**, so it fits a PC with **32 GB of RAM**, runs 262K context on 64 GB, and reads long
+prompts the fastest of all. With half the experts gone it is weaker outside code and in languages other than
+English, Chinese and other CJK text included (#438: Chinese answers came out wrong or looping where English was
+fine). For general use, or whenever your RAM allows, take a size that keeps every expert: Q2_0, IQ2_XS or IQ3_S.
 More: [details](DETAILS.md#or-the-coder-half-the-experts-for-code).
 
 ```

docs/MULTI_GPU.md

@@ -8,7 +8,8 @@ speed comes from (decode then barely touches the CPU pool).
 
 This is pipeline (layer) parallelism, not tensor parallelism: a token crosses from one card to the next once per
 verify window (a few hundred KB through pinned RAM), not twice per layer. No NVLink or peer-to-peer access is
-needed; cards on x4 or x1 slots work, and the PCIe share of each card is probed on its own link.
+needed; cards on x4 or x1 slots work, and the PCIe share of each card is probed on its own link. A `--pcie-frac` you give
+is every card's share and skips those probes; there is no per-card setting yet.
 
 ## Using it
 

docs/SECOND_GPU.md

@@ -83,3 +83,42 @@ short decode requests.
 The existing warning about the CUDA0 expert-cache GPU hit path still applies:
 its outputs diverge from cache-off runs. Treat performance as experimental
 until the generated tokens have been validated.
+
+## Peer tier (`--peer-device`)
+
+`--peer-device N` puts a second adaptive expert cache on CUDA device N. It
+takes the ranked pairs CUDA0's cache does not hold, as many as fit. The peer
+computes the rows of its own experts, for decode windows and for prompt
+chunks; the activations and the results cross NVLink or another P2P path. The
+tier adapts while the server runs, like the primary cache. It is an
+alternative to the CUDA1-3 caches above, not a third tier beside them.
+
+- `--peer-device N` (default off): enable the tier on CUDA device N (N >= 1).
+- `--peer-reserve-mib M` (default 600): leave M MiB free on the peer card; the
+  cache takes what remains. The prompt-path buffers need this headroom.
+- `--peer-slots N` (default 0): cap the tier at N experts; 0 = as many as fit.
+- `--peer-adapt-swaps N` (default -1): swaps per adaptive round on the peer;
+  -1 uses the primary's `--adapt-swaps`.
+- `--peer-prefill-rows N` (default -1): the share of each prompt chunk's rows
+  the peer computes; -1 is half of chunk x top-k, 0 keeps prompt rows on the
+  primary.
+
+`--peer-device` requires `--expert-profile` and an enabled expert cache, and
+the device must be visible; it refuses otherwise. It also refuses
+`--layer-split` (a different second-GPU mode: use one or the other) and
+`--expert-cache-device1..3` (the peer tier already caches experts on that card):
+
+    strata generate: --peer-device cannot be combined with --layer-split (use one or the other)
+    strata generate: --peer-device cannot be combined with --expert-cache-device1..3 (the peer tier already caches experts there)
+
+Without `--peer-device` the binary is unchanged; its output is byte-identical
+to the release. With `--peer-device` and the same expert set split across the
+two cards, the generated tokens are byte-identical to the single-GPU run under
+the exactness gate.
+
+With a peer the prompt path keeps the MMQ path; the fused int8 prompt path is
+not yet combined with the peer's rows. Mapped host buffers gain
+`cudaHostAllocPortable` only with a peer, since only then does a second
+context write them. The tier size is manual for now (`--peer-reserve-mib`,
+`--peer-slots`); automatic sizing on small cards wants the buffer lending of
+#216 and is a follow-up.

docs/UNSLOTH_Q4.md

@@ -25,8 +25,16 @@ What setup does differently for this model:
 - It needs 48 GB of RAM or more (with less it asks, default no; `--model UD-Q4_K_XL --yes` installs it anyway) and
   engine 0.1.32 or newer (checked before anything is downloaded), and an NVIDIA
   GPU: it has not been run on AMD cards (its prompt kernels for the Q4_K / Q5_K experts are NVIDIA-only), so with
-  `--backend hip` setup says so and asks before the download (#429; `--model UD-Q4_K_XL --yes` tries it). One GPU only: with `--gpus` it uses the first one and says so. No images (the vision encoder is not wired to
-  this file yet) and no experimental speed projection (not tested with it).
+  `--backend hip` setup says so and asks before the download (#429; `--model UD-Q4_K_XL --yes` tries it). One GPU
+  by default: the RAM budget below has no layer split (the engine refuses `--resident-budget-gib` with one). No
+  images (the vision encoder is not wired to this file yet) and no experimental speed projection (not tested with it).
+- Several GPUs (#498): when the RAM holds the GGUF files and 24 GB more (~135 GB of RAM) and two or more cards can
+  share it, setup asks (one GPU stays the default; `--gpus 0,1` takes the split). The split runs **without** the RAM budget: all 77 GB of experts are loaded into RAM from the GGUFs at
+  start, the files pass through the OS file cache while they load, and the config gets `"gpu": [0, 1]` and
+  `"layer_split": "auto"`. Measured on 2x RTX 3090 with 165 GiB (#498): decode 31 tok/s on one card with the budget,
+  64-78 tok/s split (55 tok/s at a 128K prompt), with `MemAvailable` never under 68 GiB. With less RAM, `--gpus`
+  keeps one GPU and says so, and `START-HERE.bat --gpus 0,1` on an installed UD-Q4_K_XL stops with the reason
+  (it used to keep the budget, and the engine exited with code 2).
 - It downloads the four shards below from the pinned revision `38bb39e` (resumable, like the other models), then
   checks each one's size and SHA-256 against the table below; the check takes a few minutes once and is remembered
   in the file's finish mark. A file with the wrong hash is deleted, so the next run downloads it again.

include/strata/core/expert_cache.hpp

@@ -55,6 +55,24 @@ inline constexpr int32_t kNotResident = -1;
 bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_expert,
                          std::vector<std::pair<int32_t, int32_t>>& ranked, int64_t& slots, std::string& err);
 
+/// #477 (--expert-profile-save): what the adaptive tier learned, as a profile ranking EVERY (layer, expert) pair -
+/// the experts resident in VRAM now first (where the swaps left the cache), then the rest; within each, by `heat`
+/// (the routing the adaptive tier counted since the start, descending), then by `prior` (the profile the engine
+/// started from: an expert this run never routed keeps its old place), then by index.  A start from it begins
+/// where this one ended; one with more slots adds the hottest of the rest, one with fewer keeps the hottest.
+/// `resident` and `heat`: n_layers x n_expert entries (`heat` may be empty: no counts, the prior decides).
+std::vector<std::pair<int32_t, int32_t>> rank_learned_profile(int64_t n_layers, int64_t n_expert,
+                                                              const std::vector<uint8_t>& resident,
+                                                              const std::vector<double>& heat,
+                                                              const std::vector<std::pair<int32_t, int32_t>>& prior);
+
+/// #477: writes `ranked` in tools/make_profile.py's format, byte for byte: `STRP`, `1, n_layers, n_expert,
+/// n_ranked, n_ranked` (uint32), the pairs (uint16 layer, uint16 expert), then the n_layers x n_expert int32 table
+/// of each pair's rank (-1: not ranked).  Atomically: a `<path>.tmp` beside it, renamed over `path` once complete,
+/// so a reader (the next start) never sees half a file.  False with `err` when it could not be written.
+bool write_expert_profile(const std::string& path, int64_t n_layers, int64_t n_expert,
+                          const std::vector<std::pair<int32_t, int32_t>>& ranked, std::string& err);
+
 class ExpertCache {
 public:
     ExpertCache() = default;

include/strata/core/expert_source.hpp

@@ -43,6 +43,7 @@ struct ExpertLayout;
 
 namespace strata::core {
 
+class PeerExperts;   // multi-GPU: the second GPU's expert tier (peer_experts.hpp)
 class RemoteExperts;
 struct LoadStats;
 
@@ -336,6 +337,9 @@ struct ExpertDispatch {
     std::vector<float> usage;
     int64_t multi_misses = 0;      ///< distinct (layer, expert) pairs the CPU computed in verify windows
     int64_t multi_entries = 0;     ///< routed (token, expert) entries the CPU served in verify windows
+    /// Multi-GPU: the second GPU's tier.  Its experts are computed there instead of on the CPU (kind 2).
+    PeerExperts* peer = nullptr;
+    int64_t peer_entries = 0;      ///< routed entries the peer served in verify windows
     /// Set when `dispatch` could not produce an answer.  The loop itself has no error channel, so this is
     /// where a source failure surfaces: the driver checks it after `session_loop` returns rather than the
     /// engine computing from a half-filled `parts`.
@@ -471,6 +475,16 @@ public:
     double file_ms() const { return (double) file_us_.load(std::memory_order_relaxed) / 1000.0; }
     /// Threads `prefetch` reads the GGUF with (STRATA_FETCH_THREADS, default 8).
     void set_fetch_threads(int n) { fetch_threads_ = n < 1 ? 1 : n; }
+    /// #286 (Windows): read the experts straight from the drive (FILE_FLAG_NO_BUFFERING, overlapped) instead of
+    /// through the mapped files - the GGUF in place, or a pack's experts.bin - when the file cache could not keep them
+    /// beside `ram_bytes` (the RAM budget), cached now or not (their mapped pages would land in the working set); see
+    /// experts_unbuffered.  The mapped reads' page faults are one small request each, and the pages they bring in
+    /// take the RAM the budget was sized for.  `why` says what decided.
+    bool set_unbuffered(uint64_t ram_bytes, std::string& why);
+    bool unbuffered() const { return !direct_.empty(); }
+    /// #286, unbuffered: assembles the blobs of these pairs ahead of the `blob` calls that will ask for them (the
+    /// GPU cache's fill from the profile) - one batch of reads instead of one blob at a time.  At most 64 pairs.
+    void prefetch_pairs(const std::pair<int32_t, int32_t>* pairs, int64_t n);
 
     const uint8_t* blob(int64_t layer, int64_t expert) override;
     bool pinned(int64_t layer, int64_t expert) const override;
@@ -485,7 +499,8 @@ public:
     /// CS-T: the GGUF in place asks the OS for the predicted experts' pages (PrefetchVirtualMemory on Windows,
     /// madvise(WILLNEED) elsewhere), skipping the RAM copy's.
     void warm(int64_t layer, const int64_t* experts, int64_t n) override;
-    bool warms() const override { return !role_ptr_.empty(); }
+    /// Not when the reads are unbuffered: the warmed pages would be read through the file cache, a second time.
+    bool warms() const override { return !role_ptr_.empty() && direct_.empty(); }
     /// Of the blobs the file tier read for the decode, how many had been warmed for their layer beforehand.
     int64_t warmed_hits() const { return warm_hits_.load(std::memory_order_relaxed); }
     int64_t warmed() const { return warm_count_.load(std::memory_order_relaxed); }
@@ -502,6 +517,18 @@ private:
     const uint8_t* staged_blob(int64_t layer, int64_t expert);
     bool claim_stage(int64_t key, size_t& v, bool& fill);
     bool fill_stage(size_t v, int64_t layer, int64_t expert, uint8_t* dst);
+    void publish_stage(size_t v, int64_t layer, bool ok, double us);
+    struct Fill { size_t v; int64_t layer, e; uint8_t* dst; };
+    /// The claimed buffers' blobs: one overlapped batch when unbuffered, else the fetch threads' mapped copies.
+    void fill_many(const std::vector<Fill>& todo);
+    /// #286: the blobs from the drive, unbuffered: every role's 4 KiB-aligned window is read at once (overlapped)
+    /// into this thread's aligned buffer, then copied into place.  False when a read fails.
+    bool read_direct(const Fill* fills, size_t n) const;
+    std::vector<std::string> paths_;          ///< the mapped files, as maps_
+    std::vector<void*> direct_;               ///< #286: per file, an unbuffered overlapped handle (Windows)
+    std::vector<int> role_file_;              ///< 3 x n_layers: index into maps_ / direct_
+    /// blobs assembled in the stage buffers (`blob` hands those out): the GGUF in place, or any unbuffered source
+    bool staged() const { return !role_ptr_.empty() || !direct_.empty(); }
     static constexpr uint64_t kNoComplement = detail::kNoCacheComplement;
     // ---- CS-T: the GGUF shards in place
     std::string gguf_;
@@ -648,7 +675,8 @@ private:
 /// layout's type and dimensions, and inside the file.  `native` is the --native shard (see set_gguf).
 bool check_experts_gguf(const std::string& native, const strata::kernels::cpu::ExpertLayout& lay, std::string& err);
 /// Fills `dst` (lay.total bytes, the experts.bin layout) from the GGUF files, one role at a time.
+/// `unbuffered`: each chunk read past the file cache (Windows).
 LoadStats load_experts_gguf(const std::string& native, uint8_t* dst, const strata::kernels::cpu::ExpertLayout& lay,
-                            int threads);
+                            int threads, bool unbuffered = false);
 
 }  // namespace strata::core

include/strata/core/peer_experts.hpp

@@ -0,0 +1,106 @@
+// include/strata/core/peer_experts.hpp - multi-GPU: a second expert tier on another GPU.
+//
+// The primary GPU (device 0) keeps everything it had: the dense weights, the KV cache, the MTP layer, its own
+// adaptive expert tier and the captured verify graph.  This class adds an expert tier on a SECOND card.  From
+// the verify graph's point of view nothing changes: for each layer the pool thread decides which routed experts
+// the primary computes (its VRAM tier, the PCIe share), and every remaining expert is a "CPU row" the graph reads
+// from mapped host memory after the pool raises its flag.  The peer takes the rows whose experts it holds: the
+// pool thread launches them on the peer right after publishing the primary's plan, the CPU computes the rest at
+// the same time, and the peer's rows are written into the same mapped rows before the pool returns.
+//
+// The peer computes with the SAME kernels as the primary's hit path (q8_1 activations + native_expert_grouped),
+// from the same activation values, so an expert gives bit-identical rows on either card.
+//
+// The tier follows the conversation like the primary's: `adapt` swaps the most-routed experts that neither card
+// holds into the peer's least-used slots (per layer, so a slot keeps its layer's blob size).
+#pragma once
+
+#include "strata/core/expert_cache.hpp"
+#include "strata/core/expert_source.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace strata::core {
+
+/// multi-GPU: set once at startup when a peer device is configured (--peer-device N).  The mapped host buffers
+/// the peer card writes into are then allocated with cudaHostAllocPortable; without a peer the flag stays off.
+void set_peer_portable(bool on);
+bool peer_portable();
+
+class PeerExperts {
+public:
+    PeerExperts() = default;
+    ~PeerExperts();
+    PeerExperts(const PeerExperts&) = delete;
+    PeerExperts& operator=(const PeerExperts&) = delete;
+
+    /// Opens the tier on `device`: fills it with the first pairs of `ranked` that `primary` does not hold, as many
+    /// as fit in the card's free memory minus `reserve_mib` (`max_slots` > 0 caps the count).
+    bool open(int device, const std::vector<std::pair<int32_t, int32_t>>& ranked, const ExpertCache& primary,
+              ExpertSource& src, int64_t n_layers, int64_t n_expert, int reserve_mib, int64_t max_slots,
+              std::string& err);
+    void close();
+    bool valid() const { return device_ >= 0; }
+    int device() const { return device_; }
+
+    /// Whether the peer holds (layer, expert) right now.
+    bool has(int64_t layer, int64_t expert) const { return res_[(size_t) (layer * n_expert_ + expert)] >= 0; }
+
+    /// The device address of (layer, expert)'s blob on the peer, or null when it is not resident.
+    const uint8_t* slot_ptr(int64_t layer, int64_t expert) {
+        const int32_t sl = res_[(size_t) (layer * n_expert_ + expert)];
+        return sl >= 0 ? cache_.device_slot(sl) : nullptr;
+    }
+    /// Whether the primary (device 0) and the peer can read each other's memory directly (NVLink / PCIe P2P).
+    bool p2p() const { return p2p_; }
+
+    /// Starts the peer's share of one layer: the entries i with kind[i] == 2 (x: n_tok rows of n_embd floats).
+    /// multi-GPU: with `out` (pinned, portable host rows) the peer writes its rows straight into
+    /// out[entry] with a scatter kernel (no D2H copy + host memcpy in finish); STRATA_PEER_DIRECT=0: the old way.
+    bool launch(int64_t layer, const float* x, const int32_t* ids, int64_t n_tok, int64_t k, const int32_t* kind,
+                std::string& err, float* out = nullptr);
+    /// Waits for the launched share and writes its rows into `out` (row i = entry i, n_embd floats).
+    bool finish(float* out, std::string& err);
+
+    /// Adaptive tier: swap up to `max_swaps` experts routed at least twice that neither the primary (`res0`, slot
+    /// or -1) nor the peer holds into the peer's least-used slots.  Copies run on the peer's refill stream; a
+    /// swapped-in expert becomes resident at `apply_pending`.  Call only between verify windows.
+    bool adapt(const float* usage, const int32_t* res0, int max_swaps, std::string& err);
+    void apply_pending(bool wait);
+
+    int64_t resident() const { return resident_; }
+    /// the last launch writes its rows into the host rows itself (the caller must not touch them)
+    bool launched_direct() const { return launched_direct_; }
+    double gib() const { return cache_.gib(); }
+    int64_t entries() const { return entries_; }      ///< routed entries the peer computed
+    int64_t experts() const { return experts_; }      ///< distinct (layer, expert) the peer computed
+    int64_t swaps() const { return swaps_; }
+    double ms_wait = 0;                               ///< host time spent in finish() waiting for the peer
+
+private:
+    int device_ = -1;
+    bool p2p_ = false;
+    int64_t n_layers_ = 0, n_expert_ = 0;
+    ExpertCache cache_;
+    ExpertSource* src_ = nullptr;
+    std::vector<int32_t> res_;                        ///< [n_layers * n_expert] -> peer slot or -1
+    std::vector<std::pair<int32_t, int32_t>> pending_;   ///< (residency index, slot) once the copies land
+    int64_t resident_ = 0, entries_ = 0, experts_ = 0, swaps_ = 0;
+    cudaStream_t stream_ = nullptr, refill_ = nullptr;
+    cudaEvent_t refill_ev_ = nullptr;
+    float* h_x_ = nullptr;   float* d_x_ = nullptr;   // n_tok x n_embd
+    float* h_out_ = nullptr; float* d_out_ = nullptr; // compact rows
+    void* h_meta_ = nullptr; void* d_meta_ = nullptr; // groups: ptr | start | dst | tok | count
+    uint8_t* d_q8_ = nullptr;
+    void* d_scratch_ = nullptr;
+    std::vector<int32_t> row_of_;                     ///< compact row -> original entry
+    int64_t launched_rows_ = 0;
+    bool launched_direct_ = false;
+};
+
+}  // namespace strata::core

include/strata/core/pinned.hpp

@@ -104,6 +104,22 @@ LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_byte
 LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
                               const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk);
 
+/// The same ranges read UNBUFFERED straight into `dst` (Windows): no staging buffer and no file-cache copy - the
+/// drive's DMA lands where the experts live. Every range, `dst` and `chunk` must be 4 KiB aligned; returns ok =
+/// false with an empty `error` when they are not (or off Windows), and the caller falls back to load_experts_ranges.
+LoadStats load_experts_direct(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
+                              const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk);
+
+/// Whether the expert files are better read unbuffered (Windows; false elsewhere): a timed probe of random 64 KiB
+/// reads says they are not in the OS file cache (a cached read takes ~10 us, the drive ~80), and the RAM left
+/// beside the arena (`arena_bytes`) could not keep them cached for the next start either - so a warm restart after
+/// an idle unload never gets slower, and only a start that reads the drive anyway skips the cache's copy.
+/// STRATA_UNBUFFERED_LOAD=1 / 0 forces it. `why` says what decided.
+/// `cache_counts` false (the file tier with a RAM budget): only whether the files could be kept decides - their mapped
+/// pages land in the process's working set, so a partly cached file would still take the RAM the budget was sized for.
+bool experts_unbuffered(const std::vector<std::string>& files, uint64_t arena_bytes, std::string& why,
+                        bool cache_counts = true);
+
 // FNV-1a 64.  Per layer, so a corrupt or short read names WHICH layer rather than just failing a whole-file
 // comparison - the same reason the Phase 1 tools report the first differing element.
 uint64_t fnv1a64(const uint8_t* p, uint64_t n, uint64_t seed = 1469598103934665603ull);

include/strata/core/verify.hpp

@@ -254,6 +254,7 @@ private:
     int32_t* plan_ = nullptr;                                     // device copy of the plan block
     uint8_t* staging_ = nullptr;                                  // VRAM slots for the PCIe share of the misses
     static constexpr int64_t kStagingBlobs = 16;
+    static constexpr int64_t kPcieGroupRows = 4;                  // the PCIe call's groups side by side (of <= 16)
     uint8_t* hit_xq_ = nullptr;
     uint8_t* nat_xq_ = nullptr;   // plan v0.3 P6: q8_1 activations for a native pack's grouped experts
     float* hit_xs_ = nullptr;

include/strata/hip_compat/cuda_runtime.h

@@ -20,10 +20,13 @@
 #define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor
 #define cudaDevAttrComputeCapabilityMinor hipDeviceAttributeComputeCapabilityMinor
 #define cudaDeviceGetAttribute hipDeviceGetAttribute
+#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer
+#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess
 #define cudaDeviceProp hipDeviceProp_t
 #define cudaDeviceSynchronize hipDeviceSynchronize
 #define cudaDriverGetVersion hipDriverGetVersion
 #define cudaErrorNotReady hipErrorNotReady
+#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled
 #define cudaErrorStreamCaptureUnsupported hipErrorStreamCaptureUnsupported
 #define cudaError_t hipError_t
 #define cudaEventCreate hipEventCreate
@@ -62,10 +65,22 @@
 #define cudaLaunchHostFunc hipLaunchHostFunc
 #define cudaMalloc hipMalloc
 #define cudaMallocHost(...) (::strata::hip_compat::malloc_host(__VA_ARGS__))
+#if defined(_WIN32)
+// Windows: hipMemGetInfo counts this process's own allocations only, so the desktop's and other programs' share of
+// the card is invisible to it, and VRAM sized by it overfills the card (WDDM then moves memory out to system RAM and
+// the GPU slows down).  Its free figure is lowered by what Windows' video memory budget for this process withholds
+// (src/core/device.cu; STRATA_WDDM_BUDGET=0: hipMemGetInfo as it is).
+namespace strata::hip_compat {
+hipError_t mem_get_info(size_t* free_bytes, size_t* total_bytes);
+}
+#define cudaMemGetInfo ::strata::hip_compat::mem_get_info
+#else
 #define cudaMemGetInfo hipMemGetInfo
+#endif
 #define cudaMemcpy hipMemcpy
 #define cudaMemcpy2DAsync hipMemcpy2DAsync
 #define cudaMemcpyAsync hipMemcpyAsync
+#define cudaMemcpyPeerAsync hipMemcpyPeerAsync
 #define cudaMemcpyDefault hipMemcpyDefault
 #define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice
 #define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost

include/strata/kernels/cpu/iq_avx2.hpp

@@ -1,5 +1,5 @@
 // include/strata/kernels/cpu/iq_avx2.hpp - AVX-2 multi-token dot products for the i-quant expert
-// formats (IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S) against Q8_K activations (ggml's block_q8_K).
+// formats (IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS) against Q8_K activations (ggml's block_q8_K).
 //
 // The AVX-512 kernel's scheme on the CPUs without it (AMD Zen 2/3, Intel Core 12th-14th gen): the weights of a
 // 32-value chunk are decoded once per verify window and every token applies them with five instructions.

include/strata/kernels/elementwise.hpp

@@ -104,6 +104,8 @@ void doorbell_wait(const uint32_t* d_flag, const uint32_t* d_seq, void* stream);
 void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream);
 /// `rows` rows of `width` floats from mapped memory, except the rows listed in hit_rows[0, *count) (device),
 /// which are written +0.0 instead (a verify window's GPU-computed entries: the pool leaves zeros there).
+/// multi-GPU: dst[rows[r] * width ..] = src[r * width ..] for r < n (float4; dst may be mapped host memory).
+void scatter_rows_f32(const float* src, float* dst, const int32_t* rows, int64_t n, int64_t width, void* stream);
 void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
                            const int32_t* count, void* stream);
 

include/strata/kernels/iq_kernels.hpp

@@ -56,9 +56,17 @@ size_t native_expert_scratch_bytes(int64_t cap_entries, int64_t n_ff);
 /// Grouped experts in the native format: group g's blob at device address grp_ptr[g]; its entries
 /// [grp_start[g], grp_start[g+1]) read token ent_tok[e]'s q8_1 activation (n_embd/32 blocks per token in x_q8_1)
 /// and write row ent_dst[e] of `out` (n_embd floats).  Counts are read on the device.
+/// `grid_groups` (1 .. cap_groups; 0 = cap_groups) groups run side by side, a block row each striding over the rest:
+/// a call that usually has few groups or none (the verify window's PCIe share) launches less for the ones it does
+/// not have.  The results do not depend on it.
 void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long* grp_ptr, const int32_t* grp_start,
                            const int32_t* n_groups, const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups,
-                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream);
+                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream,
+                           int64_t grid_groups = 0);
+/// true: `native_expert_grouped`'s launches before the group stride (STRATA_GROUPED_V1=1 at startup) - a block row
+/// per possible group, SwiGLU and the q8_1 quantization as two kernels over all cap_entries.  Bitwise the same results
+/// (native_grouped_parity checks it); kept for A/B timing.  Set before graph capture; captured graphs keep theirs.
+void native_grouped_set_v1(bool v1);
 
 /// `iq_mmvq` and `native_expert_grouped` decode each weight part once and apply it to every column / entry;
 /// true selects the older kernels that decode it again per column (STRATA_OLD_IQ_MMVQ=1 at startup).  Both give

include/strata/kernels/ngram.hpp

@@ -114,7 +114,7 @@ enum class PleIo { Direct, Mmap };
 
 struct PleIoOptions {
     PleIo mode = PleIo::Direct;
-    uint32_t max_inflight = 64;      ///< outstanding SSD reads (decode needs 16; prefill chunks use more)
+    uint32_t max_inflight = 256;     ///< outstanding SSD reads (decode needs 16; a prompt chunk saturates the SSD at 256)
     uint64_t cache_rows = 1u << 20;  ///< bounded row cache: 1,048,576 rows x 90 B ~ 95 MB; 0 disables
     bool io_thread = true;           ///< reads submitted by a worker thread, not the caller
     /// Mmap mode only (`--ple-io ram`): lock the whole mapped table in RAM at open, so no SSD read ever sits on

include/strata/kernels/qsa_prompt_attn.hpp

@@ -14,7 +14,8 @@
 //   * q and p are split into FP16 hi + lo parts (two MMAs each), so they keep ~22 bits: the result differs from
 //     the FP32 kernel by summation order and the exp2 rounding, not by an FP16 cast.
 // Not bitwise equal to `qsa_decode_attn_batch`; `qsa_prompt_attn_parity` bounds the difference and the prompt
-// quality gate (needles, teacher-forced top-1) checks it end to end. Q4_0 KV is not handled (returns false).
+// quality gate (needles, teacher-forced top-1) checks it end to end. Q4_0 KV (mode 4): each block's codes enter
+// exactly as int8 and its scale (one per 32 values) in FP32, as int8 KV's do (STRATA_PROMPT_ATTN_Q4=0: the old kernel).
 #pragma once
 
 #include "strata/kernels/qsa_decode_attn.hpp"
@@ -24,7 +25,7 @@
 namespace strata::kernels {
 
 /// Same arguments and output as `qsa_decode_attn_batch` minus the scratch. Returns false (nothing launched) when the
-/// pools are Q4_0 or the geometry is not 24 heads / 2 KV heads / 256: the caller then uses the old kernel.
+/// geometry is not 24 heads / 2 KV heads / 256: the caller then uses the old kernel.
 bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
                            int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream);
 

include/strata/kernels/qsa_select.hpp

@@ -11,7 +11,8 @@
 //                      a chunk it may already hold a block completed later, so it is not read).
 //   qsa_block_topk   : one block per query; a 4-pass radix select over the query's n_bid + 1 blocks, each
 //                      weighted by its cell count, then the cells emitted in ascending order with ties to the
-//                      lowest index - the same selection as topk_kernel, over a quarter of the elements.
+//                      lowest index - the same selection as topk_kernel, over a quarter of the elements.  On sm_90+
+//                      a call of up to 16 queries runs it on a cluster of 8 CTAs per query: the same ids.
 //
 // Queries carry their own step record (pos, n_kv, n_bid, width) as everywhere else in QSA.
 #pragma once
@@ -40,7 +41,14 @@ void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64
                     int64_t active_blocks = -1);   ///< > 0: no query of the call has more than this many blocks (n_bid + 1;
                                                    ///< the same contract as qsa_block_scores's). The register kernel is
                                                    ///< chosen by this, not by the capacity max_blocks (a long
-                                                   ///< --max-context otherwise sends every short prompt to the slow one)
+                                                   ///< --max-context otherwise sends every short prompt to the slow one).
+                                                   ///< CUDA uses the bound only on sm_75; HIP keeps its existing policy.
+                                                   ///< Omit it for captured graphs whose context can grow after capture.
+/// The same ids on a thread-block cluster of 8 CTAs per query (sm_90+, CUDA; S19). qsa_block_topk takes it for calls
+/// of up to 16 queries unless STRATA_QSA_CLUSTER=0. False (nothing launched) where it cannot run: HIP, a card or a
+/// build below sm_90, or a capacity whose keys do not fit one cluster's shared memory. Capturable.
+bool qsa_block_topk_cluster(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                            const QsaShapes& s, int32_t* ids, void* stream);
 /// The original kernel (keys read from memory on every radix pass), for tests: the same ids.
 void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
                         const QsaShapes& s, int32_t* ids, void* stream);

include/strata/kernels/sampler.hpp

@@ -42,6 +42,11 @@ struct SamplerParams {
 void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
                    const SamplerParams& p, int* out, void* stream);
 
+// The greedy pick (no penalties) on a thread-block cluster of 8 CTAs per row (sm_90+, CUDA; S19): the same token as
+// sample_tokens' one-block argmax, which takes it unless STRATA_ARGMAX_MULTI=0.  False (nothing launched) where it
+// cannot run: HIP, a card or a build below sm_90.  Device pointers; capturable.
+bool sample_greedy_cluster(const float* logits, int n_tokens, int n_vocab, int* out, void* stream);
+
 // ---- COUPLED DRAFT SAMPLING (include/strata/core/coupled_draft.hpp, STRATA_SPEC_COUPLED=1).  Device pointers
 // throughout; every per-request / per-round value comes from device memory, so the calls can be captured.
 //

include/strata/prefill/kernels.hpp

@@ -43,8 +43,8 @@ void gdn_gates(const float* ab, const float* dt, const float* ssm_a, float* gate
 /// The 4-tap causal conv + SiLU over the chunk (history [C][3] in, updated to the chunk's last three inputs), then
 /// the L2 norm of the q and k heads of every token.  h: [T, C].
 void gdn_conv(float* history, const float* qkv, const float* conv_w, float* h, int64_t T, float eps, void* stream);
-/// The recurrence over the chunk, block per value head, state in registers; y[t] = rmsnorm(o) * gamma * sigmoid(z)
-/// (FP32 and FP16 bits: the out projection is quantized).
+/// The recurrence over the chunk, state in registers; y16[t] = rmsnorm(o) * gamma * sigmoid(z) in FP16 (what the
+/// out projection reads); y is FP32 scratch.
 void gdn_recurrence(float* state, const float* h, const float* gate, const float* beta, const float* z,
                     const float* gamma, float eps, float* y, uint16_t* y16, int64_t T, void* stream);
 

include/strata/prefill/moe_fused.hpp

@@ -0,0 +1,64 @@
+// include/strata/prefill/moe_fused.hpp - #136 (prompt speed): the routed experts of a Q2_0 pack in the prompt path on
+// Strata's own int8 tensor-core kernels, opt-in with STRATA_PF_FUSED=1 (NVIDIA sm_80 and newer; sm_75 and HIP: MMQ).
+//
+// The MMQ path (moe_mmq.hpp) per MoE layer: a host sync to group the (token, k) pairs by expert, a gather of each
+// expert into a 16-expert group buffer, then per group the q8_1 activations of every routed slot, gate/up, SwiGLU,
+// the q8_1 of H and down - five passes over per-slot buffers.  Here:
+//   - the activations are rounded to int8 once per TOKEN (not per slot), with a scale per 32 values;
+//   - the grouping (counts, offsets, the rows of each expert, a list of 64-row tiles) runs on the GPU from the
+//     router's ids, so the host never waits for the routing;
+//   - one launch per batch of up to kMaxBatch experts reads each expert where it already is (a resident cache slot
+//     or a slot of the streamed ring: a pointer per expert, no gather) - gate/up with SwiGLU and the int8 rounding
+//     of H in its epilogue, then down into the per-slot rows the existing combine reads.
+// The 2-bit codes are unpacked in registers and multiplied with mma.sync m16n8k32 (s8 x s8 -> s32); Q2_0's offset
+// (w = d * (q - 1)) is folded into the activation blocks' sums.  The numbers differ from MMQ's (another int8
+// rounding of the activations and of H) by about as much as MMQ differs from FP32 (tests/cuda/prefill_fused_moe_test).
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace strata::prefill::fused {
+
+/// Routed rows (token, k pairs of one expert) per tile.
+constexpr int kTileRows = 64;
+/// Experts per launch: their blob pointers travel in the launch's parameters (1 KB).
+constexpr int kMaxBatch = 128;
+
+/// This build has the kernels (a CUDA build; HIP and builds without MMQ do not).
+bool built();
+/// built(), and the current device is sm_80 or newer (mma.sync with s8 operands at m16n8k32).
+bool available();
+/// available(), unless STRATA_PF_FUSED=0 (read once): the Q2_0 pack's fused experts, on by default since 0.1.36.
+bool enabled();
+/// STRATA_PF_FUSED=1 given explicitly, and available(): the native IQ packs' fused kernels (opt-in).
+bool requested();
+
+/// Bytes of `rows` activation rows of `cols` values (a multiple of 64) in the kernels' int8 form: per 64 values 64
+/// codes and {d, c} of each half (c = -(1.5 * 2^23 + the half's code sum), what the epilogue adds back).
+size_t act_bytes(int64_t rows, int64_t cols);
+/// Bytes of the grouping tables for `n` routed rows over `n_expert` experts.
+size_t group_bytes(int64_t n, int n_expert);
+
+/// x [rows][cols] FP32 -> `xa` (act_bytes(rows, cols)).
+void quantize_act(const float* x, int64_t rows, int64_t cols, void* xa, void* stream);
+
+/// The routing `ids` [n = tokens * k] -> per expert its rows (counts, offsets, the 64-row tile list, in `scratch`,
+/// group_bytes(n, n_expert)), `slot[i]` = the row of pair i (the row of the per-slot outputs the combine reads) and
+/// `src[row]` = its token (i / k).  The order of the rows within an expert is not fixed; no output depends on it.
+void group(const int32_t* ids, int64_t n, int k, int n_expert, void* scratch, int32_t* slot, int32_t* src,
+           void* stream);
+
+/// Experts [e0, e1) of one layer (e1 - e0 <= kMaxBatch); blob[e - e0]: the Strata Q2_0 blob of expert e on the
+/// device (16-byte aligned), read only for an expert with rows.
+struct Batch {
+    int e0 = 0, e1 = 0;
+    const uint8_t* blob[kMaxBatch] = {};
+};
+/// The batch's products: gate/up from `xa` (quantize_act of the layer's input, [tokens][2560]) through the rows
+/// `src` of `group`, SwiGLU, H as int8 into `ha` (act_bytes(n, 640)), down into `dm` [n][2560] FP32 at the rows of
+/// `group`.  `n`: the layer's routed rows (tokens * k), which bounds the launch grid.
+void experts(const Batch& b, int n_expert, int64_t n, const void* scratch, const void* xa, const int32_t* src, void* ha,
+             float* dm, void* stream);
+
+}  // namespace strata::prefill::fused

include/strata/prefill/moe_fused_iq.hpp

@@ -0,0 +1,41 @@
+// include/strata/prefill/moe_fused_iq.hpp - #136: the fused int8 prompt experts (moe_fused.hpp) for the native GGUF
+// packs (tools/iq_pack.py: an expert is its raw GGUF slices [gate rows | up rows | down rows], formats per layer in
+// expert_layout().fmt), opt-in with the same STRATA_PF_FUSED=1 (NVIDIA sm_80 and newer; sm_75 and HIP: MMQ).
+//
+// The grouping on the GPU (fused::group), the per-expert pointers (fused::Batch: a cache slot or a ring slot, no
+// gather) and the launch shape are the Q2_0 path's.  What differs is the load stage: the i-quant blocks are decoded to
+// int8 in shared memory before the mma (codebook and sign lookups, as llama.cpp's MMQ load_tiles does), with the
+// block's scale per 32 (or per 16: IQ2_XS, IQ2_S, multiplied at m16n8k16) beside them.  The activations and H are
+// int8 per 32 values in their natural order here (quantize_act_native), not the Q2_0 path's permuted one.
+//
+// Covered: gate/up IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS; down Q2_0, IQ4_NL - every layer of the IQ2_XS,
+// IQ3_XXS and IQ3_S packs but the IQ1_M ones (which MMQ does not cover either).  Other layers keep MMQ.
+#pragma once
+
+#include "strata/prefill/moe_fused.hpp"
+
+#include <cstddef>
+#include <cstdint>
+
+namespace strata::prefill::fused {
+
+/// One layer's native expert geometry (cpu::NativeFmt's fields the kernels read; n_embd 2560, n_ff 640).
+struct NativeGeom {
+    int gu_type = -1, d_type = -1;    ///< ggml types
+    size_t gu_row = 0, d_row = 0;     ///< bytes per gate/up row and per down row
+    size_t up_off = 0, down_off = 0;  ///< inside the blob
+};
+
+/// enabled() (STRATA_PF_FUSED=1 on sm_80+), and the kernels cover this gate/up and down pair on this device.
+bool native_supported(int gu_type, int d_type);
+
+/// x [rows][cols] FP32 -> `xa` (act_bytes(rows, cols)): int8 per 32 values in natural order, the native kernels' form.
+void quantize_act_native(const float* x, int64_t rows, int64_t cols, void* xa, void* stream);
+
+/// experts() for native blobs: b.blob[e - e0] is the native blob of expert e (gate at 0, up at g.up_off, down at
+/// g.down_off; 2-byte aligned).  `xa` from quantize_act_native; H in `ha` (act_bytes(n, 640)); down into `dm` at the
+/// rows of `group`.
+void experts_native(const Batch& b, const NativeGeom& g, int n_expert, int64_t n, const void* scratch, const void* xa,
+                    const int32_t* src, void* ha, float* dm, void* stream);
+
+}  // namespace strata::prefill::fused

include/strata/prefill/moe_mmq.hpp

@@ -64,6 +64,16 @@ private:
 /// slot: gate rows then up rows at `gu_dst`, down at `d_dst`.
 void gather_native(const void* gate, const void* up, size_t gu_half_bytes, const void* down, size_t d_bytes,
                    void* gu_dst, void* d_dst, void* stream);
+/// gather_native for an MMQ group's experts [first, n) in ONE launch: expert q's blob (`blob[q]`; gate at +0, up at
+/// +up_off, down at +down_off) to gu_dst + q * gu_stride and d_dst + q * d_stride - the same bytes as one gather_native
+/// each.  Every pointer, offset and size 16-byte aligned (false otherwise: nothing launched, gather one at a time).
+constexpr int kGatherGroupMax = 16;
+struct GatherGroup {
+    const uint8_t* blob[kGatherGroupMax] = {};
+    int first = 0, n = 0;
+};
+bool gather_native_group(const GatherGroup& g, size_t up_off, size_t gu_half_bytes, size_t down_off, size_t d_bytes,
+                         void* gu_dst, size_t gu_stride, void* d_dst, size_t d_stride, void* stream);
 /// A Strata-pack Q2_0 expert blob (codes and fp16 scales in separate planes, gate/up rows interleaved) into GGUF
 /// Q2_0 blocks: gate/up [1280, 2560] at `gu_dst` (rows stay interleaved), down [2560, 640] at `d_dst`.  Same values.
 void gather_strata_q2(const uint8_t* blob, void* gu_dst, void* d_dst, void* stream);

include/strata/prefill/prefill.hpp

@@ -21,6 +21,8 @@
 #include <memory>
 #include <string>
 
+namespace strata::core { class PeerExperts; }
+
 namespace strata::prefill {
 
 struct PrefillStats {
@@ -86,6 +88,11 @@ public:
 
     const PrefillStats& stats() const { return stats_; }
 
+    /// multi-GPU: the experts the peer GPU holds are computed THERE for every prompt chunk (up to `cap_rows` routed
+    /// rows per layer; the rest of the peer's experts are read by this GPU over P2P).  Allocates the peer's buffers for
+    /// chunks of up to init's `chunk` tokens.  Needs P2P between the two cards.
+    bool set_peer(core::PeerExperts* peer, int64_t cap_rows, std::string& err);
+
     /// Plan v0.3 P6: called after every chunk with the chunk's final multi-stream residual rows (device,
     /// T x hc*n_embd, valid until the next chunk) and the chunk's first position; the MTP draft layer builds its
     /// K/V from them.  The prefill stream is synchronized before the call.

serve/server.py

@@ -34,6 +34,7 @@ import re
 import select
 import signal
 import socket
+import struct
 import subprocess
 import sys
 import tempfile
@@ -60,11 +61,22 @@ IMAGE_PAD = "<|image_pad|>"
 VISION_START = "<|vision_start|>"
 # #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)
 REASONING_WRAP_UP = "\n\nI have thought about this long enough; time to give my answer.\n</think>\n\n"
+LOOPBACK_NAMES = ("localhost", "127.0.0.1", "::1")
 CTX_SLACK = 8               # `strata --serve` rejects prompt + max_new + 8 > context: keep the same margin here
 # The live tok/s is a rate over a window, not a mean since the first token: a mean reads ~1/elapsed at the first
 # token (the Monitor showed five-digit numbers) and then undershoots for the first second of every answer.
 RATE_WINDOW_S = 2.0
 RATE_MIN_SPAN_S = 0.25      # younger than this there is no rate yet: the mean so far, with the span floored here
+# #481: a running request whose engine prints nothing (no T, PP or any other line) for this long has lost step with the
+# server (the engine's main thread waits, untimed, for its next command): the engine is ended and the request fails;
+# the next request starts it again.  The config's "engine_silence_s" sets it (0: wait forever, as before).
+ENGINE_SILENCE_S = 300.0
+# ... except while a prompt is read: a PP line comes once per chunk (up to 32768 tokens with --prefill auto, issue
+# #282), and the slowest PCs read ~100 tok/s, so a first chunk can take minutes before the first line.  Until the first
+# PP the wait adds the chunk's tokens at PP_FLOOR_TOK_S; after one, a chunk may take PP_SLACK x the last one's time.
+PP_CHUNK_MAX = 32768
+PP_FLOOR_TOK_S = 50.0
+PP_SLACK = 3.0
 
 
 # ------------------------------------------------------------------------------------------------ engines
@@ -112,6 +124,12 @@ class ModelBusy(RuntimeError):
     """Explicit model controls must not interrupt active or queued requests."""
 
 
+class EngineSilent(EngineDied):
+    """#481: the engine said nothing for too long during a request (or never acknowledged a STOP): the two sides lost
+    step - the engine waiting for its next command, the server for this request's end - and the server ended it.  An
+    EngineDied, so the request ends with an error and the next one starts the engine again."""
+
+
 class EngineStuck(RuntimeError):
     """The engine process did not end after QUIT, terminate and kill: the server keeps it (and says so) rather than
     reporting its GPU and RAM as given back."""
@@ -123,12 +141,60 @@ class GpuBusy(RuntimeError):
 
 
 ENGINE_REQUEST = re.compile(
-    r"prompt (?P<prompt>\d+) tokens = (?P<reused>\d+) reused \+ \d+ read in (?P<read>[\d.]+) ms \((?P<pp>[\d.]+) tok/s\), "
+    # "+ 12288 of 98179 read": a request cancelled while its prompt was read (#471)
+    r"prompt (?P<prompt>\d+) tokens = (?P<reused>\d+) reused \+ \d+(?: of \d+)? read in (?P<read>[\d.]+) ms "
+    r"\((?P<pp>[\d.]+) tok/s\), "
     r"(?P<gen>\d+) generated in (?P<gen_ms>[\d.]+) ms \((?P<tg>[\d.]+) tok/s\)")
 
 
 _echoing: set[str] = set()      # the logs echo_requests already follows (restart() runs StrataEngine.__init__ again)
 
+DRAFT_HEAD_FAIL = "the draft head does not fit"
+DRAFT_HEAD_HINT = ("a smaller draft vocabulary needs less VRAM: --draft-vocab cyrillic (English, code and the Cyrillic "
+                   "script) or --draft-vocab en (English and code, ~215 MiB less than the default). Start once with "
+                   "it - START-HERE.bat --draft-vocab en (Windows) or ./setup.sh --draft-vocab en - and the model "
+                   "keeps it; or a smaller --context in setup.")
+
+
+def start_failure_hint(log: str | None, offset: int) -> str:
+    """#474: what to change when the engine stopped at the start because the MTP draft head did not fit the VRAM
+    left: the engine's own `strata mtp:` lines after that failure (0.1.36+: what it needs, what is free, the smaller
+    subsets), else the same advice in words for an older engine.  "" for any other failure: the log says why."""
+    if not log:
+        return ""
+    try:
+        with open(log, "rb") as f:
+            f.seek(offset)
+            text = f.read().decode("utf-8", "replace")
+    except (OSError, ValueError):
+        return ""
+    if DRAFT_HEAD_FAIL not in text:
+        return ""
+    said = [x.strip()[len("strata mtp: "):] for x in text.splitlines()
+            if x.strip().startswith("strata mtp: ") and ("draft head over" in x or "hint:" in x)]
+    return ". mtp: " + DRAFT_HEAD_FAIL + ". " + (" ".join(said) if said else "Hint: " + DRAFT_HEAD_HINT)
+
+
+def start_log_tail(log: str | None, offset: int, n: int = 20) -> str:
+    """#496: the last n lines this start wrote to the engine log, for the error when the engine ended before READY -
+    whatever the failure, the engine's own reason is in them (people posted the traceback without the log).  "" when
+    there is no log or nothing in it from this start."""
+    if not log:
+        return ""
+    try:
+        with open(log, "rb") as f:
+            f.seek(0, 2)
+            start = max(offset, f.tell() - 64 * 1024)   # enough for 20 lines, never an earlier start's
+            f.seek(start)
+            text = f.read().decode("utf-8", "replace")
+    except (OSError, ValueError):
+        return ""
+    lines = text.splitlines()[1 if start > offset else 0:]   # not a line cut in half
+    lines = [x.rstrip() for x in lines if x.strip()][-n:]
+    if not lines:
+        return ""
+    return "\nthe engine log's last lines:\n" + "\n".join("  " + x for x in lines)
+
 
 def echo_requests(log_path: str, offset: int) -> None:
     """STRATA_REQUEST_LINES=1: one stdout line per finished request, from the engine's own summary in its log.
@@ -153,6 +219,27 @@ def echo_requests(log_path: str, offset: int) -> None:
                                                  m["tg"]), flush=True)
 
 
+def experts_loading_words(args: list, size: str) -> str:
+    """#505: what the start does with the experts, by the engine's flags (generate.cpp's option parsing): a RAM budget
+    copies the hottest N GiB into RAM (--resident-budget-gib), the resident low-RAM mode the ones the GPU does not hold
+    (--resident-experts), plain --mmap-experts reads them from the model files through the OS file cache (nothing is
+    loaded into RAM up front); otherwise all of them go into RAM."""
+    if "--resident-budget-gib" in args:
+        try:
+            n = f"up to {float(args[args.index('--resident-budget-gib') + 1]):g} GiB"
+        except (IndexError, ValueError):
+            n = "a RAM budget"
+        return (f"loading the most-used experts into RAM ({n}; the rest are read from the model files as needed) "
+                "and locking part of them for the GPU.")
+    if "--resident-experts" in args:
+        return (f"loading the experts the GPU does not hold into RAM (of {size}) and locking part of them for the "
+                "GPU.")
+    if "--mmap-experts" in args:
+        return (f"mapping the experts from the model files ({size}, --mmap-experts): they are not loaded into RAM - "
+                "the OS file cache reads them as the GPU's expert cache fills and as requests need them.")
+    return f"loading the experts into RAM ({size}) and locking part of them for the GPU."
+
+
 def narrate_start(log_path: str, offset: int, args: list, done: threading.Event, heartbeat=20.0) -> None:
     """While the engine starts, say in the server window what it is doing, from its log: the start reads tens of GB
     into RAM and locks part of it for the GPU, and on many PCs everything is slow or frozen for a minute or more -
@@ -164,6 +251,7 @@ def narrate_start(log_path: str, offset: int, args: list, done: threading.Event,
         except (OSError, IndexError):
             pass
     size = f"about {gb:.0f} GB" if gb >= 1 else "tens of GB"
+    loading = experts_loading_words(args, size)
     t0 = last = time.time()
     said = set()
 
@@ -187,8 +275,8 @@ def narrate_start(log_path: str, offset: int, args: list, done: threading.Event,
             cut = chunk.rfind(b"\n") + 1
             pos += cut
             for line in chunk[:cut].decode("utf-8", "replace").splitlines():
-                if "PLE on" in line or "expert arena:" in line:
-                    say("arena", f"[strata] loading the experts into RAM ({size}) and locking part of them for the GPU.\n"
+                if "PLE on" in line or "expert arena:" in line or "experts via mmap" in line:   # #505: mapped
+                    say("arena", f"[strata] {loading}\n"
                                  "         YOUR PC CAN BE SLOW OR STOP RESPONDING FOR 1-3 MINUTES NOW - this is normal.\n"
                                  "         Please wait and don't close this window; the browser opens when it is ready.")
                 elif " loaded " in line and "GiB at" in line:
@@ -212,6 +300,8 @@ class StrataEngine:
     (`temperature=F top_p=F top_k=N seed=N`, the engine's own spelling).  An absent temperature keeps the
     engine's default, which is greedy; `temperature=0` means the same thing, so it is not forwarded.
     """
+    silence_s = ENGINE_SILENCE_S         # #481: main() sets the config's engine_silence_s (survives restart())
+    silent_note = None                   # #481: why the server ended a silent engine (death_note says it)
 
     def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | None = None,
                  env: dict | None = None, lazy: bool = False):
@@ -227,7 +317,8 @@ class StrataEngine:
         self.info = {}                   # INFO key=value facts (engine 0.1.8+): kv, expert slots, ... (Monitor tab)
         self.prefill_tok_s_mean = None
         self.progress = None             # (read, total) prompt tokens while a prompt is read, from PP lines
-        try:                             # a ready-made engine's BUILD.json says its version
+        self.silent_note = None
+        try:                            # a ready-made engine's BUILD.json says its version
             self.info["version"] = json.loads((Path(exe).parent / "BUILD.json").read_text()).get("version")
         except (OSError, ValueError):
             self.info["version"] = None
@@ -236,6 +327,7 @@ class StrataEngine:
         self.unloaded = False            # `ended` stays True until READY (below): not alive while starting (#344)
         self.log = open(log, "a", encoding="utf-8") if log else subprocess.DEVNULL
         loading = threading.Event()                     # set once READY: the narrator below stops
+        log_start = os.path.getsize(log) if log else 0  # where this start's lines begin (start_failure_hint)
         if log:
             threading.Thread(target=narrate_start, args=(log, os.path.getsize(log), args, loading),
                              daemon=True).start()
@@ -259,7 +351,16 @@ class StrataEngine:
                 break
         loading.set()
         if self.max_context <= 0:
-            raise RuntimeError("the engine exited before it was ready" + (f" (see {log})" if log else ""))
+            try:                                        # its pipes and our handle on its log (the log stays)
+                self.proc.wait(timeout=5)
+                self.proc.stdin.close()
+                self.proc.stdout.close()
+                if log:
+                    self.log.close()
+            except (OSError, subprocess.TimeoutExpired):
+                pass
+            raise RuntimeError("the engine exited before it was ready" + (f" (see {log})" if log else "") +
+                               start_failure_hint(log, log_start) + start_log_tail(log, log_start))
         # (from PR #41, midhatn) a locally built engine can sit next to another release's BUILD.json: engines that
         # report their own version (INFO engine=, 0.1.8+) win, the manifest stays the fallback for older ones
         if self.info.get("engine"):
@@ -280,6 +381,8 @@ class StrataEngine:
 
     def death_note(self) -> str:
         """Why the engine most likely ended, from the end of its log: its own watchdog (issue #29), else RAM."""
+        if getattr(self, "silent_note", None):          # #481: the server ended it, not the OS or the engine itself
+            return self.silent_note
         tail = ""
         try:
             with open(self.log_path, "rb") as f:
@@ -338,6 +441,8 @@ class StrataEngine:
             self.last.update(hits=int(f[9]), lookups=int(f[10]))
         if len(f) >= 14:                                  # the expert tiers (engine 0.1.31+): RAM / file blobs, file MB
             self.last.update(ram_blobs=int(f[11]), file_blobs=int(f[12]), file_mb=float(f[13]))
+        if len(f) >= 15:                                  # #471 (engine 0.1.36+): the prompt tokens actually read
+            self.last.update(prompt_read=int(f[14]))
 
     @staticmethod
     def sampling_keys(sampling: dict) -> str:
@@ -408,19 +513,34 @@ class StrataEngine:
         except OSError:                                  # the pipe is gone: the engine died (not the client)
             raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})") from None
         done = False
+        # #481: how long the engine may stay silent from here.  Until the first line: the request's first prompt
+        # chunk at the slowest prompt reading on top of silence_s; a PP line resets it to its own chunk's time.
+        silence = float(self.silence_s or 0)
+        allow = silence + min(len(ids), PP_CHUNK_MAX) / PP_FLOOR_TOK_S if silence > 0 else 0.0
+        heard, read_to = time.monotonic(), 0
         try:
             while True:
+                wait = 10.0
+                if allow > 0:
+                    wait = min(wait, allow - (time.monotonic() - heard))
+                    if wait <= 0:
+                        done = True                       # no STOP and no drain: nothing is listening
+                        raise self._silent(f"the engine said nothing for {time.monotonic() - heard:.0f} s during "
+                                           "the request")
                 try:
-                    line = self.lines.get(timeout=10)
+                    line = self.lines.get(timeout=wait)
                 except queue.Empty:
                     if cancel.is_set():
                         return
-                    yield None
+                    if wait >= 10.0:
+                        yield None                        # the 10 s heartbeat (the deadline's short waits are not)
                     continue
                 if line is None:
                     done = True
                     raise EngineDied(f"the engine stopped unexpectedly (exit code {self.exit_code()})")
+                heard = time.monotonic()                  # any line is output: T, PP, RESUME, INFO ...
                 if line.startswith("T "):
+                    allow = silence
                     if cancel.is_set():
                         return
                     yield int(line[2:])
@@ -429,9 +549,18 @@ class StrataEngine:
                     if len(f) >= 3 and f[1].isdigit() and f[2].isdigit():
                         self.progress = (int(f[1]), int(f[2]))             # prompt progress, one per chunk: also a heartbeat (the
                         self.prefill_tok_s_mean = float(f[4]) if len(f) >= 5 else None
+                        rate, chunk = self.prefill_tok_s_mean or 0.0, int(f[1]) - read_to
+                        read_to = int(f[1])
+                        if silence > 0 and rate > 0 and chunk > 0:   # #481: the next chunk, as long as this one
+                            allow = max(silence, PP_SLACK * chunk / rate)
                     if cancel.is_set():                   # lines reset the 10 s wait, so without this a long prompt
                         return                            # would send no keep-alives at all)
                     yield None
+                elif line.startswith("RESUME "):          # the reused tokens: the first chunk starts after them
+                    try:
+                        read_to = int(line.split()[1])
+                    except (IndexError, ValueError):
+                        pass
                 elif line.startswith("DONE"):
                     self._parse_done(line)
                     done = True
@@ -447,13 +576,41 @@ class StrataEngine:
                         self.proc.stdin.flush()
                     except OSError:
                         pass
+                # #481: never an untimed wait here - it holds the request FIFO, and an engine that lost step never
+                # answers.  An engine that honours STOP gets the current allowance in all (a STOP during a prompt
+                # chunk is seen after it); an older one runs on to max_new, so each line only has to come in time.
+                heard = time.monotonic()
                 while True:
-                    line = self.lines.get()
+                    left = allow - (time.monotonic() - heard) if allow > 0 else None
+                    try:
+                        if left is not None and left <= 0:
+                            raise queue.Empty
+                        line = self.lines.get(timeout=left)
+                    except queue.Empty:
+                        raise self._silent("the engine did not finish the request after it was stopped (STOP) "
+                                           f"within {allow:.0f} s") from None
                     if line is None or line.startswith("ERR"):
                         break
                     if line.startswith("DONE"):
                         self._parse_done(line)
                         break
+                    if not self.can_stop:
+                        heard = time.monotonic()
+
+    def _silent(self, what: str) -> EngineSilent:
+        """#481: end an engine that lost step with the server (its main thread waits for a command the server never
+        sends), so the next request starts it again: killed now, its GPU and RAM go back with the process."""
+        self.silent_note = ("The engine and the server lost step (issue #481; a very slow PC can raise "
+                            "\"engine_silence_s\" in the config, 0 = wait forever). If it happens again, please add "
+                            "the end of the engine log to github.com/Niko1221/Strata/issues/481.")
+        self.ended = True                               # not alive from now: the next request restarts it
+        proc = self.proc
+        try:
+            proc.kill()
+            proc.wait(timeout=20)                       # restart() -> close() handles one that is still exiting
+        except (OSError, AttributeError, subprocess.TimeoutExpired):
+            pass
+        return EngineSilent(f"{what}; the server ended the engine")
 
     def close(self):
         """End the engine process: QUIT first (the engine frees its memory itself - unpinning tens of GB can take
@@ -621,6 +778,17 @@ def gpu_list(cfg: dict) -> list[int]:
     return [int(str(x).strip()) for x in items if str(x).strip() != ""]
 
 
+def engine_silence_s(cfg: dict) -> float:
+    """#481: the config's "engine_silence_s" - seconds an engine may print nothing during a request before the server
+    ends it (default ENGINE_SILENCE_S; 0 = wait forever).  ValueError for anything but a number >= 0."""
+    v = cfg.get("engine_silence_s")
+    if v is None:
+        return ENGINE_SILENCE_S
+    if isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0:
+        raise ValueError(f'"engine_silence_s" must be a number of seconds >= 0 (0 = no limit), not {v!r}')
+    return float(v)
+
+
 def engine_args(cfg: dict) -> list[str]:
     """The engine's arguments: the config's, and with several GPUs the layer split across them ("layer_split" in the
     config: "auto" by default, or the first layer of each later GPU's share, e.g. "18" or "16,32")."""
@@ -630,6 +798,44 @@ def engine_args(cfg: dict) -> list[str]:
     # opt-in: an auto split runs on the first card alone when it holds every profiled expert and the KV
     if len(gpu_list(cfg)) > 1 and cfg.get("split_skip_if_fits") and "--split-skip-if-fits" not in args:
         args.append("--split-skip-if-fits")
+    return learned_profile_args(cfg, args)
+
+
+def profile_shape(path: str) -> tuple[int, int] | None:
+    """An expert profile's (layers, experts per layer), from its header (tools/make_profile.py's format), or None
+    when the file is missing, is not one or is shorter than the pairs its header promises."""
+    try:
+        with open(path, "rb") as f:
+            head = f.read(24)
+            size = os.fstat(f.fileno()).st_size
+    except OSError:
+        return None
+    if len(head) < 24 or head[:4] != b"STRP":
+        return None
+    _, nl, ne, _, n = struct.unpack("<5I", head[4:])
+    return (nl, ne) if size >= 24 + 4 * n else None
+
+
+def learned_profile_args(cfg: dict, args: list[str]) -> list[str]:
+    """#477 (opt-in): "expert_profile_save": "<path>" in the config has the engine (0.1.36+) save what its adaptive
+    tier learned there - on QUIT and every "expert_profile_save_every" minutes (10 by default, 0 = at QUIT only) -
+    and the next start begins from it instead of the config's --expert-profile, when it is a profile of the same
+    model (its header's layers and experts match); otherwise from the config's own, as before.  A relative path is
+    the engine's (the config's "cwd").  Without the key, the arguments are the config's, unchanged."""
+    save = cfg.get("expert_profile_save")
+    if not isinstance(save, str) or not save.strip() or "--expert-profile-save" in args:
+        return args
+    args = args + ["--expert-profile-save", save]
+    every = cfg.get("expert_profile_save_every")
+    if isinstance(every, (int, float)) and not isinstance(every, bool) and every >= 0:
+        args += ["--expert-profile-save-every", str(every)]
+    if "--expert-profile" in args[:-1]:
+        i = args.index("--expert-profile") + 1
+        here = cfg.get("cwd") or "."
+        learned = profile_shape(save if os.path.isabs(save) else os.path.join(here, save))
+        base = profile_shape(args[i] if os.path.isabs(args[i]) else os.path.join(here, args[i]))
+        if learned is not None and learned == base:
+            args[i] = save
     return args
 
 
@@ -755,6 +961,10 @@ class Service:
         # #321: origins that count as Strata's own page for /settings and MCP tools, e.g. the web app reached through a
         # reverse proxy or tunnel whose Host differs ("https://strata.example.com"); never a wildcard
         self.trusted_origins: list[str] = []
+        # DNS rebinding: extra Host names this server answers to (the config's allowed_hosts, $STRATA_ALLOWED_HOSTS;
+        # "*" = any), and every name it answers to, which serve() works out from the address it listens on
+        self.allowed_hosts: list[str] = []
+        self.host_names: set[str] = set(LOOPBACK_NAMES)
         self.status = {"busy": False, "queued": 0}      # GET /status: what the model is doing right now
         self.rate = collections.deque(maxlen=32)        # (time, generated) samples for the live tok/s window
         # #332: the API request monitor (/api-monitor) keeps the last 100 requests' prompts and answers in memory,
@@ -879,6 +1089,13 @@ class Service:
         self.engine.restart()
         print("[strata] the engine is running again", flush=True)
 
+    def _say_died(self, e: Exception) -> None:
+        """The server window's line for an engine that died (or was ended, #481) in the middle of a request."""
+        note = self.engine.death_note() if hasattr(self.engine, "death_note") else ""
+        log = getattr(self.engine, "log_path", None)
+        print(f"[strata] {e}. {note} The next request starts the engine again."
+              f"{' Its log: ' + log if log else ''}", flush=True)
+
     def load(self):
         """POST /load and every generation request: start the engine now if it is unloaded (raises GpuBusy)."""
         # a request is on its way: the idle thread must not unload between this and the request's own start
@@ -1168,7 +1385,9 @@ class Service:
         elif max_new > room:
             if not self.fit_max_tokens:
                 raise ValueError(f"prompt ({len(ids)} tokens) + max tokens ({max_new}) exceeds the context "
-                                 f"({self.engine.max_context}); requests are never truncated")
+                                 f"({self.engine.max_context}); requests are never truncated. Send a smaller "
+                                 f"max_tokens (at most {max(0, room)} here), or add \"fit_max_tokens\": true to the "
+                                 "model's strata-<model>.json to shorten it to the room left (#545)")
             max_new = max(1, room)          # --fit-max-tokens: a shorter completion beats a 400
         return ids, kwargs.get("enable_thinking", True) is not False, max_new
 
@@ -1249,7 +1468,7 @@ class Service:
                     while True:
                         gen = self.engine.generate(prompt, max_new - n, sampling, cancel, embeddings=emb) if emb \
                             else self.engine.generate(prompt, max_new - n, sampling, cancel)
-                        seg, wrap = [], False           # this pass's tokens; the budget is reached
+                        seg, wrap, leaving = [], False, False   # this pass's tokens; the budget is reached; closed
                         try:
                             for t in gen:
                                 if t is None:               # heartbeat while the engine is quiet
@@ -1279,19 +1498,25 @@ class Service:
                                         break
                         except EngineDied as e:
                             finish = "error"
-                            note = self.engine.death_note() if hasattr(self.engine, "death_note") else ""
-                            log = getattr(self.engine, "log_path", None)
-                            print(f"[strata] {e}. {note} The next request starts the engine again."
-                                  f"{' Its log: ' + log if log else ''}", flush=True)
+                            self._say_died(e)
                             raise
                         except ValueError as e:             # the engine's ERR line (it may have ended after it)
                             finish = "error"
                             print(f"[strata] the engine reported an error: {e}", flush=True)
                             raise
+                        except GeneratorExit:               # the client went away: an engine that never acknowledges
+                            leaving = True                  # the STOP below is ended, but no error replaces this
+                            raise
                         finally:
-                            gen.close()                 # STOP+drain to THIS request's DONE while still holding the
-                            #                             fifo, so a stop-token break can't leave the shared engine
-                            #                             queue mid-drain for the next request to read as its own DONE
+                            try:
+                                gen.close()             # STOP+drain to THIS request's DONE while still holding the
+                                #                         fifo, so a stop-token break can't leave the shared engine
+                                #                         queue mid-drain for the next request to read as its own DONE
+                            except EngineSilent as e:   # #481: the STOP was never acknowledged: the engine is ended
+                                finish = "error"
+                                self._say_died(e)
+                                if not leaving and not cancel.is_set():
+                                    raise
                         if not wrap or cancel.is_set():
                             break
                         # #123: the thinking reached reasoning_budget_tokens.  Close it the way the model would (a
@@ -1329,11 +1554,14 @@ class Service:
                             cvec = (getattr(self.engine, "info", {}) or {}).get("cvec", 0)
                             loaded = str(cvec) not in ("0", "", "None")
                             hit_rate = round(last["hits"] / last["lookups"], 3) if last.get("lookups") else None
+                            seen = prompt_tokens_seen(len(ids), last)   # #471: < len(ids) when cancelled mid-read
                             self.history.append({
                                 "projection": (sampling or {}).get("experimental_speed_projection") is not False
                                 if loaded else None,
                                 "time": started, "duration_s": round(time.time() - started, 1), "finish": finish,
-                                "prompt_tokens": len(ids), "reused": last.get("reused"), "output_tokens": n,
+                                "prompt_tokens": seen, "reused": last.get("reused"), "output_tokens": n,
+                                # the request's whole prompt, and the tokens read of it (None: an older engine)
+                                "prompt_total": len(ids), "prompt_read": last.get("prompt_read"),
                                 "engine_generated": last.get("generated"),
                                 "prompt_ms": last.get("prompt_ms"), "decode_ms": last.get("decode_ms"),
                                 "decode_tok_s": round(last["generated"] / (last["decode_ms"] / 1000), 1)
@@ -1345,7 +1573,7 @@ class Service:
                                 "drafts_accepted": last.get("drafts_accepted")})
                             t = self.totals
                             t["requests"] += 1
-                            t["prompt_tokens"] += len(ids)
+                            t["prompt_tokens"] += seen
                             t["reused"] += last.get("reused") or 0
                             t["output_tokens"] += n
                             t["prompt_ms"] += last.get("prompt_ms") or 0.0
@@ -1354,7 +1582,7 @@ class Service:
                             t["drafts_accepted"] += last.get("drafts_accepted") or 0
                             fresh = getattr(self.engine, "last", None)
                             if fresh is not None and fresh is not before:      # the engine's clock for THIS request
-                                timings = request_timings(len(ids), n, last)
+                                timings = request_timings(seen, n, last)
                                 self.last_timings = dict(timings, at=int(time.time())) if timings else None
                             self.last_request_at = time.time()
                             now = time.time()
@@ -1364,6 +1592,10 @@ class Service:
                             hit_msg = f", expert cache {hit_rate*100:.1f}% hit" if hit_rate is not None else ""
                             print(f"[strata] done: {n} tokens in {el:.0f} s ({rate:.1f} tok/s) "
                                   f"({finish}, cancel={cancel.is_set()}){hit_msg}", flush=True)
+                            if finish == "length" and parser.state == "reasoning":   # #530
+                                print("[strata] the reply reached max tokens while still thinking, so it has no "
+                                      "answer: a thinking budget (reasoning_budget_tokens, in the request or in "
+                                      "strata-<model>.json for every request) leaves room to answer", flush=True)
                             if os.environ.get("STRATA_DEBUG") and raw_ids:
                                 print(f"[strata] raw: {self.tok.decode(raw_ids)!r}", flush=True)
                         self.status["busy"] = False
@@ -1378,6 +1610,17 @@ class Service:
                        "timings": timings}
 
 
+def prompt_tokens_seen(prompt_tokens: int, last: dict) -> int:
+    """#471: the prompt tokens a request got through - all of them, unless the engine's DONE line says a cancel stopped
+    its prompt read part-way (then the reused ones plus those read).  /metrics' history and totals count these, so a
+    cancelled read is neither recorded as the whole prompt nor given a rate from tokens it never read.  An engine
+    before 0.1.36 does not say (no `prompt_read`): the whole prompt, as before."""
+    read = last.get("prompt_read")
+    if read is None or last.get("finish") != "cancel":
+        return prompt_tokens
+    return min(prompt_tokens, int(last.get("reused") or 0) + int(read))
+
+
 def request_timings(prompt_tokens: int, generated: int, last: dict) -> dict | None:
     """One request's `timings` in llama.cpp's names (what its clients show as speed), from the engine's own clock
     (StrataEngine.last): prompt_n is what was read, cache_n what the conversation cache already held.  None when the
@@ -1735,6 +1978,46 @@ def make_handler(svc: Service):
         def log_message(self, fmt, *args):
             pass
 
+        def parse_request(self):
+            """Without an API key, every request (any method) first passes the Host check: DNS rebinding protection
+            (host_allowed).  With a key a rebinding page cannot authenticate, so the check is skipped: tunnels and
+            proxies that pass their own name on keep working."""
+            if not super().parse_request():
+                return False
+            host = self.headers.get("Host")
+            if svc.api_key or host_allowed(host, svc.host_names, "*" in svc.allowed_hosts):
+                return True
+            print(f"[strata] refused a request for Host {host!r} from {self.client_address[0]}: not a name this server "
+                  f"answers to (add it to \"allowed_hosts\" in the config or STRATA_ALLOWED_HOSTS, or set an API key)",
+                  flush=True)
+            self._json(403, {"error": {"type": "forbidden", "message":
+                             f"Host {host!r} is not allowed (DNS rebinding protection). Reaching Strata under this "
+                             f"name on purpose? Add it to \"allowed_hosts\" in the config (strata-<model>.json) or to "
+                             f"the STRATA_ALLOWED_HOSTS environment variable, or set an API key (\"api_key\"), which "
+                             f"turns this check off"}})
+            return False
+
+        def _foreign_page(self) -> bool:
+            """Without an API key, a /v1 POST from a browser page of another site (any site can POST text/plain
+            there without a CORS preflight) would burn GPU time: an Origin header must name an allowed page, and
+            then the body must be JSON.  No Origin (curl, the SDKs, other servers): any content type, as before."""
+            origin = self.headers.get("Origin")
+            if svc.api_key or not origin:
+                return False
+            if not origin_allowed(origin, self.headers.get("Host"), svc.host_names,
+                                  [*svc.trusted_origins, *svc.cors_origins]):
+                print(f"[strata] refused an API request from the web page {origin!r} (no API key; add its host to "
+                      f"\"allowed_hosts\" or its origin to \"cors_origins\" in the config)", flush=True)
+                self._json(403, {"error": {"type": "forbidden", "message":
+                                 f"web pages of {origin} may not use this server without an API key; set \"api_key\", "
+                                 f"or add the page's host to \"allowed_hosts\" (or its origin to \"cors_origins\") "
+                                 f"in the config"}})
+                return True
+            if not self.headers.get("Content-Type", "").startswith("application/json"):
+                self._json(415, {"error": {"message": "send application/json"}})
+                return True
+            return False
+
         def _watch_client(self, cancel: threading.Event) -> None:
             """#430 #431: cancel the request as soon as its client hangs up.  A non-streamed request writes nothing
             until it ends, and a streamed one only a keep-alive per prompt chunk (and the first write after a hang-up
@@ -1938,9 +2221,14 @@ def make_handler(svc: Service):
             if not self._authorized():
                 return
             path = self.path.split("?")[0].rstrip("/")   # issue #55: Claude Code posts /v1/messages?beta=true
+            if path.startswith("/v1/") and self._foreign_page():
+                return
             if path == "/settings":
                 self._settings()
                 return
+            # JSON from Strata's own page only, as /settings: else a plain form POST from any site unloads the model
+            if path in ("/unload", "/load") and not self._own_page("the model can be loaded or unloaded"):
+                return
             if path == "/unload":                            # give the GPU back now (between requests)
                 try:
                     r = svc.unload()
@@ -2256,6 +2544,36 @@ def warn_tight_ram(arena_mib) -> None:
               + "Close other programs, or run START-HERE --setup and pick a smaller size (Q2_0 / IQ2_XS).", flush=True)
 
 
+DESKTOP_FREE_MIB = 2048          # #560 #516: below this, an AMD card that also drives a Linux desktop can run out
+DESKTOP_RESERVE_MIB = 3072       # what kept KDE/Wayland alive beside a full expert cache in both reports
+
+
+def linux_desktop(env=None) -> bool:
+    """A graphical session on Linux (Wayland or X): its compositor, browser and apps take VRAM after the model has."""
+    env = os.environ if env is None else env
+    return os.name != "nt" and sys.platform != "darwin" and bool(env.get("WAYLAND_DISPLAY") or env.get("DISPLAY"))
+
+
+def desktop_vram_note(backend, vram_free_mib, args: list, desktop: bool) -> str:
+    """#560 #516: on Linux, when the desktop needs VRAM the AMD card does not have, amdgpu moves GPU memory (the expert
+    cache, ~24 GB) to system RAM, which the experts already fill - the OOM killer then ends the compositor.  The
+    default reserve (700 MiB) is sized for a card without a desktop.  A recommendation, nothing changes: "" when it
+    does not apply."""
+    if backend != "hip" or not desktop or not isinstance(vram_free_mib, int) or vram_free_mib >= DESKTOP_FREE_MIB:
+        return ""
+    try:
+        reserve = int(args[args.index("--vram-reserve-mib") + 1]) if "--vram-reserve-mib" in args else 700
+    except (ValueError, IndexError):
+        reserve = 700
+    if reserve >= DESKTOP_RESERVE_MIB:
+        return ""
+    return (f"[strata] note: {vram_free_mib} MiB of VRAM free with the model loaded. If this AMD card also drives your "
+            "desktop and the desktop or apps crash after the start (the driver moves the expert cache to RAM and "
+            "the OOM killer ends the session), keep more VRAM free: ./setup.sh --vram-reserve-mib "
+            f"{DESKTOP_RESERVE_MIB} (remembered; the expert cache gets "
+            f"{(DESKTOP_RESERVE_MIB - reserve) / 1024:.1f} GB less, a few % of speed)")
+
+
 def lan_addresses() -> list[str]:
     """This PC's IPv4 addresses on its networks (what another device types in), without loopback/link-local."""
     import socket
@@ -2275,7 +2593,114 @@ def lan_addresses() -> list[str]:
     return ([first] if ok(first) else []) + sorted(ip for ip in ips if ok(ip) and ip != first)
 
 
+def host_name(value) -> str:
+    """The name in a Host header (or an origin's host[:port]): "Example.com:8080" -> "example.com",
+    "[::1]:8095" -> "::1"; "" when it is malformed."""
+    v = (value or "").strip().lower()
+    if v.startswith("["):
+        name, sep, rest = v[1:].partition("]")
+        return name if sep and (not rest or (rest[:1] == ":" and rest[1:].isdigit())) else ""
+    if v.count(":") == 1:
+        v, port = v.split(":")
+        if not port.isdigit():
+            return ""
+    elif ":" in v:                                       # a bare IPv6 address (a config entry)
+        return v
+    v = v.rstrip(".")
+    return v if v and all(c.isalnum() or c in "-._" for c in v) else ""
+
+
+def _is_ip(name: str) -> bool:
+    import ipaddress
+    try:
+        ipaddress.ip_address(name)
+        return True
+    except ValueError:
+        return False
+
+
+def _name_in(name: str, names) -> bool:
+    """`name` is one of `names`, or below an entry that starts with a dot (".example.com")."""
+    return name in names or any(n.startswith(".") and (name.endswith(n) or name == n[1:]) for n in names)
+
+
+def allowed_hosts_of(value, env: str = "") -> list[str]:
+    """The config's allowed_hosts (a name or a list) plus $STRATA_ALLOWED_HOSTS (comma-separated): host names, "*" or
+    ".example.com" (it and every name below it).  A scheme, port or path is dropped ("https://a.example.com:8443/"
+    -> "a.example.com"); a wrong entry stops the start (ValueError)."""
+    items = [] if value in (None, "") else [value] if isinstance(value, str) else value
+    if not isinstance(items, list) or not all(isinstance(x, str) for x in items):
+        raise ValueError("allowed_hosts: expected a host name or a list of them")
+    out = []
+    for raw in items + [x for x in env.split(",") if x.strip()]:
+        x = raw.strip().lower()
+        if x == "*":
+            out.append(x)
+            continue
+        x = x.split("://", 1)[-1].split("/", 1)[0]
+        dot = x.startswith(".")
+        name = host_name(x[1:] if dot else x)
+        if not name:
+            raise ValueError(f"allowed_hosts: {raw!r} is not a host name like strata.example.com")
+        out.append("." + name if dot else name)
+    return list(dict.fromkeys(out))
+
+
+def host_names_for(bind_host: str, allowed_hosts=(), trusted_origins=()) -> set[str]:
+    """Every name this server answers to besides an IP address: localhost, the address it listens on, the config's
+    allowed_hosts and trusted_origins' hosts, and - listening beyond this PC (0.0.0.0 or a LAN address) - this PC's
+    name and LAN addresses (the LAN addresses matter for the Origin check, which takes no IP on trust)."""
+    names = set(LOOPBACK_NAMES)
+    bind = host_name(bind_host)
+    if bind and bind not in ("0.0.0.0", "::"):
+        names.add(bind)
+    if bind not in LOOPBACK_NAMES:
+        try:
+            pc = socket.gethostname().lower()
+            names.update((pc, pc + ".local"))
+        except OSError:
+            pass
+        names.update(("host.docker.internal", *lan_addresses()))
+    names.update(x for x in allowed_hosts if x != "*")
+    names.update(filter(None, (host_name(o.split("://", 1)[-1]) for o in trusted_origins)))
+    return names
+
+
+def host_allowed(host, names, any_host=False) -> bool:
+    """DNS rebinding: a web page of another site whose name its DNS points at 127.0.0.1 reaches this server as the
+    same origin, so the browser lets it read every answer.  Its requests carry that site's name in Host, so only
+    the names this server answers to pass.  An IP address passes (a page served from an IP is that IP's own page;
+    rebinding needs a name), as does "*.localhost" (browsers never ask DNS for it) and a request without a Host
+    header (HTTP/1.0 clients; browsers always send one)."""
+    if any_host or not (host or "").strip():
+        return True
+    name = host_name(host)
+    return bool(name) and (_is_ip(name) or name == "localhost" or name.endswith(".localhost")
+                           or _name_in(name, names))
+
+
+def origin_allowed(origin: str, host, names, origins=()) -> bool:
+    """A browser page's Origin that may use the model without an API key: this server's own page (the Origin is the
+    request's own Host), a page on one of the names this server answers to (any port), or an origin the config lists
+    (trusted_origins, cors_origins).  Unlike the Host check no IP passes on trust: a page served from any other IP is
+    another site.  "null" (a sandboxed frame, a file:// page) does not pass: any site can send it.  An origin of another
+    scheme (chrome-extension://, moz-extension://, an Electron app's app://) does: no web site can send one."""
+    origin = (origin or "").strip().rstrip("/")
+    if origin in origins or "*" in origins:             # cors_origins ["*"]: the config lets every page in
+        return True
+    scheme, sep, rest = origin.lower().partition("://")
+    if not sep or not scheme:
+        return False
+    if scheme not in ("http", "https"):
+        return True
+    if host and rest == host.strip().lower():
+        return True
+    name = host_name(rest)
+    return bool(name) and (name in LOOPBACK_NAMES or name.endswith(".localhost") or _name_in(name, names))
+
+
 def serve(svc: Service, host="127.0.0.1", port=8095) -> ThreadingHTTPServer:
+    svc.host_names = host_names_for(host, svc.allowed_hosts, svc.trusted_origins)
     svc.start_telemetry()
     httpd = Server((host, port), make_handler(svc))
     threading.Thread(target=httpd.serve_forever, daemon=True).start()
@@ -2493,8 +2918,17 @@ def main() -> int:
         # a relative "exe" is the config's cwd's: Windows' CreateProcess resolves "engine/strata.exe" against nothing
         # it is told about (WinError 2), so it is made absolute here
         exe = cfg["exe"] if os.path.isabs(cfg["exe"]) else os.path.abspath(os.path.join(cfg.get("cwd") or ".", cfg["exe"]))
+        try:
+            silence = engine_silence_s(cfg)             # #481: checked before the (minutes-long) start
+        except ValueError as e:
+            raise SystemExit(f"[strata] config {e}")
         engine = StrataEngine(exe, engine_args(cfg), cwd=cfg.get("cwd"), log=cfg.get("log"), env=env, lazy=lazy)
+        engine.silence_s = silence                      # an attribute of its own: restart() keeps it
         warn_tight_ram(engine.info.get("arena_mib"))
+        note = desktop_vram_note(cfg.get("backend"), engine.info.get("vram_free_mib"), engine.spawn[1],
+                                 linux_desktop())
+        if note:                                        # #560 #516: before --open starts a browser on that card
+            print(note, flush=True)
     else:
         engine, vision, sampling_defaults = MockEngine(tok, a.script or [
             "Thinking about it.</think>\n\nHello from the mock engine."]), None, {}
@@ -2519,6 +2953,13 @@ def main() -> int:
     svc.api_key = a.api_key or cfg.get("api_key", "")
     svc.cors_origins = origins_of(cfg.get("cors_origins"), "cors_origins", wildcard=True)
     svc.trusted_origins = origins_of(cfg.get("trusted_origins"), "trusted_origins", wildcard=False)
+    try:
+        svc.allowed_hosts = allowed_hosts_of(cfg.get("allowed_hosts"), os.environ.get("STRATA_ALLOWED_HOSTS", ""))
+    except ValueError as e:
+        raise SystemExit(f"[strata] config {e}")
+    if svc.allowed_hosts:
+        print("[strata] Host check off: any name reaches this server (allowed_hosts \"*\")" if "*" in svc.allowed_hosts
+              else f"[strata] also answers to the host names {', '.join(svc.allowed_hosts)} (allowed_hosts)", flush=True)
     svc.api_monitor = a.api_monitor or cfg.get("api_monitor") is True
     if svc.api_monitor:
         print("[strata] API request monitor on (/api-monitor): the last 100 requests' prompts and answers are kept in "

serve/test_security.py

@@ -0,0 +1,255 @@
+"""serve/test_security.py - the Host check (DNS rebinding) and the Origin check for requests without an API key,
+against the mock engine (no GPU, no pack).
+
+    python -m unittest serve.test_security -v
+"""
+from __future__ import annotations
+
+import contextlib
+import http.client
+import io
+import json
+import socket
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from serve.frontend import ChatTemplate  # noqa: E402
+from serve.server import (ByteTokenizer, MockEngine, Service, allowed_hosts_of, host_allowed,  # noqa: E402
+                          host_name, host_names_for, origin_allowed, serve)
+
+ROOT = Path(__file__).resolve().parents[1]
+LOCAL = {"localhost", "127.0.0.1", "::1"}
+
+
+class HostNames(unittest.TestCase):
+    def test_host_name(self):
+        self.assertEqual(host_name("Example.COM:8080"), "example.com")
+        self.assertEqual(host_name("localhost"), "localhost")
+        self.assertEqual(host_name("[::1]:8095"), "::1")
+        self.assertEqual(host_name("[::1]"), "::1")
+        self.assertEqual(host_name("strata.example.com."), "strata.example.com")
+        for bad in ("", "a:b", "[::1", "[::1]x", "a b", "evil.com/x"):
+            self.assertEqual(host_name(bad), "", bad)
+
+    def test_allowed_hosts_of(self):
+        self.assertEqual(allowed_hosts_of(None), [])
+        self.assertEqual(allowed_hosts_of("Strata.Example.com"), ["strata.example.com"])
+        self.assertEqual(allowed_hosts_of(["https://a.example.com:8443/x", ".example.org", "*"], "box, nas.lan "),
+                         ["a.example.com", ".example.org", "*", "box", "nas.lan"])
+        for bad in (["bad name"], 5, [3], "http://"):
+            with self.assertRaises(ValueError):
+                allowed_hosts_of(bad)
+
+    def test_names_from_the_bind_address(self):
+        with mock.patch("serve.server.lan_addresses", return_value=["192.168.1.20"]):
+            self.assertEqual(host_names_for("127.0.0.1"), LOCAL)                 # this PC only: nothing else
+            self.assertEqual(host_names_for("localhost"), LOCAL)
+            names = host_names_for("0.0.0.0")                                    # other devices too
+            self.assertIn(socket.gethostname().lower(), names)
+            self.assertIn(socket.gethostname().lower() + ".local", names)
+            self.assertIn("192.168.1.20", names)
+            self.assertIn("10.0.0.7", host_names_for("10.0.0.7"))
+            names = host_names_for("127.0.0.1", ["strata.example.com", "*"], ["https://chat.example.net"])
+            self.assertTrue({"strata.example.com", "chat.example.net"} <= names)
+            self.assertNotIn("*", names)
+
+
+class HostCheck(unittest.TestCase):
+    def test_accepted(self):
+        for host in ("localhost:8095", "LOCALHOST", "127.0.0.1:1", "[::1]:8095", "192.168.1.20:8095", "[fe80::1]",
+                     "app.localhost:8095", None, "", "box:8095", "a.example.org", "example.org:443"):
+            self.assertTrue(host_allowed(host, LOCAL | {"box", ".example.org"}), host)
+
+    def test_refused(self):
+        # a rebinding page sends its own name; lookalikes of allowed names stay out
+        for host in ("evil.example.com", "evil.example.com:8095", "localhost.evil.com", "127.0.0.1.nip.io",
+                     "box.evil.com", "xexample.org", "bad host", "a:b:c:d:e:f:g:h:i"):
+            self.assertFalse(host_allowed(host, LOCAL | {"box", ".example.org"}), host)
+
+    def test_any_host(self):
+        self.assertTrue(host_allowed("evil.example.com", LOCAL, any_host=True))
+
+
+class OriginCheck(unittest.TestCase):
+    def test_accepted(self):
+        names = LOCAL | {"box"}
+        self.assertTrue(origin_allowed("http://127.0.0.1:8095", "127.0.0.1:8095", names))      # Strata's own page
+        self.assertTrue(origin_allowed("http://192.168.1.20:8095", "192.168.1.20:8095", names))  # ... by LAN IP
+        self.assertTrue(origin_allowed("http://localhost:3000", "127.0.0.1:8095", names))      # a local app
+        self.assertTrue(origin_allowed("http://[::1]:3000/", "127.0.0.1:8095", names))
+        self.assertTrue(origin_allowed("https://box", "127.0.0.1:8095", names))
+        self.assertTrue(origin_allowed("https://chat.example.com", "x", names, ["https://chat.example.com"]))
+        self.assertTrue(origin_allowed("https://any.example.com", "x", names, ["*"]))           # cors_origins ["*"]
+        # browser extensions and desktop apps: no web site can send another scheme than http(s) (or "null")
+        for origin in ("chrome-extension://abcdef", "moz-extension://1234-5678", "app://."):
+            self.assertTrue(origin_allowed(origin, "127.0.0.1:8095", names), origin)
+
+    def test_refused(self):
+        names = LOCAL | {"box"}
+        for origin in ("http://evil.example.com", "http://1.2.3.4", "http://192.168.1.99:8095", "null", "",
+                       "http://localhost.evil.com", "https://box.evil.com", "://x"):
+            self.assertFalse(origin_allowed(origin, "127.0.0.1:8095", names, ["https://chat.example.com"]), origin)
+
+
+class OverHttp(unittest.TestCase):
+    """The checks in the running server, as a browser and as curl/the SDKs send their requests."""
+
+    def setUp(self):
+        tok = ByteTokenizer()
+        self.svc = Service(MockEngine(tok, "</think>\n\nok", max_context=4096), tok,
+                           ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        self.httpd = None
+
+    def start(self, **attrs):
+        for k, v in attrs.items():
+            setattr(self.svc, k, v)
+        self.httpd = serve(self.svc, port=0)
+        self.port = self.httpd.server_address[1]
+
+    def tearDown(self):
+        if self.httpd:
+            self.httpd.shutdown()
+            self.httpd.server_close()
+
+    def req(self, method, path, body=None, headers=None, host=None):
+        """(status, JSON, what the server printed); `host` replaces the Host header http.client sends."""
+        c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
+        h = dict(headers or {})
+        if host is not None:
+            h["Host"] = host
+        data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode()
+        out = io.StringIO()
+        try:
+            with contextlib.redirect_stdout(out):
+                c.request(method, path, body=data, headers=h)
+                r = c.getresponse()
+                raw = r.read()
+            return r.status, json.loads(raw) if raw[:1] in (b"{", b"[") else raw, out.getvalue()
+        finally:
+            c.close()
+
+    def chat_body(self):
+        return {"model": "m", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 64}
+
+    # --- Host
+    def test_rebinding_name_is_refused_everywhere(self):
+        self.start()
+        for method, path in (("GET", "/status"), ("GET", "/metrics"), ("GET", "/"), ("GET", "/health"),
+                             ("POST", "/settings"), ("POST", "/v1/chat/completions"), ("OPTIONS", "/v1/models")):
+            code, body, log = self.req(method, path, {} if method == "POST" else None,
+                                       {"Content-Type": "application/json"}, host=f"evil.example.com:{self.port}")
+            self.assertEqual(code, 403, path)
+            if method != "OPTIONS":
+                self.assertIn("allowed_hosts", body["error"]["message"])
+            self.assertEqual(log.count("\n"), 1, path)                          # one log line each
+            self.assertIn("evil.example.com", log)
+
+    def test_local_names_and_ips_pass(self):
+        self.start()
+        for host in (f"127.0.0.1:{self.port}", f"localhost:{self.port}", f"[::1]:{self.port}",
+                     f"192.168.1.20:{self.port}"):
+            self.assertEqual(self.req("GET", "/status", host=host)[0], 200, host)
+
+    def test_no_host_header_passes(self):
+        self.start()
+        with socket.create_connection(("127.0.0.1", self.port), timeout=10) as s:      # an HTTP/1.0 client
+            s.sendall(b"GET /health HTTP/1.0\r\n\r\n")
+            raw = b""
+            while chunk := s.recv(65536):
+                raw += chunk
+        self.assertTrue(raw.startswith(b"HTTP/1.0 200"), raw[:40])
+
+    def test_allowed_hosts_and_wildcard(self):
+        self.start(allowed_hosts=["strata.example.com", ".home.arpa"])
+        self.assertEqual(self.req("GET", "/status", host="strata.example.com")[0], 200)
+        self.assertEqual(self.req("GET", "/status", host="nas.home.arpa:8095")[0], 200)
+        self.assertEqual(self.req("GET", "/status", host="evil.example.com")[0], 403)
+        self.svc.allowed_hosts = ["*"]
+        self.assertEqual(self.req("GET", "/status", host="evil.example.com")[0], 200)
+
+    def test_refusal_names_both_ways_out(self):
+        self.start()
+        message = self.req("GET", "/status", host="evil.example.com")[1]["error"]["message"]
+        self.assertIn("allowed_hosts", message)
+        self.assertIn("api_key", message)
+
+    def test_with_a_key_the_host_check_is_off(self):
+        # a rebinding page cannot authenticate; tunnels and proxies that pass their own name on keep working
+        self.start(api_key="s3cret")
+        tunnel = "random-words.trycloudflare.com"
+        self.assertEqual(self.req("GET", "/status", host=tunnel)[0], 401)
+        self.assertEqual(self.req("GET", "/status", headers={"Authorization": "Bearer s3cret"}, host=tunnel)[0], 200)
+        self.assertEqual(self.req("GET", "/health", host=tunnel)[0], 200)
+        code, _, log = self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                {"Content-Type": "application/json", "Authorization": "Bearer s3cret"}, host=tunnel)
+        self.assertEqual(code, 200)
+        self.assertNotIn("refused", log)
+
+    def test_a_trusted_origin_s_host_is_allowed(self):
+        self.start(trusted_origins=["https://strata.example.com"])
+        self.assertEqual(self.req("GET", "/status", host="strata.example.com")[0], 200)
+
+    # --- Origin on /v1 without an API key
+    def test_curl_and_sdks_without_origin(self):
+        self.start()
+        for ctype in ("application/json", "text/plain", None):              # curl -d sends a form type
+            code, body, _ = self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                     {"Content-Type": ctype} if ctype else {})
+            self.assertEqual(code, 200, ctype)
+            self.assertEqual(body["choices"][0]["message"]["content"], "ok")
+
+    def test_cross_site_page_is_refused(self):
+        self.start()
+        for path in ("/v1/chat/completions", "/v1/messages", "/v1/messages/count_tokens"):
+            for ctype in ("text/plain", "application/json"):
+                code, body, log = self.req("POST", path, self.chat_body(),
+                                           {"Content-Type": ctype, "Origin": "http://evil.example.com"})
+                self.assertEqual(code, 403, (path, ctype))
+                self.assertIn("api_key", body["error"]["message"])
+                self.assertIn("evil.example.com", log)
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                  {"Content-Type": "text/plain", "Origin": "null"})[0], 403)
+
+    def test_own_and_local_pages_pass(self):
+        self.start()
+        json_type = {"Content-Type": "application/json"}
+        for origin in (f"http://127.0.0.1:{self.port}", "http://localhost:3000", "chrome-extension://abcdef"):
+            self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                      {**json_type, "Origin": origin})[0], 200, origin)
+        # a browser page sends JSON; text/plain from a page is the cross-site "simple request" shape
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                  {"Content-Type": "text/plain", "Origin": f"http://127.0.0.1:{self.port}"})[0], 415)
+
+    def test_configured_origins_pass(self):
+        self.start(cors_origins=["https://chat.example.com"], trusted_origins=["https://strata.example.com"],
+                   allowed_hosts=["webui.lan"])
+        for origin in ("https://chat.example.com", "https://strata.example.com", "http://webui.lan:3000"):
+            self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                      {"Content-Type": "application/json", "Origin": origin})[0], 200, origin)
+
+    def test_with_a_key_the_key_decides(self):
+        self.start(api_key="s3cret")
+        headers = {"Content-Type": "text/plain", "Origin": "http://evil.example.com"}
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(), headers)[0], 401)
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                  {**headers, "Authorization": "Bearer s3cret"})[0], 200)
+
+    # --- /unload and /load
+    def test_unload_and_load_need_json_from_the_own_page(self):
+        self.start()
+        form = {"Content-Type": "application/x-www-form-urlencoded"}
+        for path in ("/unload", "/load"):
+            self.assertEqual(self.req("POST", path, b"a=1", form)[0], 415, path)
+            self.assertEqual(self.req("POST", path, b"", {})[0], 415, path)
+            self.assertEqual(self.req("POST", path, {}, {"Content-Type": "application/json",
+                                                         "Origin": "http://evil.example.com"})[0], 403, path)
+        self.assertEqual(self.req("POST", "/unload", {}, {"Content-Type": "application/json"})[0], 200)
+        self.assertEqual(self.req("POST", "/load", {}, {"Content-Type": "application/json",
+                                                        "Origin": f"http://127.0.0.1:{self.port}"})[0], 200)
+
+
+if __name__ == "__main__":
+    unittest.main()

serve/test_server.py

@@ -4,6 +4,8 @@
 """
 from __future__ import annotations
 
+import contextlib
+import io
 import json
 import os
 import sys
@@ -18,7 +20,9 @@ from unittest import mock
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 from serve.frontend import ChatTemplate  # noqa: E402
-from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, GpuBusy, MockEngine, Service, StrataEngine, request_timings, serve  # noqa: E402
+from serve.server import (CTX_SLACK, ByteTokenizer, EngineDied, GpuBusy, MockEngine, Service, StrataEngine,  # noqa: E402
+                          engine_args, prompt_tokens_seen, request_timings, serve, start_failure_hint)
+from types import SimpleNamespace  # noqa: E402
 
 ROOT = Path(__file__).resolve().parents[1]
 CTX = 4096
@@ -95,6 +99,13 @@ class MaxTokens(unittest.TestCase):
         m = ENGINE_REQUEST.search(line)
         self.assertIsNotNone(m)
         self.assertEqual((m["prompt"], m["reused"], m["gen"], m["tg"]), ("1200", "1000", "30", "100.0"))
+        # #471: a request cancelled while its prompt was read says how far it got
+        m = ENGINE_REQUEST.search("strata serve: prompt 98179 tokens = 0 reused + 12288 of 98179 read in 17565 ms "
+                                  "(699.6 tok/s), 0 generated in 0 ms (0.0 tok/s), drafts accepted 0 of 0, "
+                                  "0 checkpoints (cancelled)")
+        self.assertIsNotNone(m)
+        self.assertEqual((m["prompt"], m["reused"], m["read"], m["pp"], m["gen"]),
+                         ("98179", "0", "17565", "699.6", "0"))
 
     def test_unset_budget_is_the_rest_of_the_context(self):
         cases = {"openai": [{"max_tokens": -1}, {"max_tokens": 0}, {}, {"max_tokens": None},
@@ -126,6 +137,8 @@ class MaxTokens(unittest.TestCase):
                 s, b, _, _ = self.call(api, max_tokens=CTX)
                 self.assertEqual(s, 400)
                 self.assertIn("exceeds the context", b["error"]["message"])
+                self.assertIn("\"fit_max_tokens\": true", b["error"]["message"])     # #545: says how to get past it
+                self.assertRegex(b["error"]["message"], r"at most \d+ here")
 
     def test_unset_budget_with_a_near_full_prompt(self):
         _, _, pt0, _ = self.call("openai", max_tokens=1)
@@ -667,7 +680,7 @@ class ClientHangUp(unittest.TestCase):
         body = json.dumps({"model": "x", "max_tokens": 20, "stream": stream,
                            "messages": [{"role": "user", "content": "a long prompt"}]}).encode()
         c = so.create_connection(("127.0.0.1", self.port))
-        c.sendall(b"POST /v1/chat/completions HTTP/1.1\r\nHost: x\r\nContent-Type: application/json\r\n"
+        c.sendall(b"POST /v1/chat/completions HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: application/json\r\n"
                   b"Content-Length: " + str(len(body)).encode() + b"\r\n\r\n" + body)
         time.sleep(1.0)
         c.close()
@@ -792,6 +805,239 @@ class DraftCounts(unittest.TestCase):
         self.assertEqual((m["totals"]["drafts_offered"], m["totals"]["drafts_accepted"]), (17, 10))
 
 
+class LearnedProfile(unittest.TestCase):
+    """#477: "expert_profile_save" in the config: the engine saves its learned profile there, and the next start
+    begins from it when it is a profile of the same model; without the key the arguments are unchanged."""
+
+    def write(self, path, nl=48, ne=512, n=4):
+        sys.path.insert(0, str(ROOT / "tools"))
+        import make_profile
+        old = make_profile.N_LAYER
+        make_profile.N_LAYER = nl
+        try:
+            make_profile.write_profile(path, [(i % nl, i // nl) for i in range(n)], n_expert=ne)
+        finally:
+            make_profile.N_LAYER = old
+
+    def test_without_the_key_nothing_changes(self):
+        args = ["--native", "x", "--expert-profile", "data/expert-profile.bin", "--adapt-every", "4"]
+        self.assertEqual(engine_args({"args": list(args)}), args)
+        self.assertEqual(engine_args({"args": list(args), "expert_profile_save": ""}), args)
+
+    def test_save_and_start_from_it(self):
+        d = Path(tempfile.mkdtemp())
+        self.write(d / "base.bin")
+        cfg = {"args": ["--expert-profile", "base.bin"], "cwd": str(d), "expert_profile_save": "learned.bin",
+               "expert_profile_save_every": 5}
+        # nothing saved yet: the config's profile, and the engine is told where to save
+        self.assertEqual(engine_args(cfg), ["--expert-profile", "base.bin", "--expert-profile-save", "learned.bin",
+                                            "--expert-profile-save-every", "5"])
+        self.write(d / "learned.bin")
+        self.assertEqual(engine_args(cfg)[:2], ["--expert-profile", "learned.bin"])
+        self.write(d / "learned.bin", ne=256)                    # another model's: not used
+        self.assertEqual(engine_args(cfg)[:2], ["--expert-profile", "base.bin"])
+        (d / "learned.bin").write_bytes(b"STRP" + bytes(20))     # not a whole profile
+        self.assertEqual(engine_args(cfg)[:2], ["--expert-profile", "base.bin"])
+        self.write(d / "learned.bin")
+        (d / "learned.bin").write_bytes((d / "learned.bin").read_bytes()[:30])   # truncated
+        self.assertEqual(engine_args(cfg)[:2], ["--expert-profile", "base.bin"])
+
+    def test_no_profile_in_the_args(self):
+        cfg = {"args": ["--native", "x"], "expert_profile_save": "learned.bin"}
+        self.assertEqual(engine_args(cfg), ["--native", "x", "--expert-profile-save", "learned.bin"])
+
+
+class DraftHeadHint(unittest.TestCase):
+    """#474: a start that stopped at "the draft head does not fit" says what to change, from this start's log lines."""
+
+    def log(self, text, before=""):
+        d = tempfile.mkdtemp()
+        p = Path(d) / "engine.log"
+        p.write_text(before + text, encoding="utf-8")
+        return str(p), len(before.encode())
+
+    def test_the_engines_hint_is_relayed(self):
+        p, off = self.log("strata mtp: the draft head over 106299 tokens needs 348 MiB of VRAM and 120 MiB is free.\n"
+                          "strata mtp: hint: a smaller draft vocabulary needs less VRAM: --draft-vocab en (...)\n"
+                          "strata serve: mtp: the draft head does not fit\n")
+        h = start_failure_hint(p, off)
+        self.assertIn("the draft head does not fit", h)
+        self.assertIn("348 MiB", h)
+        self.assertIn("--draft-vocab en", h)
+
+    def test_an_older_engine_gets_the_advice_in_words(self):
+        p, off = self.log("strata serve: mtp: the draft head does not fit\n")
+        self.assertIn("--draft-vocab en", start_failure_hint(p, off))
+
+    def test_other_failures_and_earlier_starts_add_nothing(self):
+        p, off = self.log("strata serve: cannot open the pack\n")
+        self.assertEqual(start_failure_hint(p, off), "")
+        # an earlier start's failure (before this start's offset) is not this one's
+        p, off = self.log("strata serve: cannot open the pack\n",
+                          before="strata serve: mtp: the draft head does not fit\n")
+        self.assertEqual(start_failure_hint(p, off), "")
+        self.assertEqual(start_failure_hint(None, 0), "")
+        self.assertEqual(start_failure_hint(str(Path(tempfile.mkdtemp()) / "missing.log"), 0), "")
+
+
+class DesktopVramNote(unittest.TestCase):
+    """#560 #516: an AMD card on a Linux desktop with little VRAM left after the start gets a recommended reserve."""
+
+    def test_when_it_applies(self):
+        from serve.server import desktop_vram_note
+        note = desktop_vram_note("hip", 624, ["--kv", "int8"], True)
+        self.assertIn("624 MiB of VRAM free", note)
+        self.assertIn("--vram-reserve-mib 3072", note)
+        self.assertIn("2.3 GB less", note)
+
+    def test_when_it_does_not(self):
+        from serve.server import desktop_vram_note
+        self.assertEqual(desktop_vram_note(None, 624, [], True), "")               # NVIDIA
+        self.assertEqual(desktop_vram_note("hip", 624, [], False), "")             # no desktop session
+        self.assertEqual(desktop_vram_note("hip", 2994, [], True), "")             # room left
+        self.assertEqual(desktop_vram_note("hip", None, [], True), "")             # lazy start: no INFO yet
+        self.assertEqual(desktop_vram_note("hip", 900, ["--vram-reserve-mib", "4000"], True), "")   # already raised
+
+    def test_desktop_detection(self):
+        from serve import server
+        with mock.patch.object(server.os, "name", "posix"), mock.patch.object(server.sys, "platform", "linux"):
+            self.assertTrue(server.linux_desktop({"WAYLAND_DISPLAY": "wayland-0"}))
+            self.assertFalse(server.linux_desktop({}))
+
+
+class StartFailureLog(unittest.TestCase):
+    """#496: whatever stopped the engine before READY, the error carries this start's last log lines."""
+
+    def test_the_last_lines_of_this_start(self):
+        from serve.server import start_log_tail
+        d = tempfile.mkdtemp()
+        p = Path(d) / "engine.log"
+        before = "strata serve: an earlier start's line\n"
+        p.write_text(before + "".join(f"strata serve: line {i}\n" for i in range(30)) + "\n"
+                     "strata serve: cannot open the pack\n", encoding="utf-8")
+        tail = start_log_tail(str(p), len(before.encode()))
+        self.assertIn("the engine log's last lines:", tail)
+        self.assertIn("cannot open the pack", tail)
+        self.assertIn("line 29", tail)
+        self.assertNotIn("line 10\n", tail + "\n")                 # 20 lines: 11..29 and the last one
+        self.assertIn("line 11", tail)
+        self.assertNotIn("earlier start", tail)
+        short = start_log_tail(str(p), len(before.encode()), n=3)
+        self.assertEqual(short.count("\n  "), 3)
+        self.assertEqual(start_log_tail(str(p), p.stat().st_size), "")   # nothing from this start
+        self.assertEqual(start_log_tail(None, 0), "")
+        self.assertEqual(start_log_tail(str(Path(d) / "missing.log"), 0), "")
+
+    def test_the_start_error_has_them(self):
+        import serve.server as server
+        fake = "import sys\nsys.stderr.write('strata serve: cannot open the pack packs/x\\n')\nsys.exit(2)\n"
+        with tempfile.TemporaryDirectory() as d:
+            script, log = Path(d) / "fake_strata.py", Path(d) / "strata.log"
+            script.write_text(fake, encoding="utf-8")
+            log.write_text("an earlier start\n", encoding="utf-8")
+            real = server.subprocess.Popen
+            with mock.patch.object(server.subprocess, "Popen",
+                                   lambda cmd, **kw: real([sys.executable, str(script), *cmd[1:]], **kw)), \
+                    mock.patch.object(server, "narrate_start", lambda *a, **k: None):
+                with self.assertRaises(RuntimeError) as cm:
+                    StrataEngine("strata", [], log=str(log))
+            text = str(cm.exception)
+            self.assertIn("exited before it was ready", text)
+            self.assertIn("cannot open the pack packs/x", text)
+            self.assertNotIn("an earlier start", text)
+
+
+class StartNarrator(unittest.TestCase):
+    """#505: the start's words say what happens to the experts - --mmap-experts loads nothing into RAM up front."""
+
+    def test_the_words_follow_the_flags(self):
+        from serve.server import experts_loading_words
+        self.assertIn("loading the experts into RAM (about 38 GB)", experts_loading_words([], "about 38 GB"))
+        mapped = experts_loading_words(["--mmap-experts"], "about 47 GB")
+        self.assertIn("mapping the experts from the model files (about 47 GB", mapped)
+        self.assertIn("not loaded into RAM", mapped)
+        self.assertIn("the GPU does not hold", experts_loading_words(["--resident-experts"], "about 47 GB"))
+        budget = experts_loading_words(["--mmap-experts", "--resident-budget-gib", "71"], "about 50 GB")
+        self.assertIn("up to 71 GiB", budget)
+        self.assertNotIn("about 50 GB", budget)
+
+    def narrate(self, args, lines):
+        import contextlib
+        import io
+        from serve.server import narrate_start
+        d = tempfile.mkdtemp()
+        log = Path(d) / "engine.log"
+        log.write_text("".join(x + "\n" for x in lines), encoding="utf-8")
+        done, out = threading.Event(), io.StringIO()
+        with contextlib.redirect_stdout(out):
+            t = threading.Thread(target=narrate_start, args=(str(log), 0, args, done))
+            t.start()
+            time.sleep(0.8)
+            done.set()
+            t.join(5)
+        return out.getvalue()
+
+    def test_a_mapped_start(self):
+        said = self.narrate(["--mmap-experts"], ["strata generate: experts via mmap (--mmap-experts; the GGUF shards "
+                                                 "in place, no experts.bin)"])
+        self.assertIn("mapping the experts from the model files", said)
+        self.assertNotIn("loading the experts into RAM", said)
+
+    def test_an_arena_start(self):
+        said = self.narrate([], ["strata generate: expert arena: resident, 31.64 GiB",
+                                 "strata generate: loaded 31.64 GiB at 3.17 GiB/s"])
+        self.assertIn("loading the experts into RAM (tens of GB)", said)
+        self.assertIn("experts loaded: 31.64 GiB at 3.17 GiB/s", said)
+
+
+class CancelledRead(unittest.TestCase):
+    """#471: a request cancelled while its prompt was read is recorded with the tokens the engine read (the DONE
+    line's 15th field), not the whole prompt; an older engine's line (no such field) keeps the whole prompt."""
+
+    def test_parse_done_read_field(self):
+        e = SimpleNamespace()
+        StrataEngine._parse_done(e, "DONE 0 98179 17565.0 0.0 cancel 0 0 0 0 0 0 0 0.0 12288")
+        self.assertEqual((e.last["prompt_tokens"], e.last["prompt_read"], e.last["finish"]), (98179, 12288, "cancel"))
+        StrataEngine._parse_done(e, "DONE 0 98179 17565.0 0.0 cancel 0 0 0 0 0 0 0 0.0")
+        self.assertNotIn("prompt_read", e.last)
+
+    def test_prompt_tokens_seen(self):
+        last = {"finish": "cancel", "reused": 1000, "prompt_read": 2000}
+        self.assertEqual(prompt_tokens_seen(98179, last), 3000)
+        self.assertEqual(prompt_tokens_seen(98179, {**last, "finish": "stop"}), 98179)   # read in full: all of it
+        self.assertEqual(prompt_tokens_seen(98179, {"finish": "cancel", "reused": 0}), 98179)   # an older engine
+        self.assertEqual(prompt_tokens_seen(98179, {}), 98179)                          # no DONE at all
+        self.assertEqual(prompt_tokens_seen(10, {**last, "prompt_read": 50}), 10)       # never past the prompt
+
+    def test_history_and_totals(self):
+        tok = ByteTokenizer()
+        engine = DoneLineEngine(tok, "</think>\n\nok", max_context=CTX, done_lines=[
+            "DONE 0 20 400.0 0.0 cancel 0 0 0 0 0 0 0 0.0 8",      # stopped after 8 prompt tokens
+            "DONE 4 20 40.0 30.0 stop 0 0 0 0 0 0 0 0.0 20",
+            "DONE 0 20 400.0 0.0 cancel 0 0 0"])                   # an older engine: no read count
+        svc = Service(engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        httpd = serve(svc, port=0)
+        base = f"http://127.0.0.1:{httpd.server_address[1]}"
+        try:
+            for _ in range(3):
+                body = json.dumps({"model": "m", "max_tokens": 10, "messages": [{"role": "user", "content": "hi"}]})
+                with urllib.request.urlopen(urllib.request.Request(base + "/v1/chat/completions", data=body.encode(),
+                                                                   headers={"Content-Type": "application/json"}),
+                                            timeout=30) as r:
+                    self.assertEqual(r.status, 200)
+            with urllib.request.urlopen(base + "/metrics", timeout=10) as r:
+                m = json.loads(r.read())
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+        rows = m["requests"][::-1]                                # oldest first
+        total = rows[0]["prompt_total"]
+        self.assertGreater(total, 8)
+        self.assertEqual([r["prompt_total"] for r in rows], [total] * 3)
+        self.assertEqual([(r["prompt_tokens"], r["prompt_read"]) for r in rows], [(8, 8), (total, 20), (total, None)])
+        self.assertEqual(m["totals"]["prompt_tokens"], 8 + 2 * total)
+
+
 class LiveRate(unittest.TestCase):
     """The Monitor's Speed readout: live.tok_s is a rate, and a request that never got a DONE keeps no counters.
 
@@ -1594,6 +1840,20 @@ class ThinkingBudget(unittest.TestCase):
         self.assertEqual(len(self.engine.prompts), 1)
         self.assertEqual(b["choices"][0]["message"]["reasoning_content"], ThinkingEngine.THOUGHT[:20])
 
+    def test_a_reply_cut_while_thinking_is_named_in_the_log(self):
+        """#530: max tokens reached inside the thinking gives an empty answer; the server log says what helps."""
+        hint = "reached max tokens while still thinking"
+        for extra, said in (({"max_tokens": 10}, True), ({}, False)):
+            with self.subTest(extra=extra):
+                out = io.StringIO()
+                with contextlib.redirect_stdout(out):
+                    code, b = self.openai(**extra)
+                self.assertEqual(code, 200, b)
+                self.assertEqual(b["choices"][0]["finish_reason"], "length" if said else "stop")
+                self.assertEqual(hint in out.getvalue(), said, out.getvalue())
+                if said:
+                    self.assertIn("reasoning_budget_tokens", out.getvalue())
+
     def test_a_bad_value_is_a_400(self):
         for bad in ("lots", 2.5, True, [1]):
             with self.subTest(value=bad):
@@ -1911,5 +2171,163 @@ class AmdTelemetry(unittest.TestCase):
                 self.assertEqual(svc.free_vram_mib(), 26 << 10)
 
 
+class SilentEngine(unittest.TestCase):
+    """#481: an engine that prints nothing for engine_silence_s during a request (or never acknowledges a STOP) has
+    lost step with the server: it is ended and the request fails with EngineDied, instead of waiting forever."""
+
+    def bare(self, silence, can_stop=False):
+        import io
+        import queue
+        engine = StrataEngine.__new__(StrataEngine)
+        engine.proc = mock.Mock()
+        engine.proc.stdin = io.StringIO()
+        engine.proc.poll.return_value = None
+        engine.lines, engine.can_stop, engine.max_context = queue.Queue(), can_stop, 4096
+        engine.silence_s, engine.log_path = silence, None
+        return engine
+
+    def later(self, engine, delay, *lines):
+        def put():
+            time.sleep(delay)
+            for x in lines:
+                engine.lines.put(x)
+        threading.Thread(target=put, daemon=True).start()
+
+    def test_silence_mid_answer_ends_the_engine(self):
+        from serve.server import EngineSilent
+        engine = self.bare(0.3)
+        engine.lines.put("T 5")
+        gen = engine.generate([1], 10, {}, threading.Event())
+        self.assertEqual(next(gen), 5)
+        t0 = time.monotonic()
+        with self.assertRaises(EngineSilent) as cm:
+            next(gen)
+        self.assertIsInstance(cm.exception, EngineDied)          # every EngineDied path handles it
+        self.assertLess(time.monotonic() - t0, 5)
+        engine.proc.kill.assert_called_once()
+        self.assertFalse(engine.alive())                          # the next request restarts it
+        self.assertIn("#481", engine.death_note())
+        self.assertNotIn("STOP", engine.proc.stdin.getvalue())    # nothing is listening: no STOP, no drain
+
+    def test_prompt_chunks_set_the_wait(self):
+        # a PP line every second, at 100 tok/s: far over a 0.3 s silence, but each chunk is on time for its size
+        engine = self.bare(0.3)
+        engine.lines.put("RESUME 0")
+        engine.lines.put("PP 100 300 1000 100.0")
+        self.later(engine, 1.0, "PP 200 300 2000 100.0", "T 7", "DONE 1 300 2000 1 length")
+        self.assertEqual(list(engine.generate([1], 10, {}, threading.Event())), [None, None, 7])
+        engine.proc.kill.assert_not_called()
+
+    def test_a_long_first_chunk_is_allowed(self):
+        # 100 prompt tokens at the slowest prompt reading (50 tok/s): 2 s on top of the silence before the first PP
+        engine = self.bare(0.3)
+        self.later(engine, 1.0, "PP 100 100 1000 100.0", "DONE 0 100 1000 0 length")
+        self.assertEqual(list(engine.generate([1] * 100, 10, {}, threading.Event())), [None])
+        engine.proc.kill.assert_not_called()
+
+    def test_a_stop_never_acknowledged(self):
+        from serve.server import EngineSilent
+        engine = self.bare(0.3, can_stop=True)
+        engine.lines.put("T 5")
+        gen = engine.generate([1], 10, {}, threading.Event())
+        self.assertEqual(next(gen), 5)
+        with self.assertRaises(EngineSilent):
+            gen.close()                                           # the consumer stopped: STOP, then the drain
+        self.assertIn("STOP", engine.proc.stdin.getvalue())
+        engine.proc.kill.assert_called_once()
+        self.assertFalse(engine.alive())
+
+    def test_zero_waits_as_before(self):
+        engine = self.bare(0)
+        self.later(engine, 0.5, "T 5", "DONE 1 1 1 1 length")
+        self.assertEqual(list(engine.generate([1], 10, {}, threading.Event())), [5])
+
+    def test_config(self):
+        from serve.server import ENGINE_SILENCE_S, engine_silence_s
+        self.assertEqual(engine_silence_s({}), ENGINE_SILENCE_S)
+        self.assertEqual(engine_silence_s({"engine_silence_s": 0}), 0.0)
+        self.assertEqual(engine_silence_s({"engine_silence_s": 900}), 900.0)
+        for bad in (-1, "300", True):
+            with self.assertRaises(ValueError):
+                engine_silence_s({"engine_silence_s": bad})
+
+
+FAKE_LOST_STEP = '''import pathlib, sys, time
+mark = pathlib.Path(sys.argv[sys.argv.index("--mark") + 1])
+mode = sys.argv[sys.argv.index("--mode") + 1]
+print("READY 4096 stop", flush=True)
+for line in sys.stdin:
+    if line.startswith("QUIT"):
+        break
+    if line.startswith("GEN"):
+        if not mark.exists():                    # the first engine loses step (#481): it never says DONE
+            mark.touch()
+            print("T 104", flush=True)
+            if mode == "stop":
+                print("T 257", flush=True)       # <|im_end|>: the server STOPs and drains, and the engine is silent
+            time.sleep(3600)
+        print("T 111", flush=True)
+        print("T 107", flush=True)
+        print("DONE 2 5 1.0 1.0 length", flush=True)
+'''
+
+
+class LostStep(unittest.TestCase):
+    """#481 over HTTP, with a real process: the request with the silent engine ends with an error, and the next one
+    starts the engine again and is answered (the server used to wait forever, holding the request FIFO)."""
+
+    def run_mode(self, mode, stream):
+        import serve.server as server
+        with tempfile.TemporaryDirectory() as d:
+            script, mark = Path(d) / "fake_strata.py", Path(d) / "lost"
+            script.write_text(FAKE_LOST_STEP, encoding="utf-8")
+            real = server.subprocess.Popen
+            with mock.patch.object(server.subprocess, "Popen",
+                                   lambda cmd, **kw: real([sys.executable, str(script), *cmd[1:]], **kw)):
+                eng = StrataEngine("strata", ["--mark", str(mark), "--mode", mode])
+                eng.silence_s = 0.5
+                tok = ByteTokenizer()
+                svc = Service(eng, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+                httpd = serve(svc, port=0)
+                base = f"http://127.0.0.1:{httpd.server_address[1]}"
+                body = {"model": "m", "max_tokens": 2, "reasoning_effort": "none", "stream": stream,
+                        "messages": [{"role": "user", "content": "hi"}]}
+                try:
+                    first = eng.proc
+                    req = urllib.request.Request(base + "/v1/chat/completions", data=json.dumps(body).encode(),
+                                                 headers={"Content-Type": "application/json"})
+                    try:
+                        with urllib.request.urlopen(req, timeout=60) as r:
+                            text = r.read().decode()
+                    except urllib.error.HTTPError as e:
+                        self.assertEqual(e.code, 503)
+                        text = e.read().decode()
+                    self.assertIn("the next request restarts it", text)
+                    self.assertIsNotNone(first.poll(), "the silent engine still runs")
+                    self.assertEqual(svc.history[-1]["finish"], "error")
+                    with urllib.request.urlopen(req, timeout=60) as r:
+                        text = r.read().decode()
+                    self.assertIsNot(eng.proc, first)
+                    if stream:
+                        answer = "".join(json.loads(x[6:])["choices"][0]["delta"].get("content") or ""
+                                         for x in text.splitlines() if x.startswith("data: {") and "choices" in x)
+                    else:
+                        answer = json.loads(text)["choices"][0]["message"]["content"]
+                    self.assertEqual(answer, "ok")
+                finally:
+                    httpd.shutdown()
+                    httpd.server_close()
+                    eng.unload()
+
+    def test_silent_mid_answer(self):
+        self.run_mode("silent", stream=False)
+
+    def test_silent_mid_stream(self):
+        self.run_mode("silent", stream=True)
+
+    def test_stop_never_acknowledged(self):
+        self.run_mode("stop", stream=False)
+
+
 if __name__ == "__main__":
     unittest.main()

setup.py

@@ -50,6 +50,7 @@ import shutil
 import struct
 import subprocess
 import sys
+import textwrap
 import time
 import urllib.error
 import urllib.request
@@ -69,14 +70,23 @@ HF_REVISIONS = {
 }
 
 
+HF_DEFAULT = "https://huggingface.co"
+
+
+def hf_endpoint() -> str:
+    """#495: the Hugging Face host - HF_ENDPOINT as huggingface_hub reads it (a mirror, e.g. https://hf-mirror.com),
+    else huggingface.co.  The pinned revisions and the SHA-256 checks are the same whichever host serves the files."""
+    return (os.environ.get("HF_ENDPOINT") or "").strip().rstrip("/") or HF_DEFAULT
+
+
 def hf(repo: str) -> str:
     """The download folder of a Hugging Face repository at its pinned revision."""
-    return f"https://huggingface.co/{repo}/resolve/{HF_REVISIONS[repo]}/"
+    return f"{hf_endpoint()}/{repo}/resolve/{HF_REVISIONS[repo]}/"
 
 
 def hf_unpinned(url: str) -> str:
     """The same file at the repository's current revision (main)."""
-    return re.sub(r"(https://huggingface\.co/.+?/resolve/)[0-9a-f]{40}/", r"\1main/", url, count=1)
+    return re.sub(r"^(https?://[^/]+/.+?/resolve/)[0-9a-f]{40}/", r"\1main/", url, count=1)
 
 
 HF = hf("ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF")
@@ -94,7 +104,7 @@ PREBUILT_ASSET = "strata-windows-x64.zip" if WIN else "strata-linux-x64.zip"
 # the CUDA libraries the ready-made engine loads (the same CUDA 13.0 it is built with), from NVIDIA's pip packages
 CUDA_WHEELS = ["nvidia-cublas==13.0.2.14", "nvidia-cuda-runtime==13.0.96"]
 MIN_DRIVER = 580                       # CUDA 13.0
-MIN_ENGINE = (0, 1, 35)                # v0.1.35: Windows AMD uses its bundled HIP runtime (#468 #461), the low-RAM resident mode on Windows 32 GB (#467), fixes #460 #459 #446 #447 #457 #448 #444; v0.1.34: AMD on Windows (a ready-made HIP engine), an MCP server for AI assistants (tools/strata_mcp.py), a shorter README; v0.1.33: a portable image encoder again (#411 #412), setup recommends instead of forcing (#406 #403 #364 #384), fixes #352 #365 #369 #371 #375 #393 #408 #414; v0.1.32: split prompts faster (#340), AMD router +12%, Unsloth Q4 in setup, faster Q4 prompts, #326/#327/#342/#344 fixes, PR batch; v0.1.31: Unsloth UD-Q4_K_XL (experimental), GGUF-in-place low-RAM mode, Windows GGUF load 2x, server race + tokenizer fixes, AMD intrinsics; v0.1.30: short prompts faster (streaming from 1024 tokens), resident low-RAM variant, multi-GPU session carve, RDNA4; v0.1.29: sampled answers faster (split top-k), #154 correctness fixes; v0.1.28: the expert cache reserves the draft head, a cancelled request no longer fails the next; v0.1.27: RTX 20 (sm_75) in the ready-made engine, the HIP build without CUDA headers; v0.1.26: the draft layer's prompt pass in batches; v0.1.25: faster prompts (grouping off the copy engine, fused hyper-connection kernels), AMD HIP backend, --kv k8v4; v0.1.24: long prompts faster (QSA select on tensor cores); v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
+MIN_ENGINE = (0, 1, 38)                # v0.1.38: prompts faster (one gather per expert group #372, the first chunk's PLE rows beside layer 0 #374, DeltaNet three heads per thread #413), --kv q4_0 prompts on tensor cores (#452), Q5_0 experts on the GPU (#473), IQ4_XS on AVX-2 (#415), unbuffered expert loading on Windows (#357 #362), --peer-device (#531), a 6 GB card starts (#496), PR batch; v0.1.37: a silent engine is restarted (#481), Windows AMD counts the desktop's VRAM (#380 #377 #497), a steadier PCIe probe (#485), fixes #496 #495 #498 #505 #493; v0.1.36: a cancelled prompt logged as read so far (#471), the draft-head hint (#474), UPDATE.bat (#475), --expert-profile-save (#477); v0.1.35: Windows AMD uses its bundled HIP runtime (#468 #461), the low-RAM resident mode on Windows 32 GB (#467), fixes #460 #459 #446 #447 #457 #448 #444; v0.1.34: AMD on Windows (a ready-made HIP engine), an MCP server for AI assistants (tools/strata_mcp.py), a shorter README; v0.1.33: a portable image encoder again (#411 #412), setup recommends instead of forcing (#406 #403 #364 #384), fixes #352 #365 #369 #371 #375 #393 #408 #414; v0.1.32: split prompts faster (#340), AMD router +12%, Unsloth Q4 in setup, faster Q4 prompts, #326/#327/#342/#344 fixes, PR batch; v0.1.31: Unsloth UD-Q4_K_XL (experimental), GGUF-in-place low-RAM mode, Windows GGUF load 2x, server race + tokenizer fixes, AMD intrinsics; v0.1.30: short prompts faster (streaming from 1024 tokens), resident low-RAM variant, multi-GPU session carve, RDNA4; v0.1.29: sampled answers faster (split top-k), #154 correctness fixes; v0.1.28: the expert cache reserves the draft head, a cancelled request no longer fails the next; v0.1.27: RTX 20 (sm_75) in the ready-made engine, the HIP build without CUDA headers; v0.1.26: the draft layer's prompt pass in batches; v0.1.25: faster prompts (grouping off the copy engine, fused hyper-connection kernels), AMD HIP backend, --kv k8v4; v0.1.24: long prompts faster (QSA select on tensor cores); v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
 PY_PACKAGES = ["numpy", "jinja2", "regex", "pyyaml", "tqdm", "requests", "cmake", "ninja", "pillow", "psutil"]
 REQUIREMENTS = ROOT / "requirements.txt"   # the same packages and their dependencies, pinned (#214)
 
@@ -598,6 +608,33 @@ def split_mmap(cfg: dict) -> bool:
     return True
 
 
+def unsloth_split_need_gb(model="UD-Q4_K_XL") -> float:
+    """#498: the RAM UD-Q4_K_XL needs on several GPUs, where it has no RAM budget (the engine refuses
+    --resident-budget-gib with a layer split): its GGUF files and UNSLOTH_RAM_LEFT_GB more (~135 GB).  Measured safe
+    at 165 GiB (2x RTX 3090: MemAvailable never under 68 GiB); the 0-free case of #384 was 47 GB with a 70 GB model."""
+    return MODELS[model]["download_gb"] + UNSLOTH_RAM_LEFT_GB
+
+
+def split_budget(cfg: dict) -> bool:
+    """#498: a UD-Q4_K_XL config (its RAM budget, --resident-budget-gib) started on several GPUs.  The engine refuses
+    the budget with a layer split (it exited with code 2), so the split runs without it - all the experts loaded into
+    RAM at start - where the RAM holds the GGUFs and 24 GB more; else setup stops, before the config is saved.  True
+    when the config changed."""
+    a = cfg.get("args", [])
+    if "--resident-budget-gib" not in a:
+        return False
+    need, ram = unsloth_split_need_gb(), ram_gb()
+    if ram < need:
+        fail(f"UD-Q4_K_XL cannot share its RAM budget across GPUs (the engine has no layer split with it), and without "
+             f"the budget it needs ~{need:.0f} GB of RAM (its GGUF files and {UNSLOTH_RAM_LEFT_GB} GB more); this PC "
+             f"has {ram:.0f} GB", "start it on one GPU: START-HERE.bat --gpu N (Linux: ./setup.sh --gpu N)")
+    i = a.index("--resident-budget-gib")
+    del a[i:i + 2]
+    ok("UD-Q4_K_XL on several GPUs: no RAM budget (the engine has none with a layer split) - all its experts are "
+       "loaded into RAM from the model files at start, and the files pass through the OS file cache (#498)")
+    return True
+
+
 def offer_together(cfg_path: Path, cfg: dict, yes: bool) -> dict:
     """Starting a model set up for one card on a PC with two or more that can share it: asked once (the answer is
     saved in its config)."""
@@ -607,6 +644,10 @@ def offer_together(cfg_path: Path, cfg: dict, yes: bool) -> dict:
     can = together_ok(found)
     if not can:
         return cfg
+    # #498: UD-Q4_K_XL's RAM budget has no layer split; without it the RAM must hold the GGUFs and 24 GB more
+    budget = "--resident-budget-gib" in cfg.get("args", [])
+    if budget and ram_gb() < unsloth_split_need_gb():
+        return cfg
     pair = can[:2]
     cfg["gpus_asked"] = True
     # #364 #384: the resident low-RAM variant stays on one card unless the user says otherwise (its RAM use is steady)
@@ -618,6 +659,10 @@ def offer_together(cfg_path: Path, cfg: dict, yes: bool) -> dict:
         say("  This model runs in the low-RAM mode with its experts kept in RAM, on one GPU (recommended: steady RAM")
         say("  use). On both, the experts the GPUs do not hold are read through the OS file cache instead: faster in")
         say("  two reports (#364, #384), but RAM can fill up to 0 free during long prompts.")
+    if budget:
+        say("  This model (UD-Q4_K_XL) runs on one GPU with a RAM budget of its experts (recommended: the tested")
+        say("  setup). On both it has no budget: all its experts are loaded into RAM at start, which this PC's RAM")
+        say("  holds - about twice as fast in #498 (2x RTX 3090: 31 -> 64-78 tokens/s).")
     short = split_short(pair)             # #448: one card recommended (asked "n" by default), as for --resident
     for g in short:
         say(f"  {split_short_note(g)}.")
@@ -626,10 +671,11 @@ def offer_together(cfg_path: Path, cfg: dict, yes: bool) -> dict:
         say("  The installed engine has no code for " + ", ".join(g["name"] for g in missing) + ": to use them "
             "together, run START-HERE.bat --setup --gpus " + ",".join(str(g["index"]) for g in pair))
     elif ask("  Use both from now on? (you can change it later: START-HERE.bat --gpu N for one card)",
-             ["y", "n"], "n" if resident or short else "y", yes) == "y":
+             ["y", "n"], "n" if resident or short or budget else "y", yes) == "y":
         cfg["gpu"] = [g["index"] for g in pair]
         cfg["layer_split"] = cfg.get("layer_split") or "auto"
         split_mmap(cfg)
+        split_budget(cfg)
         ok("from now on this model runs on " + " + ".join(gpu_name(g) for g in pair))
     else:
         ok("staying on one GPU (START-HERE.bat --gpus " + ",".join(str(g["index"]) for g in pair) + " switches)")
@@ -2068,6 +2114,43 @@ def low_ram_together(a, model, ram, gpu, chosen) -> bool:
     return False
 
 
+def unsloth_together(a, model, ram, gpu, chosen) -> bool:
+    """#498: UD-Q4_K_XL with several GPUs chosen.  Its RAM budget (--resident-budget-gib) has no layer split, so a
+    split runs without it: all its experts loaded into RAM from the GGUFs at start, as with the 2-3-bit models - only
+    where the RAM holds the GGUF files and 24 GB more (unsloth_split_need_gb; 165 GiB, 2x RTX 3090: 31 -> 64-78
+    tok/s).  An explicit --gpus is honoured there; otherwise asked, one GPU by default (--yes: one GPU, as before); an
+    explicit --resident-budget-gib keeps one GPU.  True: all of them."""
+    names = " + ".join(gpu_name(g) for g in chosen)
+    need = unsloth_split_need_gb(model)
+    if ram < need:
+        warn(f"{model} runs on one GPU here: on several it has no RAM budget and needs ~{need:.0f} GB of RAM (its GGUF "
+             f"files and {UNSLOTH_RAM_LEFT_GB} GB more), this PC has {ram:.0f} - using {gpu_name(gpu)} only")
+        return False
+    if a.resident_budget_gib is not None:
+        warn(f"--resident-budget-gib has no layer split: {model} runs on one GPU with it - using {gpu_name(gpu)} only "
+             f"(leave the budget out to use {names} together)")
+        return False
+    note = (f"no RAM budget - all of its experts (~{MODELS[model]['arena_gb']:.0f} GB) are loaded into RAM from the "
+            f"model files at start, and the files pass through the OS file cache (needs ~{need:.0f} GB of RAM, this "
+            f"PC has {ram:.0f})")
+    if a.gpus:
+        ok(f"{model} on {names}, as you chose (--gpus): {note}")
+        return True
+    if not a.yes:
+        say()
+        say(f"  {model} can run on one GPU with a RAM budget of its experts, or on {names} together without one:")
+        say(f"  1) {gpu_name(gpu)} only: the most-used experts kept in RAM, the rest read from the SSD   (recommended: "
+            "the tested setup)")
+        say(f"  2) {names} together: {note};")
+        say("     about twice as fast in #498 (2x RTX 3090: 31 -> 64-78 tokens/s)")
+        if ask(f"{model}: which GPUs?", ["1", "2"], "1", a.yes) == "2":
+            ok(f"{model} on {names}: {note}")
+            return True
+    warn(f"{model} runs on one GPU: using {gpu_name(gpu)} only (--gpus " + ",".join(str(g["index"]) for g in chosen) +
+         f" shares it across {names} without the RAM budget: this PC's RAM holds it)")
+    return False
+
+
 def confirm_risk(msg, explicit, yes, stop, hint=None, question="  Go on anyway?", default="n") -> None:
     """The owner's rule: setup recommends, it never forces.  A choice setup expects to fail or run badly is said
     plainly (msg), then asked (default n; `default` keeps an older question's own default), or with --yes taken as
@@ -2332,7 +2415,10 @@ def choices_from_config(cfg_path: Path) -> dict:
             "vision": ("gpu" if vis.get("gpu") else "cpu") if isinstance(vis, dict) else "none",
             "esp": ("on" if Path(esp_path).name == ESP_VECTOR.name else esp_path) if esp_path else "off",
             "host": cfg.get("host"), "api_key": cfg.get("api_key"), "port": cfg.get("port"), "gpu": cfg.get("gpu"),
-            "layer_split": cfg.get("layer_split")}
+            "layer_split": cfg.get("layer_split"),
+            # #493: --vram-reserve-mib given at setup (images write the default 700 themselves)
+            "vram_reserve_mib": int(val("--vram-reserve-mib")) if (val("--vram-reserve-mib") or "").isdigit() and (
+                vis is None or int(val("--vram-reserve-mib")) != VISION["gpu"]["reserve_mib"]) else None}
 
 
 def find_in(roots: list, rel: str):
@@ -2344,8 +2430,25 @@ def find_in(roots: list, rel: str):
 
 
 # ------------------------------------------------------------------------------------------------ start
+def model_config(path: Path) -> bool:
+    """#549: a model's run config (a JSON object with "exe" and "args"). Any other strata-*.json in the folder (a
+    file of the user's own, a cut-off one) is skipped with a warning naming it instead of stopping setup."""
+    try:
+        cfg = json.loads(path.read_text(encoding="utf-8-sig"))
+        if isinstance(cfg, dict) and cfg.get("exe") and isinstance(cfg.get("args"), list):
+            return True
+        why = 'no "exe" or "args"'
+    except OSError as e:
+        why = e.strerror or str(e)
+    except ValueError:
+        why = "not valid JSON"
+    warn(f"skipped {path.name} ({why}): it is not a Strata model config")
+    return False
+
+
 def installed_configs():
-    return sorted(ROOT.glob("strata-*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+    return [p for p in sorted(ROOT.glob("strata-*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+            if model_config(p)]
 
 
 def source_version() -> str:
@@ -2454,14 +2557,76 @@ def upgrade_config(cfg_path: Path, cfg: dict) -> dict:
     return cfg
 
 
+def update_install(have: list, a) -> int:
+    """#475: `setup.py --update` (UPDATE.bat / update.sh, after their git pull): what a plain START-HERE.bat does to
+    an install before it starts the model, without starting it - the Python packages, the ready-made engine when this
+    setup needs a newer one (MIN_ENGINE; a compiled engine when its source changed), each installed model's config
+    upgrades and its draft subset.  No question is asked and the model files are not touched; a model still running
+    keeps its engine (update_installed_engine says to close it and run this again)."""
+    have = [p for p in have if model_config(p)]        # #549: a strata-*.json that is no model config is skipped
+    if not have:
+        say("  No model is installed in this Strata folder yet: run START-HERE.bat (Linux: ./setup.sh) to set it up -")
+        say("  it finds an earlier install's model files next to it and reuses them.")
+        return 0
+    pip_install(requirement_lines() if REQUIREMENTS.exists() else PY_PACKAGES,
+                "numpy, jinja2, regex, pyyaml, tqdm, requests, cmake, ninja, pillow, psutil")
+    if not a.build:
+        update_installed_engine(a.prebuilt)
+    for cfg_path in have:
+        cfg = upgrade_config(cfg_path, json.loads(cfg_path.read_text(encoding="utf-8-sig")))
+        if "--mtp" in cfg["args"][:-1]:
+            refresh_draft_vocab(Path(cfg["args"][cfg["args"].index("--mtp") + 1]), cfg.get("draft_vocab", "cjk"))
+        if cfg.get("backend") == "hip" and WIN:
+            hip_runtime_beside_exe(Path(cfg["exe"]).parent)   # #468 #461
+        ok(f"{cfg.get('model_name', cfg_path.stem)}: up to date")
+    ver = engine_version(Path(json.loads(have[0].read_text(encoding="utf-8-sig"))["exe"]))
+    say()
+    ok("Strata is updated" + (f" (engine {'.'.join(map(str, ver))})" if any(ver) else "") +
+       ". Start the model with " + ("START-HERE.bat" if WIN else "./setup.sh") + " when you want it.")
+    return 0
+
+
+def settings_summary(cfg: dict, port=None) -> str:
+    """#564: the settings a start uses, in one line: the config's engine options (the model's file paths left out)
+    and the server's own fields, so a change made by hand to strata-<model>.json can be checked without the log."""
+    a, out, i = [str(x) for x in cfg.get("args") or []], [], 0
+    while i < len(a):
+        flag = a[i]
+        val = a[i + 1] if i + 1 < len(a) and not a[i + 1].startswith("--") else None
+        i += 1 if val is None else 2
+        if not flag.startswith("--"):
+            continue                                   # a positional: the model file
+        if val is not None and ("/" in val or "\\" in val or val.lower().endswith((".gguf", ".bin"))):
+            continue                                   # a path: --native, --mtp, --profile ...
+        out.append(flag if val is None else f"{flag} {val}")
+    srv = [f"{cfg.get('host', '127.0.0.1')}:{port or cfg.get('port', 8080)}"]
+    if cfg.get("api_key"):
+        srv.append("api key set")
+    for k in ("gpu", "layer_split", "draft_vocab", "fit_max_tokens", "reasoning_budget_tokens", "anthropic_thinking"):
+        if cfg.get(k) is not None:
+            v = cfg[k]
+            srv.append(f"{k} {','.join(map(str, v)) if isinstance(v, list) else str(v).lower() if isinstance(v, bool) else v}")
+    return " ".join(out) + ("; " if out else "") + "server " + ", ".join(srv)
+
+
 def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_browser=True, yes=False,
           layer_split=None, keep=None) -> int:
-    """keep: settings given on this start that the model keeps from now on (--host, --api-key, --draft-vocab)."""
+    """keep: settings given on this start that the model keeps from now on (--host, --api-key, --draft-vocab,
+    --vram-reserve-mib)."""
     cfg = upgrade_config(cfg_path, json.loads(cfg_path.read_text(encoding="utf-8-sig")))
     missing = [p for p in [cfg["exe"], *[a for a in cfg["args"] if a.endswith(".gguf")]] if not Path(p).exists()]
     if missing:
         fail(f"{cfg_path.name} refers to missing files: {missing[0]}", "run it again with --setup to repair")
     keep = {k: v for k, v in (keep or {}).items() if v is not None}
+    reserve = keep.pop("vram_reserve_mib", None)       # #493: an engine argument, kept in the config's args
+    if reserve is not None:
+        args = cfg["args"]
+        if "--vram-reserve-mib" in args[:-1]:
+            args[args.index("--vram-reserve-mib") + 1] = str(reserve)
+        else:
+            args += ["--vram-reserve-mib", str(reserve)]
+        write_config(cfg_path, cfg)
+        ok(f"saved for this model: {reserve} MiB of VRAM kept free for other programs (--vram-reserve-mib)")
     if keep and any(cfg.get(k) != v for k, v in keep.items()):   # #179: a --host/--api-key on a start was ignored
         cfg.update(keep)
         write_config(cfg_path, cfg)
@@ -2485,6 +2650,7 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
                      "set it up for these cards: ./setup.sh --setup --backend hip --gpus " + ",".join(map(str, gpu)))
             cfg["gpu"], cfg["gpus_asked"] = gpu, True
             cfg["layer_split"] = layer_split or cfg.get("layer_split") or "auto"
+            split_budget(cfg)                          # #498: before it is saved (it stops when the RAM is short)
             write_config(cfg_path, cfg)
             gpu = None
         elif gpu is not None:
@@ -2508,6 +2674,7 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
         check_gpus(gpu, found, yes=yes, named=True)
         cfg["gpu"], cfg["gpus_asked"] = gpu, True
         cfg["layer_split"] = layer_split or cfg.get("layer_split") or "auto"
+        split_budget(cfg)                              # #498: before it is saved (it stops when the RAM is short)
         write_config(cfg_path, cfg)
         gpu = None
     elif gpu is not None:                              # --gpu N: this start only, on that card
@@ -2516,7 +2683,8 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
     else:
         cfg = offer_together(cfg_path, cfg, yes)
     use = gpu if gpu is not None else cfg.get("gpu")
-    if isinstance(use, list) and split_mmap(cfg):     # #364 #384: a resident low-RAM config on several GPUs
+    # #364 #384: a resident low-RAM config on several GPUs; #498: a UD-Q4_K_XL config with its RAM budget (by hand)
+    if isinstance(use, list) and (split_mmap(cfg) | split_budget(cfg)):
         write_config(cfg_path, cfg)
     if cfg.get("backend") == "hip":
         pass
@@ -2541,12 +2709,22 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
             pass
     say()
     say("  " + "-" * 100)
-    say(f"  Starting {cfg.get('model_name', 'the model')}: it loads {f'about {gb:.0f} GB' if gb >= 1 else '34-55 GB'} "
-        "into RAM and locks part of it for the GPU.")
+    size = f'about {gb:.0f} GB' if gb >= 1 else '34-55 GB'
+    a_ = cfg["args"]
+    if "--mmap-experts" in a_ and "--resident-budget-gib" not in a_ and "--resident-experts" not in a_:
+        # #505: the mapped low-RAM mode loads nothing into RAM up front (the server's narrator says the same)
+        say(f"  Starting {cfg.get('model_name', 'the model')}: it maps {size} of experts from the model files (the OS "
+            "file cache reads them).")
+    else:
+        say(f"  Starting {cfg.get('model_name', 'the model')}: it loads {size} into RAM and locks part of it for the "
+            "GPU.")
     say("  While it does, YOUR PC CAN BE SLOW OR STOP RESPONDING FOR 1-3 MINUTES (longer the first time after a")
     say("  restart). That is normal: please wait and don't close this window - the browser opens when it is ready.")
     say("  Later, closing this window stops the model.")
     say("  " + "-" * 100)
+    for n, line in enumerate(textwrap.wrap(f"Settings ({cfg_path.name}): {settings_summary(cfg, port)}", 100,
+                                           break_on_hyphens=False)):   # #564: what this start uses
+        say(("  " if n == 0 else "    ") + line)
     if not WIN and os.environ.get("STRATA_EXECV"):
         # Replace this process instead of spawning a child. The Docker image sets STRATA_EXECV=1,
         # so there the server is PID 1 and docker stop's SIGTERM reaches the process that can
@@ -2572,6 +2750,66 @@ def saved_draft_vocab(cfg_path: Path) -> str | None:
     return v if v in DRAFT_VOCABS else None
 
 
+DRAFT_VOCAB_MIB = {"cjk": 348, "cyrillic": 193, "en": 133}   # the draft head's VRAM per subset (IQ3_S: the largest)
+SMALL_DRAFT_VRAM_GB = 14   # #474: below this the default subset's head can be what does not fit
+
+
+def draft_vocab_note(vram_gb: float, chosen: str | None) -> list[str]:
+    """#474: on a card under 14 GB, the default draft subset (cjk, ~348 MiB of VRAM) can be what does not fit at the
+    start ("the draft head does not fit"), and the engine's expert cache gets what a smaller one leaves.  Setup
+    RECOMMENDS a smaller one here and changes nothing (the owner's rule, #403 #406): a subset chosen with
+    --draft-vocab, or kept from an earlier install, gets no note.  [] for every other case."""
+    if chosen or not 0 < vram_gb < SMALL_DRAFT_VRAM_GB:
+        return []
+    start = "START-HERE.bat" if WIN else "./setup.sh"
+    return [f"Tip for a {vram_gb:.0f} GB card: the draft layer's default token subset (with Chinese, Japanese and "
+            f"Korean) needs up to ~{DRAFT_VOCAB_MIB['cjk']} MiB of VRAM.",
+            f"  For English and code answers, {start} --draft-vocab en needs up to ~{DRAFT_VOCAB_MIB['en']} MiB "
+            f"(cyrillic: ~{DRAFT_VOCAB_MIB['cyrillic']}) and leaves the rest to the expert cache - and it is the",
+            "  fix when the start stops with \"the draft head does not fit\". The model keeps the choice."]
+
+
+SMALL_CARD_GB = 7.5            # #496: a card under 8 GB gets a tip (an 8 GB card lists 7.99)
+
+
+def small_card_note(ctx: int, draft_vocab: str | None) -> list[str]:
+    """#496: what frees VRAM on a card under 8 GB when the start stops with "no VRAM is left for the expert cache"
+    (the engine already lowers its own reserve on such a card) - a recommendation, setup changes none of it.  (The
+    draft layer stays: the server needs it.)"""
+    start = "START-HERE.bat --setup" if WIN else "./setup.sh"
+    tips = []
+    if ctx > 8192:
+        tips.append("an 8K context (a smaller KV cache)")
+    if draft_vocab != "en":
+        tips.append(f"--draft-vocab en (a draft head of ~{DRAFT_VOCAB_MIB['en']} MiB instead of "
+                    f"~{DRAFT_VOCAB_MIB[draft_vocab or 'cjk']})")
+    lines = ["If the start stops with \"no VRAM is left for the expert cache\" (the engine's log says how much is "
+             "short):"]
+    if tips:
+        lines.append(f"  run {start} again with " + " and ".join(tips) + ", or close other programs that use the GPU.")
+    else:
+        lines.append("  close other programs that use the GPU.")
+    return lines
+
+
+DESKTOP_RESERVE_MIB = 3072     # #560 #516: what kept a KDE/Wayland desktop alive beside a full expert cache
+
+
+def linux_desktop(env=None) -> bool:
+    """A graphical session on Linux (Wayland or X)."""
+    env = os.environ if env is None else env
+    return sys.platform.startswith("linux") and bool(env.get("WAYLAND_DISPLAY") or env.get("DISPLAY"))
+
+
+def desktop_reserve_note() -> list[str]:
+    """#560 #516: an AMD card that also drives a Linux desktop - with the default 700 MiB reserve the expert cache
+    fills it, and when the desktop needs more VRAM amdgpu moves the cache to system RAM, where the OOM killer then ends
+    the compositor.  A recommendation, setup changes nothing."""
+    return [f"If this AMD card also drives your desktop and the desktop or apps crash once the model is loaded, keep "
+            f"more VRAM free: ./setup.sh --vram-reserve-mib {DESKTOP_RESERVE_MIB}",
+            "  (remembered for this model; the expert cache gets ~2.3 GB less, a few % of speed)"]
+
+
 def mtp_corrupt(mtp: Path, env=None) -> bool:
     """#327: True when the MTP tensors an install fetched are not the pinned checkpoint's (tools/mtp_fetch.py verify,
     which hashes only files that changed since they last checked out).  A mirror that ignored range requests left the
@@ -2718,6 +2956,9 @@ def main() -> int:
     ap.add_argument("--yes", action="store_true", help="accept the recommended answers")
     ap.add_argument("--setup", action="store_true", help="install another model or change settings")
     ap.add_argument("--no-start", action="store_true", help="install only, do not start the model")
+    ap.add_argument("--update", action="store_true",
+                    help="update the installed engine, Python packages and model settings as a start would, without "
+                         "starting the model (UPDATE.bat / update.sh run it after a git pull)")
     ap.add_argument("--build", action="store_true", help="compile the engine instead of using the ready-made one")
     ap.add_argument("--prebuilt", default=os.environ.get("STRATA_PREBUILT_URL", PREBUILT_URL),
                     help="where the ready-made engine is (a URL folder or a local folder)")
@@ -2736,6 +2977,9 @@ def main() -> int:
     ap.add_argument("--resident-budget-gib", type=float, metavar="N",
                     help="UD-Q4_K_XL: the GiB of its experts kept in RAM (default: the RAM less 24 GB, 40 on 64 GB; "
                          "more is kept as you choose, with a note)")
+    ap.add_argument("--vram-reserve-mib", type=int, metavar="N",
+                    help="VRAM in MiB the engine leaves free for other programs (a game, another model; the engine's "
+                         "default: 700); the expert cache takes that much less")
     ap.add_argument("--kv-streaming", choices=["auto", "on", "off"], default="auto",
                     help="from a 64K context: keep the KV cache in RAM and only the attention's window in VRAM (more "
                          "experts fit on the GPU); auto: when the RAM has room for it")
@@ -2746,6 +2990,8 @@ def main() -> int:
     a = ap.parse_args()
     if a.resident_budget_gib is not None and not a.resident_budget_gib > 0:
         ap.error("--resident-budget-gib takes a number of GiB above 0, e.g. --resident-budget-gib 32")
+    if a.vram_reserve_mib is not None and a.vram_reserve_mib < 0:
+        ap.error("--vram-reserve-mib takes a number of MiB, 0 or more, e.g. --vram-reserve-mib 2048")
     if a.gpu is not None:                             # --gpu 0,2 means --gpus 0,2 (a user tried it: issue report)
         if "," in a.gpu:
             a.gpus, a.gpu = a.gpus or a.gpu, None
@@ -2761,6 +3007,8 @@ def main() -> int:
 
     # ---- 0. already installed: just start it
     have = installed_configs()
+    if a.update:                                       # #475: UPDATE.bat / update.sh - never starts the model
+        return update_install(have, a)
     explicit = a.setup or a.model or a.family or a.check or a.no_start
     if not have and not explicit:                      # a new copy of Strata (an update unzipped elsewhere): set it
         prev = previous_config(elsewhere, load_settings())   # up like the last one, from the files already here
@@ -2775,6 +3023,8 @@ def main() -> int:
                 a.experimental_speed_projection = a.experimental_speed_projection or ch["esp"]
                 a.host, a.api_key = a.host or ch["host"], a.api_key or ch["api_key"]
                 a.port = a.port or ch["port"]
+                if a.vram_reserve_mib is None:          # #493: an explicit reserve set up before
+                    a.vram_reserve_mib = ch.get("vram_reserve_mib")
                 if isinstance(ch.get("gpu"), list):     # a layer split: set up across the same cards again
                     a.gpus = a.gpus or ",".join(str(g) for g in ch["gpu"])
                     a.layer_split = a.layer_split or ch.get("layer_split")
@@ -2801,13 +3051,15 @@ def main() -> int:
             warn("this PC is NOT tuned: the tuning failed (the reason is above); the model "
                  + ("keeps" if a.no_start else "starts with") + " the default settings")
         return 0 if a.no_start else start(pick_cfg, a.port, run_gpu, yes=a.yes, layer_split=a.layer_split,
-                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab})
+                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab,
+                           "vram_reserve_mib": a.vram_reserve_mib})
     if have and not (a.setup or a.model or a.family or a.check or a.no_start):
         if not a.build:
             update_installed_engine(a.prebuilt)
         if len(have) == 1:
             return start(have[0], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split,
-                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab})
+                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab,
+                           "vram_reserve_mib": a.vram_reserve_mib})
         say()
         for i, c in enumerate(have, 1):
             say(f"  {i}) {json.loads(c.read_text(encoding='utf-8-sig')).get('model_name', c.stem)}")
@@ -2815,7 +3067,8 @@ def main() -> int:
         pick = int(ask("Which one?", [str(i) for i in range(1, len(have) + 2)], "1", a.yes))
         if pick <= len(have):
             return start(have[pick - 1], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split,
-                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab})
+                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab,
+                           "vram_reserve_mib": a.vram_reserve_mib})
 
     # ---- 1. the PC
     step(1, "checking your PC")
@@ -2967,10 +3220,11 @@ def main() -> int:
         say(f"  {i}) {m:8s} {d['about']}; download {d['download_gb']:.0f} GB, uses ~{d['arena_gb']:.0f} GB of RAM{fit}")
     rec = str(names.index("IQ3_XXS") + 1) if ram >= 60 and "IQ3_XXS" in names else "1"
     model = a.model or names[int(ask("Which size?", [str(i) for i in range(1, len(names) + 1)], rec, a.yes)) - 1]
-    budget = None
+    budget, q4_split = None, False
     if MODELS[model].get("budget"):
         # Unsloth's UD-Q4_K_XL: a RAM budget of experts, the rest from the GGUF on the SSD - not the low-RAM mode (no
-        # experts.bin: it would be another 77 GB on the disk), and one GPU (the budget mode has no layer split)
+        # experts.bin: it would be another 77 GB on the disk), and one GPU (the budget mode has no layer split) unless
+        # the RAM holds the GGUFs and 24 GB more: then several, without the budget, if asked for (#498)
         warn(f"{model} is EXPERIMENTAL (docs/UNSLOTH_Q4.md): most of its experts are read from the SSD while it "
              "answers, so it is several times slower than the 2-3-bit models; quality checked against llama.cpp")
         if hip:
@@ -2990,12 +3244,13 @@ def main() -> int:
                          "install it anyway", "  Install it anyway?")
             warn(f"installing {model} with {ram:.0f} GB of RAM, as you chose")
         budget = budget_choice(model, ram, a.resident_budget_gib)
-        ok(f"RAM budget: {budget:g} GiB of {model}'s experts in RAM, the rest read from the model files on the SSD")
+        if multi and not unsloth_together(a, model, ram, gpu, chosen):
+            multi, sel, chosen = [], [gpu["index"]], [gpu]
+        q4_split = bool(multi)                         # #498: on several GPUs without the RAM budget
+        if not q4_split:
+            ok(f"RAM budget: {budget:g} GiB of {model}'s experts in RAM, the rest read from the model files on the SSD")
         if a.low_ram not in ("auto", "off"):
             warn(f"--low-ram {a.low_ram} does not apply to {model}: it always reads part of its experts from the files")
-        if multi:
-            warn(f"{model} runs on one GPU: using " + gpu_name(gpu) + " only")
-            multi, sel, chosen = [], [gpu["index"]], [gpu]
     elif a.resident_budget_gib is not None:
         warn(f"--resident-budget-gib is for UD-Q4_K_XL: {model} keeps all of its experts in RAM or in the low-RAM mode")
     low_ram = budget is None and (a.low_ram in ("on", "resident", "mmap") or
@@ -3202,6 +3457,13 @@ def main() -> int:
     # ---- 5. the model files
     step(5, f"downloading {fam['title']} {model}")
     if not a.gguf_dir:
+        missing = [s.name for s in shards if not (s.exists() and done(s))]
+        if missing:                                    # #495: files downloaded by hand go here, or --gguf-dir
+            say(f"  The model files go in {models_dir}")
+            say(f"  Files you already have: put them here with their original names ({', '.join(missing)}), or use "
+                "--gguf-dir <their folder>.")
+            if hf_endpoint() != HF_DEFAULT:
+                say(f"  Downloading from {hf_endpoint()} (HF_ENDPOINT)")
         for s in shards:
             if s.exists() and done(s):
                 ok(f"{s.name} already downloaded")
@@ -3274,6 +3536,8 @@ def main() -> int:
     draft_vocab = a.draft_vocab or saved_draft_vocab(ROOT / f"strata-{tag.lower()}.json")
     refresh_draft_vocab(rt, draft_vocab or "cjk")
     ok(f"MTP draft layer: {rt}")
+    for line in draft_vocab_note(gpu.get("vram_gb", 0.0), draft_vocab):   # #474: a recommendation, nothing changes
+        say("  " + line)
 
     # ---- 7. the start script
     step(7, "writing the start script")
@@ -3325,7 +3589,9 @@ def main() -> int:
             warn(f"KV streaming needs ~{kv_ram_gb:.1f} GB of RAM beside the ~{MODELS[model]['ram_gb']} GB {model} "
                  f"uses; this PC has {ram:.0f}. Kept as you chose (--kv-streaming on): it may page or run out of RAM "
                  "under load")
-        if budget is not None and a.resident_budget_gib is None:   # its RAM comes out of the experts' budget
+        if q4_split:                                   # #498: no budget to take it out of
+            pass
+        elif budget is not None and a.resident_budget_gib is None:   # its RAM comes out of the experts' budget
             budget = resident_budget_gib(model, ram, kv_ram_gb)
             ok(f"RAM budget: {budget} GiB (less the KV cache's RAM)")
         elif budget is not None and budget > resident_budget_gib(model, ram, kv_ram_gb):
@@ -3333,10 +3599,29 @@ def main() -> int:
                  f"would take them out of it: {resident_budget_gib(model, ram, kv_ram_gb)} GiB); kept as you chose")
     elif a.kv_streaming == "on":
         warn("--kv-streaming on: a context under 64K is not streamed (the attention's window holds all of it): off")
-    if budget is not None:     # UD-Q4_K_XL: the experts read from the GGUF in place, the most-used N GiB kept in RAM
-        args += ["--resident-budget-gib", f"{budget:g}"]
+    if budget is not None and not q4_split:   # UD-Q4_K_XL: the experts read from the GGUF in place, the most-used N
+        args += ["--resident-budget-gib", f"{budget:g}"]   # GiB kept in RAM (#498: a layer split has no budget)
     if vision != "none":
         args += ["--vision", "--vram-reserve-mib", str(VISION[vision]["reserve_mib"])]
+    if a.vram_reserve_mib is not None:                 # #493: VRAM left free for other programs (only when given)
+        if "--vram-reserve-mib" in args:
+            i = args.index("--vram-reserve-mib") + 1
+            if vision == "gpu" and a.vram_reserve_mib < int(args[i]):
+                warn(f"--vram-reserve-mib {a.vram_reserve_mib}: the image encoder on the GPU needs ~{args[i]} MiB of "
+                     "it; kept as you chose (it may run out of VRAM when it reads a picture)")
+            args[i] = str(a.vram_reserve_mib)
+        else:
+            args += ["--vram-reserve-mib", str(a.vram_reserve_mib)]
+        ok(f"VRAM kept free for other programs: {a.vram_reserve_mib} MiB (--vram-reserve-mib; the expert cache takes "
+           "that much less)")
+    if not multi and 0 < gpu.get("vram_gb", 0.0) < SMALL_CARD_GB:
+        # #496: on a 6 GB card the expert cache can get no room at all; the engine lowers its own reserve when that
+        # is what it takes, and says what is short when even that is not enough.  Setup only says what helps.
+        for line in small_card_note(ctx, draft_vocab):   # a recommendation: nothing changes
+            say("  " + line)
+    elif hip and a.vram_reserve_mib is None and linux_desktop():
+        for line in desktop_reserve_note():              # #560 #516: a recommendation: nothing changes
+            say("  " + line)
     if esp is not None:
         # the package's profile, with llama.cpp's flags (the engine takes the same ones)
         args += ["--control-vector-scaled", f"{esp}:1.0", "--control-vector-layer-range", "4", "44",

src/core/device.cu

@@ -101,6 +101,97 @@ bool device_summary(int ordinal, std::string& name, std::string& detail) {
     return true;
 }
 
+}  // namespace strata::core
+
+#if defined(STRATA_USE_HIP) && defined(_WIN32)
+// The free VRAM figure on Windows HIP (include/strata/hip_compat/cuda_runtime.h maps cudaMemGetInfo here).
+//
+// hipMemGetInfo (ROCclr's PAL backend, Device::globalFreeMemory) is the card's size minus this process's own
+// allocations: it asks Windows for this process's usage only, never for what the desktop and other programs hold.
+// So on a card that also drives the desktop the engine counted ~930 MiB that was not there, the expert cache filled
+// the card, and WDDM moved memory out to system RAM: decode at 30 tok/s instead of 41 (RX 6800, HIP SDK 7.2).
+// Windows itself gives each process a video memory budget (DXGI QueryVideoMemoryInfo) that does account for the
+// others.  Measured on that card: the budget sits 0.8 GiB below the card's size while this process is small and
+// 1.8 GiB below once it holds 11 GiB; allocations past it still succeed, and are what Windows moves out.  The
+// free figure here is hipMemGetInfo's lowered by what the budget withholds (the card's size minus the budget).
+// Subtracting from HIP's own figure, rather than taking the budget minus DXGI's usage, keeps memory the HIP runtime
+// has freed and holds in its cache counted as free, as hipMemGetInfo counts it.  dxgi.dll is loaded at run time, so
+// nothing new is linked; when it or the card's adapter cannot be found, hipMemGetInfo's figure stands.
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <dxgi1_4.h>
+#include <atomic>
+#include <cstdlib>
+#include <mutex>
+
+namespace strata::hip_compat {
+namespace {
+IDXGIAdapter3* budget_adapter(int device) {
+    constexpr int kMaxDevices = 16;
+    static std::mutex mu;
+    static IDXGIAdapter3* adapters[kMaxDevices] = {};
+    static bool tried[kMaxDevices] = {};
+    if (device < 0 || device >= kMaxDevices) return nullptr;
+    std::lock_guard<std::mutex> lock(mu);
+    if (tried[device]) return adapters[device];
+    tried[device] = true;
+    hipDeviceProp_t p{};
+    if (hipGetDeviceProperties(&p, device) != hipSuccess) {
+        (void) hipGetLastError();
+        return nullptr;
+    }
+    LUID luid{};
+    std::memcpy(&luid, p.luid, sizeof luid);
+    using CreateFactory = HRESULT(WINAPI*)(REFIID, void**);
+    static HMODULE dxgi = LoadLibraryW(L"dxgi.dll");   // kept for the process' lifetime, like the adapters
+    const auto create = dxgi ? (CreateFactory) (void*) GetProcAddress(dxgi, "CreateDXGIFactory1") : nullptr;
+    IDXGIFactory4* factory = nullptr;
+    if (create == nullptr || FAILED(create(__uuidof(IDXGIFactory4), (void**) &factory))) return nullptr;
+    IDXGIAdapter3* adapter = nullptr;
+    if (FAILED(factory->EnumAdapterByLuid(luid, __uuidof(IDXGIAdapter3), (void**) &adapter))) adapter = nullptr;
+    factory->Release();
+    adapters[device] = adapter;
+    return adapter;
+}
+}  // namespace
+
+hipError_t mem_get_info(size_t* free_bytes, size_t* total_bytes) {
+    const hipError_t e = hipMemGetInfo(free_bytes, total_bytes);
+    static const bool off = [] {
+        const char* v = std::getenv("STRATA_WDDM_BUDGET");
+        return v != nullptr && std::atoi(v) == 0;
+    }();
+    if (e != hipSuccess || off || free_bytes == nullptr || total_bytes == nullptr) return e;
+    int device = 0;
+    if (hipGetDevice(&device) != hipSuccess) {
+        (void) hipGetLastError();
+        return e;
+    }
+    IDXGIAdapter3* adapter = budget_adapter(device);
+    DXGI_QUERY_VIDEO_MEMORY_INFO local{};
+    if (adapter == nullptr || FAILED(adapter->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_LOCAL, &local)) ||
+        local.Budget == 0 || local.Budget >= *total_bytes)
+        return e;
+    const size_t withheld = *total_bytes - (size_t) local.Budget;   // the desktop's and other programs' share
+    static std::atomic<bool> said{false};
+    if (!said.exchange(true)) {
+        std::fprintf(stderr, "strata: Windows budgets %llu of this card's %llu MiB for this process; free VRAM is "
+                             "counted within that (STRATA_WDDM_BUDGET=0: off)\n",
+                     (unsigned long long) (local.Budget >> 20), (unsigned long long) (*total_bytes >> 20));
+    }
+    *free_bytes = *free_bytes > withheld ? *free_bytes - withheld : 0;
+    return e;
+}
+}  // namespace strata::hip_compat
+#endif
+
+namespace strata::core {
+
 std::string gpu_arch_problem(int ordinal) {
 #if defined(STRATA_USE_HIP)
     int count = 0;

src/core/expert_cache.cpp

@@ -3,7 +3,9 @@
 
 #include <cuda_runtime.h>
 
+#include <algorithm>
 #include <cstdio>
+#include <filesystem>
 #include <utility>
 #include <cstring>
 
@@ -67,6 +69,77 @@ bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_ex
     return true;
 }
 
+std::vector<std::pair<int32_t, int32_t>> rank_learned_profile(int64_t n_layers, int64_t n_expert,
+                                                              const std::vector<uint8_t>& resident,
+                                                              const std::vector<double>& heat,
+                                                              const std::vector<std::pair<int32_t, int32_t>>& prior) {
+    const size_t n = (size_t) (n_layers * n_expert);
+    std::vector<int64_t> prior_rank(n, INT64_MAX);
+    for (size_t r = 0; r < prior.size(); ++r) {
+        const auto [l, e] = prior[r];
+        if (l >= 0 && l < n_layers && e >= 0 && e < n_expert) {
+            int64_t& pr = prior_rank[(size_t) (l * n_expert + e)];
+            if (pr == INT64_MAX) pr = (int64_t) r;
+        }
+    }
+    std::vector<int64_t> order(n);
+    for (size_t i = 0; i < n; ++i) order[i] = (int64_t) i;
+    auto res = [&](int64_t i) { return (size_t) i < resident.size() && resident[(size_t) i] != 0; };
+    auto ht = [&](int64_t i) { return (size_t) i < heat.size() ? heat[(size_t) i] : 0.0; };
+    std::stable_sort(order.begin(), order.end(), [&](int64_t a, int64_t b) {
+        if (res(a) != res(b)) return res(a);
+        if (ht(a) != ht(b)) return ht(a) > ht(b);
+        if (prior_rank[(size_t) a] != prior_rank[(size_t) b]) return prior_rank[(size_t) a] < prior_rank[(size_t) b];
+        return a < b;
+    });
+    std::vector<std::pair<int32_t, int32_t>> ranked(n);
+    for (size_t r = 0; r < n; ++r)
+        ranked[r] = {(int32_t) (order[r] / n_expert), (int32_t) (order[r] % n_expert)};
+    return ranked;
+}
+
+bool write_expert_profile(const std::string& path, int64_t n_layers, int64_t n_expert,
+                          const std::vector<std::pair<int32_t, int32_t>>& ranked, std::string& err) {
+    if (n_layers <= 0 || n_expert <= 0 || n_layers > 65535 || n_expert > 65535) {
+        err = "write_expert_profile: the model's layout does not fit the format";
+        return false;
+    }
+    std::vector<int32_t> table((size_t) (n_layers * n_expert), -1);
+    std::vector<uint16_t> pairs;
+    pairs.reserve(ranked.size() * 2);
+    for (size_t r = 0; r < ranked.size(); ++r) {
+        const auto [l, e] = ranked[r];
+        if (l < 0 || l >= n_layers || e < 0 || e >= n_expert) {
+            err = "write_expert_profile: a ranked pair is out of range";
+            return false;
+        }
+        table[(size_t) (l * n_expert + e)] = (int32_t) r;
+        pairs.push_back((uint16_t) l);
+        pairs.push_back((uint16_t) e);
+    }
+    const uint32_t hdr[5] = {1u, (uint32_t) n_layers, (uint32_t) n_expert, (uint32_t) ranked.size(),
+                             (uint32_t) ranked.size()};
+    const std::string tmp = path + ".tmp";
+    std::FILE* f = std::fopen(tmp.c_str(), "wb");
+    if (f == nullptr) {
+        err = "write_expert_profile: cannot create " + tmp;
+        return false;
+    }
+    // the format is little-endian (make_profile.py's "<"): so is every machine this engine runs on
+    bool ok = std::fwrite("STRP", 1, 4, f) == 4 && std::fwrite(hdr, 4, 5, f) == 5 &&
+              (pairs.empty() || std::fwrite(pairs.data(), 2, pairs.size(), f) == pairs.size()) &&
+              std::fwrite(table.data(), 4, table.size(), f) == table.size();
+    ok = (std::fclose(f) == 0) && ok;
+    std::error_code ec;
+    if (ok) std::filesystem::rename(tmp, path, ec);   // replaces an existing file (MoveFileEx / rename(2))
+    if (!ok || ec) {
+        std::filesystem::remove(tmp, ec);
+        err = "write_expert_profile: cannot write " + path;
+        return false;
+    }
+    return true;
+}
+
 ExpertCache::~ExpertCache() { close(); }
 
 #if defined(STRATA_USE_HIP)

src/core/expert_source.cpp

@@ -1,6 +1,7 @@
 // src/core/expert_source.cpp - the adapter.  See the header for the three clauses of the contract.
 #include "strata/core/expert_source.hpp"
 #include "strata/core/remote_experts.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 #include "strata/artifact/gguf_reader.hpp"
 
@@ -426,6 +427,7 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     file_ = f;
     mapping_ = m;
     base_ = (const uint8_t*) view;
+    paths_.assign(1, path);
 #else
     const int fd = ::open(path.c_str(), O_RDONLY);
     if (fd < 0) { err = "FileExpertSource: cannot open " + path; return false; }
@@ -506,6 +508,12 @@ void FileExpertSource::close() {
     }
     role_ptr_.clear();
     role_bytes_.clear();
+    role_file_.clear();
+    paths_.clear();
+#if defined(_WIN32)
+    for (void* h : direct_) CloseHandle((HANDLE) h);
+#endif
+    direct_.clear();
     {
         std::lock_guard<std::mutex> lk(stage_mu_);
         stage_buf_.clear();
@@ -571,6 +579,7 @@ bool FileExpertSource::open_gguf(std::string& err) {
     std::map<std::string, size_t> index;
     role_ptr_.assign((size_t) (3 * n_layers_), nullptr);
     role_bytes_.assign((size_t) (3 * n_layers_), 0);
+    role_file_.assign((size_t) (3 * n_layers_), 0);
     for (int64_t l = 0; l < n_layers_; ++l) {
         const auto& fm = lay.fmt[(size_t) l];
         const uint64_t per[3] = {fm.up_off, fm.up_off, lay.bytes[(size_t) l] - fm.down_off};
@@ -620,9 +629,11 @@ bool FileExpertSource::open_gguf(std::string& err) {
                 m.base = (const uint8_t*) view;
 #endif
                 maps_.push_back(m);
+                paths_.push_back(path);
                 it = index.emplace(path, maps_.size() - 1).first;
             }
             const Map& m = maps_[it->second];
+            role_file_[i] = (int) it->second;
             const uint64_t at = lay.gguf_off[i], bytes = per[r] * (uint64_t) n_expert_;
             if (at > m.bytes || bytes > m.bytes - at) {   // check_experts_gguf proved it; the mapping must agree
                 err = "FileExpertSource: an expert span runs past the end of " + path;
@@ -640,6 +651,10 @@ bool FileExpertSource::open_gguf(std::string& err) {
 
 bool FileExpertSource::copy_from_files(int64_t layer, int64_t expert, uint8_t* dst) const {
     if (dst == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
+    if (!direct_.empty()) {   // #286: from the drive; a failed read falls back to the mapping below
+        const Fill f{0, layer, expert, dst};
+        if (read_direct(&f, 1)) return true;
+    }
     if (!role_ptr_.empty()) {
         uint64_t at = 0;
         for (int r = 0; r < 3; ++r) {
@@ -710,7 +725,11 @@ bool FileExpertSource::claim_stage(int64_t key, size_t& v, bool& fill) {
 bool FileExpertSource::fill_stage(size_t v, int64_t layer, int64_t expert, uint8_t* dst) {
     const auto t0 = std::chrono::steady_clock::now();
     const bool ok = copy_from_files(layer, expert, dst);
-    const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+    publish_stage(v, layer, ok, std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count());
+    return ok;
+}
+
+void FileExpertSource::publish_stage(size_t v, int64_t layer, bool ok, double us) {
     file_us_.fetch_add((uint64_t) us, std::memory_order_relaxed);
     if (ok) {
         file_read_bytes_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);
@@ -725,7 +744,6 @@ bool FileExpertSource::fill_stage(size_t v, int64_t layer, int64_t expert, uint8
         }
     }
     stage_cv_.notify_all();
-    return ok;
 }
 
 const uint8_t* FileExpertSource::staged_blob(int64_t layer, int64_t expert) {
@@ -749,8 +767,7 @@ const uint8_t* FileExpertSource::staged_blob(int64_t layer, int64_t expert) {
 }
 
 void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n) {
-    if (role_ptr_.empty() || n <= 0 || layer < 0 || layer >= n_layers_) return;
-    struct Fill { size_t v; int64_t e; uint8_t* dst; };
+    if (!staged() || n <= 0 || layer < 0 || layer >= n_layers_) return;
     std::vector<Fill> todo;
     {
         std::lock_guard<std::mutex> lk(stage_mu_);
@@ -764,7 +781,7 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
             size_t v = 0;
             bool fill = false;
             if (!claim_stage(layer * n_expert_ + e, v, fill) && fill) {
-                todo.push_back({v, e, stage_buf_[v].get()});
+                todo.push_back({v, layer, e, stage_buf_[v].get()});
                 if (warm_stamp_) {
                     const uint32_t s = warm_stamp_[index].load(std::memory_order_relaxed);
                     if (s != 0 && (uint64_t) s + 3 >= epoch_ + 1) warm_hits_.fetch_add(1, std::memory_order_relaxed);
@@ -772,7 +789,52 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
             }
         }
     }
+    fill_many(todo);
+}
+
+void FileExpertSource::prefetch_pairs(const std::pair<int32_t, int32_t>* pairs, int64_t n) {
+    if (direct_.empty() || pairs == nullptr || n <= 0) return;   // unbuffered only: the mapped fill stays as it was
+    std::vector<Fill> todo;
+    {
+        std::lock_guard<std::mutex> lk(stage_mu_);
+        for (int64_t i = 0; i < std::min<int64_t>(n, 64); ++i) {
+            const int64_t l = pairs[i].first, e = pairs[i].second;
+            if (l < 0 || e < 0 || l >= n_layers_ || e >= n_expert_) continue;
+            size_t v = 0;
+            bool fill = false;
+            if (!claim_stage(l * n_expert_ + e, v, fill) && fill) todo.push_back({v, l, e, stage_buf_[v].get()});
+        }
+    }
+    fill_many(todo);
+}
+
+void FileExpertSource::fill_many(const std::vector<Fill>& todo) {
     if (todo.empty()) return;
+    if (!direct_.empty()) {
+        // #286: overlapped batches - every role window of 16 blobs in flight at once; a big batch (the profile
+        // fill) on up to 4 threads, a decode layer's few misses on this one
+        std::atomic<size_t> next{0};
+        auto work = [&] {
+            for (size_t at; (at = next.fetch_add(16)) < todo.size();) {
+                const size_t k = std::min<size_t>(16, todo.size() - at);
+                const auto t0 = std::chrono::steady_clock::now();
+                const bool ok = read_direct(todo.data() + at, k);
+                const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+                for (size_t i = at; i < at + k; ++i) {
+                    const Fill& f = todo[i];
+                    // a failed batch: each blob again on its own (copy_from_files falls back to the mapping)
+                    if (ok) publish_stage(f.v, f.layer, true, us / (double) k);
+                    else (void) fill_stage(f.v, f.layer, f.e, f.dst);
+                }
+            }
+        };
+        const size_t nt = std::min<size_t>(4, (todo.size() + 15) / 16);
+        std::vector<std::thread> th;
+        for (size_t t = 1; t < nt; ++t) th.emplace_back(work);
+        work();
+        for (auto& t : th) t.join();
+        return;
+    }
 #if defined(_WIN32)
     // One PrefetchVirtualMemory call for every slice about to be copied: the memory manager reads them in large
     // requests, all queued at once, where the copies' page faults would read a few clusters each.  The copies below
@@ -789,7 +851,7 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
             ranges.reserve(todo.size() * 3);
             for (const Fill& f : todo)
                 for (int r = 0; r < 3; ++r) {
-                    const size_t i = (size_t) (3 * layer + r);
+                    const size_t i = (size_t) (3 * f.layer + r);
                     ranges.push_back({(PVOID) (role_ptr_[i] + (size_t) ((uint64_t) f.e * role_bytes_[i])),
                                       (SIZE_T) role_bytes_[i]});
                 }
@@ -801,7 +863,7 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
     std::atomic<size_t> next{0};
     auto work = [&] {
         for (size_t i; (i = next.fetch_add(1)) < todo.size();)
-            (void) fill_stage(todo[i].v, layer, todo[i].e, todo[i].dst);
+            (void) fill_stage(todo[i].v, todo[i].layer, todo[i].e, todo[i].dst);
     };
     const size_t nt = std::min<size_t>(todo.size(), (size_t) fetch_threads_);
     std::vector<std::thread> th;
@@ -810,6 +872,159 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
     for (auto& t : th) t.join();
 }
 
+bool FileExpertSource::set_unbuffered(uint64_t ram_bytes, std::string& why) {
+#if defined(_WIN32)
+    if (base_ == nullptr || paths_.empty() || !direct_.empty()) {
+        why = !direct_.empty() ? "already unbuffered" : "no expert files open";
+        return !direct_.empty();
+    }
+    if (!experts_unbuffered(paths_, ram_bytes, why, /*cache_counts=*/false)) return false;
+    for (const std::string& path : paths_) {
+        const int wide = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, nullptr, 0);
+        std::vector<wchar_t> w((size_t) (wide > 0 ? wide : 1), L'\0');
+        if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, w.data(), wide);
+        HANDLE h = CreateFileW(w.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                               FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);
+        if (h == INVALID_HANDLE_VALUE) {
+            why += "; cannot open " + path + " unbuffered (error " + std::to_string((unsigned long long) GetLastError()) +
+                   "), read through the file cache";
+            for (void* d : direct_) CloseHandle((HANDLE) d);
+            direct_.clear();
+            return false;
+        }
+        direct_.push_back(h);
+    }
+    if (role_ptr_.empty()) {   // experts.bin: blob() now assembles into the stage buffers, sized for the largest blob
+        std::lock_guard<std::mutex> lk(stage_mu_);
+        for (uint64_t b : layer_blob_bytes_) stage_blob_ = std::max(stage_blob_, b);
+    }
+    return true;
+#else
+    (void) ram_bytes;
+    why = "through the file cache (not Windows)";
+    return false;
+#endif
+}
+
+bool FileExpertSource::read_direct(const Fill* fills, size_t n) const {
+#if defined(_WIN32)
+    // NTFS runs the unbuffered reads of a file one at a time while the file is mapped or cached anywhere (the engine
+    // maps the GGUF shards for the token embedding and the PLE table): measured on a PCIe 5 drive, 512 KiB reads at
+    // queue depth 48 make 10.4 GB/s on a file nobody maps and 3.4 GB/s on a mapped one, 8 MiB reads 8.6 GB/s.  So
+    // windows less than kGap apart (the same role of nearby experts) are merged into requests of up to kMerge bytes.
+    constexpr uint64_t kSector = 4096, kGap = 1ull << 20, kMerge = 32ull << 20;
+    struct Window { int file; uint64_t a0, size, skip, n, at; uint8_t* dst; size_t req; uint64_t in_req; };
+    struct Req { HANDLE h; uint64_t a0, size, pos; };
+    // this thread's aligned buffer and events, kept for its next batch
+    struct Scratch {
+        uint8_t* buf = nullptr;
+        size_t cap = 0;
+        std::vector<HANDLE> ev;
+        std::vector<OVERLAPPED> ov;
+        std::vector<Window> win;
+        std::vector<size_t> order;
+        std::vector<Req> req;
+        std::vector<DWORD> got;
+        ~Scratch() {
+            if (buf != nullptr) VirtualFree(buf, 0, MEM_RELEASE);
+            for (HANDLE e : ev) CloseHandle(e);
+        }
+    };
+    thread_local Scratch sc;
+    sc.win.clear();
+    for (size_t k = 0; k < n; ++k) {
+        const Fill& f = fills[k];
+        if (f.dst == nullptr || f.layer < 0 || f.e < 0 || f.layer >= n_layers_ || f.e >= n_expert_) return false;
+        if (role_ptr_.empty()) {   // experts.bin: the blob is one contiguous range
+            const uint64_t per = layer_blob_bytes_[(size_t) f.layer];
+            const uint64_t off = layer_offsets_[(size_t) f.layer] + (uint64_t) f.e * per;
+            const uint64_t a0 = off / kSector * kSector, a1 = (off + per + kSector - 1) / kSector * kSector;
+            sc.win.push_back({0, a0, a1 - a0, off - a0, per, 0, f.dst, 0, 0});
+            continue;
+        }
+        uint64_t at = 0;
+        for (int r = 0; r < 3; ++r) {
+            const size_t i = (size_t) (3 * f.layer + r);
+            const uint64_t per = role_bytes_[i];
+            const Map& m = maps_[(size_t) role_file_[i]];
+            const uint64_t off = (uint64_t) (role_ptr_[i] - m.base) + (uint64_t) f.e * per;
+            const uint64_t a0 = off / kSector * kSector, a1 = (off + per + kSector - 1) / kSector * kSector;
+            sc.win.push_back({role_file_[i], a0, a1 - a0, off - a0, per, at, f.dst, 0, 0});
+            at += per;
+        }
+    }
+    sc.order.resize(sc.win.size());
+    for (size_t w = 0; w < sc.win.size(); ++w) sc.order[w] = w;
+    std::sort(sc.order.begin(), sc.order.end(), [&](size_t a, size_t b) {
+        return sc.win[a].file != sc.win[b].file ? sc.win[a].file < sc.win[b].file : sc.win[a].a0 < sc.win[b].a0;
+    });
+    sc.req.clear();
+    uint64_t total = 0;
+    int last_file = -1;
+    for (size_t w : sc.order) {
+        Window& x = sc.win[w];
+        if (!sc.req.empty() && x.file == last_file) {
+            Req& q = sc.req.back();
+            const uint64_t end = q.a0 + q.size, xend = x.a0 + x.size;
+            if (x.a0 <= end + kGap && std::max(end, xend) - q.a0 <= kMerge) {
+                if (xend > end) {
+                    total += xend - end;
+                    q.size = xend - q.a0;
+                }
+                x.req = sc.req.size() - 1;
+                x.in_req = x.a0 - q.a0;
+                continue;
+            }
+        }
+        sc.req.push_back({(HANDLE) direct_[(size_t) x.file], x.a0, x.size, total});
+        total += x.size;
+        last_file = x.file;
+        x.req = sc.req.size() - 1;
+        x.in_req = 0;
+    }
+    if (total > sc.cap) {
+        if (sc.buf != nullptr) VirtualFree(sc.buf, 0, MEM_RELEASE);
+        sc.cap = (size_t) ((total + (1u << 20) - 1) >> 20 << 20);
+        sc.buf = (uint8_t*) VirtualAlloc(nullptr, sc.cap, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+        if (sc.buf == nullptr) { sc.cap = 0; return false; }
+    }
+    while (sc.ev.size() < sc.req.size()) {
+        HANDLE e = CreateEventW(nullptr, TRUE, FALSE, nullptr);
+        if (e == nullptr) return false;
+        sc.ev.push_back(e);
+    }
+    sc.ov.assign(sc.req.size(), OVERLAPPED{});
+    // every request in flight before the first wait: the drive sees the whole batch as one queue
+    size_t issued = 0;
+    bool ok = true;
+    for (size_t q = 0; q < sc.req.size(); ++q) {
+        const Req& r = sc.req[q];
+        OVERLAPPED& o = sc.ov[q];
+        o.Offset = (DWORD) r.a0;
+        o.OffsetHigh = (DWORD) (r.a0 >> 32);
+        o.hEvent = sc.ev[q];
+        if (!ReadFile(r.h, sc.buf + r.pos, (DWORD) r.size, nullptr, &o) && GetLastError() != ERROR_IO_PENDING) {
+            ok = false;
+            break;
+        }
+        ++issued;
+    }
+    std::vector<DWORD>& got = sc.got;
+    got.assign(sc.req.size(), 0);
+    for (size_t q = 0; q < issued; ++q)
+        if (!GetOverlappedResult(sc.req[q].h, &sc.ov[q], &got[q], TRUE)) ok = false;
+    if (!ok || issued < sc.req.size()) return false;
+    for (const Window& x : sc.win) {
+        // a request may run past the end of the file: only the role's own bytes have to arrive
+        if ((uint64_t) got[x.req] < x.in_req + x.skip + x.n) return false;
+        std::memcpy(x.dst + x.at, sc.buf + sc.req[x.req].pos + x.in_req + x.skip, (size_t) x.n);
+    }
+    return true;
+#else
+    (void) fills; (void) n;
+    return false;
+#endif
+}
 
 void FileExpertSource::warm(int64_t layer, const int64_t* experts, int64_t n) {
     if (role_ptr_.empty() || n <= 0 || layer < 0 || layer >= n_layers_) return;
@@ -932,7 +1147,7 @@ void RouterLookahead::run() {
 void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k) {
     (void) ids;
     (void) k;
-    if (role_ptr_.empty()) return;
+    if (!staged()) return;
     std::lock_guard<std::mutex> lk(stage_mu_);
     if (layer != last_layer_) {
         ++epoch_;
@@ -941,7 +1156,7 @@ void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k)
 }
 
 bool FileExpertSource::transient(int64_t layer, int64_t expert) const {
-    if (role_ptr_.empty() || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
+    if (!staged() || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
     const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
     if (complement_ready_ && index < complement_offsets_.size() && complement_offsets_[index] != kNoComplement)
         return false;
@@ -1271,17 +1486,43 @@ bool FileExpertSource::pin_cache_complement(
             const int64_t layer = next_layer.fetch_add(1);
             if (layer >= n_layers_ || failed.load()) return;
             const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer];
+            std::vector<Fill> batch;   // #286, unbuffered: 32 blobs' reads in flight at once, merged where near
+            auto flush = [&]() {
+                if (batch.empty()) return true;
+                bool ok = read_direct(batch.data(), batch.size());
+                for (size_t i = 0; !ok && i < batch.size(); ++i)
+                    if (!copy_from_files(batch[i].layer, batch[i].e, batch[i].dst)) return false;
+                copied.fetch_add(blob_bytes * (uint64_t) batch.size());
+                batch.clear();
+                return true;
+            };
             for (int64_t expert = 0; expert < n_expert_; ++expert) {
                 const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
                 const uint64_t offset = offsets[index];
                 if (offset == kNoComplement) continue;
-                if (offset > bytes || blob_bytes > bytes - offset ||
-                    !copy_from_files(layer, expert, (uint8_t*) host + (size_t) offset)) {
+                if (offset > bytes || blob_bytes > bytes - offset) {
+                    fail("FileExpertSource: invalid blob bounds while building the cache complement");
+                    return;
+                }
+                uint8_t* dst = (uint8_t*) host + (size_t) offset;
+                if (!direct_.empty()) {
+                    batch.push_back({0, layer, expert, dst});
+                    if (batch.size() == 32 && !flush()) {
+                        fail("FileExpertSource: an expert could not be read while building the cache complement");
+                        return;
+                    }
+                    continue;
+                }
+                if (!copy_from_files(layer, expert, dst)) {
                     fail("FileExpertSource: invalid blob bounds while building the cache complement");
                     return;
                 }
                 copied.fetch_add(blob_bytes);
             }
+            if (!flush()) {
+                fail("FileExpertSource: an expert could not be read while building the cache complement");
+                return;
+            }
 #if !defined(_WIN32)
             if (role_ptr_.empty()) {   // experts.bin; the GGUF in place leaves its pages to the OS
             const uint64_t layer_offset = layer_offsets_[(size_t) layer];
@@ -1444,7 +1685,7 @@ const uint8_t* FileExpertSource::blob(int64_t layer, int64_t expert) {
         }
     }
     if (from_files) {
-        if (!role_ptr_.empty()) {
+        if (staged()) {
             result = staged_blob(layer, expert);          // counts its bytes
         } else {
             result = mapped_blob(layer, expert);
@@ -1659,7 +1900,8 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
             if (first_of[i] == i) {
                 distinct[nd++] = i;
                 const int32_t e = ids[i];
-                if (e >= 0 && e < d.n_expert && d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] < 0) ++nmiss;
+                if (e >= 0 && e < d.n_expert && d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] < 0 &&
+                    !(d.peer != nullptr && d.peer->has(d.layers, e))) ++nmiss;
             }
         }
         const bool pcie_ok = d.pcie_num > 0 && d.src->pcie_layer(d.layers);
@@ -1679,6 +1921,8 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                     kd = 0;
                     ptr = (unsigned long long) (d.cache_base + (d.cache_slot_off ? (size_t) d.cache_slot_off[slot]
                                                                                  : (size_t) slot * (size_t) d.cache_blob));
+                } else if (d.peer != nullptr && d.peer->has(d.layers, e)) {
+                    kd = 2;                        // multi-GPU: the second GPU computes it
                 } else {
                     if (miss_rank >= nmiss - m && fetches < P.staging_cap && fetches < 64) {
                         const uint8_t* src = d.src->pinned(d.layers, e) ? d.src->blob(d.layers, e) : nullptr;
@@ -1733,7 +1977,18 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
         for (int64_t i = 0; i < n; ++i) {
             const int32_t e = ids[i];
             kind[i] = (e >= 0 && e < d.n_expert && d.host_res != nullptr &&
-                       d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] >= 0) ? 0 : -1;
+                       d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] >= 0) ? 0
+                    : (e >= 0 && e < d.n_expert && d.peer != nullptr && d.peer->has(d.layers, e)) ? 2 : -1;
+        }
+    }
+    if (d.peer != nullptr) {                 // multi-GPU: start the second GPU's share before the CPU's own work
+        std::string perr;
+        if (!d.peer->launch(d.layers, x_f, ids, n_tok, k, kind, perr, out)) {
+            std::fprintf(stderr, "strata: %s (layer %lld)\n", perr.c_str(), (long long) d.layers);
+            d.failed = true;
+            d.fail = "the peer GPU's experts could not be launched";
+            d.fail_layer = d.layers;
+            return;
         }
     }
     if (d.remote_count > 0) {
@@ -1776,9 +2031,12 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                 d.fail_expert = e;
                 return;
             }
-            if (kind[i] >= 0) {             // CUDA0, PCIe, or a remote result staged into this row below
+            if (kind[i] >= 0) {             // CUDA0, PCIe, or a remote/peer result staged into this row below
                 if (kind[i] == 0) ++d.cache_hits;
-                std::memset(row, 0, (size_t) H * sizeof(float));
+                else if (kind[i] == 2 && d.peer != nullptr) ++d.peer_entries;
+                // multi-GPU: a direct peer launch is writing this row right now - zeroing it would race it
+                if (!(kind[i] == 2 && d.peer != nullptr && d.peer->launched_direct()))
+                    std::memset(row, 0, (size_t) H * sizeof(float));
                 continue;
             }
             ++d.cache_refused;
@@ -1816,6 +2074,16 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                 d.failed = true; d.fail = remote_error.c_str(); d.fail_layer = d.layers; return;
             }
     }
+    if (d.peer != nullptr) {                 // multi-GPU: the second GPU's rows, into the same mapped rows
+        std::string perr;
+        if (!d.peer->finish(out, perr)) {
+            std::fprintf(stderr, "strata: %s (layer %lld)\n", perr.c_str(), (long long) d.layers);
+            d.failed = true;
+            d.fail = "the peer GPU's experts failed";
+            d.fail_layer = d.layers;
+            return;
+        }
+    }
     const auto c4 = std::chrono::steady_clock::now();
     pt("ran");
     auto ms = [](auto a, auto b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
@@ -2115,13 +2383,104 @@ bool check_experts_gguf(const std::string& gguf, const strata::kernels::cpu::Exp
 // [gate rows | up rows | down rows] - the layout tools/iq_pack.py would have written to experts.bin.  Each role
 // is read from its own file (native_experts.txt v4: a shard boundary can fall inside a layer; per role as in
 // #255, gopinath87607).  The caller checks the spans first (check_experts_gguf).
+// `unbuffered` (Windows, experts_unbuffered): each chunk's 4 KiB-aligned window is read with FILE_FLAG_NO_BUFFERING into
+// an aligned buffer and scattered into the blobs - no copy through the file cache when the drive is read anyway.
 LoadStats load_experts_gguf(const std::string& gguf, uint8_t* dst, const strata::kernels::cpu::ExpertLayout& lay,
-                            int threads) {
+                            int threads, bool unbuffered) {
     LoadStats st;
     st.layers = (uint64_t) lay.n_layers;
     const auto t0 = std::chrono::steady_clock::now();
     std::atomic<int64_t> next{0};
     std::atomic<bool> bad{false};
+#if defined(_WIN32)
+    if (unbuffered) {
+        uint64_t max_chunk = 0;
+        for (int64_t l = 0; l < lay.n_layers; ++l) {
+            const auto& fm = lay.fmt[(size_t) l];
+            max_chunk = std::max<uint64_t>(max_chunk, std::max<uint64_t>(fm.up_off, lay.bytes[(size_t) l] - fm.down_off) * 16);
+        }
+        std::mutex err_mu;
+        std::string err;
+        auto worker = [&]() {
+            constexpr uint64_t kSector = 4096;
+            const uint64_t cap = (max_chunk + 2 * kSector + kSector - 1) / kSector * kSector;
+            uint8_t* buf = (uint8_t*) VirtualAlloc(nullptr, (size_t) cap, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+            HANDLE h = INVALID_HANDLE_VALUE;
+            std::string open_name;
+            auto fail = [&](const std::string& what) {
+                std::lock_guard<std::mutex> g(err_mu);
+                if (err.empty()) err = what;
+                bad = true;
+            };
+            if (buf == nullptr) fail("cannot allocate a read buffer");
+            for (;;) {
+                const int64_t l = next.fetch_add(1);
+                if (l >= lay.n_layers || bad) break;
+                const auto& fm = lay.fmt[(size_t) l];
+                const uint64_t blob = lay.bytes[(size_t) l];
+                const uint64_t per[3] = {fm.up_off, fm.up_off, blob - fm.down_off};
+                const uint64_t at[3] = {0, fm.up_off, fm.down_off};
+                for (int r = 0; r < 3 && !bad; ++r) {
+                    // the handle is kept while consecutive roles share a file (every layer of a v3 pack)
+                    const std::string name = expert_gguf_file(gguf, lay, l, r);
+                    if (name != open_name) {
+                        if (h != INVALID_HANDLE_VALUE) CloseHandle(h);
+                        const int wide = MultiByteToWideChar(CP_UTF8, 0, name.c_str(), -1, nullptr, 0);
+                        std::vector<wchar_t> w((size_t) std::max(wide, 1), L'\0');
+                        if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, name.c_str(), -1, w.data(), wide);
+                        h = CreateFileW(w.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                                        FILE_FLAG_NO_BUFFERING | FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
+                        if (h == INVALID_HANDLE_VALUE) {
+                            fail("cannot open " + name + " (error " + std::to_string((unsigned long long) GetLastError()) + ")");
+                            break;
+                        }
+                        open_name = name;
+                    }
+                    const uint64_t src = lay.gguf_off[(size_t) (3 * l + r)];
+                    const uint64_t total = per[r] * (uint64_t) lay.n_expert;
+                    const uint64_t chunk = per[r] * 16;           // 16 experts per read
+                    for (uint64_t done = 0; done < total; done += chunk) {
+                        const uint64_t n = std::min<uint64_t>(chunk, total - done);
+                        const uint64_t a0 = (src + done) / kSector * kSector;
+                        const uint64_t a1 = (src + done + n + kSector - 1) / kSector * kSector;
+                        OVERLAPPED ov{};
+                        ov.Offset = (DWORD) a0;
+                        ov.OffsetHigh = (DWORD) (a0 >> 32);
+                        DWORD got = 0;
+                        // the window may run past the end of the file: only the tensor's own bytes have to arrive
+                        if (!ReadFile(h, buf, (DWORD) (a1 - a0), &got, &ov) || (uint64_t) got < src + done - a0 + n) {
+                            fail("short unbuffered read of layer " + std::to_string(l) + " in " + name + " (error " +
+                                 std::to_string((unsigned long long) GetLastError()) + ")");
+                            break;
+                        }
+                        const uint8_t* q = buf + (src + done - a0);
+                        for (uint64_t k = 0; k < n / per[r]; ++k) {
+                            const uint64_t e = done / per[r] + k;
+                            std::memcpy(dst + lay.blob_offset(l, (int64_t) e) + at[r], q + k * per[r], (size_t) per[r]);
+                        }
+                    }
+                }
+            }
+            if (h != INVALID_HANDLE_VALUE) CloseHandle(h);
+            if (buf != nullptr) VirtualFree(buf, 0, MEM_RELEASE);
+        };
+        std::vector<std::thread> pool;
+        for (int i = 1; i < threads; ++i) pool.emplace_back(worker);
+        worker();
+        for (auto& t : pool) t.join();
+        if (bad) {
+            st.seconds = -1.0;
+            st.ok = false;
+            st.error = err.empty() ? "unreadable shard while reading the experts from the GGUF" : err;
+            return st;
+        }
+        st.bytes = lay.total;
+        st.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+        return st;
+    }
+#else
+    (void) unbuffered;
+#endif
     auto worker = [&]() {
         // #230: `fread` on a `FILE*`, as load_experts_ranges (#89): MSVC's `std::ifstream::read` splits a request
         // into 4095-byte freads, which took this path to 0.02 GiB/s on a Windows install without experts.bin.
@@ -2245,8 +2604,31 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
               (why.empty() ? std::string{} : ": " + why);
         return false;
     }
-    const LoadStats st = from_gguf ? load_experts_gguf(gguf_, a->data(), lay, threads)
-                                   : load_experts_ranges(path, a->data(), loff, lbytes, threads, /*chunk=*/8u << 20);
+    // #285: unbuffered when the drive is read anyway and the file cache could not keep the experts for the next
+    // start either (a 64 GB PC); otherwise the buffered readers, which a warm restart serves from the cache
+    std::vector<std::string> files;
+    if (from_gguf) {
+        for (int64_t l = 0; l < n_layers; ++l)
+            for (int r = 0; r < 3; ++r) {
+                const std::string f = expert_gguf_file(gguf_, lay, l, r);
+                if (std::find(files.begin(), files.end(), f) == files.end()) files.push_back(f);
+            }
+    } else {
+        files.push_back(path);
+    }
+    std::string why;
+    const bool unbuffered = experts_unbuffered(files, want + (uint64_t) blob, why);
+    const int readers = unbuffered ? std::max(threads, 16) : threads;   // 16 keep a PCIe 5 drive's queue full
+    LoadStats st;
+    if (from_gguf) {
+        st = load_experts_gguf(gguf_, a->data(), lay, readers, unbuffered);
+    } else {
+        if (unbuffered) st = load_experts_direct(path, a->data(), loff, lbytes, readers, /*chunk=*/8u << 20);
+        if (!unbuffered || (!st.ok && st.error.empty()))   // unaligned ranges: the buffered reader
+            st = load_experts_ranges(path, a->data(), loff, lbytes, threads, /*chunk=*/8u << 20);
+    }
+    std::fprintf(stderr, "strata generate: expert arena read %s (%s)\n", unbuffered ? "unbuffered" : "through the file cache",
+                 why.c_str());
     if (!st.ok) {
         delete a;
         err = "ArenaExpertSource: the expert load was refused: " + (st.error.empty() ? std::string("unknown") : st.error);

src/core/mtp.cpp

@@ -4,6 +4,7 @@
 #include "strata/core/on_device.hpp"
 
 #include "strata/core/native_head.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
 #include "strata/kernels/cpu/expert.hpp"
 #include "strata/kernels/elementwise.hpp"
@@ -30,6 +31,7 @@
 #include <atomic>
 #include <chrono>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <exception>
 #include <fstream>
@@ -55,7 +57,7 @@ struct Bump {
 };
 
 bool mapped(size_t bytes, void** h, void** d) {
-    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped) != cudaSuccess) return false;
+    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false;
     std::memset(*h, 0, bytes);
     return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;
 }
@@ -372,6 +374,45 @@ void MtpDrafter::set_draft_history(const int32_t* tail, int64_t n_tail, int32_t
     coupled_hist_base(tail, n_tail, next, h, h_chist_ + (kCoupledHistCap - h));
 }
 
+namespace {
+// #474: what to change when the draft head's token subset does not fit the VRAM left - a smaller subset that setup
+// ships (data/draft_vocab_*.bin; their token counts below), and how to pick it.  The start used to stop at "the draft
+// head does not fit" with no hint.  Text only, after the failure: nothing changes for a start that fits.
+void draft_head_hint(int64_t n_tokens, int64_t row_bytes) {
+    size_t free_b = 0, total_b = 0;
+    const bool have_free = cudaMemGetInfo(&free_b, &total_b) == cudaSuccess;
+    auto mib = [&](int64_t n) { return (double) (n * row_bytes) / 1048576.0; };
+    std::string free_s;
+    if (have_free) {
+        char b[64];
+        std::snprintf(b, sizeof(b), " and %.0f MiB is free", (double) free_b / 1048576.0);
+        free_s = b;
+    }
+    std::fprintf(stderr, "strata mtp: the draft head over %lld tokens needs %.0f MiB of VRAM%s.\n",
+                 (long long) n_tokens, mib(n_tokens), free_s.c_str());
+    struct Subset { const char* name; int64_t tokens; const char* what; };
+    static const Subset smaller[] = {{"cyrillic", 58963, "English, code and the Cyrillic script"},
+                                     {"en", 40525, "English and code"}};
+    std::string opts;
+    for (const Subset& s : smaller)
+        if (s.tokens < n_tokens) {
+            char b[160];
+            std::snprintf(b, sizeof(b), "%s--draft-vocab %s (%s, ~%.0f MiB)", opts.empty() ? "" : " or ", s.name,
+                          s.what, mib(s.tokens));
+            opts += b;
+        }
+    if (!opts.empty())
+        std::fprintf(stderr, "strata mtp: hint: a smaller draft vocabulary needs less VRAM: %s. Start once with it - "
+                             "START-HERE.bat --draft-vocab en (Windows) or ./setup.sh --draft-vocab en - and the model "
+                             "keeps it (\"draft_vocab\" in its strata-*.json config); or a smaller --context in "
+                             "setup.\n",
+                     opts.c_str());
+    else
+        std::fprintf(stderr, "strata mtp: hint: this is already the smallest shipped draft vocabulary: a smaller "
+                             "--context (or closing what else uses the GPU) leaves it room.\n");
+}
+}  // namespace
+
 bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err) {
     const OnDevice on_device(device_);
     wt_ = &wt;
@@ -395,6 +436,7 @@ bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float
             if (cudaMalloc((void**) &dvocab_, raw.size()) != cudaSuccess ||
                 cudaMalloc((void**) &dhead_, (size_t) (n_dvocab_ * row_bytes)) != cudaSuccess) {
                 err = "mtp: the draft head does not fit";
+                draft_head_hint(n_dvocab_, row_bytes);
                 return false;
             }
             cudaMemcpy(dvocab_, raw.data(), raw.size(), cudaMemcpyHostToDevice);
@@ -688,8 +730,21 @@ bool MtpDrafter::prefill(const float* R_rows, const int32_t* next_tokens, int64_
     // E-4: every group's token / step / position records uploaded at once; each group is then device copies and a
     // graph on the one stream, with a single sync at the end (a group of <= max_t rows used to be staged in mapped
     // memory and synced before the next: ~5,500 host round trips on a 32K prompt).  The same work in the same
-    // order.  STRATA_MTP_PREFILL_SYNC=1 keeps the old loop.
-    static const bool per_group_sync = std::getenv("STRATA_MTP_PREFILL_SYNC") != nullptr;
+    // order.  STRATA_MTP_PREFILL_SYNC=1 keeps the old loop, =0 forces E-4.
+    // HIP defaults to the old loop.  This pass runs whenever E-9 (Prefill::draft_kv) declines, which it does for the
+    // drafter's ring (KV streaming, --kv-resident), and on gfx1201 / ROCm 7.2.4 the E-4 queue (a graph launch and four
+    // device copies per group, ~2,000 groups per 8192-token chunk, no sync) sometimes never completes: the prompt hangs
+    // in hipGraphLaunch or the final sync until the watchdog ends the engine.  R9700, full IQ3_XXS, --kv-resident
+    // 32768, mixed load (chats, 32K and 7K prompts, deep follow-ups): E-4 hung in the first round on 2 of 2 tries,
+    // the old loop ran 30 of 30 rounds clean, prompt speed unchanged.
+    static const bool per_group_sync = [] {
+        if (const char* v = std::getenv("STRATA_MTP_PREFILL_SYNC")) return std::atoi(v) != 0;
+#if defined(STRATA_USE_HIP)
+        return true;
+#else
+        return false;
+#endif
+    }();
     const int64_t NHp = g_->n_head, per_row = 1 + 4 + NHp;
     if (!per_group_sync && n > 0) {
         if (pf_cap_ < n * per_row) {

src/core/peer_experts.cpp

@@ -0,0 +1,325 @@
+// src/core/peer_experts.cpp - see include/strata/core/peer_experts.hpp.
+#include "strata/core/peer_experts.hpp"
+
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/elementwise.hpp"
+#include "strata/kernels/iq_kernels.hpp"
+#include "strata/kernels/quantize_act.hpp"
+#include "strata/kernels/s2_expert_grouped.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdio>
+#include <cstring>
+#include <cstdlib>
+#include <stdexcept>
+
+namespace strata::core {
+
+static bool g_peer_portable = false;
+void set_peer_portable(bool on) { g_peer_portable = on; }
+bool peer_portable() { return g_peer_portable; }
+
+namespace {
+
+constexpr int64_t H = strata::kernels::cpu::H;
+constexpr int64_t CAP = strata::kernels::cpu::MAXT * 10;   // entries per layer: MAXT tokens x top-10
+
+struct Meta {                           // one block, uploaded with one copy per layer
+    unsigned long long ptr[CAP];
+    int32_t start[CAP + 1];
+    int32_t dst[CAP];
+    int32_t tok[CAP];
+    int32_t count[4];                   // [0] groups, [1] entries
+    int32_t out_row[CAP];               // multi-GPU: compact row -> entry row in the host `out`
+};
+
+/// Switches the calling thread to `dev` and back.  The engine's own thread lives on device 0.
+struct On {
+    int prev = 0;
+    explicit On(int dev) { cudaGetDevice(&prev); if (prev != dev) cudaSetDevice(dev); }
+    ~On() { int cur = 0; cudaGetDevice(&cur); if (cur != prev) cudaSetDevice(prev); }
+};
+
+bool ck(cudaError_t e, const char* what, std::string& err) {
+    if (e == cudaSuccess) return true;
+    err = std::string("peer experts: ") + what + ": " + cudaGetErrorString(e);
+    return false;
+}
+
+}  // namespace
+
+PeerExperts::~PeerExperts() { close(); }
+
+void PeerExperts::close() {
+    if (device_ < 0) return;
+    {
+        On on(device_);
+        if (stream_) cudaStreamSynchronize(stream_);
+        if (refill_) cudaStreamSynchronize(refill_);
+        cache_.close();
+        if (d_x_) cudaFree(d_x_);
+        if (d_out_) cudaFree(d_out_);
+        if (d_meta_) cudaFree(d_meta_);
+        if (d_q8_) cudaFree(d_q8_);
+        if (d_scratch_) cudaFree(d_scratch_);
+        if (h_x_) cudaFreeHost(h_x_);
+        if (h_out_) cudaFreeHost(h_out_);
+        if (h_meta_) cudaFreeHost(h_meta_);
+        if (refill_ev_) cudaEventDestroy(refill_ev_);
+        if (stream_) cudaStreamDestroy(stream_);
+        if (refill_) cudaStreamDestroy(refill_);
+    }
+    d_x_ = d_out_ = h_x_ = h_out_ = nullptr;
+    d_meta_ = h_meta_ = d_scratch_ = nullptr;
+    d_q8_ = nullptr;
+    stream_ = refill_ = nullptr;
+    refill_ev_ = nullptr;
+    device_ = -1;
+}
+
+bool PeerExperts::open(int device, const std::vector<std::pair<int32_t, int32_t>>& ranked, const ExpertCache& primary,
+                       ExpertSource& src, int64_t n_layers, int64_t n_expert, int reserve_mib, int64_t max_slots,
+                       std::string& err) {
+    close();
+    int count = 0;
+    if (!ck(cudaGetDeviceCount(&count), "cudaGetDeviceCount", err)) return false;
+    if (device < 1 || device >= count) {
+        err = "peer experts: CUDA device " + std::to_string(device) + " is not visible (" + std::to_string(count) +
+              " devices; set CUDA_VISIBLE_DEVICES)";
+        return false;
+    }
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    n_layers_ = n_layers;
+    n_expert_ = n_expert;
+    src_ = &src;
+    res_.assign((size_t) (n_layers * n_expert), kNotResident);
+    {   // direct access both ways (NVLink or another P2P path): the prompt path copies activations and results over it
+        int a = 0, b = 0;
+        cudaDeviceCanAccessPeer(&a, 0, device);
+        cudaDeviceCanAccessPeer(&b, device, 0);
+        p2p_ = a && b;
+        if (p2p_) {
+            On on0(0);
+            cudaError_t e0 = cudaDeviceEnablePeerAccess(device, 0);
+            if (e0 == cudaErrorPeerAccessAlreadyEnabled) { cudaGetLastError(); e0 = cudaSuccess; }
+            On on1(device);
+            cudaError_t e1 = cudaDeviceEnablePeerAccess(0, 0);
+            if (e1 == cudaErrorPeerAccessAlreadyEnabled) { cudaGetLastError(); e1 = cudaSuccess; }
+            p2p_ = e0 == cudaSuccess && e1 == cudaSuccess;
+        }
+    }
+    On on(device);
+    device_ = device;
+    // scratch first, so the slots take what is really left
+    int64_t ff = strata::kernels::cpu::FF;
+    if (lay.native)
+        for (const auto& f : lay.fmt) ff = std::max<int64_t>(ff, f.n_ff);
+    const size_t scratch = std::max<size_t>((size_t) strata::kernels::moe_hit_grouped_scratch_bytes(CAP, H, ff),
+                                            strata::kernels::native_expert_scratch_bytes(CAP, ff));
+    const bool alloc_ok =
+        ck(cudaStreamCreateWithFlags(&stream_, cudaStreamNonBlocking), "stream", err) &&
+        ck(cudaStreamCreateWithFlags(&refill_, cudaStreamNonBlocking), "refill stream", err) &&
+        ck(cudaEventCreateWithFlags(&refill_ev_, cudaEventDisableTiming), "event", err) &&
+        ck(cudaHostAlloc((void**) &h_x_, (size_t) CAP * H * sizeof(float), cudaHostAllocPortable | cudaHostAllocMapped), "input staging", err) &&
+        ck(cudaHostAlloc((void**) &h_out_, (size_t) CAP * H * sizeof(float), cudaHostAllocPortable | cudaHostAllocMapped), "result staging", err) &&
+        ck(cudaHostAlloc(&h_meta_, sizeof(Meta), cudaHostAllocPortable | cudaHostAllocMapped), "plan staging", err) &&
+        ck(cudaMalloc((void**) &d_x_, (size_t) CAP * H * sizeof(float)), "input", err) &&
+        ck(cudaMalloc((void**) &d_out_, (size_t) CAP * H * sizeof(float)), "result", err) &&
+        ck(cudaMalloc(&d_meta_, sizeof(Meta)), "plan", err) &&
+        ck(cudaMalloc((void**) &d_q8_, (size_t) CAP * (H / 32) * 36), "activations", err) &&
+        ck(cudaMalloc(&d_scratch_, scratch), "scratch", err);
+    if (!alloc_ok) { close(); return false; }
+
+    // the pairs the primary does not hold, in rank order, as many as fit
+    size_t free_b = 0, total_b = 0;
+    if (!ck(cudaMemGetInfo(&free_b, &total_b), "cudaMemGetInfo", err)) { close(); return false; }
+    const uint64_t reserve = (uint64_t) std::max(reserve_mib, 128) << 20;
+    const uint64_t budget = free_b > reserve ? free_b - reserve : 0;
+    std::vector<std::pair<int32_t, int32_t>> pick;
+    std::vector<int64_t> sizes;
+    uint64_t used = 0;
+    for (const auto& pr : ranked) {
+        if (primary.slot_of(pr.first, pr.second) >= 0) continue;
+        const uint64_t b = lay.blob_bytes(pr.first);
+        const uint64_t b256 = lay.native ? (b + 255) / 256 * 256 : lay.max_blob;
+        if (used + b256 > budget) break;
+        if (max_slots > 0 && (int64_t) pick.size() >= max_slots) break;
+        used += b256;
+        pick.push_back(pr);
+        sizes.push_back((int64_t) b);
+    }
+    if (pick.empty()) { err = "peer experts: no room or no expert left for the peer"; close(); return false; }
+    const bool opened = lay.native ? cache_.open_sized(sizes, n_layers, n_expert, err)
+                                   : cache_.open((int64_t) pick.size(), n_layers, n_expert, (int64_t) lay.max_blob, err);
+    if (!opened) { err = "peer experts: " + err; close(); return false; }
+    // open() zeroes the arena with cudaMemset on the legacy stream, which is not ordered against the non-blocking
+    // refill stream: wait for it, or the zeroing can land on top of the fills
+    if (!ck(cudaDeviceSynchronize(), "arena zeroing", err)) { close(); return false; }
+    for (const auto& pr : pick) {
+        const int32_t slot = cache_.admit(pr.first, pr.second);
+        const uint8_t* b = src.blob(pr.first, pr.second);
+        if (slot < 0 || b == nullptr || !cache_.fill_slot(slot, b, refill_, err, (int64_t) lay.blob_bytes(pr.first))) {
+            err = "peer experts: fill failed: " + err;
+            close();
+            return false;
+        }
+        res_[(size_t) (pr.first * n_expert + pr.second)] = slot;
+    }
+    if (!ck(cudaStreamSynchronize(refill_), "fill", err)) { close(); return false; }
+    const auto& f0 = pick.front();
+    if (!cache_.verify_slot(res_[(size_t) (f0.first * n_expert + f0.second)], src.blob(f0.first, f0.second), err,
+                            (int64_t) lay.blob_bytes(f0.first))) {
+        err = "peer experts: " + err;
+        close();
+        return false;
+    }
+    resident_ = (int64_t) pick.size();
+    return true;
+}
+
+bool PeerExperts::launch(int64_t layer, const float* x, const int32_t* ids, int64_t n_tok, int64_t k,
+                         const int32_t* kind, std::string& err, float* out) {
+    static const bool direct_env = [] { const char* v = std::getenv("STRATA_PEER_DIRECT"); return v == nullptr || std::atoi(v) != 0; }();
+    const bool direct = direct_env && out != nullptr;
+    launched_direct_ = false;
+    launched_rows_ = 0;
+    row_of_.clear();
+    const int64_t n = n_tok * k;
+    if (n > CAP) { err = "peer experts: window too large"; return false; }
+    Meta& m = *(Meta*) h_meta_;
+    int groups = 0, rows = 0;
+    // distinct experts in routing order; each one's entries become compact rows
+    for (int64_t i = 0; i < n; ++i) {
+        if (kind[i] != 2) continue;
+        bool seen = false;
+        for (int64_t j = 0; j < i; ++j)
+            if (kind[j] == 2 && ids[j] == ids[i]) { seen = true; break; }
+        if (seen) continue;
+        const int32_t slot = res_[(size_t) (layer * n_expert_ + ids[i])];
+        if (slot < 0) { err = "peer experts: an entry was planned for the peer but is not resident"; return false; }
+        m.ptr[groups] = (unsigned long long) cache_.device_slot(slot);
+        m.start[groups] = rows;
+        for (int64_t j = i; j < n; ++j)
+            if (kind[j] == 2 && ids[j] == ids[i]) {
+                m.dst[rows] = rows;
+                m.tok[rows] = (int32_t) (j / k);
+                m.out_row[rows] = (int32_t) j;
+                row_of_.push_back((int32_t) j);
+                ++rows;
+            }
+        ++groups;
+        ++experts_;
+    }
+    if (groups == 0) return true;
+    m.start[groups] = rows;
+    m.count[0] = groups;
+    m.count[1] = rows;
+    entries_ += rows;
+    launched_rows_ = rows;
+    // direct: x is the verifier's pinned (portable) doorbell row block - copied from where it is
+    if (!direct) std::memcpy(h_x_, x, (size_t) (n_tok * H) * sizeof(float));
+    On on(device_);
+    const cudaStream_t s = stream_;
+    if (!ck(cudaMemcpyAsync(d_x_, direct ? x : h_x_, (size_t) (n_tok * H) * sizeof(float), cudaMemcpyHostToDevice, s), "input", err) ||
+        !ck(cudaMemcpyAsync(d_meta_, h_meta_, sizeof(Meta), cudaMemcpyHostToDevice, s), "plan", err))
+        return false;
+    Meta* dm = (Meta*) d_meta_;
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    if (lay.native) {
+        strata::kernels::quantize_q8_1_rows(d_x_, n_tok, H, d_q8_, s);
+        const auto& f = lay.fmt[(size_t) layer];
+        const auto L = strata::kernels::native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);
+        strata::kernels::native_expert_grouped(L, dm->ptr, dm->start, dm->count, dm->dst, dm->tok, groups, rows, d_q8_,
+                                               d_scratch_, d_out_, s);
+    } else {
+        err = "peer experts: only native packs are supported";
+        return false;
+    }
+    if (direct) {   // the rows straight into the host `out` (zero-copy, coalesced float4 writes)
+        try {
+            strata::kernels::scatter_rows_f32(d_out_, out, dm->out_row, rows, H, s);
+        } catch (const std::exception& e) { err = std::string("peer experts: scatter: ") + e.what(); return false; }
+        launched_direct_ = true;
+        return true;
+    }
+    return ck(cudaMemcpyAsync(h_out_, d_out_, (size_t) rows * H * sizeof(float), cudaMemcpyDeviceToHost, s),
+              "results", err);
+}
+
+bool PeerExperts::finish(float* out, std::string& err) {
+    if (launched_rows_ == 0) return true;
+    const auto t0 = std::chrono::steady_clock::now();
+    {
+        On on(device_);
+        // spin on the stream (a blocking sync would sleep the pool thread and wake it late)
+        cudaError_t e;
+        while ((e = cudaStreamQuery(stream_)) == cudaErrorNotReady) {
+        }
+        if (!ck(e, "finish", err)) return false;
+    }
+    ms_wait += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
+    if (launched_direct_) { launched_rows_ = 0; return true; }
+    for (size_t r = 0; r < row_of_.size(); ++r)
+        std::memcpy(out + (size_t) row_of_[r] * H, h_out_ + r * H, (size_t) H * sizeof(float));
+    launched_rows_ = 0;
+    return true;
+}
+
+bool PeerExperts::adapt(const float* usage, const int32_t* res0, int max_swaps, std::string& err) {
+    if (!pending_.empty() || max_swaps <= 0) return true;
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    struct Swap { float gain; int32_t layer, in, out; };
+    std::vector<Swap> swaps;
+    std::vector<std::pair<float, int32_t>> cand, vict;
+    for (int64_t l = 0; l < n_layers_; ++l) {
+        cand.clear();
+        vict.clear();
+        const float* u = usage + l * n_expert_;
+        const int32_t* r0 = res0 + l * n_expert_;
+        const int32_t* r1 = res_.data() + l * n_expert_;
+        for (int32_t e = 0; e < (int32_t) n_expert_; ++e) {
+            if (r1[e] >= 0) vict.emplace_back(u[e], e);
+            else if (r0[e] < 0 && u[e] >= 2.0f) cand.emplace_back(u[e], e);
+        }
+        if (cand.empty() || vict.empty()) continue;
+        std::sort(cand.begin(), cand.end(), [](auto& a, auto& b) { return a.first > b.first; });
+        const size_t nc = std::min(cand.size(), vict.size());
+        std::partial_sort(vict.begin(), vict.begin() + (ptrdiff_t) nc, vict.end(),
+                          [](auto& a, auto& b) { return a.first < b.first; });
+        for (size_t i = 0; i < nc; ++i) {
+            if (cand[i].first < vict[i].first + 1.5f) break;
+            swaps.push_back({cand[i].first - vict[i].first, (int32_t) l, cand[i].second, vict[i].second});
+        }
+    }
+    if (swaps.empty()) return true;
+    std::sort(swaps.begin(), swaps.end(), [](const Swap& a, const Swap& b) { return a.gain > b.gain; });
+    if ((int) swaps.size() > max_swaps) swaps.resize((size_t) max_swaps);
+    On on(device_);
+    for (const Swap& s : swaps) {
+        const size_t in = (size_t) (s.layer * n_expert_ + s.in), out = (size_t) (s.layer * n_expert_ + s.out);
+        const int32_t slot = res_[out];
+        const uint8_t* b = src_->blob(s.layer, s.in);
+        if (slot < 0 || b == nullptr) continue;
+        if (!ck(cudaMemcpyAsync(cache_.device_slot(slot), b, (size_t) lay.blob_bytes(s.layer), cudaMemcpyHostToDevice,
+                                refill_), "refill", err))
+            return false;
+        res_[out] = kNotResident;              // evicted now: the CPU computes it meanwhile
+        pending_.emplace_back((int32_t) in, slot);
+        ++swaps_;
+    }
+    return ck(cudaEventRecord(refill_ev_, refill_), "refill event", err);
+}
+
+void PeerExperts::apply_pending(bool wait) {
+    if (pending_.empty()) return;
+    {
+        On on(device_);
+        if (wait) cudaEventSynchronize(refill_ev_);
+        else if (cudaEventQuery(refill_ev_) != cudaSuccess) return;
+    }
+    for (const auto& [i, slot] : pending_) res_[(size_t) i] = slot;
+    pending_.clear();
+}
+
+}  // namespace strata::core

src/core/pinned.cu

@@ -199,15 +199,32 @@ void* reserve(uint64_t bytes, PageBacking& got, std::string& note, const std::st
         return (uint8_t*) map + kSharedArenaHeaderBytes;
     }
 
-    void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE,
-                   MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, -1, 0);
-    if (p != MAP_FAILED) {
-        got = PageBacking::LargePages;
-        note = "hugetlb 2 MB pages";
-        return p;
+    // STRATA_NO_LARGEPAGES=1 is the same-run A/B switch the Windows branch documents; honor it
+    // here too, so the large-page path can be compared without changing the pool or rebooting.
+    if (std::getenv("STRATA_NO_LARGEPAGES") != nullptr) {
+        note = "large pages skipped (STRATA_NO_LARGEPAGES); using 4 KB pages";
+    } else {
+        void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE,
+                       MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, -1, 0);
+        if (p != MAP_FAILED) {
+            got = PageBacking::LargePages;
+            note = "hugetlb 2 MB pages";
+            return p;
+        }
+        // MAP_HUGETLB is all-or-nothing: a pool smaller than the mapping fails exactly like an absent
+        // one, and the old message guessed "no hugetlb pool configured" either way. Name the shortfall:
+        // how many 2 MiB pages the mapping needs against what vm.nr_hugepages actually holds.
+        const unsigned long long need = ((unsigned long long) bytes + (1ull << 21) - 1) / (1ull << 21);
+        unsigned long long pool = 0;
+        bool have_pool = false;
+        if (std::FILE* f = std::fopen("/proc/sys/vm/nr_hugepages", "r")) {
+            have_pool = std::fscanf(f, "%llu", &pool) == 1;
+            std::fclose(f);
+        }
+        note = "MAP_HUGETLB unavailable (needed " + std::to_string(need) + " 2 MiB pages, vm.nr_hugepages=" +
+               (have_pool ? std::to_string(pool) : std::string("?")) + "); using 4 KB pages";
     }
-    note = "MAP_HUGETLB unavailable (no hugetlb pool configured?); using 4 KB pages";
-    p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
     got = PageBacking::NormalPages;
     return p == MAP_FAILED ? nullptr : p;
 #endif
@@ -413,6 +430,141 @@ LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_byte
     return load_experts_ranges(path, dst, off, n, threads, chunk);
 }
 
+LoadStats load_experts_direct(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
+                              const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk) {
+    LoadStats st;
+    st.ok = false;
+#ifdef _WIN32
+    constexpr uint64_t kAlign = 4096;
+    if (((uintptr_t) dst % kAlign) != 0 || chunk == 0 || chunk % kAlign != 0) return st;
+    struct Piece { uint64_t off, n; };
+    std::vector<Piece> pieces;
+    uint64_t bytes = 0;
+    for (size_t L = 0; L < layer_off.size(); ++L) {
+        if (layer_off[L] % kAlign != 0 || layer_bytes[L] % kAlign != 0) return st;
+        for (uint64_t p = 0; p < layer_bytes[L]; p += chunk)
+            pieces.push_back({layer_off[L] + p, std::min<uint64_t>(chunk, layer_bytes[L] - p)});
+        bytes += layer_bytes[L];
+    }
+    if (threads < 1) threads = 1;
+    const auto t0 = std::chrono::steady_clock::now();
+    std::atomic<size_t> next{0};
+    std::mutex err_mu;
+    std::string err;
+    const int wide = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, nullptr, 0);
+    std::vector<wchar_t> wpath((size_t) std::max(wide, 1), L'\0');
+    if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, wpath.data(), wide);
+    auto worker = [&]() {
+        HANDLE h = CreateFileW(wpath.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                               FILE_FLAG_NO_BUFFERING | FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
+        if (h == INVALID_HANDLE_VALUE) {
+            std::lock_guard<std::mutex> g(err_mu);
+            if (err.empty())
+                err = "cannot open " + path + " unbuffered (error " + std::to_string((unsigned long long) GetLastError()) + ")";
+            next = pieces.size();
+            return;
+        }
+        for (;;) {
+            const size_t i = next.fetch_add(1);
+            if (i >= pieces.size()) break;
+            OVERLAPPED ov{};
+            ov.Offset = (DWORD) pieces[i].off;
+            ov.OffsetHigh = (DWORD) (pieces[i].off >> 32);
+            DWORD got = 0;
+            if (!ReadFile(h, dst + pieces[i].off, (DWORD) pieces[i].n, &got, &ov) || got != pieces[i].n) {
+                std::lock_guard<std::mutex> g(err_mu);
+                if (err.empty())
+                    err = "short unbuffered read at offset " + std::to_string(pieces[i].off) + ": got " + std::to_string(got) +
+                          " of " + std::to_string(pieces[i].n) + " B (error " +
+                          std::to_string((unsigned long long) GetLastError()) + ")";
+                next = pieces.size();
+                break;
+            }
+        }
+        CloseHandle(h);
+    };
+    std::vector<std::thread> pool;
+    for (int i = 1; i < threads; ++i) pool.emplace_back(worker);
+    worker();
+    for (auto& t : pool) t.join();
+    st.layers = layer_off.size();
+    if (!err.empty()) {
+        st.seconds = -1.0;
+        st.error = err;
+        return st;
+    }
+    st.ok = true;
+    st.bytes = bytes;
+    st.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+#else
+    (void) path; (void) dst; (void) layer_off; (void) layer_bytes; (void) threads; (void) chunk;
+#endif
+    return st;
+}
+
+bool experts_unbuffered(const std::vector<std::string>& files, uint64_t arena_bytes, std::string& why,
+                        bool cache_counts) {
+    const char* env = std::getenv("STRATA_UNBUFFERED_LOAD");
+    if (env != nullptr && env[0] != '\0') {
+        why = std::string("STRATA_UNBUFFERED_LOAD=") + env;
+        return env[0] != '0';
+    }
+#ifdef _WIN32
+    LARGE_INTEGER freq{}, a{}, b{};
+    QueryPerformanceFrequency(&freq);
+    constexpr DWORD kRead = 64 << 10;
+    constexpr int kSamples = 16;
+    std::vector<uint8_t> buf(kRead);
+    uint64_t total_bytes = 0;
+    int fast = 0, n = 0;
+    uint64_t seed = (uint64_t) GetTickCount64() * 6364136223846793005ull + 1442695040888963407ull;
+    for (const std::string& f : files) {
+        const int wide = MultiByteToWideChar(CP_UTF8, 0, f.c_str(), -1, nullptr, 0);
+        std::vector<wchar_t> w((size_t) std::max(wide, 1), L'\0');
+        if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, f.c_str(), -1, w.data(), wide);
+        HANDLE h = CreateFileW(w.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS,
+                               nullptr);
+        if (h == INVALID_HANDLE_VALUE) continue;
+        LARGE_INTEGER size{};
+        GetFileSizeEx(h, &size);
+        total_bytes += (uint64_t) size.QuadPart;
+        // random offsets: a probe must not find the blocks an earlier probe put into the cache
+        for (int i = 0; i < kSamples && (uint64_t) size.QuadPart > 2ull * kRead; ++i) {
+            seed = seed * 6364136223846793005ull + 1442695040888963407ull;
+            const uint64_t off = ((seed >> 17) % ((uint64_t) size.QuadPart - kRead)) / kRead * kRead;
+            OVERLAPPED ov{};
+            ov.Offset = (DWORD) off;
+            ov.OffsetHigh = (DWORD) (off >> 32);
+            DWORD got = 0;
+            QueryPerformanceCounter(&a);
+            const BOOL ok = ReadFile(h, buf.data(), kRead, &got, &ov);
+            QueryPerformanceCounter(&b);
+            if (!ok) continue;
+            ++n;
+            fast += (double) (b.QuadPart - a.QuadPart) * 1e6 / (double) freq.QuadPart < 30.0;
+        }
+        CloseHandle(h);
+    }
+    const bool cached = n > 0 && fast * 4 >= n * 3;
+    MEMORYSTATUSEX ms{};
+    ms.dwLength = sizeof ms;
+    GlobalMemoryStatusEx(&ms);
+    const uint64_t avail = ms.ullAvailPhys;
+    // what the cache could keep beside the arena (~4 GiB for everything else)
+    const uint64_t room = avail > arena_bytes + (4ull << 30) ? avail - arena_bytes - (4ull << 30) : 0;
+    const bool keepable = room >= total_bytes;
+    char msg[200];
+    std::snprintf(msg, sizeof msg, "%d of %d probe reads from the file cache; %.1f GiB available, %.1f GiB of files",
+                  fast, n, (double) avail / (1ull << 30), (double) total_bytes / (1ull << 30));
+    why = msg;
+    return (!cached || !cache_counts) && !keepable;
+#else
+    (void) files; (void) arena_bytes; (void) cache_counts;
+    why = "buffered (not Windows)";
+    return false;
+#endif
+}
+
 LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
                               const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk) {
     LoadStats st;

src/core/verify.cpp

@@ -6,6 +6,7 @@
 
 #include "strata/core/native_head.hpp"
 #include "strata/core/on_device.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/kernels/iq_kernels.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
@@ -69,7 +70,7 @@ struct Bump {
 };
 
 bool mapped(size_t bytes, void** h, void** d) {
-    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped) != cudaSuccess) return false;
+    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false;   // multi-GPU: portable only with a peer - the peer card writes its rows into them
     std::memset(*h, 0, bytes);
     return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;
 }
@@ -210,7 +211,9 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
     }
     if (hits.d_res == nullptr || hits.cache_base == nullptr || hits.blob <= 0) {
         err = "verify: needs the profile-filled VRAM expert tier (--expert-profile and --expert-cache); with "
-              "--expert-cache auto, no VRAM was left for it: lower --max-context, use --kv k8v4 or images on the CPU";
+              "--expert-cache auto, no VRAM was left for it - the 'no VRAM is left for the expert cache' line above "
+              "says how much is short and what makes room (a smaller --max-context, --kv q4_0, setup --draft-vocab en, "
+              "images on the CPU)";
         return false;
     }
     std::string why;
@@ -745,20 +748,21 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
         const int32_t* p_start2 = pl + ptr_off + 4 * capx;
         float* hit_out = hit_out_ + (size_t) tb * K * N;
         const auto& lay = strata::kernels::cpu::expert_layout();
-        // plan v0.3 P6: the VRAM groups now; the PCIe groups once the copy engine has landed them in staging
-        auto grouped = [&](const unsigned long long* gp, const int32_t* gs, const int32_t* gn) {
+        // plan v0.3 P6: the VRAM groups now; the PCIe groups once the copy engine has landed them in staging.
+        // `gy`: the native launch's groups side by side (0: cap, one block row per possible group).
+        auto grouped = [&](const unsigned long long* gp, const int32_t* gs, const int32_t* gn, int64_t gy) {
             if (lay.native) {
                 // the layer's GGUF formats (i-quant gate/up, Q2_0 / IQ4_NL down)
                 const auto& f = lay.fmt[(size_t) l];
                 const NativeExpertLayout L = native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);
                 native_expert_grouped(L, gp, gs, gn, p_dst, p_tok, cap, cap,
-                                      nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, hit_out, cs);
+                                      nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, hit_out, cs, gy);
             } else {
                 moe_grouped_s2(gp, gs, gn, p_dst, p_tok, cap, cap, hit_xq_ + (size_t) tb * (N / 32) * 34,
                                hit_xs_ + (size_t) tb * (N / 32), hit_scratch_, hit_out, cs);
             }
         };
-        grouped(p_ptr, p_start, p_counts);
+        grouped(p_ptr, p_start, p_counts, 0);
         stamp(l, 20, grp);
         if (device_plan_) wait_flag_ge_or(m_flagB_, ring, skip_ + grp, cs);
         else wait_flag_ge(m_flagB_, ring, cs);                 // the PCIe share is in staging (DMA) or mapped
@@ -769,7 +773,9 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             rebase_ptrs((unsigned long long*) p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), cs);
         }
         stamp(l, 21, grp);
-        grouped(p_ptr2, p_start2, p_counts + 2);
+        // the PCIe share is pcie_frac of the misses: a few groups when the cache is cold, usually none (always none at
+        // pcie_frac 0), so its launch is kPcieGroupRows block rows striding over the groups, not cap of them
+        grouped(p_ptr2, p_start2, p_counts + 2, kPcieGroupRows);
         stamp(l, 22, grp);
         if (device_plan_) {   // no CPU share when the device planned the group: its rows are zeros
             wait_flag_ge_or(m_flag_, ring, skip_ + grp, cs);

src/kernels/cpu/iq_avx2.cpp

@@ -9,7 +9,7 @@
 // with one, `vpsignb`.  The arithmetic is ggml's (ggml-cpu/quants.c, the `_generic` references) - only the
 // order of the float additions differs.
 //
-// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22).  IQ1_M stays on ggml-cpu.
+// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22), IQ4_XS (23).  IQ1_M stays on ggml-cpu.
 #include "strata/kernels/cpu/iq_avx2.hpp"
 
 #define GGML_COMMON_DECL_CPP
@@ -177,6 +177,28 @@ template <> struct Fmt32<21> {   // IQ3_S: d, qs[64], qh[8], signs[32], scales[4
     }
 };
 
+template <> struct Fmt32<23> {   // IQ4_XS: d, scales_h, scales_l[4], qs[128] - 136 B, 8 signed sub-scales
+    // ggml's ggml_vec_dot_iq4_xs_q8_K: the same 16-value codebook as IQ4_NL, but each of the eight 32-value
+    // sub-blocks carries its own 6-bit scale, read as two nibbles of scales_l[p] plus two bits of scales_h, and
+    // used SIGNED as (ls - 32).  The scale therefore folds into the int16 operand of madd_epi16 instead of
+    // becoming a float multiply per sub-block, and the codebook sign is carried the way iq4nl_rows does it.
+    static constexpr int bytes = 136;
+    static constexpr float K = 1.0f;
+    static inline void decode(const uint8_t* b, int j, int half, __m256i& g, __m256i& sgn, __m256i& sc) {
+        const int H = 2 * j + half;   // the eight 32-value sub-blocks, in the order the activation bytes come
+        const __m128i values = _mm_loadu_si128((const __m128i*) kvalues_iq4nl);
+        const __m128i bits = _mm_loadu_si128((const __m128i*) (b + 8 + 16 * H));
+        const __m128i m4 = _mm_set1_epi8(0x0f);
+        const __m256i q4 = _mm256_inserti128_si256(
+            _mm256_castsi128_si256(_mm_shuffle_epi8(values, _mm_and_si128(bits, m4))),
+            _mm_shuffle_epi8(values, _mm_and_si128(_mm_srli_epi16(bits, 4), m4)), 1);
+        g   = _mm256_sign_epi8(q4, q4);                       // |w|, the unsigned operand of maddubs
+        sgn = _mm256_sign_epi8(_mm256_set1_epi8(1), q4);      // w's sign, applied to the activation
+        const int ls = ((b[4 + (H >> 1)] >> (4 * (H & 1))) & 0xf) | (((u16(b + 2) >> (2 * H)) & 3) << 4);
+        sc = _mm256_set1_epi16((short) (ls - 32));
+    }
+};
+
 template <int TY, int NT>
 inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res) {
     __m256 accf[NT];
@@ -357,7 +379,7 @@ void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void*
 }  // namespace
 
 bool iq256_supported(int type) noexcept {
-    return type == 16 || type == 17 || type == 18 || type == 21 || type == 22;
+    return type == 16 || type == 17 || type == 18 || type == 21 || type == 22 || type == 23;
 }
 
 void iq256_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt,
@@ -368,6 +390,7 @@ void iq256_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off,
         case 18: gu_rows_nt<18>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         case 21: gu_rows_nt<21>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         case 22: gu_rows_nt<22>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 23: gu_rows_nt<23>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         default: break;
     }
 }
@@ -380,6 +403,7 @@ void iq256_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void*
         case 18: dot_rows_nt<18>(nt, w, row_bytes, n, act, out, r0, r1); break;
         case 21: dot_rows_nt<21>(nt, w, row_bytes, n, act, out, r0, r1); break;
         case 22: dot_rows_nt<22>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 23: dot_rows_nt<23>(nt, w, row_bytes, n, act, out, r0, r1); break;
         default: break;
     }
 }

src/kernels/cpu/native_expert.cpp

@@ -98,12 +98,16 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*
         kq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
         return;
     }
-    if (nt >= mt_min && iq512_supported(f.gu_type)) {
-        if (avx512) {
+    // A format with only an AVX-2 kernel (IQ4_XS, #415) takes it on AVX-2 CPUs only: an AVX-512 CPU keeps ggml-cpu for
+    // it, as before (its rows would round differently).  Each kernel only for the formats it implements: falling
+    // through an empty switch would leave ff unwritten instead of falling back to ggml-cpu.
+    static const bool cpu512 = cpu_avx512_ok();
+    if (nt >= mt_min && (iq512_supported(f.gu_type) || (!cpu512 && iq256_supported(f.gu_type)))) {
+        if (avx512 && iq512_supported(f.gu_type)) {
             iq512_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
             return;
         }
-        if (avx2) {
+        if (avx2 && iq256_supported(f.gu_type)) {
             iq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
             return;
         }

src/kernels/cuda/elementwise.cu

@@ -251,6 +251,23 @@ __global__ void copy_rows_from_mapped_kernel(float4* __restrict__ dst, const vol
         for (int64_t i = threadIdx.x; i < row4; i += blockDim.x) d[i] = const_cast<const float4*>(sr)[i];
     }
 }
+namespace {
+__global__ void scatter_rows_kernel(const float4* __restrict__ src, float4* dst, const int32_t* __restrict__ rows,
+                                    int64_t w4) {
+    const int64_t r = blockIdx.x;
+    const float4* s = src + r * w4;
+    float4* d = dst + (int64_t) rows[r] * w4;
+    for (int64_t i = threadIdx.x; i < w4; i += blockDim.x) d[i] = s[i];
+}
+}  // namespace
+void scatter_rows_f32(const float* src, float* dst, const int32_t* rows, int64_t n, int64_t width, void* stream) {
+    if (n <= 0) return;
+    if ((width & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {
+        std::fprintf(stderr, "scatter_rows_f32: width must be a multiple of 4 and both pointers 16-byte aligned\n");
+        std::exit(1);
+    }
+    scatter_rows_kernel<<<(unsigned) n, 128, 0, (cudaStream_t) stream>>>((const float4*) src, (float4*) dst, rows, width / 4);
+}
 void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
                            const int32_t* count, void* stream) {
     if (rows <= 0) return;

src/kernels/cuda/fused_gr.cu

@@ -196,7 +196,7 @@ __global__ void __launch_bounds__(THREADS) gr_norm_multi_kernel(GrMulti m) {
 // which fits all eight tokens in one 40 KiB launch and stages 160 chunks of 8 (5 per lane, half the registers held
 // for the weight prefetch); smaller tiles raise how many blocks share an SM (the 41-block grid), e.g. three blocks
 // of four tokens instead of one on sm_75.
-template <int TILEV>
+template <int TILEV, int MAX_T = kFusedGrMaxT>
 __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) {
     constexpr int TQ = TILEV / 8 / 32;      // uint4 weight chunks per lane per tile
     extern __shared__ __align__(16) float tile[];   // [T][TILEV]
@@ -207,9 +207,9 @@ __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) {
     const bool active = !(inject_block && (m.a[0].w_inject == nullptr || warp >= HC));
     const uint16_t* wrow = (inject_block ? m.a[0].w_inject : m.a[0].w_down) + (size_t) (active ? row : 0) * D;
     const uint4* w4 = reinterpret_cast<const uint4*>(wrow);
-    float acc[kFusedGrMaxT];
+    float acc[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) acc[k] = 0.0f;
+    for (int k = 0; k < MAX_T; ++k) acc[k] = 0.0f;
     for (int base = 0; base < D; base += TILEV) {
         uint4 wv[TQ];
         if (active) {
@@ -229,17 +229,17 @@ __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) {
         for (int q = 0; q < TQ; ++q) {
             const int j = lane + 32 * q;
 #pragma unroll
-            for (int k = 0; k < kFusedGrMaxT; ++k)
+            for (int k = 0; k < MAX_T; ++k)
                 if (k < T) acc[k] += dot8(wv[q], tile + k * TILEV + j * 8);
         }
     }
     if (!active) return;
-    float s[kFusedGrMaxT];
+    float s[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
+    for (int k = 0; k < MAX_T; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
     // lane k writes token k (every lane holds every sum after the xor reduction)
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) {
+    for (int k = 0; k < MAX_T; ++k) {
         if (k >= T || lane != k) continue;
         if (inject_block) {
             m.a[k].inject_out[row] = s[k];
@@ -616,6 +616,7 @@ __device__ __forceinline__ void stage_htile(const GrMulti& m, int T, int h, floa
     }
 }
 
+template <int MAX_T = kFusedGrMaxT>
 __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
     extern __shared__ __align__(16) float4 hbuf[];      // 2 buffers x [T][2][160] float4
     const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
@@ -626,9 +627,9 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
     const uint16_t* wrow = (inject_block ? m.a[0].w_inject : m.a[0].w_down) + (size_t) (active ? row : 0) * D;
     const uint4* w4 = reinterpret_cast<const uint4*>(wrow);
     const size_t buf_f4 = (size_t) T * (H_TILE / 4);    // float4 per buffer (the second one follows the first)
-    float acc[kFusedGrMaxT];
+    float acc[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) acc[k] = 0.0f;
+    for (int k = 0; k < MAX_T; ++k) acc[k] = 0.0f;
     uint4 wv[HQ], wnext[HQ];
     if (active) {
 #pragma unroll
@@ -653,7 +654,7 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
             for (int q = 0; q < HQ; ++q) {
                 const int j = lane + 32 * q;
 #pragma unroll
-                for (int k = 0; k < kFusedGrMaxT; ++k) {
+                for (int k = 0; k < MAX_T; ++k) {
                     if (k < T) {
                         const float4* pk = cur + (size_t) k * (H_TILE / 4);
                         acc[k] += dot8v(wv[q], pk[j], pk[H_TILE / 8 + j]);
@@ -670,12 +671,12 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
         }
     }
     if (!active) return;
-    float s[kFusedGrMaxT];
+    float s[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
+    for (int k = 0; k < MAX_T; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
     // lane k writes token k (every lane holds every sum after the xor reduction)
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) {
+    for (int k = 0; k < MAX_T; ++k) {
         if (k >= T || lane != k) continue;
         if (inject_block) {
             m.a[k].inject_out[row] = s[k];
@@ -729,7 +730,7 @@ int down_chunk(bool staged, int* tile_out) {
         }
         int want_staged = (int) (2 * kFusedGrMaxT * H_TILE * sizeof(float));
         if (optin > 0 && want_staged > optin) want_staged = optin;
-        cudaFuncSetAttribute(gr_down_staged_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, want_staged);
+        cudaFuncSetAttribute(gr_down_staged_kernel<>, cudaFuncAttributeMaxDynamicSharedMemorySize, want_staged);
         cudaGetLastError();      // drop any error the attempt left behind
         // The opt-in is a promise a pre-Volta card does not keep: an sm_60 answers 65536 and accepts the
         // cudaFuncSetAttribute for 61440 B, then fails the LAUNCH with "invalid argument".  What such a card
@@ -778,9 +779,21 @@ void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long
             for (int k = 0; k < ct; ++k) c.a[k] = m.a[c0 + k];
         }
         const size_t smem = (size_t) ct * per_tok;
-        if (staged) gr_down_staged_kernel<<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
-        else if (tv == 1280) gr_down_multi_kernel<1280><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
-        else gr_down_multi_kernel<2560><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        // #443, opt-in STRATA_GR_DOWN_MAX4=1: launches of up to 4 tokens hold 4 tokens' sums per thread instead of
+        // kFusedGrMaxT - the same tile, block size, accumulation order and plain/split/staged path, so the same bits
+        // (read once: this runs per layer when decode is not captured)
+        static const bool max4_on = [] { const char* v = std::getenv("STRATA_GR_DOWN_MAX4"); return v && std::atoi(v) != 0; }();
+        const bool max4 = max4_on && ct <= 4;
+        if (staged) {
+            if (max4) gr_down_staged_kernel<4><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+            else gr_down_staged_kernel<><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        } else if (tv == 1280) {
+            if (max4) gr_down_multi_kernel<1280, 4><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+            else gr_down_multi_kernel<1280><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        } else {
+            if (max4) gr_down_multi_kernel<2560, 4><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+            else gr_down_multi_kernel<2560><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        }
     }
     if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0 + 1, (void*) st);
     gr_up_multi_kernel<<<UPM_BLOCKS, THREADS, 0, st>>>(m);

src/kernels/cuda/iq_kernels.cu

@@ -295,7 +295,7 @@ __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(const void* __restrict__ vb
 // Q4_K / Q5_K gate/up and Q5_1 / Q8_0 down: llama.cpp's vec_dot_*_q8_1 (vecdotq.cuh, VDR 2 each), transcribed; the
 // Q5_1 min term is the one departure (below).  Via eddoursul/Strata 8029fa9 (iq_dot.cuh) and #255 (Q8_0,
 // gopinath87607), which agree with llama.cpp and with each other.
-constexpr int VDR_Q4_K = 2, VDR_Q5_K = 2, VDR_Q5_1 = 2, VDR_Q8_0 = 2;
+constexpr int VDR_Q4_K = 2, VDR_Q5_K = 2, VDR_Q5_1 = 2, VDR_Q5_0 = 2, VDR_Q8_0 = 2;
 
 __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(const int* __restrict__ v, const int* __restrict__ u,
                                                              const uint8_t* __restrict__ sc, const uint8_t* __restrict__ m,
@@ -405,6 +405,36 @@ __device__ __forceinline__ float vec_dot_q5_K_q8_1(const void* __restrict__ vbq,
 // of the ORIGINAL activations.  That is ggml-cpu's convention (its q8_1 `s` is d * sum(q)) and the one the K-quant
 // mins above use; the scaled and the min term then see the same activation (eddoursul/Strata measured 1.1-1.2%
 // against 1.9% relative error per expert).  Result: sumi * (d5 * d8) + sumu * (m5 * d8).
+// Q5_0: llama.cpp's integer chain verbatim (vecdotq.cuh vec_dot_q5_0_q8_1_impl) -- symmetric (one delta, no
+// learned min), so unlike Q5_1 there is no min-term rounding choice to make: the constant -16 per-weight offset
+// is accounted for via the q8_1 block's `ds.y` (sum of the ORIGINAL activations), exactly as upstream.
+__device__ __forceinline__ float vec_dot_q5_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,
+                                                   const int& kbx, const int& iqs) {
+    const block_q5_0* bq5_0 = (const block_q5_0*) vbq + kbx;
+    int sumi = 0;
+#pragma unroll
+    for (int i = 0; i < VDR_Q5_0; ++i) {
+        const int vl = get_int_b2(bq5_0->qs, iqs + i);
+        const int vh = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i));
+        const int u0 = get_int_b4(bq8_1->qs, iqs + i), u1 = get_int_b4(bq8_1->qs, iqs + i + QI5_0);
+        int vi0 = (vl >> 0) & 0x0F0F0F0F;
+        vi0 |= (vh << 4) & 0x00000010;
+        vi0 |= (vh << 11) & 0x00001000;
+        vi0 |= (vh << 18) & 0x00100000;
+        vi0 |= (vh << 25) & 0x10000000;
+        sumi = ggml_cuda_dp4a(vi0, u0, sumi);
+        int vi1 = (vl >> 4) & 0x0F0F0F0F;
+        vi1 |= (vh >> 12) & 0x00000010;
+        vi1 |= (vh >> 5) & 0x00001000;
+        vi1 |= (vh << 2) & 0x00100000;
+        vi1 |= (vh << 9) & 0x10000000;
+        sumi = ggml_cuda_dp4a(vi1, u1, sumi);
+    }
+    const float d5 = __half2float(bq5_0->d);
+    const float d8 = __low2float(bq8_1->ds);
+    const float s8 = __high2float(bq8_1->ds);
+    return d5 * (sumi * d8 - (16.0f * VDR_Q5_0 / QI5_0) * s8);
+}
 __device__ __forceinline__ float vec_dot_q5_1_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,
                                                    const int& kbx, const int& iqs) {
     const block_q5_1* bq5_1 = (const block_q5_1*) vbq + kbx;
@@ -470,14 +500,16 @@ template<> struct Fmt<13> { static constexpr int qk = 256, ipb = QI5_K / VDR_Q5_
     __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_K_q8_1(v, y, kbx, iqs); } };
 template<> struct Fmt<7> { static constexpr int qk = 32, ipb = QI5_1 / VDR_Q5_1, step = VDR_Q5_1;
     __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_1_q8_1(v, y, kbx, iqs); } };
+template<> struct Fmt<6> { static constexpr int qk = 32, ipb = QI5_0 / VDR_Q5_0, step = VDR_Q5_0;
+    __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_0_q8_1(v, y, kbx, iqs); } };
 template<> struct Fmt<8> { static constexpr int qk = 32, ipb = QI8_0 / VDR_Q8_0, step = VDR_Q8_0;
     __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q8_0_q8_1(v, y, kbx, iqs); } };
 
 // The formats of each role, one list each so a type cannot be in one switch and missing from another.  Every
 // entry is a kernel template for each CUDA architecture of the build, hence two lists rather than one.
-#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(8)
-#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(8)
-#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(8)
+#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(6) X(8)
+#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(8)
+#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(6) X(8)
 
 __device__ __forceinline__ float warp_sum(float v) {
 #pragma unroll
@@ -854,6 +886,12 @@ __global__ void __launch_bounds__(128) mmvq_multi_kernel(const uint8_t* __restri
 // ---------------------------------------------------------------- grouped native experts
 constexpr int GU_ROWS = 8;     // rows per block (one warp each)
 
+// Group g is computed by block row g % gridDim.y: rows y, y + gridDim.y, ... < *n_groups.  The callers size the
+// launch for the most groups a call can have (cap: every entry its own expert), but the count is only known on the
+// device, and a verify window's PCIe call usually has no group at all: with one block row per possible group every
+// unused one was cap x (2 n_ff / 8 + n_embd / 8) blocks that started only to return.  A smaller gridDim.y strides
+// instead.  Each group's rows are computed by the same warp code in the same order either way, so the results do
+// not depend on gridDim.y (gridDim.y = cap_groups is the one-row-per-group launch).
 template<int TG>
 __global__ void __launch_bounds__(256) native_gu_kernel(const unsigned long long* __restrict__ grp_ptr,
                                                         const int32_t* __restrict__ grp_start,
@@ -861,20 +899,21 @@ __global__ void __launch_bounds__(256) native_gu_kernel(const unsigned long long
                                                         const int32_t* __restrict__ ent_tok,
                                                         const block_q8_1* __restrict__ xq, NativeExpertLayout L,
                                                         float* __restrict__ gate, float* __restrict__ up) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int row = blockIdx.x * GU_ROWS + warp;             // 0 .. 2*n_ff
     if (row >= 2 * L.n_ff) return;
     const bool is_up = row >= L.n_ff;
     const int r = is_up ? row - (int) L.n_ff : row;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
+    const size_t off = (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
     const int nb = (int) (L.n_embd / Fmt<TG>::qk), xb = (int) (L.n_embd / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
-    for (int e = e0; e < e1; ++e) {
-        const float s = row_dot<TG>(wr, xq + (size_t) ent_tok[e] * xb, nb, lane);
-        if (lane == 0) (is_up ? up : gate)[(size_t) e * L.n_ff + r] = s;
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; ++e) {
+            const float s = row_dot<TG>(wr, xq + (size_t) ent_tok[e] * xb, nb, lane);
+            if (lane == 0) (is_up ? up : gate)[(size_t) e * L.n_ff + r] = s;
+        }
     }
 }
 
@@ -890,28 +929,29 @@ __global__ void __launch_bounds__(256) native_gu_multi_kernel(const unsigned lon
                                                               const int32_t* __restrict__ ent_tok,
                                                               const block_q8_1* __restrict__ xq, NativeExpertLayout L,
                                                               float* __restrict__ gate, float* __restrict__ up) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int row = blockIdx.x * GU_ROWS + warp;             // 0 .. 2*n_ff
     if (row >= 2 * L.n_ff) return;
     const bool is_up = row >= L.n_ff;
     const int r = is_up ? row - (int) L.n_ff : row;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
+    const size_t w_off = (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
     const int nb = (int) (L.n_embd / Fmt<TG>::qk), xb = (int) (L.n_embd / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
     float* dst = is_up ? up : gate;
-    for (int e = e0; e < e1; e += GRP_NC) {
-        const int n = min(GRP_NC, e1 - e);
-        int off[GRP_NC];
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {   // the group stride, as native_gu_kernel
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + w_off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; e += GRP_NC) {
+            const int n = min(GRP_NC, e1 - e);
+            int off[GRP_NC];
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c) off[c] = ent_tok[e + min(c, n - 1)] * xb;
-        float s[GRP_NC];
-        row_dot_multi<TG, GRP_NC>(wr, xq, off, n, nb, lane, s);
+            for (int c = 0; c < GRP_NC; ++c) off[c] = ent_tok[e + min(c, n - 1)] * xb;
+            float s[GRP_NC];
+            row_dot_multi<TG, GRP_NC>(wr, xq, off, n, nb, lane, s);
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c)
-            if (c < n && lane == c) dst[(size_t) (e + c) * L.n_ff + r] = s[c];
+            for (int c = 0; c < GRP_NC; ++c)
+                if (c < n && lane == c) dst[(size_t) (e + c) * L.n_ff + r] = s[c];
+        }
     }
 }
 
@@ -930,18 +970,19 @@ __global__ void __launch_bounds__(256) native_down_kernel(const unsigned long lo
                                                           const int32_t* __restrict__ ent_dst,
                                                           const block_q8_1* __restrict__ hq, NativeExpertLayout L,
                                                           float* __restrict__ out) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int r = blockIdx.x * 8 + warp;
     if (r >= L.n_embd) return;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + L.down_off + (size_t) r * L.d_row;
+    const size_t off = L.down_off + (size_t) r * L.d_row;
     const int nb = (int) (L.n_ff / Fmt<TD>::qk), hb = (int) (L.n_ff / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
-    for (int e = e0; e < e1; ++e) {
-        const float s = row_dot<TD>(wr, hq + (size_t) e * hb, nb, lane);
-        if (lane == 0) out[(size_t) ent_dst[e] * L.n_embd + r] = s;
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {   // as native_gu_kernel
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; ++e) {
+            const float s = row_dot<TD>(wr, hq + (size_t) e * hb, nb, lane);
+            if (lane == 0) out[(size_t) ent_dst[e] * L.n_embd + r] = s;
+        }
     }
 }
 
@@ -953,33 +994,32 @@ __global__ void __launch_bounds__(256) native_down_multi_kernel(const unsigned l
                                                                 const int32_t* __restrict__ ent_dst,
                                                                 const block_q8_1* __restrict__ hq, NativeExpertLayout L,
                                                                 float* __restrict__ out) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int r = blockIdx.x * 8 + warp;
     if (r >= L.n_embd) return;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + L.down_off + (size_t) r * L.d_row;
+    const size_t w_off = L.down_off + (size_t) r * L.d_row;
     const int nb = (int) (L.n_ff / Fmt<TD>::qk), hb = (int) (L.n_ff / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
-    for (int e = e0; e < e1; e += GRP_NC) {
-        const int n = min(GRP_NC, e1 - e);
-        int off[GRP_NC];
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {   // the group stride, as native_gu_kernel
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + w_off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; e += GRP_NC) {
+            const int n = min(GRP_NC, e1 - e);
+            int off[GRP_NC];
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c) off[c] = (e + min(c, n - 1)) * hb;
-        float s[GRP_NC];
-        row_dot_multi<TD, GRP_NC>(wr, hq, off, n, nb, lane, s);
+            for (int c = 0; c < GRP_NC; ++c) off[c] = (e + min(c, n - 1)) * hb;
+            float s[GRP_NC];
+            row_dot_multi<TD, GRP_NC>(wr, hq, off, n, nb, lane, s);
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c)
-            if (c < n && lane == c) out[(size_t) ent_dst[e + c] * L.n_embd + r] = s[c];
+            for (int c = 0; c < GRP_NC; ++c)
+                if (c < n && lane == c) out[(size_t) ent_dst[e + c] * L.n_embd + r] = s[c];
+        }
     }
 }
 
 // ---------------------------------------------------------------- q8_1 (quantize.cu)
-__global__ void quantize_q8_1_kernel(const float* __restrict__ x, block_q8_1* __restrict__ y, long long n) {
-    const long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x;
-    if (i >= n) return;
-    const float xi = x[i];
+// value i of a q8_1 row set (the warp holds block i / 32, lane = i % 32)
+__device__ __forceinline__ void q8_1_store(const float xi, block_q8_1* __restrict__ y, const long long i) {
     float amax = fabsf(xi), sum = xi;
 #pragma unroll
     for (int o = 16; o > 0; o >>= 1) {
@@ -993,6 +1033,40 @@ __global__ void quantize_q8_1_kernel(const float* __restrict__ x, block_q8_1* __
     if (iqs == 0) y[ib].ds = make_half2(d, sum);
 }
 
+__global__ void quantize_q8_1_kernel(const float* __restrict__ x, block_q8_1* __restrict__ y, long long n) {
+    const long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= n) return;
+    q8_1_store(x[i], y, i);
+}
+
+// swiglu_entries_kernel and quantize_q8_1_kernel in one pass, over the call's own entries only:
+// [grp_start[0], grp_start[*n_groups]) (a call's entries are contiguous; the verify window's PCIe call starts after
+// its VRAM call) instead of all cap_entries twice - nothing reads the others: the down kernel reads its groups'.
+// n_ff is a multiple of 32, so a warp is one q8_1 block and the bounds are warp-uniform.  The values are the two
+// kernels': the SwiGLU product is rounded on its own - it must not contract into the first add of q8_1_store's block
+// sum, as it could not when it went through memory (CUDA: __fmul_rn; HIP's __fmul_rn is a plain product, so there
+// the product is written here under contract(off)) - then q8_1_store unchanged: the same blocks, bit for bit.
+__global__ void __launch_bounds__(256) swiglu_q8_1_entries_kernel(const float* __restrict__ gate,
+                                                                  const float* __restrict__ up,
+                                                                  const int32_t* __restrict__ grp_start,
+                                                                  const int32_t* __restrict__ n_groups, int n_ff,
+                                                                  block_q8_1* __restrict__ hq) {
+#if defined(__HIPCC__)
+#pragma clang fp contract(off)
+#endif
+    const long long lo = (long long) grp_start[0] * n_ff, hi = (long long) grp_start[*n_groups] * n_ff;
+    for (long long i = lo + (long long) blockIdx.x * blockDim.x + threadIdx.x; i < hi;
+         i += (long long) gridDim.x * blockDim.x) {
+        const float g = gate[i];
+#if defined(__HIPCC__)
+        const float h = (g / (1.0f + __expf(-g))) * up[i];
+#else
+        const float h = __fmul_rn(g / (1.0f + __expf(-g)), up[i]);
+#endif
+        q8_1_store(h, hq, i);
+    }
+}
+
 // ---------------------------------------------------------------- dequant (dequantize.cuh)
 template<typename dst_t> __device__ __forceinline__ dst_t cvt(float v);
 template<> __device__ __forceinline__ float cvt<float>(float v) { return v; }
@@ -1194,6 +1268,23 @@ __device__ void dq_q5_k(const void* vx, int64_t ibs, dst_t* yy, int tid) {
     }
 }
 template<typename dst_t>
+__device__ void dq_q5_0(const void* vx, int64_t ibs, dst_t* yy, int tid) {
+    // symmetric: one delta (no min), the high bit folded in then centered by the format's fixed -16 offset
+    const block_q5_0* x = (const block_q5_0*) vx + ibs * (QK_K / QK5_0);
+    const int ib = tid % 8, il = tid / 8;
+    const float d = __half2float(x[ib].d);
+    uint32_t qh;
+    memcpy(&qh, x[ib].qh, sizeof(qh));
+    dst_t* y = yy + 32 * ib;
+    for (int j = 0; j < 4; ++j) {
+        const int iqs = 4 * il + j;
+        const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
+        const int xh_1 = ((qh >> (iqs + 12))) & 0x10;
+        y[iqs] = cvt<dst_t>(((float) ((x[ib].qs[iqs] & 0xf) | xh_0) - 16.0f) * d);
+        y[iqs + 16] = cvt<dst_t>(((float) ((x[ib].qs[iqs] >> 4) | xh_1) - 16.0f) * d);
+    }
+}
+template<typename dst_t>
 __device__ void dq_q5_1(const void* vx, int64_t ibs, dst_t* yy, int tid) {
     const block_q5_1* x = (const block_q5_1*) vx + ibs * (QK_K / QK5_1);
     const int ib = tid % 8, il = tid / 8;
@@ -1243,6 +1334,7 @@ __device__ __forceinline__ void dq_dispatch(int ty, const void* vx, int64_t ibs,
         case 12: dq_q4_k(vx, ibs, y, tid); break;
         case 13: dq_q5_k(vx, ibs, y, tid); break;
         case 7: dq_q5_1(vx, ibs, y, tid); break;
+        case 6: dq_q5_0(vx, ibs, y, tid); break;
         case 8: dq_q8_0(vx, ibs, y, tid); break;
         case 30: dq_bf16(vx, ibs, y, tid); break;
         default: break;
@@ -1267,7 +1359,7 @@ __global__ void dequant_gu_kernel(int ty, const void* __restrict__ gate, const v
 // the types dq_dispatch dequantizes
 bool is_iq(int t) {
     return t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||
-           t == 12 || t == 13 || t == 7 || t == 8;
+           t == 12 || t == 13 || t == 7 || t == 6 || t == 8;
 }
 // values per block of the types the grouped expert kernels take (0 = none)
 int gu_qk(int t) {
@@ -1347,6 +1439,7 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {
         case 12: return (size_t) (n / 256) * sizeof(block_q4_K);
         case 13: return (size_t) (n / 256) * sizeof(block_q5_K);
         case 7: return (size_t) (n / 32) * sizeof(block_q5_1);
+        case 6: return (size_t) (n / 32) * sizeof(block_q5_0);
         case 8: return (size_t) (n / 32) * sizeof(block_q8_0);
         case 30: return (size_t) n * 2;   // BF16: the token embedding only (iq_embed_rows, iq_dequant_f32)
         default: return 0;
@@ -1439,10 +1532,18 @@ size_t native_expert_scratch_bytes(int64_t cap, int64_t n_ff) {
     return 3 * ((f + 255) & ~(size_t) 255) + (((size_t) cap * (size_t) (n_ff / 32) * sizeof(block_q8_1) + 255) & ~(size_t) 255);
 }
 
+namespace {
+bool g_grouped_v1 = env_on("STRATA_GROUPED_V1");
+}  // namespace
+
+void native_grouped_set_v1(bool v1) { g_grouped_v1 = v1; }
+
 void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long* grp_ptr, const int32_t* grp_start,
                            const int32_t* n_groups, const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups,
-                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream) {
+                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream,
+                           int64_t grid_groups) {
     if (cap_groups <= 0 || cap_entries <= 0) return;
+    if (L.n_ff % 32 != 0) { std::fprintf(stderr, "native_expert_grouped: n_ff %lld\n", (long long) L.n_ff); std::exit(1); }
     cudaStream_t s = (cudaStream_t) stream;
     const size_t f = (size_t) cap_entries * (size_t) L.n_ff * sizeof(float), fa = (f + 255) & ~(size_t) 255;
     float* gate = (float*) scratch;
@@ -1450,7 +1551,9 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
     float* h = (float*) ((uint8_t*) scratch + 2 * fa);
     block_q8_1* hq = (block_q8_1*) ((uint8_t*) scratch + 3 * fa);
     const auto* X = (const block_q8_1*) x_q8_1;
-    const dim3 ggu((unsigned) ((2 * L.n_ff + GU_ROWS - 1) / GU_ROWS), (unsigned) cap_groups);
+    const bool v1 = g_grouped_v1;
+    const int64_t gy = (v1 || grid_groups <= 0 || grid_groups > cap_groups) ? cap_groups : grid_groups;
+    const dim3 ggu((unsigned) ((2 * L.n_ff + GU_ROWS - 1) / GU_ROWS), (unsigned) gy);
     switch (L.gu_type) {
 #define STRATA_GU(T) case T: launch_gu<T>(ggu, s, grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up); break;
         STRATA_GU_FMTS(STRATA_GU)
@@ -1459,9 +1562,15 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
     }
     check("native_expert_grouped/gu");
     const long long nh = (long long) cap_entries * L.n_ff;
-    swiglu_entries_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(gate, up, h, nh);
-    quantize_q8_1_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(h, hq, nh);
-    const dim3 gd((unsigned) ((L.n_embd + 7) / 8), (unsigned) cap_groups);
+    if (v1) {
+        swiglu_entries_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(gate, up, h, nh);
+        quantize_q8_1_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(h, hq, nh);
+    } else {
+        swiglu_q8_1_entries_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(gate, up, grp_start, n_groups,
+                                                                                (int) L.n_ff, hq);
+    }
+    check("native_expert_grouped/swiglu");
+    const dim3 gd((unsigned) ((L.n_embd + 7) / 8), (unsigned) gy);
     switch (L.d_type) {
 #define STRATA_DOWN(T) case T: launch_down<T>(gd, s, grp_ptr, grp_start, n_groups, ent_dst, hq, L, out); break;
         STRATA_D_FMTS(STRATA_DOWN)

src/kernels/cuda/qsa_prompt_attn.cu

@@ -78,18 +78,22 @@ __device__ __forceinline__ uint32_t pack_h2(float lo_k, float hi_k) {   // eleme
 // KV_MODE 1: int8 codes + fp16 scale per 64 values. KV_MODE 0: fp16 values (scales 1).
 // KV_MODE 3 (hybrid K8V4): K as mode 1, V as mode 0 - the row's q4_0 blocks are dequantized to fp16 at
 // gather, so everything downstream of the load is the mode-0 V path; the caller un-rotates the output.
+// KV_MODE 4 (Q4_0 K and V, `--kv q4_0`): mode 1 with a scale per 32 values - each q4_0 block's codes enter as exact
+// int8 (code - 8) and its fp16 scale is applied in FP32, as mode 1's are; the caller rotates q and un-rotates the
+// output (kv_q4.hpp).
 template <int KV_MODE>
 struct Smem {
     using KElem = typename std::conditional<KV_MODE == 0, __half, int8_t>::type;
-    using VElem = typename std::conditional<KV_MODE == 1, int8_t, __half>::type;
+    using VElem = typename std::conditional<KV_MODE == 1 || KV_MODE == 4, int8_t, __half>::type;
     static constexpr int KROW = KV_MODE == 0 ? HD + 8 : HD + 16;   // elements; 16-byte aligned rows, banks spread
-    static constexpr int VROW = KV_MODE == 1 ? HD + 16 : HD + 8;
+    static constexpr int VROW = (KV_MODE == 1 || KV_MODE == 4) ? HD + 16 : HD + 8;
+    static constexpr int NG = KV_MODE == 4 ? HD / QK4_0 : 4;      // scale groups per row (q4_0: 8 of 32, else 4 of 64)
     __half qh[16][QS];
     __half ql[16][QS];
     KElem k[CH][KROW];
     VElem v[CH][VROW];
-    float ks[CH][4];
-    float vs[CH][4];
+    float ks[CH][NG];
+    float vs[CH][NG];
     float s[16][CH + 1];
     float qmax[THREADS / 32];
     float alpha[16];
@@ -154,7 +158,39 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
         __syncthreads();   // rows ready; the previous chunk's p.v is done with k, v, s
         // gather the chunk's K and V rows (16-byte pieces; K8V4's V as q4_0 blocks dequantized to fp16)
         // and their scales
-        {
+        if constexpr (KV_MODE == 4) {
+            // q4_0 K and V: one (K or V, cell, block) per item - the block's 16 code bytes (2-byte aligned: eight
+            // 16-bit loads) become 32 exact int8 codes (element j in the low nibble of byte j, j + 16 in the high
+            // one), its fp16 scale goes to ks/vs
+            constexpr int BLKS = HD / QK4_0;
+            constexpr int BYTES = BLKS * (int) sizeof(block_q4_0);
+            for (int i = t; i < 2 * CH * BLKS; i += THREADS) {
+                const int kv = i / (CH * BLKS), rem = i % (CH * BLKS), c = rem / BLKS, b = rem % BLKS;
+                const long long r = S.row[c];
+                uint4 lo = make_uint4(0, 0, 0, 0), hi = make_uint4(0, 0, 0, 0);
+                float d = 0.0f;
+                if (r >= 0) {
+                    const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((kv == 0 ? p.k_q4 : p.v_q4) + r * BYTES) + b;
+                    d = __half2float(__ushort_as_half(__ldg(&blk->d)));
+                    const uint16_t* q16 = reinterpret_cast<const uint16_t*>(blk->qs);
+                    uint32_t w[4];
+#pragma unroll
+                    for (int j = 0; j < 4; ++j)
+                        w[j] = (uint32_t) __ldg(q16 + 2 * j) | ((uint32_t) __ldg(q16 + 2 * j + 1) << 16);
+                    // per byte: the low nibble minus 8 is element j, the high one minus 8 element j + 16
+                    lo = make_uint4(__vsub4(w[0] & 0x0F0F0F0Fu, 0x08080808u), __vsub4(w[1] & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4(w[2] & 0x0F0F0F0Fu, 0x08080808u), __vsub4(w[3] & 0x0F0F0F0Fu, 0x08080808u));
+                    hi = make_uint4(__vsub4((w[0] >> 4) & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4((w[1] >> 4) & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4((w[2] >> 4) & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4((w[3] >> 4) & 0x0F0F0F0Fu, 0x08080808u));
+                }
+                int8_t* dst = kv == 0 ? &S.k[c][b * QK4_0] : &S.v[c][b * QK4_0];
+                reinterpret_cast<uint4*>(dst)[0] = lo;
+                reinterpret_cast<uint4*>(dst)[1] = hi;
+                (kv == 0 ? S.ks : S.vs)[c][b] = d;
+            }
+        } else {
             constexpr int KPIECES = HD * (int) sizeof(typename Smem<KV_MODE>::KElem) / 16;   // per K row
             for (int i = t; i < CH * KPIECES; i += THREADS) {
                 const int c = i / KPIECES, pc = i % KPIECES;
@@ -222,17 +258,18 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             }
         }
         __syncthreads();
-        // scores: warp w takes cells 8w..8w+7 (one n-tile) over all 256 dims, per 64-dim scale group
+        // scores: warp w takes cells 8w..8w+7 (one n-tile) over all 256 dims, per scale group (64 dims; q4_0's 32)
+        constexpr int NG = Smem<KV_MODE>::NG, KPG = HD / 16 / NG;   // groups per row, 16-dim MMA steps per group
 #pragma unroll
         for (int nt = 0; nt < CH / 32; ++nt) {
             const int cb = (warp + 4 * nt) * 8;
             float sc[4] = {0.f, 0.f, 0.f, 0.f};
 #pragma unroll
-            for (int g = 0; g < 4; ++g) {
+            for (int g = 0; g < NG; ++g) {
                 float tg[4] = {0.f, 0.f, 0.f, 0.f};
 #pragma unroll
-                for (int kk = 0; kk < 4; ++kk) {
-                    const int k0 = (g * 4 + kk) * 16;
+                for (int kk = 0; kk < KPG; ++kk) {
+                    const int k0 = (g * KPG + kk) * 16;
                     uint32_t ah[4], al[4], b[2];
                     ah[0] = *reinterpret_cast<const uint32_t*>(&S.qh[gid][k0 + 2 * tig]);
                     ah[1] = *reinterpret_cast<const uint32_t*>(&S.qh[gid + 8][k0 + 2 * tig]);
@@ -296,24 +333,31 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             }
         }
         __syncthreads();
-        // p.v: warp w owns dims [64w, 64w+64), which is int8 scale group w. The scale is folded into p relative to
-        // the chunk's largest, times 2^14 (p' <= 2^14: its lo half stays out of FP16's subnormal range); the chunk's
-        // sum is then added to the running one in FP32 with the factor taken back out
+        // p.v: warp w owns dims [64w, 64w+64), which is int8 scale group w (q4_0: groups 2w and 2w+1, four n-tiles
+        // each). A group's scale is folded into p relative to the chunk's largest magnitude (q4_0's scales are signed:
+        // ggml's d = max / -8), times 2^14 (|p'| <= 2^14: inside FP16's range, its lo half out of the subnormals); the
+        // chunk's sum is then added to the running one in FP32 with the factor taken back out
         {
-            float vmax = 0.0f;
-#pragma unroll
-            for (int c = lane; c < CH; c += 32) vmax = fmaxf(vmax, S.vs[c][warp]);
-#pragma unroll
-            for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));
-            const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f, vdown = vmax * (1.0f / 16384.0f);
+            constexpr int GPW = NG / 4, JPG = 8 / GPW;   // scale groups per warp, n-tiles per group
+            float vdown_g[GPW];
             float tmp[8][4];
 #pragma unroll
             for (int j = 0; j < 8; ++j) tmp[j][0] = tmp[j][1] = tmp[j][2] = tmp[j][3] = 0.0f;
+#pragma unroll
+            for (int gi = 0; gi < GPW; ++gi) {
+            const int vg = warp * GPW + gi;
+            float vmax = 0.0f;
+#pragma unroll
+            for (int c = lane; c < CH; c += 32) vmax = fmaxf(vmax, fabsf(S.vs[c][vg]));
+#pragma unroll
+            for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));
+            const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f;
+            vdown_g[gi] = vmax * (1.0f / 16384.0f);
 #pragma unroll
             for (int ks = 0; ks < CH / 16; ++ks) {
                 const int cA = ks * 16 + 2 * tig, cB = cA + 8;
-                const float w0 = S.vs[cA][warp] * vup, w1 = S.vs[cA + 1][warp] * vup, w2 = S.vs[cB][warp] * vup,
-                            w3 = S.vs[cB + 1][warp] * vup;
+                const float w0 = S.vs[cA][vg] * vup, w1 = S.vs[cA + 1][vg] * vup, w2 = S.vs[cB][vg] * vup,
+                            w3 = S.vs[cB + 1][vg] * vup;
                 const float p00 = S.s[gid][cA] * w0, p01 = S.s[gid][cA + 1] * w1;
                 const float p10 = S.s[gid + 8][cA] * w0, p11 = S.s[gid + 8][cA + 1] * w1;
                 const float p02 = S.s[gid][cB] * w2, p03 = S.s[gid][cB + 1] * w3;
@@ -332,10 +376,11 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
                     f = __half22float2(h[3]); al[3] = pack_h2(p12 - f.x, p13 - f.y);
                 }
 #pragma unroll
-                for (int j = 0; j < 8; ++j) {
+                for (int jj = 0; jj < JPG; ++jj) {
+                    const int j = gi * JPG + jj;
                     const int d = warp * 64 + j * 8 + gid;
                     uint32_t b[2];
-                    if constexpr (KV_MODE == 1) {
+                    if constexpr (KV_MODE == 1 || KV_MODE == 4) {
                         const uint32_t x0 = (uint8_t) S.v[cA][d] | ((uint32_t) (uint8_t) S.v[cA + 1][d] << 8);
                         const uint32_t x1 = (uint8_t) S.v[cB][d] | ((uint32_t) (uint8_t) S.v[cB + 1][d] << 8);
                         b[0] = i8x2_to_h2(x0);
@@ -352,9 +397,11 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
 #endif
                 }
             }
+            }
             const float a0 = S.alpha[gid], a1 = S.alpha[gid + 8];
 #pragma unroll
             for (int j = 0; j < 8; ++j) {
+                const float vdown = vdown_g[j / JPG];
                 acc[j][0] = fmaf(acc[j][0], a0, tmp[j][0] * vdown);
                 acc[j][1] = fmaf(acc[j][1], a0, tmp[j][1] * vdown);
                 acc[j][2] = fmaf(acc[j][2], a1, tmp[j][2] * vdown);
@@ -1008,10 +1055,18 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3
         return launch_wmma(q, pools, ids, steps, cap, s, attn, n_q, (cudaStream_t) stream);
     return false;
 #endif
-    if (pools.k_q4 != nullptr || s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !ids ||
-        !steps || !pools.page_table)
+    if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !ids || !steps || !pools.page_table)
         return false;
     cudaStream_t st = (cudaStream_t) stream;
+    if (pools.k_q4 != nullptr) {   // Q4_0 K and V (--kv q4_0): mode 4.  STRATA_PROMPT_ATTN_Q4=0: the old kernel (A/B)
+        static const bool q4_off = [] {
+            const char* v = std::getenv("STRATA_PROMPT_ATTN_Q4");
+            return v != nullptr && v[0] == '0';
+        }();
+        // sm_80+ only: on Turing mode 4 would run as pairs of m16n8k8 MMAs, which no parity run has checked yet
+        if (q4_off || turing || pools.v_q4 == nullptr) return false;
+        return launch<4>(q, pools, ids, steps, cap, s, attn, n_q, st);
+    }
     if (pools.k_q != nullptr && pools.v_q4 != nullptr) {   // hybrid K8V4: int8 K + dequantized-q4 V
         if (!pools.k_scale) return false;
         return launch<3>(q, pools, ids, steps, cap, s, attn, n_q, st);

src/kernels/cuda/qsa_select.cu

@@ -644,6 +644,229 @@ __global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(co
         }
     }
 }
+
+#if !defined(__HIPCC__)
+// ---- the decode top-k on a thread-block cluster (sm_90+; S19).  One CTA per query (block_topk_reg_kernel, or
+// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to
+// 65,538 blocks on ONE SM while the rest of the GPU idles - a decode window has 1-5 queries.  Here a cluster of CL_N
+// CTAs shares a query: CTA r holds the keys of blocks [r * per, (r + 1) * per) in shared memory, builds the digit
+// histogram of its keys, and PUSHES it into slot r of every CTA's `hin` (distributed shared memory); after one cluster
+// barrier each CTA sums the CL_N slots itself, so all of them take the same digit.  The emit pass needs each CTA's
+// cells above / at the threshold: pushed the same way, then each CTA offsets its own by the ranks before it.
+// RTX 5070, per call at capacity = context (decode_cluster_parity --bench): 21.9 -> 15.6 us at 32K, 58 -> 18 at 128K,
+// 200 -> 22 at 262K (the one-CTA kernels' digit search was serial, too: one thread over 256 bins per pass).
+//
+// IDENTICAL IDS, by construction rather than by luck: the threshold thr (the width-th largest key, cells counted with
+// their weights) and `above` (cells with a larger key) are pure functions of the multiset of (key, weight) - integer
+// histograms, summed in any order - and the digit rule is block_topk_kernel's, written as a scan (the largest digit d
+// whose cells at or above it reach `need`; 0 when none). The cells emitted are then fixed: every cell of a block with
+// key > thr, and the first eq_budget = width - above cells at thr in ascending order (a block at thr may be cut),
+// written in ascending cell order. Position of a thread's first cell = (cells above thr before it) + min(cells at thr
+// before it, eq_budget): the telescoped sum of the reference's per-thread clamp. NaN keys are 0 (order_key) as there.
+//
+// Barriers: barrier.cluster arrive / wait (each CTA's threads all take part; it also orders the CTA's own shared
+// memory, so it doubles as __syncthreads). Phase 0 is a relaxed arrive at entry, waited before the first remote store
+// (a CTA's shared memory exists once it runs). Pushes go out before the arrive that releases them, and nothing reads
+// another CTA's memory after the last wait - so a CTA may exit without a closing barrier (a wait counts the threads
+// that have not exited). `hin` alternates by pass parity: CTA x reads slot buffer p&1 in pass p before arriving at
+// pass p+1's barrier, and nobody writes buffer p&1 again (pass p+2) before waiting on that barrier.
+constexpr int CL_N = 8;        // CTAs per query (the portable cluster size)
+constexpr int CL_T = 1024;     // threads per CTA
+constexpr int CL_MAXQ = 16;    // larger calls (prefill sub-batches) fill the GPU with one CTA per query already
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
+#define STRATA_SEL_CLUSTER 1
+#else
+#define STRATA_SEL_CLUSTER 0   // an older target's code is a trap; qsa_block_topk_cluster never launches it there
+#endif
+
+#if STRATA_SEL_CLUSTER
+__device__ __forceinline__ void cl_arrive_relaxed() {
+    asm volatile("barrier.cluster.arrive.relaxed.aligned;\n" ::: "memory");
+}
+__device__ __forceinline__ void cl_arrive() { asm volatile("barrier.cluster.arrive.release.aligned;\n" ::: "memory"); }
+__device__ __forceinline__ void cl_wait() { asm volatile("barrier.cluster.wait.acquire.aligned;\n" ::: "memory"); }
+__device__ __forceinline__ unsigned cl_rank() {
+    unsigned r;
+    asm volatile("mov.u32 %0, %%cluster_ctarank;\n" : "=r"(r));
+    return r;
+}
+template <typename T> __device__ __forceinline__ T* cl_map(T* p, unsigned rank) {   // p in CTA `rank`'s shared memory
+    uint64_t o;
+    asm volatile("mapa.u64 %0, %1, %2;\n" : "=l"(o) : "l"((uint64_t) p), "r"(rank));
+    return reinterpret_cast<T*>(o);
+}
+// exclusive block scan of a 64-bit value over CL_T threads; `total` = the CTA's sum
+__device__ __forceinline__ unsigned long long cl_excl_scan64(unsigned long long v, unsigned long long* s_warp,
+                                                             unsigned long long& total) {
+    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+    unsigned long long x = v;
+#pragma unroll
+    for (int o = 1; o < 32; o <<= 1) {
+        const unsigned long long y = __shfl_up_sync(0xffffffffu, x, o);
+        if (lane >= o) x += y;
+    }
+    if (lane == 31) s_warp[warp] = x;
+    __syncthreads();
+    if (warp == 0) {
+        const unsigned long long w = s_warp[lane];
+        unsigned long long z = w;
+#pragma unroll
+        for (int o = 1; o < 32; o <<= 1) {
+            const unsigned long long y = __shfl_up_sync(0xffffffffu, z, o);
+            if (lane >= o) z += y;
+        }
+        s_warp[lane] = z - w;
+        if (lane == 31) s_warp[32] = z;
+    }
+    __syncthreads();
+    total = s_warp[32];
+    return s_warp[warp] + x - v;
+}
+#endif
+
+// grid (CL_N, nq), cluster (CL_N, 1, 1), CL_T threads, dynamic shared memory: ceil(max_blocks / CL_N) keys
+__global__ void __launch_bounds__(CL_T) block_topk_cluster_kernel(const float* __restrict__ scores,
+                                                                  const int32_t* __restrict__ steps, int64_t max_blocks,
+                                                                  int64_t cap, int32_t* __restrict__ ids) {
+#if STRATA_SEL_CLUSTER
+    extern __shared__ uint32_t keys[];                  // this CTA's blocks' keys
+    __shared__ __align__(16) int hin[2][CL_N][256];     // the cluster's histograms, slot = the pushing rank
+    __shared__ __align__(16) int hloc[2][256];          // this CTA's histogram
+    __shared__ unsigned long long cnt[CL_N];            // per rank: its cells above thr (low 32 bits) and at it (high)
+    __shared__ unsigned long long s_warp[33];
+    __shared__ uint32_t s_prefix;
+    __shared__ int s_above;
+    const unsigned rank = cl_rank();
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+    cl_arrive_relaxed();                                // phase 0: this CTA runs
+    const int64_t qi = blockIdx.y;
+    const int32_t* st = steps + qi * kStepCount;
+    const int64_t n_kv = st[kStepNKv], n_bid = st[kStepNBid], width = st[kStepWidth];
+    int32_t* out = ids + qi * cap;
+    if (n_kv <= width) {                                // the identity, split over the cluster; no remote access
+        for (int64_t j = (int64_t) rank * CL_T + t; j < n_kv; j += (int64_t) CL_N * CL_T) out[j] = (int32_t) j;
+        return;
+    }
+    // the tail block n_bid weighs its cells (0..3); with none it is not a candidate (the reference skips w == 0)
+    const int wtail = (int) (n_kv - n_bid * R);
+    const int64_t nbe = n_bid + (wtail > 0 ? 1 : 0);
+    const int64_t per = (nbe + CL_N - 1) / CL_N;       // <= ceil(max_blocks / CL_N), the keys the host sized
+    const int64_t lo = (int64_t) rank * per < nbe ? (int64_t) rank * per : nbe;
+    const int64_t hi = lo + per < nbe ? lo + per : nbe;
+    const int n = (int) (hi - lo);
+    const int tail_i = (wtail > 0 && n_bid >= lo && n_bid < hi) ? (int) (n_bid - lo) : -1;
+    const float* sc = scores + qi * max_blocks + lo;
+    for (int i = t; i < n; i += CL_T) keys[i] = order_key(sc[i]);
+    for (int i = t; i < 2 * 256; i += CL_T) (&hloc[0][0])[i] = 0;
+    __syncthreads();
+    uint32_t prefix = 0;
+    int above = 0;                                      // cells strictly above the digits fixed so far
+    for (int pass = 0; pass < 4; ++pass) {
+        const int shift = 24 - 8 * pass;
+        const uint32_t hmask = pass == 0 ? 0u : (0xffffffffu << (shift + 8));
+        int* hl = hloc[pass & 1];
+        // warp-aggregated: the scores share their top bits, so plain atomics would queue on a few bins
+        for (int i0 = 0; i0 < n; i0 += CL_T) {
+            const int i = i0 + t;
+            int bin = -1;
+            if (i < n) {
+                const uint32_t k = keys[i];
+                if ((k & hmask) == prefix) bin = (int) ((k >> shift) & 255);
+            }
+            const unsigned same = __match_any_sync(0xffffffffu, bin);
+            if (bin >= 0 && lane == __ffs(same) - 1) atomicAdd(&hl[bin], __popc(same) * R);
+            if (bin >= 0 && i == tail_i) atomicAdd(&hl[bin], wtail - R);
+        }
+        __syncthreads();
+        if (pass == 0) cl_wait();                       // phase 0 done: every CTA's shared memory is there
+        for (int i = t; i < CL_N * 64; i += CL_T) {     // the histogram into slot `rank` of every CTA, as int4
+            const int dst = i >> 6, c = i & 63;
+            *cl_map(reinterpret_cast<int4*>(&hin[pass & 1][rank][0]) + c, (unsigned) dst) =
+                reinterpret_cast<const int4*>(hl)[c];
+        }
+        for (int i = t; i < 256; i += CL_T) hloc[(pass + 1) & 1][i] = 0;
+        cl_arrive();
+        cl_wait();
+        if (warp == 0) {
+            // lane L owns digits 255 - 8L down to 248 - 8L; the digit holding the need-th cell from the top
+            const int need = (int) width - above;
+            int tot[8], part = 0;
+#pragma unroll
+            for (int k = 0; k < 8; ++k) {
+                const int d = 255 - 8 * lane - k;
+                int v = 0;
+#pragma unroll
+                for (int r = 0; r < CL_N; ++r) v += hin[pass & 1][r][d];
+                tot[k] = v;
+                part += v;
+            }
+            int incl = part;
+#pragma unroll
+            for (int o = 1; o < 32; o <<= 1) {
+                const int y = __shfl_up_sync(0xffffffffu, incl, o);
+                if (lane >= o) incl += y;
+            }
+            const int excl = incl - part;
+            const unsigned hit = __ballot_sync(0xffffffffu, excl < need && incl >= need);
+            // no hit cannot happen (the cells total n_kv > width); the reference would then take digit 0
+            const int who = hit ? __ffs(hit) - 1 : 31;
+            if (lane == who) {
+                int acc = excl, k = 0;
+                for (; k < 7; ++k) {
+                    if (acc + tot[k] >= need) break;
+                    acc += tot[k];
+                }
+                s_prefix = prefix | ((uint32_t) (255 - 8 * lane - k) << shift);
+                s_above = above + acc;
+            }
+        }
+        __syncthreads();
+        prefix = s_prefix;
+        above = s_above;
+    }
+    const uint32_t thr = prefix;
+    const int64_t eq_budget = width - above;           // cells at thr that fit, lowest index first
+    // each thread a contiguous run of the CTA's keys; an odd run length keeps the shared reads conflict-free
+    int seg = (n + CL_T - 1) / CL_T;
+    if ((seg & 1) == 0 && seg > 0) ++seg;
+    const int s0 = t * seg < n ? t * seg : n, s1 = s0 + seg < n ? s0 + seg : n;
+    uint32_t gt = 0, eq = 0;
+    for (int i = s0; i < s1; ++i) {
+        const uint32_t k = keys[i], w = i == tail_i ? (uint32_t) wtail : (uint32_t) R;
+        if (k > thr) gt += w;
+        else if (k == thr) eq += w;
+    }
+    unsigned long long cta;
+    const unsigned long long before =
+        cl_excl_scan64((unsigned long long) gt | ((unsigned long long) eq << 32), s_warp, cta);
+    if (t < CL_N) *cl_map(&cnt[rank], (unsigned) t) = cta;
+    cl_arrive();
+    cl_wait();
+    int64_t gt_off = 0, eq_off = 0;
+    for (unsigned r = 0; r < rank; ++r) {
+        gt_off += (int64_t) (cnt[r] & 0xffffffffu);
+        eq_off += (int64_t) (cnt[r] >> 32);
+    }
+    const int64_t eq_before = eq_off + (int64_t) (before >> 32);
+    int64_t wpos = gt_off + (int64_t) (before & 0xffffffffu) + (eq_before < eq_budget ? eq_before : eq_budget);
+    int64_t eq_left = eq_budget - eq_before;
+    if (eq_left < 0) eq_left = 0;
+    for (int i = s0; i < s1; ++i) {
+        const uint32_t k = keys[i];
+        const int w = i == tail_i ? wtail : R;
+        const int64_t b = lo + i;
+        if (k > thr) {
+            for (int c = 0; c < w; ++c) out[wpos++] = (int32_t) (b * R + c);
+        } else if (k == thr) {
+            for (int c = 0; c < w && eq_left > 0; ++c, --eq_left) out[wpos++] = (int32_t) (b * R + c);
+        }
+    }
+#else
+    (void) scores; (void) steps; (void) max_blocks; (void) cap; (void) ids;
+    __trap();
+#endif
+}
+#endif  // !__HIPCC__
 }  // namespace
 
 void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
@@ -745,21 +968,121 @@ void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, i
     if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
 }
 
+#if !defined(__HIPCC__)
+// Only Turing has a retained model measurement for this CUDA dispatch. Other CUDA devices keep the capacity rule
+// (the RTX 5070 regression below). Cache the properties per calling thread; layer-split device switches are checked.
+static bool topk_active_turing_device() {
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess) return false;
+    static thread_local int cached_device = -1;
+    static thread_local bool turing = false;
+    if (dev != cached_device) {
+        cudaDeviceProp prop{};
+        if (cudaGetDeviceProperties(&prop, dev) != cudaSuccess) return false;
+        turing = prop.major == 7 && prop.minor == 5;
+        cached_device = dev;
+    }
+    return turing;
+}
+#endif
+
+bool qsa_block_topk_cluster(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                            const QsaShapes& s, int32_t* ids, void* stream) {
+#if defined(__HIPCC__)
+    (void) scores; (void) steps; (void) nq; (void) max_blocks; (void) cap; (void) s; (void) ids; (void) stream;
+    return false;
+#else
+    if (nq <= 0) return true;
+    if (s.idx_block != R || cap < qsa_selection_width(kTopkMaxCells, s) || nq > 65535 || max_blocks <= 0) return false;
+    const size_t smem = (size_t) ((max_blocks + CL_N - 1) / CL_N) * sizeof(uint32_t);
+    // Per device (a layer split runs on several): 1 the cluster kernel runs here, 2 it does not. It needs sm_90+ (the
+    // card's, or STRATA_EMULATE_CC's) AND code built for it: a build with only older code JIT-compiles their PTX, whose
+    // copy of this kernel is a trap - the function's PTX version says which. `opt`: the dynamic shared memory opted in.
+    static int ok[64] = {};
+    static size_t opt[64] = {};
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+    if (ok[dev] == 0) {
+        int major = 0;
+        cudaFuncAttributes fa{};
+        const bool code = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+                          strata::cc_major_of(major) >= 9 &&
+                          cudaFuncGetAttributes(&fa, block_topk_cluster_kernel) == cudaSuccess && fa.ptxVersion >= 90 &&
+                          fa.binaryVersion >= 90;
+        cudaGetLastError();
+        ok[dev] = code ? 1 : 2;
+    }
+    if (ok[dev] != 1) return false;
+    if (smem > opt[dev]) {   // a larger capacity: opt in, and check that a cluster of CL_N such CTAs can be resident
+        int clusters = 0;
+        cudaLaunchConfig_t q{};
+        cudaLaunchAttribute qa[1];
+        qa[0].id = cudaLaunchAttributeClusterDimension;
+        qa[0].val.clusterDim.x = CL_N;
+        qa[0].val.clusterDim.y = 1;
+        qa[0].val.clusterDim.z = 1;
+        q.gridDim = dim3(CL_N, 1, 1);
+        q.blockDim = dim3(CL_T, 1, 1);
+        q.dynamicSmemBytes = smem;
+        q.attrs = qa;
+        q.numAttrs = 1;
+        if (cudaFuncSetAttribute(block_topk_cluster_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) smem) !=
+                cudaSuccess ||
+            cudaOccupancyMaxActiveClusters(&clusters, block_topk_cluster_kernel, &q) != cudaSuccess || clusters < 1) {
+            cudaGetLastError();
+            return false;    // this capacity takes the one-CTA kernels; a smaller one may still fit
+        }
+        opt[dev] = smem;
+    }
+    cudaLaunchConfig_t cfg{};
+    cudaLaunchAttribute at[1];
+    at[0].id = cudaLaunchAttributeClusterDimension;
+    at[0].val.clusterDim.x = CL_N;
+    at[0].val.clusterDim.y = 1;
+    at[0].val.clusterDim.z = 1;
+    cfg.gridDim = dim3(CL_N, (unsigned) nq, 1);
+    cfg.blockDim = dim3(CL_T, 1, 1);
+    cfg.dynamicSmemBytes = smem;
+    cfg.stream = (cudaStream_t) stream;
+    cfg.attrs = at;
+    cfg.numAttrs = 1;
+    const cudaError_t e = cudaLaunchKernelEx(&cfg, block_topk_cluster_kernel, scores, steps, max_blocks, cap, ids);
+    if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk cluster: %s\n", cudaGetErrorString(e)); std::exit(1); }
+    return true;
+#endif
+}
+
 void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
                     const QsaShapes& s, int32_t* ids, void* stream, int64_t active_blocks) {
     // keys in registers when every query's blocks fit (contexts up to ~135K cells); the same ids. STRATA_TOPK_OLD=1:
     // the kernel that reads them from memory on every pass
     static const bool old = std::getenv("STRATA_TOPK_OLD") != nullptr;
     if (nq <= 0) return;
+#if !defined(__HIPCC__)
+    // sm_90+: a cluster of CL_N CTAs per query for the calls of a few queries (decode windows); the same ids.
+    // STRATA_QSA_CLUSTER=0: the one-CTA kernels below
+    static const bool cluster = [] {
+        const char* v = std::getenv("STRATA_QSA_CLUSTER");
+        return !v || std::atoi(v) != 0;
+    }();
+    if (cluster && !old && nq <= CL_MAXQ && qsa_block_topk_cluster(scores, steps, nq, max_blocks, cap, s, ids, stream))
+        return;
+#endif
     // the blocks a query can have: the call's active count when the caller knows it (the prompt path), else the capacity.
     // Decode (no count) keeps the capacity rule and the original register width: nothing changes there.
 #if defined(__HIPCC__)
     const bool counted = active_blocks > 0;
 #else
-    // CUDA keeps 0.1.32's capacity rule: #337's dispatch was measured on RDNA4 only, and on the RTX 5070 the 64K
-    // prompts read 1-3% slower with it
-    const bool counted = false;
-    (void) active_blocks;
+    // Turing: --max-context 262144 makes the stride 65538, even while a 131K prompt's active blocks fit in
+    // TK_T * TK_PER registers. Use the prefill bound on sm_75, keeping max_blocks as the score-row stride.
+    // Other CUDA devices keep 0.1.32's capacity rule: #337 was measured on RDNA4, and RTX 5070 64K prompts were
+    // 1-3% slower. Decode/captured graphs omit the bound and never query the device here.
+    static const bool capacity_guard = std::getenv("STRATA_TOPK_CAPACITY_GUARD") != nullptr;
+    // STRATA_TOPK_ACTIVE_ANY=1 (tests): the Turing dispatch on any CUDA card, so qsa_topk_active_parity checks it
+    // on whatever card runs the tests (the kernels are the same on every architecture)
+    static const bool any_card = [] { const char* v = std::getenv("STRATA_TOPK_ACTIVE_ANY"); return v && v[0] == '1'; }();
+    const bool counted = !capacity_guard && active_blocks > 0 && active_blocks <= max_blocks &&
+                         (any_card || topk_active_turing_device());
 #endif
     const int64_t reach = counted && active_blocks < max_blocks ? active_blocks : max_blocks;
     const int64_t fit = (int64_t) TK_T * (counted ? TK_PER_MAX : TK_PER);

src/kernels/cuda/sampler.cu

@@ -10,7 +10,8 @@
 // produces a valid token, so only a comparison at the distribution level can tell them apart; the parity test
 // does that against an independently computed distribution.
 //
-// `sampler_greedy_kernel` is the plain argmax, one block per token over the vocabulary.  The sampled chain has
+// `sampler_greedy_kernel` is the plain argmax, one block per token over the vocabulary (on sm_90+ without penalties,
+// `sampler_greedy_cluster_kernel`: the same token from a cluster of 8 CTAs per row).  The sampled chain has
 // three implementations that pick the same token, bit for bit:
 //   - the SPLIT top_k (default): `sampler_split_part_kernel` cuts each row into 4,096-logit blocks over the whole
 //     GPU, each keeps its own top_k, and `sampler_split_merge_kernel` merges those lists and runs the tail;
@@ -20,6 +21,7 @@
 // The two new ones share `sampled_tail_warp` (top_p / min_p / temperature / draw on one warp).
 #include "strata/kernels/sampler.hpp"
 #include "strata/core/coupled_draft.hpp"
+#include "strata/core/emulate.hpp"
 
 #include <cuda_runtime.h>
 
@@ -166,6 +168,96 @@ __global__ void sampler_greedy_kernel(const float* __restrict__ logits, int n_vo
     }
 }
 
+#if !defined(__HIPCC__)
+/// **THE SAME ARGMAX ON A THREAD-BLOCK CLUSTER (sm_90+, S19), FOR THE CALLS WITHOUT PENALTIES.**  One block per
+/// token reads its 1 MB row (248,320 logits) on one SM with one load in flight per thread: latency-bound, 40 us per
+/// call on an RTX 5070, for every verify window and every draft step.  Here a cluster of `kAmCtas` CTAs shares the
+/// row, each thread keeps four loads in flight, and the CTAs' results meet in CTA 0's shared memory (distributed
+/// shared memory): 6 us (decode_cluster_parity --bench).
+///
+/// The answer is a function of the row alone, so it is the one-block kernel's bit for bit: the LOWEST index whose value
+/// is the largest non-NaN value above -inf (each thread walks its elements in ascending order with a strict `>`, and
+/// every merge takes the larger value or, on equality, the smaller index - an order-free rule), and 0 when there is
+/// none (all -inf / NaN), as there.  NaN never wins a `>`.
+///
+/// Barriers: a relaxed cluster arrive at entry, waited before the remote store (CTA 0 must be running); each CTA's
+/// result goes to slot `rank` of CTA 0, released by the next arrive; CTAs 1.. then exit (a cluster wait counts the
+/// threads that have not exited) and CTA 0 waits, reads its own slots, and writes the token.
+constexpr int kAmCtas = 8;      // CTAs per token (the portable cluster size)
+constexpr int kAmThreads = 1024;
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
+#define STRATA_AM_CLUSTER 1
+#else
+#define STRATA_AM_CLUSTER 0     // older targets: a trap, never launched (sample_greedy_cluster checks)
+#endif
+#if STRATA_AM_CLUSTER
+__device__ __forceinline__ void am_take(float s, int v, float& bv, int& best) {
+    if (s > bv) { bv = s; best = v; }
+}
+__device__ __forceinline__ void am_merge(float ov, int oi, float& bv, int& best) {
+    if (ov > bv || (ov == bv && oi < best)) { bv = ov; best = oi; }
+}
+#endif
+// grid (kAmCtas, n_tokens), cluster (kAmCtas, 1, 1), kAmThreads threads
+__global__ void __launch_bounds__(kAmThreads) sampler_greedy_cluster_kernel(const float* __restrict__ logits,
+                                                                            int n_vocab, int* __restrict__ out) {
+#if STRATA_AM_CLUSTER
+    __shared__ float sv[32];
+    __shared__ int si[32];
+    __shared__ float cv[kAmCtas];    // on CTA 0: each CTA's result
+    __shared__ int ci[kAmCtas];
+    unsigned rank;
+    asm volatile("mov.u32 %0, %%cluster_ctarank;\n" : "=r"(rank));
+    asm volatile("barrier.cluster.arrive.relaxed.aligned;\n" ::: "memory");
+    const float* l = logits + (size_t) blockIdx.y * n_vocab;
+    float bv = __int_as_float(0xff800000);   // -inf; `n_vocab` is "no candidate", as in sampler_greedy_kernel
+    int best = n_vocab;
+    constexpr int S = kAmCtas * kAmThreads;
+    int v = (int) rank * kAmThreads + (int) threadIdx.x;
+    for (; v + 3 * S < n_vocab; v += 4 * S) {   // four independent loads, then taken in ascending order
+        const float x0 = l[v], x1 = l[v + S], x2 = l[v + 2 * S], x3 = l[v + 3 * S];
+        am_take(x0, v, bv, best);
+        am_take(x1, v + S, bv, best);
+        am_take(x2, v + 2 * S, bv, best);
+        am_take(x3, v + 3 * S, bv, best);
+    }
+    for (; v < n_vocab; v += S) am_take(l[v], v, bv, best);
+    for (int off = 16; off > 0; off >>= 1)
+        am_merge(__shfl_down_sync(0xFFFFFFFFu, bv, off), __shfl_down_sync(0xFFFFFFFFu, best, off), bv, best);
+    const int warp = (int) (threadIdx.x >> 5), lane = (int) (threadIdx.x & 31);
+    if (lane == 0) { sv[warp] = bv; si[warp] = best; }
+    __syncthreads();
+    if (warp == 0) {
+        bv = sv[lane];
+        best = si[lane];
+        for (int off = 16; off > 0; off >>= 1)
+            am_merge(__shfl_down_sync(0xFFFFFFFFu, bv, off), __shfl_down_sync(0xFFFFFFFFu, best, off), bv, best);
+    }
+    asm volatile("barrier.cluster.wait.acquire.aligned;\n" ::: "memory");   // CTA 0 runs
+    if (threadIdx.x == 0) {
+        uint64_t a;
+        asm volatile("mapa.u64 %0, %1, %2;\n" : "=l"(a) : "l"((uint64_t) &cv[rank]), "r"(0u));
+        *reinterpret_cast<float*>(a) = bv;
+        asm volatile("mapa.u64 %0, %1, %2;\n" : "=l"(a) : "l"((uint64_t) &ci[rank]), "r"(0u));
+        *reinterpret_cast<int*>(a) = best;
+    }
+    asm volatile("barrier.cluster.arrive.release.aligned;\n" ::: "memory");
+    if (rank != 0) return;
+    asm volatile("barrier.cluster.wait.acquire.aligned;\n" ::: "memory");
+    if (warp == 0) {
+        bv = lane < kAmCtas ? cv[lane] : __int_as_float(0xff800000);
+        best = lane < kAmCtas ? ci[lane] : n_vocab;
+        for (int off = 16; off > 0; off >>= 1)
+            am_merge(__shfl_down_sync(0xFFFFFFFFu, bv, off), __shfl_down_sync(0xFFFFFFFFu, best, off), bv, best);
+        if (lane == 0) out[blockIdx.y] = (best < n_vocab) ? best : 0;
+    }
+#else
+    (void) logits; (void) n_vocab; (void) out;
+    __trap();
+#endif
+}
+#endif  // !__HIPCC__
+
 /// **THE SAMPLED PATH, ONE BLOCK PER TOKEN.**  The kernel below replaced a version that ran the whole chain
 /// in ONE THREAD per token (`<<<ceil(T/64), 64>>>`, so a 4-token window fielded four threads): `top_k` alone
 /// was `k` sequential scans of the vocabulary with an inner sweep over the already-taken list - 20 x 248,320
@@ -833,6 +925,55 @@ int2* split_scratch(void* stream, size_t entries) {
 
 }  // namespace
 
+bool sample_greedy_cluster(const float* logits, int n_tokens, int n_vocab, int* out, void* stream) {
+#if defined(__HIPCC__)
+    (void) logits; (void) n_tokens; (void) n_vocab; (void) out; (void) stream;
+    return false;
+#else
+    if (n_tokens <= 0 || n_vocab <= 0) return true;
+    if (n_tokens > 65535) return false;
+    // Per device (a layer split runs on several): 1 the cluster kernel runs here, 2 it does not - sm_90+ (the card's,
+    // or STRATA_EMULATE_CC's), code built for it (an older build's PTX holds a trap: the PTX version says which), and
+    // room for one cluster of kAmCtas CTAs.
+    static int ok[64] = {};
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+    cudaLaunchConfig_t cfg{};
+    cudaLaunchAttribute at[1];
+    at[0].id = cudaLaunchAttributeClusterDimension;
+    at[0].val.clusterDim.x = kAmCtas;
+    at[0].val.clusterDim.y = 1;
+    at[0].val.clusterDim.z = 1;
+    cfg.gridDim = dim3(kAmCtas, 1, 1);
+    cfg.blockDim = dim3(kAmThreads, 1, 1);
+    cfg.dynamicSmemBytes = 0;
+    cfg.stream = (cudaStream_t) stream;
+    cfg.attrs = at;
+    cfg.numAttrs = 1;
+    if (ok[dev] == 0) {
+        int major = 0, clusters = 0;
+        cudaFuncAttributes fa{};
+        const bool runs = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+                          strata::cc_major_of(major) >= 9 &&
+                          cudaFuncGetAttributes(&fa, sampler_greedy_cluster_kernel) == cudaSuccess &&
+                          fa.ptxVersion >= 90 && fa.binaryVersion >= 90 &&
+                          cudaOccupancyMaxActiveClusters(&clusters, sampler_greedy_cluster_kernel, &cfg) ==
+                              cudaSuccess &&
+                          clusters >= 1;
+        cudaGetLastError();
+        ok[dev] = runs ? 1 : 2;
+    }
+    if (ok[dev] != 1) return false;
+    cfg.gridDim = dim3(kAmCtas, (unsigned) n_tokens, 1);
+    const cudaError_t e = cudaLaunchKernelEx(&cfg, sampler_greedy_cluster_kernel, logits, n_vocab, out);
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "sample_tokens cluster launch: %s\n", cudaGetErrorString(e));
+        std::exit(1);
+    }
+    return true;
+#endif
+}
+
 void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
                    const SamplerParams& p, int* out, void* stream) {
     if (n_tokens <= 0 || n_vocab <= 0) return;
@@ -845,10 +986,17 @@ void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* hi
                                ? (unsigned) ((n_vocab + 31) / 32) * sizeof(unsigned)   // the penalty bitmap
                                : 0;
     if (p.greedy || p.temperature <= 0.0f) {
+        // Without penalties (shmem == 0: no window) on sm_90+, a cluster of CTAs per token - the same token; see
+        // `sampler_greedy_cluster_kernel`.  STRATA_ARGMAX_MULTI=0: always the one-block kernel.
+        static const bool multi = [] {
+            const char* v = std::getenv("STRATA_ARGMAX_MULTI");
+            return !v || std::atoi(v) != 0;
+        }();
         // One block per token, 1,024 threads over the vocabulary.  See `sampler_greedy_kernel`.
         const int gthreads = 1024;
-        sampler_greedy_kernel<<<(unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream>>>(
-            logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);
+        if (!(multi && shmem == 0 && sample_greedy_cluster(logits, n_tokens, n_vocab, out, stream)))
+            sampler_greedy_kernel<<<(unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream>>>(
+                logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);
     } else if (sampled_path() == SampledPath::Old) {
         // The same block-per-token shape: the selection's k argmax rounds reduce inside the block.  See
         // `sampler_kernel`'s header for what the old one-thread-per-token launch cost.

src/kernels/decode_cluster_parity.cpp

@@ -0,0 +1,424 @@
+// src/kernels/decode_cluster_parity.cpp - S19: the decode kernels on thread-block clusters (sm_90+) against the ones
+// they replace, bit for bit (GPU, synthetic, no model).
+//
+//   QSA top-k : qsa_block_topk_cluster vs qsa_block_topk's one-CTA dispatch (the register kernel up to 33,792 blocks,
+//               block_topk_kernel above) and vs qsa_block_topk_ref - every id of every row of the `cap`-wide buffer,
+//               including the entries past the width that neither may write.
+//   argmax    : sample_greedy_cluster vs sample_tokens' one-block greedy kernel.
+//
+// Inputs: random scores like a real indexer's, and adversarial ones - all equal, a few distinct values (ties at the
+// threshold across the CTAs of a cluster), NaN, +-inf, -0.0 against +0.0, denormals, monotone rows, all NaN; contexts
+// from below the selection width (the identity) to 262,144 cells, every tail weight (n_kv mod 4), 1-16 queries; and
+// one CUDA graph that captures both cluster launches, replayed with new inputs.
+//
+//   decode_cluster_parity --selftest     parity (exit 0 = identical everywhere); skipped (exit 0) below sm_90
+//   decode_cluster_parity --bench        per-call times, old vs new, at 32K / 128K / 262K
+//
+// The old paths are reached through the public dispatchers with STRATA_QSA_CLUSTER=0 / STRATA_ARGMAX_MULTI=0, set
+// here before their first call (they read the variables once).
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_select.hpp"
+#include "strata/kernels/sampler.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+
+void set_env(const char* name, const char* value) {
+#if defined(_WIN32)
+    _putenv_s(name, value);
+#else
+    setenv(name, value, 1);
+#endif
+}
+
+template <typename T> struct Dev {
+    T* p = nullptr;
+    size_t n = 0;
+    explicit Dev(size_t count) : n(count) { ck(cudaMalloc(&p, count * sizeof(T) + 64), "cudaMalloc"); }
+    ~Dev() { cudaFree(p); }
+    void put(const std::vector<T>& h) {
+        ck(cudaMemcpy(p, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "up");
+        // #548 #536: a pageable cudaMemcpy can return before its DMA lands, and the graph replays and the timing
+        // cases run on a cudaStreamNonBlocking stream that does not wait for the legacy stream - so wait here
+        ck(cudaDeviceSynchronize(), "up landed");
+    }
+    std::vector<T> get() const {
+        std::vector<T> h(n);
+        ck(cudaMemcpy(h.data(), p, n * sizeof(T), cudaMemcpyDeviceToHost), "down");
+        return h;
+    }
+};
+
+const float kNaN = std::numeric_limits<float>::quiet_NaN(), kInf = std::numeric_limits<float>::infinity();
+
+// ---- QSA top-k fixtures
+enum Pattern { kRandom, kAllEqual, kFewValues, kNaNs, kInfs, kSignedZero, kDenormal, kAscending, kDescending, kAllNaN,
+               kQuantized, kPatterns };
+const char* pattern_name(int p) {
+    static const char* n[] = {"random", "all-equal", "few-values", "nan", "inf", "signed-zero", "denormal", "ascending",
+                              "descending", "all-nan", "quantized"};
+    return n[p];
+}
+
+// one query's row of block scores (blocks 0..n_bid); the tail block gets the +1e9 the score kernel gives it
+void fill_row(float* row, int64_t n_kv, int64_t n_bid, int pattern, std::mt19937& rng) {
+    std::normal_distribution<float> nd(0.f, 1.f);
+    std::uniform_int_distribution<int> pick(0, 99);
+    for (int64_t b = 0; b <= n_bid; ++b) {
+        float v = 0.f;
+        const float g = nd(rng);
+        switch (pattern) {
+            case kRandom: v = std::fmax(0.f, g) * 3.f + std::fmax(0.f, nd(rng)); break;   // relu sums: exact zeros too
+            case kAllEqual: v = 1.f; break;
+            case kFewValues: v = (float) (pick(rng) % 4) * 0.5f; break;
+            case kNaNs: v = pick(rng) < 15 ? kNaN : std::fabs(g); break;
+            case kInfs: {
+                const int r = pick(rng);
+                v = r < 5 ? kInf : r < 15 ? -kInf : r < 30 ? -std::fabs(g) : g;
+                break;
+            }
+            case kSignedZero: v = pick(rng) < 50 ? -0.0f : (pick(rng) < 50 ? 0.0f : -std::fabs(g) * 1e-3f); break;
+            case kDenormal: v = (float) (pick(rng) % 7) * 1e-41f; break;
+            case kAscending: v = (float) b; break;
+            case kDescending: v = (float) (n_bid - b); break;
+            case kAllNaN: v = kNaN; break;
+            case kQuantized: v = std::round(std::fabs(g) * 8.f) / 8.f; break;
+        }
+        row[b] = v;
+    }
+    if (n_kv % 4 != 0 && pattern != kAllNaN && pattern != kAllEqual) row[n_bid] += 1e9f;
+}
+
+struct TopkCase {
+    int64_t ctx;         // the last query's n_kv
+    int64_t capacity;    // --max-context: max_blocks = capacity / 4 + 2 (the engine's rule)
+    int nq;
+    int pattern;
+};
+
+int run_topk_case(const TopkCase& c, std::mt19937& rng, bool verbose) {
+    const k::QsaShapes s = k::qsa_real_shapes();
+    const int64_t max_blocks = c.capacity / 4 + 2, cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    std::vector<float> sc((size_t) (c.nq * max_blocks), 0.f);
+    std::vector<int32_t> steps((size_t) c.nq * k::kStepCount);
+    for (int i = 0; i < c.nq; ++i) {
+        const int64_t n_kv = c.ctx - (c.nq - 1 - i);   // the window's queries, one cell apart, the last at ctx
+        int32_t* st = steps.data() + (size_t) i * k::kStepCount;
+        st[k::kStepPos] = (int32_t) (n_kv - 1);
+        st[k::kStepNKv] = (int32_t) n_kv;
+        st[k::kStepNBid] = (int32_t) (n_kv / s.idx_block);
+        st[k::kStepWidth] = (int32_t) k::qsa_selection_width(n_kv, s);
+        fill_row(sc.data() + (size_t) i * max_blocks, n_kv, n_kv / s.idx_block, c.pattern, rng);
+    }
+    Dev<float> d_sc(sc.size());
+    const size_t n_ids = (size_t) (c.nq * cap);
+    Dev<int32_t> d_st(steps.size()), d_new(n_ids), d_old(n_ids), d_ref(n_ids);
+    d_sc.put(sc);
+    d_st.put(steps);
+    ck(cudaMemset(d_new.p, 0xA5, d_new.n * 4), "memset");
+    ck(cudaMemset(d_old.p, 0xA5, d_old.n * 4), "memset");
+    ck(cudaMemset(d_ref.p, 0xA5, d_ref.n * 4), "memset");
+    if (!k::qsa_block_topk_cluster(d_sc.p, d_st.p, c.nq, max_blocks, cap, s, d_new.p, nullptr)) {
+        std::printf("FAIL topk ctx %lld cap %lld nq %d: the cluster kernel did not run\n", (long long) c.ctx,
+                    (long long) c.capacity, c.nq);
+        return 1;
+    }
+    k::qsa_block_topk(d_sc.p, d_st.p, c.nq, max_blocks, cap, s, d_old.p, nullptr);
+    k::qsa_block_topk_ref(d_sc.p, d_st.p, c.nq, max_blocks, cap, s, d_ref.p, nullptr);
+    ck(cudaDeviceSynchronize(), "topk sync");
+    const std::vector<int32_t> a = d_new.get(), b = d_old.get(), r = d_ref.get();
+    int64_t diff_old = 0, diff_ref = 0, first = -1;
+    for (size_t i = 0; i < a.size(); ++i) {
+        if (a[i] != b[i]) { ++diff_old; if (first < 0) first = (int64_t) i; }
+        if (a[i] != r[i]) ++diff_ref;
+    }
+    const bool ok = diff_old == 0 && diff_ref == 0;
+    if (!ok || verbose)
+        std::printf("%s topk ctx %7lld cap %7lld nq %2d %-11s: %lld ids differ from the one-CTA dispatch, %lld from "
+                    "the reference%s\n", ok ? "ok  " : "FAIL", (long long) c.ctx, (long long) c.capacity, c.nq,
+                    pattern_name(c.pattern), (long long) diff_old, (long long) diff_ref,
+                    first >= 0 ? (" (first at " + std::to_string(first) + ")").c_str() : "");
+    return ok ? 0 : 1;
+}
+
+// ---- argmax fixtures
+enum AmPattern { aRandom, aTies, aAllEqual, aAllNegInf, aAllNaN, aNaNMax, aPosInf, aSignedZero, aLast, aFirst,
+                 aDenormal, aTiesAcross, aPatterns };
+const char* am_name(int p) {
+    static const char* n[] = {"random", "ties", "all-equal", "all--inf", "all-nan", "nan+max", "+inf", "signed-zero",
+                              "max-last", "max-first", "denormal", "ties-across"};
+    return n[p];
+}
+
+void fill_logits(float* l, int nv, int pattern, std::mt19937& rng) {
+    std::normal_distribution<float> nd(0.f, 3.f);
+    std::uniform_int_distribution<int> at(0, nv - 1);
+    for (int v = 0; v < nv; ++v) l[v] = nd(rng);
+    switch (pattern) {
+        case aRandom: break;
+        case aTies: { const int n = 1 + at(rng) % 9; for (int i = 0; i < n; ++i) l[at(rng)] = 100.f; break; }
+        case aAllEqual: for (int v = 0; v < nv; ++v) l[v] = 0.25f; break;
+        case aAllNegInf: for (int v = 0; v < nv; ++v) l[v] = -kInf; break;
+        case aAllNaN: for (int v = 0; v < nv; ++v) l[v] = kNaN; break;
+        case aNaNMax:
+            for (int v = 0; v < nv; ++v) if (at(rng) % 3 == 0) l[v] = kNaN;
+            l[at(rng)] = 50.f;
+            l[at(rng)] = 50.f;
+            break;
+        case aPosInf: for (int i = 0; i < 4; ++i) l[at(rng)] = kInf; break;
+        case aSignedZero:
+            for (int v = 0; v < nv; ++v) l[v] = -std::fabs(l[v]) - 1.f;
+            for (int i = 0; i < 6; ++i) l[at(rng)] = (i & 1) ? 0.0f : -0.0f;
+            break;
+        case aLast: l[nv - 1] = 1e30f; break;
+        case aFirst: l[0] = 1e30f; if (nv > 1) l[nv - 1] = 1e30f; break;
+        case aDenormal: for (int v = 0; v < nv; ++v) l[v] = (float) (at(rng) % 5) * 1e-42f - 1e-30f * (v & 1); break;
+        case aTiesAcross:   // the same maximum in several CTAs' and threads' ranges (8 x 1024 stride), plus -inf, NaN
+            for (int v = 0; v < nv; v += 1 + at(rng) % 9000) l[v] = 7.f;
+            for (int v = 3; v < nv; v += 11) l[v] = kNaN;
+            for (int v = 5; v < nv; v += 13) l[v] = -kInf;
+            break;
+    }
+}
+
+int run_argmax_case(int nv, int T, int pattern, std::mt19937& rng, bool verbose) {
+    std::vector<float> l((size_t) nv * T);
+    for (int t = 0; t < T; ++t) fill_logits(l.data() + (size_t) t * nv, nv, t % 2 == 0 ? pattern : aRandom, rng);
+    Dev<float> d_l(l.size());
+    Dev<int> d_new((size_t) T), d_old((size_t) T);
+    d_l.put(l);
+    ck(cudaMemset(d_new.p, 0xA5, (size_t) T * 4), "memset");
+    ck(cudaMemset(d_old.p, 0x5A, (size_t) T * 4), "memset");
+    k::SamplerParams sp;
+    sp.greedy = true;
+    sp.temperature = 0.f;
+    if (!k::sample_greedy_cluster(d_l.p, T, nv, d_new.p, nullptr)) {
+        std::printf("FAIL argmax nv %d T %d: the cluster kernel did not run\n", nv, T);
+        return 1;
+    }
+    k::sample_tokens(d_l.p, T, nv, nullptr, 0, sp, d_old.p, nullptr);
+    ck(cudaDeviceSynchronize(), "argmax sync");
+    const std::vector<int> a = d_new.get(), b = d_old.get();
+    // and an independent host answer: the lowest index of the largest non-NaN value above -inf, else 0
+    int bad = 0;
+    for (int t = 0; t < T; ++t) {
+        const float* r = l.data() + (size_t) t * nv;
+        int hb = -1;
+        for (int v = 0; v < nv; ++v)
+            if (r[v] > -kInf && (hb < 0 || r[v] > r[hb])) hb = v;
+        if (hb < 0) hb = 0;
+        if (a[(size_t) t] != b[(size_t) t] || a[(size_t) t] != hb) ++bad;
+    }
+    if (bad || verbose)
+        std::printf("%s argmax nv %6d T %d %-11s: %d rows differ (new %d, old %d)\n", bad ? "FAIL" : "ok  ", nv, T,
+                    am_name(pattern), bad, a[0], b[0]);
+    return bad ? 1 : 0;
+}
+
+// both cluster launches captured in one graph, replayed over new inputs written into the same buffers
+int run_graph_case(std::mt19937& rng) {
+    const k::QsaShapes s = k::qsa_real_shapes();
+    const int nq = 5, nv = 248320;
+    const int64_t capacity = 131072 + 272, max_blocks = capacity / 4 + 2;
+    const int64_t cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    Dev<float> d_sc((size_t) (nq * max_blocks)), d_l((size_t) nv * nq);
+    Dev<int32_t> d_st((size_t) nq * k::kStepCount), d_new((size_t) (nq * cap)), d_old((size_t) (nq * cap));
+    Dev<int> a_new((size_t) nq), a_old((size_t) nq);
+    ck(cudaMemset(d_new.p, 0, d_new.n * 4), "memset");   // the entries past a row's width: written by neither
+    ck(cudaMemset(d_old.p, 0, d_old.n * 4), "memset");
+    cudaStream_t cs;
+    ck(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking), "stream");
+    cudaGraph_t g = nullptr;
+    cudaGraphExec_t ge = nullptr;
+    ck(cudaStreamBeginCapture(cs, cudaStreamCaptureModeThreadLocal), "begin capture");
+    const bool l1 = k::qsa_block_topk_cluster(d_sc.p, d_st.p, nq, max_blocks, cap, s, d_new.p, cs);
+    const bool l2 = k::sample_greedy_cluster(d_l.p, nq, nv, a_new.p, cs);
+    ck(cudaStreamEndCapture(cs, &g), "end capture");
+    if (!l1 && !l2) {   // no cluster code here (a card or a build below sm_90): nothing to compare
+        if (g) cudaGraphDestroy(g);
+        cudaStreamDestroy(cs);
+        return -1;
+    }
+    if (!l1 || !l2) { std::printf("FAIL graph: one cluster kernel did not run\n"); return 1; }
+    ck(cudaGraphInstantiate(&ge, g, 0), "instantiate");
+    int fails = 0;
+    for (int rep = 0; rep < 6; ++rep) {
+        const int64_t ctx = 100000 + rep * 4001 + rep % 4;
+        std::vector<float> sc((size_t) (nq * max_blocks), 0.f), l((size_t) nv * nq);
+        std::vector<int32_t> steps((size_t) nq * k::kStepCount);
+        for (int i = 0; i < nq; ++i) {
+            const int64_t n_kv = ctx - (nq - 1 - i);
+            int32_t* st = steps.data() + (size_t) i * k::kStepCount;
+            st[k::kStepPos] = (int32_t) (n_kv - 1);
+            st[k::kStepNKv] = (int32_t) n_kv;
+            st[k::kStepNBid] = (int32_t) (n_kv / 4);
+            st[k::kStepWidth] = (int32_t) k::qsa_selection_width(n_kv, s);
+            fill_row(sc.data() + (size_t) i * max_blocks, n_kv, n_kv / 4, rep % 2 ? kFewValues : kRandom, rng);
+            fill_logits(l.data() + (size_t) i * nv, nv, rep % 2 ? aTiesAcross : aRandom, rng);
+        }
+        d_sc.put(sc);
+        d_st.put(steps);
+        d_l.put(l);
+        ck(cudaGraphLaunch(ge, cs), "graph launch");
+        k::qsa_block_topk(d_sc.p, d_st.p, nq, max_blocks, cap, s, d_old.p, cs);
+        k::SamplerParams sp;
+        sp.greedy = true;
+        k::sample_tokens(d_l.p, nq, nv, nullptr, 0, sp, a_old.p, cs);
+        ck(cudaStreamSynchronize(cs), "graph sync");
+        const auto x = d_new.get(), y = d_old.get();
+        const auto u = a_new.get(), v = a_old.get();
+        int64_t d1 = 0, d2 = 0;
+        for (size_t i = 0; i < x.size(); ++i) d1 += x[i] != y[i];
+        for (size_t i = 0; i < u.size(); ++i) d2 += u[i] != v[i];
+        if (d1 || d2) ++fails;
+        std::printf("%s graph replay %d (ctx %lld): %lld ids, %lld tokens differ\n", d1 || d2 ? "FAIL" : "ok  ", rep,
+                    (long long) ctx, (long long) d1, (long long) d2);
+    }
+    cudaGraphExecDestroy(ge);
+    cudaGraphDestroy(g);
+    cudaStreamDestroy(cs);
+    return fails;
+}
+
+// back-to-back launches on `cs` (the engine's are too: one stream, inside a graph)
+template <typename F> float time_us(cudaStream_t cs, F&& f, int reps = 300) {
+    cudaEvent_t a, b;
+    cudaEventCreate(&a);
+    cudaEventCreate(&b);
+    for (int i = 0; i < 5; ++i) f();
+    cudaEventRecord(a, cs);
+    for (int i = 0; i < reps; ++i) f();
+    cudaEventRecord(b, cs);
+    ck(cudaEventSynchronize(b), "time");
+    float ms = 0.f;
+    cudaEventElapsedTime(&ms, a, b);
+    cudaEventDestroy(a);
+    cudaEventDestroy(b);
+    return ms * 1000.f / (float) reps;
+}
+
+int bench() {
+    std::mt19937 rng(11);
+    const k::QsaShapes s = k::qsa_real_shapes();
+    const int64_t cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    cudaStream_t cs;
+    ck(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking), "stream");
+    std::printf("QSA top-k, per call (us); capacity = context + 272 as the speed runs set --max-context\n");
+    for (const int64_t ctx : {32768LL, 131072LL, 262144LL - 272}) {
+        for (const int nq : {1, 5}) {
+            const int64_t capacity = ctx + 272, max_blocks = capacity / 4 + 2;
+            std::vector<float> sc((size_t) (nq * max_blocks), 0.f);
+            std::vector<int32_t> steps((size_t) nq * k::kStepCount);
+            for (int i = 0; i < nq; ++i) {
+                const int64_t n_kv = ctx - (nq - 1 - i);
+                int32_t* st = steps.data() + (size_t) i * k::kStepCount;
+                st[k::kStepPos] = (int32_t) (n_kv - 1);
+                st[k::kStepNKv] = (int32_t) n_kv;
+                st[k::kStepNBid] = (int32_t) (n_kv / 4);
+                st[k::kStepWidth] = (int32_t) k::qsa_selection_width(n_kv, s);
+                fill_row(sc.data() + (size_t) i * max_blocks, n_kv, n_kv / 4, kRandom, rng);
+            }
+            Dev<float> d_sc(sc.size());
+            Dev<int32_t> d_st(steps.size()), d_ids((size_t) (nq * cap));
+            d_sc.put(sc);
+            d_st.put(steps);
+            const float* sp = d_sc.p;
+            const int32_t* st = d_st.p;
+            int32_t* ids = d_ids.p;
+            const float t_old = time_us(cs, [&] { k::qsa_block_topk(sp, st, nq, max_blocks, cap, s, ids, cs); });
+            const float t_ref = time_us(cs, [&] { k::qsa_block_topk_ref(sp, st, nq, max_blocks, cap, s, ids, cs); });
+            const float t_new = time_us(cs, [&] { k::qsa_block_topk_cluster(sp, st, nq, max_blocks, cap, s, ids, cs); });
+            std::printf("  ctx %6lld nq %d: one-CTA dispatch %7.1f  (ref %7.1f)  cluster %6.1f  -> %.1fx\n",
+                        (long long) ctx, nq, t_old, t_ref, t_new, t_old / t_new);
+        }
+    }
+    std::printf("argmax, per call (us)\n");
+    for (const int nv : {248320, 32768}) {
+        for (const int T : {1, 5}) {
+            std::vector<float> l((size_t) nv * T);
+            for (int t = 0; t < T; ++t) fill_logits(l.data() + (size_t) t * nv, nv, aRandom, rng);
+            Dev<float> d_l(l.size());
+            Dev<int> d_o((size_t) T);
+            d_l.put(l);
+            k::SamplerParams sp;
+            sp.greedy = true;
+            const float t_old = time_us(cs, [&] { k::sample_tokens(d_l.p, T, nv, nullptr, 0, sp, d_o.p, cs); });
+            const float t_new = time_us(cs, [&] { k::sample_greedy_cluster(d_l.p, T, nv, d_o.p, cs); });
+            std::printf("  n_vocab %6d T %d: one block %6.1f  cluster %5.1f  -> %.1fx\n", nv, T, t_old, t_new,
+                        t_old / t_new);
+        }
+    }
+    cudaStreamDestroy(cs);
+    return 0;
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    set_env("STRATA_QSA_CLUSTER", "0");     // the dispatchers: the one-CTA kernels (the cluster ones called directly)
+    set_env("STRATA_ARGMAX_MULTI", "0");
+    const bool do_bench = argc > 1 && std::strcmp(argv[1], "--bench") == 0;
+    const bool verbose = argc > 2 && std::strcmp(argv[2], "-v") == 0;
+    int dev = 0, major = 0;
+    ck(cudaGetDevice(&dev), "device");
+    ck(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev), "cc");
+    std::mt19937 rng(20261002);
+    // first, so that both kernels' first calls (their per-device checks and opt-ins) happen inside a stream capture,
+    // as in the engine's decode graphs
+    const int gfails = run_graph_case(rng);
+    if (gfails < 0) {
+        std::printf("SKIP: no thread-block clusters on this card / build (compute capability %d)\n", major);
+        return 0;
+    }
+    if (do_bench) return bench();
+    int fails = 0, cases = 0;
+    // contexts: below the width (identity), at it, just past it, odd tails, the register kernel's range, past it
+    // (block_topk_kernel), up to 262,144; capacities: tight (the speed runs) and 262,144 (a long --max-context)
+    const int64_t ctxs[] = {7, 2050, 2051, 2052, 2053, 2054, 2055, 3001, 9000, 32768, 32769, 65538, 100003,
+                            131072, 135170, 135171, 200001, 262142, 262143, 262144};
+    for (const int64_t ctx : ctxs) {
+        for (const int64_t capacity : {ctx + 272, (int64_t) 262144 + 272}) {
+            if (capacity < ctx) continue;
+            for (int p = 0; p < kPatterns; ++p) {
+                const int nq = (p % 3 == 0) ? 1 : (p % 3 == 1 ? 5 : 16);
+                if (nq > ctx) continue;
+                fails += run_topk_case({ctx, capacity, nq, p}, rng, verbose);
+                ++cases;
+            }
+        }
+    }
+    std::printf("QSA top-k: %d cases, %d failed\n", cases, fails);
+    int afails = 0, acases = 0;
+    for (const int nv : {248320, 248321, 248319, 32771, 24576, 8193, 1000, 33, 1}) {
+        for (int p = 0; p < aPatterns; ++p) {
+            for (const int T : {1, 5}) {
+                afails += run_argmax_case(nv, T, p, rng, verbose);
+                ++acases;
+            }
+        }
+    }
+    std::printf("argmax: %d cases, %d failed\n", acases, afails);
+    const int total = fails + afails + gfails;
+    std::printf("%s\n", total == 0 ? "PASS: the cluster kernels are bitwise identical" : "FAIL");
+    return total == 0 ? 0 : 1;
+}

src/kernels/native_expert_parity.cpp

@@ -94,7 +94,9 @@ int check_blob(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int se
             ffp[k] = ff[k].data();
         }
         cpu::native_gu_rows(f, blob.data(), a, NT, ffp, 0, (int) FF);
-        if (cpu::iq512_supported(f.gu_type)) {
+        // either multi-token kernel: IQ4_XS has an AVX-2 one and no AVX-512 one, so the gate cannot be
+        // iq512_supported alone - that would leave the format untested on every CPU.
+        if (cpu::iq512_supported(f.gu_type) || cpu::iq256_supported(f.gu_type)) {
             // ggml's own vec_dot, same Q8_K activations: the reference for both multi-token kernels
             // (float-order differences only)
             const auto* tc = ggml_get_type_traits_cpu((ggml_type) f.gu_type);
@@ -143,8 +145,9 @@ int check_blob(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int se
                             tag, us1, 2.0 * f.up_off / us1 / 1e3, usg, 2.0 * f.up_off / usg / 1e3);
                 std::printf("          gate+up %d tokens one thread: %s %.0f us vs ggml %d x %.0f us\n", NT, tag, usn, NT, usg);
             };
-            if (cpu::cpu_avx512_ok()) check("avx512", true);   // guarded: the binary runs on AVX-2 CPUs too
-            check("avx2", false);
+            // guarded: the binary runs on AVX-2 CPUs too, and IQ4_XS has no AVX-512 kernel (an empty switch)
+            if (cpu::cpu_avx512_ok() && cpu::iq512_supported(f.gu_type)) check("avx512", true);
+            if (cpu::iq256_supported(f.gu_type)) check("avx2", false);
         }
         for (int k = 0; k < NT; ++k) {
             cpu::native_quant_h(f, ff[k].data(), hq[k].data());

src/kernels/native_grouped_parity.cpp

@@ -0,0 +1,351 @@
+// src/kernels/native_grouped_parity.cpp - native_expert_grouped's group stride (grid_groups) and its fused SwiGLU +
+// q8_1 pass, against the launches before them (native_grouped_set_v1: a block row per possible group, SwiGLU and the
+// q8_1 quantization as two kernels over all cap_entries).  Synthetic blobs, no model.
+//
+//     build/native_grouped_parity           the checks (a ctest; needs the GPU)
+//     build/native_grouped_parity --bench   the checks, then v1/new timings of 48 calls in a graph (a window's layers)
+//
+// Every gate/up format with every down format; calls of 0..cap groups of 1..T entries each (an empty call is what the
+// verify window's PCIe call usually is), entries from 0 and from past 0 (the PCIe call's follow the VRAM call's),
+// scattered destinations, stale scratch, grid_groups 0 (= cap), 1, 2, 3, 4 and above cap: the output must be BITWISE
+// equal to v1's, the rows the call must not write included (both start from the same NaN pattern).
+//
+// Random bytes are valid codes for every format here (every grid index is in range); only the fp16 block scales are
+// set, small enough that the SwiGLU outputs keep a finite fp16 q8_1 scale.
+#include "strata/kernels/iq_kernels.hpp"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <numeric>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+
+int g_fail = 0;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "CUDA error in %s: %s\n", what, cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+
+template<typename T>
+T* dalloc(size_t n) {
+    T* p = nullptr;
+    ck(cudaMalloc(&p, n * sizeof(T) + 256), "malloc");
+    ck(cudaMemset(p, 0, n * sizeof(T) + 256), "memset");
+    return p;
+}
+
+const char* name_of(int t) {
+    switch (t) {
+        case 16: return "IQ2_XXS";
+        case 17: return "IQ2_XS";
+        case 18: return "IQ3_XXS";
+        case 20: return "IQ4_NL";
+        case 21: return "IQ3_S";
+        case 22: return "IQ2_S";
+        case 23: return "IQ4_XS";
+        case 29: return "IQ1_M";
+        case 42: return "Q2_0";
+        case 12: return "Q4_K";
+        case 13: return "Q5_K";
+        case 7: return "Q5_1";
+        case 8: return "Q8_0";
+        default: return "?";
+    }
+}
+int block_values(int t) { return t == 20 || t == 7 || t == 8 ? 32 : t == 42 ? 64 : 256; }
+
+// `rows` rows of `n` values of format t: random bytes, then a finite fp16 scale in every block
+std::vector<uint8_t> random_rows(int t, int64_t rows, int64_t n, std::mt19937& rng) {
+    const size_t rb = k::iq_row_bytes(t, n), bs = k::iq_row_bytes(t, block_values(t));
+    std::vector<uint8_t> w((size_t) rows * rb);
+    std::uniform_int_distribution<int> byte(0, 255), ex(2, 8), man(0, 1023), sgn(0, 1);
+    for (auto& b : w) b = (uint8_t) byte(rng);
+    for (size_t o = 0; o < w.size(); o += bs) {
+        if (t == 29) {
+            // IQ1_M: the fp16 scale is the top nibbles of its four scale words (bytes 48..55), the sign and the
+            // exponent's top bits in the last: 0x1 / 0x2 there (0x9 / 0xA negative) keeps it in 2^-11 .. 2^-3
+            const uint8_t nib = (uint8_t) ((sgn(rng) ? 0x8 : 0x0) | (1 + (byte(rng) & 1)));
+            w[o + 55] = (uint8_t) ((w[o + 55] & 0x0F) | (nib << 4));
+        } else {   // the other formats start their block with the fp16 scale: 2^-13 .. 2^-6
+            // (Q4_K / Q5_K: 2^-14 .. 2^-11, their 6-bit sub-scales multiply it by up to 63)
+            const int e = (t == 12 || t == 13) ? 1 + ex(rng) % 4 : ex(rng);
+            const uint16_t h = (uint16_t) ((sgn(rng) ? 0x8000 : 0) | (e << 10) | man(rng));
+            std::memcpy(&w[o], &h, 2);
+            if (t == 12 || t == 13 || t == 7) {   // Q4_K / Q5_K: dmin, Q5_1: m - the second fp16, as small
+                const int em = (t == 7) ? ex(rng) : 1 + ex(rng) % 4;
+                const uint16_t m = (uint16_t) ((sgn(rng) ? 0x8000 : 0) | (em << 10) | man(rng));
+                std::memcpy(&w[o + 2], &m, 2);
+            }
+        }
+    }
+    return w;
+}
+
+// one window's worth of expert calls on one format pair: blobs, T tokens' q8_1 activations, cap = T * K entries
+struct Setup {
+    k::NativeExpertLayout L;
+    int T = 0, K = 0, cap = 0, n_blobs = 0;
+    size_t slot = 0;
+    uint8_t* blobs = nullptr;
+    uint8_t* xq = nullptr;
+    uint8_t* scr = nullptr;
+    size_t scr_bytes = 0;
+    float* out = nullptr;
+    size_t out_floats = 0;
+    unsigned long long* ptr = nullptr;
+    int32_t *start = nullptr, *n = nullptr, *dst = nullptr, *tok = nullptr;
+
+    Setup(int gu, int dt, int64_t H, int64_t FF, int tokens, int k_per_token, int blobs_n, std::mt19937& rng,
+          cudaStream_t s) {
+        L = k::native_expert_layout(gu, dt, H, FF);
+        T = tokens; K = k_per_token; cap = T * K; n_blobs = blobs_n;
+        slot = (L.bytes + 255) & ~(size_t) 255;
+        blobs = dalloc<uint8_t>(slot * (size_t) n_blobs);
+        for (int b = 0; b < n_blobs; ++b) {
+            // gate rows | up rows (format gu, H values each) | down rows (format dt, FF values each)
+            auto blob = random_rows(gu, 2 * FF, H, rng);
+            const auto down = random_rows(dt, H, FF, rng);
+            blob.insert(blob.end(), down.begin(), down.end());
+            if (blob.size() != L.bytes) { std::fprintf(stderr, "blob %zu != %zu bytes\n", blob.size(), L.bytes); std::exit(2); }
+            ck(cudaMemcpy(blobs + slot * (size_t) b, blob.data(), blob.size(), cudaMemcpyHostToDevice), "blob");
+        }
+        std::normal_distribution<float> nd(0.f, 1.f);
+        std::vector<float> x((size_t) T * H);
+        for (auto& v : x) v = nd(rng);
+        float* dx = dalloc<float>(x.size());
+        ck(cudaMemcpy(dx, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+        xq = dalloc<uint8_t>((size_t) T * (H / 32) * 36);
+        k::quantize_q8_1_rows(dx, T, H, xq, s);
+        ck(cudaStreamSynchronize(s), "quantize");
+        cudaFree(dx);
+        scr_bytes = k::native_expert_scratch_bytes(cap, FF);
+        scr = dalloc<uint8_t>(scr_bytes);
+        out_floats = (size_t) cap * H;
+        out = dalloc<float>(out_floats);
+        ptr = dalloc<unsigned long long>(cap);
+        start = dalloc<int32_t>(cap + 1);
+        n = dalloc<int32_t>(1);
+        dst = dalloc<int32_t>(cap);
+        tok = dalloc<int32_t>(cap);
+    }
+    ~Setup() {
+        cudaFree(blobs); cudaFree(xq); cudaFree(scr); cudaFree(out);
+        cudaFree(ptr); cudaFree(start); cudaFree(n); cudaFree(dst); cudaFree(tok);
+    }
+    // a plan: `groups` groups of 1..T entries from entry `base` on (as many as fit in cap)
+    int plan(int groups, int base, std::mt19937& rng) {
+        std::uniform_int_distribution<int> size(1, T);
+        std::vector<int> sizes;
+        int used = base;
+        for (int g = 0; g < groups; ++g) {
+            const int m = std::min(size(rng), cap - used);
+            if (m <= 0) break;
+            sizes.push_back(m);
+            used += m;
+        }
+        return plan_sizes(sizes, base, rng);
+    }
+    // groups of these sizes from entry `base` on, each its own blob (as long as there are enough: a window's groups
+    // are different experts); scattered destinations, random tokens
+    int plan_sizes(const std::vector<int>& sizes, int base, std::mt19937& rng) {
+        std::uniform_int_distribution<int> tk(0, T - 1);
+        std::vector<int> order(n_blobs);
+        std::iota(order.begin(), order.end(), 0);
+        std::shuffle(order.begin(), order.end(), rng);
+        std::vector<unsigned long long> p;
+        std::vector<int32_t> st(1, base), tk_v, ds(cap);
+        for (const int m : sizes) {
+            p.push_back((unsigned long long) (blobs + slot * (size_t) order[p.size() % order.size()]));
+            st.push_back(st.back() + m);
+        }
+        if (st.back() > cap) { std::fprintf(stderr, "plan: %d entries > cap %d\n", st.back(), cap); std::exit(2); }
+        const int ng = (int) p.size();
+        std::iota(ds.begin(), ds.end(), 0);
+        std::shuffle(ds.begin(), ds.end(), rng);                  // scattered rows of `out`
+        for (int e = 0; e < cap; ++e) tk_v.push_back(tk(rng));
+        if (ng) ck(cudaMemcpy(ptr, p.data(), p.size() * 8, cudaMemcpyHostToDevice), "ptr");
+        ck(cudaMemcpy(start, st.data(), st.size() * 4, cudaMemcpyHostToDevice), "start");
+        ck(cudaMemcpy(n, &ng, 4, cudaMemcpyHostToDevice), "n");
+        ck(cudaMemcpy(dst, ds.data(), ds.size() * 4, cudaMemcpyHostToDevice), "dst");
+        ck(cudaMemcpy(tok, tk_v.data(), tk_v.size() * 4, cudaMemcpyHostToDevice), "tok");
+        return ng;
+    }
+    void call(int64_t grid_groups, cudaStream_t s) {
+        k::native_expert_grouped(L, ptr, start, n, dst, tok, cap, cap, xq, scr, out, s, grid_groups);
+    }
+    // `out` after one call from the NaN pattern, the scratch holding `junk` (what earlier calls left there)
+    std::vector<uint32_t> result(bool v1, int64_t grid_groups, int junk, cudaStream_t s) {
+        ck(cudaMemset(out, 0xFF, out_floats * 4), "memset");
+        ck(cudaMemset(scr, junk, scr_bytes), "memset");
+        k::native_grouped_set_v1(v1);
+        call(grid_groups, s);
+        k::native_grouped_set_v1(false);
+        ck(cudaStreamSynchronize(s), "sync");
+        std::vector<uint32_t> o(out_floats);
+        ck(cudaMemcpy(o.data(), out, o.size() * 4, cudaMemcpyDeviceToHost), "out");
+        return o;
+    }
+};
+
+void check(int gu, int dt, int64_t H, int64_t FF, cudaStream_t s, std::mt19937& rng) {
+    Setup S(gu, dt, H, FF, 6, 4, 9, rng, s);   // 6 tokens x 4 experts: cap 24 entries
+    int calls = 0, bad = 0;
+    size_t written = 0;
+    for (const int base : {0, 7}) {
+        for (const int groups : {0, 1, 2, 3, 5, 8, 24}) {
+            const int ng = S.plan(groups, base, rng);
+            const auto ref = S.result(true, 0, 0x00, s);
+            size_t w = 0;
+            for (const uint32_t v : ref) w += v != 0xFFFFFFFFu;
+            written += w;
+            for (const int64_t gy : {(int64_t) 0, (int64_t) 1, (int64_t) 2, (int64_t) 3, (int64_t) 4, (int64_t) S.cap + 3}) {
+                const auto got = S.result(false, gy, (calls & 1) ? 0x7F : 0x00, s);
+                ++calls;
+                size_t diff = 0;
+                for (size_t i = 0; i < ref.size(); ++i) diff += ref[i] != got[i];
+                if (diff) {
+                    std::printf("  %-8s/%-7s base %d, %d groups, grid_groups %lld: %zu of %zu floats differ  FAIL\n",
+                                name_of(gu), name_of(dt), base, ng, (long long) gy, diff, ref.size());
+                    ++bad;
+                }
+            }
+        }
+    }
+    std::printf("%-8s/%-7s %5lld x %4lld  %d calls: %s (%zu output floats written in all)\n", name_of(gu), name_of(dt),
+                (long long) H, (long long) FF, calls, bad ? "FAIL" : "bitwise equal to v1", written);
+    if (written == 0) { std::printf("  nothing was written: the check checked nothing  FAIL\n"); ++bad; }
+    g_fail += bad;
+}
+
+// ------------------------------------------------------------------------------------------------ --bench
+// A window's 48 calls, one per layer, each with ITS OWN experts as the layers have: copies of the Setup's blobs in one
+// arena, so the weights stream from DRAM as in the engine.  (48 calls re-reading the same 28 experts - 73 MB, about an
+// Ada L2 - would read part of them from L2; the engine's layers share none.)
+struct Layers {
+    uint8_t* arena = nullptr;
+    std::vector<unsigned long long*> ptr;           // per layer: its groups' blobs (device)
+    Layers(Setup& S, int ng) {
+        const int per = std::max(ng, 1);
+        arena = dalloc<uint8_t>(S.slot * (size_t) (48 * per));
+        for (int l = 0; l < 48; ++l) {
+            std::vector<unsigned long long> p;
+            for (int j = 0; j < per; ++j) {
+                uint8_t* d = arena + S.slot * (size_t) (l * per + j);
+                ck(cudaMemcpy(d, S.blobs + S.slot * (size_t) ((l * per + j) % S.n_blobs), S.L.bytes,
+                              cudaMemcpyDeviceToDevice), "blob copy");
+                p.push_back((unsigned long long) d);
+            }
+            unsigned long long* dp = dalloc<unsigned long long>(p.size());
+            ck(cudaMemcpy(dp, p.data(), p.size() * 8, cudaMemcpyHostToDevice), "ptr");
+            ptr.push_back(dp);
+        }
+    }
+    Layers(const Layers&) = delete;
+    Layers& operator=(const Layers&) = delete;
+    ~Layers() {
+        for (auto* p : ptr) cudaFree(p);
+        cudaFree(arena);
+    }
+};
+
+// the 48 calls of one variant captured in one graph
+cudaGraphExec_t make_graph(cudaStream_t s, Setup& S, const Layers& Ls, bool v1, int64_t gy) {
+    k::native_grouped_set_v1(v1);
+    cudaGraph_t g;
+    cudaGraphExec_t ge;
+    ck(cudaStreamBeginCapture(s, cudaStreamCaptureModeThreadLocal), "capture");
+    for (int l = 0; l < 48; ++l)
+        k::native_expert_grouped(S.L, Ls.ptr[l], S.start, S.n, S.dst, S.tok, S.cap, S.cap, S.xq, S.scr, S.out, s, gy);
+    ck(cudaStreamEndCapture(s, &g), "capture");
+    ck(cudaGraphInstantiate(&ge, g, 0), "instantiate");
+    ck(cudaGraphDestroy(g), "graph");
+    k::native_grouped_set_v1(false);
+    return ge;
+}
+
+// one launch of the graph: microseconds per call
+float launch_us(cudaStream_t s, cudaGraphExec_t ge, cudaEvent_t e0, cudaEvent_t e1) {
+    ck(cudaEventRecord(e0, s), "record");
+    ck(cudaGraphLaunch(ge, s), "launch");
+    ck(cudaEventRecord(e1, s), "record");
+    ck(cudaEventSynchronize(e1), "bench");
+    float ms = 0;
+    ck(cudaEventElapsedTime(&ms, e0, e1), "elapsed");
+    return 1e3f * ms / 48.0f;
+}
+
+float median(std::vector<float> v) {
+    std::sort(v.begin(), v.end());
+    return v[v.size() / 2];
+}
+
+// The two variants' launches ALTERNATE (v1-new, new-v1, ...) after a warm-up, and each reports its median: a card
+// under sustained load steps its clock down after a fraction of a second (an RTX 4080 SUPER went from 80 to 89 us
+// per gate/up call within one run), and whichever variant ran before the step would look faster.
+constexpr int kWarmPairs = 20, kPairs = 40;
+
+void bench(cudaStream_t s, std::mt19937& rng) {
+    std::printf("\n--bench: microseconds per call, a window's 48 calls in a graph, each layer with its own experts (the\n"
+                "weights stream from DRAM as in the engine); v1 / new (grid_groups): medians of %d launches each,\n"
+                "alternating, after %d warm-up pairs - idle GPU assumed\n", kPairs, kWarmPairs);
+    // a verify window's shape: 4 tokens x 10 experts, 2560 x 768 IQ3_S / IQ4_XS experts
+    Setup S(21, 23, 2560, 768, 4, 10, 32, rng, s);
+    std::vector<int> vram(16, 1);
+    vram.insert(vram.end(), 12, 2);                                   // 28 groups, 40 entries
+    const struct { const char* what; std::vector<int> sizes; int64_t gy; } rows[] = {
+        {"no group (the PCIe call at pcie_frac 0)", {}, 4},
+        {"1 group of 2 entries (a PCIe call)", {2}, 4},
+        {"28 groups, 40 entries (a VRAM call)", vram, 0},
+    };
+    cudaEvent_t e0, e1;
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    for (const auto& r : rows) {
+        const int ng = S.plan_sizes(r.sizes, 0, rng);
+        const Layers Ls(S, ng);                                       // 48 x 28 experts: 3.7 GB for the VRAM call
+        cudaGraphExec_t ga = make_graph(s, S, Ls, true, 0), gb = make_graph(s, S, Ls, false, r.gy);
+        std::vector<float> ta, tb;
+        for (int i = 0; i < kWarmPairs + kPairs; ++i) {
+            const bool a_first = i % 2 == 0;
+            const float x = launch_us(s, a_first ? ga : gb, e0, e1), y = launch_us(s, a_first ? gb : ga, e0, e1);
+            if (i < kWarmPairs) continue;
+            ta.push_back(a_first ? x : y);
+            tb.push_back(a_first ? y : x);
+        }
+        ck(cudaGraphExecDestroy(ga), "graph");
+        ck(cudaGraphExecDestroy(gb), "graph");
+        std::printf("  %-42s %8.2f / %8.2f us (grid_groups %lld)\n", r.what, median(ta), median(tb), (long long) r.gy);
+    }
+    cudaEventDestroy(e0);
+    cudaEventDestroy(e1);
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    const bool do_bench = argc > 1 && std::string(argv[1]) == "--bench";
+    cudaStream_t s;
+    ck(cudaStreamCreate(&s), "stream");
+    std::mt19937 rng(316);
+    for (int gu : {16, 17, 18, 21, 22, 23, 29, 42, 12, 13, 8})        // STRATA_GU_FMTS
+        for (int dt : {20, 23, 42, 7, 8}) check(gu, dt, 512, 256, s, rng);   // STRATA_D_FMTS; IQ4_XS: n_ff % 256
+    check(21, 20, 2560, 640, s, rng);                                   // a model's shapes
+    check(21, 23, 2560, 768, s, rng);
+    if (do_bench) bench(s, rng);
+    std::printf("native_grouped_parity: %d failures\n", g_fail);
+    cudaStreamDestroy(s);
+    return g_fail ? 1 : 0;
+}

src/kernels/ngram.cpp

@@ -209,13 +209,9 @@ bool PleTable::open(const std::string& gguf_path, std::string& err, const PleIoO
         close();
         return false;
     }
-    // The direct reader has fixed 90-byte rows. Keep it for IQ4_NL; Q5_0's
-    // 110-byte rows use the mapped path until that reader supports variable rows.
-    if (impl_->q5_0 && io.mode == PleIo::Direct) {
-        err = "Q5_0 PLE requires --ple-io mmap";
-        close();
-        return false;
-    }
+    // PleReader's row_bytes has been a runtime parameter since the FP8 table (160 B rows) needed it; Q5_0's
+    // 110 B rows go through the exact same generic path (ple_reader_test --selftest covers both 90 and 110 B
+    // rows: straddling, caching, in-flight tickets, keep-alive). This refusal was stale.
     impl_->n_rows = t->shape[1];
     impl_->data = impl_->file->tensor_data(*t);
 

src/kernels/qsa_prompt_attn_parity.cpp

@@ -1,7 +1,7 @@
 // src/kernels/qsa_prompt_attn_parity.cpp - perf-review D-1: the tensor-core prompt attention (qsa_prompt_attn.hpp)
 // against the FP32 kernel it replaces (`qsa_decode_attn_batch`) and an FP64 host reference (GPU, synthetic, no model).
 //
-// Int8 and FP16 pools with random codes, scales and queries; selections shaped like the prompt path's (the 2,051
+// Int8, FP16 and Q4_0 pools with random codes, scales and queries; selections shaped like the prompt path's (the 2,051
 // widest, a recent window plus older cells that drift slowly from one query to the next, so neighbours share most
 // of them as they do in a real prompt; short contexts take every cell). Checks:
 //   1. against FP64, the new kernel's error is no larger than a small multiple of the old kernel's (both FP32 math);
@@ -12,6 +12,7 @@
 #include "strata/kernels/qsa.hpp"
 #include "strata/kernels/qsa_decode_attn.hpp"
 #include "strata/kernels/qsa_prompt_attn.hpp"
+#include "strata/kernels/kv_q4.hpp"
 
 #include <cuda_fp16.h>
 #include <cuda_runtime.h>
@@ -39,7 +40,7 @@ template <typename T> T* up(const std::vector<T>& h) {
 float h2f(uint16_t b) { __half h; *reinterpret_cast<uint16_t*>(&h) = b; return __half2float(h); }
 uint16_t f2h(float f) { __half h = __float2half(f); return *reinterpret_cast<uint16_t*>(&h); }
 
-int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
+int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16, 2 q4_0
     const k::QsaShapes s = k::qsa_real_shapes();
     const int64_t HD = s.head_dim, NKV = s.n_head_kv, NH = s.n_head, PS = s.page_size;
     const int64_t pages = (ctx + PS - 1) / PS, rows = pages * NKV * PS;
@@ -50,7 +51,22 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
     // pools; page table a shuffled permutation (the readers must follow it)
     std::vector<int8_t> kq, vq;
     std::vector<uint16_t> ks, vs, kh, vh;
-    if (fmt == 1) {
+    std::vector<uint8_t> k4, v4;   // q4_0: 8 blocks of {fp16 d, 16 code bytes} per row (kv_q4.hpp)
+    constexpr int64_t B4 = 18, ROW4 = 8 * B4;
+    if (fmt == 2) {
+        std::uniform_int_distribution<int> byte(0, 255);
+        // the scales as a q4_0 pool holds them: signed (ggml's d = max / -8) and spread over two decades, so a chunk's
+        // cells mix signs and magnitudes (a positive-only bound on them overflowed FP16 on real K/V)
+        std::uniform_real_distribution<float> lg(std::log(0.01f), std::log(1.0f));
+        auto sc4 = [&](std::mt19937& r) { return std::exp(lg(r)) * (byte(r) & 1 ? -1.0f : 1.0f); };
+        k4.resize(rows * ROW4); v4.resize(rows * ROW4);
+        for (auto* pool : {&k4, &v4})
+            for (int64_t b = 0; b < rows * 8; ++b) {
+                const uint16_t d = f2h(sc4(rng));
+                std::memcpy(pool->data() + b * B4, &d, 2);
+                for (int j = 0; j < 16; ++j) (*pool)[b * B4 + 2 + j] = (uint8_t) byte(rng);
+            }
+    } else if (fmt == 1) {
         kq.resize(rows * HD); vq.resize(rows * HD); ks.resize(rows * 4); vs.resize(rows * 4);
         for (auto& x : kq) x = (int8_t) code(rng);
         for (auto& x : vq) x = (int8_t) code(rng);
@@ -98,8 +114,19 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
         std::copy(v.begin(), v.end(), sel);
     }
     k::QsaAttnPools pl;
-    if (fmt == 1) { pl.k_q = up(kq); pl.v_q = up(vq); pl.k_scale = up(ks); pl.v_scale = up(vs); }
+    if (fmt == 2) { pl.k_q4 = up(k4); pl.v_q4 = up(v4); }
+    else if (fmt == 1) { pl.k_q = up(kq); pl.v_q = up(vq); pl.k_scale = up(ks); pl.v_scale = up(vs); }
     else { pl.k_pool = up(kh); pl.v_pool = up(vh); }
+    // a q4_0 value: block d / 32 of the row, element j = d % 32 in the low nibble of byte j (j < 16), else the high
+    // nibble of byte j - 16, minus 8, times the block's scale
+    auto q4v = [&](const std::vector<uint8_t>& pool, int64_t row, int64_t d) {
+        const uint8_t* blk = pool.data() + row * ROW4 + (d / 32) * B4;
+        uint16_t sc;
+        std::memcpy(&sc, blk, 2);
+        const int j = (int) (d % 32);
+        const int code = j < 16 ? (blk[2 + j] & 0x0F) : (blk[2 + j - 16] >> 4);
+        return (double) (code - 8) * h2f(sc);
+    };
     pl.page_table = up(table);
     const int32_t* d_ids = up(ids);
     const int32_t* d_steps = up(steps);
@@ -139,7 +166,8 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
                 const int64_t cell = sel[c], row = ((int64_t) table[cell / PS] * NKV + kvh) * PS + cell % PS;
                 double a = 0;
                 for (int64_t d = 0; d < HD; ++d) {
-                    const double kv = fmt == 1 ? (double) kq[row * HD + d] * h2f(ks[row * 4 + d / 64]) : h2f(kh[row * HD + d]);
+                    const double kv = fmt == 2 ? q4v(k4, row, d)
+                                      : fmt == 1 ? (double) kq[row * HD + d] * h2f(ks[row * 4 + d / 64]) : h2f(kh[row * HD + d]);
                     a += (double) q[(i * NH + h) * HD + d] * kv;
                 }
                 sco[c] = a / 16.0;
@@ -151,7 +179,8 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
                 double a = 0;
                 for (int64_t c = 0; c < w; ++c) {
                     const int64_t cell = sel[c], row = ((int64_t) table[cell / PS] * NKV + kvh) * PS + cell % PS;
-                    const double vv = fmt == 1 ? (double) vq[row * HD + d] * h2f(vs[row * 4 + d / 64]) : h2f(vh[row * HD + d]);
+                    const double vv = fmt == 2 ? q4v(v4, row, d)
+                                      : fmt == 1 ? (double) vq[row * HD + d] * h2f(vs[row * 4 + d / 64]) : h2f(vh[row * HD + d]);
                     a += sco[c] * vv;
                 }
                 const double r = a / l;
@@ -187,12 +216,13 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
     const bool ok2 = diff <= 1e-4 * scale;
     std::printf("%s %s ctx %lld, %lld queries: vs FP64 old %.3g new %.3g (output scale %.3g); new vs old %.3g (%.2g of "
                 "scale); %.3f -> %.3f ms per chunk (%.2fx)\n",
-                ok1 && ok2 ? "PASS" : "FAIL", fmt == 1 ? "int8" : "fp16", (long long) ctx, (long long) nq, err_old,
+                ok1 && ok2 ? "PASS" : "FAIL", fmt == 2 ? "q4_0" : fmt == 1 ? "int8" : "fp16", (long long) ctx, (long long) nq, err_old,
                 err_new, ref_scale, diff, diff / scale, ms_old / reps, ms_new / reps, ms_old / ms_new);
     cudaFree((void*) d_ids); cudaFree((void*) d_steps); cudaFree((void*) d_q); cudaFree(d_old); cudaFree(d_new);
     cudaFree(scratch);
     cudaFree((void*) pl.k_q); cudaFree((void*) pl.v_q); cudaFree((void*) pl.k_scale); cudaFree((void*) pl.v_scale);
     cudaFree((void*) pl.k_pool); cudaFree((void*) pl.v_pool); cudaFree((void*) pl.page_table);
+    cudaFree((void*) pl.k_q4); cudaFree((void*) pl.v_q4);
     return ok1 && ok2 ? 0 : 1;
 }
 }  // namespace
@@ -222,6 +252,8 @@ int main(int argc, char** argv) {
     fails += run(1, ctx, nq, reps);
 #if !defined(__HIP_PLATFORM_AMD__)
     fails += run(0, ctx, nq, reps);   // FP16 KV: the RDNA4 kernel takes int8 KV only
+    fails += run(2, ctx, nq, reps);   // Q4_0 KV (mode 4)
+    fails += run(2, 1500, std::min<int64_t>(nq, 1500), reps);
 #endif
     fails += run(1, 1500, std::min<int64_t>(nq, 1500), reps);   // short context: the selection is every cell
     fails += run(1, 2100, std::min<int64_t>(nq, 256), reps);    // the identity-to-sparse edge

src/kernels/qsa_topk_active_parity.cpp

@@ -0,0 +1,95 @@
+// Capacity 262144, live prefill up to the register boundary, and the legacy fallbacks.
+// Compare selected IDs only: unused output padding is not part of the API contract.
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_select.hpp"
+#include <cuda_runtime.h>
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+namespace {
+void ck(cudaError_t e) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "CUDA: %s\n", cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+struct Case { int64_t ctx, queries; int mode; };
+bool check(Case c) {
+    constexpr int64_t stride = 262144 / 4 + 2;
+    const auto s = k::qsa_real_shapes();
+    const int64_t cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    std::vector<int32_t> steps((size_t)(c.queries * k::kStepCount));
+    std::vector<float> scores((size_t)(c.queries * stride), -999.f);
+    std::mt19937 rng(742913);
+    std::uniform_real_distribution<float> random(0.f, 200.f);
+    for (int64_t i = 0; i < c.queries; ++i) {
+        auto* st = steps.data() + i * k::kStepCount;
+        const int64_t pos = c.ctx - c.queries + i;
+        st[k::kStepPos] = (int32_t)pos;
+        st[k::kStepNKv] = (int32_t)(pos + 1);
+        st[k::kStepNBid] = (int32_t)((pos + 1) / s.idx_block);
+        st[k::kStepWidth] = (int32_t)k::qsa_selection_width(pos + 1, s);
+        for (int64_t j = 0; j <= st[k::kStepNBid]; ++j)
+            scores[(size_t)(i * stride + j)] = c.mode == 1 ? 0.f : c.mode == 2 ? float(j % 7) : random(rng);
+        if ((pos + 1) % s.idx_block)
+            scores[(size_t)(i * stride + st[k::kStepNBid])] += 1e9f;
+    }
+    int64_t active = steps[(size_t)((c.queries - 1) * k::kStepCount + k::kStepNBid)] + 1;
+    if (c.mode == 3) active = 0;
+    if (c.mode == 4) active = stride + 1;
+    float* ds = nullptr;
+    int32_t *dt = nullptr, *da = nullptr, *db = nullptr;
+    ck(cudaMalloc(&ds, scores.size() * sizeof(float)));
+    ck(cudaMalloc(&dt, steps.size() * sizeof(int32_t)));
+    ck(cudaMalloc(&da, (size_t)(c.queries * cap) * sizeof(int32_t)));
+    ck(cudaMalloc(&db, (size_t)(c.queries * cap) * sizeof(int32_t)));
+    ck(cudaMemcpy(ds, scores.data(), scores.size() * sizeof(float), cudaMemcpyHostToDevice));
+    ck(cudaMemcpy(dt, steps.data(), steps.size() * sizeof(int32_t), cudaMemcpyHostToDevice));
+    // No bound: the exact capacity-based dispatch used by captured decode.
+    k::qsa_block_topk(ds, dt, c.queries, stride, cap, s, da, nullptr);
+    k::qsa_block_topk(ds, dt, c.queries, stride, cap, s, db, nullptr, active);
+    ck(cudaDeviceSynchronize());
+    std::vector<int32_t> a((size_t)(c.queries * cap)), b(a.size());
+    ck(cudaMemcpy(a.data(), da, a.size() * sizeof(int32_t), cudaMemcpyDeviceToHost));
+    ck(cudaMemcpy(b.data(), db, b.size() * sizeof(int32_t), cudaMemcpyDeviceToHost));
+    bool ok = true;
+    for (int64_t i = 0; i < c.queries; ++i) {
+        const int64_t w = steps[(size_t)(i * k::kStepCount + k::kStepWidth)];
+        if (!std::equal(a.begin() + i * cap, a.begin() + i * cap + w, b.begin() + i * cap)) {
+            std::fprintf(stderr, "FAIL ctx=%lld queries=%lld mode=%d row=%lld\n",
+                         (long long)c.ctx, (long long)c.queries, c.mode, (long long)i);
+            ok = false;
+            break;
+        }
+    }
+    ck(cudaFree(ds)); ck(cudaFree(dt)); ck(cudaFree(da)); ck(cudaFree(db));
+    return ok;
+}
+}
+
+int main() {
+    int devices = 0;
+    if (cudaGetDeviceCount(&devices) != cudaSuccess || devices == 0) {
+        std::puts("SKIP: no CUDA device");
+        return 77;
+    }
+    // the active-bound dispatch is Turing's in the engine; forced here so every CUDA card checks it
+#if defined(_WIN32)
+    _putenv_s("STRATA_TOPK_ACTIVE_ANY", "1");
+#else
+    setenv("STRATA_TOPK_ACTIVE_ANY", "1", 1);
+#endif
+    const Case cases[] = {
+        {1111, 256, 0}, {8192, 256, 0}, {32768, 256, 0}, {131072, 256, 0},
+        {131075, 255, 0}, {135164, 256, 0}, {135168, 256, 0}, {250022, 256, 0},
+        {262144, 256, 0}, {131072, 1, 0}, {131072, 8, 0}, {131072, 257, 0},
+        {131072, 256, 1}, {131072, 256, 2}, {131072, 256, 3}, {131072, 256, 4},
+    };
+    for (const auto c : cases) if (!check(c)) return 1;
+    std::puts("PASS: 16 top-k active-bound cases, selected IDs bitwise identical");
+    return 0;
+}

src/ngram/ple_reader_test.cpp

@@ -41,33 +41,33 @@ int g_fail = 0;
 
 constexpr uint64_t HEADER = 192;   // the real shard's data offset, so rows are misaligned the same way
 
-void expected_row(uint32_t row, uint8_t* out) {
-    for (uint32_t b = 0; b < ng::ROW_BYTES; ++b) out[b] = (uint8_t) ((row * 2654435761u + b * 97u) >> 7);
+void expected_row(uint32_t row, uint32_t rb, uint8_t* out) {
+    for (uint32_t b = 0; b < rb; ++b) out[b] = (uint8_t) ((row * 2654435761u + b * 97u) >> 7);
     std::memcpy(out, &row, 4);
 }
 
-bool make_table(const std::string& path, uint32_t rows) {
+bool make_table(const std::string& path, uint32_t rows, uint32_t rb) {
     std::ofstream f(path, std::ios::binary);
     std::vector<uint8_t> head(HEADER, 0xAB);
     f.write((const char*) head.data(), (std::streamsize) head.size());
-    uint8_t r[ng::ROW_BYTES];
+    std::vector<uint8_t> r(rb);
     for (uint32_t i = 0; i < rows; ++i) {
-        expected_row(i, r);
-        f.write((const char*) r, ng::ROW_BYTES);
+        expected_row(i, rb, r.data());
+        f.write((const char*) r.data(), rb);
     }
     return (bool) f;
 }
 
-bool check_rows(ng::PleReader& rd, const std::vector<uint32_t>& rows, uint32_t n_rows, const char* what) {
-    std::vector<uint8_t> out(rows.size() * ng::ROW_BYTES, 0xCC);
+bool check_rows(ng::PleReader& rd, const std::vector<uint32_t>& rows, uint32_t n_rows, uint32_t rb, const char* what) {
+    std::vector<uint8_t> out(rows.size() * rb, 0xCC);
     std::string err;
     const auto t = rd.issue(rows.data(), rows.size(), out.data());
     if (!rd.collect(t, err)) { CHECK(false, "%s: collect failed: %s", what, err.c_str()); return false; }
-    uint8_t want[ng::ROW_BYTES];
+    std::vector<uint8_t> want(rb);
     for (size_t i = 0; i < rows.size(); ++i) {
-        if (rows[i] >= n_rows) std::memset(want, 0, sizeof want);
-        else expected_row(rows[i], want);
-        if (std::memcmp(want, &out[i * ng::ROW_BYTES], ng::ROW_BYTES) != 0) {
+        if (rows[i] >= n_rows) std::memset(want.data(), 0, rb);
+        else expected_row(rows[i], rb, want.data());
+        if (std::memcmp(want.data(), &out[i * rb], rb) != 0) {
             CHECK(false, "%s: row %u (index %zu) differs", what, rows[i], i);
             return false;
         }
@@ -75,50 +75,53 @@ bool check_rows(ng::PleReader& rd, const std::vector<uint32_t>& rows, uint32_t n
     return true;
 }
 
-int selftest(const std::string& dir) {
+// row_bytes: ng::ROW_BYTES (90, IQ4_NL) is the production default; 110 (#296, OrcaRouter's Q5_0 PLE rows) is
+// run too, through the exact same generic row_bytes path -- nothing here is IQ4_NL-specific, so a second row
+// size run here is the correctness evidence for lifting ngram.cpp's "Q5_0 PLE requires --ple-io mmap" refusal.
+int selftest(const std::string& dir, uint32_t rb) {
     const uint32_t N = 500000;                          // 45 MB: large enough for thousands of distinct pages
-    const std::string path = dir + "/ple_reader_selftest.bin";
-    if (!make_table(path, N)) { std::fprintf(stderr, "cannot write %s\n", path.c_str()); return 2; }
+    const std::string path = dir + "/ple_reader_selftest_" + std::to_string(rb) + ".bin";
+    if (!make_table(path, N, rb)) { std::fprintf(stderr, "cannot write %s\n", path.c_str()); return 2; }
     std::mt19937 rng(7);
     for (bool thr : {false, true})
     for (uint64_t cache : {0ull, 4096ull}) {
         for (uint32_t inflight : {1u, 8u, 64u}) {
             ng::PleReader rd;
             std::string err;
-            CHECK(rd.open(path, HEADER, N, inflight, cache, err, thr), "open: %s", err.c_str());
+            CHECK(rd.open(path, HEADER, N, inflight, cache, err, thr, rb), "open: %s", err.c_str());
             // decode-shaped tickets: 16 random rows
             for (int t = 0; t < 200; ++t) {
                 std::vector<uint32_t> rows(16);
                 for (auto& r : rows) r = rng() % N;
-                check_rows(rd, rows, N, "decode");
+                check_rows(rd, rows, N, rb, "decode");
             }
-            // rows that straddle a 4 KiB boundary: byte offset of row r is HEADER + 90 r
+            // rows that straddle a 4 KiB boundary: byte offset of row r is HEADER + rb * r
             std::vector<uint32_t> straddle;
             for (uint32_t r = 0; r < N && straddle.size() < 64; ++r) {
-                const uint64_t a = HEADER + (uint64_t) r * ng::ROW_BYTES;
-                if (a / 4096 != (a + ng::ROW_BYTES - 1) / 4096) straddle.push_back(r);
+                const uint64_t a = HEADER + (uint64_t) r * rb;
+                if (a / 4096 != (a + rb - 1) / 4096) straddle.push_back(r);
             }
-            check_rows(rd, straddle, N, "straddle");
+            check_rows(rd, straddle, N, rb, "straddle");
             // duplicates, neighbours on one page, the first and last rows, and out-of-range rows
-            check_rows(rd, {5, 5, 6, 7, 5, 0, N - 1, N, 0xFFFFFFFFu, 44, 45}, N, "dedup/edges");
+            check_rows(rd, {5, 5, 6, 7, 5, 0, N - 1, N, 0xFFFFFFFFu, 44, 45}, N, rb, "dedup/edges");
             // a prefill-shaped ticket much larger than the in-flight window
             std::vector<uint32_t> bulk(20000);
             for (auto& r : bulk) r = rng() % N;
-            check_rows(rd, bulk, N, "bulk");
+            check_rows(rd, bulk, N, rb, "bulk");
             // two tickets in flight at once, collected in reverse order
             std::vector<uint32_t> a(16), b(16);
             for (auto& r : a) r = rng() % N;
             for (auto& r : b) r = rng() % N;
-            std::vector<uint8_t> oa(16 * ng::ROW_BYTES), ob(16 * ng::ROW_BYTES);
+            std::vector<uint8_t> oa(16 * rb), ob(16 * rb);
             const auto ta = rd.issue(a.data(), 16, oa.data());
             const auto tb = rd.issue(b.data(), 16, ob.data());
             CHECK(rd.collect(tb, err) && rd.collect(ta, err), "two tickets: %s", err.c_str());
-            uint8_t want[ng::ROW_BYTES];
+            std::vector<uint8_t> want(rb);
             for (int i = 0; i < 16; ++i) {
-                expected_row(a[i], want);
-                CHECK(!std::memcmp(want, &oa[i * ng::ROW_BYTES], ng::ROW_BYTES), "ticket a row %d", i);
-                expected_row(b[i], want);
-                CHECK(!std::memcmp(want, &ob[i * ng::ROW_BYTES], ng::ROW_BYTES), "ticket b row %d", i);
+                expected_row(a[i], rb, want.data());
+                CHECK(!std::memcmp(want.data(), &oa[i * rb], rb), "ticket a row %d", i);
+                expected_row(b[i], rb, want.data());
+                CHECK(!std::memcmp(want.data(), &ob[i * rb], rb), "ticket b row %d", i);
             }
             if (cache > 0) CHECK(rd.stats().cache_hits > 0, "the row cache never hit");
             CHECK(rd.cache_size() <= rd.cache_capacity(), "row cache exceeded its bound");
@@ -128,12 +131,12 @@ int selftest(const std::string& dir) {
     for (bool thr : {false, true}) {
         ng::PleReader rd;
         std::string err;
-        CHECK(rd.open(path, HEADER, N, 16, 0, err, thr), "open: %s", err.c_str());
+        CHECK(rd.open(path, HEADER, N, 16, 0, err, thr, rb), "open: %s", err.c_str());
         rd.set_injected_delay_us(3000);
         std::vector<uint32_t> rows(16);
         for (auto& r : rows) r = rng() % N;
         const double t0 = now_us();
-        check_rows(rd, rows, N, "delayed");
+        check_rows(rd, rows, N, rb, "delayed");
         CHECK(now_us() - t0 >= 3000, "injected delay not observed (%.0f us)", now_us() - t0);
         CHECK(rd.stats().late_injected > 0, "no read was held back");
     }
@@ -143,11 +146,11 @@ int selftest(const std::string& dir) {
     {
         ng::PleReader rd;
         std::string err;
-        CHECK(rd.open(path, HEADER, N, 16, 4096, err, true), "open: %s", err.c_str());
+        CHECK(rd.open(path, HEADER, N, 16, 4096, err, true, rb), "open: %s", err.c_str());
         rd.set_keepalive(20.0, 0.5);
         std::vector<uint32_t> rows(16);
         for (auto& r : rows) r = rng() % N;
-        check_rows(rd, rows, N, "keep-alive, first ticket");
+        check_rows(rd, rows, N, rb, "keep-alive, first ticket");
         const uint64_t reads0 = rd.snapshot().reads;
         const auto nap = [](int ms) { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); };
         nap(300);
@@ -161,14 +164,14 @@ int selftest(const std::string& dir) {
         CHECK(k3 == k2, "keep-alive went on after the window (%llu -> %llu)", (unsigned long long) k2,
               (unsigned long long) k3);
         CHECK(k2 <= k1 + 20, "keep-alive: %llu reads by the end of a 0.5 s window", (unsigned long long) k2);
-        check_rows(rd, rows, N, "keep-alive, cached ticket");   // the row cache serves all 16: no row read
+        check_rows(rd, rows, N, rb, "keep-alive, cached ticket");   // the row cache serves all 16: no row read
         CHECK(rd.snapshot().reads == reads0, "cached rows were read again");
         nap(200);
         CHECK(rd.snapshot().keepalive_reads >= k3 + 5, "the keep-alive did not start again (%llu -> %llu)",
               (unsigned long long) k3, (unsigned long long) rd.snapshot().keepalive_reads);
         std::vector<uint32_t> more(2000);
         for (auto& r : more) r = rng() % N;
-        check_rows(rd, more, N, "keep-alive, rows afterwards");
+        check_rows(rd, more, N, rb, "keep-alive, rows afterwards");
         rd.set_keepalive(0, 0.5);                           // off
         const uint64_t k4 = rd.snapshot().keepalive_reads;
         nap(200);
@@ -179,16 +182,16 @@ int selftest(const std::string& dir) {
     {   // the caller's thread does the reads: no worker, so no keep-alive
         ng::PleReader rd;
         std::string err;
-        CHECK(rd.open(path, HEADER, N, 16, 0, err, false), "open: %s", err.c_str());
+        CHECK(rd.open(path, HEADER, N, 16, 0, err, false, rb), "open: %s", err.c_str());
         rd.set_keepalive(20.0, 1.0);
         std::vector<uint32_t> rows(16);
         for (auto& r : rows) r = rng() % N;
-        check_rows(rd, rows, N, "keep-alive, caller thread");
+        check_rows(rd, rows, N, rb, "keep-alive, caller thread");
         std::this_thread::sleep_for(std::chrono::milliseconds(150));
         CHECK(rd.snapshot().keepalive_reads == 0, "keep-alive without the worker thread");
     }
     std::filesystem::remove(path);
-    std::printf("ple_reader selftest: %s\n", g_fail ? "FAILED" : "OK");
+    std::printf("ple_reader selftest (row_bytes=%u): %s\n", rb, g_fail ? "FAILED" : "OK");
     return g_fail ? 1 : 0;
 }
 
@@ -282,7 +285,13 @@ int main(int argc, char** argv) {
         else if (a == "--sync") sync_submit = true;
         else { std::fprintf(stderr, "usage: ple_reader_test --selftest [--dir D] | --gguf SHARD2 [--rows N] [--tokens F]\n"); return 2; }
     }
-    if (self) return selftest(dir);
+    if (self) {
+        // ng::ROW_BYTES (90, IQ4_NL, production default) and 110 (#296, OrcaRouter's Q5_0 PLE rows) through the
+        // same generic row_bytes path -- see the comment on selftest().
+        const int r90 = selftest(dir, ng::ROW_BYTES);
+        const int r110 = selftest(dir, 110);
+        return r90 != 0 ? r90 : r110;
+    }
     if (!gguf.empty()) return real(gguf, rows, tokens, inflight, direct_first, direct_only, sync_submit);
     std::fprintf(stderr, "nothing to do\n");
     return 2;

src/prefill/gdn_rec_parity.cu

@@ -0,0 +1,886 @@
+// src/prefill/gdn_rec_parity.cu - the prompt path's DeltaNet recurrence, bit for bit and timed.
+//
+// src/prefill/kernels.cu runs gdn_rec_kh_kernel (the three value heads that share a key head in one thread, the
+// inputs staged in blocks of 8 tokens) on CUDA sm_80+ cards that hold all 64 of its blocks at once, and
+// gdn_rec_cols_pipe_kernel elsewhere and with STRATA_GDN_KEYHEAD=0.  Both are copied below verbatim, with the output
+// norm (gdn_out_norm_kernel) and the norm as it was before, which also stored the normalized value in FP32 although
+// nothing reads it.  The two recurrences do the same arithmetic in the same order per value head and column, so the
+// output, the state after it and the FP16 output must be the same bits.
+//
+// Synthetic inputs of the model's shape (q/k L2-normalized per key head as gdn_l2_kernel leaves them), no model.
+//   gdn_rec_parity            the bit checks (T from 1 to 4099, a non-zero state as after an earlier chunk)
+//   gdn_rec_parity --bench    and the timings at T = 8192 and 32768 (one layer, all 48 value heads); the two
+//                             variants' launches alternate, since a card under load steps its clock down after a
+//                             fraction of a second (see native_grouped_parity); then both at 1 to 6 blocks per
+//                             SM, as on cards with more or fewer SMs (scale), and both with the engine's grids on
+//                             32 to all of this card's SMs, the others held by a sleeping kernel (fewer_sms)
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+#include <vector>
+
+namespace {
+
+constexpr int S = 128, HK = 16, HV = 48, C = 10240;   // as kernels.cu: head size, key heads, value heads, q|k|v width
+constexpr int RG = 4, RPG = S / RG;
+constexpr int CB = 32, NCB = S / CB;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+
+__device__ __forceinline__ float warp_sum(float v) {
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
+    return v;
+}
+__device__ __forceinline__ float sigm(float x) { return 1.0f / (1.0f + __expf(-x)); }
+__device__ __forceinline__ uint16_t hf(float f) { return __half_as_ushort(__float2half_rn(f)); }
+
+// ------------------------------------------------------------------ the engine's kernels (src/prefill/kernels.cu)
+// gdn_rec_cols_kernel with the next token's inputs (q/k rows, v, gate, beta) loaded into registers while this token
+// computes (software pipelining).  The same arithmetic in the same order: the same bits, and the same CB-column split.
+// STRATA_GDN_PIPELINE=0: gdn_rec_cols_kernel.
+__global__ void __launch_bounds__(CB * RG) gdn_rec_cols_pipe_kernel(float* __restrict__ state, const float* __restrict__ h,
+                                                                      const float* __restrict__ gate,
+                                                                      const float* __restrict__ beta,
+                                                                      float* __restrict__ oc_out, int64_t T) {
+    constexpr int NT = CB * RG, LPT = S / NT;   // threads, q/k rows loaded per thread
+    __shared__ float sk[S], sq[S], red[RG][CB];
+    const int head = blockIdx.x / NCB, cb = blockIdx.x % NCB;
+    const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
+    const int qh = head % HK;
+    float s[RPG];
+    float* base = state + ((size_t) (rg * RPG) * HV + head) * S + col;
+    const size_t rs = (size_t) HV * S;
+#pragma unroll
+    for (int r = 0; r < RPG; ++r) s[r] = base[r * rs];
+    float nq[LPT], nk[LPT], nv = 0.0f, ng = 0.0f, nb = 0.0f;
+    auto fetch = [&](int64_t t) {
+        const float* ht = h + t * C;
+#pragma unroll
+        for (int u = 0; u < LPT; ++u) { nq[u] = ht[qh * S + tid + u * NT]; nk[u] = ht[HK * S + qh * S + tid + u * NT]; }
+        nv = ht[2 * HK * S + head * S + col];
+        ng = gate[t * HV + head];
+        nb = beta[t * HV + head];
+    };
+    if (T > 0) fetch(0);
+    for (int64_t t = 0; t < T; ++t) {
+        float cq[LPT], ck[LPT];
+#pragma unroll
+        for (int u = 0; u < LPT; ++u) { cq[u] = nq[u]; ck[u] = nk[u]; }
+        const float cv = nv, cg = ng, cbt = nb;
+        __syncthreads();
+#pragma unroll
+        for (int u = 0; u < LPT; ++u) { sq[tid + u * NT] = cq[u]; sk[tid + u * NT] = ck[u]; }
+        __syncthreads();
+        if (t + 1 < T) fetch(t + 1);
+        const float g = __expf(cg);
+        float kv = 0.0f;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) kv = fmaf(s[r], sk[rg * RPG + r], kv);
+        red[rg][c] = kv;
+        __syncthreads();
+        const float kv_col = red[0][c] + red[1][c] + red[2][c] + red[3][c];
+        const float delta = (cv - g * kv_col) * cbt;
+        float o = 0.0f;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) {
+            s[r] = fmaf(g, s[r], sk[rg * RPG + r] * delta);
+            o = fmaf(s[r], sq[rg * RPG + r], o);
+        }
+        __syncthreads();
+        red[rg][c] = o;
+        __syncthreads();
+        if (rg == 0) oc_out[t * HV * S + head * S + col] = (red[0][c] + red[1][c] + red[2][c] + red[3][c]) * rsqrtf((float) S);
+    }
+#pragma unroll
+    for (int r = 0; r < RPG; ++r) base[r * rs] = s[r];
+}
+// The recurrence with one thread for the three value heads that share a key head (head % HK): column c of heads
+// qh, qh + 16 and qh + 32, row group rg.  gdn_rec_cols_pipe_kernel spends its time in shared memory, not in
+// arithmetic: every thread of a warp needs the same 32 q and k values per token (the k twice), and a warp receives one
+// such broadcast value per clock however wide the load.  Here every q/k value a thread loads feeds three heads, and a
+// token's k row goes into registers once for both of its uses.  The inputs come in blocks of GDN_TB tokens, copied to
+// shared memory by cp.async while the block before computes (one token ahead is shorter than a load from L2 takes),
+// and the two cross-row-group sums have their own arrays, so a token needs 2 __syncthreads instead of 5: the second
+// one of a token orders every read of rkv before the next token's writes, the next token's first one every read of ro
+// before the writes after it.  64 blocks instead of 192.  Per value head and column the same arithmetic in the same
+// order: the same bits (src/prefill/gdn_rec_parity.cu checks them and times the variants: 1.41x on a 4080 Super).
+// sm_80+ (gdn_keyhead_ok); STRATA_GDN_KEYHEAD=0: gdn_rec_cols_pipe_kernel.
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
+#define STRATA_GDN_CP_ASYNC 0   // Turing builds: plain copies (never launched there, see gdn_keyhead_ok)
+#else
+#define STRATA_GDN_CP_ASYNC 1
+#endif
+__device__ __forceinline__ void gdn_cp4(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *smem = *gmem;
+#endif
+}
+__device__ __forceinline__ void gdn_cp16(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *reinterpret_cast<float4*>(smem) = *reinterpret_cast<const float4*>(gmem);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_commit() {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.commit_group;\n" ::);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_wait_prev() {   // every group but the newest has landed
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.wait_group 1;\n" ::);
+#endif
+}
+constexpr int GDN_TB = 8, VPK = HV / HK;   // tokens per staged block, value heads per key head
+__global__ void __launch_bounds__(CB * RG) gdn_rec_kh_kernel(float* __restrict__ state, const float* __restrict__ h,
+                                                               const float* __restrict__ gate,
+                                                               const float* __restrict__ beta,
+                                                               float* __restrict__ oc_out, int64_t T) {
+    constexpr int TB = GDN_TB, NT = CB * RG, QKP = S / 4, VP = CB / 4;   // threads, 16-byte pieces of a q/k row, of v
+    __shared__ __align__(16) float sq[2][TB][S];
+    __shared__ __align__(16) float sk[2][TB][S];
+    __shared__ __align__(16) float sv[2][TB][VPK][CB];
+    __shared__ float sg[2][TB][VPK], sb[2][TB][VPK], rkv[VPK][RG][CB], ro[VPK][RG][CB];
+    const int qh = blockIdx.x / NCB, cb = blockIdx.x % NCB;
+    const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
+    float s[VPK][RPG];
+    const size_t rs = (size_t) HV * S;
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        const float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) s[j][r] = base[r * rs];
+    }
+    const int64_t nblk = (T + TB - 1) / TB;
+    auto stage = [&](int64_t k) {   // tokens [k * TB, k * TB + TB) into buffer k & 1
+        const int bb = (int) (k & 1);
+        const int64_t t0 = k * TB;
+        for (int p = tid; p < TB * 2 * QKP; p += NT) {
+            const int i = p / (2 * QKP), w = p % (2 * QKP), isk = w / QKP, jj = (w % QKP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(isk ? &sk[bb][i][jj] : &sq[bb][i][jj], h + (t0 + i) * C + (isk ? HK * S : 0) + qh * S + jj);
+        }
+        for (int p = tid; p < TB * VPK * VP; p += NT) {
+            const int i = p / (VPK * VP), w = p % (VPK * VP), j = w / VP, jj = (w % VP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(&sv[bb][i][j][jj], h + (t0 + i) * C + 2 * HK * S + (qh + j * HK) * S + cb * CB + jj);
+        }
+        for (int p = tid; p < 2 * TB * VPK; p += NT) {
+            const int isb = p / (TB * VPK), w = p % (TB * VPK), i = w / VPK, j = w % VPK;
+            if (t0 + i < T)
+                gdn_cp4(isb ? &sb[bb][i][j] : &sg[bb][i][j], (isb ? beta : gate) + (t0 + i) * HV + qh + j * HK);
+        }
+    };
+    if (nblk > 0) stage(0);
+    gdn_cp_commit();
+    for (int64_t k = 0; k < nblk; ++k) {
+        // buffer (k + 1) & 1 was read by block k - 1, whose last token's second __syncthreads every thread has passed
+        if (k + 1 < nblk) stage(k + 1);
+        gdn_cp_commit();
+        gdn_cp_wait_prev();
+        __syncthreads();
+        const int bb = (int) (k & 1);
+        const int n = (int) ((T - k * TB) < TB ? (T - k * TB) : TB);
+        for (int i = 0; i < n; ++i) {
+            const int64_t t = k * TB + i;
+            float kc[RPG];
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) kc[r] = sk[bb][i][rg * RPG + r];
+            float g[VPK], kv[VPK], delta[VPK], o[VPK];
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) { g[j] = __expf(sg[bb][i][j]); kv[j] = 0.0f; o[j] = 0.0f; }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r)
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) kv[j] = fmaf(s[j][r], kc[r], kv[j]);
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) rkv[j][rg][c] = kv[j];
+            __syncthreads();
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) {
+                const float kv_col = rkv[j][0][c] + rkv[j][1][c] + rkv[j][2][c] + rkv[j][3][c];
+                delta[j] = (sv[bb][i][j][c] - g[j] * kv_col) * sb[bb][i][j];
+            }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) {
+                const float qr = sq[bb][i][rg * RPG + r];
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) {
+                    s[j][r] = fmaf(g[j], s[j][r], kc[r] * delta[j]);
+                    o[j] = fmaf(s[j][r], qr, o[j]);
+                }
+            }
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) ro[j][rg][c] = o[j];
+            __syncthreads();
+            if (rg < VPK)   // row group j writes head j's output
+                oc_out[t * HV * S + (qh + rg * HK) * S + col] =
+                    (ro[rg][0][c] + ro[rg][1][c] + ro[rg][2][c] + ro[rg][3][c]) * rsqrtf((float) S);
+        }
+    }
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) base[r * rs] = s[j][r];
+    }
+}
+// the output norm over a head's 128 columns, into the FP16 copy the out projection reads (the FP32 output before the
+// norm stays in its scratch buffer)
+__global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
+                                                         float eps, const float* __restrict__ y,
+                                                         uint16_t* __restrict__ y16) {
+    __shared__ float wsum[4];
+    const int64_t t = blockIdx.x;
+    const int head = blockIdx.y, col = threadIdx.x;
+    const size_t at = (size_t) t * HV * S + (size_t) head * S + col;
+    const float oc = y[at];
+    float sp = warp_sum(oc * oc);
+    if ((col & 31) == 0) wsum[col >> 5] = sp;
+    __syncthreads();
+    const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
+    const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
+    y16[at] = hf(v);
+}
+
+// ------------------------------------------------------------------ the output norm before (0.1.31)
+__global__ void __launch_bounds__(S) gdn_out_norm_old_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
+                                                             float eps, float* __restrict__ y,
+                                                             uint16_t* __restrict__ y16) {
+    __shared__ float wsum[4];
+    const int64_t t = blockIdx.x;
+    const int head = blockIdx.y, col = threadIdx.x;
+    const size_t at = (size_t) t * HV * S + (size_t) head * S + col;
+    const float oc = y[at];
+    float sp = warp_sum(oc * oc);
+    if ((col & 31) == 0) wsum[col >> 5] = sp;
+    __syncthreads();
+    const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
+    const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
+    y[at] = v;
+    y16[at] = hf(v);
+}
+
+// ------------------------------------------------------------------ synthetic inputs
+__device__ __forceinline__ uint32_t mix(uint64_t x) {
+    x ^= x >> 33; x *= 0xff51afd7ed558ccdULL; x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53ULL; x ^= x >> 33;
+    return (uint32_t) x;
+}
+__device__ __forceinline__ float uni(uint64_t key) { return ((mix(key) >> 8) + 0.5f) * (1.0f / 16777216.0f); }
+__device__ __forceinline__ float gauss(uint64_t key) {
+    const float u1 = uni(key * 2 + 1), u2 = uni(key * 2 + 2);
+    return sqrtf(-2.0f * logf(u1)) * cosf(6.2831853f * u2);
+}
+__global__ void fill_gauss(float* x, int64_t n, float scale, uint64_t seed, float offset = 0.0f) {
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
+        x[i] = offset + scale * gauss(seed * 0x9E3779B97F4A7C15ULL + (uint64_t) i);
+}
+// the q and k rows L2-normalized per key head, as gdn_l2_kernel leaves them for the recurrence
+__global__ void l2_rows(float* h, float eps) {
+    float* x = h + (int64_t) blockIdx.y * C + (int64_t) blockIdx.x * S;
+    const float v = x[threadIdx.x];
+    __shared__ float part[4];
+    const float sq = warp_sum(v * v);
+    if ((threadIdx.x & 31) == 0) part[threadIdx.x >> 5] = sq;
+    __syncthreads();
+    x[threadIdx.x] = v * rsqrtf(part[0] + part[1] + part[2] + part[3] + eps);
+}
+// gate = log of the decay (mostly near 1, some strong), beta in (0, 1)
+__global__ void fill_gates(float* gate, float* beta, int64_t n, uint64_t seed) {
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) {
+        const float u = uni(seed * 0x9E3779B97F4A7C15ULL + 7 * (uint64_t) i);
+        gate[i] = -0.02f - 0.8f * u * u * u;
+        beta[i] = 1.0f / (1.0f + expf(-gauss(seed * 0xD1B54A32D192ED03ULL + (uint64_t) i)));
+    }
+}
+__global__ void count_diff(const uint32_t* a, const uint32_t* b, int64_t n, unsigned long long* out) {
+    unsigned long long d = 0;
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
+        d += a[i] != b[i];
+    if (d) atomicAdd(out, d);
+}
+
+// ------------------------------------------------------------------ holding SMs (fewer_sms)
+__device__ __forceinline__ uint64_t gtime() {
+    uint64_t t;
+    asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
+    return t;
+}
+__device__ __forceinline__ int sm_id() {
+    unsigned id;
+    asm volatile("mov.u32 %0, %%smid;" : "=r"(id));
+    return (int) id;
+}
+__device__ __forceinline__ void nap(unsigned ns) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+    __nanosleep(ns);
+#else
+    (void) ns;
+#endif
+}
+// limits that a run that works never reaches (a launch here takes milliseconds); they keep a run that does not work
+// from waiting forever, and the host drops its time
+constexpr uint64_t kHoldLimitNs = 1000000000ull, kWaitLimitNs = 500000000ull;
+// One block per SM: it asks for all the shared memory a block may have (launched with that much dynamic shared
+// memory), so no block of another kernel fits beside it.  It notes its SM and sleeps until release says gen.
+__global__ void hold_sm(unsigned long long* sitting, const volatile unsigned* release, unsigned gen, int* where,
+                        unsigned* late) {
+    where[blockIdx.x] = sm_id();
+    __threadfence();
+    atomicAdd(sitting, 1ull);
+    const uint64_t t0 = gtime();
+    while (*release != gen) {
+        if (gtime() - t0 > kHoldLimitNs) { atomicAdd(late, 1u); return; }
+        nap(20000);
+    }
+}
+// on the timed launch's stream, before it: returns once all holding blocks so far sit on their SMs
+__global__ void wait_sitting(const volatile unsigned long long* sitting, unsigned long long all, unsigned* late) {
+    const uint64_t t0 = gtime();
+    while (*sitting < all) {
+        if (gtime() - t0 > kWaitLimitNs) { atomicAdd(late, 1u); return; }
+        nap(1000);
+    }
+}
+__global__ void release_sms(volatile unsigned* release, unsigned gen) { *release = gen; }
+// Where the blocks of a grid land: launched like the recurrence it stands for (the same grid and block size, and as
+// much dynamic shared memory as lets as many blocks share an SM), every block notes its SM and stays until all of them
+// have landed, so they all sit at once as the recurrence's blocks do for the whole chunk.
+__global__ void land(int* where, unsigned* landed, unsigned* apart) {
+    extern __shared__ unsigned char land_smem[];   // only its size matters
+    if (threadIdx.x != 0 || threadIdx.y != 0) return;
+    where[blockIdx.x] = sm_id();
+    atomicAdd(landed, 1u);
+    const uint64_t t0 = gtime();
+    while (*(volatile unsigned*) landed < gridDim.x) {
+        if (gtime() - t0 > kWaitLimitNs) { atomicAdd(apart, 1u); return; }   // they did not all fit at once
+        nap(1000);
+    }
+    (void) land_smem;
+}
+
+template <typename T> T* dalloc(size_t n) {
+    T* p = nullptr;
+    ck(cudaMalloc((void**) &p, n * sizeof(T) + 16), "cudaMalloc");
+    return p;
+}
+unsigned long long diff_words(const void* a, const void* b, size_t bytes) {
+    static unsigned long long* d = dalloc<unsigned long long>(1);
+    ck(cudaMemset(d, 0, sizeof(*d)), "memset");
+    count_diff<<<1024, 256>>>((const uint32_t*) a, (const uint32_t*) b, (int64_t) (bytes / 4), d);
+    unsigned long long h = 0;
+    ck(cudaMemcpy(&h, d, sizeof(h), cudaMemcpyDeviceToHost), "diff");
+    return h;
+}
+
+struct Inputs {
+    int64_t T = 0;
+    float *h = nullptr, *gate = nullptr, *beta = nullptr, *z = nullptr, *gamma = nullptr, *state0 = nullptr;
+    explicit Inputs(int64_t t, uint64_t seed) : T(t) {
+        h = dalloc<float>((size_t) T * C);
+        gate = dalloc<float>((size_t) T * HV);
+        beta = dalloc<float>((size_t) T * HV);
+        z = dalloc<float>((size_t) T * HV * S);
+        gamma = dalloc<float>(S);
+        state0 = dalloc<float>((size_t) S * HV * S);
+        fill_gauss<<<4096, 256>>>(h, T * C, 1.0f, seed);
+        l2_rows<<<dim3(2 * HK, (unsigned) T), S>>>(h, 1e-6f);
+        fill_gates<<<1024, 256>>>(gate, beta, T * HV, seed + 1);
+        fill_gauss<<<4096, 256>>>(z, T * HV * S, 1.0f, seed + 2);
+        fill_gauss<<<1, 128>>>(gamma, S, 0.1f, seed + 3, 1.0f);
+        fill_gauss<<<1024, 256>>>(state0, (int64_t) S * HV * S, 0.05f, seed + 4);
+        ck(cudaDeviceSynchronize(), "inputs");
+    }
+    Inputs(const Inputs&) = delete;
+    Inputs& operator=(const Inputs&) = delete;
+    ~Inputs() { cudaFree(h); cudaFree(gate); cudaFree(beta); cudaFree(z); cudaFree(gamma); cudaFree(state0); }
+};
+
+enum Rec { kBefore, kKeyHead, kRecCount };
+void run_rec(Rec v, float* state, const Inputs& in, float* oc, cudaStream_t s = 0) {
+    if (v == kBefore)
+        gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, s>>>(state, in.h, in.gate, in.beta, oc, in.T);
+    else
+        gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG), 0, s>>>(state, in.h, in.gate, in.beta, oc, in.T);
+}
+
+int check_bits(int64_t T, uint64_t seed) {
+    Inputs in(T, seed);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state[kRecCount];
+    float* oc[kRecCount];
+    for (int v = 0; v < kRecCount; ++v) {
+        state[v] = dalloc<float>(st);
+        oc[v] = dalloc<float>(oc_n);
+        ck(cudaMemcpy(state[v], in.state0, st * 4, cudaMemcpyDeviceToDevice), "state copy");
+        run_rec((Rec) v, state[v], in, oc[v]);
+    }
+    ck(cudaDeviceSynchronize(), "recurrence");
+    int fails = 0;
+    const unsigned long long d_oc = diff_words(oc[kBefore], oc[kKeyHead], oc_n * 4),
+                             d_st = diff_words(state[kBefore], state[kKeyHead], st * 4);
+    if (d_oc || d_st) {
+        std::printf("FAIL T=%lld recurrence: %llu of %zu outputs and %llu of %zu state values differ\n", (long long) T,
+                    d_oc, oc_n, d_st, st);
+        ++fails;
+    }
+    // the output norm: before (in place, with the FP32 store) against now (the FP16 store only), on the same output
+    uint16_t* y16a = dalloc<uint16_t>(oc_n);
+    uint16_t* y16b = dalloc<uint16_t>(oc_n);
+    float* yb = dalloc<float>(oc_n);
+    ck(cudaMemcpy(yb, oc[kBefore], oc_n * 4, cudaMemcpyDeviceToDevice), "y copy");
+    gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, yb, y16b);
+    gdn_out_norm_old_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc[kBefore], y16a);
+    ck(cudaDeviceSynchronize(), "norm");
+    const unsigned long long d16 = diff_words(y16a, y16b, oc_n * 2);
+    if (d16) {
+        std::printf("FAIL T=%lld output norm: %llu of %zu FP16 pairs differ\n", (long long) T, d16, oc_n / 2);
+        ++fails;
+    }
+    std::printf("T=%-6lld %s\n", (long long) T, fails ? "DIFFERENT" : "the same bits (output, state, FP16 output)");
+    for (int v = 0; v < kRecCount; ++v) { cudaFree(state[v]); cudaFree(oc[v]); }
+    cudaFree(y16a); cudaFree(y16b); cudaFree(yb);
+    return fails;
+}
+
+float median(std::vector<float> v) {
+    std::sort(v.begin(), v.end());
+    return v.empty() ? 0.0f : v[v.size() / 2];
+}
+
+void bench(int64_t T) {
+    Inputs in(T, 0xB3);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state = dalloc<float>(st);
+    float* oc = dalloc<float>(oc_n);
+    uint16_t* y16 = dalloc<uint16_t>(oc_n);
+    cudaEvent_t e0, e1;
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    auto timed = [&](auto&& launch) {
+        ck(cudaMemcpy(state, in.state0, st * 4, cudaMemcpyDeviceToDevice), "state");
+        cudaEventRecord(e0);
+        launch();
+        cudaEventRecord(e1);
+        ck(cudaEventSynchronize(e1), "timed");
+        float ms = 0.0f;
+        cudaEventElapsedTime(&ms, e0, e1);
+        return ms;
+    };
+    const int warm = 3, rounds = T >= 16384 ? 10 : 20;
+    std::vector<float> t_rec[kRecCount], t_norm[2];
+    for (int i = 0; i < warm + rounds; ++i) {
+        for (int j = 0; j < kRecCount; ++j) {
+            const Rec v = (Rec) ((i + j) % kRecCount);   // a different variant first in each round
+            const float ms = timed([&] { run_rec(v, state, in, oc); });
+            if (i >= warm) t_rec[v].push_back(ms);
+        }
+    }
+    for (int i = 0; i < warm + rounds; ++i) {
+        for (int j = 0; j < 2; ++j) {
+            const int v = (i + j) % 2;
+            const float ms = timed([&] {
+                if (v == 0) gdn_out_norm_old_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc, y16);
+                else gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc, y16);
+            });
+            if (i >= warm) t_norm[v].push_back(ms);
+        }
+    }
+    const float r0 = median(t_rec[kBefore]), r1 = median(t_rec[kKeyHead]), n0 = median(t_norm[0]), n1 = median(t_norm[1]);
+    std::printf("  T = %lld tokens, one layer (48 value heads), medians of %d alternating runs:\n", (long long) T, rounds);
+    std::printf("    recurrence, before (gdn_rec_cols_pipe_kernel)  %8.2f ms  (%.3f us per token)\n", r0,
+                1000.0f * r0 / (float) T);
+    std::printf("    recurrence, gdn_rec_kh_kernel                  %8.2f ms  (%.3f us per token)  %.2fx\n", r1,
+                1000.0f * r1 / (float) T, r0 / r1);
+    std::printf("    output norm, before (FP32 and FP16 store)      %8.2f ms\n", n0);
+    std::printf("    output norm, FP16 store only                   %8.2f ms  %.2fx\n", n1, n0 / n1);
+    std::printf("    36 DeltaNet layers per %lld-token block: recurrence + norm %.0f ms -> %.0f ms (-%.0f ms)\n",
+                (long long) T, 36.0f * (r0 + n0), 36.0f * (r1 + n1), 36.0f * (r0 + n0 - r1 - n1));
+    cudaEventDestroy(e0);
+    cudaEventDestroy(e1);
+    cudaFree(state); cudaFree(oc); cudaFree(y16);
+}
+
+// The two kernels at a given number of blocks per SM, as on cards with more or fewer SMs than this one: the kernel
+// before puts ceil(192 / SMs) of its blocks on an SM, gdn_rec_kh_kernel ceil(64 / SMs), and the busiest SM sets the
+// pace (every block walks the whole chunk).  b blocks per SM here: b rounds of one block per SM, each round one launch
+// of SMs blocks (the first heads; split in launches of at most the engine's grid), every launch on its own stream with
+// its own state and output, all started together, so the block scheduler puts one block of each round on every SM.
+// The kernels are the ones above, unchanged.
+void scale(int64_t T) {
+    int dev = 0, sms = 0, per_sm[kRecCount] = {};
+    ck(cudaGetDevice(&dev), "device");
+    ck(cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev), "SMs");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kBefore], gdn_rec_cols_pipe_kernel, CB * RG, 0), "occupancy");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kKeyHead], gdn_rec_kh_kernel, CB * RG, 0), "occupancy");
+    constexpr int kMaxB = 6, kMaxLaunches = 6;
+    const int full[kRecCount] = {HV * NCB, HK * NCB};   // the grids as the engine launches them
+    auto parts = [&](int v) { return (sms + full[v] - 1) / full[v]; };   // launches per round
+    Inputs in(T, 0xC7);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state[kMaxLaunches];
+    float* oc[kMaxLaunches];
+    cudaStream_t stream[kMaxLaunches];
+    cudaEvent_t e0, e1, done[kMaxLaunches];
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    for (int c = 0; c < kMaxLaunches; ++c) {
+        state[c] = dalloc<float>(st);
+        oc[c] = dalloc<float>(oc_n);
+        ck(cudaStreamCreateWithFlags(&stream[c], cudaStreamNonBlocking), "stream");
+        ck(cudaEventCreate(&done[c]), "event");
+    }
+    struct Case { int v, b; };
+    std::vector<Case> cases;
+    for (int v = 0; v < kRecCount; ++v)
+        for (int b = 1; b <= std::min(kMaxB, per_sm[v]); ++b)
+            if (b * parts(v) <= kMaxLaunches) cases.push_back({v, b});
+    auto run = [&](const Case& cs) {
+        for (int c = 0; c < kMaxLaunches; ++c) ck(cudaMemcpy(state[c], in.state0, st * 4, cudaMemcpyDeviceToDevice), "state");
+        ck(cudaDeviceSynchronize(), "sync");
+        const int np = parts(cs.v), n = cs.b * np;
+        cudaEventRecord(e0, stream[0]);
+        for (int c = 0; c < n; ++c) {
+            const int k = c % np, grid = sms / np + (k < sms % np ? 1 : 0);   // round c / np, part k
+            if (c > 0) cudaStreamWaitEvent(stream[c], e0, 0);
+            if (cs.v == kBefore)
+                gdn_rec_cols_pipe_kernel<<<grid, dim3(CB, RG), 0, stream[c]>>>(state[c], in.h, in.gate, in.beta, oc[c], T);
+            else
+                gdn_rec_kh_kernel<<<grid, dim3(CB, RG), 0, stream[c]>>>(state[c], in.h, in.gate, in.beta, oc[c], T);
+            cudaEventRecord(done[c], stream[c]);
+        }
+        for (int c = 1; c < n; ++c) cudaStreamWaitEvent(stream[0], done[c], 0);
+        cudaEventRecord(e1, stream[0]);
+        ck(cudaEventSynchronize(e1), "scale");
+        float ms = 0.0f;
+        cudaEventElapsedTime(&ms, e0, e1);
+        return ms;
+    };
+    const int warm = 2, rounds = 9;
+    std::vector<std::vector<float>> t(cases.size());
+    for (int i = 0; i < warm + rounds; ++i)
+        for (size_t j = 0; j < cases.size(); ++j) {
+            const size_t k = (i + j) % cases.size();   // a different case first in each round
+            const float ms = run(cases[k]);
+            if (i >= warm) t[k].push_back(ms);
+        }
+    auto us = [&](int v, int b) {   // per token, or < 0 if not measured
+        for (size_t k = 0; k < cases.size(); ++k)
+            if (cases[k].v == v && cases[k].b == b) return 1000.0f * median(t[k]) / (float) T;
+        return -1.0f;
+    };
+    std::printf("  per token at a given number of blocks per SM (%lld tokens, medians of %d alternating runs; this card has "
+                "%d SMs):\n", (long long) T, rounds, sms);
+    std::printf("    blocks per SM   before      gdn_rec_kh_kernel\n");
+    for (int b = 1; b <= kMaxB; ++b) {
+        const float u0 = us(kBefore, b), u1 = us(kKeyHead, b);
+        if (u0 < 0 && u1 < 0) continue;
+        char a0[32] = "-", a1[32] = "-";
+        if (u0 >= 0) std::snprintf(a0, sizeof a0, "%.3f us", u0);
+        if (u1 >= 0) std::snprintf(a1, sizeof a1, "%.3f us", u1);
+        std::printf("    %8d        %-11s %s\n", b, a0, a1);
+    }
+    std::printf("  so on a card with N SMs (the kernel before puts ceil(192 / N) blocks on an SM, gdn_rec_kh_kernel "
+                "ceil(64 / N), at most %d):\n", per_sm[kKeyHead]);
+    const int bands[][2] = {{192, 0}, {96, 191}, {64, 95}, {48, 63}, {39, 47}, {32, 38}};
+    for (const auto& bd : bands) {
+        const int nsm = bd[0], b0 = (192 + nsm - 1) / nsm, b1 = (64 + nsm - 1) / nsm;
+        const float u0 = us(kBefore, b0), u1 = us(kKeyHead, b1);
+        char range[32];
+        if (bd[1]) std::snprintf(range, sizeof range, "%d-%d SMs", bd[0], bd[1]);
+        else std::snprintf(range, sizeof range, "%d+ SMs", bd[0]);
+        if (u0 < 0 || u1 < 0)
+            std::printf("    %-12s before %d / new %d per SM: not measured on this card\n", range, b0, b1);
+        else
+            std::printf("    %-12s before %d / new %d per SM: %.3f -> %.3f us per token, %.2fx\n", range, b0, b1, u0, u1,
+                        u0 / u1);
+    }
+    for (int c = 0; c < kMaxLaunches; ++c) {
+        cudaFree(state[c]);
+        cudaFree(oc[c]);
+        cudaStreamDestroy(stream[c]);
+        cudaEventDestroy(done[c]);
+    }
+    cudaEventDestroy(e0);
+    cudaEventDestroy(e1);
+}
+
+// The engine's grids on N of this card's SMs.  scale() puts as many blocks on every SM as the busiest SM of a card with
+// N SMs gets; on that card only some SMs get that many (192 blocks on 80 SMs: 3 on 32 SMs, 2 on the other 48), the
+// rest of the card has less to do, and the estimate came out 4-9 % above the real launch on two cards.  Here the
+// recurrences run as the engine launches them, while hold_sm keeps all but N SMs: its blocks go out first, a small
+// kernel on the timed launch's stream waits until they all sit, and one after the timed launch lets them go.  land()
+// shows for every N where the two grids land: none on a held SM, and how many blocks share the busiest SM.  The
+// recurrences are the ones above, unchanged; L2, memory and clock stay this card's.
+void fewer_sms(int64_t T) {
+    int dev = 0, sms = 0, optin = 0, smem_sm = 0, reserved = 0, hold_per_sm = 0, per_sm[kRecCount] = {};
+    ck(cudaGetDevice(&dev), "device");
+    ck(cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev), "SMs");
+    ck(cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev), "shared memory");
+    ck(cudaDeviceGetAttribute(&smem_sm, cudaDevAttrMaxSharedMemoryPerMultiprocessor, dev), "shared memory");
+    ck(cudaDeviceGetAttribute(&reserved, cudaDevAttrReservedSharedMemoryPerBlock, dev), "shared memory");
+    for (const void* f : {(const void*) hold_sm, (const void*) land}) {
+        ck(cudaFuncSetAttribute(f, cudaFuncAttributeMaxDynamicSharedMemorySize, optin), "shared memory");
+        ck(cudaFuncSetAttribute(f, cudaFuncAttributePreferredSharedMemoryCarveout, (int) cudaSharedmemCarveoutMaxShared),
+           "carveout");
+    }
+    // Loaded now rather than at a first launch: with lazy loading (CUDA 12's default) a kernel's first launch may wait
+    // until the device is idle, and so for hold_sm, which waits for release_sms.
+    cudaFuncAttributes fa[kRecCount], fx;
+    ck(cudaFuncGetAttributes(&fa[kBefore], gdn_rec_cols_pipe_kernel), "attributes");
+    ck(cudaFuncGetAttributes(&fa[kKeyHead], gdn_rec_kh_kernel), "attributes");
+    for (const void* f : {(const void*) hold_sm, (const void*) wait_sitting, (const void*) release_sms, (const void*) land})
+        ck(cudaFuncGetAttributes(&fx, f), "attributes");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&hold_per_sm, hold_sm, 1, optin), "occupancy");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kBefore], gdn_rec_cols_pipe_kernel, CB * RG, 0), "occupancy");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kKeyHead], gdn_rec_kh_kernel, CB * RG, 0), "occupancy");
+    const int grid[kRecCount] = {HV * NCB, HK * NCB};   // as the engine launches them
+    std::printf("  the engine's grids on N of this card's %d SMs (%lld tokens; the other SMs held by a sleeping kernel, see "
+                "fewer_sms):\n", sms, (long long) T);
+    const int left = smem_sm - optin - reserved;   // shared memory a holding block leaves on its SM
+    if (hold_per_sm != 1 || (int) fa[kBefore].sharedSizeBytes + reserved <= left ||
+        (int) fa[kKeyHead].sharedSizeBytes + reserved <= left) {
+        std::printf("    not on this card: a block of a recurrence would fit beside a holding block (%d bytes left)\n",
+                    left);
+        return;
+    }
+    // land(): as much dynamic shared memory as still lets as many of its blocks share an SM as of the recurrence
+    int land_smem[kRecCount];
+    for (int v = 0; v < kRecCount; ++v) {
+        int lo = 0, hi = optin;
+        while (lo < hi) {
+            const int mid = (lo + hi + 1) / 2;
+            int nb = 0;
+            ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&nb, land, CB * RG, mid), "occupancy");
+            if (nb >= per_sm[v]) lo = mid;
+            else hi = mid - 1;
+        }
+        land_smem[v] = lo;
+    }
+    // N: all SMs, and below that the SM counts of cards and the band edges, as long as both grids fit at once
+    std::vector<int> ns = {sms};
+    for (const int n : {170, 144, 132, 128, 112, 108, 96, 95, 84, 82, 80, 76, 72, 70, 66, 64, 63, 60, 56, 52, 48, 47, 46, 44,
+                        40, 39, 38, 36, 34, 32})
+        if (n < sms && (int64_t) n * per_sm[kBefore] >= grid[kBefore] && (int64_t) n * per_sm[kKeyHead] >= grid[kKeyHead])
+            ns.push_back(n);
+
+    Inputs in(T, 0xD5);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state = dalloc<float>(st);
+    float* oc = dalloc<float>(oc_n);
+    unsigned long long* sitting = dalloc<unsigned long long>(1);
+    unsigned* flags = dalloc<unsigned>(4);
+    unsigned *release = flags, *late = flags + 1, *landed = flags + 2, *apart = flags + 3;
+    int* where = dalloc<int>((size_t) sms + grid[kBefore]);   // the holding blocks' SMs, then land()'s
+    ck(cudaMemset(sitting, 0, sizeof(*sitting)), "memset");
+    ck(cudaMemset(flags, 0, 4 * sizeof(unsigned)), "memset");
+    cudaStream_t s_hold, s_run;
+    ck(cudaStreamCreateWithFlags(&s_hold, cudaStreamNonBlocking), "stream");
+    ck(cudaStreamCreateWithFlags(&s_run, cudaStreamNonBlocking), "stream");
+    cudaEvent_t e0, e1;
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    unsigned long long all = 0;   // holding blocks launched so far: sitting counts every one that sat down
+    unsigned gen = 0;
+    auto hold = [&](int n) {   // all but n SMs, before the next launch on s_run
+        if (n >= sms) return;
+        ++gen;
+        all += (unsigned long long) (sms - n);
+        hold_sm<<<sms - n, 1, optin, s_hold>>>(sitting, release, gen, where, late);
+        ck(cudaGetLastError(), "hold_sm");
+        const cudaError_t q = cudaStreamQuery(s_hold);   // hands the launch to the GPU now (Windows collects launches)
+        if (q != cudaErrorNotReady) ck(q, "hold_sm");
+        (void) cudaGetLastError();   // not ready is no error
+        wait_sitting<<<1, 1, 0, s_run>>>(sitting, all, late);
+    };
+    auto let_go = [&](int n) {
+        if (n < sms) release_sms<<<1, 1, 0, s_run>>>(release, gen);
+    };
+    auto settle = [&](const char* what) {
+        ck(cudaStreamSynchronize(s_run), what);
+        ck(cudaStreamSynchronize(s_hold), what);
+    };
+    auto count = [&](const unsigned* c) {   // after settle()
+        unsigned h = 0;
+        ck(cudaMemcpy(&h, c, sizeof(h), cudaMemcpyDeviceToHost), "count");
+        return h;
+    };
+    auto done = [&] {
+        cudaEventDestroy(e0);
+        cudaEventDestroy(e1);
+        cudaStreamDestroy(s_hold);
+        cudaStreamDestroy(s_run);
+        cudaFree(state); cudaFree(oc); cudaFree(sitting); cudaFree(flags); cudaFree(where);
+    };
+
+    // where the grids land, once per N
+    struct Landing { bool held = true; int busiest[kRecCount] = {}, on[kRecCount] = {}; };
+    std::vector<Landing> lands(ns.size());
+    for (size_t i = 0; i < ns.size(); ++i) {
+        const int n = ns[i];
+        for (int v = 0; v < kRecCount; ++v) {
+            bool sat = false;
+            unsigned apart0 = 0;
+            for (int attempt = 0; attempt < 2 && !sat; ++attempt) {
+                const unsigned late0 = count(late);
+                apart0 = count(apart);
+                ck(cudaMemsetAsync(landed, 0, sizeof(unsigned), s_run), "memset");
+                hold(n);
+                land<<<grid[v], dim3(CB, RG), land_smem[v], s_run>>>(where + sms, landed, apart);
+                ck(cudaGetLastError(), "land");
+                let_go(n);
+                settle("land");
+                sat = count(late) == late0;
+            }
+            if (!sat) {   // twice: the holding kernel and the launches beside it do not run at the same time here
+                std::printf("    not on this card: the other SMs could not be held (the kernel holding them and the "
+                            "launches beside it did not run at the same time)\n");
+                done();
+                return;
+            }
+            std::vector<int> w((size_t) sms + grid[v]);
+            ck(cudaMemcpy(w.data(), where, w.size() * sizeof(int), cudaMemcpyDeviceToHost), "where");
+            std::map<int, int> held, blocks;
+            for (int b = 0; b < (n < sms ? sms - n : 0); ++b) ++held[w[b]];
+            for (int b = 0; b < grid[v]; ++b) ++blocks[w[sms + b]];
+            bool ok = count(apart) == apart0 && (int) held.size() == (n < sms ? sms - n : 0) && (int) blocks.size() <= n;
+            int most = 0, on = 0;
+            for (const auto& sb : blocks) {
+                if (held.count(sb.first)) ok = false;
+                if (sb.second > most) { most = sb.second; on = 0; }
+                if (sb.second == most) ++on;
+            }
+            lands[i].held = lands[i].held && ok;
+            lands[i].busiest[v] = most;
+            lands[i].on[v] = on;
+        }
+    }
+
+    // the timings: every N and recurrence once per round, a different one first in each round
+    struct Case { size_t i; Rec v; };
+    std::vector<Case> cases;
+    for (size_t i = 0; i < ns.size(); ++i)
+        if (lands[i].held)
+            for (int v = 0; v < kRecCount; ++v) cases.push_back({i, (Rec) v});
+    auto run = [&](const Case& cs) {   // ms, or < 0 if the SMs were not held for the whole launch
+        const int n = ns[cs.i];
+        const unsigned late0 = count(late);
+        ck(cudaMemcpyAsync(state, in.state0, st * 4, cudaMemcpyDeviceToDevice, s_run), "state");
+        hold(n);
+        ck(cudaEventRecord(e0, s_run), "event");
+        run_rec(cs.v, state, in, oc, s_run);
+        ck(cudaEventRecord(e1, s_run), "event");
+        let_go(n);
+        settle("fewer SMs");
+        if (count(late) != late0) return -1.0f;
+        float ms = 0.0f;
+        ck(cudaEventElapsedTime(&ms, e0, e1), "event");
+        return ms;
+    };
+    const int warm = 2, rounds = 15, max_dropped = 5;
+    std::vector<std::vector<float>> t(cases.size());
+    int dropped = 0;
+    for (int r = 0; r < warm + rounds && dropped <= max_dropped; ++r)
+        for (size_t j = 0; j < cases.size() && dropped <= max_dropped; ++j) {
+            const size_t k = (r + j) % cases.size();
+            const float ms = run(cases[k]);
+            if (ms < 0.0f) ++dropped;
+            else if (r >= warm) t[k].push_back(ms);
+        }
+    if (dropped > max_dropped) {   // each such run waits for a limit of a second or so
+        std::printf("    stopped: in %d runs the other SMs were not held for the whole launch\n", dropped);
+        done();
+        return;
+    }
+    auto us = [&](size_t i, int v) {   // per token, or < 0 if not measured
+        for (size_t k = 0; k < cases.size(); ++k)
+            if (cases[k].i == i && cases[k].v == v && !t[k].empty()) return 1000.0f * median(t[k]) / (float) T;
+        return -1.0f;
+    };
+    auto blocks_on = [](char* out, size_t size, int most, int on) {
+        std::snprintf(out, size, "%d block%s on %d SM%s", most, most == 1 ? "" : "s", on, on == 1 ? "" : "s");
+    };
+    std::printf("  (medians of %d alternating runs; L2, memory and clock stay this card's)\n", rounds);
+    std::printf("      N   before: busiest SM   new: busiest SM      us per token, before -> new\n");
+    for (size_t i = 0; i < ns.size(); ++i) {
+        if (!lands[i].held) {
+            std::printf("    %3d   the other SMs were not held (a block landed on one, or a wait ran out)\n", ns[i]);
+            continue;
+        }
+        char b0[40], b1[40];
+        blocks_on(b0, sizeof b0, lands[i].busiest[kBefore], lands[i].on[kBefore]);
+        blocks_on(b1, sizeof b1, lands[i].busiest[kKeyHead], lands[i].on[kKeyHead]);
+        const float u0 = us(i, kBefore), u1 = us(i, kKeyHead);
+        if (u0 < 0.0f || u1 < 0.0f) std::printf("    %3d   %-20s %-20s not measured\n", ns[i], b0, b1);
+        else std::printf("    %3d   %-20s %-20s %.3f -> %.3f   %.2fx\n", ns[i], b0, b1, u0, u1, u0 / u1);
+    }
+    if (dropped) std::printf("    (%d runs dropped: the other SMs were not held for the whole launch)\n", dropped);
+    std::printf("  so on a card with N SMs, as measured here (the kernel before puts ceil(192 / N) blocks on the busiest SM, "
+                "gdn_rec_kh_kernel ceil(64 / N)):\n");
+    const int bands[][2] = {{96, 191}, {64, 95}, {48, 63}, {39, 47}, {32, 38}};
+    for (const auto& bd : bands) {
+        const int b0 = (192 + bd[0] - 1) / bd[0], b1 = (64 + bd[0] - 1) / bd[0];
+        float lo = 0.0f, hi = 0.0f;
+        int cnt = 0, nmin = 0, nmax = 0;
+        for (size_t i = 0; i < ns.size(); ++i) {
+            const float u0 = us(i, kBefore), u1 = us(i, kKeyHead);
+            if (ns[i] < bd[0] || ns[i] > bd[1] || u0 < 0.0f || u1 < 0.0f) continue;
+            const float x = u0 / u1;
+            if (cnt == 0 || x < lo) lo = x;
+            if (cnt == 0 || x > hi) hi = x;
+            if (cnt == 0 || ns[i] < nmin) nmin = ns[i];
+            if (cnt == 0 || ns[i] > nmax) nmax = ns[i];
+            ++cnt;
+        }
+        char range[32];
+        std::snprintf(range, sizeof range, "%d-%d SMs", bd[0], bd[1]);
+        if (cnt > 0)
+            std::printf("    %-12s before %d / new %d per SM: %.2fx to %.2fx (%d to %d SMs)\n", range, b0, b1, lo, hi, nmin,
+                        nmax);
+        else if (bd[0] > sms)
+            std::printf("    %-12s before %d / new %d per SM: more SMs than this card has (see the estimate above)\n", range,
+                        b0, b1);
+        else
+            std::printf("    %-12s before %d / new %d per SM: not measured\n", range, b0, b1);
+    }
+    done();
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    bool do_bench = false;
+    for (int i = 1; i < argc; ++i)
+        if (std::strcmp(argv[i], "--bench") == 0) do_bench = true;
+    cudaDeviceProp p{};
+    ck(cudaGetDeviceProperties(&p, 0), "device");
+    std::printf("gdn_rec_parity on %s (sm_%d%d)\n", p.name, p.major, p.minor);
+    int fails = 0;
+    const int64_t Ts[] = {1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 33, 257, 1000, 4099};
+    uint64_t seed = 11;
+    for (const int64_t T : Ts) fails += check_bits(T, seed++);
+    if (do_bench) {
+        std::printf("--bench: the prompt path's DeltaNet recurrence\n");
+        bench(8192);
+        bench(32768);
+        scale(8192);
+        fewer_sms(8192);
+    }
+    std::printf("gdn_rec_parity: %d failures\n", fails);
+    return fails ? 1 : 0;
+}

src/prefill/ggml_cuda_host.cu

@@ -80,17 +80,30 @@ const ggml_cuda_device_info & ggml_cuda_info() {
             d.smpbo = prop.sharedMemPerBlock;
             d.integrated = false;
             d.supports_cooperative_launch = false;
-#else
-            d.cc = 100 * strata::cc_major_of(prop.major) + 10 * strata::cc_minor_of(prop.minor);   // STRATA_EMULATE_CC
-            d.smpbo = strata::smem_optin_of((int) prop.sharedMemPerBlockOptin);
-            d.integrated = prop.integrated != 0;
-            d.supports_cooperative_launch = prop.cooperativeLaunch != 0;
-#endif
             d.nsm = prop.multiProcessorCount;
             d.smpb = prop.sharedMemPerBlock;
+            d.warp_size = prop.warpSize;
+#else
+            // #542: the fields MMQ's choices read come from cudaDeviceGetAttribute, which is ABI-stable: a
+            // cudaDeviceProp filled by an older libcudart than the headers (a CUDA 13 build that links a CUDA 12
+            // libcudart.so) is shifted - sharedMemPerBlockOptin read 1 there, and every i-quant lost MMQ (#420's
+            // fits()).  In a matched build both give the same values.
+            auto attr = [id](cudaDeviceAttr a) {
+                int v = 0;
+                CUDA_CHECK(cudaDeviceGetAttribute(&v, a, id));
+                return v;
+            };
+            d.cc = 100 * strata::cc_major_of(attr(cudaDevAttrComputeCapabilityMajor)) +
+                   10 * strata::cc_minor_of(attr(cudaDevAttrComputeCapabilityMinor));   // STRATA_EMULATE_CC
+            d.smpbo = strata::smem_optin_of(attr(cudaDevAttrMaxSharedMemoryPerBlockOptin));
+            d.integrated = attr(cudaDevAttrIntegrated) != 0;
+            d.supports_cooperative_launch = attr(cudaDevAttrCooperativeLaunch) != 0;
+            d.nsm = attr(cudaDevAttrMultiProcessorCount);
+            d.smpb = (size_t) attr(cudaDevAttrMaxSharedMemoryPerBlock);
+            d.warp_size = attr(cudaDevAttrWarpSize);
+#endif
             d.vmm = false;
             d.total_vram = prop.totalGlobalMem;
-            d.warp_size = prop.warpSize;
             d.physical_device = id;
             d.physical_share_count = 1;
             d.virtual_index = 0;

src/prefill/kernels.cu

@@ -433,8 +433,172 @@ __global__ void __launch_bounds__(CB * RG) gdn_rec_cols_pipe_kernel(float* __res
 #pragma unroll
     for (int r = 0; r < RPG; ++r) base[r * rs] = s[r];
 }
+#if !defined(__HIPCC__)
+// The recurrence with one thread for the three value heads that share a key head (head % HK): column c of heads
+// qh, qh + 16 and qh + 32, row group rg.  gdn_rec_cols_pipe_kernel spends its time in shared memory, not in
+// arithmetic: every thread of a warp needs the same 32 q and k values per token (the k twice), and a warp receives one
+// such broadcast value per clock however wide the load.  Here every q/k value a thread loads feeds three heads, and a
+// token's k row goes into registers once for both of its uses.  The inputs come in blocks of GDN_TB tokens, copied to
+// shared memory by cp.async while the block before computes (one token ahead is shorter than a load from L2 takes),
+// and the two cross-row-group sums have their own arrays, so a token needs 2 __syncthreads instead of 5: the second
+// one of a token orders every read of rkv before the next token's writes, the next token's first one every read of ro
+// before the writes after it.  64 blocks instead of 192.  Per value head and column the same arithmetic in the same
+// order: the same bits (src/prefill/gdn_rec_parity.cu checks them and times the variants: 1.41x on a 4080 Super).
+// sm_80+ cards that hold its 64 blocks at once (gdn_keyhead_ok); STRATA_GDN_KEYHEAD=0: gdn_rec_cols_pipe_kernel.
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
+#define STRATA_GDN_CP_ASYNC 0   // Turing builds: plain copies (never launched there, see gdn_keyhead_ok)
+#else
+#define STRATA_GDN_CP_ASYNC 1
+#endif
+__device__ __forceinline__ void gdn_cp4(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *smem = *gmem;
+#endif
+}
+__device__ __forceinline__ void gdn_cp16(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *reinterpret_cast<float4*>(smem) = *reinterpret_cast<const float4*>(gmem);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_commit() {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.commit_group;\n" ::);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_wait_prev() {   // every group but the newest has landed
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.wait_group 1;\n" ::);
+#endif
+}
+constexpr int GDN_TB = 8, VPK = HV / HK;   // tokens per staged block, value heads per key head
+__global__ void __launch_bounds__(CB * RG) gdn_rec_kh_kernel(float* __restrict__ state, const float* __restrict__ h,
+                                                               const float* __restrict__ gate,
+                                                               const float* __restrict__ beta,
+                                                               float* __restrict__ oc_out, int64_t T) {
+    constexpr int TB = GDN_TB, NT = CB * RG, QKP = S / 4, VP = CB / 4;   // threads, 16-byte pieces of a q/k row, of v
+    __shared__ __align__(16) float sq[2][TB][S];
+    __shared__ __align__(16) float sk[2][TB][S];
+    __shared__ __align__(16) float sv[2][TB][VPK][CB];
+    __shared__ float sg[2][TB][VPK], sb[2][TB][VPK], rkv[VPK][RG][CB], ro[VPK][RG][CB];
+    const int qh = blockIdx.x / NCB, cb = blockIdx.x % NCB;
+    const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
+    float s[VPK][RPG];
+    const size_t rs = (size_t) HV * S;
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        const float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) s[j][r] = base[r * rs];
+    }
+    const int64_t nblk = (T + TB - 1) / TB;
+    auto stage = [&](int64_t k) {   // tokens [k * TB, k * TB + TB) into buffer k & 1
+        const int bb = (int) (k & 1);
+        const int64_t t0 = k * TB;
+        for (int p = tid; p < TB * 2 * QKP; p += NT) {
+            const int i = p / (2 * QKP), w = p % (2 * QKP), isk = w / QKP, jj = (w % QKP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(isk ? &sk[bb][i][jj] : &sq[bb][i][jj], h + (t0 + i) * C + (isk ? HK * S : 0) + qh * S + jj);
+        }
+        for (int p = tid; p < TB * VPK * VP; p += NT) {
+            const int i = p / (VPK * VP), w = p % (VPK * VP), j = w / VP, jj = (w % VP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(&sv[bb][i][j][jj], h + (t0 + i) * C + 2 * HK * S + (qh + j * HK) * S + cb * CB + jj);
+        }
+        for (int p = tid; p < 2 * TB * VPK; p += NT) {
+            const int isb = p / (TB * VPK), w = p % (TB * VPK), i = w / VPK, j = w % VPK;
+            if (t0 + i < T)
+                gdn_cp4(isb ? &sb[bb][i][j] : &sg[bb][i][j], (isb ? beta : gate) + (t0 + i) * HV + qh + j * HK);
+        }
+    };
+    if (nblk > 0) stage(0);
+    gdn_cp_commit();
+    for (int64_t k = 0; k < nblk; ++k) {
+        // buffer (k + 1) & 1 was read by block k - 1, whose last token's second __syncthreads every thread has passed
+        if (k + 1 < nblk) stage(k + 1);
+        gdn_cp_commit();
+        gdn_cp_wait_prev();
+        __syncthreads();
+        const int bb = (int) (k & 1);
+        const int n = (int) ((T - k * TB) < TB ? (T - k * TB) : TB);
+        for (int i = 0; i < n; ++i) {
+            const int64_t t = k * TB + i;
+            float kc[RPG];
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) kc[r] = sk[bb][i][rg * RPG + r];
+            float g[VPK], kv[VPK], delta[VPK], o[VPK];
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) { g[j] = __expf(sg[bb][i][j]); kv[j] = 0.0f; o[j] = 0.0f; }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r)
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) kv[j] = fmaf(s[j][r], kc[r], kv[j]);
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) rkv[j][rg][c] = kv[j];
+            __syncthreads();
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) {
+                const float kv_col = rkv[j][0][c] + rkv[j][1][c] + rkv[j][2][c] + rkv[j][3][c];
+                delta[j] = (sv[bb][i][j][c] - g[j] * kv_col) * sb[bb][i][j];
+            }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) {
+                const float qr = sq[bb][i][rg * RPG + r];
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) {
+                    s[j][r] = fmaf(g[j], s[j][r], kc[r] * delta[j]);
+                    o[j] = fmaf(s[j][r], qr, o[j]);
+                }
+            }
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) ro[j][rg][c] = o[j];
+            __syncthreads();
+            if (rg < VPK)   // row group j writes head j's output
+                oc_out[t * HV * S + (qh + rg * HK) * S + col] =
+                    (ro[rg][0][c] + ro[rg][1][c] + ro[rg][2][c] + ro[rg][3][c]) * rsqrtf((float) S);
+        }
+    }
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) base[r * rs] = s[j][r];
+    }
+}
+// gdn_rec_kh_kernel where it pays: a CUDA card with cp.async (sm_80+) that holds all 64 of its blocks at once (each
+// walks the whole chunk, so blocks left for a second wave would double the time).  The busiest SM sets the pace: from
+// 64 SMs up this kernel has one block per SM, below that two on some SMs, while the kernel before has ceil(192 / SMs).
+// With the engine's grids on fewer SMs (gdn_rec_parity --bench), on a 4080 SUPER / a 3090: 1.40-1.42x / 1.28-1.29x at
+// 64 SMs and more, 1.02-1.04x / 1.03-1.06x at 48 to 63 (two blocks against four: a draw, slower on neither),
+// 1.28-1.31x / 1.28-1.30x at 39 to 47, 1.53-1.57x / 1.54-1.55x at 32 to 38.  Per call, from the current device (a
+// layer split can mix cards).
+bool gdn_keyhead_ok() {
+    static const bool off = [] { const char* v = std::getenv("STRATA_GDN_KEYHEAD"); return v != nullptr && std::atoi(v) == 0; }();
+    if (off) return false;
+    static int known[64] = {};   // per device: 0 not asked yet, 1 yes, 2 no
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+    if (known[dev] == 0) {
+        int major = 0, sms = 0, per_sm = 0;
+        const bool yes = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+                         cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev) == cudaSuccess && major >= 8 &&
+                         cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm, gdn_rec_kh_kernel, CB * RG, 0) ==
+                             cudaSuccess &&
+                         (int64_t) per_sm * sms >= (int64_t) HK * NCB;
+        if (!yes) cudaGetLastError();
+        known[dev] = yes ? 1 : 2;
+    }
+    return known[dev] == 1;
+}
+#endif
+// the output norm over a head's 128 columns, into the FP16 copy the out projection reads (the FP32 output before the
+// norm stays in its scratch buffer)
 __global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
-                                                         float eps, float* __restrict__ y, uint16_t* __restrict__ y16) {
+                                                         float eps, const float* __restrict__ y,
+                                                         uint16_t* __restrict__ y16) {
     __shared__ float wsum[4];
     const int64_t t = blockIdx.x;
     const int head = blockIdx.y, col = threadIdx.x;
@@ -445,7 +609,6 @@ __global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict
     __syncthreads();
     const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
     const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
-    y[at] = v;
     y16[at] = hf(v);
 }
 
@@ -748,6 +911,11 @@ void gdn_recurrence(float* state, const float* h, const float* gate, const float
         gdn_rec_kernel<<<HV, dim3(S, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, z, gamma, eps, y, y16, T);
     } else {
         static const bool pipe = [] { const char* v = std::getenv("STRATA_GDN_PIPELINE"); return v == nullptr || std::atoi(v) != 0; }();
+#if !defined(__HIPCC__)
+        if (pipe && gdn_keyhead_ok())   // the value heads of a key head in one thread (same bits)
+            gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);
+        else
+#endif
         if (pipe)   // the software-pipelined loads (same bits)
             gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);
         else

src/prefill/moe_fused.cu

@@ -0,0 +1,459 @@
+// src/prefill/moe_fused.cu - see include/strata/prefill/moe_fused.hpp (#136: the Q2_0 pack's prompt experts on
+// Strata's own int8 tensor-core kernels, with the grouping on the GPU).
+//
+// The arithmetic.  A Q2_0 block is 64 weights w = d_w * (q - 1), q in 0..3, four codes per byte, weight j of a block
+// in byte j / 4 at bits 2 * (j % 4).  An activation block of 32 values is x = d_x * a with a = round(x / d_x) in
+// -127..127.  Over one 32-value half-block
+//     sum w x = d_w * d_x * (sum q a - sum a),
+// so the int8 product takes the codes as they are (0..3 is a valid s8) and the offset is the activations' own sum,
+// stored with the block.  The dot product (|sum q a| <= 32 * 3 * 127 < 2^22) becomes a float exactly with one
+// integer add: as_float(0x4B400000 + dot) = 1.5 * 2^23 + dot, and the block stores c = -(1.5 * 2^23 + sum a), so
+// as_float(0x4B400000 + dot) + c = dot - sum a with no rounding.  Per output and 64-block that is
+//     acc += d_w * (d_x0 * t0 + d_x1 * t1)        (t = dot - sum a of each half)
+// - two integer adds, two float adds and three multiply-adds.
+//
+// The K order.  mma.sync m16n8k32 takes four consecutive k of a row (A) or column (B) per register.  Four consecutive
+// weights are one code byte; four codes of one register instead come from the same 2-bit field of four consecutive
+// bytes - (word >> 2t) & 0x03030303 holds weights t, t+4, t+8, t+12 of a 16-weight word.  So the kernels use that order
+// for the k of a half-block: mma k index 16h + 4t + j stands for weight (and activation) 16h + t + 4j.  The
+// activations are stored in it (perm32), and a lane's two B registers (h = 0 and 1) are adjacent: one 8-byte load.
+#include "strata/prefill/moe_fused.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <mutex>
+
+namespace strata::prefill::fused {
+namespace {
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "prefill fused experts: %s: %s\n", what, cudaGetErrorString(e));
+        std::exit(1);
+    }
+}
+
+constexpr int AB = 80;                  // activation bytes per 64 values: 64 codes, then {d0, c0, d1, c1}
+constexpr int MAGIC = 0x4B400000;       // the bits of 1.5 * 2^23
+constexpr float MAGICF = 12582912.0f;
+// The Strata Q2_0 blob (moe_mmq.cu strata_q2_kernel, kernels.cu blob_dequant_kernel): gate/up codes [1280][640 B]
+// (rows interleaved: gate of feature f at row 2f, its up at 2f + 1), down codes [2560][160 B], gate/up scales
+// [1280][40] fp16, down scales [2560][10] fp16.
+constexpr size_t O_GU_CODES = 0, O_D_CODES = (size_t) 1280 * 640, O_GU_SC = O_D_CODES + (size_t) 2560 * 160,
+                 O_D_SC = O_GU_SC + (size_t) 1280 * 40 * 2;
+constexpr int THREADS = 512;            // 16 warps: 4 along the weight rows (64 each) x 4 along the tile rows (16 each)
+constexpr int WROWS = 256;              // weight rows per work item: 128 features of gate/up, or 256 outputs of down
+constexpr int STAGES = 4;               // cp.async pipeline depth, one 64-weight block of K per stage
+constexpr int STAGE_BYTES = WROWS * 16 + kTileRows * AB;
+constexpr size_t smem_bytes(bool gu) {
+    return (size_t) WROWS * (gu ? 40 : 10) * 2 + STAGES * STAGE_BYTES + kTileRows * 4;
+}
+
+// the stored position of value k of a 32-value half-block (see "The K order")
+__host__ __device__ __forceinline__ int perm32(int k) { return 8 * (k & 3) + 4 * (k >> 4) + ((k & 15) >> 2); }
+
+struct Tables {
+    int32_t *cnt, *off, *fill, *ts;      // per expert: rows, first row (E + 1), placement cursor, first tile (E + 1)
+    int2* tiles;                         // {expert, first row}, in expert order
+};
+size_t tiles_at(int n_expert) { return ((size_t) (4 * n_expert + 2) * 4 + 15) / 16 * 16; }
+Tables tables(void* scratch, int n_expert) {
+    int32_t* p = (int32_t*) scratch;
+    return {p, p + n_expert, p + 2 * n_expert + 1, p + 3 * n_expert + 1,
+            (int2*) ((uint8_t*) scratch + tiles_at(n_expert))};
+}
+
+// ---- activations: one warp per 64 values
+__global__ void quant_act_kernel(const float* __restrict__ x, int64_t nblk, uint8_t* __restrict__ xa) {
+    const int64_t w = (int64_t) blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32;
+    const int lane = threadIdx.x & 31;
+    if (w >= nblk) return;
+    const float v0 = x[w * 64 + lane], v1 = x[w * 64 + 32 + lane];
+    float a0 = fabsf(v0), a1 = fabsf(v1);
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) {
+        a0 = fmaxf(a0, __shfl_xor_sync(0xffffffffu, a0, o));
+        a1 = fmaxf(a1, __shfl_xor_sync(0xffffffffu, a1, o));
+    }
+    const int q0 = a0 > 0.0f ? __float2int_rn(v0 * (127.0f / a0)) : 0;
+    const int q1 = a1 > 0.0f ? __float2int_rn(v1 * (127.0f / a1)) : 0;
+    int s0 = q0, s1 = q1;
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) {
+        s0 += __shfl_xor_sync(0xffffffffu, s0, o);
+        s1 += __shfl_xor_sync(0xffffffffu, s1, o);
+    }
+    uint8_t* out = xa + w * AB;
+    out[perm32(lane)] = (uint8_t) (int8_t) q0;
+    out[32 + perm32(lane)] = (uint8_t) (int8_t) q1;
+    if (lane == 0)
+        *(float4*) (out + 64) = make_float4(a0 / 127.0f, -(MAGICF + (float) s0), a1 / 127.0f, -(MAGICF + (float) s1));
+}
+
+// ---- grouping: counts, offsets and tiles, placement
+__global__ void count_kernel(const int32_t* __restrict__ ids, int64_t n, int E, int32_t* __restrict__ cnt) {
+    extern __shared__ int32_t hist[];
+    for (int e = threadIdx.x; e < E; e += blockDim.x) hist[e] = 0;
+    __syncthreads();
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) {
+        const int e = ids[i];
+        if ((unsigned) e < (unsigned) E) atomicAdd(&hist[e], 1);
+    }
+    __syncthreads();
+    for (int e = threadIdx.x; e < E; e += blockDim.x)
+        if (hist[e]) atomicAdd(&cnt[e], hist[e]);
+}
+
+// exclusive prefix sum over the block (blockDim a multiple of 32); *total = the block's sum
+__device__ int block_excl_sum(int v, int* total, int* sh) {
+    const int lane = threadIdx.x & 31, w = threadIdx.x >> 5, nw = blockDim.x >> 5;
+    int x = v;
+#pragma unroll
+    for (int o = 1; o < 32; o <<= 1) {
+        const int y = __shfl_up_sync(0xffffffffu, x, o);
+        if (lane >= o) x += y;
+    }
+    if (lane == 31) sh[w] = x;
+    __syncthreads();
+    if (w == 0) {
+        int s = lane < nw ? sh[lane] : 0;
+#pragma unroll
+        for (int o = 1; o < 32; o <<= 1) {
+            const int y = __shfl_up_sync(0xffffffffu, s, o);
+            if (lane >= o) s += y;
+        }
+        sh[lane] = s;
+    }
+    __syncthreads();
+    const int r = x - v + (w > 0 ? sh[w - 1] : 0);
+    *total = sh[nw - 1];
+    __syncthreads();   // sh is reused by the next call
+    return r;
+}
+
+// one block, a thread per expert (E <= 1024)
+__global__ void scan_kernel(Tables tb, int E) {
+    __shared__ int sh[32];
+    const int e = threadIdx.x;
+    const int c = e < E ? tb.cnt[e] : 0, nt = (c + kTileRows - 1) / kTileRows;
+    int rows = 0, tiles = 0;
+    const int o = block_excl_sum(c, &rows, sh), to = block_excl_sum(nt, &tiles, sh);
+    if (e < E) {
+        tb.off[e] = o;
+        tb.fill[e] = o;
+        tb.ts[e] = to;
+        for (int i = 0; i < nt; ++i) tb.tiles[to + i] = make_int2(e, o + i * kTileRows);
+    }
+    if (e == 0) {
+        tb.off[E] = rows;
+        tb.ts[E] = tiles;
+    }
+}
+
+__global__ void place_kernel(const int32_t* __restrict__ ids, int64_t n, int k, int E, Tables tb,
+                             int32_t* __restrict__ slot, int32_t* __restrict__ src) {
+    const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= n) return;
+    const int e = ids[i];
+    if ((unsigned) e >= (unsigned) E) { slot[i] = 0; return; }   // (the router never writes one)
+    const int p = atomicAdd(&tb.fill[e], 1);
+    slot[i] = p;
+    src[p] = (int32_t) (i / k);
+}
+
+// ---- the expert products
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+__device__ __forceinline__ void cp16(void* dst, const void* src) {
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(dst)),
+                 "l"(src));
+}
+__device__ __forceinline__ void cp4(void* dst, const void* src) {
+    asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"((unsigned) __cvta_generic_to_shared(dst)),
+                 "l"(src));
+}
+__device__ __forceinline__ void cp_commit() { asm volatile("cp.async.commit_group;\n" ::); }
+template <int N> __device__ __forceinline__ void cp_wait() { asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); }
+// d = A (16 x 32 s8, row) * B (32 x 8 s8, col), int32, from zero
+__device__ __forceinline__ void mma_s8(int (&d)[4], const uint32_t (&a)[4], uint32_t b0, uint32_t b1) {
+    asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%10,%10,%10};\n"
+        : "=r"(d[0]), "=r"(d[1]), "=r"(d[2]), "=r"(d[3])
+        : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1), "r"(0));
+}
+#endif
+
+// One work item = (tile of up to 64 routed rows of expert e, a block of 256 weight rows); a persistent grid walks
+// the batch's items (their count is on the device).  GU: gate/up rows of 128 features against the rows' tokens'
+// activations (`act` per token, 2560 values), SwiGLU, H to int8 per 32 features into `out` (per row, 640 values).
+// !GU: down rows (256 outputs) against H (`act` per row), FP32 into `dm` per row.
+// Warp (wf, wt): weight rows 64 wf.. as four m16 tiles (GU: a tile = 8 features, its gate rows as mma rows 0-7 and
+// its up rows as 8-15, so a lane holds gate and up of one feature: SwiGLU in registers), tile rows 16 wt.. as two n8.
+template <bool GU>
+__global__ void __launch_bounds__(THREADS, 1)
+expert_kernel(const Batch b, const Tables tb, const uint8_t* __restrict__ act, const int32_t* __restrict__ src,
+              uint8_t* __restrict__ out, float* __restrict__ dm) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+    constexpr int NKB = GU ? 40 : 10;                 // 64-weight blocks along K
+    constexpr int NFB = GU ? 5 : 10;                  // weight-row blocks per tile
+    constexpr int CODE_LD = GU ? 640 : 160;           // code bytes per weight row
+    constexpr size_t O_CODES = GU ? O_GU_CODES : O_D_CODES, O_SC = GU ? O_GU_SC : O_D_SC;
+    constexpr int ACT_LD = NKB * AB;                  // bytes per activation row: 3200 (a token), 800 (a row's H)
+    constexpr unsigned M2 = 0x03030303u;
+    extern __shared__ __align__(16) uint8_t smem[];
+    const __half* wsc = (const __half*) smem;                     // [256][NKB] the work item's weight scales
+    uint8_t* stages = smem + WROWS * NKB * 2;                     // [STAGES] x {codes [256][16], act [64][AB]}
+    int* srow = (int*) (stages + STAGES * STAGE_BYTES);           // [64] the activation row of each tile row
+
+    const int tid = threadIdx.x, lane = tid & 31, warp = tid >> 5, g = lane >> 2, tig = lane & 3;
+    const int wf = warp & 3, nb0 = 16 * (warp >> 2);
+    int ra[4], rb[4];                                             // the local weight rows of mma rows g and g + 8
+#pragma unroll
+    for (int i = 0; i < 4; ++i) {
+        if (GU) { ra[i] = 2 * (32 * wf + 8 * i + g); rb[i] = ra[i] + 1; }
+        else { ra[i] = 64 * wf + 16 * i + g; rb[i] = ra[i] + 8; }
+    }
+    const int t0 = tb.ts[b.e0], nwork = (tb.ts[b.e1] - t0) * NFB;
+    for (int w = blockIdx.x; w < nwork; w += gridDim.x) {
+        const int2 tl = tb.tiles[t0 + w / NFB];
+        const int fb = w % NFB, e = tl.x, row0 = tl.y, nrows = min(kTileRows, tb.off[e + 1] - row0);
+        const uint8_t* blob = b.blob[e - b.e0];
+        const int rbase = fb * WROWS;
+        __syncthreads();                                          // the previous item is done with the buffers
+        if (tid < kTileRows) {
+            const int r = row0 + min(tid, nrows - 1);             // rows past the tile's end repeat its last one
+            srow[tid] = GU ? src[r] : r;
+        }
+        for (int c = tid; c < WROWS * 5; c += THREADS) {          // the scales: 5 x 16 B (GU) or 5 x 4 B a row
+            const int r = c / 5, q = c % 5;
+            if (GU) cp16(smem + r * 80 + q * 16, blob + O_SC + (size_t) (rbase + r) * 80 + q * 16);
+            else cp4(smem + r * 20 + q * 4, blob + O_SC + (size_t) (rbase + r) * 20 + q * 4);
+        }
+        __syncthreads();                                          // srow
+        auto load = [&](int kb) {
+            uint8_t* st = stages + (kb % STAGES) * STAGE_BYTES;
+            if (tid < WROWS) cp16(st + tid * 16, blob + O_CODES + (size_t) (rbase + tid) * CODE_LD + kb * 16);
+            const int a = tid - (THREADS - kTileRows * 5);        // the last 320 threads: 64 rows x 5 x 16 B
+            if (a >= 0) {
+                const int r = a / 5, q = a % 5;
+                cp16(st + WROWS * 16 + r * AB + q * 16, act + (size_t) srow[r] * ACT_LD + kb * AB + q * 16);
+            }
+        };
+#pragma unroll
+        for (int s = 0; s < STAGES - 1; ++s) {
+            if (s < NKB) load(s);
+            cp_commit();
+        }
+        // a warp whose rows are all past the tile's end only takes part in the loads
+        const bool on0 = nb0 < nrows, on1 = nb0 + 8 < nrows;
+        float acc[4][2][4];
+#pragma unroll
+        for (int i = 0; i < 4; ++i)
+#pragma unroll
+            for (int n = 0; n < 2; ++n)
+#pragma unroll
+                for (int q = 0; q < 4; ++q) acc[i][n][q] = 0.0f;
+        for (int kb = 0; kb < NKB; ++kb) {
+            cp_wait<STAGES - 2>();
+            __syncthreads();
+            if (kb + STAGES - 1 < NKB) load(kb + STAGES - 1);
+            cp_commit();
+            if (!on0) continue;
+            const uint8_t* st = stages + (kb % STAGES) * STAGE_BYTES;
+            const uint8_t* sa = st + WROWS * 16;
+            uint2 bq[2][2];                                       // [n8 tile][half]: the B registers
+            float4 sc[2][2];                                      // [n8 tile][column 2 tig + c]: {d0, c0, d1, c1}
+#pragma unroll
+            for (int n = 0; n < 2; ++n) {
+                if (n == 1 && !on1) break;
+                const uint8_t* r = sa + (nb0 + 8 * n + g) * AB + 8 * tig;
+                bq[n][0] = *(const uint2*) r;
+                bq[n][1] = *(const uint2*) (r + 32);
+                sc[n][0] = *(const float4*) (sa + (nb0 + 8 * n + 2 * tig) * AB + 64);
+                sc[n][1] = *(const float4*) (sa + (nb0 + 8 * n + 2 * tig + 1) * AB + 64);
+            }
+#pragma unroll
+            for (int i = 0; i < 4; ++i) {
+                const uint4 ca = *(const uint4*) (st + ra[i] * 16), cb = *(const uint4*) (st + rb[i] * 16);
+                const float da = __half2float(wsc[ra[i] * NKB + kb]), db = __half2float(wsc[rb[i] * NKB + kb]);
+                const int s = 2 * tig;
+                const uint32_t a0[4] = {(ca.x >> s) & M2, (cb.x >> s) & M2, (ca.y >> s) & M2, (cb.y >> s) & M2};
+                const uint32_t a1[4] = {(ca.z >> s) & M2, (cb.z >> s) & M2, (ca.w >> s) & M2, (cb.w >> s) & M2};
+#pragma unroll
+                for (int n = 0; n < 2; ++n) {
+                    if (n == 1 && !on1) break;
+                    int d0[4], d1[4];
+                    mma_s8(d0, a0, bq[n][0].x, bq[n][0].y);
+                    mma_s8(d1, a1, bq[n][1].x, bq[n][1].y);
+#pragma unroll
+                    for (int q = 0; q < 4; ++q) {
+                        const float4 z = sc[n][q & 1];
+                        const float u0 = __int_as_float(MAGIC + d0[q]) + z.y, u1 = __int_as_float(MAGIC + d1[q]) + z.w;
+                        acc[i][n][q] = fmaf(q < 2 ? da : db, fmaf(z.z, u1, z.x * u0), acc[i][n][q]);
+                    }
+                }
+            }
+        }
+        if (!on0) continue;
+        if (GU) {
+            // SwiGLU of feature 32 wf + 8 i + g (lane g of m tile i), tile rows nb0 + 8 n + 2 tig + c; H to int8 per
+            // row over the warp's 32 features (= one 32-value half-block of the down product's K)
+            const int blk = 4 * fb + wf;
+#pragma unroll
+            for (int n = 0; n < 2; ++n) {
+                if (n == 1 && !on1) break;
+#pragma unroll
+                for (int c = 0; c < 2; ++c) {
+                    float h[4], am = 0.0f;
+#pragma unroll
+                    for (int i = 0; i < 4; ++i) {
+                        const float gt = acc[i][n][c], up = acc[i][n][2 + c];
+                        h[i] = gt / (1.0f + __expf(-gt)) * up;
+                        am = fmaxf(am, fabsf(h[i]));
+                    }
+#pragma unroll
+                    for (int o = 4; o < 32; o <<= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
+                    const float inv = am > 0.0f ? 127.0f / am : 0.0f;
+                    int qv[4], sum = 0;
+#pragma unroll
+                    for (int i = 0; i < 4; ++i) { qv[i] = __float2int_rn(h[i] * inv); sum += qv[i]; }
+#pragma unroll
+                    for (int o = 4; o < 32; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
+                    const int r = nb0 + 8 * n + 2 * tig + c;
+                    if (r < nrows) {
+                        uint8_t* o = out + (size_t) (row0 + r) * (10 * AB) + (blk >> 1) * AB;
+                        const int hh = blk & 1;
+#pragma unroll
+                        for (int i = 0; i < 4; ++i) o[32 * hh + perm32(8 * i + g)] = (uint8_t) (int8_t) qv[i];
+                        if (g == 0) *(float2*) (o + 64 + 8 * hh) = make_float2(am / 127.0f, -(MAGICF + (float) sum));
+                    }
+                }
+            }
+        } else {
+#pragma unroll
+            for (int i = 0; i < 4; ++i)
+#pragma unroll
+                for (int n = 0; n < 2; ++n) {
+                    if (n == 1 && !on1) break;
+#pragma unroll
+                    for (int q = 0; q < 4; ++q) {
+                        const int r = nb0 + 8 * n + 2 * tig + (q & 1);
+                        if (r < nrows) dm[(size_t) (row0 + r) * 2560 + rbase + (q < 2 ? ra[i] : rb[i])] = acc[i][n][q];
+                    }
+                }
+        }
+    }
+#endif
+}
+
+unsigned blocks(int64_t n, int per) { return (unsigned) ((n + per - 1) / per); }
+
+struct DevInfo {
+    bool done = false;
+    int cc = 0, sms = 0, occ_gu = 0, occ_d = 0;
+};
+std::mutex g_mu;
+DevInfo g_dev[32];
+
+const DevInfo& dev_info() {
+    int dev = 0;
+    cudaGetDevice(&dev);
+    std::lock_guard<std::mutex> lk(g_mu);
+    DevInfo& d = g_dev[dev & 31];
+    if (d.done) return d;
+    d.done = true;
+    int major = 0, minor = 0;
+    cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+    cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);
+    cudaDeviceGetAttribute(&d.sms, cudaDevAttrMultiProcessorCount, dev);
+    d.cc = 10 * major + minor;
+    if (d.cc < 80) return d;
+    // the device code of this card's arch must have the kernels (a build without an sm_80+ target has an empty body)
+    cudaFuncAttributes fa{};
+    if (cudaFuncGetAttributes(&fa, expert_kernel<true>) != cudaSuccess || fa.ptxVersion < 80) {
+        cudaGetLastError();
+        d.cc = 0;
+        return d;
+    }
+    ck(cudaFuncSetAttribute(expert_kernel<true>, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) smem_bytes(true)),
+       "smem gu");
+    ck(cudaFuncSetAttribute(expert_kernel<false>, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) smem_bytes(false)),
+       "smem down");
+    cudaOccupancyMaxActiveBlocksPerMultiprocessor(&d.occ_gu, expert_kernel<true>, THREADS, smem_bytes(true));
+    cudaOccupancyMaxActiveBlocksPerMultiprocessor(&d.occ_d, expert_kernel<false>, THREADS, smem_bytes(false));
+    if (d.occ_gu < 1 || d.occ_d < 1) d.cc = 0;                   // does not fit this card: the MMQ path
+    cudaGetLastError();
+    return d;
+}
+
+}  // namespace
+
+bool built() { return true; }
+bool available() { return dev_info().cc >= 80; }
+// 0.1.36: on by default for the Q2_0 pack (+13-23% prompts on an RTX 5070; teacher-forced against the FP16 path as
+// close as MMQ at 8K and closer at 32K: phaseA-tests s18-tf); STRATA_PF_FUSED=0 keeps MMQ.  The native packs' kernels
+// (moe_fused_iq) stay opt-in: STRATA_PF_FUSED=1 (requested()).
+bool enabled() {
+    static const bool env = [] {
+        const char* v = std::getenv("STRATA_PF_FUSED");
+        return v == nullptr || v[0] != '0';
+    }();
+    return env && available();
+}
+bool requested() {
+    static const bool env = [] {
+        const char* v = std::getenv("STRATA_PF_FUSED");
+        return v != nullptr && v[0] == '1';
+    }();
+    return env && available();
+}
+
+size_t act_bytes(int64_t rows, int64_t cols) { return (size_t) rows * (size_t) (cols / 64) * AB; }
+size_t group_bytes(int64_t n, int n_expert) {
+    return tiles_at(n_expert) + ((size_t) (n + kTileRows - 1) / kTileRows + (size_t) n_expert) * sizeof(int2);
+}
+
+void quantize_act(const float* x, int64_t rows, int64_t cols, void* xa, void* stream) {
+    if (rows <= 0) return;
+    const int64_t nblk = rows * (cols / 64);
+    quant_act_kernel<<<blocks(nblk, 8), 256, 0, (cudaStream_t) stream>>>(x, nblk, (uint8_t*) xa);
+    ck(cudaGetLastError(), "quantize_act");
+}
+
+void group(const int32_t* ids, int64_t n, int k, int n_expert, void* scratch, int32_t* slot, int32_t* src,
+           void* stream) {
+    if (n_expert < 1 || n_expert > 1024) {
+        std::fprintf(stderr, "prefill fused experts: %d experts (the grouping takes 1 to 1024)\n", n_expert);
+        std::exit(1);
+    }
+    const cudaStream_t s = (cudaStream_t) stream;
+    const Tables tb = tables(scratch, n_expert);
+    ck(cudaMemsetAsync(tb.cnt, 0, (size_t) n_expert * 4, s), "group memset");
+    if (n > 0) {
+        const unsigned nb = std::min<unsigned>(blocks(n, 1024), 4u * (unsigned) std::max(dev_info().sms, 1));
+        count_kernel<<<nb, 1024, (size_t) n_expert * 4, s>>>(ids, n, n_expert, tb.cnt);
+    }
+    scan_kernel<<<1, (unsigned) ((n_expert + 31) / 32 * 32), 0, s>>>(tb, n_expert);
+    if (n > 0) place_kernel<<<blocks(n, 256), 256, 0, s>>>(ids, n, k, n_expert, tb, slot, src);
+    ck(cudaGetLastError(), "group");
+}
+
+void experts(const Batch& b, int n_expert, int64_t n, const void* scratch, const void* xa, const int32_t* src,
+             void* ha, float* dm, void* stream) {
+    if (b.e1 <= b.e0 || n <= 0) return;
+    const DevInfo& d = dev_info();
+    const cudaStream_t s = (cudaStream_t) stream;
+    const Tables tb = tables(const_cast<void*>(scratch), n_expert);
+    // the most tiles the batch can have: every row in it, plus a partial tile per expert
+    const int64_t tiles = (n + kTileRows - 1) / kTileRows + (b.e1 - b.e0);
+    const unsigned g_gu = (unsigned) std::min<int64_t>(tiles * 5, (int64_t) d.sms * d.occ_gu);
+    const unsigned g_d = (unsigned) std::min<int64_t>(tiles * 10, (int64_t) d.sms * d.occ_d);
+    expert_kernel<true><<<g_gu, THREADS, smem_bytes(true), s>>>(b, tb, (const uint8_t*) xa, src, (uint8_t*) ha,
+                                                                nullptr);
+    expert_kernel<false><<<g_d, THREADS, smem_bytes(false), s>>>(b, tb, (const uint8_t*) ha, src, nullptr, dm);
+    ck(cudaGetLastError(), "experts");
+}
+
+}  // namespace strata::prefill::fused

src/prefill/moe_fused_iq.cu

@@ -0,0 +1,630 @@
+// src/prefill/moe_fused_iq.cu - see include/strata/prefill/moe_fused_iq.hpp (#136: the native packs' prompt experts on
+// the fused int8 kernels of moe_fused.cu).
+//
+// The arithmetic.  An activation block of 32 values is x = d_x * a, a = round(x / d_x) in -127..127, stored in natural
+// order (per 64 values: 64 codes, then {d_0, d_1} and 8 unused bytes - 80 bytes, the Q2_0 path's size).  A weight
+// block of 32 (or 16) values is w = d_w * q with q an int8: the i-quants' codebook entries with their signs applied
+// (IQ2_XXS / IQ2_XS / IQ2_S: 0..43, IQ3_XXS: 0..62, IQ3_S: 0..15, IQ4_XS / IQ4_NL: -127..113), Q2_0's code - 1.  So
+//     sum w x = d_w * d_x * sum q a,     |sum q a| <= 32 * 127 * 127 < 2^22,
+// and the int32 dot becomes a float exactly with one float add: the mma starts from C = 0x4B400000 (the bits of
+// 1.5 * 2^23), so as_float(D) - 1.5 * 2^23 is the dot.  Per output and 32 values: that add, d_w * d_x, one fma (the
+// formats with a scale per 16 values: two dots, each times its d_w, then d_x).  d_w is llama.cpp's own: IQ2_XXS
+// d (2s + 1) / 8, IQ2_XS / IQ2_S d (2s + 1) / 8 per 16, IQ3_XXS d (2s + 1) / 4, IQ3_S d (2s + 1), IQ4_XS d (s - 32),
+// IQ4_NL and Q2_0 d (dequantize_row_* in ggml-quants.c, mmq-load-tiles.cuh).
+//
+// The load stage.  Per 64 values of K (a stage), a thread decodes one 32-value sub-block of one of the work item's
+// weight rows: its block bytes are read from the blob with 16-bit loads (the GGUF blocks are 2-byte aligned: 66 / 74 /
+// 82 / 98 / 110 / 136 / 18 bytes) one stage ahead into registers (and two 256-value blocks ahead into L2), then turned
+// into 32 int8 and the sub-block's scales in shared memory, double-buffered.  The codebooks live in shared memory (up
+// to 8 KB, IQ2_S).  The activations arrive by cp.async, four stages deep, as in moe_fused.cu; the fragments come by
+// ldmatrix.  The products are mma.sync m16n8k32 (m16n8k16 for the formats with a scale per 16 values); the epilogues
+// (SwiGLU, H to int8 per 32 features; down into the per-slot rows) are moe_fused.cu's.
+#include "strata/prefill/moe_fused_iq.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include "ggml.h"
+
+#define GGML_COMMON_DECL_CUDA
+#define GGML_COMMON_IMPL_CUDA
+#include "ggml-common.h"
+
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <mutex>
+
+namespace strata::prefill::fused {
+namespace {
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "prefill fused experts (native): %s: %s\n", what, cudaGetErrorString(e));
+        std::exit(1);
+    }
+}
+
+constexpr int AB = 80;                  // activation bytes per 64 values: 64 codes, {d0, d1}, 8 unused
+constexpr int MAGIC = 0x4B400000;       // the bits of 1.5 * 2^23
+constexpr float MAGICF = 12582912.0f;
+// 16 warps, each 64 weight rows x 16 routed rows; WW of them along the weight rows and 16 / WW along the routed rows: a
+// work item is (WR = 64 WW weight rows) x (TR = 256 / WW routed rows of one expert) - 256 x 64 or 128 x 128, chosen
+// per launch by the rows per expert (pick_ww)
+constexpr int THREADS = 512;
+constexpr int WLD = 80;                 // bytes per decoded weight row of a stage: 64 int8 + 16 (no bank conflicts)
+__host__ __device__ constexpr int tile_rows(int ww) { return 256 / ww; }
+__host__ __device__ constexpr int weight_rows(int ww) { return 64 * ww; }
+constexpr int ASTAGES = 4;              // cp.async depth of the activations (64 values of K a stage)
+constexpr int GU_ROWS_K = 2560, D_ROWS_K = 640;   // K of gate/up (n_embd) and of down (n_ff)
+
+// the formats (ggml type ids)
+constexpr int T_IQ2_XXS = GGML_TYPE_IQ2_XXS, T_IQ2_XS = GGML_TYPE_IQ2_XS, T_IQ2_S = GGML_TYPE_IQ2_S,
+              T_IQ3_XXS = GGML_TYPE_IQ3_XXS, T_IQ3_S = GGML_TYPE_IQ3_S, T_IQ4_XS = GGML_TYPE_IQ4_XS,
+              T_IQ4_NL = GGML_TYPE_IQ4_NL, T_Q2_0 = GGML_TYPE_Q2_0;
+static_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_xs) == 74 && sizeof(block_iq2_s) == 82 &&
+              sizeof(block_iq3_xxs) == 98 && sizeof(block_iq3_s) == 110 && sizeof(block_iq4_xs) == 136 &&
+              sizeof(block_iq4_nl) == 18 && sizeof(block_q2_0) == 18, "the block layouts this file decodes");
+
+// block bytes, scale per 16 values, codebook bytes in shared memory
+__host__ __device__ constexpr int block_bytes(int t) {
+    return t == T_IQ2_XXS ? 66 : t == T_IQ2_XS ? 74 : t == T_IQ2_S ? 82 : t == T_IQ3_XXS ? 98 : t == T_IQ3_S ? 110
+         : t == T_IQ4_XS ? 136 : 18;
+}
+__host__ __device__ constexpr bool per16(int t) { return t == T_IQ2_XS || t == T_IQ2_S; }
+__host__ __device__ constexpr int grid_bytes(int t) {
+    return t == T_IQ2_XXS ? 256 * 8 : t == T_IQ2_XS ? 512 * 8 : t == T_IQ2_S ? 1024 * 8 : t == T_IQ3_XXS ? 256 * 4
+         : t == T_IQ3_S ? 512 * 4 : 0;
+}
+__host__ __device__ constexpr size_t smem_bytes(int t, int ww) {
+    return (size_t) 2 * weight_rows(ww) * (WLD + 16) + (size_t) ASTAGES * tile_rows(ww) * AB +
+           (size_t) tile_rows(ww) * 4 + grid_bytes(t);
+}
+
+// moe_fused.cu's grouping tables (the same layout: fused::group writes them)
+struct Tables {
+    int32_t *cnt, *off, *fill, *ts;
+    int2* tiles;
+};
+size_t tiles_at(int n_expert) { return ((size_t) (4 * n_expert + 2) * 4 + 15) / 16 * 16; }
+Tables tables(void* scratch, int n_expert) {
+    int32_t* p = (int32_t*) scratch;
+    return {p, p + n_expert, p + 2 * n_expert + 1, p + 3 * n_expert + 1,
+            (int2*) ((uint8_t*) scratch + tiles_at(n_expert))};
+}
+
+// ---- activations in natural order: one warp per 64 values
+__global__ void quant_act_nat_kernel(const float* __restrict__ x, int64_t nblk, uint8_t* __restrict__ xa) {
+    const int64_t w = (int64_t) blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32;
+    const int lane = threadIdx.x & 31;
+    if (w >= nblk) return;
+    const float v0 = x[w * 64 + lane], v1 = x[w * 64 + 32 + lane];
+    float a0 = fabsf(v0), a1 = fabsf(v1);
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) {
+        a0 = fmaxf(a0, __shfl_xor_sync(0xffffffffu, a0, o));
+        a1 = fmaxf(a1, __shfl_xor_sync(0xffffffffu, a1, o));
+    }
+    uint8_t* out = xa + w * AB;
+    out[lane] = (uint8_t) (int8_t) (a0 > 0.0f ? __float2int_rn(v0 * (127.0f / a0)) : 0);
+    out[32 + lane] = (uint8_t) (int8_t) (a1 > 0.0f ? __float2int_rn(v1 * (127.0f / a1)) : 0);
+    if (lane == 0) *(float4*) (out + 64) = make_float4(a0 / 127.0f, a1 / 127.0f, 0.0f, 0.0f);
+}
+
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+__device__ __forceinline__ void cp16(void* dst, const void* src) {
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(dst)),
+                 "l"(src));
+}
+__device__ __forceinline__ void cp_commit() { asm volatile("cp.async.commit_group;\n" ::); }
+__device__ __forceinline__ void pf_l2(const void* p) { asm volatile("prefetch.global.L2 [%0];\n" ::"l"(p)); }
+template <int N> __device__ __forceinline__ void cp_wait() { asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); }
+// d = MAGIC + A (16 x 32 s8, row) * B (32 x 8 s8, col): the int32 dot with the magic bias already added (the C
+// operand), so as_float(d) - 1.5 * 2^23 is the dot as a float
+__device__ __forceinline__ void mma32(int (&d)[4], const uint32_t (&a)[4], uint32_t b0, uint32_t b1) {
+    asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%10,%10,%10};\n"
+        : "=r"(d[0]), "=r"(d[1]), "=r"(d[2]), "=r"(d[3])
+        : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1), "r"(MAGIC));
+}
+// the same at m16n8k16 (A 16 x 16, B 16 x 8)
+__device__ __forceinline__ void mma16(int (&d)[4], uint32_t a0, uint32_t a1, uint32_t b0) {
+    asm("mma.sync.aligned.m16n8k16.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5}, {%6}, {%7,%7,%7,%7};\n"
+        : "=r"(d[0]), "=r"(d[1]), "=r"(d[2]), "=r"(d[3])
+        : "r"(a0), "r"(a1), "r"(b0), "r"(MAGIC));
+}
+__device__ __forceinline__ float dotf(int d) { return __int_as_float(d) - MAGICF; }
+// four 8 x 16-byte matrices, row addresses from lanes 0-7, 8-15, 16-23, 24-31: lane (g, t) gets bytes 4t..4t+3 of row g
+// of each - an mma fragment
+__device__ __forceinline__ void ldsm4(uint32_t (&r)[4], const void* p) {
+    asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
+                 : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3])
+                 : "r"((unsigned) __cvta_generic_to_shared(p))
+                 : "memory");
+}
+
+// ---- the load stage: a 32-value sub-block's bytes into registers, then int8 and scales
+__device__ __forceinline__ uint32_t ld16(const uint8_t* p) { return *(const uint16_t*) p; }
+__device__ __forceinline__ uint32_t ld32(const uint8_t* p) { return ld16(p) | (ld16(p + 2) << 16); }
+__device__ __forceinline__ float half_at(uint32_t w) { return __half2float(__ushort_as_half((unsigned short) w)); }
+// llama.cpp's sign unpacking: 7 bits of signs, the 8th their parity (bit 7 of v may be anything)
+__device__ __forceinline__ uint32_t unpack_ksigns(uint32_t v) {
+    v &= 0xFF;
+    const uint32_t p = __popc(v) & 1;
+    return (v ^ p << 7) * 0x01010101u;
+}
+// 8 bytes of a codebook entry with the sign byte `s` (broadcast) applied: bits 0-3 to .x, 4-7 to .y
+__device__ __forceinline__ void signed8(uint32_t gx, uint32_t gy, uint32_t s, uint32_t& qx, uint32_t& qy) {
+    const uint32_t m0 = __vcmpne4(s & 0x08040201u, 0), m1 = __vcmpne4(s & 0x80402010u, 0);
+    qx = __vsub4(gx ^ m0, m0);
+    qy = __vsub4(gy ^ m1, m1);
+}
+// 8 nibbles of q4 through a 16-entry int8 table (4 words): the low nibbles' values in .x, the high ones' in .y
+__device__ __forceinline__ void table16(uint32_t q4, const uint32_t (&t)[4], uint32_t& lo, uint32_t& hi) {
+    uint32_t tmp[2];
+    const uint32_t sel = 0x32103210u | ((q4 & 0x88888888u) >> 1);
+#pragma unroll
+    for (int i = 0; i < 2; ++i) {
+        const uint32_t sh = 16 * i;
+        const uint32_t l = __byte_perm(t[0], t[1], q4 >> sh), h = __byte_perm(t[2], t[3], q4 >> sh);
+        tmp[i] = __byte_perm(l, h, sel >> sh);
+    }
+    lo = __byte_perm(tmp[0], tmp[1], 0x6420);
+    hi = __byte_perm(tmp[0], tmp[1], 0x7531);
+}
+
+// Raw bytes of sub-block `ib` of the block at `bp` (IQ: the 256-value super-block, ib 0..7; Q2_0: the 64-value block,
+// ib 0..1; IQ4_NL: the 32-value block).
+template <int T> __device__ __forceinline__ void load_unit(const uint8_t* bp, int ib, uint32_t (&w)[5]) {
+    if constexpr (T == T_IQ2_XXS) {
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld16(bp);
+    } else if constexpr (T == T_IQ2_XS) {
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld16(bp) | ((uint32_t) bp[66 + ib] << 16);
+    } else if constexpr (T == T_IQ2_S) {
+        w[0] = ld32(bp + 2 + 4 * ib); w[1] = ld32(bp + 34 + 4 * ib);
+        w[2] = ld16(bp) | ((uint32_t) bp[66 + ib] << 16) | ((uint32_t) bp[74 + ib] << 24);
+    } else if constexpr (T == T_IQ3_XXS) {
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld32(bp + 66 + 4 * ib); w[3] = ld16(bp);
+    } else if constexpr (T == T_IQ3_S) {
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld32(bp + 74 + 4 * ib);
+        w[3] = ld16(bp) | ((uint32_t) bp[66 + ib] << 16) | ((uint32_t) ((bp[106 + ib / 2] >> (4 * (ib & 1))) & 15) << 24);
+    } else if constexpr (T == T_IQ4_XS) {
+#pragma unroll
+        for (int k = 0; k < 4; ++k) w[k] = ld32(bp + 8 + 16 * ib + 4 * k);
+        const uint32_t ls = ((bp[4 + ib / 2] >> (4 * (ib & 1))) & 15) | (((ld16(bp + 2) >> (2 * ib)) & 3) << 4);
+        w[4] = ld16(bp) | (ls << 16);
+    } else if constexpr (T == T_IQ4_NL) {
+#pragma unroll
+        for (int k = 0; k < 4; ++k) w[k] = ld32(bp + 2 + 4 * k);
+        w[4] = ld16(bp);
+    } else {   // Q2_0
+        w[0] = ld32(bp + 2 + 8 * ib); w[1] = ld32(bp + 6 + 8 * ib); w[2] = ld16(bp);
+    }
+}
+
+// The sub-block as 32 int8 (q[0..7], natural order) and its scales (s0: values 0-15, s1: 16-31).
+template <int T>
+__device__ __forceinline__ void convert(const uint32_t (&w)[5], const uint8_t* grid, const uint32_t (&kv)[4],
+                                        uint32_t (&q)[8], float& s0, float& s1) {
+    if constexpr (T == T_IQ2_XXS) {
+        const uint2* g = (const uint2*) grid;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint2 e = g[(w[0] >> (8 * l)) & 255];
+            signed8(e.x, e.y, unpack_ksigns(w[1] >> (7 * l)), q[2 * l], q[2 * l + 1]);
+        }
+        s0 = s1 = half_at(w[2]) * (float) ((w[1] >> 27) | 1) * 0.125f;
+    } else if constexpr (T == T_IQ2_XS) {
+        const uint2* g = (const uint2*) grid;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint32_t c = (w[l >> 1] >> (16 * (l & 1))) & 0xFFFF;
+            const uint2 e = g[c & 511];
+            signed8(e.x, e.y, unpack_ksigns(c >> 9), q[2 * l], q[2 * l + 1]);
+        }
+        const float d = half_at(w[2]);
+        const uint32_t sc = w[2] >> 16;
+        s0 = d * (float) (2 * (sc & 15) + 1) * 0.125f;
+        s1 = d * (float) (2 * ((sc >> 4) & 15) + 1) * 0.125f;
+    } else if constexpr (T == T_IQ2_S) {
+        const uint2* g = (const uint2*) grid;
+        const uint32_t qh = (w[2] >> 16) & 255;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint2 e = g[((w[0] >> (8 * l)) & 255) | ((qh << (8 - 2 * l)) & 0x300)];
+            signed8(e.x, e.y, ((w[1] >> (8 * l)) & 255) * 0x01010101u, q[2 * l], q[2 * l + 1]);
+        }
+        const float d = half_at(w[2]);
+        const uint32_t sc = w[2] >> 24;
+        s0 = d * (float) (2 * (sc & 15) + 1) * 0.125f;
+        s1 = d * (float) (2 * (sc >> 4) + 1) * 0.125f;
+    } else if constexpr (T == T_IQ3_XXS) {
+        const uint32_t* g = (const uint32_t*) grid;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint32_t i0 = (w[l >> 1] >> (16 * (l & 1))) & 255, i1 = (w[l >> 1] >> (16 * (l & 1) + 8)) & 255;
+            signed8(g[i0], g[i1], unpack_ksigns(w[2] >> (7 * l)), q[2 * l], q[2 * l + 1]);
+        }
+        s0 = s1 = half_at(w[3]) * (float) (2 * (w[2] >> 28) + 1) * 0.25f;
+    } else if constexpr (T == T_IQ3_S) {
+        const uint32_t* g = (const uint32_t*) grid;
+        const uint32_t qh = (w[3] >> 16) & 255;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint32_t i0 = (w[l >> 1] >> (16 * (l & 1))) & 255, i1 = (w[l >> 1] >> (16 * (l & 1) + 8)) & 255;
+            signed8(g[i0 | ((qh << (8 - 2 * l)) & 256)], g[i1 | ((qh << (7 - 2 * l)) & 256)],
+                    ((w[2] >> (8 * l)) & 255) * 0x01010101u, q[2 * l], q[2 * l + 1]);
+        }
+        s0 = s1 = half_at(w[3]) * (float) (1 + 2 * (w[3] >> 24));
+    } else if constexpr (T == T_IQ4_XS || T == T_IQ4_NL) {
+#pragma unroll
+        for (int k = 0; k < 4; ++k) table16(w[k], kv, q[k], q[4 + k]);
+        s0 = s1 = T == T_IQ4_XS ? half_at(w[4]) * (float) ((int) (w[4] >> 16) - 32) : half_at(w[4]);
+    } else {   // Q2_0: code - 1 via a byte table {-1, 0, 1, 2}
+#pragma unroll
+        for (int h = 0; h < 4; ++h) {
+            const uint32_t c = (w[h >> 1] >> (16 * (h & 1))) & 0xFFFF;
+            const uint32_t qe = __byte_perm(0x020100FFu, 0x020100FFu, c & 0x7777);
+            const uint32_t qo = __byte_perm(0x020100FFu, 0x020100FFu, (c >> 2) & 0x7777);
+            q[2 * h] = __byte_perm(qe, qo, 0x5140);
+            q[2 * h + 1] = __byte_perm(qe, qo, 0x7362);
+        }
+        s0 = s1 = half_at(w[2]);
+    }
+}
+
+template <int T> __device__ __forceinline__ const void* grid_src() {
+    if constexpr (T == T_IQ2_XXS) return iq2xxs_grid;
+    else if constexpr (T == T_IQ2_XS) return iq2xs_grid;
+    else if constexpr (T == T_IQ2_S) return iq2s_grid;
+    else if constexpr (T == T_IQ3_XXS) return iq3xxs_grid;
+    else if constexpr (T == T_IQ3_S) return iq3s_grid;
+    else return nullptr;
+}
+#endif
+
+// One work item = (TR routed rows of expert e - one or two of group()'s 64-row tiles -, a block of WR weight rows);
+// a persistent grid walks the batch's items.  GU: gate/up rows of WR / 2 features (format WT) against the rows'
+// tokens' activations, SwiGLU, H to int8 per 32 features into `out`.  !GU: down rows (format WT) against H, FP32 into
+// `dm`.  Warp (wf, wt): weight rows 64 wf.. as four m16 tiles (GU: a tile = 8 features, gate rows as mma rows 0-7 and
+// up rows as 8-15, so a lane holds gate and up of one feature), routed rows 16 wt.. as two n8.
+template <int WT, bool GU, int WW>
+__global__ void __launch_bounds__(THREADS, 1)
+native_kernel(const Batch b, const NativeGeom geo, const Tables tb, const uint8_t* __restrict__ act,
+              const int32_t* __restrict__ src, uint8_t* __restrict__ out, float* __restrict__ dm) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+    constexpr int TR = tile_rows(WW), WR = weight_rows(WW);
+    constexpr int WT_BYTES = WR * WLD, WS_FLOATS = WR * 4, ACT_STAGE = TR * AB;
+    constexpr int NS = (GU ? GU_ROWS_K : D_ROWS_K) / 64;   // 64-value stages along K
+    constexpr int NFB = (GU ? 1280 : 2560) / WR;          // weight-row blocks per tile
+    constexpr int ACT_LD = NS * AB;                       // bytes per activation row: 3200 (a token), 800 (a row's H)
+    constexpr int BS = block_bytes(WT);
+    constexpr bool K16 = per16(WT);
+    extern __shared__ __align__(16) uint8_t smem[];
+    uint8_t* wt = smem;                                               // [2][WR][WLD] decoded int8 weights
+    float* ws = (float*) (smem + 2 * WT_BYTES);                       // [2][WR][4] their scales per 16 values
+    uint8_t* stages = (uint8_t*) (ws + 2 * WS_FLOATS);                // [ASTAGES][TR][AB] activations
+    int* srow = (int*) (stages + ASTAGES * ACT_STAGE);                // [TR] the activation row of each tile row
+    uint8_t* sgrid = (uint8_t*) (srow + TR);                          // the codebook
+
+    const int tid = threadIdx.x, lane = tid & 31, warp = tid >> 5, g = lane >> 2, tig = lane & 3;
+    const int wf = warp % WW, nb0 = 16 * (warp / WW);
+    if constexpr (grid_bytes(WT) > 0) {
+        const uint32_t* gs = (const uint32_t*) grid_src<WT>();
+        for (int i = tid; i < grid_bytes(WT) / 4; i += THREADS) ((uint32_t*) sgrid)[i] = gs[i];
+    }
+    uint32_t kv[4] = {0, 0, 0, 0};
+    if constexpr (WT == T_IQ4_XS || WT == T_IQ4_NL) {
+#pragma unroll
+        for (int k = 0; k < 16; ++k) kv[k >> 2] |= (uint32_t) (uint8_t) kvalues_iq4nl[k] << (8 * (k & 3));
+    }
+    // Local weight rows: m tile i of warp wf is rows 64 wf + 16 i.. (mma rows g and g + 8: rows +g and +8+g).  For
+    // gate/up a tile is 8 features, its gate rows then its up rows, so a lane holds gate and up of one feature.  The
+    // fragments come by ldmatrix: lane l gives the address of row (l & 7) of matrix l >> 3 - A: {rows 0-7, rows 8-15}
+    // x {bytes 0-15, 16-31} of a 32-value chunk; B: {tokens nb0 + 0-7, + 8-15} x {bytes 0-15, 16-31}.
+    const int lm = lane >> 3, l8 = lane & 7;
+    const int a_off = (64 * wf + 8 * (lm & 1) + l8) * WLD + 16 * (lm >> 1);
+    const int b_off = (nb0 + 8 * (lm >> 1) + l8) * AB + 16 * (lm & 1);
+    // this thread's decode unit (the first 2 WR threads): local weight row ur, sub-block uj of each stage
+    const bool dec = tid < 2 * WR;
+    const int ur = dec ? tid >> 1 : 0, uj = tid & 1;
+    const int t0 = tb.ts[b.e0], nwork = (tb.ts[b.e1] - t0) * NFB;
+    for (int w = blockIdx.x; w < nwork; w += gridDim.x) {
+        const int2 tl = tb.tiles[t0 + w / NFB];
+        const int fb = w % NFB, e = tl.x, row0 = tl.y;
+        if ((row0 - tb.off[e]) % TR != 0) continue;              // a 64-row tile inside an earlier item's TR rows
+        const int nrows = min(TR, tb.off[e + 1] - row0);
+        const uint8_t* blob = b.blob[e - b.e0];
+        const int rbase = fb * WR;
+        // local row ur: gate/up - feature (WR / 2) fb + 8 (ur >> 4) + (ur & 7), its up row when bit 3 is set
+        const uint8_t* wrow = GU ? blob + ((ur & 8) ? geo.up_off : 0) +
+                                       (size_t) (fb * (WR / 2) + 8 * (ur >> 4) + (ur & 7)) * geo.gu_row
+                                 : blob + geo.down_off + (size_t) (rbase + ur) * geo.d_row;
+        auto unit = [&](int s) -> const uint8_t* {                  // the block of stage s's sub-block
+            if (GU) return wrow + (s >> 2) * BS;
+            return WT == T_IQ4_NL ? wrow + (2 * s + uj) * BS : wrow + s * BS;
+        };
+        auto sub = [&](int s) { return GU ? 2 * (s & 3) + uj : uj; };
+        // the weight bytes into L2 ahead of the register loads (one stage ahead only - less than a DRAM round trip):
+        // gate/up PF super-blocks ahead (the row's two threads take a block's first and last line), down the whole row
+        // slice at once (180 or 360 bytes a row)
+        constexpr int PF = 2;
+        auto prefetch_sb = [&](int sb) {
+            if (dec && sb < GU_ROWS_K / 256) pf_l2(wrow + sb * BS + (uj ? BS - 1 : 0));
+        };
+        if (GU) {
+#pragma unroll
+            for (int sb = 0; sb < PF; ++sb) prefetch_sb(sb);
+        } else if (dec) {
+            for (size_t o = 128 * (size_t) uj; o < geo.d_row + 127; o += 256) pf_l2(wrow + min(o, geo.d_row - 1));
+        }
+        __syncthreads();                                          // the previous item is done with the buffers
+        if (tid < TR) srow[tid] = tid < nrows ? (GU ? src[row0 + tid] : row0 + tid) : -1;
+        __syncthreads();                                          // srow
+        // the rows past the item's end are not loaded: a warp entirely past it skips the products, and the columns
+        // of a partial one are never written (no reduction mixes columns)
+        auto load_act = [&](int s) {
+            uint8_t* st = stages + (s % ASTAGES) * ACT_STAGE;
+            for (int c = tid; c < TR * 5; c += THREADS) {
+                const int r = c / 5, q = c % 5;
+                if (r < nrows) cp16(st + r * AB + q * 16, act + (size_t) srow[r] * ACT_LD + s * AB + q * 16);
+            }
+        };
+        auto put = [&](const uint32_t (&raw)[5], int buf) {
+            if (!dec) return;
+            uint32_t q[8];
+            float s0, s1;
+            convert<WT>(raw, sgrid, kv, q, s0, s1);
+            uint4* d = (uint4*) (wt + buf * WT_BYTES + ur * WLD + 32 * uj);
+            d[0] = make_uint4(q[0], q[1], q[2], q[3]);
+            d[1] = make_uint4(q[4], q[5], q[6], q[7]);
+            *(float2*) (ws + buf * WS_FLOATS + ur * 4 + 2 * uj) = make_float2(s0, s1);
+        };
+#pragma unroll
+        for (int s = 0; s < ASTAGES - 1; ++s) {
+            if (s < NS) load_act(s);
+            cp_commit();
+        }
+        uint32_t raw[5] = {0, 0, 0, 0, 0};
+        if (dec) load_unit<WT>(unit(0), sub(0), raw);
+        put(raw, 0);
+        if (dec) load_unit<WT>(unit(1), sub(1), raw);
+        // a warp whose rows are all past the tile's end only takes part in the loads
+        const bool on0 = nb0 < nrows, on1 = nb0 + 8 < nrows;
+        float acc[4][2][4];
+#pragma unroll
+        for (int i = 0; i < 4; ++i)
+#pragma unroll
+            for (int n = 0; n < 2; ++n)
+#pragma unroll
+                for (int q = 0; q < 4; ++q) acc[i][n][q] = 0.0f;
+        for (int s = 0; s < NS; ++s) {
+            cp_wait<ASTAGES - 2>();
+            __syncthreads();
+            if (s + ASTAGES - 1 < NS) load_act(s + ASTAGES - 1);
+            cp_commit();
+            if (GU && (s & 3) == 0) prefetch_sb((s >> 2) + PF);
+            if (on0) {
+                const uint8_t* W = wt + (s & 1) * WT_BYTES;
+                const float* S = ws + (s & 1) * WS_FLOATS;
+                const uint8_t* sa = stages + (s % ASTAGES) * ACT_STAGE;
+                // the stage's scales: per weight row 4 (per 16 values), per routed row 2 (per 32)
+                float2 dx[2][2];                                  // [n8 tile][column 2 tig + cc]
+#pragma unroll
+                for (int n = 0; n < 2; ++n)
+#pragma unroll
+                    for (int cc = 0; cc < 2; ++cc) dx[n][cc] = *(const float2*) (sa + (nb0 + 8 * n + 2 * tig + cc) * AB + 64);
+#pragma unroll
+                for (int h = 0; h < 2; ++h) {                     // the stage's two 32-value halves
+                    uint32_t bq[4];                               // {n0 bytes 0-15, n0 16-31, n1 0-15, n1 16-31}
+                    ldsm4(bq, sa + b_off + 32 * h);
+#pragma unroll
+                    for (int i = 0; i < 4; ++i) {
+                        uint32_t a[4];                            // {rows g, g + 8} x {bytes 0-15, 16-31}
+                        ldsm4(a, W + a_off + 16 * i * WLD + 32 * h);
+                        const float2 swa = *(const float2*) (S + (64 * wf + 16 * i + g) * 4 + 2 * h);
+                        const float2 swb = *(const float2*) (S + (64 * wf + 16 * i + 8 + g) * 4 + 2 * h);
+                        const float wa0 = swa.x, wa1 = swa.y, wb0 = swb.x, wb1 = swb.y;
+#pragma unroll
+                        for (int n = 0; n < 2; ++n) {
+                            if (n == 1 && !on1) break;
+                            if constexpr (K16) {
+                                // a scale per 16 weights: the two halves' dots, each times its scale, then d_x
+                                int d0[4], d1[4];
+                                mma16(d0, a[0], a[1], bq[2 * n]);
+                                mma16(d1, a[2], a[3], bq[2 * n + 1]);
+#pragma unroll
+                                for (int q = 0; q < 4; ++q) {
+                                    const float v = fmaf(q < 2 ? wa1 : wb1, dotf(d1[q]), (q < 2 ? wa0 : wb0) * dotf(d0[q]));
+                                    acc[i][n][q] = fmaf(h ? dx[n][q & 1].y : dx[n][q & 1].x, v, acc[i][n][q]);
+                                }
+                            } else {
+                                int d[4];
+                                mma32(d, a, bq[2 * n], bq[2 * n + 1]);
+#pragma unroll
+                                for (int q = 0; q < 4; ++q) {
+                                    const float p = (q < 2 ? wa0 : wb0) * (h ? dx[n][q & 1].y : dx[n][q & 1].x);
+                                    acc[i][n][q] = fmaf(p, dotf(d[q]), acc[i][n][q]);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            // the next stage's weights (into the other buffer, read after the next barrier).  (Decoding them before
+            // this stage's products instead was slower on the RTX 5070: a layer of 2048 / 3584 / 8192 tokens, IQ2_S
+            // 5.2 / 9.5 / 17.5 ms -> 6.0 / 10.2 / 18.3.)
+            if (s + 1 < NS) {
+                put(raw, (s + 1) & 1);
+                if (dec && s + 2 < NS) load_unit<WT>(unit(s + 2), sub(s + 2), raw);
+            }
+        }
+        if (!on0) continue;
+        if (GU) {
+            // SwiGLU of feature (WR / 2) fb + 32 wf + 8 i + g (lane g of m tile i: rows 64 wf + 16 i + g and + 8), tile
+            // rows nb0 + 8 n + 2 tig + c; H to int8 per row over the warp's 32 features (one 32-value half-block of the
+            // down product's K)
+            const int blk = WW * fb + wf;
+#pragma unroll
+            for (int n = 0; n < 2; ++n) {
+                if (n == 1 && !on1) break;
+#pragma unroll
+                for (int c = 0; c < 2; ++c) {
+                    float h[4], am = 0.0f;
+#pragma unroll
+                    for (int i = 0; i < 4; ++i) {
+                        const float gt = acc[i][n][c], up = acc[i][n][2 + c];
+                        h[i] = gt / (1.0f + __expf(-gt)) * up;
+                        am = fmaxf(am, fabsf(h[i]));
+                    }
+#pragma unroll
+                    for (int o = 4; o < 32; o <<= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
+                    const float inv = am > 0.0f ? 127.0f / am : 0.0f;
+                    const int r = nb0 + 8 * n + 2 * tig + c;
+                    if (r < nrows) {
+                        uint8_t* o = out + (size_t) (row0 + r) * (10 * AB) + (blk >> 1) * AB;
+                        const int hh = blk & 1;
+#pragma unroll
+                        for (int i = 0; i < 4; ++i) o[32 * hh + 8 * i + g] = (uint8_t) (int8_t) __float2int_rn(h[i] * inv);
+                        if (g == 0) *(float*) (o + 64 + 4 * hh) = am / 127.0f;
+                    }
+                }
+            }
+        } else {
+#pragma unroll
+            for (int i = 0; i < 4; ++i)
+#pragma unroll
+                for (int n = 0; n < 2; ++n) {
+                    if (n == 1 && !on1) break;
+#pragma unroll
+                    for (int q = 0; q < 4; ++q) {
+                        const int r = nb0 + 8 * n + 2 * tig + (q & 1);
+                        if (r < nrows) dm[(size_t) (row0 + r) * 2560 + rbase + 64 * wf + 16 * i + 8 * (q >> 1) + g] = acc[i][n][q];
+                    }
+                }
+        }
+    }
+#endif
+}
+
+unsigned blocks(int64_t n, int per) { return (unsigned) ((n + per - 1) / per); }
+
+// per device: whether every kernel here runs (sm_80+, device code in this build, fits), and their occupancy
+struct DevInfo {
+    bool done = false, ok = false;
+    int sms = 0, occ = 1;
+};
+std::mutex g_mu;
+DevInfo g_dev[32];
+
+template <int T, bool GU, int WW> bool setup_ww(int& occ) {
+    cudaFuncAttributes fa{};
+    if (cudaFuncGetAttributes(&fa, native_kernel<T, GU, WW>) != cudaSuccess || fa.ptxVersion < 80) return false;
+    if (cudaFuncSetAttribute(native_kernel<T, GU, WW>, cudaFuncAttributeMaxDynamicSharedMemorySize,
+                             (int) smem_bytes(T, WW)) != cudaSuccess)
+        return false;
+    int o = 0;
+    if (cudaOccupancyMaxActiveBlocksPerMultiprocessor(&o, native_kernel<T, GU, WW>, THREADS, smem_bytes(T, WW)) !=
+            cudaSuccess || o < 1)
+        return false;
+    occ = std::min(occ, o);
+    return true;
+}
+template <int T, bool GU> bool setup_one(int& occ) { return setup_ww<T, GU, 4>(occ) && setup_ww<T, GU, 2>(occ); }
+
+const DevInfo& dev_info() {
+    int dev = 0;
+    cudaGetDevice(&dev);
+    std::lock_guard<std::mutex> lk(g_mu);
+    DevInfo& d = g_dev[dev & 31];
+    if (d.done) return d;
+    d.done = true;
+    int major = 0;
+    cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+    cudaDeviceGetAttribute(&d.sms, cudaDevAttrMultiProcessorCount, dev);
+    if (major < 8) return d;
+    int occ = 1 << 20;
+    d.ok = setup_one<T_IQ2_XXS, true>(occ) && setup_one<T_IQ2_XS, true>(occ) && setup_one<T_IQ2_S, true>(occ) &&
+           setup_one<T_IQ3_XXS, true>(occ) && setup_one<T_IQ3_S, true>(occ) && setup_one<T_IQ4_XS, true>(occ) &&
+           setup_one<T_Q2_0, false>(occ) && setup_one<T_IQ4_NL, false>(occ);
+    d.occ = d.ok ? occ : 1;
+    cudaGetLastError();
+    return d;
+}
+
+bool gu_covered(int t) {
+    return t == T_IQ2_XXS || t == T_IQ2_XS || t == T_IQ2_S || t == T_IQ3_XXS || t == T_IQ3_S || t == T_IQ4_XS;
+}
+bool d_covered(int t) { return t == T_Q2_0 || t == T_IQ4_NL; }
+
+template <int T, bool GU>
+void launch(int ww, unsigned grid, const Batch& b, const NativeGeom& g, const Tables& tb, const void* act,
+            const int32_t* src, void* out, float* dm, cudaStream_t s) {
+    const uint8_t* a = (const uint8_t*) act;
+    uint8_t* o = (uint8_t*) out;
+    if (ww == 4) native_kernel<T, GU, 4><<<grid, THREADS, smem_bytes(T, 4), s>>>(b, g, tb, a, src, o, dm);
+    else native_kernel<T, GU, 2><<<grid, THREADS, smem_bytes(T, 2), s>>>(b, g, tb, a, src, o, dm);
+}
+
+// The work item's shape for a layer of `n` routed rows over `n_expert` experts: 128 routed rows when an expert has
+// about one such tile (56 to 112 rows on average), else 64.  Measured on the RTX 5070 (tests/cuda/prefill_fused_iq_test,
+// a layer of 2048 / 3584 / 8192 tokens, ms, 64 -> 128): IQ2_S 5.2 -> 6.7, 9.5 -> 9.3, 17.5 -> 19.5; IQ2_XXS 4.2 ->
+// 5.8, 7.4 -> 7.6, 13.6 -> 16.5; IQ3_S 5.6 -> 6.6, 10.2 -> 8.8, 18.4 -> 19.0; IQ3_XXS 5.1 -> 6.3, 9.1 -> 8.3, 16.5 ->
+// 17.5 (256 rows was slower still).  STRATA_PF_FUSED_TILE=64|128 forces one.
+int pick_ww(int64_t n, int n_expert) {
+    static const int forced = [] {
+        const char* v = std::getenv("STRATA_PF_FUSED_TILE");
+        const int t = v ? std::atoi(v) : 0;
+        return t == 64 ? 4 : t == 128 ? 2 : 0;
+    }();
+    if (forced) return forced;
+    const double avg = (double) n / std::max(n_expert, 1);
+    return avg > 56.0 && avg <= 112.0 ? 2 : 4;
+}
+
+}  // namespace
+
+bool native_supported(int gu_type, int d_type) {
+    static const bool off = [] {   // STRATA_PF_FUSED_NATIVE=0: the native packs keep MMQ under STRATA_PF_FUSED=1 (A/B)
+        const char* v = std::getenv("STRATA_PF_FUSED_NATIVE");
+        return v != nullptr && v[0] == '0';
+    }();
+    return !off && requested() && gu_covered(gu_type) && d_covered(d_type) && dev_info().ok;   // opt-in (=1)
+}
+
+void quantize_act_native(const float* x, int64_t rows, int64_t cols, void* xa, void* stream) {
+    if (rows <= 0) return;
+    const int64_t nblk = rows * (cols / 64);
+    quant_act_nat_kernel<<<blocks(nblk, 8), 256, 0, (cudaStream_t) stream>>>(x, nblk, (uint8_t*) xa);
+    ck(cudaGetLastError(), "quantize_act_native");
+}
+
+void experts_native(const Batch& b, const NativeGeom& g, int n_expert, int64_t n, const void* scratch, const void* xa,
+                    const int32_t* src, void* ha, float* dm, void* stream) {
+    if (b.e1 <= b.e0 || n <= 0) return;
+    const DevInfo& d = dev_info();
+    if (!d.ok || !gu_covered(g.gu_type) || !d_covered(g.d_type)) {
+        std::fprintf(stderr, "prefill fused experts (native): types %d / %d are not covered here\n", g.gu_type, g.d_type);
+        std::exit(1);
+    }
+    const cudaStream_t s = (cudaStream_t) stream;
+    const Tables tb = tables(const_cast<void*>(scratch), n_expert);
+    const int ww = pick_ww(n, n_expert);
+    // the most 64-row tiles the batch can have (every row in it, plus a partial tile per expert) - the items of the
+    // ones inside a larger item end at once
+    const int64_t tiles = (n + kTileRows - 1) / kTileRows + (b.e1 - b.e0);
+    const unsigned g_gu = (unsigned) std::min<int64_t>(tiles * (1280 / weight_rows(ww)), (int64_t) d.sms * d.occ);
+    const unsigned g_d = (unsigned) std::min<int64_t>(tiles * (2560 / weight_rows(ww)), (int64_t) d.sms * d.occ);
+    switch (g.gu_type) {
+        case T_IQ2_XXS: launch<T_IQ2_XXS, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        case T_IQ2_XS: launch<T_IQ2_XS, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        case T_IQ2_S: launch<T_IQ2_S, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        case T_IQ3_XXS: launch<T_IQ3_XXS, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        case T_IQ3_S: launch<T_IQ3_S, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+        default: launch<T_IQ4_XS, true>(ww, g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+    }
+    if (g.d_type == T_Q2_0) launch<T_Q2_0, false>(ww, g_d, b, g, tb, ha, src, nullptr, dm, s);
+    else launch<T_IQ4_NL, false>(ww, g_d, b, g, tb, ha, src, nullptr, dm, s);
+    ck(cudaGetLastError(), "experts_native");
+}
+
+}  // namespace strata::prefill::fused

src/prefill/moe_mmq.cu

@@ -29,6 +29,22 @@ __global__ void copy16_kernel(const uint4* __restrict__ a, int64_t na, const uin
     else if (i < na + nb) ab_dst[i] = b[i - na];
     else if (i < na + nb + nc) c_dst[i - na - nb] = c[i - na - nb];
 }
+// copy16_kernel for an MMQ group: blockIdx.y is the expert (first + y)
+struct GroupArgs {
+    const uint8_t* blob[kGatherGroupMax];
+    int64_t up_off, down_off, gu_stride, d_stride;   // in uint4
+};
+__global__ void copy16_group_kernel(GroupArgs ga, int first, int64_t na, int64_t nc, uint4* __restrict__ gu_dst,
+                                    uint4* __restrict__ d_dst) {
+    const int q = first + (int) blockIdx.y;
+    const uint4* src = (const uint4*) ga.blob[q];
+    uint4* ab = gu_dst + (int64_t) q * ga.gu_stride;
+    uint4* cd = d_dst + (int64_t) q * ga.d_stride;
+    const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < na) ab[i] = src[i];
+    else if (i < 2 * na) ab[i] = src[ga.up_off + (i - na)];
+    else if (i < 2 * na + nc) cd[i - 2 * na] = src[ga.down_off + (i - 2 * na)];
+}
 __global__ void copy1_kernel(const uint8_t* __restrict__ a, int64_t na, const uint8_t* __restrict__ b, int64_t nb,
                              uint8_t* __restrict__ ab_dst, const uint8_t* __restrict__ c, int64_t nc, uint8_t* __restrict__ c_dst) {
     const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
@@ -210,6 +226,25 @@ void gather_native(const void* gate, const void* up, size_t gu_half_bytes, const
     ck(cudaGetLastError(), "gather_native");
 }
 
+bool gather_native_group(const GatherGroup& g, size_t up_off, size_t gu_half_bytes, size_t down_off, size_t d_bytes,
+                         void* gu_dst, size_t gu_stride, void* d_dst, size_t d_stride, void* stream) {
+    if (g.first < 0 || g.n <= g.first || g.n > kGatherGroupMax) return false;
+    uintptr_t a = (uintptr_t) gu_dst | (uintptr_t) d_dst | up_off | gu_half_bytes | down_off | d_bytes | gu_stride | d_stride;
+    for (int q = g.first; q < g.n; ++q) a |= (uintptr_t) g.blob[q];
+    if (a % 16 != 0) return false;
+    GroupArgs ga{};
+    for (int q = g.first; q < g.n; ++q) ga.blob[q] = g.blob[q];
+    ga.up_off = (int64_t) up_off / 16;
+    ga.down_off = (int64_t) down_off / 16;
+    ga.gu_stride = (int64_t) gu_stride / 16;
+    ga.d_stride = (int64_t) d_stride / 16;
+    const int64_t na = (int64_t) gu_half_bytes / 16, nc = (int64_t) d_bytes / 16;
+    copy16_group_kernel<<<dim3(blocks(2 * na + nc), (unsigned) (g.n - g.first)), 256, 0, (cudaStream_t) stream>>>(
+        ga, g.first, na, nc, (uint4*) gu_dst, (uint4*) d_dst);
+    ck(cudaGetLastError(), "gather_native_group");
+    return true;
+}
+
 void gather_strata_q2(const uint8_t* blob, void* gu_dst, void* d_dst, void* stream) {
     strata_q2_kernel<<<blocks(1280LL * 40 + 2560LL * 10), 256, 0, (cudaStream_t) stream>>>(blob, (uint16_t*) gu_dst,
                                                                                          (uint16_t*) d_dst);

src/program/generate.cpp

@@ -23,6 +23,7 @@
 #include "strata/core/pinned.hpp"
 #include "strata/core/remote_experts.hpp"
 #include "strata/core/on_device.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/core/layer.hpp"
 #include "strata/core/layout.hpp"
 #include "strata/core/session.hpp"
@@ -77,6 +78,7 @@
 #include <chrono>
 #include <algorithm>
 #include <iostream>
+#include <future>
 #include <thread>
 #include <atomic>
 #include <condition_variable>
@@ -221,7 +223,7 @@ struct Options {
     /// Plan v0.3 P2: how the n-gram table is read. Direct (default) = unbuffered SSD reads, table never in RAM.
     std::string ple_io = "direct";
     int64_t ple_row_cache = 1 << 20;   ///< bounded row cache (rows of 90 B); 0 disables
-    int ple_inflight = 64;
+    int ple_inflight = 256;   // the prompt path reads a chunk's rows at once: 64 left the SSD half idle (32K: 303 -> 189 ms)
     double ple_delay_us = 0;           ///< fault injection: every row read completes no earlier than this
     bool ple_sync_submit = false;      ///< A/B arm: submit reads on the token thread, no I/O worker
     std::string kv = "fp16";           ///< plan v0.3 P7: KV storage, fp16 (default) or int8 (half the VRAM)
@@ -295,6 +297,16 @@ struct Options {
     /// to them.  Measured at 256 slots: **1781 of 60000 = 2.97%**, against **21.4%** for 8 slots per layer and
     /// **70.4%** for 64, from `Memory/cache_allocation.py` on the same run's routing.  Off by default.
     bool expert_cache_per_layer = false;
+    /// Multi-GPU: a second expert tier on CUDA device `peer_device` (-1 = off), `peer_reserve_mib` left free
+    /// there, `peer_slots` caps its size (0 = as many as fit), `peer_adapt_swaps` per adaptive round
+    /// (-1 = adapt_swaps).
+    int peer_device = -1;
+    int peer_reserve_mib = 600;
+    int64_t peer_slots = 0;
+    int peer_adapt_swaps = -1;
+    /// the prompt path's rows per layer the peer computes: -1 = half of chunk x top-k, 0 = the prompt path stays on
+    /// the primary
+    int64_t peer_prefill_rows = -1;
     /// The PLE gather's prefetch, as an A/B arm.  The gather measured 2.10-2.61 ms/token because its sixteen
     /// row reads are sixteen SEPARATE page faults into a 26.8 GB mapping; see `ple_prefetch_enable`.
     bool no_ple_prefetch = false;
@@ -303,6 +315,11 @@ struct Options {
     /// trace is what the plan's `h = 0.6447` refers to, and compulsory-miss measured 0.4864 because it fills
     /// with whatever the prompt touched FIRST.  Empty means no profile.
     std::string expert_profile;
+    /// #477 (--serve, opt-in): where to save what the adaptive tier learned, as a profile `--expert-profile` reads
+    /// (the resident experts first, then the routing counted since the start); on QUIT and every
+    /// `expert_profile_save_min` minutes between requests.  Empty (the default): nothing is counted or written.
+    std::string expert_profile_save;
+    double expert_profile_save_min = 10.0;
     /// R4.2d: **ON by default**, because the measurement is unambiguous and the alternative is known-broken.
     /// Without it, 17 of 10,562 layers had the hit work done when the pool returned; with it, 9,190.  The
     /// A/B arm is `--no-hit-poke`.
@@ -322,6 +339,7 @@ struct Options {
     /// Plan v0.3 P4: `--expert-cache auto` sizes the VRAM tier from what is free after the weights, the session
     /// and the KV state, minus this reserve for the graphs, the hit scratch and the head.
     int vram_reserve_mib = 700;
+    bool vram_reserve_given = false;   ///< --vram-reserve-mib on the command line (#496: no smaller automatic reserve)
     /// Plan v0.3 P5: batched prompt processing in chunks of this many tokens (0 = the token path).
     int64_t prefill_chunk = 0;
     /// `--prefill auto`: the largest chunk (up to 8192) whose buffers the expert cache can lend.  Every expert a chunk
@@ -357,6 +375,7 @@ struct Options {
     /// Plan v0.3 P6: every `adapt_every` rounds, swap up to `adapt_swaps` of the most-routed missing experts into
     /// the VRAM tier in place of the least-routed resident ones (decayed counts).  0 = static residency.
     int adapt_every = 4;
+    float adapt_decay = 0.7f;   ///< the usage counts are multiplied by this after each adaptation (--adapt-decay)
     /// Plan v0.3 P6: a draft enters the verify window only while every draft before it (and itself) has at least
     /// this probability under the draft layer; 0 = always --spec-1 drafts.
     double spec_min_p = 0.0;
@@ -428,7 +447,7 @@ void usage() {
                  "                       reads, the table never enters RAM or the file cache; mmap: A/B arm;\n"
                  "                       ram: mmap with the whole table locked in RAM at start (Linux/macOS)\n"
                  "  --ple-row-cache N    bounded cache of fetched rows, 90 B each (default 1048576; 0 = off)\n"
-                 "  --ple-inflight N     outstanding SSD reads (default 64)\n"
+                 "  --ple-inflight N     outstanding SSD reads (default 256)\n"
                  "  --ple-delay-us U     fault injection: each row read completes no earlier than U us\n"
                  "  --ple-sync-submit    A/B arm: submit table reads on the token thread (default: an I/O thread)\n"
                  "  --kv fp16|int8       KV storage (plan v0.3 P7): int8 codes + fp16 scale per 64 values, half the\n"
@@ -530,6 +549,10 @@ void usage() {
                  "                       real graph that --stage-timing cannot give.  Prints and exits.\n"
                  "  --expert-profile P   R4.2e: pre-load the VRAM tier from a `profile.bin` (see\n"
                  "                       tools/make_profile.py) instead of admitting on first use.\n"
+                 "  --expert-profile-save P  --serve, #477: save what the adaptive tier learned (the experts in\n"
+                 "                       VRAM, then the routing counted since the start) as a profile at P, on\n"
+                 "                       QUIT and every --expert-profile-save-every MIN minutes (default 10;\n"
+                 "                       0 = on QUIT only) between requests; start from it with --expert-profile P\n"
                  "  --no-hit-poke        R4.2d's A/B arm.  The hit path pokes the driver once right after its\n"
                  "                       launch so the GPU starts while the CPU pool runs; without it the work\n"
                  "                       waits for the next driver entry and does not overlap at all.\n"
@@ -544,6 +567,14 @@ void usage() {
                  "  --expert-cache-device3 N  pre-fill N more experts on CUDA3\n"
                  "  --expert-cache-remote-placement stripe|layer  distribute expert ranks or whole\n"
                  "                       layers across CUDA1..3 (default: stripe)\n"
+                 "  --peer-device N      a second GPU as an adaptive expert-cache tier (rows over P2P; it also\n"
+                 "                       computes its experts' prompt rows).  Not with --layer-split or\n"
+                 "                       --expert-cache-device1..3.  Default output unchanged without it.\n"
+                 "  --peer-reserve-mib N  VRAM the peer tier leaves free on its card (default 600)\n"
+                 "  --peer-slots N       expert slots on the peer (default: what fits)\n"
+                 "  --peer-adapt-swaps N  peer cache swaps per adaptation step (default: --adapt-swaps)\n"
+                 "  --peer-prefill-rows N  prompt rows per layer the peer computes (default half of chunk x top-k;\n"
+                 "                       0 = the prompt path stays on the primary)\n"
                  "  --expert-cache-per-layer  R4.2g: give each layer its OWN slots instead of letting the first\n"
                  "                       position take all of them.  The default policy fills in arrival order\n"
                  "                       from one shared counter, so 256 slots went to ~26 layers of position 0\n"
@@ -815,6 +846,12 @@ void mem_mark(const char* where) {
     std::fprintf(stderr, "strata trace: %lld MiB free after %s\n", (long long) (free_b >> 20), where);
 }
 
+/// #463's A/B: STRATA_ADAPT_NOWAIT=1 lets a verify window start before the adaptive tier's copies have landed (0.1.37)
+bool adapt_nowait() {
+    static const bool v = [] { const char* e = std::getenv("STRATA_ADAPT_NOWAIT"); return e && e[0] == '1'; }();
+    return v;
+}
+
 int argmax(const std::vector<float>& v) {
     int best = 0;
     for (size_t i = 1; i < v.size(); ++i)
@@ -947,35 +984,81 @@ bool load_control_vectors(const Options& o, const strata::core::ModelGeometry& g
 // The effective host->device bandwidth of the PCIe link: copies from pinned host memory, as the expert arena's
 // reads are.  The native default share (0.55) was measured on x16 links (~26-28 GB/s); a x8 card in a x8 slot
 // carries about half of that.  Returns < 0 when the probe cannot run (then the caller keeps the default).
-double probe_pcie_h2d_gbps() {
+//
+// #485: a single timed burst read an x16 PCIe 4 link (RTX A3000 laptop) at 18.5, 6.9 and 5.8 GB/s in three starts -
+// a link still in a low-power state, other DMA, a context not yet up to speed - and the low reading set the share.
+// Such a disturbance only ever slows a copy: nothing makes a correctly timed copy faster than the link carries.  So
+// the same 1 GiB is now copied as four bursts of 256 MiB, timed one by one, and the fastest is the link's figure (the
+// median would still follow a disturbance that lasts through half the bursts).  Each burst takes ~10 ms on an x16
+// PCIe 4 link, so the probe takes no longer than the one 1 GiB burst did.  `samples`, when given, gets every
+// burst's reading for the log.
+double probe_pcie_h2d_gbps(std::string* samples = nullptr) {
     constexpr size_t kBytes = 256ull << 20;
-    constexpr int kIters = 4;
+    constexpr int kBursts = 4;
     void* h = nullptr;
     void* d = nullptr;
-    cudaEvent_t ev0, ev1;
     if (cudaMallocHost(&h, kBytes) != cudaSuccess) return -1.0;
-    if (cudaMalloc(&d, kBytes) != cudaSuccess || cudaEventCreate(&ev0) != cudaSuccess ||
-        cudaEventCreate(&ev1) != cudaSuccess) {
-        if (d != nullptr) cudaFree(d);
+    if (cudaMalloc(&d, kBytes) != cudaSuccess) {
         cudaFreeHost(h);
         return -1.0;
     }
     std::memset(h, 0, kBytes);   // fault the pages in before timing
     cudaMemcpyAsync(d, h, kBytes, cudaMemcpyHostToDevice);   // warmup: context up, copy engine primed
-    cudaEventRecord(ev0);
-    for (int i = 0; i < kIters; ++i) cudaMemcpyAsync(d, h, kBytes, cudaMemcpyHostToDevice);
-    cudaEventRecord(ev1);
-    const bool ok = cudaEventSynchronize(ev1) == cudaSuccess;
-    float ms = 0.f;
-    const bool timed = ok && cudaEventElapsedTime(&ms, ev0, ev1) == cudaSuccess && ms > 0.01f;
-    const double bw = timed ? ((double) kIters * (double) kBytes / (ms * 1e-3)) / 1e9 : -1.0;
-    cudaEventDestroy(ev0);
-    cudaEventDestroy(ev1);
+    float ms[kBursts] = {};
+#if defined(STRATA_USE_HIP) && defined(_WIN32)
+    // Windows HIP: the events do not bracket the copies there (an RX 6800 read 3,300-26,000 GB/s, so every link kept
+    // the x16 share), so each burst is timed on the host between two synchronizes: 256 MiB takes ~10 ms, so the
+    // synchronize around it hardly matters
+    bool ok = cudaDeviceSynchronize() == cudaSuccess;
+    for (int b = 0; b < kBursts && ok; ++b) {
+        const Clock::time_point t0 = Clock::now();
+        cudaMemcpyAsync(d, h, kBytes, cudaMemcpyHostToDevice);
+        ok = cudaDeviceSynchronize() == cudaSuccess;
+        ms[b] = (float) std::chrono::duration<double, std::milli>(Clock::now() - t0).count();
+    }
+#else
+    // the bursts run back to back on the stream, an event between each two
+    cudaEvent_t ev[kBursts + 1];
+    int n_ev = 0;
+    while (n_ev <= kBursts && cudaEventCreate(&ev[n_ev]) == cudaSuccess) ++n_ev;
+    bool ok = n_ev == kBursts + 1;
+    if (ok) {
+        cudaEventRecord(ev[0]);
+        for (int b = 0; b < kBursts; ++b) {
+            cudaMemcpyAsync(d, h, kBytes, cudaMemcpyHostToDevice);
+            cudaEventRecord(ev[b + 1]);
+        }
+        ok = cudaEventSynchronize(ev[kBursts]) == cudaSuccess;
+        for (int b = 0; b < kBursts && ok; ++b) ok = cudaEventElapsedTime(&ms[b], ev[b], ev[b + 1]) == cudaSuccess;
+    }
+    for (int i = 0; i < n_ev; ++i) cudaEventDestroy(ev[i]);
+#endif
+    double bw = -1.0;
+    if (samples != nullptr) samples->clear();
+    for (int b = 0; b < kBursts && ok; ++b) {
+        const double s = ms[b] > 0.01f ? (double) kBytes / (ms[b] * 1e-3) / 1e9 : -1.0;
+        bw = std::max(bw, s);
+        if (samples != nullptr) {
+            char buf[32];
+            std::snprintf(buf, sizeof(buf), "%s%.1f", b == 0 ? "" : " ", s);
+            *samples += buf;
+        }
+    }
+    if (!ok) (void) cudaGetLastError();
     cudaFree(d);
     cudaFreeHost(h);
     return bw;
 }
 
+// The PCIe share of the missed experts for a link measured at `gbps`: `base` (the share measured on x16 links) from
+// 20 GB/s up, and below that in proportion to the bandwidth, so the time the link spends on its share stays about
+// what the x16 share costs.  Continuous (#485): before, 19.9 GB/s gave 0.42 and 20.0 the full 0.55 (and 4.0 GB/s
+// gave 0.08, 3.9 none), so a reading near either edge moved the share by a quarter of its range.  From 20 GB/s up
+// (an x16 PCIe 4/5 link: ~26-28 GB/s) the share is unchanged.
+double pcie_frac_for_gbps(double gbps, double base) {
+    return gbps <= 0.0 ? base : base * std::min(1.0, gbps / 20.0);
+}
+
 }  // namespace
 
 int main(int argc, char** argv) {
@@ -1115,7 +1198,7 @@ int main(int argc, char** argv) {
         else if (a == "--expert-cache-device3") o.expert_cache_remote[2] = std::atoi(next("--expert-cache-device3"));
         else if (a == "--expert-cache-remote-placement")
             o.expert_cache_remote_placement = next("--expert-cache-remote-placement");
-        else if (a == "--vram-reserve-mib") o.vram_reserve_mib = std::atoi(next("--vram-reserve-mib"));
+        else if (a == "--vram-reserve-mib") { o.vram_reserve_mib = std::atoi(next("--vram-reserve-mib")); o.vram_reserve_given = true; }
         else if (a == "--prefill") {
             const std::string v = next("--prefill");
             o.prefill_auto = v == "auto" || v.rfind("auto:", 0) == 0;
@@ -1137,6 +1220,7 @@ int main(int argc, char** argv) {
         else if (a == "--mtp-window") o.mtp_window = std::atoll(next("--mtp-window"));
         else if (a == "--pcie-frac") o.pcie_frac = std::atof(next("--pcie-frac"));
         else if (a == "--adapt-every") o.adapt_every = std::atoi(next("--adapt-every"));
+        else if (a == "--adapt-decay") o.adapt_decay = (float) std::atof(next("--adapt-decay"));
         else if (a == "--spec-min-p") o.spec_min_p = std::atof(next("--spec-min-p"));
         else if (a == "--stop-eos") o.stop_eos = true;
         else if (a == "--spec-split") o.spec_split = true;
@@ -1208,8 +1292,16 @@ int main(int argc, char** argv) {
         else if (a == "--adapt-swaps") o.adapt_swaps = std::atoi(next("--adapt-swaps"));
         else if (a == "--expert-cache-cpu-order") o.expert_cache_cpu_order = true;
         else if (a == "--expert-cache-per-layer") o.expert_cache_per_layer = true;
+        else if (a == "--peer-device") o.peer_device = std::atoi(next("--peer-device"));
+        else if (a == "--peer-reserve-mib") o.peer_reserve_mib = std::atoi(next("--peer-reserve-mib"));
+        else if (a == "--peer-slots") o.peer_slots = std::atoll(next("--peer-slots"));
+        else if (a == "--peer-adapt-swaps") o.peer_adapt_swaps = std::atoi(next("--peer-adapt-swaps"));
+        else if (a == "--peer-prefill-rows") o.peer_prefill_rows = std::atoll(next("--peer-prefill-rows"));
         else if (a == "--no-hit-poke") o.no_hit_poke = true;
         else if (a == "--expert-profile") o.expert_profile = next("--expert-profile");
+        else if (a == "--expert-profile-save") o.expert_profile_save = next("--expert-profile-save");
+        else if (a == "--expert-profile-save-every")
+            o.expert_profile_save_min = std::atof(next("--expert-profile-save-every"));
         else if (a == "--gpu-stages") o.gpu_stages = true;
         else if (a == "--mmap-experts") o.mmap_experts = true;
         else if (a == "--shared-expert-arena") o.shared_expert_arena = next("--shared-expert-arena");
@@ -1254,6 +1346,7 @@ int main(int argc, char** argv) {
         }
     }
     strata::core::set_coupled_draft(o.coupled_draft);
+    strata::core::set_peer_portable(o.peer_device >= 1);   // multi-GPU: the Portable flag on mapped host buffers only with a peer device (before any allocation)
     if (o.serve && o.conversation_cache_mib > 0 && (o.prompt_cache == 0 || o.conversation_cache_slots == 0))
         std::fprintf(stderr, "strata serve: warning: conversation caching is disabled by %s\n",
                      o.prompt_cache == 0 ? "--prompt-cache 0" : "--conversation-cache-slots 0");
@@ -1265,6 +1358,8 @@ int main(int argc, char** argv) {
     // the next visible ones); "auto" places the K from each GPU's free VRAM once the weights are in (below).  Across
     // GPUs, not yet: KV streaming, images, control vectors, the helper caches (--expert-cache-remote), and lending
     // cache slots to the prompt path (each stage's prompt path has its own buffers).
+    // --pcie-frac given: that one share is every stage's (the stages' own link probes are skipped), so a split over
+    // a fast and a slow link cannot set the two apart from the command line (#485)
     const bool pcie_given = o.pcie_frac >= 0.0;
     std::vector<int64_t> split_at;
     std::vector<int> split_devs;
@@ -1618,6 +1713,14 @@ int main(int argc, char** argv) {
         std::fprintf(stderr, "strata generate: --expert-cache-remote-placement must be stripe or layer\n");
         return 2;
     }
+    // the peer tier is the second card's only user: a layer split or a remote expert cache would put a second engine
+    // part (and a second copy of the same experts) on it
+    if (o.peer_device >= 1 && (!o.layer_split.empty() || o.expert_cache_remote[0] > 0)) {
+        std::fprintf(stderr, "strata generate: --peer-device cannot be combined with %s\n",
+                     !o.layer_split.empty() ? "--layer-split (use one or the other)"
+                                            : "--expert-cache-device1..3 (the peer tier already caches experts there)");
+        return 2;
+    }
 
     if (o.native_flash_attn_short && o.max_context > 256) {
         std::fprintf(stderr, "strata generate: --native-flash-attn-short requires --max-context <=256\n");
@@ -1720,15 +1823,14 @@ int main(int argc, char** argv) {
     // pack's 0.2 was never measured against the link, so it is left alone.  `--calibrate` measures it outright.
     if (o.pcie_frac < 0.0) {
         const double base = native_pack ? 0.55 : 0.2;
-        const double bw = native_pack ? probe_pcie_h2d_gbps() : -1.0;
+        std::string bursts;
+        const double bw = native_pack ? probe_pcie_h2d_gbps(&bursts) : -1.0;
         if (!native_pack) {
             o.pcie_frac = base;
         } else if (bw > 0.0) {
-            // below ~4 GB/s (an x1 link: ~0.9 GB/s) a missed expert's 1.4 MB takes longer to cross than the CPU
-            // pool takes to compute it, so none of them go over the link
-            o.pcie_frac = bw >= 20.0 ? base : bw < 4.0 ? 0.0 : std::min(base, std::max(0.05, base * (bw / 26.0)));
-            std::fprintf(stderr, "strata generate: PCIe probe: %.1f GB/s host->device -> pcie_frac %.2f (default %.2f)\n",
-                         bw, o.pcie_frac, base);
+            o.pcie_frac = pcie_frac_for_gbps(bw, base);
+            std::fprintf(stderr, "strata generate: PCIe probe: %.1f GB/s host->device (best of %s) -> pcie_frac %.2f "
+                                 "(default %.2f)\n", bw, bursts.c_str(), o.pcie_frac, base);
         } else {
             o.pcie_frac = base;
             std::fprintf(stderr, "strata generate: PCIe probe failed -> pcie_frac default %.2f\n", base);
@@ -1888,9 +1990,18 @@ int main(int argc, char** argv) {
         return 1;
     }
     void* arena = nullptr;
-    if (cudaMalloc(&arena, pool_bytes) != cudaSuccess) {
-        std::fprintf(stderr, "strata generate: cudaMalloc(%llu) for the weight arena failed\n",
-                     (unsigned long long) pool_bytes);
+    if (const cudaError_t ce = cudaMalloc(&arena, pool_bytes); ce != cudaSuccess) {
+        // #486: the arena is the first large allocation and its size does not depend on the context, so what is
+        // missing is held by something else: say how much was free
+        cudaGetLastError();
+        size_t free_b = 0, total_b = 0;
+        cudaMemGetInfo(&free_b, &total_b);
+        std::fprintf(stderr, "strata generate: cudaMalloc(%llu) for the weight arena failed (%s): %llu MiB of %llu "
+                             "MiB VRAM free on this GPU. The arena is allocated first, before the KV and expert "
+                             "caches: another program (or an engine that is still exiting) holds the rest - "
+                             "nvidia-smi / rocm-smi lists them\n",
+                     (unsigned long long) pool_bytes, cudaGetErrorString(ce), (unsigned long long) (free_b >> 20),
+                     (unsigned long long) (total_b >> 20));
         return 1;
     }
     strata::core::WeightTable wt;
@@ -2195,8 +2306,14 @@ int main(int argc, char** argv) {
         void* arena_s = nullptr;
         if (cudaMalloc(&arena_s, pool_bytes) != cudaSuccess ||
             !st.wt.load(o.pack, arena_s, pool_bytes, err, skip.empty() ? nullptr : &skip)) {
-            std::fprintf(stderr, "strata generate: layer split, CUDA%d weights: %s\n", st.dev,
-                         err.empty() ? "the weight arena does not fit" : err.c_str());
+            cudaGetLastError();
+            size_t free_b = 0, total_b = 0;   // #486: what that card had free
+            cudaMemGetInfo(&free_b, &total_b);
+            std::fprintf(stderr, "strata generate: layer split, CUDA%d weights: %s (%llu MiB needed, %llu MiB of %llu "
+                                 "MiB free on that card)\n", st.dev,
+                         err.empty() ? "the weight arena does not fit" : err.c_str(),
+                         (unsigned long long) (pool_bytes >> 20), (unsigned long long) (free_b >> 20),
+                         (unsigned long long) (total_b >> 20));
             return 1;
         }
         if (!o.native_dense_gguf.empty() && !st.dense.load(o.native_dense_gguf, st.wt, err, o.native_ple_key)) {
@@ -2229,13 +2346,15 @@ int main(int argc, char** argv) {
             }
             strata::kernels::mrope_table_set(st.mrope);
         }
-        // its own PCIe share of the missed experts (the same rule as CUDA0's above: its link is probed)
+        // its own PCIe share of the missed experts (the same rule as CUDA0's above: its link is probed).  A given
+        // --pcie-frac is every stage's share and skips these probes (pcie_given); there is no per-stage setting yet.
         st.pcie_frac = o.pcie_frac;
         if (!pcie_given && native_pack) {
-            const double bw = probe_pcie_h2d_gbps();
-            if (bw > 0.0) st.pcie_frac = bw >= 20.0 ? 0.55 : bw < 4.0 ? 0.0 : std::min(0.55, std::max(0.05, 0.55 * (bw / 26.0)));
-            std::fprintf(stderr, "strata generate: layer split: CUDA%d PCIe probe %.1f GB/s -> pcie_frac %.2f\n", st.dev,
-                         bw, st.pcie_frac);
+            std::string bursts;
+            const double bw = probe_pcie_h2d_gbps(&bursts);
+            if (bw > 0.0) st.pcie_frac = pcie_frac_for_gbps(bw, 0.55);
+            std::fprintf(stderr, "strata generate: layer split: CUDA%d PCIe probe %.1f GB/s (best of %s) -> pcie_frac "
+                                 "%.2f\n", st.dev, bw, bursts.c_str(), st.pcie_frac);
         }
         size_t fb = 0, tb = 0;
         cudaMemGetInfo(&fb, &tb);
@@ -2255,9 +2374,43 @@ int main(int argc, char** argv) {
         // file).  `--expert-cache 0` used to take the count the profile was built for; the profile now ranks
         // every pair (issue #46: a card that holds more than the old 8,000 used to stop there), so it means auto.
         if (o.expert_cache == 0) o.expert_cache = -1;
+        // Multi-GPU: STRATA_PEER_HOT=<f> gives the peer card a share f of the HOT pairs, so both cards
+        // compute routed experts every layer (the primary alone did ~25 of ~30 per layer-window).  Of the first
+        // STRATA_PEER_HOT_AT (default 8700, ~ the primary's slots) ranks, every pair with floor((r+1)f) > floor(rf)
+        // moves to just after that point: the primary fills past them, the peer (which takes what the primary does
+        // not hold, in order) gets them first.
+        if (o.peer_device >= 1) {   // default 0.45 (measured: 0.3-0.6 all better than 0; 0 = off, e.g. for the gate)
+            const char* ph = std::getenv("STRATA_PEER_HOT");
+            const double f = ph ? std::atof(ph) : 0.45;
+            const char* pa = std::getenv("STRATA_PEER_HOT_AT");
+            const size_t at = std::min(profile.size(), (size_t) (pa ? std::atoll(pa) : 8700));
+            if (f > 0.0 && f < 1.0 && at > 0) {
+                std::vector<std::pair<int32_t, int32_t>> keep, moved;
+                for (size_t r = 0; r < at; ++r) {
+                    const bool to_peer = (int64_t) ((double) (r + 1) * f) > (int64_t) ((double) r * f);
+                    (to_peer ? moved : keep).push_back(profile[r]);
+                }
+                const size_t n_moved = moved.size();
+                // the primary's share continues with the ranks after `at` until it is full; then the moved ones
+                std::vector<std::pair<int32_t, int32_t>> out;
+                out.reserve(profile.size());
+                out.insert(out.end(), keep.begin(), keep.end());
+                const size_t fill = std::min(profile.size(), at + n_moved);   // what the primary still takes
+                out.insert(out.end(), profile.begin() + (long) at, profile.begin() + (long) fill);
+                out.insert(out.end(), moved.begin(), moved.end());
+                out.insert(out.end(), profile.begin() + (long) fill, profile.end());
+                profile.swap(out);
+                std::fprintf(stderr, "strata generate: STRATA_PEER_HOT %.2f: %zu of the first %zu ranked pairs moved "
+                                     "behind rank %zu (the peer's)\n", f, n_moved, at, fill);
+            }
+        }
         std::fprintf(stderr, "strata generate: profile %s: %zu ranked pairs, built for %lld slots\n",
                      o.expert_profile.c_str(), profile.size(), (long long) pslots);
     }
+    // #477: the whole ranking as loaded, the prior of --expert-profile-save's order (a layer split keeps only
+    // CUDA0's pairs in `profile` below).  Empty without --expert-profile-save.
+    std::vector<std::pair<int32_t, int32_t>> profile_loaded;
+    if (!o.expert_profile_save.empty()) profile_loaded = profile;
     // ---- layer split across GPUs: "auto" places the split points by a cost model of one decode window, measured on
     // the 5080 + 3090 rig (bench/results/2026-09-29-layer-split):
     //   - every layer costs its GPU a time inversely proportional to SMs x clock (0.33 ms on an RTX 5080, 0.50 on a
@@ -2565,6 +2718,16 @@ int main(int argc, char** argv) {
         std::fprintf(stderr, "strata generate: GPU %d: %s, compute capability %d.%d%s\n", dev, name,
                      strata::cc_major_of(p.major), strata::cc_minor_of(p.minor),
                      strata::emulated_cc() ? " (STRATA_EMULATE_CC: a test mode, the card is emulated)" : "");
+        {   // #542: a build whose libcudart is older than its headers (a CUDA 13 kit with a dangling libcudart.so that
+            // CMake resolved to the system's CUDA 12 one) reads cudaDeviceProp shifted - silently, and slowly
+            int rt = 0;
+            if (cudaRuntimeGetVersion(&rt) == cudaSuccess && rt / 1000 != CUDART_VERSION / 1000)
+                std::fprintf(stderr, "strata generate: WARNING: this engine was compiled with CUDA %d.%d headers but "
+                                     "loaded a CUDA %d.%d runtime (libcudart): GPU properties can read wrong and some "
+                                     "kernels go unused. Rebuild it against one toolkit (cmake -DCUDAToolkit_ROOT=<the "
+                                     "toolkit>, with its libcudart.so present) (#542)\n",
+                             CUDART_VERSION / 1000, CUDART_VERSION % 1000 / 10, rt / 1000, rt % 1000 / 10);
+        }
 #endif
         const std::string e = strata::core::device_code_error();
         if (!e.empty()) {
@@ -2591,6 +2754,15 @@ int main(int argc, char** argv) {
         }
         std::fprintf(stderr, "strata generate: experts via mmap (--mmap-experts; %s)\n",
                      src.gguf_mode() ? "the GGUF shards in place, no experts.bin" : "the A/B arm of R2.1");
+        // #286: with a RAM budget the hottest experts live in it, and the rest are read from the drive unbuffered
+        // when the file cache could not keep them beside the budget anyway (a 32 GB PC) - the mapped reads' page
+        // faults are small requests on the critical path, and their pages take the RAM the budget was sized for
+        if (o.resident_budget > 0 || std::getenv("STRATA_UNBUFFERED_LOAD") != nullptr) {
+            std::string why;
+            const bool ub = src.set_unbuffered(o.resident_budget, why);
+            std::fprintf(stderr, "strata generate: the file tier reads %s (%s)\n",
+                         ub ? "unbuffered" : "through the file cache", why.c_str());
+        }
         srcp = &src;
     } else {
         arena_src.set_gguf(o.native_preset);   // plan v0.3 P6: a native pack may take its experts from shard 1
@@ -2670,6 +2842,7 @@ int main(int argc, char** argv) {
         return 1;
     }
     const bool auto_cache = o.expert_cache < 0;
+    bool reserve_adapted = false;   // #496: the auto sizing lowered the reserve so a small card's cache fits
     if (o.expert_cache < 0) {
         size_t free_b = 0, total_b = 0;
         cudaMemGetInfo(&free_b, &total_b);
@@ -2683,15 +2856,63 @@ int main(int argc, char** argv) {
         const int64_t mtp_bind = (!o.mtp.empty() && native_head.loaded())
                                      ? (int64_t) mtp.bind_bytes(native_head.row_bytes(), n_vocab) : 0;
         const int64_t reserve = (((int64_t) o.vram_reserve_mib + prefill_mib) << 20) + mtp_bind;
-        int64_t slots = ((int64_t) free_b - reserve) / (int64_t) strata::kernels::cpu::expert_layout().max_blob;
+        const int64_t blob = (int64_t) strata::kernels::cpu::expert_layout().max_blob;
+        int64_t slots = ((int64_t) free_b - reserve) / blob;
         if (!profile.empty()) slots = std::min<int64_t>(slots, (int64_t) profile.size());
         o.expert_cache = (int) std::max<int64_t>(slots, 0);
         std::fprintf(stderr, "strata generate: expert cache auto: %.2f GiB free, %d MiB reserved (+%lld MiB for the "
                              "draft head) -> %d slots\n",
                      (double) free_b / 1073741824.0, o.vram_reserve_mib, (long long) (mtp_bind >> 20), o.expert_cache);
-        if (o.expert_cache == 0)   // the verify window cannot start without it (#174): say what makes room
-            std::fprintf(stderr, "strata generate: no VRAM is left for the expert cache: lower --max-context, use "
-                                 "--kv k8v4, run images on the CPU, or close other programs that use the GPU\n");
+        // #496: the verify window cannot start without a cache (#174), and a cache too small to lend the prompt path
+        // a 256-token chunk's buffers (plus the 128 slots a loan leaves; one slot without --prefill) makes it
+        // allocate its own on top - more than the reserve.  When the default reserve leaves less than that (a 6 GB
+        // card), the reserve shrinks to what leaves exactly that cache, down to kSmallReserveMib: what is allocated
+        // after the cache - the prompt path's own part, the verify buffers, the draft head - comes out of the reserve,
+        // and below ~550 MiB a card ends with less than the 256 MiB the serve check calls LOW (IQ3_XXS, 32K, a 300 MiB
+        // reserve: 5 MiB left), so the cache gets no more than it needs, and the serve check says so plainly when it
+        // ends LOW (`reserve_adapted`).  A reserve given on the command line is kept.  No slot at all: the start
+        // stops, saying what is short and what makes room.  A card the default reserve leaves that much is sized as
+        // before.
+        constexpr int kSmallReserveMib = 300;
+        const int64_t min_slots = (o.prefill_chunk > 0 && pf_borrow)
+            ? ((int64_t) strata::prefill::Prefill::bytes_needed(g, ss, 256) + blob - 1) / blob + 128 : 1;
+        if (o.expert_cache < min_slots && !o.vram_reserve_given && o.vram_reserve_mib > kSmallReserveMib) {
+            // the largest reserve (in MiB) that still leaves min_slots
+            const int64_t fit_mib = ((int64_t) free_b - mtp_bind - min_slots * blob) / (1 << 20) - prefill_mib;
+            if (fit_mib >= kSmallReserveMib) {
+                const int r = (int) std::min<int64_t>(fit_mib, o.vram_reserve_mib);
+                int64_t s2 = ((int64_t) free_b - ((((int64_t) r + prefill_mib) << 20) + mtp_bind)) / blob;
+                if (!profile.empty()) s2 = std::min<int64_t>(s2, (int64_t) profile.size());
+                std::fprintf(stderr, "strata generate: expert cache auto: the %d MiB reserve leaves too few slots on "
+                                     "this card (a working cache needs %lld): a %d MiB reserve instead -> %lld slots\n",
+                             o.vram_reserve_mib, (long long) min_slots, r, (long long) s2);
+                o.vram_reserve_mib = r;
+                o.expert_cache = (int) s2;
+                reserve_adapted = true;
+            }
+        }
+        if (o.expert_cache == 0) {
+            // what is short, and what makes room: the numbers a small card picks from
+            const int64_t at_reserve = o.vram_reserve_given ? o.vram_reserve_mib
+                                                            : std::min(o.vram_reserve_mib, kSmallReserveMib);
+            const int64_t need_b = (((int64_t) at_reserve + prefill_mib) << 20) + mtp_bind + min_slots * blob;
+            const int64_t short_mib = std::max<int64_t>(1, (need_b - (int64_t) free_b + (1 << 20) - 1) >> 20);
+            const int64_t session_mib =
+                (int64_t) (strata::core::session_bytes(g, o.max_context, K, 0, g.n_layers) >> 20);
+            const std::string reserve_tip =
+                o.vram_reserve_given && o.vram_reserve_mib > kSmallReserveMib
+                    ? ", a smaller --vram-reserve-mib (" + std::to_string(o.vram_reserve_mib) + " now; " +
+                          std::to_string(kSmallReserveMib) + " is enough on a small card)"
+                    : std::string();
+            std::fprintf(stderr, "strata generate: no VRAM is left for the expert cache: it needs at least %lld slots "
+                                 "(%lld MiB), about %lld MiB more than this card has free. To make room: a smaller "
+                                 "--max-context (the session, mostly its KV cache, takes %lld MiB at %lld tokens), "
+                                 "--kv q4_0, the English draft subset (setup --draft-vocab en; the draft head takes "
+                                 "%lld MiB now)%s, images on the CPU, or close other programs that use the GPU\n",
+                         (long long) min_slots, (long long) ((min_slots * blob) >> 20), (long long) short_mib,
+                         (long long) session_mib, (long long) o.max_context, (long long) (mtp_bind >> 20),
+                         reserve_tip.c_str());
+        }
     } else if (multi_gpu && o.expert_cache > 0) {
         // an explicit cache size leaves room for the prompt path's buffers and the reserve, or the first prompt
         // fails with "device buffers ... do not fit" (with borrowing - the default with a profile - the path lends
@@ -2859,7 +3080,15 @@ int main(int argc, char** argv) {
         const bool per_layer = xcache.per_layer_admission();
         const int64_t want = per_layer ? (int64_t) profile.size()
                                        : std::min<int64_t>((int64_t) profile.size(), xcache.slots());
+        // #286: an unbuffered file tier reads the pairs in batches of 64, the next batch while this one is copied
+        std::future<void> ahead;
+        auto read_batch = [&](int64_t at) { src.prefetch_pairs(profile.data() + at, std::min<int64_t>(64, want - at)); };
         for (int64_t i = 0; i < want; ++i) {
+            if (!per_layer && srcp == &src && src.unbuffered() && i % 64 == 0) {
+                if (ahead.valid()) ahead.get();
+                else read_batch(i);
+                if (i + 64 < want) ahead = std::async(std::launch::async, read_batch, i + 64);
+            }
             const int32_t slot = xcache.admit(profile[(size_t) i].first, profile[(size_t) i].second);
             if (slot == strata::core::kNotResident) {
                 if (per_layer) continue;
@@ -3021,9 +3250,29 @@ int main(int argc, char** argv) {
         }
     }
 
+    // ---- Multi-GPU: the second GPU's expert tier, filled with the ranked pairs the primary does not hold
+    strata::core::PeerExperts peer;
+    if (o.peer_device >= 1) {
+        if (profile.empty() || srcp == nullptr || o.expert_cache <= 0) {
+            std::fprintf(stderr, "strata generate: --peer-device needs --expert-profile and the expert cache\n");
+            return 1;
+        }
+        const auto tp0 = Clock::now();
+        if (!peer.open(o.peer_device, profile, xcache, *srcp, g.n_layers, g.n_expert, o.peer_reserve_mib, o.peer_slots,
+                       err)) {
+            std::fprintf(stderr, "strata generate: %s\n", err.c_str());
+            return 1;
+        }
+        std::fprintf(stderr, "strata generate: peer GPU %d: %lld experts, %.2f GiB (filled in %.1f s); with the primary's "
+                             "%lld that is %lld of %lld on the GPUs\n", o.peer_device, (long long) peer.resident(), peer.gib(),
+                     std::chrono::duration<double>(Clock::now() - tp0).count(), (long long) xcache.slots(),
+                     (long long) (peer.resident() + xcache.slots()), (long long) (g.n_layers * g.n_expert));
+    }
+
     Drive drive;
     for (int r = 0; r < 3; ++r) if (o.expert_cache_remote[(size_t) r] > 0)
         drive.d.remote[drive.d.remote_count++] = &remote_experts[(size_t) r];
+    drive.d.peer = peer.valid() ? &peer : nullptr;
     drive.d.hit_cpu_order = o.expert_cache_cpu_order;
     drive.d.split_rows = !o.no_split_rows;
     drive.d.pool = &pool;
@@ -3288,6 +3537,10 @@ int main(int argc, char** argv) {
     }
 
     std::FILE* dump = nullptr;
+    // The logits header is written WITH THE FIRST ROW, not at open: a native pack never reaches the
+    // per-token dump site, and a header promising rows that were never written is worse than no file.
+    int32_t hdr[2] = {0, 0};
+    bool hdr_written = false;
     const int64_t dump_positions = (int64_t) o.tokens.size() - 1 + o.max_new;
     if (!o.dump_logits.empty()) {
         if (dump_positions > INT32_MAX || n_vocab > INT32_MAX) {
@@ -3307,12 +3560,7 @@ int main(int argc, char** argv) {
         // of the size gets a wrong answer that looks authoritative.  `tools/logits_identical.py` caught it by
         // parsing the header and refusing the file.
         const int32_t n_rows = (int32_t) strata::program::logits_selection::row_count(dump_positions, o.logits_stride);
-        const int32_t hdr[2] = {(int32_t) n_vocab, n_rows};
-        if (std::fwrite(hdr, sizeof hdr, 1, dump) != 1) {
-            std::fprintf(stderr, "strata generate: cannot write logits header\n");
-            std::fclose(dump);
-            return 1;
-        }
+        hdr[0] = n_vocab; hdr[1] = n_rows;
     }
 
     // ---- THE C1 ORACLE: ONE RESIDUAL SNAPSHOT PER LAYER PER POSITION, so the engine can be bisected against
@@ -3697,7 +3945,7 @@ int main(int argc, char** argv) {
     // and each generated token is written to stdout as `T <id>` as soon as its verify window is done, followed by
     //
     //     DONE <generated> <prompt_tokens> <prompt_ms> <decode_ms> <stop|length|cancel> <drafts accepted>
-    //          <drafts offered> <prompt tokens reused>
+    //          <drafts offered> <prompt tokens reused> ... <prompt tokens read>   (see the DONE line below; #471)
     //
     // Before that, `RESUME <n>` (n prompt tokens are not read again), `PP <position> <prompt_tokens> <ms> <tok/s>`
     // after every prompt chunk, and `REUSED <n>` once the prompt is read.  (`ERR <message>` instead when a request
@@ -4146,6 +4394,13 @@ int main(int argc, char** argv) {
                                      "context, or read prompts in smaller chunks (--prefill 512)\n");
             return 1;
         }
+        if (peer.valid() && o.peer_prefill_rows != 0) {
+            const int64_t rows = o.peer_prefill_rows > 0 ? o.peer_prefill_rows : o.prefill_chunk * K / 2;
+            if (!sp.set_peer(&peer, rows, err)) {
+                std::fprintf(stderr, "strata serve: %s - the prompt path stays on the primary GPU\n", err.c_str());
+                err.clear();
+            }
+        }
         mem_mark("the head and the prompt path");
         // #340: STRATA_SPLIT_SMALL_OWN=S (tokens): on a layer split, every stage that borrows keeps the slots for an
         // S-token chunk's buffers for the whole session (0.1.29's own buffers, carved from the tail of its cache):
@@ -4373,6 +4628,9 @@ int main(int argc, char** argv) {
             return true;
         };
         int64_t pp_total = 0, pp_from = 0, pp_next_check = 0;
+        // #471: the position the prompt pass has read up to (a chunk's or a window's end): what a request cancelled
+        // mid-read reports as read, instead of the whole prompt
+        int64_t pp_reached = 0;
         Clock::time_point pp_t0 = Clock::now();
         auto imgs_below = [&](const std::vector<ImgKey>& all, int64_t L) {
             std::vector<ImgKey> v;
@@ -4436,6 +4694,7 @@ int main(int argc, char** argv) {
                              batched ? "batched" : "token", mtp.kv_state().kv_mode, (long long) T);
             // progress for the server window: PP <position reached> <prompt tokens> <ms> <fresh tokens/s>
             const int64_t done = p0 + T;
+            pp_reached = done;
             const double ms = std::chrono::duration<double, std::milli>(Clock::now() - pp_t0).count();
             std::printf("PP %lld %lld %.0f %.1f\n", (long long) done, (long long) pp_total, ms,
                         ms > 0.0 ? 1000.0 * (double) (done - pp_from) / ms : 0.0);
@@ -4486,6 +4745,17 @@ int main(int argc, char** argv) {
         drive.d.plan = ver.plan_sink();
         drive.d.pcie_num = std::max(0, std::min(256, (int) (o.pcie_frac * 256.0 + 0.5)));
         if (o.adapt_every > 0 && o.adapt_swaps > 0) drive.d.usage.assign((size_t) (g.n_layers * g.n_expert), 0.0f);
+        // #477 --expert-profile-save: what the adaptive tier learned, kept across restarts (opt-in; off: `heat` stays
+        // empty and nothing below runs).  It needs the adaptive tier's counts and the residency table.
+        std::vector<double> heat;
+        if (!o.expert_profile_save.empty()) {
+            if (drive.d.usage.empty() || host_res.empty())
+                std::fprintf(stderr, "strata serve: --expert-profile-save needs the adaptive tier (--adapt-every and "
+                                     "--adapt-swaps above 0) and --expert-profile: nothing will be saved\n");
+            else
+                heat.assign(drive.d.usage.size(), 0.0);
+        }
+        Clock::time_point profile_saved_at = Clock::now();
         cudaStream_t adapt_stream = nullptr;
         if (cudaStreamCreateWithFlags(&adapt_stream, cudaStreamNonBlocking) != cudaSuccess) {
             std::fprintf(stderr, "strata serve: cannot create the refill stream\n");
@@ -4505,6 +4775,7 @@ int main(int argc, char** argv) {
             }
         };
         auto apply_pending = [&](bool wait) {
+            if (peer.valid()) peer.apply_pending(wait);
             if (pending.empty()) return;
             if (wait) cudaEventSynchronize(adapt_ev);
             else if (cudaEventQuery(adapt_ev) != cudaSuccess) return;
@@ -4531,7 +4802,7 @@ int main(int argc, char** argv) {
                 const float* u = drive.d.usage.data() + l * g.n_expert;
                 const int32_t* r = host_res.data() + l * g.n_expert;
                 for (int32_t e = 0; e < (int32_t) g.n_expert; ++e) {
-                    if (r[e] < 0) { if (u[e] >= 2.0f) cand.emplace_back(u[e], e); }
+                    if (r[e] < 0) { if (u[e] >= 2.0f && !(peer.valid() && peer.has(l, e))) cand.emplace_back(u[e], e); }
                     else vict.emplace_back(u[e], e);
                 }
                 if (cand.empty() || vict.empty()) continue;
@@ -4572,9 +4843,40 @@ int main(int argc, char** argv) {
                     const strata::core::OnDevice on(st->dev);
                     cudaEventRecord(st->adapt_ev, st->adapt_stream);
                 }
-            for (float& v : drive.d.usage) v *= 0.7f;
+            // #477: the routing counted since the start (each count adds up to 1 / (1 - --adapt-decay) over its
+            // decays: the sum is proportional to the routing itself) - only with --expert-profile-save, else `heat`
+            // is empty
+            for (size_t i = 0; i < heat.size(); ++i) heat[i] += (double) drive.d.usage[i];
+            if (peer.valid()) {                   // multi-GPU: the peer takes the next most-routed CPU misses
+                std::vector<int32_t> r0 = host_res;
+                for (const auto& [i, slot] : pending) r0[(size_t) i] = slot;   // swapped into the primary already
+                std::string perr;
+                if (!peer.adapt(drive.d.usage.data(), r0.data(),
+                                o.peer_adapt_swaps >= 0 ? o.peer_adapt_swaps : o.adapt_swaps, perr)) {
+                    std::fprintf(stderr, "strata serve: %s\n", perr.c_str());
+                    return false;
+                }
+            }
+            for (float& v : drive.d.usage) v *= o.adapt_decay;
             return true;
         };
+        // #477: write the learned profile (between requests and at QUIT: a prompt's lent slots are back by then).
+        // A swap still in flight counts as done - its expert is resident once the copy lands.  `why`: for the log.
+        auto save_profile = [&](const char* why) {
+            if (heat.empty()) return;
+            std::vector<uint8_t> resident(host_res.size(), 0);
+            for (size_t i = 0; i < host_res.size(); ++i) resident[i] = host_res[i] >= 0;
+            for (const auto& p : pending) resident[(size_t) p.first] = 1;
+            std::string e;
+            const auto ranked = strata::core::rank_learned_profile(g.n_layers, g.n_expert, resident, heat,
+                                                                   profile_loaded);
+            if (strata::core::write_expert_profile(o.expert_profile_save, g.n_layers, g.n_expert, ranked, e))
+                std::fprintf(stderr, "strata serve: expert profile saved to %s (%s)\n", o.expert_profile_save.c_str(),
+                             why);
+            else
+                std::fprintf(stderr, "strata serve: the expert profile was not saved: %s\n", e.c_str());
+            profile_saved_at = Clock::now();
+        };
         // stdin is read on its own thread, so a STOP line reaches a request that is still running (the client went
         // away, or pressed Esc): the flag is checked between prompt chunks and between verify windows.
         std::atomic<bool> stop_req{false};
@@ -4648,6 +4950,12 @@ int main(int argc, char** argv) {
             const int64_t free_mib = (int64_t) (free_b >> 20);
             if (free_mib >= 256) {
                 std::fprintf(stderr, "strata serve: %lld MiB of VRAM free with everything loaded\n", (long long) free_mib);
+            } else if (reserve_adapted) {
+                // #496: the reserve was already lowered to make the cache fit - a bigger one would leave it no room
+                std::fprintf(stderr, "strata serve: WARNING: %lld MiB of VRAM free with everything loaded - this card "
+                                     "only just fits the model (the VRAM reserve was lowered to %d MiB so the expert "
+                                     "cache fits): requests may stall. Close other programs that use the GPU, or lower "
+                                     "--max-context\n", (long long) free_mib, o.vram_reserve_mib);
             } else {
                 std::fprintf(stderr, "strata serve: %lld MiB of VRAM free with everything loaded - LOW: requests may stall;"
                                      " add --vram-reserve-mib %lld to the config's args (or lower --max-context)\n",
@@ -4737,6 +5045,10 @@ int main(int argc, char** argv) {
         std::vector<float> img_rows;
         std::vector<const float*> row_ptr;
         while (next_line(line)) {
+            // #477: every --expert-profile-save-every minutes, before the next request (at QUIT: after the loop)
+            if (!heat.empty() && line != "QUIT" && o.expert_profile_save_min > 0 &&
+                Clock::now() - profile_saved_at >= std::chrono::duration<double>(o.expert_profile_save_min * 60.0))
+                save_profile("periodic");
             if (line == "QUIT") break;
             // the watchdog watches a request from here until this iteration ends, whichever way it ends
             struct BusyScope {
@@ -5063,6 +5375,7 @@ int main(int argc, char** argv) {
             conversations.limit_reuse(read_from);
             pp_total = n;
             pp_from = read_from;
+            pp_reached = read_from;
             pp_t0 = r0;
             pp_next_check = reread_to > 0 ? INT64_MAX : resume + o.prompt_cache_every;
             {
@@ -5128,6 +5441,7 @@ int main(int argc, char** argv) {
                         return false;
                     if (!ver.commit(T, e) || !mtp.prefill(ver.final_R_all(), nxt.data(), T, q, e)) return false;
                     q += T;
+                    pp_reached = q;   // #471
                 }
                 // the batched prompt path (other streams), checkpoints and snapshots may follow: the last commit first
                 if (!ver.wait_commit(e)) return false;
@@ -5412,7 +5726,10 @@ int main(int argc, char** argv) {
                 drive.d.layers = 0;
                 drive.d.experts = 0;
                 drive.d.failed = false;
-                apply_pending(false);
+                // #463: the previous adapt round's copies land first - with a non-blocking query, whether a swapped-in
+                // expert ran on the GPU or the CPU (they round differently) depended on the copy's timing
+                // (STRATA_ADAPT_NOWAIT=1: 0.1.37's non-blocking query, the A/B)
+                apply_pending(!adapt_nowait());
                 if (hist_n > 0) {
                     // the tails the penalties count over, ONE PER ROW: the tokens the state has consumed, the
                     // fed-back head `x` (it joins `consumed` only after this window commits), then the drafts
@@ -5628,20 +5945,30 @@ int main(int argc, char** argv) {
             }
             const int64_t req_hits = drive.d.cache_hits - decode_hits0;
             const int64_t req_look = (drive.d.cache_hits + drive.d.cache_admitted + drive.d.cache_refused) - decode_look0;
+            // #471: the prompt tokens this request read - all the fresh ones, or as far as the prompt pass got when a
+            // cancel stopped it part-way (a cancelled request used to be logged and counted as having read them all)
+            const int64_t fresh = n - resume;
+            const int64_t read_n = cancelled ? std::clamp<int64_t>(pp_reached - resume, 0, fresh) : fresh;
             // DONE <generated> <prompt> <prompt ms> <decode ms> <finish> <drafts accepted> <drafts offered> <reused> [hits] [lookups]
             //      [RAM blobs] [file blobs] [file MB]   (CS-T tiers; appended, so an older server reads the rest)
-            std::printf("DONE %lld %lld %.1f %.1f %s %lld %lld %lld %lld %lld %lld %lld %.1f\n", (long long) produced_n,
+            //      [prompt tokens read]   (#471: fewer than <prompt> - <reused> when a cancel stopped the read)
+            std::printf("DONE %lld %lld %.1f %.1f %s %lld %lld %lld %lld %lld %lld %lld %.1f %lld\n", (long long) produced_n,
                         (long long) n, prompt_ms, decode_ms, finish, (long long) draft_accepted, (long long) draft_offered,
                         (long long) resume, (long long) req_hits, (long long) req_look,
                         (long long) (src.ram_reads() - ram0), (long long) (src.file_reads() - files0),
-                        (double) (src.file_read_bytes() - file_bytes0) / 1e6);
+                        (double) (src.file_read_bytes() - file_bytes0) / 1e6, (long long) read_n);
             std::fflush(stdout);
             if (drive.routing != nullptr) std::fflush(drive.routing);   // the routing trace survives a crash and is watchable mid-session
-            const int64_t fresh = n - resume;
-            std::fprintf(stderr, "strata serve: prompt %lld tokens = %lld reused + %lld read in %.0f ms (%.1f tok/s), "
+            // "12288 of 98179" when cancelled mid-read (#471), the rate from what was read
+            char read_txt[64];
+            if (cancelled)
+                std::snprintf(read_txt, sizeof(read_txt), "%lld of %lld", (long long) read_n, (long long) fresh);
+            else
+                std::snprintf(read_txt, sizeof(read_txt), "%lld", (long long) fresh);
+            std::fprintf(stderr, "strata serve: prompt %lld tokens = %lld reused + %s read in %.0f ms (%.1f tok/s), "
                                  "%lld generated in %.0f ms (%.1f tok/s), drafts accepted %lld of %lld, %zu checkpoints%s\n",
-                         (long long) n, (long long) resume, (long long) fresh, prompt_ms,
-                         prompt_ms > 0 ? 1000.0 * fresh / prompt_ms : 0.0, (long long) produced_n, decode_ms,
+                         (long long) n, (long long) resume, read_txt, prompt_ms,
+                         prompt_ms > 0 ? 1000.0 * read_n / prompt_ms : 0.0, (long long) produced_n, decode_ms,
                          decode_ms > 0 ? 1000.0 * produced_n / decode_ms : 0.0, (long long) draft_accepted,
                          (long long) draft_offered, checks.size(), cancelled ? " (cancelled)" : "");
             // the VRAM share of the experts the pool looked up while decoding; experts it sent over PCIe for the GPU
@@ -5703,6 +6030,7 @@ int main(int argc, char** argv) {
                              remote_experts[(size_t) r].ms_begin() - begin_before[(size_t) r],
                              remote_experts[(size_t) r].ms_wait() - wait_before[(size_t) r]);
         }
+        save_profile("exit");   // #477: QUIT, or the server closed stdin
         return 0;
     }
 
@@ -5950,6 +6278,13 @@ int main(int argc, char** argv) {
                          (long long) n_vocab, (long long) pos);
             return 1;
         }
+        // the header with the first row (see `hdr`): a run that dumps nothing leaves an empty file
+        if (emit_logits && !hdr_written && std::fwrite(hdr, sizeof hdr, 1, dump) != 1) {
+            std::fprintf(stderr, "strata generate: cannot write logits header\n");
+            std::fclose(dump);
+            return 1;
+        }
+        if (emit_logits) hdr_written = true;
         if (emit_logits && std::fwrite(logits.data(), sizeof(float), (size_t) n_vocab, dump) != (size_t) n_vocab) {
             std::fprintf(stderr, "strata generate: cannot write logits at position %lld\n", (long long) pos);
             std::fclose(dump);
@@ -6098,10 +6433,20 @@ int main(int argc, char** argv) {
         std::vector<std::pair<int32_t, int32_t>> pending;
         cudaEvent_t adapt_ev = nullptr;
         cudaEventCreateWithFlags(&adapt_ev, cudaEventDisableTiming);
+        int64_t adapt_rounds = 0;   // counted here: `rounds` is declared below the adapt lambda
         auto apply_pending = [&](bool wait) {
             if (pending.empty()) return;
+            // STRATA_TRACE_ADAPT=1: whether a round's copies had landed when the next window read the table
+            static const bool trace_pending = std::getenv("STRATA_TRACE_ADAPT") != nullptr;
             if (wait) cudaEventSynchronize(adapt_ev);
-            else if (cudaEventQuery(adapt_ev) != cudaSuccess) return;
+            else if (cudaEventQuery(adapt_ev) != cudaSuccess) {
+                if (trace_pending)
+                    std::fprintf(stderr, "strata: PENDING not landed, %zu stay non-resident this window\n",
+                                 pending.size());
+                return;
+            }
+            if (trace_pending)
+                std::fprintf(stderr, "strata: PENDING landed, %zu experts become resident\n", pending.size());
             src.commit_exchanges();   // the resident RAM mode: the evicted experts take their places in RAM
             for (const auto& [i, slot] : pending) host_res[(size_t) i] = slot;
             pending.clear();
@@ -6113,7 +6458,18 @@ int main(int argc, char** argv) {
         // routed clearly more often.  Copies run between rounds, when the GPU is idle.
         auto adapt = [&]() -> bool {
             const Clock::time_point ta = Clock::now();
-            if (!pending.empty()) return true;   // the previous swaps are still in flight
+            ++adapt_rounds;
+            // STRATA_TRACE_ADAPT: why an adapt round did or did not swap.  Default off, one getenv, and it
+            // reports the only thing that can make an adapt round a coin flip: whether the PREVIOUS round's
+            // asynchronous expert copies had landed by the time this round started.
+            static const bool trace_adapt = std::getenv("STRATA_TRACE_ADAPT") != nullptr;
+            if (!pending.empty()) {
+                if (trace_adapt)
+                    std::fprintf(stderr, "strata: ADAPT round=%lld SKIPPED, %zu swaps still in flight\n",
+                                 (long long) adapt_rounds, pending.size());
+                return true;   // the previous swaps are still in flight
+            }
+            if (trace_adapt) std::fprintf(stderr, "strata: ADAPT round=%lld considering\n", (long long) adapt_rounds);
             struct Swap { float gain; int32_t layer, in, out; };
             std::vector<Swap> swaps;
             std::vector<std::pair<float, int32_t>> cand, vict;
@@ -6157,7 +6513,10 @@ int main(int argc, char** argv) {
                 pending.emplace_back((int32_t) in, slot);      // resident once the copy has landed
             }
             if (!swaps.empty()) cudaEventRecord(adapt_ev, adapt_stream);
-            for (float& v : drive.d.usage) v *= 0.7f;
+            if (trace_adapt)
+                std::fprintf(stderr, "strata: ADAPT round=%lld swapped %zu of %d slots, usage decayed\n",
+                             (long long) adapt_rounds, swaps.size(), o.adapt_swaps);
+            for (float& v : drive.d.usage) v *= o.adapt_decay;
             swaps_total += (int64_t) swaps.size();
             ms_adapt += std::chrono::duration<double, std::milli>(Clock::now() - ta).count();
             return true;
@@ -6225,7 +6584,10 @@ int main(int argc, char** argv) {
             drive.d.layers = 0;
             drive.d.experts = 0;
             drive.d.failed = false;
-            apply_pending(false);
+            // #463: the previous adapt round's copies land first - with a non-blocking query, whether a swapped-in
+            // expert ran on the GPU or the CPU (they round differently) depended on the copy's timing
+            // (STRATA_ADAPT_NOWAIT=1: 0.1.37's non-blocking query, the A/B)
+            apply_pending(!adapt_nowait());
             if (!ver.run(T, window.data(), p, &drive_pool_multi, &drive, outv.data(), err)) {
                 std::fprintf(stderr, "strata generate: %s\n", err.c_str());
                 return 1;

tests/core/expert_profile_save_test.cpp

@@ -0,0 +1,80 @@
+// #477: --expert-profile-save's profile - rank_learned_profile's order (resident first, then the counted routing, then
+// the profile the engine started from) and write_expert_profile's file, byte for byte tools/make_profile.py's format,
+// read back by read_expert_profile.  CPU only: no device is touched.
+#include "strata/core/expert_cache.hpp"
+
+#include <cstdio>
+#include <cstring>
+#include <filesystem>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+int fails = 0;
+void check(bool ok, const char* what) {
+    if (!ok) {
+        std::fprintf(stderr, "FAIL: %s\n", what);
+        ++fails;
+    }
+}
+}  // namespace
+
+int main() {
+    using Pair = std::pair<int32_t, int32_t>;
+    const int64_t L = 3, E = 4;   // 12 pairs, index = layer * 4 + expert
+    std::vector<uint8_t> resident(12, 0);
+    resident[5] = 1;              // (1, 1)
+    resident[2] = 1;              // (0, 2)
+    std::vector<double> heat(12, 0.0);
+    heat[2] = 1.0;
+    heat[5] = 7.0;
+    heat[11] = 9.0;               // (2, 3): the hottest, but not resident
+    heat[0] = 3.0;
+    const std::vector<Pair> prior = {{2, 0}, {1, 3}, {0, 1}};   // a profile that ranks three pairs
+    const std::vector<Pair> r = strata::core::rank_learned_profile(L, E, resident, heat, prior);
+    check(r.size() == 12, "every pair is ranked");
+    const std::vector<Pair> head = {{1, 1}, {0, 2}, {2, 3}, {0, 0}, {2, 0}, {1, 3}, {0, 1}, {0, 3}};
+    for (size_t i = 0; i < head.size() && i < r.size(); ++i)
+        check(r[i] == head[i], "resident by heat, then the rest by heat, then the prior, then the index");
+    // no counts at all: the resident ones, then the prior's order
+    const std::vector<Pair> r0 = strata::core::rank_learned_profile(L, E, resident, {}, prior);
+    check(r0.size() == 12 && r0[0] == Pair(0, 2) && r0[1] == Pair(1, 1) && r0[2] == Pair(2, 0) &&
+          r0[3] == Pair(1, 3) && r0[4] == Pair(0, 1) && r0[5] == Pair(0, 0), "without heat: resident, then prior");
+
+    const std::filesystem::path dir = std::filesystem::temp_directory_path() / "strata_profile_save_test";
+    std::filesystem::create_directories(dir);
+    const std::string path = (dir / "learned.bin").string();
+    std::string err;
+    check(strata::core::write_expert_profile(path, L, E, r, err), "written");
+    check(!std::filesystem::exists(path + ".tmp"), "no temporary file left");
+    // the bytes make_profile.write_profile would write for the same ranking
+    std::vector<uint8_t> want(4);
+    std::memcpy(want.data(), "STRP", 4);
+    auto u32 = [&](uint32_t v) { for (int b = 0; b < 4; ++b) want.push_back((uint8_t) (v >> (8 * b))); };
+    auto u16 = [&](uint32_t v) { want.push_back((uint8_t) v); want.push_back((uint8_t) (v >> 8)); };
+    for (uint32_t v : {1u, 3u, 4u, 12u, 12u}) u32(v);
+    for (const Pair& p : r) { u16((uint32_t) p.first); u16((uint32_t) p.second); }
+    std::vector<int32_t> table(12, -1);
+    for (size_t i = 0; i < r.size(); ++i) table[(size_t) (r[i].first * E + r[i].second)] = (int32_t) i;
+    for (int32_t v : table) u32((uint32_t) v);
+    std::vector<uint8_t> got(want.size() + 1);
+    if (std::FILE* f = std::fopen(path.c_str(), "rb")) {
+        got.resize(std::fread(got.data(), 1, got.size(), f));
+        std::fclose(f);
+    }
+    check(got == want, "the file is make_profile.py's format, byte for byte");
+    std::vector<Pair> back;
+    int64_t slots = 0;
+    check(strata::core::read_expert_profile(path, L, E, back, slots, err) && back == r && slots == 12,
+          "read back by the loader");
+    // a second save replaces the first (the rename over an existing file)
+    check(strata::core::write_expert_profile(path, L, E, r0, err), "written again");
+    check(strata::core::read_expert_profile(path, L, E, back, slots, err) && back == r0, "replaced");
+    check(!strata::core::write_expert_profile(path, L, E, {{3, 0}}, err), "a pair out of range is refused");
+    check(strata::core::read_expert_profile(path, L, E, back, slots, err) && back == r0,
+          "a refused save keeps the file");
+    std::filesystem::remove_all(dir);
+    if (fails == 0) std::puts("expert_profile_save_test: OK");
+    return fails == 0 ? 0 : 1;
+}

tests/cuda/prefill_fused_iq_test.cpp

@@ -0,0 +1,633 @@
+// prefill_fused_iq_test - #136: the fused int8 expert kernels for the native packs (moe_fused_iq.hpp, STRATA_PF_FUSED=1)
+// on random native expert blobs ([gate rows | up rows | down rows], GGUF blocks) and random routing, per format pair,
+// against
+//   - a double-precision reference: the weights dequantized by ggml's own to_float, times the FP32 activations
+//     (gate/up, SwiGLU, down), and
+//   - the MMQ path in prefill.cpp's sequence (gather_native into 16-expert groups, q8_1 of the slots' activations,
+//     gate/up, SwiGLU, q8_1 of H, down).
+// Both round the activations and H to int8 per 32 values, differently, so this is not a bitwise test: the fused
+// path's error against the reference has to be comparable to MMQ's own (at most 1.5x its RMS and 2x its worst row).
+// The pairs cover every format the kernels take: gate/up IQ2_XXS IQ2_XS IQ2_S IQ3_XXS IQ3_S IQ4_XS, down Q2_0 IQ4_NL.
+// Part 1 (reference), per pair: 256 tokens over 64 experts, skewed routing - experts with 0 rows, with one, with
+// several 64-row tiles - an all-zero token, per-expert blobs at unrelated (2-byte aligned) addresses, three launches of
+// different expert ranges.  Part 2 (timing): one layer at the engine's chunks (2048, 3584, 8192 tokens; 512 experts,
+// top 10) on both paths and on MMQ's products alone (no gathers), for the IQ2_XS, IQ3_XXS and IQ3_S packs' most
+// common layers, with the grouping checked and a sample of rows against the reference.  --no-ref, --no-timing,
+// --only=NAME, --chunks=A,B.  Exit 77 without a CUDA device of sm_80 or newer.
+#include "strata/prefill/moe_fused_iq.hpp"
+#include "strata/prefill/moe_mmq.hpp"
+
+#include "ggml.h"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <random>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace {
+namespace mmq = strata::prefill::mmq;
+namespace fused = strata::prefill::fused;
+
+constexpr int N = 2560, FF = 640, K = 10, GROUP = 16;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) throw std::runtime_error(std::string(what) + ": " + cudaGetErrorString(e));
+}
+
+struct Dev {
+    void* p = nullptr;
+    explicit Dev(size_t n) { ck(cudaMalloc(&p, n), "cudaMalloc"); }
+    ~Dev() { cudaFree(p); }
+    Dev(const Dev&) = delete;
+    Dev& operator=(const Dev&) = delete;
+    template <typename T> T* as() const { return (T*) p; }
+};
+
+struct Pair {
+    ggml_type gu, d;
+    const char* name;
+};
+
+struct Geo {
+    size_t gu_row, d_row, up_off, down_off, bytes;
+    explicit Geo(const Pair& p) {
+        gu_row = ggml_row_size(p.gu, N);
+        d_row = ggml_row_size(p.d, FF);
+        up_off = gu_row * FF;
+        down_off = 2 * up_off;
+        bytes = down_off + d_row * N;
+    }
+    fused::NativeGeom native(const Pair& p) const {
+        fused::NativeGeom g;
+        g.gu_type = (int) p.gu; g.d_type = (int) p.d; g.gu_row = gu_row; g.d_row = d_row; g.up_off = up_off;
+        g.down_off = down_off;
+        return g;
+    }
+};
+
+// random blocks with a sane fp16 scale `d` at each block's start (every format here begins with it): scales chosen so
+// that the weights' typical size is that of a 2-3 bit expert (a few hundredths)
+void fill_matrix(uint8_t* m, ggml_type t, int64_t rows, int64_t cols, std::mt19937& rng) {
+    const size_t bs = ggml_type_size(t), nb = (size_t) rows * (size_t) (cols / ggml_blck_size(t));
+    std::uniform_int_distribution<int> byte(0, 255);
+    const float lo = t == GGML_TYPE_IQ3_S ? 0.0004f : t == GGML_TYPE_IQ4_XS ? 0.00005f
+                   : t == GGML_TYPE_IQ4_NL ? 0.0002f : t == GGML_TYPE_Q2_0 ? 0.004f : 0.0008f;
+    std::uniform_real_distribution<float> sc(lo, 6.0f * lo);
+    for (size_t i = 0; i < nb * bs; ++i) m[i] = (uint8_t) byte(rng);
+    for (size_t b = 0; b < nb; ++b) {
+        const ggml_fp16_t h = ggml_fp32_to_fp16(sc(rng));
+        std::memcpy(m + b * bs, &h, 2);
+    }
+}
+std::vector<uint8_t> make_blob(const Pair& p, const Geo& g, std::mt19937& rng) {
+    std::vector<uint8_t> b(g.bytes);
+    fill_matrix(b.data(), p.gu, 2 * FF, N, rng);
+    fill_matrix(b.data() + g.down_off, p.d, N, FF, rng);
+    return b;
+}
+
+// ggml's dequantized weights: gate [640][2560], up [640][2560], down [2560][640]
+void dequant(const Pair& p, const Geo& g, const std::vector<uint8_t>& b, std::vector<float>& gu, std::vector<float>& dn) {
+    const auto* tg = ggml_get_type_traits(p.gu);
+    const auto* td = ggml_get_type_traits(p.d);
+    gu.resize((size_t) 2 * FF * N);
+    dn.resize((size_t) N * FF);
+    for (int r = 0; r < 2 * FF; ++r) tg->to_float(b.data() + (size_t) r * g.gu_row, gu.data() + (size_t) r * N, N);
+    for (int r = 0; r < N; ++r) td->to_float(b.data() + g.down_off + (size_t) r * g.d_row, dn.data() + (size_t) r * FF, FF);
+}
+
+// activations: N(0, 1) with a few large values (the hidden state has outliers); token `zero` all zero
+std::vector<float> make_x(int T, std::mt19937& rng, int zero) {
+    std::vector<float> x((size_t) T * N);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    std::uniform_int_distribution<int> pick(0, 199);
+    for (float& v : x) v = nd(rng) * (pick(rng) == 0 ? 12.0f : 1.0f);
+    if (zero >= 0) std::fill(x.begin() + (size_t) zero * N, x.begin() + (size_t) (zero + 1) * N, 0.0f);
+    return x;
+}
+
+// K distinct experts per token; `hot` > 0 skews the first choices towards the lowest ids
+std::vector<int32_t> make_ids(int T, int E, std::mt19937& rng, int hot, int unused) {
+    std::vector<int32_t> ids((size_t) T * K);
+    std::uniform_int_distribution<int> any(0, E - 1 - unused);
+    for (int t = 0; t < T; ++t)
+        for (int k = 0; k < K; ++k) {
+            int e;
+            bool dup;
+            do {
+                e = (hot > 0 && k < 3 && (int) (rng() % 4) != 0) ? (int) (rng() % (unsigned) hot) : any(rng);
+                dup = false;
+                for (int j = 0; j < k; ++j) dup |= ids[(size_t) t * K + j] == e;
+            } while (dup);
+            ids[(size_t) t * K + k] = e;
+        }
+    return ids;
+}
+
+struct Routing {
+    std::vector<int32_t> cnt, off, src;   // per expert; sorted row -> token (rows in pair order within an expert)
+    std::vector<int32_t> row_of;          // pair -> sorted row
+};
+Routing sort_rows(const std::vector<int32_t>& ids, int E) {
+    Routing r;
+    r.cnt.assign((size_t) E, 0);
+    for (int32_t e : ids) ++r.cnt[(size_t) e];
+    r.off.assign((size_t) E + 1, 0);
+    for (int e = 0; e < E; ++e) r.off[(size_t) e + 1] = r.off[(size_t) e] + r.cnt[(size_t) e];
+    std::vector<int32_t> fill(r.off.begin(), r.off.end() - 1);
+    r.src.resize(ids.size());
+    r.row_of.resize(ids.size());
+    for (size_t i = 0; i < ids.size(); ++i) {
+        const int32_t p = fill[(size_t) ids[i]]++;
+        r.row_of[i] = p;
+        r.src[(size_t) p] = (int32_t) (i / K);
+    }
+    return r;
+}
+
+// MMQ, as prefill.cpp runs a native layer: per group of 16 routed experts a gather, gate/up, SwiGLU (gate rows, then
+// up rows), q8_1 of H, down.  Returns Dm [rows][N] in sorted-row order.
+struct MmqBufs {
+    Dev xq, gu, h, hq, dm, ident, bounds, grp_gu, grp_d;
+    MmqBufs(const Pair& p, int64_t rows, int E)
+        : xq(mmq::q8_bytes(rows, N)), gu((size_t) rows * 1280 * 4), h((size_t) rows * FF * 4),
+          hq(mmq::q8_bytes(rows, FF)), dm((size_t) rows * N * 4), ident((size_t) rows * 4),
+          bounds((size_t) (2 * (E + E / GROUP + 2)) * 4), grp_gu(GROUP * mmq::matrix_bytes(p.gu, 1280, N) + 4096),
+          grp_d(GROUP * mmq::matrix_bytes(p.d, N, FF) + 4096) {}
+};
+// `direct`: no gathers - the products read the blobs where they are (they must lie at a fixed stride, every expert
+// with rows: the timing part's layout), the cost of MMQ's own kernels alone
+void run_mmq(const Pair& p, const Geo& geo, mmq::Context& ctx, MmqBufs& b, const Routing& r, const float* x_dev,
+             const int32_t* src_dev, const std::vector<const uint8_t*>& blob, int64_t rows, cudaStream_t s,
+             bool direct = false, size_t stride = 0) {
+    const size_t gub = mmq::matrix_bytes(p.gu, 1280, N), db = mmq::matrix_bytes(p.d, N, FF);
+    const int E = (int) r.cnt.size();
+    std::vector<int32_t> order;
+    for (int e = 0; e < E; ++e) if (r.cnt[(size_t) e] > 0) order.push_back(e);
+    const size_t n = order.size(), ng = (n + GROUP - 1) / GROUP;
+    std::vector<int32_t> bh(n + 1 + ng * (GROUP + 1));
+    for (size_t j = 0; j < n; ++j) bh[j] = r.off[(size_t) order[j]];
+    bh[n] = (int32_t) rows;
+    for (size_t g = 0; g < ng; ++g)
+        for (size_t i = 0; i <= GROUP; ++i)
+            bh[n + 1 + g * (GROUP + 1) + i] = bh[std::min(n, g * GROUP + i)] - bh[g * GROUP];
+    ck(cudaMemcpyAsync(b.bounds.p, bh.data(), bh.size() * 4, cudaMemcpyHostToDevice, s), "bounds");
+    mmq::iota(b.ident.as<int32_t>(), rows, s);
+    mmq::quantize(x_dev, src_dev, b.xq.p, (int) p.gu, N, N, rows, s);
+    for (size_t j = 0; j < n; ++j) {
+        const size_t q = j % GROUP;
+        const uint8_t* bl = blob[(size_t) order[j]];
+        if (!direct)
+            mmq::gather_native(bl, bl + geo.up_off, gub / 2, bl + geo.down_off, db, b.grp_gu.as<uint8_t>() + q * gub,
+                               b.grp_d.as<uint8_t>() + q * db, s);
+        if (q + 1 < GROUP && j + 1 < n) continue;
+        const size_t j0 = j - q, g = j0 / GROUP;
+        const int ngx = (int) (q + 1);
+        const int64_t r0 = bh[j0], nr = bh[j + 1] - r0;
+        int64_t maxr = 0;
+        for (size_t i = j0; i <= j; ++i) maxr = std::max<int64_t>(maxr, r.cnt[(size_t) order[i]]);
+        if (!direct) {
+            ck(cudaMemsetAsync(b.grp_gu.as<uint8_t>() + (size_t) ngx * gub, 0, 4096, s), "tail");
+            ck(cudaMemsetAsync(b.grp_d.as<uint8_t>() + (size_t) ngx * db, 0, 4096, s), "tail");
+        }
+        const uint8_t* b0 = blob[(size_t) order[j0]];
+        mmq::Product gu;
+        gu.w = direct ? (const void*) b0 : b.grp_gu.p; gu.type = (int) p.gu; gu.w_rows = 1280; gu.w_cols = N;
+        gu.expert_bytes = direct ? stride : gub;
+        gu.n = ngx; gu.xq = b.xq.p; gu.bounds = b.bounds.as<int32_t>() + j0; gu.ids = b.ident.as<int32_t>();
+        gu.total_rows = rows; gu.max_rows = maxr; gu.dst = b.gu.as<float>(); gu.ld_dst = 1280;
+        ctx.run(gu, s);
+        mmq::swiglu(b.gu.as<float>() + r0 * 1280, b.h.as<float>() + r0 * FF, nr, FF, false, s);
+        mmq::quantize(b.h.as<float>() + r0 * FF, nullptr, b.hq.p, (int) p.d, FF, FF, nr, s);
+        mmq::Product dn;
+        dn.w = direct ? (const void*) (b0 + geo.down_off) : b.grp_d.p; dn.type = (int) p.d; dn.w_rows = N; dn.w_cols = FF;
+        dn.expert_bytes = direct ? stride : db;
+        dn.n = ngx; dn.xq = b.hq.p; dn.bounds = b.bounds.as<int32_t>() + n + 1 + g * (GROUP + 1);
+        dn.ids = b.ident.as<int32_t>(); dn.total_rows = nr; dn.max_rows = maxr; dn.dst = b.dm.as<float>() + r0 * N;
+        dn.ld_dst = N;
+        ctx.run(dn, s);
+    }
+}
+
+// the fused path: quantize, group on the device, then launches over expert ranges `cuts` (e.g. {0, 20, 64})
+struct FusedBufs {
+    Dev xa, ha, dm, scratch, slot, src, ids;
+    FusedBufs(int T, int64_t rows, int E)
+        : xa(fused::act_bytes(T, N)), ha(fused::act_bytes(rows, FF)), dm((size_t) rows * N * 4),
+          scratch(fused::group_bytes(rows, E)), slot((size_t) rows * 4), src((size_t) rows * 4),
+          ids((size_t) rows * 4) {}
+};
+void run_fused(const Pair& p, const Geo& geo, FusedBufs& b, const float* x_dev, int T, int E,
+               const std::vector<int>& cuts, const std::vector<const uint8_t*>& blob, cudaStream_t s) {
+    const int64_t rows = (int64_t) T * K;
+    const fused::NativeGeom ng = geo.native(p);
+    fused::quantize_act_native(x_dev, T, N, b.xa.p, s);
+    fused::group(b.ids.as<int32_t>(), rows, K, E, b.scratch.p, b.slot.as<int32_t>(), b.src.as<int32_t>(), s);
+    for (size_t c = 0; c + 1 < cuts.size(); ++c) {
+        for (int e0 = cuts[c]; e0 < cuts[c + 1]; e0 += fused::kMaxBatch) {
+            fused::Batch bt;
+            bt.e0 = e0;
+            bt.e1 = std::min(cuts[c + 1], e0 + fused::kMaxBatch);
+            for (int e = bt.e0; e < bt.e1; ++e) bt.blob[e - bt.e0] = blob[(size_t) e];
+            fused::experts_native(bt, ng, E, rows, b.scratch.p, b.xa.p, b.src.as<int32_t>(), b.ha.p, b.dm.as<float>(), s);
+        }
+    }
+}
+
+struct Err {
+    double rms = 0, worst = 0;   // RMS error / RMS of the reference; the worst row's max |error| / its max |ref|
+};
+// y_a[row_a(i)] against y_b[row_b(i)] for every pair i (rows of N values)
+template <typename RA, typename RB>
+Err compare(const std::vector<float>& a, RA row_a, const std::vector<float>& b, RB row_b, size_t pairs) {
+    double e2 = 0, r2 = 0;
+    Err out;
+    for (size_t i = 0; i < pairs; ++i) {
+        const float* ya = a.data() + (size_t) row_a(i) * N;
+        const float* yb = b.data() + (size_t) row_b(i) * N;
+        double me = 0, mr = 0;
+        for (int o = 0; o < N; ++o) {
+            if (!std::isfinite(ya[o])) throw std::runtime_error("a non-finite output");
+            const double d = (double) ya[o] - yb[o];
+            e2 += d * d;
+            r2 += (double) yb[o] * yb[o];
+            me = std::max(me, std::fabs(d));
+            mr = std::max(mr, (double) std::fabs(yb[o]));
+        }
+        if (mr > 0) out.worst = std::max(out.worst, me / mr);
+        else if (me > 0) out.worst = 1e30;   // a zero row must stay exactly zero
+    }
+    out.rms = r2 > 0 ? std::sqrt(e2 / r2) : 0;
+    return out;
+}
+
+std::vector<float> download(const Dev& d, size_t n) {
+    std::vector<float> h(n);
+    ck(cudaMemcpy(h.data(), d.p, n * 4, cudaMemcpyDeviceToHost), "download");
+    return h;
+}
+std::vector<int32_t> download_i(const Dev& d, size_t n) {
+    std::vector<int32_t> h(n);
+    ck(cudaMemcpy(h.data(), d.p, n * 4, cudaMemcpyDeviceToHost), "download");
+    return h;
+}
+
+// part 1: against the double-precision reference
+void reference_part(const Pair& p, cudaStream_t s) {
+    constexpr int T = 256, E = 64, ZERO = 77;
+    const Geo geo(p);
+    std::mt19937 rng(136 + (unsigned) p.gu * 7 + (unsigned) p.d);
+    std::vector<std::vector<uint8_t>> host((size_t) E);
+    std::vector<std::unique_ptr<Dev>> dev;
+    std::vector<const uint8_t*> blob((size_t) E);
+    for (int e = 0; e < E; ++e) {
+        host[(size_t) e] = make_blob(p, geo, rng);
+        dev.push_back(std::make_unique<Dev>(geo.bytes + 4096 * (size_t) (e % 3) + 64));   // unrelated addresses
+        uint8_t* q = dev.back()->as<uint8_t>() + 256 * (size_t) (e % 3) + 2 * (size_t) (e % 2);   // 2-byte aligned
+        ck(cudaMemcpy(q, host[(size_t) e].data(), geo.bytes, cudaMemcpyHostToDevice), "blob");
+        blob[(size_t) e] = q;
+    }
+    const std::vector<float> x = make_x(T, rng, ZERO);
+    const std::vector<int32_t> ids = make_ids(T, E, rng, 2, 4);   // experts 0, 1 hot (several tiles), 60-63 unused
+    const int64_t rows = (int64_t) T * K;
+    const Routing r = sort_rows(ids, E);
+    std::printf("%s - reference part: %d tokens, %d experts, rows per expert: max %d, experts without rows %d\n", p.name,
+                T, E, *std::max_element(r.cnt.begin(), r.cnt.end()), (int) std::count(r.cnt.begin(), r.cnt.end(), 0));
+
+    Dev x_dev(x.size() * 4), src_dev((size_t) rows * 4);
+    ck(cudaMemcpy(x_dev.p, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+    ck(cudaMemcpy(src_dev.p, r.src.data(), r.src.size() * 4, cudaMemcpyHostToDevice), "src");
+    // (the uploads above are on the legacy stream and the work below on a non-blocking one: a cudaMemcpy from pageable
+    // memory may return before its DMA lands - on Windows it can wait in the queue until the next synchronous call)
+    ck(cudaDeviceSynchronize(), "uploads");
+    mmq::Context ctx;
+    MmqBufs mb(p, rows, E);
+    run_mmq(p, geo, ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s);
+    FusedBufs fb(T, rows, E);
+    ck(cudaMemcpy(fb.ids.p, ids.data(), ids.size() * 4, cudaMemcpyHostToDevice), "ids");
+    ck(cudaMemset(fb.dm.p, 0xff, (size_t) rows * N * 4), "sentinel");   // an unwritten row is NaN
+    ck(cudaDeviceSynchronize(), "uploads");
+    run_fused(p, geo, fb, x_dev.as<float>(), T, E, {0, 1, 20, E}, blob, s);
+    ck(cudaStreamSynchronize(s), "sync");
+    const std::vector<float> y_mmq = download(mb.dm, (size_t) rows * N), y_f = download(fb.dm, (size_t) rows * N);
+    const std::vector<int32_t> slot = download_i(fb.slot, (size_t) rows), fsrc = download_i(fb.src, (size_t) rows);
+    for (int64_t i = 0; i < rows; ++i)
+        if (fsrc[(size_t) slot[(size_t) i]] != (int32_t) (i / K)) throw std::runtime_error("group: slot/src disagree");
+
+    // the reference, an expert at a time on all cores
+    std::vector<float> ref((size_t) rows * N);
+    std::vector<std::thread> th;
+    const int nth = std::max(1, std::min(16, (int) std::thread::hardware_concurrency()));
+    for (int w = 0; w < nth; ++w)
+        th.emplace_back([&, w] {
+            std::vector<float> gu, dn, h(FF);
+            for (int e = w; e < E; e += nth) {
+                if (r.cnt[(size_t) e] == 0) continue;
+                dequant(p, geo, host[(size_t) e], gu, dn);
+                for (int64_t i = 0; i < rows; ++i) {
+                    if (ids[(size_t) i] != e) continue;
+                    const float* xr = x.data() + (size_t) (i / K) * N;
+                    for (int f = 0; f < FF; ++f) {
+                        double gt = 0, up = 0;
+                        const float *wg = gu.data() + (size_t) f * N, *wu = gu.data() + (size_t) (FF + f) * N;
+                        for (int k = 0; k < N; ++k) { gt += (double) wg[k] * xr[k]; up += (double) wu[k] * xr[k]; }
+                        h[(size_t) f] = (float) (gt / (1.0 + std::exp(-gt)) * up);
+                    }
+                    for (int o = 0; o < N; ++o) {
+                        double a = 0;
+                        const float* wd = dn.data() + (size_t) o * FF;
+                        for (int f = 0; f < FF; ++f) a += (double) wd[f] * h[(size_t) f];
+                        ref[(size_t) i * N + o] = (float) a;
+                    }
+                }
+            }
+        });
+    for (auto& t : th) t.join();
+
+    auto pair = [](size_t i) { return (int64_t) i; };
+    auto mrow = [&](size_t i) { return (int64_t) r.row_of[i]; };
+    auto frow = [&](size_t i) { return (int64_t) slot[i]; };
+    const Err em = compare(y_mmq, mrow, ref, pair, (size_t) rows);
+    const Err ef = compare(y_f, frow, ref, pair, (size_t) rows);
+    const Err efm = compare(y_f, frow, y_mmq, mrow, (size_t) rows);
+    std::printf("  MMQ   vs FP32 reference: rel RMS %.3e  worst row max rel %.3e\n", em.rms, em.worst);
+    std::printf("  fused vs FP32 reference: rel RMS %.3e  worst row max rel %.3e\n", ef.rms, ef.worst);
+    std::printf("  fused vs MMQ           : rel RMS %.3e  worst row max rel %.3e\n", efm.rms, efm.worst);
+    double zmax = 0;
+    for (int k = 0; k < K; ++k)
+        for (int o = 0; o < N; ++o)
+            zmax = std::max(zmax, (double) std::fabs(y_f[(size_t) slot[(size_t) (ZERO * K + k)] * N + o]));
+    if (zmax != 0) throw std::runtime_error(std::string(p.name) + ": the all-zero token's outputs are not zero");
+    if (ef.rms > 1.5 * em.rms || ef.worst > 2.0 * em.worst)
+        throw std::runtime_error(std::string(p.name) + ": the fused path's error is not comparable to MMQ's");
+}
+
+// part 2: one layer at a real chunk, timed
+void timing_part(const Pair& p, int T, cudaStream_t s) {
+    constexpr int E = 512;
+    const Geo geo(p);
+    std::mt19937 rng(1360);
+    const int64_t rows = (int64_t) T * K;
+    // 512 distinct blobs - the products read each once per tile, as the engine does
+    // at a stride of whole blocks of both formats (MMQ's expert stride counts blocks), + MMQ's read past the end
+    size_t unit = ggml_type_size(p.gu);
+    while (unit % ggml_type_size(p.d) != 0 || unit % 16 != 0) unit += ggml_type_size(p.gu);
+    const size_t stride = (geo.bytes + unit - 1) / unit * unit;
+    Dev blobs((size_t) E * stride + 4096);
+    ck(cudaMemset(blobs.p, 0, (size_t) E * stride + 4096), "zero");
+    {
+        std::vector<uint8_t> b = make_blob(p, geo, rng);
+        for (int e = 0; e < E; ++e) {
+            b[4 + (size_t) (e % 16)] ^= (uint8_t) e;   // not all equal (inside the first block, not its scale)
+            ck(cudaMemcpy(blobs.as<uint8_t>() + (size_t) e * stride, b.data(), geo.bytes, cudaMemcpyHostToDevice),
+               "blob");
+        }
+    }
+    std::vector<const uint8_t*> blob((size_t) E);
+    for (int e = 0; e < E; ++e) blob[(size_t) e] = blobs.as<uint8_t>() + (size_t) e * stride;
+    const std::vector<float> x = make_x(T, rng, -1);
+    const std::vector<int32_t> ids = make_ids(T, E, rng, 0, 0);
+    const Routing r = sort_rows(ids, E);
+    Dev x_dev(x.size() * 4), src_dev((size_t) rows * 4);
+    ck(cudaMemcpy(x_dev.p, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+    ck(cudaMemcpy(src_dev.p, r.src.data(), r.src.size() * 4, cudaMemcpyHostToDevice), "src");
+    mmq::Context ctx;
+    MmqBufs mb(p, rows, E);
+    FusedBufs fb(T, rows, E);
+    ck(cudaMemcpy(fb.ids.p, ids.data(), ids.size() * 4, cudaMemcpyHostToDevice), "ids");
+    ck(cudaDeviceSynchronize(), "uploads");   // (see reference_part)
+    cudaEvent_t a, b;
+    ck(cudaEventCreate(&a), "event");
+    ck(cudaEventCreate(&b), "event");
+    auto once = [&](auto&& f) {
+        ck(cudaEventRecord(a, s), "record");
+        f();
+        ck(cudaEventRecord(b, s), "record");
+        ck(cudaEventSynchronize(b), "sync");
+        float t = 0;
+        ck(cudaEventElapsedTime(&t, a, b), "elapsed");
+        return t;
+    };
+    auto mmq_run = [&] { run_mmq(p, geo, ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s); };
+    auto fused_run = [&] { run_fused(p, geo, fb, x_dev.as<float>(), T, E, {0, E}, blob, s); };
+    const bool all_routed = std::count(r.cnt.begin(), r.cnt.end(), 0) == 0;
+    auto direct_run = [&] {
+        run_mmq(p, geo, ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s, true, stride);
+    };
+    if (std::getenv("S20_IDSDIAG")) {   // which step overwrites the routing ids on the device
+        auto check = [&](const char* after) {
+            ck(cudaStreamSynchronize(s), "sync");
+            const std::vector<int32_t> d = download_i(fb.ids, (size_t) rows);
+            int64_t bad = 0;
+            for (size_t i = 0; i < (size_t) rows; ++i) bad += d[i] != ids[i];
+            std::printf("    ids on the device after %s: %lld of %lld differ (fb.ids %p, mb.dm %p..%p, fb.dm %p..%p)\n",
+                        after, (long long) bad, (long long) rows, fb.ids.p, mb.dm.p,
+                        (void*) (mb.dm.as<uint8_t>() + (size_t) rows * N * 4), fb.dm.p,
+                        (void*) (fb.dm.as<uint8_t>() + (size_t) rows * N * 4));
+        };
+        check("the upload");
+        mmq_run();
+        check("an MMQ run");
+        fused_run();
+        check("a fused run");
+        if (all_routed) {
+            direct_run();
+            check("an MMQ run without gathers");
+        }
+    }
+    // ~1 s of both first (the clocks ramp up), then alternating rounds; the medians
+    const bool no_mmq = std::getenv("S20_NOMMQ") != nullptr;   // (debug: the fused runs alone)
+    for (float spent = 0; spent < 1000.0f;) spent += (no_mmq ? 0.0f : once(mmq_run)) + once(fused_run);
+    std::vector<float> tm, tf, td;
+    for (int rep = 0; rep < 8; ++rep) {
+        tm.push_back(no_mmq ? 1.0f : once(mmq_run));
+        tf.push_back(once(fused_run));
+        if (all_routed && !no_mmq) td.push_back(once(direct_run));
+    }
+    if (td.empty()) td.push_back(0.0f);
+    std::sort(tm.begin(), tm.end());
+    std::sort(tf.begin(), tf.end());
+    std::sort(td.begin(), td.end());
+    const float t_mmq = tm[tm.size() / 2], t_f = tf[tf.size() / 2];
+    if (all_routed)
+        std::printf("  (MMQ's products alone, no gathers: %.3f ms - fused %.2fx)\n", td[td.size() / 2],
+                    td[td.size() / 2] / t_f);
+    std::vector<float> tf2;   // back to back
+    for (int rep = 0; rep < 8; ++rep) tf2.push_back(once(fused_run));
+    std::sort(tf2.begin(), tf2.end());
+    std::printf("  (fused back to back: median %.3f ms, min %.3f; alternating min %.3f; MMQ min %.3f)\n", tf2[4],
+                tf2[0], tf[0], tm[0]);
+    const std::vector<float> y_mmq = download(mb.dm, (size_t) rows * N), y_f = download(fb.dm, (size_t) rows * N);
+    const std::vector<int32_t> slot = download_i(fb.slot, (size_t) rows);
+    const Err efm = compare(y_f, [&](size_t i) { return (int64_t) slot[i]; }, y_mmq,
+                            [&](size_t i) { return (int64_t) r.row_of[i]; }, (size_t) rows);
+    std::printf("%s - timing part: %d tokens x top %d over %d experts, one layer: MMQ path %.3f ms, fused %.3f ms "
+                "(%.2fx); fused vs MMQ rel RMS %.3e worst row %.3e\n",
+                p.name, T, K, E, t_mmq, t_f, t_mmq / t_f, efm.rms, efm.worst);
+    {   // the grouping: every pair in its expert's rows (the near-identical blobs here would hide a wrong expert)
+        const std::vector<int32_t> fsrc = download_i(fb.src, (size_t) rows);
+        int64_t bad = 0, bad_src = 0;
+        for (size_t i = 0; i < (size_t) rows; ++i) {
+            const int e = ids[i];
+            bad += slot[i] < r.off[(size_t) e] || slot[i] >= r.off[(size_t) e + 1];
+            bad_src += fsrc[(size_t) slot[i]] != (int32_t) (i / K);
+        }
+        if (bad || bad_src) {
+            std::vector<int32_t> tbl((size_t) 4 * E + 2);
+            ck(cudaMemcpy(tbl.data(), fb.scratch.p, tbl.size() * 4, cudaMemcpyDeviceToHost), "tables");
+            int64_t bad_cnt = 0, bad_off = 0, bad_ids = 0;
+            const std::vector<int32_t> d_ids = download_i(fb.ids, (size_t) rows);
+            for (size_t i = 0; i < (size_t) rows; ++i) bad_ids += d_ids[i] != ids[i];
+            std::printf("  the routing ids on the device: %lld of %lld differ (first %d %d %d)\n", (long long) bad_ids,
+                        (long long) rows, d_ids[0], d_ids[1], d_ids[2]);
+            const int32_t smin = *std::min_element(slot.begin(), slot.end()), smax = *std::max_element(slot.begin(), slot.end());
+            std::printf("  slots %d..%d; off[0..3] %d %d %d %d off[E] %d; fill[0..3] %d %d %d %d; ts[0..3] %d %d %d %d "
+                        "ts[E] %d\n", smin, smax, tbl[E], tbl[E + 1], tbl[E + 2], tbl[E + 3], tbl[2 * E], tbl[2 * E + 1],
+                        tbl[2 * E + 2], tbl[2 * E + 3], tbl[2 * E + 4], tbl[3 * E + 1], tbl[3 * E + 2], tbl[3 * E + 3],
+                        tbl[3 * E + 4], tbl[4 * E + 1]);
+            // the group again on these ids, synchronously
+            fused::group(fb.ids.as<int32_t>(), rows, K, E, fb.scratch.p, fb.slot.as<int32_t>(), fb.src.as<int32_t>(), s);
+            ck(cudaStreamSynchronize(s), "sync");
+            std::vector<int32_t> t2((size_t) 4 * E + 2);
+            ck(cudaMemcpy(t2.data(), fb.scratch.p, t2.size() * 4, cudaMemcpyDeviceToHost), "tables");
+            std::printf("  group() again, synchronized: cnt[0..3] %d %d %d %d off[E] %d\n", t2[0], t2[1], t2[2], t2[3],
+                        t2[(size_t) 2 * E]);
+            for (int e = 0; e < E; ++e) {
+                bad_cnt += tbl[(size_t) e] != r.cnt[(size_t) e];
+                bad_off += tbl[(size_t) E + e] != r.off[(size_t) e];
+            }
+            std::printf("  GROUPING WRONG: %lld pairs outside their expert's rows, %lld src mismatches; tables: %lld counts "
+                        "and %lld offsets differ from the host's (cnt[0..3] %d %d %d %d vs %d %d %d %d)\n",
+                        (long long) bad, (long long) bad_src, (long long) bad_cnt, (long long) bad_off, tbl[0], tbl[1],
+                        tbl[2], tbl[3], r.cnt[0], r.cnt[1], r.cnt[2], r.cnt[3]);
+            throw std::runtime_error(std::string(p.name) + ": the grouping is wrong");
+        }
+    }
+    if (std::getenv("S20_ROWDIAG")) {   // the pairs whose rows differ most: expert, its rows, the row's place
+        const std::vector<int32_t> fsrc = download_i(fb.src, (size_t) rows);
+        std::vector<std::pair<double, size_t>> worst;
+        for (size_t i = 0; i < (size_t) rows; ++i) {
+            const float* a = y_f.data() + (size_t) slot[i] * N;
+            const float* m = y_mmq.data() + (size_t) r.row_of[i] * N;
+            double me = 0, mr = 0;
+            for (int o = 0; o < N; ++o) { me = std::max(me, (double) std::fabs(a[o] - m[o])); mr = std::max(mr, (double) std::fabs(m[o])); }
+            worst.push_back({mr > 0 ? me / mr : me, i});
+        }
+        std::sort(worst.rbegin(), worst.rend());
+        for (int k = 0; k < 12; ++k) {
+            const size_t i = worst[(size_t) k].second;
+            const int e = ids[i];
+            // the expert's rows start at its first fused row: the smallest slot of its pairs
+            int32_t first = INT32_MAX;
+            for (size_t j = 0; j < (size_t) rows; ++j) if (ids[j] == e) first = std::min(first, slot[j]);
+            std::printf("    pair %zu token %zu expert %d (%d rows) fused row %d (+%d in the expert) max rel %.3e\n", i,
+                        i / K, e, r.cnt[(size_t) e], slot[i], slot[i] - first, worst[(size_t) k].first);
+        }
+        (void) fsrc;
+    }
+    // both against the double-precision reference on a sample of 96 pairs
+    {
+        std::vector<size_t> pick;
+        for (size_t i = 0; i < 96; ++i) pick.push_back((i * 2654435761u) % (size_t) rows);
+        std::vector<float> ref(pick.size() * N), ym(pick.size() * N), yf(pick.size() * N);
+        std::vector<uint8_t> hb(geo.bytes);
+        std::vector<float> gu, dn, h(FF);
+        for (size_t j = 0; j < pick.size(); ++j) {
+            const size_t i = pick[j];
+            const int e = ids[i];
+            ck(cudaMemcpy(hb.data(), blob[(size_t) e], geo.bytes, cudaMemcpyDeviceToHost), "blob back");
+            dequant(p, geo, hb, gu, dn);
+            const float* xr = x.data() + (i / K) * N;
+            for (int f = 0; f < FF; ++f) {
+                double gt = 0, up = 0;
+                const float *wg = gu.data() + (size_t) f * N, *wu = gu.data() + (size_t) (FF + f) * N;
+                for (int k = 0; k < N; ++k) { gt += (double) wg[k] * xr[k]; up += (double) wu[k] * xr[k]; }
+                h[(size_t) f] = (float) (gt / (1.0 + std::exp(-gt)) * up);
+            }
+            for (int o = 0; o < N; ++o) {
+                double a = 0;
+                const float* wd = dn.data() + (size_t) o * FF;
+                for (int f = 0; f < FF; ++f) a += (double) wd[f] * h[(size_t) f];
+                ref[j * N + o] = (float) a;
+            }
+            std::copy_n(y_mmq.begin() + (ptrdiff_t) ((size_t) r.row_of[i] * N), N, ym.begin() + (ptrdiff_t) (j * N));
+            std::copy_n(y_f.begin() + (ptrdiff_t) ((size_t) slot[i] * N), N, yf.begin() + (ptrdiff_t) (j * N));
+        }
+        auto id = [](size_t j) { return (int64_t) j; };
+        const Err em = compare(ym, id, ref, id, pick.size()), ef = compare(yf, id, ref, id, pick.size());
+        std::printf("  sample of %zu pairs vs FP32 reference: MMQ rel RMS %.3e worst %.3e, fused rel RMS %.3e worst "
+                    "%.3e\n", pick.size(), em.rms, em.worst, ef.rms, ef.worst);
+        if (ef.rms > 1.5 * em.rms || ef.worst > 2.0 * em.worst)
+            throw std::runtime_error(std::string(p.name) + ": the fused path's error is not comparable to MMQ's (timing)");
+    }
+    cudaEventDestroy(a);
+    cudaEventDestroy(b);
+    if (efm.rms > 0.05) throw std::runtime_error(std::string(p.name) + ": the paths disagree at the real shape");
+}
+}  // namespace
+
+int main(int argc, char** argv) {
+    try {
+#ifdef _WIN32
+        _putenv_s("STRATA_PF_FUSED", "1");
+#else
+        setenv("STRATA_PF_FUSED", "1", 1);
+#endif
+        int n = 0;
+        if (cudaGetDeviceCount(&n) != cudaSuccess || n == 0) { std::printf("no CUDA device: skipped\n"); return 77; }
+        if (!fused::available()) { std::printf("the fused kernels need sm_80 or newer: skipped\n"); return 77; }
+        if (!mmq::built()) { std::printf("no MMQ in this build\n"); return 1; }
+        const std::vector<Pair> pairs = {
+            {GGML_TYPE_IQ2_S, GGML_TYPE_Q2_0, "IQ2_S / Q2_0"},       {GGML_TYPE_IQ2_XXS, GGML_TYPE_Q2_0, "IQ2_XXS / Q2_0"},
+            {GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_NL, "IQ2_XS / IQ4_NL"}, {GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ4_NL, "IQ3_XXS / IQ4_NL"},
+            {GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_NL, "IQ3_S / IQ4_NL"},   {GGML_TYPE_IQ4_XS, GGML_TYPE_Q2_0, "IQ4_XS / Q2_0"}};
+        const char* only = nullptr;   // --only=NAME: the pairs whose name contains NAME
+        for (int i = 1; i < argc; ++i)
+            if (std::strncmp(argv[i], "--only=", 7) == 0) only = argv[i] + 7;
+        bool timing = true, ref = true;   // --no-timing, --no-ref: skip a part
+        std::vector<int> chunks = {2048, 3584, 8192};   // --chunks=A,B: the timing part's chunk sizes
+        for (int i = 1; i < argc; ++i) {
+            if (std::strcmp(argv[i], "--no-timing") == 0) timing = false;
+            if (std::strcmp(argv[i], "--no-ref") == 0) ref = false;
+            if (std::strncmp(argv[i], "--chunks=", 9) == 0) {
+                chunks.clear();
+                for (const char* c = argv[i] + 9; *c;) {
+                    chunks.push_back(std::atoi(c));
+                    while (*c && *c != ',') ++c;
+                    if (*c == ',') ++c;
+                }
+            }
+        }
+        cudaStream_t s = nullptr;
+        ck(cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking), "stream");
+        for (const Pair& p : pairs) {
+            if (only && std::string(p.name).find(only) == std::string::npos) continue;
+            if (!fused::native_supported((int) p.gu, (int) p.d))
+                throw std::runtime_error(std::string(p.name) + ": not covered by the native kernels");
+            if (ref) reference_part(p, s);
+        }
+        if (timing) {
+            // the IQ2_XS pack's layers (34 of 48 IQ2_S / Q2_0, 11 IQ2_XXS / Q2_0), the IQ3_S pack's most common, IQ3_XXS's
+            for (const Pair& p : {pairs[0], pairs[1], pairs[4], pairs[3]}) {
+                if (only && std::string(p.name).find(only) == std::string::npos) continue;
+                for (int T : chunks) timing_part(p, T, s);
+            }
+        }
+        ck(cudaStreamDestroy(s), "destroy");
+        std::printf("prefill fused MoE (native formats) parity passed\n");
+        return 0;
+    } catch (const std::exception& e) {
+        std::fprintf(stderr, "prefill fused MoE (native formats) parity failed: %s\n", e.what());
+        return 1;
+    }
+}

tests/cuda/prefill_fused_moe_test.cpp

@@ -0,0 +1,420 @@
+// prefill_fused_moe_test - #136: the fused int8 expert kernels of the prompt path (moe_fused.hpp, STRATA_PF_FUSED=1)
+// on random Strata Q2_0 expert blobs and random routing, against
+//   - a double-precision reference: the dequantized weights times the FP32 activations (gate/up, SwiGLU, down), and
+//   - the MMQ path in prefill.cpp's sequence (gather_strata_q2 into 16-expert groups, q8_1 of the slots' activations,
+//     gate/up, SwiGLU, q8_1 of H, down).
+// Both round the activations and H to int8 per 32 values, differently, so this is not a bitwise test: the fused
+// path's error against the reference has to be comparable to MMQ's own (at most 1.5x its RMS and 2x its worst row).
+// Part 1 (reference): 256 tokens over 64 experts, skewed routing - experts with 0 rows, with one, with several 64-row
+// tiles - an all-zero token, per-expert blobs at unrelated addresses, three launches of different expert ranges.
+// Part 2 (timing): one layer at a real chunk (2048 tokens, 512 experts, top 10) on both paths, with their agreement.
+// Exit 77 without a CUDA device of sm_80 or newer.
+#include "strata/prefill/moe_fused.hpp"
+#include "strata/prefill/moe_mmq.hpp"
+
+#include "ggml.h"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <memory>
+#include <random>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace {
+namespace mmq = strata::prefill::mmq;
+namespace fused = strata::prefill::fused;
+
+constexpr int N = 2560, FF = 640, K = 10, GROUP = 16, Q2 = 42;   // GGML_TYPE_Q2_0
+constexpr size_t BLOB = 1382400, O_D_CODES = (size_t) 1280 * 640, O_GU_SC = O_D_CODES + (size_t) 2560 * 160,
+                 O_D_SC = O_GU_SC + (size_t) 1280 * 40 * 2;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) throw std::runtime_error(std::string(what) + ": " + cudaGetErrorString(e));
+}
+
+struct Dev {
+    void* p = nullptr;
+    explicit Dev(size_t n) { ck(cudaMalloc(&p, n), "cudaMalloc"); }
+    ~Dev() { cudaFree(p); }
+    Dev(const Dev&) = delete;
+    Dev& operator=(const Dev&) = delete;
+    template <typename T> T* as() const { return (T*) p; }
+};
+
+// a Strata Q2_0 blob: random codes, scales in [0.004, 0.03] (a 2-bit expert's range)
+std::vector<uint8_t> make_blob(std::mt19937& rng) {
+    std::vector<uint8_t> b(BLOB);
+    std::uniform_int_distribution<int> byte(0, 255);
+    std::uniform_real_distribution<float> sc(0.004f, 0.03f);
+    for (size_t i = 0; i < O_GU_SC; ++i) b[i] = (uint8_t) byte(rng);
+    for (size_t i = O_GU_SC; i < BLOB; i += 2) {
+        const ggml_fp16_t h = ggml_fp32_to_fp16(sc(rng));
+        std::memcpy(&b[i], &h, 2);
+    }
+    return b;
+}
+
+// the blob's weights as floats: gate/up [1280][2560] (interleaved rows), down [2560][640]
+void dequant(const std::vector<uint8_t>& b, std::vector<float>& gu, std::vector<float>& dn) {
+    auto row = [&](size_t codes, size_t ld, size_t scales, int nb, int r, float* out) {
+        for (int k = 0; k < nb * 64; ++k) {
+            ggml_fp16_t h;
+            std::memcpy(&h, &b[scales + ((size_t) r * nb + k / 64) * 2], 2);
+            const int q = (b[codes + (size_t) r * ld + k / 4] >> (2 * (k % 4))) & 3;
+            out[k] = ggml_fp16_to_fp32(h) * (float) (q - 1);
+        }
+    };
+    gu.resize((size_t) 1280 * N);
+    dn.resize((size_t) N * FF);
+    for (int r = 0; r < 1280; ++r) row(0, 640, O_GU_SC, 40, r, gu.data() + (size_t) r * N);
+    for (int r = 0; r < N; ++r) row(O_D_CODES, 160, O_D_SC, 10, r, dn.data() + (size_t) r * FF);
+}
+
+// activations: N(0, 1) with a few large values (the hidden state has outliers); token `zero` all zero
+std::vector<float> make_x(int T, std::mt19937& rng, int zero) {
+    std::vector<float> x((size_t) T * N);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    std::uniform_int_distribution<int> pick(0, 199);
+    for (float& v : x) v = nd(rng) * (pick(rng) == 0 ? 12.0f : 1.0f);
+    if (zero >= 0) std::fill(x.begin() + (size_t) zero * N, x.begin() + (size_t) (zero + 1) * N, 0.0f);
+    return x;
+}
+
+// K distinct experts per token; `hot` > 0 skews the first choices towards the lowest ids
+std::vector<int32_t> make_ids(int T, int E, std::mt19937& rng, int hot, int unused) {
+    std::vector<int32_t> ids((size_t) T * K);
+    std::uniform_int_distribution<int> any(0, E - 1 - unused);
+    for (int t = 0; t < T; ++t)
+        for (int k = 0; k < K; ++k) {
+            int e;
+            bool dup;
+            do {
+                e = (hot > 0 && k < 3 && (int) (rng() % 4) != 0) ? (int) (rng() % (unsigned) hot) : any(rng);
+                dup = false;
+                for (int j = 0; j < k; ++j) dup |= ids[(size_t) t * K + j] == e;
+            } while (dup);
+            ids[(size_t) t * K + k] = e;
+        }
+    return ids;
+}
+
+struct Routing {
+    std::vector<int32_t> cnt, off, src;   // per expert; sorted row -> token (rows in pair order within an expert)
+    std::vector<int32_t> row_of;          // pair -> sorted row
+};
+Routing sort_rows(const std::vector<int32_t>& ids, int E) {
+    Routing r;
+    r.cnt.assign((size_t) E, 0);
+    for (int32_t e : ids) ++r.cnt[(size_t) e];
+    r.off.assign((size_t) E + 1, 0);
+    for (int e = 0; e < E; ++e) r.off[(size_t) e + 1] = r.off[(size_t) e] + r.cnt[(size_t) e];
+    std::vector<int32_t> fill(r.off.begin(), r.off.end() - 1);
+    r.src.resize(ids.size());
+    r.row_of.resize(ids.size());
+    for (size_t i = 0; i < ids.size(); ++i) {
+        const int32_t p = fill[(size_t) ids[i]]++;
+        r.row_of[i] = p;
+        r.src[(size_t) p] = (int32_t) (i / K);
+    }
+    return r;
+}
+
+// MMQ, as prefill.cpp runs it: per group of 16 routed experts a gather, gate/up, SwiGLU, q8_1 of H, down.  Returns
+// Dm [rows][N] in sorted-row order.
+struct MmqBufs {
+    Dev xq, gu, h, hq, dm, ident, bounds, grp_gu, grp_d;
+    MmqBufs(int64_t rows, int E)
+        : xq(mmq::q8_bytes(rows, N)), gu((size_t) rows * 1280 * 4), h((size_t) rows * FF * 4),
+          hq(mmq::q8_bytes(rows, FF)),
+          dm((size_t) rows * N * 4), ident((size_t) rows * 4), bounds((size_t) (2 * (E + E / GROUP + 2)) * 4),
+          grp_gu(GROUP * mmq::matrix_bytes(Q2, 1280, N) + 4096), grp_d(GROUP * mmq::matrix_bytes(Q2, N, FF) + 4096) {}
+};
+void run_mmq(mmq::Context& ctx, MmqBufs& b, const Routing& r, const float* x_dev, const int32_t* src_dev,
+             const std::vector<const uint8_t*>& blob, int64_t rows, cudaStream_t s) {
+    const size_t gub = mmq::matrix_bytes(Q2, 1280, N), db = mmq::matrix_bytes(Q2, N, FF);
+    const int E = (int) r.cnt.size();
+    std::vector<int32_t> order;
+    for (int e = 0; e < E; ++e) if (r.cnt[(size_t) e] > 0) order.push_back(e);
+    const size_t n = order.size(), ng = (n + GROUP - 1) / GROUP;
+    std::vector<int32_t> bh(n + 1 + ng * (GROUP + 1));
+    for (size_t j = 0; j < n; ++j) bh[j] = r.off[(size_t) order[j]];
+    bh[n] = (int32_t) rows;
+    for (size_t g = 0; g < ng; ++g)
+        for (size_t i = 0; i <= GROUP; ++i)
+            bh[n + 1 + g * (GROUP + 1) + i] = bh[std::min(n, g * GROUP + i)] - bh[g * GROUP];
+    ck(cudaMemcpyAsync(b.bounds.p, bh.data(), bh.size() * 4, cudaMemcpyHostToDevice, s), "bounds");
+    mmq::iota(b.ident.as<int32_t>(), rows, s);
+    mmq::quantize(x_dev, src_dev, b.xq.p, Q2, N, N, rows, s);
+    for (size_t j = 0; j < n; ++j) {
+        const size_t q = j % GROUP;
+        mmq::gather_strata_q2(blob[(size_t) order[j]], b.grp_gu.as<uint8_t>() + q * gub, b.grp_d.as<uint8_t>() + q * db,
+                              s);
+        if (q + 1 < GROUP && j + 1 < n) continue;
+        const size_t j0 = j - q, g = j0 / GROUP;
+        const int ngx = (int) (q + 1);
+        const int64_t r0 = bh[j0], nr = bh[j + 1] - r0;
+        int64_t maxr = 0;
+        for (size_t i = j0; i <= j; ++i) maxr = std::max<int64_t>(maxr, r.cnt[(size_t) order[i]]);
+        ck(cudaMemsetAsync(b.grp_gu.as<uint8_t>() + (size_t) ngx * gub, 0, 4096, s), "tail");
+        ck(cudaMemsetAsync(b.grp_d.as<uint8_t>() + (size_t) ngx * db, 0, 4096, s), "tail");
+        mmq::Product gu;
+        gu.w = b.grp_gu.p; gu.type = Q2; gu.w_rows = 1280; gu.w_cols = N; gu.expert_bytes = gub;
+        gu.n = ngx; gu.xq = b.xq.p; gu.bounds = b.bounds.as<int32_t>() + j0; gu.ids = b.ident.as<int32_t>();
+        gu.total_rows = rows; gu.max_rows = maxr; gu.dst = b.gu.as<float>(); gu.ld_dst = 1280;
+        ctx.run(gu, s);
+        mmq::swiglu(b.gu.as<float>() + r0 * 1280, b.h.as<float>() + r0 * FF, nr, FF, true, s);
+        mmq::quantize(b.h.as<float>() + r0 * FF, nullptr, b.hq.p, Q2, FF, FF, nr, s);
+        mmq::Product dn;
+        dn.w = b.grp_d.p; dn.type = Q2; dn.w_rows = N; dn.w_cols = FF; dn.expert_bytes = db;
+        dn.n = ngx; dn.xq = b.hq.p; dn.bounds = b.bounds.as<int32_t>() + n + 1 + g * (GROUP + 1);
+        dn.ids = b.ident.as<int32_t>(); dn.total_rows = nr; dn.max_rows = maxr; dn.dst = b.dm.as<float>() + r0 * N;
+        dn.ld_dst = N;
+        ctx.run(dn, s);
+    }
+}
+
+// the fused path: quantize, group on the device, then launches over expert ranges `cuts` (e.g. {0, 20, 64})
+struct FusedBufs {
+    Dev xa, ha, dm, scratch, slot, src, ids;
+    FusedBufs(int T, int64_t rows, int E)
+        : xa(fused::act_bytes(T, N)), ha(fused::act_bytes(rows, FF)), dm((size_t) rows * N * 4),
+          scratch(fused::group_bytes(rows, E)), slot((size_t) rows * 4), src((size_t) rows * 4),
+          ids((size_t) rows * 4) {}
+};
+void run_fused(FusedBufs& b, const float* x_dev, int T, int E, const std::vector<int>& cuts,
+               const std::vector<const uint8_t*>& blob, cudaStream_t s) {
+    const int64_t rows = (int64_t) T * K;
+    fused::quantize_act(x_dev, T, N, b.xa.p, s);
+    fused::group(b.ids.as<int32_t>(), rows, K, E, b.scratch.p, b.slot.as<int32_t>(), b.src.as<int32_t>(), s);
+    for (size_t c = 0; c + 1 < cuts.size(); ++c) {
+        for (int e0 = cuts[c]; e0 < cuts[c + 1]; e0 += fused::kMaxBatch) {
+            fused::Batch bt;
+            bt.e0 = e0;
+            bt.e1 = std::min(cuts[c + 1], e0 + fused::kMaxBatch);
+            for (int e = bt.e0; e < bt.e1; ++e) bt.blob[e - bt.e0] = blob[(size_t) e];
+            fused::experts(bt, E, rows, b.scratch.p, b.xa.p, b.src.as<int32_t>(), b.ha.p, b.dm.as<float>(), s);
+        }
+    }
+}
+
+struct Err {
+    double rms = 0, worst = 0;   // RMS error / RMS of the reference; the worst row's max |error| / its max |ref|
+};
+// y_a[row_a(i)] against y_b[row_b(i)] for every pair i (rows of N values)
+template <typename RA, typename RB>
+Err compare(const std::vector<float>& a, RA row_a, const std::vector<float>& b, RB row_b, size_t pairs) {
+    double e2 = 0, r2 = 0;
+    Err out;
+    for (size_t i = 0; i < pairs; ++i) {
+        const float* ya = a.data() + (size_t) row_a(i) * N;
+        const float* yb = b.data() + (size_t) row_b(i) * N;
+        double me = 0, mr = 0;
+        for (int o = 0; o < N; ++o) {
+            if (!std::isfinite(ya[o])) throw std::runtime_error("a non-finite output");
+            const double d = (double) ya[o] - yb[o];
+            e2 += d * d;
+            r2 += (double) yb[o] * yb[o];
+            me = std::max(me, std::fabs(d));
+            mr = std::max(mr, (double) std::fabs(yb[o]));
+        }
+        if (mr > 0) out.worst = std::max(out.worst, me / mr);
+        else if (me > 0) out.worst = 1e30;   // a zero row must stay exactly zero
+    }
+    out.rms = r2 > 0 ? std::sqrt(e2 / r2) : 0;
+    return out;
+}
+
+std::vector<float> download(const Dev& d, size_t n) {
+    std::vector<float> h(n);
+    ck(cudaMemcpy(h.data(), d.p, n * 4, cudaMemcpyDeviceToHost), "download");
+    return h;
+}
+std::vector<int32_t> download_i(const Dev& d, size_t n) {
+    std::vector<int32_t> h(n);
+    ck(cudaMemcpy(h.data(), d.p, n * 4, cudaMemcpyDeviceToHost), "download");
+    return h;
+}
+
+// part 1: against the double-precision reference
+void reference_part(cudaStream_t s) {
+    constexpr int T = 256, E = 64, ZERO = 77;
+    std::mt19937 rng(136);
+    std::vector<std::vector<uint8_t>> host((size_t) E);
+    std::vector<std::unique_ptr<Dev>> dev;
+    std::vector<const uint8_t*> blob((size_t) E);
+    for (int e = 0; e < E; ++e) {
+        host[(size_t) e] = make_blob(rng);
+        dev.push_back(std::make_unique<Dev>(BLOB + 4096 * (size_t) (e % 3)));   // unrelated addresses
+        uint8_t* p = dev.back()->as<uint8_t>() + 256 * (size_t) (e % 3);       // 16-byte aligned, not 512
+        ck(cudaMemcpy(p, host[(size_t) e].data(), BLOB, cudaMemcpyHostToDevice), "blob");
+        blob[(size_t) e] = p;
+    }
+    const std::vector<float> x = make_x(T, rng, ZERO);
+    const std::vector<int32_t> ids = make_ids(T, E, rng, 2, 4);   // experts 0, 1 hot (several tiles), 60-63 unused
+    const int64_t rows = (int64_t) T * K;
+    const Routing r = sort_rows(ids, E);
+    std::printf("reference part: %d tokens, %d experts, rows per expert: max %d, experts without rows %d\n", T, E,
+                *std::max_element(r.cnt.begin(), r.cnt.end()), (int) std::count(r.cnt.begin(), r.cnt.end(), 0));
+
+    Dev x_dev(x.size() * 4), src_dev((size_t) rows * 4);
+    ck(cudaMemcpy(x_dev.p, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+    ck(cudaMemcpy(src_dev.p, r.src.data(), r.src.size() * 4, cudaMemcpyHostToDevice), "src");
+    mmq::Context ctx;
+    MmqBufs mb(rows, E);
+    // the uploads (legacy stream) must land before the non-blocking stream reads them (S20 found the kernels could run
+    // on all-zero routing otherwise - every pair on one expert: fast and wrong)
+    ck(cudaDeviceSynchronize(), "uploads");
+    run_mmq(ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s);
+    FusedBufs fb(T, rows, E);
+    ck(cudaMemcpy(fb.ids.p, ids.data(), ids.size() * 4, cudaMemcpyHostToDevice), "ids");
+    ck(cudaMemset(fb.dm.p, 0xff, (size_t) rows * N * 4), "sentinel");   // an unwritten row is NaN
+    ck(cudaDeviceSynchronize(), "uploads");
+    run_fused(fb, x_dev.as<float>(), T, E, {0, 1, 20, E}, blob, s);
+    ck(cudaStreamSynchronize(s), "sync");
+    const std::vector<float> y_mmq = download(mb.dm, (size_t) rows * N), y_f = download(fb.dm, (size_t) rows * N);
+    const std::vector<int32_t> slot = download_i(fb.slot, (size_t) rows), fsrc = download_i(fb.src, (size_t) rows);
+    for (int64_t i = 0; i < rows; ++i)
+        if (fsrc[(size_t) slot[(size_t) i]] != (int32_t) (i / K)) throw std::runtime_error("group: slot/src disagree");
+
+    // the reference, an expert at a time on all cores
+    std::vector<float> ref((size_t) rows * N);
+    std::vector<std::thread> th;
+    const int nth = std::max(1, std::min(16, (int) std::thread::hardware_concurrency()));
+    for (int w = 0; w < nth; ++w)
+        th.emplace_back([&, w] {
+            std::vector<float> gu, dn, h(FF);
+            for (int e = w; e < E; e += nth) {
+                if (r.cnt[(size_t) e] == 0) continue;
+                dequant(host[(size_t) e], gu, dn);
+                for (int64_t i = 0; i < rows; ++i) {
+                    if (ids[(size_t) i] != e) continue;
+                    const float* xr = x.data() + (size_t) (i / K) * N;
+                    for (int f = 0; f < FF; ++f) {
+                        double gt = 0, up = 0;
+                        const float *wg = gu.data() + (size_t) (2 * f) * N, *wu = wg + N;
+                        for (int k = 0; k < N; ++k) { gt += (double) wg[k] * xr[k]; up += (double) wu[k] * xr[k]; }
+                        h[(size_t) f] = (float) (gt / (1.0 + std::exp(-gt)) * up);
+                    }
+                    for (int o = 0; o < N; ++o) {
+                        double a = 0;
+                        const float* wd = dn.data() + (size_t) o * FF;
+                        for (int f = 0; f < FF; ++f) a += (double) wd[f] * h[(size_t) f];
+                        ref[(size_t) i * N + o] = (float) a;
+                    }
+                }
+            }
+        });
+    for (auto& t : th) t.join();
+
+    auto pair = [](size_t i) { return (int64_t) i; };
+    auto mrow = [&](size_t i) { return (int64_t) r.row_of[i]; };
+    auto frow = [&](size_t i) { return (int64_t) slot[i]; };
+    const Err em = compare(y_mmq, mrow, ref, pair, (size_t) rows);
+    const Err ef = compare(y_f, frow, ref, pair, (size_t) rows);
+    const Err efm = compare(y_f, frow, y_mmq, mrow, (size_t) rows);
+    std::printf("  MMQ   vs FP32 reference: rel RMS %.3e  worst row max rel %.3e\n", em.rms, em.worst);
+    std::printf("  fused vs FP32 reference: rel RMS %.3e  worst row max rel %.3e\n", ef.rms, ef.worst);
+    std::printf("  fused vs MMQ           : rel RMS %.3e  worst row max rel %.3e\n", efm.rms, efm.worst);
+    double zmax = 0;
+    for (int k = 0; k < K; ++k)
+        for (int o = 0; o < N; ++o)
+            zmax = std::max(zmax, (double) std::fabs(y_f[(size_t) slot[(size_t) (ZERO * K + k)] * N + o]));
+    if (zmax != 0) throw std::runtime_error("the all-zero token's outputs are not zero");
+    if (ef.rms > 1.5 * em.rms || ef.worst > 2.0 * em.worst)
+        throw std::runtime_error("the fused path's error is not comparable to MMQ's");
+}
+
+// part 2: one layer at a real chunk, timed
+void timing_part(cudaStream_t s) {
+    constexpr int T = 2048, E = 512;
+    std::mt19937 rng(1360);
+    const int64_t rows = (int64_t) T * K;
+    // 512 distinct blobs (708 MB) - the products read each once per tile, as the engine does
+    Dev blobs((size_t) E * BLOB);
+    {
+        std::vector<uint8_t> b = make_blob(rng);
+        for (int e = 0; e < E; ++e) {
+            b[(size_t) e * 7919 % O_GU_SC] ^= (uint8_t) e;   // not all equal
+            ck(cudaMemcpy(blobs.as<uint8_t>() + (size_t) e * BLOB, b.data(), BLOB, cudaMemcpyHostToDevice), "blob");
+        }
+    }
+    std::vector<const uint8_t*> blob((size_t) E);
+    for (int e = 0; e < E; ++e) blob[(size_t) e] = blobs.as<uint8_t>() + (size_t) e * BLOB;
+    const std::vector<float> x = make_x(T, rng, -1);
+    const std::vector<int32_t> ids = make_ids(T, E, rng, 0, 0);
+    const Routing r = sort_rows(ids, E);
+    Dev x_dev(x.size() * 4), src_dev((size_t) rows * 4);
+    ck(cudaMemcpy(x_dev.p, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+    ck(cudaMemcpy(src_dev.p, r.src.data(), r.src.size() * 4, cudaMemcpyHostToDevice), "src");
+    mmq::Context ctx;
+    MmqBufs mb(rows, E);
+    FusedBufs fb(T, rows, E);
+    ck(cudaMemcpy(fb.ids.p, ids.data(), ids.size() * 4, cudaMemcpyHostToDevice), "ids");
+    ck(cudaDeviceSynchronize(), "uploads");   // the legacy-stream uploads before the timed non-blocking stream
+    cudaEvent_t a, b;
+    ck(cudaEventCreate(&a), "event");
+    ck(cudaEventCreate(&b), "event");
+    auto time = [&](auto&& f) {
+        std::vector<float> ms;
+        for (int rep = 0; rep < 6; ++rep) {
+            ck(cudaEventRecord(a, s), "record");
+            f();
+            ck(cudaEventRecord(b, s), "record");
+            ck(cudaEventSynchronize(b), "sync");
+            float t = 0;
+            ck(cudaEventElapsedTime(&t, a, b), "elapsed");
+            if (rep > 0) ms.push_back(t);   // the first is the warm-up
+        }
+        std::sort(ms.begin(), ms.end());
+        return ms[ms.size() / 2];
+    };
+    // the engine's batches: up to 128 experts per launch (the streamed ring decides; 4 launches here)
+    const float t_mmq = time([&] { run_mmq(ctx, mb, r, x_dev.as<float>(), src_dev.as<int32_t>(), blob, rows, s); });
+    const float t_f = time([&] { run_fused(fb, x_dev.as<float>(), T, E, {0, E}, blob, s); });
+    const float t_f1 = time([&] {   // one launch per 16 experts, MMQ's group size
+        std::vector<int> cuts;
+        for (int e = 0; e <= E; e += 16) cuts.push_back(e);
+        run_fused(fb, x_dev.as<float>(), T, E, cuts, blob, s);
+    });
+    const std::vector<float> y_mmq = download(mb.dm, (size_t) rows * N), y_f = download(fb.dm, (size_t) rows * N);
+    const std::vector<int32_t> slot = download_i(fb.slot, (size_t) rows);
+    const Err efm = compare(y_f, [&](size_t i) { return (int64_t) slot[i]; }, y_mmq,
+                            [&](size_t i) { return (int64_t) r.row_of[i]; }, (size_t) rows);
+    std::printf("timing part: %d tokens x top %d over %d experts (%.1f rows per expert), one layer:\n", T, K, E,
+                (double) rows / E);
+    std::printf("  MMQ path (gathers, q8_1, products, SwiGLU, 32 groups): %.3f ms\n", t_mmq);
+    std::printf("  fused path (quantize, group, 4 launches of 128):       %.3f ms  (%.2fx)\n", t_f, t_mmq / t_f);
+    std::printf("  fused path, launches of 16 experts:                    %.3f ms\n", t_f1);
+    std::printf("  fused vs MMQ: rel RMS %.3e  worst row max rel %.3e\n", efm.rms, efm.worst);
+    cudaEventDestroy(a);
+    cudaEventDestroy(b);
+    if (efm.rms > 0.05) throw std::runtime_error("the paths disagree at the real shape");
+}
+}  // namespace
+
+int main(int argc, char** argv) {
+    try {
+        int n = 0;
+        if (cudaGetDeviceCount(&n) != cudaSuccess || n == 0) { std::printf("no CUDA device: skipped\n"); return 77; }
+        if (!fused::available()) { std::printf("the fused kernels need sm_80 or newer: skipped\n"); return 77; }
+        if (!mmq::built()) { std::printf("no MMQ in this build\n"); return 1; }
+        cudaStream_t s = nullptr;
+        ck(cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking), "stream");
+        reference_part(s);
+        if (!(argc > 1 && std::strcmp(argv[1], "--no-timing") == 0)) timing_part(s);
+        ck(cudaStreamDestroy(s), "destroy");
+        std::printf("prefill fused MoE parity passed\n");
+        return 0;
+    } catch (const std::exception& e) {
+        std::fprintf(stderr, "prefill fused MoE parity failed: %s\n", e.what());
+        return 1;
+    }
+}

tools/hip/gfx1200-hipblaslt-100202.txt

@@ -0,0 +1,69 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1200 100202
+bf16 10240 2560 10240 2560 30831
+bf16 10240 2560 10240 256 31013
+bf16 10240 2560 10240 512 31013
+bf16 10240 320 10240 2560 30902
+bf16 10240 320 10240 256 30831
+bf16 10240 320 10240 512 30779
+bf16 12288 2560 12288 2560 30969
+bf16 12288 2560 12288 256 31013
+bf16 2560 10240 10240 256 30654
+bf16 2560 10240 10240 512 30998
+bf16 2560 12288 2560 256 30654
+bf16 2560 2560 2560 2560 30963
+bf16 2560 2560 2560 256 30789
+bf16 2560 2560 2560 512 30844
+bf16 2560 512 4096 512 30654
+bf16 2560 6144 2560 2560 30654
+bf16 2560 640 2560 2560 30726
+bf16 2560 640 2560 256 30654
+bf16 2560 640 2560 512 30654
+bf16 2560 96 4096 37 30681
+bf16 2560 96 4096 48 30664
+bf16 320 10240 320 2560 31046
+bf16 320 10240 320 256 30668
+bf16 48 2560 96 4096 30904
+bf16 6144 2560 6144 2560 30831
+bf16 6144 2560 6144 256 30969
+bf16 6144 2560 6144 512 31013
+bf16 640 2560 640 2560 31171
+bf16 640 2560 640 256 30912
+bf16 640 2560 640 512 30686
+f16 10240 2560 10240 2560 55549
+f16 10240 2560 10240 256 55715
+f16 10240 2560 10240 512 55549
+f16 10240 320 10240 2560 55604
+f16 10240 320 10240 256 55430
+f16 10240 320 10240 512 55481
+f16 12288 2560 12288 2560 55671
+f16 12288 2560 12288 256 55715
+f16 12288 2560 12288 512 55533
+f16 2560 10240 10240 2560 55731
+f16 2560 10240 10240 256 55356
+f16 2560 10240 10240 512 55730
+f16 2560 12288 2560 2560 55729
+f16 2560 12288 2560 256 55356
+f16 2560 12288 2560 512 55693
+f16 2560 2560 2560 2560 55665
+f16 2560 2560 2560 256 55491
+f16 2560 2560 2560 512 55581
+f16 2560 512 4096 37 55381
+f16 2560 512 4096 512 55356
+f16 2560 6144 2560 2560 55356
+f16 2560 6144 2560 256 55356
+f16 2560 6144 2560 512 55356
+f16 2560 640 2560 2560 55554
+f16 2560 640 2560 256 55356
+f16 2560 640 2560 512 55356
+f16 2560 96 4096 48 55383
+f16 320 10240 320 2560 55545
+f16 320 10240 320 256 55517
+f16 320 10240 320 512 55606
+f16 512 2560 512 4096 55575
+f16 6144 2560 6144 2560 55533
+f16 6144 2560 6144 256 55671
+f16 6144 2560 6144 512 55715
+f16 640 2560 640 2560 55873
+f16 640 2560 640 256 55614
+f16 640 2560 640 512 55826

tools/hip/gfx1201-hipblaslt-100202.txt

@@ -0,0 +1,26 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1201 100202
+bf16 1 2560 1 4096 74196
+bf16 48 2560 96 4096 73754
+bf16 48 2560 96 8192 73754
+bf16 128 2560 128 4096 73754
+bf16 128 2560 128 8192 73715
+bf16 320 10240 320 4096 73838
+bf16 320 10240 320 8192 73838
+bf16 512 2560 512 8192 73803
+bf16 2560 2560 2560 8192 73778
+bf16 10240 320 10240 8192 73761
+f16 512 2560 512 4096 88193
+f16 512 2560 512 8192 88307
+f16 640 2560 640 4096 88251
+f16 640 2560 640 8192 88192
+f16 2560 640 2560 4096 88216
+f16 2560 640 2560 8192 88307
+f16 2560 6144 2560 4096 88281
+f16 2560 6144 2560 8192 88281
+f16 6144 2560 6144 4096 88255
+f16 6144 2560 6144 8192 88307
+f16 10240 2560 10240 4096 88255
+f16 10240 2560 10240 8192 88255
+f16 12288 2560 12288 4096 88255
+f16 12288 2560 12288 8192 88255

tools/mtp_fetch.py

@@ -31,8 +31,10 @@ import urllib.request
 # repository no longer has it, the current files are read instead, with a message (resolve_repo).
 PINNED_REVISION = "de4b8e4d43b917e7706784d8bb445c9af86a3540"
 REVISION = os.environ.get("STRATA_MTP_REVISION") or PINNED_REVISION
-REPO = "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/resolve/%s/" % REVISION
-PINNED = "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/resolve/%s/" % PINNED_REVISION   # SHA256's revision
+# #495: HF_ENDPOINT (a mirror, e.g. https://hf-mirror.com) serves the same revision; the SHA256 checks below still apply
+HF_ENDPOINT = (os.environ.get("HF_ENDPOINT") or "").strip().rstrip("/") or "https://huggingface.co"
+REPO = HF_ENDPOINT + "/Qwen/Qwen3.8-Flash-Next/resolve/%s/" % REVISION
+PINNED = HF_ENDPOINT + "/Qwen/Qwen3.8-Flash-Next/resolve/%s/" % PINNED_REVISION   # SHA256's revision
 DTYPE_BYTES = {"BF16": 2, "F16": 2, "F32": 4, "F8_E4M3": 1, "I64": 8, "I32": 4}
 BAD = 3                                             # `verify`'s exit code: a tensor is missing or corrupt
 

tools/test_setup_choices.py

@@ -267,5 +267,124 @@ class HipVision(unittest.TestCase):
         self.assertNotIn("vision_src", meta)
 
 
+class VramReserve(unittest.TestCase):
+    """#493: --vram-reserve-mib N - VRAM the engine leaves free for other programs - goes into the config only when
+    given (the default config is tools/test_setup_golden.py's, unchanged), at setup and on a start."""
+
+    def setUp(self):
+        from test_setup_golden import PROFILES
+        self.ram, self.found = PROFILES["64GB-1x32GB"]
+
+    def install(self, *extra):
+        from test_setup_golden import install
+        return install(self.ram, self.found, ["--family", "qwen", "--model", "Q2_0", "--no-start", *extra])
+
+    def test_given_at_setup(self):
+        code, out, cfg, _ = self.install("--vram-reserve-mib", "2048")
+        self.assertEqual(code, 0, out)
+        a = cfg["args"]
+        self.assertEqual(a.count("--vram-reserve-mib"), 1)
+        self.assertEqual(a[a.index("--vram-reserve-mib") + 1], "2048")
+        self.assertIn("VRAM kept free for other programs: 2048 MiB", out)
+        code, out, cfg, _ = self.install()
+        self.assertNotIn("--vram-reserve-mib", cfg["args"])          # not given: the engine's default, as before
+
+    def test_with_images_one_value(self):
+        code, out, cfg, _ = self.install("--vision", "cpu", "--vram-reserve-mib", "1500")
+        self.assertEqual(code, 0, out)
+        a = cfg["args"]
+        self.assertEqual(a.count("--vram-reserve-mib"), 1)
+        self.assertEqual(a[a.index("--vram-reserve-mib") + 1], "1500")
+
+    def test_a_negative_value_is_refused(self):
+        with contextlib.redirect_stderr(io.StringIO()) as err:
+            code, out, cfg, _ = self.install("--vram-reserve-mib", "-1")
+        self.assertEqual(code, 2)
+        self.assertIn("--vram-reserve-mib takes a number of MiB", err.getvalue())
+
+    def test_kept_from_an_earlier_install(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = Path(d) / "strata-q2_0.json"
+            p.write_text(json.dumps({"args": ["--max-context", "8192", "--vram-reserve-mib", "2048"]}))
+            self.assertEqual(setup.choices_from_config(p)["vram_reserve_mib"], 2048)
+            p.write_text(json.dumps({"args": ["--vision", "--vram-reserve-mib", "700"], "vision": {"gpu": True}}))
+            self.assertIsNone(setup.choices_from_config(p)["vram_reserve_mib"])   # the images' own reserve
+            p.write_text(json.dumps({"args": ["--max-context", "8192"]}))
+            self.assertIsNone(setup.choices_from_config(p)["vram_reserve_mib"])
+
+    def test_given_on_a_start(self):
+        with tempfile.TemporaryDirectory() as d:
+            exe = Path(d) / "strata.exe"
+            exe.write_bytes(b"")
+            p = Path(d) / "strata-q2_0.json"
+            p.write_text(json.dumps({"exe": str(exe), "args": ["--kv", "int8"], "gpu": 0, "gpus_asked": True}))
+            call = mock.Mock(return_value=0)
+            with mock.patch.object(setup, "gpus", lambda: self.found), \
+                    mock.patch.object(setup, "is_wsl", lambda: False), \
+                    mock.patch.object(setup, "ensure_engine_for", lambda cards, path, cfg, yes: cfg), \
+                    mock.patch.object(setup.subprocess, "call", call):
+                for n in ("2048", "1024"):
+                    _, out = quiet(setup.start, p, None, None, False, True, None, {"vram_reserve_mib": int(n)})
+                    self.assertEqual(json.loads(p.read_text())["args"], ["--kv", "int8", "--vram-reserve-mib", n])
+        self.assertIn("1024 MiB of VRAM kept free", out)
+        self.assertTrue(call.called)
+
+
+class SmallCardTip(unittest.TestCase):
+    """#496: a card under 8 GB (a 6 GB laptop RTX 3060) gets a tip for when the start has no room for the expert cache;
+    the engine chooses its own reserve there, so the config is the one any card gets.  An 8 GB card: no tip."""
+
+    def install(self, vram, *extra):
+        from test_setup_golden import card, install
+        return install(63.7, [card(0, "NVIDIA GeForce RTX 3060 Laptop GPU", vram, "86")],
+                       ["--family", "qwen", "--model", "Q2_0", "--no-start", *extra])
+
+    def test_a_6gb_card(self):
+        code, out, cfg, _ = self.install(6.0)
+        self.assertEqual(code, 0, out)
+        self.assertNotIn("--vram-reserve-mib", cfg["args"])          # the engine decides (no fixed reserve)
+        self.assertIn("no VRAM is left for the expert cache", out)
+        self.assertIn("--draft-vocab en", out)
+        self.assertIn("--mtp", cfg["args"])                          # the draft layer stays: the server needs it
+
+    def test_an_8gb_card_has_no_tip(self):
+        for vram in (8188 / 1024, 8.0):                               # nvidia-smi lists an 8 GB card as 8188 MiB
+            code, out, cfg, _ = self.install(vram)
+            self.assertEqual(code, 0, out)
+            self.assertNotIn("no VRAM is left for the expert cache", out)
+            self.assertNotIn("--vram-reserve-mib", cfg["args"])
+
+    def test_a_given_reserve_is_kept(self):
+        code, out, cfg, _ = self.install(6.0, "--vram-reserve-mib", "500")
+        self.assertEqual(code, 0, out)
+        a = cfg["args"]
+        self.assertEqual(a[a.index("--vram-reserve-mib") + 1], "500")
+
+    def test_the_tip(self):
+        self.assertIn("an 8K context", " ".join(setup.small_card_note(32768, None)))
+        self.assertIn("--draft-vocab en", " ".join(setup.small_card_note(32768, "cjk")))
+        tip = " ".join(setup.small_card_note(8192, "en"))
+        self.assertNotIn("--draft-vocab", tip)
+        self.assertIn("close other programs", tip)
+
+
+class DesktopReserveTip(unittest.TestCase):
+    """#560 #516: an AMD card on a Linux desktop gets a recommended reserve (3072 MiB) - a tip, the config is not
+    changed."""
+
+    def test_desktop_detection(self):
+        with mock.patch.object(setup.sys, "platform", "linux"):
+            self.assertTrue(setup.linux_desktop({"WAYLAND_DISPLAY": "wayland-0"}))
+            self.assertTrue(setup.linux_desktop({"DISPLAY": ":0"}))
+            self.assertFalse(setup.linux_desktop({}))                 # a headless box / ssh
+        with mock.patch.object(setup.sys, "platform", "win32"):
+            self.assertFalse(setup.linux_desktop({"DISPLAY": ":0"}))  # Windows counts the desktop itself (#497)
+
+    def test_the_tip(self):
+        tip = " ".join(setup.desktop_reserve_note())
+        self.assertIn("--vram-reserve-mib 3072", tip)
+        self.assertIn("desktop", tip)
+
+
 if __name__ == "__main__":
     unittest.main()

tools/test_setup_draft_vocab.py

@@ -49,5 +49,29 @@ class Refresh(unittest.TestCase):
             self.assertEqual(hashlib.sha256((rt / "draft_vocab.bin").read_bytes()).hexdigest(), want)
 
 
+class SmallCardNote(unittest.TestCase):
+    """#474: a card under 14 GB is told about a smaller draft subset - a note only, and not when one was chosen."""
+
+    def test_note_on_a_small_card_only(self):
+        note = setup.draft_vocab_note(12.0, None)
+        self.assertTrue(note)
+        self.assertIn("--draft-vocab en", " ".join(note))
+        self.assertIn("the draft head does not fit", " ".join(note))
+        for vram, chosen in ((16.0, None), (24.0, None), (12.0, "en"), (12.0, "cyrillic"), (12.0, "cjk"), (0.0, None)):
+            self.assertEqual(setup.draft_vocab_note(vram, chosen), [], (vram, chosen))
+
+    def test_sizes_follow_the_shipped_subsets(self):
+        # the MiB the note gives scale with the subsets' token counts (4 bytes per token id in data/)
+        sizes = {}
+        for choice, name in setup.DRAFT_VOCABS.items():
+            p = ROOT / "data" / name
+            if not p.exists():
+                self.skipTest(f"data/{name} is not in this checkout")
+            sizes[choice] = p.stat().st_size // 4
+        for choice in ("en", "cyrillic"):
+            want = setup.DRAFT_VOCAB_MIB["cjk"] * sizes[choice] / sizes["cjk"]
+            self.assertAlmostEqual(setup.DRAFT_VOCAB_MIB[choice], want, delta=3)
+
+
 if __name__ == "__main__":
     unittest.main()

tools/test_setup_pins.py

@@ -98,6 +98,48 @@ class HuggingFacePins(unittest.TestCase):
         finally:
             mtp_fetch.REPO = pinned
 
+    def test_hf_endpoint(self):
+        # #495: HF_ENDPOINT (a mirror) serves the same pinned revision; a trailing slash and blanks are dropped
+        repo = "ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF"
+        with mock.patch.dict(setup.os.environ, {}, clear=False):
+            setup.os.environ.pop("HF_ENDPOINT", None)
+            self.assertEqual(setup.hf(repo), f"https://huggingface.co/{repo}/resolve/{setup.HF_REVISIONS[repo]}/")
+            for value in ("https://hf-mirror.com", "https://hf-mirror.com/", " https://hf-mirror.com/ "):
+                setup.os.environ["HF_ENDPOINT"] = value
+                url = setup.hf(repo)
+                self.assertEqual(url, f"https://hf-mirror.com/{repo}/resolve/{setup.HF_REVISIONS[repo]}/")
+                self.assertRegex(url, SHA)
+                self.assertEqual(setup.hf_unpinned(url + "x.gguf"), f"https://hf-mirror.com/{repo}/resolve/main/x.gguf")
+            setup.os.environ["HF_ENDPOINT"] = ""
+            self.assertEqual(setup.hf_endpoint(), "https://huggingface.co")
+
+    def test_mtp_fetch_honours_hf_endpoint(self):
+        import importlib
+        import mtp_fetch
+        try:
+            with mock.patch.dict(mtp_fetch.os.environ, {"HF_ENDPOINT": "https://hf-mirror.com/"}):
+                importlib.reload(mtp_fetch)
+                self.assertTrue(mtp_fetch.REPO.startswith("https://hf-mirror.com/Qwen/Qwen3.8-Flash-Next/resolve/"))
+                self.assertRegex(mtp_fetch.REPO, SHA)
+                self.assertTrue(mtp_fetch.pinned())                 # the tensors' SHA-256 checks still apply
+        finally:
+            importlib.reload(mtp_fetch)
+        self.assertTrue(mtp_fetch.REPO.startswith("https://huggingface.co/"))
+
+    def test_step5_names_the_folder(self):
+        # #495: where setup expects the model files, and how to give it files downloaded by hand
+        from test_setup_golden import PROFILES, install
+        ram, found = PROFILES["96GB-1x16GB"]
+        with mock.patch.dict(setup.os.environ, {"HF_ENDPOINT": "https://hf-mirror.com"}):
+            code, out, cfg, _ = install(ram, found, ["--family", "qwen", "--model", "IQ3_XXS", "--no-start"])
+        self.assertEqual(code, 0)
+        text = out.replace("\\", "/")
+        self.assertRegex(text, r"The model files go in .*/models/IQ3_XXS")
+        self.assertIn("put them here with their original names (Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+                      text)
+        self.assertIn("--gguf-dir", text)
+        self.assertIn("Downloading from https://hf-mirror.com (HF_ENDPOINT)", text)
+
 
 class Engine(unittest.TestCase):
     def setUp(self):

tools/test_setup_unsloth.py

@@ -319,6 +319,117 @@ class Main(unittest.TestCase):
         self.assertNotIn("--mmap-experts", cfg["args"])
         self.assertFalse(any("--experts-bin" in r for r in self.runs))
 
+    def test_split_when_the_ram_holds_it(self):
+        # #498: 165 GiB and --gpus 0,1: the split without the RAM budget (the engine refuses the pair)
+        code, out, cfg = self.main(["--context", "8192", "--gpus", "0,1"], n_gpus=2, ram=165.0)
+        self.assertEqual(code, 0, out)
+        self.assertEqual(cfg["gpu"], [0, 1])
+        self.assertEqual(cfg["layer_split"], "auto")
+        for flag in ("--resident-budget-gib", "--mmap-experts", "--resident-experts"):
+            self.assertNotIn(flag, cfg["args"])
+        self.assertIn("as you chose (--gpus): no RAM budget", out)
+        self.assertIn("OS file cache", out)
+        self.assertNotIn("RAM budget: ", out)
+
+    def test_split_refused_without_the_ram(self):
+        need = setup.unsloth_split_need_gb()
+        self.assertAlmostEqual(need, setup.MODELS[M]["download_gb"] + setup.UNSLOTH_RAM_LEFT_GB)
+        code, out, cfg = self.main(["--context", "8192", "--gpus", "0,1"], n_gpus=2, ram=127.8)
+        self.assertEqual(code, 0, out)
+        self.assertIn("runs on one GPU here", out)
+        self.assertNotIn("layer_split", cfg)
+        self.assertEqual(cfg["args"][cfg["args"].index("--resident-budget-gib") + 1], "71")   # all of them
+
+    def test_an_explicit_budget_keeps_one_gpu(self):
+        code, out, cfg = self.main(["--context", "8192", "--gpus", "0,1", "--resident-budget-gib", "60"], n_gpus=2,
+                                   ram=165.0)
+        self.assertEqual(code, 0, out)
+        self.assertIn("--resident-budget-gib has no layer split", out)
+        self.assertNotIn("layer_split", cfg)
+        self.assertEqual(cfg["args"][cfg["args"].index("--resident-budget-gib") + 1], "60")
+
+    def test_yes_alone_keeps_one_gpu(self):
+        code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=165.0)
+        self.assertEqual(code, 0, out)
+        self.assertIn("runs on one GPU: using", out)
+        self.assertIn("--gpus 0,1 shares it", out)
+        self.assertNotIn("layer_split", cfg)
+        self.assertIn("--resident-budget-gib", cfg["args"])
+
+
+class LayerSplit(unittest.TestCase):
+    """#498: UD-Q4_K_XL across GPUs - asked at setup (one GPU by default), and a start with --gpus no longer keeps
+    the RAM budget the engine refuses with a split (it exited with code 2)."""
+
+    def setUp(self):
+        sys.path.insert(0, str(ROOT / "tools"))
+        from test_setup_golden import card
+        self.found = [card(i, "NVIDIA GeForce RTX 3090", 24.0, "86") for i in range(2)]
+
+    def test_asked_one_gpu_by_default(self):
+        from test_setup_golden import install
+        argv = ["--family", "unsloth", "--model", M, "--context", "32768", "--no-start"]
+        code, out, cfg, asked = install(165.0, self.found, argv, answers={"which GPUs?": "1", "Which GPUs?": "1"})
+        self.assertEqual(code, 0, out)
+        self.assertTrue(any(f"{M}: which GPUs?" in q and "[1]" in q for q in asked), asked)
+        self.assertEqual(cfg["gpu"], 0)
+        self.assertIn("--resident-budget-gib", cfg["args"])
+        code, out, cfg, asked = install(165.0, self.found, argv, answers={f"{M}: which GPUs?": "2"})
+        self.assertEqual(code, 0, out)
+        self.assertEqual(cfg["gpu"], [0, 1])
+        self.assertEqual(cfg["layer_split"], "auto")
+        self.assertNotIn("--resident-budget-gib", cfg["args"])
+
+    def start(self, ram, gpu=(0, 1), offered=None):
+        from test_setup_risk import run
+        with tempfile.TemporaryDirectory() as d:
+            exe = Path(d) / "strata.exe"
+            exe.write_bytes(b"")
+            p = Path(d) / "strata-unsloth-ud-q4_k_xl.json"
+            before = {"exe": str(exe), "args": ["--pack", "p", "--resident-budget-gib", "40", "--kv", "int8"],
+                      "gpu": 0, "gpus_asked": offered is None}
+            p.write_text(json.dumps(before))
+            call = mock.Mock(return_value=0)
+            with mock.patch.object(setup, "gpus", lambda: self.found), \
+                    mock.patch.object(setup, "ram_gb", lambda: ram), \
+                    mock.patch.object(setup, "engine_runs_on", lambda g: True), \
+                    mock.patch.object(setup, "ensure_engine_for", lambda cards, path, cfg, yes: cfg), \
+                    mock.patch.object(setup, "is_wsl", lambda: False), \
+                    mock.patch.object(setup.subprocess, "call", call):
+                code, out, asked = run(setup.start, p, None, list(gpu) if gpu else None, False, offered is None,
+                                       stdin=offered)
+            return code, out, json.loads(p.read_text()), call.called
+
+    def test_start_with_gpus_drops_the_budget(self):
+        code, out, cfg, started = self.start(165.0)
+        self.assertIsNone(code, out)
+        self.assertTrue(started)
+        self.assertEqual(cfg["gpu"], [0, 1])
+        self.assertEqual(cfg["args"], ["--pack", "p", "--kv", "int8"])
+        self.assertIn("no RAM budget", out)
+
+    def test_start_with_gpus_refused_without_the_ram(self):
+        code, out, cfg, started = self.start(63.7)
+        self.assertEqual(code, 1)
+        self.assertFalse(started)
+        self.assertIn("cannot share its RAM budget across GPUs", out)
+        self.assertIn("--gpu N", out)
+        self.assertEqual(cfg["gpu"], 0)                                # nothing saved
+        self.assertIn("--resident-budget-gib", cfg["args"])
+
+    def test_offered_at_a_start(self):
+        code, out, cfg, _ = self.start(165.0, gpu=None, offered="")    # Enter: one GPU (the default here)
+        self.assertIsNone(code, out)
+        self.assertEqual(cfg["gpu"], 0)
+        self.assertIn("--resident-budget-gib", cfg["args"])
+        self.assertIn("about twice as fast in #498", out)
+        code, out, cfg, _ = self.start(165.0, gpu=None, offered="y")
+        self.assertEqual(cfg["gpu"], [0, 1])
+        self.assertNotIn("--resident-budget-gib", cfg["args"])
+        code, out, cfg, _ = self.start(63.7, gpu=None, offered="y")    # not offered: the RAM does not hold it
+        self.assertEqual(cfg["gpu"], 0)
+        self.assertNotIn("Use both", out)
+
 
 if __name__ == "__main__":
     unittest.main()

tools/test_setup_update.py

@@ -0,0 +1,145 @@
+"""Tests for `setup.py --update` (#475, UPDATE.bat / update.sh): it refreshes what a start would - the Python packages,
+the engine, each model's config and draft subset - and never starts the model.  Every outside effect is mocked: no
+GPU, no downloads, nothing written outside a temp folder.
+
+    python -m unittest tools.test_setup_update
+"""
+from __future__ import annotations
+
+import contextlib
+import io
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+import setup  # noqa: E402
+
+
+class Update(unittest.TestCase):
+    def config(self, d: Path, **extra) -> Path:
+        rt = d / "mtp" / "rt"
+        rt.mkdir(parents=True)
+        p = d / "strata-iq3_s.json"
+        p.write_text(json.dumps({"model_name": "Qwen IQ3_S", "exe": str(d / "engine" / setup.EXE),
+                                 "args": ["--native", "x", "--mtp", str(rt)], **extra}), encoding="utf-8")
+        return p
+
+    def run_update(self, have, argv=()):
+        a = mock.Mock(**{"build": False, "prebuilt": "URL", **dict(argv)})
+        out = io.StringIO()
+        with mock.patch.object(setup, "pip_install") as pip, \
+                mock.patch.object(setup, "update_installed_engine") as eng, \
+                mock.patch.object(setup, "refresh_draft_vocab") as dv, \
+                mock.patch.object(setup, "start") as start, \
+                mock.patch("subprocess.call") as call, \
+                contextlib.redirect_stdout(out):
+            rc = setup.update_install(have, a)
+        return rc, pip, eng, dv, start, call, out.getvalue()
+
+    def test_refreshes_without_starting(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = self.config(Path(d), draft_vocab="en")
+            rc, pip, eng, dv, start, call, out = self.run_update([p])
+        self.assertEqual(rc, 0)
+        pip.assert_called_once()
+        eng.assert_called_once_with("URL")
+        dv.assert_called_once()
+        self.assertEqual(dv.call_args[0][1], "en")                 # the model's own subset is kept
+        start.assert_not_called()
+        call.assert_not_called()
+        self.assertIn("Strata is updated", out)
+
+    def test_build_keeps_the_compiled_engine_path(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = self.config(Path(d))
+            rc, pip, eng, dv, *_ = self.run_update([p], {"build": True})
+        self.assertEqual(rc, 0)
+        eng.assert_not_called()
+        self.assertEqual(dv.call_args[0][1], "cjk")
+
+    def test_nothing_installed(self):
+        rc, pip, eng, dv, start, call, out = self.run_update([])
+        self.assertEqual(rc, 0)
+        for m in (pip, eng, dv, start, call):
+            m.assert_not_called()
+        self.assertIn("START-HERE.bat", out)
+
+    def test_a_json_that_is_no_model_config_is_skipped(self):
+        """#549: a strata-*.json without "args" (not written by setup) stopped update.sh with KeyError: 'args'."""
+        with tempfile.TemporaryDirectory() as d:
+            p = self.config(Path(d))
+            other = Path(d) / "strata-notes.json"
+            other.write_text(json.dumps({"note": "mine"}), encoding="utf-8")
+            broken = Path(d) / "strata-cut.json"
+            broken.write_text("{\"args\": [", encoding="utf-8")
+            rc, pip, eng, dv, start, call, out = self.run_update([other, broken, p])
+        self.assertEqual(rc, 0, out)
+        self.assertIn('skipped strata-notes.json (no "exe" or "args"): it is not a Strata model config', out)
+        self.assertIn("skipped strata-cut.json (not valid JSON)", out)
+        dv.assert_called_once()                                     # the real model is still refreshed
+        self.assertIn("Qwen IQ3_S: up to date", out)
+        self.assertIn("Strata is updated", out)
+
+    def test_installed_configs_lists_only_model_configs(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = self.config(Path(d))
+            (Path(d) / "strata-notes.json").write_text("{}", encoding="utf-8")
+            with mock.patch.object(setup, "ROOT", Path(d)), contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(setup.installed_configs(), [p])
+
+    def test_main_update_never_starts(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = self.config(Path(d))
+            with mock.patch.object(sys, "argv", ["setup.py", "--update"]), \
+                    mock.patch.object(setup, "data_folder", return_value=(Path(d), [])), \
+                    mock.patch.object(setup, "installed_configs", return_value=[p]), \
+                    mock.patch.object(setup, "update_install", return_value=0) as up, \
+                    mock.patch.object(setup, "start") as start, \
+                    contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(setup.main(), 0)
+        up.assert_called_once()
+        start.assert_not_called()
+
+
+class SettingsLine(unittest.TestCase):
+    """#564: a start prints the settings it uses (the engine options without the model's paths, and the server's
+    fields), so a change made by hand to strata-<model>.json shows without reading the log."""
+
+    CFG = {"exe": "x", "host": "0.0.0.0", "port": 8081, "api_key": "secret", "gpu": [0, 1], "fit_max_tokens": True,
+           "args": ["--native", "E:\\Strata\\packs\\iq3_s", "--mtp", "/s/mtp/rt", "m.gguf", "--kv", "int8",
+                    "--kv-resident", "32768", "--spec-min-p", "0.5", "--vram-reserve-mib", "2048", "--mmap-experts",
+                    "--prefill", "auto"]}
+
+    def test_summary(self):
+        s = setup.settings_summary(self.CFG)
+        self.assertEqual(s, "--kv int8 --kv-resident 32768 --spec-min-p 0.5 --vram-reserve-mib 2048 --mmap-experts "
+                            "--prefill auto; server 0.0.0.0:8081, api key set, gpu 0,1, fit_max_tokens true")
+        self.assertNotIn("secret", s)
+        self.assertIn("127.0.0.1:9000", setup.settings_summary({"args": []}, 9000))
+
+    def test_a_start_prints_it(self):
+        with tempfile.TemporaryDirectory() as d:
+            exe = Path(d) / "strata.exe"
+            exe.write_bytes(b"")
+            p = Path(d) / "strata-iq3_s.json"
+            args = [x for x in self.CFG["args"] if x != "m.gguf"]           # no model file here
+            p.write_text(json.dumps({**self.CFG, "exe": str(exe), "gpu": 0, "args": args}), encoding="utf-8")
+            out = io.StringIO()
+            with mock.patch.object(setup, "gpus", lambda: []), \
+                    mock.patch.object(setup, "refresh_draft_vocab"), \
+                    mock.patch.object(setup, "is_wsl", lambda: False), \
+                    mock.patch.object(setup.subprocess, "call", return_value=0), \
+                    contextlib.redirect_stdout(out):
+                setup.start(p, None, open_browser=False, yes=True)
+        text = " ".join(out.getvalue().split())
+        self.assertIn("Settings (strata-iq3_s.json): --kv int8 --kv-resident 32768", text)
+        self.assertIn("--vram-reserve-mib 2048", text)
+
+
+if __name__ == "__main__":
+    unittest.main()

update.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+# Update Strata without starting the model (#475): the newest code (git pull, when this folder is a git clone), then
+# what ./setup.sh does before a start - the engine (a new ready-made one when this version needs it, or compiled
+# again when its source changed), the Python packages, each installed model's settings and draft subset. The model
+# files are not touched. Start the model later with ./setup.sh. Options are passed on to setup.py.
+cd "$(dirname "$0")" || exit 1
+# all of it in a function, read before it runs: the git pull below can change this very file
+main() {
+  if [ -e .git ]; then
+    if ! command -v git >/dev/null 2>&1; then
+      echo "This folder is a git clone, but git is not installed: install it (sudo apt install git) or run"
+      echo "\"git pull\" here yourself, then run ./update.sh again."
+      exit 1
+    fi
+    echo "Getting the newest Strata (git pull) ..."
+    if ! git pull --ff-only; then
+      echo
+      echo "git pull did not succeed (the reason is above): nothing was updated. Files you changed here can stop it:"
+      echo "\"git status\" lists them."
+      exit 1
+    fi
+  else
+    echo "This copy of Strata was not made with git, so it cannot fetch new files itself. Download the newest one:"
+    echo "  https://github.com/Niko1221/Strata/archive/refs/heads/main.zip"
+    echo "unzip it anywhere and run ./setup.sh (or ./update.sh) in it: it finds the model files in Strata-data and"
+    echo "sets itself up the same way - nothing big is downloaded again."
+    echo "Checking this copy's engine and settings meanwhile ..."
+  fi
+  exec sh ./setup.sh --update "$@"
+}
+main "$@"