Merge branch 'f-138-fixB' into f-138-prs

A rebinding page cannot send the key, so with one configured the check only
broke tunnels and proxies that pass their own name on (cloudflared does).
Keyless setups keep it, with allowed_hosts; the 403 now also says that
setting an API key turns the check off.

CMakeLists.txt

@@ -8,7 +8,7 @@
 # below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
 # and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.37 LANGUAGES CXX)
+project(strata VERSION 0.1.38 LANGUAGES CXX)
 add_compile_definitions(STRATA_VERSION="${PROJECT_VERSION}")
 # issue #31: release builds keep a PDB beside the exe (not shipped in it), so a minidump from a stalled engine can
 # be read back to the exact source line
@@ -274,6 +274,15 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     target_include_directories(qsa_prompt_attn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
     target_link_libraries(qsa_prompt_attn_parity PRIVATE strata_kernels CUDA::cudart)
   endif()
+  # #512: the top-k's active-block bound (Turing's dispatch, forced on any card by STRATA_TOPK_ACTIVE_ANY) against
+  # the capacity rule, selected ids bitwise
+  if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA)
+    add_executable(qsa_topk_active_parity src/kernels/qsa_topk_active_parity.cpp)
+    target_include_directories(qsa_topk_active_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+    target_link_libraries(qsa_topk_active_parity PRIVATE strata_kernels CUDA::cudart)
+    add_test(NAME qsa_topk_active_parity COMMAND qsa_topk_active_parity)
+    set_tests_properties(qsa_topk_active_parity PROPERTIES SKIP_RETURN_CODE 77)
+  endif()
   # the QSA selection (block scores on the warp kernel and the tensor-core one, top-k): a GPU bench with a
   # scorer-vs-scorer comparison and an FP64 accuracy gate (synthetic, no model); the ctest runs a 32K context under a
   # 262144-cell capacity (the top-k dispatch follows the capacity) and exits non-zero on a failed gate
@@ -340,7 +349,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   # about arithmetic, and this one knows the ORDER the arithmetic happens in.
   add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
     src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp
-    src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp)
+    src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/peer_experts.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
   target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
   if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/file_expert_source_test.cpp")
@@ -558,6 +567,19 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   target_include_directories(iq_multi_parity PRIVATE ${_strata_gpu_include_directories})
   target_link_libraries(iq_multi_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME iq_multi_parity COMMAND iq_multi_parity)
+  # the prompt path's DeltaNet recurrence: gdn_rec_kh_kernel against gdn_rec_cols_pipe_kernel, bitwise (GPU,
+  # synthetic, no model; --bench adds timings)
+  if(STRATA_ENABLE_CUDA)
+    add_executable(gdn_rec_parity src/prefill/gdn_rec_parity.cu)
+    target_link_libraries(gdn_rec_parity PRIVATE ${_strata_gpu_runtime_target})
+    add_test(NAME gdn_rec_parity COMMAND gdn_rec_parity)
+  endif()
+  # native_expert_grouped's group stride and fused SwiGLU + q8_1, bitwise vs the launches before them (GPU, synthetic,
+  # no model; --bench adds v1/new timings of a window's 48 calls in a graph)
+  add_executable(native_grouped_parity src/kernels/native_grouped_parity.cpp)
+  target_include_directories(native_grouped_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(native_grouped_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  add_test(NAME native_grouped_parity COMMAND native_grouped_parity)
   add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)
   target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})
   target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
@@ -983,8 +1005,13 @@ if(STRATA_NATIVE_EXPERTS)
     target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact ${_strata_gpu_runtime_target})
     if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA)
       # Unsloth UD-Q4_K_XL's three expert pairs (and Q8_0 alone, its embedding) on synthetic ggml-quantized weights
-      # at 2560/640: GPU and CPU experts against the float reference, GPU dequantizers against to_float
-      foreach(_pair q4_K/q5_1 q4_K/q8_0 q5_K/q8_0 q8_0/q8_0)
+      # at 2560/640: GPU and CPU experts against the float reference, GPU dequantizers against to_float.
+      # q4_K/q5_0 and q5_0/q8_0: a plain llama-quantize "_M" tier GGUF (not Unsloth's dynamic method) commonly
+      # puts Q5_0 on down-projections (sometimes gate/up too) where this model's native_experts.txt had no GPU
+      # kernel for it before; q5_0/q5_0 (both roles Q5_0) is left out of this list since stacking the same
+      # lower-precision format on both roles pushes the GPU path over this test's 3% threshold (quantization
+      # noise, not a kernel bug -- see the PR).
+      foreach(_pair q4_K/q5_1 q4_K/q8_0 q5_K/q8_0 q8_0/q8_0 q4_K/q5_0 q5_0/q8_0)
         string(REPLACE "/" "_" _name "${_pair}")
         add_test(NAME native_expert_parity_${_name} COMMAND native_expert_parity --synthetic ${_pair})
         set_tests_properties(native_expert_parity_${_name} PROPERTIES TIMEOUT 120)

docs/AMD_HIP.md

@@ -190,6 +190,11 @@ The worker count above was used on a 16-core CPU; measure it for your CPU.
 The 4K context is a smoke-test starting point, not a model limit. The expert cache
 sizes itself automatically and leaves 1 GiB of VRAM headroom.
 
+**The card also drives a Linux desktop (#560 #516):** keep more VRAM free than the default 700 MiB, e.g.
+`./setup.sh --vram-reserve-mib 3072`. With the cache filling the card, the desktop's next VRAM need makes amdgpu move
+GPU memory to system RAM (GTT), the OOM killer then ends KWin/plasmashell or `systemd-oomd` ends apps, or the
+compositor fails with "Failed to pin framebuffer with error -12".
+
 The installer supports this backend (see "Install with setup" above). Images run through the CPU encoder for now (`--vision cpu`).
 Setup installs one AMD card, or several with `--gpus` (the engine's layer split; see RDNA4 below).
 
@@ -229,6 +234,10 @@ an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 39
   so none was shipped for it: there the plain hipBLAS path is already close. For hipBLASLt 1.5.0 (ROCm
   10.2.0a nightly) `tools/hip/gfx1201-hipblaslt-100500.txt` is shipped (see "Tuning table" below); on one R9700 it
   measured +3.9% prompt speed on 4,210-token prompts (1,590 vs 1,531 tok/s), a modest gain.
+  With the hipBLASLt 1.2.2 of a system ROCm 7.2.4 the plain path is far off, and `tools/hip/gfx1201-hipblaslt-100202.txt`
+  is shipped for it (24 of the gfx1100 table's 26 shapes; setup uses it only with that exact version): on an R9700
+  with the full IQ3_XXS, `--kv int8 --kv-resident 32768`, a fresh 32K prompt read at 638 -> 1,177 tok/s and a 7K one
+  at 656 -> 1,164 tok/s with it, decode unchanged.  `hip_prefill_hipblaslt_gemm` passes with it.
 - **Both cards in one run (layer split, engine 0.1.30):** the config's `"backend": "hip", "gpu": [1, 0]` (R9700
   first) runs through `serve/server.py` (setup writes it with `--gpus 1,0` since 0.1.31). Auto split put layers 0-27 on the R9700 and
   28-47 on the 9070 XT. With every expert on the GPUs the split gives exactly the tokens of the R9700 alone (4K and
@@ -274,8 +283,9 @@ an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 39
 
 ## Community-validated cards
 
-Run by their owners, not on the maintainers' machines; setup accepts them like gfx1100 / gfx1201. No hipBLASLt table
-is shipped for them (make one with [Tuning table](#tuning-table) and compare the prompt speed with and without it).
+Run by their owners, not on the maintainers' machines; setup accepts them like gfx1100 / gfx1201. setup ships a
+hipBLASLt table for gfx1200 (the numbers below); gfx1101 has none (make one with [Tuning table](#tuning-table)
+and compare the prompt speed with and without it).
 
 - **gfx1101, RX 7800 XT 16 GB** (jhohertz, #254; engine 0.1.29, Ryzen 9 5950X, 121 GiB RAM, system ROCm with
   hipBLASLt 1.4.1): `./setup.sh --backend hip` detected the card and compiled the engine; `strata-device --selftest`
@@ -295,6 +305,17 @@ is shipped for them (make one with [Tuning table](#tuning-table) and compare the
   Greedy output was the same across runs. For comparison, llama.cpp's HIP build measured 20 tok/s decode and
   450 tok/s prompt on that card.
 
+  Since 0.1.38 setup ships that table (`tools/hip/gfx1200-hipblaslt-100202.txt`). On engine 0.1.31, one binary (ctest
+  37/37, without `ple_parity` and `platform_memory_test`), prompts of repeated code blocks, prefill 2560 for
+  the short prompt and 16384 with `--kv int8 --kv-resident 65536` otherwise, two runs each - prompt speed
+  with the table vs plain hipBLAS (no table):
+
+  | prompt | with the table | plain hipBLAS | gain |
+  |---|---|---|---|
+  | 2,374 tokens | 597-599 tok/s | 396-398 tok/s | 1.50x |
+  | 65,045 tokens | 754-757 tok/s | 380 tok/s | 1.99x |
+  | 130,091 tokens (the same flags) | 719 tok/s | 363-370 tok/s | 1.98x |
+
 ## RDNA2 (gfx1030)
 
 The RX 6800 / 6800 XT / 6900 XT / 6950 XT run the same kernels: wave32 and 64 KiB of LDS per workgroup. The one

docs/DETAILS.md

@@ -405,7 +405,8 @@ server, three server options (all off by default; also as keys in `strata-<model
 | `--min-free-vram-mib 11000` | `"min_free_vram_mib": 11000` | load an unloaded model only when that much VRAM is free (it waits up to 15 s for memory being given back), else answer **503** "the GPU is in use by another program" instead of starting into what a game left (with several GPUs it checks the first one) |
 | `--before-load "cmd"` | `"before_load": "cmd"` or `["cmd", "arg"]` | a command run before the model is loaded again, e.g. one that unloads another server's model |
 
-`POST /unload` unloads it now (`409` while a request is running) and `POST /load` loads it ahead of a request;
+`POST /unload` unloads it now (`409` while a request is running) and `POST /load` loads it ahead of a request (both
+with `Content-Type: application/json`, e.g. `curl -X POST -H "Content-Type: application/json" localhost:8080/unload`);
 `/health` says `"loaded"`, `/v1/models` lists it as `unloaded` (like llama.cpp's router), `/props` sets
 `is_sleeping` and the Monitor shows the state. Unloading ends the engine process - and the image encoder, when images
 are on; it is started again first, as at a start - so their VRAM and RAM go straight back. The model files stay in
@@ -482,6 +483,40 @@ print(r.choices[0].message.content)
   15 s, and `GET /status` says what it is doing (`reading the prompt`, `answering`, tokens so far). Closing the
   connection or pressing stop in your app really stops the model, so the next request starts at once.
 - **Chat apps.** Any app with an "OpenAI-compatible" provider works: base URL `http://127.0.0.1:8080/v1`, any API key.
+- **OpenCode** (#543). A starting point for `opencode.jsonc` (in your project, or `~/.config/opencode/`); the field
+  names are OpenCode's, so check its config docs if your version differs:
+
+  ```jsonc
+  {
+    "$schema": "https://opencode.ai/config.json",
+    "provider": {
+      "strata": {
+        "npm": "@ai-sdk/openai-compatible",
+        "name": "Strata (local)",
+        "options": { "baseURL": "http://127.0.0.1:8080/v1", "apiKey": "none" },  // or your api_key
+        "models": {
+          "strata": {
+            "name": "Qwen3.8-Flash-Next (Strata)",
+            // context: what you chose in setup; output: what one reply may use (prompt + output must fit)
+            "limit": { "context": 262144, "output": 32768 },
+            "options": { "reasoningEffort": "high" },                  // sent as reasoning_effort
+            "variants": {                                              // switch between them in OpenCode
+              "low": { "reasoningEffort": "low" },
+              "medium": { "reasoningEffort": "medium" },
+              "none": { "reasoningEffort": "none" }
+            }
+          }
+        }
+      }
+    },
+    "model": "strata/strata"
+  }
+  ```
+
+  Set `limit.context` to the context you chose in setup: OpenCode compacts the conversation before it gets there.
+  Keep `limit.output` well under it: a request whose prompt plus `max_tokens` runs past the context is refused (see
+  **Context** below), or add `"fit_max_tokens": true` to `strata-<model>.json`. For a hard cap on the thinking, add
+  `"reasoning_budget_tokens": N` to `strata-<model>.json` (see above).
 - **Claude Code** (Strata 0.1.17 or newer): set `ANTHROPIC_BASE_URL=http://127.0.0.1:8080` and
   `ANTHROPIC_MODEL` to a Claude model name it knows (it refuses names it doesn't; Strata ignores the name), plus any
   `ANTHROPIC_AUTH_TOKEN` (or your `api_key`, if you set one).
@@ -505,9 +540,30 @@ print(r.choices[0].message.content)
   clients then send it as their API key. Streamed answers carry `X-Accel-Buffering: no`, so nginx-style proxies pass
   each token on at once. The web app's settings and MCP tools only answer Strata's own page: when you open it through
   a proxy or tunnel whose address differs, add that address, e.g. `"trusted_origins": ["https://strata.example.com"]`.
+  With the key set, any `Host` name reaches the server (see Host names below).
 - **From web apps in a browser (CORS).** Off by default. `"cors_origins": ["https://chat.example.com"]` lets pages of
   those origins call `/v1/*` from the browser (Open WebUI's direct connections, browser extensions); `["*"]` lets any
   page do it - only sensible with an API key. It never opens `/settings`, `/unload` or the MCP tools.
+- **Host names (DNS rebinding).** A web page of another site can point its own name at `127.0.0.1` and then reach
+  this server as if it were its own, so without an API key the server answers only requests whose `Host` is a name
+  it knows (with a key the check is off: such a page cannot send the key, and tunnels and proxies that pass their
+  own name on keep working):
+  `localhost` (and `*.localhost`), any IP address (`127.0.0.1`, `[::1]`, `192.168.x.x`, ...), the address it
+  listens on and, when it listens beyond this PC (`0.0.0.0` or a LAN address), this PC's name (`mypc`, `mypc.local`)
+  and `host.docker.internal`; any port. Others get **403** naming the setting, and the server window prints one line
+  for each. Reach it under another name (a reverse proxy that keeps the name, a tunnel, a DNS name on your network,
+  another container's name for it)? Add the name: `"allowed_hosts": ["strata.example.com"]` in
+  `strata-<model>.json` or `STRATA_ALLOWED_HOSTS=strata.example.com` (comma-separated); `".example.com"` allows that
+  name and every name below it, and `["*"]` turns the check off (so does setting `api_key`). The hosts of
+  `trusted_origins` count as allowed. Requests without a `Host` header (HTTP/1.0 clients) pass.
+- **Web pages without an API key.** Without `api_key`, a `POST` to `/v1/*` that carries an `Origin` header (a
+  browser page sent it) is answered only for Strata's own page, pages on `localhost` or an allowed host name (any
+  port), the origins in `trusted_origins` or `cors_origins`, and browser extensions and desktop apps
+  (`chrome-extension://`, `moz-extension://`, `app://`: no web site can send those), and only with a JSON body; any
+  other page, and `Origin: null`, gets **403**. Clients that send no `Origin` (curl, the OpenAI and Anthropic SDKs,
+  other servers) are not affected. With
+  an API key, the key decides. `POST /unload` and `POST /load` take `Content-Type: application/json` from Strata's
+  own page (or no `Origin`), like `/settings`.
 
 **Conversation cache.** A request that continues a chat reads only the part after what the engine already holds: the
 live session, or one of the checkpoints it keeps in RAM (up to 6, ~118 MB each, taken at the start of each new

docs/INSTALL.md

@@ -201,6 +201,17 @@ By hand: add `"--vram-reserve-mib", "2048"` to the config's `"args"` list and re
 much smaller, so answers can be a little slower. The engine sizes its cache from the VRAM free when it starts: what
 another program already holds then is left alone anyway; the reserve is room for what it takes later.
 
+**An AMD card that also drives a Linux desktop (#560 #516):** with the default reserve the expert cache fills the
+card, and when the desktop (the compositor, a browser, a new app) needs more VRAM, the amdgpu driver moves GPU memory
+to system RAM, which the model's experts already fill: the OOM killer then ends the desktop session (KWin, plasmashell)
+or the compositor fails ("Failed to pin framebuffer"). `./setup.sh --vram-reserve-mib 3072` fixed it in both reports
+(about 2.3 GB fewer experts in VRAM, a few % of speed). Setup and the server window say so on such a PC.
+
+**A card under 8 GB (#496):** when the default reserve leaves the expert cache too little room, the engine lowers the
+reserve (down to 300 MiB) until it fits, and warns if the card then ends nearly full. If the start stops with "no VRAM
+is left for the expert cache", that log line says how much is short; an 8K context and `--draft-vocab en` at setup
+free the most.
+
 **Model files downloaded by hand, or from a mirror (#495):** setup's step 5 prints the folder it expects them in
 (`Strata-data\models\<SIZE>\`, e.g. `Strata-data\models\IQ3_XXS\`): put them there with their original names, or
 point setup at them with `--gguf-dir`. To let setup download from a Hugging Face mirror itself, set `HF_ENDPOINT`

docs/MODELS.md

@@ -11,12 +11,14 @@ The installer recommends one for your PC; this page explains the choice. Back to
 
 | Your RAM | Take | Why |
 | --- | --- | --- |
-| **32 GB** | **Coder** | it fits 32 GB, and it is made for code (with a 24 GB card, Q2_0 and IQ2_XS run too: [low-RAM mode](#a-big-graphics-card-and-little-ram)) |
+| **32 GB** | **Coder** | it fits 32 GB; best at code, weaker at everything else ([why](#coder)). With a 24 GB card, Q2_0 and IQ2_XS run too ([low-RAM mode](#a-big-graphics-card-and-little-ram)) and are the better pick for general use |
 | **48 GB** | **IQ2_XS** (or Q2_0, the fastest) | the larger sizes do not fit |
 | **64 GB** | **IQ2_XS** (recommended), or IQ3_XXS / IQ3_S | every size fits (IQ3_S with little else open) |
 | **96 GB or more** | **IQ3_S**, or [Unsloth's 4-bit](#unsloth-ud-q4_k_xl-experimental) (experimental) | room for the largest sizes |
 
-Not sure? Take **IQ2_XS** - or the **Coder** if you mainly write code, or have 32-48 GB of RAM.
+Not sure? Take **IQ2_XS**. The **Coder** is the one that fits a 32 GB PC, but it keeps only 256 of the 512 experts,
+chosen on code data, so it is weaker outside code and in languages other than English. For general use, or whenever
+your RAM allows, take a full model (Q2_0, IQ2_XS or IQ3_S).
 
 ## How fast is each size
 
@@ -91,12 +93,12 @@ The original, in all four sizes. With images, and with the
 ### Coder
 
 **[Coder](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF)** - ISTA-DASLab's coding version:
-half of the experts removed, keeping the ones that code, tool use and images need (91% of the full model's SWE-bench
-Verified score, 99% of LiveCodeBench, by its authors). One size (IQ1_M: its experts stored like IQ3_S): shard 1 is
-**29.6 GB**, so it fits a PC with **32 GB of RAM**, runs 262K context on 64 GB, and reads long prompts the fastest of
-all. It is weaker outside code, and that includes Chinese and other CJK text (#438: Chinese answers came out wrong
-or looping where English was fine). For general chat or CJK text, take a size that keeps every expert: Q2_0,
-IQ2_XS or IQ3_S.
+it keeps 256 of each layer's 512 experts, chosen on code (with agentic and image) data, and drops the rest (91% of
+the full model's SWE-bench Verified score, 99% of LiveCodeBench, by its authors). One size (IQ1_M: its experts stored
+like IQ3_S): shard 1 is **29.6 GB**, so it fits a PC with **32 GB of RAM**, runs 262K context on 64 GB, and reads long
+prompts the fastest of all. With half the experts gone it is weaker outside code and in languages other than
+English, Chinese and other CJK text included (#438: Chinese answers came out wrong or looping where English was
+fine). For general use, or whenever your RAM allows, take a size that keeps every expert: Q2_0, IQ2_XS or IQ3_S.
 More: [details](DETAILS.md#or-the-coder-half-the-experts-for-code).
 
 ```

docs/SECOND_GPU.md

@@ -83,3 +83,42 @@ short decode requests.
 The existing warning about the CUDA0 expert-cache GPU hit path still applies:
 its outputs diverge from cache-off runs. Treat performance as experimental
 until the generated tokens have been validated.
+
+## Peer tier (`--peer-device`)
+
+`--peer-device N` puts a second adaptive expert cache on CUDA device N. It
+takes the ranked pairs CUDA0's cache does not hold, as many as fit. The peer
+computes the rows of its own experts, for decode windows and for prompt
+chunks; the activations and the results cross NVLink or another P2P path. The
+tier adapts while the server runs, like the primary cache. It is an
+alternative to the CUDA1-3 caches above, not a third tier beside them.
+
+- `--peer-device N` (default off): enable the tier on CUDA device N (N >= 1).
+- `--peer-reserve-mib M` (default 600): leave M MiB free on the peer card; the
+  cache takes what remains. The prompt-path buffers need this headroom.
+- `--peer-slots N` (default 0): cap the tier at N experts; 0 = as many as fit.
+- `--peer-adapt-swaps N` (default -1): swaps per adaptive round on the peer;
+  -1 uses the primary's `--adapt-swaps`.
+- `--peer-prefill-rows N` (default -1): the share of each prompt chunk's rows
+  the peer computes; -1 is half of chunk x top-k, 0 keeps prompt rows on the
+  primary.
+
+`--peer-device` requires `--expert-profile` and an enabled expert cache, and
+the device must be visible; it refuses otherwise. It also refuses
+`--layer-split` (a different second-GPU mode: use one or the other) and
+`--expert-cache-device1..3` (the peer tier already caches experts on that card):
+
+    strata generate: --peer-device cannot be combined with --layer-split (use one or the other)
+    strata generate: --peer-device cannot be combined with --expert-cache-device1..3 (the peer tier already caches experts there)
+
+Without `--peer-device` the binary is unchanged; its output is byte-identical
+to the release. With `--peer-device` and the same expert set split across the
+two cards, the generated tokens are byte-identical to the single-GPU run under
+the exactness gate.
+
+With a peer the prompt path keeps the MMQ path; the fused int8 prompt path is
+not yet combined with the peer's rows. Mapped host buffers gain
+`cudaHostAllocPortable` only with a peer, since only then does a second
+context write them. The tier size is manual for now (`--peer-reserve-mib`,
+`--peer-slots`); automatic sizing on small cards wants the buffer lending of
+#216 and is a follow-up.

include/strata/core/expert_source.hpp

@@ -43,6 +43,7 @@ struct ExpertLayout;
 
 namespace strata::core {
 
+class PeerExperts;   // multi-GPU: the second GPU's expert tier (peer_experts.hpp)
 class RemoteExperts;
 struct LoadStats;
 
@@ -336,6 +337,9 @@ struct ExpertDispatch {
     std::vector<float> usage;
     int64_t multi_misses = 0;      ///< distinct (layer, expert) pairs the CPU computed in verify windows
     int64_t multi_entries = 0;     ///< routed (token, expert) entries the CPU served in verify windows
+    /// Multi-GPU: the second GPU's tier.  Its experts are computed there instead of on the CPU (kind 2).
+    PeerExperts* peer = nullptr;
+    int64_t peer_entries = 0;      ///< routed entries the peer served in verify windows
     /// Set when `dispatch` could not produce an answer.  The loop itself has no error channel, so this is
     /// where a source failure surfaces: the driver checks it after `session_loop` returns rather than the
     /// engine computing from a half-filled `parts`.
@@ -471,6 +475,16 @@ public:
     double file_ms() const { return (double) file_us_.load(std::memory_order_relaxed) / 1000.0; }
     /// Threads `prefetch` reads the GGUF with (STRATA_FETCH_THREADS, default 8).
     void set_fetch_threads(int n) { fetch_threads_ = n < 1 ? 1 : n; }
+    /// #286 (Windows): read the experts straight from the drive (FILE_FLAG_NO_BUFFERING, overlapped) instead of
+    /// through the mapped files - the GGUF in place, or a pack's experts.bin - when the file cache could not keep them
+    /// beside `ram_bytes` (the RAM budget), cached now or not (their mapped pages would land in the working set); see
+    /// experts_unbuffered.  The mapped reads' page faults are one small request each, and the pages they bring in
+    /// take the RAM the budget was sized for.  `why` says what decided.
+    bool set_unbuffered(uint64_t ram_bytes, std::string& why);
+    bool unbuffered() const { return !direct_.empty(); }
+    /// #286, unbuffered: assembles the blobs of these pairs ahead of the `blob` calls that will ask for them (the
+    /// GPU cache's fill from the profile) - one batch of reads instead of one blob at a time.  At most 64 pairs.
+    void prefetch_pairs(const std::pair<int32_t, int32_t>* pairs, int64_t n);
 
     const uint8_t* blob(int64_t layer, int64_t expert) override;
     bool pinned(int64_t layer, int64_t expert) const override;
@@ -485,7 +499,8 @@ public:
     /// CS-T: the GGUF in place asks the OS for the predicted experts' pages (PrefetchVirtualMemory on Windows,
     /// madvise(WILLNEED) elsewhere), skipping the RAM copy's.
     void warm(int64_t layer, const int64_t* experts, int64_t n) override;
-    bool warms() const override { return !role_ptr_.empty(); }
+    /// Not when the reads are unbuffered: the warmed pages would be read through the file cache, a second time.
+    bool warms() const override { return !role_ptr_.empty() && direct_.empty(); }
     /// Of the blobs the file tier read for the decode, how many had been warmed for their layer beforehand.
     int64_t warmed_hits() const { return warm_hits_.load(std::memory_order_relaxed); }
     int64_t warmed() const { return warm_count_.load(std::memory_order_relaxed); }
@@ -502,6 +517,18 @@ private:
     const uint8_t* staged_blob(int64_t layer, int64_t expert);
     bool claim_stage(int64_t key, size_t& v, bool& fill);
     bool fill_stage(size_t v, int64_t layer, int64_t expert, uint8_t* dst);
+    void publish_stage(size_t v, int64_t layer, bool ok, double us);
+    struct Fill { size_t v; int64_t layer, e; uint8_t* dst; };
+    /// The claimed buffers' blobs: one overlapped batch when unbuffered, else the fetch threads' mapped copies.
+    void fill_many(const std::vector<Fill>& todo);
+    /// #286: the blobs from the drive, unbuffered: every role's 4 KiB-aligned window is read at once (overlapped)
+    /// into this thread's aligned buffer, then copied into place.  False when a read fails.
+    bool read_direct(const Fill* fills, size_t n) const;
+    std::vector<std::string> paths_;          ///< the mapped files, as maps_
+    std::vector<void*> direct_;               ///< #286: per file, an unbuffered overlapped handle (Windows)
+    std::vector<int> role_file_;              ///< 3 x n_layers: index into maps_ / direct_
+    /// blobs assembled in the stage buffers (`blob` hands those out): the GGUF in place, or any unbuffered source
+    bool staged() const { return !role_ptr_.empty() || !direct_.empty(); }
     static constexpr uint64_t kNoComplement = detail::kNoCacheComplement;
     // ---- CS-T: the GGUF shards in place
     std::string gguf_;
@@ -648,7 +675,8 @@ private:
 /// layout's type and dimensions, and inside the file.  `native` is the --native shard (see set_gguf).
 bool check_experts_gguf(const std::string& native, const strata::kernels::cpu::ExpertLayout& lay, std::string& err);
 /// Fills `dst` (lay.total bytes, the experts.bin layout) from the GGUF files, one role at a time.
+/// `unbuffered`: each chunk read past the file cache (Windows).
 LoadStats load_experts_gguf(const std::string& native, uint8_t* dst, const strata::kernels::cpu::ExpertLayout& lay,
-                            int threads);
+                            int threads, bool unbuffered = false);
 
 }  // namespace strata::core

include/strata/core/peer_experts.hpp

@@ -0,0 +1,106 @@
+// include/strata/core/peer_experts.hpp - multi-GPU: a second expert tier on another GPU.
+//
+// The primary GPU (device 0) keeps everything it had: the dense weights, the KV cache, the MTP layer, its own
+// adaptive expert tier and the captured verify graph.  This class adds an expert tier on a SECOND card.  From
+// the verify graph's point of view nothing changes: for each layer the pool thread decides which routed experts
+// the primary computes (its VRAM tier, the PCIe share), and every remaining expert is a "CPU row" the graph reads
+// from mapped host memory after the pool raises its flag.  The peer takes the rows whose experts it holds: the
+// pool thread launches them on the peer right after publishing the primary's plan, the CPU computes the rest at
+// the same time, and the peer's rows are written into the same mapped rows before the pool returns.
+//
+// The peer computes with the SAME kernels as the primary's hit path (q8_1 activations + native_expert_grouped),
+// from the same activation values, so an expert gives bit-identical rows on either card.
+//
+// The tier follows the conversation like the primary's: `adapt` swaps the most-routed experts that neither card
+// holds into the peer's least-used slots (per layer, so a slot keeps its layer's blob size).
+#pragma once
+
+#include "strata/core/expert_cache.hpp"
+#include "strata/core/expert_source.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace strata::core {
+
+/// multi-GPU: set once at startup when a peer device is configured (--peer-device N).  The mapped host buffers
+/// the peer card writes into are then allocated with cudaHostAllocPortable; without a peer the flag stays off.
+void set_peer_portable(bool on);
+bool peer_portable();
+
+class PeerExperts {
+public:
+    PeerExperts() = default;
+    ~PeerExperts();
+    PeerExperts(const PeerExperts&) = delete;
+    PeerExperts& operator=(const PeerExperts&) = delete;
+
+    /// Opens the tier on `device`: fills it with the first pairs of `ranked` that `primary` does not hold, as many
+    /// as fit in the card's free memory minus `reserve_mib` (`max_slots` > 0 caps the count).
+    bool open(int device, const std::vector<std::pair<int32_t, int32_t>>& ranked, const ExpertCache& primary,
+              ExpertSource& src, int64_t n_layers, int64_t n_expert, int reserve_mib, int64_t max_slots,
+              std::string& err);
+    void close();
+    bool valid() const { return device_ >= 0; }
+    int device() const { return device_; }
+
+    /// Whether the peer holds (layer, expert) right now.
+    bool has(int64_t layer, int64_t expert) const { return res_[(size_t) (layer * n_expert_ + expert)] >= 0; }
+
+    /// The device address of (layer, expert)'s blob on the peer, or null when it is not resident.
+    const uint8_t* slot_ptr(int64_t layer, int64_t expert) {
+        const int32_t sl = res_[(size_t) (layer * n_expert_ + expert)];
+        return sl >= 0 ? cache_.device_slot(sl) : nullptr;
+    }
+    /// Whether the primary (device 0) and the peer can read each other's memory directly (NVLink / PCIe P2P).
+    bool p2p() const { return p2p_; }
+
+    /// Starts the peer's share of one layer: the entries i with kind[i] == 2 (x: n_tok rows of n_embd floats).
+    /// multi-GPU: with `out` (pinned, portable host rows) the peer writes its rows straight into
+    /// out[entry] with a scatter kernel (no D2H copy + host memcpy in finish); STRATA_PEER_DIRECT=0: the old way.
+    bool launch(int64_t layer, const float* x, const int32_t* ids, int64_t n_tok, int64_t k, const int32_t* kind,
+                std::string& err, float* out = nullptr);
+    /// Waits for the launched share and writes its rows into `out` (row i = entry i, n_embd floats).
+    bool finish(float* out, std::string& err);
+
+    /// Adaptive tier: swap up to `max_swaps` experts routed at least twice that neither the primary (`res0`, slot
+    /// or -1) nor the peer holds into the peer's least-used slots.  Copies run on the peer's refill stream; a
+    /// swapped-in expert becomes resident at `apply_pending`.  Call only between verify windows.
+    bool adapt(const float* usage, const int32_t* res0, int max_swaps, std::string& err);
+    void apply_pending(bool wait);
+
+    int64_t resident() const { return resident_; }
+    /// the last launch writes its rows into the host rows itself (the caller must not touch them)
+    bool launched_direct() const { return launched_direct_; }
+    double gib() const { return cache_.gib(); }
+    int64_t entries() const { return entries_; }      ///< routed entries the peer computed
+    int64_t experts() const { return experts_; }      ///< distinct (layer, expert) the peer computed
+    int64_t swaps() const { return swaps_; }
+    double ms_wait = 0;                               ///< host time spent in finish() waiting for the peer
+
+private:
+    int device_ = -1;
+    bool p2p_ = false;
+    int64_t n_layers_ = 0, n_expert_ = 0;
+    ExpertCache cache_;
+    ExpertSource* src_ = nullptr;
+    std::vector<int32_t> res_;                        ///< [n_layers * n_expert] -> peer slot or -1
+    std::vector<std::pair<int32_t, int32_t>> pending_;   ///< (residency index, slot) once the copies land
+    int64_t resident_ = 0, entries_ = 0, experts_ = 0, swaps_ = 0;
+    cudaStream_t stream_ = nullptr, refill_ = nullptr;
+    cudaEvent_t refill_ev_ = nullptr;
+    float* h_x_ = nullptr;   float* d_x_ = nullptr;   // n_tok x n_embd
+    float* h_out_ = nullptr; float* d_out_ = nullptr; // compact rows
+    void* h_meta_ = nullptr; void* d_meta_ = nullptr; // groups: ptr | start | dst | tok | count
+    uint8_t* d_q8_ = nullptr;
+    void* d_scratch_ = nullptr;
+    std::vector<int32_t> row_of_;                     ///< compact row -> original entry
+    int64_t launched_rows_ = 0;
+    bool launched_direct_ = false;
+};
+
+}  // namespace strata::core

include/strata/core/pinned.hpp

@@ -104,6 +104,22 @@ LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_byte
 LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
                               const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk);
 
+/// The same ranges read UNBUFFERED straight into `dst` (Windows): no staging buffer and no file-cache copy - the
+/// drive's DMA lands where the experts live. Every range, `dst` and `chunk` must be 4 KiB aligned; returns ok =
+/// false with an empty `error` when they are not (or off Windows), and the caller falls back to load_experts_ranges.
+LoadStats load_experts_direct(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
+                              const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk);
+
+/// Whether the expert files are better read unbuffered (Windows; false elsewhere): a timed probe of random 64 KiB
+/// reads says they are not in the OS file cache (a cached read takes ~10 us, the drive ~80), and the RAM left
+/// beside the arena (`arena_bytes`) could not keep them cached for the next start either - so a warm restart after
+/// an idle unload never gets slower, and only a start that reads the drive anyway skips the cache's copy.
+/// STRATA_UNBUFFERED_LOAD=1 / 0 forces it. `why` says what decided.
+/// `cache_counts` false (the file tier with a RAM budget): only whether the files could be kept decides - their mapped
+/// pages land in the process's working set, so a partly cached file would still take the RAM the budget was sized for.
+bool experts_unbuffered(const std::vector<std::string>& files, uint64_t arena_bytes, std::string& why,
+                        bool cache_counts = true);
+
 // FNV-1a 64.  Per layer, so a corrupt or short read names WHICH layer rather than just failing a whole-file
 // comparison - the same reason the Phase 1 tools report the first differing element.
 uint64_t fnv1a64(const uint8_t* p, uint64_t n, uint64_t seed = 1469598103934665603ull);

include/strata/core/verify.hpp

@@ -254,6 +254,7 @@ private:
     int32_t* plan_ = nullptr;                                     // device copy of the plan block
     uint8_t* staging_ = nullptr;                                  // VRAM slots for the PCIe share of the misses
     static constexpr int64_t kStagingBlobs = 16;
+    static constexpr int64_t kPcieGroupRows = 4;                  // the PCIe call's groups side by side (of <= 16)
     uint8_t* hit_xq_ = nullptr;
     uint8_t* nat_xq_ = nullptr;   // plan v0.3 P6: q8_1 activations for a native pack's grouped experts
     float* hit_xs_ = nullptr;

include/strata/hip_compat/cuda_runtime.h

@@ -20,10 +20,13 @@
 #define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor
 #define cudaDevAttrComputeCapabilityMinor hipDeviceAttributeComputeCapabilityMinor
 #define cudaDeviceGetAttribute hipDeviceGetAttribute
+#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer
+#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess
 #define cudaDeviceProp hipDeviceProp_t
 #define cudaDeviceSynchronize hipDeviceSynchronize
 #define cudaDriverGetVersion hipDriverGetVersion
 #define cudaErrorNotReady hipErrorNotReady
+#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled
 #define cudaErrorStreamCaptureUnsupported hipErrorStreamCaptureUnsupported
 #define cudaError_t hipError_t
 #define cudaEventCreate hipEventCreate
@@ -77,6 +80,7 @@ hipError_t mem_get_info(size_t* free_bytes, size_t* total_bytes);
 #define cudaMemcpy hipMemcpy
 #define cudaMemcpy2DAsync hipMemcpy2DAsync
 #define cudaMemcpyAsync hipMemcpyAsync
+#define cudaMemcpyPeerAsync hipMemcpyPeerAsync
 #define cudaMemcpyDefault hipMemcpyDefault
 #define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice
 #define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost

include/strata/kernels/cpu/iq_avx2.hpp

@@ -1,5 +1,5 @@
 // include/strata/kernels/cpu/iq_avx2.hpp - AVX-2 multi-token dot products for the i-quant expert
-// formats (IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S) against Q8_K activations (ggml's block_q8_K).
+// formats (IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS) against Q8_K activations (ggml's block_q8_K).
 //
 // The AVX-512 kernel's scheme on the CPUs without it (AMD Zen 2/3, Intel Core 12th-14th gen): the weights of a
 // 32-value chunk are decoded once per verify window and every token applies them with five instructions.

include/strata/kernels/elementwise.hpp

@@ -104,6 +104,8 @@ void doorbell_wait(const uint32_t* d_flag, const uint32_t* d_seq, void* stream);
 void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream);
 /// `rows` rows of `width` floats from mapped memory, except the rows listed in hit_rows[0, *count) (device),
 /// which are written +0.0 instead (a verify window's GPU-computed entries: the pool leaves zeros there).
+/// multi-GPU: dst[rows[r] * width ..] = src[r * width ..] for r < n (float4; dst may be mapped host memory).
+void scatter_rows_f32(const float* src, float* dst, const int32_t* rows, int64_t n, int64_t width, void* stream);
 void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
                            const int32_t* count, void* stream);
 

include/strata/kernels/iq_kernels.hpp

@@ -56,9 +56,17 @@ size_t native_expert_scratch_bytes(int64_t cap_entries, int64_t n_ff);
 /// Grouped experts in the native format: group g's blob at device address grp_ptr[g]; its entries
 /// [grp_start[g], grp_start[g+1]) read token ent_tok[e]'s q8_1 activation (n_embd/32 blocks per token in x_q8_1)
 /// and write row ent_dst[e] of `out` (n_embd floats).  Counts are read on the device.
+/// `grid_groups` (1 .. cap_groups; 0 = cap_groups) groups run side by side, a block row each striding over the rest:
+/// a call that usually has few groups or none (the verify window's PCIe share) launches less for the ones it does
+/// not have.  The results do not depend on it.
 void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long* grp_ptr, const int32_t* grp_start,
                            const int32_t* n_groups, const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups,
-                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream);
+                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream,
+                           int64_t grid_groups = 0);
+/// true: `native_expert_grouped`'s launches before the group stride (STRATA_GROUPED_V1=1 at startup) - a block row
+/// per possible group, SwiGLU and the q8_1 quantization as two kernels over all cap_entries.  Bitwise the same results
+/// (native_grouped_parity checks it); kept for A/B timing.  Set before graph capture; captured graphs keep theirs.
+void native_grouped_set_v1(bool v1);
 
 /// `iq_mmvq` and `native_expert_grouped` decode each weight part once and apply it to every column / entry;
 /// true selects the older kernels that decode it again per column (STRATA_OLD_IQ_MMVQ=1 at startup).  Both give

include/strata/kernels/ngram.hpp

@@ -114,7 +114,7 @@ enum class PleIo { Direct, Mmap };
 
 struct PleIoOptions {
     PleIo mode = PleIo::Direct;
-    uint32_t max_inflight = 64;      ///< outstanding SSD reads (decode needs 16; prefill chunks use more)
+    uint32_t max_inflight = 256;     ///< outstanding SSD reads (decode needs 16; a prompt chunk saturates the SSD at 256)
     uint64_t cache_rows = 1u << 20;  ///< bounded row cache: 1,048,576 rows x 90 B ~ 95 MB; 0 disables
     bool io_thread = true;           ///< reads submitted by a worker thread, not the caller
     /// Mmap mode only (`--ple-io ram`): lock the whole mapped table in RAM at open, so no SSD read ever sits on

include/strata/kernels/qsa_prompt_attn.hpp

@@ -14,7 +14,8 @@
 //   * q and p are split into FP16 hi + lo parts (two MMAs each), so they keep ~22 bits: the result differs from
 //     the FP32 kernel by summation order and the exp2 rounding, not by an FP16 cast.
 // Not bitwise equal to `qsa_decode_attn_batch`; `qsa_prompt_attn_parity` bounds the difference and the prompt
-// quality gate (needles, teacher-forced top-1) checks it end to end. Q4_0 KV is not handled (returns false).
+// quality gate (needles, teacher-forced top-1) checks it end to end. Q4_0 KV (mode 4): each block's codes enter
+// exactly as int8 and its scale (one per 32 values) in FP32, as int8 KV's do (STRATA_PROMPT_ATTN_Q4=0: the old kernel).
 #pragma once
 
 #include "strata/kernels/qsa_decode_attn.hpp"
@@ -24,7 +25,7 @@
 namespace strata::kernels {
 
 /// Same arguments and output as `qsa_decode_attn_batch` minus the scratch. Returns false (nothing launched) when the
-/// pools are Q4_0 or the geometry is not 24 heads / 2 KV heads / 256: the caller then uses the old kernel.
+/// geometry is not 24 heads / 2 KV heads / 256: the caller then uses the old kernel.
 bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
                            int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream);
 

include/strata/kernels/qsa_select.hpp

@@ -41,7 +41,9 @@ void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64
                     int64_t active_blocks = -1);   ///< > 0: no query of the call has more than this many blocks (n_bid + 1;
                                                    ///< the same contract as qsa_block_scores's). The register kernel is
                                                    ///< chosen by this, not by the capacity max_blocks (a long
-                                                   ///< --max-context otherwise sends every short prompt to the slow one)
+                                                   ///< --max-context otherwise sends every short prompt to the slow one).
+                                                   ///< CUDA uses the bound only on sm_75; HIP keeps its existing policy.
+                                                   ///< Omit it for captured graphs whose context can grow after capture.
 /// The same ids on a thread-block cluster of 8 CTAs per query (sm_90+, CUDA; S19). qsa_block_topk takes it for calls
 /// of up to 16 queries unless STRATA_QSA_CLUSTER=0. False (nothing launched) where it cannot run: HIP, a card or a
 /// build below sm_90, or a capacity whose keys do not fit one cluster's shared memory. Capturable.

include/strata/prefill/kernels.hpp

@@ -43,8 +43,8 @@ void gdn_gates(const float* ab, const float* dt, const float* ssm_a, float* gate
 /// The 4-tap causal conv + SiLU over the chunk (history [C][3] in, updated to the chunk's last three inputs), then
 /// the L2 norm of the q and k heads of every token.  h: [T, C].
 void gdn_conv(float* history, const float* qkv, const float* conv_w, float* h, int64_t T, float eps, void* stream);
-/// The recurrence over the chunk, block per value head, state in registers; y[t] = rmsnorm(o) * gamma * sigmoid(z)
-/// (FP32 and FP16 bits: the out projection is quantized).
+/// The recurrence over the chunk, state in registers; y16[t] = rmsnorm(o) * gamma * sigmoid(z) in FP16 (what the
+/// out projection reads); y is FP32 scratch.
 void gdn_recurrence(float* state, const float* h, const float* gate, const float* beta, const float* z,
                     const float* gamma, float eps, float* y, uint16_t* y16, int64_t T, void* stream);
 

include/strata/prefill/moe_mmq.hpp

@@ -64,6 +64,16 @@ private:
 /// slot: gate rows then up rows at `gu_dst`, down at `d_dst`.
 void gather_native(const void* gate, const void* up, size_t gu_half_bytes, const void* down, size_t d_bytes,
                    void* gu_dst, void* d_dst, void* stream);
+/// gather_native for an MMQ group's experts [first, n) in ONE launch: expert q's blob (`blob[q]`; gate at +0, up at
+/// +up_off, down at +down_off) to gu_dst + q * gu_stride and d_dst + q * d_stride - the same bytes as one gather_native
+/// each.  Every pointer, offset and size 16-byte aligned (false otherwise: nothing launched, gather one at a time).
+constexpr int kGatherGroupMax = 16;
+struct GatherGroup {
+    const uint8_t* blob[kGatherGroupMax] = {};
+    int first = 0, n = 0;
+};
+bool gather_native_group(const GatherGroup& g, size_t up_off, size_t gu_half_bytes, size_t down_off, size_t d_bytes,
+                         void* gu_dst, size_t gu_stride, void* d_dst, size_t d_stride, void* stream);
 /// A Strata-pack Q2_0 expert blob (codes and fp16 scales in separate planes, gate/up rows interleaved) into GGUF
 /// Q2_0 blocks: gate/up [1280, 2560] at `gu_dst` (rows stay interleaved), down [2560, 640] at `d_dst`.  Same values.
 void gather_strata_q2(const uint8_t* blob, void* gu_dst, void* d_dst, void* stream);

include/strata/prefill/prefill.hpp

@@ -21,6 +21,8 @@
 #include <memory>
 #include <string>
 
+namespace strata::core { class PeerExperts; }
+
 namespace strata::prefill {
 
 struct PrefillStats {
@@ -86,6 +88,11 @@ public:
 
     const PrefillStats& stats() const { return stats_; }
 
+    /// multi-GPU: the experts the peer GPU holds are computed THERE for every prompt chunk (up to `cap_rows` routed
+    /// rows per layer; the rest of the peer's experts are read by this GPU over P2P).  Allocates the peer's buffers for
+    /// chunks of up to init's `chunk` tokens.  Needs P2P between the two cards.
+    bool set_peer(core::PeerExperts* peer, int64_t cap_rows, std::string& err);
+
     /// Plan v0.3 P6: called after every chunk with the chunk's final multi-stream residual rows (device,
     /// T x hc*n_embd, valid until the next chunk) and the chunk's first position; the MTP draft layer builds its
     /// K/V from them.  The prefill stream is synchronized before the call.

serve/server.py

@@ -61,6 +61,7 @@ IMAGE_PAD = "<|image_pad|>"
 VISION_START = "<|vision_start|>"
 # #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)
 REASONING_WRAP_UP = "\n\nI have thought about this long enough; time to give my answer.\n</think>\n\n"
+LOOPBACK_NAMES = ("localhost", "127.0.0.1", "::1")
 CTX_SLACK = 8               # `strata --serve` rejects prompt + max_new + 8 > context: keep the same margin here
 # The live tok/s is a rate over a window, not a mean since the first token: a mean reads ~1/elapsed at the first
 # token (the Monitor showed five-digit numbers) and then undershoots for the first second of every answer.
@@ -960,6 +961,10 @@ class Service:
         # #321: origins that count as Strata's own page for /settings and MCP tools, e.g. the web app reached through a
         # reverse proxy or tunnel whose Host differs ("https://strata.example.com"); never a wildcard
         self.trusted_origins: list[str] = []
+        # DNS rebinding: extra Host names this server answers to (the config's allowed_hosts, $STRATA_ALLOWED_HOSTS;
+        # "*" = any), and every name it answers to, which serve() works out from the address it listens on
+        self.allowed_hosts: list[str] = []
+        self.host_names: set[str] = set(LOOPBACK_NAMES)
         self.status = {"busy": False, "queued": 0}      # GET /status: what the model is doing right now
         self.rate = collections.deque(maxlen=32)        # (time, generated) samples for the live tok/s window
         # #332: the API request monitor (/api-monitor) keeps the last 100 requests' prompts and answers in memory,
@@ -1380,7 +1385,9 @@ class Service:
         elif max_new > room:
             if not self.fit_max_tokens:
                 raise ValueError(f"prompt ({len(ids)} tokens) + max tokens ({max_new}) exceeds the context "
-                                 f"({self.engine.max_context}); requests are never truncated")
+                                 f"({self.engine.max_context}); requests are never truncated. Send a smaller "
+                                 f"max_tokens (at most {max(0, room)} here), or add \"fit_max_tokens\": true to the "
+                                 "model's strata-<model>.json to shorten it to the room left (#545)")
             max_new = max(1, room)          # --fit-max-tokens: a shorter completion beats a 400
         return ids, kwargs.get("enable_thinking", True) is not False, max_new
 
@@ -1585,6 +1592,10 @@ class Service:
                             hit_msg = f", expert cache {hit_rate*100:.1f}% hit" if hit_rate is not None else ""
                             print(f"[strata] done: {n} tokens in {el:.0f} s ({rate:.1f} tok/s) "
                                   f"({finish}, cancel={cancel.is_set()}){hit_msg}", flush=True)
+                            if finish == "length" and parser.state == "reasoning":   # #530
+                                print("[strata] the reply reached max tokens while still thinking, so it has no "
+                                      "answer: a thinking budget (reasoning_budget_tokens, in the request or in "
+                                      "strata-<model>.json for every request) leaves room to answer", flush=True)
                             if os.environ.get("STRATA_DEBUG") and raw_ids:
                                 print(f"[strata] raw: {self.tok.decode(raw_ids)!r}", flush=True)
                         self.status["busy"] = False
@@ -1967,6 +1978,46 @@ def make_handler(svc: Service):
         def log_message(self, fmt, *args):
             pass
 
+        def parse_request(self):
+            """Without an API key, every request (any method) first passes the Host check: DNS rebinding protection
+            (host_allowed).  With a key a rebinding page cannot authenticate, so the check is skipped: tunnels and
+            proxies that pass their own name on keep working."""
+            if not super().parse_request():
+                return False
+            host = self.headers.get("Host")
+            if svc.api_key or host_allowed(host, svc.host_names, "*" in svc.allowed_hosts):
+                return True
+            print(f"[strata] refused a request for Host {host!r} from {self.client_address[0]}: not a name this server "
+                  f"answers to (add it to \"allowed_hosts\" in the config or STRATA_ALLOWED_HOSTS, or set an API key)",
+                  flush=True)
+            self._json(403, {"error": {"type": "forbidden", "message":
+                             f"Host {host!r} is not allowed (DNS rebinding protection). Reaching Strata under this "
+                             f"name on purpose? Add it to \"allowed_hosts\" in the config (strata-<model>.json) or to "
+                             f"the STRATA_ALLOWED_HOSTS environment variable, or set an API key (\"api_key\"), which "
+                             f"turns this check off"}})
+            return False
+
+        def _foreign_page(self) -> bool:
+            """Without an API key, a /v1 POST from a browser page of another site (any site can POST text/plain
+            there without a CORS preflight) would burn GPU time: an Origin header must name an allowed page, and
+            then the body must be JSON.  No Origin (curl, the SDKs, other servers): any content type, as before."""
+            origin = self.headers.get("Origin")
+            if svc.api_key or not origin:
+                return False
+            if not origin_allowed(origin, self.headers.get("Host"), svc.host_names,
+                                  [*svc.trusted_origins, *svc.cors_origins]):
+                print(f"[strata] refused an API request from the web page {origin!r} (no API key; add its host to "
+                      f"\"allowed_hosts\" or its origin to \"cors_origins\" in the config)", flush=True)
+                self._json(403, {"error": {"type": "forbidden", "message":
+                                 f"web pages of {origin} may not use this server without an API key; set \"api_key\", "
+                                 f"or add the page's host to \"allowed_hosts\" (or its origin to \"cors_origins\") "
+                                 f"in the config"}})
+                return True
+            if not self.headers.get("Content-Type", "").startswith("application/json"):
+                self._json(415, {"error": {"message": "send application/json"}})
+                return True
+            return False
+
         def _watch_client(self, cancel: threading.Event) -> None:
             """#430 #431: cancel the request as soon as its client hangs up.  A non-streamed request writes nothing
             until it ends, and a streamed one only a keep-alive per prompt chunk (and the first write after a hang-up
@@ -2170,9 +2221,14 @@ def make_handler(svc: Service):
             if not self._authorized():
                 return
             path = self.path.split("?")[0].rstrip("/")   # issue #55: Claude Code posts /v1/messages?beta=true
+            if path.startswith("/v1/") and self._foreign_page():
+                return
             if path == "/settings":
                 self._settings()
                 return
+            # JSON from Strata's own page only, as /settings: else a plain form POST from any site unloads the model
+            if path in ("/unload", "/load") and not self._own_page("the model can be loaded or unloaded"):
+                return
             if path == "/unload":                            # give the GPU back now (between requests)
                 try:
                     r = svc.unload()
@@ -2488,6 +2544,36 @@ def warn_tight_ram(arena_mib) -> None:
               + "Close other programs, or run START-HERE --setup and pick a smaller size (Q2_0 / IQ2_XS).", flush=True)
 
 
+DESKTOP_FREE_MIB = 2048          # #560 #516: below this, an AMD card that also drives a Linux desktop can run out
+DESKTOP_RESERVE_MIB = 3072       # what kept KDE/Wayland alive beside a full expert cache in both reports
+
+
+def linux_desktop(env=None) -> bool:
+    """A graphical session on Linux (Wayland or X): its compositor, browser and apps take VRAM after the model has."""
+    env = os.environ if env is None else env
+    return os.name != "nt" and sys.platform != "darwin" and bool(env.get("WAYLAND_DISPLAY") or env.get("DISPLAY"))
+
+
+def desktop_vram_note(backend, vram_free_mib, args: list, desktop: bool) -> str:
+    """#560 #516: on Linux, when the desktop needs VRAM the AMD card does not have, amdgpu moves GPU memory (the expert
+    cache, ~24 GB) to system RAM, which the experts already fill - the OOM killer then ends the compositor.  The
+    default reserve (700 MiB) is sized for a card without a desktop.  A recommendation, nothing changes: "" when it
+    does not apply."""
+    if backend != "hip" or not desktop or not isinstance(vram_free_mib, int) or vram_free_mib >= DESKTOP_FREE_MIB:
+        return ""
+    try:
+        reserve = int(args[args.index("--vram-reserve-mib") + 1]) if "--vram-reserve-mib" in args else 700
+    except (ValueError, IndexError):
+        reserve = 700
+    if reserve >= DESKTOP_RESERVE_MIB:
+        return ""
+    return (f"[strata] note: {vram_free_mib} MiB of VRAM free with the model loaded. If this AMD card also drives your "
+            "desktop and the desktop or apps crash after the start (the driver moves the expert cache to RAM and "
+            "the OOM killer ends the session), keep more VRAM free: ./setup.sh --vram-reserve-mib "
+            f"{DESKTOP_RESERVE_MIB} (remembered; the expert cache gets "
+            f"{(DESKTOP_RESERVE_MIB - reserve) / 1024:.1f} GB less, a few % of speed)")
+
+
 def lan_addresses() -> list[str]:
     """This PC's IPv4 addresses on its networks (what another device types in), without loopback/link-local."""
     import socket
@@ -2507,7 +2593,114 @@ def lan_addresses() -> list[str]:
     return ([first] if ok(first) else []) + sorted(ip for ip in ips if ok(ip) and ip != first)
 
 
+def host_name(value) -> str:
+    """The name in a Host header (or an origin's host[:port]): "Example.com:8080" -> "example.com",
+    "[::1]:8095" -> "::1"; "" when it is malformed."""
+    v = (value or "").strip().lower()
+    if v.startswith("["):
+        name, sep, rest = v[1:].partition("]")
+        return name if sep and (not rest or (rest[:1] == ":" and rest[1:].isdigit())) else ""
+    if v.count(":") == 1:
+        v, port = v.split(":")
+        if not port.isdigit():
+            return ""
+    elif ":" in v:                                       # a bare IPv6 address (a config entry)
+        return v
+    v = v.rstrip(".")
+    return v if v and all(c.isalnum() or c in "-._" for c in v) else ""
+
+
+def _is_ip(name: str) -> bool:
+    import ipaddress
+    try:
+        ipaddress.ip_address(name)
+        return True
+    except ValueError:
+        return False
+
+
+def _name_in(name: str, names) -> bool:
+    """`name` is one of `names`, or below an entry that starts with a dot (".example.com")."""
+    return name in names or any(n.startswith(".") and (name.endswith(n) or name == n[1:]) for n in names)
+
+
+def allowed_hosts_of(value, env: str = "") -> list[str]:
+    """The config's allowed_hosts (a name or a list) plus $STRATA_ALLOWED_HOSTS (comma-separated): host names, "*" or
+    ".example.com" (it and every name below it).  A scheme, port or path is dropped ("https://a.example.com:8443/"
+    -> "a.example.com"); a wrong entry stops the start (ValueError)."""
+    items = [] if value in (None, "") else [value] if isinstance(value, str) else value
+    if not isinstance(items, list) or not all(isinstance(x, str) for x in items):
+        raise ValueError("allowed_hosts: expected a host name or a list of them")
+    out = []
+    for raw in items + [x for x in env.split(",") if x.strip()]:
+        x = raw.strip().lower()
+        if x == "*":
+            out.append(x)
+            continue
+        x = x.split("://", 1)[-1].split("/", 1)[0]
+        dot = x.startswith(".")
+        name = host_name(x[1:] if dot else x)
+        if not name:
+            raise ValueError(f"allowed_hosts: {raw!r} is not a host name like strata.example.com")
+        out.append("." + name if dot else name)
+    return list(dict.fromkeys(out))
+
+
+def host_names_for(bind_host: str, allowed_hosts=(), trusted_origins=()) -> set[str]:
+    """Every name this server answers to besides an IP address: localhost, the address it listens on, the config's
+    allowed_hosts and trusted_origins' hosts, and - listening beyond this PC (0.0.0.0 or a LAN address) - this PC's
+    name and LAN addresses (the LAN addresses matter for the Origin check, which takes no IP on trust)."""
+    names = set(LOOPBACK_NAMES)
+    bind = host_name(bind_host)
+    if bind and bind not in ("0.0.0.0", "::"):
+        names.add(bind)
+    if bind not in LOOPBACK_NAMES:
+        try:
+            pc = socket.gethostname().lower()
+            names.update((pc, pc + ".local"))
+        except OSError:
+            pass
+        names.update(("host.docker.internal", *lan_addresses()))
+    names.update(x for x in allowed_hosts if x != "*")
+    names.update(filter(None, (host_name(o.split("://", 1)[-1]) for o in trusted_origins)))
+    return names
+
+
+def host_allowed(host, names, any_host=False) -> bool:
+    """DNS rebinding: a web page of another site whose name its DNS points at 127.0.0.1 reaches this server as the
+    same origin, so the browser lets it read every answer.  Its requests carry that site's name in Host, so only
+    the names this server answers to pass.  An IP address passes (a page served from an IP is that IP's own page;
+    rebinding needs a name), as does "*.localhost" (browsers never ask DNS for it) and a request without a Host
+    header (HTTP/1.0 clients; browsers always send one)."""
+    if any_host or not (host or "").strip():
+        return True
+    name = host_name(host)
+    return bool(name) and (_is_ip(name) or name == "localhost" or name.endswith(".localhost")
+                           or _name_in(name, names))
+
+
+def origin_allowed(origin: str, host, names, origins=()) -> bool:
+    """A browser page's Origin that may use the model without an API key: this server's own page (the Origin is the
+    request's own Host), a page on one of the names this server answers to (any port), or an origin the config lists
+    (trusted_origins, cors_origins).  Unlike the Host check no IP passes on trust: a page served from any other IP is
+    another site.  "null" (a sandboxed frame, a file:// page) does not pass: any site can send it.  An origin of another
+    scheme (chrome-extension://, moz-extension://, an Electron app's app://) does: no web site can send one."""
+    origin = (origin or "").strip().rstrip("/")
+    if origin in origins or "*" in origins:             # cors_origins ["*"]: the config lets every page in
+        return True
+    scheme, sep, rest = origin.lower().partition("://")
+    if not sep or not scheme:
+        return False
+    if scheme not in ("http", "https"):
+        return True
+    if host and rest == host.strip().lower():
+        return True
+    name = host_name(rest)
+    return bool(name) and (name in LOOPBACK_NAMES or name.endswith(".localhost") or _name_in(name, names))
+
+
 def serve(svc: Service, host="127.0.0.1", port=8095) -> ThreadingHTTPServer:
+    svc.host_names = host_names_for(host, svc.allowed_hosts, svc.trusted_origins)
     svc.start_telemetry()
     httpd = Server((host, port), make_handler(svc))
     threading.Thread(target=httpd.serve_forever, daemon=True).start()
@@ -2732,6 +2925,10 @@ def main() -> int:
         engine = StrataEngine(exe, engine_args(cfg), cwd=cfg.get("cwd"), log=cfg.get("log"), env=env, lazy=lazy)
         engine.silence_s = silence                      # an attribute of its own: restart() keeps it
         warn_tight_ram(engine.info.get("arena_mib"))
+        note = desktop_vram_note(cfg.get("backend"), engine.info.get("vram_free_mib"), engine.spawn[1],
+                                 linux_desktop())
+        if note:                                        # #560 #516: before --open starts a browser on that card
+            print(note, flush=True)
     else:
         engine, vision, sampling_defaults = MockEngine(tok, a.script or [
             "Thinking about it.</think>\n\nHello from the mock engine."]), None, {}
@@ -2756,6 +2953,13 @@ def main() -> int:
     svc.api_key = a.api_key or cfg.get("api_key", "")
     svc.cors_origins = origins_of(cfg.get("cors_origins"), "cors_origins", wildcard=True)
     svc.trusted_origins = origins_of(cfg.get("trusted_origins"), "trusted_origins", wildcard=False)
+    try:
+        svc.allowed_hosts = allowed_hosts_of(cfg.get("allowed_hosts"), os.environ.get("STRATA_ALLOWED_HOSTS", ""))
+    except ValueError as e:
+        raise SystemExit(f"[strata] config {e}")
+    if svc.allowed_hosts:
+        print("[strata] Host check off: any name reaches this server (allowed_hosts \"*\")" if "*" in svc.allowed_hosts
+              else f"[strata] also answers to the host names {', '.join(svc.allowed_hosts)} (allowed_hosts)", flush=True)
     svc.api_monitor = a.api_monitor or cfg.get("api_monitor") is True
     if svc.api_monitor:
         print("[strata] API request monitor on (/api-monitor): the last 100 requests' prompts and answers are kept in "

serve/test_security.py

@@ -0,0 +1,255 @@
+"""serve/test_security.py - the Host check (DNS rebinding) and the Origin check for requests without an API key,
+against the mock engine (no GPU, no pack).
+
+    python -m unittest serve.test_security -v
+"""
+from __future__ import annotations
+
+import contextlib
+import http.client
+import io
+import json
+import socket
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from serve.frontend import ChatTemplate  # noqa: E402
+from serve.server import (ByteTokenizer, MockEngine, Service, allowed_hosts_of, host_allowed,  # noqa: E402
+                          host_name, host_names_for, origin_allowed, serve)
+
+ROOT = Path(__file__).resolve().parents[1]
+LOCAL = {"localhost", "127.0.0.1", "::1"}
+
+
+class HostNames(unittest.TestCase):
+    def test_host_name(self):
+        self.assertEqual(host_name("Example.COM:8080"), "example.com")
+        self.assertEqual(host_name("localhost"), "localhost")
+        self.assertEqual(host_name("[::1]:8095"), "::1")
+        self.assertEqual(host_name("[::1]"), "::1")
+        self.assertEqual(host_name("strata.example.com."), "strata.example.com")
+        for bad in ("", "a:b", "[::1", "[::1]x", "a b", "evil.com/x"):
+            self.assertEqual(host_name(bad), "", bad)
+
+    def test_allowed_hosts_of(self):
+        self.assertEqual(allowed_hosts_of(None), [])
+        self.assertEqual(allowed_hosts_of("Strata.Example.com"), ["strata.example.com"])
+        self.assertEqual(allowed_hosts_of(["https://a.example.com:8443/x", ".example.org", "*"], "box, nas.lan "),
+                         ["a.example.com", ".example.org", "*", "box", "nas.lan"])
+        for bad in (["bad name"], 5, [3], "http://"):
+            with self.assertRaises(ValueError):
+                allowed_hosts_of(bad)
+
+    def test_names_from_the_bind_address(self):
+        with mock.patch("serve.server.lan_addresses", return_value=["192.168.1.20"]):
+            self.assertEqual(host_names_for("127.0.0.1"), LOCAL)                 # this PC only: nothing else
+            self.assertEqual(host_names_for("localhost"), LOCAL)
+            names = host_names_for("0.0.0.0")                                    # other devices too
+            self.assertIn(socket.gethostname().lower(), names)
+            self.assertIn(socket.gethostname().lower() + ".local", names)
+            self.assertIn("192.168.1.20", names)
+            self.assertIn("10.0.0.7", host_names_for("10.0.0.7"))
+            names = host_names_for("127.0.0.1", ["strata.example.com", "*"], ["https://chat.example.net"])
+            self.assertTrue({"strata.example.com", "chat.example.net"} <= names)
+            self.assertNotIn("*", names)
+
+
+class HostCheck(unittest.TestCase):
+    def test_accepted(self):
+        for host in ("localhost:8095", "LOCALHOST", "127.0.0.1:1", "[::1]:8095", "192.168.1.20:8095", "[fe80::1]",
+                     "app.localhost:8095", None, "", "box:8095", "a.example.org", "example.org:443"):
+            self.assertTrue(host_allowed(host, LOCAL | {"box", ".example.org"}), host)
+
+    def test_refused(self):
+        # a rebinding page sends its own name; lookalikes of allowed names stay out
+        for host in ("evil.example.com", "evil.example.com:8095", "localhost.evil.com", "127.0.0.1.nip.io",
+                     "box.evil.com", "xexample.org", "bad host", "a:b:c:d:e:f:g:h:i"):
+            self.assertFalse(host_allowed(host, LOCAL | {"box", ".example.org"}), host)
+
+    def test_any_host(self):
+        self.assertTrue(host_allowed("evil.example.com", LOCAL, any_host=True))
+
+
+class OriginCheck(unittest.TestCase):
+    def test_accepted(self):
+        names = LOCAL | {"box"}
+        self.assertTrue(origin_allowed("http://127.0.0.1:8095", "127.0.0.1:8095", names))      # Strata's own page
+        self.assertTrue(origin_allowed("http://192.168.1.20:8095", "192.168.1.20:8095", names))  # ... by LAN IP
+        self.assertTrue(origin_allowed("http://localhost:3000", "127.0.0.1:8095", names))      # a local app
+        self.assertTrue(origin_allowed("http://[::1]:3000/", "127.0.0.1:8095", names))
+        self.assertTrue(origin_allowed("https://box", "127.0.0.1:8095", names))
+        self.assertTrue(origin_allowed("https://chat.example.com", "x", names, ["https://chat.example.com"]))
+        self.assertTrue(origin_allowed("https://any.example.com", "x", names, ["*"]))           # cors_origins ["*"]
+        # browser extensions and desktop apps: no web site can send another scheme than http(s) (or "null")
+        for origin in ("chrome-extension://abcdef", "moz-extension://1234-5678", "app://."):
+            self.assertTrue(origin_allowed(origin, "127.0.0.1:8095", names), origin)
+
+    def test_refused(self):
+        names = LOCAL | {"box"}
+        for origin in ("http://evil.example.com", "http://1.2.3.4", "http://192.168.1.99:8095", "null", "",
+                       "http://localhost.evil.com", "https://box.evil.com", "://x"):
+            self.assertFalse(origin_allowed(origin, "127.0.0.1:8095", names, ["https://chat.example.com"]), origin)
+
+
+class OverHttp(unittest.TestCase):
+    """The checks in the running server, as a browser and as curl/the SDKs send their requests."""
+
+    def setUp(self):
+        tok = ByteTokenizer()
+        self.svc = Service(MockEngine(tok, "</think>\n\nok", max_context=4096), tok,
+                           ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        self.httpd = None
+
+    def start(self, **attrs):
+        for k, v in attrs.items():
+            setattr(self.svc, k, v)
+        self.httpd = serve(self.svc, port=0)
+        self.port = self.httpd.server_address[1]
+
+    def tearDown(self):
+        if self.httpd:
+            self.httpd.shutdown()
+            self.httpd.server_close()
+
+    def req(self, method, path, body=None, headers=None, host=None):
+        """(status, JSON, what the server printed); `host` replaces the Host header http.client sends."""
+        c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
+        h = dict(headers or {})
+        if host is not None:
+            h["Host"] = host
+        data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode()
+        out = io.StringIO()
+        try:
+            with contextlib.redirect_stdout(out):
+                c.request(method, path, body=data, headers=h)
+                r = c.getresponse()
+                raw = r.read()
+            return r.status, json.loads(raw) if raw[:1] in (b"{", b"[") else raw, out.getvalue()
+        finally:
+            c.close()
+
+    def chat_body(self):
+        return {"model": "m", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 64}
+
+    # --- Host
+    def test_rebinding_name_is_refused_everywhere(self):
+        self.start()
+        for method, path in (("GET", "/status"), ("GET", "/metrics"), ("GET", "/"), ("GET", "/health"),
+                             ("POST", "/settings"), ("POST", "/v1/chat/completions"), ("OPTIONS", "/v1/models")):
+            code, body, log = self.req(method, path, {} if method == "POST" else None,
+                                       {"Content-Type": "application/json"}, host=f"evil.example.com:{self.port}")
+            self.assertEqual(code, 403, path)
+            if method != "OPTIONS":
+                self.assertIn("allowed_hosts", body["error"]["message"])
+            self.assertEqual(log.count("\n"), 1, path)                          # one log line each
+            self.assertIn("evil.example.com", log)
+
+    def test_local_names_and_ips_pass(self):
+        self.start()
+        for host in (f"127.0.0.1:{self.port}", f"localhost:{self.port}", f"[::1]:{self.port}",
+                     f"192.168.1.20:{self.port}"):
+            self.assertEqual(self.req("GET", "/status", host=host)[0], 200, host)
+
+    def test_no_host_header_passes(self):
+        self.start()
+        with socket.create_connection(("127.0.0.1", self.port), timeout=10) as s:      # an HTTP/1.0 client
+            s.sendall(b"GET /health HTTP/1.0\r\n\r\n")
+            raw = b""
+            while chunk := s.recv(65536):
+                raw += chunk
+        self.assertTrue(raw.startswith(b"HTTP/1.0 200"), raw[:40])
+
+    def test_allowed_hosts_and_wildcard(self):
+        self.start(allowed_hosts=["strata.example.com", ".home.arpa"])
+        self.assertEqual(self.req("GET", "/status", host="strata.example.com")[0], 200)
+        self.assertEqual(self.req("GET", "/status", host="nas.home.arpa:8095")[0], 200)
+        self.assertEqual(self.req("GET", "/status", host="evil.example.com")[0], 403)
+        self.svc.allowed_hosts = ["*"]
+        self.assertEqual(self.req("GET", "/status", host="evil.example.com")[0], 200)
+
+    def test_refusal_names_both_ways_out(self):
+        self.start()
+        message = self.req("GET", "/status", host="evil.example.com")[1]["error"]["message"]
+        self.assertIn("allowed_hosts", message)
+        self.assertIn("api_key", message)
+
+    def test_with_a_key_the_host_check_is_off(self):
+        # a rebinding page cannot authenticate; tunnels and proxies that pass their own name on keep working
+        self.start(api_key="s3cret")
+        tunnel = "random-words.trycloudflare.com"
+        self.assertEqual(self.req("GET", "/status", host=tunnel)[0], 401)
+        self.assertEqual(self.req("GET", "/status", headers={"Authorization": "Bearer s3cret"}, host=tunnel)[0], 200)
+        self.assertEqual(self.req("GET", "/health", host=tunnel)[0], 200)
+        code, _, log = self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                {"Content-Type": "application/json", "Authorization": "Bearer s3cret"}, host=tunnel)
+        self.assertEqual(code, 200)
+        self.assertNotIn("refused", log)
+
+    def test_a_trusted_origin_s_host_is_allowed(self):
+        self.start(trusted_origins=["https://strata.example.com"])
+        self.assertEqual(self.req("GET", "/status", host="strata.example.com")[0], 200)
+
+    # --- Origin on /v1 without an API key
+    def test_curl_and_sdks_without_origin(self):
+        self.start()
+        for ctype in ("application/json", "text/plain", None):              # curl -d sends a form type
+            code, body, _ = self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                     {"Content-Type": ctype} if ctype else {})
+            self.assertEqual(code, 200, ctype)
+            self.assertEqual(body["choices"][0]["message"]["content"], "ok")
+
+    def test_cross_site_page_is_refused(self):
+        self.start()
+        for path in ("/v1/chat/completions", "/v1/messages", "/v1/messages/count_tokens"):
+            for ctype in ("text/plain", "application/json"):
+                code, body, log = self.req("POST", path, self.chat_body(),
+                                           {"Content-Type": ctype, "Origin": "http://evil.example.com"})
+                self.assertEqual(code, 403, (path, ctype))
+                self.assertIn("api_key", body["error"]["message"])
+                self.assertIn("evil.example.com", log)
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                  {"Content-Type": "text/plain", "Origin": "null"})[0], 403)
+
+    def test_own_and_local_pages_pass(self):
+        self.start()
+        json_type = {"Content-Type": "application/json"}
+        for origin in (f"http://127.0.0.1:{self.port}", "http://localhost:3000", "chrome-extension://abcdef"):
+            self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                      {**json_type, "Origin": origin})[0], 200, origin)
+        # a browser page sends JSON; text/plain from a page is the cross-site "simple request" shape
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                  {"Content-Type": "text/plain", "Origin": f"http://127.0.0.1:{self.port}"})[0], 415)
+
+    def test_configured_origins_pass(self):
+        self.start(cors_origins=["https://chat.example.com"], trusted_origins=["https://strata.example.com"],
+                   allowed_hosts=["webui.lan"])
+        for origin in ("https://chat.example.com", "https://strata.example.com", "http://webui.lan:3000"):
+            self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                      {"Content-Type": "application/json", "Origin": origin})[0], 200, origin)
+
+    def test_with_a_key_the_key_decides(self):
+        self.start(api_key="s3cret")
+        headers = {"Content-Type": "text/plain", "Origin": "http://evil.example.com"}
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(), headers)[0], 401)
+        self.assertEqual(self.req("POST", "/v1/chat/completions", self.chat_body(),
+                                  {**headers, "Authorization": "Bearer s3cret"})[0], 200)
+
+    # --- /unload and /load
+    def test_unload_and_load_need_json_from_the_own_page(self):
+        self.start()
+        form = {"Content-Type": "application/x-www-form-urlencoded"}
+        for path in ("/unload", "/load"):
+            self.assertEqual(self.req("POST", path, b"a=1", form)[0], 415, path)
+            self.assertEqual(self.req("POST", path, b"", {})[0], 415, path)
+            self.assertEqual(self.req("POST", path, {}, {"Content-Type": "application/json",
+                                                         "Origin": "http://evil.example.com"})[0], 403, path)
+        self.assertEqual(self.req("POST", "/unload", {}, {"Content-Type": "application/json"})[0], 200)
+        self.assertEqual(self.req("POST", "/load", {}, {"Content-Type": "application/json",
+                                                        "Origin": f"http://127.0.0.1:{self.port}"})[0], 200)
+
+
+if __name__ == "__main__":
+    unittest.main()

serve/test_server.py

@@ -4,6 +4,8 @@
 """
 from __future__ import annotations
 
+import contextlib
+import io
 import json
 import os
 import sys
@@ -135,6 +137,8 @@ class MaxTokens(unittest.TestCase):
                 s, b, _, _ = self.call(api, max_tokens=CTX)
                 self.assertEqual(s, 400)
                 self.assertIn("exceeds the context", b["error"]["message"])
+                self.assertIn("\"fit_max_tokens\": true", b["error"]["message"])     # #545: says how to get past it
+                self.assertRegex(b["error"]["message"], r"at most \d+ here")
 
     def test_unset_budget_with_a_near_full_prompt(self):
         _, _, pt0, _ = self.call("openai", max_tokens=1)
@@ -676,7 +680,7 @@ class ClientHangUp(unittest.TestCase):
         body = json.dumps({"model": "x", "max_tokens": 20, "stream": stream,
                            "messages": [{"role": "user", "content": "a long prompt"}]}).encode()
         c = so.create_connection(("127.0.0.1", self.port))
-        c.sendall(b"POST /v1/chat/completions HTTP/1.1\r\nHost: x\r\nContent-Type: application/json\r\n"
+        c.sendall(b"POST /v1/chat/completions HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: application/json\r\n"
                   b"Content-Length: " + str(len(body)).encode() + b"\r\n\r\n" + body)
         time.sleep(1.0)
         c.close()
@@ -876,6 +880,31 @@ class DraftHeadHint(unittest.TestCase):
         self.assertEqual(start_failure_hint(str(Path(tempfile.mkdtemp()) / "missing.log"), 0), "")
 
 
+class DesktopVramNote(unittest.TestCase):
+    """#560 #516: an AMD card on a Linux desktop with little VRAM left after the start gets a recommended reserve."""
+
+    def test_when_it_applies(self):
+        from serve.server import desktop_vram_note
+        note = desktop_vram_note("hip", 624, ["--kv", "int8"], True)
+        self.assertIn("624 MiB of VRAM free", note)
+        self.assertIn("--vram-reserve-mib 3072", note)
+        self.assertIn("2.3 GB less", note)
+
+    def test_when_it_does_not(self):
+        from serve.server import desktop_vram_note
+        self.assertEqual(desktop_vram_note(None, 624, [], True), "")               # NVIDIA
+        self.assertEqual(desktop_vram_note("hip", 624, [], False), "")             # no desktop session
+        self.assertEqual(desktop_vram_note("hip", 2994, [], True), "")             # room left
+        self.assertEqual(desktop_vram_note("hip", None, [], True), "")             # lazy start: no INFO yet
+        self.assertEqual(desktop_vram_note("hip", 900, ["--vram-reserve-mib", "4000"], True), "")   # already raised
+
+    def test_desktop_detection(self):
+        from serve import server
+        with mock.patch.object(server.os, "name", "posix"), mock.patch.object(server.sys, "platform", "linux"):
+            self.assertTrue(server.linux_desktop({"WAYLAND_DISPLAY": "wayland-0"}))
+            self.assertFalse(server.linux_desktop({}))
+
+
 class StartFailureLog(unittest.TestCase):
     """#496: whatever stopped the engine before READY, the error carries this start's last log lines."""
 
@@ -1811,6 +1840,20 @@ class ThinkingBudget(unittest.TestCase):
         self.assertEqual(len(self.engine.prompts), 1)
         self.assertEqual(b["choices"][0]["message"]["reasoning_content"], ThinkingEngine.THOUGHT[:20])
 
+    def test_a_reply_cut_while_thinking_is_named_in_the_log(self):
+        """#530: max tokens reached inside the thinking gives an empty answer; the server log says what helps."""
+        hint = "reached max tokens while still thinking"
+        for extra, said in (({"max_tokens": 10}, True), ({}, False)):
+            with self.subTest(extra=extra):
+                out = io.StringIO()
+                with contextlib.redirect_stdout(out):
+                    code, b = self.openai(**extra)
+                self.assertEqual(code, 200, b)
+                self.assertEqual(b["choices"][0]["finish_reason"], "length" if said else "stop")
+                self.assertEqual(hint in out.getvalue(), said, out.getvalue())
+                if said:
+                    self.assertIn("reasoning_budget_tokens", out.getvalue())
+
     def test_a_bad_value_is_a_400(self):
         for bad in ("lots", 2.5, True, [1]):
             with self.subTest(value=bad):

setup.py

@@ -50,6 +50,7 @@ import shutil
 import struct
 import subprocess
 import sys
+import textwrap
 import time
 import urllib.error
 import urllib.request
@@ -103,7 +104,7 @@ PREBUILT_ASSET = "strata-windows-x64.zip" if WIN else "strata-linux-x64.zip"
 # the CUDA libraries the ready-made engine loads (the same CUDA 13.0 it is built with), from NVIDIA's pip packages
 CUDA_WHEELS = ["nvidia-cublas==13.0.2.14", "nvidia-cuda-runtime==13.0.96"]
 MIN_DRIVER = 580                       # CUDA 13.0
-MIN_ENGINE = (0, 1, 37)                # v0.1.37: a silent engine is restarted (#481), Windows AMD counts the desktop's VRAM (#380 #377 #497), a steadier PCIe probe (#485), fixes #496 #495 #498 #505 #493; v0.1.36: a cancelled prompt logged as read so far (#471), the draft-head hint (#474), UPDATE.bat (#475), --expert-profile-save (#477); v0.1.35: Windows AMD uses its bundled HIP runtime (#468 #461), the low-RAM resident mode on Windows 32 GB (#467), fixes #460 #459 #446 #447 #457 #448 #444; v0.1.34: AMD on Windows (a ready-made HIP engine), an MCP server for AI assistants (tools/strata_mcp.py), a shorter README; v0.1.33: a portable image encoder again (#411 #412), setup recommends instead of forcing (#406 #403 #364 #384), fixes #352 #365 #369 #371 #375 #393 #408 #414; v0.1.32: split prompts faster (#340), AMD router +12%, Unsloth Q4 in setup, faster Q4 prompts, #326/#327/#342/#344 fixes, PR batch; v0.1.31: Unsloth UD-Q4_K_XL (experimental), GGUF-in-place low-RAM mode, Windows GGUF load 2x, server race + tokenizer fixes, AMD intrinsics; v0.1.30: short prompts faster (streaming from 1024 tokens), resident low-RAM variant, multi-GPU session carve, RDNA4; v0.1.29: sampled answers faster (split top-k), #154 correctness fixes; v0.1.28: the expert cache reserves the draft head, a cancelled request no longer fails the next; v0.1.27: RTX 20 (sm_75) in the ready-made engine, the HIP build without CUDA headers; v0.1.26: the draft layer's prompt pass in batches; v0.1.25: faster prompts (grouping off the copy engine, fused hyper-connection kernels), AMD HIP backend, --kv k8v4; v0.1.24: long prompts faster (QSA select on tensor cores); v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
+MIN_ENGINE = (0, 1, 38)                # v0.1.38: prompts faster (one gather per expert group #372, the first chunk's PLE rows beside layer 0 #374, DeltaNet three heads per thread #413), --kv q4_0 prompts on tensor cores (#452), Q5_0 experts on the GPU (#473), IQ4_XS on AVX-2 (#415), unbuffered expert loading on Windows (#357 #362), --peer-device (#531), a 6 GB card starts (#496), PR batch; v0.1.37: a silent engine is restarted (#481), Windows AMD counts the desktop's VRAM (#380 #377 #497), a steadier PCIe probe (#485), fixes #496 #495 #498 #505 #493; v0.1.36: a cancelled prompt logged as read so far (#471), the draft-head hint (#474), UPDATE.bat (#475), --expert-profile-save (#477); v0.1.35: Windows AMD uses its bundled HIP runtime (#468 #461), the low-RAM resident mode on Windows 32 GB (#467), fixes #460 #459 #446 #447 #457 #448 #444; v0.1.34: AMD on Windows (a ready-made HIP engine), an MCP server for AI assistants (tools/strata_mcp.py), a shorter README; v0.1.33: a portable image encoder again (#411 #412), setup recommends instead of forcing (#406 #403 #364 #384), fixes #352 #365 #369 #371 #375 #393 #408 #414; v0.1.32: split prompts faster (#340), AMD router +12%, Unsloth Q4 in setup, faster Q4 prompts, #326/#327/#342/#344 fixes, PR batch; v0.1.31: Unsloth UD-Q4_K_XL (experimental), GGUF-in-place low-RAM mode, Windows GGUF load 2x, server race + tokenizer fixes, AMD intrinsics; v0.1.30: short prompts faster (streaming from 1024 tokens), resident low-RAM variant, multi-GPU session carve, RDNA4; v0.1.29: sampled answers faster (split top-k), #154 correctness fixes; v0.1.28: the expert cache reserves the draft head, a cancelled request no longer fails the next; v0.1.27: RTX 20 (sm_75) in the ready-made engine, the HIP build without CUDA headers; v0.1.26: the draft layer's prompt pass in batches; v0.1.25: faster prompts (grouping off the copy engine, fused hyper-connection kernels), AMD HIP backend, --kv k8v4; v0.1.24: long prompts faster (QSA select on tensor cores); v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
 PY_PACKAGES = ["numpy", "jinja2", "regex", "pyyaml", "tqdm", "requests", "cmake", "ninja", "pillow", "psutil"]
 REQUIREMENTS = ROOT / "requirements.txt"   # the same packages and their dependencies, pinned (#214)
 
@@ -2429,8 +2430,25 @@ def find_in(roots: list, rel: str):
 
 
 # ------------------------------------------------------------------------------------------------ start
+def model_config(path: Path) -> bool:
+    """#549: a model's run config (a JSON object with "exe" and "args"). Any other strata-*.json in the folder (a
+    file of the user's own, a cut-off one) is skipped with a warning naming it instead of stopping setup."""
+    try:
+        cfg = json.loads(path.read_text(encoding="utf-8-sig"))
+        if isinstance(cfg, dict) and cfg.get("exe") and isinstance(cfg.get("args"), list):
+            return True
+        why = 'no "exe" or "args"'
+    except OSError as e:
+        why = e.strerror or str(e)
+    except ValueError:
+        why = "not valid JSON"
+    warn(f"skipped {path.name} ({why}): it is not a Strata model config")
+    return False
+
+
 def installed_configs():
-    return sorted(ROOT.glob("strata-*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+    return [p for p in sorted(ROOT.glob("strata-*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+            if model_config(p)]
 
 
 def source_version() -> str:
@@ -2545,6 +2563,7 @@ def update_install(have: list, a) -> int:
     setup needs a newer one (MIN_ENGINE; a compiled engine when its source changed), each installed model's config
     upgrades and its draft subset.  No question is asked and the model files are not touched; a model still running
     keeps its engine (update_installed_engine says to close it and run this again)."""
+    have = [p for p in have if model_config(p)]        # #549: a strata-*.json that is no model config is skipped
     if not have:
         say("  No model is installed in this Strata folder yet: run START-HERE.bat (Linux: ./setup.sh) to set it up -")
         say("  it finds an earlier install's model files next to it and reuses them.")
@@ -2567,6 +2586,29 @@ def update_install(have: list, a) -> int:
     return 0
 
 
+def settings_summary(cfg: dict, port=None) -> str:
+    """#564: the settings a start uses, in one line: the config's engine options (the model's file paths left out)
+    and the server's own fields, so a change made by hand to strata-<model>.json can be checked without the log."""
+    a, out, i = [str(x) for x in cfg.get("args") or []], [], 0
+    while i < len(a):
+        flag = a[i]
+        val = a[i + 1] if i + 1 < len(a) and not a[i + 1].startswith("--") else None
+        i += 1 if val is None else 2
+        if not flag.startswith("--"):
+            continue                                   # a positional: the model file
+        if val is not None and ("/" in val or "\\" in val or val.lower().endswith((".gguf", ".bin"))):
+            continue                                   # a path: --native, --mtp, --profile ...
+        out.append(flag if val is None else f"{flag} {val}")
+    srv = [f"{cfg.get('host', '127.0.0.1')}:{port or cfg.get('port', 8080)}"]
+    if cfg.get("api_key"):
+        srv.append("api key set")
+    for k in ("gpu", "layer_split", "draft_vocab", "fit_max_tokens", "reasoning_budget_tokens", "anthropic_thinking"):
+        if cfg.get(k) is not None:
+            v = cfg[k]
+            srv.append(f"{k} {','.join(map(str, v)) if isinstance(v, list) else str(v).lower() if isinstance(v, bool) else v}")
+    return " ".join(out) + ("; " if out else "") + "server " + ", ".join(srv)
+
+
 def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_browser=True, yes=False,
           layer_split=None, keep=None) -> int:
     """keep: settings given on this start that the model keeps from now on (--host, --api-key, --draft-vocab,
@@ -2680,6 +2722,9 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
     say("  restart). That is normal: please wait and don't close this window - the browser opens when it is ready.")
     say("  Later, closing this window stops the model.")
     say("  " + "-" * 100)
+    for n, line in enumerate(textwrap.wrap(f"Settings ({cfg_path.name}): {settings_summary(cfg, port)}", 100,
+                                           break_on_hyphens=False)):   # #564: what this start uses
+        say(("  " if n == 0 else "    ") + line)
     if not WIN and os.environ.get("STRATA_EXECV"):
         # Replace this process instead of spawning a child. The Docker image sets STRATA_EXECV=1,
         # so there the server is PID 1 and docker stop's SIGTERM reaches the process that can
@@ -2724,6 +2769,47 @@ def draft_vocab_note(vram_gb: float, chosen: str | None) -> list[str]:
             "  fix when the start stops with \"the draft head does not fit\". The model keeps the choice."]
 
 
+SMALL_CARD_GB = 7.5            # #496: a card under 8 GB gets a tip (an 8 GB card lists 7.99)
+
+
+def small_card_note(ctx: int, draft_vocab: str | None) -> list[str]:
+    """#496: what frees VRAM on a card under 8 GB when the start stops with "no VRAM is left for the expert cache"
+    (the engine already lowers its own reserve on such a card) - a recommendation, setup changes none of it.  (The
+    draft layer stays: the server needs it.)"""
+    start = "START-HERE.bat --setup" if WIN else "./setup.sh"
+    tips = []
+    if ctx > 8192:
+        tips.append("an 8K context (a smaller KV cache)")
+    if draft_vocab != "en":
+        tips.append(f"--draft-vocab en (a draft head of ~{DRAFT_VOCAB_MIB['en']} MiB instead of "
+                    f"~{DRAFT_VOCAB_MIB[draft_vocab or 'cjk']})")
+    lines = ["If the start stops with \"no VRAM is left for the expert cache\" (the engine's log says how much is "
+             "short):"]
+    if tips:
+        lines.append(f"  run {start} again with " + " and ".join(tips) + ", or close other programs that use the GPU.")
+    else:
+        lines.append("  close other programs that use the GPU.")
+    return lines
+
+
+DESKTOP_RESERVE_MIB = 3072     # #560 #516: what kept a KDE/Wayland desktop alive beside a full expert cache
+
+
+def linux_desktop(env=None) -> bool:
+    """A graphical session on Linux (Wayland or X)."""
+    env = os.environ if env is None else env
+    return sys.platform.startswith("linux") and bool(env.get("WAYLAND_DISPLAY") or env.get("DISPLAY"))
+
+
+def desktop_reserve_note() -> list[str]:
+    """#560 #516: an AMD card that also drives a Linux desktop - with the default 700 MiB reserve the expert cache
+    fills it, and when the desktop needs more VRAM amdgpu moves the cache to system RAM, where the OOM killer then ends
+    the compositor.  A recommendation, setup changes nothing."""
+    return [f"If this AMD card also drives your desktop and the desktop or apps crash once the model is loaded, keep "
+            f"more VRAM free: ./setup.sh --vram-reserve-mib {DESKTOP_RESERVE_MIB}",
+            "  (remembered for this model; the expert cache gets ~2.3 GB less, a few % of speed)"]
+
+
 def mtp_corrupt(mtp: Path, env=None) -> bool:
     """#327: True when the MTP tensors an install fetched are not the pinned checkpoint's (tools/mtp_fetch.py verify,
     which hashes only files that changed since they last checked out).  A mirror that ignored range requests left the
@@ -3528,6 +3614,14 @@ def main() -> int:
             args += ["--vram-reserve-mib", str(a.vram_reserve_mib)]
         ok(f"VRAM kept free for other programs: {a.vram_reserve_mib} MiB (--vram-reserve-mib; the expert cache takes "
            "that much less)")
+    if not multi and 0 < gpu.get("vram_gb", 0.0) < SMALL_CARD_GB:
+        # #496: on a 6 GB card the expert cache can get no room at all; the engine lowers its own reserve when that
+        # is what it takes, and says what is short when even that is not enough.  Setup only says what helps.
+        for line in small_card_note(ctx, draft_vocab):   # a recommendation: nothing changes
+            say("  " + line)
+    elif hip and a.vram_reserve_mib is None and linux_desktop():
+        for line in desktop_reserve_note():              # #560 #516: a recommendation: nothing changes
+            say("  " + line)
     if esp is not None:
         # the package's profile, with llama.cpp's flags (the engine takes the same ones)
         args += ["--control-vector-scaled", f"{esp}:1.0", "--control-vector-layer-range", "4", "44",

src/core/expert_source.cpp

@@ -1,6 +1,7 @@
 // src/core/expert_source.cpp - the adapter.  See the header for the three clauses of the contract.
 #include "strata/core/expert_source.hpp"
 #include "strata/core/remote_experts.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 #include "strata/artifact/gguf_reader.hpp"
 
@@ -426,6 +427,7 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     file_ = f;
     mapping_ = m;
     base_ = (const uint8_t*) view;
+    paths_.assign(1, path);
 #else
     const int fd = ::open(path.c_str(), O_RDONLY);
     if (fd < 0) { err = "FileExpertSource: cannot open " + path; return false; }
@@ -506,6 +508,12 @@ void FileExpertSource::close() {
     }
     role_ptr_.clear();
     role_bytes_.clear();
+    role_file_.clear();
+    paths_.clear();
+#if defined(_WIN32)
+    for (void* h : direct_) CloseHandle((HANDLE) h);
+#endif
+    direct_.clear();
     {
         std::lock_guard<std::mutex> lk(stage_mu_);
         stage_buf_.clear();
@@ -571,6 +579,7 @@ bool FileExpertSource::open_gguf(std::string& err) {
     std::map<std::string, size_t> index;
     role_ptr_.assign((size_t) (3 * n_layers_), nullptr);
     role_bytes_.assign((size_t) (3 * n_layers_), 0);
+    role_file_.assign((size_t) (3 * n_layers_), 0);
     for (int64_t l = 0; l < n_layers_; ++l) {
         const auto& fm = lay.fmt[(size_t) l];
         const uint64_t per[3] = {fm.up_off, fm.up_off, lay.bytes[(size_t) l] - fm.down_off};
@@ -620,9 +629,11 @@ bool FileExpertSource::open_gguf(std::string& err) {
                 m.base = (const uint8_t*) view;
 #endif
                 maps_.push_back(m);
+                paths_.push_back(path);
                 it = index.emplace(path, maps_.size() - 1).first;
             }
             const Map& m = maps_[it->second];
+            role_file_[i] = (int) it->second;
             const uint64_t at = lay.gguf_off[i], bytes = per[r] * (uint64_t) n_expert_;
             if (at > m.bytes || bytes > m.bytes - at) {   // check_experts_gguf proved it; the mapping must agree
                 err = "FileExpertSource: an expert span runs past the end of " + path;
@@ -640,6 +651,10 @@ bool FileExpertSource::open_gguf(std::string& err) {
 
 bool FileExpertSource::copy_from_files(int64_t layer, int64_t expert, uint8_t* dst) const {
     if (dst == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
+    if (!direct_.empty()) {   // #286: from the drive; a failed read falls back to the mapping below
+        const Fill f{0, layer, expert, dst};
+        if (read_direct(&f, 1)) return true;
+    }
     if (!role_ptr_.empty()) {
         uint64_t at = 0;
         for (int r = 0; r < 3; ++r) {
@@ -710,7 +725,11 @@ bool FileExpertSource::claim_stage(int64_t key, size_t& v, bool& fill) {
 bool FileExpertSource::fill_stage(size_t v, int64_t layer, int64_t expert, uint8_t* dst) {
     const auto t0 = std::chrono::steady_clock::now();
     const bool ok = copy_from_files(layer, expert, dst);
-    const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+    publish_stage(v, layer, ok, std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count());
+    return ok;
+}
+
+void FileExpertSource::publish_stage(size_t v, int64_t layer, bool ok, double us) {
     file_us_.fetch_add((uint64_t) us, std::memory_order_relaxed);
     if (ok) {
         file_read_bytes_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);
@@ -725,7 +744,6 @@ bool FileExpertSource::fill_stage(size_t v, int64_t layer, int64_t expert, uint8
         }
     }
     stage_cv_.notify_all();
-    return ok;
 }
 
 const uint8_t* FileExpertSource::staged_blob(int64_t layer, int64_t expert) {
@@ -749,8 +767,7 @@ const uint8_t* FileExpertSource::staged_blob(int64_t layer, int64_t expert) {
 }
 
 void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n) {
-    if (role_ptr_.empty() || n <= 0 || layer < 0 || layer >= n_layers_) return;
-    struct Fill { size_t v; int64_t e; uint8_t* dst; };
+    if (!staged() || n <= 0 || layer < 0 || layer >= n_layers_) return;
     std::vector<Fill> todo;
     {
         std::lock_guard<std::mutex> lk(stage_mu_);
@@ -764,7 +781,7 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
             size_t v = 0;
             bool fill = false;
             if (!claim_stage(layer * n_expert_ + e, v, fill) && fill) {
-                todo.push_back({v, e, stage_buf_[v].get()});
+                todo.push_back({v, layer, e, stage_buf_[v].get()});
                 if (warm_stamp_) {
                     const uint32_t s = warm_stamp_[index].load(std::memory_order_relaxed);
                     if (s != 0 && (uint64_t) s + 3 >= epoch_ + 1) warm_hits_.fetch_add(1, std::memory_order_relaxed);
@@ -772,7 +789,52 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
             }
         }
     }
+    fill_many(todo);
+}
+
+void FileExpertSource::prefetch_pairs(const std::pair<int32_t, int32_t>* pairs, int64_t n) {
+    if (direct_.empty() || pairs == nullptr || n <= 0) return;   // unbuffered only: the mapped fill stays as it was
+    std::vector<Fill> todo;
+    {
+        std::lock_guard<std::mutex> lk(stage_mu_);
+        for (int64_t i = 0; i < std::min<int64_t>(n, 64); ++i) {
+            const int64_t l = pairs[i].first, e = pairs[i].second;
+            if (l < 0 || e < 0 || l >= n_layers_ || e >= n_expert_) continue;
+            size_t v = 0;
+            bool fill = false;
+            if (!claim_stage(l * n_expert_ + e, v, fill) && fill) todo.push_back({v, l, e, stage_buf_[v].get()});
+        }
+    }
+    fill_many(todo);
+}
+
+void FileExpertSource::fill_many(const std::vector<Fill>& todo) {
     if (todo.empty()) return;
+    if (!direct_.empty()) {
+        // #286: overlapped batches - every role window of 16 blobs in flight at once; a big batch (the profile
+        // fill) on up to 4 threads, a decode layer's few misses on this one
+        std::atomic<size_t> next{0};
+        auto work = [&] {
+            for (size_t at; (at = next.fetch_add(16)) < todo.size();) {
+                const size_t k = std::min<size_t>(16, todo.size() - at);
+                const auto t0 = std::chrono::steady_clock::now();
+                const bool ok = read_direct(todo.data() + at, k);
+                const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+                for (size_t i = at; i < at + k; ++i) {
+                    const Fill& f = todo[i];
+                    // a failed batch: each blob again on its own (copy_from_files falls back to the mapping)
+                    if (ok) publish_stage(f.v, f.layer, true, us / (double) k);
+                    else (void) fill_stage(f.v, f.layer, f.e, f.dst);
+                }
+            }
+        };
+        const size_t nt = std::min<size_t>(4, (todo.size() + 15) / 16);
+        std::vector<std::thread> th;
+        for (size_t t = 1; t < nt; ++t) th.emplace_back(work);
+        work();
+        for (auto& t : th) t.join();
+        return;
+    }
 #if defined(_WIN32)
     // One PrefetchVirtualMemory call for every slice about to be copied: the memory manager reads them in large
     // requests, all queued at once, where the copies' page faults would read a few clusters each.  The copies below
@@ -789,7 +851,7 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
             ranges.reserve(todo.size() * 3);
             for (const Fill& f : todo)
                 for (int r = 0; r < 3; ++r) {
-                    const size_t i = (size_t) (3 * layer + r);
+                    const size_t i = (size_t) (3 * f.layer + r);
                     ranges.push_back({(PVOID) (role_ptr_[i] + (size_t) ((uint64_t) f.e * role_bytes_[i])),
                                       (SIZE_T) role_bytes_[i]});
                 }
@@ -801,7 +863,7 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
     std::atomic<size_t> next{0};
     auto work = [&] {
         for (size_t i; (i = next.fetch_add(1)) < todo.size();)
-            (void) fill_stage(todo[i].v, layer, todo[i].e, todo[i].dst);
+            (void) fill_stage(todo[i].v, todo[i].layer, todo[i].e, todo[i].dst);
     };
     const size_t nt = std::min<size_t>(todo.size(), (size_t) fetch_threads_);
     std::vector<std::thread> th;
@@ -810,6 +872,159 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
     for (auto& t : th) t.join();
 }
 
+bool FileExpertSource::set_unbuffered(uint64_t ram_bytes, std::string& why) {
+#if defined(_WIN32)
+    if (base_ == nullptr || paths_.empty() || !direct_.empty()) {
+        why = !direct_.empty() ? "already unbuffered" : "no expert files open";
+        return !direct_.empty();
+    }
+    if (!experts_unbuffered(paths_, ram_bytes, why, /*cache_counts=*/false)) return false;
+    for (const std::string& path : paths_) {
+        const int wide = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, nullptr, 0);
+        std::vector<wchar_t> w((size_t) (wide > 0 ? wide : 1), L'\0');
+        if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, w.data(), wide);
+        HANDLE h = CreateFileW(w.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                               FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);
+        if (h == INVALID_HANDLE_VALUE) {
+            why += "; cannot open " + path + " unbuffered (error " + std::to_string((unsigned long long) GetLastError()) +
+                   "), read through the file cache";
+            for (void* d : direct_) CloseHandle((HANDLE) d);
+            direct_.clear();
+            return false;
+        }
+        direct_.push_back(h);
+    }
+    if (role_ptr_.empty()) {   // experts.bin: blob() now assembles into the stage buffers, sized for the largest blob
+        std::lock_guard<std::mutex> lk(stage_mu_);
+        for (uint64_t b : layer_blob_bytes_) stage_blob_ = std::max(stage_blob_, b);
+    }
+    return true;
+#else
+    (void) ram_bytes;
+    why = "through the file cache (not Windows)";
+    return false;
+#endif
+}
+
+bool FileExpertSource::read_direct(const Fill* fills, size_t n) const {
+#if defined(_WIN32)
+    // NTFS runs the unbuffered reads of a file one at a time while the file is mapped or cached anywhere (the engine
+    // maps the GGUF shards for the token embedding and the PLE table): measured on a PCIe 5 drive, 512 KiB reads at
+    // queue depth 48 make 10.4 GB/s on a file nobody maps and 3.4 GB/s on a mapped one, 8 MiB reads 8.6 GB/s.  So
+    // windows less than kGap apart (the same role of nearby experts) are merged into requests of up to kMerge bytes.
+    constexpr uint64_t kSector = 4096, kGap = 1ull << 20, kMerge = 32ull << 20;
+    struct Window { int file; uint64_t a0, size, skip, n, at; uint8_t* dst; size_t req; uint64_t in_req; };
+    struct Req { HANDLE h; uint64_t a0, size, pos; };
+    // this thread's aligned buffer and events, kept for its next batch
+    struct Scratch {
+        uint8_t* buf = nullptr;
+        size_t cap = 0;
+        std::vector<HANDLE> ev;
+        std::vector<OVERLAPPED> ov;
+        std::vector<Window> win;
+        std::vector<size_t> order;
+        std::vector<Req> req;
+        std::vector<DWORD> got;
+        ~Scratch() {
+            if (buf != nullptr) VirtualFree(buf, 0, MEM_RELEASE);
+            for (HANDLE e : ev) CloseHandle(e);
+        }
+    };
+    thread_local Scratch sc;
+    sc.win.clear();
+    for (size_t k = 0; k < n; ++k) {
+        const Fill& f = fills[k];
+        if (f.dst == nullptr || f.layer < 0 || f.e < 0 || f.layer >= n_layers_ || f.e >= n_expert_) return false;
+        if (role_ptr_.empty()) {   // experts.bin: the blob is one contiguous range
+            const uint64_t per = layer_blob_bytes_[(size_t) f.layer];
+            const uint64_t off = layer_offsets_[(size_t) f.layer] + (uint64_t) f.e * per;
+            const uint64_t a0 = off / kSector * kSector, a1 = (off + per + kSector - 1) / kSector * kSector;
+            sc.win.push_back({0, a0, a1 - a0, off - a0, per, 0, f.dst, 0, 0});
+            continue;
+        }
+        uint64_t at = 0;
+        for (int r = 0; r < 3; ++r) {
+            const size_t i = (size_t) (3 * f.layer + r);
+            const uint64_t per = role_bytes_[i];
+            const Map& m = maps_[(size_t) role_file_[i]];
+            const uint64_t off = (uint64_t) (role_ptr_[i] - m.base) + (uint64_t) f.e * per;
+            const uint64_t a0 = off / kSector * kSector, a1 = (off + per + kSector - 1) / kSector * kSector;
+            sc.win.push_back({role_file_[i], a0, a1 - a0, off - a0, per, at, f.dst, 0, 0});
+            at += per;
+        }
+    }
+    sc.order.resize(sc.win.size());
+    for (size_t w = 0; w < sc.win.size(); ++w) sc.order[w] = w;
+    std::sort(sc.order.begin(), sc.order.end(), [&](size_t a, size_t b) {
+        return sc.win[a].file != sc.win[b].file ? sc.win[a].file < sc.win[b].file : sc.win[a].a0 < sc.win[b].a0;
+    });
+    sc.req.clear();
+    uint64_t total = 0;
+    int last_file = -1;
+    for (size_t w : sc.order) {
+        Window& x = sc.win[w];
+        if (!sc.req.empty() && x.file == last_file) {
+            Req& q = sc.req.back();
+            const uint64_t end = q.a0 + q.size, xend = x.a0 + x.size;
+            if (x.a0 <= end + kGap && std::max(end, xend) - q.a0 <= kMerge) {
+                if (xend > end) {
+                    total += xend - end;
+                    q.size = xend - q.a0;
+                }
+                x.req = sc.req.size() - 1;
+                x.in_req = x.a0 - q.a0;
+                continue;
+            }
+        }
+        sc.req.push_back({(HANDLE) direct_[(size_t) x.file], x.a0, x.size, total});
+        total += x.size;
+        last_file = x.file;
+        x.req = sc.req.size() - 1;
+        x.in_req = 0;
+    }
+    if (total > sc.cap) {
+        if (sc.buf != nullptr) VirtualFree(sc.buf, 0, MEM_RELEASE);
+        sc.cap = (size_t) ((total + (1u << 20) - 1) >> 20 << 20);
+        sc.buf = (uint8_t*) VirtualAlloc(nullptr, sc.cap, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+        if (sc.buf == nullptr) { sc.cap = 0; return false; }
+    }
+    while (sc.ev.size() < sc.req.size()) {
+        HANDLE e = CreateEventW(nullptr, TRUE, FALSE, nullptr);
+        if (e == nullptr) return false;
+        sc.ev.push_back(e);
+    }
+    sc.ov.assign(sc.req.size(), OVERLAPPED{});
+    // every request in flight before the first wait: the drive sees the whole batch as one queue
+    size_t issued = 0;
+    bool ok = true;
+    for (size_t q = 0; q < sc.req.size(); ++q) {
+        const Req& r = sc.req[q];
+        OVERLAPPED& o = sc.ov[q];
+        o.Offset = (DWORD) r.a0;
+        o.OffsetHigh = (DWORD) (r.a0 >> 32);
+        o.hEvent = sc.ev[q];
+        if (!ReadFile(r.h, sc.buf + r.pos, (DWORD) r.size, nullptr, &o) && GetLastError() != ERROR_IO_PENDING) {
+            ok = false;
+            break;
+        }
+        ++issued;
+    }
+    std::vector<DWORD>& got = sc.got;
+    got.assign(sc.req.size(), 0);
+    for (size_t q = 0; q < issued; ++q)
+        if (!GetOverlappedResult(sc.req[q].h, &sc.ov[q], &got[q], TRUE)) ok = false;
+    if (!ok || issued < sc.req.size()) return false;
+    for (const Window& x : sc.win) {
+        // a request may run past the end of the file: only the role's own bytes have to arrive
+        if ((uint64_t) got[x.req] < x.in_req + x.skip + x.n) return false;
+        std::memcpy(x.dst + x.at, sc.buf + sc.req[x.req].pos + x.in_req + x.skip, (size_t) x.n);
+    }
+    return true;
+#else
+    (void) fills; (void) n;
+    return false;
+#endif
+}
 
 void FileExpertSource::warm(int64_t layer, const int64_t* experts, int64_t n) {
     if (role_ptr_.empty() || n <= 0 || layer < 0 || layer >= n_layers_) return;
@@ -932,7 +1147,7 @@ void RouterLookahead::run() {
 void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k) {
     (void) ids;
     (void) k;
-    if (role_ptr_.empty()) return;
+    if (!staged()) return;
     std::lock_guard<std::mutex> lk(stage_mu_);
     if (layer != last_layer_) {
         ++epoch_;
@@ -941,7 +1156,7 @@ void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k)
 }
 
 bool FileExpertSource::transient(int64_t layer, int64_t expert) const {
-    if (role_ptr_.empty() || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
+    if (!staged() || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
     const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
     if (complement_ready_ && index < complement_offsets_.size() && complement_offsets_[index] != kNoComplement)
         return false;
@@ -1271,17 +1486,43 @@ bool FileExpertSource::pin_cache_complement(
             const int64_t layer = next_layer.fetch_add(1);
             if (layer >= n_layers_ || failed.load()) return;
             const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer];
+            std::vector<Fill> batch;   // #286, unbuffered: 32 blobs' reads in flight at once, merged where near
+            auto flush = [&]() {
+                if (batch.empty()) return true;
+                bool ok = read_direct(batch.data(), batch.size());
+                for (size_t i = 0; !ok && i < batch.size(); ++i)
+                    if (!copy_from_files(batch[i].layer, batch[i].e, batch[i].dst)) return false;
+                copied.fetch_add(blob_bytes * (uint64_t) batch.size());
+                batch.clear();
+                return true;
+            };
             for (int64_t expert = 0; expert < n_expert_; ++expert) {
                 const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
                 const uint64_t offset = offsets[index];
                 if (offset == kNoComplement) continue;
-                if (offset > bytes || blob_bytes > bytes - offset ||
-                    !copy_from_files(layer, expert, (uint8_t*) host + (size_t) offset)) {
+                if (offset > bytes || blob_bytes > bytes - offset) {
+                    fail("FileExpertSource: invalid blob bounds while building the cache complement");
+                    return;
+                }
+                uint8_t* dst = (uint8_t*) host + (size_t) offset;
+                if (!direct_.empty()) {
+                    batch.push_back({0, layer, expert, dst});
+                    if (batch.size() == 32 && !flush()) {
+                        fail("FileExpertSource: an expert could not be read while building the cache complement");
+                        return;
+                    }
+                    continue;
+                }
+                if (!copy_from_files(layer, expert, dst)) {
                     fail("FileExpertSource: invalid blob bounds while building the cache complement");
                     return;
                 }
                 copied.fetch_add(blob_bytes);
             }
+            if (!flush()) {
+                fail("FileExpertSource: an expert could not be read while building the cache complement");
+                return;
+            }
 #if !defined(_WIN32)
             if (role_ptr_.empty()) {   // experts.bin; the GGUF in place leaves its pages to the OS
             const uint64_t layer_offset = layer_offsets_[(size_t) layer];
@@ -1444,7 +1685,7 @@ const uint8_t* FileExpertSource::blob(int64_t layer, int64_t expert) {
         }
     }
     if (from_files) {
-        if (!role_ptr_.empty()) {
+        if (staged()) {
             result = staged_blob(layer, expert);          // counts its bytes
         } else {
             result = mapped_blob(layer, expert);
@@ -1659,7 +1900,8 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
             if (first_of[i] == i) {
                 distinct[nd++] = i;
                 const int32_t e = ids[i];
-                if (e >= 0 && e < d.n_expert && d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] < 0) ++nmiss;
+                if (e >= 0 && e < d.n_expert && d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] < 0 &&
+                    !(d.peer != nullptr && d.peer->has(d.layers, e))) ++nmiss;
             }
         }
         const bool pcie_ok = d.pcie_num > 0 && d.src->pcie_layer(d.layers);
@@ -1679,6 +1921,8 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                     kd = 0;
                     ptr = (unsigned long long) (d.cache_base + (d.cache_slot_off ? (size_t) d.cache_slot_off[slot]
                                                                                  : (size_t) slot * (size_t) d.cache_blob));
+                } else if (d.peer != nullptr && d.peer->has(d.layers, e)) {
+                    kd = 2;                        // multi-GPU: the second GPU computes it
                 } else {
                     if (miss_rank >= nmiss - m && fetches < P.staging_cap && fetches < 64) {
                         const uint8_t* src = d.src->pinned(d.layers, e) ? d.src->blob(d.layers, e) : nullptr;
@@ -1733,7 +1977,18 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
         for (int64_t i = 0; i < n; ++i) {
             const int32_t e = ids[i];
             kind[i] = (e >= 0 && e < d.n_expert && d.host_res != nullptr &&
-                       d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] >= 0) ? 0 : -1;
+                       d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] >= 0) ? 0
+                    : (e >= 0 && e < d.n_expert && d.peer != nullptr && d.peer->has(d.layers, e)) ? 2 : -1;
+        }
+    }
+    if (d.peer != nullptr) {                 // multi-GPU: start the second GPU's share before the CPU's own work
+        std::string perr;
+        if (!d.peer->launch(d.layers, x_f, ids, n_tok, k, kind, perr, out)) {
+            std::fprintf(stderr, "strata: %s (layer %lld)\n", perr.c_str(), (long long) d.layers);
+            d.failed = true;
+            d.fail = "the peer GPU's experts could not be launched";
+            d.fail_layer = d.layers;
+            return;
         }
     }
     if (d.remote_count > 0) {
@@ -1776,9 +2031,12 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                 d.fail_expert = e;
                 return;
             }
-            if (kind[i] >= 0) {             // CUDA0, PCIe, or a remote result staged into this row below
+            if (kind[i] >= 0) {             // CUDA0, PCIe, or a remote/peer result staged into this row below
                 if (kind[i] == 0) ++d.cache_hits;
-                std::memset(row, 0, (size_t) H * sizeof(float));
+                else if (kind[i] == 2 && d.peer != nullptr) ++d.peer_entries;
+                // multi-GPU: a direct peer launch is writing this row right now - zeroing it would race it
+                if (!(kind[i] == 2 && d.peer != nullptr && d.peer->launched_direct()))
+                    std::memset(row, 0, (size_t) H * sizeof(float));
                 continue;
             }
             ++d.cache_refused;
@@ -1816,6 +2074,16 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                 d.failed = true; d.fail = remote_error.c_str(); d.fail_layer = d.layers; return;
             }
     }
+    if (d.peer != nullptr) {                 // multi-GPU: the second GPU's rows, into the same mapped rows
+        std::string perr;
+        if (!d.peer->finish(out, perr)) {
+            std::fprintf(stderr, "strata: %s (layer %lld)\n", perr.c_str(), (long long) d.layers);
+            d.failed = true;
+            d.fail = "the peer GPU's experts failed";
+            d.fail_layer = d.layers;
+            return;
+        }
+    }
     const auto c4 = std::chrono::steady_clock::now();
     pt("ran");
     auto ms = [](auto a, auto b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
@@ -2115,13 +2383,104 @@ bool check_experts_gguf(const std::string& gguf, const strata::kernels::cpu::Exp
 // [gate rows | up rows | down rows] - the layout tools/iq_pack.py would have written to experts.bin.  Each role
 // is read from its own file (native_experts.txt v4: a shard boundary can fall inside a layer; per role as in
 // #255, gopinath87607).  The caller checks the spans first (check_experts_gguf).
+// `unbuffered` (Windows, experts_unbuffered): each chunk's 4 KiB-aligned window is read with FILE_FLAG_NO_BUFFERING into
+// an aligned buffer and scattered into the blobs - no copy through the file cache when the drive is read anyway.
 LoadStats load_experts_gguf(const std::string& gguf, uint8_t* dst, const strata::kernels::cpu::ExpertLayout& lay,
-                            int threads) {
+                            int threads, bool unbuffered) {
     LoadStats st;
     st.layers = (uint64_t) lay.n_layers;
     const auto t0 = std::chrono::steady_clock::now();
     std::atomic<int64_t> next{0};
     std::atomic<bool> bad{false};
+#if defined(_WIN32)
+    if (unbuffered) {
+        uint64_t max_chunk = 0;
+        for (int64_t l = 0; l < lay.n_layers; ++l) {
+            const auto& fm = lay.fmt[(size_t) l];
+            max_chunk = std::max<uint64_t>(max_chunk, std::max<uint64_t>(fm.up_off, lay.bytes[(size_t) l] - fm.down_off) * 16);
+        }
+        std::mutex err_mu;
+        std::string err;
+        auto worker = [&]() {
+            constexpr uint64_t kSector = 4096;
+            const uint64_t cap = (max_chunk + 2 * kSector + kSector - 1) / kSector * kSector;
+            uint8_t* buf = (uint8_t*) VirtualAlloc(nullptr, (size_t) cap, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+            HANDLE h = INVALID_HANDLE_VALUE;
+            std::string open_name;
+            auto fail = [&](const std::string& what) {
+                std::lock_guard<std::mutex> g(err_mu);
+                if (err.empty()) err = what;
+                bad = true;
+            };
+            if (buf == nullptr) fail("cannot allocate a read buffer");
+            for (;;) {
+                const int64_t l = next.fetch_add(1);
+                if (l >= lay.n_layers || bad) break;
+                const auto& fm = lay.fmt[(size_t) l];
+                const uint64_t blob = lay.bytes[(size_t) l];
+                const uint64_t per[3] = {fm.up_off, fm.up_off, blob - fm.down_off};
+                const uint64_t at[3] = {0, fm.up_off, fm.down_off};
+                for (int r = 0; r < 3 && !bad; ++r) {
+                    // the handle is kept while consecutive roles share a file (every layer of a v3 pack)
+                    const std::string name = expert_gguf_file(gguf, lay, l, r);
+                    if (name != open_name) {
+                        if (h != INVALID_HANDLE_VALUE) CloseHandle(h);
+                        const int wide = MultiByteToWideChar(CP_UTF8, 0, name.c_str(), -1, nullptr, 0);
+                        std::vector<wchar_t> w((size_t) std::max(wide, 1), L'\0');
+                        if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, name.c_str(), -1, w.data(), wide);
+                        h = CreateFileW(w.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                                        FILE_FLAG_NO_BUFFERING | FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
+                        if (h == INVALID_HANDLE_VALUE) {
+                            fail("cannot open " + name + " (error " + std::to_string((unsigned long long) GetLastError()) + ")");
+                            break;
+                        }
+                        open_name = name;
+                    }
+                    const uint64_t src = lay.gguf_off[(size_t) (3 * l + r)];
+                    const uint64_t total = per[r] * (uint64_t) lay.n_expert;
+                    const uint64_t chunk = per[r] * 16;           // 16 experts per read
+                    for (uint64_t done = 0; done < total; done += chunk) {
+                        const uint64_t n = std::min<uint64_t>(chunk, total - done);
+                        const uint64_t a0 = (src + done) / kSector * kSector;
+                        const uint64_t a1 = (src + done + n + kSector - 1) / kSector * kSector;
+                        OVERLAPPED ov{};
+                        ov.Offset = (DWORD) a0;
+                        ov.OffsetHigh = (DWORD) (a0 >> 32);
+                        DWORD got = 0;
+                        // the window may run past the end of the file: only the tensor's own bytes have to arrive
+                        if (!ReadFile(h, buf, (DWORD) (a1 - a0), &got, &ov) || (uint64_t) got < src + done - a0 + n) {
+                            fail("short unbuffered read of layer " + std::to_string(l) + " in " + name + " (error " +
+                                 std::to_string((unsigned long long) GetLastError()) + ")");
+                            break;
+                        }
+                        const uint8_t* q = buf + (src + done - a0);
+                        for (uint64_t k = 0; k < n / per[r]; ++k) {
+                            const uint64_t e = done / per[r] + k;
+                            std::memcpy(dst + lay.blob_offset(l, (int64_t) e) + at[r], q + k * per[r], (size_t) per[r]);
+                        }
+                    }
+                }
+            }
+            if (h != INVALID_HANDLE_VALUE) CloseHandle(h);
+            if (buf != nullptr) VirtualFree(buf, 0, MEM_RELEASE);
+        };
+        std::vector<std::thread> pool;
+        for (int i = 1; i < threads; ++i) pool.emplace_back(worker);
+        worker();
+        for (auto& t : pool) t.join();
+        if (bad) {
+            st.seconds = -1.0;
+            st.ok = false;
+            st.error = err.empty() ? "unreadable shard while reading the experts from the GGUF" : err;
+            return st;
+        }
+        st.bytes = lay.total;
+        st.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+        return st;
+    }
+#else
+    (void) unbuffered;
+#endif
     auto worker = [&]() {
         // #230: `fread` on a `FILE*`, as load_experts_ranges (#89): MSVC's `std::ifstream::read` splits a request
         // into 4095-byte freads, which took this path to 0.02 GiB/s on a Windows install without experts.bin.
@@ -2245,8 +2604,31 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
               (why.empty() ? std::string{} : ": " + why);
         return false;
     }
-    const LoadStats st = from_gguf ? load_experts_gguf(gguf_, a->data(), lay, threads)
-                                   : load_experts_ranges(path, a->data(), loff, lbytes, threads, /*chunk=*/8u << 20);
+    // #285: unbuffered when the drive is read anyway and the file cache could not keep the experts for the next
+    // start either (a 64 GB PC); otherwise the buffered readers, which a warm restart serves from the cache
+    std::vector<std::string> files;
+    if (from_gguf) {
+        for (int64_t l = 0; l < n_layers; ++l)
+            for (int r = 0; r < 3; ++r) {
+                const std::string f = expert_gguf_file(gguf_, lay, l, r);
+                if (std::find(files.begin(), files.end(), f) == files.end()) files.push_back(f);
+            }
+    } else {
+        files.push_back(path);
+    }
+    std::string why;
+    const bool unbuffered = experts_unbuffered(files, want + (uint64_t) blob, why);
+    const int readers = unbuffered ? std::max(threads, 16) : threads;   // 16 keep a PCIe 5 drive's queue full
+    LoadStats st;
+    if (from_gguf) {
+        st = load_experts_gguf(gguf_, a->data(), lay, readers, unbuffered);
+    } else {
+        if (unbuffered) st = load_experts_direct(path, a->data(), loff, lbytes, readers, /*chunk=*/8u << 20);
+        if (!unbuffered || (!st.ok && st.error.empty()))   // unaligned ranges: the buffered reader
+            st = load_experts_ranges(path, a->data(), loff, lbytes, threads, /*chunk=*/8u << 20);
+    }
+    std::fprintf(stderr, "strata generate: expert arena read %s (%s)\n", unbuffered ? "unbuffered" : "through the file cache",
+                 why.c_str());
     if (!st.ok) {
         delete a;
         err = "ArenaExpertSource: the expert load was refused: " + (st.error.empty() ? std::string("unknown") : st.error);

src/core/mtp.cpp

@@ -4,6 +4,7 @@
 #include "strata/core/on_device.hpp"
 
 #include "strata/core/native_head.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
 #include "strata/kernels/cpu/expert.hpp"
 #include "strata/kernels/elementwise.hpp"
@@ -30,6 +31,7 @@
 #include <atomic>
 #include <chrono>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <exception>
 #include <fstream>
@@ -55,7 +57,7 @@ struct Bump {
 };
 
 bool mapped(size_t bytes, void** h, void** d) {
-    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped) != cudaSuccess) return false;
+    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false;
     std::memset(*h, 0, bytes);
     return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;
 }
@@ -728,8 +730,21 @@ bool MtpDrafter::prefill(const float* R_rows, const int32_t* next_tokens, int64_
     // E-4: every group's token / step / position records uploaded at once; each group is then device copies and a
     // graph on the one stream, with a single sync at the end (a group of <= max_t rows used to be staged in mapped
     // memory and synced before the next: ~5,500 host round trips on a 32K prompt).  The same work in the same
-    // order.  STRATA_MTP_PREFILL_SYNC=1 keeps the old loop.
-    static const bool per_group_sync = std::getenv("STRATA_MTP_PREFILL_SYNC") != nullptr;
+    // order.  STRATA_MTP_PREFILL_SYNC=1 keeps the old loop, =0 forces E-4.
+    // HIP defaults to the old loop.  This pass runs whenever E-9 (Prefill::draft_kv) declines, which it does for the
+    // drafter's ring (KV streaming, --kv-resident), and on gfx1201 / ROCm 7.2.4 the E-4 queue (a graph launch and four
+    // device copies per group, ~2,000 groups per 8192-token chunk, no sync) sometimes never completes: the prompt hangs
+    // in hipGraphLaunch or the final sync until the watchdog ends the engine.  R9700, full IQ3_XXS, --kv-resident
+    // 32768, mixed load (chats, 32K and 7K prompts, deep follow-ups): E-4 hung in the first round on 2 of 2 tries,
+    // the old loop ran 30 of 30 rounds clean, prompt speed unchanged.
+    static const bool per_group_sync = [] {
+        if (const char* v = std::getenv("STRATA_MTP_PREFILL_SYNC")) return std::atoi(v) != 0;
+#if defined(STRATA_USE_HIP)
+        return true;
+#else
+        return false;
+#endif
+    }();
     const int64_t NHp = g_->n_head, per_row = 1 + 4 + NHp;
     if (!per_group_sync && n > 0) {
         if (pf_cap_ < n * per_row) {

src/core/peer_experts.cpp

@@ -0,0 +1,325 @@
+// src/core/peer_experts.cpp - see include/strata/core/peer_experts.hpp.
+#include "strata/core/peer_experts.hpp"
+
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/elementwise.hpp"
+#include "strata/kernels/iq_kernels.hpp"
+#include "strata/kernels/quantize_act.hpp"
+#include "strata/kernels/s2_expert_grouped.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdio>
+#include <cstring>
+#include <cstdlib>
+#include <stdexcept>
+
+namespace strata::core {
+
+static bool g_peer_portable = false;
+void set_peer_portable(bool on) { g_peer_portable = on; }
+bool peer_portable() { return g_peer_portable; }
+
+namespace {
+
+constexpr int64_t H = strata::kernels::cpu::H;
+constexpr int64_t CAP = strata::kernels::cpu::MAXT * 10;   // entries per layer: MAXT tokens x top-10
+
+struct Meta {                           // one block, uploaded with one copy per layer
+    unsigned long long ptr[CAP];
+    int32_t start[CAP + 1];
+    int32_t dst[CAP];
+    int32_t tok[CAP];
+    int32_t count[4];                   // [0] groups, [1] entries
+    int32_t out_row[CAP];               // multi-GPU: compact row -> entry row in the host `out`
+};
+
+/// Switches the calling thread to `dev` and back.  The engine's own thread lives on device 0.
+struct On {
+    int prev = 0;
+    explicit On(int dev) { cudaGetDevice(&prev); if (prev != dev) cudaSetDevice(dev); }
+    ~On() { int cur = 0; cudaGetDevice(&cur); if (cur != prev) cudaSetDevice(prev); }
+};
+
+bool ck(cudaError_t e, const char* what, std::string& err) {
+    if (e == cudaSuccess) return true;
+    err = std::string("peer experts: ") + what + ": " + cudaGetErrorString(e);
+    return false;
+}
+
+}  // namespace
+
+PeerExperts::~PeerExperts() { close(); }
+
+void PeerExperts::close() {
+    if (device_ < 0) return;
+    {
+        On on(device_);
+        if (stream_) cudaStreamSynchronize(stream_);
+        if (refill_) cudaStreamSynchronize(refill_);
+        cache_.close();
+        if (d_x_) cudaFree(d_x_);
+        if (d_out_) cudaFree(d_out_);
+        if (d_meta_) cudaFree(d_meta_);
+        if (d_q8_) cudaFree(d_q8_);
+        if (d_scratch_) cudaFree(d_scratch_);
+        if (h_x_) cudaFreeHost(h_x_);
+        if (h_out_) cudaFreeHost(h_out_);
+        if (h_meta_) cudaFreeHost(h_meta_);
+        if (refill_ev_) cudaEventDestroy(refill_ev_);
+        if (stream_) cudaStreamDestroy(stream_);
+        if (refill_) cudaStreamDestroy(refill_);
+    }
+    d_x_ = d_out_ = h_x_ = h_out_ = nullptr;
+    d_meta_ = h_meta_ = d_scratch_ = nullptr;
+    d_q8_ = nullptr;
+    stream_ = refill_ = nullptr;
+    refill_ev_ = nullptr;
+    device_ = -1;
+}
+
+bool PeerExperts::open(int device, const std::vector<std::pair<int32_t, int32_t>>& ranked, const ExpertCache& primary,
+                       ExpertSource& src, int64_t n_layers, int64_t n_expert, int reserve_mib, int64_t max_slots,
+                       std::string& err) {
+    close();
+    int count = 0;
+    if (!ck(cudaGetDeviceCount(&count), "cudaGetDeviceCount", err)) return false;
+    if (device < 1 || device >= count) {
+        err = "peer experts: CUDA device " + std::to_string(device) + " is not visible (" + std::to_string(count) +
+              " devices; set CUDA_VISIBLE_DEVICES)";
+        return false;
+    }
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    n_layers_ = n_layers;
+    n_expert_ = n_expert;
+    src_ = &src;
+    res_.assign((size_t) (n_layers * n_expert), kNotResident);
+    {   // direct access both ways (NVLink or another P2P path): the prompt path copies activations and results over it
+        int a = 0, b = 0;
+        cudaDeviceCanAccessPeer(&a, 0, device);
+        cudaDeviceCanAccessPeer(&b, device, 0);
+        p2p_ = a && b;
+        if (p2p_) {
+            On on0(0);
+            cudaError_t e0 = cudaDeviceEnablePeerAccess(device, 0);
+            if (e0 == cudaErrorPeerAccessAlreadyEnabled) { cudaGetLastError(); e0 = cudaSuccess; }
+            On on1(device);
+            cudaError_t e1 = cudaDeviceEnablePeerAccess(0, 0);
+            if (e1 == cudaErrorPeerAccessAlreadyEnabled) { cudaGetLastError(); e1 = cudaSuccess; }
+            p2p_ = e0 == cudaSuccess && e1 == cudaSuccess;
+        }
+    }
+    On on(device);
+    device_ = device;
+    // scratch first, so the slots take what is really left
+    int64_t ff = strata::kernels::cpu::FF;
+    if (lay.native)
+        for (const auto& f : lay.fmt) ff = std::max<int64_t>(ff, f.n_ff);
+    const size_t scratch = std::max<size_t>((size_t) strata::kernels::moe_hit_grouped_scratch_bytes(CAP, H, ff),
+                                            strata::kernels::native_expert_scratch_bytes(CAP, ff));
+    const bool alloc_ok =
+        ck(cudaStreamCreateWithFlags(&stream_, cudaStreamNonBlocking), "stream", err) &&
+        ck(cudaStreamCreateWithFlags(&refill_, cudaStreamNonBlocking), "refill stream", err) &&
+        ck(cudaEventCreateWithFlags(&refill_ev_, cudaEventDisableTiming), "event", err) &&
+        ck(cudaHostAlloc((void**) &h_x_, (size_t) CAP * H * sizeof(float), cudaHostAllocPortable | cudaHostAllocMapped), "input staging", err) &&
+        ck(cudaHostAlloc((void**) &h_out_, (size_t) CAP * H * sizeof(float), cudaHostAllocPortable | cudaHostAllocMapped), "result staging", err) &&
+        ck(cudaHostAlloc(&h_meta_, sizeof(Meta), cudaHostAllocPortable | cudaHostAllocMapped), "plan staging", err) &&
+        ck(cudaMalloc((void**) &d_x_, (size_t) CAP * H * sizeof(float)), "input", err) &&
+        ck(cudaMalloc((void**) &d_out_, (size_t) CAP * H * sizeof(float)), "result", err) &&
+        ck(cudaMalloc(&d_meta_, sizeof(Meta)), "plan", err) &&
+        ck(cudaMalloc((void**) &d_q8_, (size_t) CAP * (H / 32) * 36), "activations", err) &&
+        ck(cudaMalloc(&d_scratch_, scratch), "scratch", err);
+    if (!alloc_ok) { close(); return false; }
+
+    // the pairs the primary does not hold, in rank order, as many as fit
+    size_t free_b = 0, total_b = 0;
+    if (!ck(cudaMemGetInfo(&free_b, &total_b), "cudaMemGetInfo", err)) { close(); return false; }
+    const uint64_t reserve = (uint64_t) std::max(reserve_mib, 128) << 20;
+    const uint64_t budget = free_b > reserve ? free_b - reserve : 0;
+    std::vector<std::pair<int32_t, int32_t>> pick;
+    std::vector<int64_t> sizes;
+    uint64_t used = 0;
+    for (const auto& pr : ranked) {
+        if (primary.slot_of(pr.first, pr.second) >= 0) continue;
+        const uint64_t b = lay.blob_bytes(pr.first);
+        const uint64_t b256 = lay.native ? (b + 255) / 256 * 256 : lay.max_blob;
+        if (used + b256 > budget) break;
+        if (max_slots > 0 && (int64_t) pick.size() >= max_slots) break;
+        used += b256;
+        pick.push_back(pr);
+        sizes.push_back((int64_t) b);
+    }
+    if (pick.empty()) { err = "peer experts: no room or no expert left for the peer"; close(); return false; }
+    const bool opened = lay.native ? cache_.open_sized(sizes, n_layers, n_expert, err)
+                                   : cache_.open((int64_t) pick.size(), n_layers, n_expert, (int64_t) lay.max_blob, err);
+    if (!opened) { err = "peer experts: " + err; close(); return false; }
+    // open() zeroes the arena with cudaMemset on the legacy stream, which is not ordered against the non-blocking
+    // refill stream: wait for it, or the zeroing can land on top of the fills
+    if (!ck(cudaDeviceSynchronize(), "arena zeroing", err)) { close(); return false; }
+    for (const auto& pr : pick) {
+        const int32_t slot = cache_.admit(pr.first, pr.second);
+        const uint8_t* b = src.blob(pr.first, pr.second);
+        if (slot < 0 || b == nullptr || !cache_.fill_slot(slot, b, refill_, err, (int64_t) lay.blob_bytes(pr.first))) {
+            err = "peer experts: fill failed: " + err;
+            close();
+            return false;
+        }
+        res_[(size_t) (pr.first * n_expert + pr.second)] = slot;
+    }
+    if (!ck(cudaStreamSynchronize(refill_), "fill", err)) { close(); return false; }
+    const auto& f0 = pick.front();
+    if (!cache_.verify_slot(res_[(size_t) (f0.first * n_expert + f0.second)], src.blob(f0.first, f0.second), err,
+                            (int64_t) lay.blob_bytes(f0.first))) {
+        err = "peer experts: " + err;
+        close();
+        return false;
+    }
+    resident_ = (int64_t) pick.size();
+    return true;
+}
+
+bool PeerExperts::launch(int64_t layer, const float* x, const int32_t* ids, int64_t n_tok, int64_t k,
+                         const int32_t* kind, std::string& err, float* out) {
+    static const bool direct_env = [] { const char* v = std::getenv("STRATA_PEER_DIRECT"); return v == nullptr || std::atoi(v) != 0; }();
+    const bool direct = direct_env && out != nullptr;
+    launched_direct_ = false;
+    launched_rows_ = 0;
+    row_of_.clear();
+    const int64_t n = n_tok * k;
+    if (n > CAP) { err = "peer experts: window too large"; return false; }
+    Meta& m = *(Meta*) h_meta_;
+    int groups = 0, rows = 0;
+    // distinct experts in routing order; each one's entries become compact rows
+    for (int64_t i = 0; i < n; ++i) {
+        if (kind[i] != 2) continue;
+        bool seen = false;
+        for (int64_t j = 0; j < i; ++j)
+            if (kind[j] == 2 && ids[j] == ids[i]) { seen = true; break; }
+        if (seen) continue;
+        const int32_t slot = res_[(size_t) (layer * n_expert_ + ids[i])];
+        if (slot < 0) { err = "peer experts: an entry was planned for the peer but is not resident"; return false; }
+        m.ptr[groups] = (unsigned long long) cache_.device_slot(slot);
+        m.start[groups] = rows;
+        for (int64_t j = i; j < n; ++j)
+            if (kind[j] == 2 && ids[j] == ids[i]) {
+                m.dst[rows] = rows;
+                m.tok[rows] = (int32_t) (j / k);
+                m.out_row[rows] = (int32_t) j;
+                row_of_.push_back((int32_t) j);
+                ++rows;
+            }
+        ++groups;
+        ++experts_;
+    }
+    if (groups == 0) return true;
+    m.start[groups] = rows;
+    m.count[0] = groups;
+    m.count[1] = rows;
+    entries_ += rows;
+    launched_rows_ = rows;
+    // direct: x is the verifier's pinned (portable) doorbell row block - copied from where it is
+    if (!direct) std::memcpy(h_x_, x, (size_t) (n_tok * H) * sizeof(float));
+    On on(device_);
+    const cudaStream_t s = stream_;
+    if (!ck(cudaMemcpyAsync(d_x_, direct ? x : h_x_, (size_t) (n_tok * H) * sizeof(float), cudaMemcpyHostToDevice, s), "input", err) ||
+        !ck(cudaMemcpyAsync(d_meta_, h_meta_, sizeof(Meta), cudaMemcpyHostToDevice, s), "plan", err))
+        return false;
+    Meta* dm = (Meta*) d_meta_;
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    if (lay.native) {
+        strata::kernels::quantize_q8_1_rows(d_x_, n_tok, H, d_q8_, s);
+        const auto& f = lay.fmt[(size_t) layer];
+        const auto L = strata::kernels::native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);
+        strata::kernels::native_expert_grouped(L, dm->ptr, dm->start, dm->count, dm->dst, dm->tok, groups, rows, d_q8_,
+                                               d_scratch_, d_out_, s);
+    } else {
+        err = "peer experts: only native packs are supported";
+        return false;
+    }
+    if (direct) {   // the rows straight into the host `out` (zero-copy, coalesced float4 writes)
+        try {
+            strata::kernels::scatter_rows_f32(d_out_, out, dm->out_row, rows, H, s);
+        } catch (const std::exception& e) { err = std::string("peer experts: scatter: ") + e.what(); return false; }
+        launched_direct_ = true;
+        return true;
+    }
+    return ck(cudaMemcpyAsync(h_out_, d_out_, (size_t) rows * H * sizeof(float), cudaMemcpyDeviceToHost, s),
+              "results", err);
+}
+
+bool PeerExperts::finish(float* out, std::string& err) {
+    if (launched_rows_ == 0) return true;
+    const auto t0 = std::chrono::steady_clock::now();
+    {
+        On on(device_);
+        // spin on the stream (a blocking sync would sleep the pool thread and wake it late)
+        cudaError_t e;
+        while ((e = cudaStreamQuery(stream_)) == cudaErrorNotReady) {
+        }
+        if (!ck(e, "finish", err)) return false;
+    }
+    ms_wait += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
+    if (launched_direct_) { launched_rows_ = 0; return true; }
+    for (size_t r = 0; r < row_of_.size(); ++r)
+        std::memcpy(out + (size_t) row_of_[r] * H, h_out_ + r * H, (size_t) H * sizeof(float));
+    launched_rows_ = 0;
+    return true;
+}
+
+bool PeerExperts::adapt(const float* usage, const int32_t* res0, int max_swaps, std::string& err) {
+    if (!pending_.empty() || max_swaps <= 0) return true;
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    struct Swap { float gain; int32_t layer, in, out; };
+    std::vector<Swap> swaps;
+    std::vector<std::pair<float, int32_t>> cand, vict;
+    for (int64_t l = 0; l < n_layers_; ++l) {
+        cand.clear();
+        vict.clear();
+        const float* u = usage + l * n_expert_;
+        const int32_t* r0 = res0 + l * n_expert_;
+        const int32_t* r1 = res_.data() + l * n_expert_;
+        for (int32_t e = 0; e < (int32_t) n_expert_; ++e) {
+            if (r1[e] >= 0) vict.emplace_back(u[e], e);
+            else if (r0[e] < 0 && u[e] >= 2.0f) cand.emplace_back(u[e], e);
+        }
+        if (cand.empty() || vict.empty()) continue;
+        std::sort(cand.begin(), cand.end(), [](auto& a, auto& b) { return a.first > b.first; });
+        const size_t nc = std::min(cand.size(), vict.size());
+        std::partial_sort(vict.begin(), vict.begin() + (ptrdiff_t) nc, vict.end(),
+                          [](auto& a, auto& b) { return a.first < b.first; });
+        for (size_t i = 0; i < nc; ++i) {
+            if (cand[i].first < vict[i].first + 1.5f) break;
+            swaps.push_back({cand[i].first - vict[i].first, (int32_t) l, cand[i].second, vict[i].second});
+        }
+    }
+    if (swaps.empty()) return true;
+    std::sort(swaps.begin(), swaps.end(), [](const Swap& a, const Swap& b) { return a.gain > b.gain; });
+    if ((int) swaps.size() > max_swaps) swaps.resize((size_t) max_swaps);
+    On on(device_);
+    for (const Swap& s : swaps) {
+        const size_t in = (size_t) (s.layer * n_expert_ + s.in), out = (size_t) (s.layer * n_expert_ + s.out);
+        const int32_t slot = res_[out];
+        const uint8_t* b = src_->blob(s.layer, s.in);
+        if (slot < 0 || b == nullptr) continue;
+        if (!ck(cudaMemcpyAsync(cache_.device_slot(slot), b, (size_t) lay.blob_bytes(s.layer), cudaMemcpyHostToDevice,
+                                refill_), "refill", err))
+            return false;
+        res_[out] = kNotResident;              // evicted now: the CPU computes it meanwhile
+        pending_.emplace_back((int32_t) in, slot);
+        ++swaps_;
+    }
+    return ck(cudaEventRecord(refill_ev_, refill_), "refill event", err);
+}
+
+void PeerExperts::apply_pending(bool wait) {
+    if (pending_.empty()) return;
+    {
+        On on(device_);
+        if (wait) cudaEventSynchronize(refill_ev_);
+        else if (cudaEventQuery(refill_ev_) != cudaSuccess) return;
+    }
+    for (const auto& [i, slot] : pending_) res_[(size_t) i] = slot;
+    pending_.clear();
+}
+
+}  // namespace strata::core

src/core/pinned.cu

@@ -199,15 +199,32 @@ void* reserve(uint64_t bytes, PageBacking& got, std::string& note, const std::st
         return (uint8_t*) map + kSharedArenaHeaderBytes;
     }
 
-    void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE,
-                   MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, -1, 0);
-    if (p != MAP_FAILED) {
-        got = PageBacking::LargePages;
-        note = "hugetlb 2 MB pages";
-        return p;
+    // STRATA_NO_LARGEPAGES=1 is the same-run A/B switch the Windows branch documents; honor it
+    // here too, so the large-page path can be compared without changing the pool or rebooting.
+    if (std::getenv("STRATA_NO_LARGEPAGES") != nullptr) {
+        note = "large pages skipped (STRATA_NO_LARGEPAGES); using 4 KB pages";
+    } else {
+        void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE,
+                       MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, -1, 0);
+        if (p != MAP_FAILED) {
+            got = PageBacking::LargePages;
+            note = "hugetlb 2 MB pages";
+            return p;
+        }
+        // MAP_HUGETLB is all-or-nothing: a pool smaller than the mapping fails exactly like an absent
+        // one, and the old message guessed "no hugetlb pool configured" either way. Name the shortfall:
+        // how many 2 MiB pages the mapping needs against what vm.nr_hugepages actually holds.
+        const unsigned long long need = ((unsigned long long) bytes + (1ull << 21) - 1) / (1ull << 21);
+        unsigned long long pool = 0;
+        bool have_pool = false;
+        if (std::FILE* f = std::fopen("/proc/sys/vm/nr_hugepages", "r")) {
+            have_pool = std::fscanf(f, "%llu", &pool) == 1;
+            std::fclose(f);
+        }
+        note = "MAP_HUGETLB unavailable (needed " + std::to_string(need) + " 2 MiB pages, vm.nr_hugepages=" +
+               (have_pool ? std::to_string(pool) : std::string("?")) + "); using 4 KB pages";
     }
-    note = "MAP_HUGETLB unavailable (no hugetlb pool configured?); using 4 KB pages";
-    p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
     got = PageBacking::NormalPages;
     return p == MAP_FAILED ? nullptr : p;
 #endif
@@ -413,6 +430,141 @@ LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_byte
     return load_experts_ranges(path, dst, off, n, threads, chunk);
 }
 
+LoadStats load_experts_direct(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
+                              const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk) {
+    LoadStats st;
+    st.ok = false;
+#ifdef _WIN32
+    constexpr uint64_t kAlign = 4096;
+    if (((uintptr_t) dst % kAlign) != 0 || chunk == 0 || chunk % kAlign != 0) return st;
+    struct Piece { uint64_t off, n; };
+    std::vector<Piece> pieces;
+    uint64_t bytes = 0;
+    for (size_t L = 0; L < layer_off.size(); ++L) {
+        if (layer_off[L] % kAlign != 0 || layer_bytes[L] % kAlign != 0) return st;
+        for (uint64_t p = 0; p < layer_bytes[L]; p += chunk)
+            pieces.push_back({layer_off[L] + p, std::min<uint64_t>(chunk, layer_bytes[L] - p)});
+        bytes += layer_bytes[L];
+    }
+    if (threads < 1) threads = 1;
+    const auto t0 = std::chrono::steady_clock::now();
+    std::atomic<size_t> next{0};
+    std::mutex err_mu;
+    std::string err;
+    const int wide = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, nullptr, 0);
+    std::vector<wchar_t> wpath((size_t) std::max(wide, 1), L'\0');
+    if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, wpath.data(), wide);
+    auto worker = [&]() {
+        HANDLE h = CreateFileW(wpath.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                               FILE_FLAG_NO_BUFFERING | FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
+        if (h == INVALID_HANDLE_VALUE) {
+            std::lock_guard<std::mutex> g(err_mu);
+            if (err.empty())
+                err = "cannot open " + path + " unbuffered (error " + std::to_string((unsigned long long) GetLastError()) + ")";
+            next = pieces.size();
+            return;
+        }
+        for (;;) {
+            const size_t i = next.fetch_add(1);
+            if (i >= pieces.size()) break;
+            OVERLAPPED ov{};
+            ov.Offset = (DWORD) pieces[i].off;
+            ov.OffsetHigh = (DWORD) (pieces[i].off >> 32);
+            DWORD got = 0;
+            if (!ReadFile(h, dst + pieces[i].off, (DWORD) pieces[i].n, &got, &ov) || got != pieces[i].n) {
+                std::lock_guard<std::mutex> g(err_mu);
+                if (err.empty())
+                    err = "short unbuffered read at offset " + std::to_string(pieces[i].off) + ": got " + std::to_string(got) +
+                          " of " + std::to_string(pieces[i].n) + " B (error " +
+                          std::to_string((unsigned long long) GetLastError()) + ")";
+                next = pieces.size();
+                break;
+            }
+        }
+        CloseHandle(h);
+    };
+    std::vector<std::thread> pool;
+    for (int i = 1; i < threads; ++i) pool.emplace_back(worker);
+    worker();
+    for (auto& t : pool) t.join();
+    st.layers = layer_off.size();
+    if (!err.empty()) {
+        st.seconds = -1.0;
+        st.error = err;
+        return st;
+    }
+    st.ok = true;
+    st.bytes = bytes;
+    st.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+#else
+    (void) path; (void) dst; (void) layer_off; (void) layer_bytes; (void) threads; (void) chunk;
+#endif
+    return st;
+}
+
+bool experts_unbuffered(const std::vector<std::string>& files, uint64_t arena_bytes, std::string& why,
+                        bool cache_counts) {
+    const char* env = std::getenv("STRATA_UNBUFFERED_LOAD");
+    if (env != nullptr && env[0] != '\0') {
+        why = std::string("STRATA_UNBUFFERED_LOAD=") + env;
+        return env[0] != '0';
+    }
+#ifdef _WIN32
+    LARGE_INTEGER freq{}, a{}, b{};
+    QueryPerformanceFrequency(&freq);
+    constexpr DWORD kRead = 64 << 10;
+    constexpr int kSamples = 16;
+    std::vector<uint8_t> buf(kRead);
+    uint64_t total_bytes = 0;
+    int fast = 0, n = 0;
+    uint64_t seed = (uint64_t) GetTickCount64() * 6364136223846793005ull + 1442695040888963407ull;
+    for (const std::string& f : files) {
+        const int wide = MultiByteToWideChar(CP_UTF8, 0, f.c_str(), -1, nullptr, 0);
+        std::vector<wchar_t> w((size_t) std::max(wide, 1), L'\0');
+        if (wide > 0) MultiByteToWideChar(CP_UTF8, 0, f.c_str(), -1, w.data(), wide);
+        HANDLE h = CreateFileW(w.data(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS,
+                               nullptr);
+        if (h == INVALID_HANDLE_VALUE) continue;
+        LARGE_INTEGER size{};
+        GetFileSizeEx(h, &size);
+        total_bytes += (uint64_t) size.QuadPart;
+        // random offsets: a probe must not find the blocks an earlier probe put into the cache
+        for (int i = 0; i < kSamples && (uint64_t) size.QuadPart > 2ull * kRead; ++i) {
+            seed = seed * 6364136223846793005ull + 1442695040888963407ull;
+            const uint64_t off = ((seed >> 17) % ((uint64_t) size.QuadPart - kRead)) / kRead * kRead;
+            OVERLAPPED ov{};
+            ov.Offset = (DWORD) off;
+            ov.OffsetHigh = (DWORD) (off >> 32);
+            DWORD got = 0;
+            QueryPerformanceCounter(&a);
+            const BOOL ok = ReadFile(h, buf.data(), kRead, &got, &ov);
+            QueryPerformanceCounter(&b);
+            if (!ok) continue;
+            ++n;
+            fast += (double) (b.QuadPart - a.QuadPart) * 1e6 / (double) freq.QuadPart < 30.0;
+        }
+        CloseHandle(h);
+    }
+    const bool cached = n > 0 && fast * 4 >= n * 3;
+    MEMORYSTATUSEX ms{};
+    ms.dwLength = sizeof ms;
+    GlobalMemoryStatusEx(&ms);
+    const uint64_t avail = ms.ullAvailPhys;
+    // what the cache could keep beside the arena (~4 GiB for everything else)
+    const uint64_t room = avail > arena_bytes + (4ull << 30) ? avail - arena_bytes - (4ull << 30) : 0;
+    const bool keepable = room >= total_bytes;
+    char msg[200];
+    std::snprintf(msg, sizeof msg, "%d of %d probe reads from the file cache; %.1f GiB available, %.1f GiB of files",
+                  fast, n, (double) avail / (1ull << 30), (double) total_bytes / (1ull << 30));
+    why = msg;
+    return (!cached || !cache_counts) && !keepable;
+#else
+    (void) files; (void) arena_bytes; (void) cache_counts;
+    why = "buffered (not Windows)";
+    return false;
+#endif
+}
+
 LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::vector<uint64_t>& layer_off,
                               const std::vector<uint64_t>& layer_bytes, int threads, uint64_t chunk) {
     LoadStats st;

src/core/verify.cpp

@@ -6,6 +6,7 @@
 
 #include "strata/core/native_head.hpp"
 #include "strata/core/on_device.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/kernels/iq_kernels.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
@@ -69,7 +70,7 @@ struct Bump {
 };
 
 bool mapped(size_t bytes, void** h, void** d) {
-    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped) != cudaSuccess) return false;
+    if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false;   // multi-GPU: portable only with a peer - the peer card writes its rows into them
     std::memset(*h, 0, bytes);
     return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;
 }
@@ -210,7 +211,9 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
     }
     if (hits.d_res == nullptr || hits.cache_base == nullptr || hits.blob <= 0) {
         err = "verify: needs the profile-filled VRAM expert tier (--expert-profile and --expert-cache); with "
-              "--expert-cache auto, no VRAM was left for it: lower --max-context, use --kv k8v4 or images on the CPU";
+              "--expert-cache auto, no VRAM was left for it - the 'no VRAM is left for the expert cache' line above "
+              "says how much is short and what makes room (a smaller --max-context, --kv q4_0, setup --draft-vocab en, "
+              "images on the CPU)";
         return false;
     }
     std::string why;
@@ -745,20 +748,21 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
         const int32_t* p_start2 = pl + ptr_off + 4 * capx;
         float* hit_out = hit_out_ + (size_t) tb * K * N;
         const auto& lay = strata::kernels::cpu::expert_layout();
-        // plan v0.3 P6: the VRAM groups now; the PCIe groups once the copy engine has landed them in staging
-        auto grouped = [&](const unsigned long long* gp, const int32_t* gs, const int32_t* gn) {
+        // plan v0.3 P6: the VRAM groups now; the PCIe groups once the copy engine has landed them in staging.
+        // `gy`: the native launch's groups side by side (0: cap, one block row per possible group).
+        auto grouped = [&](const unsigned long long* gp, const int32_t* gs, const int32_t* gn, int64_t gy) {
             if (lay.native) {
                 // the layer's GGUF formats (i-quant gate/up, Q2_0 / IQ4_NL down)
                 const auto& f = lay.fmt[(size_t) l];
                 const NativeExpertLayout L = native_expert_layout(f.gu_type, f.d_type, f.n_embd, f.n_ff);
                 native_expert_grouped(L, gp, gs, gn, p_dst, p_tok, cap, cap,
-                                      nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, hit_out, cs);
+                                      nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, hit_out, cs, gy);
             } else {
                 moe_grouped_s2(gp, gs, gn, p_dst, p_tok, cap, cap, hit_xq_ + (size_t) tb * (N / 32) * 34,
                                hit_xs_ + (size_t) tb * (N / 32), hit_scratch_, hit_out, cs);
             }
         };
-        grouped(p_ptr, p_start, p_counts);
+        grouped(p_ptr, p_start, p_counts, 0);
         stamp(l, 20, grp);
         if (device_plan_) wait_flag_ge_or(m_flagB_, ring, skip_ + grp, cs);
         else wait_flag_ge(m_flagB_, ring, cs);                 // the PCIe share is in staging (DMA) or mapped
@@ -769,7 +773,9 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             rebase_ptrs((unsigned long long*) p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), cs);
         }
         stamp(l, 21, grp);
-        grouped(p_ptr2, p_start2, p_counts + 2);
+        // the PCIe share is pcie_frac of the misses: a few groups when the cache is cold, usually none (always none at
+        // pcie_frac 0), so its launch is kPcieGroupRows block rows striding over the groups, not cap of them
+        grouped(p_ptr2, p_start2, p_counts + 2, kPcieGroupRows);
         stamp(l, 22, grp);
         if (device_plan_) {   // no CPU share when the device planned the group: its rows are zeros
             wait_flag_ge_or(m_flag_, ring, skip_ + grp, cs);

src/kernels/cpu/iq_avx2.cpp

@@ -9,7 +9,7 @@
 // with one, `vpsignb`.  The arithmetic is ggml's (ggml-cpu/quants.c, the `_generic` references) - only the
 // order of the float additions differs.
 //
-// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22).  IQ1_M stays on ggml-cpu.
+// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22), IQ4_XS (23).  IQ1_M stays on ggml-cpu.
 #include "strata/kernels/cpu/iq_avx2.hpp"
 
 #define GGML_COMMON_DECL_CPP
@@ -177,6 +177,28 @@ template <> struct Fmt32<21> {   // IQ3_S: d, qs[64], qh[8], signs[32], scales[4
     }
 };
 
+template <> struct Fmt32<23> {   // IQ4_XS: d, scales_h, scales_l[4], qs[128] - 136 B, 8 signed sub-scales
+    // ggml's ggml_vec_dot_iq4_xs_q8_K: the same 16-value codebook as IQ4_NL, but each of the eight 32-value
+    // sub-blocks carries its own 6-bit scale, read as two nibbles of scales_l[p] plus two bits of scales_h, and
+    // used SIGNED as (ls - 32).  The scale therefore folds into the int16 operand of madd_epi16 instead of
+    // becoming a float multiply per sub-block, and the codebook sign is carried the way iq4nl_rows does it.
+    static constexpr int bytes = 136;
+    static constexpr float K = 1.0f;
+    static inline void decode(const uint8_t* b, int j, int half, __m256i& g, __m256i& sgn, __m256i& sc) {
+        const int H = 2 * j + half;   // the eight 32-value sub-blocks, in the order the activation bytes come
+        const __m128i values = _mm_loadu_si128((const __m128i*) kvalues_iq4nl);
+        const __m128i bits = _mm_loadu_si128((const __m128i*) (b + 8 + 16 * H));
+        const __m128i m4 = _mm_set1_epi8(0x0f);
+        const __m256i q4 = _mm256_inserti128_si256(
+            _mm256_castsi128_si256(_mm_shuffle_epi8(values, _mm_and_si128(bits, m4))),
+            _mm_shuffle_epi8(values, _mm_and_si128(_mm_srli_epi16(bits, 4), m4)), 1);
+        g   = _mm256_sign_epi8(q4, q4);                       // |w|, the unsigned operand of maddubs
+        sgn = _mm256_sign_epi8(_mm256_set1_epi8(1), q4);      // w's sign, applied to the activation
+        const int ls = ((b[4 + (H >> 1)] >> (4 * (H & 1))) & 0xf) | (((u16(b + 2) >> (2 * H)) & 3) << 4);
+        sc = _mm256_set1_epi16((short) (ls - 32));
+    }
+};
+
 template <int TY, int NT>
 inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res) {
     __m256 accf[NT];
@@ -357,7 +379,7 @@ void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void*
 }  // namespace
 
 bool iq256_supported(int type) noexcept {
-    return type == 16 || type == 17 || type == 18 || type == 21 || type == 22;
+    return type == 16 || type == 17 || type == 18 || type == 21 || type == 22 || type == 23;
 }
 
 void iq256_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt,
@@ -368,6 +390,7 @@ void iq256_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off,
         case 18: gu_rows_nt<18>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         case 21: gu_rows_nt<21>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         case 22: gu_rows_nt<22>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 23: gu_rows_nt<23>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         default: break;
     }
 }
@@ -380,6 +403,7 @@ void iq256_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void*
         case 18: dot_rows_nt<18>(nt, w, row_bytes, n, act, out, r0, r1); break;
         case 21: dot_rows_nt<21>(nt, w, row_bytes, n, act, out, r0, r1); break;
         case 22: dot_rows_nt<22>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 23: dot_rows_nt<23>(nt, w, row_bytes, n, act, out, r0, r1); break;
         default: break;
     }
 }

src/kernels/cpu/native_expert.cpp

@@ -98,12 +98,16 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*
         kq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
         return;
     }
-    if (nt >= mt_min && iq512_supported(f.gu_type)) {
-        if (avx512) {
+    // A format with only an AVX-2 kernel (IQ4_XS, #415) takes it on AVX-2 CPUs only: an AVX-512 CPU keeps ggml-cpu for
+    // it, as before (its rows would round differently).  Each kernel only for the formats it implements: falling
+    // through an empty switch would leave ff unwritten instead of falling back to ggml-cpu.
+    static const bool cpu512 = cpu_avx512_ok();
+    if (nt >= mt_min && (iq512_supported(f.gu_type) || (!cpu512 && iq256_supported(f.gu_type)))) {
+        if (avx512 && iq512_supported(f.gu_type)) {
             iq512_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
             return;
         }
-        if (avx2) {
+        if (avx2 && iq256_supported(f.gu_type)) {
             iq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
             return;
         }

src/kernels/cuda/elementwise.cu

@@ -251,6 +251,23 @@ __global__ void copy_rows_from_mapped_kernel(float4* __restrict__ dst, const vol
         for (int64_t i = threadIdx.x; i < row4; i += blockDim.x) d[i] = const_cast<const float4*>(sr)[i];
     }
 }
+namespace {
+__global__ void scatter_rows_kernel(const float4* __restrict__ src, float4* dst, const int32_t* __restrict__ rows,
+                                    int64_t w4) {
+    const int64_t r = blockIdx.x;
+    const float4* s = src + r * w4;
+    float4* d = dst + (int64_t) rows[r] * w4;
+    for (int64_t i = threadIdx.x; i < w4; i += blockDim.x) d[i] = s[i];
+}
+}  // namespace
+void scatter_rows_f32(const float* src, float* dst, const int32_t* rows, int64_t n, int64_t width, void* stream) {
+    if (n <= 0) return;
+    if ((width & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {
+        std::fprintf(stderr, "scatter_rows_f32: width must be a multiple of 4 and both pointers 16-byte aligned\n");
+        std::exit(1);
+    }
+    scatter_rows_kernel<<<(unsigned) n, 128, 0, (cudaStream_t) stream>>>((const float4*) src, (float4*) dst, rows, width / 4);
+}
 void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
                            const int32_t* count, void* stream) {
     if (rows <= 0) return;

src/kernels/cuda/fused_gr.cu

@@ -196,7 +196,7 @@ __global__ void __launch_bounds__(THREADS) gr_norm_multi_kernel(GrMulti m) {
 // which fits all eight tokens in one 40 KiB launch and stages 160 chunks of 8 (5 per lane, half the registers held
 // for the weight prefetch); smaller tiles raise how many blocks share an SM (the 41-block grid), e.g. three blocks
 // of four tokens instead of one on sm_75.
-template <int TILEV>
+template <int TILEV, int MAX_T = kFusedGrMaxT>
 __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) {
     constexpr int TQ = TILEV / 8 / 32;      // uint4 weight chunks per lane per tile
     extern __shared__ __align__(16) float tile[];   // [T][TILEV]
@@ -207,9 +207,9 @@ __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) {
     const bool active = !(inject_block && (m.a[0].w_inject == nullptr || warp >= HC));
     const uint16_t* wrow = (inject_block ? m.a[0].w_inject : m.a[0].w_down) + (size_t) (active ? row : 0) * D;
     const uint4* w4 = reinterpret_cast<const uint4*>(wrow);
-    float acc[kFusedGrMaxT];
+    float acc[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) acc[k] = 0.0f;
+    for (int k = 0; k < MAX_T; ++k) acc[k] = 0.0f;
     for (int base = 0; base < D; base += TILEV) {
         uint4 wv[TQ];
         if (active) {
@@ -229,17 +229,17 @@ __global__ void __launch_bounds__(THREADS) gr_down_multi_kernel(GrMulti m) {
         for (int q = 0; q < TQ; ++q) {
             const int j = lane + 32 * q;
 #pragma unroll
-            for (int k = 0; k < kFusedGrMaxT; ++k)
+            for (int k = 0; k < MAX_T; ++k)
                 if (k < T) acc[k] += dot8(wv[q], tile + k * TILEV + j * 8);
         }
     }
     if (!active) return;
-    float s[kFusedGrMaxT];
+    float s[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
+    for (int k = 0; k < MAX_T; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
     // lane k writes token k (every lane holds every sum after the xor reduction)
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) {
+    for (int k = 0; k < MAX_T; ++k) {
         if (k >= T || lane != k) continue;
         if (inject_block) {
             m.a[k].inject_out[row] = s[k];
@@ -616,6 +616,7 @@ __device__ __forceinline__ void stage_htile(const GrMulti& m, int T, int h, floa
     }
 }
 
+template <int MAX_T = kFusedGrMaxT>
 __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
     extern __shared__ __align__(16) float4 hbuf[];      // 2 buffers x [T][2][160] float4
     const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
@@ -626,9 +627,9 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
     const uint16_t* wrow = (inject_block ? m.a[0].w_inject : m.a[0].w_down) + (size_t) (active ? row : 0) * D;
     const uint4* w4 = reinterpret_cast<const uint4*>(wrow);
     const size_t buf_f4 = (size_t) T * (H_TILE / 4);    // float4 per buffer (the second one follows the first)
-    float acc[kFusedGrMaxT];
+    float acc[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) acc[k] = 0.0f;
+    for (int k = 0; k < MAX_T; ++k) acc[k] = 0.0f;
     uint4 wv[HQ], wnext[HQ];
     if (active) {
 #pragma unroll
@@ -653,7 +654,7 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
             for (int q = 0; q < HQ; ++q) {
                 const int j = lane + 32 * q;
 #pragma unroll
-                for (int k = 0; k < kFusedGrMaxT; ++k) {
+                for (int k = 0; k < MAX_T; ++k) {
                     if (k < T) {
                         const float4* pk = cur + (size_t) k * (H_TILE / 4);
                         acc[k] += dot8v(wv[q], pk[j], pk[H_TILE / 8 + j]);
@@ -670,12 +671,12 @@ __global__ void __launch_bounds__(THREADS) gr_down_staged_kernel(GrMulti m) {
         }
     }
     if (!active) return;
-    float s[kFusedGrMaxT];
+    float s[MAX_T];
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
+    for (int k = 0; k < MAX_T; ++k) s[k] = k < T ? warp_sum(acc[k]) : 0.0f;
     // lane k writes token k (every lane holds every sum after the xor reduction)
 #pragma unroll
-    for (int k = 0; k < kFusedGrMaxT; ++k) {
+    for (int k = 0; k < MAX_T; ++k) {
         if (k >= T || lane != k) continue;
         if (inject_block) {
             m.a[k].inject_out[row] = s[k];
@@ -729,7 +730,7 @@ int down_chunk(bool staged, int* tile_out) {
         }
         int want_staged = (int) (2 * kFusedGrMaxT * H_TILE * sizeof(float));
         if (optin > 0 && want_staged > optin) want_staged = optin;
-        cudaFuncSetAttribute(gr_down_staged_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, want_staged);
+        cudaFuncSetAttribute(gr_down_staged_kernel<>, cudaFuncAttributeMaxDynamicSharedMemorySize, want_staged);
         cudaGetLastError();      // drop any error the attempt left behind
         // The opt-in is a promise a pre-Volta card does not keep: an sm_60 answers 65536 and accepts the
         // cudaFuncSetAttribute for 61440 B, then fails the LAUNCH with "invalid argument".  What such a card
@@ -778,9 +779,21 @@ void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long
             for (int k = 0; k < ct; ++k) c.a[k] = m.a[c0 + k];
         }
         const size_t smem = (size_t) ct * per_tok;
-        if (staged) gr_down_staged_kernel<<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
-        else if (tv == 1280) gr_down_multi_kernel<1280><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
-        else gr_down_multi_kernel<2560><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        // #443, opt-in STRATA_GR_DOWN_MAX4=1: launches of up to 4 tokens hold 4 tokens' sums per thread instead of
+        // kFusedGrMaxT - the same tile, block size, accumulation order and plain/split/staged path, so the same bits
+        // (read once: this runs per layer when decode is not captured)
+        static const bool max4_on = [] { const char* v = std::getenv("STRATA_GR_DOWN_MAX4"); return v && std::atoi(v) != 0; }();
+        const bool max4 = max4_on && ct <= 4;
+        if (staged) {
+            if (max4) gr_down_staged_kernel<4><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+            else gr_down_staged_kernel<><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        } else if (tv == 1280) {
+            if (max4) gr_down_multi_kernel<1280, 4><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+            else gr_down_multi_kernel<1280><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        } else {
+            if (max4) gr_down_multi_kernel<2560, 4><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+            else gr_down_multi_kernel<2560><<<DOWN_BLOCKS + 1, THREADS, smem, st>>>(c);
+        }
     }
     if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0 + 1, (void*) st);
     gr_up_multi_kernel<<<UPM_BLOCKS, THREADS, 0, st>>>(m);

src/kernels/cuda/iq_kernels.cu

@@ -295,7 +295,7 @@ __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(const void* __restrict__ vb
 // Q4_K / Q5_K gate/up and Q5_1 / Q8_0 down: llama.cpp's vec_dot_*_q8_1 (vecdotq.cuh, VDR 2 each), transcribed; the
 // Q5_1 min term is the one departure (below).  Via eddoursul/Strata 8029fa9 (iq_dot.cuh) and #255 (Q8_0,
 // gopinath87607), which agree with llama.cpp and with each other.
-constexpr int VDR_Q4_K = 2, VDR_Q5_K = 2, VDR_Q5_1 = 2, VDR_Q8_0 = 2;
+constexpr int VDR_Q4_K = 2, VDR_Q5_K = 2, VDR_Q5_1 = 2, VDR_Q5_0 = 2, VDR_Q8_0 = 2;
 
 __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(const int* __restrict__ v, const int* __restrict__ u,
                                                              const uint8_t* __restrict__ sc, const uint8_t* __restrict__ m,
@@ -405,6 +405,36 @@ __device__ __forceinline__ float vec_dot_q5_K_q8_1(const void* __restrict__ vbq,
 // of the ORIGINAL activations.  That is ggml-cpu's convention (its q8_1 `s` is d * sum(q)) and the one the K-quant
 // mins above use; the scaled and the min term then see the same activation (eddoursul/Strata measured 1.1-1.2%
 // against 1.9% relative error per expert).  Result: sumi * (d5 * d8) + sumu * (m5 * d8).
+// Q5_0: llama.cpp's integer chain verbatim (vecdotq.cuh vec_dot_q5_0_q8_1_impl) -- symmetric (one delta, no
+// learned min), so unlike Q5_1 there is no min-term rounding choice to make: the constant -16 per-weight offset
+// is accounted for via the q8_1 block's `ds.y` (sum of the ORIGINAL activations), exactly as upstream.
+__device__ __forceinline__ float vec_dot_q5_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,
+                                                   const int& kbx, const int& iqs) {
+    const block_q5_0* bq5_0 = (const block_q5_0*) vbq + kbx;
+    int sumi = 0;
+#pragma unroll
+    for (int i = 0; i < VDR_Q5_0; ++i) {
+        const int vl = get_int_b2(bq5_0->qs, iqs + i);
+        const int vh = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i));
+        const int u0 = get_int_b4(bq8_1->qs, iqs + i), u1 = get_int_b4(bq8_1->qs, iqs + i + QI5_0);
+        int vi0 = (vl >> 0) & 0x0F0F0F0F;
+        vi0 |= (vh << 4) & 0x00000010;
+        vi0 |= (vh << 11) & 0x00001000;
+        vi0 |= (vh << 18) & 0x00100000;
+        vi0 |= (vh << 25) & 0x10000000;
+        sumi = ggml_cuda_dp4a(vi0, u0, sumi);
+        int vi1 = (vl >> 4) & 0x0F0F0F0F;
+        vi1 |= (vh >> 12) & 0x00000010;
+        vi1 |= (vh >> 5) & 0x00001000;
+        vi1 |= (vh << 2) & 0x00100000;
+        vi1 |= (vh << 9) & 0x10000000;
+        sumi = ggml_cuda_dp4a(vi1, u1, sumi);
+    }
+    const float d5 = __half2float(bq5_0->d);
+    const float d8 = __low2float(bq8_1->ds);
+    const float s8 = __high2float(bq8_1->ds);
+    return d5 * (sumi * d8 - (16.0f * VDR_Q5_0 / QI5_0) * s8);
+}
 __device__ __forceinline__ float vec_dot_q5_1_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,
                                                    const int& kbx, const int& iqs) {
     const block_q5_1* bq5_1 = (const block_q5_1*) vbq + kbx;
@@ -470,14 +500,16 @@ template<> struct Fmt<13> { static constexpr int qk = 256, ipb = QI5_K / VDR_Q5_
     __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_K_q8_1(v, y, kbx, iqs); } };
 template<> struct Fmt<7> { static constexpr int qk = 32, ipb = QI5_1 / VDR_Q5_1, step = VDR_Q5_1;
     __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_1_q8_1(v, y, kbx, iqs); } };
+template<> struct Fmt<6> { static constexpr int qk = 32, ipb = QI5_0 / VDR_Q5_0, step = VDR_Q5_0;
+    __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q5_0_q8_1(v, y, kbx, iqs); } };
 template<> struct Fmt<8> { static constexpr int qk = 32, ipb = QI8_0 / VDR_Q8_0, step = VDR_Q8_0;
     __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_q8_0_q8_1(v, y, kbx, iqs); } };
 
 // The formats of each role, one list each so a type cannot be in one switch and missing from another.  Every
 // entry is a kernel template for each CUDA architecture of the build, hence two lists rather than one.
-#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(8)
-#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(8)
-#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(8)
+#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(6) X(8)
+#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(8)
+#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(6) X(8)
 
 __device__ __forceinline__ float warp_sum(float v) {
 #pragma unroll
@@ -854,6 +886,12 @@ __global__ void __launch_bounds__(128) mmvq_multi_kernel(const uint8_t* __restri
 // ---------------------------------------------------------------- grouped native experts
 constexpr int GU_ROWS = 8;     // rows per block (one warp each)
 
+// Group g is computed by block row g % gridDim.y: rows y, y + gridDim.y, ... < *n_groups.  The callers size the
+// launch for the most groups a call can have (cap: every entry its own expert), but the count is only known on the
+// device, and a verify window's PCIe call usually has no group at all: with one block row per possible group every
+// unused one was cap x (2 n_ff / 8 + n_embd / 8) blocks that started only to return.  A smaller gridDim.y strides
+// instead.  Each group's rows are computed by the same warp code in the same order either way, so the results do
+// not depend on gridDim.y (gridDim.y = cap_groups is the one-row-per-group launch).
 template<int TG>
 __global__ void __launch_bounds__(256) native_gu_kernel(const unsigned long long* __restrict__ grp_ptr,
                                                         const int32_t* __restrict__ grp_start,
@@ -861,20 +899,21 @@ __global__ void __launch_bounds__(256) native_gu_kernel(const unsigned long long
                                                         const int32_t* __restrict__ ent_tok,
                                                         const block_q8_1* __restrict__ xq, NativeExpertLayout L,
                                                         float* __restrict__ gate, float* __restrict__ up) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int row = blockIdx.x * GU_ROWS + warp;             // 0 .. 2*n_ff
     if (row >= 2 * L.n_ff) return;
     const bool is_up = row >= L.n_ff;
     const int r = is_up ? row - (int) L.n_ff : row;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
+    const size_t off = (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
     const int nb = (int) (L.n_embd / Fmt<TG>::qk), xb = (int) (L.n_embd / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
-    for (int e = e0; e < e1; ++e) {
-        const float s = row_dot<TG>(wr, xq + (size_t) ent_tok[e] * xb, nb, lane);
-        if (lane == 0) (is_up ? up : gate)[(size_t) e * L.n_ff + r] = s;
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; ++e) {
+            const float s = row_dot<TG>(wr, xq + (size_t) ent_tok[e] * xb, nb, lane);
+            if (lane == 0) (is_up ? up : gate)[(size_t) e * L.n_ff + r] = s;
+        }
     }
 }
 
@@ -890,28 +929,29 @@ __global__ void __launch_bounds__(256) native_gu_multi_kernel(const unsigned lon
                                                               const int32_t* __restrict__ ent_tok,
                                                               const block_q8_1* __restrict__ xq, NativeExpertLayout L,
                                                               float* __restrict__ gate, float* __restrict__ up) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int row = blockIdx.x * GU_ROWS + warp;             // 0 .. 2*n_ff
     if (row >= 2 * L.n_ff) return;
     const bool is_up = row >= L.n_ff;
     const int r = is_up ? row - (int) L.n_ff : row;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
+    const size_t w_off = (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
     const int nb = (int) (L.n_embd / Fmt<TG>::qk), xb = (int) (L.n_embd / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
     float* dst = is_up ? up : gate;
-    for (int e = e0; e < e1; e += GRP_NC) {
-        const int n = min(GRP_NC, e1 - e);
-        int off[GRP_NC];
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {   // the group stride, as native_gu_kernel
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + w_off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; e += GRP_NC) {
+            const int n = min(GRP_NC, e1 - e);
+            int off[GRP_NC];
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c) off[c] = ent_tok[e + min(c, n - 1)] * xb;
-        float s[GRP_NC];
-        row_dot_multi<TG, GRP_NC>(wr, xq, off, n, nb, lane, s);
+            for (int c = 0; c < GRP_NC; ++c) off[c] = ent_tok[e + min(c, n - 1)] * xb;
+            float s[GRP_NC];
+            row_dot_multi<TG, GRP_NC>(wr, xq, off, n, nb, lane, s);
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c)
-            if (c < n && lane == c) dst[(size_t) (e + c) * L.n_ff + r] = s[c];
+            for (int c = 0; c < GRP_NC; ++c)
+                if (c < n && lane == c) dst[(size_t) (e + c) * L.n_ff + r] = s[c];
+        }
     }
 }
 
@@ -930,18 +970,19 @@ __global__ void __launch_bounds__(256) native_down_kernel(const unsigned long lo
                                                           const int32_t* __restrict__ ent_dst,
                                                           const block_q8_1* __restrict__ hq, NativeExpertLayout L,
                                                           float* __restrict__ out) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int r = blockIdx.x * 8 + warp;
     if (r >= L.n_embd) return;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + L.down_off + (size_t) r * L.d_row;
+    const size_t off = L.down_off + (size_t) r * L.d_row;
     const int nb = (int) (L.n_ff / Fmt<TD>::qk), hb = (int) (L.n_ff / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
-    for (int e = e0; e < e1; ++e) {
-        const float s = row_dot<TD>(wr, hq + (size_t) e * hb, nb, lane);
-        if (lane == 0) out[(size_t) ent_dst[e] * L.n_embd + r] = s;
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {   // as native_gu_kernel
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; ++e) {
+            const float s = row_dot<TD>(wr, hq + (size_t) e * hb, nb, lane);
+            if (lane == 0) out[(size_t) ent_dst[e] * L.n_embd + r] = s;
+        }
     }
 }
 
@@ -953,33 +994,32 @@ __global__ void __launch_bounds__(256) native_down_multi_kernel(const unsigned l
                                                                 const int32_t* __restrict__ ent_dst,
                                                                 const block_q8_1* __restrict__ hq, NativeExpertLayout L,
                                                                 float* __restrict__ out) {
-    const int g = blockIdx.y;
-    if (g >= *n_groups) return;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int r = blockIdx.x * 8 + warp;
     if (r >= L.n_embd) return;
-    const uint8_t* blob = (const uint8_t*) grp_ptr[g];
-    const uint8_t* wr = blob + L.down_off + (size_t) r * L.d_row;
+    const size_t w_off = L.down_off + (size_t) r * L.d_row;
     const int nb = (int) (L.n_ff / Fmt<TD>::qk), hb = (int) (L.n_ff / 32);
-    const int e0 = grp_start[g], e1 = grp_start[g + 1];
-    for (int e = e0; e < e1; e += GRP_NC) {
-        const int n = min(GRP_NC, e1 - e);
-        int off[GRP_NC];
+    const int ng = *n_groups;
+    for (int g = blockIdx.y; g < ng; g += gridDim.y) {   // the group stride, as native_gu_kernel
+        const uint8_t* wr = (const uint8_t*) grp_ptr[g] + w_off;
+        const int e0 = grp_start[g], e1 = grp_start[g + 1];
+        for (int e = e0; e < e1; e += GRP_NC) {
+            const int n = min(GRP_NC, e1 - e);
+            int off[GRP_NC];
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c) off[c] = (e + min(c, n - 1)) * hb;
-        float s[GRP_NC];
-        row_dot_multi<TD, GRP_NC>(wr, hq, off, n, nb, lane, s);
+            for (int c = 0; c < GRP_NC; ++c) off[c] = (e + min(c, n - 1)) * hb;
+            float s[GRP_NC];
+            row_dot_multi<TD, GRP_NC>(wr, hq, off, n, nb, lane, s);
 #pragma unroll
-        for (int c = 0; c < GRP_NC; ++c)
-            if (c < n && lane == c) out[(size_t) ent_dst[e + c] * L.n_embd + r] = s[c];
+            for (int c = 0; c < GRP_NC; ++c)
+                if (c < n && lane == c) out[(size_t) ent_dst[e + c] * L.n_embd + r] = s[c];
+        }
     }
 }
 
 // ---------------------------------------------------------------- q8_1 (quantize.cu)
-__global__ void quantize_q8_1_kernel(const float* __restrict__ x, block_q8_1* __restrict__ y, long long n) {
-    const long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x;
-    if (i >= n) return;
-    const float xi = x[i];
+// value i of a q8_1 row set (the warp holds block i / 32, lane = i % 32)
+__device__ __forceinline__ void q8_1_store(const float xi, block_q8_1* __restrict__ y, const long long i) {
     float amax = fabsf(xi), sum = xi;
 #pragma unroll
     for (int o = 16; o > 0; o >>= 1) {
@@ -993,6 +1033,40 @@ __global__ void quantize_q8_1_kernel(const float* __restrict__ x, block_q8_1* __
     if (iqs == 0) y[ib].ds = make_half2(d, sum);
 }
 
+__global__ void quantize_q8_1_kernel(const float* __restrict__ x, block_q8_1* __restrict__ y, long long n) {
+    const long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= n) return;
+    q8_1_store(x[i], y, i);
+}
+
+// swiglu_entries_kernel and quantize_q8_1_kernel in one pass, over the call's own entries only:
+// [grp_start[0], grp_start[*n_groups]) (a call's entries are contiguous; the verify window's PCIe call starts after
+// its VRAM call) instead of all cap_entries twice - nothing reads the others: the down kernel reads its groups'.
+// n_ff is a multiple of 32, so a warp is one q8_1 block and the bounds are warp-uniform.  The values are the two
+// kernels': the SwiGLU product is rounded on its own - it must not contract into the first add of q8_1_store's block
+// sum, as it could not when it went through memory (CUDA: __fmul_rn; HIP's __fmul_rn is a plain product, so there
+// the product is written here under contract(off)) - then q8_1_store unchanged: the same blocks, bit for bit.
+__global__ void __launch_bounds__(256) swiglu_q8_1_entries_kernel(const float* __restrict__ gate,
+                                                                  const float* __restrict__ up,
+                                                                  const int32_t* __restrict__ grp_start,
+                                                                  const int32_t* __restrict__ n_groups, int n_ff,
+                                                                  block_q8_1* __restrict__ hq) {
+#if defined(__HIPCC__)
+#pragma clang fp contract(off)
+#endif
+    const long long lo = (long long) grp_start[0] * n_ff, hi = (long long) grp_start[*n_groups] * n_ff;
+    for (long long i = lo + (long long) blockIdx.x * blockDim.x + threadIdx.x; i < hi;
+         i += (long long) gridDim.x * blockDim.x) {
+        const float g = gate[i];
+#if defined(__HIPCC__)
+        const float h = (g / (1.0f + __expf(-g))) * up[i];
+#else
+        const float h = __fmul_rn(g / (1.0f + __expf(-g)), up[i]);
+#endif
+        q8_1_store(h, hq, i);
+    }
+}
+
 // ---------------------------------------------------------------- dequant (dequantize.cuh)
 template<typename dst_t> __device__ __forceinline__ dst_t cvt(float v);
 template<> __device__ __forceinline__ float cvt<float>(float v) { return v; }
@@ -1194,6 +1268,23 @@ __device__ void dq_q5_k(const void* vx, int64_t ibs, dst_t* yy, int tid) {
     }
 }
 template<typename dst_t>
+__device__ void dq_q5_0(const void* vx, int64_t ibs, dst_t* yy, int tid) {
+    // symmetric: one delta (no min), the high bit folded in then centered by the format's fixed -16 offset
+    const block_q5_0* x = (const block_q5_0*) vx + ibs * (QK_K / QK5_0);
+    const int ib = tid % 8, il = tid / 8;
+    const float d = __half2float(x[ib].d);
+    uint32_t qh;
+    memcpy(&qh, x[ib].qh, sizeof(qh));
+    dst_t* y = yy + 32 * ib;
+    for (int j = 0; j < 4; ++j) {
+        const int iqs = 4 * il + j;
+        const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
+        const int xh_1 = ((qh >> (iqs + 12))) & 0x10;
+        y[iqs] = cvt<dst_t>(((float) ((x[ib].qs[iqs] & 0xf) | xh_0) - 16.0f) * d);
+        y[iqs + 16] = cvt<dst_t>(((float) ((x[ib].qs[iqs] >> 4) | xh_1) - 16.0f) * d);
+    }
+}
+template<typename dst_t>
 __device__ void dq_q5_1(const void* vx, int64_t ibs, dst_t* yy, int tid) {
     const block_q5_1* x = (const block_q5_1*) vx + ibs * (QK_K / QK5_1);
     const int ib = tid % 8, il = tid / 8;
@@ -1243,6 +1334,7 @@ __device__ __forceinline__ void dq_dispatch(int ty, const void* vx, int64_t ibs,
         case 12: dq_q4_k(vx, ibs, y, tid); break;
         case 13: dq_q5_k(vx, ibs, y, tid); break;
         case 7: dq_q5_1(vx, ibs, y, tid); break;
+        case 6: dq_q5_0(vx, ibs, y, tid); break;
         case 8: dq_q8_0(vx, ibs, y, tid); break;
         case 30: dq_bf16(vx, ibs, y, tid); break;
         default: break;
@@ -1267,7 +1359,7 @@ __global__ void dequant_gu_kernel(int ty, const void* __restrict__ gate, const v
 // the types dq_dispatch dequantizes
 bool is_iq(int t) {
     return t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||
-           t == 12 || t == 13 || t == 7 || t == 8;
+           t == 12 || t == 13 || t == 7 || t == 6 || t == 8;
 }
 // values per block of the types the grouped expert kernels take (0 = none)
 int gu_qk(int t) {
@@ -1347,6 +1439,7 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {
         case 12: return (size_t) (n / 256) * sizeof(block_q4_K);
         case 13: return (size_t) (n / 256) * sizeof(block_q5_K);
         case 7: return (size_t) (n / 32) * sizeof(block_q5_1);
+        case 6: return (size_t) (n / 32) * sizeof(block_q5_0);
         case 8: return (size_t) (n / 32) * sizeof(block_q8_0);
         case 30: return (size_t) n * 2;   // BF16: the token embedding only (iq_embed_rows, iq_dequant_f32)
         default: return 0;
@@ -1439,10 +1532,18 @@ size_t native_expert_scratch_bytes(int64_t cap, int64_t n_ff) {
     return 3 * ((f + 255) & ~(size_t) 255) + (((size_t) cap * (size_t) (n_ff / 32) * sizeof(block_q8_1) + 255) & ~(size_t) 255);
 }
 
+namespace {
+bool g_grouped_v1 = env_on("STRATA_GROUPED_V1");
+}  // namespace
+
+void native_grouped_set_v1(bool v1) { g_grouped_v1 = v1; }
+
 void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long* grp_ptr, const int32_t* grp_start,
                            const int32_t* n_groups, const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups,
-                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream) {
+                           int64_t cap_entries, const void* x_q8_1, void* scratch, float* out, void* stream,
+                           int64_t grid_groups) {
     if (cap_groups <= 0 || cap_entries <= 0) return;
+    if (L.n_ff % 32 != 0) { std::fprintf(stderr, "native_expert_grouped: n_ff %lld\n", (long long) L.n_ff); std::exit(1); }
     cudaStream_t s = (cudaStream_t) stream;
     const size_t f = (size_t) cap_entries * (size_t) L.n_ff * sizeof(float), fa = (f + 255) & ~(size_t) 255;
     float* gate = (float*) scratch;
@@ -1450,7 +1551,9 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
     float* h = (float*) ((uint8_t*) scratch + 2 * fa);
     block_q8_1* hq = (block_q8_1*) ((uint8_t*) scratch + 3 * fa);
     const auto* X = (const block_q8_1*) x_q8_1;
-    const dim3 ggu((unsigned) ((2 * L.n_ff + GU_ROWS - 1) / GU_ROWS), (unsigned) cap_groups);
+    const bool v1 = g_grouped_v1;
+    const int64_t gy = (v1 || grid_groups <= 0 || grid_groups > cap_groups) ? cap_groups : grid_groups;
+    const dim3 ggu((unsigned) ((2 * L.n_ff + GU_ROWS - 1) / GU_ROWS), (unsigned) gy);
     switch (L.gu_type) {
 #define STRATA_GU(T) case T: launch_gu<T>(ggu, s, grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up); break;
         STRATA_GU_FMTS(STRATA_GU)
@@ -1459,9 +1562,15 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
     }
     check("native_expert_grouped/gu");
     const long long nh = (long long) cap_entries * L.n_ff;
-    swiglu_entries_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(gate, up, h, nh);
-    quantize_q8_1_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(h, hq, nh);
-    const dim3 gd((unsigned) ((L.n_embd + 7) / 8), (unsigned) cap_groups);
+    if (v1) {
+        swiglu_entries_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(gate, up, h, nh);
+        quantize_q8_1_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(h, hq, nh);
+    } else {
+        swiglu_q8_1_entries_kernel<<<(unsigned) ((nh + 255) / 256), 256, 0, s>>>(gate, up, grp_start, n_groups,
+                                                                                (int) L.n_ff, hq);
+    }
+    check("native_expert_grouped/swiglu");
+    const dim3 gd((unsigned) ((L.n_embd + 7) / 8), (unsigned) gy);
     switch (L.d_type) {
 #define STRATA_DOWN(T) case T: launch_down<T>(gd, s, grp_ptr, grp_start, n_groups, ent_dst, hq, L, out); break;
         STRATA_D_FMTS(STRATA_DOWN)

src/kernels/cuda/qsa_prompt_attn.cu

@@ -78,18 +78,22 @@ __device__ __forceinline__ uint32_t pack_h2(float lo_k, float hi_k) {   // eleme
 // KV_MODE 1: int8 codes + fp16 scale per 64 values. KV_MODE 0: fp16 values (scales 1).
 // KV_MODE 3 (hybrid K8V4): K as mode 1, V as mode 0 - the row's q4_0 blocks are dequantized to fp16 at
 // gather, so everything downstream of the load is the mode-0 V path; the caller un-rotates the output.
+// KV_MODE 4 (Q4_0 K and V, `--kv q4_0`): mode 1 with a scale per 32 values - each q4_0 block's codes enter as exact
+// int8 (code - 8) and its fp16 scale is applied in FP32, as mode 1's are; the caller rotates q and un-rotates the
+// output (kv_q4.hpp).
 template <int KV_MODE>
 struct Smem {
     using KElem = typename std::conditional<KV_MODE == 0, __half, int8_t>::type;
-    using VElem = typename std::conditional<KV_MODE == 1, int8_t, __half>::type;
+    using VElem = typename std::conditional<KV_MODE == 1 || KV_MODE == 4, int8_t, __half>::type;
     static constexpr int KROW = KV_MODE == 0 ? HD + 8 : HD + 16;   // elements; 16-byte aligned rows, banks spread
-    static constexpr int VROW = KV_MODE == 1 ? HD + 16 : HD + 8;
+    static constexpr int VROW = (KV_MODE == 1 || KV_MODE == 4) ? HD + 16 : HD + 8;
+    static constexpr int NG = KV_MODE == 4 ? HD / QK4_0 : 4;      // scale groups per row (q4_0: 8 of 32, else 4 of 64)
     __half qh[16][QS];
     __half ql[16][QS];
     KElem k[CH][KROW];
     VElem v[CH][VROW];
-    float ks[CH][4];
-    float vs[CH][4];
+    float ks[CH][NG];
+    float vs[CH][NG];
     float s[16][CH + 1];
     float qmax[THREADS / 32];
     float alpha[16];
@@ -154,7 +158,39 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
         __syncthreads();   // rows ready; the previous chunk's p.v is done with k, v, s
         // gather the chunk's K and V rows (16-byte pieces; K8V4's V as q4_0 blocks dequantized to fp16)
         // and their scales
-        {
+        if constexpr (KV_MODE == 4) {
+            // q4_0 K and V: one (K or V, cell, block) per item - the block's 16 code bytes (2-byte aligned: eight
+            // 16-bit loads) become 32 exact int8 codes (element j in the low nibble of byte j, j + 16 in the high
+            // one), its fp16 scale goes to ks/vs
+            constexpr int BLKS = HD / QK4_0;
+            constexpr int BYTES = BLKS * (int) sizeof(block_q4_0);
+            for (int i = t; i < 2 * CH * BLKS; i += THREADS) {
+                const int kv = i / (CH * BLKS), rem = i % (CH * BLKS), c = rem / BLKS, b = rem % BLKS;
+                const long long r = S.row[c];
+                uint4 lo = make_uint4(0, 0, 0, 0), hi = make_uint4(0, 0, 0, 0);
+                float d = 0.0f;
+                if (r >= 0) {
+                    const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((kv == 0 ? p.k_q4 : p.v_q4) + r * BYTES) + b;
+                    d = __half2float(__ushort_as_half(__ldg(&blk->d)));
+                    const uint16_t* q16 = reinterpret_cast<const uint16_t*>(blk->qs);
+                    uint32_t w[4];
+#pragma unroll
+                    for (int j = 0; j < 4; ++j)
+                        w[j] = (uint32_t) __ldg(q16 + 2 * j) | ((uint32_t) __ldg(q16 + 2 * j + 1) << 16);
+                    // per byte: the low nibble minus 8 is element j, the high one minus 8 element j + 16
+                    lo = make_uint4(__vsub4(w[0] & 0x0F0F0F0Fu, 0x08080808u), __vsub4(w[1] & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4(w[2] & 0x0F0F0F0Fu, 0x08080808u), __vsub4(w[3] & 0x0F0F0F0Fu, 0x08080808u));
+                    hi = make_uint4(__vsub4((w[0] >> 4) & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4((w[1] >> 4) & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4((w[2] >> 4) & 0x0F0F0F0Fu, 0x08080808u),
+                                    __vsub4((w[3] >> 4) & 0x0F0F0F0Fu, 0x08080808u));
+                }
+                int8_t* dst = kv == 0 ? &S.k[c][b * QK4_0] : &S.v[c][b * QK4_0];
+                reinterpret_cast<uint4*>(dst)[0] = lo;
+                reinterpret_cast<uint4*>(dst)[1] = hi;
+                (kv == 0 ? S.ks : S.vs)[c][b] = d;
+            }
+        } else {
             constexpr int KPIECES = HD * (int) sizeof(typename Smem<KV_MODE>::KElem) / 16;   // per K row
             for (int i = t; i < CH * KPIECES; i += THREADS) {
                 const int c = i / KPIECES, pc = i % KPIECES;
@@ -222,17 +258,18 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             }
         }
         __syncthreads();
-        // scores: warp w takes cells 8w..8w+7 (one n-tile) over all 256 dims, per 64-dim scale group
+        // scores: warp w takes cells 8w..8w+7 (one n-tile) over all 256 dims, per scale group (64 dims; q4_0's 32)
+        constexpr int NG = Smem<KV_MODE>::NG, KPG = HD / 16 / NG;   // groups per row, 16-dim MMA steps per group
 #pragma unroll
         for (int nt = 0; nt < CH / 32; ++nt) {
             const int cb = (warp + 4 * nt) * 8;
             float sc[4] = {0.f, 0.f, 0.f, 0.f};
 #pragma unroll
-            for (int g = 0; g < 4; ++g) {
+            for (int g = 0; g < NG; ++g) {
                 float tg[4] = {0.f, 0.f, 0.f, 0.f};
 #pragma unroll
-                for (int kk = 0; kk < 4; ++kk) {
-                    const int k0 = (g * 4 + kk) * 16;
+                for (int kk = 0; kk < KPG; ++kk) {
+                    const int k0 = (g * KPG + kk) * 16;
                     uint32_t ah[4], al[4], b[2];
                     ah[0] = *reinterpret_cast<const uint32_t*>(&S.qh[gid][k0 + 2 * tig]);
                     ah[1] = *reinterpret_cast<const uint32_t*>(&S.qh[gid + 8][k0 + 2 * tig]);
@@ -296,24 +333,31 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             }
         }
         __syncthreads();
-        // p.v: warp w owns dims [64w, 64w+64), which is int8 scale group w. The scale is folded into p relative to
-        // the chunk's largest, times 2^14 (p' <= 2^14: its lo half stays out of FP16's subnormal range); the chunk's
-        // sum is then added to the running one in FP32 with the factor taken back out
+        // p.v: warp w owns dims [64w, 64w+64), which is int8 scale group w (q4_0: groups 2w and 2w+1, four n-tiles
+        // each). A group's scale is folded into p relative to the chunk's largest magnitude (q4_0's scales are signed:
+        // ggml's d = max / -8), times 2^14 (|p'| <= 2^14: inside FP16's range, its lo half out of the subnormals); the
+        // chunk's sum is then added to the running one in FP32 with the factor taken back out
         {
-            float vmax = 0.0f;
-#pragma unroll
-            for (int c = lane; c < CH; c += 32) vmax = fmaxf(vmax, S.vs[c][warp]);
-#pragma unroll
-            for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));
-            const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f, vdown = vmax * (1.0f / 16384.0f);
+            constexpr int GPW = NG / 4, JPG = 8 / GPW;   // scale groups per warp, n-tiles per group
+            float vdown_g[GPW];
             float tmp[8][4];
 #pragma unroll
             for (int j = 0; j < 8; ++j) tmp[j][0] = tmp[j][1] = tmp[j][2] = tmp[j][3] = 0.0f;
+#pragma unroll
+            for (int gi = 0; gi < GPW; ++gi) {
+            const int vg = warp * GPW + gi;
+            float vmax = 0.0f;
+#pragma unroll
+            for (int c = lane; c < CH; c += 32) vmax = fmaxf(vmax, fabsf(S.vs[c][vg]));
+#pragma unroll
+            for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));
+            const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f;
+            vdown_g[gi] = vmax * (1.0f / 16384.0f);
 #pragma unroll
             for (int ks = 0; ks < CH / 16; ++ks) {
                 const int cA = ks * 16 + 2 * tig, cB = cA + 8;
-                const float w0 = S.vs[cA][warp] * vup, w1 = S.vs[cA + 1][warp] * vup, w2 = S.vs[cB][warp] * vup,
-                            w3 = S.vs[cB + 1][warp] * vup;
+                const float w0 = S.vs[cA][vg] * vup, w1 = S.vs[cA + 1][vg] * vup, w2 = S.vs[cB][vg] * vup,
+                            w3 = S.vs[cB + 1][vg] * vup;
                 const float p00 = S.s[gid][cA] * w0, p01 = S.s[gid][cA + 1] * w1;
                 const float p10 = S.s[gid + 8][cA] * w0, p11 = S.s[gid + 8][cA + 1] * w1;
                 const float p02 = S.s[gid][cB] * w2, p03 = S.s[gid][cB + 1] * w3;
@@ -332,10 +376,11 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
                     f = __half22float2(h[3]); al[3] = pack_h2(p12 - f.x, p13 - f.y);
                 }
 #pragma unroll
-                for (int j = 0; j < 8; ++j) {
+                for (int jj = 0; jj < JPG; ++jj) {
+                    const int j = gi * JPG + jj;
                     const int d = warp * 64 + j * 8 + gid;
                     uint32_t b[2];
-                    if constexpr (KV_MODE == 1) {
+                    if constexpr (KV_MODE == 1 || KV_MODE == 4) {
                         const uint32_t x0 = (uint8_t) S.v[cA][d] | ((uint32_t) (uint8_t) S.v[cA + 1][d] << 8);
                         const uint32_t x1 = (uint8_t) S.v[cB][d] | ((uint32_t) (uint8_t) S.v[cB + 1][d] << 8);
                         b[0] = i8x2_to_h2(x0);
@@ -352,9 +397,11 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
 #endif
                 }
             }
+            }
             const float a0 = S.alpha[gid], a1 = S.alpha[gid + 8];
 #pragma unroll
             for (int j = 0; j < 8; ++j) {
+                const float vdown = vdown_g[j / JPG];
                 acc[j][0] = fmaf(acc[j][0], a0, tmp[j][0] * vdown);
                 acc[j][1] = fmaf(acc[j][1], a0, tmp[j][1] * vdown);
                 acc[j][2] = fmaf(acc[j][2], a1, tmp[j][2] * vdown);
@@ -1008,10 +1055,18 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3
         return launch_wmma(q, pools, ids, steps, cap, s, attn, n_q, (cudaStream_t) stream);
     return false;
 #endif
-    if (pools.k_q4 != nullptr || s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !ids ||
-        !steps || !pools.page_table)
+    if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !ids || !steps || !pools.page_table)
         return false;
     cudaStream_t st = (cudaStream_t) stream;
+    if (pools.k_q4 != nullptr) {   // Q4_0 K and V (--kv q4_0): mode 4.  STRATA_PROMPT_ATTN_Q4=0: the old kernel (A/B)
+        static const bool q4_off = [] {
+            const char* v = std::getenv("STRATA_PROMPT_ATTN_Q4");
+            return v != nullptr && v[0] == '0';
+        }();
+        // sm_80+ only: on Turing mode 4 would run as pairs of m16n8k8 MMAs, which no parity run has checked yet
+        if (q4_off || turing || pools.v_q4 == nullptr) return false;
+        return launch<4>(q, pools, ids, steps, cap, s, attn, n_q, st);
+    }
     if (pools.k_q != nullptr && pools.v_q4 != nullptr) {   // hybrid K8V4: int8 K + dequantized-q4 V
         if (!pools.k_scale) return false;
         return launch<3>(q, pools, ids, steps, cap, s, attn, n_q, st);

src/kernels/cuda/qsa_select.cu

@@ -968,6 +968,24 @@ void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, i
     if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
 }
 
+#if !defined(__HIPCC__)
+// Only Turing has a retained model measurement for this CUDA dispatch. Other CUDA devices keep the capacity rule
+// (the RTX 5070 regression below). Cache the properties per calling thread; layer-split device switches are checked.
+static bool topk_active_turing_device() {
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess) return false;
+    static thread_local int cached_device = -1;
+    static thread_local bool turing = false;
+    if (dev != cached_device) {
+        cudaDeviceProp prop{};
+        if (cudaGetDeviceProperties(&prop, dev) != cudaSuccess) return false;
+        turing = prop.major == 7 && prop.minor == 5;
+        cached_device = dev;
+    }
+    return turing;
+}
+#endif
+
 bool qsa_block_topk_cluster(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
                             const QsaShapes& s, int32_t* ids, void* stream) {
 #if defined(__HIPCC__)
@@ -1055,10 +1073,16 @@ void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64
 #if defined(__HIPCC__)
     const bool counted = active_blocks > 0;
 #else
-    // CUDA keeps 0.1.32's capacity rule: #337's dispatch was measured on RDNA4 only, and on the RTX 5070 the 64K
-    // prompts read 1-3% slower with it
-    const bool counted = false;
-    (void) active_blocks;
+    // Turing: --max-context 262144 makes the stride 65538, even while a 131K prompt's active blocks fit in
+    // TK_T * TK_PER registers. Use the prefill bound on sm_75, keeping max_blocks as the score-row stride.
+    // Other CUDA devices keep 0.1.32's capacity rule: #337 was measured on RDNA4, and RTX 5070 64K prompts were
+    // 1-3% slower. Decode/captured graphs omit the bound and never query the device here.
+    static const bool capacity_guard = std::getenv("STRATA_TOPK_CAPACITY_GUARD") != nullptr;
+    // STRATA_TOPK_ACTIVE_ANY=1 (tests): the Turing dispatch on any CUDA card, so qsa_topk_active_parity checks it
+    // on whatever card runs the tests (the kernels are the same on every architecture)
+    static const bool any_card = [] { const char* v = std::getenv("STRATA_TOPK_ACTIVE_ANY"); return v && v[0] == '1'; }();
+    const bool counted = !capacity_guard && active_blocks > 0 && active_blocks <= max_blocks &&
+                         (any_card || topk_active_turing_device());
 #endif
     const int64_t reach = counted && active_blocks < max_blocks ? active_blocks : max_blocks;
     const int64_t fit = (int64_t) TK_T * (counted ? TK_PER_MAX : TK_PER);

src/kernels/decode_cluster_parity.cpp

@@ -58,6 +58,9 @@ template <typename T> struct Dev {
     ~Dev() { cudaFree(p); }
     void put(const std::vector<T>& h) {
         ck(cudaMemcpy(p, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "up");
+        // #548 #536: a pageable cudaMemcpy can return before its DMA lands, and the graph replays and the timing
+        // cases run on a cudaStreamNonBlocking stream that does not wait for the legacy stream - so wait here
+        ck(cudaDeviceSynchronize(), "up landed");
     }
     std::vector<T> get() const {
         std::vector<T> h(n);

src/kernels/native_expert_parity.cpp

@@ -94,7 +94,9 @@ int check_blob(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int se
             ffp[k] = ff[k].data();
         }
         cpu::native_gu_rows(f, blob.data(), a, NT, ffp, 0, (int) FF);
-        if (cpu::iq512_supported(f.gu_type)) {
+        // either multi-token kernel: IQ4_XS has an AVX-2 one and no AVX-512 one, so the gate cannot be
+        // iq512_supported alone - that would leave the format untested on every CPU.
+        if (cpu::iq512_supported(f.gu_type) || cpu::iq256_supported(f.gu_type)) {
             // ggml's own vec_dot, same Q8_K activations: the reference for both multi-token kernels
             // (float-order differences only)
             const auto* tc = ggml_get_type_traits_cpu((ggml_type) f.gu_type);
@@ -143,8 +145,9 @@ int check_blob(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int se
                             tag, us1, 2.0 * f.up_off / us1 / 1e3, usg, 2.0 * f.up_off / usg / 1e3);
                 std::printf("          gate+up %d tokens one thread: %s %.0f us vs ggml %d x %.0f us\n", NT, tag, usn, NT, usg);
             };
-            if (cpu::cpu_avx512_ok()) check("avx512", true);   // guarded: the binary runs on AVX-2 CPUs too
-            check("avx2", false);
+            // guarded: the binary runs on AVX-2 CPUs too, and IQ4_XS has no AVX-512 kernel (an empty switch)
+            if (cpu::cpu_avx512_ok() && cpu::iq512_supported(f.gu_type)) check("avx512", true);
+            if (cpu::iq256_supported(f.gu_type)) check("avx2", false);
         }
         for (int k = 0; k < NT; ++k) {
             cpu::native_quant_h(f, ff[k].data(), hq[k].data());

src/kernels/native_grouped_parity.cpp

@@ -0,0 +1,351 @@
+// src/kernels/native_grouped_parity.cpp - native_expert_grouped's group stride (grid_groups) and its fused SwiGLU +
+// q8_1 pass, against the launches before them (native_grouped_set_v1: a block row per possible group, SwiGLU and the
+// q8_1 quantization as two kernels over all cap_entries).  Synthetic blobs, no model.
+//
+//     build/native_grouped_parity           the checks (a ctest; needs the GPU)
+//     build/native_grouped_parity --bench   the checks, then v1/new timings of 48 calls in a graph (a window's layers)
+//
+// Every gate/up format with every down format; calls of 0..cap groups of 1..T entries each (an empty call is what the
+// verify window's PCIe call usually is), entries from 0 and from past 0 (the PCIe call's follow the VRAM call's),
+// scattered destinations, stale scratch, grid_groups 0 (= cap), 1, 2, 3, 4 and above cap: the output must be BITWISE
+// equal to v1's, the rows the call must not write included (both start from the same NaN pattern).
+//
+// Random bytes are valid codes for every format here (every grid index is in range); only the fp16 block scales are
+// set, small enough that the SwiGLU outputs keep a finite fp16 q8_1 scale.
+#include "strata/kernels/iq_kernels.hpp"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <numeric>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+
+int g_fail = 0;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "CUDA error in %s: %s\n", what, cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+
+template<typename T>
+T* dalloc(size_t n) {
+    T* p = nullptr;
+    ck(cudaMalloc(&p, n * sizeof(T) + 256), "malloc");
+    ck(cudaMemset(p, 0, n * sizeof(T) + 256), "memset");
+    return p;
+}
+
+const char* name_of(int t) {
+    switch (t) {
+        case 16: return "IQ2_XXS";
+        case 17: return "IQ2_XS";
+        case 18: return "IQ3_XXS";
+        case 20: return "IQ4_NL";
+        case 21: return "IQ3_S";
+        case 22: return "IQ2_S";
+        case 23: return "IQ4_XS";
+        case 29: return "IQ1_M";
+        case 42: return "Q2_0";
+        case 12: return "Q4_K";
+        case 13: return "Q5_K";
+        case 7: return "Q5_1";
+        case 8: return "Q8_0";
+        default: return "?";
+    }
+}
+int block_values(int t) { return t == 20 || t == 7 || t == 8 ? 32 : t == 42 ? 64 : 256; }
+
+// `rows` rows of `n` values of format t: random bytes, then a finite fp16 scale in every block
+std::vector<uint8_t> random_rows(int t, int64_t rows, int64_t n, std::mt19937& rng) {
+    const size_t rb = k::iq_row_bytes(t, n), bs = k::iq_row_bytes(t, block_values(t));
+    std::vector<uint8_t> w((size_t) rows * rb);
+    std::uniform_int_distribution<int> byte(0, 255), ex(2, 8), man(0, 1023), sgn(0, 1);
+    for (auto& b : w) b = (uint8_t) byte(rng);
+    for (size_t o = 0; o < w.size(); o += bs) {
+        if (t == 29) {
+            // IQ1_M: the fp16 scale is the top nibbles of its four scale words (bytes 48..55), the sign and the
+            // exponent's top bits in the last: 0x1 / 0x2 there (0x9 / 0xA negative) keeps it in 2^-11 .. 2^-3
+            const uint8_t nib = (uint8_t) ((sgn(rng) ? 0x8 : 0x0) | (1 + (byte(rng) & 1)));
+            w[o + 55] = (uint8_t) ((w[o + 55] & 0x0F) | (nib << 4));
+        } else {   // the other formats start their block with the fp16 scale: 2^-13 .. 2^-6
+            // (Q4_K / Q5_K: 2^-14 .. 2^-11, their 6-bit sub-scales multiply it by up to 63)
+            const int e = (t == 12 || t == 13) ? 1 + ex(rng) % 4 : ex(rng);
+            const uint16_t h = (uint16_t) ((sgn(rng) ? 0x8000 : 0) | (e << 10) | man(rng));
+            std::memcpy(&w[o], &h, 2);
+            if (t == 12 || t == 13 || t == 7) {   // Q4_K / Q5_K: dmin, Q5_1: m - the second fp16, as small
+                const int em = (t == 7) ? ex(rng) : 1 + ex(rng) % 4;
+                const uint16_t m = (uint16_t) ((sgn(rng) ? 0x8000 : 0) | (em << 10) | man(rng));
+                std::memcpy(&w[o + 2], &m, 2);
+            }
+        }
+    }
+    return w;
+}
+
+// one window's worth of expert calls on one format pair: blobs, T tokens' q8_1 activations, cap = T * K entries
+struct Setup {
+    k::NativeExpertLayout L;
+    int T = 0, K = 0, cap = 0, n_blobs = 0;
+    size_t slot = 0;
+    uint8_t* blobs = nullptr;
+    uint8_t* xq = nullptr;
+    uint8_t* scr = nullptr;
+    size_t scr_bytes = 0;
+    float* out = nullptr;
+    size_t out_floats = 0;
+    unsigned long long* ptr = nullptr;
+    int32_t *start = nullptr, *n = nullptr, *dst = nullptr, *tok = nullptr;
+
+    Setup(int gu, int dt, int64_t H, int64_t FF, int tokens, int k_per_token, int blobs_n, std::mt19937& rng,
+          cudaStream_t s) {
+        L = k::native_expert_layout(gu, dt, H, FF);
+        T = tokens; K = k_per_token; cap = T * K; n_blobs = blobs_n;
+        slot = (L.bytes + 255) & ~(size_t) 255;
+        blobs = dalloc<uint8_t>(slot * (size_t) n_blobs);
+        for (int b = 0; b < n_blobs; ++b) {
+            // gate rows | up rows (format gu, H values each) | down rows (format dt, FF values each)
+            auto blob = random_rows(gu, 2 * FF, H, rng);
+            const auto down = random_rows(dt, H, FF, rng);
+            blob.insert(blob.end(), down.begin(), down.end());
+            if (blob.size() != L.bytes) { std::fprintf(stderr, "blob %zu != %zu bytes\n", blob.size(), L.bytes); std::exit(2); }
+            ck(cudaMemcpy(blobs + slot * (size_t) b, blob.data(), blob.size(), cudaMemcpyHostToDevice), "blob");
+        }
+        std::normal_distribution<float> nd(0.f, 1.f);
+        std::vector<float> x((size_t) T * H);
+        for (auto& v : x) v = nd(rng);
+        float* dx = dalloc<float>(x.size());
+        ck(cudaMemcpy(dx, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+        xq = dalloc<uint8_t>((size_t) T * (H / 32) * 36);
+        k::quantize_q8_1_rows(dx, T, H, xq, s);
+        ck(cudaStreamSynchronize(s), "quantize");
+        cudaFree(dx);
+        scr_bytes = k::native_expert_scratch_bytes(cap, FF);
+        scr = dalloc<uint8_t>(scr_bytes);
+        out_floats = (size_t) cap * H;
+        out = dalloc<float>(out_floats);
+        ptr = dalloc<unsigned long long>(cap);
+        start = dalloc<int32_t>(cap + 1);
+        n = dalloc<int32_t>(1);
+        dst = dalloc<int32_t>(cap);
+        tok = dalloc<int32_t>(cap);
+    }
+    ~Setup() {
+        cudaFree(blobs); cudaFree(xq); cudaFree(scr); cudaFree(out);
+        cudaFree(ptr); cudaFree(start); cudaFree(n); cudaFree(dst); cudaFree(tok);
+    }
+    // a plan: `groups` groups of 1..T entries from entry `base` on (as many as fit in cap)
+    int plan(int groups, int base, std::mt19937& rng) {
+        std::uniform_int_distribution<int> size(1, T);
+        std::vector<int> sizes;
+        int used = base;
+        for (int g = 0; g < groups; ++g) {
+            const int m = std::min(size(rng), cap - used);
+            if (m <= 0) break;
+            sizes.push_back(m);
+            used += m;
+        }
+        return plan_sizes(sizes, base, rng);
+    }
+    // groups of these sizes from entry `base` on, each its own blob (as long as there are enough: a window's groups
+    // are different experts); scattered destinations, random tokens
+    int plan_sizes(const std::vector<int>& sizes, int base, std::mt19937& rng) {
+        std::uniform_int_distribution<int> tk(0, T - 1);
+        std::vector<int> order(n_blobs);
+        std::iota(order.begin(), order.end(), 0);
+        std::shuffle(order.begin(), order.end(), rng);
+        std::vector<unsigned long long> p;
+        std::vector<int32_t> st(1, base), tk_v, ds(cap);
+        for (const int m : sizes) {
+            p.push_back((unsigned long long) (blobs + slot * (size_t) order[p.size() % order.size()]));
+            st.push_back(st.back() + m);
+        }
+        if (st.back() > cap) { std::fprintf(stderr, "plan: %d entries > cap %d\n", st.back(), cap); std::exit(2); }
+        const int ng = (int) p.size();
+        std::iota(ds.begin(), ds.end(), 0);
+        std::shuffle(ds.begin(), ds.end(), rng);                  // scattered rows of `out`
+        for (int e = 0; e < cap; ++e) tk_v.push_back(tk(rng));
+        if (ng) ck(cudaMemcpy(ptr, p.data(), p.size() * 8, cudaMemcpyHostToDevice), "ptr");
+        ck(cudaMemcpy(start, st.data(), st.size() * 4, cudaMemcpyHostToDevice), "start");
+        ck(cudaMemcpy(n, &ng, 4, cudaMemcpyHostToDevice), "n");
+        ck(cudaMemcpy(dst, ds.data(), ds.size() * 4, cudaMemcpyHostToDevice), "dst");
+        ck(cudaMemcpy(tok, tk_v.data(), tk_v.size() * 4, cudaMemcpyHostToDevice), "tok");
+        return ng;
+    }
+    void call(int64_t grid_groups, cudaStream_t s) {
+        k::native_expert_grouped(L, ptr, start, n, dst, tok, cap, cap, xq, scr, out, s, grid_groups);
+    }
+    // `out` after one call from the NaN pattern, the scratch holding `junk` (what earlier calls left there)
+    std::vector<uint32_t> result(bool v1, int64_t grid_groups, int junk, cudaStream_t s) {
+        ck(cudaMemset(out, 0xFF, out_floats * 4), "memset");
+        ck(cudaMemset(scr, junk, scr_bytes), "memset");
+        k::native_grouped_set_v1(v1);
+        call(grid_groups, s);
+        k::native_grouped_set_v1(false);
+        ck(cudaStreamSynchronize(s), "sync");
+        std::vector<uint32_t> o(out_floats);
+        ck(cudaMemcpy(o.data(), out, o.size() * 4, cudaMemcpyDeviceToHost), "out");
+        return o;
+    }
+};
+
+void check(int gu, int dt, int64_t H, int64_t FF, cudaStream_t s, std::mt19937& rng) {
+    Setup S(gu, dt, H, FF, 6, 4, 9, rng, s);   // 6 tokens x 4 experts: cap 24 entries
+    int calls = 0, bad = 0;
+    size_t written = 0;
+    for (const int base : {0, 7}) {
+        for (const int groups : {0, 1, 2, 3, 5, 8, 24}) {
+            const int ng = S.plan(groups, base, rng);
+            const auto ref = S.result(true, 0, 0x00, s);
+            size_t w = 0;
+            for (const uint32_t v : ref) w += v != 0xFFFFFFFFu;
+            written += w;
+            for (const int64_t gy : {(int64_t) 0, (int64_t) 1, (int64_t) 2, (int64_t) 3, (int64_t) 4, (int64_t) S.cap + 3}) {
+                const auto got = S.result(false, gy, (calls & 1) ? 0x7F : 0x00, s);
+                ++calls;
+                size_t diff = 0;
+                for (size_t i = 0; i < ref.size(); ++i) diff += ref[i] != got[i];
+                if (diff) {
+                    std::printf("  %-8s/%-7s base %d, %d groups, grid_groups %lld: %zu of %zu floats differ  FAIL\n",
+                                name_of(gu), name_of(dt), base, ng, (long long) gy, diff, ref.size());
+                    ++bad;
+                }
+            }
+        }
+    }
+    std::printf("%-8s/%-7s %5lld x %4lld  %d calls: %s (%zu output floats written in all)\n", name_of(gu), name_of(dt),
+                (long long) H, (long long) FF, calls, bad ? "FAIL" : "bitwise equal to v1", written);
+    if (written == 0) { std::printf("  nothing was written: the check checked nothing  FAIL\n"); ++bad; }
+    g_fail += bad;
+}
+
+// ------------------------------------------------------------------------------------------------ --bench
+// A window's 48 calls, one per layer, each with ITS OWN experts as the layers have: copies of the Setup's blobs in one
+// arena, so the weights stream from DRAM as in the engine.  (48 calls re-reading the same 28 experts - 73 MB, about an
+// Ada L2 - would read part of them from L2; the engine's layers share none.)
+struct Layers {
+    uint8_t* arena = nullptr;
+    std::vector<unsigned long long*> ptr;           // per layer: its groups' blobs (device)
+    Layers(Setup& S, int ng) {
+        const int per = std::max(ng, 1);
+        arena = dalloc<uint8_t>(S.slot * (size_t) (48 * per));
+        for (int l = 0; l < 48; ++l) {
+            std::vector<unsigned long long> p;
+            for (int j = 0; j < per; ++j) {
+                uint8_t* d = arena + S.slot * (size_t) (l * per + j);
+                ck(cudaMemcpy(d, S.blobs + S.slot * (size_t) ((l * per + j) % S.n_blobs), S.L.bytes,
+                              cudaMemcpyDeviceToDevice), "blob copy");
+                p.push_back((unsigned long long) d);
+            }
+            unsigned long long* dp = dalloc<unsigned long long>(p.size());
+            ck(cudaMemcpy(dp, p.data(), p.size() * 8, cudaMemcpyHostToDevice), "ptr");
+            ptr.push_back(dp);
+        }
+    }
+    Layers(const Layers&) = delete;
+    Layers& operator=(const Layers&) = delete;
+    ~Layers() {
+        for (auto* p : ptr) cudaFree(p);
+        cudaFree(arena);
+    }
+};
+
+// the 48 calls of one variant captured in one graph
+cudaGraphExec_t make_graph(cudaStream_t s, Setup& S, const Layers& Ls, bool v1, int64_t gy) {
+    k::native_grouped_set_v1(v1);
+    cudaGraph_t g;
+    cudaGraphExec_t ge;
+    ck(cudaStreamBeginCapture(s, cudaStreamCaptureModeThreadLocal), "capture");
+    for (int l = 0; l < 48; ++l)
+        k::native_expert_grouped(S.L, Ls.ptr[l], S.start, S.n, S.dst, S.tok, S.cap, S.cap, S.xq, S.scr, S.out, s, gy);
+    ck(cudaStreamEndCapture(s, &g), "capture");
+    ck(cudaGraphInstantiate(&ge, g, 0), "instantiate");
+    ck(cudaGraphDestroy(g), "graph");
+    k::native_grouped_set_v1(false);
+    return ge;
+}
+
+// one launch of the graph: microseconds per call
+float launch_us(cudaStream_t s, cudaGraphExec_t ge, cudaEvent_t e0, cudaEvent_t e1) {
+    ck(cudaEventRecord(e0, s), "record");
+    ck(cudaGraphLaunch(ge, s), "launch");
+    ck(cudaEventRecord(e1, s), "record");
+    ck(cudaEventSynchronize(e1), "bench");
+    float ms = 0;
+    ck(cudaEventElapsedTime(&ms, e0, e1), "elapsed");
+    return 1e3f * ms / 48.0f;
+}
+
+float median(std::vector<float> v) {
+    std::sort(v.begin(), v.end());
+    return v[v.size() / 2];
+}
+
+// The two variants' launches ALTERNATE (v1-new, new-v1, ...) after a warm-up, and each reports its median: a card
+// under sustained load steps its clock down after a fraction of a second (an RTX 4080 SUPER went from 80 to 89 us
+// per gate/up call within one run), and whichever variant ran before the step would look faster.
+constexpr int kWarmPairs = 20, kPairs = 40;
+
+void bench(cudaStream_t s, std::mt19937& rng) {
+    std::printf("\n--bench: microseconds per call, a window's 48 calls in a graph, each layer with its own experts (the\n"
+                "weights stream from DRAM as in the engine); v1 / new (grid_groups): medians of %d launches each,\n"
+                "alternating, after %d warm-up pairs - idle GPU assumed\n", kPairs, kWarmPairs);
+    // a verify window's shape: 4 tokens x 10 experts, 2560 x 768 IQ3_S / IQ4_XS experts
+    Setup S(21, 23, 2560, 768, 4, 10, 32, rng, s);
+    std::vector<int> vram(16, 1);
+    vram.insert(vram.end(), 12, 2);                                   // 28 groups, 40 entries
+    const struct { const char* what; std::vector<int> sizes; int64_t gy; } rows[] = {
+        {"no group (the PCIe call at pcie_frac 0)", {}, 4},
+        {"1 group of 2 entries (a PCIe call)", {2}, 4},
+        {"28 groups, 40 entries (a VRAM call)", vram, 0},
+    };
+    cudaEvent_t e0, e1;
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    for (const auto& r : rows) {
+        const int ng = S.plan_sizes(r.sizes, 0, rng);
+        const Layers Ls(S, ng);                                       // 48 x 28 experts: 3.7 GB for the VRAM call
+        cudaGraphExec_t ga = make_graph(s, S, Ls, true, 0), gb = make_graph(s, S, Ls, false, r.gy);
+        std::vector<float> ta, tb;
+        for (int i = 0; i < kWarmPairs + kPairs; ++i) {
+            const bool a_first = i % 2 == 0;
+            const float x = launch_us(s, a_first ? ga : gb, e0, e1), y = launch_us(s, a_first ? gb : ga, e0, e1);
+            if (i < kWarmPairs) continue;
+            ta.push_back(a_first ? x : y);
+            tb.push_back(a_first ? y : x);
+        }
+        ck(cudaGraphExecDestroy(ga), "graph");
+        ck(cudaGraphExecDestroy(gb), "graph");
+        std::printf("  %-42s %8.2f / %8.2f us (grid_groups %lld)\n", r.what, median(ta), median(tb), (long long) r.gy);
+    }
+    cudaEventDestroy(e0);
+    cudaEventDestroy(e1);
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    const bool do_bench = argc > 1 && std::string(argv[1]) == "--bench";
+    cudaStream_t s;
+    ck(cudaStreamCreate(&s), "stream");
+    std::mt19937 rng(316);
+    for (int gu : {16, 17, 18, 21, 22, 23, 29, 42, 12, 13, 8})        // STRATA_GU_FMTS
+        for (int dt : {20, 23, 42, 7, 8}) check(gu, dt, 512, 256, s, rng);   // STRATA_D_FMTS; IQ4_XS: n_ff % 256
+    check(21, 20, 2560, 640, s, rng);                                   // a model's shapes
+    check(21, 23, 2560, 768, s, rng);
+    if (do_bench) bench(s, rng);
+    std::printf("native_grouped_parity: %d failures\n", g_fail);
+    cudaStreamDestroy(s);
+    return g_fail ? 1 : 0;
+}

src/kernels/ngram.cpp

@@ -209,13 +209,9 @@ bool PleTable::open(const std::string& gguf_path, std::string& err, const PleIoO
         close();
         return false;
     }
-    // The direct reader has fixed 90-byte rows. Keep it for IQ4_NL; Q5_0's
-    // 110-byte rows use the mapped path until that reader supports variable rows.
-    if (impl_->q5_0 && io.mode == PleIo::Direct) {
-        err = "Q5_0 PLE requires --ple-io mmap";
-        close();
-        return false;
-    }
+    // PleReader's row_bytes has been a runtime parameter since the FP8 table (160 B rows) needed it; Q5_0's
+    // 110 B rows go through the exact same generic path (ple_reader_test --selftest covers both 90 and 110 B
+    // rows: straddling, caching, in-flight tickets, keep-alive). This refusal was stale.
     impl_->n_rows = t->shape[1];
     impl_->data = impl_->file->tensor_data(*t);
 

src/kernels/qsa_prompt_attn_parity.cpp

@@ -1,7 +1,7 @@
 // src/kernels/qsa_prompt_attn_parity.cpp - perf-review D-1: the tensor-core prompt attention (qsa_prompt_attn.hpp)
 // against the FP32 kernel it replaces (`qsa_decode_attn_batch`) and an FP64 host reference (GPU, synthetic, no model).
 //
-// Int8 and FP16 pools with random codes, scales and queries; selections shaped like the prompt path's (the 2,051
+// Int8, FP16 and Q4_0 pools with random codes, scales and queries; selections shaped like the prompt path's (the 2,051
 // widest, a recent window plus older cells that drift slowly from one query to the next, so neighbours share most
 // of them as they do in a real prompt; short contexts take every cell). Checks:
 //   1. against FP64, the new kernel's error is no larger than a small multiple of the old kernel's (both FP32 math);
@@ -12,6 +12,7 @@
 #include "strata/kernels/qsa.hpp"
 #include "strata/kernels/qsa_decode_attn.hpp"
 #include "strata/kernels/qsa_prompt_attn.hpp"
+#include "strata/kernels/kv_q4.hpp"
 
 #include <cuda_fp16.h>
 #include <cuda_runtime.h>
@@ -39,7 +40,7 @@ template <typename T> T* up(const std::vector<T>& h) {
 float h2f(uint16_t b) { __half h; *reinterpret_cast<uint16_t*>(&h) = b; return __half2float(h); }
 uint16_t f2h(float f) { __half h = __float2half(f); return *reinterpret_cast<uint16_t*>(&h); }
 
-int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
+int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16, 2 q4_0
     const k::QsaShapes s = k::qsa_real_shapes();
     const int64_t HD = s.head_dim, NKV = s.n_head_kv, NH = s.n_head, PS = s.page_size;
     const int64_t pages = (ctx + PS - 1) / PS, rows = pages * NKV * PS;
@@ -50,7 +51,22 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
     // pools; page table a shuffled permutation (the readers must follow it)
     std::vector<int8_t> kq, vq;
     std::vector<uint16_t> ks, vs, kh, vh;
-    if (fmt == 1) {
+    std::vector<uint8_t> k4, v4;   // q4_0: 8 blocks of {fp16 d, 16 code bytes} per row (kv_q4.hpp)
+    constexpr int64_t B4 = 18, ROW4 = 8 * B4;
+    if (fmt == 2) {
+        std::uniform_int_distribution<int> byte(0, 255);
+        // the scales as a q4_0 pool holds them: signed (ggml's d = max / -8) and spread over two decades, so a chunk's
+        // cells mix signs and magnitudes (a positive-only bound on them overflowed FP16 on real K/V)
+        std::uniform_real_distribution<float> lg(std::log(0.01f), std::log(1.0f));
+        auto sc4 = [&](std::mt19937& r) { return std::exp(lg(r)) * (byte(r) & 1 ? -1.0f : 1.0f); };
+        k4.resize(rows * ROW4); v4.resize(rows * ROW4);
+        for (auto* pool : {&k4, &v4})
+            for (int64_t b = 0; b < rows * 8; ++b) {
+                const uint16_t d = f2h(sc4(rng));
+                std::memcpy(pool->data() + b * B4, &d, 2);
+                for (int j = 0; j < 16; ++j) (*pool)[b * B4 + 2 + j] = (uint8_t) byte(rng);
+            }
+    } else if (fmt == 1) {
         kq.resize(rows * HD); vq.resize(rows * HD); ks.resize(rows * 4); vs.resize(rows * 4);
         for (auto& x : kq) x = (int8_t) code(rng);
         for (auto& x : vq) x = (int8_t) code(rng);
@@ -98,8 +114,19 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
         std::copy(v.begin(), v.end(), sel);
     }
     k::QsaAttnPools pl;
-    if (fmt == 1) { pl.k_q = up(kq); pl.v_q = up(vq); pl.k_scale = up(ks); pl.v_scale = up(vs); }
+    if (fmt == 2) { pl.k_q4 = up(k4); pl.v_q4 = up(v4); }
+    else if (fmt == 1) { pl.k_q = up(kq); pl.v_q = up(vq); pl.k_scale = up(ks); pl.v_scale = up(vs); }
     else { pl.k_pool = up(kh); pl.v_pool = up(vh); }
+    // a q4_0 value: block d / 32 of the row, element j = d % 32 in the low nibble of byte j (j < 16), else the high
+    // nibble of byte j - 16, minus 8, times the block's scale
+    auto q4v = [&](const std::vector<uint8_t>& pool, int64_t row, int64_t d) {
+        const uint8_t* blk = pool.data() + row * ROW4 + (d / 32) * B4;
+        uint16_t sc;
+        std::memcpy(&sc, blk, 2);
+        const int j = (int) (d % 32);
+        const int code = j < 16 ? (blk[2 + j] & 0x0F) : (blk[2 + j - 16] >> 4);
+        return (double) (code - 8) * h2f(sc);
+    };
     pl.page_table = up(table);
     const int32_t* d_ids = up(ids);
     const int32_t* d_steps = up(steps);
@@ -139,7 +166,8 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
                 const int64_t cell = sel[c], row = ((int64_t) table[cell / PS] * NKV + kvh) * PS + cell % PS;
                 double a = 0;
                 for (int64_t d = 0; d < HD; ++d) {
-                    const double kv = fmt == 1 ? (double) kq[row * HD + d] * h2f(ks[row * 4 + d / 64]) : h2f(kh[row * HD + d]);
+                    const double kv = fmt == 2 ? q4v(k4, row, d)
+                                      : fmt == 1 ? (double) kq[row * HD + d] * h2f(ks[row * 4 + d / 64]) : h2f(kh[row * HD + d]);
                     a += (double) q[(i * NH + h) * HD + d] * kv;
                 }
                 sco[c] = a / 16.0;
@@ -151,7 +179,8 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
                 double a = 0;
                 for (int64_t c = 0; c < w; ++c) {
                     const int64_t cell = sel[c], row = ((int64_t) table[cell / PS] * NKV + kvh) * PS + cell % PS;
-                    const double vv = fmt == 1 ? (double) vq[row * HD + d] * h2f(vs[row * 4 + d / 64]) : h2f(vh[row * HD + d]);
+                    const double vv = fmt == 2 ? q4v(v4, row, d)
+                                      : fmt == 1 ? (double) vq[row * HD + d] * h2f(vs[row * 4 + d / 64]) : h2f(vh[row * HD + d]);
                     a += sco[c] * vv;
                 }
                 const double r = a / l;
@@ -187,12 +216,13 @@ int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
     const bool ok2 = diff <= 1e-4 * scale;
     std::printf("%s %s ctx %lld, %lld queries: vs FP64 old %.3g new %.3g (output scale %.3g); new vs old %.3g (%.2g of "
                 "scale); %.3f -> %.3f ms per chunk (%.2fx)\n",
-                ok1 && ok2 ? "PASS" : "FAIL", fmt == 1 ? "int8" : "fp16", (long long) ctx, (long long) nq, err_old,
+                ok1 && ok2 ? "PASS" : "FAIL", fmt == 2 ? "q4_0" : fmt == 1 ? "int8" : "fp16", (long long) ctx, (long long) nq, err_old,
                 err_new, ref_scale, diff, diff / scale, ms_old / reps, ms_new / reps, ms_old / ms_new);
     cudaFree((void*) d_ids); cudaFree((void*) d_steps); cudaFree((void*) d_q); cudaFree(d_old); cudaFree(d_new);
     cudaFree(scratch);
     cudaFree((void*) pl.k_q); cudaFree((void*) pl.v_q); cudaFree((void*) pl.k_scale); cudaFree((void*) pl.v_scale);
     cudaFree((void*) pl.k_pool); cudaFree((void*) pl.v_pool); cudaFree((void*) pl.page_table);
+    cudaFree((void*) pl.k_q4); cudaFree((void*) pl.v_q4);
     return ok1 && ok2 ? 0 : 1;
 }
 }  // namespace
@@ -222,6 +252,8 @@ int main(int argc, char** argv) {
     fails += run(1, ctx, nq, reps);
 #if !defined(__HIP_PLATFORM_AMD__)
     fails += run(0, ctx, nq, reps);   // FP16 KV: the RDNA4 kernel takes int8 KV only
+    fails += run(2, ctx, nq, reps);   // Q4_0 KV (mode 4)
+    fails += run(2, 1500, std::min<int64_t>(nq, 1500), reps);
 #endif
     fails += run(1, 1500, std::min<int64_t>(nq, 1500), reps);   // short context: the selection is every cell
     fails += run(1, 2100, std::min<int64_t>(nq, 256), reps);    // the identity-to-sparse edge

src/kernels/qsa_topk_active_parity.cpp

@@ -0,0 +1,95 @@
+// Capacity 262144, live prefill up to the register boundary, and the legacy fallbacks.
+// Compare selected IDs only: unused output padding is not part of the API contract.
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_select.hpp"
+#include <cuda_runtime.h>
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+namespace {
+void ck(cudaError_t e) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "CUDA: %s\n", cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+struct Case { int64_t ctx, queries; int mode; };
+bool check(Case c) {
+    constexpr int64_t stride = 262144 / 4 + 2;
+    const auto s = k::qsa_real_shapes();
+    const int64_t cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    std::vector<int32_t> steps((size_t)(c.queries * k::kStepCount));
+    std::vector<float> scores((size_t)(c.queries * stride), -999.f);
+    std::mt19937 rng(742913);
+    std::uniform_real_distribution<float> random(0.f, 200.f);
+    for (int64_t i = 0; i < c.queries; ++i) {
+        auto* st = steps.data() + i * k::kStepCount;
+        const int64_t pos = c.ctx - c.queries + i;
+        st[k::kStepPos] = (int32_t)pos;
+        st[k::kStepNKv] = (int32_t)(pos + 1);
+        st[k::kStepNBid] = (int32_t)((pos + 1) / s.idx_block);
+        st[k::kStepWidth] = (int32_t)k::qsa_selection_width(pos + 1, s);
+        for (int64_t j = 0; j <= st[k::kStepNBid]; ++j)
+            scores[(size_t)(i * stride + j)] = c.mode == 1 ? 0.f : c.mode == 2 ? float(j % 7) : random(rng);
+        if ((pos + 1) % s.idx_block)
+            scores[(size_t)(i * stride + st[k::kStepNBid])] += 1e9f;
+    }
+    int64_t active = steps[(size_t)((c.queries - 1) * k::kStepCount + k::kStepNBid)] + 1;
+    if (c.mode == 3) active = 0;
+    if (c.mode == 4) active = stride + 1;
+    float* ds = nullptr;
+    int32_t *dt = nullptr, *da = nullptr, *db = nullptr;
+    ck(cudaMalloc(&ds, scores.size() * sizeof(float)));
+    ck(cudaMalloc(&dt, steps.size() * sizeof(int32_t)));
+    ck(cudaMalloc(&da, (size_t)(c.queries * cap) * sizeof(int32_t)));
+    ck(cudaMalloc(&db, (size_t)(c.queries * cap) * sizeof(int32_t)));
+    ck(cudaMemcpy(ds, scores.data(), scores.size() * sizeof(float), cudaMemcpyHostToDevice));
+    ck(cudaMemcpy(dt, steps.data(), steps.size() * sizeof(int32_t), cudaMemcpyHostToDevice));
+    // No bound: the exact capacity-based dispatch used by captured decode.
+    k::qsa_block_topk(ds, dt, c.queries, stride, cap, s, da, nullptr);
+    k::qsa_block_topk(ds, dt, c.queries, stride, cap, s, db, nullptr, active);
+    ck(cudaDeviceSynchronize());
+    std::vector<int32_t> a((size_t)(c.queries * cap)), b(a.size());
+    ck(cudaMemcpy(a.data(), da, a.size() * sizeof(int32_t), cudaMemcpyDeviceToHost));
+    ck(cudaMemcpy(b.data(), db, b.size() * sizeof(int32_t), cudaMemcpyDeviceToHost));
+    bool ok = true;
+    for (int64_t i = 0; i < c.queries; ++i) {
+        const int64_t w = steps[(size_t)(i * k::kStepCount + k::kStepWidth)];
+        if (!std::equal(a.begin() + i * cap, a.begin() + i * cap + w, b.begin() + i * cap)) {
+            std::fprintf(stderr, "FAIL ctx=%lld queries=%lld mode=%d row=%lld\n",
+                         (long long)c.ctx, (long long)c.queries, c.mode, (long long)i);
+            ok = false;
+            break;
+        }
+    }
+    ck(cudaFree(ds)); ck(cudaFree(dt)); ck(cudaFree(da)); ck(cudaFree(db));
+    return ok;
+}
+}
+
+int main() {
+    int devices = 0;
+    if (cudaGetDeviceCount(&devices) != cudaSuccess || devices == 0) {
+        std::puts("SKIP: no CUDA device");
+        return 77;
+    }
+    // the active-bound dispatch is Turing's in the engine; forced here so every CUDA card checks it
+#if defined(_WIN32)
+    _putenv_s("STRATA_TOPK_ACTIVE_ANY", "1");
+#else
+    setenv("STRATA_TOPK_ACTIVE_ANY", "1", 1);
+#endif
+    const Case cases[] = {
+        {1111, 256, 0}, {8192, 256, 0}, {32768, 256, 0}, {131072, 256, 0},
+        {131075, 255, 0}, {135164, 256, 0}, {135168, 256, 0}, {250022, 256, 0},
+        {262144, 256, 0}, {131072, 1, 0}, {131072, 8, 0}, {131072, 257, 0},
+        {131072, 256, 1}, {131072, 256, 2}, {131072, 256, 3}, {131072, 256, 4},
+    };
+    for (const auto c : cases) if (!check(c)) return 1;
+    std::puts("PASS: 16 top-k active-bound cases, selected IDs bitwise identical");
+    return 0;
+}

src/ngram/ple_reader_test.cpp

@@ -41,33 +41,33 @@ int g_fail = 0;
 
 constexpr uint64_t HEADER = 192;   // the real shard's data offset, so rows are misaligned the same way
 
-void expected_row(uint32_t row, uint8_t* out) {
-    for (uint32_t b = 0; b < ng::ROW_BYTES; ++b) out[b] = (uint8_t) ((row * 2654435761u + b * 97u) >> 7);
+void expected_row(uint32_t row, uint32_t rb, uint8_t* out) {
+    for (uint32_t b = 0; b < rb; ++b) out[b] = (uint8_t) ((row * 2654435761u + b * 97u) >> 7);
     std::memcpy(out, &row, 4);
 }
 
-bool make_table(const std::string& path, uint32_t rows) {
+bool make_table(const std::string& path, uint32_t rows, uint32_t rb) {
     std::ofstream f(path, std::ios::binary);
     std::vector<uint8_t> head(HEADER, 0xAB);
     f.write((const char*) head.data(), (std::streamsize) head.size());
-    uint8_t r[ng::ROW_BYTES];
+    std::vector<uint8_t> r(rb);
     for (uint32_t i = 0; i < rows; ++i) {
-        expected_row(i, r);
-        f.write((const char*) r, ng::ROW_BYTES);
+        expected_row(i, rb, r.data());
+        f.write((const char*) r.data(), rb);
     }
     return (bool) f;
 }
 
-bool check_rows(ng::PleReader& rd, const std::vector<uint32_t>& rows, uint32_t n_rows, const char* what) {
-    std::vector<uint8_t> out(rows.size() * ng::ROW_BYTES, 0xCC);
+bool check_rows(ng::PleReader& rd, const std::vector<uint32_t>& rows, uint32_t n_rows, uint32_t rb, const char* what) {
+    std::vector<uint8_t> out(rows.size() * rb, 0xCC);
     std::string err;
     const auto t = rd.issue(rows.data(), rows.size(), out.data());
     if (!rd.collect(t, err)) { CHECK(false, "%s: collect failed: %s", what, err.c_str()); return false; }
-    uint8_t want[ng::ROW_BYTES];
+    std::vector<uint8_t> want(rb);
     for (size_t i = 0; i < rows.size(); ++i) {
-        if (rows[i] >= n_rows) std::memset(want, 0, sizeof want);
-        else expected_row(rows[i], want);
-        if (std::memcmp(want, &out[i * ng::ROW_BYTES], ng::ROW_BYTES) != 0) {
+        if (rows[i] >= n_rows) std::memset(want.data(), 0, rb);
+        else expected_row(rows[i], rb, want.data());
+        if (std::memcmp(want.data(), &out[i * rb], rb) != 0) {
             CHECK(false, "%s: row %u (index %zu) differs", what, rows[i], i);
             return false;
         }
@@ -75,50 +75,53 @@ bool check_rows(ng::PleReader& rd, const std::vector<uint32_t>& rows, uint32_t n
     return true;
 }
 
-int selftest(const std::string& dir) {
+// row_bytes: ng::ROW_BYTES (90, IQ4_NL) is the production default; 110 (#296, OrcaRouter's Q5_0 PLE rows) is
+// run too, through the exact same generic row_bytes path -- nothing here is IQ4_NL-specific, so a second row
+// size run here is the correctness evidence for lifting ngram.cpp's "Q5_0 PLE requires --ple-io mmap" refusal.
+int selftest(const std::string& dir, uint32_t rb) {
     const uint32_t N = 500000;                          // 45 MB: large enough for thousands of distinct pages
-    const std::string path = dir + "/ple_reader_selftest.bin";
-    if (!make_table(path, N)) { std::fprintf(stderr, "cannot write %s\n", path.c_str()); return 2; }
+    const std::string path = dir + "/ple_reader_selftest_" + std::to_string(rb) + ".bin";
+    if (!make_table(path, N, rb)) { std::fprintf(stderr, "cannot write %s\n", path.c_str()); return 2; }
     std::mt19937 rng(7);
     for (bool thr : {false, true})
     for (uint64_t cache : {0ull, 4096ull}) {
         for (uint32_t inflight : {1u, 8u, 64u}) {
             ng::PleReader rd;
             std::string err;
-            CHECK(rd.open(path, HEADER, N, inflight, cache, err, thr), "open: %s", err.c_str());
+            CHECK(rd.open(path, HEADER, N, inflight, cache, err, thr, rb), "open: %s", err.c_str());
             // decode-shaped tickets: 16 random rows
             for (int t = 0; t < 200; ++t) {
                 std::vector<uint32_t> rows(16);
                 for (auto& r : rows) r = rng() % N;
-                check_rows(rd, rows, N, "decode");
+                check_rows(rd, rows, N, rb, "decode");
             }
-            // rows that straddle a 4 KiB boundary: byte offset of row r is HEADER + 90 r
+            // rows that straddle a 4 KiB boundary: byte offset of row r is HEADER + rb * r
             std::vector<uint32_t> straddle;
             for (uint32_t r = 0; r < N && straddle.size() < 64; ++r) {
-                const uint64_t a = HEADER + (uint64_t) r * ng::ROW_BYTES;
-                if (a / 4096 != (a + ng::ROW_BYTES - 1) / 4096) straddle.push_back(r);
+                const uint64_t a = HEADER + (uint64_t) r * rb;
+                if (a / 4096 != (a + rb - 1) / 4096) straddle.push_back(r);
             }
-            check_rows(rd, straddle, N, "straddle");
+            check_rows(rd, straddle, N, rb, "straddle");
             // duplicates, neighbours on one page, the first and last rows, and out-of-range rows
-            check_rows(rd, {5, 5, 6, 7, 5, 0, N - 1, N, 0xFFFFFFFFu, 44, 45}, N, "dedup/edges");
+            check_rows(rd, {5, 5, 6, 7, 5, 0, N - 1, N, 0xFFFFFFFFu, 44, 45}, N, rb, "dedup/edges");
             // a prefill-shaped ticket much larger than the in-flight window
             std::vector<uint32_t> bulk(20000);
             for (auto& r : bulk) r = rng() % N;
-            check_rows(rd, bulk, N, "bulk");
+            check_rows(rd, bulk, N, rb, "bulk");
             // two tickets in flight at once, collected in reverse order
             std::vector<uint32_t> a(16), b(16);
             for (auto& r : a) r = rng() % N;
             for (auto& r : b) r = rng() % N;
-            std::vector<uint8_t> oa(16 * ng::ROW_BYTES), ob(16 * ng::ROW_BYTES);
+            std::vector<uint8_t> oa(16 * rb), ob(16 * rb);
             const auto ta = rd.issue(a.data(), 16, oa.data());
             const auto tb = rd.issue(b.data(), 16, ob.data());
             CHECK(rd.collect(tb, err) && rd.collect(ta, err), "two tickets: %s", err.c_str());
-            uint8_t want[ng::ROW_BYTES];
+            std::vector<uint8_t> want(rb);
             for (int i = 0; i < 16; ++i) {
-                expected_row(a[i], want);
-                CHECK(!std::memcmp(want, &oa[i * ng::ROW_BYTES], ng::ROW_BYTES), "ticket a row %d", i);
-                expected_row(b[i], want);
-                CHECK(!std::memcmp(want, &ob[i * ng::ROW_BYTES], ng::ROW_BYTES), "ticket b row %d", i);
+                expected_row(a[i], rb, want.data());
+                CHECK(!std::memcmp(want.data(), &oa[i * rb], rb), "ticket a row %d", i);
+                expected_row(b[i], rb, want.data());
+                CHECK(!std::memcmp(want.data(), &ob[i * rb], rb), "ticket b row %d", i);
             }
             if (cache > 0) CHECK(rd.stats().cache_hits > 0, "the row cache never hit");
             CHECK(rd.cache_size() <= rd.cache_capacity(), "row cache exceeded its bound");
@@ -128,12 +131,12 @@ int selftest(const std::string& dir) {
     for (bool thr : {false, true}) {
         ng::PleReader rd;
         std::string err;
-        CHECK(rd.open(path, HEADER, N, 16, 0, err, thr), "open: %s", err.c_str());
+        CHECK(rd.open(path, HEADER, N, 16, 0, err, thr, rb), "open: %s", err.c_str());
         rd.set_injected_delay_us(3000);
         std::vector<uint32_t> rows(16);
         for (auto& r : rows) r = rng() % N;
         const double t0 = now_us();
-        check_rows(rd, rows, N, "delayed");
+        check_rows(rd, rows, N, rb, "delayed");
         CHECK(now_us() - t0 >= 3000, "injected delay not observed (%.0f us)", now_us() - t0);
         CHECK(rd.stats().late_injected > 0, "no read was held back");
     }
@@ -143,11 +146,11 @@ int selftest(const std::string& dir) {
     {
         ng::PleReader rd;
         std::string err;
-        CHECK(rd.open(path, HEADER, N, 16, 4096, err, true), "open: %s", err.c_str());
+        CHECK(rd.open(path, HEADER, N, 16, 4096, err, true, rb), "open: %s", err.c_str());
         rd.set_keepalive(20.0, 0.5);
         std::vector<uint32_t> rows(16);
         for (auto& r : rows) r = rng() % N;
-        check_rows(rd, rows, N, "keep-alive, first ticket");
+        check_rows(rd, rows, N, rb, "keep-alive, first ticket");
         const uint64_t reads0 = rd.snapshot().reads;
         const auto nap = [](int ms) { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); };
         nap(300);
@@ -161,14 +164,14 @@ int selftest(const std::string& dir) {
         CHECK(k3 == k2, "keep-alive went on after the window (%llu -> %llu)", (unsigned long long) k2,
               (unsigned long long) k3);
         CHECK(k2 <= k1 + 20, "keep-alive: %llu reads by the end of a 0.5 s window", (unsigned long long) k2);
-        check_rows(rd, rows, N, "keep-alive, cached ticket");   // the row cache serves all 16: no row read
+        check_rows(rd, rows, N, rb, "keep-alive, cached ticket");   // the row cache serves all 16: no row read
         CHECK(rd.snapshot().reads == reads0, "cached rows were read again");
         nap(200);
         CHECK(rd.snapshot().keepalive_reads >= k3 + 5, "the keep-alive did not start again (%llu -> %llu)",
               (unsigned long long) k3, (unsigned long long) rd.snapshot().keepalive_reads);
         std::vector<uint32_t> more(2000);
         for (auto& r : more) r = rng() % N;
-        check_rows(rd, more, N, "keep-alive, rows afterwards");
+        check_rows(rd, more, N, rb, "keep-alive, rows afterwards");
         rd.set_keepalive(0, 0.5);                           // off
         const uint64_t k4 = rd.snapshot().keepalive_reads;
         nap(200);
@@ -179,16 +182,16 @@ int selftest(const std::string& dir) {
     {   // the caller's thread does the reads: no worker, so no keep-alive
         ng::PleReader rd;
         std::string err;
-        CHECK(rd.open(path, HEADER, N, 16, 0, err, false), "open: %s", err.c_str());
+        CHECK(rd.open(path, HEADER, N, 16, 0, err, false, rb), "open: %s", err.c_str());
         rd.set_keepalive(20.0, 1.0);
         std::vector<uint32_t> rows(16);
         for (auto& r : rows) r = rng() % N;
-        check_rows(rd, rows, N, "keep-alive, caller thread");
+        check_rows(rd, rows, N, rb, "keep-alive, caller thread");
         std::this_thread::sleep_for(std::chrono::milliseconds(150));
         CHECK(rd.snapshot().keepalive_reads == 0, "keep-alive without the worker thread");
     }
     std::filesystem::remove(path);
-    std::printf("ple_reader selftest: %s\n", g_fail ? "FAILED" : "OK");
+    std::printf("ple_reader selftest (row_bytes=%u): %s\n", rb, g_fail ? "FAILED" : "OK");
     return g_fail ? 1 : 0;
 }
 
@@ -282,7 +285,13 @@ int main(int argc, char** argv) {
         else if (a == "--sync") sync_submit = true;
         else { std::fprintf(stderr, "usage: ple_reader_test --selftest [--dir D] | --gguf SHARD2 [--rows N] [--tokens F]\n"); return 2; }
     }
-    if (self) return selftest(dir);
+    if (self) {
+        // ng::ROW_BYTES (90, IQ4_NL, production default) and 110 (#296, OrcaRouter's Q5_0 PLE rows) through the
+        // same generic row_bytes path -- see the comment on selftest().
+        const int r90 = selftest(dir, ng::ROW_BYTES);
+        const int r110 = selftest(dir, 110);
+        return r90 != 0 ? r90 : r110;
+    }
     if (!gguf.empty()) return real(gguf, rows, tokens, inflight, direct_first, direct_only, sync_submit);
     std::fprintf(stderr, "nothing to do\n");
     return 2;

src/prefill/gdn_rec_parity.cu

@@ -0,0 +1,886 @@
+// src/prefill/gdn_rec_parity.cu - the prompt path's DeltaNet recurrence, bit for bit and timed.
+//
+// src/prefill/kernels.cu runs gdn_rec_kh_kernel (the three value heads that share a key head in one thread, the
+// inputs staged in blocks of 8 tokens) on CUDA sm_80+ cards that hold all 64 of its blocks at once, and
+// gdn_rec_cols_pipe_kernel elsewhere and with STRATA_GDN_KEYHEAD=0.  Both are copied below verbatim, with the output
+// norm (gdn_out_norm_kernel) and the norm as it was before, which also stored the normalized value in FP32 although
+// nothing reads it.  The two recurrences do the same arithmetic in the same order per value head and column, so the
+// output, the state after it and the FP16 output must be the same bits.
+//
+// Synthetic inputs of the model's shape (q/k L2-normalized per key head as gdn_l2_kernel leaves them), no model.
+//   gdn_rec_parity            the bit checks (T from 1 to 4099, a non-zero state as after an earlier chunk)
+//   gdn_rec_parity --bench    and the timings at T = 8192 and 32768 (one layer, all 48 value heads); the two
+//                             variants' launches alternate, since a card under load steps its clock down after a
+//                             fraction of a second (see native_grouped_parity); then both at 1 to 6 blocks per
+//                             SM, as on cards with more or fewer SMs (scale), and both with the engine's grids on
+//                             32 to all of this card's SMs, the others held by a sleeping kernel (fewer_sms)
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+#include <vector>
+
+namespace {
+
+constexpr int S = 128, HK = 16, HV = 48, C = 10240;   // as kernels.cu: head size, key heads, value heads, q|k|v width
+constexpr int RG = 4, RPG = S / RG;
+constexpr int CB = 32, NCB = S / CB;
+
+void ck(cudaError_t e, const char* what) {
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e));
+        std::exit(2);
+    }
+}
+
+__device__ __forceinline__ float warp_sum(float v) {
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
+    return v;
+}
+__device__ __forceinline__ float sigm(float x) { return 1.0f / (1.0f + __expf(-x)); }
+__device__ __forceinline__ uint16_t hf(float f) { return __half_as_ushort(__float2half_rn(f)); }
+
+// ------------------------------------------------------------------ the engine's kernels (src/prefill/kernels.cu)
+// gdn_rec_cols_kernel with the next token's inputs (q/k rows, v, gate, beta) loaded into registers while this token
+// computes (software pipelining).  The same arithmetic in the same order: the same bits, and the same CB-column split.
+// STRATA_GDN_PIPELINE=0: gdn_rec_cols_kernel.
+__global__ void __launch_bounds__(CB * RG) gdn_rec_cols_pipe_kernel(float* __restrict__ state, const float* __restrict__ h,
+                                                                      const float* __restrict__ gate,
+                                                                      const float* __restrict__ beta,
+                                                                      float* __restrict__ oc_out, int64_t T) {
+    constexpr int NT = CB * RG, LPT = S / NT;   // threads, q/k rows loaded per thread
+    __shared__ float sk[S], sq[S], red[RG][CB];
+    const int head = blockIdx.x / NCB, cb = blockIdx.x % NCB;
+    const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
+    const int qh = head % HK;
+    float s[RPG];
+    float* base = state + ((size_t) (rg * RPG) * HV + head) * S + col;
+    const size_t rs = (size_t) HV * S;
+#pragma unroll
+    for (int r = 0; r < RPG; ++r) s[r] = base[r * rs];
+    float nq[LPT], nk[LPT], nv = 0.0f, ng = 0.0f, nb = 0.0f;
+    auto fetch = [&](int64_t t) {
+        const float* ht = h + t * C;
+#pragma unroll
+        for (int u = 0; u < LPT; ++u) { nq[u] = ht[qh * S + tid + u * NT]; nk[u] = ht[HK * S + qh * S + tid + u * NT]; }
+        nv = ht[2 * HK * S + head * S + col];
+        ng = gate[t * HV + head];
+        nb = beta[t * HV + head];
+    };
+    if (T > 0) fetch(0);
+    for (int64_t t = 0; t < T; ++t) {
+        float cq[LPT], ck[LPT];
+#pragma unroll
+        for (int u = 0; u < LPT; ++u) { cq[u] = nq[u]; ck[u] = nk[u]; }
+        const float cv = nv, cg = ng, cbt = nb;
+        __syncthreads();
+#pragma unroll
+        for (int u = 0; u < LPT; ++u) { sq[tid + u * NT] = cq[u]; sk[tid + u * NT] = ck[u]; }
+        __syncthreads();
+        if (t + 1 < T) fetch(t + 1);
+        const float g = __expf(cg);
+        float kv = 0.0f;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) kv = fmaf(s[r], sk[rg * RPG + r], kv);
+        red[rg][c] = kv;
+        __syncthreads();
+        const float kv_col = red[0][c] + red[1][c] + red[2][c] + red[3][c];
+        const float delta = (cv - g * kv_col) * cbt;
+        float o = 0.0f;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) {
+            s[r] = fmaf(g, s[r], sk[rg * RPG + r] * delta);
+            o = fmaf(s[r], sq[rg * RPG + r], o);
+        }
+        __syncthreads();
+        red[rg][c] = o;
+        __syncthreads();
+        if (rg == 0) oc_out[t * HV * S + head * S + col] = (red[0][c] + red[1][c] + red[2][c] + red[3][c]) * rsqrtf((float) S);
+    }
+#pragma unroll
+    for (int r = 0; r < RPG; ++r) base[r * rs] = s[r];
+}
+// The recurrence with one thread for the three value heads that share a key head (head % HK): column c of heads
+// qh, qh + 16 and qh + 32, row group rg.  gdn_rec_cols_pipe_kernel spends its time in shared memory, not in
+// arithmetic: every thread of a warp needs the same 32 q and k values per token (the k twice), and a warp receives one
+// such broadcast value per clock however wide the load.  Here every q/k value a thread loads feeds three heads, and a
+// token's k row goes into registers once for both of its uses.  The inputs come in blocks of GDN_TB tokens, copied to
+// shared memory by cp.async while the block before computes (one token ahead is shorter than a load from L2 takes),
+// and the two cross-row-group sums have their own arrays, so a token needs 2 __syncthreads instead of 5: the second
+// one of a token orders every read of rkv before the next token's writes, the next token's first one every read of ro
+// before the writes after it.  64 blocks instead of 192.  Per value head and column the same arithmetic in the same
+// order: the same bits (src/prefill/gdn_rec_parity.cu checks them and times the variants: 1.41x on a 4080 Super).
+// sm_80+ (gdn_keyhead_ok); STRATA_GDN_KEYHEAD=0: gdn_rec_cols_pipe_kernel.
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
+#define STRATA_GDN_CP_ASYNC 0   // Turing builds: plain copies (never launched there, see gdn_keyhead_ok)
+#else
+#define STRATA_GDN_CP_ASYNC 1
+#endif
+__device__ __forceinline__ void gdn_cp4(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *smem = *gmem;
+#endif
+}
+__device__ __forceinline__ void gdn_cp16(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *reinterpret_cast<float4*>(smem) = *reinterpret_cast<const float4*>(gmem);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_commit() {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.commit_group;\n" ::);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_wait_prev() {   // every group but the newest has landed
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.wait_group 1;\n" ::);
+#endif
+}
+constexpr int GDN_TB = 8, VPK = HV / HK;   // tokens per staged block, value heads per key head
+__global__ void __launch_bounds__(CB * RG) gdn_rec_kh_kernel(float* __restrict__ state, const float* __restrict__ h,
+                                                               const float* __restrict__ gate,
+                                                               const float* __restrict__ beta,
+                                                               float* __restrict__ oc_out, int64_t T) {
+    constexpr int TB = GDN_TB, NT = CB * RG, QKP = S / 4, VP = CB / 4;   // threads, 16-byte pieces of a q/k row, of v
+    __shared__ __align__(16) float sq[2][TB][S];
+    __shared__ __align__(16) float sk[2][TB][S];
+    __shared__ __align__(16) float sv[2][TB][VPK][CB];
+    __shared__ float sg[2][TB][VPK], sb[2][TB][VPK], rkv[VPK][RG][CB], ro[VPK][RG][CB];
+    const int qh = blockIdx.x / NCB, cb = blockIdx.x % NCB;
+    const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
+    float s[VPK][RPG];
+    const size_t rs = (size_t) HV * S;
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        const float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) s[j][r] = base[r * rs];
+    }
+    const int64_t nblk = (T + TB - 1) / TB;
+    auto stage = [&](int64_t k) {   // tokens [k * TB, k * TB + TB) into buffer k & 1
+        const int bb = (int) (k & 1);
+        const int64_t t0 = k * TB;
+        for (int p = tid; p < TB * 2 * QKP; p += NT) {
+            const int i = p / (2 * QKP), w = p % (2 * QKP), isk = w / QKP, jj = (w % QKP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(isk ? &sk[bb][i][jj] : &sq[bb][i][jj], h + (t0 + i) * C + (isk ? HK * S : 0) + qh * S + jj);
+        }
+        for (int p = tid; p < TB * VPK * VP; p += NT) {
+            const int i = p / (VPK * VP), w = p % (VPK * VP), j = w / VP, jj = (w % VP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(&sv[bb][i][j][jj], h + (t0 + i) * C + 2 * HK * S + (qh + j * HK) * S + cb * CB + jj);
+        }
+        for (int p = tid; p < 2 * TB * VPK; p += NT) {
+            const int isb = p / (TB * VPK), w = p % (TB * VPK), i = w / VPK, j = w % VPK;
+            if (t0 + i < T)
+                gdn_cp4(isb ? &sb[bb][i][j] : &sg[bb][i][j], (isb ? beta : gate) + (t0 + i) * HV + qh + j * HK);
+        }
+    };
+    if (nblk > 0) stage(0);
+    gdn_cp_commit();
+    for (int64_t k = 0; k < nblk; ++k) {
+        // buffer (k + 1) & 1 was read by block k - 1, whose last token's second __syncthreads every thread has passed
+        if (k + 1 < nblk) stage(k + 1);
+        gdn_cp_commit();
+        gdn_cp_wait_prev();
+        __syncthreads();
+        const int bb = (int) (k & 1);
+        const int n = (int) ((T - k * TB) < TB ? (T - k * TB) : TB);
+        for (int i = 0; i < n; ++i) {
+            const int64_t t = k * TB + i;
+            float kc[RPG];
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) kc[r] = sk[bb][i][rg * RPG + r];
+            float g[VPK], kv[VPK], delta[VPK], o[VPK];
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) { g[j] = __expf(sg[bb][i][j]); kv[j] = 0.0f; o[j] = 0.0f; }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r)
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) kv[j] = fmaf(s[j][r], kc[r], kv[j]);
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) rkv[j][rg][c] = kv[j];
+            __syncthreads();
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) {
+                const float kv_col = rkv[j][0][c] + rkv[j][1][c] + rkv[j][2][c] + rkv[j][3][c];
+                delta[j] = (sv[bb][i][j][c] - g[j] * kv_col) * sb[bb][i][j];
+            }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) {
+                const float qr = sq[bb][i][rg * RPG + r];
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) {
+                    s[j][r] = fmaf(g[j], s[j][r], kc[r] * delta[j]);
+                    o[j] = fmaf(s[j][r], qr, o[j]);
+                }
+            }
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) ro[j][rg][c] = o[j];
+            __syncthreads();
+            if (rg < VPK)   // row group j writes head j's output
+                oc_out[t * HV * S + (qh + rg * HK) * S + col] =
+                    (ro[rg][0][c] + ro[rg][1][c] + ro[rg][2][c] + ro[rg][3][c]) * rsqrtf((float) S);
+        }
+    }
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) base[r * rs] = s[j][r];
+    }
+}
+// the output norm over a head's 128 columns, into the FP16 copy the out projection reads (the FP32 output before the
+// norm stays in its scratch buffer)
+__global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
+                                                         float eps, const float* __restrict__ y,
+                                                         uint16_t* __restrict__ y16) {
+    __shared__ float wsum[4];
+    const int64_t t = blockIdx.x;
+    const int head = blockIdx.y, col = threadIdx.x;
+    const size_t at = (size_t) t * HV * S + (size_t) head * S + col;
+    const float oc = y[at];
+    float sp = warp_sum(oc * oc);
+    if ((col & 31) == 0) wsum[col >> 5] = sp;
+    __syncthreads();
+    const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
+    const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
+    y16[at] = hf(v);
+}
+
+// ------------------------------------------------------------------ the output norm before (0.1.31)
+__global__ void __launch_bounds__(S) gdn_out_norm_old_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
+                                                             float eps, float* __restrict__ y,
+                                                             uint16_t* __restrict__ y16) {
+    __shared__ float wsum[4];
+    const int64_t t = blockIdx.x;
+    const int head = blockIdx.y, col = threadIdx.x;
+    const size_t at = (size_t) t * HV * S + (size_t) head * S + col;
+    const float oc = y[at];
+    float sp = warp_sum(oc * oc);
+    if ((col & 31) == 0) wsum[col >> 5] = sp;
+    __syncthreads();
+    const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
+    const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
+    y[at] = v;
+    y16[at] = hf(v);
+}
+
+// ------------------------------------------------------------------ synthetic inputs
+__device__ __forceinline__ uint32_t mix(uint64_t x) {
+    x ^= x >> 33; x *= 0xff51afd7ed558ccdULL; x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53ULL; x ^= x >> 33;
+    return (uint32_t) x;
+}
+__device__ __forceinline__ float uni(uint64_t key) { return ((mix(key) >> 8) + 0.5f) * (1.0f / 16777216.0f); }
+__device__ __forceinline__ float gauss(uint64_t key) {
+    const float u1 = uni(key * 2 + 1), u2 = uni(key * 2 + 2);
+    return sqrtf(-2.0f * logf(u1)) * cosf(6.2831853f * u2);
+}
+__global__ void fill_gauss(float* x, int64_t n, float scale, uint64_t seed, float offset = 0.0f) {
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
+        x[i] = offset + scale * gauss(seed * 0x9E3779B97F4A7C15ULL + (uint64_t) i);
+}
+// the q and k rows L2-normalized per key head, as gdn_l2_kernel leaves them for the recurrence
+__global__ void l2_rows(float* h, float eps) {
+    float* x = h + (int64_t) blockIdx.y * C + (int64_t) blockIdx.x * S;
+    const float v = x[threadIdx.x];
+    __shared__ float part[4];
+    const float sq = warp_sum(v * v);
+    if ((threadIdx.x & 31) == 0) part[threadIdx.x >> 5] = sq;
+    __syncthreads();
+    x[threadIdx.x] = v * rsqrtf(part[0] + part[1] + part[2] + part[3] + eps);
+}
+// gate = log of the decay (mostly near 1, some strong), beta in (0, 1)
+__global__ void fill_gates(float* gate, float* beta, int64_t n, uint64_t seed) {
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) {
+        const float u = uni(seed * 0x9E3779B97F4A7C15ULL + 7 * (uint64_t) i);
+        gate[i] = -0.02f - 0.8f * u * u * u;
+        beta[i] = 1.0f / (1.0f + expf(-gauss(seed * 0xD1B54A32D192ED03ULL + (uint64_t) i)));
+    }
+}
+__global__ void count_diff(const uint32_t* a, const uint32_t* b, int64_t n, unsigned long long* out) {
+    unsigned long long d = 0;
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
+        d += a[i] != b[i];
+    if (d) atomicAdd(out, d);
+}
+
+// ------------------------------------------------------------------ holding SMs (fewer_sms)
+__device__ __forceinline__ uint64_t gtime() {
+    uint64_t t;
+    asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
+    return t;
+}
+__device__ __forceinline__ int sm_id() {
+    unsigned id;
+    asm volatile("mov.u32 %0, %%smid;" : "=r"(id));
+    return (int) id;
+}
+__device__ __forceinline__ void nap(unsigned ns) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+    __nanosleep(ns);
+#else
+    (void) ns;
+#endif
+}
+// limits that a run that works never reaches (a launch here takes milliseconds); they keep a run that does not work
+// from waiting forever, and the host drops its time
+constexpr uint64_t kHoldLimitNs = 1000000000ull, kWaitLimitNs = 500000000ull;
+// One block per SM: it asks for all the shared memory a block may have (launched with that much dynamic shared
+// memory), so no block of another kernel fits beside it.  It notes its SM and sleeps until release says gen.
+__global__ void hold_sm(unsigned long long* sitting, const volatile unsigned* release, unsigned gen, int* where,
+                        unsigned* late) {
+    where[blockIdx.x] = sm_id();
+    __threadfence();
+    atomicAdd(sitting, 1ull);
+    const uint64_t t0 = gtime();
+    while (*release != gen) {
+        if (gtime() - t0 > kHoldLimitNs) { atomicAdd(late, 1u); return; }
+        nap(20000);
+    }
+}
+// on the timed launch's stream, before it: returns once all holding blocks so far sit on their SMs
+__global__ void wait_sitting(const volatile unsigned long long* sitting, unsigned long long all, unsigned* late) {
+    const uint64_t t0 = gtime();
+    while (*sitting < all) {
+        if (gtime() - t0 > kWaitLimitNs) { atomicAdd(late, 1u); return; }
+        nap(1000);
+    }
+}
+__global__ void release_sms(volatile unsigned* release, unsigned gen) { *release = gen; }
+// Where the blocks of a grid land: launched like the recurrence it stands for (the same grid and block size, and as
+// much dynamic shared memory as lets as many blocks share an SM), every block notes its SM and stays until all of them
+// have landed, so they all sit at once as the recurrence's blocks do for the whole chunk.
+__global__ void land(int* where, unsigned* landed, unsigned* apart) {
+    extern __shared__ unsigned char land_smem[];   // only its size matters
+    if (threadIdx.x != 0 || threadIdx.y != 0) return;
+    where[blockIdx.x] = sm_id();
+    atomicAdd(landed, 1u);
+    const uint64_t t0 = gtime();
+    while (*(volatile unsigned*) landed < gridDim.x) {
+        if (gtime() - t0 > kWaitLimitNs) { atomicAdd(apart, 1u); return; }   // they did not all fit at once
+        nap(1000);
+    }
+    (void) land_smem;
+}
+
+template <typename T> T* dalloc(size_t n) {
+    T* p = nullptr;
+    ck(cudaMalloc((void**) &p, n * sizeof(T) + 16), "cudaMalloc");
+    return p;
+}
+unsigned long long diff_words(const void* a, const void* b, size_t bytes) {
+    static unsigned long long* d = dalloc<unsigned long long>(1);
+    ck(cudaMemset(d, 0, sizeof(*d)), "memset");
+    count_diff<<<1024, 256>>>((const uint32_t*) a, (const uint32_t*) b, (int64_t) (bytes / 4), d);
+    unsigned long long h = 0;
+    ck(cudaMemcpy(&h, d, sizeof(h), cudaMemcpyDeviceToHost), "diff");
+    return h;
+}
+
+struct Inputs {
+    int64_t T = 0;
+    float *h = nullptr, *gate = nullptr, *beta = nullptr, *z = nullptr, *gamma = nullptr, *state0 = nullptr;
+    explicit Inputs(int64_t t, uint64_t seed) : T(t) {
+        h = dalloc<float>((size_t) T * C);
+        gate = dalloc<float>((size_t) T * HV);
+        beta = dalloc<float>((size_t) T * HV);
+        z = dalloc<float>((size_t) T * HV * S);
+        gamma = dalloc<float>(S);
+        state0 = dalloc<float>((size_t) S * HV * S);
+        fill_gauss<<<4096, 256>>>(h, T * C, 1.0f, seed);
+        l2_rows<<<dim3(2 * HK, (unsigned) T), S>>>(h, 1e-6f);
+        fill_gates<<<1024, 256>>>(gate, beta, T * HV, seed + 1);
+        fill_gauss<<<4096, 256>>>(z, T * HV * S, 1.0f, seed + 2);
+        fill_gauss<<<1, 128>>>(gamma, S, 0.1f, seed + 3, 1.0f);
+        fill_gauss<<<1024, 256>>>(state0, (int64_t) S * HV * S, 0.05f, seed + 4);
+        ck(cudaDeviceSynchronize(), "inputs");
+    }
+    Inputs(const Inputs&) = delete;
+    Inputs& operator=(const Inputs&) = delete;
+    ~Inputs() { cudaFree(h); cudaFree(gate); cudaFree(beta); cudaFree(z); cudaFree(gamma); cudaFree(state0); }
+};
+
+enum Rec { kBefore, kKeyHead, kRecCount };
+void run_rec(Rec v, float* state, const Inputs& in, float* oc, cudaStream_t s = 0) {
+    if (v == kBefore)
+        gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, s>>>(state, in.h, in.gate, in.beta, oc, in.T);
+    else
+        gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG), 0, s>>>(state, in.h, in.gate, in.beta, oc, in.T);
+}
+
+int check_bits(int64_t T, uint64_t seed) {
+    Inputs in(T, seed);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state[kRecCount];
+    float* oc[kRecCount];
+    for (int v = 0; v < kRecCount; ++v) {
+        state[v] = dalloc<float>(st);
+        oc[v] = dalloc<float>(oc_n);
+        ck(cudaMemcpy(state[v], in.state0, st * 4, cudaMemcpyDeviceToDevice), "state copy");
+        run_rec((Rec) v, state[v], in, oc[v]);
+    }
+    ck(cudaDeviceSynchronize(), "recurrence");
+    int fails = 0;
+    const unsigned long long d_oc = diff_words(oc[kBefore], oc[kKeyHead], oc_n * 4),
+                             d_st = diff_words(state[kBefore], state[kKeyHead], st * 4);
+    if (d_oc || d_st) {
+        std::printf("FAIL T=%lld recurrence: %llu of %zu outputs and %llu of %zu state values differ\n", (long long) T,
+                    d_oc, oc_n, d_st, st);
+        ++fails;
+    }
+    // the output norm: before (in place, with the FP32 store) against now (the FP16 store only), on the same output
+    uint16_t* y16a = dalloc<uint16_t>(oc_n);
+    uint16_t* y16b = dalloc<uint16_t>(oc_n);
+    float* yb = dalloc<float>(oc_n);
+    ck(cudaMemcpy(yb, oc[kBefore], oc_n * 4, cudaMemcpyDeviceToDevice), "y copy");
+    gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, yb, y16b);
+    gdn_out_norm_old_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc[kBefore], y16a);
+    ck(cudaDeviceSynchronize(), "norm");
+    const unsigned long long d16 = diff_words(y16a, y16b, oc_n * 2);
+    if (d16) {
+        std::printf("FAIL T=%lld output norm: %llu of %zu FP16 pairs differ\n", (long long) T, d16, oc_n / 2);
+        ++fails;
+    }
+    std::printf("T=%-6lld %s\n", (long long) T, fails ? "DIFFERENT" : "the same bits (output, state, FP16 output)");
+    for (int v = 0; v < kRecCount; ++v) { cudaFree(state[v]); cudaFree(oc[v]); }
+    cudaFree(y16a); cudaFree(y16b); cudaFree(yb);
+    return fails;
+}
+
+float median(std::vector<float> v) {
+    std::sort(v.begin(), v.end());
+    return v.empty() ? 0.0f : v[v.size() / 2];
+}
+
+void bench(int64_t T) {
+    Inputs in(T, 0xB3);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state = dalloc<float>(st);
+    float* oc = dalloc<float>(oc_n);
+    uint16_t* y16 = dalloc<uint16_t>(oc_n);
+    cudaEvent_t e0, e1;
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    auto timed = [&](auto&& launch) {
+        ck(cudaMemcpy(state, in.state0, st * 4, cudaMemcpyDeviceToDevice), "state");
+        cudaEventRecord(e0);
+        launch();
+        cudaEventRecord(e1);
+        ck(cudaEventSynchronize(e1), "timed");
+        float ms = 0.0f;
+        cudaEventElapsedTime(&ms, e0, e1);
+        return ms;
+    };
+    const int warm = 3, rounds = T >= 16384 ? 10 : 20;
+    std::vector<float> t_rec[kRecCount], t_norm[2];
+    for (int i = 0; i < warm + rounds; ++i) {
+        for (int j = 0; j < kRecCount; ++j) {
+            const Rec v = (Rec) ((i + j) % kRecCount);   // a different variant first in each round
+            const float ms = timed([&] { run_rec(v, state, in, oc); });
+            if (i >= warm) t_rec[v].push_back(ms);
+        }
+    }
+    for (int i = 0; i < warm + rounds; ++i) {
+        for (int j = 0; j < 2; ++j) {
+            const int v = (i + j) % 2;
+            const float ms = timed([&] {
+                if (v == 0) gdn_out_norm_old_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc, y16);
+                else gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc, y16);
+            });
+            if (i >= warm) t_norm[v].push_back(ms);
+        }
+    }
+    const float r0 = median(t_rec[kBefore]), r1 = median(t_rec[kKeyHead]), n0 = median(t_norm[0]), n1 = median(t_norm[1]);
+    std::printf("  T = %lld tokens, one layer (48 value heads), medians of %d alternating runs:\n", (long long) T, rounds);
+    std::printf("    recurrence, before (gdn_rec_cols_pipe_kernel)  %8.2f ms  (%.3f us per token)\n", r0,
+                1000.0f * r0 / (float) T);
+    std::printf("    recurrence, gdn_rec_kh_kernel                  %8.2f ms  (%.3f us per token)  %.2fx\n", r1,
+                1000.0f * r1 / (float) T, r0 / r1);
+    std::printf("    output norm, before (FP32 and FP16 store)      %8.2f ms\n", n0);
+    std::printf("    output norm, FP16 store only                   %8.2f ms  %.2fx\n", n1, n0 / n1);
+    std::printf("    36 DeltaNet layers per %lld-token block: recurrence + norm %.0f ms -> %.0f ms (-%.0f ms)\n",
+                (long long) T, 36.0f * (r0 + n0), 36.0f * (r1 + n1), 36.0f * (r0 + n0 - r1 - n1));
+    cudaEventDestroy(e0);
+    cudaEventDestroy(e1);
+    cudaFree(state); cudaFree(oc); cudaFree(y16);
+}
+
+// The two kernels at a given number of blocks per SM, as on cards with more or fewer SMs than this one: the kernel
+// before puts ceil(192 / SMs) of its blocks on an SM, gdn_rec_kh_kernel ceil(64 / SMs), and the busiest SM sets the
+// pace (every block walks the whole chunk).  b blocks per SM here: b rounds of one block per SM, each round one launch
+// of SMs blocks (the first heads; split in launches of at most the engine's grid), every launch on its own stream with
+// its own state and output, all started together, so the block scheduler puts one block of each round on every SM.
+// The kernels are the ones above, unchanged.
+void scale(int64_t T) {
+    int dev = 0, sms = 0, per_sm[kRecCount] = {};
+    ck(cudaGetDevice(&dev), "device");
+    ck(cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev), "SMs");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kBefore], gdn_rec_cols_pipe_kernel, CB * RG, 0), "occupancy");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kKeyHead], gdn_rec_kh_kernel, CB * RG, 0), "occupancy");
+    constexpr int kMaxB = 6, kMaxLaunches = 6;
+    const int full[kRecCount] = {HV * NCB, HK * NCB};   // the grids as the engine launches them
+    auto parts = [&](int v) { return (sms + full[v] - 1) / full[v]; };   // launches per round
+    Inputs in(T, 0xC7);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state[kMaxLaunches];
+    float* oc[kMaxLaunches];
+    cudaStream_t stream[kMaxLaunches];
+    cudaEvent_t e0, e1, done[kMaxLaunches];
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    for (int c = 0; c < kMaxLaunches; ++c) {
+        state[c] = dalloc<float>(st);
+        oc[c] = dalloc<float>(oc_n);
+        ck(cudaStreamCreateWithFlags(&stream[c], cudaStreamNonBlocking), "stream");
+        ck(cudaEventCreate(&done[c]), "event");
+    }
+    struct Case { int v, b; };
+    std::vector<Case> cases;
+    for (int v = 0; v < kRecCount; ++v)
+        for (int b = 1; b <= std::min(kMaxB, per_sm[v]); ++b)
+            if (b * parts(v) <= kMaxLaunches) cases.push_back({v, b});
+    auto run = [&](const Case& cs) {
+        for (int c = 0; c < kMaxLaunches; ++c) ck(cudaMemcpy(state[c], in.state0, st * 4, cudaMemcpyDeviceToDevice), "state");
+        ck(cudaDeviceSynchronize(), "sync");
+        const int np = parts(cs.v), n = cs.b * np;
+        cudaEventRecord(e0, stream[0]);
+        for (int c = 0; c < n; ++c) {
+            const int k = c % np, grid = sms / np + (k < sms % np ? 1 : 0);   // round c / np, part k
+            if (c > 0) cudaStreamWaitEvent(stream[c], e0, 0);
+            if (cs.v == kBefore)
+                gdn_rec_cols_pipe_kernel<<<grid, dim3(CB, RG), 0, stream[c]>>>(state[c], in.h, in.gate, in.beta, oc[c], T);
+            else
+                gdn_rec_kh_kernel<<<grid, dim3(CB, RG), 0, stream[c]>>>(state[c], in.h, in.gate, in.beta, oc[c], T);
+            cudaEventRecord(done[c], stream[c]);
+        }
+        for (int c = 1; c < n; ++c) cudaStreamWaitEvent(stream[0], done[c], 0);
+        cudaEventRecord(e1, stream[0]);
+        ck(cudaEventSynchronize(e1), "scale");
+        float ms = 0.0f;
+        cudaEventElapsedTime(&ms, e0, e1);
+        return ms;
+    };
+    const int warm = 2, rounds = 9;
+    std::vector<std::vector<float>> t(cases.size());
+    for (int i = 0; i < warm + rounds; ++i)
+        for (size_t j = 0; j < cases.size(); ++j) {
+            const size_t k = (i + j) % cases.size();   // a different case first in each round
+            const float ms = run(cases[k]);
+            if (i >= warm) t[k].push_back(ms);
+        }
+    auto us = [&](int v, int b) {   // per token, or < 0 if not measured
+        for (size_t k = 0; k < cases.size(); ++k)
+            if (cases[k].v == v && cases[k].b == b) return 1000.0f * median(t[k]) / (float) T;
+        return -1.0f;
+    };
+    std::printf("  per token at a given number of blocks per SM (%lld tokens, medians of %d alternating runs; this card has "
+                "%d SMs):\n", (long long) T, rounds, sms);
+    std::printf("    blocks per SM   before      gdn_rec_kh_kernel\n");
+    for (int b = 1; b <= kMaxB; ++b) {
+        const float u0 = us(kBefore, b), u1 = us(kKeyHead, b);
+        if (u0 < 0 && u1 < 0) continue;
+        char a0[32] = "-", a1[32] = "-";
+        if (u0 >= 0) std::snprintf(a0, sizeof a0, "%.3f us", u0);
+        if (u1 >= 0) std::snprintf(a1, sizeof a1, "%.3f us", u1);
+        std::printf("    %8d        %-11s %s\n", b, a0, a1);
+    }
+    std::printf("  so on a card with N SMs (the kernel before puts ceil(192 / N) blocks on an SM, gdn_rec_kh_kernel "
+                "ceil(64 / N), at most %d):\n", per_sm[kKeyHead]);
+    const int bands[][2] = {{192, 0}, {96, 191}, {64, 95}, {48, 63}, {39, 47}, {32, 38}};
+    for (const auto& bd : bands) {
+        const int nsm = bd[0], b0 = (192 + nsm - 1) / nsm, b1 = (64 + nsm - 1) / nsm;
+        const float u0 = us(kBefore, b0), u1 = us(kKeyHead, b1);
+        char range[32];
+        if (bd[1]) std::snprintf(range, sizeof range, "%d-%d SMs", bd[0], bd[1]);
+        else std::snprintf(range, sizeof range, "%d+ SMs", bd[0]);
+        if (u0 < 0 || u1 < 0)
+            std::printf("    %-12s before %d / new %d per SM: not measured on this card\n", range, b0, b1);
+        else
+            std::printf("    %-12s before %d / new %d per SM: %.3f -> %.3f us per token, %.2fx\n", range, b0, b1, u0, u1,
+                        u0 / u1);
+    }
+    for (int c = 0; c < kMaxLaunches; ++c) {
+        cudaFree(state[c]);
+        cudaFree(oc[c]);
+        cudaStreamDestroy(stream[c]);
+        cudaEventDestroy(done[c]);
+    }
+    cudaEventDestroy(e0);
+    cudaEventDestroy(e1);
+}
+
+// The engine's grids on N of this card's SMs.  scale() puts as many blocks on every SM as the busiest SM of a card with
+// N SMs gets; on that card only some SMs get that many (192 blocks on 80 SMs: 3 on 32 SMs, 2 on the other 48), the
+// rest of the card has less to do, and the estimate came out 4-9 % above the real launch on two cards.  Here the
+// recurrences run as the engine launches them, while hold_sm keeps all but N SMs: its blocks go out first, a small
+// kernel on the timed launch's stream waits until they all sit, and one after the timed launch lets them go.  land()
+// shows for every N where the two grids land: none on a held SM, and how many blocks share the busiest SM.  The
+// recurrences are the ones above, unchanged; L2, memory and clock stay this card's.
+void fewer_sms(int64_t T) {
+    int dev = 0, sms = 0, optin = 0, smem_sm = 0, reserved = 0, hold_per_sm = 0, per_sm[kRecCount] = {};
+    ck(cudaGetDevice(&dev), "device");
+    ck(cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev), "SMs");
+    ck(cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev), "shared memory");
+    ck(cudaDeviceGetAttribute(&smem_sm, cudaDevAttrMaxSharedMemoryPerMultiprocessor, dev), "shared memory");
+    ck(cudaDeviceGetAttribute(&reserved, cudaDevAttrReservedSharedMemoryPerBlock, dev), "shared memory");
+    for (const void* f : {(const void*) hold_sm, (const void*) land}) {
+        ck(cudaFuncSetAttribute(f, cudaFuncAttributeMaxDynamicSharedMemorySize, optin), "shared memory");
+        ck(cudaFuncSetAttribute(f, cudaFuncAttributePreferredSharedMemoryCarveout, (int) cudaSharedmemCarveoutMaxShared),
+           "carveout");
+    }
+    // Loaded now rather than at a first launch: with lazy loading (CUDA 12's default) a kernel's first launch may wait
+    // until the device is idle, and so for hold_sm, which waits for release_sms.
+    cudaFuncAttributes fa[kRecCount], fx;
+    ck(cudaFuncGetAttributes(&fa[kBefore], gdn_rec_cols_pipe_kernel), "attributes");
+    ck(cudaFuncGetAttributes(&fa[kKeyHead], gdn_rec_kh_kernel), "attributes");
+    for (const void* f : {(const void*) hold_sm, (const void*) wait_sitting, (const void*) release_sms, (const void*) land})
+        ck(cudaFuncGetAttributes(&fx, f), "attributes");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&hold_per_sm, hold_sm, 1, optin), "occupancy");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kBefore], gdn_rec_cols_pipe_kernel, CB * RG, 0), "occupancy");
+    ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kKeyHead], gdn_rec_kh_kernel, CB * RG, 0), "occupancy");
+    const int grid[kRecCount] = {HV * NCB, HK * NCB};   // as the engine launches them
+    std::printf("  the engine's grids on N of this card's %d SMs (%lld tokens; the other SMs held by a sleeping kernel, see "
+                "fewer_sms):\n", sms, (long long) T);
+    const int left = smem_sm - optin - reserved;   // shared memory a holding block leaves on its SM
+    if (hold_per_sm != 1 || (int) fa[kBefore].sharedSizeBytes + reserved <= left ||
+        (int) fa[kKeyHead].sharedSizeBytes + reserved <= left) {
+        std::printf("    not on this card: a block of a recurrence would fit beside a holding block (%d bytes left)\n",
+                    left);
+        return;
+    }
+    // land(): as much dynamic shared memory as still lets as many of its blocks share an SM as of the recurrence
+    int land_smem[kRecCount];
+    for (int v = 0; v < kRecCount; ++v) {
+        int lo = 0, hi = optin;
+        while (lo < hi) {
+            const int mid = (lo + hi + 1) / 2;
+            int nb = 0;
+            ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&nb, land, CB * RG, mid), "occupancy");
+            if (nb >= per_sm[v]) lo = mid;
+            else hi = mid - 1;
+        }
+        land_smem[v] = lo;
+    }
+    // N: all SMs, and below that the SM counts of cards and the band edges, as long as both grids fit at once
+    std::vector<int> ns = {sms};
+    for (const int n : {170, 144, 132, 128, 112, 108, 96, 95, 84, 82, 80, 76, 72, 70, 66, 64, 63, 60, 56, 52, 48, 47, 46, 44,
+                        40, 39, 38, 36, 34, 32})
+        if (n < sms && (int64_t) n * per_sm[kBefore] >= grid[kBefore] && (int64_t) n * per_sm[kKeyHead] >= grid[kKeyHead])
+            ns.push_back(n);
+
+    Inputs in(T, 0xD5);
+    const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
+    float* state = dalloc<float>(st);
+    float* oc = dalloc<float>(oc_n);
+    unsigned long long* sitting = dalloc<unsigned long long>(1);
+    unsigned* flags = dalloc<unsigned>(4);
+    unsigned *release = flags, *late = flags + 1, *landed = flags + 2, *apart = flags + 3;
+    int* where = dalloc<int>((size_t) sms + grid[kBefore]);   // the holding blocks' SMs, then land()'s
+    ck(cudaMemset(sitting, 0, sizeof(*sitting)), "memset");
+    ck(cudaMemset(flags, 0, 4 * sizeof(unsigned)), "memset");
+    cudaStream_t s_hold, s_run;
+    ck(cudaStreamCreateWithFlags(&s_hold, cudaStreamNonBlocking), "stream");
+    ck(cudaStreamCreateWithFlags(&s_run, cudaStreamNonBlocking), "stream");
+    cudaEvent_t e0, e1;
+    ck(cudaEventCreate(&e0), "event");
+    ck(cudaEventCreate(&e1), "event");
+    unsigned long long all = 0;   // holding blocks launched so far: sitting counts every one that sat down
+    unsigned gen = 0;
+    auto hold = [&](int n) {   // all but n SMs, before the next launch on s_run
+        if (n >= sms) return;
+        ++gen;
+        all += (unsigned long long) (sms - n);
+        hold_sm<<<sms - n, 1, optin, s_hold>>>(sitting, release, gen, where, late);
+        ck(cudaGetLastError(), "hold_sm");
+        const cudaError_t q = cudaStreamQuery(s_hold);   // hands the launch to the GPU now (Windows collects launches)
+        if (q != cudaErrorNotReady) ck(q, "hold_sm");
+        (void) cudaGetLastError();   // not ready is no error
+        wait_sitting<<<1, 1, 0, s_run>>>(sitting, all, late);
+    };
+    auto let_go = [&](int n) {
+        if (n < sms) release_sms<<<1, 1, 0, s_run>>>(release, gen);
+    };
+    auto settle = [&](const char* what) {
+        ck(cudaStreamSynchronize(s_run), what);
+        ck(cudaStreamSynchronize(s_hold), what);
+    };
+    auto count = [&](const unsigned* c) {   // after settle()
+        unsigned h = 0;
+        ck(cudaMemcpy(&h, c, sizeof(h), cudaMemcpyDeviceToHost), "count");
+        return h;
+    };
+    auto done = [&] {
+        cudaEventDestroy(e0);
+        cudaEventDestroy(e1);
+        cudaStreamDestroy(s_hold);
+        cudaStreamDestroy(s_run);
+        cudaFree(state); cudaFree(oc); cudaFree(sitting); cudaFree(flags); cudaFree(where);
+    };
+
+    // where the grids land, once per N
+    struct Landing { bool held = true; int busiest[kRecCount] = {}, on[kRecCount] = {}; };
+    std::vector<Landing> lands(ns.size());
+    for (size_t i = 0; i < ns.size(); ++i) {
+        const int n = ns[i];
+        for (int v = 0; v < kRecCount; ++v) {
+            bool sat = false;
+            unsigned apart0 = 0;
+            for (int attempt = 0; attempt < 2 && !sat; ++attempt) {
+                const unsigned late0 = count(late);
+                apart0 = count(apart);
+                ck(cudaMemsetAsync(landed, 0, sizeof(unsigned), s_run), "memset");
+                hold(n);
+                land<<<grid[v], dim3(CB, RG), land_smem[v], s_run>>>(where + sms, landed, apart);
+                ck(cudaGetLastError(), "land");
+                let_go(n);
+                settle("land");
+                sat = count(late) == late0;
+            }
+            if (!sat) {   // twice: the holding kernel and the launches beside it do not run at the same time here
+                std::printf("    not on this card: the other SMs could not be held (the kernel holding them and the "
+                            "launches beside it did not run at the same time)\n");
+                done();
+                return;
+            }
+            std::vector<int> w((size_t) sms + grid[v]);
+            ck(cudaMemcpy(w.data(), where, w.size() * sizeof(int), cudaMemcpyDeviceToHost), "where");
+            std::map<int, int> held, blocks;
+            for (int b = 0; b < (n < sms ? sms - n : 0); ++b) ++held[w[b]];
+            for (int b = 0; b < grid[v]; ++b) ++blocks[w[sms + b]];
+            bool ok = count(apart) == apart0 && (int) held.size() == (n < sms ? sms - n : 0) && (int) blocks.size() <= n;
+            int most = 0, on = 0;
+            for (const auto& sb : blocks) {
+                if (held.count(sb.first)) ok = false;
+                if (sb.second > most) { most = sb.second; on = 0; }
+                if (sb.second == most) ++on;
+            }
+            lands[i].held = lands[i].held && ok;
+            lands[i].busiest[v] = most;
+            lands[i].on[v] = on;
+        }
+    }
+
+    // the timings: every N and recurrence once per round, a different one first in each round
+    struct Case { size_t i; Rec v; };
+    std::vector<Case> cases;
+    for (size_t i = 0; i < ns.size(); ++i)
+        if (lands[i].held)
+            for (int v = 0; v < kRecCount; ++v) cases.push_back({i, (Rec) v});
+    auto run = [&](const Case& cs) {   // ms, or < 0 if the SMs were not held for the whole launch
+        const int n = ns[cs.i];
+        const unsigned late0 = count(late);
+        ck(cudaMemcpyAsync(state, in.state0, st * 4, cudaMemcpyDeviceToDevice, s_run), "state");
+        hold(n);
+        ck(cudaEventRecord(e0, s_run), "event");
+        run_rec(cs.v, state, in, oc, s_run);
+        ck(cudaEventRecord(e1, s_run), "event");
+        let_go(n);
+        settle("fewer SMs");
+        if (count(late) != late0) return -1.0f;
+        float ms = 0.0f;
+        ck(cudaEventElapsedTime(&ms, e0, e1), "event");
+        return ms;
+    };
+    const int warm = 2, rounds = 15, max_dropped = 5;
+    std::vector<std::vector<float>> t(cases.size());
+    int dropped = 0;
+    for (int r = 0; r < warm + rounds && dropped <= max_dropped; ++r)
+        for (size_t j = 0; j < cases.size() && dropped <= max_dropped; ++j) {
+            const size_t k = (r + j) % cases.size();
+            const float ms = run(cases[k]);
+            if (ms < 0.0f) ++dropped;
+            else if (r >= warm) t[k].push_back(ms);
+        }
+    if (dropped > max_dropped) {   // each such run waits for a limit of a second or so
+        std::printf("    stopped: in %d runs the other SMs were not held for the whole launch\n", dropped);
+        done();
+        return;
+    }
+    auto us = [&](size_t i, int v) {   // per token, or < 0 if not measured
+        for (size_t k = 0; k < cases.size(); ++k)
+            if (cases[k].i == i && cases[k].v == v && !t[k].empty()) return 1000.0f * median(t[k]) / (float) T;
+        return -1.0f;
+    };
+    auto blocks_on = [](char* out, size_t size, int most, int on) {
+        std::snprintf(out, size, "%d block%s on %d SM%s", most, most == 1 ? "" : "s", on, on == 1 ? "" : "s");
+    };
+    std::printf("  (medians of %d alternating runs; L2, memory and clock stay this card's)\n", rounds);
+    std::printf("      N   before: busiest SM   new: busiest SM      us per token, before -> new\n");
+    for (size_t i = 0; i < ns.size(); ++i) {
+        if (!lands[i].held) {
+            std::printf("    %3d   the other SMs were not held (a block landed on one, or a wait ran out)\n", ns[i]);
+            continue;
+        }
+        char b0[40], b1[40];
+        blocks_on(b0, sizeof b0, lands[i].busiest[kBefore], lands[i].on[kBefore]);
+        blocks_on(b1, sizeof b1, lands[i].busiest[kKeyHead], lands[i].on[kKeyHead]);
+        const float u0 = us(i, kBefore), u1 = us(i, kKeyHead);
+        if (u0 < 0.0f || u1 < 0.0f) std::printf("    %3d   %-20s %-20s not measured\n", ns[i], b0, b1);
+        else std::printf("    %3d   %-20s %-20s %.3f -> %.3f   %.2fx\n", ns[i], b0, b1, u0, u1, u0 / u1);
+    }
+    if (dropped) std::printf("    (%d runs dropped: the other SMs were not held for the whole launch)\n", dropped);
+    std::printf("  so on a card with N SMs, as measured here (the kernel before puts ceil(192 / N) blocks on the busiest SM, "
+                "gdn_rec_kh_kernel ceil(64 / N)):\n");
+    const int bands[][2] = {{96, 191}, {64, 95}, {48, 63}, {39, 47}, {32, 38}};
+    for (const auto& bd : bands) {
+        const int b0 = (192 + bd[0] - 1) / bd[0], b1 = (64 + bd[0] - 1) / bd[0];
+        float lo = 0.0f, hi = 0.0f;
+        int cnt = 0, nmin = 0, nmax = 0;
+        for (size_t i = 0; i < ns.size(); ++i) {
+            const float u0 = us(i, kBefore), u1 = us(i, kKeyHead);
+            if (ns[i] < bd[0] || ns[i] > bd[1] || u0 < 0.0f || u1 < 0.0f) continue;
+            const float x = u0 / u1;
+            if (cnt == 0 || x < lo) lo = x;
+            if (cnt == 0 || x > hi) hi = x;
+            if (cnt == 0 || ns[i] < nmin) nmin = ns[i];
+            if (cnt == 0 || ns[i] > nmax) nmax = ns[i];
+            ++cnt;
+        }
+        char range[32];
+        std::snprintf(range, sizeof range, "%d-%d SMs", bd[0], bd[1]);
+        if (cnt > 0)
+            std::printf("    %-12s before %d / new %d per SM: %.2fx to %.2fx (%d to %d SMs)\n", range, b0, b1, lo, hi, nmin,
+                        nmax);
+        else if (bd[0] > sms)
+            std::printf("    %-12s before %d / new %d per SM: more SMs than this card has (see the estimate above)\n", range,
+                        b0, b1);
+        else
+            std::printf("    %-12s before %d / new %d per SM: not measured\n", range, b0, b1);
+    }
+    done();
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    bool do_bench = false;
+    for (int i = 1; i < argc; ++i)
+        if (std::strcmp(argv[i], "--bench") == 0) do_bench = true;
+    cudaDeviceProp p{};
+    ck(cudaGetDeviceProperties(&p, 0), "device");
+    std::printf("gdn_rec_parity on %s (sm_%d%d)\n", p.name, p.major, p.minor);
+    int fails = 0;
+    const int64_t Ts[] = {1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 33, 257, 1000, 4099};
+    uint64_t seed = 11;
+    for (const int64_t T : Ts) fails += check_bits(T, seed++);
+    if (do_bench) {
+        std::printf("--bench: the prompt path's DeltaNet recurrence\n");
+        bench(8192);
+        bench(32768);
+        scale(8192);
+        fewer_sms(8192);
+    }
+    std::printf("gdn_rec_parity: %d failures\n", fails);
+    return fails ? 1 : 0;
+}

src/prefill/ggml_cuda_host.cu

@@ -80,17 +80,30 @@ const ggml_cuda_device_info & ggml_cuda_info() {
             d.smpbo = prop.sharedMemPerBlock;
             d.integrated = false;
             d.supports_cooperative_launch = false;
-#else
-            d.cc = 100 * strata::cc_major_of(prop.major) + 10 * strata::cc_minor_of(prop.minor);   // STRATA_EMULATE_CC
-            d.smpbo = strata::smem_optin_of((int) prop.sharedMemPerBlockOptin);
-            d.integrated = prop.integrated != 0;
-            d.supports_cooperative_launch = prop.cooperativeLaunch != 0;
-#endif
             d.nsm = prop.multiProcessorCount;
             d.smpb = prop.sharedMemPerBlock;
+            d.warp_size = prop.warpSize;
+#else
+            // #542: the fields MMQ's choices read come from cudaDeviceGetAttribute, which is ABI-stable: a
+            // cudaDeviceProp filled by an older libcudart than the headers (a CUDA 13 build that links a CUDA 12
+            // libcudart.so) is shifted - sharedMemPerBlockOptin read 1 there, and every i-quant lost MMQ (#420's
+            // fits()).  In a matched build both give the same values.
+            auto attr = [id](cudaDeviceAttr a) {
+                int v = 0;
+                CUDA_CHECK(cudaDeviceGetAttribute(&v, a, id));
+                return v;
+            };
+            d.cc = 100 * strata::cc_major_of(attr(cudaDevAttrComputeCapabilityMajor)) +
+                   10 * strata::cc_minor_of(attr(cudaDevAttrComputeCapabilityMinor));   // STRATA_EMULATE_CC
+            d.smpbo = strata::smem_optin_of(attr(cudaDevAttrMaxSharedMemoryPerBlockOptin));
+            d.integrated = attr(cudaDevAttrIntegrated) != 0;
+            d.supports_cooperative_launch = attr(cudaDevAttrCooperativeLaunch) != 0;
+            d.nsm = attr(cudaDevAttrMultiProcessorCount);
+            d.smpb = (size_t) attr(cudaDevAttrMaxSharedMemoryPerBlock);
+            d.warp_size = attr(cudaDevAttrWarpSize);
+#endif
             d.vmm = false;
             d.total_vram = prop.totalGlobalMem;
-            d.warp_size = prop.warpSize;
             d.physical_device = id;
             d.physical_share_count = 1;
             d.virtual_index = 0;

src/prefill/kernels.cu

@@ -433,8 +433,172 @@ __global__ void __launch_bounds__(CB * RG) gdn_rec_cols_pipe_kernel(float* __res
 #pragma unroll
     for (int r = 0; r < RPG; ++r) base[r * rs] = s[r];
 }
+#if !defined(__HIPCC__)
+// The recurrence with one thread for the three value heads that share a key head (head % HK): column c of heads
+// qh, qh + 16 and qh + 32, row group rg.  gdn_rec_cols_pipe_kernel spends its time in shared memory, not in
+// arithmetic: every thread of a warp needs the same 32 q and k values per token (the k twice), and a warp receives one
+// such broadcast value per clock however wide the load.  Here every q/k value a thread loads feeds three heads, and a
+// token's k row goes into registers once for both of its uses.  The inputs come in blocks of GDN_TB tokens, copied to
+// shared memory by cp.async while the block before computes (one token ahead is shorter than a load from L2 takes),
+// and the two cross-row-group sums have their own arrays, so a token needs 2 __syncthreads instead of 5: the second
+// one of a token orders every read of rkv before the next token's writes, the next token's first one every read of ro
+// before the writes after it.  64 blocks instead of 192.  Per value head and column the same arithmetic in the same
+// order: the same bits (src/prefill/gdn_rec_parity.cu checks them and times the variants: 1.41x on a 4080 Super).
+// sm_80+ cards that hold its 64 blocks at once (gdn_keyhead_ok); STRATA_GDN_KEYHEAD=0: gdn_rec_cols_pipe_kernel.
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
+#define STRATA_GDN_CP_ASYNC 0   // Turing builds: plain copies (never launched there, see gdn_keyhead_ok)
+#else
+#define STRATA_GDN_CP_ASYNC 1
+#endif
+__device__ __forceinline__ void gdn_cp4(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *smem = *gmem;
+#endif
+}
+__device__ __forceinline__ void gdn_cp16(float* smem, const float* gmem) {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
+#else
+    *reinterpret_cast<float4*>(smem) = *reinterpret_cast<const float4*>(gmem);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_commit() {
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.commit_group;\n" ::);
+#endif
+}
+__device__ __forceinline__ void gdn_cp_wait_prev() {   // every group but the newest has landed
+#if STRATA_GDN_CP_ASYNC
+    asm volatile("cp.async.wait_group 1;\n" ::);
+#endif
+}
+constexpr int GDN_TB = 8, VPK = HV / HK;   // tokens per staged block, value heads per key head
+__global__ void __launch_bounds__(CB * RG) gdn_rec_kh_kernel(float* __restrict__ state, const float* __restrict__ h,
+                                                               const float* __restrict__ gate,
+                                                               const float* __restrict__ beta,
+                                                               float* __restrict__ oc_out, int64_t T) {
+    constexpr int TB = GDN_TB, NT = CB * RG, QKP = S / 4, VP = CB / 4;   // threads, 16-byte pieces of a q/k row, of v
+    __shared__ __align__(16) float sq[2][TB][S];
+    __shared__ __align__(16) float sk[2][TB][S];
+    __shared__ __align__(16) float sv[2][TB][VPK][CB];
+    __shared__ float sg[2][TB][VPK], sb[2][TB][VPK], rkv[VPK][RG][CB], ro[VPK][RG][CB];
+    const int qh = blockIdx.x / NCB, cb = blockIdx.x % NCB;
+    const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
+    float s[VPK][RPG];
+    const size_t rs = (size_t) HV * S;
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        const float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) s[j][r] = base[r * rs];
+    }
+    const int64_t nblk = (T + TB - 1) / TB;
+    auto stage = [&](int64_t k) {   // tokens [k * TB, k * TB + TB) into buffer k & 1
+        const int bb = (int) (k & 1);
+        const int64_t t0 = k * TB;
+        for (int p = tid; p < TB * 2 * QKP; p += NT) {
+            const int i = p / (2 * QKP), w = p % (2 * QKP), isk = w / QKP, jj = (w % QKP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(isk ? &sk[bb][i][jj] : &sq[bb][i][jj], h + (t0 + i) * C + (isk ? HK * S : 0) + qh * S + jj);
+        }
+        for (int p = tid; p < TB * VPK * VP; p += NT) {
+            const int i = p / (VPK * VP), w = p % (VPK * VP), j = w / VP, jj = (w % VP) * 4;
+            if (t0 + i < T)
+                gdn_cp16(&sv[bb][i][j][jj], h + (t0 + i) * C + 2 * HK * S + (qh + j * HK) * S + cb * CB + jj);
+        }
+        for (int p = tid; p < 2 * TB * VPK; p += NT) {
+            const int isb = p / (TB * VPK), w = p % (TB * VPK), i = w / VPK, j = w % VPK;
+            if (t0 + i < T)
+                gdn_cp4(isb ? &sb[bb][i][j] : &sg[bb][i][j], (isb ? beta : gate) + (t0 + i) * HV + qh + j * HK);
+        }
+    };
+    if (nblk > 0) stage(0);
+    gdn_cp_commit();
+    for (int64_t k = 0; k < nblk; ++k) {
+        // buffer (k + 1) & 1 was read by block k - 1, whose last token's second __syncthreads every thread has passed
+        if (k + 1 < nblk) stage(k + 1);
+        gdn_cp_commit();
+        gdn_cp_wait_prev();
+        __syncthreads();
+        const int bb = (int) (k & 1);
+        const int n = (int) ((T - k * TB) < TB ? (T - k * TB) : TB);
+        for (int i = 0; i < n; ++i) {
+            const int64_t t = k * TB + i;
+            float kc[RPG];
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) kc[r] = sk[bb][i][rg * RPG + r];
+            float g[VPK], kv[VPK], delta[VPK], o[VPK];
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) { g[j] = __expf(sg[bb][i][j]); kv[j] = 0.0f; o[j] = 0.0f; }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r)
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) kv[j] = fmaf(s[j][r], kc[r], kv[j]);
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) rkv[j][rg][c] = kv[j];
+            __syncthreads();
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) {
+                const float kv_col = rkv[j][0][c] + rkv[j][1][c] + rkv[j][2][c] + rkv[j][3][c];
+                delta[j] = (sv[bb][i][j][c] - g[j] * kv_col) * sb[bb][i][j];
+            }
+#pragma unroll
+            for (int r = 0; r < RPG; ++r) {
+                const float qr = sq[bb][i][rg * RPG + r];
+#pragma unroll
+                for (int j = 0; j < VPK; ++j) {
+                    s[j][r] = fmaf(g[j], s[j][r], kc[r] * delta[j]);
+                    o[j] = fmaf(s[j][r], qr, o[j]);
+                }
+            }
+#pragma unroll
+            for (int j = 0; j < VPK; ++j) ro[j][rg][c] = o[j];
+            __syncthreads();
+            if (rg < VPK)   // row group j writes head j's output
+                oc_out[t * HV * S + (qh + rg * HK) * S + col] =
+                    (ro[rg][0][c] + ro[rg][1][c] + ro[rg][2][c] + ro[rg][3][c]) * rsqrtf((float) S);
+        }
+    }
+#pragma unroll
+    for (int j = 0; j < VPK; ++j) {
+        float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
+#pragma unroll
+        for (int r = 0; r < RPG; ++r) base[r * rs] = s[j][r];
+    }
+}
+// gdn_rec_kh_kernel where it pays: a CUDA card with cp.async (sm_80+) that holds all 64 of its blocks at once (each
+// walks the whole chunk, so blocks left for a second wave would double the time).  The busiest SM sets the pace: from
+// 64 SMs up this kernel has one block per SM, below that two on some SMs, while the kernel before has ceil(192 / SMs).
+// With the engine's grids on fewer SMs (gdn_rec_parity --bench), on a 4080 SUPER / a 3090: 1.40-1.42x / 1.28-1.29x at
+// 64 SMs and more, 1.02-1.04x / 1.03-1.06x at 48 to 63 (two blocks against four: a draw, slower on neither),
+// 1.28-1.31x / 1.28-1.30x at 39 to 47, 1.53-1.57x / 1.54-1.55x at 32 to 38.  Per call, from the current device (a
+// layer split can mix cards).
+bool gdn_keyhead_ok() {
+    static const bool off = [] { const char* v = std::getenv("STRATA_GDN_KEYHEAD"); return v != nullptr && std::atoi(v) == 0; }();
+    if (off) return false;
+    static int known[64] = {};   // per device: 0 not asked yet, 1 yes, 2 no
+    int dev = 0;
+    if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+    if (known[dev] == 0) {
+        int major = 0, sms = 0, per_sm = 0;
+        const bool yes = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+                         cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev) == cudaSuccess && major >= 8 &&
+                         cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm, gdn_rec_kh_kernel, CB * RG, 0) ==
+                             cudaSuccess &&
+                         (int64_t) per_sm * sms >= (int64_t) HK * NCB;
+        if (!yes) cudaGetLastError();
+        known[dev] = yes ? 1 : 2;
+    }
+    return known[dev] == 1;
+}
+#endif
+// the output norm over a head's 128 columns, into the FP16 copy the out projection reads (the FP32 output before the
+// norm stays in its scratch buffer)
 __global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
-                                                         float eps, float* __restrict__ y, uint16_t* __restrict__ y16) {
+                                                         float eps, const float* __restrict__ y,
+                                                         uint16_t* __restrict__ y16) {
     __shared__ float wsum[4];
     const int64_t t = blockIdx.x;
     const int head = blockIdx.y, col = threadIdx.x;
@@ -445,7 +609,6 @@ __global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict
     __syncthreads();
     const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
     const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
-    y[at] = v;
     y16[at] = hf(v);
 }
 
@@ -748,6 +911,11 @@ void gdn_recurrence(float* state, const float* h, const float* gate, const float
         gdn_rec_kernel<<<HV, dim3(S, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, z, gamma, eps, y, y16, T);
     } else {
         static const bool pipe = [] { const char* v = std::getenv("STRATA_GDN_PIPELINE"); return v == nullptr || std::atoi(v) != 0; }();
+#if !defined(__HIPCC__)
+        if (pipe && gdn_keyhead_ok())   // the value heads of a key head in one thread (same bits)
+            gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);
+        else
+#endif
         if (pipe)   // the software-pipelined loads (same bits)
             gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);
         else

src/prefill/moe_mmq.cu

@@ -29,6 +29,22 @@ __global__ void copy16_kernel(const uint4* __restrict__ a, int64_t na, const uin
     else if (i < na + nb) ab_dst[i] = b[i - na];
     else if (i < na + nb + nc) c_dst[i - na - nb] = c[i - na - nb];
 }
+// copy16_kernel for an MMQ group: blockIdx.y is the expert (first + y)
+struct GroupArgs {
+    const uint8_t* blob[kGatherGroupMax];
+    int64_t up_off, down_off, gu_stride, d_stride;   // in uint4
+};
+__global__ void copy16_group_kernel(GroupArgs ga, int first, int64_t na, int64_t nc, uint4* __restrict__ gu_dst,
+                                    uint4* __restrict__ d_dst) {
+    const int q = first + (int) blockIdx.y;
+    const uint4* src = (const uint4*) ga.blob[q];
+    uint4* ab = gu_dst + (int64_t) q * ga.gu_stride;
+    uint4* cd = d_dst + (int64_t) q * ga.d_stride;
+    const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < na) ab[i] = src[i];
+    else if (i < 2 * na) ab[i] = src[ga.up_off + (i - na)];
+    else if (i < 2 * na + nc) cd[i - 2 * na] = src[ga.down_off + (i - 2 * na)];
+}
 __global__ void copy1_kernel(const uint8_t* __restrict__ a, int64_t na, const uint8_t* __restrict__ b, int64_t nb,
                              uint8_t* __restrict__ ab_dst, const uint8_t* __restrict__ c, int64_t nc, uint8_t* __restrict__ c_dst) {
     const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
@@ -210,6 +226,25 @@ void gather_native(const void* gate, const void* up, size_t gu_half_bytes, const
     ck(cudaGetLastError(), "gather_native");
 }
 
+bool gather_native_group(const GatherGroup& g, size_t up_off, size_t gu_half_bytes, size_t down_off, size_t d_bytes,
+                         void* gu_dst, size_t gu_stride, void* d_dst, size_t d_stride, void* stream) {
+    if (g.first < 0 || g.n <= g.first || g.n > kGatherGroupMax) return false;
+    uintptr_t a = (uintptr_t) gu_dst | (uintptr_t) d_dst | up_off | gu_half_bytes | down_off | d_bytes | gu_stride | d_stride;
+    for (int q = g.first; q < g.n; ++q) a |= (uintptr_t) g.blob[q];
+    if (a % 16 != 0) return false;
+    GroupArgs ga{};
+    for (int q = g.first; q < g.n; ++q) ga.blob[q] = g.blob[q];
+    ga.up_off = (int64_t) up_off / 16;
+    ga.down_off = (int64_t) down_off / 16;
+    ga.gu_stride = (int64_t) gu_stride / 16;
+    ga.d_stride = (int64_t) d_stride / 16;
+    const int64_t na = (int64_t) gu_half_bytes / 16, nc = (int64_t) d_bytes / 16;
+    copy16_group_kernel<<<dim3(blocks(2 * na + nc), (unsigned) (g.n - g.first)), 256, 0, (cudaStream_t) stream>>>(
+        ga, g.first, na, nc, (uint4*) gu_dst, (uint4*) d_dst);
+    ck(cudaGetLastError(), "gather_native_group");
+    return true;
+}
+
 void gather_strata_q2(const uint8_t* blob, void* gu_dst, void* d_dst, void* stream) {
     strata_q2_kernel<<<blocks(1280LL * 40 + 2560LL * 10), 256, 0, (cudaStream_t) stream>>>(blob, (uint16_t*) gu_dst,
                                                                                          (uint16_t*) d_dst);

src/prefill/prefill.cpp

@@ -26,6 +26,7 @@
 #include "strata/prefill/moe_fused.hpp"
 #include "strata/prefill/moe_fused_iq.hpp"
 #include "strata/prefill/moe_mmq.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/prefill/kernels.hpp"
 
 #include <cuda_runtime.h>
@@ -56,6 +57,7 @@ Context::Context() {}
 Context::~Context() {}
 void Context::run(const Product&, void*) {}
 void gather_native(const void*, const void*, size_t, const void*, size_t, void*, void*, void*) {}
+bool gather_native_group(const GatherGroup&, size_t, size_t, size_t, size_t, void*, size_t, void*, size_t, void*) { return false; }
 void gather_strata_q2(const uint8_t*, void*, void*, void*) {}
 void swiglu(const float*, float*, int64_t, int64_t, bool, void*) {}
 void iota(int32_t*, int64_t, void*) {}
@@ -118,14 +120,19 @@ int g_ring_override = 0;   // #340: set by a layer split (Prefill::set_ring_over
 // take any of its layers: IQ2_XS, 4K / 32K, their first version at 384 slots -3% / -6% against MMQ, at 512 +8% / 0%.
 inline bool fused_ring() {
     if (!fused::enabled()) return false;
+    if (core::peer_portable()) return false;   // multi-GPU: --peer-device keeps the MMQ path and its buffer sizes
     const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();
     if (!lay.native) return true;
-    static const bool any = [&lay] {
+    // EVERY layer: fused_layout() shrinks the MoE buffers to the fused path's needs, so a layer the native kernels do
+    // not cover (Unsloth UD-IQ4_XS's Q8_0 down projections) would run MMQ in them at the full chunk and overflow them
+    // (garbage, an illegal memory access or a hung prompt on gfx1151).  A pack with such a layer keeps MMQ's buffers;
+    // its covered layers still take the fused kernels.
+    static const bool all = [&lay] {
         for (const auto& f : lay.fmt)
-            if (fused::native_supported(f.gu_type, f.d_type)) return true;
-        return false;
+            if (!fused::native_supported(f.gu_type, f.d_type)) return false;
+        return !lay.fmt.empty();
     }();
-    return any;
+    return all;
 }
 // the largest ring: 512 slots; 1024 with the Q2_0 pack's fused experts (P3's smaller buffers, measured there) - the
 // native packs' fused layers were measured at 512
@@ -271,10 +278,12 @@ struct Stager {
                 const int j = claim(seen);
                 if (j < 0) { active.fetch_sub(1, std::memory_order_acq_rel); break; }
                 const int b = j % kRing;
-                if (j >= kRing) {
+                if (j >= kRing)   // job j - kRing's DMA from this buffer is queued
                     while (issued.load(std::memory_order_acquire) <= j - kRing) std::this_thread::yield();
-                    cudaEventSynchronize(dma_done[b]);
-                }
+                // and done - for a generation's first kRing jobs that is the previous generation's last DMA from
+                // the buffer, which nothing else waits for when a chunk ends without a sync (no MTP) or the DMA
+                // was a ring entry the routing skipped (an event never recorded returns at once)
+                cudaEventSynchronize(dma_done[b]);
                 const Job& jb = jobs[(size_t) j];
                 if (jb.from == nullptr) std::memcpy(buf[b], jb.src, jb.bytes);
                 else if (!jb.from->copy_blob(jb.l, jb.e, buf[b])) {
@@ -328,6 +337,81 @@ struct Stager {
     }
 };
 
+// multi-GPU: the peer GPU's share of a prompt chunk's experts.  Per MoE layer the primary copies its normed
+// activations over P2P, the peer quantizes the rows routed to the experts it holds, runs the same MMQ products the
+// primary would (gathered from its own slots), and copies the result rows back into the primary's Dm rows - the rows
+// are laid out local-first, so the peer's are one contiguous block at the end.
+struct PeerPrefill {
+    core::PeerExperts* peer = nullptr;
+    int dev = -1;
+    int64_t cap_rows = 0, T_max = 0;
+    cudaStream_t s = nullptr;
+    cudaEvent_t ev_in = nullptr;    // on the primary: the activations and the row tables are ready
+    cudaEvent_t ev_done = nullptr;  // on the peer: its rows have landed in the primary's Dm
+    // multi-GPU: the result rows go back group by group on a second stream while the next group computes
+    cudaStream_t s_out = nullptr;
+    static constexpr int kGrpEv = NE / 16 + 2;
+    cudaEvent_t ev_grp[kGrpEv] = {};
+    bool out_pending = false;       // s_out may still read Dm (the next layer's products wait for it)
+    bool out_pipe = true;
+    // multi-GPU COMPACT: the peer's per-row buffers hold one GROUP's rows instead of the layer's (a group
+    // = up to 16 experts and at most G rows), the result rows double-buffered: ~0.9 GB -> ~0.26 GB on the helper
+    // card at 8192-token chunks, and no row cap (every peer-held expert's rows go to the peer).
+    // STRATA_PF_PEER_COMPACT=0: the layer-sized buffers (the A/B).
+    bool compact = true;
+    int64_t G = 0;
+    void *Xq_g = nullptr, *Hq_g = nullptr;
+    float *GU_g = nullptr, *H_g = nullptr, *Dm_b[2] = {};
+    cudaEvent_t ev_dm[2] = {};
+    bool dm_live[2] = {};
+    std::vector<std::pair<size_t, size_t>> groups;
+    // multi-GPU PEER STREAMING: a share of the experts the primary would stream over ITS PCIe link in a
+    // big chunk is streamed by the peer over its own link instead (into its own ring) and computed there - it halves
+    // the primary's copy load and moves expert work to the card that idles through most of the MoE half.
+    // STRATA_PF_PEER_STREAM = the share (0 = off), STRATA_PF_PEER_RING = its ring slots.
+    double ps_frac = 0.0;
+    int RP = 0;
+    std::vector<uint8_t*> pstage;
+    std::vector<cudaEvent_t> pcopied, pused;
+    std::vector<char> plive;
+    cudaStream_t s_cp = nullptr;
+    struct PsEntry { int32_t l, e; const uint8_t* blob; };
+    std::vector<PsEntry> pseq;            // this chunk's peer-streamed experts, layer by layer in id order
+    std::vector<size_t> pseq_start;
+    std::vector<char> ps_flag;            // [layer * NE + e]: streamed by the peer in this chunk
+    size_t p_issued = 0, pk = 0;
+    int64_t ps_experts = 0;
+    float *mixed = nullptr, *GU = nullptr, *H = nullptr, *Dm = nullptr;
+    void *Xq = nullptr, *Hq = nullptr;
+    int32_t *src = nullptr, *bounds = nullptr, *ident = nullptr;
+    uint8_t *grp_gu = nullptr, *grp_d = nullptr;
+    std::unique_ptr<mmq::Context> ctx;
+    std::vector<int32_t> bounds_host;
+    std::vector<void*> owned;
+    int64_t layers = 0, experts = 0, rows = 0, over_cap = 0;   // stats
+    ~PeerPrefill() {
+        if (dev < 0) return;
+        int prev = 0;
+        cudaGetDevice(&prev);
+        cudaSetDevice(dev);
+        if (s) cudaStreamSynchronize(s);
+        ctx.reset();
+        for (void* p : owned) cudaFree(p);
+        if (ev_done) cudaEventDestroy(ev_done);
+        if (s_out) cudaStreamSynchronize(s_out);
+        for (cudaEvent_t e : ev_grp) if (e) cudaEventDestroy(e);
+        for (cudaEvent_t e : ev_dm) if (e) cudaEventDestroy(e);
+        if (s_cp) cudaStreamSynchronize(s_cp);
+        for (cudaEvent_t e : pcopied) if (e) cudaEventDestroy(e);
+        for (cudaEvent_t e : pused) if (e) cudaEventDestroy(e);
+        if (s_cp) cudaStreamDestroy(s_cp);
+        if (s_out) cudaStreamDestroy(s_out);
+        if (s) cudaStreamDestroy(s);
+        cudaSetDevice(prev);
+        if (ev_in) cudaEventDestroy(ev_in);
+    }
+};
+
 struct Prefill::Impl {
     const core::WeightTable* wt = nullptr;
     const core::ModelGeometry* g = nullptr;
@@ -389,6 +473,9 @@ struct Prefill::Impl {
     std::unique_ptr<Stager> stager;          // the unpinned experts' host copies (step 4)
     cudaEvent_t copied[RING_MAX] = {}, used[RING_MAX] = {};
     bool stage_live[RING_MAX] = {};
+    // the event that releases each ring slot: its own `used`, or - when an MMQ group is gathered in one launch - the
+    // `used` of the last slot gathered with it, recorded once for all of them (a later record only waits longer)
+    int used_of[RING_MAX] = {};
     // PLE
     float* ple_emb = nullptr;
     std::vector<float> ple_pageable[2];      // the fallback when no more RAM can be pinned
@@ -408,6 +495,7 @@ struct Prefill::Impl {
     // C-4: the chunk's token ids on the device, for one batched embedding gather
     int32_t* tok_dev = nullptr;
     std::vector<int32_t> tok_host;
+    std::unique_ptr<PeerPrefill> pp;         // multi-GPU: the peer GPU's expert share (set_peer)
 };
 
 namespace {
@@ -636,7 +724,7 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, co
             m.grp_host = m.grp_dev = nullptr;
             m.grp_n = m.grp_tk = 0;
             void *h = nullptr, *d = nullptr;
-            if (cudaHostAlloc(&h, need * 4, cudaHostAllocMapped) == cudaSuccess &&
+            if (cudaHostAlloc(&h, need * 4, cudaHostAllocMapped | (core::peer_portable() ? cudaHostAllocPortable : 0)) == cudaSuccess &&
                 cudaHostGetDevicePointer(&d, h, 0) == cudaSuccess) {
                 m.grp_host = (int32_t*) h;
                 m.grp_dev = (int32_t*) d;
@@ -765,6 +853,7 @@ bool Prefill::carve(size_t T, void* alloc) {
     for (int i = 0; i < m.ring; ++i) {
         m.stage_dev[i] = o.take<uint8_t>((size_t) MAXBLOB(), ok);
         m.stage_live[i] = false;                        // a new buffer: nothing of an earlier layout to wait for
+        m.used_of[i] = i;
     }
     m.ple_emb = o.take<float>(T * N, ok);
     m.ple_norm = o.take<float>((size_t) strata::kernels::NG_HC_DIM, ok);
@@ -804,9 +893,14 @@ bool Prefill::draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t
                        std::string& err) {
     Impl& m = *impl_;
     static const bool off = [] { const char* v = std::getenv("STRATA_MTP_BATCH"); return v != nullptr && v[0] == '0'; }();
+    // A ring (KV streaming: the drafter's window, page p in slot p % n_slots over a host copy) takes the same appends
+    // with its own page table and host copy, as a streamed main layer does; the cells written are those the window can
+    // still reach (r0 below), which the ring holds, so no two of them share a slot.  STRATA_MTP_BATCH_RING=0: the
+    // drafter's own pass for a ring (the A/B).
+    static const bool ring_ok = [] { const char* v = std::getenv("STRATA_MTP_BATCH_RING"); return v == nullptr || v[0] != '0'; }();
     core::QsaState& st = mtp.kv_state_rw();
-    if (off || n <= 0 || m.g == nullptr || m.region == nullptr || st.kv_mode != 0 || st.kv_hybrid ||
-        mtp.device() != m.device)
+    if (off || n <= 0 || m.g == nullptr || m.region == nullptr || (st.kv_mode != 0 && !(st.kv_mode == 2 && ring_ok)) ||
+        st.kv_hybrid || mtp.device() != m.device)
         return false;
     const auto t0 = Clock::now();
     const core::ModelGeometry& g = *m.g;
@@ -839,7 +933,9 @@ bool Prefill::draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t
     const bool q8 = !f16_only && mmq::built() && mmq::fits(kQ8_0, Nn) && mmq::fits(kQ8_0, KV);   // #420
     const uint64_t per_row = 4 * (2 * Nn + 4 * HCN + LR + HC + Nn + 2 * KV + 1) + 2 * (Nn + 2 * HCN + LR + Nn) + 64 +
                              (q8 ? (uint64_t) mmq::q8_bytes(g.hc, Nn) + 4 * g.hc : 0);
-    const int64_t B = std::min<int64_t>(n - r0, (int64_t) (m.region_bytes / per_row) & ~(int64_t) 63);
+    int64_t B = std::min<int64_t>(n - r0, (int64_t) (m.region_bytes / per_row) & ~(int64_t) 63);
+    if (st.kv_mode == 2)   // a ring: one batch's cells must not share a slot (a batch can straddle one page more)
+        B = std::min<int64_t>(B, ((st.n_slots - 1) * strata::kernels::qsa_real_shapes().page_size) & ~(int64_t) 63);
     if (B < 64) return false;
     uint8_t* q = m.region;
     auto carve = [&](size_t bytes) { void* p = q; q += (bytes + 255) & ~(size_t) 255; return p; };
@@ -864,7 +960,12 @@ bool Prefill::draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t
     int32_t* ident = q8 ? (int32_t*) carve((size_t) B * g.hc * 4) : nullptr;
     int32_t* bnd = q8 ? (int32_t*) carve(16) : nullptr;
     if ((uint64_t) (q - m.region) > m.region_bytes) return false;
+    static const bool timing = std::getenv("STRATA_DRAFT_TIMING") != nullptr;   // debug: where this pass's time goes
+    if (timing) cudaStreamSynchronize(m.cs);
+    const auto ti0 = Clock::now();
     if (!mtp.idle(err)) return false;   // the drafter's own stream (its graph uploads) before this writes its K/V
+    const double ms_idle = ms_since(ti0);
+    const auto tl0 = Clock::now();
     strata::kernels::QsaShapes s = strata::kernels::qsa_real_shapes();
     s.n_head = g.n_head; s.n_head_kv = g.n_head_kv; s.head_dim = g.head_dim; s.idx_n_head = g.idx_q_heads;
     s.idx_dim = g.idx_key_dim;
@@ -944,8 +1045,110 @@ bool Prefill::draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t
         return false;
     }
     mtp.ms_prefill += ms_since(t0);
+    if (timing)
+        std::fprintf(stderr, "strata draft kv: %lld cells from %lld (first needed %lld), batch %lld: drafter idle %.1f ms, "
+                     "the batches %.1f ms, all %.1f ms\n", (long long) n, (long long) cell0, (long long) (cell0 + r0),
+                     (long long) B, ms_idle, ms_since(tl0), ms_since(t0));
     return true;
 }
+bool Prefill::set_peer(core::PeerExperts* peer, int64_t cap_rows, std::string& err) {
+    Impl& m = *impl_;
+    if (peer == nullptr || !peer->valid()) { m.pp.reset(); return true; }
+    if (!peer->p2p()) { err = "prefill peer: the two GPUs cannot access each other (no P2P)"; return false; }
+    if (!mmq_plan().any) { err = "prefill peer: needs the MMQ prompt path"; return false; }
+    auto pp = std::make_unique<PeerPrefill>();
+    pp->peer = peer;
+    pp->T_max = m.T_max;
+    pp->cap_rows = std::max<int64_t>(1, std::min<int64_t>(cap_rows, m.T_max * K));
+    const int64_t R = pp->cap_rows;
+    const MmqPlan& mp = mmq_plan();
+    if (cudaEventCreateWithFlags(&pp->ev_in, cudaEventDisableTiming) != cudaSuccess) { err = "prefill peer: event"; return false; }
+    int prev = 0;
+    cudaGetDevice(&prev);
+    pp->dev = peer->device();
+    cudaSetDevice(pp->dev);
+    bool ok = cudaStreamCreateWithFlags(&pp->s, cudaStreamNonBlocking) == cudaSuccess &&
+              cudaEventCreateWithFlags(&pp->ev_done, cudaEventDisableTiming) == cudaSuccess;
+    {
+        const char* v = std::getenv("STRATA_PF_PEER_OUT_PIPE");   // =0: one copy after the last group (the A/B)
+        pp->out_pipe = (v == nullptr || std::atoi(v) != 0) || pp->compact;   // compact always pipes
+    }
+    if (ok && pp->out_pipe) {
+        ok = cudaStreamCreateWithFlags(&pp->s_out, cudaStreamNonBlocking) == cudaSuccess;
+        for (int i = 0; ok && i < PeerPrefill::kGrpEv; ++i)
+            ok = cudaEventCreateWithFlags(&pp->ev_grp[i], cudaEventDisableTiming) == cudaSuccess;
+    }
+    auto take = [&](size_t bytes) -> void* {
+        void* p = nullptr;
+        if (!ok || cudaMalloc(&p, bytes) != cudaSuccess) { ok = false; return nullptr; }
+        pp->owned.push_back(p);
+        return p;
+    };
+    pp->mixed = (float*) take((size_t) m.T_max * N * 4);
+    {
+        const char* v = std::getenv("STRATA_PF_PEER_COMPACT");
+        pp->compact = v == nullptr || std::atoi(v) != 0;
+    }
+    if (pp->compact) {
+        pp->cap_rows = m.T_max * K;   // no row cap: only the row tables grow with it
+        pp->G = m.T_max;              // one expert never has more rows than the chunk has tokens
+        const int64_t Gr = pp->G;
+        pp->Xq_g = take(mmq::q8_bytes(Gr, N));
+        pp->Hq_g = take(mmq::q8_bytes(Gr, 640));
+        pp->GU_g = (float*) take((size_t) Gr * 1280 * 4);
+        pp->H_g = (float*) take((size_t) Gr * 640 * 4);
+        pp->Dm_b[0] = (float*) take((size_t) Gr * N * 4);
+        pp->Dm_b[1] = (float*) take((size_t) Gr * N * 4);
+        for (int b = 0; b < 2 && ok; ++b) ok = cudaEventCreateWithFlags(&pp->ev_dm[b], cudaEventDisableTiming) == cudaSuccess;
+        const char* fs = std::getenv("STRATA_PF_PEER_STREAM");
+        pp->ps_frac = fs ? std::atof(fs) : 0.35;   // measured: 0.25-0.5 all ~1950-1970 at 32K, 0.35 best
+        if (pp->ps_frac > 0.0 && !mp.fallback && m.T_max >= stream_all_min()) {
+            pp->RP = 48;
+            if (const char* pr = std::getenv("STRATA_PF_PEER_RING"); pr != nullptr) pp->RP = std::atoi(pr);
+            pp->pstage.assign((size_t) pp->RP, nullptr);
+            pp->pcopied.assign((size_t) pp->RP, nullptr);
+            pp->pused.assign((size_t) pp->RP, nullptr);
+            pp->plive.assign((size_t) pp->RP, 0);
+            ok = ok && cudaStreamCreateWithFlags(&pp->s_cp, cudaStreamNonBlocking) == cudaSuccess;
+            for (int i = 0; ok && i < pp->RP; ++i) {
+                pp->pstage[(size_t) i] = (uint8_t*) take((size_t) MAXBLOB());
+                ok = ok && cudaEventCreateWithFlags(&pp->pcopied[(size_t) i], cudaEventDisableTiming) == cudaSuccess &&
+                     cudaEventCreateWithFlags(&pp->pused[(size_t) i], cudaEventDisableTiming) == cudaSuccess;
+            }
+        } else {
+            pp->ps_frac = 0.0;
+        }
+    } else {
+        pp->Xq = take(mmq::q8_bytes(R, N));
+        pp->Hq = take(mmq::q8_bytes(R, 640));
+        pp->GU = (float*) take((size_t) R * 1280 * 4);
+        pp->H = (float*) take((size_t) R * 640 * 4);
+        pp->Dm = (float*) take((size_t) R * N * 4);
+    }
+    const int64_t Rt = pp->cap_rows;
+    pp->src = (int32_t*) take((size_t) Rt * 4);
+    pp->ident = (int32_t*) take((size_t) Rt * 4);
+    pp->bounds = (int32_t*) take((size_t) (2 * (NE + NE / MMQ_GROUP + 2)) * 4);
+    pp->grp_gu = (uint8_t*) take(MMQ_GROUP * mp.gu_max + MMQ_TAIL);
+    pp->grp_d = (uint8_t*) take(MMQ_GROUP * mp.d_max + MMQ_TAIL);
+    if (ok) {
+        pp->ctx = std::make_unique<mmq::Context>();
+        mmq::iota(pp->ident, pp->cap_rows, pp->s);
+        ok = cudaStreamSynchronize(pp->s) == cudaSuccess;
+    }
+    size_t fb = 0, tb = 0;
+    cudaMemGetInfo(&fb, &tb);
+    cudaSetDevice(prev);
+    if (!ok) { err = "prefill peer: the peer's buffers do not fit (raise --peer-reserve-mib or lower --peer-prefill-rows)"; return false; }
+    std::fprintf(stderr, "strata prefill: peer GPU %d computes its experts' rows of each prompt chunk (up to %lld rows per "
+                         "layer%s); %zu MiB left free on it\n", pp->dev, (long long) pp->cap_rows,
+                 pp->compact ? (pp->ps_frac > 0.0 ? (", compact group buffers, streams " + std::to_string((int) (pp->ps_frac * 100 + 0.5)) +
+                                                    "% of the primary's streamed experts through a " + std::to_string(pp->RP) + "-slot ring").c_str()
+                                                 : ", compact group buffers") : "", fb >> 20);
+    m.pp = std::move(pp);
+    return true;
+}
+
 void Prefill::set_pinned_share(double share) { g_pinned_share = share; }
 void Prefill::set_ring_override(int slots) { g_ring_override = slots > 0 ? slots : 0; }
 double Prefill::pinned_share() { return g_pinned_share; }
@@ -1055,6 +1258,47 @@ struct PfTimer {
         for (cudaEvent_t e : ev) cudaEventDestroy(e);
     }
 };
+// multi-GPU: the peer's own timeline (STRATA_PREFILL_TIMING): marks on the peer stream, folded with the primary's
+enum PePhase { kPeIdle, kPeMoeIn, kPeMoeGemm, kPeMoeOut, kPeCount };
+const char* const kPeNames[kPeCount] = {"idle", "moe in", "moe gemm", "moe out"};
+struct PeTimer {
+    bool on = std::getenv("STRATA_PREFILL_TIMING") != nullptr;
+    int dev = -1;
+    std::vector<cudaEvent_t> ev;
+    std::vector<int> ph;
+    size_t used = 0;
+    double ms[kPeCount] = {};
+    void mark(int phase, cudaStream_t s) {   // the peer device is current
+        if (!on) return;
+        if (used == ev.size()) {
+            cudaEvent_t e = nullptr;
+            cudaEventCreate(&e);
+            ev.push_back(e);
+            ph.push_back(0);
+        }
+        ph[used] = phase;
+        cudaEventRecord(ev[used], s);
+        ++used;
+    }
+    void fold() {   // every mark has completed
+        if (!on || used < 2) return;
+        for (size_t i = 0; i + 1 < used; ++i) {
+            float t = 0.0f;
+            if (cudaEventElapsedTime(&t, ev[i], ev[i + 1]) == cudaSuccess) ms[ph[i]] += t;
+        }
+        std::swap(ev[0], ev[used - 1]);
+        std::swap(ph[0], ph[used - 1]);
+        used = 1;
+    }
+    ~PeTimer() {
+        if (dev < 0) return;
+        int prev = 0;
+        cudaGetDevice(&prev);
+        cudaSetDevice(dev);
+        for (cudaEvent_t e : ev) cudaEventDestroy(e);
+        cudaSetDevice(prev);
+    }
+};
 }  // namespace
 
 bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err) {
@@ -1078,6 +1322,8 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                                 (uint64_t) g.ssm_conv_channels * (g.ssm_d_conv - 1);
     int32_t prev[2] = {ss.ple_prev[0], ss.ple_prev[1]};
     PfTimer pt;
+    PeTimer pe;
+    if (m.pp) pe.dev = m.pp->dev; else pe.on = false;
     const cudaStream_t cs = (cudaStream_t) m.cs;
     // the MMQ row table lives in the borrowed cache slots, which the refill after a prompt overwrites with experts:
     // write it again for every prompt (a layout is reused as long as the chunk and the slots are the same)
@@ -1175,26 +1421,43 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
             }
         }
         if (hand_in_ == nullptr) gr_broadcast(m.emb, m.R, T, m.cs);
-        // ---- the PLE rows of the whole chunk, one batched SSD request (read ahead on a thread, see ple_gather)
-        if (ple_on) {
+        // ---- the PLE rows of the whole chunk, one batched SSD request on a thread (see ple_gather).  A later chunk's
+        // were read ahead during the previous chunk; the first chunk's are read beside layer 0 - they are needed from
+        // layer 1 on, and gathering them here first left the GPU idle for the whole read (~0.4 s of a 32K prompt)
+        if (ple_on && !ple_next.valid())
+            ple_next = std::async(std::launch::async, [&ple_gather, &ple_next_err, c0, b = ple_buf] {
+                return ple_gather(c0, b, ple_next_err);
+            });
+        bool ple_pending = ple_on;
+        // the chunk's rows onto the device just before layer 1 reads them, and the next chunk's gather started
+        auto ple_land = [&]() -> bool {
+            if (!ple_pending) return true;
+            ple_pending = false;
             const auto tp = Clock::now();
-            if (!ple_next.valid()) {
-                if (!ple_gather(c0, ple_buf, err)) return false;
-            } else if (!ple_next.get()) {
+            if (!ple_next.get()) {
                 err = ple_next_err;
                 return false;
             }
-            cudaMemcpyAsync(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, cudaMemcpyHostToDevice, m.cs);
-            cudaEventRecord(m.ple_copied[ple_buf], m.cs);
+            if (cudaMemcpyAsync(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, cudaMemcpyHostToDevice, m.cs) !=
+                    cudaSuccess ||
+                cudaEventRecord(m.ple_copied[ple_buf], m.cs) != cudaSuccess) {
+                err = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError());
+                return false;
+            }
             if (c0 + m.T < n) {
-                cudaEventSynchronize(m.ple_copied[ple_buf ^ 1]);   // the other buffer's upload (a chunk ago) is done
+                // the other buffer's upload (a chunk ago) is done before the SSD thread refills it
+                if (cudaEventSynchronize(m.ple_copied[ple_buf ^ 1]) != cudaSuccess) {
+                    err = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError());
+                    return false;
+                }
                 ple_next = std::async(std::launch::async, [&ple_gather, &ple_next_err, c1 = c0 + m.T, b = ple_buf ^ 1] {
                     return ple_gather(c1, b, ple_next_err);
                 });
             }
             ple_buf ^= 1;
             stats_.ms_ple += ms_since(tp);
-        }
+            return true;
+        };
         for (int64_t t = 0; t < T; ++t) { prev[0] = prev[1]; prev[1] = (int32_t) tokens[c0 + t]; }
         // ---- the QSA step records of every position in the chunk
         for (int64_t t = 0; t < T; ++t) strata::kernels::qsa_step_fill(m.steps_host.data() + t * strata::kernels::kStepCount, p0 + t, s);
@@ -1208,6 +1471,31 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
         // `used` event recorded), so the copy stream never waits on an event that is not queued yet
         const strata::kernels::cpu::ExpertLayout& lay0 = strata::kernels::cpu::expert_layout();
         const bool stream_all = m.ring > STAGE && T >= stream_all_min() && m.src != nullptr;
+        const bool ps_on = stream_all && m.pp && m.pp->ps_frac > 0.0;
+        if (m.pp && !ps_on) m.pp->ps_flag.clear();
+        // multi-GPU: the peer's ring - issue its copies up to `limit` / give entries back (the peer device is current)
+        auto p_issue_until = [&](size_t limit) {
+            PeerPrefill& P = *m.pp;
+            limit = std::min(limit, P.pseq.size());
+            while (P.p_issued < limit) {
+                const PeerPrefill::PsEntry& en = P.pseq[P.p_issued];
+                const size_t sl = P.p_issued % (size_t) P.RP;
+                if (P.plive[sl]) cudaStreamWaitEvent(P.s_cp, P.pused[sl], 0);
+                cudaMemcpyAsync(P.pstage[sl], en.blob, (size_t) strata::kernels::cpu::expert_layout().blob_bytes(en.l),
+                                cudaMemcpyHostToDevice, P.s_cp);
+                cudaEventRecord(P.pcopied[sl], P.s_cp);
+                P.plive[sl] = 1;
+                ++P.p_issued;
+            }
+        };
+        auto p_release_to = [&](int64_t l, int32_t e_stop, cudaStream_t ps) {
+            PeerPrefill& P = *m.pp;
+            while (P.pk < P.pseq_start[(size_t) l + 1] && P.pseq[P.pk].e < e_stop) {
+                cudaEventRecord(P.pused[P.pk % (size_t) P.RP], ps);
+                ++P.pk;
+                p_issue_until(P.pk + (size_t) P.RP);
+            }
+        };
         struct StreamEntry { int32_t l, e; const uint8_t* blob; int job; };
         std::vector<StreamEntry> seq;
         std::vector<size_t> seq_start;
@@ -1215,10 +1503,19 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
         if (stream_all) {
             seq_start.assign((size_t) g.n_layers + 1, 0);
             std::vector<Stager::Job> js;
+            if (ps_on) {
+                m.pp->pseq.clear();
+                m.pp->pseq_start.assign((size_t) g.n_layers + 1, 0);
+                m.pp->ps_flag.assign((size_t) g.n_layers * m.g->n_expert, 0);
+                m.pp->p_issued = m.pp->pk = 0;
+            }
+            double ps_acc = 0.0;
             for (int64_t l = LB; l < LE; ++l) {
                 seq_start[(size_t) l] = seq.size();
+                if (ps_on) m.pp->pseq_start[(size_t) l] = m.pp->pseq.size();
                 for (int32_t e = 0; e < m.g->n_expert; ++e) {
                     if (m.host_res && m.cache && m.host_res[(size_t) l * m.g->n_expert + e] >= 0) continue;
+                    if (m.pp && m.pp->peer->has(l, e)) continue;   // multi-GPU: computed on (or read from) the peer
                     int job = -1;
                     const uint8_t* b = nullptr;
                     if (m.src->transient(l, e)) {   // CS-T: copied by the source into the stager's buffer
@@ -1227,6 +1524,15 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     } else {
                         b = m.src->blob(l, e);
                         if (!b) { err = "prefill: expert source has no blob"; return false; }
+                        if (ps_on && m.src->pinned(l, e)) {   // multi-GPU: every ps_frac-th one goes to the peer's ring
+                            ps_acc += m.pp->ps_frac;
+                            if (ps_acc >= 1.0) {
+                                ps_acc -= 1.0;
+                                m.pp->pseq.push_back({(int32_t) l, e, b});
+                                m.pp->ps_flag[(size_t) l * m.g->n_expert + e] = 1;
+                                continue;
+                            }
+                        }
                         if (!m.src->pinned(l, e)) {
                             job = (int) js.size();
                             js.push_back({b, (size_t) lay0.blob_bytes(l)});
@@ -1235,7 +1541,10 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     seq.push_back({(int32_t) l, e, b, job});
                 }
             }
-            for (int64_t l = LE; l <= g.n_layers; ++l) seq_start[(size_t) l] = seq.size();
+            for (int64_t l = LE; l <= g.n_layers; ++l) {
+                seq_start[(size_t) l] = seq.size();
+                if (ps_on) m.pp->pseq_start[(size_t) l] = m.pp->pseq.size();
+            }
             m.stager->start(std::move(js));
         }
         struct StagerDone {
@@ -1249,7 +1558,7 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                 const int sl = (int) (issued % (size_t) m.ring);
                 const auto th = Clock::now();
                 const size_t bytes = (size_t) lay0.blob_bytes(en.l);
-                if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[sl], 0);
+                if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[m.used_of[sl]], 0);
                 if (en.job < 0) {
                     cudaMemcpyAsync(m.stage_dev[sl], en.blob, bytes, cudaMemcpyHostToDevice, m.copy);
                     ++stats_.experts_dma;
@@ -1296,7 +1605,7 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     const int sl = (int) (idx % (size_t) m.ring);
                     const auto th = Clock::now();
                     const size_t bytes = (size_t) lay0.blob_bytes(en.l);
-                    if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[sl], 0);
+                    if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[m.used_of[sl]], 0);
                     if (en.job < 0) {
                         cudaMemcpyAsync(m.stage_dev[sl], en.blob, bytes, cudaMemcpyHostToDevice, m.copy);
                         ++iss_dma;
@@ -1324,12 +1633,20 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
             if (threaded_issue) a_consumed.store(upto, std::memory_order_release);
             else issue_until(upto + (size_t) m.ring);
         };
+        if (ps_on) {
+            int pd = 0;
+            cudaGetDevice(&pd);
+            cudaSetDevice(m.pp->dev);
+            p_issue_until((size_t) m.pp->RP);
+            cudaSetDevice(pd);
+        }
         host_setup_ms += ms_since(tsetup);
         bool normed = false;   // F-2: the previous half's write already normed R for this half (grs, xn16)
         for (int64_t l = LB; l < LE; ++l) {
             core::progress_beat();   // the serve watchdog: a prompt chunk of 8192 tokens is still moving
             core::progress_at("reading the prompt (batched): layer", l, p0);   // #251: a stall names layer and chunk
             const core::LayerView v(*m.wt, l);
+            if (l == std::max<int64_t>(LB, 1) && !ple_land()) return false;   // the PLE rows, read from layer 1 on
             // ---- the PLE block at layer 1, token by token (its conv reads the previous tokens' rows)
             if (l == 1 && ple_on && ple_batch) {
                 // the whole chunk at once, in sub-batches carved from the idle scratch region: the key and value
@@ -1656,9 +1973,13 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l];
                     const int mmq_gt = lay.native ? lay.fmt[(size_t) l].gu_type : 42;
                     const int mmq_dt = lay.native ? lay.fmt[(size_t) l].d_type : 42;
-                    const bool fused_nat = use_mmq && stream_all && lay.native && fused::native_supported(mmq_gt, mmq_dt);
-                    const bool fused_l = (use_mmq && stream_all && !lay.native && fused::enabled()) || fused_nat;
+                    // --peer-device: MMQ only, whether or not the peer took the prompt path (set_peer can decline), as
+                    // fused_ring() sized the ring and the buffers for it
+                    const bool no_peer = !core::peer_portable();
+                    const bool fused_nat = use_mmq && stream_all && no_peer && lay.native && fused::native_supported(mmq_gt, mmq_dt);
+                    const bool fused_l = (use_mmq && stream_all && no_peer && !lay.native && fused::enabled()) || fused_nat;
                     size_t n_order = 0;                   // the routed experts (the debug report; unknown when fused)
+                    bool peer_now = false;                // multi-GPU: the peer computed rows of this layer (MMQ path only)
                     if (fused_l) {
                         if (static bool said = false; !said) {
                             said = true;
@@ -1708,7 +2029,10 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             } else {
                                 fused::experts(b, (int) m.g->n_expert, T * K, m.GU, m.Xq, m.src_dev, m.H, m.Dm, m.cs);
                             }
-                            for (size_t kk = k0; kk < k; ++kk) cudaEventRecord(m.used[kk % (size_t) m.ring], m.cs);
+                            for (size_t kk = k0; kk < k; ++kk) {
+                                cudaEventRecord(m.used[kk % (size_t) m.ring], m.cs);
+                                m.used_of[kk % (size_t) m.ring] = (int) (kk % (size_t) m.ring);
+                            }
                             if (k > k0) {
                                 consumed = k;
                                 give_back(consumed);
@@ -1727,14 +2051,47 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                         else cudaMemcpyAsync(m.ids_host.data(), m.ids, (size_t) T * K * 4, cudaMemcpyDeviceToHost, m.cs);
                         cudaStreamSynchronize(m.cs);
                         pt.fold();
+                        if (pe.on) {   // the peer's marks so far are done: the primary waited for its last rows
+                            int pd = 0; cudaGetDevice(&pd); cudaSetDevice(pe.dev); cudaStreamSynchronize(m.pp->s); pe.fold(); cudaSetDevice(pd);
+                        }
                         std::fill(m.cnt.begin(), m.cnt.end(), 0);
                         for (int64_t i = 0; i < T * K; ++i) {
                             const int32_t e = ids_h[(size_t) i];
                             if (e < 0 || e >= m.g->n_expert) { err = "prefill: routed id out of range"; return false; }
                             ++m.cnt[(size_t) e];
                         }
-                        m.off[0] = 0;
-                        for (int64_t e = 0; e < m.g->n_expert; ++e) m.off[(size_t) e + 1] = m.off[(size_t) e] + m.cnt[(size_t) e];
+                        // multi-GPU: the rows of the experts the peer computes go last, as one block [rows_local, T*K)
+                        const bool pre_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l];
+                        std::vector<char> on_peer;
+                        int64_t rows_local = T * K, rows_peer = 0;
+                        if (m.pp && pre_mmq) {
+                            on_peer.assign((size_t) m.g->n_expert, 0);
+                            for (int32_t e = 0; e < m.g->n_expert; ++e) {
+                                const int32_t c = m.cnt[(size_t) e];
+                                if (c > 0 && !m.pp->ps_flag.empty() && m.pp->ps_flag[(size_t) l * m.g->n_expert + e]) {   // peer-streamed
+                                    on_peer[(size_t) e] = 2;
+                                    rows_peer += c;
+                                    continue;
+                                }
+                                if (c == 0 || (m.host_res && m.cache && m.host_res[(size_t) l * m.g->n_expert + e] >= 0) ||
+                                    !m.pp->peer->has(l, e))
+                                    continue;
+                                if (rows_peer + c > m.pp->cap_rows) { ++m.pp->over_cap; continue; }
+                                on_peer[(size_t) e] = 1;
+                                rows_peer += c;
+                            }
+                            rows_local = T * K - rows_peer;
+                        }
+                        {
+                            int32_t r = 0;
+                            for (int32_t e = 0; e < m.g->n_expert; ++e)
+                                if (on_peer.empty() || !on_peer[(size_t) e]) { m.off[(size_t) e] = r; r += m.cnt[(size_t) e]; }
+                            // the peer's rows in order_peer's order (peer-held, then peer-streamed): its groups need them contiguous
+                            for (int kind = 1; kind <= 2 && !on_peer.empty(); ++kind)
+                                for (int32_t e = 0; e < m.g->n_expert; ++e)
+                                    if (on_peer[(size_t) e] == kind) { m.off[(size_t) e] = r; r += m.cnt[(size_t) e]; }
+                            m.off[(size_t) m.g->n_expert] = r;
+                        }
                         std::vector<int32_t> fill(m.off.begin(), m.off.end() - 1);
                         for (int64_t i = 0; i < T * K; ++i) {
                             const int32_t e = ids_h[(size_t) i];
@@ -1750,8 +2107,12 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             cudaMemcpyAsync(m.src_dev, m.src_host.data(), (size_t) T * K * 4, cudaMemcpyHostToDevice, m.cs);
                         }
                         // the experts, in id order: resident ones from VRAM, the others through the staging ring
-                        std::vector<int32_t> order;
-                        for (int32_t e = 0; e < m.g->n_expert; ++e) if (m.cnt[(size_t) e] > 0) order.push_back(e);
+                        std::vector<int32_t> order, order_peer;
+                        for (int32_t e = 0; e < m.g->n_expert; ++e)
+                            if (m.cnt[(size_t) e] > 0 && (on_peer.empty() || on_peer[(size_t) e] != 2))
+                                (!on_peer.empty() && on_peer[(size_t) e] ? order_peer : order).push_back(e);
+                        for (int32_t e = 0; e < m.g->n_expert && !on_peer.empty(); ++e)   // the peer-streamed ones last:
+                            if (on_peer[(size_t) e] == 2) order_peer.push_back(e);        // their copies get the most time
                         n_order = order.size();
                         const size_t mmq_gub = use_mmq ? mmq::matrix_bytes(mmq_gt, 1280, N) : 0;
                         const size_t mmq_db = use_mmq ? mmq::matrix_bytes(mmq_dt, N, 640) : 0;
@@ -1764,7 +2125,7 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             const size_t n = order.size(), ng = (n + MMQ_GROUP - 1) / MMQ_GROUP;
                             m.bounds_host.resize(n + 1 + ng * (MMQ_GROUP + 1));
                             for (size_t j = 0; j < n; ++j) m.bounds_host[j] = m.off[(size_t) order[j]];
-                            m.bounds_host[n] = (int32_t) (T * K);
+                            m.bounds_host[n] = (int32_t) rows_local;
                             for (size_t g = 0; g < ng; ++g)
                                 for (size_t i = 0; i <= MMQ_GROUP; ++i)
                                     m.bounds_host[n + 1 + g * (MMQ_GROUP + 1) + i] =
@@ -1780,6 +2141,182 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                         } else {
                             gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs);
                         }
+                        // multi-GPU: the peer's share, enqueued before the primary's own experts so both cards work at once
+                        peer_now = use_mmq && !order_peer.empty();
+                        if (peer_now) {
+                            PeerPrefill& P = *m.pp;
+                            cudaEventRecord(P.ev_in, m.cs);
+                            int prevd = 0;
+                            cudaGetDevice(&prevd);
+                            cudaSetDevice(P.dev);
+                            const cudaStream_t ps = P.s;
+                            cudaStreamWaitEvent(ps, P.ev_in, 0);
+                            if (P.out_pending) { cudaStreamWaitEvent(ps, P.ev_done, 0); P.out_pending = false; }
+                            pe.mark(kPeMoeIn, ps);
+                            cudaMemcpyPeerAsync(P.mixed, P.dev, m.mixed, prevd, (size_t) T * N * 4, ps);
+                            cudaMemcpyAsync(P.src, src_h + rows_local,   // multi-GPU: the mapped table when on
+                                             (size_t) rows_peer * 4, cudaMemcpyHostToDevice, ps);
+                            const size_t n = order_peer.size();
+                            if (P.compact) {
+                                // groups of up to MMQ_GROUP experts and at most G rows, each computed from its own rows
+                                P.groups.clear();
+                                {
+                                    size_t g0 = 0;
+                                    int64_t gr = 0;
+                                    for (size_t j = 0; j < n; ++j) {
+                                        const int64_t c = m.cnt[(size_t) order_peer[j]];
+                                        if (j > g0 && (j - g0 == (size_t) MMQ_GROUP || gr + c > P.G)) { P.groups.push_back({g0, j}); g0 = j; gr = 0; }
+                                        gr += c;
+                                    }
+                                    if (n > g0) P.groups.push_back({g0, n});
+                                }
+                                const size_t ng = P.groups.size();
+                                P.bounds_host.resize(n + 1 + ng * (MMQ_GROUP + 1));
+                                for (size_t j = 0; j < n; ++j) P.bounds_host[j] = m.off[(size_t) order_peer[j]] - (int32_t) rows_local;
+                                P.bounds_host[n] = (int32_t) rows_peer;
+                                for (size_t g2 = 0; g2 < ng; ++g2)
+                                    for (size_t i = 0; i <= MMQ_GROUP; ++i)
+                                        P.bounds_host[n + 1 + g2 * (MMQ_GROUP + 1) + i] =
+                                            P.bounds_host[std::min(P.groups[g2].second, P.groups[g2].first + i)] -
+                                            P.bounds_host[P.groups[g2].first];
+                                cudaMemcpyAsync(P.bounds, P.bounds_host.data(), P.bounds_host.size() * 4, cudaMemcpyHostToDevice, ps);
+                                pe.mark(kPeMoeGemm, ps);
+                                const auto& f = lay.fmt[(size_t) l];
+                                for (size_t g2 = 0; g2 < ng; ++g2) {
+                                    const size_t j0 = P.groups[g2].first, j1 = P.groups[g2].second;
+                                    const int ngx = (int) (j1 - j0);
+                                    int64_t maxr = 0;
+                                    for (size_t j = j0; j < j1; ++j) {
+                                        const int32_t e = order_peer[j];
+                                        maxr = std::max<int64_t>(maxr, m.cnt[(size_t) e]);
+                                        const uint8_t* bd = nullptr;
+                                        int psl = -1;
+                                        if (on_peer[(size_t) e] == 2) {   // from the peer's ring
+                                            p_release_to(l, e, ps);
+                                            if (P.pk >= P.pseq_start[(size_t) l + 1] || P.pseq[P.pk].e != e) {
+                                                err = "prefill: a peer-streamed expert is not next in the peer's ring";
+                                                cudaSetDevice(prevd);
+                                                return false;
+                                            }
+                                            psl = (int) (P.pk % (size_t) P.RP);
+                                            cudaStreamWaitEvent(ps, P.pcopied[(size_t) psl], 0);
+                                            bd = P.pstage[(size_t) psl];
+                                            ++P.ps_experts;
+                                        } else {
+                                            bd = P.peer->slot_ptr(l, e);
+                                        }
+                                        const size_t q = j - j0;
+                                        if (lay.native)
+                                            mmq::gather_native(bd, bd + f.up_off, mmq_gub / 2, bd + f.down_off, mmq_db,
+                                                               P.grp_gu + q * mmq_gub, P.grp_d + q * mmq_db, ps);
+                                        else
+                                            mmq::gather_strata_q2(bd, P.grp_gu + q * mmq_gub, P.grp_d + q * mmq_db, ps);
+                                        if (psl >= 0) {   // the slot is free once gathered
+                                            cudaEventRecord(P.pused[(size_t) psl], ps);
+                                            ++P.pk;
+                                            p_issue_until(P.pk + (size_t) P.RP);
+                                        }
+                                    }
+                                    const int64_t r0 = P.bounds_host[j0], nr = P.bounds_host[j1] - r0;
+                                    if (nr <= 0) continue;
+                                    const int32_t* rel = P.bounds + n + 1 + g2 * (MMQ_GROUP + 1);
+                                    cudaMemsetAsync(P.grp_gu + (size_t) ngx * mmq_gub, 0, MMQ_TAIL, ps);
+                                    cudaMemsetAsync(P.grp_d + (size_t) ngx * mmq_db, 0, MMQ_TAIL, ps);
+                                    mmq::quantize(P.mixed, P.src + r0, P.Xq_g, mmq_gt, N, N, nr, ps);
+                                    mmq::Product gu;
+                                    gu.w = P.grp_gu; gu.type = mmq_gt; gu.w_rows = 1280; gu.w_cols = N; gu.expert_bytes = mmq_gub;
+                                    gu.n = ngx; gu.xq = P.Xq_g; gu.bounds = rel; gu.ids = P.ident;
+                                    gu.total_rows = nr; gu.max_rows = maxr; gu.dst = P.GU_g; gu.ld_dst = 1280;
+                                    P.ctx->run(gu, ps);
+                                    mmq::swiglu(P.GU_g, P.H_g, nr, 640, !lay.native, ps);
+                                    mmq::quantize(P.H_g, nullptr, P.Hq_g, mmq_dt, 640, 640, nr, ps);
+                                    const int b = (int) (g2 & 1);
+                                    if (P.dm_live[b]) cudaStreamWaitEvent(ps, P.ev_dm[b], 0);   // its last rows have left
+                                    mmq::Product dn;
+                                    dn.w = P.grp_d; dn.type = mmq_dt; dn.w_rows = N; dn.w_cols = 640; dn.expert_bytes = mmq_db;
+                                    dn.n = ngx; dn.xq = P.Hq_g; dn.bounds = rel;
+                                    dn.ids = P.ident; dn.total_rows = nr; dn.max_rows = maxr; dn.dst = P.Dm_b[b];
+                                    dn.ld_dst = N;
+                                    P.ctx->run(dn, ps);
+                                    cudaEventRecord(P.ev_grp[g2 % PeerPrefill::kGrpEv], ps);
+                                    cudaStreamWaitEvent(P.s_out, P.ev_grp[g2 % PeerPrefill::kGrpEv], 0);
+                                    cudaMemcpyPeerAsync(m.Dm + (size_t) (rows_local + r0) * N, prevd, P.Dm_b[b], P.dev,
+                                                        (size_t) nr * N * 4, P.s_out);
+                                    cudaEventRecord(P.ev_dm[b], P.s_out);
+                                    P.dm_live[b] = true;
+                                }
+                            } else {
+                                const size_t ng = (n + MMQ_GROUP - 1) / MMQ_GROUP;
+                                P.bounds_host.resize(n + 1 + ng * (MMQ_GROUP + 1));
+                                for (size_t j = 0; j < n; ++j) P.bounds_host[j] = m.off[(size_t) order_peer[j]] - (int32_t) rows_local;
+                                P.bounds_host[n] = (int32_t) rows_peer;
+                                for (size_t g2 = 0; g2 < ng; ++g2)
+                                    for (size_t i = 0; i <= MMQ_GROUP; ++i)
+                                        P.bounds_host[n + 1 + g2 * (MMQ_GROUP + 1) + i] =
+                                            P.bounds_host[std::min(n, g2 * MMQ_GROUP + i)] - P.bounds_host[g2 * MMQ_GROUP];
+                                cudaMemcpyAsync(P.bounds, P.bounds_host.data(), P.bounds_host.size() * 4, cudaMemcpyHostToDevice, ps);
+                                mmq::quantize(P.mixed, P.src, P.Xq, mmq_gt, N, N, rows_peer, ps);
+                                pe.mark(kPeMoeGemm, ps);
+                                const auto& f = lay.fmt[(size_t) l];
+                                for (size_t j = 0; j < n; ++j) {
+                                    const int32_t e = order_peer[j];
+                                    const uint8_t* bd = P.peer->slot_ptr(l, e);
+                                    const size_t q = j % MMQ_GROUP;
+                                    if (lay.native)
+                                        mmq::gather_native(bd, bd + f.up_off, mmq_gub / 2, bd + f.down_off, mmq_db,
+                                                           P.grp_gu + q * mmq_gub, P.grp_d + q * mmq_db, ps);
+                                    else
+                                        mmq::gather_strata_q2(bd, P.grp_gu + q * mmq_gub, P.grp_d + q * mmq_db, ps);
+                                    if (q + 1 < MMQ_GROUP && j + 1 < n) continue;
+                                    const size_t j0 = j - q, g2 = j0 / MMQ_GROUP;
+                                    const int ngx = (int) (q + 1);
+                                    const int64_t r0 = P.bounds_host[j0], nr = P.bounds_host[j + 1] - r0;
+                                    int64_t maxr = 0;
+                                    for (size_t i = j0; i <= j; ++i) maxr = std::max<int64_t>(maxr, m.cnt[(size_t) order_peer[i]]);
+                                    cudaMemsetAsync(P.grp_gu + (size_t) ngx * mmq_gub, 0, MMQ_TAIL, ps);
+                                    cudaMemsetAsync(P.grp_d + (size_t) ngx * mmq_db, 0, MMQ_TAIL, ps);
+                                    mmq::Product gu;
+                                    gu.w = P.grp_gu; gu.type = mmq_gt; gu.w_rows = 1280; gu.w_cols = N; gu.expert_bytes = mmq_gub;
+                                    gu.n = ngx; gu.xq = P.Xq; gu.bounds = P.bounds + j0; gu.ids = P.ident;
+                                    gu.total_rows = rows_peer; gu.max_rows = maxr; gu.dst = P.GU; gu.ld_dst = 1280;
+                                    P.ctx->run(gu, ps);
+                                    mmq::swiglu(P.GU + r0 * 1280, P.H + r0 * 640, nr, 640, !lay.native, ps);
+                                    mmq::quantize(P.H + r0 * 640, nullptr, P.Hq, mmq_dt, 640, 640, nr, ps);
+                                    mmq::Product dn;
+                                    dn.w = P.grp_d; dn.type = mmq_dt; dn.w_rows = N; dn.w_cols = 640; dn.expert_bytes = mmq_db;
+                                    dn.n = ngx; dn.xq = P.Hq; dn.bounds = P.bounds + n + 1 + g2 * (MMQ_GROUP + 1);
+                                    dn.ids = P.ident; dn.total_rows = nr; dn.max_rows = maxr; dn.dst = P.Dm + r0 * N;
+                                    dn.ld_dst = N;
+                                    P.ctx->run(dn, ps);
+                                    if (P.out_pipe && nr > 0) {   // this group's rows go back while the next group computes
+                                        cudaEventRecord(P.ev_grp[g2], ps);
+                                        cudaStreamWaitEvent(P.s_out, P.ev_grp[g2], 0);
+                                        cudaMemcpyPeerAsync(m.Dm + (size_t) (rows_local + r0) * N, prevd, P.Dm + (size_t) r0 * N,
+                                                            P.dev, (size_t) nr * N * 4, P.s_out);
+                                    }
+                                }
+                            }
+                            pe.mark(kPeMoeOut, ps);
+                            if (P.out_pipe) {
+                                cudaEventRecord(P.ev_done, P.s_out);
+                                P.out_pending = true;
+                            } else {
+                                cudaMemcpyPeerAsync(m.Dm + (size_t) rows_local * N, prevd, P.Dm, P.dev, (size_t) rows_peer * N * 4, ps);
+                                cudaEventRecord(P.ev_done, ps);
+                            }
+                            pe.mark(kPeIdle, ps);
+                            cudaSetDevice(prevd);
+                            ++P.layers;
+                            P.experts += (int64_t) n;
+                            P.rows += rows_peer;
+                        }
+                        if (ps_on) {   // multi-GPU: this layer's peer-ring entries the routing did not pick give their slots back
+                            int pd = 0;
+                            cudaGetDevice(&pd);
+                            cudaSetDevice(m.pp->dev);
+                            p_release_to(l, (int32_t) m.g->n_expert, m.pp->s);
+                            cudaSetDevice(pd);
+                        }
                         // Stage ahead: the copy stream moves blobs host -> device while the compute stream works.
                         int stage_next = 0;
                         std::vector<int> stage_of(order.size(), -1);
@@ -1815,13 +2352,13 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             if (pinned && !b) { err = "prefill: expert source has no blob"; return false; }
                             if (pinned) {
                                 // DMA straight from the page-locked arena: the copy stream only waits for the slot
-                                if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[sl], 0);
+                                if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[m.used_of[sl]], 0);
                                 cudaMemcpyAsync(m.stage_dev[sl], b, (size_t) lay.blob_bytes(l), cudaMemcpyHostToDevice, m.copy);
                                 ++stats_.experts_dma;
                             } else {
                                 // copied to a pinned buffer by the stager (waits only if it is behind), then DMA
                                 const uint8_t* hb = m.stager->wait(job_of[j]);
-                                if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[sl], 0);
+                                if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[m.used_of[sl]], 0);
                                 cudaMemcpyAsync(m.stage_dev[sl], hb, (size_t) lay.blob_bytes(l), cudaMemcpyHostToDevice, m.copy);
                                 m.stager->issued_one(job_of[j], m.copy);
                             }
@@ -1832,6 +2369,46 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             ++stats_.experts_streamed;
                             return true;
                         };
+                        // In the streamed walk an MMQ group is gathered in ONE launch, after ONE wait on its last
+                        // streamed copy (the copy stream is in order), and its ring slots are released by ONE event.  A
+                        // wait and a record per expert put ~10 us of GPU bubble each on the compute stream under WDDM
+                        // (226 ms of a 32K prompt on an NVFP4 pack).  The same bytes into the same group slots.  A ring
+                        // entry the routing skipped inside an open group first gathers what the group holds so far
+                        // (`flush`), so no more than a group's entries are ever held back from the issuer.
+                        // STRATA_PREFILL_GROUP_GATHER=0: one gather, one wait and one record per expert.
+                        static const bool group_env = [] {
+                            const char* v = std::getenv("STRATA_PREFILL_GROUP_GATHER");
+                            return v == nullptr || std::atoi(v) != 0;
+                        }();
+                        const bool group_gather = group_env && stream_all && use_mmq && lay.native &&
+                                                  MMQ_GROUP <= mmq::kGatherGroupMax;
+                        mmq::GatherGroup gg;
+                        int gg_slots[MMQ_GROUP];
+                        int gg_nslots = 0;   // ring slots gathered by the next flush
+                        auto flush = [&]() {
+                            if (gg.n <= gg.first) return;
+                            const auto& f = lay.fmt[(size_t) l];
+                            if (gg_nslots > 0) {   // the copies land in order: the last one covers the others
+                                pt.mark(kPfWaitCopy, cs);
+                                cudaStreamWaitEvent(m.cs, m.copied[gg_slots[gg_nslots - 1]], 0);
+                                pt.mark(kPfDequant, cs);
+                            }
+                            if (!mmq::gather_native_group(gg, f.up_off, mmq_gub / 2, f.down_off, mmq_db, m.grp_gu, mmq_gub,
+                                                          m.grp_d, mmq_db, m.cs)) {
+                                for (int i = gg.first; i < gg.n; ++i) {   // not 16-byte aligned: one at a time
+                                    const uint8_t* b = gg.blob[i];
+                                    mmq::gather_native(b, b + f.up_off, mmq_gub / 2, b + f.down_off, mmq_db,
+                                                       m.grp_gu + i * mmq_gub, m.grp_d + i * mmq_db, m.cs);
+                                }
+                            }
+                            if (gg_nslots > 0) {
+                                const int rel = gg_slots[gg_nslots - 1];
+                                cudaEventRecord(m.used[rel], m.cs);
+                                for (int i = 0; i < gg_nslots; ++i) m.used_of[gg_slots[i]] = rel;
+                            }
+                            gg_nslots = 0;
+                            gg.first = gg.n;
+                        };
                         // one expert's products from its blob on the device; `slot` (a ring slot, or -1 for a resident
                         // expert) is released once the blob is read
                         auto compute = [&](size_t j, const uint8_t* blob_dev, int slot) -> bool {
@@ -1840,14 +2417,21 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             if (use_mmq) {
                                 // gather the expert into its group slot (GGUF blocks, unchanged or converted)
                                 const size_t q = j % MMQ_GROUP;
-                                if (lay.native) {
+                                if (group_gather) {
+                                    gg.blob[q] = blob_dev;
+                                    gg.n = (int) q + 1;
+                                    if (slot >= 0) gg_slots[gg_nslots++] = slot;
+                                    if (q + 1 < MMQ_GROUP && j + 1 < order.size()) return true;
+                                    flush();
+                                    gg = mmq::GatherGroup{};
+                                } else if (lay.native) {
                                     const auto& f = lay.fmt[(size_t) l];
                                     mmq::gather_native(blob_dev, blob_dev + f.up_off, mmq_gub / 2, blob_dev + f.down_off,
                                                        mmq_db, m.grp_gu + q * mmq_gub, m.grp_d + q * mmq_db, m.cs);
                                 } else {
                                     mmq::gather_strata_q2(blob_dev, m.grp_gu + q * mmq_gub, m.grp_d + q * mmq_db, m.cs);
                                 }
-                                if (slot >= 0) cudaEventRecord(m.used[slot], m.cs);
+                                if (slot >= 0 && !group_gather) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }
                                 if (q + 1 < MMQ_GROUP && j + 1 < order.size()) return true;
                                 // the group's products: gate/up, swiglu, the group's H to q8_1, down
                                 const size_t j0 = j - q, g = j0 / MMQ_GROUP, n = order.size();
@@ -1885,7 +2469,7 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             } else {
                                 blob_dequant_f16(blob_dev, m.dq_gu[q], m.dq_d[q], m.cs);
                             }
-                            if (slot >= 0) cudaEventRecord(m.used[slot], m.cs);
+                            if (slot >= 0) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }
                             const int64_t o0 = m.off[(size_t) e], ne = m.cnt[(size_t) e];
                             pt.mark(kPfGemmGU, cs);
                             m.gemm.f16(m.Xs + o0 * N, m.dq_gu[q], m.GU + o0 * 1280, ne, 1280, N);
@@ -1919,7 +2503,10 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             const size_t kend = seq_start[(size_t) l + 1];
                             auto release_to = [&](int32_t e_stop) {
                                 while (k < kend && seq[k].e < e_stop) {
-                                    cudaEventRecord(m.used[k % (size_t) m.ring], m.cs);
+                                    if (gg_nslots > 0) flush();   // the open group's slots get their event first
+                                    const int sl = (int) (k % (size_t) m.ring);
+                                    cudaEventRecord(m.used[sl], m.cs);
+                                    m.used_of[sl] = sl;
                                     consumed = ++k;
                                     give_back(consumed);
                                 }
@@ -1929,21 +2516,29 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                                 release_to(e);
                                 if (k < kend && seq[k].e == e) {
                                     const int sl = (int) (k % (size_t) m.ring);
-                                    pt.mark(kPfWaitCopy, cs);
                                     wait_issued(k);
-                                    cudaStreamWaitEvent(m.cs, m.copied[sl], 0);
+                                    if (!group_gather) {
+                                        pt.mark(kPfWaitCopy, cs);
+                                        cudaStreamWaitEvent(m.cs, m.copied[sl], 0);
+                                    }
                                     if (!compute(j, m.stage_dev[sl], sl)) return false;
                                     consumed = ++k;
-                                    give_back(consumed);
+                                    if (!group_gather || gg_nslots == 0) give_back(consumed);   // its group was gathered
                                 } else {
+                                    const bool r0 = m.host_res && m.cache && m.host_res[(size_t) l * m.g->n_expert + e] >= 0;
+                                    const uint8_t* bp = r0 ? m.cache->device_slot(m.host_res[(size_t) l * m.g->n_expert + e])
+                                                           : (m.pp ? m.pp->peer->slot_ptr(l, e) : nullptr);   // over the cap: P2P
+                                    if (bp == nullptr) { err = "prefill: an expert is neither resident, streamed nor on the peer"; return false; }
                                     ++stats_.experts_resident;
-                                    if (!compute(j, m.cache->device_slot(m.host_res[(size_t) l * m.g->n_expert + e]), -1)) return false;
+                                    if (!compute(j, bp, -1)) return false;
+                                    if (group_gather && gg_nslots == 0) give_back(consumed);
                                 }
                             }
                             release_to(m.g->n_expert);
                         }
                     }
                     pt.mark(kPfCombine, cs);
+                    if (peer_now) cudaStreamWaitEvent(m.cs, m.pp->ev_done, 0);   // multi-GPU: the peer's rows are in Dm
                     moe_combine(m.Dm, m.slot_dev, m.w, m.shared, m.sg, m.bo, T, m.cs);
                     // debug: STRATA_DBG_NAN=1 reports the first layer of a chunk whose MoE produced non-finite values
                     if (static const bool dbg = std::getenv("STRATA_DBG_NAN") != nullptr; dbg) {
@@ -1992,6 +2587,7 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     strata::kernels::cvec_apply(m.R, l, T, D, nullptr, 0, nullptr, 0, false, m.cs);
             }
         }
+        if (!ple_land()) return false;   // a stage that ends before layer 1: the rows land anyway, the next gather starts
         if (issuer.joinable()) {
             issuer.join();
             stats_.ms_experts_host += iss_ms;
@@ -2092,6 +2688,21 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
         std::fprintf(stderr, "strata prefill timing: host: chunk setup (PLE rows, the expert stream plan) %.0f ms, "
                              "waiting for each chunk %.0f ms, after each chunk (the draft layer, progress) %.0f ms, "
                              "PLE %.0f ms\n", host_setup_ms, host_sync_ms, host_chunk_ms, stats_.ms_ple);
+        if (pe.on) {
+            int pd = 0; cudaGetDevice(&pd); cudaSetDevice(pe.dev); cudaStreamSynchronize(m.pp->s); pe.fold(); cudaSetDevice(pd);
+            std::string pl;
+            for (int i = 0; i < kPeCount; ++i) {
+                std::snprintf(b, sizeof b, " %s %.0f", kPeNames[i], pe.ms[i]);
+                pl += b;
+            }
+            std::fprintf(stderr, "strata prefill timing (peer GPU, ms):%s; MoE layers %lld, rows/layer %.0f of %lld cap, experts/layer %.0f, "
+                                 "over the cap %lld\n", pl.c_str(), (long long) m.pp->layers,
+                         m.pp->layers ? (double) m.pp->rows / m.pp->layers : 0.0, (long long) m.pp->cap_rows,
+                         m.pp->layers ? (double) m.pp->experts / m.pp->layers : 0.0, (long long) m.pp->over_cap);
+            if (m.pp->ps_frac > 0.0)
+                std::fprintf(stderr, "strata prefill timing: peer-streamed experts %lld (%.0f per MoE layer)\n",
+                             (long long) m.pp->ps_experts, m.pp->layers ? (double) m.pp->ps_experts / m.pp->layers : 0.0);
+        }
     }
     if (std::getenv("STRATA_STATE_HASH_GDN") != nullptr) {   // debug: the GDN states as the prompt path leaves them
         cudaStreamSynchronize(m.cs);

src/program/generate.cpp

@@ -23,6 +23,7 @@
 #include "strata/core/pinned.hpp"
 #include "strata/core/remote_experts.hpp"
 #include "strata/core/on_device.hpp"
+#include "strata/core/peer_experts.hpp"
 #include "strata/core/layer.hpp"
 #include "strata/core/layout.hpp"
 #include "strata/core/session.hpp"
@@ -77,6 +78,7 @@
 #include <chrono>
 #include <algorithm>
 #include <iostream>
+#include <future>
 #include <thread>
 #include <atomic>
 #include <condition_variable>
@@ -221,7 +223,7 @@ struct Options {
     /// Plan v0.3 P2: how the n-gram table is read. Direct (default) = unbuffered SSD reads, table never in RAM.
     std::string ple_io = "direct";
     int64_t ple_row_cache = 1 << 20;   ///< bounded row cache (rows of 90 B); 0 disables
-    int ple_inflight = 64;
+    int ple_inflight = 256;   // the prompt path reads a chunk's rows at once: 64 left the SSD half idle (32K: 303 -> 189 ms)
     double ple_delay_us = 0;           ///< fault injection: every row read completes no earlier than this
     bool ple_sync_submit = false;      ///< A/B arm: submit reads on the token thread, no I/O worker
     std::string kv = "fp16";           ///< plan v0.3 P7: KV storage, fp16 (default) or int8 (half the VRAM)
@@ -295,6 +297,16 @@ struct Options {
     /// to them.  Measured at 256 slots: **1781 of 60000 = 2.97%**, against **21.4%** for 8 slots per layer and
     /// **70.4%** for 64, from `Memory/cache_allocation.py` on the same run's routing.  Off by default.
     bool expert_cache_per_layer = false;
+    /// Multi-GPU: a second expert tier on CUDA device `peer_device` (-1 = off), `peer_reserve_mib` left free
+    /// there, `peer_slots` caps its size (0 = as many as fit), `peer_adapt_swaps` per adaptive round
+    /// (-1 = adapt_swaps).
+    int peer_device = -1;
+    int peer_reserve_mib = 600;
+    int64_t peer_slots = 0;
+    int peer_adapt_swaps = -1;
+    /// the prompt path's rows per layer the peer computes: -1 = half of chunk x top-k, 0 = the prompt path stays on
+    /// the primary
+    int64_t peer_prefill_rows = -1;
     /// The PLE gather's prefetch, as an A/B arm.  The gather measured 2.10-2.61 ms/token because its sixteen
     /// row reads are sixteen SEPARATE page faults into a 26.8 GB mapping; see `ple_prefetch_enable`.
     bool no_ple_prefetch = false;
@@ -327,6 +339,7 @@ struct Options {
     /// Plan v0.3 P4: `--expert-cache auto` sizes the VRAM tier from what is free after the weights, the session
     /// and the KV state, minus this reserve for the graphs, the hit scratch and the head.
     int vram_reserve_mib = 700;
+    bool vram_reserve_given = false;   ///< --vram-reserve-mib on the command line (#496: no smaller automatic reserve)
     /// Plan v0.3 P5: batched prompt processing in chunks of this many tokens (0 = the token path).
     int64_t prefill_chunk = 0;
     /// `--prefill auto`: the largest chunk (up to 8192) whose buffers the expert cache can lend.  Every expert a chunk
@@ -362,6 +375,7 @@ struct Options {
     /// Plan v0.3 P6: every `adapt_every` rounds, swap up to `adapt_swaps` of the most-routed missing experts into
     /// the VRAM tier in place of the least-routed resident ones (decayed counts).  0 = static residency.
     int adapt_every = 4;
+    float adapt_decay = 0.7f;   ///< the usage counts are multiplied by this after each adaptation (--adapt-decay)
     /// Plan v0.3 P6: a draft enters the verify window only while every draft before it (and itself) has at least
     /// this probability under the draft layer; 0 = always --spec-1 drafts.
     double spec_min_p = 0.0;
@@ -433,7 +447,7 @@ void usage() {
                  "                       reads, the table never enters RAM or the file cache; mmap: A/B arm;\n"
                  "                       ram: mmap with the whole table locked in RAM at start (Linux/macOS)\n"
                  "  --ple-row-cache N    bounded cache of fetched rows, 90 B each (default 1048576; 0 = off)\n"
-                 "  --ple-inflight N     outstanding SSD reads (default 64)\n"
+                 "  --ple-inflight N     outstanding SSD reads (default 256)\n"
                  "  --ple-delay-us U     fault injection: each row read completes no earlier than U us\n"
                  "  --ple-sync-submit    A/B arm: submit table reads on the token thread (default: an I/O thread)\n"
                  "  --kv fp16|int8       KV storage (plan v0.3 P7): int8 codes + fp16 scale per 64 values, half the\n"
@@ -553,6 +567,14 @@ void usage() {
                  "  --expert-cache-device3 N  pre-fill N more experts on CUDA3\n"
                  "  --expert-cache-remote-placement stripe|layer  distribute expert ranks or whole\n"
                  "                       layers across CUDA1..3 (default: stripe)\n"
+                 "  --peer-device N      a second GPU as an adaptive expert-cache tier (rows over P2P; it also\n"
+                 "                       computes its experts' prompt rows).  Not with --layer-split or\n"
+                 "                       --expert-cache-device1..3.  Default output unchanged without it.\n"
+                 "  --peer-reserve-mib N  VRAM the peer tier leaves free on its card (default 600)\n"
+                 "  --peer-slots N       expert slots on the peer (default: what fits)\n"
+                 "  --peer-adapt-swaps N  peer cache swaps per adaptation step (default: --adapt-swaps)\n"
+                 "  --peer-prefill-rows N  prompt rows per layer the peer computes (default half of chunk x top-k;\n"
+                 "                       0 = the prompt path stays on the primary)\n"
                  "  --expert-cache-per-layer  R4.2g: give each layer its OWN slots instead of letting the first\n"
                  "                       position take all of them.  The default policy fills in arrival order\n"
                  "                       from one shared counter, so 256 slots went to ~26 layers of position 0\n"
@@ -824,6 +846,12 @@ void mem_mark(const char* where) {
     std::fprintf(stderr, "strata trace: %lld MiB free after %s\n", (long long) (free_b >> 20), where);
 }
 
+/// #463's A/B: STRATA_ADAPT_NOWAIT=1 lets a verify window start before the adaptive tier's copies have landed (0.1.37)
+bool adapt_nowait() {
+    static const bool v = [] { const char* e = std::getenv("STRATA_ADAPT_NOWAIT"); return e && e[0] == '1'; }();
+    return v;
+}
+
 int argmax(const std::vector<float>& v) {
     int best = 0;
     for (size_t i = 1; i < v.size(); ++i)
@@ -1170,7 +1198,7 @@ int main(int argc, char** argv) {
         else if (a == "--expert-cache-device3") o.expert_cache_remote[2] = std::atoi(next("--expert-cache-device3"));
         else if (a == "--expert-cache-remote-placement")
             o.expert_cache_remote_placement = next("--expert-cache-remote-placement");
-        else if (a == "--vram-reserve-mib") o.vram_reserve_mib = std::atoi(next("--vram-reserve-mib"));
+        else if (a == "--vram-reserve-mib") { o.vram_reserve_mib = std::atoi(next("--vram-reserve-mib")); o.vram_reserve_given = true; }
         else if (a == "--prefill") {
             const std::string v = next("--prefill");
             o.prefill_auto = v == "auto" || v.rfind("auto:", 0) == 0;
@@ -1192,6 +1220,7 @@ int main(int argc, char** argv) {
         else if (a == "--mtp-window") o.mtp_window = std::atoll(next("--mtp-window"));
         else if (a == "--pcie-frac") o.pcie_frac = std::atof(next("--pcie-frac"));
         else if (a == "--adapt-every") o.adapt_every = std::atoi(next("--adapt-every"));
+        else if (a == "--adapt-decay") o.adapt_decay = (float) std::atof(next("--adapt-decay"));
         else if (a == "--spec-min-p") o.spec_min_p = std::atof(next("--spec-min-p"));
         else if (a == "--stop-eos") o.stop_eos = true;
         else if (a == "--spec-split") o.spec_split = true;
@@ -1263,6 +1292,11 @@ int main(int argc, char** argv) {
         else if (a == "--adapt-swaps") o.adapt_swaps = std::atoi(next("--adapt-swaps"));
         else if (a == "--expert-cache-cpu-order") o.expert_cache_cpu_order = true;
         else if (a == "--expert-cache-per-layer") o.expert_cache_per_layer = true;
+        else if (a == "--peer-device") o.peer_device = std::atoi(next("--peer-device"));
+        else if (a == "--peer-reserve-mib") o.peer_reserve_mib = std::atoi(next("--peer-reserve-mib"));
+        else if (a == "--peer-slots") o.peer_slots = std::atoll(next("--peer-slots"));
+        else if (a == "--peer-adapt-swaps") o.peer_adapt_swaps = std::atoi(next("--peer-adapt-swaps"));
+        else if (a == "--peer-prefill-rows") o.peer_prefill_rows = std::atoll(next("--peer-prefill-rows"));
         else if (a == "--no-hit-poke") o.no_hit_poke = true;
         else if (a == "--expert-profile") o.expert_profile = next("--expert-profile");
         else if (a == "--expert-profile-save") o.expert_profile_save = next("--expert-profile-save");
@@ -1312,6 +1346,7 @@ int main(int argc, char** argv) {
         }
     }
     strata::core::set_coupled_draft(o.coupled_draft);
+    strata::core::set_peer_portable(o.peer_device >= 1);   // multi-GPU: the Portable flag on mapped host buffers only with a peer device (before any allocation)
     if (o.serve && o.conversation_cache_mib > 0 && (o.prompt_cache == 0 || o.conversation_cache_slots == 0))
         std::fprintf(stderr, "strata serve: warning: conversation caching is disabled by %s\n",
                      o.prompt_cache == 0 ? "--prompt-cache 0" : "--conversation-cache-slots 0");
@@ -1678,6 +1713,14 @@ int main(int argc, char** argv) {
         std::fprintf(stderr, "strata generate: --expert-cache-remote-placement must be stripe or layer\n");
         return 2;
     }
+    // the peer tier is the second card's only user: a layer split or a remote expert cache would put a second engine
+    // part (and a second copy of the same experts) on it
+    if (o.peer_device >= 1 && (!o.layer_split.empty() || o.expert_cache_remote[0] > 0)) {
+        std::fprintf(stderr, "strata generate: --peer-device cannot be combined with %s\n",
+                     !o.layer_split.empty() ? "--layer-split (use one or the other)"
+                                            : "--expert-cache-device1..3 (the peer tier already caches experts there)");
+        return 2;
+    }
 
     if (o.native_flash_attn_short && o.max_context > 256) {
         std::fprintf(stderr, "strata generate: --native-flash-attn-short requires --max-context <=256\n");
@@ -1947,9 +1990,18 @@ int main(int argc, char** argv) {
         return 1;
     }
     void* arena = nullptr;
-    if (cudaMalloc(&arena, pool_bytes) != cudaSuccess) {
-        std::fprintf(stderr, "strata generate: cudaMalloc(%llu) for the weight arena failed\n",
-                     (unsigned long long) pool_bytes);
+    if (const cudaError_t ce = cudaMalloc(&arena, pool_bytes); ce != cudaSuccess) {
+        // #486: the arena is the first large allocation and its size does not depend on the context, so what is
+        // missing is held by something else: say how much was free
+        cudaGetLastError();
+        size_t free_b = 0, total_b = 0;
+        cudaMemGetInfo(&free_b, &total_b);
+        std::fprintf(stderr, "strata generate: cudaMalloc(%llu) for the weight arena failed (%s): %llu MiB of %llu "
+                             "MiB VRAM free on this GPU. The arena is allocated first, before the KV and expert "
+                             "caches: another program (or an engine that is still exiting) holds the rest - "
+                             "nvidia-smi / rocm-smi lists them\n",
+                     (unsigned long long) pool_bytes, cudaGetErrorString(ce), (unsigned long long) (free_b >> 20),
+                     (unsigned long long) (total_b >> 20));
         return 1;
     }
     strata::core::WeightTable wt;
@@ -2254,8 +2306,14 @@ int main(int argc, char** argv) {
         void* arena_s = nullptr;
         if (cudaMalloc(&arena_s, pool_bytes) != cudaSuccess ||
             !st.wt.load(o.pack, arena_s, pool_bytes, err, skip.empty() ? nullptr : &skip)) {
-            std::fprintf(stderr, "strata generate: layer split, CUDA%d weights: %s\n", st.dev,
-                         err.empty() ? "the weight arena does not fit" : err.c_str());
+            cudaGetLastError();
+            size_t free_b = 0, total_b = 0;   // #486: what that card had free
+            cudaMemGetInfo(&free_b, &total_b);
+            std::fprintf(stderr, "strata generate: layer split, CUDA%d weights: %s (%llu MiB needed, %llu MiB of %llu "
+                                 "MiB free on that card)\n", st.dev,
+                         err.empty() ? "the weight arena does not fit" : err.c_str(),
+                         (unsigned long long) (pool_bytes >> 20), (unsigned long long) (free_b >> 20),
+                         (unsigned long long) (total_b >> 20));
             return 1;
         }
         if (!o.native_dense_gguf.empty() && !st.dense.load(o.native_dense_gguf, st.wt, err, o.native_ple_key)) {
@@ -2316,6 +2374,36 @@ int main(int argc, char** argv) {
         // file).  `--expert-cache 0` used to take the count the profile was built for; the profile now ranks
         // every pair (issue #46: a card that holds more than the old 8,000 used to stop there), so it means auto.
         if (o.expert_cache == 0) o.expert_cache = -1;
+        // Multi-GPU: STRATA_PEER_HOT=<f> gives the peer card a share f of the HOT pairs, so both cards
+        // compute routed experts every layer (the primary alone did ~25 of ~30 per layer-window).  Of the first
+        // STRATA_PEER_HOT_AT (default 8700, ~ the primary's slots) ranks, every pair with floor((r+1)f) > floor(rf)
+        // moves to just after that point: the primary fills past them, the peer (which takes what the primary does
+        // not hold, in order) gets them first.
+        if (o.peer_device >= 1) {   // default 0.45 (measured: 0.3-0.6 all better than 0; 0 = off, e.g. for the gate)
+            const char* ph = std::getenv("STRATA_PEER_HOT");
+            const double f = ph ? std::atof(ph) : 0.45;
+            const char* pa = std::getenv("STRATA_PEER_HOT_AT");
+            const size_t at = std::min(profile.size(), (size_t) (pa ? std::atoll(pa) : 8700));
+            if (f > 0.0 && f < 1.0 && at > 0) {
+                std::vector<std::pair<int32_t, int32_t>> keep, moved;
+                for (size_t r = 0; r < at; ++r) {
+                    const bool to_peer = (int64_t) ((double) (r + 1) * f) > (int64_t) ((double) r * f);
+                    (to_peer ? moved : keep).push_back(profile[r]);
+                }
+                const size_t n_moved = moved.size();
+                // the primary's share continues with the ranks after `at` until it is full; then the moved ones
+                std::vector<std::pair<int32_t, int32_t>> out;
+                out.reserve(profile.size());
+                out.insert(out.end(), keep.begin(), keep.end());
+                const size_t fill = std::min(profile.size(), at + n_moved);   // what the primary still takes
+                out.insert(out.end(), profile.begin() + (long) at, profile.begin() + (long) fill);
+                out.insert(out.end(), moved.begin(), moved.end());
+                out.insert(out.end(), profile.begin() + (long) fill, profile.end());
+                profile.swap(out);
+                std::fprintf(stderr, "strata generate: STRATA_PEER_HOT %.2f: %zu of the first %zu ranked pairs moved "
+                                     "behind rank %zu (the peer's)\n", f, n_moved, at, fill);
+            }
+        }
         std::fprintf(stderr, "strata generate: profile %s: %zu ranked pairs, built for %lld slots\n",
                      o.expert_profile.c_str(), profile.size(), (long long) pslots);
     }
@@ -2630,6 +2718,16 @@ int main(int argc, char** argv) {
         std::fprintf(stderr, "strata generate: GPU %d: %s, compute capability %d.%d%s\n", dev, name,
                      strata::cc_major_of(p.major), strata::cc_minor_of(p.minor),
                      strata::emulated_cc() ? " (STRATA_EMULATE_CC: a test mode, the card is emulated)" : "");
+        {   // #542: a build whose libcudart is older than its headers (a CUDA 13 kit with a dangling libcudart.so that
+            // CMake resolved to the system's CUDA 12 one) reads cudaDeviceProp shifted - silently, and slowly
+            int rt = 0;
+            if (cudaRuntimeGetVersion(&rt) == cudaSuccess && rt / 1000 != CUDART_VERSION / 1000)
+                std::fprintf(stderr, "strata generate: WARNING: this engine was compiled with CUDA %d.%d headers but "
+                                     "loaded a CUDA %d.%d runtime (libcudart): GPU properties can read wrong and some "
+                                     "kernels go unused. Rebuild it against one toolkit (cmake -DCUDAToolkit_ROOT=<the "
+                                     "toolkit>, with its libcudart.so present) (#542)\n",
+                             CUDART_VERSION / 1000, CUDART_VERSION % 1000 / 10, rt / 1000, rt % 1000 / 10);
+        }
 #endif
         const std::string e = strata::core::device_code_error();
         if (!e.empty()) {
@@ -2656,6 +2754,15 @@ int main(int argc, char** argv) {
         }
         std::fprintf(stderr, "strata generate: experts via mmap (--mmap-experts; %s)\n",
                      src.gguf_mode() ? "the GGUF shards in place, no experts.bin" : "the A/B arm of R2.1");
+        // #286: with a RAM budget the hottest experts live in it, and the rest are read from the drive unbuffered
+        // when the file cache could not keep them beside the budget anyway (a 32 GB PC) - the mapped reads' page
+        // faults are small requests on the critical path, and their pages take the RAM the budget was sized for
+        if (o.resident_budget > 0 || std::getenv("STRATA_UNBUFFERED_LOAD") != nullptr) {
+            std::string why;
+            const bool ub = src.set_unbuffered(o.resident_budget, why);
+            std::fprintf(stderr, "strata generate: the file tier reads %s (%s)\n",
+                         ub ? "unbuffered" : "through the file cache", why.c_str());
+        }
         srcp = &src;
     } else {
         arena_src.set_gguf(o.native_preset);   // plan v0.3 P6: a native pack may take its experts from shard 1
@@ -2735,6 +2842,7 @@ int main(int argc, char** argv) {
         return 1;
     }
     const bool auto_cache = o.expert_cache < 0;
+    bool reserve_adapted = false;   // #496: the auto sizing lowered the reserve so a small card's cache fits
     if (o.expert_cache < 0) {
         size_t free_b = 0, total_b = 0;
         cudaMemGetInfo(&free_b, &total_b);
@@ -2748,15 +2856,63 @@ int main(int argc, char** argv) {
         const int64_t mtp_bind = (!o.mtp.empty() && native_head.loaded())
                                      ? (int64_t) mtp.bind_bytes(native_head.row_bytes(), n_vocab) : 0;
         const int64_t reserve = (((int64_t) o.vram_reserve_mib + prefill_mib) << 20) + mtp_bind;
-        int64_t slots = ((int64_t) free_b - reserve) / (int64_t) strata::kernels::cpu::expert_layout().max_blob;
+        const int64_t blob = (int64_t) strata::kernels::cpu::expert_layout().max_blob;
+        int64_t slots = ((int64_t) free_b - reserve) / blob;
         if (!profile.empty()) slots = std::min<int64_t>(slots, (int64_t) profile.size());
         o.expert_cache = (int) std::max<int64_t>(slots, 0);
         std::fprintf(stderr, "strata generate: expert cache auto: %.2f GiB free, %d MiB reserved (+%lld MiB for the "
                              "draft head) -> %d slots\n",
                      (double) free_b / 1073741824.0, o.vram_reserve_mib, (long long) (mtp_bind >> 20), o.expert_cache);
-        if (o.expert_cache == 0)   // the verify window cannot start without it (#174): say what makes room
-            std::fprintf(stderr, "strata generate: no VRAM is left for the expert cache: lower --max-context, use "
-                                 "--kv k8v4, run images on the CPU, or close other programs that use the GPU\n");
+        // #496: the verify window cannot start without a cache (#174), and a cache too small to lend the prompt path
+        // a 256-token chunk's buffers (plus the 128 slots a loan leaves; one slot without --prefill) makes it
+        // allocate its own on top - more than the reserve.  When the default reserve leaves less than that (a 6 GB
+        // card), the reserve shrinks to what leaves exactly that cache, down to kSmallReserveMib: what is allocated
+        // after the cache - the prompt path's own part, the verify buffers, the draft head - comes out of the reserve,
+        // and below ~550 MiB a card ends with less than the 256 MiB the serve check calls LOW (IQ3_XXS, 32K, a 300 MiB
+        // reserve: 5 MiB left), so the cache gets no more than it needs, and the serve check says so plainly when it
+        // ends LOW (`reserve_adapted`).  A reserve given on the command line is kept.  No slot at all: the start
+        // stops, saying what is short and what makes room.  A card the default reserve leaves that much is sized as
+        // before.
+        constexpr int kSmallReserveMib = 300;
+        const int64_t min_slots = (o.prefill_chunk > 0 && pf_borrow)
+            ? ((int64_t) strata::prefill::Prefill::bytes_needed(g, ss, 256) + blob - 1) / blob + 128 : 1;
+        if (o.expert_cache < min_slots && !o.vram_reserve_given && o.vram_reserve_mib > kSmallReserveMib) {
+            // the largest reserve (in MiB) that still leaves min_slots
+            const int64_t fit_mib = ((int64_t) free_b - mtp_bind - min_slots * blob) / (1 << 20) - prefill_mib;
+            if (fit_mib >= kSmallReserveMib) {
+                const int r = (int) std::min<int64_t>(fit_mib, o.vram_reserve_mib);
+                int64_t s2 = ((int64_t) free_b - ((((int64_t) r + prefill_mib) << 20) + mtp_bind)) / blob;
+                if (!profile.empty()) s2 = std::min<int64_t>(s2, (int64_t) profile.size());
+                std::fprintf(stderr, "strata generate: expert cache auto: the %d MiB reserve leaves too few slots on "
+                                     "this card (a working cache needs %lld): a %d MiB reserve instead -> %lld slots\n",
+                             o.vram_reserve_mib, (long long) min_slots, r, (long long) s2);
+                o.vram_reserve_mib = r;
+                o.expert_cache = (int) s2;
+                reserve_adapted = true;
+            }
+        }
+        if (o.expert_cache == 0) {
+            // what is short, and what makes room: the numbers a small card picks from
+            const int64_t at_reserve = o.vram_reserve_given ? o.vram_reserve_mib
+                                                            : std::min(o.vram_reserve_mib, kSmallReserveMib);
+            const int64_t need_b = (((int64_t) at_reserve + prefill_mib) << 20) + mtp_bind + min_slots * blob;
+            const int64_t short_mib = std::max<int64_t>(1, (need_b - (int64_t) free_b + (1 << 20) - 1) >> 20);
+            const int64_t session_mib =
+                (int64_t) (strata::core::session_bytes(g, o.max_context, K, 0, g.n_layers) >> 20);
+            const std::string reserve_tip =
+                o.vram_reserve_given && o.vram_reserve_mib > kSmallReserveMib
+                    ? ", a smaller --vram-reserve-mib (" + std::to_string(o.vram_reserve_mib) + " now; " +
+                          std::to_string(kSmallReserveMib) + " is enough on a small card)"
+                    : std::string();
+            std::fprintf(stderr, "strata generate: no VRAM is left for the expert cache: it needs at least %lld slots "
+                                 "(%lld MiB), about %lld MiB more than this card has free. To make room: a smaller "
+                                 "--max-context (the session, mostly its KV cache, takes %lld MiB at %lld tokens), "
+                                 "--kv q4_0, the English draft subset (setup --draft-vocab en; the draft head takes "
+                                 "%lld MiB now)%s, images on the CPU, or close other programs that use the GPU\n",
+                         (long long) min_slots, (long long) ((min_slots * blob) >> 20), (long long) short_mib,
+                         (long long) session_mib, (long long) o.max_context, (long long) (mtp_bind >> 20),
+                         reserve_tip.c_str());
+        }
     } else if (multi_gpu && o.expert_cache > 0) {
         // an explicit cache size leaves room for the prompt path's buffers and the reserve, or the first prompt
         // fails with "device buffers ... do not fit" (with borrowing - the default with a profile - the path lends
@@ -2924,7 +3080,15 @@ int main(int argc, char** argv) {
         const bool per_layer = xcache.per_layer_admission();
         const int64_t want = per_layer ? (int64_t) profile.size()
                                        : std::min<int64_t>((int64_t) profile.size(), xcache.slots());
+        // #286: an unbuffered file tier reads the pairs in batches of 64, the next batch while this one is copied
+        std::future<void> ahead;
+        auto read_batch = [&](int64_t at) { src.prefetch_pairs(profile.data() + at, std::min<int64_t>(64, want - at)); };
         for (int64_t i = 0; i < want; ++i) {
+            if (!per_layer && srcp == &src && src.unbuffered() && i % 64 == 0) {
+                if (ahead.valid()) ahead.get();
+                else read_batch(i);
+                if (i + 64 < want) ahead = std::async(std::launch::async, read_batch, i + 64);
+            }
             const int32_t slot = xcache.admit(profile[(size_t) i].first, profile[(size_t) i].second);
             if (slot == strata::core::kNotResident) {
                 if (per_layer) continue;
@@ -3086,9 +3250,29 @@ int main(int argc, char** argv) {
         }
     }
 
+    // ---- Multi-GPU: the second GPU's expert tier, filled with the ranked pairs the primary does not hold
+    strata::core::PeerExperts peer;
+    if (o.peer_device >= 1) {
+        if (profile.empty() || srcp == nullptr || o.expert_cache <= 0) {
+            std::fprintf(stderr, "strata generate: --peer-device needs --expert-profile and the expert cache\n");
+            return 1;
+        }
+        const auto tp0 = Clock::now();
+        if (!peer.open(o.peer_device, profile, xcache, *srcp, g.n_layers, g.n_expert, o.peer_reserve_mib, o.peer_slots,
+                       err)) {
+            std::fprintf(stderr, "strata generate: %s\n", err.c_str());
+            return 1;
+        }
+        std::fprintf(stderr, "strata generate: peer GPU %d: %lld experts, %.2f GiB (filled in %.1f s); with the primary's "
+                             "%lld that is %lld of %lld on the GPUs\n", o.peer_device, (long long) peer.resident(), peer.gib(),
+                     std::chrono::duration<double>(Clock::now() - tp0).count(), (long long) xcache.slots(),
+                     (long long) (peer.resident() + xcache.slots()), (long long) (g.n_layers * g.n_expert));
+    }
+
     Drive drive;
     for (int r = 0; r < 3; ++r) if (o.expert_cache_remote[(size_t) r] > 0)
         drive.d.remote[drive.d.remote_count++] = &remote_experts[(size_t) r];
+    drive.d.peer = peer.valid() ? &peer : nullptr;
     drive.d.hit_cpu_order = o.expert_cache_cpu_order;
     drive.d.split_rows = !o.no_split_rows;
     drive.d.pool = &pool;
@@ -3353,6 +3537,10 @@ int main(int argc, char** argv) {
     }
 
     std::FILE* dump = nullptr;
+    // The logits header is written WITH THE FIRST ROW, not at open: a native pack never reaches the
+    // per-token dump site, and a header promising rows that were never written is worse than no file.
+    int32_t hdr[2] = {0, 0};
+    bool hdr_written = false;
     const int64_t dump_positions = (int64_t) o.tokens.size() - 1 + o.max_new;
     if (!o.dump_logits.empty()) {
         if (dump_positions > INT32_MAX || n_vocab > INT32_MAX) {
@@ -3372,12 +3560,7 @@ int main(int argc, char** argv) {
         // of the size gets a wrong answer that looks authoritative.  `tools/logits_identical.py` caught it by
         // parsing the header and refusing the file.
         const int32_t n_rows = (int32_t) strata::program::logits_selection::row_count(dump_positions, o.logits_stride);
-        const int32_t hdr[2] = {(int32_t) n_vocab, n_rows};
-        if (std::fwrite(hdr, sizeof hdr, 1, dump) != 1) {
-            std::fprintf(stderr, "strata generate: cannot write logits header\n");
-            std::fclose(dump);
-            return 1;
-        }
+        hdr[0] = n_vocab; hdr[1] = n_rows;
     }
 
     // ---- THE C1 ORACLE: ONE RESIDUAL SNAPSHOT PER LAYER PER POSITION, so the engine can be bisected against
@@ -4211,6 +4394,13 @@ int main(int argc, char** argv) {
                                      "context, or read prompts in smaller chunks (--prefill 512)\n");
             return 1;
         }
+        if (peer.valid() && o.peer_prefill_rows != 0) {
+            const int64_t rows = o.peer_prefill_rows > 0 ? o.peer_prefill_rows : o.prefill_chunk * K / 2;
+            if (!sp.set_peer(&peer, rows, err)) {
+                std::fprintf(stderr, "strata serve: %s - the prompt path stays on the primary GPU\n", err.c_str());
+                err.clear();
+            }
+        }
         mem_mark("the head and the prompt path");
         // #340: STRATA_SPLIT_SMALL_OWN=S (tokens): on a layer split, every stage that borrows keeps the slots for an
         // S-token chunk's buffers for the whole session (0.1.29's own buffers, carved from the tail of its cache):
@@ -4585,6 +4775,7 @@ int main(int argc, char** argv) {
             }
         };
         auto apply_pending = [&](bool wait) {
+            if (peer.valid()) peer.apply_pending(wait);
             if (pending.empty()) return;
             if (wait) cudaEventSynchronize(adapt_ev);
             else if (cudaEventQuery(adapt_ev) != cudaSuccess) return;
@@ -4611,7 +4802,7 @@ int main(int argc, char** argv) {
                 const float* u = drive.d.usage.data() + l * g.n_expert;
                 const int32_t* r = host_res.data() + l * g.n_expert;
                 for (int32_t e = 0; e < (int32_t) g.n_expert; ++e) {
-                    if (r[e] < 0) { if (u[e] >= 2.0f) cand.emplace_back(u[e], e); }
+                    if (r[e] < 0) { if (u[e] >= 2.0f && !(peer.valid() && peer.has(l, e))) cand.emplace_back(u[e], e); }
                     else vict.emplace_back(u[e], e);
                 }
                 if (cand.empty() || vict.empty()) continue;
@@ -4652,10 +4843,21 @@ int main(int argc, char** argv) {
                     const strata::core::OnDevice on(st->dev);
                     cudaEventRecord(st->adapt_ev, st->adapt_stream);
                 }
-            // #477: the routing counted since the start (each count adds up to 1 / (1 - 0.7) over its decays: the
-            // sum is proportional to the routing itself) - only with --expert-profile-save, else `heat` is empty
+            // #477: the routing counted since the start (each count adds up to 1 / (1 - --adapt-decay) over its
+            // decays: the sum is proportional to the routing itself) - only with --expert-profile-save, else `heat`
+            // is empty
             for (size_t i = 0; i < heat.size(); ++i) heat[i] += (double) drive.d.usage[i];
-            for (float& v : drive.d.usage) v *= 0.7f;
+            if (peer.valid()) {                   // multi-GPU: the peer takes the next most-routed CPU misses
+                std::vector<int32_t> r0 = host_res;
+                for (const auto& [i, slot] : pending) r0[(size_t) i] = slot;   // swapped into the primary already
+                std::string perr;
+                if (!peer.adapt(drive.d.usage.data(), r0.data(),
+                                o.peer_adapt_swaps >= 0 ? o.peer_adapt_swaps : o.adapt_swaps, perr)) {
+                    std::fprintf(stderr, "strata serve: %s\n", perr.c_str());
+                    return false;
+                }
+            }
+            for (float& v : drive.d.usage) v *= o.adapt_decay;
             return true;
         };
         // #477: write the learned profile (between requests and at QUIT: a prompt's lent slots are back by then).
@@ -4748,6 +4950,12 @@ int main(int argc, char** argv) {
             const int64_t free_mib = (int64_t) (free_b >> 20);
             if (free_mib >= 256) {
                 std::fprintf(stderr, "strata serve: %lld MiB of VRAM free with everything loaded\n", (long long) free_mib);
+            } else if (reserve_adapted) {
+                // #496: the reserve was already lowered to make the cache fit - a bigger one would leave it no room
+                std::fprintf(stderr, "strata serve: WARNING: %lld MiB of VRAM free with everything loaded - this card "
+                                     "only just fits the model (the VRAM reserve was lowered to %d MiB so the expert "
+                                     "cache fits): requests may stall. Close other programs that use the GPU, or lower "
+                                     "--max-context\n", (long long) free_mib, o.vram_reserve_mib);
             } else {
                 std::fprintf(stderr, "strata serve: %lld MiB of VRAM free with everything loaded - LOW: requests may stall;"
                                      " add --vram-reserve-mib %lld to the config's args (or lower --max-context)\n",
@@ -5518,7 +5726,10 @@ int main(int argc, char** argv) {
                 drive.d.layers = 0;
                 drive.d.experts = 0;
                 drive.d.failed = false;
-                apply_pending(false);
+                // #463: the previous adapt round's copies land first - with a non-blocking query, whether a swapped-in
+                // expert ran on the GPU or the CPU (they round differently) depended on the copy's timing
+                // (STRATA_ADAPT_NOWAIT=1: 0.1.37's non-blocking query, the A/B)
+                apply_pending(!adapt_nowait());
                 if (hist_n > 0) {
                     // the tails the penalties count over, ONE PER ROW: the tokens the state has consumed, the
                     // fed-back head `x` (it joins `consumed` only after this window commits), then the drafts
@@ -6067,6 +6278,13 @@ int main(int argc, char** argv) {
                          (long long) n_vocab, (long long) pos);
             return 1;
         }
+        // the header with the first row (see `hdr`): a run that dumps nothing leaves an empty file
+        if (emit_logits && !hdr_written && std::fwrite(hdr, sizeof hdr, 1, dump) != 1) {
+            std::fprintf(stderr, "strata generate: cannot write logits header\n");
+            std::fclose(dump);
+            return 1;
+        }
+        if (emit_logits) hdr_written = true;
         if (emit_logits && std::fwrite(logits.data(), sizeof(float), (size_t) n_vocab, dump) != (size_t) n_vocab) {
             std::fprintf(stderr, "strata generate: cannot write logits at position %lld\n", (long long) pos);
             std::fclose(dump);
@@ -6215,10 +6433,20 @@ int main(int argc, char** argv) {
         std::vector<std::pair<int32_t, int32_t>> pending;
         cudaEvent_t adapt_ev = nullptr;
         cudaEventCreateWithFlags(&adapt_ev, cudaEventDisableTiming);
+        int64_t adapt_rounds = 0;   // counted here: `rounds` is declared below the adapt lambda
         auto apply_pending = [&](bool wait) {
             if (pending.empty()) return;
+            // STRATA_TRACE_ADAPT=1: whether a round's copies had landed when the next window read the table
+            static const bool trace_pending = std::getenv("STRATA_TRACE_ADAPT") != nullptr;
             if (wait) cudaEventSynchronize(adapt_ev);
-            else if (cudaEventQuery(adapt_ev) != cudaSuccess) return;
+            else if (cudaEventQuery(adapt_ev) != cudaSuccess) {
+                if (trace_pending)
+                    std::fprintf(stderr, "strata: PENDING not landed, %zu stay non-resident this window\n",
+                                 pending.size());
+                return;
+            }
+            if (trace_pending)
+                std::fprintf(stderr, "strata: PENDING landed, %zu experts become resident\n", pending.size());
             src.commit_exchanges();   // the resident RAM mode: the evicted experts take their places in RAM
             for (const auto& [i, slot] : pending) host_res[(size_t) i] = slot;
             pending.clear();
@@ -6230,7 +6458,18 @@ int main(int argc, char** argv) {
         // routed clearly more often.  Copies run between rounds, when the GPU is idle.
         auto adapt = [&]() -> bool {
             const Clock::time_point ta = Clock::now();
-            if (!pending.empty()) return true;   // the previous swaps are still in flight
+            ++adapt_rounds;
+            // STRATA_TRACE_ADAPT: why an adapt round did or did not swap.  Default off, one getenv, and it
+            // reports the only thing that can make an adapt round a coin flip: whether the PREVIOUS round's
+            // asynchronous expert copies had landed by the time this round started.
+            static const bool trace_adapt = std::getenv("STRATA_TRACE_ADAPT") != nullptr;
+            if (!pending.empty()) {
+                if (trace_adapt)
+                    std::fprintf(stderr, "strata: ADAPT round=%lld SKIPPED, %zu swaps still in flight\n",
+                                 (long long) adapt_rounds, pending.size());
+                return true;   // the previous swaps are still in flight
+            }
+            if (trace_adapt) std::fprintf(stderr, "strata: ADAPT round=%lld considering\n", (long long) adapt_rounds);
             struct Swap { float gain; int32_t layer, in, out; };
             std::vector<Swap> swaps;
             std::vector<std::pair<float, int32_t>> cand, vict;
@@ -6274,7 +6513,10 @@ int main(int argc, char** argv) {
                 pending.emplace_back((int32_t) in, slot);      // resident once the copy has landed
             }
             if (!swaps.empty()) cudaEventRecord(adapt_ev, adapt_stream);
-            for (float& v : drive.d.usage) v *= 0.7f;
+            if (trace_adapt)
+                std::fprintf(stderr, "strata: ADAPT round=%lld swapped %zu of %d slots, usage decayed\n",
+                             (long long) adapt_rounds, swaps.size(), o.adapt_swaps);
+            for (float& v : drive.d.usage) v *= o.adapt_decay;
             swaps_total += (int64_t) swaps.size();
             ms_adapt += std::chrono::duration<double, std::milli>(Clock::now() - ta).count();
             return true;
@@ -6342,7 +6584,10 @@ int main(int argc, char** argv) {
             drive.d.layers = 0;
             drive.d.experts = 0;
             drive.d.failed = false;
-            apply_pending(false);
+            // #463: the previous adapt round's copies land first - with a non-blocking query, whether a swapped-in
+            // expert ran on the GPU or the CPU (they round differently) depended on the copy's timing
+            // (STRATA_ADAPT_NOWAIT=1: 0.1.37's non-blocking query, the A/B)
+            apply_pending(!adapt_nowait());
             if (!ver.run(T, window.data(), p, &drive_pool_multi, &drive, outv.data(), err)) {
                 std::fprintf(stderr, "strata generate: %s\n", err.c_str());
                 return 1;

tools/hip/gfx1200-hipblaslt-100202.txt

@@ -0,0 +1,69 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1200 100202
+bf16 10240 2560 10240 2560 30831
+bf16 10240 2560 10240 256 31013
+bf16 10240 2560 10240 512 31013
+bf16 10240 320 10240 2560 30902
+bf16 10240 320 10240 256 30831
+bf16 10240 320 10240 512 30779
+bf16 12288 2560 12288 2560 30969
+bf16 12288 2560 12288 256 31013
+bf16 2560 10240 10240 256 30654
+bf16 2560 10240 10240 512 30998
+bf16 2560 12288 2560 256 30654
+bf16 2560 2560 2560 2560 30963
+bf16 2560 2560 2560 256 30789
+bf16 2560 2560 2560 512 30844
+bf16 2560 512 4096 512 30654
+bf16 2560 6144 2560 2560 30654
+bf16 2560 640 2560 2560 30726
+bf16 2560 640 2560 256 30654
+bf16 2560 640 2560 512 30654
+bf16 2560 96 4096 37 30681
+bf16 2560 96 4096 48 30664
+bf16 320 10240 320 2560 31046
+bf16 320 10240 320 256 30668
+bf16 48 2560 96 4096 30904
+bf16 6144 2560 6144 2560 30831
+bf16 6144 2560 6144 256 30969
+bf16 6144 2560 6144 512 31013
+bf16 640 2560 640 2560 31171
+bf16 640 2560 640 256 30912
+bf16 640 2560 640 512 30686
+f16 10240 2560 10240 2560 55549
+f16 10240 2560 10240 256 55715
+f16 10240 2560 10240 512 55549
+f16 10240 320 10240 2560 55604
+f16 10240 320 10240 256 55430
+f16 10240 320 10240 512 55481
+f16 12288 2560 12288 2560 55671
+f16 12288 2560 12288 256 55715
+f16 12288 2560 12288 512 55533
+f16 2560 10240 10240 2560 55731
+f16 2560 10240 10240 256 55356
+f16 2560 10240 10240 512 55730
+f16 2560 12288 2560 2560 55729
+f16 2560 12288 2560 256 55356
+f16 2560 12288 2560 512 55693
+f16 2560 2560 2560 2560 55665
+f16 2560 2560 2560 256 55491
+f16 2560 2560 2560 512 55581
+f16 2560 512 4096 37 55381
+f16 2560 512 4096 512 55356
+f16 2560 6144 2560 2560 55356
+f16 2560 6144 2560 256 55356
+f16 2560 6144 2560 512 55356
+f16 2560 640 2560 2560 55554
+f16 2560 640 2560 256 55356
+f16 2560 640 2560 512 55356
+f16 2560 96 4096 48 55383
+f16 320 10240 320 2560 55545
+f16 320 10240 320 256 55517
+f16 320 10240 320 512 55606
+f16 512 2560 512 4096 55575
+f16 6144 2560 6144 2560 55533
+f16 6144 2560 6144 256 55671
+f16 6144 2560 6144 512 55715
+f16 640 2560 640 2560 55873
+f16 640 2560 640 256 55614
+f16 640 2560 640 512 55826

tools/hip/gfx1201-hipblaslt-100202.txt

@@ -0,0 +1,26 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1201 100202
+bf16 1 2560 1 4096 74196
+bf16 48 2560 96 4096 73754
+bf16 48 2560 96 8192 73754
+bf16 128 2560 128 4096 73754
+bf16 128 2560 128 8192 73715
+bf16 320 10240 320 4096 73838
+bf16 320 10240 320 8192 73838
+bf16 512 2560 512 8192 73803
+bf16 2560 2560 2560 8192 73778
+bf16 10240 320 10240 8192 73761
+f16 512 2560 512 4096 88193
+f16 512 2560 512 8192 88307
+f16 640 2560 640 4096 88251
+f16 640 2560 640 8192 88192
+f16 2560 640 2560 4096 88216
+f16 2560 640 2560 8192 88307
+f16 2560 6144 2560 4096 88281
+f16 2560 6144 2560 8192 88281
+f16 6144 2560 6144 4096 88255
+f16 6144 2560 6144 8192 88307
+f16 10240 2560 10240 4096 88255
+f16 10240 2560 10240 8192 88255
+f16 12288 2560 12288 4096 88255
+f16 12288 2560 12288 8192 88255

tools/test_setup_choices.py

@@ -330,5 +330,61 @@ class VramReserve(unittest.TestCase):
         self.assertTrue(call.called)
 
 
+class SmallCardTip(unittest.TestCase):
+    """#496: a card under 8 GB (a 6 GB laptop RTX 3060) gets a tip for when the start has no room for the expert cache;
+    the engine chooses its own reserve there, so the config is the one any card gets.  An 8 GB card: no tip."""
+
+    def install(self, vram, *extra):
+        from test_setup_golden import card, install
+        return install(63.7, [card(0, "NVIDIA GeForce RTX 3060 Laptop GPU", vram, "86")],
+                       ["--family", "qwen", "--model", "Q2_0", "--no-start", *extra])
+
+    def test_a_6gb_card(self):
+        code, out, cfg, _ = self.install(6.0)
+        self.assertEqual(code, 0, out)
+        self.assertNotIn("--vram-reserve-mib", cfg["args"])          # the engine decides (no fixed reserve)
+        self.assertIn("no VRAM is left for the expert cache", out)
+        self.assertIn("--draft-vocab en", out)
+        self.assertIn("--mtp", cfg["args"])                          # the draft layer stays: the server needs it
+
+    def test_an_8gb_card_has_no_tip(self):
+        for vram in (8188 / 1024, 8.0):                               # nvidia-smi lists an 8 GB card as 8188 MiB
+            code, out, cfg, _ = self.install(vram)
+            self.assertEqual(code, 0, out)
+            self.assertNotIn("no VRAM is left for the expert cache", out)
+            self.assertNotIn("--vram-reserve-mib", cfg["args"])
+
+    def test_a_given_reserve_is_kept(self):
+        code, out, cfg, _ = self.install(6.0, "--vram-reserve-mib", "500")
+        self.assertEqual(code, 0, out)
+        a = cfg["args"]
+        self.assertEqual(a[a.index("--vram-reserve-mib") + 1], "500")
+
+    def test_the_tip(self):
+        self.assertIn("an 8K context", " ".join(setup.small_card_note(32768, None)))
+        self.assertIn("--draft-vocab en", " ".join(setup.small_card_note(32768, "cjk")))
+        tip = " ".join(setup.small_card_note(8192, "en"))
+        self.assertNotIn("--draft-vocab", tip)
+        self.assertIn("close other programs", tip)
+
+
+class DesktopReserveTip(unittest.TestCase):
+    """#560 #516: an AMD card on a Linux desktop gets a recommended reserve (3072 MiB) - a tip, the config is not
+    changed."""
+
+    def test_desktop_detection(self):
+        with mock.patch.object(setup.sys, "platform", "linux"):
+            self.assertTrue(setup.linux_desktop({"WAYLAND_DISPLAY": "wayland-0"}))
+            self.assertTrue(setup.linux_desktop({"DISPLAY": ":0"}))
+            self.assertFalse(setup.linux_desktop({}))                 # a headless box / ssh
+        with mock.patch.object(setup.sys, "platform", "win32"):
+            self.assertFalse(setup.linux_desktop({"DISPLAY": ":0"}))  # Windows counts the desktop itself (#497)
+
+    def test_the_tip(self):
+        tip = " ".join(setup.desktop_reserve_note())
+        self.assertIn("--vram-reserve-mib 3072", tip)
+        self.assertIn("desktop", tip)
+
+
 if __name__ == "__main__":
     unittest.main()

tools/test_setup_update.py

@@ -69,6 +69,29 @@ class Update(unittest.TestCase):
             m.assert_not_called()
         self.assertIn("START-HERE.bat", out)
 
+    def test_a_json_that_is_no_model_config_is_skipped(self):
+        """#549: a strata-*.json without "args" (not written by setup) stopped update.sh with KeyError: 'args'."""
+        with tempfile.TemporaryDirectory() as d:
+            p = self.config(Path(d))
+            other = Path(d) / "strata-notes.json"
+            other.write_text(json.dumps({"note": "mine"}), encoding="utf-8")
+            broken = Path(d) / "strata-cut.json"
+            broken.write_text("{\"args\": [", encoding="utf-8")
+            rc, pip, eng, dv, start, call, out = self.run_update([other, broken, p])
+        self.assertEqual(rc, 0, out)
+        self.assertIn('skipped strata-notes.json (no "exe" or "args"): it is not a Strata model config', out)
+        self.assertIn("skipped strata-cut.json (not valid JSON)", out)
+        dv.assert_called_once()                                     # the real model is still refreshed
+        self.assertIn("Qwen IQ3_S: up to date", out)
+        self.assertIn("Strata is updated", out)
+
+    def test_installed_configs_lists_only_model_configs(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = self.config(Path(d))
+            (Path(d) / "strata-notes.json").write_text("{}", encoding="utf-8")
+            with mock.patch.object(setup, "ROOT", Path(d)), contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(setup.installed_configs(), [p])
+
     def test_main_update_never_starts(self):
         with tempfile.TemporaryDirectory() as d:
             p = self.config(Path(d))
@@ -83,5 +106,40 @@ class Update(unittest.TestCase):
         start.assert_not_called()
 
 
+class SettingsLine(unittest.TestCase):
+    """#564: a start prints the settings it uses (the engine options without the model's paths, and the server's
+    fields), so a change made by hand to strata-<model>.json shows without reading the log."""
+
+    CFG = {"exe": "x", "host": "0.0.0.0", "port": 8081, "api_key": "secret", "gpu": [0, 1], "fit_max_tokens": True,
+           "args": ["--native", "E:\\Strata\\packs\\iq3_s", "--mtp", "/s/mtp/rt", "m.gguf", "--kv", "int8",
+                    "--kv-resident", "32768", "--spec-min-p", "0.5", "--vram-reserve-mib", "2048", "--mmap-experts",
+                    "--prefill", "auto"]}
+
+    def test_summary(self):
+        s = setup.settings_summary(self.CFG)
+        self.assertEqual(s, "--kv int8 --kv-resident 32768 --spec-min-p 0.5 --vram-reserve-mib 2048 --mmap-experts "
+                            "--prefill auto; server 0.0.0.0:8081, api key set, gpu 0,1, fit_max_tokens true")
+        self.assertNotIn("secret", s)
+        self.assertIn("127.0.0.1:9000", setup.settings_summary({"args": []}, 9000))
+
+    def test_a_start_prints_it(self):
+        with tempfile.TemporaryDirectory() as d:
+            exe = Path(d) / "strata.exe"
+            exe.write_bytes(b"")
+            p = Path(d) / "strata-iq3_s.json"
+            args = [x for x in self.CFG["args"] if x != "m.gguf"]           # no model file here
+            p.write_text(json.dumps({**self.CFG, "exe": str(exe), "gpu": 0, "args": args}), encoding="utf-8")
+            out = io.StringIO()
+            with mock.patch.object(setup, "gpus", lambda: []), \
+                    mock.patch.object(setup, "refresh_draft_vocab"), \
+                    mock.patch.object(setup, "is_wsl", lambda: False), \
+                    mock.patch.object(setup.subprocess, "call", return_value=0), \
+                    contextlib.redirect_stdout(out):
+                setup.start(p, None, open_browser=False, yes=True)
+        text = " ".join(out.getvalue().split())
+        self.assertIn("Settings (strata-iq3_s.json): --kv int8 --kv-resident 32768", text)
+        self.assertIn("--vram-reserve-mib 2048", text)
+
+
 if __name__ == "__main__":
     unittest.main()