Engine 0.1.30: short prompts up to 28% faster (every expert streams from 1024-token chunks), the resident low-RAM variant, the multi-GPU session carve (#216), AMD RDNA4, idle unload (#208); opt-ins: rope scaling (#84), conversation cache (#189), STRATA_IQ_MT_MIN=1 (#152); setup requires it

# Conflicts:
#	setup.py

.dockerignore

@@ -0,0 +1,27 @@
+# Keep the image to the source only: everything generated is re-created on
+# first start (inside the container or on the /data volume).
+.git
+
+# local Python environment (rebuilt in the image)
+.venv/
+__pycache__/
+*.pyc
+.pytest_cache/
+
+# setup.py outputs and downloads (engine, packs, models, logs)
+/engine/
+/build/
+/build*/
+/dist/
+/packs/
+/mtp/
+/models/
+/Strata-data/
+third_party/llama.cpp/
+third_party/*.zip
+*.done
+*.log
+strata-*.json
+strata-*.txt
+run-*.sh
+run-*.bat

.gitignore

@@ -18,6 +18,8 @@ __pycache__/
 
 # plan v0.3 P1 (repository hygiene, review E1): profiler captures and large diagnostic dumps stay out of git.
 *.nsys-rep
+
+# Pi harness metadata (local session state, not project source)
 *.sqlite
 *.npz
 *.occ1
@@ -26,7 +28,7 @@ td.log
 # created by setup.py on the user's machine
 /.venv/
 /build/
-/third_party/llama.cpp/
+/third_party/llama.cpp
 /models/
 /packs/
 /mtp/

CMakeLists.txt

@@ -8,7 +8,7 @@
 # below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
 # and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.25 LANGUAGES CXX)
+project(strata VERSION 0.1.30 LANGUAGES CXX)
 add_compile_definitions(STRATA_VERSION="${PROJECT_VERSION}")
 # issue #31: release builds keep a PDB beside the exe (not shipped in it), so a minidump from a stalled engine can
 # be read back to the exact source line
@@ -33,7 +33,7 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 option(STRATA_ENABLE_CUDA "Build the CUDA targets (Phase 2+)" OFF)
-option(STRATA_ENABLE_HIP "Build the HIP targets for gfx1100 wave32 hardware" OFF)
+option(STRATA_ENABLE_HIP "Build the HIP targets for wave32 AMD GPUs (gfx1100, gfx1201)" OFF)
 option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF)
 if(STRATA_PREFILL_MMQ AND NOT STRATA_ENABLE_HIP)
   message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_ENABLE_HIP=ON")
@@ -50,8 +50,19 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt AND EXISTS ${CMAKE_CU
   set(_strata_tests_default ON)
 endif()
 option(STRATA_BUILD_TESTS "Build the test suite" ${_strata_tests_default})
+option(STRATA_BUILD_CONVERSATION_TESTS "Build standalone conversation cache tests" OFF)
 option(STRATA_WERROR "Treat compiler warnings as errors" OFF)
 
+if(STRATA_BUILD_CONVERSATION_TESTS)
+  enable_testing()
+  add_executable(conversation_cache_test src/core/conversation_cache_test.cpp)
+  target_include_directories(conversation_cache_test PRIVATE include)
+  add_test(NAME conversation_cache_test COMMAND conversation_cache_test)
+  add_executable(conversation_memory_test src/core/conversation_memory_test.cpp src/core/conversation_memory.cpp)
+  target_include_directories(conversation_memory_test PRIVATE include)
+  add_test(NAME conversation_memory_test COMMAND conversation_memory_test)
+endif()
+
 if(STRATA_ENABLE_CUDA)
   enable_language(CUDA)
   # `enable_language(CUDA)` gives the .cu compile rules but NOT the imported targets, so `CUDA::cudart` has to
@@ -62,15 +73,26 @@ if(STRATA_ENABLE_CUDA)
   endif()
   foreach(_a IN LISTS CMAKE_CUDA_ARCHITECTURES)
     string(REGEX REPLACE "-.*$" "" _base "${_a}")     # 120-real / 120-virtual are still 120
-    # Plan v0.3 P6: developed and measured on sm_120 (RTX 50); the kernels need sm_80 or newer (tf32 mma in
-    # the attention scorer, bf16 math), so RTX 30 (86) and RTX 40 (89) build too.  Older cards are refused.
+    # Turing port: developed and measured on sm_120 (RTX 50); the tf32 mma in the QSA scorer has a
+    # portable fp32-FMA fallback, so RTX 20 (75), 30 (86), RTX 40 (89) and RTX 50 (120) build too.
+    # Pre-sm_75 is refused.
     if(_base STREQUAL "native" OR _base STREQUAL "all" OR _base STREQUAL "all-major")
       continue()
     endif()
-    if(_base LESS 80 AND NOT STRATA_EXPERIMENTAL_SM75)   # the A/B of a Turing card as a layer-split stage
+    # STRATA_EXPERIMENTAL_SM60: a Pascal (sm_60, GP100-class) build.  It sits below every architecture
+    # these kernels were written for, so two intrinsics are replaced for it - `__dp4a` (6.1+) and
+    # `__nanosleep` (7.0+) - in include/strata/kernels/dp4a.hpp, which is bit-exact for the first and
+    # merely a shorter pause for the second.  RTX 20/30/40/50 cards must use the release engine instead.
+    if(_base LESS 75 AND NOT STRATA_EXPERIMENTAL_SM60)
       message(FATAL_ERROR
-        "Strata needs an NVIDIA GPU of compute capability 8.0 or newer (RTX 30 / 40 / 50 series), but "
-        "CMAKE_CUDA_ARCHITECTURES is '${CMAKE_CUDA_ARCHITECTURES}'.")
+        "Strata needs an NVIDIA GPU of compute capability 7.5 or newer (RTX 20 / 30 / 40 / 50 series), but "
+        "CMAKE_CUDA_ARCHITECTURES is '${CMAKE_CUDA_ARCHITECTURES}'.  (A Pascal build is "
+        "-DSTRATA_EXPERIMENTAL_SM60=ON; upstream does not support it.)")
+    endif()
+    # #220 / #224: RTX 50 engines built with CUDA 12.8 crashed in the prompt path on Linux (13.0 and newer did not)
+    if(_base GREATER_EQUAL 120 AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 13.0)
+      message(WARNING "Building for sm_${_base} (RTX 50) with CUDA ${CMAKE_CUDA_COMPILER_VERSION}: engines built with "
+                      "CUDA 12.8 crashed on long prompts on these cards (issues #220, #224). Use CUDA 13.0 or newer.")
     endif()
   endforeach()
   message(STATUS "Strata: CUDA enabled, arch ${CMAKE_CUDA_ARCHITECTURES}")
@@ -226,8 +248,8 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
                           src/platform/direct_file.cpp)
   target_link_libraries(strata_kernels PUBLIC strata_core strata_artifact)
   # perf-review D-1: the tensor-core prompt attention vs the FP32 kernel and FP64 (GPU, synthetic, no model); with
-  # the tests, or alone with -DSTRATA_PARITY_PROMPT_ATTN=ON
-  if(STRATA_BUILD_TESTS OR STRATA_PARITY_PROMPT_ATTN)
+  # the tests, or alone with -DSTRATA_PARITY_PROMPT_ATTN=ON.  CUDA only: HIP compiles the tensor-core kernel out.
+  if((STRATA_BUILD_TESTS OR STRATA_PARITY_PROMPT_ATTN) AND STRATA_ENABLE_CUDA)
     add_executable(qsa_prompt_attn_parity src/kernels/qsa_prompt_attn_parity.cpp)
     target_include_directories(qsa_prompt_attn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
     target_link_libraries(qsa_prompt_attn_parity PRIVATE strata_kernels CUDA::cudart)
@@ -288,7 +310,8 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   # library is the honest shape of the dependency graph: core knows about devices and weights, kernels know
   # about arithmetic, and this one knows the ORDER the arithmetic happens in.
   add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
-    src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp)
+    src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp
+    src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
   target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
   if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/file_expert_source_test.cpp")
@@ -298,6 +321,25 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     add_test(NAME file_expert_source_test COMMAND file_expert_source_test)
     set_tests_properties(file_expert_source_test PROPERTIES TIMEOUT 60)
   endif()
+  if(STRATA_BUILD_CONVERSATION_TESTS)
+    add_executable(conversation_snapshot_test src/core/conversation_snapshot_test.cpp)
+    target_link_libraries(conversation_snapshot_test PRIVATE strata_engine)
+    add_test(NAME conversation_snapshot_test COMMAND conversation_snapshot_test)
+    set_tests_properties(conversation_snapshot_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
+    add_executable(conversation_validation_test src/core/conversation_validation_test.cpp)
+    target_link_libraries(conversation_validation_test PRIVATE strata_engine)
+    add_test(NAME conversation_validation_test COMMAND conversation_validation_test)
+    set_tests_properties(conversation_validation_test PROPERTIES ENVIRONMENT "CUDA_VISIBLE_DEVICES=-1" TIMEOUT 120)
+    if(NOT STRATA_ENABLE_HIP AND UNIX AND NOT APPLE AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+      add_executable(conversation_transfer_test src/core/conversation_validation_test.cpp)
+      target_compile_definitions(conversation_transfer_test PRIVATE CONVERSATION_TEST_TRANSFERS)
+      target_link_options(conversation_transfer_test PRIVATE
+        -Wl,--wrap=cudaMemcpy -Wl,--wrap=cudaDeviceSynchronize -Wl,--wrap=cudaGetLastError)
+      target_link_libraries(conversation_transfer_test PRIVATE strata_engine)
+      add_test(NAME conversation_transfer_test COMMAND conversation_transfer_test)
+      set_tests_properties(conversation_transfer_test PROPERTIES ENVIRONMENT "CUDA_VISIBLE_DEVICES=-1" TIMEOUT 120)
+    endif()
+  endif()
 
   # `strata generate` (P2.S6).  The FIRST user-facing program: everything under it is a component, and this is
   # what composes them into a token.  No tokenizer - the prompt arrives as ids - because Gate C1 compares logits
@@ -384,6 +426,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   add_executable(iq_parity src/kernels/iq_parity.cpp)
   target_include_directories(iq_parity PRIVATE ${_strata_gpu_include_directories})
   target_link_libraries(iq_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  if(STRATA_ENABLE_CUDA)
+    # native_mmvq's multi_exact contract, bitwise: a T-column call against T single-column calls on the same Q8_1
+    # bytes, with the flag off as the negative control (synthetic, GPU, no model; the control assumes 32-lane warps)
+    add_executable(mmvq_multi_parity src/kernels/mmvq_multi_parity.cpp)
+    target_include_directories(mmvq_multi_parity PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(mmvq_multi_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+    add_test(NAME mmvq_multi_parity COMMAND mmvq_multi_parity)
+  endif()
   add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)
   target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})
   target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
@@ -408,6 +458,12 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   target_include_directories(sampler_parity PRIVATE ${_strata_gpu_include_directories})
   target_link_libraries(sampler_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME sampler_parity COMMAND sampler_parity --selftest)
+  set_tests_properties(sampler_parity PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=0;STRATA_SAMPLER_ONE_BLOCK=0")
+  # the same fixtures on the other two sampled paths: the one-block kernel and the 0.1.20 kernel
+  add_test(NAME sampler_parity_one_block COMMAND sampler_parity --selftest)
+  set_tests_properties(sampler_parity_one_block PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=0;STRATA_SAMPLER_ONE_BLOCK=1")
+  add_test(NAME sampler_parity_old COMMAND sampler_parity --selftest)
+  set_tests_properties(sampler_parity_old PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=1;STRATA_SAMPLER_ONE_BLOCK=0")
 
   add_executable(rope_parity src/kernels/rope_parity.cpp)
   target_include_directories(rope_parity PRIVATE ${_strata_gpu_include_directories})
@@ -494,11 +550,11 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   add_test(NAME cvec_parity COMMAND cvec_parity)
 
   # Plan v0.3 P3: native MMVQ for 1..8 tokens (bitwise selftest + --bench timing); synthetic data, no model.
-  if(STRATA_BUILD_TESTS)
-  add_executable(native_mmvq_multi bench/micro/native_mmvq_multi.cpp)
-  target_include_directories(native_mmvq_multi PRIVATE ${_strata_gpu_include_directories})
-  target_link_libraries(native_mmvq_multi PRIVATE strata_kernels ${_strata_gpu_runtime_target})
-  add_test(NAME native_mmvq_multi COMMAND native_mmvq_multi --selftest)
+  if(STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/native_mmvq_multi.cpp")
+    add_executable(native_mmvq_multi bench/micro/native_mmvq_multi.cpp)
+    target_include_directories(native_mmvq_multi PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(native_mmvq_multi PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+    add_test(NAME native_mmvq_multi COMMAND native_mmvq_multi --selftest)
   endif()
 
   # Plan v0.3 P0.1: working-set lock of a host region (CPU only).
@@ -506,6 +562,12 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   target_link_libraries(platform_memory_test PRIVATE strata_core ${_strata_gpu_runtime_target})
   add_test(NAME platform_memory_test COMMAND platform_memory_test)
 
+  if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    add_executable(pinned_shared_test src/core/pinned_shared_test.cpp)
+    target_link_libraries(pinned_shared_test PRIVATE strata_core ${_strata_gpu_runtime_target})
+    add_test(NAME pinned_shared_test COMMAND pinned_shared_test)
+  endif()
+
   # Plan v0.3 P2: the n-gram table read straight from the SSD. The selftest needs only the CPU and a temp file.
   add_executable(ple_reader_test src/ngram/ple_reader_test.cpp)
   target_include_directories(ple_reader_test PRIVATE ${_strata_gpu_include_directories})
@@ -574,10 +636,20 @@ if(STRATA_BUILD_TESTS)
   add_executable(draft_policy_test src/spec/draft_policy_test.cpp)
   target_link_libraries(draft_policy_test PRIVATE strata_spec)
   add_test(NAME draft_policy_test COMMAND draft_policy_test)
+endif()
+if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)
   # the serve loop's conversation-cache retention policy (header-only, no library to link)
   add_executable(conv_cache_test src/program/conv_cache_test.cpp)
   target_include_directories(conv_cache_test PRIVATE include)
   add_test(NAME conv_cache_test COMMAND conv_cache_test)
+  # coupled draft sampling's counter and penalty-ring arithmetic (header-only, CPU)
+  add_executable(coupled_draft_test src/core/coupled_draft_test.cpp)
+  target_include_directories(coupled_draft_test PRIVATE include)
+  add_test(NAME coupled_draft_test COMMAND coupled_draft_test)
+  # the f32 -> bf16 rounding against ggml's, NaN included, over every f32 bit pattern (header-only, CPU)
+  add_executable(bf16_bits_test src/kernels/bf16_bits_test.cpp)
+  target_include_directories(bf16_bits_test PRIVATE include)
+  add_test(NAME bf16_bits_test COMMAND bf16_bits_test)
 endif()
 
 add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp
@@ -664,7 +736,7 @@ if(STRATA_NATIVE_EXPERTS)
     endif()
     set(_strata_mmq ${_strata_gg}/src/ggml-cuda)
     add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
-    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs)
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)   # q8_0: the drafter's matrices (E-9)
       target_sources(strata_mmq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu)
     endforeach()
     target_include_directories(strata_mmq PRIVATE ${_strata_gg}/include ${_strata_gg}/src ${_strata_mmq}
@@ -692,7 +764,7 @@ if(STRATA_NATIVE_EXPERTS)
       src/prefill/moe_mmq.cu
       src/prefill/ggml_cuda_host.cu
       "${_strata_mmq}/quantize.cu")
-    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs)
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)
       list(APPEND _strata_mmq_sources "${_strata_mmq}/template-instances/mmq-instance-${_t}.cu")
     endforeach()
     set_source_files_properties(${_strata_mmq_sources} PROPERTIES LANGUAGE HIP)
@@ -725,7 +797,7 @@ if(STRATA_NATIVE_EXPERTS)
   endif()
 endif()
 
-if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS)
+if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/hit_cpu_order_parity.cu")
   add_executable(hit_cpu_order_parity bench/micro/hit_cpu_order_parity.cu)
   target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
 endif()

Dockerfile

@@ -0,0 +1,116 @@
+# syntax=docker/dockerfile:1
+#
+# Strata: Qwen3.8-Flash-Next on NVIDIA GPUs (RTX 30/40/50, 12+ GB VRAM; two or
+# three cards can share one model, 8 GB each - docs/MULTI_GPU.md).
+#
+# The engine is compiled during docker build, so the first container start only
+# downloads the model (~70 GB) and starts the server. docker build has no GPU,
+# so the CUDA architectures are fixed here instead of read from nvidia-smi: the
+# engine is a fat binary with a cubin per listed arch, and the runtime picks the
+# one matching your card. Narrow CUDA_ARCHITECTURES to your card for a faster
+# build; a card outside the set needs a rebuild with its own arch.
+#
+# Build:
+#   docker build -t strata .
+#   docker build -t strata --build-arg CUDA_ARCHITECTURES=89 .        # RTX 40 only
+#
+# Run (host needs an NVIDIA driver >= 580 and nvidia-container-toolkit):
+#   docker run --rm --gpus all \
+#     -p 8080:8080 \
+#     --ulimit memlock=-1 \
+#     -v strata-data:/data \
+#     -e MODEL=IQ2_XS \
+#     strata
+#
+# Setup choices are env vars, read by docker-entrypoint.sh: FAMILY, MODEL, CONTEXT,
+# VISION (no | yes | cpu), KV (int8 | q4_0 | k8v4), GPU (one card) or GPUS ("0,2"
+# or "all", with LAYER_SPLIT), LOW_RAM (auto | on | off), HOST, PORT, API_KEY.
+#
+# Only the model files, the prepared pack, the MTP layer and the install config
+# live in the /data volume; the engine is part of the image. Strata loads 32-62 GB
+# into RAM, so a capped container needs -e LOW_RAM=on: setup.py reads the RAM from
+# /proc/meminfo, which here is the host's total, not the container's limit. Add an
+# API key before exposing the port to a network: -e API_KEY=<secret>. Pass
+# -e REINSTALL=1 to change the model settings later.
+#
+# --gpus all on a host with two usable cards: setup takes both (the layer split is
+# its recommended default). Pin one card with -e GPU=0, or name them with
+# -e GPUS=0,2. A volume set up for one card switches to the pair on its first start
+# on a two-card host unless GPU or GPUS pins it. LOW_RAM=on runs on one card.
+
+FROM nvidia/cuda:13.0.0-devel-ubuntu24.04
+
+# STRATA_EXECV=1: setup.py replaces itself with the server, so the server is PID 1
+# and docker stop's SIGTERM reaches it (see setup.start). Normal Linux starts, which
+# don't set it, keep spawning the server as a child.
+ENV DEBIAN_FRONTEND=noninteractive PYTHONUNBUFFERED=1 LANG=C.UTF-8 STRATA_EXECV=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential ca-certificates curl git libatomic1 libgomp1 \
+        python3 python3-pip python3-venv unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/strata
+COPY . .
+
+# RTX 20 (75), RTX 30 (86), RTX 40 (89), RTX 50 (120), plus 80 for A-series. CMakeLists
+# refuses anything below 75. BUILD_VISION=0 skips the image encoder build.
+ARG CUDA_ARCHITECTURES=75;80;86;89;120
+ARG BUILD_VISION=1
+
+RUN python3 -m venv .venv \
+    && .venv/bin/pip install --no-cache-dir --upgrade pip \
+    && .venv/bin/pip install --no-cache-dir \
+        numpy jinja2 regex pyyaml tqdm requests cmake ninja pillow psutil \
+    && chmod +x setup.sh docker-entrypoint.sh
+
+# llama.cpp at the pinned commit, then the engine and the image encoder, built
+# exactly the way setup.py builds them. BUILD.json is what setup.py reads to
+# decide whether an engine is current: source=local with a matching src hash
+# means the first start reuses it instead of recompiling.
+RUN .venv/bin/python - <<'PYEOF'
+import json, os, pathlib, shutil
+import setup
+
+llama = setup.get_llama_cpp()
+nvcc, _ = setup.find_nvcc()
+arch = os.environ.get("CUDA_ARCHITECTURES", "75;80;86;89;120").strip().strip('"').replace(",", ";")
+vision = "gpu" if os.environ.get("BUILD_VISION", "1") == "1" else "none"
+
+setup.cmake_build(setup.ROOT, setup.ROOT / "build", "strata",
+    ["-DSTRATA_ENABLE_CUDA=ON", "-DSTRATA_BUILD_TESTS=OFF",
+     f"-DCMAKE_CUDA_ARCHITECTURES={arch}", f"-DCMAKE_CUDA_COMPILER={nvcc}",
+     f"-DSTRATA_GGML_DIR={llama}"], None, "build-strata.bat")
+if vision != "none":
+    setup.cmake_build(setup.ROOT / "tools" / "vision", setup.ROOT / "build-vision", "strata-vision",
+        [f"-DLLAMA_DIR={llama}", "-DSTRATA_VISION_CUDA=ON",
+         f"-DCMAKE_CUDA_ARCHITECTURES={arch}", f"-DCMAKE_CUDA_COMPILER={nvcc}"], None, "build-vision.bat")
+
+eng = setup.ROOT / "engine"
+eng.mkdir(exist_ok=True)
+shutil.copy2(setup.ROOT / "build" / setup.EXE, eng / setup.EXE)
+if vision != "none":
+    shutil.copy2(setup.ROOT / "build-vision" / "bin" / setup.VEXE, eng / setup.VEXE)
+bindir = pathlib.Path(nvcc).parent
+meta = {"source": "local", "version": setup.source_version(),
+        "archs": [int(a.split("-")[0]) for a in arch.split(";") if a.split("-")[0].isdigit()], "vision": vision,
+        "cuda_dirs": [str(d) for d in (bindir, bindir / "x64", bindir.parent / "lib64") if d.is_dir()],
+        "src": setup.source_hash(setup.ENGINE_SOURCES),
+        "vision_src": setup.source_hash(setup.VISION_SOURCES) if vision != "none" else None}
+(eng / "BUILD.json").write_text(json.dumps(meta, indent=1))
+PYEOF
+
+# the cmake trees are build-time only; the engine itself is what the container needs
+RUN rm -rf build build-vision
+
+VOLUME ["/data"]
+EXPOSE 8080
+
+# /health is answered before the API key gate, so it works with or without one.
+# The port only opens after the model loads (1-3 minutes, longer on a first run),
+# so the start period is generous: a too short one marks a still-loading container
+# unhealthy and a restart policy would kill it mid-download.
+HEALTHCHECK --interval=30s --timeout=5s --start-period=600s --retries=3 \
+  CMD curl -fs "http://127.0.0.1:${PORT:-8080}/health" || exit 1
+
+ENTRYPOINT ["./docker-entrypoint.sh"]

README.md

@@ -25,15 +25,15 @@ Measured on an RTX 5070 (12 GB), a Ryzen 5 7600 and 64 GB of RAM:
 
 | Size | Writes answers (short chat) | Writes answers (128K context) | Reads your prompt |
 | --- | ---: | ---: | ---: |
-| **Q2_0** | 90 tokens/s | 67 tokens/s | 1,840 tokens/s |
-| **IQ2_XS** | 74 tokens/s | 60 tokens/s | 1,800 tokens/s |
-| **IQ3_XXS** | 62 tokens/s | 46 tokens/s | 1,550 tokens/s |
-| **IQ3_S** | 52 tokens/s | 41 tokens/s | 1,500 tokens/s |
-| **Coder** (IQ1_M) | 51 tokens/s | 44 tokens/s | 1,870 tokens/s |
+| **Q2_0** | 93 tokens/s | 74 tokens/s | 2,170 tokens/s |
+| **IQ2_XS** | 79 tokens/s | 63 tokens/s | 2,090 tokens/s |
+| **IQ3_XXS** | 62 tokens/s | 49 tokens/s | 1,750 tokens/s |
+| **IQ3_S** | 53 tokens/s | 46 tokens/s | 1,620 tokens/s |
+| **Coder** (IQ1_M) | 55 tokens/s | 43 tokens/s | 2,180 tokens/s |
 
 - **Writes answers** = how fast the reply appears (tokens per second).
 - **Reads your prompt** = how fast it takes in what you send (long documents, code, chat history), measured on a
-  32K-token prompt; a 4K prompt reads at 890-1,520 tokens/s. A 32K prompt takes about 17 seconds with Q2_0.
+  32K-token prompt; a 4K prompt reads at 910-1,580 tokens/s. A 32K prompt takes about 15 seconds with Q2_0.
 
 A card with more VRAM is faster, because more of the model fits on the GPU: an RTX 3090 (24 GB) should do roughly
 100-140 tokens per second. All measurements, long-context numbers and estimates for other cards are in the
@@ -47,7 +47,7 @@ asks whether to share the model across them (recommended when two can). An insta
 its next start. Or choose yourself: `START-HERE.bat --gpus 0,2` (both, remembered), `--gpus all`, or `--gpu 0` (one
 card, this start only). Each card keeps the experts of its own layers, and prompts flow through the cards in a
 pipeline: on an RTX 5080 + RTX 3090 prompts were read 18-20% faster than on the 5080 alone, decoding on par.
-Every card must be an RTX 30 series or newer with 8 GB or more. See [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
+Every card must be an RTX 20 series or newer with 8 GB or more. See [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
 
 ## Which model should I pick?
 
@@ -84,13 +84,14 @@ one later with `SETUP.bat` (the same as `START-HERE.bat --setup`; on Linux `./se
 For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](docs/ORCA.md).
 It needs an explicit packing conversion and is not an installer menu option.
 
-An **AMD Radeon RX 7900 XT / XTX on Linux** works too (experimental): `./setup.sh --backend hip`, chosen by itself on
-a PC with no NVIDIA card Strata can use. It installs ROCm without sudo and compiles the engine (one GPU, no images
-yet). Details: [AMD HIP](docs/AMD_HIP.md).
+An **AMD Radeon RX 7900 XT / XTX, RX 9070 / 9070 XT or Radeon AI PRO R9700 on Linux** works too (experimental):
+`./setup.sh --backend hip`, chosen by itself on a PC with no NVIDIA card Strata can use. It installs ROCm without sudo
+and compiles the engine (one GPU, no images yet). Details: [AMD HIP](docs/AMD_HIP.md).
 
 ## Install
 
-**You need:** an NVIDIA RTX 30, 40 or 50 card with 12 GB of VRAM or more, enough RAM for the size you pick (above),
+**You need:** an NVIDIA RTX 20, 30, 40 or 50 card with 12 GB of VRAM or more (RTX 20 since 0.1.27), enough RAM for the size you pick (above;
+a big GPU makes up for less RAM - the [low-RAM mode](docs/DETAILS.md)),
 ~80 GB of free disk space (an SSD makes the first start much faster), and Windows 10/11 or Linux. The only thing you
 install yourself is a current **NVIDIA driver** ([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA
 App). Everything else - Python, the engine, the model - is set up for you.
@@ -101,7 +102,9 @@ App). Everything else - Python, the engine, the model - is set up for you.
 2. Double-click **`START-HERE.bat`**.
 3. Answer a few questions - or just press Enter each time for the recommended choice:
    - **Which model and size?** The original or Swift 1.5, and Q2_0, IQ2_XS, IQ3_XXS or IQ3_S - see [above](#which-model-should-i-pick)
-   - **How much context?** How much text it can keep in mind at once (it suggests one for your card)
+   - **How much context?** How much text it can keep in mind at once (it suggests one for your card). 384K and
+     512K (experimental) extend the model past its trained 262K by rope scaling - the setup turns it on itself (yarn and a
+     covering factor; `--rope-scaling`/`--rope-scale` override) ([details](docs/DETAILS.md))
    - **Images?** Whether it should also read pictures
    - **Experimental speed projection?** Off unless you say yes - [read what it does](docs/DETAILS.md#experimental-speed-projection-experimental-off-by-default) first
 
@@ -121,6 +124,38 @@ the same way - nothing big is downloaded again.
 
 **Linux:** run `./setup.sh` - same questions, same result.
 
+**Docker (Linux):** the same idea, in a container.
+
+1. Host: Docker with the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
+   and a driver **580 or newer** (CUDA 13.0).
+2. Build (this compiles the engine into the image, so the container never compiles):
+   `docker build -t strata .`
+   `docker build -t strata --build-arg CUDA_ARCHITECTURES=89 .` builds for one card only (faster).
+   The default covers RTX 30 (86), RTX 40 (89), RTX 50 (120) and A-series (80); a card outside that
+   set needs a rebuild with its own arch. Add `--build-arg BUILD_VISION=0` to skip the image encoder.
+3. Run (the first start downloads the ~70 GB model, then starts; later starts go straight to serving):
+   `docker run --rm --gpus all -p 8080:8080 --ulimit memlock=-1 -v strata-data:/data strata`
+
+   The setup choices are env vars: `-e MODEL=IQ2_XS -e FAMILY=qwen -e CONTEXT=32768 -e VISION=no`
+   (or `MODEL=Q2_0|IQ3_XXS|IQ3_S`, `FAMILY=swift|coder`; the defaults above are the recommended ones).
+   `-e VISION=cpu` keeps the image encoder on the CPU. `-e KV=int8|q4_0|k8v4` picks the KV cache
+   precision; `k8v4` is INT8 K with 4-bit V and keeps its KV in VRAM from 64K up.
+   Only the model files, the prepared pack, the MTP layer and the install config live in the
+   `strata-data` volume; the engine is part of the image. Switching between models already on the
+   volume needs no setup pass: `-e MODEL=Q2_0 -e FAMILY=coder` picks that model's config. Add
+   `-e REINSTALL=1` only to change settings for a model already set up (context, vision, KV, host,
+   api_key, LOW_RAM), since those are recorded in its config.
+   Strata loads 32-62 GB into RAM. `--gpus all` on a host with two usable cards takes both: the
+   layer split is setup's recommended default ([docs/MULTI_GPU.md](docs/MULTI_GPU.md)), and a volume
+   set up for one card switches to the pair on its first start there. Pin one card with `-e GPU=0`,
+   or name them with `-e GPUS=0,2` and where the later card's layers start with `-e LAYER_SPLIT=18`.
+   A memory limit needs `-e LOW_RAM=on`, which maps the model's experts from the pack instead of
+   keeping them in RAM: setup.py measures the host's RAM, not the container's limit, so it cannot
+   see a cap. LOW_RAM runs on one card.
+   The server listens on `0.0.0.0:8080` by default; set `-e API_KEY=<secret>` before exposing the port
+   to a network. The image has a `HEALTHCHECK` on `/health`, so `docker ps` shows the container
+   healthy once the model is loaded, and `GET /v1/status` says what it is running.
+
 ## Using it
 
 <p align="center"><img src="docs/media/runpagoda.png" width="900" alt="The Strata app's Monitor tab next to a coding agent"><br>

bench/results/2026-09-28-rope-scaling/README.md

@@ -0,0 +1,49 @@
+# Rope scaling: needle recall past the trained 262K
+
+The feature (PR #84, experimental): `--rope-scaling none|linear|yarn` extends the context past the model's
+trained 262,144 positions by rescaling the rotary angles - llama.cpp's types, ggml's `rope_yarn`
+arithmetic, one process config applied at the cos/sin table (default path) and as kernel constants
+(`--native-rope`, the native indexer, prefill). The three tables here are the first end-to-end check
+that a scaled run actually recalls past the trained end.
+
+**Hardware:** RTX 4070 Ti SUPER 16 GB, Ryzen 9 5900X (no AVX-512 - the IQ3_XXS native pack runs its
+expert kernels on AVX-2), 125 GB RAM, Linux. **Engine:** this branch built from source (the log records
+`rope scaling ... --max-context ... against a trained context of 262144` at startup). The runs below
+were taken on the rope tree built over the 0.1.17-era base; the branch then rebased onto main (engine
+0.1.20) - the drift touches none of the rope files - and the rebase was re-validated: `rope_parity`
+green, a 64-token greedy run token-identical to the plain 0.1.20 build, and the yarn 262k probe
+repeated green (see the last table row). Baseline arm A also re-checked the default is the stock
+model: a 64-token greedy run produces the same token ids as the ready-made 0.1.17 engine, and
+`rope_parity` holds the `none` table bit-identical to the pre-scaling builder.
+**Method:** `tools/needle_bench.py` against a running server (launch recipes in `config-*.json`),
+`temperature 0`, thinking off, depth 50%, one probe per length.
+
+| Arm | Scaling | Context | Probe | Prompt tokens | Result | Probe time |
+| --- | --- | --- | --- | --- | --- | --- |
+| A | none | 262,144 | 1k @ 50% | 1,442 | FOUND | 4 s |
+| A | none | 262,144 | 32k @ 50% | 31,763 | FOUND | 27 s |
+| A | none | 262,144 | 128k @ 50% | 128,328 | FOUND | 86 s |
+| B | linear, factor 2 | 560,000 | 262k @ 50% | 274,222 | FOUND | 232 s |
+| B | linear, factor 2 | 560,000 | 512k @ 50% | 527,051 | FOUND | 619 s |
+| C | yarn, factor 2 | 560,000 | 262k @ 50% | 274,215 | FOUND | 236 s |
+| C | yarn, factor 2 | 560,000 | 512k @ 50% | 526,988 | FOUND | 615 s |
+| C | yarn, factor 2 | 560,000 | 262k @ 50% (the rebase re-check) | 273,708 | FOUND | 235 s |
+
+All 8 of 8 probes found the needle; JSON per arm: `needle-A-none.json`, `needle-B-linear2.json`,
+`needle-B-linear2-512k.json`, `needle-C-yarn2.json` (+ the rebase re-check in
+`needle-C-yarn2-rebased.json`); server and engine logs alongside.
+
+What this establishes, and what it does not:
+
+- **Past the trained end the model still reads.** 512k probes are 2.01x the trained 262,144 - position
+  interpolation (B) and YaRN (C) both hold a mid-context lookup there.
+- **Scaled runs within the trained range are a slightly different model.** `yarn`'s mscale applies at
+  every position (a 64-token greedy prompt diverges from the stock run after ~19 tokens); arm A is the
+  stock model, arms B/C are not, by design.
+- **One depth, one needle, one machine.** A single probe per (length, arm) at depth 50% says "recall
+  works there", not "no degradation anywhere" - a full depth sweep and a longer-length ladder (768K,
+  1M) on the bench machine (64 GB RAM there vs 125 GB here, so 262K IQ3_XXS fits either way) is the
+  natural follow-up. 512K prompts prefill at ~850-950 tokens/s on this card; the probe time is almost
+  entirely prefill.
+- **The `560,000` context** in arms B/C is headroom for the needle harness's tokenizer overshoot
+  (a "512k" probe tokenizes to ~527k); 524,288 exactly would refuse the probe.

bench/results/2026-09-28-rope-scaling/config-A-none-262144.json

@@ -0,0 +1,38 @@
+{
+ "exe": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata/build/strata",
+ "cwd": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata",
+ "tokenizer": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/packs/iq3_xxs/tokenizer",
+ "model_name": "qwen3.8-flash-next-iq3_xxs-rope-scaling",
+ "lib_dirs": [
+  "/usr/local/cuda/bin",
+  "/usr/local/cuda/lib64"
+ ],
+ "port": 8130,
+ "args": [
+  "--pack",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/packs/iq3_xxs",
+  "--native",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+  "--ple-gguf",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+  "--expert-profile",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata/data/expert-profile.bin",
+  "--expert-cache",
+  "auto",
+  "--prefill",
+  "auto",
+  "--spec",
+  "4",
+  "--spec-min-p",
+  "0.5",
+  "--mtp",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/mtp/rt",
+  "--kv",
+  "int8",
+  "--kv-resident",
+  "32768",
+  "--max-context",
+  "262144"
+ ],
+ "log": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata/bench/results/2026-09-28-rope-scaling/engine-A-none-262144.log"
+}

bench/results/2026-09-28-rope-scaling/config-B-linear2.json

@@ -0,0 +1,42 @@
+{
+ "exe": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata/build/strata",
+ "cwd": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata",
+ "tokenizer": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/packs/iq3_xxs/tokenizer",
+ "model_name": "qwen3.8-flash-next-iq3_xxs-rope-scaling",
+ "lib_dirs": [
+  "/usr/local/cuda/bin",
+  "/usr/local/cuda/lib64"
+ ],
+ "port": 8130,
+ "args": [
+  "--pack",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/packs/iq3_xxs",
+  "--native",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+  "--ple-gguf",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+  "--expert-profile",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata/data/expert-profile.bin",
+  "--expert-cache",
+  "auto",
+  "--prefill",
+  "auto",
+  "--spec",
+  "4",
+  "--spec-min-p",
+  "0.5",
+  "--mtp",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/mtp/rt",
+  "--kv",
+  "int8",
+  "--kv-resident",
+  "32768",
+  "--max-context",
+  "560000",
+  "--rope-scaling",
+  "linear",
+  "--rope-scale",
+  "2"
+ ],
+ "log": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata/bench/results/2026-09-28-rope-scaling/engine-B-linear2-524288.log"
+}

bench/results/2026-09-28-rope-scaling/config-C-yarn2.json

@@ -0,0 +1,42 @@
+{
+ "exe": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata/build/strata",
+ "cwd": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata",
+ "tokenizer": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/packs/iq3_xxs/tokenizer",
+ "model_name": "qwen3.8-flash-next-iq3_xxs-rope-scaling",
+ "lib_dirs": [
+  "/usr/local/cuda/bin",
+  "/usr/local/cuda/lib64"
+ ],
+ "port": 8130,
+ "args": [
+  "--pack",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/packs/iq3_xxs",
+  "--native",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf",
+  "--ple-gguf",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/models/IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf",
+  "--expert-profile",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata/data/expert-profile.bin",
+  "--expert-cache",
+  "auto",
+  "--prefill",
+  "auto",
+  "--spec",
+  "4",
+  "--spec-min-p",
+  "0.5",
+  "--mtp",
+  "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata-data/mtp/rt",
+  "--kv",
+  "int8",
+  "--kv-resident",
+  "32768",
+  "--max-context",
+  "560000",
+  "--rope-scaling",
+  "yarn",
+  "--rope-scale",
+  "2"
+ ],
+ "log": "/home/jakub/workspace/deepseek-harness/harnes_workspace/Strata/bench/results/2026-09-28-rope-scaling/engine-C-yarn2-524288.log"
+}

bench/results/2026-09-28-rope-scaling/kv-sweep-tgs.json

@@ -0,0 +1,121 @@
+[
+ {
+  "length": "128k",
+  "round": 0,
+  "kind": "decode",
+  "prompt_tok": 128401,
+  "read": 52,
+  "gen": 357,
+  "read_tok_s": 79.2,
+  "gen_tok_s": 56.3,
+  "drafts_ok": 182,
+  "drafts_all": 305,
+  "seg_blocks": 3284340,
+  "seg_mib": 370.0,
+  "seg_misses": 91744,
+  "seg_hit_pct": 97.207,
+  "kib_per_token": 1061.29
+ },
+ {
+  "length": "128k",
+  "round": 1,
+  "kind": "read",
+  "prompt_tok": 128357,
+  "read": 128357,
+  "gen": 24,
+  "read_tok_s": 1389.2,
+  "gen_tok_s": 41.7,
+  "drafts_ok": 11,
+  "drafts_all": 23,
+  "seg_blocks": -3253560,
+  "seg_mib": -391.7,
+  "seg_misses": -97147,
+  "seg_hit_pct": 97.014,
+  "kib_per_token": -3.12
+ },
+ {
+  "length": "128k",
+  "round": 1,
+  "kind": "decode",
+  "prompt_tok": 128402,
+  "read": 52,
+  "gen": 388,
+  "read_tok_s": 76.5,
+  "gen_tok_s": 54.6,
+  "drafts_ok": 195,
+  "drafts_all": 363,
+  "seg_blocks": 3752616,
+  "seg_mib": 463.5,
+  "seg_misses": 114911,
+  "seg_hit_pct": 96.938,
+  "kib_per_token": 1223.26
+ },
+ {
+  "length": "512k",
+  "round": 0,
+  "kind": "read",
+  "prompt_tok": 522983,
+  "read": 522983,
+  "gen": 24,
+  "read_tok_s": 868.4,
+  "gen_tok_s": 42.4,
+  "drafts_ok": 13,
+  "drafts_all": 16,
+  "seg_blocks": -3808056,
+  "seg_mib": -387.8,
+  "seg_misses": -96113,
+  "seg_hit_pct": 97.476,
+  "kib_per_token": -0.76
+ },
+ {
+  "length": "512k",
+  "round": 0,
+  "kind": "decode",
+  "prompt_tok": 523028,
+  "read": 52,
+  "gen": 402,
+  "read_tok_s": 63.3,
+  "gen_tok_s": 48.4,
+  "drafts_ok": 221,
+  "drafts_all": 348,
+  "seg_blocks": 3580056,
+  "seg_mib": 920.4,
+  "seg_misses": 228658,
+  "seg_hit_pct": 93.613,
+  "kib_per_token": 2344.5
+ },
+ {
+  "length": "512k",
+  "round": 1,
+  "kind": "read",
+  "prompt_tok": 522984,
+  "read": 522984,
+  "gen": 24,
+  "read_tok_s": 857.1,
+  "gen_tok_s": 35.5,
+  "drafts_ok": 9,
+  "drafts_all": 18,
+  "seg_blocks": -3555408,
+  "seg_mib": -905.8,
+  "seg_misses": -225016,
+  "seg_hit_pct": 93.671,
+  "kib_per_token": -1.77
+ },
+ {
+  "length": "512k",
+  "round": 1,
+  "kind": "decode",
+  "prompt_tok": 523029,
+  "read": 52,
+  "gen": 402,
+  "read_tok_s": 61.3,
+  "gen_tok_s": 42.8,
+  "drafts_ok": 192,
+  "drafts_all": 388,
+  "seg_blocks": 4005216,
+  "seg_mib": 1079.9,
+  "seg_misses": 268221,
+  "seg_hit_pct": 93.303,
+  "kib_per_token": 2750.79
+ }
+]

bench/results/2026-09-28-rope-scaling/needle-A-none.json

@@ -0,0 +1,29 @@
+[
+ {
+  "length": "1k",
+  "depth": 50,
+  "prompt_tokens": 1442,
+  "seconds": 3.7,
+  "found": true,
+  "word": "meadow-copper-504",
+  "answer": "meadow-copper-504"
+ },
+ {
+  "length": "32k",
+  "depth": 50,
+  "prompt_tokens": 31763,
+  "seconds": 27.5,
+  "found": true,
+  "word": "falcon-quartz-940",
+  "answer": "falcon-quartz-940"
+ },
+ {
+  "length": "128k",
+  "depth": 50,
+  "prompt_tokens": 128328,
+  "seconds": 86.1,
+  "found": true,
+  "word": "willow-cobalt-696",
+  "answer": "willow-cobalt-696"
+ }
+]

bench/results/2026-09-28-rope-scaling/needle-B-linear2-512k.json

@@ -0,0 +1,11 @@
+[
+ {
+  "length": "512k",
+  "depth": 50,
+  "prompt_tokens": 527051,
+  "seconds": 619.0,
+  "found": true,
+  "word": "meadow-copper-504",
+  "answer": "meadow-copper-504"
+ }
+]

bench/results/2026-09-28-rope-scaling/needle-B-linear2.json

@@ -0,0 +1,16 @@
+[
+ {
+  "length": "262k",
+  "depth": 50,
+  "prompt_tokens": 274222,
+  "seconds": 232.2,
+  "found": true,
+  "word": "meadow-copper-504",
+  "answer": "meadow-copper-504"
+ },
+ {
+  "length": "512k",
+  "depth": 50,
+  "error": "HTTP Error 400: Bad Request"
+ }
+]

bench/results/2026-09-28-rope-scaling/needle-C-yarn2-rebased.json

@@ -0,0 +1,11 @@
+[
+ {
+  "length": "262k",
+  "depth": 50,
+  "prompt_tokens": 273708,
+  "seconds": 234.8,
+  "found": true,
+  "word": "meadow-copper-504",
+  "answer": "meadow-copper-504"
+ }
+]

bench/results/2026-09-28-rope-scaling/needle-C-yarn2.json

@@ -0,0 +1,20 @@
+[
+ {
+  "length": "262k",
+  "depth": 50,
+  "prompt_tokens": 274215,
+  "seconds": 235.5,
+  "found": true,
+  "word": "meadow-copper-504",
+  "answer": "meadow-copper-504"
+ },
+ {
+  "length": "512k",
+  "depth": 50,
+  "prompt_tokens": 526988,
+  "seconds": 615.0,
+  "found": true,
+  "word": "falcon-quartz-940",
+  "answer": "falcon-quartz-940"
+ }
+]

bench/results/2026-09-29-speed-0126/README.md

@@ -0,0 +1,36 @@
+# Speed with engine 0.1.26
+
+RTX 5070 12 GB (PCIe 5.0 x16), Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. The ready-made 0.1.26 engine, one-shot runs
+with what setup writes: `--prefill auto`, the shipped expert profile, `--expert-cache auto`, 8-bit KV above 4K, KV
+streaming (`--kv-resident 32768`) from 64K, MTP (`--spec 4 --spec-min-p 0.5`), greedy, 256 generated tokens, images
+off, no timing marks. The same code-agent prompts as `2026-09-29-speed-0122`. Every run: [`matrix.json`](matrix.json).
+
+## Prompt (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 536 | 1,299 | 2,171 | 2,126 | 2,107 |
+| **IQ2_XS** | 534 | 1,256 | 2,092 | 1,754 | 1,752 |
+| **IQ3_XXS** | 482 | 1,007 | 1,745 | 1,609 | 1,602 |
+| **IQ3_S** | 427 | 913 | 1,624 | 1,640 | 1,443 |
+| **Coder** | 656 | 1,583 | 2,177 | 2,236 | 2,208 |
+
+## Output (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 87.3 | 93.0 | 81.8 | 76.2 | 73.7 |
+| **IQ2_XS** | 79.6 | 78.6 | 76.3 | 63.7 | 62.7 |
+| **IQ3_XXS** | 61.9 | 61.6 | 58.5 | 57.2 | 49.0 |
+| **IQ3_S** | 52.4 | 53.3 | 48.3 | 46.3 | 45.5 |
+| **Coder** | 58.9 | 55.1 | 54.9 | 53.2 | 43.0 |
+
+- Against 0.1.22 (`2026-09-29-speed-0122`), prompts at 32K-128K are 8-28% faster: Q2_0 1,844 -> 2,171 at 32K and
+  1,682 -> 2,107 at 128K, the Coder 1,779 -> 2,208 at 128K. The gains come from:
+  - 0.1.24's QSA selection on tensor cores;
+  - 0.1.25's grouping tables in mapped memory and fused norms;
+  - 0.1.26's batched draft-layer pass.
+- 262K was not run again (0.1.22: Q2_0 1,304, IQ2_XS 1,181 tokens/s).
+- Output speed is the same decode as before. One run per cell moves several percent with the text (the share of
+  accepted drafts, `spec_accept` in `matrix.json`). For example, the Coder's 128K answer accepted 55% of its drafts
+  and its 64K answer 71%.

bench/results/2026-09-29-speed-0126/matrix.json

@@ -0,0 +1,352 @@
+[
+ {
+  "model": "Q2_0",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 23.9,
+  "decode_tok_s": 87.33,
+  "decoded": 256,
+  "prefill_tok_s": 536.42,
+  "ttft_s": 3.4,
+  "spec_accept": 0.681,
+  "tokens_per_round": 2.72,
+  "vram_slots": 4174
+ },
+ {
+  "model": "Q2_0",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 27.6,
+  "decode_tok_s": 92.97,
+  "decoded": 256,
+  "prefill_tok_s": 1299.44,
+  "ttft_s": 4.21,
+  "spec_accept": 0.838,
+  "tokens_per_round": 3.23,
+  "vram_slots": 4121
+ },
+ {
+  "model": "Q2_0",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 40.8,
+  "decode_tok_s": 81.75,
+  "decoded": 256,
+  "prefill_tok_s": 2170.85,
+  "ttft_s": 16.02,
+  "spec_accept": 0.788,
+  "tokens_per_round": 2.99,
+  "vram_slots": 3832
+ },
+ {
+  "model": "Q2_0",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 57.3,
+  "decode_tok_s": 76.23,
+  "decoded": 256,
+  "prefill_tok_s": 2126.03,
+  "ttft_s": 32.06,
+  "spec_accept": 0.707,
+  "tokens_per_round": 2.8,
+  "vram_slots": 3778
+ },
+ {
+  "model": "Q2_0",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 87.7,
+  "decode_tok_s": 73.73,
+  "decoded": 256,
+  "prefill_tok_s": 2106.85,
+  "ttft_s": 62.48,
+  "spec_accept": 0.679,
+  "tokens_per_round": 2.71,
+  "vram_slots": 3695
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 42.2,
+  "decode_tok_s": 79.62,
+  "decoded": 256,
+  "prefill_tok_s": 533.83,
+  "ttft_s": 3.51,
+  "spec_accept": 0.774,
+  "tokens_per_round": 2.89,
+  "vram_slots": 4359
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 40.6,
+  "decode_tok_s": 78.6,
+  "decoded": 256,
+  "prefill_tok_s": 1256.02,
+  "ttft_s": 4.31,
+  "spec_accept": 0.81,
+  "tokens_per_round": 3.29,
+  "vram_slots": 4308
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 53.6,
+  "decode_tok_s": 76.34,
+  "decoded": 256,
+  "prefill_tok_s": 2092.02,
+  "ttft_s": 16.57,
+  "spec_accept": 0.85,
+  "tokens_per_round": 3.2,
+  "vram_slots": 4029
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 78.3,
+  "decode_tok_s": 63.69,
+  "decoded": 256,
+  "prefill_tok_s": 1753.65,
+  "ttft_s": 38.08,
+  "spec_accept": 0.636,
+  "tokens_per_round": 2.59,
+  "vram_slots": 3977
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 115.1,
+  "decode_tok_s": 62.72,
+  "decoded": 256,
+  "prefill_tok_s": 1751.79,
+  "ttft_s": 74.85,
+  "spec_accept": 0.696,
+  "tokens_per_round": 2.6,
+  "vram_slots": 3896
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 46.9,
+  "decode_tok_s": 61.94,
+  "decoded": 256,
+  "prefill_tok_s": 482.04,
+  "ttft_s": 3.67,
+  "spec_accept": 0.845,
+  "tokens_per_round": 3.57,
+  "vram_slots": 3279
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 47.4,
+  "decode_tok_s": 61.64,
+  "decoded": 256,
+  "prefill_tok_s": 1007.48,
+  "ttft_s": 5.03,
+  "spec_accept": 0.801,
+  "tokens_per_round": 3.19,
+  "vram_slots": 3242
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 65.6,
+  "decode_tok_s": 58.49,
+  "decoded": 256,
+  "prefill_tok_s": 1745.16,
+  "ttft_s": 19.63,
+  "spec_accept": 0.789,
+  "tokens_per_round": 2.85,
+  "vram_slots": 3007
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 88.5,
+  "decode_tok_s": 57.21,
+  "decoded": 256,
+  "prefill_tok_s": 1609.31,
+  "ttft_s": 41.45,
+  "spec_accept": 0.74,
+  "tokens_per_round": 2.87,
+  "vram_slots": 2963
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 127.0,
+  "decode_tok_s": 48.95,
+  "decoded": 256,
+  "prefill_tok_s": 1602.33,
+  "ttft_s": 81.71,
+  "spec_accept": 0.643,
+  "tokens_per_round": 2.37,
+  "vram_slots": 2899
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 54.2,
+  "decode_tok_s": 52.4,
+  "decoded": 256,
+  "prefill_tok_s": 426.6,
+  "ttft_s": 4.03,
+  "spec_accept": 0.822,
+  "tokens_per_round": 3.17,
+  "vram_slots": 2617
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 58.9,
+  "decode_tok_s": 53.3,
+  "decoded": 256,
+  "prefill_tok_s": 912.99,
+  "ttft_s": 5.58,
+  "spec_accept": 0.85,
+  "tokens_per_round": 3.38,
+  "vram_slots": 2579
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 72.8,
+  "decode_tok_s": 48.31,
+  "decoded": 256,
+  "prefill_tok_s": 1624.07,
+  "ttft_s": 21.07,
+  "spec_accept": 0.827,
+  "tokens_per_round": 3.0,
+  "vram_slots": 2383
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 93.3,
+  "decode_tok_s": 46.25,
+  "decoded": 256,
+  "prefill_tok_s": 1640.34,
+  "ttft_s": 40.77,
+  "spec_accept": 0.693,
+  "tokens_per_round": 2.56,
+  "vram_slots": 2346
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 144.6,
+  "decode_tok_s": 45.51,
+  "decoded": 256,
+  "prefill_tok_s": 1443.21,
+  "ttft_s": 90.72,
+  "spec_accept": 0.697,
+  "tokens_per_round": 2.73,
+  "vram_slots": 2291
+ },
+ {
+  "model": "Coder",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 37.0,
+  "decode_tok_s": 58.91,
+  "decoded": 256,
+  "prefill_tok_s": 656.36,
+  "ttft_s": 2.48,
+  "spec_accept": 0.743,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2649
+ },
+ {
+  "model": "Coder",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 18.3,
+  "decode_tok_s": 55.1,
+  "decoded": 256,
+  "prefill_tok_s": 1583.1,
+  "ttft_s": 3.09,
+  "spec_accept": 0.713,
+  "tokens_per_round": 2.88,
+  "vram_slots": 2614
+ },
+ {
+  "model": "Coder",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 30.4,
+  "decode_tok_s": 54.89,
+  "decoded": 256,
+  "prefill_tok_s": 2177.17,
+  "ttft_s": 15.33,
+  "spec_accept": 0.731,
+  "tokens_per_round": 2.75,
+  "vram_slots": 2417
+ },
+ {
+  "model": "Coder",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 44.8,
+  "decode_tok_s": 53.22,
+  "decoded": 256,
+  "prefill_tok_s": 2236.49,
+  "ttft_s": 29.51,
+  "spec_accept": 0.709,
+  "tokens_per_round": 2.68,
+  "vram_slots": 2382
+ },
+ {
+  "model": "Coder",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 75.9,
+  "decode_tok_s": 43.04,
+  "decoded": 256,
+  "prefill_tok_s": 2208.27,
+  "ttft_s": 59.02,
+  "spec_accept": 0.553,
+  "tokens_per_round": 2.23,
+  "vram_slots": 2324
+ }
+]

cmake/hip_backend.cmake

@@ -1,12 +1,34 @@
-# Opt-in HIP configuration. Strata's CUDA-shaped kernels currently target RDNA3 gfx1100 wave32.
-# CMake/compiler discovery stays machine-independent; pass CMAKE_HIP_COMPILER when it is not on PATH.
+# Opt-in HIP configuration. Strata's CUDA-shaped kernels target wave32 RDNA3 / RDNA4 (64 KiB LDS per workgroup,
+# the signed dot4 instruction). CMake/compiler discovery stays machine-independent; pass CMAKE_HIP_COMPILER when it
+# is not on PATH.
 if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "")
-  set(CMAKE_HIP_ARCHITECTURES gfx1100 CACHE STRING "Strata HIP target architecture")
+  set(CMAKE_HIP_ARCHITECTURES gfx1100 CACHE STRING "Strata HIP target architecture(s), e.g. gfx1100 or gfx1100;gfx1201")
 endif()
-if(NOT CMAKE_HIP_ARCHITECTURES STREQUAL "gfx1100")
-  message(FATAL_ERROR
-    "Strata HIP currently supports only gfx1100 wave32; CMAKE_HIP_ARCHITECTURES is '${CMAKE_HIP_ARCHITECTURES}'")
+# Validated on real cards: gfx1100 (RX 7900 XT / XTX) and gfx1201 (RX 9070 / 9070 XT, Radeon AI PRO R9700).
+# The other RDNA3 / RDNA4 wave32 chips have the same LDS limit and dot4 instruction and build the same code, but
+# the maintainers have not run them (community reports: gfx1102 #192, gfx1200 #176).
+set(_strata_hip_validated gfx1100 gfx1201)
+set(_strata_hip_unvalidated gfx1101 gfx1102 gfx1200)
+set(STRATA_HIP_ARCH_LIST "")
+foreach(_arch IN LISTS CMAKE_HIP_ARCHITECTURES)
+  string(REGEX REPLACE ":.*$" "" _base "${_arch}")      # gfx1100:xnack- -> gfx1100
+  if(_base IN_LIST _strata_hip_validated)
+  elseif(_base IN_LIST _strata_hip_unvalidated)
+    message(WARNING "Strata HIP: ${_base} builds, but it is not validated on a real card yet; please report results")
+  else()
+    message(FATAL_ERROR
+      "Strata HIP supports wave32 gfx1100 and gfx1201 (unvalidated: ${_strata_hip_unvalidated}); "
+      "CMAKE_HIP_ARCHITECTURES is '${CMAKE_HIP_ARCHITECTURES}'")
+  endif()
+  list(APPEND STRATA_HIP_ARCH_LIST "${_base}")
+endforeach()
+list(REMOVE_DUPLICATES STRATA_HIP_ARCH_LIST)
+if(NOT STRATA_HIP_ARCH_LIST)
+  message(FATAL_ERROR "Strata HIP: CMAKE_HIP_ARCHITECTURES is empty")
 endif()
+# The compiled architectures reach the runtime device check (src/core/device.cu) as "gfx1100,gfx1201": a binary
+# carried to a card it has no code for stops at startup with a clear message instead of "invalid device function".
+string(REPLACE ";" "," STRATA_HIP_ARCHS "${STRATA_HIP_ARCH_LIST}")
 
 enable_language(HIP)
 find_package(hip CONFIG REQUIRED)
@@ -33,7 +55,7 @@ set(STRATA_HIP_COMPAT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/strata/hi
 add_library(strata_hip_runtime INTERFACE)
 target_include_directories(strata_hip_runtime BEFORE INTERFACE
   "${STRATA_HIP_COMPAT_INCLUDE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/include")
-target_compile_definitions(strata_hip_runtime INTERFACE STRATA_USE_HIP=1)
+target_compile_definitions(strata_hip_runtime INTERFACE STRATA_USE_HIP=1 "STRATA_HIP_ARCHS=\"${STRATA_HIP_ARCHS}\"")
 target_link_libraries(strata_hip_runtime INTERFACE hip::host)
 foreach(_language IN ITEMS CXX HIP)
   target_compile_options(strata_hip_runtime INTERFACE
@@ -55,4 +77,4 @@ foreach(_source IN ITEMS tests/hip/intrinsics.cpp tests/hip/native_qsa_score.cpp
   endif()
 endforeach()
 
-message(STATUS "Strata: HIP enabled, arch ${CMAKE_HIP_ARCHITECTURES}")
+message(STATUS "Strata: HIP enabled, arch ${STRATA_HIP_ARCHS}")

docker-entrypoint.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+# Entry point for the Strata container. The engine is compiled during docker
+# build and lives in the image, so the first start only downloads the model.
+# The install config is kept on the /data volume so a recreated container skips
+# the setup pass and goes straight to serving.
+set -e
+cd /opt/strata || exit 1
+
+STRATA_DATA="${STRATA_DATA:-/data}"
+FAMILY="${FAMILY:-qwen}"
+MODEL="${MODEL:-IQ2_XS}"
+CONTEXT="${CONTEXT:-32768}"
+VISION="${VISION:-no}"          # no | yes | cpu (the image encoder on the CPU)
+HOST="${HOST:-0.0.0.0}"
+PORT="${PORT:-8080}"
+API_KEY="${API_KEY:-}"
+KV="${KV:-}"                    # int8 | q4_0 | k8v4; empty: setup.py's own default (int8)
+GPUS="${GPUS:-}"                # "0,2" or "all": one model across several cards (docs/MULTI_GPU.md)
+GPU="${GPU:-}"                  # one card, numbered as nvidia-smi numbers them
+LAYER_SPLIT="${LAYER_SPLIT:-}"  # with GPUS: where each later card's layers start (default: auto)
+LOW_RAM="${LOW_RAM:-auto}"      # on: the experts come from the pack's experts.bin, not from RAM
+
+# setup.py starts the newest strata-*.json it finds, so link in exactly the one
+# this family and model were set up with. The config is the recorded output of
+# that setup (the pack, the profile, the quant, the KV decision), not settings
+# the entry point could rebuild from env vars. qwen has an empty family tag.
+case "$FAMILY" in qwen) prefix="" ;; *) prefix="${FAMILY}-" ;; esac
+tag="${prefix}$(printf '%s' "$MODEL" | tr 'A-Z' 'a-z')"
+cfg="$STRATA_DATA/config/strata-$tag.json"
+mkdir -p "$STRATA_DATA/config"
+
+# REINSTALL is only needed to change settings for a model that is already set up
+# (context, vision, KV, host, api_key). Switching between models already on the
+# volume needs no setup pass: their config is already there.
+#
+# KV / GPUS / GPU / LAYER_SPLIT are passed only when set, so an unset one keeps
+# setup.py's own default. LOW_RAM is always passed: setup.py measures the PC's RAM
+# from /proc/meminfo, which in a container is the host's total, not the container's
+# limit, so a memory-capped container has to ask for the low-RAM mode itself.
+if [ "${REINSTALL:-0}" = "1" ] || [ ! -f "$cfg" ]; then
+  echo "Setting up $tag: downloading the model (~70 GB; the engine is already in the image)."
+  set -- --family "$FAMILY" --model "$MODEL" --context "$CONTEXT" --vision "$VISION" \
+    --data-dir "$STRATA_DATA" --host "$HOST" --api-key "$API_KEY" \
+    --port "$PORT" --no-start --low-ram "$LOW_RAM"
+  if [ -n "$KV" ]; then set -- "$@" --kv "$KV"; fi
+  if [ -n "$GPUS" ]; then set -- "$@" --gpus "$GPUS"; fi
+  if [ -n "$GPU" ]; then set -- "$@" --gpu "$GPU"; fi
+  if [ -n "$LAYER_SPLIT" ]; then set -- "$@" --layer-split "$LAYER_SPLIT"; fi
+  .venv/bin/python setup.py --setup --yes "$@"
+  [ -e "/opt/strata/strata-$tag.json" ] && { cmp -s "/opt/strata/strata-$tag.json" "$cfg" || cp -f "/opt/strata/strata-$tag.json" "$cfg"; }
+else
+  [ -e "/opt/strata/strata-$tag.json" ] || ln -s "$cfg" "/opt/strata/strata-$tag.json"
+fi
+
+# Later starts skip straight here: setup.py finds the installed config and
+# launches serve/server.py (OpenAI- and Anthropic-compatible API on :8080).
+# GPUS / GPU / LAYER_SPLIT are repeated on purpose. Given at the start they pin the
+# cards for this model, and setup.py saves them in its config; without them a config
+# that names one card is offered once to a pair, on its own, when the host has two
+# cards that can share the model (setup.py's offer_together, docs/MULTI_GPU.md).
+set -- --port "$PORT"
+if [ -n "$GPUS" ]; then set -- "$@" --gpus "$GPUS"; fi
+if [ -n "$GPU" ]; then set -- "$@" --gpu "$GPU"; fi
+if [ -n "$LAYER_SPLIT" ]; then set -- "$@" --layer-split "$LAYER_SPLIT"; fi
+exec .venv/bin/python setup.py "$@"

docs/AMD_HIP.md

@@ -1,20 +1,22 @@
-# Experimental AMD HIP backend (gfx1100)
+# Experimental AMD HIP backend (gfx1100, gfx1201)
 
-This is a manual Linux source build for the RX 7900 XTX. It is opt-in; the
-NVIDIA installer and CUDA build remain the default. Other AMD architectures,
-wave64, Windows HIP, and mixed AMD/NVIDIA execution are outside this contribution.
+This is a Linux source build for the RX 7900 XT / XTX (RDNA3, gfx1100) and the
+RX 9070 / 9070 XT / Radeon AI PRO R9700 (RDNA4, gfx1201; see [RDNA4](#rdna4-gfx1201)).
+It is opt-in; the NVIDIA installer and CUDA build remain the default. Other AMD
+architectures, wave64, Windows HIP, and mixed AMD/NVIDIA execution are outside this contribution.
 
 The backend maps the CUDA-shaped runtime and BLAS calls to HIP/hipBLAS, uses
-RDNA3's signed integer dot instruction for quantized kernels, and supplies
+RDNA3/RDNA4's signed integer dot instruction for quantized kernels, and supplies
 wave32 shuffle/packed-byte operations. CUDA-only QSA matrix instructions have
 an ordered FP32 fallback. Prefill supports both dequantization plus hipBLAS GEMM and opt-in HIP ggml MMQ.
-An optional, calibrated hipBLASLt path accelerates dense projections on gfx1100.
+An optional, calibrated hipBLASLt path accelerates dense projections (per-architecture tables in `tools/hip`).
 This does not claim bit-identical model answers across backends. See
 [performance settings and evidence](AMD_HIP_PERFORMANCE.md).
 
 ## Install with setup (recommended)
 
-On Linux with an RX 7900 XT / XTX and the kernel's amdgpu driver (no ROCm install needed):
+On Linux with an RX 7900 XT / XTX or an RX 9070 / 9070 XT / Radeon AI PRO R9700 and the kernel's amdgpu driver
+(no ROCm install needed):
 
 ```sh
 ./setup.sh --backend hip
@@ -22,11 +24,17 @@ On Linux with an RX 7900 XT / XTX and the kernel's amdgpu driver (no ROCm instal
 
 - **Detection:** setup finds the card through the kernel's KFD topology. Integrated Radeon GPUs are listed as not
   supported. On a PC without an NVIDIA card Strata can use, `--backend hip` is chosen automatically.
-- **ROCm:** installed into `.venv` from AMD's TheRock wheels (~10 GB, no sudo), pinned to the version this backend was
-  tested with (`STRATA_ROCM_VERSION` / `STRATA_ROCM_INDEX` override it). A system ROCm in `/opt/rocm` (or
-  `$ROCM_PATH`) with hipcc and hipBLAS is used instead when present.
-- **Engine:** compiled on your PC (10-20 minutes, once; again after a `git pull` that changes it). This needs a C++
-  compiler and git (`sudo apt install build-essential git`).
+- **ROCm:** a system ROCm 7 in `/opt/rocm` (or `$ROCM_PATH`) with hipcc and hipBLAS is used when present. Otherwise
+  (or when it is older than 7.0) ROCm is installed into `.venv` from AMD's TheRock wheels (~10 GB, no sudo), pinned
+  to the version this backend was tested with, from the card family's index: `gfx110X-dgpu` for gfx1100,
+  `gfx120X-all` for gfx1201 (`STRATA_ROCM_VERSION` / `STRATA_ROCM_INDEX` override them).
+- **Engine:** compiled on your PC for the card's architecture (10-20 minutes, once; again after a `git pull` that
+  changes it, or when you pick a card of another architecture). This needs a C++ compiler and git
+  (`sudo apt install build-essential git`).
+- **hipBLASLt tuning table:** setup uses `tools/hip/<arch>-hipblaslt-<version>.txt` only when it matches both the
+  card's architecture and the installed hipBLASLt version (read from `hipblaslt-version.h`; 1.2.0 is `100200`).
+  Otherwise it says so and the prompt's dense matrix products use plain hipBLAS (slower prompts, same answers).
+  A table's solution ids are valid only for that pair, and the engine refuses any other table.
 - **Limits for now:** one GPU, no images, no calibration. The Monitor shows no GPU statistics.
 
 The rest of setup is the same as on NVIDIA: the model download, the start script, the server.
@@ -46,6 +54,13 @@ cmake -S . -B build-hip \
 cmake --build build-hip --target strata -j2
 ```
 
+`CMAKE_HIP_ARCHITECTURES` is `gfx1100`, `gfx1201`, or a list such as `"gfx1100;gfx1201"` (one binary for both).
+gfx1101, gfx1102 and gfx1200 (the same wave32, 64 KiB LDS and dot4 instruction) build with a warning: they have
+not been validated on a real card here. At startup the engine and `strata-device` compare each GPU they use
+(`gcnArchName` up to the `:` feature suffix) with the architectures the binary was compiled for, and require
+wave32. A binary carried to another card stops with the card's name, its architecture and the build's list,
+instead of failing later with "invalid device function".
+
 If CMake cannot find the HIP compiler, add
 `-DCMAKE_HIP_COMPILER=/path/to/rocm/llvm/bin/clang++`. On the Fedora-family test
 host this was `/usr/lib64/rocm/llvm/bin/clang++`. Use the compiler and libraries
@@ -98,8 +113,77 @@ The worker count above was used on a 16-core CPU; measure it for your CPU.
 The 4K context is a smoke-test starting point, not a model limit. The expert cache
 sizes itself automatically and leaves 1 GiB of VRAM headroom.
 
-The installer supports this backend (see "Install with setup" above). The vision helper and multi-GPU layer
-splits are NVIDIA-only for now.
+The installer supports this backend (see "Install with setup" above). The vision helper is NVIDIA-only for now.
+Setup installs one AMD card; the engine's layer split also runs on two AMD cards when the config is written by hand
+(see RDNA4 below).
+
+## RDNA4 (gfx1201)
+
+The RX 9070 / 9070 XT and the Radeon AI PRO R9700 run the same kernels as gfx1100: wave32, 64 KiB of LDS per
+workgroup, the signed dot4 instruction (`v_dot4_i32_iu8` through `__builtin_amdgcn_sudot4`) and a 100 MHz wall
+clock. The first report and patch came from doplxyz (#178). Validated on 2026-09-30 on doplxyz's test machine:
+an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 3900X (16 threads, no AVX-512),
+47 GB RAM, Ubuntu 24.04 in a KVM/VFIO guest.
+
+- **Build:** complete HIP build with tests (`-DCMAKE_HIP_ARCHITECTURES=gfx1201 -DSTRATA_BUILD_TESTS=ON
+  -DSTRATA_PREFILL_MMQ=ON`), with the system ROCm 7.14 (hipBLASLt 1.4.1) and with setup's own path: TheRock
+  7.10.0a20251120 wheels from `gfx120X-all` (hipBLASLt 1.2.0) and `build_engine_hip`.
+- **ctest** (all 45 registered tests, R9700): 42 pass, `hip_prefill_hipblaslt_gemm` skips (no table), and two
+  fail for reasons outside the GPU: `ple_parity` needs the Q2_0 model fixture, `expert_multi_test` needs an
+  AVX-512 CPU. `hip_handoff` needed the volatile ring store: on gfx1201 a plain store to mapped pinned memory
+  stays in the GPU's L2 until the stream is synchronized.
+- **Arch check:** a gfx1100-only build stops on the gfx1201 card at startup with the message above (engine and
+  `strata-device`).
+- **End to end** (Coder IQ1_M, setup's arguments: `--expert-cache auto --prefill auto --spec 4 --spec-min-p 0.5`,
+  MTP, `--kv int8`, `--max-context 32768`; 128 greedy tokens, system ROCm 7.14, no hipBLASLt table). The answers
+  are coherent and the same on both cards. The live server smoke (`serve/server.py`: web page, models, props,
+  chat, prefix reuse, streaming, the Anthropic endpoint, a sampled code answer) passed 9/9 on each card.
+
+  | card | expert cache | peak VRAM | 4K prompt | 4K decode | 16K prompt | 16K decode |
+  |---|---|---|---|---|---|---|
+  | Radeon AI PRO R9700 32 GB | 12,288 slots, 23.4 GiB | 29.7 GiB | 982 tok/s | 45.5 tok/s | 1,402 tok/s | 48.3 tok/s |
+  | RX 9070 XT 16 GB | 4,931 slots, 9.4 GiB | 15.6 GiB | 782 tok/s | 30.8 tok/s | 1,235 tok/s | 35.4 tok/s |
+  | R9700, TheRock 7.10 wheels | 12,288 slots | | 957 tok/s | 44.2 tok/s | 1,284 tok/s | 43.9 tok/s |
+
+  The engine's resident memory was about 26 GB in every run.
+- **hipBLASLt:** there is no gfx1201 table in `tools/hip`. A table calibrated on the R9700 at the engine's shapes
+  (hipBLASLt 1.4.1; 0.98-1.76x per GEMM over hipBLAS) changed the end-to-end prompt speed by 0-3%, within noise,
+  so none is shipped: on gfx1201 the plain hipBLAS path is already close.
+- **Both cards in one run (layer split, engine 0.1.30):** the config's `"backend": "hip", "gpu": [1, 0]` (R9700
+  first) runs through `serve/server.py`; setup does not offer it yet. Auto split put layers 0-27 on the R9700 and
+  28-47 on the 9070 XT. With every expert on the GPUs the split gives exactly the tokens of the R9700 alone (4K and
+  16K prompts); checkpoints on the split (second turn, rewind, a prompt sharing a prefix, a cancelled prompt
+  retried) give exactly the tokens of a fresh read. The conversation cache refuses a split at start (exit 2).
+  On this pair the split does not pay: the R9700 alone already holds all 12,288 expert pairs.
+
+  | run (the same session, warm) | 4K prompt | 16K prompt | decode |
+  |---|---|---|---|
+  | R9700 alone | 1,794 tok/s | 1,804 tok/s | 51-52 tok/s |
+  | R9700 + 9070 XT, auto split | 1,384 tok/s | 1,906 tok/s | 42-43 tok/s |
+  | RX 9070 XT alone | 1,016 tok/s | 1,519 tok/s | 38-40 tok/s |
+
+  A split is worth it when no single card holds the model's experts. On Linux the split pins at most 8 GiB of the
+  expert arena (a Windows limit that also applies here).
+- **Known:** rarely (about 1 start in 10) a HIP run's greedy output differs from another start's at some token, on
+  one card or two and on engine 0.1.29 as well; not yet explained.
+- **Not validated:** gfx1200 (RX 9060 XT; a community report is #176), images, long contexts beyond 16K,
+  answer-quality benchmarks.
+
+## Tuning table
+
+A hipBLASLt table holds solution ids that are valid only for one GPU architecture and one hipBLASLt version, so it
+is calibrated on the card with the ROCm the engine runs with. The shipped gfx1100 table's rows are the engine's
+dense GEMM shapes; to calibrate them for another card or version (a few minutes):
+
+```sh
+cmake --build build-hip --target tune_hipblaslt
+CASES=$(awk 'NR>2 {printf " --case %s,%s,%s,%s,%s", $1, $5, $2, $3, $4}' tools/hip/gfx1100-hipblaslt-100200.txt)
+./build-hip/tune_hipblaslt $CASES --tuning-out table.txt
+```
+
+The file's second line names the architecture and version (`STRATA_HIPBLASLT_TUNING_V1 gfx1201 100401`); save it
+as `tools/hip/<arch>-hipblaslt-<version>.txt` for setup, or point `STRATA_HIPBLASLT_TUNING` at it. Compare the
+prompt speed with and without it before keeping it.
 
 ## Original backend validation (PR #94)
 

docs/AMD_HIP_PERFORMANCE.md

@@ -34,9 +34,12 @@ GGUF/tokenizer/template, `--mmap-experts --resident-cpu-experts`, fixed ranked
 and `--vram-reserve-mib 1024`. Keep PLE on SSD.
 
 `--resident-cpu-experts` copies the complement of the static GPU cache to ordinary
-RAM. It requires mmap, a fixed expert profile, and disabled adaptation. It does
-not lock all data into RAM: temporary prefill borrowing and cache refill may
-still use the mapped fallback. Allocation needs sufficient available RAM;
+RAM. It requires mmap and a fixed expert profile. The slots the prompt path may
+borrow keep their experts in RAM too as far as the budget below allows (the rest
+use the mapped fallback while borrowed and for their refill), and adaptive swaps
+(`--adapt-every`) exchange experts between VRAM and the RAM copy without reading
+the file. The copy is pageable; `--resident-experts` (CUDA, chosen by setup)
+page-locks it and uses 4 GiB of headroom. Allocation needs sufficient available RAM;
 on Linux this option requires readable standard cgroup-v2 mounts. For each
 finite cgroup ancestor, the guard credits only `inactive_file` after subtracting
 `file_dirty` and `file_writeback`, capped by current usage; it remains bounded by

docs/DETAILS.md

@@ -12,7 +12,7 @@ New here? Start with the [README](../README.md) - it has everything you need to
 
 ## Speed (measured)
 
-RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1.22 (prompts) / 0.1.14 (output) with the settings setup writes
+RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1.26 with the settings setup writes
 (`--prefill auto`, 8-bit KV above 4K, KV streaming from 64K). One code-agent prompt per length, 256 generated tokens,
 MTP speculative decoding on. "262K" is the model's full context window (a 259,943-token prompt). The IQ2_XS row was
 measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
@@ -21,24 +21,27 @@ measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| **Q2_0** | 519 | 1,226 | 1,844 | 1,836 | 1,682 | 1,304 |
-| **IQ2_XS** | 524 | 1,196 | 1,799 | 1,611 | 1,495 | 1,181* |
-| **IQ3_XXS** | 472 | 974 | 1,555 | 1,449 | 1,386 | - |
-| **IQ3_S** | 419 | 893 | 1,499 | 1,285 | 1,245 | - |
-| **Coder** | 660 | 1,522 | 1,871 | 1,938 | 1,779 | 1,034** |
+| **Q2_0** | 536 | 1,299 | 2,171 | 2,126 | 2,107 | 1,304† |
+| **IQ2_XS** | 534 | 1,256 | 2,092 | 1,754 | 1,752 | 1,181*† |
+| **IQ3_XXS** | 482 | 1,007 | 1,745 | 1,609 | 1,602 | - |
+| **IQ3_S** | 427 | 913 | 1,624 | 1,640 | 1,443 | - |
+| **Coder** | 656 | 1,583 | 2,177 | 2,236 | 2,208 | 1,034** |
 
-Engine 0.1.22 (the prompt path of 0.1.23 is the same); `bench/results/2026-09-29-speed-0122`. \* measured with
-images on (the image encoder's VRAM reserve leaves fewer experts cached). \*\* not measured again: 0.1.14.
+Engine 0.1.26; `bench/results/2026-09-29-speed-0126`. At 32K-128K that is 8-28% faster than 0.1.22. † not measured
+again: 0.1.22. \* measured with images on (the image encoder's VRAM reserve leaves fewer experts cached). \*\* not
+measured again: 0.1.14.
 
 ### Output (tokens/s)
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| **Q2_0** | 84.3 | 90.3 | 73.6 | 69.0 | 67.2 | 60.3 |
-| **IQ2_XS** | 74.4 | 73.8 | 71.5 | 64.3 | 59.8 | 52.8 |
-| **IQ3_XXS** | 60.3 | 62.1 | 51.4 | 50.0 | 45.8 | - |
-| **IQ3_S** | 51.5 | 51.6 | 48.2 | 48.8 | 40.5 | - |
-| **Coder** | 53.3 | 50.6 | 53.3 | 50.8 | 44.0 | 42.8 |
+| **Q2_0** | 87.3 | 93.0 | 81.8 | 76.2 | 73.7 | 60.3† |
+| **IQ2_XS** | 79.6 | 78.6 | 76.3 | 63.7 | 62.7 | 52.8† |
+| **IQ3_XXS** | 61.9 | 61.6 | 58.5 | 57.2 | 49.0 | - |
+| **IQ3_S** | 52.4 | 53.3 | 48.3 | 46.3 | 45.5 | - |
+| **Coder** | 58.9 | 55.1 | 54.9 | 53.2 | 43.0 | 42.8† |
+
+Engine 0.1.26, the same runs. † not measured again: 0.1.14.
 
 Output speed depends on the text as well: speculative decoding runs faster when more of the drafted tokens are
 accepted, so a different answer to the same prompt moves it by several percent. Run back to back on the 4K prompt,
@@ -67,6 +70,48 @@ VRAM. RTX 3090, the Coder at 198K context: 99 instead of 85 tokens/s output, the
 slower. It does not stream its KV cache (KV streaming is on by default from 64K), so it pays off mostly on large
 cards at long contexts.
 
+**Reproducible greedy output (0.1.30, opt-in, `STRATA_IQ_MT_MIN=1`):** with the IQ models, the CPU computes an
+expert for one token with ggml's dot product and for several tokens with Strata's multi-token kernels, which round
+slightly differently. How many tokens share an expert depends on the drafts in a verify window, so the same prompt
+at temperature 0 can end in a different (equally good) answer when the drafting, the cache state or a resumed
+conversation differ (issue #152). `STRATA_IQ_MT_MIN=1` (in the config's `env`) uses the multi-token kernels for
+every group: the answer then no longer depends on the drafting. Measured on a Ryzen 7600 (AVX-512): IQ3_S decode
+-1..-3%, the other models the same; the default stays the fastest rule.
+
+**The draft layer's tokens (0.1.27, `--draft-vocab`):** the MTP draft layer can only propose tokens from a subset
+of the vocabulary (`mtp/rt/draft_vocab.bin`). Since 0.1.27 the subset includes every Chinese, Japanese and Korean
+token (106,299 ids), so answers in those languages are 15-38% faster (Q2_0, RTX 5070). Its head takes ~180 MiB of
+VRAM, which the expert cache leaves free for it (0.1.28). `START-HERE.bat --setup --draft-vocab en` keeps the
+English/code subset from before (40,525 ids, ~110 MiB less VRAM, English answers 1-2% faster; CJK answers get
+almost no drafts). `tools/draft_vocab.py` builds and inspects subsets.
+
+**Low-RAM mode (engine 0.1.26, chosen by setup):** normally all of a model's experts are copied into RAM (23-50 GB,
+pinned) and the GPU holds a copy of the most-used ones. On a PC whose RAM cannot hold them beside the system (the
+experts plus ~10 GB), setup instead maps them from one file in the model's folder (`--mmap-experts`, the pack's
+`experts.bin`, +23-50 GB of disk). The OS file cache holds what the GPU does not, and it can give that memory back.
+On the Coder the engine's committed memory drops from 36 to ~13 GB, with the same answers. With a big GPU (an RTX
+5090 holds all of the Coder's experts, most of Q2_0's) it runs at nearly the usual speed. With a small one, most
+experts come from the SSD and it is much slower (setup says so). `START-HERE.bat --setup --low-ram on|off` overrides
+the choice.
+
+**Low-RAM mode, resident (engine 0.1.30):** when the experts the GPU does not hold fit the RAM (with the same ~10 GB
+beside them), setup picks the resident variant instead (`--resident-experts`): at start the engine copies exactly those
+experts from `experts.bin` into RAM (page-locked when the driver allows, else locked in RAM), so while it answers
+nothing is read from the SSD, however little RAM the OS leaves for its file cache. Examples with setup's context: a
+32 GB PC with a 24 GB GPU runs Q2_0, IQ2_XS and the Coder this way (~16-18 GB of experts in RAM, the GPU holds the
+other ~18 GB), a 32 GB PC with a 12-16 GB GPU the Coder; IQ3_XXS on a 32 GB PC stays mapped. The details:
+- The prompt path borrows room in the GPU's expert cache for its buffers and puts those experts back after the prompt;
+  as far as the RAM allows, their experts are kept in RAM too (so a prompt reads nothing from the SSD either).
+- The cache still follows the conversation (`--adapt-every`): a swap copies the evicted expert back from VRAM into the
+  RAM place of the one that replaces it, so the RAM copy keeps holding exactly what the GPU does not.
+- The answers are the plain mapped mode's for the same expert placement: the bytes are the file's. With a page-locked
+  copy the GPU also takes its usual share of the misses over PCIe (`--pcie-frac`), as with enough RAM; `--pcie-frac 0`
+  (or `STRATA_RESIDENT_PIN=0`) gives the mapped mode's exact tokens.
+- The engine leaves 4 GB of the RAM it finds free (`STRATA_RESIDENT_HEADROOM_GIB`); when even the experts the GPU does
+  not hold do not fit, it says so and runs the plain mapped mode. The server log shows, per request, how many expert
+  reads went to the file (`resident RAM: ... blob reads from the file`: 0 in steady use).
+- `--low-ram resident|mmap` forces one variant (also on a PC with enough RAM, e.g. to try it).
+
 Time to first token is prompt length / prompt speed: with Q2_0 about 4 s at 4K, 25 s at 32K, under 2 minutes at 128K
 and 4.5 minutes at 262K (engine 0.1.13 made long prompts about twice as fast, below).
 
@@ -157,7 +202,7 @@ You need **only an NVIDIA driver** (version 580 or newer; update it with the NVI
 
 | | |
 | --- | --- |
-| GPU | NVIDIA **RTX 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070; RTX 30/40 are untested. |
+| GPU | NVIDIA **RTX 20, 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070 and an RTX 3090; RTX 20 (Turing, since 0.1.27) was tested by a contributor on an RTX 2070. |
 | RAM | **64 GB** recommended (see the table above). |
 | CPU | x86-64 with AVX2 (any Intel/AMD desktop CPU from the last ~8 years). AVX-512 (Ryzen 7000/9000) is a bit faster. |
 | Disk | ~70-80 GB free for the model, ~6 GB for the MTP layer (+1 GB with images). **Q2_0 on an AVX-512 CPU** also writes a one-time ~40 GB copy of its experts for the fast CPU kernel. An NVMe SSD is strongly recommended. |
@@ -169,7 +214,7 @@ elsewhere) finds them and sets itself up the same way. The place is remembered p
 `~/.config/strata/settings.json`); `--data-dir` chooses another. Installs from before 0.1.16 are moved there by the next
 start (a rename on the same drive; files on another drive are used where they are).
 Python 3.12 if you have none (for your user account, no admin), a private Python environment, NVIDIA's CUDA libraries
-(from pip, ~0.4 GB), the ready-made Strata engine for RTX 30/40/50, the model and the MTP draft layer. If no
+(from pip, ~0.4 GB), the ready-made Strata engine for RTX 20/30/40/50, the model and the MTP draft layer. If no
 ready-made engine fits your PC, it offers to install the build tools (Visual Studio Build Tools + CUDA Toolkit on
 Windows, `build-essential` + CUDA on Ubuntu) and compiles the engine for your GPU (asks first; 20-40 minutes once).
 
@@ -277,6 +322,27 @@ Terminal chat: `.venv/bin/python chat.py`.
 
 ---
 
+## Sharing the GPU with other programs (optional)
+
+By default the model stays loaded until you close Strata. On a PC that also games, renders or runs another model
+server, three server options (all off by default; also as keys in `strata-<model>.json`) give the VRAM back:
+
+| Option | Config key | What it does |
+| --- | --- | --- |
+| `--idle-unload 600` | `"idle_unload_s": 600` | unload the model after 600 s without requests; the next request loads it again |
+| `--min-free-vram-mib 11000` | `"min_free_vram_mib": 11000` | load an unloaded model only when that much VRAM is free (it waits up to 15 s for memory being given back), else answer **503** "the GPU is in use by another program" instead of starting into what a game left (with several GPUs it checks the first one) |
+| `--before-load "cmd"` | `"before_load": "cmd"` or `["cmd", "arg"]` | a command run before the model is loaded again, e.g. one that unloads another server's model |
+
+`POST /unload` unloads it now (`409` while a request is running) and `POST /load` loads it ahead of a request;
+`/health` says `"loaded"`, `/v1/models` lists it as `unloaded` (like llama.cpp's router), `/props` sets
+`is_sleeping` and the Monitor shows the state. Unloading ends the engine process - and the image encoder, when images
+are on; it is started again first, as at a start - so their VRAM and RAM go straight back. The model files stay in
+the OS file cache, so loading again takes seconds while that RAM is not needed elsewhere. Measured on an RTX 5060 Ti
+16 GB with Q2_0 in the low-RAM mode: unloading takes ~0.3 s, and a request to an unloaded model answered after
+4.6 s (text) or 14.7 s (a picture, image encoder on the CPU).
+
+---
+
 ## Using it
 
 The server listens on `http://127.0.0.1:8080` (change with `--port` in setup, or edit the run script).
@@ -352,9 +418,48 @@ prompt when that is 2,048 tokens or more (engine 0.1.20; PR #62 + #65), so that
 system prompts and tool lists. Engine options: `--prompt-cache N` (0 = off), `--prompt-cache-every N`,
 `--prompt-cache-root N` (0 = no system-prompt checkpoint), `--turn-token ID`.
 
-**Current limits (v1):** one request at a time, and one conversation's history in the KV cache at a time (switching
-between two chats re-reads the part where they diverge; the shared prefix, such as the system prompt, is reused); images
-only when set up with them (below); no video. **Temperature / top_p / top_k / min_p /
+**Multiple conversations (opt-in).** Add `--conversation-cache-mib 8192
+--conversation-cache-slots 4` to the engine arguments to park up to four conversations
+in a bounded 8 GiB host-RAM cache. This preserves controller/worker histories when
+their requests alternate; it does not execute requests concurrently. No client session
+ID is required: only exact token/image prefixes with matching steering mode are reused.
+The default budget is 0 (disabled); `--prompt-cache 0` also disables parking.
+The initial shared-core integration supports a single session GPU: combining
+enabled parking with `--layer-split` is rejected before model loading. Ordinary
+upstream layer-split checkpoints remain available with parking disabled. FP16,
+INT8, Q4_0 and identity-layout K8V4 snapshots are supported; the K8V4 draft ring
+remains INT8, as in upstream. Windows/HIP and multi-GPU runtime coverage must be
+reported separately from Linux/CUDA evidence.
+
+Snapshots contain running state, checkpoints, used K/V pages, and draft-layer K/V.
+They add host RAM, not another model or VRAM allocation. The byte budget also counts
+an incoming snapshot during a switch. After a restore, unchanged K/V pages can be
+retained for the next parking operation; growth appends storage without copying
+the existing pages. Rewinds refresh the affected pages, and running state and
+checkpoints are captured again. Retained active K/V counts against the same byte
+budget and is discarded before evicting parked entries under memory pressure.
+If reserving space for growth would evict another conversation, parking uses a
+full capture instead.
+Oldest parked entries are evicted first.
+Oversized snapshots or host allocation failures fall back to ordinary prompt processing.
+`--conversation-cache-min-free-mib N` (default 2560) additionally requires that
+physical-RAM headroom remain available: the engine checks before allocation and
+again after capture. Unknown telemetry or insufficient RAM skips parking. Windows
+uses `GlobalMemoryStatusEx`, Linux uses `MemAvailable`; these are host-level samples,
+not a reservation or enforcement of container/job memory limits. An 8 GiB budget
+is a cap, not a recommendation for every machine.
+
+The shared snapshot core validates all layers and checkpoints before applying any
+state. Invalid entries are discarded; transfer/synchronization failure is fatal
+rather than permission to continue with partial state. Indexer spare keys and the
+moving spare row are preserved, including checkpoint rewinds.
+The engine log reports parking, restoration, bytes, evictions, individual snapshot
+sizes and K/V bytes reused during capture. `STRATA_SNAPSHOT_FULL_CAPTURE=1` disables
+retention for diagnostic comparisons. Snapshots are not
+persisted across restarts.
+
+**Current limits (v1):** one request at a time, and one conversation cached at a time (switching between two chats
+re-reads the other one unless the opt-in cache above is enabled); images only when set up with them (below); no video. **Temperature / top_p / top_k / min_p /
 seed** are honored per request (OpenAI and Anthropic fields); with the default adaptive expert tier a sampled result
 is not reproducible run to run - for seed-reproducible output add `--adapt-every 100000` (static residency) to the
 engine arguments. The run config's optional `sampling` block sets the defaults for requests that leave the fields out
@@ -409,6 +514,43 @@ it needs, prefer read-only tools, and don't add servers you don't trust. The too
 page itself (a request with another site's Origin or without a JSON content type is refused); if Strata is reachable
 from other devices, set an API key.
 
+**Context extension past 262K (rope scaling, EXPERIMENTAL, off unless you pick it).** The model was trained on
+262,144 positions (rotary base 1e7). Rope scaling rescales the rotation angles so that longer contexts stay usable,
+with llama.cpp's types and flag names. `linear` is Position Interpolation: every angle is shrunk by the factor.
+`yarn` keeps the high-frequency angles, interpolates the low-frequency ones, and adds the magnitude correction
+that keeps the attention temperature where training put it. **Without the flags nothing changes:** an unscaled
+run computes exactly what it did before the feature existed, bit for bit. Scaled contexts need proportionally
+more VRAM/RAM for the KV cache and the rope tables (~13 KB and ~0.26 KB per token).
+
+What was measured (contributors' runs, RTX 5080 + IQ3_S, native path, in PR #84): per-position perplexity on the
+same tokens, with only the scaling flag changed. At 293K tokens (1.12x the trained length), `yarn` with factor 2
+lowered the NLL by 0.18 nats against both `none` and `linear` 2 (2.04 vs 2.22 / 2.22). That is 3-4x the path noise
+measured at the same length. `linear` 2 was indistinguishable from `none`. At 2.7K and 32K no arm separated from the
+noise. Needle tests do not tell the arms apart: the unscaled model also finds a needle at 413K. Long real-document
+Q&A worked with `yarn` 2 at 421K and `yarn` 4 at 714K (8/8 each), and a 1M-token `yarn` 4 run read end to end.
+Taken together, use **yarn**. It is still experimental: the numbers come from one machine and one quant.
+
+- setup: `START-HERE.bat --setup --context 393216` asks nothing extra - it picks the method (yarn; one
+  question when run interactively) and derives the factor from the final context for you (final context /
+  262,144, at least 1: 1.5 at 393K, 2 at 512K, 1 inside the trained range; `--rope-scaling`/`--rope-scale`
+  override; an explicit `--rope-scale` is kept as given even when it is too small for the context actually
+  served, so check it if you set one). An explicit `--rope-scaling none` for a context past
+  262,144 is refused: the setup will not configure a run with the stock angles past the trained range. If
+  the RAM check reduces a chosen 384K/512K back inside the trained range, an omitted method adds no
+  scaling, and an explicitly chosen one stays at factor 1 - the trained angles, no expansion (not a
+  switch for rope as a whole: explicitly supplied rope settings keep their behavior).
+- engine: `--rope-scaling none|linear|yarn`, `--rope-scale F`, and the raw ggml knobs `--rope-freq-base`,
+  `--rope-freq-scale`, `--yarn-orig-ctx` (default 262,144), `--yarn-ext-factor`, `--yarn-attn-factor`,
+  `--yarn-beta-fast` (32), `--yarn-beta-slow` (1). The model file's `rope.scaling.*` keys, when a
+  fine-tune ships them, are the defaults the flags override.
+
+The scaling is fixed for the whole run - the engine stores keys in its cache after rotating them, so one
+cache must never mix two scalings, and there is no per-request form. Within the trained 262,144 a scaled
+run is a slightly different model: the rescaled angles, and `yarn`'s magnitude correction, apply at every
+position, not only past the trained end. That is why the setup turns scaling on only for a context past
+262,144. Pictures read the same scaled table (their (t, h, w) positions feed it). That should work, but it is
+unmeasured: all the runs above are text.
+
 ---
 
 ## Images (vision)
@@ -538,6 +680,7 @@ the document, +0.4% on the chat. Details: `bench/results/2026-09-27-esp/`.
 | `the engine stopped unexpectedly (exit code ...)` | The engine process ended mid-answer - usually out of RAM (Linux ends the biggest program: `sudo dmesg \| grep -i -E 'killed process\|out of memory'`). The next request starts it again by itself. If it repeats: close other programs or pick a smaller size. The server also warns at start when the model's experts leave less than ~6 GB of RAM for everything else. |
 | Slow output, disk light busy | Not enough free RAM: close other programs, or choose Q2_0 / IQ2_XS. |
 | `prompt ... exceeds the context` | The request is longer than the context you chose: run setup again with a bigger `--context`. |
+| `the setup refuses --rope-scaling none for a past-trained context` | A context past the trained 262,144 needs the rotary angles rescaled (experimental rope scaling), and the setup will not configure one with the stock angles there. Let it pick (`START-HERE.bat --setup --context 393216` adds yarn and a covering factor), or pass `--rope-scaling linear` or `yarn` yourself. |
 | Slower than the tables | The monitor plugged into the GPU and other GPU programs take VRAM from the expert cache; RAM running below its rated speed (enable EXPO/XMP in the BIOS) slows the CPU half. |
 | `this server was started without the vision encoder` | The model was set up for text only: run setup again with `--vision gpu`. |
 | A picture is refused or `cannot read the image` | The file is not a picture Pillow can open (JPEG, PNG, WebP, GIF, BMP, TIFF, AVIF work). |

docs/MULTI_GPU.md

@@ -18,7 +18,7 @@ Strata can use it:
 ```
   Your NVIDIA GPUs:
     GPU 0: NVIDIA GeForce RTX 5080, 16 GB VRAM - can be used
-    GPU 1: NVIDIA GeForce RTX 2080 Ti, 22 GB VRAM - not supported - older than the RTX 30 series (compute capability 7.5; Strata needs 8.0 or newer)
+    GPU 1: NVIDIA GeForce GTX 1080 Ti, 11 GB VRAM - not supported - older than the RTX 20 series (compute capability 6.1; Strata needs 7.5 or newer)
     GPU 2: NVIDIA GeForce RTX 3090, 24 GB VRAM - can be used
   ...
   1) GPU 0 (NVIDIA GeForce RTX 5080, 16 GB) + GPU 2 (NVIDIA GeForce RTX 3090, 24 GB) together   (recommended)
@@ -41,10 +41,11 @@ now on; the answer is kept.
 ```
 
 **Not supported** (setup says so and names the cards that can be used instead):
-- a card older than the RTX 30 series (compute capability below 8.0: RTX 20, GTX 16/10);
+- a card older than the RTX 20 series (compute capability below 7.5: GTX 10 and older);
 - a card with less than 8 GB of VRAM, together with others (each card holds a copy of the dense weights and its
   own prompt buffers);
-- AMD and Intel GPUs, and a mix of NVIDIA with them.
+- AMD and Intel GPUs, and a mix of NVIDIA with them. (Two AMD RDNA4 cards run the split from a hand-written
+  config; see [AMD_HIP.md](AMD_HIP.md#rdna4-gfx1201).)
 
 Or edit an existing config (`strata-*.json`), then restart:
 
@@ -96,8 +97,8 @@ into the card that owns the layer.
   capped to leave room for them.
 - On Windows only 8 GiB of the expert arena is pinned (more, mapped into two GPU contexts, leaves WDDM refusing
   allocations); the PCIe share covers those layers.
-- Every card needs compute capability 8.0 (RTX 30 or newer). A Turing card (RTX 20, sm_75) builds only with the
-  experimental `-DSTRATA_EXPERIMENTAL_SM75=ON`.
+- Every card needs compute capability 7.5 (RTX 20 or newer). The pre-sm_80 QSA scorer path is fp32 FMAs, so a
+  Turing card runs the same kernels instead of the tensor-core prompt attention.
 
 ## Measured
 

include/strata/artifact/gguf_reader.hpp

@@ -32,6 +32,9 @@
 
 #ifdef _WIN32
 #define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX   // windows.h's min/max macros would break std::min/std::max in every file that includes this one
+#endif
 #include <windows.h>
 #else
 #include <fcntl.h>

include/strata/core/conversation_buffer.hpp

@@ -0,0 +1,154 @@
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <initializer_list>
+#include <utility>
+#include <vector>
+
+namespace strata::core {
+
+// Snapshot-owned host storage. Appending never relocates existing payloads;
+// only the small segment directory can move. No sharing or copy-on-write is
+// needed because a parked image and its retained active buffers have one owner.
+class ConversationBuffer {
+public:
+    static constexpr size_t segment_bytes = 16 * 1024 * 1024;
+    ConversationBuffer() = default;
+    ConversationBuffer(std::initializer_list<uint8_t> values) {
+        resize(values.size());
+        visit(0, size(), [&](uint8_t* p, size_t n, size_t at) {
+            std::memcpy(p, values.begin() + at, n);
+            return true;
+        });
+    }
+    ConversationBuffer(const ConversationBuffer&) = default;
+    ConversationBuffer& operator=(const ConversationBuffer&) = default;
+    ConversationBuffer(ConversationBuffer&& other) noexcept
+        : segments_(std::move(other.segments_)), size_(std::exchange(other.size_, 0)) {}
+    ConversationBuffer& operator=(ConversationBuffer&& other) noexcept {
+        if (this == &other) return *this;
+        segments_ = std::move(other.segments_);
+        size_ = std::exchange(other.size_, 0);
+        return *this;
+    }
+
+    size_t size() const { return size_; }
+    bool empty() const { return size_ == 0; }
+    size_t bytes() const {
+        size_t n = segments_.capacity() * sizeof(Segment);
+        for (const auto& segment : segments_) n += segment.capacity();
+        return n;
+    }
+    // Includes the old directory while reserve allocates its replacement.
+    // Saturation makes an overflowing estimate fail admission before resize.
+    size_t allocation_peak(size_t n) const {
+        size_t total = bytes();
+        if (n <= size_) return total;
+        const size_t spare = segments_.empty() ? 0 : segments_.back().capacity() - segments_.back().size();
+        const size_t extra = n - size_ - std::min(n - size_, spare);
+        const size_t full = extra / segment_bytes, tail = extra % segment_bytes;
+        if (!add(total, full * segment_bytes)) return SIZE_MAX;
+        if (tail && !add(total, segment_capacity(tail, full ? segment_bytes :
+                segments_.empty() ? 0 : segments_.back().capacity(), size_ != 0))) return SIZE_MAX;
+        size_t count = segments_.size();
+        if (!add(count, full + (tail != 0))) return SIZE_MAX;
+        if (count > segments_.capacity() &&
+            (count > SIZE_MAX / sizeof(Segment) || !add(total, count * sizeof(Segment)))) return SIZE_MAX;
+        return total;
+    }
+    void resize(size_t n, uint8_t value = 0) {
+        if (n > size_) {
+            const bool growing = size_ != 0;
+            const size_t spare = segments_.empty() ? 0 : segments_.back().capacity() - segments_.back().size();
+            const size_t extend = std::min(n - size_, spare);
+            const size_t extra = n - size_ - extend;
+            segments_.reserve(segments_.size() + extra / segment_bytes + (extra % segment_bytes != 0));
+            if (extend) {
+                segments_.back().resize(segments_.back().size() + extend, value);
+                size_ += extend;
+            }
+            while (size_ < n) {
+                const size_t count = std::min(segment_bytes, n - size_);
+                const size_t previous = segments_.empty() ? 0 : segments_.back().capacity();
+                Segment segment;
+                segment.reserve(segment_capacity(count, previous, growing));
+                segment.resize(count, value);
+                segments_.push_back(std::move(segment));
+                size_ += count;
+            }
+        } else {
+            while (!segments_.empty() && size_ - segments_.back().size() >= n) {
+                size_ -= segments_.back().size();
+                segments_.pop_back();
+            }
+            if (size_ > n) segments_.back().resize(segments_.back().size() - (size_ - n));
+            size_ = n;
+        }
+    }
+    void pop_back() { resize(size_ - 1); }
+
+    // fn(pointer, count, absolute_offset); false aborts a transfer immediately.
+    template<class Fn> bool visit(size_t offset, size_t count, Fn fn) {
+        return visit_impl(*this, offset, count, fn);
+    }
+    template<class Fn> bool visit(size_t offset, size_t count, Fn fn) const {
+        return visit_impl(*this, offset, count, fn);
+    }
+    bool read(void* target, size_t offset, size_t count) const {
+        return visit(offset, count, [&](const uint8_t* p, size_t n, size_t at) {
+            std::memcpy(static_cast<uint8_t*>(target) + at - offset, p, n);
+            return true;
+        });
+    }
+    bool operator==(const ConversationBuffer& other) const {
+        if (size_ != other.size_) return false;
+        size_t a = 0, b = 0, x = 0, y = 0;
+        while (a < segments_.size() && b < other.segments_.size()) {
+            const auto& left = segments_[a];
+            const auto& right = other.segments_[b];
+            const size_t n = std::min(left.size() - x, right.size() - y);
+            if (std::memcmp(left.data() + x, right.data() + y, n)) return false;
+            x += n; y += n;
+            if (x == left.size()) { ++a; x = 0; }
+            if (y == right.size()) { ++b; y = 0; }
+        }
+        return true;
+    }
+
+private:
+    using Segment = std::vector<uint8_t>;
+    // A fresh capture allocates exactly its payload. Later appends reserve
+    // geometrically growing segments, bounded by 16 MiB. Small chat turns then
+    // extend the final allocation instead of adding one transfer per turn.
+    static size_t segment_capacity(size_t count, size_t previous, bool growing) {
+        if (!growing) return count;
+        return std::max(count, std::min(segment_bytes, std::max<size_t>(65536, std::min(previous, segment_bytes/2)*2)));
+    }
+    static bool add(size_t& n, size_t extra) {
+        if (extra > SIZE_MAX - n) return false;
+        n += extra;
+        return true;
+    }
+    template<class Self, class Fn> static bool visit_impl(Self& self, size_t offset, size_t count, Fn fn) {
+        if (offset > self.size_ || count > self.size_ - offset) return false;
+        size_t begin = 0;
+        for (auto& segment : self.segments_) {
+            if (!count) break;
+            const size_t end = begin + segment.size();
+            if (offset < end) {
+                const size_t n = std::min(count, end - offset);
+                if (!fn(segment.data() + offset - begin, n, offset)) return false;
+                offset += n; count -= n;
+            }
+            begin = end;
+        }
+        return count == 0;
+    }
+    std::vector<Segment> segments_;
+    size_t size_ = 0;
+};
+
+} // namespace strata::core

include/strata/core/conversation_cache.hpp

@@ -0,0 +1,181 @@
+// CPU-only ownership and matching policy for --serve's parked conversations.
+// Token equality, image identity, and steering mode are all required for reuse.
+#pragma once
+
+#include "strata/core/conversation_buffer.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <deque>
+#include <utility>
+#include <vector>
+
+namespace strata::core {
+
+struct ConversationImageKey {
+    int64_t start = 0;
+    uint64_t hash = 0;
+    bool operator==(const ConversationImageKey&) const = default;
+};
+
+struct ConversationCheckpoint {
+    std::vector<int32_t> ids;
+    std::vector<ConversationImageKey> imgs;
+    std::vector<uint8_t> gdn, ple, tails, dead, block_pos;
+    uint64_t used = 0; // upstream root-pinned/LRU checkpoint retention
+    // Ordinary layer-split checkpoints retain each device's running state.
+    // Whole-session parking is currently single-GPU and rejects these parts.
+    std::vector<ConversationCheckpoint> stage_parts;
+
+    size_t bytes() const {
+        size_t n = ids.capacity() * sizeof(int32_t) + imgs.capacity() * sizeof(ConversationImageKey) +
+               gdn.capacity() + ple.capacity() + tails.capacity() + dead.capacity() + block_pos.capacity() +
+               stage_parts.capacity() * sizeof(ConversationCheckpoint);
+        for (const auto& part : stage_parts) n += part.bytes();
+        return n;
+    }
+};
+
+// Identity-layout K/V pages and completed indexer rows. For streamed layers the
+// source is the authoritative host pool, NOT the replaceable VRAM slots.
+struct ConversationKv {
+    int format = 0;
+    int64_t cells = 0, heads = 0, head_dim = 0, page_size = 0, pooled_rows = 0, idx_dim = 0;
+    ConversationBuffer k, v, k_scale, v_scale, pooled;
+    size_t bytes() const {
+        return k.bytes() + v.bytes() + k_scale.bytes() + v_scale.bytes() + pooled.bytes();
+    }
+};
+
+struct ConversationKvReuse {
+    std::vector<ConversationKv> kv;
+    // Original image extent for validation, and the earliest subsequent rewrite.
+    int64_t captured_tokens = 0, unchanged_tokens = 0;
+    size_t bytes() const {
+        size_t n = kv.capacity() * sizeof(ConversationKv);
+        for (const auto& layer : kv) n += layer.bytes();
+        return n;
+    }
+};
+
+struct SavedConversation {
+    // Runtime compatibility only; NOT a model/weights identity or disk schema.
+    std::array<int64_t, 18> geometry{};
+    // The session's layer carve the image was captured from ([0, n_layers) on one GPU); restore requires the same.
+    int64_t layer_lo = 0, layer_hi = 0;
+    ConversationCheckpoint live;
+    std::vector<ConversationCheckpoint> checkpoints;
+    std::vector<ConversationKv> kv; // main layers followed by the draft layer
+    bool cvec = true;
+
+    size_t bytes() const {
+        size_t n = live.bytes() + checkpoints.capacity() * sizeof(ConversationCheckpoint) +
+                   kv.capacity() * sizeof(ConversationKv);
+        for (const auto& c : checkpoints) n += c.bytes();
+        for (const auto& k : kv) n += k.bytes();
+        return n;
+    }
+};
+
+template<class Token>
+int64_t conversation_prefix(const ConversationCheckpoint& c, const std::vector<Token>& prompt,
+                            const std::vector<ConversationImageKey>& images) {
+    const size_t n = c.ids.size();
+    // The last prompt token always starts the next verify window.
+    if (n == 0 || n >= prompt.size() || !std::equal(c.ids.begin(), c.ids.end(), prompt.begin())) return 0;
+    size_t j = 0;
+    for (const auto& image : images) {
+        if (image.start >= (int64_t) n) continue;
+        if (j == c.imgs.size() || !(c.imgs[j++] == image)) return 0;
+    }
+    if (j != c.imgs.size()) return 0;
+    return (int64_t) n;
+}
+
+class ConversationCache {
+public:
+    struct Match {
+        size_t index = 0;
+        int64_t tokens = 0;
+        bool live = false;
+    };
+
+    ConversationCache(size_t budget, size_t slots) : budget_(budget), slots_(slots) {}
+    bool enabled() const { return budget_ != 0 && slots_ != 0; }
+    size_t bytes() const { return bytes_ + reuse_.bytes(); }
+    size_t size() const { return entries_.size(); }
+    size_t evictions() const { return evictions_; }
+
+    // Retain only the restored K/V buffers, not duplicate running checkpoints.
+    // This optimization never evicts a parked conversation to make itself fit.
+    void retain(std::vector<ConversationKv>&& kv, int64_t tokens) {
+        reuse_ = {};
+        ConversationKvReuse candidate{std::move(kv), tokens, tokens};
+        if (enabled() && candidate.bytes() <= budget_ - bytes_) reuse_ = std::move(candidate);
+    }
+    void limit_reuse(int64_t first_dirty) {
+        reuse_.unchanged_tokens = std::min(reuse_.unchanged_tokens, first_dirty);
+        if (reuse_.unchanged_tokens <= 0) reuse_ = {};
+    }
+    ConversationKvReuse take_reuse() { return std::exchange(reuse_, {}); }
+    size_t retained_bytes() const { return reuse_.bytes(); }
+    bool can_fit(size_t incoming, size_t held = 0) const {
+        return enabled() && held <= budget_ && incoming <= budget_ - held &&
+               entries_.size() < slots_ && bytes() <= budget_ - held - incoming;
+    }
+
+    template<class Token>
+    Match best(const std::vector<Token>& prompt, const std::vector<ConversationImageKey>& images, bool cvec) const {
+        Match best;
+        // Ties prefer the most recently parked branch. The caller prefers its
+        // already-active state when that offers the same prefix length.
+        for (size_t i = entries_.size(); i-- > 0;) {
+            const auto& e = entries_[i];
+            if (e.cvec != cvec) continue;
+            auto consider = [&](const ConversationCheckpoint& c, bool live) {
+                const int64_t n = conversation_prefix(c, prompt, images);
+                if (n > best.tokens) best = {i, n, live};
+            };
+            consider(e.live, true);
+            for (const auto& c : e.checkpoints) consider(c, false);
+        }
+        return best;
+    }
+
+    SavedConversation take(size_t index) {
+        SavedConversation out = std::move(entries_.at(index));
+        bytes_ -= out.bytes();
+        entries_.erase(entries_.begin() + (std::ptrdiff_t) index);
+        return out;
+    }
+
+    // Reserve before allocating a snapshot. held is an incoming image removed
+    // with take() but still alive during the exchange; count it against RAM too.
+    bool make_room(size_t incoming, size_t held = 0) {
+        if (!enabled() || held > budget_ || incoming > budget_ - held) return false;
+        if (bytes() > budget_ - held - incoming) reuse_ = {};
+        while (!entries_.empty() && (entries_.size() >= slots_ || bytes_ > budget_ - held - incoming)) {
+            bytes_ -= entries_.front().bytes();
+            entries_.pop_front();
+            ++evictions_;
+        }
+        return true;
+    }
+
+    bool put(SavedConversation&& image, size_t held = 0) {
+        const size_t n = image.bytes();
+        if (!make_room(n, held)) return false;
+        entries_.push_back(std::move(image));
+        bytes_ += n;
+        return true;
+    }
+
+private:
+    size_t budget_ = 0, slots_ = 0, bytes_ = 0, evictions_ = 0;
+    std::deque<SavedConversation> entries_; // least recently active first
+    ConversationKvReuse reuse_;
+};
+
+} // namespace strata::core

include/strata/core/conversation_memory.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <cstdint>
+#include <istream>
+#include <optional>
+
+namespace strata::core {
+
+// Host physical memory, not swap/commit or a container/job memory reservation.
+// Unknown telemetry is deliberately distinct from a measured zero.
+std::optional<uint64_t> conversation_available_memory();
+std::optional<uint64_t> conversation_mem_available(std::istream& meminfo);
+
+inline bool conversation_memory_admit(std::optional<uint64_t> available,
+                                      uint64_t allocation, uint64_t floor) {
+    return available && *available >= floor && allocation <= *available - floor;
+}
+
+} // namespace strata::core

include/strata/core/conversation_snapshot.hpp

@@ -0,0 +1,79 @@
+#pragma once
+
+#include "strata/core/conversation_cache.hpp"
+#include "strata/core/layer.hpp"
+#include "strata/core/session.hpp"
+
+#include <string>
+
+namespace strata::core {
+
+// Caller synchronizes the device before saving, and after restoring all layers.
+// include_index is false for the draft layer (its attention has no indexer).
+size_t conversation_kv_bytes(const QsaState& state, const ModelGeometry& g, int64_t upto, bool include_index);
+// A nonzero unchanged_tokens is valid only for storage retained from an image
+// actually restored into this session, bounded by every subsequent rewrite.
+// Equal token IDs alone do not establish that its K/V bytes are unchanged.
+bool conversation_kv_save(ConversationKv& image, const QsaState& state, const ModelGeometry& g,
+                          int64_t upto, bool include_index, std::string& error,
+                          int64_t unchanged_tokens = 0, size_t* reused_bytes = nullptr);
+bool conversation_kv_capture_bytes(const ConversationKv& image, const QsaState& state, const ModelGeometry& g,
+                                   int64_t upto, bool include_index, size_t& bytes, std::string& error);
+// No CUDA calls or destination writes. Used for whole-session prevalidation.
+bool conversation_kv_validate(const ConversationKv& image, const QsaState& state, const ModelGeometry& g,
+                              int64_t upto, bool include_index, std::string& error);
+bool conversation_kv_restore(const ConversationKv& image, const QsaState& state, const ModelGeometry& g,
+                             int64_t upto, bool include_index, std::string& error);
+// Diagnostic read-back after a synchronized restore. Uses 64 KiB of stack
+// workspace, compares authoritative bytes and resident draft-ring pages, and
+// fingerprints the authoritative payload only. Never changes model state.
+bool conversation_kv_verify(const ConversationKv& image, const QsaState& state, const ModelGeometry& g,
+                            int64_t upto, bool include_index, uint64_t& fingerprint, std::string& error);
+
+struct ConversationStateSizes {
+    size_t gdn = 0, ple = 0, tail = 0, dead = 0, block_pos = 0;
+};
+/// Whole-model sizes: `gdn` covers every GDN layer, the indexer sizes are per QSA layer.
+bool conversation_state_sizes(const ModelGeometry& g, ConversationStateSizes& sizes, std::string& error);
+/// The same for one session's layer carve (#216): `gdn` covers its `gdn_alloc` rows; the per-QSA-layer sizes
+/// apply to each owned state [qsa_ord0, qsa_ord0 + qsa_alloc).  Rejects an inconsistent carve.  Checkpoints,
+/// snapshots and their validation all use this: a session saves and restores only the state it owns.
+bool conversation_session_sizes(const ModelGeometry& g, const SessionState& session, ConversationStateSizes& sizes,
+                                std::string& error);
+bool conversation_checkpoint_validate(const ConversationCheckpoint& checkpoint, const SessionState& session,
+                                      const ModelGeometry& g, std::string& error);
+bool conversation_checkpoint_save(ConversationCheckpoint& checkpoint, const SessionState& session,
+                                  const ModelGeometry& g, std::string& error);
+bool conversation_checkpoint_restore(const ConversationCheckpoint& checkpoint, SessionState& session,
+                                     const ModelGeometry& g, std::string& error);
+
+struct ConversationView {
+    const std::vector<int32_t>& ids;
+    const std::vector<ConversationImageKey>& images;
+    const std::vector<ConversationCheckpoint>& checkpoints;
+    bool cvec;
+};
+bool conversation_snapshot_bytes(const ConversationView& view, const SessionState& session,
+                                 const ModelGeometry& g, const QsaState& draft, size_t& bytes, std::string& error);
+bool conversation_snapshot_capture_bytes(const ConversationKvReuse& reuse, const ConversationView& view,
+                                         const SessionState& session, const ModelGeometry& g,
+                                         const QsaState& draft, size_t& bytes, std::string& error);
+// The capture estimate includes retained capacity and transient segment directories;
+// only estimate - reuse.bytes() requires additional physical RAM. Capture consumes
+// the uniquely owned reusable buffers, including on failure.
+// Caller admits the estimate before invoking capture. Allocation failures propagate
+// to the RAM policy; the active session is never modified by capture.
+bool conversation_snapshot_save(SavedConversation& image, const ConversationView& view,
+                                const SessionState& session, const ModelGeometry& g,
+                                const QsaState& draft, std::string& error,
+                                ConversationKvReuse reuse = {}, size_t* reused_bytes = nullptr);
+bool conversation_snapshot_validate(const SavedConversation& image, const SessionState& session,
+                                    const ModelGeometry& g, const QsaState& draft, std::string& error);
+enum class ConversationRestore { restored, invalid, transfer_failed };
+// Invalid images are rejected before any CUDA call/write. Transfer failure may
+// leave partial state: caller MUST NOT continue inference from that session.
+ConversationRestore conversation_snapshot_restore(const SavedConversation& image, SessionState& session,
+                                                   const ModelGeometry& g, const QsaState& draft,
+                                                   std::string& error);
+
+} // namespace strata::core

include/strata/core/coupled_draft.hpp

@@ -0,0 +1,82 @@
+// include/strata/core/coupled_draft.hpp - COUPLED DRAFT SAMPLING (STRATA_SPEC_COUPLED=1, off by default).
+//
+// Under sampling the verify window samples row t of a window at position pos0 as Philox(seed, pos0 + t) over the
+// sampler chain (penalties -> top_k -> top_p -> min_p -> temperature -> inverse-CDF pick; verify.cpp run(),
+// sampler.cu), and a draft is kept only when it EQUALS that row's pick.  The output is therefore a function of the
+// seed alone, whatever the drafts are: drafts only decide how many tokens a window yields.
+//
+// The MTP draft layer used to propose its ARGMAX.  In coupled mode it SAMPLES instead, with the target's own chain
+// (the request's top_k / top_p / min_p / temperature and penalties over the same history) and the SAME uniform the
+// target will draw for the row that verifies the draft.  When the draft's distribution is close to the target's,
+// the two inverse-CDF picks from one uniform agree far more often than an argmax agrees with a sample; the target's
+// pick - and so the text - does not change, because verification is still exact-match against the target's sample.
+//
+// THE COUNTER ARITHMETIC (all positions are sequence indices; a window's row t holds the token at pos0 + t and its
+// pick is the token at pos0 + t + 1, drawn with counter pos0 + t):
+//   * MTP cell c pairs the main model's residual at position c with the token at c + 1 and predicts the token at
+//     c + 2 (mtp.hpp).
+//   * After a window at p with a drafts accepted, the next window starts at p' = p + a + 1 (its row 0 is the token
+//     the window picked at row a).  Chain step j (j = 0 ..) runs at cell p + a + j and predicts the token at
+//     p + a + j + 2 = p' + j + 1: row j + 1 of the next window, VERIFIED BY ROW j's pick, counter p' + j.
+//   * So a draft made at cell c is verified with counter c + 1 - whatever the window size T, the accepted count, a
+//     suffix-drafter window in between (its drafts are then simply not used), or `draft_first` (cell p - 1, a = 0 for
+//     a window at p).  The kernel reads c from the step record the host already stages for the cell (device memory,
+//     so the captured graphs replay with each round's own counter), and the seed and chain from a device copy of the
+//     request's SamplerParams, refreshed at the start of every round graph.
+//
+// THE PENALTY HISTORY.  The verify row j of the next window counts penalties over penalty_rows(consumed', window')
+// row j: the last h of (consumed', x', drafts 0 .. j-1), where consumed' is what the session holds after the commit
+// and x' the window's pick at row a.  The drafter keeps a device RING of kCoupledHistCap + max_t slots: the host
+// stages the base (the last h of consumed' + x', -1 padded in front) in [cap - h, cap) before the round, and draft j
+// writes its token to cap + j - so draft j's window is [cap + j - h, cap + j), exactly the row the target will use.
+#pragma once
+
+#include "strata/kernels/sampler.hpp"
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+
+#if defined(__CUDACC__) || defined(__HIPCC__)
+#define STRATA_COUPLED_HD __host__ __device__
+#else
+#define STRATA_COUPLED_HD
+#endif
+
+namespace strata::core {
+
+/// The longest penalty window the coupled drafter mirrors (serve's kPenaltyWindowCap).
+inline constexpr int kCoupledHistCap = 4096;
+
+/// STRATA_SPEC_COUPLED=1 (any value but empty or "0"): read once.
+inline bool coupled_draft_env() {
+    static const bool on = [] {
+        const char* e = std::getenv("STRATA_SPEC_COUPLED");
+        return e != nullptr && *e != '\0' && std::strcmp(e, "0") != 0;
+    }();
+    return on;
+}
+
+/// The MTP cell of chain step j after a window at `p` with `a` drafts accepted (draft() runs step j there).
+STRATA_COUPLED_HD inline int64_t coupled_draft_cell(int64_t p, int a, int j) { return p + a + j; }
+
+/// The Philox counter the verify window will draw for the row that checks the draft made at MTP cell `cell`.
+STRATA_COUPLED_HD inline uint64_t coupled_draft_counter(int64_t cell) { return (uint64_t) (cell + 1); }
+
+/// The penalty window the draft layer and the target use: min(penalty_last_n, cap), 0 when penalties are off.
+STRATA_COUPLED_HD inline int coupled_hist_len(int penalty_last_n, int cap) {
+    return penalty_last_n <= 0 ? 0 : (penalty_last_n < cap ? penalty_last_n : cap);
+}
+
+/// Where draft j's penalty window starts in the ring (base in [cap - h, cap), draft i at cap + i).
+STRATA_COUPLED_HD inline int coupled_hist_start(int cap, int j, int h) { return cap + j - h; }
+
+/// The ring's base on the host: the last `h` tokens of (tail[0..n_tail), next), -1 padded in front, into
+/// out[0..h) - row 0 of `penalty_rows` for the one-token window [next].
+inline void coupled_hist_base(const int32_t* tail, int64_t n_tail, int32_t next, int h, int32_t* out) {
+    if (h <= 0) return;
+    strata::kernels::penalty_rows(tail, n_tail, &next, 1, h, out);
+}
+
+}  // namespace strata::core

include/strata/core/device.hpp

@@ -26,8 +26,18 @@ struct DeviceInfo {
     uint64_t free_bytes = 0;
     int driver_version = 0, runtime_version = 0;
     int multi_processor_count = 0;
+    std::string arch;              // HIP: gcnArchName without its feature suffix ("gfx1201"); empty on CUDA
 };
 
+// HIP builds: whether GPU `ordinal` can run this binary - its architecture must be one the binary was COMPILED
+// for (STRATA_HIP_ARCHS, set by cmake/hip_backend.cmake) and it must run wave32.  "" when it can (or when there
+// is no such device: the caller's own device errors apply), else the reason in a sentence.  A binary carried to
+// another card would otherwise fail later with "invalid device function".  CUDA builds: always "".
+std::string gpu_arch_problem(int ordinal);
+
+// The GPU architectures this binary was compiled for ("gfx1100,gfx1201"); "" on CUDA builds.
+const char* compiled_gpu_archs();
+
 // Throws when there is no CUDA device.  The engine targets sm_120 specifically and must say so rather than
 // run slowly on something else: `CMakeLists.txt` already refuses to COMPILE for another architecture, and
 // this is the matching check at run time (a binary can be carried to a different machine).

include/strata/core/expert_source.hpp

@@ -25,6 +25,7 @@
 #include "strata/core/hit_hook.hpp"
 #include "strata/kernels/cpu/pool.hpp"
 
+#include <atomic>
 #include <cstdint>
 #include <cstddef>
 #include <utility>
@@ -66,6 +67,18 @@ const uint8_t* cache_complement_blob_or_fallback(
     size_t index, const std::vector<uint64_t>& offsets, const uint8_t* complement_host,
     const uint8_t* mapped_fallback);
 
+/// The resident RAM mode: which GPU-cache slots' experts are kept in RAM too.  The prompt path lends the cache's
+/// LAST slots (from `lend_from` on; a short prompt lends only the last few), and a lent slot's expert is streamed
+/// from RAM during the prompt and copied back into its slot after it.  `base_bytes` (every expert no slot holds)
+/// must fit `budget`; slots are then added from the end down to `lend_from` while they still fit.  Returns the
+/// first slot kept in RAM (`slot_bytes.size()` = none), or -1 when `base_bytes` alone exceeds `budget`.
+int64_t choose_resident_keep_from(const std::vector<uint64_t>& slot_bytes, uint64_t base_bytes, uint64_t budget,
+                                  int64_t lend_from);
+
+/// The adaptive tier swapped `in` into a GPU slot and `out` out of it: `out` takes `in`'s place in the compact copy
+/// (the caller copies out's bytes there).  False, and nothing changed, unless `in` is in the copy and `out` is not.
+bool exchange_cache_complement(std::vector<uint64_t>& offsets, size_t in, size_t out);
+
 }  // namespace detail
 
 
@@ -98,6 +111,10 @@ public:
     virtual void begin_layer(int64_t layer, const int32_t* ids, int64_t k) { (void) layer; (void) ids; (void) k; }
     /// Plan v0.3 P6: the DEVICE address of a pinned, mapped blob (the GPU can read it over PCIe), or null.
     virtual const uint8_t* device_alias(int64_t layer, int64_t expert) const { (void) layer; (void) expert; return nullptr; }
+    /// Whether the verify window may give the GPU a PCIe share of this layer's misses at all (each expert is still
+    /// checked with `pinned`).  The arena answers per layer through its expert 0; the resident RAM mode's compact
+    /// copy has no expert 0 when the GPU cache holds it, so it answers for the whole copy.
+    virtual bool pcie_layer(int64_t layer) const { return device_alias(layer, 0) != nullptr; }
 };
 
 /// Plan v0.3 P6: what the GPU computes in a verify window's layer, written by the pool (mapped host memory) right
@@ -315,9 +332,20 @@ public:
     bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err);
     /// Pin a compact host mirror of experts absent from a fully filled static GPU cache. The mmap remains open
     /// as a fallback for later cache reloads. This is opt-in because the complement may still be a large allocation.
+    ///
+    /// The resident RAM mode (`--resident-experts`, `--resident-cpu-experts`):
+    ///   - `pin`: page-locked and mapped (`cudaHostAlloc`), so the prompt path copies it by DMA and the verify
+    ///     window may read a share of the misses over PCIe; when the driver refuses, ordinary memory locked in the
+    ///     working set instead.  `pin = false` is ordinary pageable memory (the ROCm arm: large pinned allocations
+    ///     can fail there, and it is what the HIP measurements used).
+    ///   - `lend_from_slot` >= 0: the GPU-cache slots from there to the end are the prompt path's lend region; their
+    ///     experts are kept in RAM too, from the last slot down, as far as `available RAM - headroom_bytes` allows
+    ///     (a lent slot's expert is streamed during the prompt and copied back after it).
+    ///   - the rest (the experts no slot holds) must fit that budget, or nothing is allocated and this returns false.
     bool pin_cache_complement(
         const ExpertCache& cache, std::string& err, bool pin = true,
-        const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs = {});
+        const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs = {}, int64_t lend_from_slot = -1,
+        uint64_t headroom_bytes = 8ull << 30);
     void close();
 
     bool mapped() const { return base_ != nullptr; }
@@ -325,10 +353,35 @@ public:
     uint64_t pinned_bytes() const { return complement_pinned_ ? complement_bytes_ : 0; }
     uint64_t resident_bytes() const { return complement_bytes_; }
     bool complement_pinned() const { return complement_pinned_; }
+    bool complement_ready() const { return complement_ready_; }
+    uint64_t locked_bytes() const { return complement_locked_; }
+    /// Lend-region slots whose experts the compact copy holds (the last ones of the cache).
+    int64_t resident_lent_slots() const { return complement_lent_slots_; }
+
+    // ---- the resident RAM mode and the adaptive tier.  A swap puts `in` (held here) into a GPU slot and evicts
+    // `out` (held only by that slot).  Before the slot is overwritten the caller copies it back into an exchange
+    // buffer and calls `stage_exchange`: `out` is then read from that buffer, and `in` still from here (the CPU
+    // computes both until the swap lands).  Once the slot copy has landed, `commit_exchanges` moves `out` into
+    // `in`'s place, so the copy keeps holding exactly the experts the GPU does not - with no read of the file.
+    /// Whether the compact copy holds `(layer, expert)`.
+    bool has_resident(int64_t layer, int64_t expert) const;
+    /// Host room for `n` evicted blobs (page-locked when possible).  Idempotent for the same or a smaller `n`.
+    bool reserve_exchanges(int64_t n, std::string& err);
+    int64_t exchange_capacity() const { return xstage_cap_; }
+    uint8_t* exchange_buffer(int64_t q) const;
+    /// Requires `has_resident(layer, in)`, `!has_resident(layer, out)` and `exchange_buffer(q)` holding out's blob.
+    bool stage_exchange(int64_t layer, int64_t in, int64_t out, int64_t q);
+    /// After the GPU copies of every staged swap have landed.  Returns how many exchanges were applied.
+    int64_t commit_exchanges();
+    int64_t exchanges() const { return exchanges_; }
+    /// With the compact copy ready: blobs read from the mapped file since (what the plain mmap mode may read from
+    /// the SSD).  0 in a steady resident mode; lend-region experts that did not fit the RAM count here.
+    int64_t file_reads() const { return file_reads_.load(std::memory_order_relaxed); }
 
     const uint8_t* blob(int64_t layer, int64_t expert) override;
     bool pinned(int64_t layer, int64_t expert) const override;
     const uint8_t* device_alias(int64_t layer, int64_t expert) const override;
+    bool pcie_layer(int64_t layer) const override;
 
     /// Blobs touched, for the driver to report.  With `h = 0` this is `48 * k` per token and the number is only
     /// interesting once Phase 3 makes it not so.
@@ -350,6 +403,17 @@ private:
     std::vector<uint64_t> complement_offsets_;
     bool complement_pinned_ = false;
     bool complement_ready_ = false;
+    uint64_t complement_locked_ = 0;          ///< bytes held in the working set (pin refused)
+    int64_t complement_lent_slots_ = 0;
+    std::vector<const uint8_t*> override_;    ///< staged exchanges: an evicted expert read from its exchange buffer
+    struct Exchange { size_t in, out; int64_t q; uint64_t bytes; };
+    std::vector<Exchange> staged_;
+    uint8_t* xstage_ = nullptr;               ///< exchange buffers, `xstage_cap_ x xstage_blob_`
+    bool xstage_pinned_ = false;
+    int64_t xstage_cap_ = 0;
+    uint64_t xstage_blob_ = 0;
+    int64_t exchanges_ = 0;
+    std::atomic<int64_t> file_reads_{0};
     int64_t reads_ = 0;
 #if defined(_WIN32)
     void* file_ = nullptr;
@@ -385,7 +449,7 @@ public:
     /// Allocates and loads `<pack_dir>/experts.bin`.  Prints nothing; the caller reports `note()` and the load
     /// rate, because those are the two numbers that say whether the arena is the one that was asked for.
     bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, int threads, std::string& err,
-              uint64_t max_pinned_bytes = 0);
+              uint64_t max_pinned_bytes = 0, const std::string& shared_arena_file = {});
     /// Plan v0.3 P6: a native pack without experts.bin takes its experts from the model's shard 1.
     void set_gguf(const std::string& shard1) { gguf_ = shard1; }
     void close();

include/strata/core/mtp.hpp

@@ -21,6 +21,7 @@
 
 #include "strata/core/layer.hpp"
 #include "strata/core/session.hpp"
+#include "strata/kernels/sampler.hpp"
 
 #include <cuda_runtime.h>
 
@@ -53,6 +54,9 @@ public:
     /// KV streaming: refill the ring of the drafter's window from its host copy for a sequence that continues at
     /// `upto` (a conversation-cache resume). No-op unless the drafter's K/V is a ring.
     void kv_restore(int64_t upto);
+    /// The VRAM bind() will allocate for a native head of `head_row_bytes` per vocabulary row: the draft logits and
+    /// the draft head over rt/draft_vocab.bin's subset.  The expert cache is sized before bind(), so it reserves this.
+    uint64_t bind_bytes(uint64_t head_row_bytes, int64_t n_vocab) const;
     /// The main model's embedding and head, and the verify window's final residuals (T rows, hc*n_embd each).
     bool bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err);
 
@@ -70,16 +74,50 @@ public:
     bool draft_first(int T, const float* R_row, int32_t token, int64_t cell, int32_t* drafts, std::string& err,
                      float* probs = nullptr, float min_p = 0.0f, int* n_drafts = nullptr);
 
+    /// COUPLED DRAFT SAMPLING (core/coupled_draft.hpp; STRATA_SPEC_COUPLED=1, set up by bind()): the request's
+    /// sampling.  A sampled request (temperature > 0, not greedy) then drafts by SAMPLING with the target's chain and
+    /// the target's Philox draw for the verifying row, and `probs` is the draft's probability under that chain; a
+    /// greedy one keeps the argmax drafts and their graphs.  A no-op when the switch is off.
+    void set_draft_sampling(const strata::kernels::SamplerParams& sp);
+    /// Coupled mode with penalties: the history the next draft() chain starts from - what the session holds after the
+    /// window's commit (`tail`) and the window's pick at its last accepted row (`next`, the next window's row 0).
+    void set_draft_history(const int32_t* tail, int64_t n_tail, int32_t next);
+    bool coupled() const { return coupled_active_; }
+
     double ms_draft = 0, ms_prefill = 0;
     int64_t rounds = 0;
 
+    /// E-9: the prompt path computes this layer's prompt K/V in batches (Prefill::draft_kv): its tensors, its K/V
+    /// state, the first cell a round can still read, its device, and a wait for its own stream.
+    const float* tensor_f32(const char* name) const { return f32(name); }
+    const uint16_t* tensor_bf16(const char* name) const { return bf16(name); }
+    const void* tensor_q8(const char* name) const { return q8(name); }
+    QsaState& kv_state_rw() { return st_; }
+    int64_t first_needed() const { return (window_ > 0 && prompt_len_ > 0) ? prompt_len_ - window_ - 64 : 0; }
+    int device() const { return device_; }
+    bool idle(std::string& err) {
+        if (cs_ && cudaStreamSynchronize(cs_) != cudaSuccess) { err = "mtp: its stream failed"; return false; }
+        return true;
+    }
+
 private:
     bool record_forward(int T, int step_row0, cudaStream_t cs, std::string& err);
     bool capture_prefill(int T, std::string& err);
     bool capture_prefill_dev(int T, std::string& err);   ///< E-4: without the mapped staging (inputs copied on device)
-    bool capture_round(int T, std::string& err);
-    bool capture_step(int j, std::string& err);
+    bool capture_round(int T, bool coupled, std::string& err);
+    bool capture_step(int j, bool coupled, std::string& err);
     cudaGraphExec_t step_exec_[9] = {};
+    // coupled draft sampling: its own round/step graphs (the argmax ones stay as they were), the request's
+    // parameters and the penalty ring (mapped staging + device copies), the split scratch, token id -> subset index
+    bool setup_coupled(std::string& err);
+    cudaGraphExec_t round_exec_c_[9] = {};
+    cudaGraphExec_t step_exec_c_[9] = {};
+    bool coupled_ok_ = false, coupled_active_ = false;
+    bool coupled_rec_ = false;   ///< record_forward: the full layer ends in the coupled sampler (draft coupled_j_)
+    int coupled_j_ = 0;
+    strata::kernels::SamplerParams *h_cparams_ = nullptr, *m_cparams_ = nullptr, *cparams_ = nullptr;
+    int32_t *h_chist_ = nullptr, *m_chist_ = nullptr, *cring_ = nullptr, *dinv_ = nullptr;
+    void* cscratch_ = nullptr;
     const float* f32(const char* name) const;
     const uint16_t* bf16(const char* name) const;
     const void* q8(const char* name) const;

include/strata/core/pinned.hpp

@@ -40,8 +40,15 @@ struct PinnedArena {
     /// Plan v0.3 P6: slices of different sizes (one per layer of a native pack), given as their start offsets
     /// followed by the end of the last one.  `slice_starts` holds the registered ones.
     /// `max_pinned_bytes`: optional cap on CUDA registration. 0 preserves the normal unrestricted path.
-    PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds, uint64_t max_pinned_bytes = 0);
+    /// `shared_file`: on Linux, use a file-backed MAP_SHARED mapping instead of anonymous memory.
+    /// `shared_pack_hash` identifies the pack that is allowed to populate that backing.  The file carries a
+    /// small header and is refused when its stored hash does not match.  Empty `shared_file` preserves the
+    /// existing allocation path.  Population/coordination and backing-file lifetime remain the caller's job.
+    PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds, uint64_t max_pinned_bytes = 0,
+                const std::string& shared_file = {}, uint64_t shared_pack_hash = 0);
     std::vector<uint64_t> slice_starts;
+    void* mapping_base = nullptr;     ///< actual mapping start; differs from base when a shared-file header exists
+    uint64_t mapping_bytes = 0;       ///< bytes to release from mapping_base
     ~PinnedArena();
     PinnedArena(const PinnedArena&) = delete;
     PinnedArena& operator=(const PinnedArena&) = delete;

include/strata/core/session.hpp

@@ -45,6 +45,20 @@ struct SessionState {
 
     float* R = nullptr;             ///< alias of `block.R`, named for what it means at this level
     int64_t k = 10;                 ///< experts per token
+
+    // **THE LAYER-RANGE CARVE (multi-GPU).**  A split stage's session used to carve state for ALL 48 layers
+    // (every QSA KV pool at max_cells, every GDN recurrence) while running only its slice - the same
+    // whole-model-sized-buffer disease the QSA chunked-prefill PR cured upstream.  A session now owns
+    // [layer_lo, layer_hi): `qsa_states`/`gdn_state` keep GLOBAL ordinal indexing, the arrays simply hold
+    // fewer rows, so per-layer consumers subtract `gdn_ord0`, and every old model-level `qsa_states[0]` use
+    // (the shared RoPE table, the prefill staging identity, the MTP rope borrow) reads `qsa_states[primary]`.
+    // A range with no QSA layer still carves ONE primary state so those model-level uses stay valid.
+    int64_t layer_lo = 0, layer_hi = 0;   ///< resolved by `session_init` (hi = n_layers for the full range)
+    int64_t qsa_ord0 = 0;                 ///< global QSA ordinal of the first allocated state
+    int64_t qsa_alloc = 0;                ///< allocated QSA states (>= 1 whenever the model has any)
+    int64_t gdn_ord0 = 0;                 ///< global GDN ordinal of `gdn_state` row 0
+    int64_t gdn_alloc = 0;                ///< allocated GDN rows
+    int qsa_primary() const { return (int) qsa_ord0; }
     /// The doorbell the host loop polls.  Null until a caller provides one - the engine runs without it, and
     /// neither `session_token` nor `session_replay` looks at it.
     const Doorbell* db = nullptr;
@@ -76,9 +90,15 @@ struct SessionState {
 /// Bytes for a whole session at `max_cells` of context.  Every layer's state is sized at once, because P2.T10
 /// requires ZERO token-path allocations - a `cudaMalloc` that happened on the first token of a longer sequence
 /// would satisfy every test here and fail that one.
-uint64_t session_bytes(const ModelGeometry& g, int64_t max_cells, int64_t k);
-/// Carves `base` (DEVICE memory) into `s`.  Returns the bytes used.
-uint64_t session_init(const ModelGeometry& g, int64_t max_cells, int64_t k, void* base, SessionState& s);
+///
+/// [layer_lo, layer_hi) carves only that range's per-layer state (a split stage runs a slice of the model);
+/// the default full range is byte-identical to the old whole-model carve.  Pure arithmetic - safe to call for
+/// a candidate range before anything is allocated, which is how the layer-split search prices a placement.
+uint64_t session_bytes(const ModelGeometry& g, int64_t max_cells, int64_t k, int64_t layer_lo = 0,
+                       int64_t layer_hi = -1);
+/// Carves `base` (DEVICE memory) into `s`.  Returns the bytes used.  Same range convention as `session_bytes`.
+uint64_t session_init(const ModelGeometry& g, int64_t max_cells, int64_t k, void* base, SessionState& s,
+                      int64_t layer_lo = 0, int64_t layer_hi = -1);
 /// Zeroes every layer's state - the residual to `R_init`, everything else to zero, so a fresh sequence starts
 /// from the reference's own `zeros()`.
 void session_zero(SessionState& s, const ModelGeometry& g, const float* R_init, void* stream);
@@ -207,7 +227,8 @@ struct SessionGraphs {
 /// replay re-reads every token - so the position is data, not an argument.  That is the whole reason rounds
 /// 211-214 moved the per-token counts into device buffers.
 bool session_capture(const WeightTable& tables, const ModelGeometry& g, SessionState& s, const float* parts,
-                     SessionGraphs& gr, std::string& err, bool split = false);
+                     SessionGraphs& gr, std::string& err, bool split = false, int64_t layer_lo = 0,
+                     int64_t layer_hi = -1);
 
 /// One token by REPLAYING the captured graphs.  Identical arithmetic to `session_token`; the only difference is
 /// that ~2,000 kernel launches become 48 graph launches.

include/strata/hip_compat/cuda_runtime.h

@@ -67,10 +67,13 @@
 #define cudaSetDevice hipSetDevice
 #define cudaStreamBeginCapture hipStreamBeginCapture
 #define cudaStreamCaptureModeThreadLocal hipStreamCaptureModeThreadLocal
+#define cudaStreamCaptureStatus hipStreamCaptureStatus
+#define cudaStreamCaptureStatusNone hipStreamCaptureStatusNone
 #define cudaStreamCreate hipStreamCreate
 #define cudaStreamCreateWithFlags hipStreamCreateWithFlags
 #define cudaStreamDestroy hipStreamDestroy
 #define cudaStreamEndCapture hipStreamEndCapture
+#define cudaStreamIsCapturing hipStreamIsCapturing
 #define cudaStreamNonBlocking hipStreamNonBlocking
 #define cudaStreamQuery hipStreamQuery
 #define cudaStreamSynchronize hipStreamSynchronize

include/strata/hip_compat/intrinsics.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-// CUDA device intrinsics used by Strata kernels that HIP does not provide on gfx1100.
+// CUDA device intrinsics used by Strata kernels that HIP does not provide on RDNA3 / RDNA4 (wave32).
 // This header is included only from the HIP cuda_runtime compatibility shim.
 #if defined(__HIPCC__)
 
@@ -15,10 +15,11 @@ __device__ __forceinline__ int signed_byte(uint32_t word, int lane) {
 
 // CUDA's signed __dp4a: four signed byte products accumulated modulo 2^32.
 __device__ __forceinline__ int dp4a(int a, int b, int c) {
-#if defined(__gfx1100__) && __has_builtin(__builtin_amdgcn_sudot4)
-    // RDNA3 exposes the signed/unsigned dot4 form. Mark both packed operands
-    // signed to preserve CUDA __dp4a semantics; keep the portable path for
-    // other HIP compilers/targets.
+#if (defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1200__) || \
+     defined(__gfx1201__)) && __has_builtin(__builtin_amdgcn_sudot4)
+    // RDNA3 and RDNA4 expose the signed/unsigned dot4 form (v_dot4_i32_iu8). Mark
+    // both packed operands signed to preserve CUDA __dp4a semantics; keep the
+    // portable path for other HIP compilers/targets.
     return __builtin_amdgcn_sudot4(true, a, true, b, c, false);
 #else
     const uint32_t ua = static_cast<uint32_t>(a);

include/strata/kernels/bf16_bits.hpp

@@ -28,6 +28,12 @@
 namespace strata::kernels {
 
 /// Round-to-nearest-even f32 -> bf16, returned as raw bits in the LOW half of the uint16.
+///
+/// **A NaN IS PASSED THROUGH, QUIETED, AS `ggml_compute_fp32_to_bf16` DOES.**  The rounding add alone
+/// carries out of the mantissa for a NaN: 0x7FFFFFFF became 0x8000 (-0) and 0x7F800001 became 0x7F80 (+inf), so
+/// a NaN upstream was masked instead of propagated.  The test is on the magnitude bits, the result keeps the
+/// sign and the top mantissa bits and sets the quiet bit (64 = bit 6 of the bf16).  Every non-NaN input,
+/// infinities included, rounds exactly as before.
 STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) {
     uint32_t i;
 #if defined(__HIP_DEVICE_COMPILE__)
@@ -35,6 +41,7 @@ STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) {
 #else
     std::memcpy(&i, &f, 4);
 #endif
+    if ((i & 0x7FFFFFFFu) > 0x7F800000u) return (uint16_t) ((i >> 16) | 64u);
     i = (i + ((i >> 16) & 1u) + 0x7FFFu) & 0xFFFF0000u;
     return (uint16_t) (i >> 16);
 }

include/strata/kernels/dp4a.hpp

@@ -0,0 +1,42 @@
+// include/strata/kernels/dp4a.hpp - the two instructions the kernels use that Pascal (sm_60) does not have.
+//
+// The engine is written for compute capability 7.5 and newer and refuses older cards at CMake configure time
+// (a Pascal build passes -DSTRATA_EXPERIMENTAL_SM60=ON).  A GP100-class card is compute capability 6.0, which is
+// missing exactly two things the i-quant kernels rely on:
+//
+//   * `__dp4a`, a byte-wise dot product, available from 6.1 (Pascal GP10x/GV11x).  Its fallback below is
+//     llama.cpp's own (`ggml/src/ggml-cuda/common.cuh`), which is the reference for the kernels in this
+//     directory: they are transcribed from llama.cpp's vecdotq.cuh, and that wrapper's operands are the same
+//     sites.  It reinterprets the operands as SIGNED bytes and accumulates in int32, which is what `__dp4a`
+//     does for these calls, so the fallback is bit-exact rather than merely close.
+//   * `__nanosleep`, available from 7.0 (Volta).  It only paces single-thread doorbell waits, so a loop that
+//     spins without it is correct, merely busier.
+//
+// Both are compile-time selections on `__CUDA_ARCH__`, so one source tree builds for Pascal and for RTX
+// 20/30/40/50 without a #define at the call sites.  A HIP build never takes either fallback: `__CUDA_ARCH__` is
+// undefined there, so `STRATA_DP4A` is `__dp4a` (which `hip_compat/intrinsics.hpp` provides) and the pause is
+// HIP's own `__nanosleep`.
+#pragma once
+
+#include <cstdint>
+
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 610
+__device__ __forceinline__ int strata_dp4a(const int a, const int b, const int c) {
+    const int8_t* a8 = (const int8_t*) &a;
+    const int8_t* b8 = (const int8_t*) &b;
+    return c + a8[0] * b8[0] + a8[1] * b8[1] + a8[2] * b8[2] + a8[3] * b8[3];
+}
+#define STRATA_DP4A(a, b, c) strata_dp4a((a), (b), (c))
+#else
+#define STRATA_DP4A(a, b, c) __dp4a((a), (b), (c))
+#endif
+
+/// Yield the thread while a doorbell flag is being polled.  The length of the pause is a backoff hint, not a
+/// contract, and only pre-Volta CUDA has no `__nanosleep` at all.
+__device__ __forceinline__ void strata_spin_pause() {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 700
+    // sm_6x: the loop spins.  Every call site is a single-thread doorbell wait, so nothing else is delayed.
+#else
+    __nanosleep(100);
+#endif
+}

include/strata/kernels/native_qsa_indexer.hpp

@@ -1,5 +1,6 @@
 #pragma once
 #include "strata/kernels/qsa.hpp"
+#include "strata/kernels/rope_scaling.hpp"
 
 namespace strata::kernels {
 
@@ -20,6 +21,9 @@ bool native_qsa_indexer_enabled();
 // contiguous, with nonnegative pos_base divisible by four. Clear state before
 // starting a new sequence. Completed block b rotates at pos_base+4*b; the spare
 // always rotates at zero, including nonzero sequence bases, as in the oracle.
+// Under rope scaling (rope_scaling.hpp) the zero angle keeps the spare's
+// rotation an identity; YaRN's mscale rides in through cos(0) on both the pooled
+// keys and the indexer queries, so selection is unchanged.
 //
 // Explicit nonnull stream; no allocation or synchronization. All spans must be
 // aligned and disjoint, remain valid through replay, and contain finite values;
@@ -32,11 +36,11 @@ bool native_qsa_indexer_enabled();
 // Host-side positions (not for a captured graph).
 void native_qsa_indexer_append_batch(const float* raw, int64_t n, int64_t p0, int32_t pos_base, const float* gamma,
                                      float epsilon, const QsaIndexerBuffers& b, const QsaShapes& s, int64_t max_cells,
-                                     float freq_base, void* stream);
+                                     const RopeScaling& scaling, void* stream);
 
 void native_qsa_indexer_append(const float* raw, const int32_t* relative_pos_device,
                                int32_t pos_base, const float* gamma, float epsilon,
                                const QsaIndexerBuffers& buffers, const QsaShapes& shapes,
-                               int64_t max_cells, float freq_base, void* stream);
+                               int64_t max_cells, const RopeScaling& scaling, void* stream);
 
 } // namespace strata::kernels

include/strata/kernels/native_rope.hpp

@@ -1,14 +1,20 @@
 #pragma once
 
+#include "strata/kernels/rope_scaling.hpp"
+
 namespace strata::kernels {
 void native_rope_set_enabled(bool enabled);
 bool native_rope_enabled();
 
-// Pinned CUDA text-only IMRoPE: F32 rows, 64 rotated channels, no YaRN/frequency
-// factors, equal text positions in all four IMRoPE sections. Each device position
-// must be nonnegative. The position buffer remains live through graph replay.
-// Supports head_dim 128/256 and exact x==out; partial overlap is rejected.
+// Pinned CUDA text-only IMRoPE: F32 rows, 64 rotated channels, equal text positions in all four
+// IMRoPE sections. Each device position must be nonnegative. The position buffer remains live
+// through graph replay. Supports head_dim 128/256 and exact x==out; partial overlap is rejected.
 // Explicit stream required. No allocation or synchronization.
+//
+// The scaling (rope_scaling.hpp) rides in as the resolved process config: none reproduces the
+// original contract exactly, linear/YaRN apply ggml's rope_yarn - the interpolation mix, the
+// corr_dims ramp along the pairs, and the mscale magnitude correction folded into cos and sin.
+// The struct's knobs are process constants, so kernel arguments baked at graph capture stay valid.
 void native_rope_apply(const float* x, float* out, int rows, int head_dim,
-                       int n_rot, float freq_base, const int* positions, void* stream);
+                       int n_rot, const RopeScaling& scaling, const int* positions, void* stream);
 }

include/strata/kernels/ngram.hpp

@@ -110,6 +110,9 @@ struct PleIoOptions {
     uint32_t max_inflight = 64;      ///< outstanding SSD reads (decode needs 16; prefill chunks use more)
     uint64_t cache_rows = 1u << 20;  ///< bounded row cache: 1,048,576 rows x 90 B ~ 95 MB; 0 disables
     bool io_thread = true;           ///< reads submitted by a worker thread, not the caller
+    /// Mmap mode only (`--ple-io ram`): lock the whole mapped table in RAM at open, so no SSD read ever sits on
+    /// the prompt or token path. Needs RAM for the full table. POSIX only (mlock); `locked()` reports the outcome.
+    bool lock = false;
 };
 
 /// The PLE table.  Held by pointer-to-impl so this header does not drag `<windows.h>` into every
@@ -144,6 +147,8 @@ public:
     bool open(const std::string& gguf_path, std::string& err);
     void close();
     bool is_open() const;
+    /// True when `PleIoOptions::lock` was asked for and mlock succeeded (false: pages only pre-touched).
+    bool locked() const;
     uint64_t rows() const;
 
     /// 16 row indices -> 2560 floats.  The gathered rows are flattened HEAD-SLOWEST: row h's 160 values

include/strata/kernels/qsa.hpp

@@ -118,7 +118,10 @@ inline QsaShapes qsa_real_shapes() {
 
 /// The one legal RMSNorm epsilon for this artifact (`attention.layer_norm_rms_epsilon`).
 inline float qsa_rms_eps() { return 1e-6f; }
-/// `rope.freq_base`, no rope.scaling keys, so freq_scale = 1 and no YaRN.
+/// `rope.freq_base`, the DEFAULT frequency base.  The artifact ships no `rope.scaling` keys, so the
+/// process's rope scaling starts at none - the runtime configuration lives in `rope_scaling.hpp`
+/// (`rope_scaling_set`, once at startup, before the table is built and any graph captured), and this
+/// constant is what that config's `freq_base` defaults to and what CLI `--rope-freq-base` overrides.
 inline double qsa_freq_base() { return 1e7; }
 
 /// The selection width: `min(n_kv, idx_top_k + idx_block - 1)`.

include/strata/kernels/rope.hpp

@@ -6,10 +6,18 @@
 // The cos/sin table is built on the HOST in float64 (see src/kernels/cuda/rope.cu for why) and the kernel is
 // a pure rotation over it.  The table depends on (n_rot, theta, position) and not on the token, so it is worth
 // caching; `build_rope_table` fills `max_pos` positions of `n_rot/2` pairs each.
+//
+// ROPE SCALING RIDES IN THE TABLE, NOT IN THE KERNEL.  The kernels below read cos/sin values and
+// cannot tell a scaled table from an unscaled one - with scaling on, the scaled angles (and YaRN's
+// mscale magnitude correction, folded into the same values) simply ARE the table.  That is why
+// neither `rope_neox_apply` nor the indexer's pooling kernel takes any scaling argument: the config
+// enters once, here, at build time (`rope_scaling.hpp`).
 #pragma once
 
 #include <cstdint>
 
+#include "strata/kernels/rope_scaling.hpp"
+
 #if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_ROPE_HD __host__ __device__
 #else
@@ -38,6 +46,12 @@ STRATA_ROPE_HD inline void rope_neox_pair(float a, float b, float c, float s, fl
 
 void build_rope_table(int n_rot, double theta, int max_pos, float* cos_tab, float* sin_tab);
 
+/// The same table under the process's rope scaling (rope_scaling.hpp).  For `RopeScalingType::None`
+/// this IS the five-argument builder above - the parity test holds the two to be bit-identical - and
+/// for linear/YaRN the scaled angles (YaRN's mscale included) replace them.  The math is ggml's,
+/// transcribed; see the header of `rope_scaling.hpp` for what arrives from where.
+void build_rope_table(int n_rot, const RopeScaling& scaling, int max_pos, float* cos_tab, float* sin_tab);
+
 // `x` and `out` are (rows, head_dim) and `pos` is (rows,) - one position per row, so a batch of heads at
 // different sequence positions is one call.  `cos_tab`/`sin_tab` are (max_pos, n_rot/2).  `out` may equal `x`.
 //

include/strata/kernels/rope_scaling.hpp

@@ -0,0 +1,193 @@
+// include/strata/kernels/rope_scaling.hpp - rope scaling: extending the context past the trained one.
+//
+// WHAT IT IS.  The artifact's RoPE is trained over `orig_ctx` positions (the artifact ships no
+// `rope.scaling` keys, so this is a pure runtime feature).  Scaling re-uses positions past that by
+// rescaling the rotation angles.  Three types, the llama.cpp set:
+//
+//   none    freq_scale 1, no correction                      - today's behaviour, exactly
+//   linear  Position Interpolation: angles shrunk by 1/factor - every position fits again, all
+//           frequencies compressed uniformly
+//   yarn    YaRN: high-frequency pairs keep their trained angles (extrapolate), low-frequency
+//           pairs are interpolated, and the pairs in between blend - plus a magnitude correction
+//           `mscale` that keeps the attention temperature where training put it
+//
+// THE MATH IS GGML'S, TRANSCRIBED, NOT RE-DERIVED.  `third_party/llama.cpp` is vendored and is the
+// numerical contract (the same standing `native_rope.cu` declares): ramp, correction dims, the
+// interpolation mix and the mscale formula below are line-for-line that source, with the MIT
+// attribution at each helper.  The one deliberate deviation is bookkeeping: ggml's device kernel
+// speaks in even dim indices (`i0`) and halves them inside the ramp, while every Strata rope kernel
+// speaks in PAIR indices - so `rope_yarn_ramp` here takes the pair index and compares it against
+// `corr_dims` directly.  Same numbers, one less division per call site.
+//
+// HOW IT REACHES THE ROPE.  Two forms, one config:
+//
+//   - the TABLE path (the default decode path, the indexer's pooled key) never changes its kernels:
+//     scaling is baked into the cos/sin contents by `build_rope_table`, which the parity test checks
+//     bit-exactly against a float64 reference;
+//   - the ANALYTIC paths (`--native-rope`, the native indexer, prefill) get the resolved constants
+//     as kernel arguments.
+//
+// **PROCESS CONSTANTS, SET ONCE BEFORE CAPTURE.**  A CUDA graph bakes kernel ARGUMENTS at capture
+// time, so the scaling must be fixed before `session_init` captures the layers - which is exactly
+// the constraint the per-token values already respect by living in device memory (`st.pos_dev`).
+// `rope_scaling_set()` is called once from the engine's startup, after CLI and GGUF defaults meet,
+// before any graph exists.  There is deliberately no way to change it mid-process: the K already in
+// the cache is stored POST-RoPE, so a mid-run change would mix two scalings in one cache.
+#pragma once
+
+#include <cmath>
+
+#if defined(__CUDACC__) || defined(__HIPCC__)
+#define STRATA_ROPE_SCALING_HD __host__ __device__
+#else
+#define STRATA_ROPE_SCALING_HD
+#endif
+
+namespace strata::kernels {
+
+enum class RopeScalingType { None, Linear, YaRN };
+
+/// The float32 constants the ANALYTIC rotation kernels take (`rope_scaled_angle`'s arguments).  The defaults are
+/// the identity: freq_scale 1, no correction, magnitude 1 - with them the helper computes `cosf(theta) * 1.0f`
+/// on `1.0f * theta`, which is exactly today's unscaled `cosf(theta)` bit for bit.
+struct RopeKernelArgs {
+    float freq_scale = 1.0f;
+    float corr_low = 0.0f;
+    float corr_high = 0.0f;
+    float ext_factor = 0.0f;
+    float attn_factor = 1.0f;
+};
+
+/// The one rope configuration, as the CLI flag family and the (future) GGUF rope keys resolve to.
+///
+/// Default-constructed this is EXACTLY today's engine: freq_scale 1, mscale 1, no correction.  That
+/// identity is a regression gate, not a coincidence - the parity test compares the default's table
+/// against the pre-scaling builder bit for bit.
+struct RopeScaling {
+    RopeScalingType type = RopeScalingType::None;
+    double freq_base = 1e7;    ///< `rope.freq_base` (the value pinned by `qsa_freq_base()`)
+    double factor = 1.0;       ///< `--rope-scale`: how far past the trained context the run goes
+    double freq_scale_in = 0;  ///< `--rope-freq-scale`: the raw ggml knob; 0 = 1/factor
+    double orig_ctx = 262144;  ///< `--yarn-orig-ctx`: the context the model was trained over
+    double ext_factor = 0;     ///< `--yarn-ext-factor`: correction depth; 0 = off, 1 = full YaRN
+    double attn_factor = 1.0;  ///< `--yarn-attn-factor`: extra magnitude scale (ggml's attn_factor)
+    double beta_fast = 32.0;   ///< `--yarn-beta-fast`
+    double beta_slow = 1.0;    ///< `--yarn-beta-slow`
+
+    /// The angle shrink.  ggml's `freq_scale`: 1 means "the trained angles", 1/factor squeezes
+    /// `factor` positions into the trained range.
+    double freq_scale() const {
+        if (type == RopeScalingType::None) return 1.0;
+        return freq_scale_in > 0 ? freq_scale_in : 1.0 / factor;
+    }
+
+    /// The magnitude correction.  ggml's `rope_yarn` multiplies cos and sin by this; the log term
+    /// rides on `ext_factor != 0` (not on the type), exactly as that source has it.  With the
+    /// default knobs YaRN factor F gives `1 + 0.1*ln(F)` - YaRN's temperature correction.
+    double mscale() const {
+        if (ext_factor == 0) return attn_factor;
+        return attn_factor * (1.0 + 0.1 * std::log(1.0 / freq_scale()));
+    }
+
+    /// The correction window, in PAIR indices: below `out[0]` pairs extrapolate (keep the trained
+    /// angles), above `out[1]` they interpolate, in between they blend.  Transcribed from ggml's
+    /// `ggml_rope_yarn_corr_dims` (see the derivation comment at `rope_yarn_corr_dim` below).
+    void corr_dims(int n_rot, double out[2]) const {
+        const double start = std::floor(rope_yarn_corr_dim(n_rot, orig_ctx, beta_fast, freq_base));
+        const double end = std::ceil(rope_yarn_corr_dim(n_rot, orig_ctx, beta_slow, freq_base));
+        out[0] = start < 0 ? 0 : start;
+        out[1] = end > (double) (n_rot - 1) ? (double) (n_rot - 1) : end;
+    }
+
+    /// The analytic kernels' constants.  `none` returns the identity constants WHATEVER the other knobs hold -
+    /// the table builder ignores them for `none` too, so the two rotation forms cannot disagree there, and an
+    /// unscaled run computes exactly what it computed before scaling existed.  Linear/YaRN: ggml's `rope_yarn`
+    /// arguments (`attn_factor` is the raw knob; the helper applies the log term when correcting).
+    RopeKernelArgs kernel_args(int n_rot) const {
+        RopeKernelArgs a;
+        if (type == RopeScalingType::None) return a;
+        double cd[2];
+        corr_dims(n_rot, cd);
+        a.freq_scale = (float) freq_scale();
+        a.corr_low = (float) cd[0];
+        a.corr_high = (float) cd[1];
+        a.ext_factor = (float) ext_factor;
+        a.attn_factor = (float) attn_factor;
+        return a;
+    }
+
+private:
+    /// ggml/src/ggml.c: "Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)`
+    /// for x, we get `corr_dim(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`".
+    /// float64 rather than ggml's float32: the table path computes in float64 and is held to a
+    /// bit-exact float64 reference, so its inputs are float64 too.
+    static double rope_yarn_corr_dim(int n_dims, double n_ctx_orig, double n_rot, double base) {
+        return (double) n_dims * std::log(n_ctx_orig / (n_rot * 2.0 * 3.14159265358979323846)) /
+               (2.0 * std::log(base));
+    }
+};
+
+/// The ramp between extrapolation (1) and interpolation (0).
+///
+/// ggml/src/ggml-cuda/rope.cu `rope_yarn_ramp`: `1 - clamp(y, 0, 1)` with
+/// `y = (i0 / 2 - low) / max(0.001, high - low)`.  ggml passes the even dim index `i0`; this takes
+/// the pair index (ggml's `i0/2`) so Strata's pair-indexed kernels don't re-derive the mapping.  The
+/// 0.001 floor on the denominator is ggml's, kept verbatim: `corr_dims` can produce low == high.
+STRATA_ROPE_SCALING_HD inline float rope_yarn_ramp(float low, float high, int pair) {
+    const float y = ((float) pair - low) / (high - low > 0.001f ? high - low : 0.001f);
+    const float clamped = y < 0.0f ? 0.0f : (y > 1.0f ? 1.0f : y);
+    return 1.0f - clamped;
+}
+
+/// The scaled angle for one pair, float32, the ANALYTIC paths' contract (pinned ggml arithmetic).
+///
+/// Takes `theta_extrap` - the position times that pair's trained `inv_freq` - and returns the angle
+/// to take cos/sin of, with ggml's mscale already folded in.  `mscale_in` must be `attn_factor`
+/// (ggml's kernels receive the raw knob and apply the log term here, where it is per-pair constant;
+/// this helper keeps that shape so the native path reads against ggml line by line).
+STRATA_ROPE_SCALING_HD inline void rope_scaled_angle(float theta_extrap, float freq_scale, float corr_low,
+                                                     float corr_high, float ext_factor, float mscale_in,
+                                                     int pair, float& cos_out, float& sin_out) {
+    // The interpolation, always applied; the correction mixed in on top (ggml-cuda/rope.cu rope_yarn).
+    float theta = freq_scale * theta_extrap;
+    float mscale = mscale_in;
+    if (ext_factor != 0.0f) {
+        const float ramp_mix = rope_yarn_ramp(corr_low, corr_high, pair) * ext_factor;
+        theta = theta * (1.0f - ramp_mix) + theta_extrap * ramp_mix;
+        // "Get n-d magnitude scaling corrected for interpolation" - the log term only when correcting.
+        mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);
+    }
+    cos_out = cosf(theta) * mscale;
+    sin_out = sinf(theta) * mscale;
+}
+
+/// The ONE validity rule for a resolved configuration, shared by the engine's startup check and every rotation
+/// site that takes the struct, so a bad knob cannot pass one site and trip (or silently NaN) another.  Returns
+/// nullptr when the configuration is usable, else what is wrong.  Every value is tested with `std::isfinite`
+/// FIRST: a NaN passes every `<`/`<=` range comparison, which is how a `--rope-scale nan` would otherwise slip
+/// through.  `none` passes with any knob values - the knobs are inert there (the resolution resets them).
+inline const char* rope_scaling_invalid(const RopeScaling& s) {
+    if (!std::isfinite(s.freq_base) || s.freq_base <= 1.0) return "the frequency base must be a finite value above 1";
+    if (s.type == RopeScalingType::None) return nullptr;
+    if (!std::isfinite(s.factor) || s.factor < 1.0) return "the scaling factor must be a finite value >= 1";
+    if (!std::isfinite(s.freq_scale_in) || s.freq_scale_in < 0.0)
+        return "the frequency scale must be finite and positive (0 = 1/factor)";
+    const double fs = s.freq_scale();
+    if (!std::isfinite(fs) || fs <= 0.0) return "the resolved frequency scale must be a finite positive value";
+    if (!std::isfinite(s.orig_ctx) || s.orig_ctx < 1.0) return "the original (trained) context must be finite and > 0";
+    if (!std::isfinite(s.ext_factor) || s.ext_factor < 0.0) return "the YaRN ext factor must be finite and >= 0";
+    if (!std::isfinite(s.attn_factor) || s.attn_factor <= 0.0) return "the attention factor must be finite and > 0";
+    if (!std::isfinite(s.beta_fast) || s.beta_fast <= 0.0 || !std::isfinite(s.beta_slow) || s.beta_slow <= 0.0)
+        return "the YaRN betas must be finite and > 0";
+    if (!std::isfinite(s.mscale()) || s.mscale() <= 0.0) return "the resolved magnitude correction must be finite and > 0";
+    return nullptr;
+}
+
+/// The process's one rope configuration.  Set it ONCE at startup, after the CLI and the model file
+/// have agreed on the values and BEFORE `session_init` (which builds the rope table and captures the
+/// graphs); read it at table build time and when launching the analytic kernels.  Plain storage, no
+/// atomics: one writer on the startup thread, every reader after it.
+void rope_scaling_set(const RopeScaling& scaling);
+const RopeScaling& rope_scaling();
+
+}  // namespace strata::kernels

include/strata/kernels/sampler.hpp

@@ -42,6 +42,24 @@ struct SamplerParams {
 void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
                    const SamplerParams& p, int* out, void* stream);
 
+// ---- COUPLED DRAFT SAMPLING (include/strata/core/coupled_draft.hpp, STRATA_SPEC_COUPLED=1).  Device pointers
+// throughout; every per-request / per-round value comes from device memory, so the calls can be captured.
+//
+// The scratch `coupled_draft_sample` needs for `nv` logits (0: too wide for the split merge, coupled cannot run).
+size_t coupled_draft_scratch_bytes(int nv);
+// A round's inputs: `*params` <- `*mapped_params`, ring[cap - h, cap) <- mapped_hist[cap - h, cap) with h =
+// coupled_hist_len(penalty_last_n, cap).  `mapped_*` are mapped host memory.
+void coupled_draft_stage(const SamplerParams* mapped_params, const int32_t* mapped_hist, SamplerParams* params,
+                         int32_t* ring, int cap, void* stream);
+// Draft j of the chain from ONE row of `nv` logits (modified in place: the penalties): the target's chain with
+// `*params`, the penalty window ring[cap + j - h, cap + j), Philox counter coupled_draft_counter(step_rec[0]).
+// `sub_to_id` / `id_to_sub` map the draft head's subset to token ids and back (null: the whole vocabulary,
+// `id_vocab` ids).  Writes the token id to *out_id, its probability under the final distribution to *out_prob, and
+// the id to ring[cap + j].
+void coupled_draft_sample(float* logits, int nv, const int32_t* sub_to_id, const int32_t* id_to_sub, int id_vocab,
+                          const SamplerParams* params, int32_t* ring, int cap, int j, const int32_t* step_rec,
+                          void* scratch, int32_t* out_id, float* out_prob, void* stream);
+
 // The penalty-history rows of a verify window, on the host: row t of `out` (T rows of `h` slots) is the last `h`
 // tokens of `tail[0..n_tail)` followed by `window[0..t]`, most recent LAST, -1 in the unused front slots.
 // `tail` is what the state consumed before the window, `window[0]` the fed-back token and `window[1..]` the

include/strata/prefill/kernels.hpp

@@ -6,6 +6,7 @@
 #pragma once
 
 #include "strata/kernels/kv_stream.hpp"
+#include "strata/kernels/rope_scaling.hpp"
 
 #include <cstdint>
 
@@ -65,8 +66,10 @@ void moe_combine(const float* D, const int32_t* slot, const float* w, const floa
 // ---- QSA helpers
 /// In place: x[r, :] = x[r, :] * rsqrt(mean x^2 + eps) * w  over rows of `cols` (row stride `ld`).
 void rms_rows(float* x, const float* w, int64_t rows, int64_t cols, int64_t ld, float eps, void* stream);
-/// NEOX rotary on the first 64 dims of each head: x [T, heads, dim] at positions pos0 + t.
-void rope(float* x, int64_t T, int64_t heads, int64_t dim, int64_t ld, int64_t pos0, float freq_base, void* stream);
+/// NEOX rotary on the first 64 dims of each head: x [T, heads, dim] at positions pos0 + t.  The rope
+/// scaling (rope_scaling.hpp) rides in as the process config - none is today's arithmetic exactly.
+void rope(float* x, int64_t T, int64_t heads, int64_t dim, int64_t ld, int64_t pos0,
+          const strata::kernels::RopeScaling& scaling, void* stream);
 /// q_full [T, 24, 512] (q | gate per head) -> q [T, 24, 256]
 void split_q(const float* q_full, float* q, int64_t T, void* stream);
 /// attn[t, h, d] *= sigmoid(q_full[t, h, 256 + d]) -> out16 (fp16 bits: the o-projection is quantized)

include/strata/prefill/prefill.hpp

@@ -34,8 +34,19 @@ struct PrefillStats {
     double ms_ple = 0;
 };
 
+}  // namespace strata::prefill
+namespace strata::core { class MtpDrafter; }
+namespace strata::prefill {
+
 class Prefill {
 public:
+    /// E-9: the draft layer's K/V for prompt cells [cell0, cell0 + n) from their final residual rows `R_rows`
+    /// (device) and `next_tokens` (host: the token at cell+1), in batches through this path's GEMMs and its idle
+    /// scratch - call it from on_chunk.  false with `err` empty: not applicable here (a ring or hybrid K/V, another
+    /// device, too little scratch; STRATA_MTP_BATCH=0), the caller runs the drafter's own pass.  Not bit-identical to
+    /// that pass (FP16 GEMMs instead of Q8_1 activations): the drafts may differ, never the target's tokens' logits.
+    bool draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t* next_tokens, int64_t n, int64_t cell0,
+                  std::string& err);
     Prefill();
     ~Prefill();
     Prefill(const Prefill&) = delete;

serve/frontend.py

@@ -242,6 +242,61 @@ CALL_START = "<tool_call>"
 CALL_END = "</tool_call>"
 
 
+PARAM_END = "</parameter>"
+FUNC_END = "</function>"
+
+
+def param_end(text: str, final: bool = False) -> int:
+    """Where a parameter value in `text` ends: the first `</parameter>` followed (after whitespace) by the next
+    `<parameter=` or `</function>` - the same text inside a value (a file that documents the call format, #210) is
+    part of the value.  -1: none yet; -2: a candidate whose follower has not arrived (streaming; `final` accepts it)."""
+    at = text.find(PARAM_END)
+    while at >= 0:
+        after = text[at + len(PARAM_END):].lstrip()
+        if after.startswith(("<parameter=", FUNC_END)):
+            return at
+        if not after or "<parameter=".startswith(after) or FUNC_END.startswith(after):
+            return at if final else -2
+        at = text.find(PARAM_END, at + 1)
+    return -1
+
+
+def call_end(text: str) -> int:
+    """Where a tool call's body ends (#210): the `</tool_call>` after the call's own `</function>`, found by walking
+    its parameters with param_end, so a value may contain either tag.  -1: not complete yet.  A body that is not in
+    the call format ends at the first `</tool_call>`, as before."""
+    s = text.lstrip()
+    pos = len(text) - len(s)
+    if not s.startswith("<function="):
+        return text.find(CALL_END) if not "<function=".startswith(s) else -1
+    gt = text.find(">", pos)
+    if gt < 0:
+        return -1
+    pos = gt + 1
+    while True:
+        rest = text[pos:]
+        s = rest.lstrip()
+        pos += len(rest) - len(s)
+        if s.startswith("<parameter="):
+            gt = text.find(">", pos)
+            if gt < 0:
+                return -1
+            end = param_end(text[gt + 1:])
+            if end < 0:
+                return -1
+            pos = gt + 1 + end + len(PARAM_END)
+        elif s.startswith(FUNC_END):
+            rest = text[pos + len(FUNC_END):]
+            s = rest.lstrip()
+            if s.startswith(CALL_END):
+                return pos + len(FUNC_END) + len(rest) - len(s)
+            return -1 if CALL_END.startswith(s) else text.find(CALL_END, pos)
+        elif not s or "<parameter=".startswith(s) or FUNC_END.startswith(s):
+            return -1
+        else:
+            return text.find(CALL_END, pos)
+
+
 def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
     """`<function=NAME>\\n<parameter=P>\\nVALUE\\n</parameter>...</function>` -> ToolCall. Values are JSON-decoded
     when the tool's schema says the parameter is not a string (or, without a schema, when they parse as JSON
@@ -257,9 +312,9 @@ def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
         rest = rest[rest.index("<parameter=") + len("<parameter="):]
         pname = rest[:rest.index(">")]
         rest = rest[rest.index(">") + 1:]
-        end = rest.find("</parameter>")
+        end = param_end(rest, final=True)
         value = rest[:end] if end >= 0 else rest
-        rest = rest[end + len("</parameter>"):] if end >= 0 else ""
+        rest = rest[end + len(PARAM_END):] if end >= 0 else ""
         if value.startswith("\n"):
             value = value[1:]
         if value.endswith("\n"):
@@ -351,16 +406,17 @@ class OutputParser:
                         self.sp += 1
                         rest = rest[1:]
                     self.sval_started = True
-                end = rest.find("</parameter>")
+                end = param_end(rest)
                 if end >= 0:
                     value = rest[:end]
                     if value.endswith("\n"):
                         value = value[:-1]
                     args(json.dumps(value)[1:-1] + '"')
-                    self.sp += end + len("</parameter>")
+                    self.sp += end + len(PARAM_END)
                     self.ss = "between"
                     continue
-                safe = len(rest) - self._hold(rest, ("</parameter>",))
+                # an undecided </parameter> (-2) is held from its start, like a tag still arriving
+                safe = rest.find(PARAM_END) if end == -2 else len(rest) - self._hold(rest, (PARAM_END,))
                 if safe > 0 and rest[safe - 1] == "\n":   # may be the trailing newline before </parameter>
                     safe -= 1
                 if safe > 0:
@@ -368,7 +424,7 @@ class OutputParser:
                     self.sp += safe
                 return out
             elif self.ss == "raw":
-                end = rest.find("</parameter>")
+                end = param_end(rest)
                 if end < 0:
                     return out
                 value = rest[:end]
@@ -381,7 +437,7 @@ class OutputParser:
                 except ValueError:
                     v = value
                 args(json.dumps(v, ensure_ascii=False))
-                self.sp += end + len("</parameter>")
+                self.sp += end + len(PARAM_END)
                 self.ss = "between"
             else:
                 return out
@@ -449,7 +505,7 @@ class OutputParser:
                 self.buf = self.buf[i + len(CALL_START):]
                 self.state = "call"
             else:
-                i = self.buf.find(CALL_END)
+                i = call_end(self.buf)
                 if self.stream_tools:
                     if i >= 0:
                         whole, self.buf = self.buf, self.buf[:i]     # scan only the body

serve/mcp.py

@@ -31,6 +31,8 @@ import time
 import urllib.error
 import urllib.request
 
+from serve.winjob import contain
+
 PROTOCOL = "2025-06-18"               # the MCP revision Strata asks for; the server's answer is used as given
 DEFAULTS = {"timeout_s": 60.0, "max_result_chars": 20000, "max_rounds": 8, "start_timeout_s": 120.0}
 
@@ -105,6 +107,7 @@ class StdioTransport:
                                          env=env, **extra)
         except OSError as e:
             raise McpError(f"could not start {command!r}: {e}") from None
+        contain(self.proc)                              # ends with the server, however it ends (Windows)
         threading.Thread(target=self._read, daemon=True).start()
         threading.Thread(target=self._read_stderr, daemon=True).start()
 

serve/server.py

@@ -24,10 +24,13 @@ import argparse
 import collections
 import base64
 import hashlib
+import hmac
 import codecs
+import ctypes
 import json
 import os
 import queue
+import signal
 import subprocess
 import sys
 import tempfile
@@ -46,9 +49,11 @@ sys.path.insert(0, str(ROOT))   # run as a script (run-<model>.bat) as well as a
 from serve.frontend import (ChatTemplate, Event, OutputParser, anthropic_to_messages,  # noqa: E402
                             images_of, openai_to_messages)
 from serve.mcp import McpCancelled, hub_from_config  # noqa: E402
+from serve.winjob import contain  # noqa: E402
 
 IM_END = "<|im_end|>"
 IMAGE_PAD = "<|image_pad|>"
+VISION_START = "<|vision_start|>"
 CTX_SLACK = 8               # `strata --serve` rejects prompt + max_new + 8 > context: keep the same margin here
 # The live tok/s is a rate over a window, not a mean since the first token: a mean reads ~1/elapsed at the first
 # token (the Monitor showed five-digit numbers) and then undershoots for the first second of every answer.
@@ -92,6 +97,11 @@ class EngineDied(RuntimeError):
     """The engine process ended in the middle of a request (issue #27: on Linux, the out-of-memory killer)."""
 
 
+class GpuBusy(RuntimeError):
+    """The model is unloaded and the GPU has less free VRAM than min_free_vram_mib: something else (a game, another
+    model server) is using it, so the engine is not started into the little that is left."""
+
+
 def narrate_start(log_path: str, offset: int, args: list, done: threading.Event, heartbeat=20.0) -> None:
     """While the engine starts, say in the server window what it is doing, from its log: the start reads tens of GB
     into RAM and locks part of it for the GPU, and on many PCs everything is slow or frozen for a minute or more -
@@ -165,10 +175,13 @@ class StrataEngine:
                              daemon=True).start()
         self.proc = subprocess.Popen([exe, "--serve", *args], cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                      stderr=self.log, text=True, encoding="utf-8", bufsize=1, env=env)
+        contain(self.proc)                               # ends with the server, however it ends (Windows)
         self.max_context = 0
+        self.unloaded = False            # stopped on purpose (idle unload, POST /unload), not crashed
         self.can_stop = False            # the engine honours a STOP line mid-request (READY <ctx> stop)
         self.last = {}
         self.info = {}                   # INFO key=value facts (engine 0.1.8+): kv, expert slots, ... (Monitor tab)
+        self.prefill_tok_s_mean = None
         self.progress = None             # (read, total) prompt tokens while a prompt is read, from PP lines
         try:                             # a ready-made engine's BUILD.json says its version
             self.info["version"] = json.loads((Path(exe).parent / "BUILD.json").read_text()).get("version")
@@ -215,6 +228,11 @@ class StrataEngine:
             if "issue #29" in line:
                 return ("The engine stopped itself because it had stopped making progress - a hang it caught. Its log "
                         "line: " + line.strip() + " - please report it at github.com/Niko1221/Strata/issues.")
+        rc = self.proc.poll()
+        last = next((x.strip() for x in reversed(tail.splitlines()) if x.strip().startswith(("strata", "ERR"))), "")
+        if rc is not None and rc >= 0 and last:          # it ended by itself: its own last words say why (#215)
+            return (f"The engine exited (code {rc}). Its last log line: {last} - if that does not explain it, please "
+                    "report it at github.com/Niko1221/Strata/issues with the log.")
         return ("The usual cause is running out of RAM: Linux then ends the biggest program (check: sudo dmesg | "
                 "grep -i -E 'killed process|out of memory'); Windows slows down instead. Close other programs or use a "
                 "smaller model (Q2_0 / IQ2_XS).")
@@ -228,6 +246,25 @@ class StrataEngine:
         except subprocess.TimeoutExpired:
             return None
 
+    def unload(self):
+        """Stop the engine process so its VRAM and RAM go back to the system (idle unload, POST /unload); the next
+        request starts it again with restart().  Only between requests: the caller holds the service's fifo."""
+        try:
+            try:                                        # QUIT first, as close() does: the engine frees its memory
+                self.proc.stdin.write("QUIT\n")
+                self.proc.stdin.flush()
+                self.proc.wait(timeout=20)
+            except (OSError, ValueError, subprocess.TimeoutExpired):
+                self.proc.terminate()
+                self.proc.wait(timeout=20)
+        except subprocess.TimeoutExpired:
+            self.proc.kill()
+            self.proc.wait(timeout=20)
+        except OSError:
+            pass
+        self.ended = True
+        self.unloaded = True
+
     def restart(self):
         """Start the engine again (the same command) after it died; the new process has its own line queue."""
         try:
@@ -307,6 +344,7 @@ class StrataEngine:
         the HTTP layer turns it into an SSE comment, which keeps clients' watchdogs calm and notices a client that
         has gone.  A consumer that stops early (or `cancel`) makes the engine STOP, so it does not run to max_new."""
         self.progress = None
+        self.prefill_tok_s_mean = None
         # an image request takes the same sampling keys as text (#75: it used to decode greedily whatever was asked)
         head = f"GENI {int(max_new)}{self.sampling_keys(sampling or {})} {embeddings}" if embeddings else \
             f"GEN {int(max_new)}{self.sampling_keys(sampling or {})}"
@@ -336,6 +374,7 @@ class StrataEngine:
                     f = line.split()
                     if len(f) >= 3 and f[1].isdigit() and f[2].isdigit():
                         self.progress = (int(f[1]), int(f[2]))             # prompt progress, one per chunk: also a heartbeat (the
+                        self.prefill_tok_s_mean = float(f[4]) if len(f) >= 5 else None
                     if cancel.is_set():                   # lines reset the 10 s wait, so without this a long prompt
                         return                            # would send no keep-alives at all)
                     yield None
@@ -385,13 +424,37 @@ class Vision:
         if cfg.get("max_tokens"):
             args += ["--max-tokens", str(cfg["max_tokens"])]
         self.dir = Path(tempfile.mkdtemp(prefix="strata-vision-"))
+        self.spawn = (args, log, env)                   # to start it again after an unload
+        self.stopped = False
+        self._start()
+        self.lock = threading.Lock()
+        self.cache: dict[str, tuple[Path, int]] = {}
+
+    def _start(self):
+        args, log, env = self.spawn
         self.proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=log or subprocess.DEVNULL,
                                      text=True, encoding="utf-8", bufsize=1, env=env)
+        contain(self.proc)
         line = self.proc.stdout.readline()
         if not line.startswith("READY"):
             raise RuntimeError("the vision encoder did not start: " + line.strip())
-        self.lock = threading.Lock()
-        self.cache: dict[str, tuple[Path, int]] = {}
+        self.stopped = False
+
+    def alive(self) -> bool:
+        return not self.stopped and self.proc.poll() is None
+
+    def unload(self):
+        """Stop the encoder process (its VRAM or RAM goes back); the encoded images stay cached on disk."""
+        self.close()
+        self.stopped = True
+
+    def restart(self):
+        """Start the encoder again after an unload (or if it died); the cache of encoded images is kept."""
+        try:
+            self.proc.kill()
+        except OSError:
+            pass
+        self._start()
 
     @staticmethod
     def load(source: str) -> bytes:
@@ -572,9 +635,114 @@ class Service:
         self.started_at = time.time()
         self.status_lock = threading.Lock()
         self.mcp = None                                  # serve/mcp.py's McpHub when MCP servers are configured
+        # sharing the GPU with other programs (all off by default): unload the engine after this many idle seconds,
+        # only start it again when this much VRAM is free, and run this command first (e.g. to unload another
+        # server's model); the next request after an unload starts the engine again
+        self.idle_unload_s = 0
+        self.min_free_vram_mib = 0
+        self.before_load = None
         self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
                             tokenizer.encode("<|endoftext|>", parse_special=True))
 
+    def loaded(self) -> bool:
+        return not hasattr(self.engine, "alive") or self.engine.alive()
+
+    def _vision_down(self) -> bool:
+        return self.vision is not None and hasattr(self.vision, "alive") and not self.vision.alive()
+
+    def free_vram_mib(self) -> int | None:
+        """Free VRAM on the engine's (first) GPU, from NVML; None when it can't be read (then nothing is refused)."""
+        try:
+            from serve.telemetry import _Nvml
+            nv = _Nvml(int(getattr(self, "gpu_index", 0) or 0))
+            if not nv.ok():
+                return None
+            m = nv.Mem()
+            if nv.lib.nvmlDeviceGetMemoryInfo(nv.dev, ctypes.byref(m)) != 0:
+                return None
+            return int(m.free >> 20)
+        except Exception:
+            return None
+
+    def ensure_loaded(self):
+        """Start the engine if it is not running (unloaded, or it died - issue #27), after the before_load hook and
+        the free-VRAM check.  The caller holds self.fifo."""
+        if self.loaded() and not self._vision_down():
+            return
+        if self.before_load:
+            cmd = self.before_load
+            print(f"[strata] before loading: {cmd if isinstance(cmd, str) else ' '.join(map(str, cmd))}", flush=True)
+            try:
+                subprocess.run(cmd, shell=isinstance(cmd, str), timeout=120, stdin=subprocess.DEVNULL)
+            except (OSError, subprocess.SubprocessError) as e:
+                print(f"[strata] the before_load command failed ({e}); loading anyway", flush=True)
+        if self.min_free_vram_mib:
+            free = self.free_vram_mib()
+            deadline = time.time() + 15                 # memory another process just gave back can take a moment
+            while free is not None and free < self.min_free_vram_mib and time.time() < deadline:
+                time.sleep(0.5)
+                free = self.free_vram_mib()
+            if free is not None and free < self.min_free_vram_mib:
+                raise GpuBusy(f"the GPU is in use by another program: {free} MiB of VRAM free, the model needs "
+                              f"{self.min_free_vram_mib} (min_free_vram_mib) - it stays unloaded until that is free")
+        if self._vision_down():                         # first, as at a start: a GPU encoder takes its VRAM before
+            print("[strata] starting the vision encoder again ...", flush=True)   # the engine sizes its cache
+            self.vision.restart()
+        if self.loaded():
+            return
+        if getattr(self.engine, "unloaded", False):
+            print("[strata] loading the model again (it was unloaded) ...", flush=True)
+        else:
+            code = self.engine.exit_code() if hasattr(self.engine, "exit_code") else None
+            print(f"[strata] the engine had stopped (exit code {code}); starting it again "
+                  "(a minute or two) ...", flush=True)
+        self.engine.restart()
+        print("[strata] the engine is running again", flush=True)
+
+    def load(self):
+        """POST /load and every generation request: start the engine now if it is unloaded (raises GpuBusy)."""
+        # a request is on its way: the idle thread must not unload between this and the request's own start
+        self.last_request_at = time.time()
+        if self.loaded() and not self._vision_down():
+            return
+        with self.fifo:
+            self.ensure_loaded()
+
+    def unload(self, idle_for: float | None = None) -> str:
+        """Stop the engine between requests: "unloaded", "not loaded", "busy" (a request is running or waiting, or
+        with idle_for: one ran more recently than that) or "unsupported"."""
+        if not hasattr(self.engine, "unload"):
+            return "unsupported"
+        if not self.fifo.acquire(blocking=False):
+            return "busy"
+        try:
+            if not self.engine.alive():
+                return "not loaded"
+            with self.status_lock:
+                if self.status.get("busy") or self.status.get("queued"):
+                    return "busy"
+            if idle_for is not None and time.time() - (self.last_request_at or self.started_at) < idle_for:
+                return "busy"
+            self.engine.unload()
+            if self.vision is not None and hasattr(self.vision, "unload"):
+                self.vision.unload()
+            print(f"[strata] model unloaded{f' after {idle_for:.0f} s idle' if idle_for else ''}; "
+                  "the next request loads it again", flush=True)
+            return "unloaded"
+        finally:
+            self.fifo.release()
+
+    def start_idle_unload(self):
+        if not self.idle_unload_s or not hasattr(self.engine, "unload"):
+            return
+        print(f"[strata] the model unloads after {self.idle_unload_s} s without requests", flush=True)
+
+        def loop():
+            while True:
+                time.sleep(max(1.0, min(30.0, self.idle_unload_s / 4)))
+                self.unload(idle_for=self.idle_unload_s)
+        threading.Thread(target=loop, daemon=True).start()
+
     def set_shared(self, defaults) -> dict:
         """The Chat settings every client gets for what it leaves out; {} / None = clients use their own again."""
         self.shared = clean_shared_defaults(defaults)
@@ -614,7 +782,8 @@ class Service:
         """The hardware sampler behind GET /metrics (serve/telemetry.py), recording this server's tok/s too."""
         if getattr(self, "telemetry", None) is None:
             from serve.telemetry import Telemetry
-            self.telemetry = Telemetry(extra=lambda: {"tok_s": self._tok_s(), "tok_s_mean": self._tok_s_mean()},
+            self.telemetry = Telemetry(extra=lambda: {"tok_s": self._tok_s(), "tok_s_mean": self._tok_s_mean(),
+                                                    "prefill_tok_s_mean": self._prefill_tok_s_mean()},
                                        gpu_index=int(getattr(self, "gpu_index", 0) or 0),
                                        gpu_indices=getattr(self, "gpu_indices", None))
 
@@ -640,6 +809,12 @@ class Service:
             return 0.0
         return s["generated"] / max(1e-6, time.time() - s["first_token"])
 
+    def _prefill_tok_s_mean(self):
+        """Engine-reported mean over newly read tokens, excluding the cached prefix."""
+        with self.status_lock:
+            reading = self.status.get("busy") and self.status.get("first_token") is None
+        return getattr(self.engine, "prefill_tok_s_mean", None) if reading else 0.0
+
     def metrics(self, all_requests=False) -> dict:
         """GET /metrics: what the Monitor tab shows - the engine's facts, what it is doing, the last requests, and
         the hardware (with a minute of history per series)."""
@@ -653,6 +828,8 @@ class Service:
             state = "reading"
         elif s.get("busy"):
             state = "generating"
+        elif not self.loaded():
+            state = "unloaded"
         else:
             state = "idle"
         live = {"state": state, "queued": s.get("queued", 0), "phase": s.get("phase") if s.get("busy") else None,
@@ -662,6 +839,7 @@ class Service:
                 "elapsed_s": round(now - s["started"], 1) if s.get("busy") and s.get("started") else None,
                 "tok_s": round(self._tok_s(), 1) if state == "generating" else None,
                 "tok_s_mean": round(self._tok_s_mean(), 1) if state == "generating" else None,
+                "prefill_tok_s_mean": getattr(self.engine, "prefill_tok_s_mean", None) if s.get("busy") else None,
                 "tok_s_window_s": RATE_WINDOW_S if state == "generating" else None}
         if state == "reading" and progress:
             live["prompt_read"], live["prompt_total"] = progress
@@ -720,17 +898,24 @@ class Service:
                 raise ValueError("this server was started without the vision encoder (run setup again and choose "
                                  "'vision'), so it cannot read images")
             pad = self.tok.encode(IMAGE_PAD, parse_special=True)[0]
+            start = self.tok.encode(VISION_START, parse_special=True)[0]
             # Encode only while the engine is idle: the engine and the image encoder (a separate process) must not
             # run on the GPU at the same time - an encode during a running request left that request stuck at
             # "reading the prompt" with CPU and GPU busy, for good (reproduced).  So encoding takes its turn in the
             # same FIFO as the requests.
             with self.fifo:
                 encoded = [self.vision.encode(src) for src in images]
+            # one <|image_pad|> per image -> one per image token.  Only the markers the template writes for an image
+            # (right after <|vision_start|>) are images: the same text inside a message (an agent reading these docs,
+            # #150) is kept as plain text, or it took an image's place and the counts no longer matched.
+            literal = self.tok.encode(IMAGE_PAD, parse_special=False)
             out, k = [], 0
-            for t in ids:                               # one <|image_pad|> per image -> one per image token
-                if t == pad and k < len(encoded):
+            for j, t in enumerate(ids):
+                if t == pad and j > 0 and ids[j - 1] == start and k < len(encoded):
                     out += [pad] * encoded[k][1]
                     k += 1
+                elif t == pad:
+                    out += literal
                 else:
                     out.append(t)
             if k != len(encoded):
@@ -810,13 +995,8 @@ class Service:
             with self.fifo:
                 with self.status_lock:
                     self.status["queued"] -= 1
-                if hasattr(self.engine, "alive") and not self.engine.alive():
-                    # issue #27: it died in an earlier request - start it again instead of failing every request
-                    code = self.engine.exit_code() if hasattr(self.engine, "exit_code") else None
-                    print(f"[strata] the engine had stopped (exit code {code}); starting it again "
-                          "(a minute or two) ...", flush=True)
-                    self.engine.restart()
-                    print("[strata] the engine is running again", flush=True)
+                # issue #27: it died in an earlier request (or was unloaded) - start it again instead of failing
+                self.ensure_loaded()
                 with self.status_lock:
                     self.status.update(busy=True, phase="reading the prompt", prompt_tokens=len(ids), generated=0,
                                        started=time.time(), first_token=None, tool=None, tail="", max_tokens=max_new)
@@ -906,6 +1086,8 @@ class Service:
                     if os.environ.get("STRATA_DEBUG") and raw_ids:
                         print(f"[strata] raw: {self.tok.decode(raw_ids)!r}", flush=True)
                 self.status["busy"] = False
+                self.status.pop("tail", None)            # #212: the answer's end is not kept once it is done
+                self.status.pop("tool", None)
         for ev in parser.finish():
             yield "event", ev
         yield "done", {"finish": finish, "completion_tokens": n, "reused": (timings or {}).get("cache_n", 0),
@@ -1237,7 +1419,7 @@ def make_handler(svc: Service):
                 return True
             auth = self.headers.get("Authorization", "")
             given = auth[7:].strip() if auth.lower().startswith("bearer ") else self.headers.get("x-api-key", "")
-            if given == svc.api_key:
+            if hmac.compare_digest(given.encode(), svc.api_key.encode()):   # #213: constant-time
                 return True
             self._json(401, {"error": {"type": "authentication_error", "message": "missing or wrong API key"}})
             return False
@@ -1300,8 +1482,11 @@ def make_handler(svc: Service):
                 self.wfile.write(body)
             elif path == "/health":
                 self._json(200, {"status": "ok", "max_context": svc.engine.max_context, "model": svc.model,
-                                 "images": svc.vision is not None, "api_key": bool(svc.api_key)})
+                                 "images": svc.vision is not None, "api_key": bool(svc.api_key),
+                                 "loaded": svc.loaded()})
             elif path == "/status":
+                if not self._authorized():                  # #212: it shows the end of the last answer
+                    return
                 with svc.status_lock:
                     s = dict(svc.status)
                 now = time.time()
@@ -1315,11 +1500,14 @@ def make_handler(svc: Service):
                 self._json(200, s)
             elif path in ("/v1/models", "/models"):
                 if self._authorized():
-                    loaded = not hasattr(svc.engine, "alive") or svc.engine.alive()
+                    loaded = svc.loaded()
                     model = {"id": svc.model, "object": "model", "status": {"value": "loaded"},
                              "meta": {"n_ctx": svc.engine.max_context},
                              "architecture": {"input_modalities": ["text", "image"] if svc.vision is not None else ["text"],
                                               "output_modalities": ["text"]}}
+                    if not loaded and (svc.idle_unload_s or getattr(svc.engine, "unloaded", False)):
+                        model["status"] = {"value": "unloaded"}   # like llama-server's router: listed, loads on use
+                        loaded = True
                     self._json(200, {"object": "list", "data": [model] if loaded else []})
             elif path == "/props":
                 if self._authorized():
@@ -1344,8 +1532,21 @@ def make_handler(svc: Service):
             if path == "/settings":
                 self._settings()
                 return
+            if path == "/unload":                            # give the GPU back now (between requests)
+                r = svc.unload()
+                self._json(409 if r == "busy" else 200, {"status": r})
+                return
+            if path == "/load":                              # load now, e.g. ahead of a request
+                try:
+                    svc.load()
+                    self._json(200, {"status": "loaded"})
+                except GpuBusy as e:
+                    self._json(503, {"error": {"type": "server_error", "message": str(e)}})
+                return
             try:
                 req = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}")
+                if path in ("/v1/chat/completions", "/v1/messages"):
+                    svc.load()                               # unloaded: load first (or 503 while the GPU is busy)
                 if path == "/v1/chat/completions":
                     self._openai(req)
                 elif path == "/v1/messages":
@@ -1354,6 +1555,8 @@ def make_handler(svc: Service):
                     self._json(404, {"error": {"message": "not found"}})
             except ValueError as e:
                 self._json(400, {"error": {"type": "invalid_request_error", "message": str(e)}})
+            except GpuBusy as e:
+                self._json(503, {"error": {"type": "server_error", "message": str(e)}})
             except EngineDied as e:                          # before the answer started (not streamed)
                 self._json(503, {"error": {"type": "server_error", "message": f"{e}; the next request restarts it"}})
 
@@ -1362,7 +1565,7 @@ def make_handler(svc: Service):
             if model != svc.model:
                 self._json(404, {"error": {"message": "model not found"}})
                 return
-            if hasattr(svc.engine, "alive") and not svc.engine.alive():
+            if not svc.loaded() and not getattr(svc.engine, "unloaded", False):
                 self._json(503, {"error": {"message": "the engine is not running"}})
                 return
             defaults = {**svc.sampling_defaults, **svc.shared}
@@ -1374,7 +1577,7 @@ def make_handler(svc: Service):
             props = {"default_generation_settings": {"n_ctx": svc.engine.max_context, "params": params},
                      "total_slots": 1, "model_alias": svc.model, "chat_template": svc.template.source,
                      "modalities": {"vision": svc.vision is not None}, "models_autoload": False,
-                     "is_sleeping": False}
+                     "is_sleeping": not svc.loaded()}
             if getattr(svc.engine, "model_path", None):
                 props["model_path"] = svc.engine.model_path
             version = getattr(svc.engine, "info", {}).get("version")
@@ -1493,6 +1696,10 @@ class Server(ThreadingHTTPServer):
     # either one (a forgotten second start of run-<model>.bat).  Without it the second start fails loudly instead.
     allow_reuse_address = os.name != "nt"
 
+    def handle_error(self, request, client_address):
+        if not isinstance(sys.exc_info()[1], ConnectionError):   # a client that hangs up needs no stack trace
+            super().handle_error(request, client_address)
+
 
 def warn_tight_ram(arena_mib) -> None:
     """The model's experts live in RAM (INFO arena_mib, engine 0.1.10+).  With less than ~6 GB left beside them for the
@@ -1663,6 +1870,15 @@ def main() -> int:
     ap.add_argument("--mcp-config", help="a JSON file with MCP servers in Claude Desktop's format ({\"mcpServers\": "
                                          "{...}}); the web app's chat can use their tools (also \"mcp_servers\" in "
                                          "the config)")
+    ap.add_argument("--idle-unload", type=float, default=None, metavar="SECONDS",
+                    help="unload the model after this many seconds without requests, so other programs (games, other "
+                         "model servers) can use the VRAM; the next request loads it again (also \"idle_unload_s\" "
+                         "in the config; default: never)")
+    ap.add_argument("--min-free-vram-mib", type=int, default=None,
+                    help="load an unloaded model only when this much VRAM is free, else answer 503 (also "
+                         "\"min_free_vram_mib\" in the config; default: always load)")
+    ap.add_argument("--before-load", help="a command run before the model is loaded again (e.g. to unload another "
+                                          "server's model; also \"before_load\" in the config, a string or a list)")
     a = ap.parse_args()
     cfg = json.loads(Path(a.config).read_text(encoding="utf-8-sig")) if a.config else {}   # Notepad adds a BOM
     if a.gpu is not None:
@@ -1716,7 +1932,17 @@ def main() -> int:
                   model_name=cfg.get("model_name", "qwen3.8-flash-next"), vision=vision,
                   sampling_defaults=sampling_defaults,
                   fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)
+    if ("STRATA_API_KEY" in os.environ and not os.environ["STRATA_API_KEY"].strip()) or             any(x == "--api-key" and i + 1 < len(sys.argv) and not sys.argv[i + 1].strip() or x.strip() == "--api-key="
+                for i, x in enumerate(sys.argv)):
+        # #213: an empty key would switch authentication off without a word
+        print("[strata] an API key was given but it is empty: set a key, or leave --api-key / STRATA_API_KEY out",
+              file=sys.stderr)
+        return 2
     svc.api_key = a.api_key or cfg.get("api_key", "")
+    svc.idle_unload_s = a.idle_unload if a.idle_unload is not None else float(cfg.get("idle_unload_s") or 0)
+    svc.min_free_vram_mib = a.min_free_vram_mib if a.min_free_vram_mib is not None else \
+        int(cfg.get("min_free_vram_mib") or 0)
+    svc.before_load = a.before_load or cfg.get("before_load") or None
     svc.gpu_index = (gpu_list(cfg) or [0])[0]           # the Monitor reads the card the engine runs on (issue #51)
     svc.gpu_indices = gpu_list(cfg)                     # ... or every card of a layer split (issue #112)
     if a.config:                                        # the Chat settings shared with other apps, from last time
@@ -1736,6 +1962,7 @@ def main() -> int:
         hub.start()
         atexit.register(hub.close)                      # the servers Strata started end with it
     httpd = serve(svc, host=a.host, port=a.port)
+    svc.start_idle_unload()
     here = "127.0.0.1" if a.host in ("0.0.0.0", "", "::") else a.host
     print(f"ready: http://{here}:{a.port}/v1  (OpenAI: /v1/chat/completions, Anthropic: /v1/messages, "
           f"context {engine.max_context} tokens{', images on' if vision else ''}"
@@ -1759,16 +1986,29 @@ def main() -> int:
     if a.open:
         import webbrowser
         webbrowser.open(f"http://{'127.0.0.1' if a.host in ('0.0.0.0', '') else a.host}:{a.port}/")
+    # #96: docker stop sends SIGTERM, which Python ignores by default, so the container's PID 1 would be killed after
+    # the grace period with the engine still running. SIGTERM takes Ctrl+C's path below (QUIT to the engine).
+    # SIGINT keeps Python's own handler, so Ctrl+C and a second Ctrl+C work as before.
+    def on_sigterm(signum, frame):
+        raise KeyboardInterrupt
     try:
-        threading.Event().wait()
+        signal.signal(signal.SIGTERM, on_sigterm)
+    except (ValueError, OSError, AttributeError):         # not the main thread
+        pass
+    try:
+        while True:
+            time.sleep(1)                               # Windows never delivers Ctrl+C to an untimed Event.wait()
     except KeyboardInterrupt:
-        httpd.shutdown()
-        if hasattr(engine, "close"):
-            engine.close()
-        if vision:
-            vision.close()
-        if hub is not None:
-            hub.close()
+        print("\n[strata] stopping (Ctrl+C again to end the engine at once) ...", flush=True)
+        closers = [httpd.shutdown, getattr(engine, "close", None), vision.close if vision else None,
+                   hub.close if hub is not None else None]
+        for close in filter(None, closers):
+            try:
+                close()
+            except KeyboardInterrupt:                   # a second Ctrl+C: don't wait for the engine to free its memory
+                if getattr(engine, "proc", None):
+                    engine.proc.kill()
+        print("[strata] stopped", flush=True)
     return 0
 
 

serve/telemetry.py

@@ -256,7 +256,7 @@ class Telemetry:
             with self.lock:
                 self.now = s
                 for k in ("gpu_util", "gpu_mem_used", "gpu_temp", "gpu_power", "gpu_pcie_rx_mb", "cpu", "ram_used",
-                          "disk_read_mb", "tok_s"):
+                          "disk_read_mb", "tok_s", "prefill_tok_s_mean"):
                     v = s.get(k)
                     self.hist[k].append(round(v, 2) if isinstance(v, float) else v)
             time.sleep(1.0)

serve/test_server.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 import json
 import os
 import sys
+import tempfile
 import threading
 import time
 import unittest
@@ -16,7 +17,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 from serve.frontend import ChatTemplate  # noqa: E402
-from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, MockEngine, Service, StrataEngine, request_timings, serve  # noqa: E402
+from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, GpuBusy, MockEngine, Service, StrataEngine, request_timings, serve  # noqa: E402
 
 ROOT = Path(__file__).resolve().parents[1]
 CTX = 4096
@@ -167,6 +168,90 @@ class FitMaxTokens(unittest.TestCase):
         self.assertIn("no room to answer", b["error"]["message"])
 
 
+class ImageMarkers(unittest.TestCase):
+    """#150: the text "<|image_pad|>" inside a message is text, not an image's place."""
+
+    class FakeVision:
+        def __init__(self, d):
+            self.dir = Path(d)
+            self.rows = self.dir / "img.sve"
+            self.rows.write_bytes(b"rows")
+
+        def encode(self, source):
+            return self.rows, 3
+
+    def test_literal_marker_with_an_image(self):
+        import tempfile
+        tok = ByteTokenizer()
+        with tempfile.TemporaryDirectory() as d:
+            svc = Service(MockEngine(tok, "ok", max_context=CTX), tok, ChatTemplate(ROOT / "serve/chat_template.jinja"),
+                          vision=self.FakeVision(d))
+            pad = tok.encode("<|image_pad|>", parse_special=True)[0]
+            for text in ("the docs say <|image_pad|> marks an image", "plain"):
+                with self.subTest(text=text):
+                    msgs = [{"role": "user", "content": [{"type": "text", "text": text},
+                                                         {"type": "image", "source": "x.png"}]}]
+                    ids, _, _ = svc.prepare(msgs, None, {})
+                    self.assertEqual(ids.count(pad), 3)          # the image's three rows, nothing else
+                    self.assertIn("<|image_pad|> marks" if "docs" in text else "plain", tok.decode(ids))
+            svc.embeddings.path.unlink(missing_ok=True)
+
+
+class StatusNeedsTheKey(unittest.TestCase):
+    """#212: /status shows the end of the answer being written, so it needs the key like /v1/*."""
+
+    def test_status(self):
+        tok = ByteTokenizer()
+        svc = Service(MockEngine(tok, "ok", max_context=CTX), tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        svc.api_key = "k3y"
+        httpd = serve(svc, port=0)
+        base = f"http://127.0.0.1:{httpd.server_address[1]}/status"
+        try:
+            with self.assertRaises(urllib.error.HTTPError) as e:
+                urllib.request.urlopen(base, timeout=10)
+            self.assertEqual(e.exception.code, 401)
+            e.exception.close()
+            req = urllib.request.Request(base, headers={"Authorization": "Bearer k3y"})
+            with urllib.request.urlopen(req, timeout=10) as r:
+                self.assertEqual(r.status, 200)
+                self.assertNotIn("tail", json.loads(r.read()))
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+
+
+class ToolCallTerminators(unittest.TestCase):
+    """#210: a value that contains </parameter> or </tool_call> (a file documenting the call format) is kept whole."""
+    CONTENT = ("Close each value with </parameter> and the call with </function></tool_call>.\n"
+               "<parameter=x>\nnot a parameter\n</parameter>\nend")
+    SCHEMA = [{"name": "write", "parameters": {"properties": {"path": {"type": "string"},
+                                                              "content": {"type": "string"}}}}]
+
+    def run_parser(self, stream_tools, step):
+        from serve.frontend import OutputParser
+        text = ("</think>\n\n<tool_call>\n<function=write>\n<parameter=path>\ndoc.md\n</parameter>\n"
+                f"<parameter=content>\n{self.CONTENT}\n</parameter>\n</function>\n</tool_call>")
+        p = OutputParser(thinking=True, tools=self.SCHEMA, stream_tools=stream_tools)
+        evs = []
+        for i in range(0, len(text), step):
+            evs += p.feed(text[i:i + step])
+        evs += p.finish()
+        return evs
+
+    def test_values_keep_the_terminators(self):
+        for stream_tools in (False, True):
+            for step in (1, 7, 10_000):
+                with self.subTest(stream_tools=stream_tools, step=step):
+                    evs = self.run_parser(stream_tools, step)
+                    calls = [e.call for e in evs if e.kind == "tool_call"]
+                    self.assertEqual(len(calls), 1)
+                    self.assertEqual(calls[0].arguments, {"path": "doc.md", "content": self.CONTENT})
+                    self.assertFalse([e for e in evs if e.kind == "content" and e.text.strip()])
+                    if stream_tools:
+                        streamed = "".join(e.text for e in evs if e.kind == "tool_args")
+                        self.assertEqual(json.loads(streamed), {"path": "doc.md", "content": self.CONTENT})
+
+
 class ClientShapes(unittest.TestCase):
     """What real clients send: Claude Code posts /v1/messages?beta=true (issue #55) and puts hook context into the
     conversation as a mid-conversation system message (issue #56); some OpenAI clients send a late developer message."""
@@ -396,6 +481,41 @@ class LiveRate(unittest.TestCase):
         with urllib.request.urlopen(self.base + "/metrics", timeout=10) as r:
             return json.loads(r.read())
 
+    def test_prefill_rate_excludes_cached_tokens(self):
+        import io
+        import queue
+        from types import SimpleNamespace
+        engine = StrataEngine.__new__(StrataEngine)
+        engine.proc = SimpleNamespace(stdin=io.StringIO(), poll=lambda: None)   # alive() asks it (#208)
+        engine.lines = queue.Queue()
+        engine.can_stop = False
+        engine.max_context = 262144
+        engine.prefill_tok_s_mean = 9999.0
+        engine.lines.put("PP 10000 12000 2000 1000.0")  # 8000 cached, 2000 newly read in two seconds
+        engine.lines.put("DONE 1 12000 4000 10 stop 0 0 8000")
+        gen = engine.generate([1], 1, {}, threading.Event())
+        self.assertIsNone(next(gen))
+        self.assertEqual(engine.progress, (10000, 12000))
+        self.assertEqual(engine.prefill_tok_s_mean, 1000.0)
+        self.svc.engine = engine
+        self.svc.status.update(busy=True, first_token=None)
+        self.assertEqual(self.metrics()["live"]["prefill_tok_s_mean"], 1000.0)
+        self.assertNotIn("prefill_tok_s", self.metrics()["live"])
+        self.assertEqual(self.svc._prefill_tok_s_mean(), 1000.0)
+        self.svc.status.update(first_token=time.time(), generated=1)
+        self.assertEqual(self.svc._prefill_tok_s_mean(), 0.0)
+        self.assertEqual(list(gen), [])
+        timings = request_timings(12000, 1, engine.last)
+        self.assertEqual(timings["prompt_per_second"], 1000.0)
+        engine.lines.put("PP 8000 12000")
+        engine.lines.put("DONE 0 12000 0 0 stop 0 0 12000")
+        gen = engine.generate([1], 1, {}, threading.Event())
+        next(gen)
+        self.assertIsNone(engine.prefill_tok_s_mean)
+        list(gen)
+        self.svc.status["busy"] = False
+        self.assertIsNone(self.metrics()["live"]["prefill_tok_s_mean"])
+
     def test_the_live_number_is_a_rate(self):
         live_samples, stop = [], threading.Event()
 
@@ -813,5 +933,142 @@ class TimingsDrafts(unittest.TestCase):
         self.assertNotIn("draft_n", request_timings(24, 20, base))
         self.assertIsNone(request_timings(24, 20, {}))
 
+
+class UnloadableEngine(MockEngine):
+    """A mock engine that can be stopped and started again like StrataEngine (alive / unload / restart)."""
+
+    def __init__(self, *a, **kw):
+        super().__init__(*a, **kw)
+        self.running, self.unloaded, self.starts = True, False, 0
+
+    def alive(self):
+        return self.running
+
+    def unload(self):
+        self.running, self.unloaded = False, True
+
+    def restart(self):
+        self.running, self.unloaded = True, False
+        self.starts += 1
+
+
+class SharingTheGpu(unittest.TestCase):
+    """Idle unload, POST /unload and /load, the free-VRAM guard and the before_load hook (all off by default)."""
+
+    def setUp(self):
+        tok = ByteTokenizer()
+        self.engine = UnloadableEngine(tok, "</think>\n\nok", max_context=CTX)
+        self.svc = Service(self.engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        self.httpd = serve(self.svc, port=0)
+        self.base = f"http://127.0.0.1:{self.httpd.server_address[1]}"
+
+    def tearDown(self):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+    def req(self, path, body=None):
+        r = urllib.request.Request(self.base + path, data=None if body is None else json.dumps(body).encode(),
+                                   headers={"Content-Type": "application/json"}, method="GET" if body is None else "POST")
+        try:
+            with urllib.request.urlopen(r, timeout=30) as resp:
+                return resp.status, json.loads(resp.read())
+        except urllib.error.HTTPError as e:
+            with e:
+                return e.code, json.loads(e.read())
+
+    def chat(self):
+        return self.req("/v1/chat/completions", {"model": "m", "messages": [{"role": "user", "content": "hi"}]})
+
+    def test_unload_then_the_next_request_loads(self):
+        self.assertEqual(self.req("/unload", {}), (200, {"status": "unloaded"}))
+        self.assertFalse(self.engine.alive())
+        self.assertEqual(self.req("/health")[1]["loaded"], False)
+        self.assertEqual(self.req("/v1/models")[1]["data"][0]["status"]["value"], "unloaded")
+        self.assertEqual(self.req("/unload", {}), (200, {"status": "not loaded"}))
+        s, b = self.chat()
+        self.assertEqual(s, 200)
+        self.assertEqual(b["choices"][0]["message"]["content"], "ok")
+        self.assertEqual(self.engine.starts, 1)
+        self.assertEqual(self.req("/health")[1]["loaded"], True)
+
+    def test_load_endpoint(self):
+        self.svc.unload()
+        self.assertEqual(self.req("/load", {}), (200, {"status": "loaded"}))
+        self.assertTrue(self.engine.alive())
+        self.assertEqual(self.req("/load", {}), (200, {"status": "loaded"}))
+        self.assertEqual(self.engine.starts, 1)
+
+    def test_unload_refused_while_a_request_runs(self):
+        with self.svc.fifo:
+            self.assertEqual(self.svc.unload(), "busy")
+        self.assertTrue(self.engine.alive())
+
+    def test_idle_unload(self):
+        self.svc.idle_unload_s = 1
+        self.svc.last_request_at = time.time()
+        self.assertEqual(self.svc.unload(idle_for=1), "busy")       # a request just now: not idle yet
+        self.svc.start_idle_unload()
+        deadline = time.time() + 10
+        while self.engine.alive() and time.time() < deadline:
+            time.sleep(0.1)
+        self.assertFalse(self.engine.alive())
+        self.assertEqual(self.chat()[0], 200)
+
+    def test_min_free_vram_refuses_to_load(self):
+        self.svc.min_free_vram_mib = 8000
+        self.svc.free_vram_mib = lambda: 2000
+        self.svc.unload()
+        t0 = time.time()
+        s, b = self.chat()
+        self.assertEqual(s, 503)
+        self.assertIn("in use by another program", b["error"]["message"])
+        self.assertFalse(self.engine.alive())
+        self.assertGreater(time.time() - t0, 10)                    # waited for memory being given back first
+        self.svc.free_vram_mib = lambda: 9000
+        self.assertEqual(self.chat()[0], 200)
+
+    def test_min_free_vram_unreadable_loads(self):
+        self.svc.min_free_vram_mib = 8000
+        self.svc.free_vram_mib = lambda: None                       # no NVML: never refuse
+        self.svc.unload()
+        self.assertEqual(self.chat()[0], 200)
+
+    def test_before_load_runs_first(self):
+        mark = Path(tempfile.mkdtemp()) / "ran"
+        self.svc.before_load = [sys.executable, "-c", f"open({str(mark)!r}, 'w').close()"]
+        self.svc.unload()
+        self.assertFalse(mark.exists())
+        self.assertEqual(self.chat()[0], 200)
+        self.assertTrue(mark.exists())
+
+    def test_vision_encoder_unloads_and_starts_first(self):
+        order = []
+
+        class FakeVision:
+            running = True
+
+            def alive(self):
+                return self.running
+
+            def unload(self):
+                self.running = False
+
+            def restart(self):
+                order.append("vision")
+                self.running = True
+
+        engine_restart = self.engine.restart
+        self.engine.restart = lambda: (order.append("engine"), engine_restart())
+        self.svc.vision = FakeVision()
+        self.assertEqual(self.svc.unload(), "unloaded")
+        self.assertFalse(self.svc.vision.alive())
+        self.assertEqual(self.chat()[0], 200)
+        self.assertEqual(order, ["vision", "engine"])             # the encoder first, as at a start
+        self.assertTrue(self.svc.vision.alive())
+
+    def test_off_by_default(self):
+        self.assertEqual((self.svc.idle_unload_s, self.svc.min_free_vram_mib, self.svc.before_load), (0, 0, None))
+        self.assertEqual(self.req("/health")[1]["loaded"], True)
+
 if __name__ == "__main__":
     unittest.main()

serve/test_winjob.py

@@ -0,0 +1,67 @@
+"""serve/winjob.py: a contained child - and whatever it starts - ends when the process that contained it ends, even
+when that process is killed outright (the console window closed, Task Manager), with no cleanup of its own.
+
+    python -m unittest serve.test_winjob      (Windows only; skipped elsewhere)
+"""
+import os
+import subprocess
+import sys
+import time
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+# stands in for the server: contains a child that starts a grandchild, prints both pids, then waits to be killed
+PARENT = f"""
+import subprocess, sys, time
+sys.path.insert(0, {str(ROOT)!r})
+from serve.winjob import contain
+code = ("import subprocess, sys, time; g = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)']); "
+        "print(g.pid, flush=True); time.sleep(120)")
+child = subprocess.Popen([sys.executable, "-c", code], stdout=subprocess.PIPE, text=True)
+ok = contain(child)
+print(ok, child.pid, child.stdout.readline().strip(), flush=True)
+time.sleep(120)
+"""
+
+
+def running(pid: int) -> bool:
+    out = subprocess.run(["tasklist", "/FI", f"PID eq {pid}", "/NH"], capture_output=True, text=True).stdout
+    return str(pid) in out
+
+
+@unittest.skipUnless(os.name == "nt", "job objects are Windows-only")
+class TestWinJob(unittest.TestCase):
+    def test_children_end_with_a_killed_server(self):
+        # a venv's python.exe is a launcher that runs the real interpreter as its child: kill() must reach the latter
+        python = getattr(sys, "_base_executable", None) or sys.executable
+        parent = subprocess.Popen([python, "-c", PARENT], stdout=subprocess.PIPE, text=True)
+        child = grandchild = None
+        try:
+            ok, child, grandchild = parent.stdout.readline().split()
+            child, grandchild = int(child), int(grandchild)
+            self.assertEqual(ok, "True")
+            self.assertTrue(running(child) and running(grandchild))
+            parent.kill()                       # TerminateProcess: no atexit, no finally - like closing the window
+            parent.wait(10)
+            deadline = time.monotonic() + 10
+            while (running(child) or running(grandchild)) and time.monotonic() < deadline:
+                time.sleep(0.2)
+            self.assertFalse(running(child), "the child outlived the server")
+            self.assertFalse(running(grandchild), "the grandchild outlived the server")
+        finally:
+            if parent.poll() is None:
+                parent.kill()
+            for pid in (child, grandchild):              # on a failure, don't leave them sleeping
+                if pid and running(pid):
+                    subprocess.run(["taskkill", "/PID", str(pid), "/F"], capture_output=True)
+
+    def test_elsewhere_or_without_a_process_it_is_a_no_op(self):
+        from serve.winjob import contain
+        self.assertFalse(contain(None))
+
+
+if __name__ == "__main__":
+    unittest.main()

serve/web/app.css

@@ -92,6 +92,11 @@ main { flex: 1; min-height: 0; display: flex; }
 .st-metric__value small { font-size: var(--st-fs-md); font-weight: var(--st-fw-medium); color: var(--st-ink-muted); margin-left: 4px; }
 .st-metric__sub { font-size: var(--st-fs-xs); color: var(--st-ink-muted); font-variant-numeric: tabular-nums; min-height: 1em; }
 .st-metric__spark { display: block; overflow: visible; }
+.speed-values { display: flex; justify-content: space-between; gap: var(--st-s-3); }
+.speed-values > div { display: flex; flex-direction: column; gap: 10px; min-width: 0; }
+.speed-values > div:last-child { text-align: right; }
+.speed-values .st-metric__value { font-size: clamp(20px, 2.2vw, var(--st-fs-stat)); }
+.st-metric__spark .speed-prefill { color: color-mix(in srgb, var(--st-accent) 65%, black); }
 .st-metric__spark[data-tone="warn"] { color: var(--st-warn); }
 .st-metric__spark[data-tone="info"] { color: var(--st-info); }
 .monitor-row { display: grid; grid-template-columns: minmax(260px, 1fr) minmax(0, 2.4fr); gap: var(--st-s-4); }

serve/web/app.js

@@ -109,7 +109,7 @@ async function loadHealth() {
 
 // ------------------------------------------------------------------ Monitor
 const METRICS = [
-  {key: "speed", label: "Speed", icon: "gauge", unit: "tok/s", series: "tok_s"},
+  {key: "speed", label: "Speed", icon: "gauge", unit: "t/s", series: "tok_s"},
   {key: "gpu", label: "GPU load", icon: "gpu", unit: "%", series: "gpu_util", max: 100},
   {key: "vram", label: "VRAM", icon: "layers", unit: "GB", series: "gpu_mem_used"},
   {key: "temp", label: "GPU temp", icon: "thermometer", unit: "°C", series: "gpu_temp", tone: "warn"},
@@ -121,11 +121,17 @@ const METRICS = [
 $("metrics").innerHTML = METRICS.map((m) => `
   <div class="st-card metric-card"><div class="st-metric">
     <span class="st-metric__label">${icon(m.icon, "st-icon st-icon--sm")}${esc(m.label)}</span>
-    <span class="st-metric__value" id="mv-${m.key}">–</span>
-    <span class="st-metric__sub" id="ms-${m.key}"></span>
+    ${m.key === "speed" ? `<div class="speed-values">
+      <div><span class="st-metric__value" id="mv-speed">-</span><span class="st-metric__sub" id="ms-speed">Decode</span></div>
+      <div class="speed-prefill"><span class="st-metric__value" id="mv-prefill">-</span><span class="st-metric__sub" id="ms-prefill">Prefill</span></div>
+    </div>` : `<span class="st-metric__value" id="mv-${m.key}">–</span>
+    <span class="st-metric__sub" id="ms-${m.key}"></span>`}
     <svg class="st-metric__spark" id="sp-${m.key}" viewBox="0 0 100 32" preserveAspectRatio="none"${m.tone ? ` data-tone="${m.tone}"` : ""}>
       <path class="area" fill="currentColor" opacity=".12"/><path class="line" fill="none" stroke="currentColor"
-      stroke-width="1.6" stroke-linejoin="round" stroke-linecap="round" vector-effect="non-scaling-stroke"/></svg>
+      stroke-width="1.6" stroke-linejoin="round" stroke-linecap="round" vector-effect="non-scaling-stroke"/>
+      ${m.key === "speed" ? `<g id="sp-prefill" class="speed-prefill"><path class="area" fill="currentColor" opacity=".12"/>
+        <path class="line" fill="none" stroke="currentColor" stroke-width="1.6" stroke-linejoin="round"
+        stroke-linecap="round" vector-effect="non-scaling-stroke"/></g>` : ""}</svg>
   </div></div>`).join("");
 
 function spark(id, values, max) {
@@ -226,8 +232,14 @@ function renderMonitor(live, hw, st, eng, h, last, requests, totals, kept) {
 
   // the eight cards
   const speed = live.state === "generating" ? live.tok_s : last ? last.decode_tok_s : null;
-  setMetric("speed", speed == null ? null : fmt(speed, 1), "tok/s", live.state === "generating" ? "now" : last ? "last request" : "");
+  setMetric("speed", speed == null ? null : fmt(speed, 1), "t/s",
+            live.state === "generating" ? "Decode now" : last ? "Decode last request" : "Decode");
+  const prefill = live.state !== "idle" ? live.prefill_tok_s_mean
+                : last && last.prompt_ms > 0 ? Math.max(0, last.prompt_tokens - (last.reused || 0)) / (last.prompt_ms / 1000) : null;
+  setMetric("prefill", prefill == null ? null : fmt(prefill), "t/s",
+            live.state === "reading" ? "Prefill now" : live.state === "generating" ? "Prefill this request" : last ? "Prefill last request" : "Prefill");
   spark("sp-speed", h.tok_s);
+  spark("sp-prefill", h.prefill_tok_s_mean);
   // a model split across several cards (issue #112): the cards show their total / mean / hottest, and each card's own
   const per = (f) => (hw.gpus || []).map((g) => `GPU ${g.index} ${f(g)}`).join(" · ");
   const multi = (hw.gpus || []).length > 1;

serve/winjob.py

@@ -0,0 +1,74 @@
+"""serve/winjob.py - on Windows, tie the processes Strata starts to the server's own lifetime.
+
+Closing the console window, Task Manager or a crash end the server without running its cleanup, and the engine,
+the vision encoder and the MCP servers kept running on their own. `contain(proc)` puts a child in a job object that
+is set to kill everything in it when its last handle closes - and the only handle is this process's, which the OS
+closes however the server ends. Processes the child starts later join the same job. The server itself stays out of
+the job, so a browser that `--open` starts is not tied to it. Elsewhere, and if the job cannot be made, a no-op.
+"""
+from __future__ import annotations
+
+import os
+import threading
+
+_job = None
+_lock = threading.Lock()
+
+if os.name == "nt":
+    import ctypes
+    from ctypes import wintypes
+
+    _k32 = ctypes.WinDLL("kernel32", use_last_error=True)
+
+    class _IoCounters(ctypes.Structure):
+        _fields_ = [(n, ctypes.c_ulonglong) for n in ("ReadOperationCount", "WriteOperationCount",
+                                                       "OtherOperationCount", "ReadTransferCount",
+                                                       "WriteTransferCount", "OtherTransferCount")]
+
+    class _BasicLimits(ctypes.Structure):
+        _fields_ = [("PerProcessUserTimeLimit", ctypes.c_longlong), ("PerJobUserTimeLimit", ctypes.c_longlong),
+                    ("LimitFlags", wintypes.DWORD), ("MinimumWorkingSetSize", ctypes.c_size_t),
+                    ("MaximumWorkingSetSize", ctypes.c_size_t), ("ActiveProcessLimit", wintypes.DWORD),
+                    ("Affinity", ctypes.c_size_t), ("PriorityClass", wintypes.DWORD),
+                    ("SchedulingClass", wintypes.DWORD)]
+
+    class _ExtendedLimits(ctypes.Structure):
+        _fields_ = [("BasicLimitInformation", _BasicLimits), ("IoInfo", _IoCounters),
+                    ("ProcessMemoryLimit", ctypes.c_size_t), ("JobMemoryLimit", ctypes.c_size_t),
+                    ("PeakProcessMemoryUsed", ctypes.c_size_t), ("PeakJobMemoryUsed", ctypes.c_size_t)]
+
+    _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
+    _JobObjectExtendedLimitInformation = 9
+
+    _k32.CreateJobObjectW.restype = wintypes.HANDLE
+    _k32.CreateJobObjectW.argtypes = (wintypes.LPVOID, wintypes.LPCWSTR)
+    _k32.SetInformationJobObject.argtypes = (wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD)
+    _k32.AssignProcessToJobObject.argtypes = (wintypes.HANDLE, wintypes.HANDLE)
+
+    def _make_job():
+        job = _k32.CreateJobObjectW(None, None)     # no security attributes: the handle is not inherited
+        if not job:
+            return None
+        info = _ExtendedLimits()
+        info.BasicLimitInformation.LimitFlags = _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
+        if not _k32.SetInformationJobObject(job, _JobObjectExtendedLimitInformation, ctypes.byref(info),
+                                            ctypes.sizeof(info)):
+            _k32.CloseHandle(job)
+            return None
+        return job
+
+
+def contain(proc) -> bool:
+    """Put a started subprocess.Popen in the kill-on-close job. True if it is in; False (harmlessly) otherwise."""
+    global _job
+    if os.name != "nt" or proc is None:
+        return False
+    with _lock:
+        if _job is None:
+            _job = _make_job() or 0
+        if not _job:
+            return False
+    try:
+        return bool(_k32.AssignProcessToJobObject(_job, int(proc._handle)))
+    except (AttributeError, OSError, ValueError):
+        return False

setup.py

@@ -13,13 +13,16 @@ What the first run does (each step is skipped when it is already done):
   1. checks your PC: NVIDIA GPU and driver, RAM, CPU, free disk space
   2. asks the questions
   3. installs the Python packages it needs into .venv (numpy, jinja2, ..., and NVIDIA's CUDA libraries)
-  4. gets the Strata engine: a ready-made build for RTX 30/40/50 cards (no compiler needed); if none fits your PC,
+  4. gets the Strata engine: a ready-made build for RTX 20/30/40/50 cards (no compiler needed); if none fits your PC,
      it installs the build tools (asks first) and compiles the engine for your GPU
   5. downloads the model from Hugging Face (resumable), and the vision encoder if you want images
   6. prepares the model for Strata and fetches the MTP draft layer (~5 GB, from the original Qwen checkpoint)
   7. writes run-<model>.bat / run-<model>.sh and starts the model
 
-Options: --family qwen|swift, --model Q2_0|IQ2_XS|IQ3_XXS|IQ3_S, --context 32768, --vision yes|no|gpu|cpu, --port 8080, --yes (recommended
+Options: --family qwen|swift, --model Q2_0|IQ2_XS|IQ3_XXS|IQ3_S, --context 32768, --rope-scaling none|linear|yarn
+(--rope-scale F; past the trained 262144 the setup adds yarn and the factor is the final context over 262144,
+at least 1 - an explicit --rope-scaling none is refused for such a context), --vision yes|no|gpu|cpu, --port
+8080, --yes (recommended
 answers, no questions), --setup (install another model / change settings instead of starting), --no-start,
 --host 0.0.0.0 --api-key KEY (reach it from other devices on your network), --experimental-speed-projection on|off
 (EXPERIMENTAL, off by default),
@@ -58,11 +61,13 @@ PREBUILT_ASSET = "strata-windows-x64.zip" if WIN else "strata-linux-x64.zip"
 # the CUDA libraries the ready-made engine loads (the same CUDA 13.0 it is built with), from NVIDIA's pip packages
 CUDA_WHEELS = ["nvidia-cublas==13.0.2.14", "nvidia-cuda-runtime==13.0.96"]
 MIN_DRIVER = 580                       # CUDA 13.0
-MIN_ENGINE = (0, 1, 25)                # v0.1.25: faster prompts (grouping off the copy engine, fused hyper-connection kernels), AMD HIP backend, --kv k8v4; v0.1.24: long prompts faster (QSA select on tensor cores); v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
+MIN_ENGINE = (0, 1, 30)                # v0.1.30: short prompts faster (streaming from 1024 tokens), resident low-RAM variant, multi-GPU session carve, RDNA4; v0.1.29: sampled answers faster (split top-k), #154 correctness fixes; v0.1.28: the expert cache reserves the draft head, a cancelled request no longer fails the next; v0.1.27: RTX 20 (sm_75) in the ready-made engine, the HIP build without CUDA headers; v0.1.26: the draft layer's prompt pass in batches; v0.1.25: faster prompts (grouping off the copy engine, fused hyper-connection kernels), AMD HIP backend, --kv k8v4; v0.1.24: long prompts faster (QSA select on tensor cores); v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
 PY_PACKAGES = ["numpy", "jinja2", "regex", "pyyaml", "tqdm", "requests", "cmake", "ninja", "pillow", "psutil"]
 
 MODELS = {
-    "Q2_0": {"about": "2-bit, the fastest", "download_gb": 66.4, "ram_gb": 48, "arena_gb": 34.0},
+    # the original model only for now: Swift 1.5's Q2_0 files split one layer's experts across the two shards, which
+    # the pack tool (tools/iq_pack.py) cannot prepare yet (#171)
+    "Q2_0": {"about": "2-bit, the fastest", "download_gb": 66.4, "ram_gb": 48, "arena_gb": 34.0, "families": ("qwen",)},
     "IQ2_XS": {"about": "2-bit i-quant, a little better quality, close in speed", "download_gb": 68.0, "ram_gb": 48,
                "arena_gb": 35.5},
     "IQ3_XXS": {"about": "3-bit i-quant, better quality, slower (more CPU work per token)", "download_gb": 75.8,
@@ -75,7 +80,12 @@ MODELS = {
     "IQ1_M": {"about": "the Coder's only size: half the experts, stored like IQ3_S (3.5 bits)", "download_gb": 58.4,
               "ram_gb": 32, "arena_gb": 23.4, "families": ("coder",)},
 }
-CONTEXTS = [8192, 32768, 65536, 131072, 262144]
+# Contexts past 262144 (the model's trained length) extend it by rope scaling: for the context it will
+# serve the setup resolves the method (yarn, or one question when interactive) and derives the factor
+# from the final context (final / 262144, at least 1) itself (below), keeps an explicit
+# --rope-scaling/--rope-scale, and refuses an explicit --rope-scaling none there - the stock angles past
+# the trained range are out of spec.
+CONTEXTS = [8192, 32768, 65536, 131072, 262144, 393216, 524288]
 # The model families: the same architecture, weights in the same three GSQ-RCO sizes, different files.
 FAMILIES = {
     "qwen": {"title": "Qwen3.8-Flash-Next", "by": "Qwen; GSQ-RCO quants by ISTA-DASLab",
@@ -221,7 +231,7 @@ def cpu_info():
     name, avx2, avx512 = platform.processor() or "unknown CPU", False, False
     if WIN:
         pf = ctypes.windll.kernel32.IsProcessorFeaturePresent
-        avx2 = bool(pf(40))                       # PF_AVX2_INSTRUCTIONS_AVAILABLE
+        avx2 = bool(pf(40)) or _cpuid_avx2()      # PF_AVX2_INSTRUCTIONS_AVAILABLE, else the CPU itself (#159)
         n = out(["powershell", "-NoProfile", "-Command", "(Get-CimInstance Win32_Processor).Name"]).strip()
         name = n or name
         avx512 = bool(pf(41)) and _cpuid_avx512_full()
@@ -259,6 +269,47 @@ def _cpuid_avx512_full() -> bool:
         return False
 
 
+def _run_stub(code: bytes, *args) -> None:
+    """Runs a few bytes of x64 machine code (Windows calling convention: the arguments in rcx, rdx)."""
+    k32 = ctypes.windll.kernel32
+    k32.VirtualAlloc.restype = ctypes.c_void_p
+    k32.VirtualFree.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32)
+    buf = k32.VirtualAlloc(None, len(code), 0x3000, 0x40)
+    if not buf:
+        raise OSError("VirtualAlloc failed")
+    try:
+        ctypes.memmove(buf, code, len(code))
+        ctypes.CFUNCTYPE(None, *[ctypes.c_void_p] * len(args))(buf)(*args)
+    finally:
+        k32.VirtualFree(buf, 0, 0x8000)
+
+
+def _cpuid_avx2() -> bool:
+    """AVX2 asked from the CPU (CPUID leaf 7 EBX bit 5), with the OS saving the YMM registers (OSXSAVE + XCR0):
+    Windows' IsProcessorFeaturePresent(PF_AVX2) says no on some PCs whose CPU has it (a Ryzen 9 3950X, #159)."""
+    try:
+        def cpuid(leaf):
+            regs = (ctypes.c_uint32 * 4)()
+            _run_stub(bytes([0x53, 0x49, 0x89, 0xC8, 0x89, 0xD0, 0x31, 0xC9, 0x0F, 0xA2,      # push rbx; r8=rcx; eax=edx; ecx=0; cpuid
+                             0x41, 0x89, 0x00, 0x41, 0x89, 0x58, 0x04, 0x41, 0x89, 0x48, 0x08,  # [r8]=eax, [r8+4]=ebx, [r8+8]=ecx
+                             0x41, 0x89, 0x50, 0x0C, 0x5B, 0xC3]),                             # [r8+12]=edx; pop rbx
+                      ctypes.addressof(regs), leaf)
+            return list(regs)
+        if cpuid(0)[0] < 7:
+            return False
+        ecx1 = cpuid(1)[2]
+        if not (ecx1 >> 27) & 1 or not (ecx1 >> 28) & 1:             # OSXSAVE, AVX
+            return False
+        xcr0 = (ctypes.c_uint32 * 2)()
+        _run_stub(bytes([0x49, 0x89, 0xC8, 0x31, 0xC9, 0x0F, 0x01, 0xD0,                        # r8=rcx; ecx=0; xgetbv
+                         0x41, 0x89, 0x00, 0x41, 0x89, 0x50, 0x04, 0xC3]), ctypes.addressof(xcr0))
+        if xcr0[0] & 6 != 6:                                           # the OS saves XMM and YMM
+            return False
+        return bool((cpuid(7)[1] >> 5) & 1)
+    except Exception:
+        return False
+
+
 def gpus():
     """Every NVIDIA GPU, numbered as nvidia-smi numbers them (by PCI bus, the order the engine is told to use)."""
     s = out(["nvidia-smi", "--query-gpu=index,name,memory.total,compute_cap,driver_version",
@@ -285,8 +336,8 @@ def cc(g) -> str:
 
 def gpu_problem(g, together=False):
     """Why Strata cannot use this card, in plain words (None: it can)."""
-    if int(g["arch"]) < 80:
-        return (f"not supported - older than the RTX 30 series (compute capability {cc(g)}; Strata needs 8.0 or "
+    if int(g["arch"]) < 75:
+        return (f"not supported - older than the RTX 20 series (compute capability {cc(g)}; Strata needs 7.5 or "
                 "newer)")
     if together and g["vram_gb"] < SPLIT_MIN_VRAM_GB - 0.5:
         return (f"not supported together with other GPUs - {g['vram_gb']:.0f} GB of VRAM (a card sharing the model "
@@ -349,7 +400,7 @@ def check_gpus(sel, found, what="") -> None:
         ones = " or ".join(f"--gpu {x['index']}" for x in single)
         both = "--gpus " + ",".join(str(x["index"]) for x in can) if can else ""
         hint = ((f"use these together: {both}" + (f" (or one card: {ones})" if not together else "")) if can else
-                f"use one card: {ones}" if single else "Strata needs an NVIDIA RTX 30 series or newer card")
+                f"use one card: {ones}" if single else "Strata needs an NVIDIA RTX 20 series or newer card")
         fail(f"GPU {i}{'' if g is None else ' (' + g['name'] + ')'} {what}cannot be used: {p}", hint)
 
 
@@ -384,7 +435,7 @@ def choose_gpus(a, found) -> list:
     single = sorted([g for g in found if gpu_problem(g) is None], key=lambda x: (-round(x["vram_gb"]), x["index"]))
     if not single:
         gpu_table(found)
-        fail("none of your GPUs can run Strata", "it needs an NVIDIA RTX 30 series or newer (compute capability 8.0+)")
+        fail("none of your GPUs can run Strata", "it needs an NVIDIA RTX 20 series or newer (compute capability 7.5+)")
     can = together_ok(found)
     if not can:
         return [single[0]["index"]]
@@ -566,6 +617,18 @@ def download(url, dst: Path, what=None):
     ok(f"{what or dst.name} downloaded")
 
 
+def whole_shard(s: Path) -> bool:
+    """A shard as long as its own tensor directory says (check_shards' test, without stopping setup)."""
+    sys.path.insert(0, str(ROOT / "tools"))
+    from gguf_reader import GGUFFile
+    try:
+        g = GGUFFile(s)
+        return s.stat().st_size >= g.data_start + max((t.offset + (t.expected_bytes() or 0) for t in g.tensors),
+                                                     default=0)
+    except (OSError, ValueError, struct.error):
+        return False
+
+
 def check_shards(shards):
     """Every shard present and whole, or setup stops naming the file and the numbers.  Whole means as long as
     its own tensor directory says (the header is read, the data is not): a truncated copy (--gguf-dir, a .part
@@ -597,7 +660,9 @@ def get_llama_cpp():
     tmp = ROOT / "third_party" / "_unpack"
     shutil.rmtree(tmp, ignore_errors=True)
     with zipfile.ZipFile(z) as f:
-        f.extractall(tmp)
+        # llama.cpp's own web UI (tools/ui) is not used, and its deep paths passed Windows' 260-character limit in a
+        # folder like Downloads\Strata-main\Strata-main (#206)
+        f.extractall(tmp, [m for m in f.namelist() if "/tools/ui/" not in m])
     top = next(tmp.iterdir())
     shutil.rmtree(llama, ignore_errors=True)
     # PR #63: on Windows a rename can fail with PermissionError while an antivirus scanner still holds a file of the
@@ -644,13 +709,22 @@ def cuda_lib_dirs():
 
 
 # ------------------------------------------------------------------------------------------------ AMD (experimental)
-# The RX 7900 XT / XTX (gfx1100) on Linux, through the HIP backend (docs/AMD_HIP.md).  There is no ready-made AMD
-# engine: ROCm comes from AMD's TheRock Python wheels into .venv (no sudo; a system ROCm in /opt/rocm is used when it
-# has hipcc and hipBLAS) and the engine is compiled here.  One GPU, no images yet.
-ROCM_INDEX = os.environ.get("STRATA_ROCM_INDEX", "https://rocm.nightlies.amd.com/v2/gfx110X-dgpu/")
+# The RX 7900 XT / XTX (gfx1100) and the RX 9070 series / Radeon AI PRO R9700 (gfx1201) on Linux, through the HIP
+# backend (docs/AMD_HIP.md).  There is no ready-made AMD engine: ROCm comes from AMD's TheRock Python wheels into .venv
+# (no sudo; a system ROCm 7 in /opt/rocm is used when it has hipcc and hipBLAS) and the engine is compiled here for
+# the card.  One GPU, no images yet.
+ROCM_INDEXES = {"gfx1100": "https://rocm.nightlies.amd.com/v2/gfx110X-dgpu/",   # TheRock's wheels per GPU family
+                "gfx1201": "https://rocm.nightlies.amd.com/v2/gfx120X-all/"}
 ROCM_VERSION = os.environ.get("STRATA_ROCM_VERSION", "7.10.0a20251120")   # what Strata's HIP build was tested with
-AMD_ARCHS = ("gfx1100",)
-AMD_NAMES = {"gfx1100": "AMD Radeon RX 7900 series (gfx1100)"}   # when sysfs has no product name
+ROCM_SYSTEM_MIN = (7, 0)       # an older system ROCm is passed over for the wheels (gfx1201 needs ROCm 6.4 or newer)
+AMD_ARCHS = ("gfx1100", "gfx1201")
+AMD_NAMES = {"gfx1100": "AMD Radeon RX 7900 series (gfx1100)",   # when sysfs has no product name
+             "gfx1201": "AMD Radeon RX 9070 series / AI PRO R9700 (gfx1201)"}
+AMD_CARDS = "the RX 7900 XT / XTX (gfx1100) and the RX 9070 / 9070 XT / Radeon AI PRO R9700 (gfx1201)"
+
+
+def rocm_index(arch):
+    return os.environ.get("STRATA_ROCM_INDEX") or ROCM_INDEXES[arch]
 
 
 def amd_gpus():
@@ -690,35 +764,88 @@ def amd_gpus():
 
 def amd_problem(g):
     if g["arch"] not in AMD_ARCHS:
-        return (f"not supported - Strata's AMD backend runs on the RX 7900 XT / XTX ({', '.join(AMD_ARCHS)}) only, "
-                f"this is {g['arch']}")
+        return f"not supported - Strata's AMD backend runs on {AMD_CARDS} only, this is {g['arch']}"
     return None
 
 
-def rocm_root():
-    """ROCm for compiling and running the HIP engine: (root, library folders).  A system ROCm with hipcc and hipBLAS,
-    else AMD's TheRock wheels (ROCM_VERSION, from ROCM_INDEX) installed into .venv."""
+def rocm_version(root):
+    """(major, minor) of a ROCm install, from rocm-core's header; None when it has none."""
+    try:
+        text = (Path(root) / "include" / "rocm-core" / "rocm_version.h").read_text()
+        return tuple(int(re.search(rf"#define\s+ROCM_VERSION_{k}\s+(\d+)", text).group(1)) for k in ("MAJOR", "MINOR"))
+    except (OSError, AttributeError, ValueError):
+        return None
+
+
+def rocm_root(arch):
+    """ROCm for compiling and running the HIP engine for `arch`: (root, library folders).  A system ROCm 7 with hipcc
+    and hipBLAS, else AMD's TheRock wheels (ROCM_VERSION, from the card family's index) installed into .venv."""
     sysroot = Path(os.environ.get("ROCM_PATH") or "/opt/rocm")
     if (sysroot / "bin" / "hipcc").exists() and list((sysroot / "lib").glob("libhipblas.so*")):
-        return sysroot, [str(sysroot / "lib")]
+        ver = rocm_version(sysroot)
+        if ver is None or ver >= ROCM_SYSTEM_MIN:
+            return sysroot, [str(sysroot / "lib")]
+        warn(f"the ROCm in {sysroot} is {ver[0]}.{ver[1]}; Strata needs {ROCM_SYSTEM_MIN[0]}.{ROCM_SYSTEM_MIN[1]} or "
+             "newer: using AMD's wheels in .venv instead")
+    index = rocm_index(arch)
     stamp = Path(sys.prefix) / ".strata-rocm.json"
-    if not stamp.exists() or json.loads(stamp.read_text()).get("version") != ROCM_VERSION:
+    have = json.loads(stamp.read_text()) if stamp.exists() else {}
+    if have.get("version") != ROCM_VERSION or have.get("index") != index:
         say(f"  Installing ROCm {ROCM_VERSION} for AMD GPUs into .venv (AMD's TheRock wheels, ~10 GB, no sudo) ...")
-        run([sys.executable, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", "--index-url",
-             ROCM_INDEX, f"rocm[libraries,devel]=={ROCM_VERSION}"])
-        stamp.write_text(json.dumps({"version": ROCM_VERSION, "index": ROCM_INDEX}))
+        pip = [sys.executable, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", "--index-url", index]
+        if have.get("version") == ROCM_VERSION:        # the same version for another GPU family: its own libraries
+            run(pip + ["--force-reinstall", "--no-deps", f"rocm=={ROCM_VERSION}"])
+        run(pip + [f"rocm[libraries,devel]=={ROCM_VERSION}"])
+        stamp.write_text(json.dumps({"version": ROCM_VERSION, "index": index}))
     sdk = Path(sys.executable).parent / "rocm-sdk"
     root = Path(out([str(sdk), "path", "--root"]).strip())
     if not (root / "llvm" / "bin" / "clang++").exists():
         fail(f"ROCm was installed but its compiler is missing ({root})",
              f"remove {stamp} and run this again; or install ROCm 7 system-wide")
+    # the card family's libraries only (gfx120X-all -> _rocm_sdk_libraries_gfx120X_all): another family's
+    # libhipblaslt.so first on the path would have no kernels for this card
+    family = "_rocm_sdk_libraries_" + index.rstrip("/").rsplit("/", 1)[-1].replace("-", "_")
     dirs = [str(root / "lib")]
     for sp in {Path(p) for p in sys.path if p.endswith("site-packages")}:
-        dirs += [str(d / "lib") for d in sorted(sp.glob("_rocm_sdk_libraries_*")) if (d / "lib").is_dir()]
+        libs = sorted(sp.glob("_rocm_sdk_libraries_*"))
+        libs = [d for d in libs if d.name.lower() == family.lower()] or libs
+        dirs += [str(d / "lib") for d in libs if (d / "lib").is_dir()]
     ok(f"ROCm: {root}")
     return root, list(dict.fromkeys(dirs))
 
 
+def hipblaslt_version(lib_dirs):
+    """The installed hipBLASLt's version as the engine reads it (hipblasLtGetVersion: 1.4.1 -> 100401), from the
+    header of the ROCm whose libraries the engine loads; None when not found."""
+    for d in lib_dirs:
+        try:
+            text = (Path(d).parent / "include" / "hipblaslt" / "hipblaslt-version.h").read_text()
+            v = [int(re.search(rf"#define\s+HIPBLASLT_VERSION_{k}\s+(\d+)", text).group(1))
+                 for k in ("MAJOR", "MINOR", "PATCH")]
+        except (OSError, AttributeError, ValueError):
+            continue
+        return v[0] * 100000 + v[1] * 100 + v[2]
+    return None
+
+
+def hipblaslt_table(arch, lib_dirs):
+    """tools/hip/<arch>-hipblaslt-<version>.txt for this card AND the installed hipBLASLt, else None: its solution
+    ids are valid only for that pair (the engine refuses any other table and uses plain hipBLAS)."""
+    ver = hipblaslt_version(lib_dirs)
+    table = ROOT / "tools" / "hip" / f"{arch}-hipblaslt-{ver}.txt"
+    if ver is not None and table.exists():
+        head = table.read_text().split("\n", 2)[:2]
+        if f"STRATA_HIPBLASLT_TUNING_V1 {arch} {ver}" in (h.strip() for h in head):
+            ok(f"hipBLASLt tuning table: {table.name} (faster prompts)")
+            return table
+    have = sorted(p.name for p in (ROOT / "tools" / "hip").glob(f"{arch}-hipblaslt-*.txt"))
+    warn(f"no hipBLASLt tuning table for {arch} with hipBLASLt {ver or '(version unknown)'}"
+       + (f" (have: {', '.join(have)})" if have else "")
+       + ": the prompt's dense matrix products use plain hipBLAS (tools/hip/tune_hipblaslt makes a table: "
+         "docs/AMD_HIP.md, Tuning table)")
+    return None
+
+
 def build_engine_hip(gpu, llama) -> Path:
     """Compile the HIP engine for this AMD GPU into engine/ (again only when its source changed: a `git pull`)."""
     eng = ROOT / "engine"
@@ -726,13 +853,13 @@ def build_engine_hip(gpu, llama) -> Path:
     stamp = eng / "BUILD.json"
     meta = json.loads(stamp.read_text()) if stamp.exists() else {}
     src = source_hash(ENGINE_SOURCES)
-    if meta.get("backend") == "hip" and (eng / EXE).exists() and meta.get("src") == src:
+    if meta.get("backend") == "hip" and (eng / EXE).exists() and meta.get("src") == src and             gpu["arch"] in meta.get("archs", []):
         ok("engine already built for this PC")
         return eng
     if not (shutil.which("c++") or shutil.which("g++")) or not shutil.which("git"):
         fail("a C++ compiler and git are needed to compile the AMD engine",
              "Ubuntu/Debian: sudo apt install build-essential git   Fedora: sudo dnf install gcc-c++ git")
-    root, dirs = rocm_root()
+    root, dirs = rocm_root(gpu["arch"])
     libs = [str(Path(d).parent) for d in dirs[1:]]
     bitcode = next((p for p in (root / "lib" / "llvm" / "amdgcn" / "bitcode", root / "amdgcn" / "bitcode") if p.is_dir()),
                    root / "amdgcn" / "bitcode")
@@ -741,7 +868,7 @@ def build_engine_hip(gpu, llama) -> Path:
     os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(dirs + [os.environ.get("LD_LIBRARY_PATH", "")]).rstrip(os.pathsep)
     os.environ["PATH"] = os.pathsep.join([str(root / "bin"), str(root / "llvm" / "bin"), os.environ.get("PATH", "")])
     say("  The engine's source changed: compiling it again (only what changed, a few minutes) ..."
-        if meta.get("backend") == "hip" and (eng / EXE).exists()
+        if meta.get("backend") == "hip" and (eng / EXE).exists() and gpu["arch"] in meta.get("archs", [])
         else "  Compiling the Strata engine for your AMD GPU (10-20 minutes, once) ...")
     cmake_build(ROOT, ROOT / "build-hip", "strata",
                 ["-DSTRATA_ENABLE_HIP=ON", "-DSTRATA_ENABLE_CUDA=OFF", "-DSTRATA_BUILD_TESTS=OFF",
@@ -773,7 +900,16 @@ def get_prebuilt(url_base, gpu, vision, updating=False) -> Path | None:
     if info.exists() and (eng / EXE).exists():
         meta = json.loads(info.read_text())
         ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:3] if x.isdigit())
-        if meta.get("source") == "local" or ver >= MIN_ENGINE:
+        if meta.get("source") == "local":              # compiled here: build_engine checks its source and cards
+            return None
+        have = [int(a) for a in meta.get("archs", [])]
+        miss = [int(x) for x in gpu.get("archs", [gpu["arch"]])
+                if have and int(x) not in have and not (meta.get("ptx") and int(x) > max(have))]
+        if miss:                                       # a card it has no code for (#128): compiled here instead
+            warn(f"the installed engine is built for {', '.join(str(a) for a in have)}; your GPU is "
+                 f"{', '.join(str(x) for x in miss)}: compiling instead")
+            return None
+        if ver >= MIN_ENGINE:
             ok("ready-made engine already installed")
             return eng
         say(f"  Updating the ready-made engine ({meta.get('version')} -> {'.'.join(map(str, MIN_ENGINE))} or newer) ...")
@@ -846,7 +982,8 @@ def update_installed_engine(url_base) -> None:
     if meta.get("backend") == "hip":                   # AMD: compiled here, again when its source changed
         if meta.get("src") != source_hash(ENGINE_SOURCES):
             try:
-                g = next((x for x in amd_gpus() if amd_problem(x) is None), None)
+                usable = [x for x in amd_gpus() if amd_problem(x) is None]
+                g = next((x for x in usable if x["arch"] in meta.get("archs", [])), usable[0] if usable else None)
                 if g is None:
                     raise RuntimeError("no supported AMD GPU found")
                 build_engine_hip(g, get_llama_cpp())
@@ -898,7 +1035,8 @@ def update_installed_engine(url_base) -> None:
 def install_build_tools(gpu, yes):
     """The compiler and the CUDA toolkit, installed for the user (asks once).  Returns (nvcc, vcvars)."""
     nvcc, cuda_v = find_nvcc()
-    need_cuda = (12, 8) if max(int(x) for x in gpu.get("archs", [gpu["arch"]])) >= 120 else (12, 0)
+    # RTX 50 (sm_120): CUDA 13.0 - an engine built with 12.8 crashed in the prompt path on Linux (#220)
+    need_cuda = (13, 0) if max(int(x) for x in gpu.get("archs", [gpu["arch"]])) >= 120 else (12, 0)
     vcvars = find_vcvars() if WIN else None
     have_cc = vcvars is not None if WIN else shutil.which("g++") is not None
     missing = []
@@ -1004,16 +1142,24 @@ def build_engine(gpu, vision, yes, llama) -> Path:
     want_vision = vision != "none"
     local = meta.get("source") == "local"
     src, vsrc = source_hash(ENGINE_SOURCES), source_hash(VISION_SOURCES)
-    engine_ok = local and (eng / EXE).exists() and meta.get("src") == src
+    archs = sorted({int(x) for x in gpu.get("archs", [gpu["arch"]])})    # every card the model runs on
+    built = {int(x) for x in meta.get("archs", [])}
+    # a card the engine has no code for (a GPU added with --gpus, #128) needs a compile even when the source is the
+    # same; the compile keeps the generations it was built for
+    new_arch = local and not set(archs) <= built
+    engine_ok = local and (eng / EXE).exists() and meta.get("src") == src and not new_arch
     vision_ok = not want_vision or ((eng / VEXE).exists() and (not local or meta.get("vision_src") == vsrc))
     if engine_ok and vision_ok:
         ok("engine already built for this PC")
         return eng
-    nvcc, vcvars = install_build_tools(gpu, yes)
-    archs = sorted({int(x) for x in gpu.get("archs", [gpu["arch"]])})    # every card the model runs on
+    if local:
+        archs = sorted(built | set(archs))
+    nvcc, vcvars = install_build_tools({**gpu, "archs": archs}, yes)
     cuda_archs = ";".join(str(x) for x in archs)
     if not engine_ok:
-        say("  The engine's source changed: compiling it again (only what changed, a few minutes) ..."
+        say("  Compiling the engine for " + ", ".join(f"sm_{x}" for x in archs) + " (a card it had no code for; "
+            "10-20 minutes, once) ..." if new_arch else
+            "  The engine's source changed: compiling it again (only what changed, a few minutes) ..."
             if local and (eng / EXE).exists() else "  Compiling the Strata engine for your GPU (10-20 minutes, once) ...")
         cmake_build(ROOT, ROOT / "build", "strata",
                     ["-DSTRATA_ENABLE_CUDA=ON", "-DSTRATA_BUILD_TESTS=OFF", f"-DCMAKE_CUDA_ARCHITECTURES={cuda_archs}",
@@ -1043,6 +1189,44 @@ def build_engine(gpu, vision, yes, llama) -> Path:
 DATA_ITEMS = ("models", "packs", "mtp")
 
 
+LOW_RAM_HEADROOM_GB = 10   # RAM beside the experts: the OS, the engine's other buffers, the server
+RESIDENT_ENGINE = (0, 1, 30)   # the first engine with --resident-experts (the low-RAM mode's resident variant)
+
+
+def low_ram_needed(model, ram) -> bool:
+    """The model's experts do not fit this PC's RAM with room left for the rest: they are then mapped from the pack's
+    experts.bin instead of copied into RAM (the low-RAM mode)."""
+    return ram < MODELS[model]["arena_gb"] + LOW_RAM_HEADROOM_GB
+
+
+def low_ram_gpu_gb(model, vram_gb, ctx=32768, kv="int8") -> float:
+    """About how many GB of the model's experts the GPU's cache holds: its VRAM minus ~5 GB for the dense weights,
+    buffers and a 32K context's KV cache, minus the KV cache of a longer context (in VRAM in the low-RAM mode: its RAM
+    has no room for KV streaming)."""
+    kv_tok = 13 * (576 if kv == "q4_0" else 1056)       # bytes per context token: 12 QSA layers + the draft layer
+    longer = max(0, ctx - 32768) * kv_tok / 1e9
+    return max(0.0, min(MODELS[model]["arena_gb"], vram_gb - 5 - longer))
+
+
+def low_ram_gpu_share(model, vram_gb, ctx=32768, kv="int8") -> float:
+    """About how much of the model's experts the GPU holds."""
+    return low_ram_gpu_gb(model, vram_gb, ctx, kv) / MODELS[model]["arena_gb"]
+
+
+def low_ram_resident(model, ram, vram_gb, ctx=32768, kv="int8") -> bool:
+    """In the low-RAM mode: the experts the GPU does not hold fit the RAM with the usual room beside them, so they are
+    copied into RAM once (the resident variant, `--resident-experts`) instead of being read through the OS file cache
+    (plain `--mmap-experts`, which a PC this short of RAM keeps re-reading from the SSD)."""
+    rest = MODELS[model]["arena_gb"] - low_ram_gpu_gb(model, vram_gb, ctx, kv)
+    return ram >= rest + LOW_RAM_HEADROOM_GB
+
+
+def low_ram_fits(model, ram, vram_gb) -> bool:
+    """In the low-RAM mode: the experts the GPU does not hold fit the RAM left beside the rest (as file cache)."""
+    arena = MODELS[model]["arena_gb"]
+    return ram - 6 + max(0.0, vram_gb - 5) >= arena
+
+
 def settings_path() -> Path:
     if WIN:
         return Path(os.environ.get("APPDATA") or Path.home() / "AppData" / "Roaming") / "Strata" / "settings.json"
@@ -1165,7 +1349,18 @@ def data_folder(requested: str | None) -> tuple:
         warn(f"cannot use {dest} for the model files ({e}): keeping them in {ROOT}")
         dest = ROOT
     elsewhere = []
-    for folder in [ROOT, *other_installs(settings)]:
+    # #198: the data folder remembered before (a --data-dir to a new place) is a source too, and so is a Strata-data
+    # folder nested in any of them (an install that kept its models one level down)
+    sources = [ROOT, *other_installs(settings)]
+    if settings.get("data_dir") and Path(settings["data_dir"]) != dest:
+        sources.append(Path(settings["data_dir"]))
+    sources += [f / "Strata-data" for f in list(sources) if (f / "Strata-data") != dest]
+    seen = set()
+    for folder in sources:
+        key = os.path.normcase(str(folder))
+        if key in seen:
+            continue
+        seen.add(key)
         if folder == dest or not has_data(folder):
             continue
         if not same_drive(folder, dest):
@@ -1338,12 +1533,21 @@ def upgrade_config(cfg_path: Path, cfg: dict) -> dict:
 
 
 def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_browser=True, yes=False,
-          layer_split=None) -> int:
+          layer_split=None, keep=None) -> int:
+    """keep: settings given on this start that the model keeps from now on (--host, --api-key, --draft-vocab)."""
     cfg = upgrade_config(cfg_path, json.loads(cfg_path.read_text(encoding="utf-8-sig")))
     missing = [p for p in [cfg["exe"], *[a for a in cfg["args"] if a.endswith(".gguf")]] if not Path(p).exists()]
     if missing:
         fail(f"{cfg_path.name} refers to missing files: {missing[0]}", "run it again with --setup to repair")
+    keep = {k: v for k, v in (keep or {}).items() if v is not None}
+    if keep and any(cfg.get(k) != v for k, v in keep.items()):   # #179: a --host/--api-key on a start was ignored
+        cfg.update(keep)
+        cfg_path.write_text(json.dumps(cfg, indent=1), encoding="utf-8")
+        ok("saved for this model: " + ", ".join("api key" if k == "api_key" else f"{k.replace('_', ' ')} {v}"
+                                                for k, v in keep.items()))
     cfg_path.touch()                                     # the most recently used model
+    if "--mtp" in cfg["args"][:-1]:
+        refresh_draft_vocab(Path(cfg["args"][cfg["args"].index("--mtp") + 1]), cfg.get("draft_vocab", "cjk"))
     cmd = [sys.executable, str(ROOT / "serve" / "server.py"), "--engine", "strata", "--config", str(cfg_path),
            "--port", str(port or cfg.get("port", 8080))]
     if cfg.get("backend") == "hip":                    # AMD: one card, numbered as HIP numbers them
@@ -1378,11 +1582,13 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
     elif isinstance(use, list):
         check_gpus(use, found, "(chosen for this model) ")
         byid = {g["index"]: g for g in found}
+        cfg = ensure_engine_for([byid[i] for i in use], cfg_path, cfg, yes)
         ok("GPUs: " + " + ".join(gpu_name(byid[i]) for i in use) + f" together (layers split {cfg.get('layer_split') or 'auto'})")
     elif found:
         g = next((x for x in found if x["index"] == use), None) if use is not None else max(
             found, key=lambda x: (round(x["vram_gb"]), -x["index"]))
         if g is not None:
+            cfg = ensure_engine_for([g], cfg_path, cfg, yes)
             ok("GPU: " + gpu_name(g))
     if open_browser:
         cmd.append("--open")
@@ -1400,16 +1606,68 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
     say("  restart). That is normal: please wait and don't close this window - the browser opens when it is ready.")
     say("  Later, closing this window stops the model.")
     say("  " + "-" * 100)
+    if not WIN and os.environ.get("STRATA_EXECV"):
+        # Replace this process instead of spawning a child. The Docker image sets STRATA_EXECV=1,
+        # so there the server is PID 1 and docker stop's SIGTERM reaches the process that can
+        # answer the engine with QUIT. Normal Linux starts keep spawning the server as a child.
+        os.execv(cmd[0], cmd)
     return subprocess.call(cmd)
 
 
+# the draft subsets setup copied before (sha256): replaced by the current one, a subset made by hand is kept
+OLD_DRAFT_VOCABS = {"369151522226a5edaa5f12cfd1e2ae7db8f4fbdbd222f3dcf327dced9597fb25"}   # to 0.1.26: 27 Han tokens
+
+
+DRAFT_VOCABS = {"cjk": "draft_vocab.bin", "en": "draft_vocab_en.bin"}
+
+
+def refresh_draft_vocab(rt: Path, choice: str = "cjk") -> None:
+    """The draft layer's token subset in the MTP folder: `cjk` (data/draft_vocab.bin, since 0.1.27, #137) or `en`
+    (data/draft_vocab_en.bin, the English/code subset before it: ~110 MiB less VRAM, English answers 1-2% faster).
+    Copied when missing or when a shipped subset other than the chosen one is there; a subset made by hand is kept."""
+    new, dst = ROOT / "data" / DRAFT_VOCABS.get(choice, "draft_vocab.bin"), rt / "draft_vocab.bin"
+    if not new.exists() or not rt.is_dir():
+        return
+    if dst.exists():
+        old = hashlib.sha256(dst.read_bytes()).hexdigest()
+        shipped = OLD_DRAFT_VOCABS | {hashlib.sha256((ROOT / "data" / f).read_bytes()).hexdigest()
+                                      for f in DRAFT_VOCABS.values() if (ROOT / "data" / f).exists()}
+        if old not in shipped or old == hashlib.sha256(new.read_bytes()).hexdigest():
+            return
+        ok("draft layer: the token subset " + ("with Chinese, Japanese and Korean" if choice == "cjk" else
+                                               "for English and code (less VRAM)"))
+    shutil.copyfile(new, dst)
+
+
+def ensure_engine_for(cards, cfg_path: Path, cfg: dict, yes: bool) -> dict:
+    """The installed engine must have code for every card the model starts on: a card added later (--gpus with an
+    older or newer generation, #128) or a new GPU in the PC otherwise stops the start with 'no kernel image'.  Such a
+    card gets the engine compiled for all of them, before the start."""
+    missing = [g for g in cards if not engine_runs_on(g)]
+    if not missing:
+        return cfg
+    info = ROOT / "engine" / "BUILD.json"
+    meta = json.loads(info.read_text())
+    say()
+    say("  The installed engine has no code for " + ", ".join(f"{g['name']} (sm_{g['arch']})" for g in missing) +
+        ": it is compiled for " + ("these cards" if len(cards) > 1 else "it") + " now.")
+    main = gpu_info(cards[0]["index"])
+    archs = sorted({int(x) for x in meta.get("archs", [])} | {int(g["arch"]) for g in cards})
+    vision = meta.get("vision") or ("gpu" if (ROOT / "engine" / VEXE).exists() else "none")
+    build_engine({**main, "archs": archs}, vision, yes, get_llama_cpp())
+    dirs = json.loads(info.read_text()).get("cuda_dirs") or []
+    cfg["lib_dirs"] = dirs + [d for d in cfg.get("lib_dirs") or [] if d not in dirs]
+    cfg_path.write_text(json.dumps(cfg, indent=1), encoding="utf-8")
+    return cfg
+
+
 def write_run_script(model, cfg_path, port):
     serve = [sys.executable, str(ROOT / "serve" / "server.py"), "--engine", "strata", "--config", str(cfg_path),
              "--port", str(port), "--open"]
     if WIN:
         script = ROOT / f"run-{model.lower()}.bat"
         script.write_text("@echo off\r\ntitle Strata " + model + "\r\ncd /d \"" + str(ROOT) + "\"\r\n" +
-                          " ".join(f'"{x}"' for x in serve) + "\r\npause\r\n", encoding="utf-8")
+                          " ".join(f'"{x}"' for x in serve) + "\r\nif errorlevel 1 pause\r\n", encoding="utf-8")
     else:
         script = ROOT / f"run-{model.lower()}.sh"
         script.write_text("#!/bin/sh\ncd \"" + str(ROOT) + "\"\nexec " + " ".join(f'"{x}"' for x in serve) + "\n",
@@ -1418,12 +1676,56 @@ def write_run_script(model, cfg_path, port):
     return script
 
 
+# ------------------------------------------------------------------------------------------------ the rope config
+def derived_factor(ctx: int, trained: int = 262144) -> float:
+    """The automatic extension factor: the FINAL context over the trained one, at least 1.
+
+    Factor 1 removes the automatic expansion - the trained angles stand as they are - but it is not a
+    switch for rope as a whole: an explicitly chosen method's settings keep their defined behavior.
+    """
+    return max(1.0, float(ctx) / float(trained))
+
+
+def resolve_rope(ctx: int, scaling, scale, trained: int = 262144):
+    """The rope config for the context ACTUALLY SERVED: (scaling, scale); scaling None = no scaling flags.
+
+    An explicit --rope-scaling/--rope-scale always wins - a user-supplied factor is kept verbatim even
+    when a reduction changed the context.  Past the trained range an omitted method defaults to yarn -
+    llama.cpp's extension method: the trained angles survive on the high-frequency pairs and the
+    magnitude correction keeps the attention temperature - and an omitted factor is derived from the
+    final context (final / trained, at least 1), as is an explicitly chosen method's missing factor
+    inside the trained range: factor 1, the trained angles, no expansion.  An explicit none is refused
+    past the trained range (the setup will not configure a run it knows is out of spec) rather than
+    silently overridden.
+    """
+    if ctx <= trained:
+        if scale is not None and scaling in (None, "none"):
+            raise ValueError("--rope-scale needs --rope-scaling linear or yarn (the chosen context fits the "
+                             "trained 262144, so there is nothing to scale)")
+        if scaling in (None, "none"):
+            return None, None          # the stock model, by choice or by default
+        return scaling, scale if scale is not None else derived_factor(ctx, trained)
+    if scaling == "none":
+        raise ValueError(f"a {ctx // 1024}K context is past the model's trained 262144, and --rope-scaling none "
+                         "keeps the stock angles there - the model has never seen those positions, so the setup "
+                         "refuses the combination instead of quietly overriding it. Pick --rope-scaling yarn or "
+                         "linear, or rerun with --context 262144 or lower")
+    return scaling or "yarn", scale if scale is not None else derived_factor(ctx, trained)
+
+
 # ------------------------------------------------------------------------------------------------ main
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument("--family", choices=list(FAMILIES), help="qwen = Qwen3.8-Flash-Next, swift = Swift 1.5")
     ap.add_argument("--model", choices=list(MODELS))
     ap.add_argument("--context", type=int)
+    ap.add_argument("--rope-scaling", choices=["none", "linear", "yarn"],
+                    help="the RoPE extension for a context past the model's trained 262144: linear (position "
+                         "interpolation) or yarn - llama.cpp's types. Omitted with a scaled context, the setup "
+                         "picks yarn; 'none' is refused for such a context")
+    ap.add_argument("--rope-scale", type=float,
+                    help="the extension factor (default: the final context over the trained 262144, at least 1 - "
+                         "1.5 for 384K, 2 for 512K, 1 inside the trained range)")
     ap.add_argument("--kv", choices=["int8", "q4_0", "k8v4"],
                     help="KV cache precision above 8K context: int8 (default), q4_0 (half the memory, a little less "
                          "precise) or k8v4 (hybrid: INT8 K + 4-bit V, 816 B/cell)")
@@ -1456,8 +1758,16 @@ def main() -> int:
     ap.add_argument("--check", action="store_true", help="only check this PC and exit")
     ap.add_argument("--calibrate", action="store_true",
                     help="tune the engine's settings for this PC (about 5-10 minutes), then start the model")
+    ap.add_argument("--draft-vocab", choices=list(DRAFT_VOCABS),
+                    help="the draft layer's tokens: cjk = with Chinese, Japanese and Korean (default), en = English "
+                         "and code only (~110 MiB less VRAM, English answers 1-2%% faster)")
+    ap.add_argument("--low-ram", choices=["auto", "on", "off", "resident", "mmap"], default="auto",
+                    help="read the model's experts from one file in its folder instead of copying them all into RAM "
+                         "(for a PC with a big GPU and little RAM); auto: when the experts would not fit the RAM. In "
+                         "this mode the experts the GPU does not hold are copied into RAM once when they fit (resident), "
+                         "else read through the OS file cache (mmap); resident / mmap force one of the two")
     ap.add_argument("--backend", choices=["cuda", "hip"],
-                    help="cuda = NVIDIA (default), hip = AMD RX 7900 XT/XTX on Linux (experimental; chosen by itself "
+                    help="cuda = NVIDIA (default), hip = AMD RX 7900 XT/XTX or RX 9070 / AI PRO R9700 on Linux (experimental; chosen by itself "
                          "when the PC has no NVIDIA card Strata can use)")
     ap.add_argument("--skip-build", action="store_true", help=argparse.SUPPRESS)
     a = ap.parse_args()
@@ -1512,35 +1822,50 @@ def main() -> int:
                 say(f"  {i}) {json.loads(c.read_text(encoding='utf-8-sig')).get('model_name', c.stem)}")
             pick_cfg = have[int(ask("Tune which one?", [str(i) for i in range(1, len(have) + 1)], "1", a.yes)) - 1]
         calibrate_config(pick_cfg)
-        return 0 if a.no_start else start(pick_cfg, a.port, run_gpu, yes=a.yes, layer_split=a.layer_split)
+        return 0 if a.no_start else start(pick_cfg, a.port, run_gpu, yes=a.yes, layer_split=a.layer_split,
+                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab})
     if have and not (a.setup or a.model or a.family or a.check or a.no_start):
         if not a.build:
             update_installed_engine(a.prebuilt)
         if len(have) == 1:
-            return start(have[0], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split)
+            return start(have[0], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split,
+                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab})
         say()
         for i, c in enumerate(have, 1):
             say(f"  {i}) {json.loads(c.read_text(encoding='utf-8-sig')).get('model_name', c.stem)}")
         say(f"  {len(have) + 1}) install another model / change settings")
         pick = int(ask("Which one?", [str(i) for i in range(1, len(have) + 2)], "1", a.yes))
         if pick <= len(have):
-            return start(have[pick - 1], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split)
+            return start(have[pick - 1], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split,
+                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab})
 
     # ---- 1. the PC
     step(1, "checking your PC")
     found = gpus()
     amd = [] if WIN else amd_gpus()
-    hip = a.backend == "hip" or (a.backend is None and not any(gpu_problem(g) is None for g in found)
-                                 and any(amd_problem(g) is None for g in amd))
+    nv_ok = any(gpu_problem(g) is None for g in found)
+    amd_ok = [g for g in amd if amd_problem(g) is None]
+    hip = a.backend == "hip" or (a.backend is None and not nv_ok and bool(amd_ok))
+    if a.backend is None and nv_ok and amd_ok:
+        # both kinds of card: asked (a first run on such a PC used to take NVIDIA without mentioning the Radeon)
+        say()
+        say("  This PC has NVIDIA and AMD cards Strata can use:")
+        say("  1) NVIDIA: " + ", ".join(f"{g['name']} ({g['vram_gb']:.0f} GB)" for g in found if gpu_problem(g) is None)
+            + "   (recommended)")
+        say("  2) AMD: " + ", ".join(f"{g['name']} ({g['vram_gb']:.0f} GB)" for g in amd_ok)
+            + "   (experimental: compiled here, one GPU, no images - docs/AMD_HIP.md)")
+        hip = ask("Which cards?", ["1", "2"], "1", a.yes or a.check) == "2"
+        if a.check and not hip:
+            say("  (the AMD card: ./setup.sh --backend hip)")
     if hip:                                            # AMD (experimental): one card, compiled here
         if WIN:
-            fail("Strata's AMD backend runs on Linux only", "use an NVIDIA RTX 30 series or newer card on Windows")
+            fail("Strata's AMD backend runs on Linux only", "use an NVIDIA RTX 20 series or newer card on Windows")
         say("  Your AMD GPUs:" if amd else "  No AMD GPU found (the amdgpu driver's KFD topology is empty).")
         for g in amd:
             say(f"    GPU {g['index']}: {g['name']}, {g['vram_gb']:.0f} GB VRAM - " + (amd_problem(g) or "can be used"))
         usable = [g for g in amd if amd_problem(g) is None]
         if not usable:
-            fail("no AMD GPU Strata can use", "the AMD backend runs on the RX 7900 XT / XTX (gfx1100) on Linux")
+            fail("no AMD GPU Strata can use", f"the AMD backend runs on {AMD_CARDS} on Linux")
         if a.gpus:
             fail("several GPUs sharing one model: NVIDIA only for now", "use one AMD card (--gpu N)")
         if a.gpu is not None:
@@ -1557,7 +1882,7 @@ def main() -> int:
         if not found:
             fail("no NVIDIA GPU found (nvidia-smi did not answer)",
                  "install the NVIDIA driver from https://www.nvidia.com/drivers and restart the PC"
-                 + ("; an AMD RX 7900 XT/XTX: --backend hip" if amd else ""))
+                 + ("; an AMD RX 7900 XT/XTX or RX 9070 / AI PRO R9700: --backend hip" if amd else ""))
         if len(found) > 1 or gpu_problem(found[0]) is not None:
             gpu_table(found)
         sel = choose_gpus(a, found)                    # asked when two or more cards can share the model
@@ -1578,13 +1903,15 @@ def main() -> int:
     ram = ram_gb()
     cpu, avx2, avx512 = cpu_info()
     need = min(d["ram_gb"] for d in MODELS.values())
-    if ram < need - 4 and not a.check:
+    low_ok = low_ram_fits("IQ1_M", ram, gpu["vram_gb"]) and a.low_ram != "off"   # the smallest model, mapped
+    if ram < need - 4 and not a.check and not low_ok:
         # every model keeps ALL its experts in RAM (23+ GB); VRAM only holds a copy of the most-used ones, so a
         # bigger GPU does not lower this
         fail(f"RAM: {ram:.0f} GB - the smallest model (the Coder) needs about {need} GB",
              "Strata keeps all of the model's experts in RAM (23-50 GB, whatever the GPU) and the GPU holds a copy "
              "of the most-used ones: it needs 32 GB of RAM or more (48 GB for the full model)")
-    ok(f"RAM: {ram:.0f} GB" if ram >= need - 4 else f"RAM: {ram:.0f} GB (less than the {need} GB the smallest model needs)")
+    ok(f"RAM: {ram:.0f} GB" if ram >= need - 4 else f"RAM: {ram:.0f} GB (less than the {need} GB the smallest model needs)"
+       + ("; the GPU's VRAM makes up for it (the low-RAM mode)" if ram < need - 4 and low_ok else ""))
     pf = page_file_gb()
     if pf is not None and pf < 4:
         warn(f"Windows' page file is {pf:.1f} GB: the graphics card's memory needs room there too (issue #60), so "
@@ -1597,6 +1924,10 @@ def main() -> int:
         say()
         for m, d in MODELS.items():
             verdict = "fits" if ram >= d["ram_gb"] else "tight" if ram >= d["ram_gb"] - 8 else "does not fit"
+            if low_ram_needed(m, ram) and low_ram_fits(m, ram, gpu["vram_gb"]) and a.low_ram != "off":
+                verdict = (f"fits in the low-RAM mode (the GPU holds ~{100 * low_ram_gpu_share(m, gpu['vram_gb']):.0f}% "
+                           "of its experts, " + ("the rest stays in RAM)" if low_ram_resident(m, ram, gpu["vram_gb"])
+                                                 else "the rest is read from the SSD as needed)"))
             say(f"  {m:8s} needs ~{d['ram_gb']} GB RAM: {verdict}")
         say("\nThis PC can run Strata. Run it again without --check to install.")
         return 0
@@ -1622,10 +1953,18 @@ def main() -> int:
     for i, m in enumerate(names, 1):
         d = MODELS[m]
         fit = "" if ram >= d["ram_gb"] else f"   <- needs {d['ram_gb']} GB RAM, you have {ram:.0f}"
+        if low_ram_needed(m, ram) and low_ram_fits(m, ram, gpu["vram_gb"]) and a.low_ram != "off":
+            fit = (f"   <- fits in the low-RAM mode (the GPU holds ~{100 * low_ram_gpu_share(m, gpu['vram_gb']):.0f}%, "
+                   + ("the rest in RAM)" if low_ram_resident(m, ram, gpu["vram_gb"]) else "the rest from the SSD)"))
         say(f"  {i}) {m:8s} {d['about']}; download {d['download_gb']:.0f} GB, uses ~{d['arena_gb']:.0f} GB of RAM{fit}")
     rec = str(names.index("IQ3_XXS") + 1) if ram >= 60 and "IQ3_XXS" in names else "1"
     model = a.model or names[int(ask("Which size?", [str(i) for i in range(1, len(names) + 1)], rec, a.yes)) - 1]
-    if ram < MODELS[model]["ram_gb"] - 4:
+    low_ram = a.low_ram in ("on", "resident", "mmap") or (a.low_ram == "auto" and low_ram_needed(model, ram))
+    if low_ram and multi:
+        warn("the low-RAM mode runs on one GPU: using " + gpu_name(gpu) + " only")
+        multi, sel, chosen = [], [gpu["index"]], [gpu]
+    # (the low-RAM mode's variant is decided once the context is known, below)
+    if not low_ram and ram < MODELS[model]["ram_gb"] - 4:
         # #125: a warning and a question, not a stop: the user may accept paging (asked, "no" by default, so an
         # unattended --yes install still stops here)
         need_gb, arena = MODELS[model]["ram_gb"], MODELS[model]["arena_gb"]
@@ -1648,12 +1987,43 @@ def main() -> int:
         say("  Context length = how much text the model can see at once (your chat, files, tool output).")
         say("  Longer needs more VRAM for it, so fewer experts fit on the GPU:")
         for i, c in enumerate(CONTEXTS, 1):
-            say(f"  {i}) {c // 1024}K tokens" + ("   (recommended for your GPU)" if c == rec_ctx else ""))
-        ctx = CONTEXTS[int(ask("Context?", [str(i) for i in range(1, 6)], str(CONTEXTS.index(rec_ctx) + 1), a.yes)) - 1]
-    if model in ("IQ3_XXS", "IQ3_S") and ram < 90 and ctx > 131072:
-        warn(f"{model} with a 262K context needs more than 64 GB of RAM ({MODELS[model]['arena_gb']:.0f} GB of experts "
-             "+ the context): using 128K")
+            note = ("   (recommended for your GPU)" if c == rec_ctx else "") + \
+                   ("   (experimental: setup adds rope scaling)" if c > 262144 else "")
+            say(f"  {i}) {c // 1024}K tokens{note}")
+        ctx = CONTEXTS[int(ask("Context?", [str(i) for i in range(1, len(CONTEXTS) + 1)],
+                               str(CONTEXTS.index(rec_ctx) + 1), a.yes)) - 1]
+    # the RAM limit comes FIRST: the rope config must cover the context the engine will actually serve -
+    # a reduced 384K can land back inside the trained 262144, and then nothing needs scaling (§4).  Counted,
+    # not a fixed 90 GB (the 0.1.29 arithmetic): arena + the context's KV + 24 GB of room for everything else.
+    asked_ctx = ctx
+    need_gb = MODELS[model].get("arena_gb", 0) + ctx * 13 * 1056 / 1e9 + 24
+    if model in ("IQ3_XXS", "IQ3_S") and ram < need_gb and ctx > 131072:
+        warn(f"{model} with a {ctx // 1024}K context needs about {need_gb:.0f} GB of RAM "
+             f"({MODELS[model]['arena_gb']:.0f} GB of experts + the context + room for the rest): using 128K")
         ctx = 131072
+    scaling = a.rope_scaling
+    if ctx > 262144 and scaling is None and not a.yes:
+        # the interactive path: one question, yarn preselected (llama.cpp's extension method, recall-tested
+        # here at 512K).  With --yes nothing prints: resolve_rope takes yarn below and the ok() line says so.
+        say()
+        say(f"  A {ctx // 1024}K context runs the model past its trained 262,144 positions: the rotary angles")
+        say("  get rescaled (llama.cpp's RoPE extension). yarn keeps the trained angles on the high-frequency")
+        say("  pairs and corrects the magnitudes; linear shrinks every angle. Override any time with")
+        say("  --rope-scaling.")
+        scaling = ask("RoPE extension method?", ["yarn", "linear"], "yarn", a.yes)
+    try:
+        scaling, rope_scale = resolve_rope(ctx, scaling, a.rope_scale)
+    except ValueError as e:
+        fail(str(e))
+    if scaling is not None:
+        origin = ("final context / trained 262144; override with --rope-scale" if a.rope_scale is None
+                  else "as requested")
+        ok(f"rope scaling: {scaling}, factor {rope_scale:g} ({origin})")
+    elif asked_ctx > 262144:
+        ok(f"the RAM limit brings the context to {ctx // 1024}K, inside the trained 262,144: no rope scaling")
+    if scaling is not None and asked_ctx > 262144 and ctx <= 262144:
+        warn(f"the RAM limit reduced the context to {ctx // 1024}K, inside the trained 262,144; keeping your "
+             f"explicit rope scaling ({scaling}) anyway - drop --rope-scaling for the completely stock model")
     ok(f"context: {ctx} tokens")
     # the KV cache (the model's memory of the conversation): 8-bit, or 4-bit after a Hadamard rotation (PR #21)
     kv = "fp16" if ctx <= 8192 else (a.kv or "int8")
@@ -1678,6 +2048,27 @@ def main() -> int:
         say("  download and keeps ~1.4 GB of VRAM free for the image encoder, so text is a few % slower.")
         vision = "gpu" if ask("Do you want images?", ["y", "n"], "n", a.yes) == "y" else "none"
     ok("images: " + {"none": "off", "gpu": "on", "cpu": "on (encoder on the CPU)"}[vision])
+    # The low-RAM mode's two variants.  resident: the experts the GPU's cache does not hold (and, as far as RAM allows,
+    # the ones the prompt path borrows cache room from) are copied from the pack's experts.bin into RAM once, so
+    # nothing is read from the SSD while it answers (engine 0.1.30, --resident-experts; the engine falls back to mmap
+    # with a warning when they do not fit the RAM it finds free).  mmap: they are read through the OS file cache.
+    # The GPU's share: its VRAM less the dense weights and buffers, this context's KV cache and the image encoder's room.
+    resident = False
+    if low_ram:
+        arena = MODELS[model]["arena_gb"]
+        vram = gpu["vram_gb"] - (VISION[vision]["reserve_mib"] / 1024 if vision != "none" else 0)
+        share = low_ram_gpu_share(model, vram, ctx, kv)
+        rest = arena - low_ram_gpu_gb(model, vram, ctx, kv)
+        resident = a.low_ram == "resident" or (a.low_ram != "mmap" and low_ram_resident(model, ram, vram, ctx, kv))
+        if resident:
+            ok(f"low-RAM mode: the GPU holds ~{100 * share:.0f}% of {model}'s experts ({arena:.0f} GB) and the other "
+               f"~{rest:.0f} GB stay in RAM ({ram:.0f} GB), read once from a copy in the model folder")
+        else:
+            ok(f"low-RAM mode: {model}'s experts ({arena:.0f} GB) are read from the model folder through the OS file "
+               f"cache instead of a copy in RAM ({ram:.0f} GB); the GPU holds ~{100 * share:.0f}% of them")
+            if share < 0.6:
+                warn("most of the experts are read from the SSD while it answers: expect it to be much slower than "
+                     "with enough RAM (a faster SSD and a smaller size help)")
     # EXPERIMENTAL: the experimental-speed-projection control vector (data/experimental-speed-projection), off unless
     # chosen here; with it loaded, the web app and the API switch it off per request
     esp = None
@@ -1706,9 +2097,13 @@ def main() -> int:
                 models_dir, shards = cand[0].parent, cand
                 ok(f"model files found in {models_dir}")
                 break
+    for s in shards:                                   # #173: a whole file copied in by hand has no finish mark
+        if s.exists() and not done(s) and whole_shard(s):
+            mark(s, "whole (checked against its own tensor directory)")
     have_model = all(s.exists() and (done(s) or a.gguf_dir) for s in shards)
     need = (0 if a.gguf_dir or have_model else MODELS[model]["download_gb"]) + 8 + \
-        (40 if model == "Q2_0" and avx512 and family == "qwen" else 0) + (1 if vision != "none" else 0)
+        (40 if model == "Q2_0" and avx512 and family == "qwen" else 0) + (1 if vision != "none" else 0) + \
+        (MODELS[model]["arena_gb"] + 1 if low_ram and not (model == "Q2_0" and avx512 and family == "qwen") else 0)
     if free_gb(models_dir) < need:
         fail(f"not enough free disk space in {models_dir}: need ~{need:.0f} GB", "use --models-dir on a bigger drive")
 
@@ -1726,6 +2121,12 @@ def main() -> int:
         if vision != "none" and not (eng / VEXE).exists():
             warn("the ready-made engine has no image encoder: compiling it")
             eng = None
+        elif vision == "gpu":                          # the encoder can cover fewer cards than the engine (RTX 20)
+            m = json.loads((eng / "BUILD.json").read_text())
+            va = [int(x) for x in m.get("vision_archs", m.get("archs", []))]
+            if va and int(gpu["arch"]) not in va and not (m.get("ptx") and int(gpu["arch"]) > max(va)):
+                warn(f"the ready-made image encoder has no code for your GPU (sm_{gpu['arch']}): compiling it")
+                eng = None
     if eng is None:
         eng = build_engine_hip(gpu, llama) if hip else build_engine(gpu, vision, a.yes, llama)
     meta = json.loads((eng / "BUILD.json").read_text())
@@ -1779,6 +2180,10 @@ def main() -> int:
     elif not (pack / "native_experts.txt").exists() or not (pack / "tokenizer" / "vocab.json").exists():
         # every tensor as the GGUF stores it; the experts are read from the GGUF at start (seconds to build)
         run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(shards[0]), "--out", str(pack)], env=env)
+    if low_ram and not (pack / "experts.bin").exists():
+        say(f"  Writing the experts into one file for the low-RAM mode (one time, {MODELS[model]['arena_gb']:.0f} GB) ...")
+        run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(shards[0]), "--out", str(pack),
+             "--experts-bin"], env=env)
     ok(f"model prepared: {pack}")
     mtp = (find_in(roots, "mtp/rt/experts.bin") or data / "mtp/rt/experts.bin").parent.parent
     rt = mtp / "rt"
@@ -1790,8 +2195,7 @@ def main() -> int:
              "--out", str(mtp / "mtp-q2_0.gguf")], env=env)
         run([sys.executable, str(ROOT / "tools" / "mtp_rt.py"), "--gguf", str(mtp / "mtp-q2_0.gguf"), "--out", str(rt)],
             env=env)
-    if not (rt / "draft_vocab.bin").exists():
-        shutil.copyfile(ROOT / "data" / "draft_vocab.bin", rt / "draft_vocab.bin")
+    refresh_draft_vocab(rt, a.draft_vocab or "cjk")
     ok(f"MTP draft layer: {rt}")
 
     # ---- 7. the start script
@@ -1805,8 +2209,17 @@ def main() -> int:
             "--expert-profile", str(ROOT / "data" / fam.get("profile", "expert-profile.bin")), "--expert-cache", "auto",
             "--prefill", "auto", "--spec", "4", "--spec-min-p", "0.5", "--mtp", str(rt),
             "--max-context", str(ctx)]
+    if scaling is not None:     # the resolved config: explicit flags as given, or the automatic yarn+factor
+        args += ["--rope-scaling", scaling, "--rope-scale", f"{rope_scale:g}"]
     if ctx > 8192:
         args += ["--kv", kv]
+    engine_ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:3] if x.isdigit())
+    if resident and a.low_ram != "resident" and engine_ver < RESIDENT_ENGINE:
+        resident = False                               # an engine from before --resident-experts would refuse it
+        ok(f"low-RAM mode: engine {meta.get('version')} has no resident variant yet; the experts are read through "
+           "the OS file cache (run setup again after the next engine update)")
+    if low_ram:   # the experts from the pack's experts.bin: the ones the GPU does not hold copied into RAM, or mapped
+        args += ["--resident-experts" if resident else "--mmap-experts"]
     # KV streaming: from 64K up the whole KV cache lives in RAM and only the part the attention reads (32K positions
     # per layer) stays in VRAM; the VRAM it frees holds more experts (+6% at 128K, +23% at 262K with Q2_0). It
     # costs ~13.7 KB of RAM per context token with 8-bit KV (1.7 GB at 128K), 7.5 KB with 4-bit, so only when it fits.
@@ -1834,10 +2247,13 @@ def main() -> int:
     if hip:
         cfg["backend"] = "hip"
         # the dense prompt GEMMs through hipBLASLt with kernels measured on this GPU generation (tools/hip; +40-60%
-        # prompt speed on the 7900 XTX); the engine refuses a table made for another hipBLASLt version and falls back
-        tables = sorted((ROOT / "tools" / "hip").glob(f"{gpu['arch']}-hipblaslt-*.txt"))
-        if tables:
-            cfg["env"] = {"STRATA_HIPBLASLT_TUNING": str(tables[-1])}
+        # prompt speed on the 7900 XTX): only a table for this card's arch AND the installed hipBLASLt version (the
+        # engine refuses any other one and falls back to plain hipBLAS)
+        table = hipblaslt_table(gpu["arch"], lib_dirs)
+        if table:
+            cfg["env"] = {"STRATA_HIPBLASLT_TUNING": str(table)}
+        if resident:   # ROCm: large page-locked host allocations can fail or be slow for the CPU; keep the copy pageable
+            cfg.setdefault("env", {})["STRATA_RESIDENT_PIN"] = "0"
     if gpu["count"] > 1 or a.gpu is not None:
         cfg["gpu"] = gpu["index"]                      # the engine is told this card (issue #51)
         cfg["gpus_asked"] = True                       # chosen at setup: not asked again at start
@@ -1849,6 +2265,8 @@ def main() -> int:
         cfg["host"] = a.host
     if a.api_key:
         cfg["api_key"] = a.api_key
+    if a.draft_vocab:
+        cfg["draft_vocab"] = a.draft_vocab
     if vision != "none":
         cfg["vision"] = {"exe": str(eng / VEXE), "mmproj": str(mmproj), "model": str(shards[0]),
                          "gpu": vision == "gpu", "max_tokens": VISION[vision]["max_tokens"]}

src/core/conversation_cache_test.cpp

@@ -0,0 +1,152 @@
+#include "strata/core/conversation_cache.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+
+using namespace strata::core;
+
+namespace {
+int checks = 0;
+void check(bool value, const char* description) {
+    ++checks;
+    if (!value) { std::fprintf(stderr, "FAIL: %s\n", description); std::exit(1); }
+}
+SavedConversation image(std::initializer_list<int32_t> ids, bool cvec = true) {
+    SavedConversation s;
+    s.live.ids = ids;
+    s.live.gdn.resize(64, 7);
+    s.cvec = cvec;
+    return s;
+}
+}
+
+int main() {
+    {
+        ConversationBuffer bytes;
+        const size_t first = ConversationBuffer::segment_bytes + 17;
+        const size_t peak = bytes.allocation_peak(first);
+        bytes.resize(first, 7);
+        check(bytes.bytes() <= peak, "segmented payload and directory fit admitted bytes");
+        uint8_t* original = nullptr;
+        bytes.visit(0, 1, [&](uint8_t* p, size_t, size_t) { original = p; return true; });
+        const size_t grown_peak = bytes.allocation_peak(first + 99);
+        bytes.resize(first + 99, 9);
+        bytes.visit(0, 1, [&](uint8_t* p, size_t, size_t) {
+            check(p == original, "appending preserves existing payload addresses"); return true;
+        });
+        check(bytes.bytes() <= grown_peak, "growth including transient directory fits admission");
+        std::vector<uint8_t> tail(116);
+        check(bytes.read(tail.data(), ConversationBuffer::segment_bytes, tail.size()), "read spans segment boundaries");
+        check(std::all_of(tail.begin(), tail.begin()+17, [](auto v){return v==7;}) &&
+              std::all_of(tail.begin()+17, tail.end(), [](auto v){return v==9;}), "growth preserves prefix and initializes only suffix");
+        auto copy = bytes;
+        bytes.resize(ConversationBuffer::segment_bytes + 10);
+        bytes.resize(first, 7); bytes.resize(first + 99, 9);
+        check(bytes == copy, "equality ignores differing segmentation after rewind and regrowth");
+        check(!bytes.read(tail.data(), bytes.size()-1, 2), "range check rejects a truncated payload");
+        check(bytes.allocation_peak(SIZE_MAX) == SIZE_MAX, "allocation estimate rejects overflow");
+    }
+    {
+        ConversationBuffer bytes;
+        for (size_t n=1;n<=4096;++n) {
+            const size_t peak = bytes.allocation_peak(n*17);
+            bytes.resize(n*17,7);
+            check(bytes.bytes() <= peak,"small append stays within the predicted allocation peak");
+        }
+        size_t segments = 0;
+        bytes.visit(0,bytes.size(),[&](const uint8_t*,size_t,size_t){++segments;return true;});
+        check(segments <= 4,"thousands of small turns do not create thousands of restore transfers");
+    }
+    {
+        ConversationKv layer;
+        layer.k.resize(400);
+        std::vector<ConversationKv> layers;
+        layers.push_back(std::move(layer));
+        const size_t retained = layers.capacity()*sizeof(ConversationKv) + layers[0].bytes();
+        const size_t parked = image({1,2,3}).bytes();
+        ConversationCache cache(retained + parked, 4);
+        check(cache.put(image({1,2,3})), "park before retaining active storage");
+        cache.retain(std::move(layers), 16);
+        check(cache.bytes() == retained + parked && cache.size() == 1, "active retained storage consumes bytes but no parked slot");
+        cache.limit_reuse(9); cache.limit_reuse(12);
+        auto reuse = cache.take_reuse();
+        check(reuse.unchanged_tokens == 9, "a later continuation cannot undo a rewind's dirty boundary");
+        check(cache.bytes() == parked, "taking retained buffers releases their budget accounting");
+        cache.retain(std::move(reuse.kv), 16);
+        check(!cache.can_fit(parked),"retained storage is counted when checking a capture without eviction");
+        check(cache.retained_bytes() == retained && cache.size() == 1 && cache.evictions() == 0,
+              "optional reuse admission does not evict or release anything");
+        check(!cache.can_fit(SIZE_MAX) && !cache.can_fit(1,SIZE_MAX),"non-mutating reservation rejects overflow");
+        check(cache.make_room(parked), "reservation can discard optional active buffers");
+        check(cache.retained_bytes() == 0 && cache.size() == 1 && cache.evictions() == 0,
+              "pressure drops retained storage before evicting parked conversations");
+    }
+    const std::vector<int64_t> a = {1, 2, 3, 4}, b = {9, 8, 7, 6};
+    {
+        ConversationCache cache(1024, 2);
+        check(cache.put(image({1, 2, 3})), "park A");
+        check(cache.put(image({9, 8, 7})), "park B");
+        auto match = cache.best(a, {}, true);
+        check(match.tokens == 3 && match.live, "A/B/A: recover A");
+        auto restored = cache.take(match.index);
+        check(restored.live.ids == std::vector<int32_t>({1, 2, 3}), "taking selected A preserves identity");
+        check(cache.size() == 1 && cache.best(b, {}, true).tokens == 3, "B remains parked");
+        check(cache.bytes() == image({9,8,7}).bytes(), "byte accounting after take");
+        check(cache.put(std::move(restored)), "park returned A as newest");
+        check(cache.put(image({5, 6})), "evict oldest by slot limit");
+        check(cache.best(b, {}, true).tokens == 0 && cache.best(a, {}, true).tokens == 3, "B evicted before A");
+        check(cache.evictions() == 1, "eviction counter");
+    }
+    {
+        auto s = image({1, 2, 3});
+        ConversationCheckpoint cp;
+        cp.ids = {1, 2};
+        s.checkpoints.push_back(cp);
+        ConversationCache cache(4096, 4);
+        cache.put(std::move(s));
+        auto match = cache.best(std::vector<int64_t>{1, 2, 9, 4}, {}, true);
+        check(match.tokens == 2 && !match.live, "edited suffix falls back to parked checkpoint");
+        check(cache.best(a, {}, true).tokens == 3, "live prefix beats shorter checkpoint");
+        check(cache.best(a, {}, false).tokens == 0, "steering mode is isolated");
+        check(cache.best(std::vector<int64_t>{1, 2}, {}, true).tokens == 0, "equal-length checkpoint cannot consume last token");
+        check(cache.best(std::vector<int64_t>{1}, {}, true).tokens == 0, "short prompt cannot match");
+        check(cache.best(std::vector<int64_t>{}, {}, true).tokens == 0, "empty prompt cannot match");
+        cache.put(image({1, 2, 3}));
+        check(cache.best(a, {}, true).index == 1, "ties prefer most recently parked");
+    }
+    {
+        auto s = image({1, 2, 3});
+        s.live.imgs = {{1, 123}};
+        ConversationCache cache(1024, 3);
+        cache.put(std::move(s));
+        check(cache.best(a, {{1,123}}, true).tokens == 3, "same image can resume");
+        check(cache.best(a, {{1,124}}, true).tokens == 0, "different image pixels invalidate same pad tokens");
+        check(cache.best(a, {}, true).tokens == 0, "missing image invalidates prefix");
+        check(cache.best(a, {{1,123},{2,45}}, true).tokens == 0, "additional image in cached prefix invalidates");
+        check(cache.best(a, {{1,123},{3,45}}, true).tokens == 3, "image after cached prefix does not invalidate");
+    }
+    {
+        const size_t one = image({1,2,3}).bytes();
+        ConversationCache cache(one*2, 8);
+        cache.put(image({1,2,3})); cache.put(image({9,8,7}));
+        check(cache.bytes() == one*2, "budget holds two exact-sized images");
+        auto held = cache.take(cache.best(a, {}, true).index);
+        check(cache.make_room(one, held.bytes()), "count in-flight image when reserving outgoing snapshot");
+        check(cache.size() == 0, "in-flight reservation evicts otherwise fitting B");
+        check(cache.put(image({5,6,7}), held.bytes()), "insert with in-flight accounting");
+        check(cache.bytes()+held.bytes() <= one*2, "exchange obeys byte budget");
+        check(!cache.make_room(one+1, one), "oversized exchange rejected");
+        check(cache.size() == 1, "oversized snapshot does not evict useful entries");
+        auto huge = image({1}); huge.live.gdn.resize(one*3);
+        check(!cache.put(std::move(huge)), "oversized image rejected");
+        check(cache.size() == 1, "oversized put leaves cache unchanged");
+        check(!cache.make_room(0, one*2+1), "held larger than budget cannot underflow");
+    }
+    {
+        ConversationCache disabled(0,4), no_slots(1024,0);
+        check(!disabled.enabled() && !no_slots.enabled(), "both disable switches");
+        check(!disabled.put(image({1,2,3})) && !no_slots.put(image({1,2,3})), "disabled cache stores nothing");
+        check(disabled.best(a,{},true).tokens == 0, "disabled cache has no matches");
+    }
+    std::printf("conversation_cache_test: %d checks passed\n", checks);
+}

src/core/conversation_checked.hpp

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <initializer_list>
+#include <limits>
+
+namespace strata::core::conversation_detail {
+inline bool add(size_t& total, size_t n) {
+    if (n > std::numeric_limits<size_t>::max() - total) return false;
+    total += n;
+    return true;
+}
+inline bool product(size_t& out, std::initializer_list<uint64_t> factors) {
+    size_t value = 1;
+    for (uint64_t factor : factors) {
+        if (factor > std::numeric_limits<size_t>::max() ||
+            (factor && value > std::numeric_limits<size_t>::max() / factor)) return false;
+        value *= (size_t) factor;
+    }
+    out = value;
+    return true;
+}
+} // namespace strata::core::conversation_detail

src/core/conversation_memory.cpp

@@ -0,0 +1,51 @@
+#include "strata/core/conversation_memory.hpp"
+
+#include <charconv>
+#include <fstream>
+#include <limits>
+#include <sstream>
+#include <string>
+
+#if defined(_WIN32)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+
+namespace strata::core {
+
+std::optional<uint64_t> conversation_mem_available(std::istream& meminfo) {
+    std::optional<uint64_t> result;
+    std::string line;
+    while (std::getline(meminfo, line)) {
+        std::istringstream fields(line);
+        std::string key, value, unit, extra;
+        if (!(fields >> key) || key != "MemAvailable:") continue;
+        if (result || !(fields >> value >> unit) || unit != "kB" || (fields >> extra)) return {};
+        uint64_t kb = 0;
+        const auto parsed = std::from_chars(value.data(), value.data() + value.size(), kb);
+        if (parsed.ec != std::errc{} || parsed.ptr != value.data() + value.size() ||
+            kb > std::numeric_limits<uint64_t>::max() / 1024) return {};
+        result = kb * 1024;
+    }
+    if (meminfo.bad() || (meminfo.fail() && !meminfo.eof())) return {};
+    return result;
+}
+
+std::optional<uint64_t> conversation_available_memory() {
+#if defined(_WIN32)
+    MEMORYSTATUSEX status{};
+    status.dwLength = sizeof status;
+    if (GlobalMemoryStatusEx(&status)) return status.ullAvailPhys;
+    return {};
+#elif defined(__linux__)
+    std::ifstream meminfo("/proc/meminfo");
+    if (!meminfo) return {};
+    return conversation_mem_available(meminfo);
+#else
+    return {};
+#endif
+}
+
+} // namespace strata::core

src/core/conversation_memory_test.cpp

@@ -0,0 +1,66 @@
+#include "strata/core/conversation_memory.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+#include <limits>
+#include <sstream>
+
+#if defined(_WIN32)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+
+using namespace strata::core;
+namespace {
+int checks = 0;
+void check(bool ok, const char* label) {
+    ++checks;
+    if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
+}
+}
+
+int main() {
+    for (const char* text : {"", "MemFree: 100 kB\n", "MemAvailable: -1 kB\n",
+                            "MemAvailable: +1 kB\n", "MemAvailable: 1 MB\n",
+                            "MemAvailable: 1\n", "MemAvailable: 1x kB\n",
+                            "MemAvailable: 18446744073709551615 kB\n",
+                            "MemAvailable: 18446744073709551616 kB\n",
+                            "MemAvailable: 1 kB trailing\n",
+                            "MemAvailable: 1 kB\nMemAvailable: 2 kB\n"}) {
+        std::istringstream input(text);
+        check(!conversation_mem_available(input), "missing/malformed telemetry is unknown");
+    }
+    std::istringstream normal("MemTotal: 999999 kB\nMemAvailable:    12345 kB\nSwapFree: 777 kB\n");
+    check(conversation_mem_available(normal) == 12345ULL * 1024, "only MemAvailable is counted");
+    std::istringstream zero("MemAvailable: 0 kB");
+    check(conversation_mem_available(zero) == 0, "zero available memory is known");
+    std::istringstream broken("MemAvailable: 123 kB\n");
+    broken.setstate(std::ios::badbit);
+    check(!conversation_mem_available(broken), "I/O failure declines admission");
+    check(!conversation_memory_admit({}, 0, 0), "unknown fails closed even with zero floor");
+    check(conversation_memory_admit(100, 40, 60), "exact allocation plus floor fits");
+    check(!conversation_memory_admit(99, 40, 60), "one byte below required memory rejected");
+    check(!conversation_memory_admit(59, 0, 60), "floor subtraction cannot underflow");
+    check(conversation_memory_admit(60, 0, 60), "post-capture floor check");
+    check(!conversation_memory_admit(100, std::numeric_limits<uint64_t>::max(), 1), "allocation arithmetic cannot overflow");
+    check(conversation_memory_admit(std::numeric_limits<uint64_t>::max(),
+                                   std::numeric_limits<uint64_t>::max(), 0), "maximal exact bound");
+    check(!conversation_memory_admit(std::numeric_limits<uint64_t>::max(),
+                                    std::numeric_limits<uint64_t>::max(), 1), "maximal sum overflow rejected");
+    // Test the real provider without assuming any particular amount of free RAM.
+    const auto available = conversation_available_memory();
+    check(!available || conversation_memory_admit(available, 0, 0), "provider returns bytes or unknown");
+#if defined(_WIN32)
+    // Unknown must fail closed in production, but must not let a broken Windows provider pass this test.
+    // Compare with total physical memory, not a second available reading: other processes can allocate
+    // between calls. No pressure allocation or fixed free-memory assumption is needed.
+    MEMORYSTATUSEX physical{};
+    physical.dwLength = sizeof physical;
+    check(GlobalMemoryStatusEx(&physical) != 0, "Windows physical-memory API is available");
+    check(available.has_value() && *available <= physical.ullTotalPhys,
+          "Windows provider returns a known, physically bounded sample");
+#endif
+    std::printf("conversation_memory_test: %d checks passed\n", checks);
+}

src/core/conversation_snapshot.cpp

@@ -0,0 +1,264 @@
+#include "strata/core/conversation_snapshot.hpp"
+#include "strata/kernels/kv_q4.hpp"
+#include "conversation_checked.hpp"
+#include <cuda_runtime.h>
+
+#include <array>
+#include <cstring>
+#include <limits>
+
+namespace strata::core {
+namespace {
+struct Layout {
+    int format;
+    int64_t cells, pooled_rows, page_size;
+    size_t data, scales, pooled, value_data, value_scales;
+};
+
+bool valid_extent(const QsaState& st, int64_t upto, std::string& error) {
+    const int64_t page = strata::kernels::qsa_real_shapes().page_size;
+    if (upto < 0 || upto > st.max_cells || upto > std::numeric_limits<int64_t>::max() - (page - 1)) {
+        error = "conversation snapshot: invalid K/V extent";
+        return false;
+    }
+    return true;
+}
+
+bool layout(const QsaState& st, const ModelGeometry& g, int64_t upto, bool index, Layout& l, std::string& error) {
+    if (!valid_extent(st, upto, error)) return false;
+    if (st.kv_hybrid && (st.kv_mode != 0 || st.kv_q4 || st.kv_int8)) {
+        error = "conversation snapshot: hybrid K8V4 requires an identity layout and distinct format flags";
+        return false;
+    }
+    const auto s = strata::kernels::qsa_real_shapes();
+    const bool int8_keys = st.kv_int8 || st.kv_hybrid;
+    if (g.n_head_kv <= 0 || g.head_dim <= 0 || g.head_dim > INT32_MAX || g.idx_key_dim <= 0 ||
+        (st.kv_q4 && g.head_dim % 32) || (int8_keys && !st.kv_q4 && g.head_dim % 64)) {
+        error = "conversation snapshot: invalid K/V geometry";
+        return false;
+    }
+    const int64_t cells = ((upto + s.page_size - 1) / s.page_size) * s.page_size;
+    const size_t per = st.kv_q4 ? (size_t) strata::kernels::kv_q4_bytes_per_head((int) g.head_dim)
+                              : (size_t) g.head_dim * (int8_keys ? 1 : 2);
+    // Include the moving spare row, not only completed blocks. The checkpoint
+    // restore reconstructs that row when rewinding to an earlier prefix.
+    const int64_t pooled = index && upto > 0 ? upto / s.idx_block + 1 : 0;
+    // Format 3 identifies snapshot K8V4 only; never pass it to the block movers.
+    l = {st.kv_hybrid ? 3 : qsa_kv_format(st), cells, pooled, s.page_size, 0, 0, 0, 0, 0};
+    using conversation_detail::product;
+    if (!product(l.data, {(uint64_t) cells, (uint64_t) g.n_head_kv, per}) ||
+        !product(l.scales, {(uint64_t) cells, (uint64_t) g.n_head_kv,
+                           int8_keys && !st.kv_q4 ? (uint64_t) (g.head_dim / 64) * 2 : 0}) ||
+        !product(l.pooled, {(uint64_t) pooled, (uint64_t) g.idx_key_dim, sizeof(float)})) {
+        error = "conversation snapshot: K/V byte count overflow";
+        return false;
+    }
+    l.value_data = l.data;
+    l.value_scales = l.scales;
+    if (st.kv_hybrid) {
+        if (!product(l.value_data, {(uint64_t) cells, (uint64_t) g.n_head_kv,
+                                   (uint64_t) strata::kernels::kv_q4_bytes_per_head((int) g.head_dim)})) {
+            error = "conversation snapshot: hybrid V byte count overflow";
+            return false;
+        }
+        l.value_scales = 0;
+    }
+    return true;
+}
+
+std::array<void*, 5> pools(const QsaState& st, bool resident = false) {
+    const bool host = st.kv_mode != 0 && !resident;
+    if (st.kv_hybrid) return {st.k_q, st.v_q4, st.k_scale, nullptr, st.idx_pooled};
+    if (st.kv_q4)
+        return {host ? st.host.k_q4 : st.k_q4, host ? st.host.v_q4 : st.v_q4, nullptr, nullptr, st.idx_pooled};
+    if (st.kv_int8)
+        return {host ? st.host.k_q : st.k_q, host ? st.host.v_q : st.v_q,
+                host ? st.host.k_scale : st.k_scale, host ? st.host.v_scale : st.v_scale, st.idx_pooled};
+    return {host ? st.host.k_pool : st.k_pool, host ? st.host.v_pool : st.v_pool, nullptr, nullptr, st.idx_pooled};
+}
+
+bool valid(const QsaState& st, const Layout& l, int64_t upto, std::string& error) {
+    if (upto < 0 || upto > st.max_cells || st.n_pages < 0 || l.cells / l.page_size > st.n_pages ||
+        st.kv_mode < 0 || st.kv_mode > 2 || st.n_slots <= 0 ||
+        (st.kv_mode == 0 && st.n_slots < st.n_pages) ||
+        l.pooled_rows > st.idx_pooled_rows || (st.kv_mode != 0 && !st.host.present())) {
+        error = "conversation snapshot: invalid K/V extent or missing authoritative host pool";
+        return false;
+    }
+    if (st.kv_mode == 1) {
+        const auto& m = st.map;
+        if (m.n_blocks != st.n_pages || m.n_slots != st.n_slots || !m.page_table || !m.slot_block ||
+            !m.slot_stamp || !m.slot_ref || !m.ctl || !m.miss_block || !m.miss_slot) {
+            error = "conversation snapshot: invalid streaming map";
+            return false;
+        }
+    }
+    if (st.kv_mode == 2 && (!st.page_table ||
+        (st.kv_q4 ? !st.k_q4 || !st.v_q4 : st.kv_int8 ? !st.k_q || !st.v_q || !st.k_scale || !st.v_scale
+                                                    : !st.k_pool || !st.v_pool))) {
+        error = "conversation snapshot: invalid draft ring";
+        return false;
+    }
+    return true;
+}
+
+bool transfer(void* dst, const void* src, size_t n, std::string& error) {
+    if (!n) return true;
+    if (!src || !dst) { error = "conversation snapshot: missing state buffer"; return false; }
+    // Default handles both device allocations and device-mapped host pool aliases.
+    const cudaError_t e = cudaMemcpy(dst, src, n, cudaMemcpyDefault);
+    if (e == cudaSuccess) return true;
+    error = std::string("conversation snapshot copy: ") + cudaGetErrorString(e);
+    return false;
+}
+} // namespace
+
+bool conversation_kv_capture_bytes(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,
+                                   int64_t upto, bool index, size_t& bytes, std::string& error) {
+    Layout l{};
+    if (!layout(st, g, upto, index, l, error) || !valid(st, l, upto, error)) return false;
+    const std::array<size_t,5> sizes = {l.data, l.value_data, l.scales, l.value_scales, l.pooled};
+    const std::array<const ConversationBuffer*,5> buffers = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};
+    bytes = 0;
+    for (size_t i = 0; i < buffers.size(); ++i) {
+        const size_t n = buffers[i]->allocation_peak(sizes[i]);
+        if (n == SIZE_MAX || !conversation_detail::add(bytes, n)) {
+            error = "conversation snapshot: segmented K/V allocation overflow";
+            return false;
+        }
+    }
+    return true;
+}
+
+size_t conversation_kv_bytes(const QsaState& st, const ModelGeometry& g, int64_t upto, bool index) {
+    size_t bytes = 0;
+    std::string error;
+    return conversation_kv_capture_bytes({}, st, g, upto, index, bytes, error) ? bytes : 0;
+}
+
+bool conversation_kv_save(ConversationKv& image, const QsaState& st, const ModelGeometry& g,
+                          int64_t upto, bool index, std::string& error,
+                          int64_t unchanged_tokens, size_t* reused_bytes) {
+    Layout l{};
+    if (!layout(st, g, upto, index, l, error)) return false;
+    if (!valid(st, l, upto, error)) return false;
+    if (unchanged_tokens < 0 || unchanged_tokens > upto || unchanged_tokens > image.cells) {
+        error = "conversation snapshot: invalid unchanged prefix";
+        return false;
+    }
+    const int64_t whole_cells = (unchanged_tokens / l.page_size) * l.page_size;
+    image.format = l.format;
+    image.cells = l.cells;
+    image.heads = g.n_head_kv;
+    image.head_dim = g.head_dim;
+    image.page_size = l.page_size;
+    image.pooled_rows = l.pooled_rows;
+    image.idx_dim = g.idx_key_dim;
+    const auto src = pools(st);
+    const std::array<size_t,5> sizes = {l.data, l.value_data, l.scales, l.value_scales, l.pooled};
+    const std::array<ConversationBuffer*,5> dst = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};
+    for (size_t i = 0; i < dst.size(); ++i) {
+        // Recopy the partial page and the indexer's moving spare row. Completed
+        // pages/rows strictly before the first rewritten token remain identical.
+        const size_t keep = i == 4 ? (index ? size_t(unchanged_tokens / strata::kernels::qsa_real_shapes().idx_block) * g.idx_key_dim * 4 : 0)
+                                  : l.cells ? (sizes[i] / size_t(l.cells)) * size_t(whole_cells) : 0;
+        if (keep > dst[i]->size()) { error = "conversation snapshot: missing reusable prefix"; return false; }
+        dst[i]->resize(sizes[i]);
+        if (!dst[i]->visit(keep, sizes[i] - keep, [&](uint8_t* p, size_t n, size_t at) {
+                return transfer(p, src[i] ? static_cast<const uint8_t*>(src[i]) + at : nullptr, n, error);
+            })) return false;
+        if (reused_bytes) *reused_bytes += keep;
+    }
+    return true;
+}
+
+bool conversation_kv_validate(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,
+                              int64_t upto, bool index, std::string& error) {
+    Layout l{};
+    if (!layout(st, g, upto, index, l, error)) return false;
+    if (!valid(st, l, upto, error)) return false;
+    const std::array<size_t,5> sizes = {l.data, l.value_data, l.scales, l.value_scales, l.pooled};
+    const std::array<const ConversationBuffer*,5> src = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};
+    if (image.format != l.format || image.cells != l.cells || image.heads != g.n_head_kv ||
+        image.head_dim != g.head_dim || image.page_size != l.page_size || image.pooled_rows != l.pooled_rows ||
+        image.idx_dim != g.idx_key_dim) {
+        error = "conversation snapshot: incompatible K/V geometry";
+        return false;
+    }
+    for (size_t i = 0; i < src.size(); ++i)
+        if (src[i]->size() != sizes[i]) { error = "conversation snapshot: invalid K/V payload size"; return false; }
+    const auto dst = pools(st);
+    for (size_t i = 0; i < src.size(); ++i)
+        if (sizes[i] && !dst[i]) { error = "conversation snapshot: missing target state buffer"; return false; }
+    return true;
+}
+
+bool conversation_kv_restore(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,
+                             int64_t upto, bool index, std::string& error) {
+    if (!conversation_kv_validate(image, st, g, upto, index, error)) return false;
+    const std::array<const ConversationBuffer*,5> src = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};
+    const auto dst = pools(st);
+    for (size_t i = 0; i < src.size(); ++i)
+        if (!src[i]->visit(0, src[i]->size(), [&](const uint8_t* p, size_t n, size_t at) {
+                return transfer(static_cast<uint8_t*>(dst[i]) + at, p, n, error);
+            })) return false;
+    // VRAM slots still contain the outgoing conversation. Resolve must refill
+    // them from the restored authoritative pools before any attention reads.
+    if (st.kv_mode == 1) strata::kernels::kv_stream_reset(st.map, nullptr);
+    if (st.kv_mode == 2 && upto > 0) {
+        auto shapes = strata::kernels::qsa_real_shapes();
+        shapes.n_head_kv = g.n_head_kv; shapes.head_dim = g.head_dim;
+        const int64_t end = (upto + shapes.page_size - 1) / shapes.page_size;
+        strata::kernels::kv_ring_restore(qsa_attn_pools(st), st.host, qsa_kv_format(st),
+                                        std::max<int64_t>(0, end - st.n_slots), end, st.n_slots, shapes, nullptr);
+    }
+    const auto status = cudaGetLastError();
+    if (status == cudaSuccess) return true;
+    error = std::string("conversation snapshot residency restore: ") + cudaGetErrorString(status);
+    return false;
+}
+
+bool conversation_kv_verify(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,
+                            int64_t upto, bool index, uint64_t& fingerprint, std::string& error) {
+    if (!conversation_kv_validate(image, st, g, upto, index, error)) return false;
+    std::array<uint8_t, 65536> buffer;
+    uint64_t hash = 1469598103934665603ull;
+    auto compare = [&](const void* device, const uint8_t* expected, size_t bytes, bool include_hash) {
+        for (size_t offset = 0; offset < bytes;) {
+            const size_t n = std::min(buffer.size(), bytes - offset);
+            if (!transfer(buffer.data(), static_cast<const uint8_t*>(device) + offset, n, error)) return false;
+            if (std::memcmp(buffer.data(), expected + offset, n) != 0) {
+                error = "conversation snapshot: restored K/V bytes differ";
+                return false;
+            }
+            if (include_hash) for (size_t i = 0; i < n; ++i) { hash ^= buffer[i]; hash *= 1099511628211ull; }
+            offset += n;
+        }
+        return true;
+    };
+    const std::array<const ConversationBuffer*, 5> saved = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};
+    const auto authoritative = pools(st);
+    for (size_t i = 0; i < saved.size(); ++i)
+        if (!saved[i]->visit(0, saved[i]->size(), [&](const uint8_t* p, size_t n, size_t at) {
+                return compare(static_cast<const uint8_t*>(authoritative[i]) + at, p, n, true);
+            })) return false;
+    if (st.kv_mode == 2 && image.cells > 0) {
+        const auto resident = pools(st, true);
+        const int64_t end = image.cells / image.page_size;
+        const int64_t begin = std::max<int64_t>(0, end - st.n_slots);
+        for (size_t i = 0; i < 4; ++i) {
+            if (saved[i]->empty()) continue;
+            const size_t page_bytes = saved[i]->size() / size_t(end);
+            for (int64_t page = begin; page < end; ++page)
+                if (!saved[i]->visit(size_t(page) * page_bytes, page_bytes,
+                        [&](const uint8_t* p, size_t n, size_t at) {
+                            const size_t offset = size_t(page % st.n_slots) * page_bytes + at - size_t(page) * page_bytes;
+                            return compare(static_cast<const uint8_t*>(resident[i]) + offset, p, n, false);
+                        })) return false;
+        }
+    }
+    fingerprint = hash;
+    return true;
+}
+
+} // namespace strata::core

src/core/conversation_snapshot_test.cpp

@@ -0,0 +1,272 @@
+#include "strata/core/conversation_snapshot.hpp"
+#include "strata/kernels/kv_q4.hpp"
+#include <cuda_runtime.h>
+
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <limits>
+#include <vector>
+
+using namespace strata::core;
+using namespace strata::kernels;
+
+namespace {
+int checks = 0;
+void check(bool ok, const char* label) {
+    ++checks;
+    if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
+}
+void cuda_check(cudaError_t e) {
+    if (e != cudaSuccess) { std::fprintf(stderr, "CUDA: %s\n", cudaGetErrorString(e)); std::exit(1); }
+}
+struct Fixture {
+    ModelGeometry g;
+    QsaState state;
+    std::vector<void*> device, host;
+    std::array<void*,5> sources{};
+    std::array<size_t,5> sizes{};
+
+    template<class T> void alloc(T*& p, size_t n, bool pinned = false) {
+        if (!n) return;
+        void* raw = nullptr;
+        if (pinned) {
+            cuda_check(cudaHostAlloc(&raw, n, cudaHostAllocMapped));
+            host.push_back(raw);
+            void* mapped = nullptr;
+            cuda_check(cudaHostGetDevicePointer(&mapped, raw, 0));
+            p = static_cast<T*>(mapped);
+        } else {
+            cuda_check(cudaMalloc(&raw, n)); device.push_back(raw); p = static_cast<T*>(raw);
+        }
+    }
+    Fixture(int fmt, int mode) {
+        auto& st = state;
+        st.kv_mode = mode; st.kv_int8 = fmt==kKvInt8; st.kv_q4 = fmt==kKvQ4; st.kv_hybrid = fmt==3;
+        st.n_pages = 24; st.n_slots = mode ? 4 : st.n_pages;
+        st.max_cells = st.n_pages * 4; st.idx_pooled_rows = mode==2 ? 2 : st.max_cells/4+2;
+        const size_t per = fmt==kKvQ4 ? kv_q4_bytes_per_head((int)g.head_dim) : g.head_dim*((st.kv_int8 || st.kv_hybrid) ? 1:2);
+        const size_t rows = st.max_cells*g.n_head_kv, slot_rows = st.n_slots*4*g.n_head_kv;
+        sizes = {rows*per, rows*per, fmt==kKvInt8 ? rows*(g.head_dim/64)*2:0,
+                 fmt==kKvInt8 ? rows*(g.head_dim/64)*2:0, (size_t)st.idx_pooled_rows*g.idx_key_dim*4};
+        if (fmt==3) {
+            sizes = {rows*per, rows*kv_q4_bytes_per_head((int)g.head_dim), rows*(g.head_dim/64)*2, 0,
+                     (size_t)st.idx_pooled_rows*g.idx_key_dim*4};
+            alloc(st.k_q,sizes[0]); alloc(st.v_q4,sizes[1]); alloc(st.k_scale,sizes[2]);
+            sources[0]=st.k_q; sources[1]=st.v_q4; sources[2]=st.k_scale;
+        } else if (fmt==kKvQ4) {
+            alloc(st.k_q4,slot_rows*per); alloc(st.v_q4,slot_rows*per);
+            if (mode) {alloc(st.host.k_q4,sizes[0],true); alloc(st.host.v_q4,sizes[1],true);}
+            sources[0]=mode?st.host.k_q4:st.k_q4; sources[1]=mode?st.host.v_q4:st.v_q4;
+        } else if (fmt==kKvInt8) {
+            alloc(st.k_q,slot_rows*per); alloc(st.v_q,slot_rows*per);
+            alloc(st.k_scale,slot_rows*(g.head_dim/64)*2); alloc(st.v_scale,slot_rows*(g.head_dim/64)*2);
+            if (mode) {
+                alloc(st.host.k_q,sizes[0],true); alloc(st.host.v_q,sizes[1],true);
+                alloc(st.host.k_scale,sizes[2],true); alloc(st.host.v_scale,sizes[3],true);
+            }
+            sources[0]=mode?st.host.k_q:st.k_q; sources[1]=mode?st.host.v_q:st.v_q;
+            sources[2]=mode?st.host.k_scale:st.k_scale; sources[3]=mode?st.host.v_scale:st.v_scale;
+        } else {
+            alloc(st.k_pool,slot_rows*per); alloc(st.v_pool,slot_rows*per);
+            if (mode) {alloc(st.host.k_pool,sizes[0],true); alloc(st.host.v_pool,sizes[1],true);}
+            sources[0]=mode?st.host.k_pool:st.k_pool; sources[1]=mode?st.host.v_pool:st.v_pool;
+        }
+        alloc(st.idx_pooled,sizes[4]); sources[4]=st.idx_pooled;
+        alloc(st.page_table,st.n_pages*4);
+        if (mode==1) {
+            auto& m=st.map;
+            m.page_table=st.page_table; m.n_blocks=st.n_pages; m.n_slots=st.n_slots;
+            alloc(m.slot_block,st.n_slots*4); alloc(m.slot_stamp,st.n_slots*4); alloc(m.slot_ref,st.n_slots*4);
+            alloc(m.miss_block,st.n_slots*4); alloc(m.miss_slot,st.n_slots*4); alloc(m.ctl,kKvCtlInts*4);
+        } else if (mode==2) {
+            kv_ring_table(st.page_table,st.n_pages,st.n_slots,nullptr);
+        }
+    }
+    void fill(uint8_t salt) {
+        for (size_t i=0;i<sources.size();++i) {
+            std::vector<uint8_t> data(sizes[i]);
+            for (size_t j=0;j<data.size();++j) data[j]=(uint8_t)(salt+i*31+j*7+j/257);
+            if (!data.empty()) cuda_check(cudaMemcpy(sources[i],data.data(),data.size(),cudaMemcpyDefault));
+        }
+    }
+    void fill_after(uint8_t salt, int64_t first_dirty) {
+        for (size_t i=0;i<sources.size();++i) {
+            const size_t offset = i == 4 ? size_t(first_dirty/4)*g.idx_key_dim*4
+                                         : (sizes[i]/size_t(state.max_cells))*size_t((first_dirty/4)*4);
+            if (sizes[i] > offset)
+                cuda_check(cudaMemset(static_cast<uint8_t*>(sources[i])+offset, salt, sizes[i]-offset));
+        }
+    }
+    ~Fixture() { for (void* p:device) cudaFree(p); for (void* p:host) cudaFreeHost(p); }
+};
+bool equal(const ConversationKv& a,const ConversationKv& b) {
+    return a.k==b.k && a.v==b.v && a.k_scale==b.k_scale && a.v_scale==b.v_scale && a.pooled==b.pooled;
+}
+
+void full_session(int fmt, int mode, int experts) {
+    Fixture main(fmt,mode), draft(fmt==3?kKvInt8:fmt,2);
+    auto& g=main.g;
+    g.n_layers=4; g.n_expert=experts;
+    g.ssm_state_size=2; g.ssm_v_heads=2; g.ssm_conv_channels=8;
+    SessionState ss;
+    ss.max_cells=96; ss.qsa_states=&main.state;
+    ss.layer_hi=g.n_layers; ss.gdn_alloc=g.n_gdn_layers(); ss.qsa_alloc=g.n_qsa_layers();   // the whole-model carve
+    std::string err;
+    ConversationStateSizes sizes;
+    check(conversation_state_sizes(g,sizes,err),"whole-session geometry sizes");
+    main.alloc(ss.gdn_state,sizes.gdn); main.alloc(ss.ple_hist,sizes.ple);
+    main.alloc(main.state.idx_tail,sizes.tail); main.alloc(main.state.idx_dead,sizes.dead);
+    main.alloc(main.state.idx_block_pos,sizes.block_pos);
+    std::vector<int32_t> ids(65);
+    for (size_t i=0;i<ids.size();++i) ids[i]=(int32_t)i+1;
+    std::vector<ConversationImageKey> images;
+    std::vector<ConversationCheckpoint> checkpoints;
+    auto fill=[&](uint8_t salt) {
+        main.fill(salt); draft.fill(salt);
+        for (const auto& [p,n] : std::vector<std::pair<void*,size_t>>{
+                 {ss.gdn_state,sizes.gdn},{ss.ple_hist,sizes.ple},{main.state.idx_tail,sizes.tail},
+                 {main.state.idx_dead,sizes.dead},{main.state.idx_block_pos,sizes.block_pos}})
+            cuda_check(cudaMemset(p,salt,n));
+        // A real indexer maintains its spare pooled row as a copy of idx_dead.
+        cuda_check(cudaMemcpy(main.state.idx_pooled+(ids.size()/4)*g.idx_key_dim,
+                              main.state.idx_dead,sizes.dead,cudaMemcpyDeviceToDevice));
+        cuda_check(cudaDeviceSynchronize());
+    };
+    fill(13);
+    ConversationCheckpoint checkpoint;
+    checkpoint.ids.assign(ids.begin(),ids.begin()+3);
+    check(conversation_checkpoint_save(checkpoint,ss,g,err),"save complete running checkpoint");
+    checkpoint.used = 17;
+    checkpoints.push_back(std::move(checkpoint));
+    const ConversationView view{ids,images,checkpoints,true};
+    SavedConversation a,b,restored;
+    check(conversation_snapshot_save(a,view,ss,g,draft.state,err),"capture complete A");
+    check(a.checkpoints[0].used == 17,"upstream checkpoint LRU stamp survives capture");
+    fill(177);
+    check(conversation_snapshot_save(b,view,ss,g,draft.state,err),"capture complete B");
+    check(a.live.dead!=b.live.dead,"different opening-state spare keys in regression fixture");
+    auto bad=a; bad.kv.back().k.pop_back();
+    check(conversation_snapshot_restore(bad,ss,g,draft.state,err)==ConversationRestore::invalid,
+          "reject invalid late draft before any main-layer write");
+    check(conversation_snapshot_save(restored,view,ss,g,draft.state,err),"capture B after refused restore");
+    check(restored.live.gdn==b.live.gdn && restored.live.dead==b.live.dead &&
+          equal(restored.kv[0],b.kv[0]) && equal(restored.kv[1],b.kv[1]),"refusal preserves all tested state");
+    check(conversation_snapshot_restore(a,ss,g,draft.state,err)==ConversationRestore::restored,"restore complete A");
+    check(conversation_snapshot_save(restored,view,ss,g,draft.state,err),"capture restored A");
+    check(restored.live.gdn==a.live.gdn && restored.live.ple==a.live.ple && restored.live.tails==a.live.tails &&
+          restored.live.dead==a.live.dead && restored.live.block_pos==a.live.block_pos &&
+          equal(restored.kv[0],a.kv[0]) && equal(restored.kv[1],a.kv[1]),"whole-session A/B/A exactness including spare key");
+    check(ss.ple_prev[0]==64 && ss.ple_prev[1]==65,"PLE token window reconstructed");
+    for (int64_t dirty : {65, 3, 0}) {
+        check(conversation_snapshot_restore(a,ss,g,draft.state,err)==ConversationRestore::restored,"restore growth fixture base");
+        ConversationKvReuse reuse{a.kv,65,dirty};
+        const uint8_t* original = nullptr;
+        reuse.kv[0].k.visit(0,1,[&](const uint8_t* p,size_t,size_t){original=p;return true;});
+        main.fill_after(91,dirty); draft.fill_after(91,std::max<int64_t>(0,dirty-1));
+        cuda_check(cudaMemset(ss.gdn_state,91,sizes.gdn));
+        ids.resize(70);
+        for (size_t i=65;i<ids.size();++i) ids[i]=int32_t(i+1);
+        cuda_check(cudaMemcpy(main.state.idx_pooled+(ids.size()/4)*g.idx_key_dim,
+                              main.state.idx_dead,sizes.dead,cudaMemcpyDeviceToDevice));
+        SavedConversation fresh,incremental;
+        check(conversation_snapshot_save(fresh,view,ss,g,draft.state,err),"full capture reference after growth or rewind");
+        size_t peak=0,reused=0;
+        check(conversation_snapshot_capture_bytes(reuse,view,ss,g,draft.state,peak,err),"admit incremental capture peak");
+        check(conversation_snapshot_save(incremental,view,ss,g,draft.state,err,std::move(reuse),&reused),"capture with retained pages");
+        check(incremental.bytes() <= peak,"incremental allocation stays within admitted bound");
+        check(incremental.live.gdn==fresh.live.gdn && incremental.live.ple==fresh.live.ple &&
+              incremental.live.dead==fresh.live.dead && equal(incremental.kv[0],fresh.kv[0]) &&
+              equal(incremental.kv[1],fresh.kv[1]),"incremental capture equals full capture after growth or rewind");
+        check((dirty>=4)==(reused>0),"only complete unchanged pages or rows are retained");
+        incremental.kv[0].k.visit(0,1,[&](const uint8_t* p,size_t,size_t){
+            check(p==original,"growth never reallocates the retained payload");return true;
+        });
+        check(conversation_snapshot_restore(incremental,ss,g,draft.state,err)==ConversationRestore::restored,"restore segmented incremental snapshot");
+        uint64_t fingerprint=0;
+        check(conversation_kv_verify(incremental.kv.back(),draft.state,g,70,false,fingerprint,err),"incremental draft authoritative and ring read-back");
+        ids.resize(65);
+    }
+    check(conversation_checkpoint_restore(a.checkpoints[0],ss,g,err),"restore early running checkpoint");
+    std::vector<uint8_t> spare(sizes.dead);
+    cuda_check(cudaMemcpy(spare.data(),main.state.idx_pooled,sizes.dead,cudaMemcpyDeviceToHost));
+    check(spare==a.checkpoints[0].dead,"checkpoint rebuilds spare row over a later completed block");
+    check(ss.ple_prev[0]==2 && ss.ple_prev[1]==3,"checkpoint PLE token window");
+}
+}
+
+int main() {
+    int devices=0;
+    if (cudaGetDeviceCount(&devices)!=cudaSuccess || !devices) return 77;
+    for (int fmt : std::array<int,4>{kKvF16,kKvInt8,kKvQ4,3}) for (int mode : {0,1,2}) {
+        if (fmt==3 && mode!=0) continue;
+        Fixture f(fmt,mode);
+        if (fmt==3) {
+            std::string err;
+            ConversationKv invalid;
+            for (int unsupported : {1,2}) {
+                f.state.kv_mode=unsupported;
+                check(conversation_kv_bytes(f.state,f.g,9,true)==0,"hybrid streaming/ring rejected without block-mover abort");
+                check(!conversation_kv_save(invalid,f.state,f.g,9,true,err),"hybrid streaming/ring capture fails closed");
+            }
+            f.state.kv_mode=0;
+        }
+        {
+            std::string err;
+            ConversationKv invalid;
+            for (const int64_t bad : {-1LL, 97LL, (long long) std::numeric_limits<int64_t>::max()}) {
+                check(!conversation_kv_save(invalid,f.state,f.g,bad,mode!=2,err),"reject invalid save extent before allocation");
+                check(!conversation_kv_restore(invalid,f.state,f.g,bad,mode!=2,err),"reject invalid restore extent before copying");
+                check(conversation_kv_bytes(f.state,f.g,bad,mode!=2)==0,"invalid extent cannot overflow byte estimate");
+            }
+        }
+        for (int64_t upto : {0,1,3,4,5,63,64,65,96}) {
+            const bool index=mode!=2;
+            std::string err;
+            f.fill(13);
+            ConversationKv a,b,restored;
+            cuda_check(cudaDeviceSynchronize());
+            check(conversation_kv_save(a,f.state,f.g,upto,index,err),"save A");
+            check(a.bytes()==conversation_kv_bytes(f.state,f.g,upto,index),"size estimate equals snapshot payload");
+            f.fill(177);
+            check(conversation_kv_save(b,f.state,f.g,upto,index,err),"save B");
+            check(upto==0 || !equal(a,b),"fixture changes state");
+            check(conversation_kv_restore(a,f.state,f.g,upto,index,err),"restore A over B");
+            cuda_check(cudaDeviceSynchronize());
+            check(conversation_kv_save(restored,f.state,f.g,upto,index,err),"read back restored A");
+            check(equal(a,restored),"A/B/A byte-exact K/V and indexer state");
+            uint64_t fingerprint = 0;
+            check(conversation_kv_verify(a,f.state,f.g,upto,index,fingerprint,err),"verify restored authoritative and resident bytes without rewriting them");
+            if (mode==1) {
+                std::vector<int32_t> table((size_t)f.state.n_pages);
+                cuda_check(cudaMemcpy(table.data(),f.state.page_table,table.size()*4,cudaMemcpyDeviceToHost));
+                check(std::all_of(table.begin(),table.end(),[](int32_t x){return x==-1;}),"stale VRAM pages invalidated");
+            }
+            if (mode==2 && upto>0) {
+                const auto s=qsa_real_shapes();
+                const int64_t b1=(upto+s.page_size-1)/s.page_size,b0=std::max<int64_t>(0,b1-f.state.n_slots);
+                const size_t block=kv_block_bytes(s,fmt)/2;
+                // Codes and scales are separate in INT8; check code bytes here.
+                const size_t code_block=fmt==kKvInt8 ? s.page_size*s.n_head_kv*s.head_dim : block;
+                const void* slots=fmt==kKvQ4?(void*)f.state.k_q4:fmt==kKvInt8?(void*)f.state.k_q:(void*)f.state.k_pool;
+                std::vector<uint8_t> got(code_block);
+                for (int64_t page=b0;page<b1;++page) {
+                    cuda_check(cudaMemcpy(got.data(),(const uint8_t*)slots+(page%f.state.n_slots)*code_block,code_block,cudaMemcpyDeviceToHost));
+                    std::vector<uint8_t> expected(code_block);
+                    check(a.k.read(expected.data(), page*code_block, code_block),"read segmented expected draft page");
+                    check(got == expected,"draft ring contains restored pages");
+                }
+            }
+            auto bad=a; bad.head_dim++;
+            check(!conversation_kv_restore(bad,f.state,f.g,upto,index,err),"reject incompatible geometry");
+            if (!a.k.empty()) {
+                bad=a; bad.k.pop_back();
+                check(!conversation_kv_restore(bad,f.state,f.g,upto,index,err),"reject malformed payload");
+            }
+        }
+    }
+    for (int fmt : {kKvF16,kKvInt8,kKvQ4}) for (int mode : {0,1}) for (int experts : {256,512})
+        full_session(fmt,mode,experts);
+    for (int experts : {256,512}) full_session(3,0,experts);
+    std::printf("conversation_snapshot_test: %d checks passed\n",checks);
+}

src/core/conversation_state.cpp

@@ -0,0 +1,305 @@
+#include "strata/core/conversation_snapshot.hpp"
+#include "conversation_checked.hpp"
+
+#include <algorithm>
+#include <array>
+#include <limits>
+
+namespace strata::core {
+namespace {
+using conversation_detail::add;
+using conversation_detail::product;
+
+std::array<int64_t, 18> geometry_key(const ModelGeometry& g) {
+    return {g.n_embd, g.n_layers, g.qsa_interval, g.ssm_state_size, g.ssm_k_heads,
+            g.ssm_v_heads, g.ssm_d_conv, g.ssm_conv_channels, g.ssm_value_dim,
+            g.n_head, g.n_head_kv, g.head_dim, g.idx_q_heads, g.idx_key_dim,
+            g.hc, g.hc_lr, g.n_expert, g.n_ff};
+}
+
+bool fail(std::string& error, const char* message) {
+    error = std::string("conversation snapshot: ") + message;
+    return false;
+}
+
+bool sync(std::string& error) {
+    const auto status = cudaDeviceSynchronize();
+    if (status == cudaSuccess) return true;
+    error = std::string("conversation snapshot synchronize: ") + cudaGetErrorString(status);
+    return false;
+}
+
+bool copy(void* dst, const void* src, size_t bytes, std::string& error) {
+    if (!bytes) return true;
+    if (!dst || !src) return fail(error, "missing running-state buffer");
+    const auto status = cudaMemcpy(dst, src, bytes, cudaMemcpyDefault);
+    if (status == cudaSuccess) return true;
+    error = std::string("conversation snapshot running-state copy: ") + cudaGetErrorString(status);
+    return false;
+}
+
+// The session's carve (#216): `qsa_states` keeps global ordinals and holds the owned ones
+// [qsa_ord0, qsa_ord0 + qsa_alloc); `gdn_state` holds `gdn_alloc` rows. A whole-model session owns every layer.
+size_t owned_qsa(const SessionState& ss) { return (size_t) std::max<int64_t>(ss.qsa_alloc, 0); }
+const QsaState& owned(const SessionState& ss, size_t j) { return ss.qsa_states[(size_t) ss.qsa_ord0 + j]; }
+
+bool image_keys(const std::vector<ConversationImageKey>& images, size_t tokens) {
+    int64_t previous = -1;
+    for (const auto& image : images) {
+        if (image.start <= previous || image.start < 0 || (uint64_t) image.start >= tokens) return false;
+        previous = image.start;
+    }
+    return true;
+}
+
+bool checkpoint_targets(const SessionState& ss, const ModelGeometry& g, size_t tokens,
+                        ConversationStateSizes& z, std::string& error) {
+    if (!conversation_session_sizes(g, ss, z, error)) return false;
+    if (ss.max_cells < 0 || tokens > (uint64_t) ss.max_cells || (z.gdn && !ss.gdn_state) ||
+        (owned_qsa(ss) && !ss.qsa_states)) return fail(error, "invalid session running-state targets");
+    for (size_t j = 0; j < owned_qsa(ss); ++j) {
+        const auto& st = owned(ss, j);
+        const auto block = strata::kernels::qsa_real_shapes().idx_block;
+        size_t pooled_bytes = 0;
+        if (!st.idx_tail || !st.idx_dead || !st.idx_block_pos || !st.idx_pooled ||
+            st.max_cells < 0 || tokens > (uint64_t) st.max_cells ||
+            (tokens && tokens / (uint64_t) block >= (uint64_t) std::max<int64_t>(0, st.idx_pooled_rows)) ||
+            !product(pooled_bytes, {tokens / (uint64_t) block + 1, (uint64_t) g.idx_key_dim, sizeof(float)}))
+            return fail(error, "invalid indexer running-state target");
+    }
+    return true;
+}
+
+bool view_validate(const ConversationView& view, const SessionState& ss,
+                   const ModelGeometry& g, std::string& error) {
+    ConversationStateSizes z;
+    if (!checkpoint_targets(ss, g, view.ids.size(), z, error)) return false;
+    if (view.ids.empty() || !image_keys(view.images, view.ids.size()) ||
+        std::any_of(view.ids.begin(), view.ids.end(), [](int32_t id) { return id < 0; }))
+        return fail(error, "invalid live token/image metadata");
+    for (const auto& c : view.checkpoints) {
+        if (!c.stage_parts.empty() || c.ids.empty() || c.ids.size() > view.ids.size() ||
+            !std::equal(c.ids.begin(), c.ids.end(), view.ids.begin()))
+            return fail(error, "checkpoint is not a live token prefix");
+        if (!conversation_checkpoint_validate(c, ss, g, error)) return false;
+        size_t image = 0;
+        for (const auto& key : view.images) {
+            if ((uint64_t) key.start >= c.ids.size()) break;
+            if (image >= c.imgs.size() || !(c.imgs[image++] == key))
+                return fail(error, "checkpoint image identity differs");
+        }
+        if (image != c.imgs.size()) return fail(error, "checkpoint image prefix differs");
+    }
+    return true;
+}
+
+bool metadata_bytes(const ConversationCheckpoint& c, size_t& total) {
+    size_t ids = 0, images = 0;
+    if (!product(ids, {c.ids.size(), sizeof(int32_t)}) ||
+        !product(images, {c.imgs.size(), sizeof(ConversationImageKey)})) return false;
+    for (size_t n : {ids, images, c.gdn.size(), c.ple.size(), c.tails.size(), c.dead.size(), c.block_pos.size()})
+        if (!add(total, n)) return false;
+    return true;
+}
+} // namespace
+
+bool conversation_state_sizes(const ModelGeometry& g, ConversationStateSizes& z, std::string& error) {
+    z = {};
+    const auto key = geometry_key(g);
+    for (size_t i = 0; i < key.size(); ++i)
+        if (key[i] < 0 || (i != 1 && key[i] == 0)) return fail(error, "invalid model geometry");
+    size_t recurrence = 0, convolution = 0;
+    if (!product(recurrence, {(uint64_t) g.ssm_state_size, (uint64_t) g.ssm_v_heads, (uint64_t) g.ssm_state_size}) ||
+        !product(convolution, {(uint64_t) g.ssm_conv_channels, (uint64_t) (g.ssm_d_conv - 1)}) ||
+        !add(recurrence, convolution) ||
+        !product(z.gdn, {(uint64_t) g.n_gdn_layers(), recurrence, sizeof(float)}) ||
+        !product(z.ple, {strata::kernels::NG_HIST, strata::kernels::NG_HC_DIM, sizeof(float)}) ||
+        !product(z.tail, {(uint64_t) (strata::kernels::qsa_real_shapes().idx_block - 1),
+                          (uint64_t) g.idx_key_dim, sizeof(float)}) ||
+        !product(z.dead, {(uint64_t) g.idx_key_dim, sizeof(float)}))
+        return fail(error, "running-state byte count overflow");
+    z.block_pos = sizeof(int32_t);
+    size_t total = 0;
+    if (!product(total, {(uint64_t) g.n_qsa_layers(), z.tail}) ||
+        !product(total, {(uint64_t) g.n_qsa_layers(), z.dead}) ||
+        !product(total, {(uint64_t) g.n_qsa_layers(), z.block_pos}))
+        return fail(error, "indexer byte count overflow");
+    return true;
+}
+
+bool conversation_session_sizes(const ModelGeometry& g, const SessionState& ss, ConversationStateSizes& z,
+                                std::string& error) {
+    if (!conversation_state_sizes(g, z, error)) return false;
+    const int64_t n_gdn = g.n_gdn_layers(), n_qsa = g.n_qsa_layers();
+    if (ss.gdn_alloc < 0 || ss.gdn_alloc > n_gdn || ss.qsa_ord0 < 0 || ss.qsa_alloc < 0 ||
+        ss.qsa_ord0 > n_qsa || ss.qsa_alloc > n_qsa - ss.qsa_ord0)
+        return fail(error, "invalid session layer carve");
+    // z.gdn is n_gdn whole rows; the session holds gdn_alloc of them (all of them for the default full range)
+    z.gdn = n_gdn > 0 ? z.gdn / (size_t) n_gdn * (size_t) ss.gdn_alloc : 0;
+    return true;
+}
+
+bool conversation_checkpoint_validate(const ConversationCheckpoint& c, const SessionState& ss,
+                                      const ModelGeometry& g, std::string& error) {
+    ConversationStateSizes z;
+    if (!checkpoint_targets(ss, g, c.ids.size(), z, error)) return false;
+    const size_t layers = owned_qsa(ss);
+    if (c.gdn.size() != z.gdn || c.ple.size() != (ss.ple_hist ? z.ple : 0) ||
+        c.tails.size() != layers * z.tail || c.dead.size() != layers * z.dead ||
+        c.block_pos.size() != layers * z.block_pos || !image_keys(c.imgs, c.ids.size()))
+        return fail(error, "invalid checkpoint running-state payload");
+    return true;
+}
+
+bool conversation_checkpoint_save(ConversationCheckpoint& c, const SessionState& ss,
+                                  const ModelGeometry& g, std::string& error) {
+    ConversationStateSizes z;
+    if (!checkpoint_targets(ss, g, c.ids.size(), z, error)) return false;
+    const size_t layers = owned_qsa(ss);
+    c.gdn.resize(z.gdn); c.ple.resize(ss.ple_hist ? z.ple : 0);
+    c.tails.resize(layers * z.tail); c.dead.resize(layers * z.dead); c.block_pos.resize(layers * z.block_pos);
+    if (!copy(c.gdn.data(), ss.gdn_state, c.gdn.size(), error) ||
+        !copy(c.ple.data(), ss.ple_hist, c.ple.size(), error)) return false;
+    for (size_t j = 0; j < layers; ++j) {
+        const auto& st = owned(ss, j);
+        if (!copy(c.tails.data() + j * z.tail, st.idx_tail, z.tail, error) ||
+            !copy(c.dead.data() + j * z.dead, st.idx_dead, z.dead, error) ||
+            !copy(c.block_pos.data() + j * z.block_pos, st.idx_block_pos, z.block_pos, error)) return false;
+    }
+    return true;
+}
+
+bool conversation_checkpoint_restore(const ConversationCheckpoint& c, SessionState& ss,
+                                     const ModelGeometry& g, std::string& error) {
+    if (!conversation_checkpoint_validate(c, ss, g, error)) return false;
+    ConversationStateSizes z;
+    if (!conversation_session_sizes(g, ss, z, error)) return false;
+    if (!copy(ss.gdn_state, c.gdn.data(), c.gdn.size(), error) ||
+        !copy(ss.ple_hist, c.ple.data(), c.ple.size(), error)) return false;
+    for (size_t j = 0; j < owned_qsa(ss); ++j) {
+        const auto& st = owned(ss, j);
+        if (!copy(st.idx_tail, c.tails.data() + j * z.tail, z.tail, error) ||
+            !copy(st.idx_dead, c.dead.data() + j * z.dead, z.dead, error) ||
+            !copy(st.idx_block_pos, c.block_pos.data() + j * z.block_pos, z.block_pos, error)) return false;
+        if (!c.ids.empty()) {
+            const size_t row = c.ids.size() / strata::kernels::qsa_real_shapes().idx_block;
+            if (!copy(st.idx_pooled + row * g.idx_key_dim, c.dead.data() + j * z.dead, z.dead, error)) return false;
+        }
+    }
+    const size_t tokens = c.ids.size();
+    ss.ple_prev[0] = tokens >= 2 ? c.ids[tokens - 2] : -1;
+    ss.ple_prev[1] = tokens >= 1 ? c.ids[tokens - 1] : -1;
+    return sync(error);
+}
+
+bool conversation_snapshot_bytes(const ConversationView& view, const SessionState& ss,
+                                 const ModelGeometry& g, const QsaState& draft, size_t& bytes, std::string& error) {
+    bytes = 0;
+    if (!view_validate(view, ss, g, error)) return false;
+    ConversationStateSizes z;
+    if (!conversation_session_sizes(g, ss, z, error)) return false;
+    size_t ids = 0, images = 0, checkpoints = 0, layers = 0, tails = 0, dead = 0, positions = 0;
+    const auto qsa = (uint64_t) owned_qsa(ss);
+    if (!product(ids, {view.ids.size(), sizeof(int32_t)}) ||
+        !product(images, {view.images.size(), sizeof(ConversationImageKey)}) ||
+        !product(checkpoints, {view.checkpoints.size(), sizeof(ConversationCheckpoint)}) ||
+        !product(layers, {qsa + 1, sizeof(ConversationKv)}) ||
+        !product(tails, {qsa, z.tail}) || !product(dead, {qsa, z.dead}) ||
+        !product(positions, {qsa, z.block_pos})) return fail(error, "snapshot metadata byte count overflow");
+    for (size_t n : {ids, images, checkpoints, layers, tails, dead, positions, z.gdn, ss.ple_hist ? z.ple : 0})
+        if (!add(bytes, n)) return fail(error, "snapshot byte count overflow");
+    for (const auto& c : view.checkpoints)
+        if (!metadata_bytes(c, bytes)) return fail(error, "checkpoint byte count overflow");
+    const int64_t upto = (int64_t) view.ids.size(); // view_validate bounds this by signed max_cells
+    for (uint64_t i = 0; i <= qsa; ++i) {
+        const auto& st = i == qsa ? draft : owned(ss, (size_t) i);
+        const size_t n = conversation_kv_bytes(st, g, upto, i != qsa);
+        if (!n || !add(bytes, n)) return fail(error, "invalid or overflowing K/V byte estimate");
+    }
+    return true;
+}
+
+bool conversation_snapshot_capture_bytes(const ConversationKvReuse& reuse, const ConversationView& view,
+                                         const SessionState& ss, const ModelGeometry& g,
+                                         const QsaState& draft, size_t& bytes, std::string& error) {
+    if (!conversation_snapshot_bytes(view, ss, g, draft, bytes, error)) return false;
+    if (reuse.kv.empty()) return true;
+    const size_t layers = owned_qsa(ss) + 1;
+    if (reuse.kv.size() != layers || reuse.unchanged_tokens < 0 ||
+        reuse.unchanged_tokens > reuse.captured_tokens || reuse.unchanged_tokens > int64_t(view.ids.size()))
+        return fail(error, "invalid retained K/V prefix");
+    for (size_t i = 0; i < layers; ++i) {
+        const bool index = i + 1 != layers;
+        const auto& st = index ? owned(ss, i) : draft;
+        if (!conversation_kv_validate(reuse.kv[i], st, g, reuse.captured_tokens, index, error)) return false;
+        const size_t fresh = conversation_kv_bytes(st, g, int64_t(view.ids.size()), index);
+        size_t retained = 0;
+        if (!conversation_kv_capture_bytes(reuse.kv[i], st, g, int64_t(view.ids.size()), index, retained, error)) return false;
+        bytes -= fresh;
+        if (!add(bytes, retained)) return fail(error, "retained K/V allocation overflow");
+    }
+    size_t directory = 0;
+    if (!product(directory, {reuse.kv.capacity() - layers, sizeof(ConversationKv)}) || !add(bytes, directory))
+        return fail(error, "retained K/V directory overflow");
+    return true;
+}
+
+bool conversation_snapshot_save(SavedConversation& image, const ConversationView& view,
+                                const SessionState& ss, const ModelGeometry& g,
+                                const QsaState& draft, std::string& error,
+                                ConversationKvReuse reuse, size_t* reused_bytes) {
+    size_t estimate = 0;
+    if (!conversation_snapshot_capture_bytes(reuse, view, ss, g, draft, estimate, error) || !sync(error)) return false;
+    // Build into a new object so a failure cannot publish a partial snapshot.
+    SavedConversation captured;
+    captured.geometry = geometry_key(g);
+    captured.layer_lo = ss.layer_lo; captured.layer_hi = ss.layer_hi;
+    captured.live.ids = view.ids; captured.live.imgs = view.images;
+    captured.cvec = view.cvec; captured.checkpoints = view.checkpoints;
+    const int64_t unchanged = reuse.kv.empty() ? 0 : reuse.unchanged_tokens;
+    captured.kv = std::move(reuse.kv);
+    const size_t layers = owned_qsa(ss);
+    captured.kv.resize(layers + 1);
+    if (!conversation_checkpoint_save(captured.live, ss, g, error)) return false;
+    const int64_t upto = (int64_t) view.ids.size();
+    for (size_t j = 0; j < layers; ++j)
+        if (!conversation_kv_save(captured.kv[j], owned(ss, j), g, upto, true, error,
+                                  unchanged, reused_bytes)) return false;
+    // The draft's final cell may not have been computed when the output cap was
+    // reached. Refresh that page even when the main prefix continued unchanged.
+    if (!conversation_kv_save(captured.kv.back(), draft, g, upto, false, error,
+                              std::max<int64_t>(0, unchanged - 1), reused_bytes)) return false;
+    image = std::move(captured);
+    return true;
+}
+
+bool conversation_snapshot_validate(const SavedConversation& image, const SessionState& ss,
+                                    const ModelGeometry& g, const QsaState& draft, std::string& error) {
+    if (!image.live.stage_parts.empty()) return fail(error, "layer-split parking is not supported");
+    if (image.geometry != geometry_key(g)) return fail(error, "incompatible runtime geometry");
+    // an image holds exactly one carve's running state and K/V: same layer range or nothing
+    if (image.layer_lo != ss.layer_lo || image.layer_hi != ss.layer_hi)
+        return fail(error, "snapshot from another session layer range");
+    const ConversationView view{image.live.ids, image.live.imgs, image.checkpoints, image.cvec};
+    if (!view_validate(view, ss, g, error) || !conversation_checkpoint_validate(image.live, ss, g, error)) return false;
+    const size_t layers = owned_qsa(ss);
+    if (image.kv.size() != layers + 1) return fail(error, "invalid K/V layer count");
+    const int64_t upto = (int64_t) image.live.ids.size();
+    for (size_t j = 0; j < layers; ++j)
+        if (!conversation_kv_validate(image.kv[j], owned(ss, j), g, upto, true, error)) return false;
+    return conversation_kv_validate(image.kv.back(), draft, g, upto, false, error);
+}
+
+ConversationRestore conversation_snapshot_restore(const SavedConversation& image, SessionState& ss,
+                                                   const ModelGeometry& g, const QsaState& draft, std::string& error) {
+    if (!conversation_snapshot_validate(image, ss, g, draft, error)) return ConversationRestore::invalid;
+    if (!sync(error)) return ConversationRestore::transfer_failed;
+    const int64_t upto = (int64_t) image.live.ids.size();
+    for (size_t j = 0; j < owned_qsa(ss); ++j)
+        if (!conversation_kv_restore(image.kv[j], owned(ss, j), g, upto, true, error))
+            return ConversationRestore::transfer_failed;
+    if (!conversation_kv_restore(image.kv.back(), draft, g, upto, false, error) ||
+        !conversation_checkpoint_restore(image.live, ss, g, error)) return ConversationRestore::transfer_failed;
+    return ConversationRestore::restored;
+}
+} // namespace strata::core

src/core/conversation_validation_test.cpp

@@ -0,0 +1,239 @@
+// Host-only validation fixtures. CUDA is linked but must never be initialized:
+// invalid restores must return before the first CUDA call or destination write.
+#include "strata/core/conversation_snapshot.hpp"
+#include "strata/kernels/kv_q4.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+#include <functional>
+#include <limits>
+#include <cstring>
+
+#if defined(CONVERSATION_TEST_TRANSFERS)
+// GNU/ELF link wrapping exercises the actual restore control flow without
+// initializing CUDA or deliberately poisoning a real device context. This is
+// host-backend fault injection, not a claim of hardware-failure recovery.
+namespace {
+int copy_calls = 0, sync_calls = 0, fail_copy = 0, fail_sync = 0;
+size_t copied_bytes = 0;
+}
+extern "C" cudaError_t __wrap_cudaMemcpy(void* dst, const void* src, size_t n, cudaMemcpyKind) {
+    if (++copy_calls == fail_copy) return cudaErrorInvalidValue;
+    copied_bytes += n;
+    std::memcpy(dst, src, n);
+    return cudaSuccess;
+}
+extern "C" cudaError_t __wrap_cudaDeviceSynchronize() {
+    return ++sync_calls == fail_sync ? cudaErrorUnknown : cudaSuccess;
+}
+extern "C" cudaError_t __wrap_cudaGetLastError() { return cudaSuccess; }
+#endif
+
+using namespace strata::core;
+namespace {
+int checks = 0;
+void check(bool ok, const char* label) {
+    ++checks;
+    if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
+}
+struct Pools {
+    QsaState st;
+    std::array<std::vector<uint8_t>, 8> data;
+    Pools(const ModelGeometry& g, int format) {
+        st.max_cells = 96; st.n_pages = st.n_slots = 24; st.idx_pooled_rows = 26;
+        st.kv_int8 = format == 1; st.kv_q4 = format == 2;
+        const size_t per = format == 2 ? strata::kernels::kv_q4_bytes_per_head((int) g.head_dim)
+                                      : g.head_dim * (format == 1 ? 1 : 2);
+        data[0].resize(96 * g.n_head_kv * per, 0xa5); data[1] = data[0];
+        data[2].resize(format == 1 ? 96 * g.n_head_kv * (g.head_dim / 64) * 2 : 0, 0xa5); data[3] = data[2];
+        data[4].resize(26 * g.idx_key_dim * 4, 0xa5);
+        data[5].resize(3 * g.idx_key_dim * 4, 0xa5);
+        data[6].resize(g.idx_key_dim * 4, 0xa5); data[7].resize(4, 0xa5);
+        if (format == 2) { st.k_q4 = data[0].data(); st.v_q4 = data[1].data(); }
+        else if (format == 1) {
+            st.k_q = (int8_t*) data[0].data(); st.v_q = (int8_t*) data[1].data();
+            st.k_scale = (uint16_t*) data[2].data(); st.v_scale = (uint16_t*) data[3].data();
+        } else { st.k_pool = (uint16_t*) data[0].data(); st.v_pool = (uint16_t*) data[1].data(); }
+        st.idx_pooled = (float*) data[4].data(); st.idx_tail = (float*) data[5].data();
+        st.idx_dead = (float*) data[6].data(); st.idx_block_pos = (int32_t*) data[7].data();
+    }
+    ConversationKv image(const ModelGeometry& g, bool index) const {
+        ConversationKv k;
+        k.format = qsa_kv_format(st); k.cells = 12; k.page_size = 4;
+        k.heads = g.n_head_kv; k.head_dim = g.head_dim; k.idx_dim = g.idx_key_dim;
+        k.pooled_rows = index ? 3 : 0;
+        k.k.resize(data[0].size() / 8, 13); k.v = k.k;
+        k.k_scale.resize(data[2].size() / 8, 13); k.v_scale = k.k_scale;
+        k.pooled.resize(k.pooled_rows * g.idx_key_dim * 4, 13);
+        return k;
+    }
+};
+
+void fixture(int format, int experts, bool zero_qsa, bool ple) {
+    ModelGeometry g;
+    g.n_layers = zero_qsa ? 3 : 8; g.n_expert = experts;
+    g.ssm_state_size = 2; g.ssm_v_heads = 2; g.ssm_conv_channels = 8;
+    g.n_head_kv = 1; g.head_dim = 64; g.idx_key_dim = 8;
+    std::string error;
+    ConversationStateSizes z;
+    check(conversation_state_sizes(g, z, error), "checked canonical/pruned/zero-QSA sizing");
+    Pools first(g, format), last(g, format), draft(g, format);
+    std::array<QsaState, 2> layers{first.st, last.st};
+    std::vector<uint8_t> gdn(z.gdn, 0xa5), history(ple ? z.ple : 0, 0xa5);
+    SessionState ss;
+    ss.max_cells = 96; ss.gdn_state = (float*) gdn.data();
+    // the whole-model carve, as session_init leaves a one-GPU session
+    ss.layer_hi = g.n_layers; ss.gdn_alloc = g.n_gdn_layers(); ss.qsa_alloc = g.n_qsa_layers();
+    ss.ple_hist = ple ? (float*) history.data() : nullptr;
+    ss.qsa_states = zero_qsa ? nullptr : layers.data();
+    SavedConversation image;
+    image.geometry = {g.n_embd,g.n_layers,g.qsa_interval,g.ssm_state_size,g.ssm_k_heads,g.ssm_v_heads,
+                      g.ssm_d_conv,g.ssm_conv_channels,g.ssm_value_dim,g.n_head,g.n_head_kv,g.head_dim,
+                      g.idx_q_heads,g.idx_key_dim,g.hc,g.hc_lr,g.n_expert,g.n_ff};
+    image.layer_hi = g.n_layers;
+    auto checkpoint = [&](size_t tokens) {
+        ConversationCheckpoint c;
+        c.ids.resize(tokens);
+        for (size_t i = 0; i < tokens; ++i) c.ids[i] = (int32_t) i + 1;
+        c.imgs = {{1, 44}};
+        c.gdn.resize(z.gdn, 13); c.ple.resize(ple ? z.ple : 0, 13);
+        c.tails.resize(g.n_qsa_layers() * z.tail, 13); c.dead.resize(g.n_qsa_layers() * z.dead, 13);
+        c.block_pos.resize(g.n_qsa_layers() * z.block_pos, 13);
+        return c;
+    };
+    image.live = checkpoint(9); image.checkpoints.push_back(checkpoint(5));
+    image.kv.reserve((size_t) g.n_qsa_layers() + 1);
+    if (!zero_qsa) { image.kv.push_back(first.image(g, true)); image.kv.push_back(last.image(g, true)); }
+    image.kv.push_back(draft.image(g, false));
+    check(conversation_snapshot_validate(image, ss, g, draft.st, error), "complete image validates without CUDA");
+    size_t estimate = 0;
+    check(conversation_snapshot_bytes({image.live.ids,image.live.imgs,image.checkpoints,true},ss,g,draft.st,estimate,error),
+          "capture estimate works without CUDA");
+    check(estimate == image.bytes(), "estimate covers checkpoint/indexer/spare payloads");
+    auto unchanged = [&] {
+        auto pristine = [](const auto& bytes) { return std::all_of(bytes.begin(),bytes.end(),[](uint8_t b){return b==0xa5;}); };
+        if (!pristine(gdn) || !pristine(history) || ss.ple_prev[0] != -1 || ss.ple_prev[1] != -1) return false;
+        for (const auto* p : {&first,&last,&draft}) for (const auto& bytes : p->data) if (!pristine(bytes)) return false;
+        return true;
+    };
+    auto reject = [&](const std::function<void(SavedConversation&)>& mutate, const char* label) {
+        auto bad = image; mutate(bad);
+        check(conversation_snapshot_restore(bad,ss,g,draft.st,error) == ConversationRestore::invalid,label);
+        check(unchanged(), "invalid restore did not touch any layer or running-state buffer");
+    };
+    reject([](auto& s){s.kv.back().k.pop_back();}, "late draft corruption rejected before first layer write");
+    reject([](auto& s){s.geometry[16] = 128;}, "different expert geometry rejected");
+    reject([](auto& s){s.live.stage_parts.emplace_back();}, "layer-split live state rejected before writes");
+    reject([](auto& s){s.checkpoints[0].stage_parts.emplace_back();}, "layer-split checkpoint rejected before writes");
+    reject([](auto& s){s.kv.pop_back();}, "missing KV layer rejected");
+    reject([](auto& s){s.live.gdn.pop_back();}, "bad live recurrence rejected");
+    reject([](auto& s){s.checkpoints.back().gdn.pop_back();}, "bad retained checkpoint rejected");
+    reject([](auto& s){s.checkpoints.back().ids[0] = 99;}, "foreign checkpoint prefix rejected");
+    reject([](auto& s){s.checkpoints.back().imgs[0].hash++;}, "foreign checkpoint image rejected");
+    reject([](auto& s){s.live.imgs[0].start = -1;}, "negative image position rejected");
+    reject([](auto& s){s.live.ids[0] = -1;}, "negative live token rejected");
+    reject([](auto& s){s.live.ple.push_back(0);}, "PLE size/enabled mismatch rejected");
+    reject([](auto& s){s.live.tails.push_back(0);}, "tail size/zero-QSA mismatch rejected");
+    if (!zero_qsa) {
+        reject([](auto& s){s.kv[1].pooled.pop_back();}, "late indexer spare row corruption rejected");
+        reject([](auto& s){s.live.dead.pop_back();}, "missing indexer spare key rejected");
+        reject([](auto& s){s.live.block_pos.pop_back();}, "missing indexer metadata rejected");
+        ss.qsa_states = nullptr;
+        check(!conversation_snapshot_validate(image,ss,g,draft.st,error), "missing QSA targets rejected");
+        ss.qsa_states = layers.data();
+        reject([](auto& s){s.layer_lo = 4;}, "snapshot from another layer range rejected");
+        // #216's carve: a split stage owning layers [4, 8) holds GDN rows 3..5 and QSA ordinal 1 only
+        SessionState stage = ss;
+        stage.layer_lo = 4; stage.gdn_alloc = 3; stage.qsa_ord0 = 1; stage.qsa_alloc = 1;
+        ConversationStateSizes zs;
+        check(conversation_session_sizes(g,stage,zs,error) && zs.gdn == z.gdn / 6 * 3 && zs.tail == z.tail,
+              "carved session sizes cover the owned rows only");
+        check(!conversation_checkpoint_validate(image.live,stage,g,error), "whole-model checkpoint rejected by a carve");
+        ConversationCheckpoint part = image.live;
+        part.gdn.resize(zs.gdn); part.tails.resize(zs.tail); part.dead.resize(zs.dead);
+        part.block_pos.resize(zs.block_pos);
+        check(conversation_checkpoint_validate(part,stage,g,error), "carve-sized checkpoint validates");
+        check(!conversation_snapshot_validate(image,stage,g,draft.st,error), "whole-model snapshot rejected by a carve");
+        stage.qsa_alloc = 2;
+        check(!conversation_session_sizes(g,stage,zs,error), "carve past the last QSA ordinal rejected");
+        stage.qsa_alloc = 1; stage.gdn_alloc = 7;
+        check(!conversation_session_sizes(g,stage,zs,error), "carve past the GDN rows rejected");
+    }
+    auto bad_geometry = g; bad_geometry.ssm_state_size = std::numeric_limits<int64_t>::max();
+    check(!conversation_state_sizes(bad_geometry,z,error), "running-state arithmetic overflow rejected");
+    bad_geometry = g; bad_geometry.qsa_interval = 0;
+    check(!conversation_state_sizes(bad_geometry,z,error), "zero layer interval rejected before division");
+    bad_geometry = g; bad_geometry.n_head_kv = std::numeric_limits<int64_t>::max();
+    check(conversation_kv_bytes(draft.st,bad_geometry,9,false)==0, "KV byte overflow rejected");
+#if defined(CONVERSATION_TEST_TRANSFERS)
+    auto reset = [&] {
+        for (auto* p : {&first,&last,&draft}) for (auto& bytes : p->data) std::fill(bytes.begin(),bytes.end(),0xa5);
+        std::fill(gdn.begin(),gdn.end(),0xa5); std::fill(history.begin(),history.end(),0xa5);
+        ss.ple_prev[0] = ss.ple_prev[1] = -1;
+        copy_calls = sync_calls = fail_copy = fail_sync = 0;
+    };
+    reset(); fail_sync = 1;
+    check(conversation_snapshot_restore(image,ss,g,draft.st,error)==ConversationRestore::transfer_failed,
+          "pre-transfer CUDA synchronization failure is fatal");
+    check(copy_calls==0 && unchanged(), "pre-sync failure did not mutate state");
+    reset(); fail_copy = 1;
+    check(conversation_snapshot_restore(image,ss,g,draft.st,error)==ConversationRestore::transfer_failed,
+          "first CUDA transfer failure is fatal");
+    check(unchanged(), "first-copy failure did not mutate state");
+    reset(); fail_copy = 2;
+    check(conversation_snapshot_restore(image,ss,g,draft.st,error)==ConversationRestore::transfer_failed,
+          "partial CUDA transfer failure is fatal, not an invalid-image fallback");
+    check(!unchanged(), "fault fixture genuinely produced partial state");
+    reset(); fail_sync = 2;
+    check(conversation_snapshot_restore(image,ss,g,draft.st,error)==ConversationRestore::transfer_failed,
+          "post-transfer CUDA synchronization failure is fatal");
+    check(copy_calls>0 && !unchanged(), "post-sync failure occurred after state writes");
+    reset();
+    check(conversation_snapshot_restore(image,ss,g,draft.st,error)==ConversationRestore::restored,
+          "injected host transfer backend completes a valid restore");
+    check(gdn==image.live.gdn && history==image.live.ple && ss.ple_prev[0]==8 && ss.ple_prev[1]==9,
+          "successful restore publishes correct running state and PLE window");
+    uint64_t fingerprint = 0;
+    check(conversation_kv_verify(image.kv.back(),draft.st,g,9,false,fingerprint,error), "read-back verifies complete draft payload");
+    draft.data[0][0] ^= 1;
+    check(!conversation_kv_verify(image.kv.back(),draft.st,g,9,false,fingerprint,error), "read-back detects corrupted draft byte");
+    draft.data[0][0] ^= 1;
+    fail_copy = copy_calls + 1;
+    check(!conversation_kv_verify(image.kv.back(),draft.st,g,9,false,fingerprint,error), "read-back transfer failure is not a successful fingerprint");
+    reset(); copied_bytes = 0;
+    const ConversationView view{image.live.ids,image.live.imgs,image.checkpoints,true};
+    SavedConversation full,incremental;
+    check(conversation_snapshot_save(full,view,ss,g,draft.st,error), "full capture through host transfer backend");
+    const size_t full_copies = copied_bytes;
+    ConversationKvReuse reuse{full.kv,9,9};
+    size_t peak = 0, reused = 0;
+    check(conversation_snapshot_capture_bytes(reuse,view,ss,g,draft.st,peak,error), "incremental capture admission without transfers");
+    copied_bytes = 0;
+    const bool saved_incrementally = conversation_snapshot_save(incremental,view,ss,g,draft.st,error,std::move(reuse),&reused);
+    if (!saved_incrementally) std::fprintf(stderr,"capture error: %s\n",error.c_str());
+    check(saved_incrementally, "incremental capture through host backend");
+    check(reused > 0 && copied_bytes + reused == full_copies, "reused byte count measures transfers actually omitted");
+    check(incremental.bytes() <= peak && incremental.live.gdn == full.live.gdn, "running state refreshed within admitted allocation");
+    for (size_t i=0;i<full.kv.size();++i)
+        check(incremental.kv[i].k==full.kv[i].k && incremental.kv[i].v==full.kv[i].v &&
+              incremental.kv[i].k_scale==full.kv[i].k_scale && incremental.kv[i].v_scale==full.kv[i].v_scale &&
+              incremental.kv[i].pooled==full.kv[i].pooled, "incremental and full capture agree across every payload");
+    reuse = {full.kv,9,9};
+    reuse.kv.back().k.pop_back();
+    copy_calls = sync_calls = 0;
+    check(!conversation_snapshot_capture_bytes(reuse,view,ss,g,draft.st,peak,error), "malformed retained draft rejected before admission");
+    check(!copy_calls && !sync_calls, "invalid retained storage performs no CUDA calls");
+    reuse = {full.kv,9,9};
+    fail_copy = copy_calls + 1;
+    SavedConversation unpublished;
+    check(!conversation_snapshot_save(unpublished,view,ss,g,draft.st,error,std::move(reuse)), "incremental capture copy failure is reported");
+    check(unpublished.kv.empty() && unpublished.live.ids.empty(), "failed incremental capture cannot publish a partial image");
+#endif
+}
+}
+
+int main() {
+    for (int format : {0,1,2}) for (int experts : {256,512})
+        for (bool zero_qsa : {false,true}) for (bool ple : {false,true}) fixture(format,experts,zero_qsa,ple);
+    std::printf("conversation_validation_test: %d host-only checks passed\n", checks);
+}

src/core/coupled_draft_test.cpp

@@ -0,0 +1,115 @@
+// src/core/coupled_draft_test.cpp - coupled draft sampling's host-side arithmetic (include/strata/core/coupled_draft.hpp).
+//
+// No GPU.  A simulated decode - windows of random size, random accepted counts, the serve loop's first window of one
+// token and the CLI's `draft_first` - checks that
+//   1. every draft the MTP chain makes is drawn with the Philox counter of the verify row that checks it: the verify
+//      window at pos0 draws row t with pos0 + t (verify.cpp run()), and row t checks the draft in row t + 1;
+//   2. the MTP cell of each draft predicts exactly the position that draft occupies in the next window;
+//   3. the penalty window each draft sees in the device ring (the staged base, then the chain's earlier drafts) is
+//      token for token the history row `penalty_rows` gives the verify row that checks it;
+//   4. the penalty window length matches the target's (serve caps the history at 4,096).
+#include "strata/core/coupled_draft.hpp"
+
+#include <cstdio>
+#include <random>
+#include <vector>
+
+using namespace strata::core;
+
+namespace {
+int g_fail = 0;
+int64_t g_checks = 0;
+void check(bool ok, const char* what, long long a = 0, long long b = 0) {
+    ++g_checks;
+    if (!ok) {
+        if (++g_fail <= 20) std::printf("  FAIL %s (%lld vs %lld)\n", what, a, b);
+    }
+}
+
+// verify.cpp run(): sp.counter = pos0, and the kernel draws row t with sp.counter + t
+uint64_t verify_row_counter(int64_t pos0, int t) { return (uint64_t) pos0 + (uint64_t) t; }
+
+// The counters and positions of a chain of `n` drafts made after a window at `p` with `a` accepted (draft()), or for
+// draft_first (cell0 = p - 1, a = 0), checked against the next window at `p_next`.
+void check_chain(int64_t p, int a, int n, int64_t p_next) {
+    for (int j = 0; j < n; ++j) {
+        const int64_t cell = coupled_draft_cell(p, a, j);
+        // mtp.hpp: cell c predicts the token at c + 2; the next window's row j + 1 holds position p_next + j + 1
+        check(cell + 2 == p_next + j + 1, "draft position = its row in the next window", cell + 2, p_next + j + 1);
+        // row j + 1 is kept iff it equals row j's pick, drawn at p_next + j
+        check(coupled_draft_counter(cell) == verify_row_counter(p_next, j), "draft counter = verifying row's counter",
+              (long long) coupled_draft_counter(cell), (long long) verify_row_counter(p_next, j));
+    }
+}
+
+// The device ring for one chain: base staged at [cap - h, cap) by the host, draft i at cap + i; draft j's window is
+// [coupled_hist_start(cap, j, h), + h).  Compared with penalty_rows over (consumed, [next, drafts...]).
+void check_history(std::mt19937& rng, int cap, int h, int n_consumed, int n_drafts) {
+    std::uniform_int_distribution<int> tok(0, 999);
+    std::vector<int32_t> consumed((size_t) n_consumed), drafts((size_t) n_drafts);
+    for (auto& t : consumed) t = tok(rng);
+    for (auto& t : drafts) t = tok(rng);
+    const int32_t next = tok(rng);
+    std::vector<int32_t> mapped((size_t) cap, 777777), ring((size_t) (cap + n_drafts), 888888);
+    coupled_hist_base(consumed.data(), (int64_t) consumed.size(), next, h, mapped.data() + (cap - h));   // set_draft_history
+    for (int i = cap - h; i < cap; ++i) ring[(size_t) i] = mapped[(size_t) i];                           // coupled_stage_kernel
+    // the next window: [next, drafts...], T = n_drafts + 1 rows; row j checks draft j (row j + 1)
+    std::vector<int32_t> window;
+    window.push_back(next);
+    window.insert(window.end(), drafts.begin(), drafts.end());
+    const int T = (int) window.size();
+    std::vector<int32_t> rows((size_t) T * (size_t) h);
+    strata::kernels::penalty_rows(consumed.data(), (int64_t) consumed.size(), window.data(), T, h, rows.data());
+    for (int j = 0; j < n_drafts; ++j) {
+        const int s = coupled_hist_start(cap, j, h);
+        check(s >= 0 && s + h <= cap + n_drafts, "the window lies inside the ring", s, cap + n_drafts);
+        for (int i = 0; i < h; ++i)
+            check(ring[(size_t) (s + i)] == rows[(size_t) j * (size_t) h + (size_t) i], "draft history = verify row history",
+                  ring[(size_t) (s + i)], rows[(size_t) j * (size_t) h + (size_t) i]);
+        ring[(size_t) (cap + j)] = drafts[(size_t) j];   // coupled_merge_kernel appends the draft after sampling it
+    }
+}
+}  // namespace
+
+int main() {
+    std::printf("coupled_draft_test\n");
+    std::mt19937 rng(20260930);
+    // ---- 1 + 2: a serve-style decode (the first window is the last prompt token alone, then MTP windows)
+    for (int run = 0; run < 200; ++run) {
+        std::uniform_int_distribution<int> tdist(1, 8);
+        int64_t p = 100 + run * 37;   // n - 1
+        int T = 1;
+        for (int round = 0; round < 300; ++round) {
+            std::uniform_int_distribution<int> adist(0, T - 1);
+            const int a = adist(rng);
+            const int64_t p_next = p + a + 1;
+            const int n = 7;   // the chain makes up to max_t - 1 drafts, whatever T the next window takes
+            check_chain(p, a, n, p_next);
+            p = p_next;
+            T = tdist(rng);    // --spec-min-p / the draft policy choose the next window's size
+        }
+    }
+    // the CLI's draft_first: the first window at p, drafts from cell p - 1 with a = 0
+    for (int64_t p = 1; p < 5000; p += 13) check_chain(p - 1, 0, 7, p);
+    // ---- 3: the penalty ring against penalty_rows
+    for (int cap : {8, 64, kCoupledHistCap}) {
+        for (int h : {1, 3, 8}) {
+            if (h > cap) continue;
+            for (int n_consumed : {0, 1, 2, 5, 7, 20, 100})
+                for (int n_drafts : {1, 3, 7}) check_history(rng, cap, h, n_consumed, n_drafts);
+        }
+        check_history(rng, cap, cap, 3, 7);        // the window is the whole ring base
+        check_history(rng, cap, cap, 5000, 7);
+    }
+    // ---- 4: the window length the target uses: min(penalty_last_n, history_len) with history_len =
+    // min(penalty_last_n, 4096) in serve (0 = off)
+    for (int pln : {-1, 0, 1, 64, 4095, 4096, 4097, 100000}) {
+        const int hist_n = std::min(std::max(pln, 0), 4096);
+        const int target = hist_n > 0 ? std::min(pln, hist_n) : 0;
+        check(coupled_hist_len(pln, kCoupledHistCap) == target, "penalty window length = the target's",
+              coupled_hist_len(pln, kCoupledHistCap), target);
+    }
+    std::printf("  %lld checks, %d failed\n", (long long) g_checks, g_fail);
+    std::printf("%s\n", g_fail == 0 ? "PASS" : "FAIL");
+    return g_fail == 0 ? 0 : 1;
+}

src/core/device.cu

@@ -24,16 +24,82 @@ __global__ void poison_kernel(float* p, uint64_t n_floats) {
     if (i < n_floats) p[i] = __int_as_float(0x7fc00000);
 }
 
+#if defined(STRATA_USE_HIP)
+#if !defined(STRATA_HIP_ARCHS)
+#error "STRATA_HIP_ARCHS (the compiled HIP architectures) is set by cmake/hip_backend.cmake"
+#endif
+// "gfx1201:sramecc-:xnack-" -> "gfx1201"
+std::string base_arch(const char* gcn_arch_name) {
+    std::string arch(gcn_arch_name);
+    const size_t colon = arch.find(':');
+    if (colon != std::string::npos) arch.resize(colon);
+    return arch;
+}
+
+bool compiled_for(const std::string& arch) {
+    const std::string list = STRATA_HIP_ARCHS;
+    size_t a = 0;
+    while (a <= list.size()) {
+        size_t b = list.find(',', a);
+        if (b == std::string::npos) b = list.size();
+        if (!arch.empty() && list.compare(a, b - a, arch) == 0 && b - a == arch.size()) return true;
+        a = b + 1;
+    }
+    return false;
+}
+
+std::string arch_problem(const cudaDeviceProp& p, int ordinal) {
+    const std::string arch = base_arch(p.gcnArchName);
+    const std::string card = "GPU " + std::to_string(ordinal) + " (" + p.name + ", " + arch + ")";
+    if (!compiled_for(arch)) {
+        return card + " is not an architecture this Strata engine was compiled for (" + STRATA_HIP_ARCHS +
+               "); compile it for this card (./setup.sh --backend hip, or -DCMAKE_HIP_ARCHITECTURES=" + arch +
+               ", docs/AMD_HIP.md) or choose another GPU with HIP_VISIBLE_DEVICES";
+    }
+    if (p.warpSize != 32) {
+        return card + " runs wave" + std::to_string(p.warpSize) + "; Strata's HIP kernels need wave32";
+    }
+    return "";
+}
+#endif
+
 }  // namespace
 
+const char* compiled_gpu_archs() {
+#if defined(STRATA_USE_HIP)
+    return STRATA_HIP_ARCHS;
+#else
+    return "";
+#endif
+}
+
+std::string gpu_arch_problem(int ordinal) {
+#if defined(STRATA_USE_HIP)
+    int count = 0;
+    if (cudaGetDeviceCount(&count) != cudaSuccess || ordinal < 0 || ordinal >= count) {
+        cudaGetLastError();
+        return "";
+    }
+    cudaDeviceProp p{};
+    if (cudaGetDeviceProperties(&p, ordinal) != cudaSuccess) {
+        cudaGetLastError();
+        return "";
+    }
+    return arch_problem(p, ordinal);
+#else
+    (void) ordinal;
+    return "";
+#endif
+}
+
 DeviceInfo device_info(int ordinal) {
     int count = 0;
     check(cudaGetDeviceCount(&count), "cudaGetDeviceCount");
     if (count == 0) {
 #if defined(STRATA_USE_HIP)
-        throw CudaError("no HIP device is present; this backend targets gfx1100 wave32", -1);
+        throw CudaError(std::string("no HIP device is present; this engine was compiled for ") + STRATA_HIP_ARCHS, -1);
 #else
-        throw CudaError("no CUDA device is present; Strata needs an NVIDIA GPU of compute capability 8.0 or newer", -1);
+        throw CudaError("no CUDA device is present; Strata needs an NVIDIA GPU (RTX 20 series or newer)", -1);
 #endif
     }
     if (ordinal < 0 || ordinal >= count) {
@@ -60,19 +126,19 @@ DeviceInfo device_info(int ordinal) {
     check(cudaDriverGetVersion(&d.driver_version), "cudaDriverGetVersion");
     check(cudaRuntimeGetVersion(&d.runtime_version), "cudaRuntimeGetVersion");
 
-    // The kernels need sm_80 or newer (tf32 mma in the attention scorer, bf16 math) - the same floor the
-    // arch guard in CMakeLists enforces at build time (RTX 30 / 40 / 50).  Anything older is caught here,
-    // because a binary can be carried to a machine with an older card and would otherwise silently take
-    // whatever path the driver chose.  The HIP backend is validated on gfx1100 (wave32) only.
+    // The engine supports compute capability 7.5 and newer (Turing: the QSA scorer's tf32 mma has a portable
+    // fp32-FMA fallback below sm_80, the tensor-core prompt kernels refuse and fall back).  Compiling for a
+    // supported arch is enforced by CMake; RUNNING on an older card is caught here, because a binary can be carried
+    // to a machine with an older card and would otherwise silently take whatever path the driver chose.  The HIP
+    // backend checks the card against the architectures the binary was compiled for (and wave32).
 #if defined(STRATA_USE_HIP)
-    if (std::strncmp(p.gcnArchName, "gfx1100", 7) != 0 || p.warpSize != 32) {
-        throw CudaError("HIP backend requires validated gfx1100 wave32 hardware", -1);
-    }
+    d.arch = base_arch(p.gcnArchName);
+    if (const std::string why = arch_problem(p, ordinal); !why.empty()) throw CudaError(why, -1);
 #else
-    if (d.cc_major < 8) {
+    if (d.cc_major * 10 + d.cc_minor < 75) {
         throw CudaError("device " + d.name + " reports compute capability " + std::to_string(d.cc_major) +
                             "." + std::to_string(d.cc_minor) +
-                            "; Strata needs an NVIDIA GPU of compute capability 8.0 or newer (RTX 30 / 40 / 50)",
+                            "; Strata needs compute capability 7.5 or newer (RTX 20 / 30 / 40 / 50 series)",
                         -1);
     }
 #endif

src/core/device_main.cpp

@@ -34,7 +34,8 @@ int main(int argc, char** argv) {
         const strata::core::DeviceInfo d = strata::core::device_info(0);
         std::printf("device %d: %s\n", d.ordinal, d.name.c_str());
 #if defined(STRATA_USE_HIP)
-        std::printf("  HIP target          gfx1100 wave32\n");
+        std::printf("  HIP arch            %s wave32 (compiled for %s)\n", d.arch.c_str(),
+                    strata::core::compiled_gpu_archs());
 #else
         std::printf("  compute capability  %d.%d   (sm_%d%d)\n", d.cc_major, d.cc_minor, d.cc_major, d.cc_minor);
 #endif

src/core/expert_source.cpp

@@ -4,6 +4,7 @@
 #include "strata/kernels/cpu/expert_layout.hpp"
 
 #include "strata/core/pinned.hpp"
+#include "strata/platform/memory.hpp"
 #include "strata/kernels/elementwise.hpp"
 #include "strata/kernels/quantize_act.hpp"
 #include "strata/kernels/s2_expert_grouped.hpp"
@@ -20,6 +21,7 @@
 #include <filesystem>
 #include <sstream>
 #include <limits>
+#include <mutex>
 #include <thread>
 #include <utility>
 #include <vector>
@@ -136,10 +138,33 @@ const uint8_t* cache_complement_blob_or_fallback(
     return mapped_fallback;
 }
 
+int64_t choose_resident_keep_from(const std::vector<uint64_t>& slot_bytes, uint64_t base_bytes, uint64_t budget,
+                                  int64_t lend_from) {
+    if (base_bytes > budget) return -1;
+    const int64_t slots = (int64_t) slot_bytes.size();
+    if (lend_from < 0 || lend_from > slots) lend_from = slots;   // no lend region: only the experts no slot holds
+    int64_t keep = slots;
+    uint64_t bytes = base_bytes;
+    while (keep > lend_from) {
+        const uint64_t b = slot_bytes[(size_t) keep - 1];
+        if (b > budget - bytes) break;
+        bytes += b;
+        --keep;
+    }
+    return keep;
+}
+
+bool exchange_cache_complement(std::vector<uint64_t>& offsets, size_t in, size_t out) {
+    if (in == out || in >= offsets.size() || out >= offsets.size() || offsets[in] == kNoCacheComplement ||
+        offsets[out] != kNoCacheComplement) return false;
+    offsets[out] = offsets[in];
+    offsets[in] = kNoCacheComplement;
+    return true;
+}
+
 }  // namespace detail
 
 namespace {
-constexpr uint64_t kPinnedMemoryHeadroom = 8ull << 30;
 
 #if defined(__linux__)
 bool read_cgroup_memory_stat(const std::filesystem::path& path, uint64_t current,
@@ -411,8 +436,23 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
 void FileExpertSource::close() {
     if (complement_arena_ != nullptr) {
         if (complement_pinned_) (void) cudaFreeHost(complement_arena_);
-        else std::free(complement_arena_);
+        else {
+            if (complement_locked_ > 0) strata::platform::unlock_resident(complement_arena_, complement_locked_);
+            std::free(complement_arena_);
+        }
     }
+    if (xstage_ != nullptr) {
+        if (xstage_pinned_) (void) cudaFreeHost(xstage_);
+        else std::free(xstage_);
+    }
+    xstage_ = nullptr;
+    xstage_pinned_ = false;
+    xstage_cap_ = 0;
+    xstage_blob_ = 0;
+    override_.clear();
+    staged_.clear();
+    exchanges_ = 0;
+    file_reads_.store(0);
     complement_arena_ = nullptr;
     complement_host_ = nullptr;
     complement_device_ = nullptr;
@@ -420,6 +460,8 @@ void FileExpertSource::close() {
     complement_offsets_.clear();
     complement_pinned_ = false;
     complement_ready_ = false;
+    complement_locked_ = 0;
+    complement_lent_slots_ = 0;
 #if defined(_WIN32)
     if (base_ != nullptr) UnmapViewOfFile((LPCVOID) base_);
     if (mapping_ != nullptr) CloseHandle((HANDLE) mapping_);
@@ -457,7 +499,8 @@ const uint8_t* FileExpertSource::mapped_blob(int64_t layer, int64_t expert) cons
 
 bool FileExpertSource::pin_cache_complement(
     const ExpertCache& cache, std::string& err, bool pin,
-    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs) {
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs, int64_t lend_from_slot,
+    uint64_t headroom_bytes) {
     err.clear();
     if (base_ == nullptr) { err = "FileExpertSource: open the mapped experts before pinning a complement"; return false; }
     if (complement_ready_) { err = "FileExpertSource: the cache complement is already pinned"; return false; }
@@ -473,12 +516,20 @@ bool FileExpertSource::pin_cache_complement(
         return false;
     }
 
+    // The GPU cache's experts, and the bytes each slot's expert takes here (for the lend region below).
+    const int64_t n_slots = cache.slots();
     std::vector<std::pair<int32_t, int32_t>> primary_gpu_pairs;
+    std::vector<int32_t> pair_slot;
+    std::vector<uint64_t> slot_bytes((size_t) std::max<int64_t>(n_slots, 0), 0);
     primary_gpu_pairs.reserve((size_t) cache.resident());
+    pair_slot.reserve((size_t) cache.resident());
     for (int64_t layer = 0; layer < n_layers_; ++layer) {
         for (int64_t expert = 0; expert < n_expert_; ++expert) {
-            if (cache.slot_of(layer, expert) != kNotResident)
-                primary_gpu_pairs.emplace_back((int32_t) layer, (int32_t) expert);
+            const int32_t slot = cache.slot_of(layer, expert);
+            if (slot == kNotResident) continue;
+            primary_gpu_pairs.emplace_back((int32_t) layer, (int32_t) expert);
+            pair_slot.push_back(slot);
+            if (slot >= 0 && slot < n_slots) slot_bytes[(size_t) slot] = layer_blob_bytes_[(size_t) layer];
         }
     }
     std::vector<uint64_t> offsets;
@@ -486,136 +537,199 @@ bool FileExpertSource::pin_cache_complement(
     if (!detail::make_cache_complement_plan(n_layers_, n_expert_, layer_blob_bytes_, primary_gpu_pairs,
                                             additional_gpu_pairs, offsets, bytes, err)) return false;
 
-    if (bytes > 0) {
+    const bool lend = lend_from_slot >= 0 && lend_from_slot < n_slots && additional_gpu_pairs.empty();
+    uint64_t budget = std::numeric_limits<uint64_t>::max();
+    if (bytes > 0 || lend) {
         uint64_t physical = 0;
         if (!available_memory_bytes(physical)) {
             err = "FileExpertSource: cannot determine available RAM for the resident-memory safety check";
             return false;
         }
-        if (physical <= kPinnedMemoryHeadroom || bytes > physical - kPinnedMemoryHeadroom) {
-            char message[256];
+        budget = physical > headroom_bytes ? physical - headroom_bytes : 0;
+        if (bytes > budget) {
+            char message[320];
             std::snprintf(message, sizeof message,
-                          "FileExpertSource: resident complement %.2f GiB exceeds available RAM minus the 8 GiB safety headroom",
-                          (double) bytes / 1073741824.0);
+                          "FileExpertSource: resident complement %.2f GiB exceeds available RAM (%.2f GiB) minus the "
+                          "%.0f GiB safety headroom",
+                          (double) bytes / 1073741824.0, (double) physical / 1073741824.0,
+                          (double) headroom_bytes / 1073741824.0);
             err = message;
             return false;
         }
     }
+    // The prompt path's lend region: its slots' experts are streamed from here during a prompt and copied back into
+    // their slots after it, so the ones that fit are kept here too (from the last slot down: a short prompt lends
+    // only the last few).  The rest keep the mapped-file fallback.
+    int64_t keep_from = n_slots;
+    if (lend) {
+        keep_from = detail::choose_resident_keep_from(slot_bytes, bytes, budget, lend_from_slot);
+        if (keep_from < 0) keep_from = n_slots;
+        if (keep_from < n_slots) {
+            std::vector<std::pair<int32_t, int32_t>> core;
+            core.reserve(primary_gpu_pairs.size());
+            for (size_t i = 0; i < primary_gpu_pairs.size(); ++i)
+                if (pair_slot[i] < keep_from) core.push_back(primary_gpu_pairs[i]);
+            if (!detail::make_cache_complement_plan(n_layers_, n_expert_, layer_blob_bytes_, core,
+                                                    additional_gpu_pairs, offsets, bytes, err)) return false;
+        }
+    }
 
     void* arena = nullptr;
     const uint8_t* host = nullptr;
     const uint8_t* device = nullptr;
+    bool pinned_ok = false;
+    uint64_t locked = 0;
+    std::string note;
+    auto release = [&]() {
+        if (arena == nullptr) return;
+        if (pinned_ok) (void) cudaFreeHost(arena);
+        else {
+            if (locked > 0) strata::platform::unlock_resident(arena, locked);
+            std::free(arena);
+        }
+        arena = nullptr;
+    };
     if (bytes > 0) {
         std::fprintf(stderr, "FileExpertSource: allocating %.2f GiB %s cache complement\n",
-                     (double) bytes / 1073741824.0, pin ? "mapped pinned" : "pageable resident");
+                     (double) bytes / 1073741824.0, pin ? "page-locked" : "pageable resident");
         std::fflush(stderr);
         if (pin) {
             const cudaError_t allocated = cudaHostAlloc(&arena, (size_t) bytes,
                                                          cudaHostAllocMapped | cudaHostAllocPortable);
-            if (allocated != cudaSuccess) {
-                err = std::string("FileExpertSource: mapped pinned complement allocation failed: ") +
-                      cudaGetErrorString(allocated);
-                std::fprintf(stderr, "%s\n", err.c_str());
-                std::fflush(stderr);
+            if (allocated == cudaSuccess) {
+                void* alias = nullptr;
+                const cudaError_t aliased = cudaHostGetDevicePointer(&alias, arena, 0);
+                if (aliased == cudaSuccess && alias != nullptr) {
+                    device = (const uint8_t*) alias;
+                    pinned_ok = true;
+                    note = "page-locked and mapped";
+                } else {
+                    note = std::string("no device alias (") + cudaGetErrorString(aliased) + ")";
+                    (void) cudaGetLastError();
+                    (void) cudaFreeHost(arena);
+                    arena = nullptr;
+                }
+            } else {
+                // Refused (the driver's page-locked limit): the same bytes in ordinary memory, locked in the working
+                // set instead, as the arena does - resident either way, only copied by the CPU instead of by DMA.
+                note = std::string("page-locking refused (") + cudaGetErrorString(allocated) + ")";
                 (void) cudaGetLastError();
-                return false;
+                arena = nullptr;
             }
-            host = (const uint8_t*) arena;
-            void* alias = nullptr;
-            const cudaError_t aliased = cudaHostGetDevicePointer(&alias, arena, 0);
-            if (aliased != cudaSuccess || alias == nullptr) {
-                err = std::string("FileExpertSource: mapped pinned complement device alias failed: ") +
-                      cudaGetErrorString(aliased);
-                std::fprintf(stderr, "%s\n", err.c_str());
-                std::fflush(stderr);
-                (void) cudaGetLastError();
-                (void) cudaFreeHost(arena);
-                return false;
-            }
-            device = (const uint8_t*) alias;
-        } else {
+        }
+        if (arena == nullptr) {
             arena = std::malloc((size_t) bytes);
             if (arena == nullptr) {
                 err = "FileExpertSource: pageable resident complement allocation failed";
                 return false;
             }
-            host = (const uint8_t*) arena;
+            if (pin) {
+                const strata::platform::LockResult lr = strata::platform::lock_resident(arena, bytes);
+                locked = lr.locked_bytes;
+                note += (note.empty() ? "" : "; ") + lr.note;
+            }
         }
+        host = (const uint8_t*) arena;
     }
 
-    uint64_t copied = 0;
-    for (int64_t layer = 0; layer < n_layers_; ++layer) {
-        const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer];
-        for (int64_t expert = 0; expert < n_expert_; ++expert) {
-            const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
-            const uint64_t offset = offsets[index];
-            if (offset == kNoComplement) continue;
-            const uint8_t* source = mapped_blob(layer, expert);
-            if (source == nullptr || offset > bytes || blob_bytes > bytes - offset) {
-                err = "FileExpertSource: invalid blob bounds while building the cache complement";
-                if (arena != nullptr) {
-                    if (pin) (void) cudaFreeHost(arena);
-                    else std::free(arena);
-                }
-                return false;
-            }
-            std::memcpy((uint8_t*) host + (size_t) offset, source, (size_t) blob_bytes);
-            copied += blob_bytes;
-        }
+    // Copied layer by layer on a few threads: the page faults of the mapped file are the cost, and they overlap.
 #if !defined(_WIN32)
-        const uint64_t layer_offset = layer_offsets_[(size_t) layer];
-        const uint64_t layer_bytes = blob_bytes * (uint64_t) n_expert_;
-        const uint64_t layer_end = layer_offset + layer_bytes;
-        const long page_bytes = sysconf(_SC_PAGESIZE);
-        if (page_bytes <= 0) {
-            err = "FileExpertSource: cannot determine page size for mapped-page release";
-            if (arena != nullptr) {
-                if (pin) (void) cudaFreeHost(arena);
-                else std::free(arena);
-            }
-            return false;
-        }
-        const uint64_t page = (uint64_t) page_bytes;
-        const uint64_t advice_start = layer_offset - layer_offset % page;
-        const uint64_t end_remainder = layer_end % page;
-        const uint64_t extra = end_remainder == 0 ? 0 : page - end_remainder;
-        const uint64_t advice_end = extra > mapped_bytes_ - layer_end ? mapped_bytes_ : layer_end + extra;
-        if (advice_end > advice_start &&
-            madvise((void*) (base_ + (size_t) advice_start), (size_t) (advice_end - advice_start), MADV_DONTNEED) != 0) {
-            err = "FileExpertSource: madvise could not release mapped expert layer " + std::to_string(layer);
-            if (arena != nullptr) {
-                if (pin) (void) cudaFreeHost(arena);
-                else std::free(arena);
-            }
-            return false;
-        }
-        const int advice = posix_fadvise(fd_, (off_t) layer_offset, (off_t) layer_bytes, POSIX_FADV_DONTNEED);
-        if (advice != 0) {
-            err = "FileExpertSource: posix_fadvise could not release expert layer " + std::to_string(layer);
-            if (arena != nullptr) {
-                if (pin) (void) cudaFreeHost(arena);
-                else std::free(arena);
-            }
-            return false;
-        }
-#endif
-        if ((layer + 1) % 8 == 0 || layer + 1 == n_layers_) {
-            std::fprintf(stderr, "FileExpertSource: copied cache complement through layer %lld/%lld (%.2f GiB)\n",
-                         (long long) (layer + 1), (long long) n_layers_, (double) copied / 1073741824.0);
-            std::fflush(stderr);
-        }
+    const long page_size = sysconf(_SC_PAGESIZE);
+    if (page_size <= 0) {
+        err = "FileExpertSource: cannot determine page size for mapped-page release";
+        release();
+        return false;
     }
+#endif
+    std::atomic<int64_t> next_layer{0}, layers_done{0};
+    std::atomic<uint64_t> copied{0};
+    std::atomic<bool> failed{false};
+    std::mutex fail_mu;
+    std::string fail_msg;
+    auto fail = [&](const std::string& m) {
+        std::lock_guard<std::mutex> lock(fail_mu);
+        if (fail_msg.empty()) fail_msg = m;
+        failed.store(true);
+    };
+    auto worker = [&]() {
+        for (;;) {
+            const int64_t layer = next_layer.fetch_add(1);
+            if (layer >= n_layers_ || failed.load()) return;
+            const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer];
+            for (int64_t expert = 0; expert < n_expert_; ++expert) {
+                const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+                const uint64_t offset = offsets[index];
+                if (offset == kNoComplement) continue;
+                const uint8_t* source = mapped_blob(layer, expert);
+                if (source == nullptr || offset > bytes || blob_bytes > bytes - offset) {
+                    fail("FileExpertSource: invalid blob bounds while building the cache complement");
+                    return;
+                }
+                std::memcpy((uint8_t*) host + (size_t) offset, source, (size_t) blob_bytes);
+                copied.fetch_add(blob_bytes);
+            }
+#if !defined(_WIN32)
+            const uint64_t layer_offset = layer_offsets_[(size_t) layer];
+            const uint64_t layer_bytes = blob_bytes * (uint64_t) n_expert_;
+            const uint64_t layer_end = layer_offset + layer_bytes;
+            const uint64_t page = (uint64_t) page_size;
+            const uint64_t advice_start = layer_offset - layer_offset % page;
+            const uint64_t end_remainder = layer_end % page;
+            const uint64_t extra = end_remainder == 0 ? 0 : page - end_remainder;
+            const uint64_t advice_end = extra > mapped_bytes_ - layer_end ? mapped_bytes_ : layer_end + extra;
+            if (advice_end > advice_start &&
+                madvise((void*) (base_ + (size_t) advice_start), (size_t) (advice_end - advice_start), MADV_DONTNEED) != 0) {
+                fail("FileExpertSource: madvise could not release mapped expert layer " + std::to_string(layer));
+                return;
+            }
+            if (posix_fadvise(fd_, (off_t) layer_offset, (off_t) layer_bytes, POSIX_FADV_DONTNEED) != 0) {
+                fail("FileExpertSource: posix_fadvise could not release expert layer " + std::to_string(layer));
+                return;
+            }
+#endif
+            const int64_t done = layers_done.fetch_add(1) + 1;
+            if (done % 8 == 0 || done == n_layers_)
+                std::fprintf(stderr, "FileExpertSource: copied cache complement through layer %lld/%lld (%.2f GiB)\n",
+                             (long long) done, (long long) n_layers_, (double) copied.load() / 1073741824.0);
+        }
+    };
+    {
+        const int threads = (int) std::max<int64_t>(1, std::min<int64_t>(6, n_layers_));
+        std::vector<std::thread> pool;
+        for (int i = 1; i < threads; ++i) pool.emplace_back(worker);
+        worker();
+        for (auto& t : pool) t.join();
+    }
+    std::fflush(stderr);
+    if (failed.load()) {
+        err = fail_msg;
+        release();
+        return false;
+    }
+#if defined(_WIN32)
+    // The mapped pages this process touched (the GPU cache's fill and this copy) leave its working set for the
+    // standby list: VirtualUnlock on pages that are not locked does exactly that (it then reports ERROR_NOT_LOCKED).
+    (void) VirtualUnlock((LPVOID) base_, (SIZE_T) mapped_bytes_);
+#endif
 
     complement_arena_ = arena;
     complement_host_ = host;
     complement_device_ = device;
     complement_bytes_ = bytes;
     complement_offsets_ = std::move(offsets);
-    complement_pinned_ = pin && bytes > 0;
+    complement_pinned_ = pinned_ok && bytes > 0;
+    complement_locked_ = locked;
+    complement_lent_slots_ = lend ? n_slots - keep_from : 0;
     complement_ready_ = true;
-    std::fprintf(stderr, "FileExpertSource: %s cache complement ready: resident %.2f GiB, pinned %.2f GiB\n",
-                 complement_pinned_ ? "mapped pinned" : "pageable resident",
-                 (double) resident_bytes() / 1073741824.0, (double) pinned_bytes() / 1073741824.0);
+    std::fprintf(stderr, "FileExpertSource: %s cache complement ready: resident %.2f GiB, pinned %.2f GiB%s%s\n",
+                 complement_pinned_ ? "mapped pinned" : pin ? "locked resident" : "pageable resident",
+                 (double) resident_bytes() / 1073741824.0, (double) pinned_bytes() / 1073741824.0,
+                 note.empty() ? "" : "; ", note.c_str());
+    if (lend)
+        std::fprintf(stderr, "FileExpertSource: %lld of the prompt path's %lld lendable slots keep their experts in RAM "
+                             "too%s\n", (long long) complement_lent_slots_, (long long) (n_slots - lend_from_slot),
+                     complement_lent_slots_ < n_slots - lend_from_slot
+                         ? " (the others are read from the file when lent: not enough RAM for them)" : "");
     if (!additional_gpu_pairs.empty()) {
         std::fprintf(stderr, "FileExpertSource: %zu verified additional-GPU experts remain on the mmap fallback\n",
                      additional_gpu_pairs.size());
@@ -624,13 +738,88 @@ bool FileExpertSource::pin_cache_complement(
     return true;
 }
 
+bool FileExpertSource::has_resident(int64_t layer, int64_t expert) const {
+    if (!complement_ready_ || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return false;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    return index < complement_offsets_.size() && complement_offsets_[index] != kNoComplement;
+}
+
+bool FileExpertSource::reserve_exchanges(int64_t n, std::string& err) {
+    err.clear();
+    if (n <= xstage_cap_) return true;
+    if (!staged_.empty()) { err = "FileExpertSource: exchange buffers are in use"; return false; }
+    uint64_t blob = 0;
+    for (const uint64_t b : layer_blob_bytes_) blob = std::max(blob, b);
+    if (blob == 0 || n <= 0) { err = "FileExpertSource: no expert geometry for the exchange buffers"; return false; }
+    if (xstage_ != nullptr) {
+        if (xstage_pinned_) (void) cudaFreeHost(xstage_);
+        else std::free(xstage_);
+        xstage_ = nullptr;
+        xstage_cap_ = 0;
+    }
+    const size_t total = (size_t) n * (size_t) blob;
+    void* p = nullptr;
+    if (cudaHostAlloc(&p, total, cudaHostAllocDefault) == cudaSuccess && p != nullptr) {
+        xstage_pinned_ = true;
+    } else {
+        (void) cudaGetLastError();
+        p = std::malloc(total);
+        xstage_pinned_ = false;
+        if (p == nullptr) { err = "FileExpertSource: cannot allocate the exchange buffers"; return false; }
+    }
+    xstage_ = (uint8_t*) p;
+    xstage_cap_ = n;
+    xstage_blob_ = blob;
+    return true;
+}
+
+uint8_t* FileExpertSource::exchange_buffer(int64_t q) const {
+    if (xstage_ == nullptr || q < 0 || q >= xstage_cap_) return nullptr;
+    return xstage_ + (size_t) q * (size_t) xstage_blob_;
+}
+
+bool FileExpertSource::stage_exchange(int64_t layer, int64_t in, int64_t out, int64_t q) {
+    if (!has_resident(layer, in) || has_resident(layer, out) || exchange_buffer(q) == nullptr) return false;
+    const size_t i_in = (size_t) layer * (size_t) n_expert_ + (size_t) in;
+    const size_t i_out = (size_t) layer * (size_t) n_expert_ + (size_t) out;
+    if (override_.empty()) override_.assign((size_t) blobs_, nullptr);
+    if (override_[i_out] != nullptr) return false;
+    for (const Exchange& x : staged_)
+        if (x.in == i_in || x.q == q) return false;
+    override_[i_out] = exchange_buffer(q);
+    staged_.push_back({i_in, i_out, q, layer_blob_bytes_[(size_t) layer]});
+    return true;
+}
+
+int64_t FileExpertSource::commit_exchanges() {
+    int64_t n = 0;
+    for (const Exchange& x : staged_) {
+        const uint8_t* src = override_[x.out];
+        const uint64_t at = complement_offsets_[x.in];
+        if (src != nullptr && at != kNoComplement && at <= complement_bytes_ && x.bytes <= complement_bytes_ - at &&
+            complement_host_ != nullptr) {
+            std::memcpy((uint8_t*) complement_host_ + (size_t) at, src, (size_t) x.bytes);
+            if (detail::exchange_cache_complement(complement_offsets_, x.in, x.out)) ++n;
+        }
+        override_[x.out] = nullptr;
+    }
+    staged_.clear();
+    exchanges_ += n;
+    return n;
+}
+
 const uint8_t* FileExpertSource::blob(int64_t layer, int64_t expert) {
     if (base_ == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return nullptr;
     const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
     const uint8_t* mapped_fallback = mapped_blob(layer, expert);
-    const uint8_t* result = complement_ready_
-        ? detail::cache_complement_blob_or_fallback(index, complement_offsets_, complement_host_, mapped_fallback)
-        : mapped_fallback;
+    const uint8_t* result = mapped_fallback;
+    if (complement_ready_) {
+        result = detail::cache_complement_blob_or_fallback(index, complement_offsets_, complement_host_, nullptr);
+        if (result == nullptr) {
+            result = !override_.empty() && override_[index] != nullptr ? override_[index] : mapped_fallback;
+            if (result == mapped_fallback && result != nullptr) file_reads_.fetch_add(1, std::memory_order_relaxed);
+        }
+    }
     if (result != nullptr) ++reads_;
     return result;
 }
@@ -648,6 +837,12 @@ const uint8_t* FileExpertSource::device_alias(int64_t layer, int64_t expert) con
     return complement_device_ + (size_t) complement_offsets_[index];
 }
 
+bool FileExpertSource::pcie_layer(int64_t layer) const {
+    if (complement_ready_ && complement_pinned_ && complement_device_ != nullptr)
+        return layer >= 0 && layer < n_layers_;
+    return device_alias(layer, 0) != nullptr;
+}
+
 // ================================ THE ADAPTER ================================
 
 void expert_pool_dispatch(void* user, const float* x_f, const int32_t* ids, const float* weights, int64_t n_embd,
@@ -772,6 +967,14 @@ void expert_pool_dispatch(void* user, const float* x_f, const int32_t* ids, cons
     d.experts += k;
 }
 
+namespace {
+// the verify window's per-entry tables in `expert_pool_dispatch_multi` (`kind`, `distinct`, `first_of`)
+// are fixed arrays of this many entries: MAXT tokens of the model's 10 routed experts must fit, and a larger k is
+// refused at run time rather than written past them.
+constexpr int64_t kMaxWindowEntries = 128;
+static_assert(strata::kernels::cpu::MAXT * 10 <= kMaxWindowEntries, "a verify window's entries overflow the tables");
+}  // namespace
+
 void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32_t* ids, int64_t n_tok, int64_t k,
                                 float* out) {
     using namespace strata::kernels::cpu;
@@ -782,6 +985,12 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
         d.fail_layer = d.layers;
         return;
     }
+    if (k < 1 || n_tok * k > kMaxWindowEntries) {
+        d.failed = true;
+        d.fail = "a verify window routes more entries than the expert pool's window tables hold";
+        d.fail_layer = d.layers;
+        return;
+    }
     if ((int64_t) d.act_multi.size() < n_tok) d.act_multi.resize((size_t) MAXT);
     const ExpertLayout& lay = expert_layout();
     const bool native = lay.native;
@@ -802,9 +1011,9 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
     // ---- plan v0.3 P6: the GPU's share, decided and published FIRST so the GPU starts while the CPU works.
     // Distinct experts in routing order; resident ones and the last pcie_num/256 of the missed ones go to the GPU.
     const int64_t n = n_tok * k;
-    int32_t kind[128];                     // per entry: -1 CPU, 0 VRAM, 1 PCIe
-    if (d.plan != nullptr && n <= 128 && n <= d.plan->cap) {
-        int64_t distinct[128], first_of[128];
+    int32_t kind[kMaxWindowEntries];       // per entry: -1 CPU, 0 VRAM, 1 PCIe
+    if (d.plan != nullptr && n <= kMaxWindowEntries && n <= d.plan->cap) {
+        int64_t distinct[kMaxWindowEntries], first_of[kMaxWindowEntries];
         int nd = 0, nmiss = 0;
         for (int64_t i = 0; i < n; ++i) {
             first_of[i] = i;
@@ -816,7 +1025,7 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                 if (e >= 0 && e < d.n_expert && d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] < 0) ++nmiss;
             }
         }
-        const bool pcie_ok = d.pcie_num > 0 && d.src->device_alias(d.layers, 0) != nullptr;
+        const bool pcie_ok = d.pcie_num > 0 && d.src->pcie_layer(d.layers);
         const int m = pcie_ok ? (nmiss * d.pcie_num) >> 8 : 0;
         int miss_rank = 0, groups = 0, entries = 0, fetches = 0;
         GpuPlanSink& P = *d.plan;
@@ -1086,6 +1295,111 @@ void expert_hit_run(void* user, void* stream, HitPhase phase, const int32_t* ids
 
 // ================================ THE RESIDENT ARENA (R2.1) ================================
 
+namespace {
+
+void hash_u64(uint64_t& h, uint64_t v) {
+    h = fnv1a64((const uint8_t*) &v, sizeof v, h);
+}
+
+void hash_text(uint64_t& h, const std::string& s) {
+    h = fnv1a64((const uint8_t*) s.data(), (uint64_t) s.size(), h);
+}
+
+bool hash_small_file(const std::filesystem::path& path, uint64_t& h, std::string& err) {
+    hash_text(h, path.filename().string());
+    std::ifstream f(path, std::ios::binary);
+    if (!f) {
+        hash_u64(h, 0);
+        return true;
+    }
+    hash_u64(h, 1);
+    std::vector<uint8_t> buf(64u << 10);
+    for (;;) {
+        f.read((char*) buf.data(), (std::streamsize) buf.size());
+        const std::streamsize n = f.gcount();
+        if (n > 0) h = fnv1a64(buf.data(), (uint64_t) n, h);
+        if (f.eof()) break;
+        if (!f) {
+            err = "ArenaExpertSource: cannot hash pack metadata " + path.string();
+            return false;
+        }
+    }
+    return true;
+}
+
+bool hash_sampled_file(const std::filesystem::path& path, uint64_t& h, std::string& err) {
+    std::ifstream f(path, std::ios::binary | std::ios::ate);
+    if (!f) {
+        err = "ArenaExpertSource: cannot sample pack source " + path.string();
+        return false;
+    }
+    const std::streamoff end = f.tellg();
+    if (end < 0) {
+        err = "ArenaExpertSource: cannot size pack source " + path.string();
+        return false;
+    }
+    const uint64_t bytes = (uint64_t) end;
+    hash_text(h, path.filename().string());
+    hash_u64(h, bytes);
+    constexpr uint64_t sample = 64u << 10;
+    const uint64_t starts[3] = {0, bytes / 2, bytes > sample ? bytes - sample : 0};
+    std::vector<uint8_t> buf((size_t) std::min<uint64_t>(sample, bytes));
+    for (uint64_t off : starts) {
+        if (buf.empty()) break;
+        const uint64_t at = std::min<uint64_t>(off, bytes - (uint64_t) buf.size());
+        f.clear();
+        f.seekg((std::streamoff) at);
+        f.read((char*) buf.data(), (std::streamsize) buf.size());
+        if ((size_t) f.gcount() != buf.size()) {
+            err = "ArenaExpertSource: short read while hashing pack source " + path.string();
+            return false;
+        }
+        hash_u64(h, at);
+        h = fnv1a64(buf.data(), (uint64_t) buf.size(), h);
+    }
+    return true;
+}
+
+bool shared_arena_pack_hash(const std::string& pack_dir, const std::string& experts_path,
+                            const std::string& gguf, const strata::kernels::cpu::ExpertLayout& lay,
+                            uint64_t& out, std::string& err) {
+    uint64_t h = 1469598103934665603ull;
+    hash_text(h, "strata-shared-expert-arena-pack-v1");
+    hash_u64(h, (uint64_t) lay.n_layers);
+    hash_u64(h, (uint64_t) lay.n_expert);
+    hash_u64(h, lay.total);
+    hash_u64(h, lay.max_blob);
+    hash_u64(h, lay.native ? 1 : 0);
+
+    const std::filesystem::path pack(pack_dir);
+    for (const char* name : {"manifest.json", "index.txt", "native_experts.txt"}) {
+        if (!hash_small_file(pack / name, h, err)) return false;
+    }
+
+    if (std::filesystem::exists(experts_path)) {
+        if (!hash_sampled_file(experts_path, h, err)) return false;
+    } else if (!gguf.empty()) {
+        // Native packs may read experts straight from one or more GGUF shards.  Sample every distinct source
+        // file named by native_experts.txt; this keeps the fingerprint cheap while still tying it to the model
+        // bytes rather than only to an equal-size layout.
+        const std::filesystem::path first(gguf);
+        std::vector<std::filesystem::path> sources{first};
+        for (const std::string& name : lay.gguf_file) {
+            if (name.empty()) continue;
+            const std::filesystem::path p = first.parent_path() / name;
+            if (std::find(sources.begin(), sources.end(), p) == sources.end()) sources.push_back(p);
+        }
+        for (const auto& p : sources) {
+            if (!hash_sampled_file(p, h, err)) return false;
+        }
+    }
+
+    out = h == 0 ? 1 : h;
+    return true;
+}
+
+}  // namespace
+
 // Plan v0.3 P6: the arena from the model's shard 1.  Each layer's gate, up and down tensors hold the 512 experts
 // one after another; they are read in chunks and each expert's slice lands at its place in the blob
 // [gate rows | up rows | down rows] - the layout tools/iq_pack.py would have written to experts.bin.
@@ -1158,7 +1472,8 @@ LoadStats load_experts_gguf(const std::string& gguf, uint8_t* dst, const strata:
 ArenaExpertSource::~ArenaExpertSource() { close(); }
 
 bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, int threads,
-                             std::string& err, uint64_t max_pinned_bytes) {
+                             std::string& err, uint64_t max_pinned_bytes,
+                             const std::string& shared_arena_file) {
     close();
     const std::string path = pack_dir + "/experts.bin";
     // plan v0.3 P6: the layout (canonical, or a native pack's per-layer blobs) was loaded by the driver
@@ -1190,6 +1505,10 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
         }
     }
 
+    uint64_t pack_hash = 0;
+    if (!shared_arena_file.empty() &&
+        !shared_arena_pack_hash(pack_dir, path, gguf_, lay, pack_hash, err)) return false;
+
     // one layer per registration slice, so no expert straddles two registrations.  The arena is one blob
     // longer than the file: a copy of a whole VRAM slot (the largest blob) may then start at any expert.
     std::vector<uint64_t> bounds, loff, lbytes;
@@ -1199,10 +1518,14 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
         lbytes.push_back(lay.blob_bytes(l) * (uint64_t) n_expert);
     }
     bounds.push_back(want);
-    PinnedArena* a = new PinnedArena(want + (uint64_t) blob, bounds, max_pinned_bytes);
+    PinnedArena* a = new PinnedArena(want + (uint64_t) blob, bounds, max_pinned_bytes,
+                                     shared_arena_file, pack_hash);
     if (!a->valid()) {
+        const std::string why = a->note;
         delete a;
-        err = "ArenaExpertSource: the arena could not be reserved (" + std::to_string(want) + " B)";
+        err = "ArenaExpertSource: the arena could not be reserved (" +
+              std::to_string(want + (uint64_t) blob) + " B)" +
+              (why.empty() ? std::string{} : ": " + why);
         return false;
     }
     const LoadStats st = from_gguf ? load_experts_gguf(gguf_, a->data(), lay, threads)

src/core/layer.cpp

@@ -655,9 +655,10 @@ uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, Q
     // THE ROPE TABLE IS BUILT ON THE HOST IN FLOAT64 and uploaded once, because the reference computes its
     // frequencies in float64 and reproducing that on device means double-precision `pow`/`cos` that need not
     // agree with the host's libm.  A table shorter than the sequence would have the rotation read past it.
+    // The process's rope scaling (none by default) is INSIDE the table - the rotation kernels cannot tell.
     if (share_rope == nullptr) {
         std::vector<float> hc((size_t) max_cells * (s.n_rot / 2)), hs((size_t) max_cells * (s.n_rot / 2));
-        strata::kernels::build_rope_table((int) s.n_rot, strata::kernels::qsa_freq_base(), (int) max_cells,
+        strata::kernels::build_rope_table((int) s.n_rot, strata::kernels::rope_scaling(), (int) max_cells,
                                           hc.data(), hs.data());
         cudaMemcpy(st.cos_tab, hc.data(), hc.size() * 4, cudaMemcpyHostToDevice);
         cudaMemcpy(st.sin_tab, hs.data(), hs.size() * 4, cudaMemcpyHostToDevice);
@@ -843,7 +844,7 @@ const auto normalize_rotate = [&](float* data, const WeightRef* norm, int rows,
     try {
         if (native_qsa_enabled()) native_qsa_rms_norm_weighted(data, (const float*) norm->data, data, cols, rows, RMS_EPS, stream);
         else rms_norm_weighted(data, (const float*) norm->data, rows, cols, RMS_EPS, stream);
-        if (native_rope_enabled()) native_rope_apply(data, data, rows, cols, (int) s.n_rot, (float) qsa_freq_base(), st.pos_dev, stream);
+        if (native_rope_enabled()) native_rope_apply(data, data, rows, cols, (int) s.n_rot, rope_scaling(), st.pos_dev, stream);
         else rope_neox_apply(data, data, rows, cols, (int) s.n_rot, st.cos_tab, st.sin_tab, st.pos_dev, stream);
         return true;
     } catch (const std::exception& error) {
@@ -905,7 +906,7 @@ if (st.kv_hybrid) {
 } else if (st.kv_int8) kv_append_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    else kv_append_step(st.k_pool, st.v_pool, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    {        const uint64_t nvk = (uint64_t) g.n_head_kv * g.head_dim;        const uint64_t base = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim + 2 * nvk + 8;        if (!st.kv_int8 && !st.kv_hybrid) dump_slot(dump, g, layer, (const float*) st.k_pool, base, nvk / 2, stream);        if (!st.kv_int8 && !st.kv_hybrid) dump_slot(dump, g, layer, (const float*) st.v_pool, base + nvk / 2, nvk / 2, stream);        dump_slot(dump, g, layer, b.vcur, base + nvk, nvk, stream);        dump_slot(dump, g, layer, b.kcur, base + 2 * nvk, nvk, stream);    }    {        const QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};        if (native_qsa_indexer_enabled()) {
     try {
         native_qsa_indexer_append(b.idx_raw, st.step + kStepPos, pos_base,
-            (const float*) w_ikn->data, RMS_EPS, ib, s, st.max_cells, (float) qsa_freq_base(), stream);
+            (const float*) w_ikn->data, RMS_EPS, ib, s, st.max_cells, rope_scaling(), stream);
     } catch (const std::exception& error) { err = v.name("native_indexer") + ": " + error.what(); return false; }
 } else indexer_key_append(b.idx_raw, st.pos_dev, pos_base, (const float*) w_ikn->data, RMS_EPS, ib, s,
                           st.cos_tab, st.sin_tab, stream);    }

src/core/mtp.cpp

@@ -1,5 +1,6 @@
 // src/core/mtp.cpp - see include/strata/core/mtp.hpp.
 #include "strata/core/mtp.hpp"
+#include "strata/core/coupled_draft.hpp"
 #include "strata/core/on_device.hpp"
 
 #include "strata/core/native_head.hpp"
@@ -105,6 +106,14 @@ MtpDrafter::~MtpDrafter() {
     if (pf_dev_) cudaFree(pf_dev_);
     for (auto& e : round_exec_) if (e) cudaGraphExecDestroy(e);
     for (auto& e : step_exec_) if (e) cudaGraphExecDestroy(e);
+    for (auto& e : round_exec_c_) if (e) cudaGraphExecDestroy(e);
+    for (auto& e : step_exec_c_) if (e) cudaGraphExecDestroy(e);
+    if (cparams_) cudaFree(cparams_);
+    if (cring_) cudaFree(cring_);
+    if (dinv_) cudaFree(dinv_);
+    if (cscratch_) cudaFree(cscratch_);
+    if (h_cparams_) cudaFreeHost(h_cparams_);
+    if (h_chist_) cudaFreeHost(h_chist_);
     if (cs_) cudaStreamDestroy(cs_);
     if (dense_) cudaFree(dense_);
     if (experts_) cudaFree(experts_);
@@ -196,7 +205,7 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
 
     // ---- the layer's own K/V (dense attention: no indexer state is read)
     const strata::kernels::QsaShapes s = shapes_of(g);
-    const int64_t max_cells = ss.qsa_states[0].max_cells;
+    const int64_t max_cells = ss.qsa_states[ss.qsa_primary()].max_cells;
     // KV streaming: the drafter only reads its last `window` cells, so with streaming on its K/V is a ring of the
     // window (plus the cells a round writes ahead of its queries) over a host copy, refilled on a resume. The host copy
     // is pinned after the expert arena has pinned what it could: if it does not fit, the K/V stays whole in VRAM.
@@ -209,7 +218,7 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
     if (kv_hybrid_was) qsa_set_kv_int8(true);   // the drafter under --kv k8v4: plain INT8
     uint64_t sb = qsa_state_bytes(g, max_cells, false, ring);
     if (cudaMalloc(&state_arena_, sb) != cudaSuccess) { err = "mtp: the K/V state does not fit"; return false; }
-    if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[0], ring) == 0) {
+    if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[ss.qsa_primary()], ring) == 0) {
         if (st_.kv_mode == 0) { err = "mtp: state init failed"; return false; }
         std::fprintf(stderr, "strata mtp: no pinned RAM left for the draft layer's K/V copy; keeping it in VRAM\n");
         cudaGetLastError();
@@ -218,7 +227,7 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
         ring = -1;   // fully resident
         sb = qsa_state_bytes(g, max_cells, false, ring);
         if (cudaMalloc(&state_arena_, sb) != cudaSuccess) { err = "mtp: the K/V state does not fit"; return false; }
-        if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[0], ring) == 0) { err = "mtp: state init failed"; return false; }
+        if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[ss.qsa_primary()], ring) == 0) { err = "mtp: state init failed"; return false; }
     }
     qsa_set_kv_int8(kv_int8_was);
     qsa_set_kv_hybrid(kv_hybrid_was);
@@ -292,6 +301,77 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
     return true;
 }
 
+uint64_t MtpDrafter::bind_bytes(uint64_t head_row_bytes, int64_t n_vocab) const {
+    uint64_t bytes = head_logits_ ? 0 : (uint64_t) max_t_ * (uint64_t) n_vocab * sizeof(float);
+    if (dhead_ == nullptr) {
+        if (FILE* f = std::fopen((rt_dir_ + "/draft_vocab.bin").c_str(), "rb")) {
+            std::fseek(f, 0, SEEK_END);
+            const long size = std::ftell(f);
+            std::fclose(f);
+            if (size >= 4 && size % 4 == 0) bytes += (uint64_t) (size / 4) * head_row_bytes + (uint64_t) size;
+        }
+    }
+    // coupled draft sampling (STRATA_SPEC_COUPLED=1 only): an upper bound - the id -> subset map, the penalty ring,
+    // the split scratch (64 lists of 64 entries at most) and the parameters
+    if (coupled_draft_env() && cparams_ == nullptr)
+        bytes += (uint64_t) n_vocab * 4 + (uint64_t) (kCoupledHistCap + max_t_) * 4 + 64ull * 64ull * 8ull + 4096;
+    return bytes;
+}
+
+bool MtpDrafter::setup_coupled(std::string& err) {
+    const int64_t nv = dhead_ != nullptr ? n_dvocab_ : n_vocab_;
+    const size_t scratch = strata::kernels::coupled_draft_scratch_bytes((int) nv);
+    if (scratch == 0) {
+        std::fprintf(stderr, "strata mtp: STRATA_SPEC_COUPLED: %lld draft logits are too wide for the coupled sampler; "
+                             "argmax drafts\n", (long long) nv);
+        return true;
+    }
+    const size_t ring = (size_t) (kCoupledHistCap + max_t_) * sizeof(int32_t);
+    if (cudaMalloc((void**) &cparams_, sizeof(strata::kernels::SamplerParams)) != cudaSuccess ||
+        cudaMalloc((void**) &cring_, ring) != cudaSuccess || cudaMalloc(&cscratch_, scratch) != cudaSuccess ||
+        !mapped(sizeof(strata::kernels::SamplerParams), (void**) &h_cparams_, (void**) &m_cparams_) ||
+        !mapped((size_t) kCoupledHistCap * sizeof(int32_t), (void**) &h_chist_, (void**) &m_chist_)) {
+        err = "mtp: the coupled draft sampler's buffers do not fit";
+        return false;
+    }
+    cudaMemset(cring_, 0xff, ring);   // -1: no token
+    std::fill(h_chist_, h_chist_ + kCoupledHistCap, -1);
+    vram_ += sizeof(strata::kernels::SamplerParams) + ring + scratch;
+    if (dhead_ != nullptr) {   // token id -> subset index (-1: not in the draft head), for the penalties
+        std::vector<int32_t> sub((size_t) n_dvocab_), inv((size_t) n_vocab_, -1);
+        cudaMemcpy(sub.data(), dvocab_, sub.size() * sizeof(int32_t), cudaMemcpyDeviceToHost);
+        for (size_t i = 0; i < sub.size(); ++i)
+            if (sub[i] >= 0 && sub[i] < n_vocab_ && inv[(size_t) sub[i]] < 0) inv[(size_t) sub[i]] = (int32_t) i;
+        if (cudaMalloc((void**) &dinv_, inv.size() * sizeof(int32_t)) != cudaSuccess) {
+            err = "mtp: the coupled draft sampler's token map does not fit";
+            return false;
+        }
+        cudaMemcpy(dinv_, inv.data(), inv.size() * sizeof(int32_t), cudaMemcpyHostToDevice);
+        vram_ += inv.size() * sizeof(int32_t);
+    }
+    if (cudaDeviceSynchronize() != cudaSuccess) {
+        err = std::string("mtp: coupled setup: ") + cudaGetErrorString(cudaGetLastError());
+        return false;
+    }
+    coupled_ok_ = true;
+    std::fprintf(stderr, "strata mtp: coupled draft sampling on (STRATA_SPEC_COUPLED): sampled requests draft with the "
+                         "target's chain and Philox draw over %lld tokens\n", (long long) nv);
+    return true;
+}
+
+void MtpDrafter::set_draft_sampling(const strata::kernels::SamplerParams& sp) {
+    coupled_active_ = coupled_ok_ && !sp.greedy && sp.temperature > 0.0f;
+    if (!coupled_active_) return;
+    *h_cparams_ = sp;   // read by the next round graph (after the previous one has synced)
+    std::fill(h_chist_, h_chist_ + kCoupledHistCap, -1);
+}
+
+void MtpDrafter::set_draft_history(const int32_t* tail, int64_t n_tail, int32_t next) {
+    if (!coupled_active_) return;
+    const int h = coupled_hist_len(h_cparams_->penalty_last_n, kCoupledHistCap);
+    coupled_hist_base(tail, n_tail, next, h, h_chist_ + (kCoupledHistCap - h));
+}
+
 bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err) {
     const OnDevice on_device(device_);
     wt_ = &wt;
@@ -325,6 +405,7 @@ bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float
                          (double) (n_dvocab_ * row_bytes) / 1048576.0);
         }
     }
+    if (coupled_draft_env() && cparams_ == nullptr && !setup_coupled(err)) return false;
     return true;
 }
 
@@ -382,7 +463,7 @@ bool MtpDrafter::record_forward(int T, int step_row0, cudaStream_t cs, std::stri
         // ---- attention: K/V into the layer's own cache, then (full) dense attention over every cell
         auto norm_rope = [&](float* data, const float* gamma, int rows, int cols, const int32_t* p) {
             native_qsa_rms_norm_weighted(data, gamma, data, cols, rows, EPS, cs);
-            if (native_rope_enabled()) native_rope_apply(data, data, rows, cols, (int) s.n_rot, (float) qsa_freq_base(), p, cs);
+            if (native_rope_enabled()) native_rope_apply(data, data, rows, cols, (int) s.n_rot, rope_scaling(), p, cs);
             else rope_neox_apply(data, data, rows, cols, (int) s.n_rot, st_.cos_tab, st_.sin_tab, p, cs);
         };
         native_quantize_q8_1(mixed_, xq_, (int) N, T, cs);
@@ -475,6 +556,13 @@ bool MtpDrafter::record_forward(int T, int step_row0, cudaStream_t cs, std::stri
         const bool sub = dhead_ != nullptr;
         const int64_t nv = sub ? n_dvocab_ : n_vocab_;
         native_mmvq(head_->type(), sub ? dhead_ : head_->weights(), xq_, head_logits_, (int) N, (int) nv, T, cs);
+        if (coupled_rec_) {
+            // coupled draft sampling: the target's chain and Philox draw for the row that will verify this draft
+            // (counter = this cell + 1, from its step record), penalties over the ring; T == 1 (full layer)
+            coupled_draft_sample(head_logits_, (int) nv, sub ? dvocab_ : nullptr, sub ? dinv_ : nullptr, (int) n_vocab_,
+                                 cparams_, cring_, kCoupledHistCap, coupled_j_, step, cscratch_, out_ids_, probs_, cs);
+            return true;
+        }
         SamplerParams sp;
         sp.greedy = true;
         sp.temperature = 0.0f;
@@ -527,12 +615,15 @@ bool MtpDrafter::capture_prefill_dev(int T, std::string& err) {
     return finish_capture(cs_, ok, prefill_dev_exec_[T], "prefill (device inputs)", err);
 }
 
-bool MtpDrafter::capture_round(int T, std::string& err) {
-    if (round_exec_[T]) return true;
+bool MtpDrafter::capture_round(int T, bool coupled, std::string& err) {
+    cudaGraphExec_t& exec = coupled ? round_exec_c_[T] : round_exec_[T];
+    if (exec) return true;
     using namespace strata::kernels;
     const int64_t HCN = g_->hc * g_->n_embd;
     if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "mtp: begin capture"; return false; }
     bool ok = true;
+    // coupled: the request's chain and the penalty history's base, for this round's drafts
+    if (coupled) coupled_draft_stage(m_cparams_, m_chist_, cparams_, cring_, kCoupledHistCap, cs_);
     copy_i32_from_mapped(tok_, m_tok_, T, cs_);
     copy_i32_from_mapped(step_, m_step_, (int64_t) 2 * T * 4, cs_);
     copy_i32_from_mapped(pos_, m_pos_, (int64_t) 2 * T * g_->n_head, cs_);
@@ -547,25 +638,32 @@ bool MtpDrafter::capture_round(int T, std::string& err) {
     if (ok) {
         copy_i32_from_mapped(step_ + ra * 4, m_step_ + ra * 4, 4, cs_);
         copy_i32_from_mapped(pos_ + ra * g_->n_head, m_pos_ + ra * g_->n_head, g_->n_head, cs_);
+        coupled_rec_ = coupled;
+        coupled_j_ = 0;
         ok = record_forward(1, ra, cs_, err);
+        coupled_rec_ = false;
     }
     if (ok) mtp_select(R_, HCN, out_ids_, row_ + 1, Rin_, tok_, m_out_, 0, cs_, probs_, m_prob_);
-    return finish_capture(cs_, ok, round_exec_[T], "round", err);
+    return finish_capture(cs_, ok, exec, coupled ? "round (coupled)" : "round", err);
 }
 
 // Chain step j (1..max_t-2): one row at the cell staged in step row `max_t + j - 1`, from the previous step's
 // residual and token (left in Rin_[0] / tok_[0] by mtp_select); draft j and its probability to the mapped outputs.
-bool MtpDrafter::capture_step(int j, std::string& err) {
-    if (step_exec_[j]) return true;
+bool MtpDrafter::capture_step(int j, bool coupled, std::string& err) {
+    cudaGraphExec_t& exec = coupled ? step_exec_c_[j] : step_exec_[j];
+    if (exec) return true;
     using namespace strata::kernels;
     const int64_t HCN = g_->hc * g_->n_embd;
     const int row = max_t_ + j - 1;
     if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "mtp: begin capture"; return false; }
     copy_i32_from_mapped(step_ + row * 4, m_step_ + row * 4, 4, cs_);
     copy_i32_from_mapped(pos_ + row * g_->n_head, m_pos_ + row * g_->n_head, g_->n_head, cs_);
+    coupled_rec_ = coupled;
+    coupled_j_ = j;
     bool ok = record_forward(1, row, cs_, err);
+    coupled_rec_ = false;
     if (ok) mtp_select(R_, HCN, out_ids_, row_ + 1, Rin_, tok_, m_out_, j, cs_, probs_, m_prob_);
-    return finish_capture(cs_, ok, step_exec_[j], "step", err);
+    return finish_capture(cs_, ok, exec, coupled ? "step (coupled)" : "step", err);
 }
 
 void MtpDrafter::kv_restore(int64_t upto) {
@@ -671,7 +769,8 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
                        float* probs, float min_p, int* n_drafts) {
     const OnDevice on_device(device_);
     if (T < 1 || T > max_t_ || a < 0 || a >= T) { err = "mtp: draft arguments out of range"; return false; }
-    if (!capture_round(T, err)) return false;
+    const bool cp = coupled_active_;   // coupled draft sampling for this request: its own graphs
+    if (!capture_round(T, cp, err)) return false;
     const Clock::time_point t0 = Clock::now();
     const int64_t NH = g_->n_head;
     auto put = [&](int row, int64_t cell) {
@@ -685,11 +784,12 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
         h_tok_[t] = tokens[t];
         put(t, p + t);
     }
-    put(2 * max_t_ - 1, p + a);
+    put(2 * max_t_ - 1, coupled_draft_cell(p, a, 0));   // p + a: draft 0's cell
     h_row_[0] = a;
     h_row_[1] = 0;
     std::atomic_thread_fence(std::memory_order_seq_cst);
-    if (cudaGraphLaunch(round_exec_[T], cs_) != cudaSuccess || cudaStreamSynchronize(cs_) != cudaSuccess) {
+    if (cudaGraphLaunch(cp ? round_exec_c_[T] : round_exec_[T], cs_) != cudaSuccess ||
+        cudaStreamSynchronize(cs_) != cudaSuccess) {
         err = std::string("mtp draft: ") + cudaGetErrorString(cudaGetLastError());
         return false;
     }
@@ -699,10 +799,11 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
     int n = 1;
     // the chain continues while the last draft is likely enough to be verified
     for (int j = 1; j < std::min(max_t_ - 1, max_drafts_) && pj >= min_p; ++j) {
-        if (!capture_step(j, err)) return false;
-        put(max_t_ + j - 1, p + a + j);
+        if (!capture_step(j, cp, err)) return false;
+        put(max_t_ + j - 1, coupled_draft_cell(p, a, j));   // p + a + j; coupled: drawn at counter cell + 1
         std::atomic_thread_fence(std::memory_order_seq_cst);
-        if (cudaGraphLaunch(step_exec_[j], cs_) != cudaSuccess || cudaStreamSynchronize(cs_) != cudaSuccess) {
+        if (cudaGraphLaunch(cp ? step_exec_c_[j] : step_exec_[j], cs_) != cudaSuccess ||
+            cudaStreamSynchronize(cs_) != cudaSuccess) {
             err = std::string("mtp draft step: ") + cudaGetErrorString(cudaGetLastError());
             return false;
         }

src/core/native_head.cpp

@@ -5,6 +5,7 @@
 
 #include <cuda_runtime.h>
 #include <climits>
+#include <cstdio>
 #include <cstring>
 #include <exception>
 
@@ -102,6 +103,7 @@ const NativeEmbed* native_embed() { return g_embed; }
 
 NativeEmbed::~NativeEmbed() {
     if (host_) cudaFreeHost(host_);
+    else if (dev_) cudaFree(const_cast<void*>(dev_));   // the VRAM fallback below
 }
 
 bool NativeEmbed::load(const std::string& path, int64_t n_embd, int64_t n_vocab, std::string& err) {
@@ -119,17 +121,31 @@ bool NativeEmbed::load(const std::string& path, int64_t n_embd, int64_t n_vocab,
         row_ = strata::kernels::iq_row_bytes((int) t->type, n_embd);
         bytes_ = (uint64_t) row_ * (uint64_t) n_vocab;
         if (cudaHostAlloc(&host_, bytes_, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+            // Under WSL2 the driver's pinned/mapped host budget (~1 GiB) can be spent by the GPU contexts
+            // themselves (three cards). The table is only gathered from, so keep it in the current device's VRAM
+            // instead: it costs its size there and reads faster than over PCIe.
+            cudaGetLastError();
             host_ = nullptr;
-            err = "native embedding: cannot pin " + std::to_string(bytes_ >> 20) + " MiB";
-            return false;
+            void* d = nullptr;
+            if (cudaMalloc(&d, bytes_) != cudaSuccess ||
+                cudaMemcpy(d, gguf.tensor_data(*t), bytes_, cudaMemcpyHostToDevice) != cudaSuccess) {
+                if (d) cudaFree(d);
+                cudaGetLastError();
+                err = "native embedding: cannot pin " + std::to_string(bytes_ >> 20) + " MiB, nor place it in VRAM";
+                return false;
+            }
+            std::fprintf(stderr, "strata: native embedding: cannot pin %llu MiB, kept in VRAM instead\n",
+                         (unsigned long long) (bytes_ >> 20));
+            dev_ = d;
+        } else {
+            std::memcpy(host_, gguf.tensor_data(*t), bytes_);
+            void* d = nullptr;
+            if (cudaHostGetDevicePointer(&d, host_, 0) != cudaSuccess) {
+                err = "native embedding: no device alias for the mapped table";
+                return false;
+            }
+            dev_ = d;
         }
-        std::memcpy(host_, gguf.tensor_data(*t), bytes_);
-        void* d = nullptr;
-        if (cudaHostGetDevicePointer(&d, host_, 0) != cudaSuccess) {
-            err = "native embedding: no device alias for the mapped table";
-            return false;
-        }
-        dev_ = d;
         type_ = (int) t->type;
         n_embd_ = n_embd;
         n_vocab_ = n_vocab;

src/core/pinned.cu

@@ -5,6 +5,7 @@
 #include <cuda_runtime.h>
 
 #include <atomic>
+#include <cerrno>
 #include <cstdlib>
 #include <cstdio>
 #include <cstring>
@@ -23,7 +24,10 @@
 #ifdef _WIN32
 #include <windows.h>
 #else
+#include <fcntl.h>
 #include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
 #include <linux/mman.h>
 #ifndef MAP_HUGE_2MB
 #define MAP_HUGE_2MB (21 << 26)
@@ -34,9 +38,32 @@ namespace strata::core {
 
 namespace {
 
+constexpr uint64_t kSharedArenaHeaderBytes = 4096;
+constexpr char kSharedArenaMagic[16] = "STRATA-ARENA-V1";
+
+struct SharedArenaHeader {
+    char magic[16];
+    uint32_t version;
+    uint32_t header_bytes;
+    uint64_t arena_bytes;
+    uint64_t pack_hash;
+    uint64_t reserved[4];
+};
+static_assert(sizeof(SharedArenaHeader) <= kSharedArenaHeaderBytes);
+
 // A 2 MB-aligned reservation.  Large pages first, then the largest alignment the OS will give us for free.
-void* reserve(uint64_t bytes, PageBacking& got, std::string& note) {
+// A non-empty shared_file instead maps one file whose first 4 KiB identify the pack and whose remaining bytes
+// are the resident arena.  This shared-file layout is intended for tmpfs (/dev/shm); hugetlbfs would need
+// hugepage-aligned file size and arena offset rather than the 4 KiB header layout used here.
+void* reserve(uint64_t bytes, PageBacking& got, std::string& note, const std::string& shared_file,
+              uint64_t shared_pack_hash, void*& mapping_base, uint64_t& mapping_bytes) {
+    mapping_base = nullptr;
+    mapping_bytes = 0;
 #ifdef _WIN32
+    if (!shared_file.empty()) {
+        note = "shared-file arena backing is not implemented on Windows";
+        return nullptr;
+    }
     // MEM_LARGE_PAGES needs SeLockMemoryPrivilege.  Having it assigned to the account is not enough: the
     // PROCESS must enable it in its own token (AdjustTokenPrivileges) before VirtualAlloc, or the call fails.
     // An account without the assignment, or a failure to enable, leaves the process as it was: VirtualAlloc
@@ -84,6 +111,94 @@ void* reserve(uint64_t bytes, PageBacking& got, std::string& note) {
     got = PageBacking::NormalPages;
     return p;
 #else
+    if (!shared_file.empty()) {
+        if (shared_pack_hash == 0) {
+            note = "shared arena requires a nonzero pack hash";
+            return nullptr;
+        }
+        if (bytes > UINT64_MAX - kSharedArenaHeaderBytes) {
+            note = "shared arena size overflows its header";
+            return nullptr;
+        }
+        const uint64_t file_bytes = kSharedArenaHeaderBytes + bytes;
+        const int fd = open(shared_file.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+        if (fd < 0) {
+            note = "cannot open shared arena " + shared_file + ": " + std::strerror(errno);
+            return nullptr;
+        }
+        struct stat st{};
+        if (fstat(fd, &st) != 0) {
+            const int e = errno;
+            close(fd);
+            note = "cannot stat shared arena " + shared_file + ": " + std::strerror(e);
+            return nullptr;
+        }
+
+        const bool fresh = st.st_size == 0;
+        if (fresh) {
+            if (ftruncate(fd, (off_t) file_bytes) != 0) {
+                const int e = errno;
+                close(fd);
+                note = "cannot size shared arena " + shared_file + ": " + std::strerror(e);
+                return nullptr;
+            }
+            SharedArenaHeader hdr{};
+            std::memcpy(hdr.magic, kSharedArenaMagic, sizeof hdr.magic);
+            hdr.version = 1;
+            hdr.header_bytes = (uint32_t) kSharedArenaHeaderBytes;
+            hdr.arena_bytes = bytes;
+            hdr.pack_hash = shared_pack_hash;
+            const ssize_t written = pwrite(fd, &hdr, sizeof hdr, 0);
+            if (written != (ssize_t) sizeof hdr) {
+                const int e = errno;
+                close(fd);
+                note = "cannot write shared arena header " + shared_file + ": " +
+                       (written < 0 ? std::string(std::strerror(e)) : std::string("short write"));
+                return nullptr;
+            }
+        } else {
+            if (st.st_size < 0 || (uint64_t) st.st_size != file_bytes) {
+                close(fd);
+                note = "shared arena " + shared_file + " is " +
+                       std::to_string((unsigned long long) st.st_size) + " B, expected " +
+                       std::to_string((unsigned long long) file_bytes) + " B including its header";
+                return nullptr;
+            }
+            SharedArenaHeader hdr{};
+            const ssize_t got_header = pread(fd, &hdr, sizeof hdr, 0);
+            if (got_header != (ssize_t) sizeof hdr ||
+                std::memcmp(hdr.magic, kSharedArenaMagic, sizeof hdr.magic) != 0 ||
+                hdr.version != 1 || hdr.header_bytes != kSharedArenaHeaderBytes || hdr.arena_bytes != bytes) {
+                close(fd);
+                note = "shared arena " + shared_file + " has an incompatible or missing header";
+                return nullptr;
+            }
+            if (hdr.pack_hash != shared_pack_hash) {
+                close(fd);
+                char b[256];
+                std::snprintf(b, sizeof b,
+                              "shared arena %s was written for pack hash %016llx, expected %016llx",
+                              shared_file.c_str(), (unsigned long long) hdr.pack_hash,
+                              (unsigned long long) shared_pack_hash);
+                note = b;
+                return nullptr;
+            }
+        }
+
+        void* map = mmap(nullptr, (size_t) file_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+        const int e = errno;
+        close(fd);
+        if (map == MAP_FAILED) {
+            note = "cannot map shared arena " + shared_file + ": " + std::strerror(e);
+            return nullptr;
+        }
+        mapping_base = map;
+        mapping_bytes = file_bytes;
+        got = PageBacking::NormalPages;
+        note = "MAP_SHARED file " + shared_file + " (pack hash checked)";
+        return (uint8_t*) map + kSharedArenaHeaderBytes;
+    }
+
     void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE,
                    MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, -1, 0);
     if (p != MAP_FAILED) {
@@ -138,9 +253,14 @@ PinnedArena::PinnedArena(uint64_t bytes, uint64_t slice) : PinnedArena(bytes, un
 }
 
 PinnedArena::PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds,
-                         uint64_t max_pinned_bytes) : capacity(bytes) {
+                         uint64_t max_pinned_bytes, const std::string& shared_file,
+                         uint64_t shared_pack_hash) : capacity(bytes) {
     if (bytes == 0) return;
-    base = reserve(bytes, backing, note);
+    base = reserve(bytes, backing, note, shared_file, shared_pack_hash, mapping_base, mapping_bytes);
+    if (base != nullptr && mapping_base == nullptr) {
+        mapping_base = base;
+        mapping_bytes = bytes;
+    }
 
     // Register with CUDA BEFORE any page is touched: cudaHostRegister pins what is resident now, and a region
     // that has already been faulted in page by page is far more expensive to register and may fail outright.
@@ -218,8 +338,10 @@ PinnedArena::~PinnedArena() {
         } else {
             cudaHostUnregister(base);
         }
-        release(base, capacity);
+        release(mapping_base ? mapping_base : base, mapping_bytes ? mapping_bytes : capacity);
         base = nullptr;
+        mapping_base = nullptr;
+        mapping_bytes = 0;
     }
 }
 

src/core/pinned_shared_test.cpp

@@ -0,0 +1,61 @@
+// Linux selftest for the optional MAP_SHARED backing of PinnedArena.
+#include "strata/core/pinned.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+#include <vector>
+
+#include <unistd.h>
+
+int main() {
+    char path[] = "/tmp/strata-shared-arena-XXXXXX";
+    const int fd = mkstemp(path);
+    if (fd < 0) {
+        std::perror("mkstemp");
+        return 2;
+    }
+    close(fd);
+
+    int fail = 0;
+    const uint64_t bytes = 64u << 10;
+    const uint64_t pack_hash = 0x0123456789abcdefull;
+    const std::vector<uint64_t> no_bounds;
+    {
+        strata::core::PinnedArena first(bytes, no_bounds, 0, path, pack_hash);
+        if (!first.valid()) {
+            std::fprintf(stderr, "first shared arena failed: %s\n", first.note.c_str());
+            fail = 1;
+        } else {
+            first.data()[123] = 0x5a;
+
+            strata::core::PinnedArena second(bytes, no_bounds, 0, path, pack_hash);
+            if (!second.valid()) {
+                std::fprintf(stderr, "second shared arena failed: %s\n", second.note.c_str());
+                fail = 1;
+            } else if (second.data()[123] != 0x5a) {
+                std::fprintf(stderr, "shared mapping did not expose the same bytes\n");
+                fail = 1;
+            }
+
+            strata::core::PinnedArena wrong_pack(bytes, no_bounds, 0, path, pack_hash + 1);
+            if (wrong_pack.valid() || wrong_pack.note.find("pack hash") == std::string::npos) {
+                std::fprintf(stderr, "pack mismatch was not refused: %s\n", wrong_pack.note.c_str());
+                fail = 1;
+            }
+
+            strata::core::PinnedArena wrong_size(bytes * 2, no_bounds, 0, path, pack_hash);
+            if (wrong_size.valid() || wrong_size.note.find("expected") == std::string::npos) {
+                std::fprintf(stderr, "size mismatch was not refused: %s\n", wrong_size.note.c_str());
+                fail = 1;
+            }
+        }
+    }
+
+    if (unlink(path) != 0) {
+        std::perror("unlink");
+        fail = 1;
+    }
+    std::printf("pinned_shared_test: %s\n", fail ? "FAILED" : "OK");
+    return fail;
+}

src/core/session.cpp

@@ -50,13 +50,20 @@ static uint64_t ple_hist_bytes() {
     return (uint64_t) strata::kernels::NG_HIST * strata::kernels::NG_HC_DIM * sizeof(float);
 }
 
-uint64_t session_bytes(const ModelGeometry& g, int64_t max_cells, int64_t k) {
+uint64_t session_bytes(const ModelGeometry& g, int64_t max_cells, int64_t k, int64_t layer_lo, int64_t layer_hi) {
+    if (layer_hi < 0 || layer_hi > g.n_layers) layer_hi = g.n_layers;
+    if (layer_lo < 0) layer_lo = 0;
+    // QSA layers are `l % interval == interval-1`, so exactly `bound / interval` of them live below `bound`
+    const int64_t I = std::max<int64_t>(g.qsa_interval, 1);
+    const int64_t q_lo = layer_lo / I, q_hi = layer_hi / I;
+    const int64_t q_n = std::max<int64_t>(q_hi - q_lo, g.n_qsa_layers() > 0 ? 1 : 0);
+    const int64_t gdn_n = std::max<int64_t>((layer_hi - layer_lo) - std::max<int64_t>(q_hi - q_lo, 0), 0);
     uint64_t n = 0;
     n += gdn_buffers_bytes(g);
-    n += (uint64_t) g.n_gdn_layers() * gdn_state_floats(g) * 4;
+    n += (uint64_t) gdn_n * gdn_state_floats(g) * 4;
     // One QSA state carries the RoPE table; the others borrow it (P7: 64 MiB per layer at 262K).
     if (g.n_qsa_layers() > 0)
-        n += qsa_state_bytes(g, max_cells, true) + (uint64_t) (g.n_qsa_layers() - 1) * qsa_state_bytes(g, max_cells, false);
+        n += qsa_state_bytes(g, max_cells, true) + (uint64_t) (q_n - 1) * qsa_state_bytes(g, max_cells, false);
     n += qsa_buffers_bytes(g, max_cells);
     n += moe_buffers_bytes(g, k);
     n += block_buffers_bytes(g);
@@ -64,7 +71,10 @@ uint64_t session_bytes(const ModelGeometry& g, int64_t max_cells, int64_t k) {
     return align_up(n, SESSION_STATE_ALIGN) + 4096;
 }
 
-uint64_t session_init(const ModelGeometry& g, int64_t max_cells, int64_t k, void* base, SessionState& s) {
+uint64_t session_init(const ModelGeometry& g, int64_t max_cells, int64_t k, void* base, SessionState& s,
+                      int64_t layer_lo, int64_t layer_hi) {
+    if (layer_hi < 0 || layer_hi > g.n_layers) layer_hi = g.n_layers;
+    if (layer_lo < 0) layer_lo = 0;
     uint8_t* p = (uint8_t*) base;
     uint64_t used = 0;
     auto take = [&](uint64_t bytes) {
@@ -75,24 +85,34 @@ uint64_t session_init(const ModelGeometry& g, int64_t max_cells, int64_t k, void
 
     s.max_cells = max_cells;
     s.k = k;
+    s.layer_lo = layer_lo;
+    s.layer_hi = layer_hi;
+    const int64_t I = std::max<int64_t>(g.qsa_interval, 1);
+    const int64_t q_n_range = layer_hi / I - layer_lo / I;
+    s.qsa_ord0 = layer_lo / I;
+    s.qsa_alloc = std::max<int64_t>(q_n_range, g.n_qsa_layers() > 0 ? 1 : 0);
+    s.gdn_ord0 = layer_lo - layer_lo / I;
+    s.gdn_alloc = std::max<int64_t>((layer_hi - layer_lo) - q_n_range, 0);
 
     gdn_buffers_init(g, take(gdn_buffers_bytes(g)), s.gdn);
-    s.gdn_state = (float*) take((uint64_t) g.n_gdn_layers() * gdn_state_floats(g) * 4);
+    s.gdn_state = (float*) take((uint64_t) s.gdn_alloc * gdn_state_floats(g) * 4);
 
-    // the 12 QSA states are separate allocations carved from one arena, because `QsaState` is a struct of
+    // the QSA states are separate allocations carved from one arena, because `QsaState` is a struct of
     // pointers and `qsa_state_init` writes them - a contiguous array would need the arena to be laid out the
-    // same way, which is a coupling with nothing to gain.
+    // same way, which is a coupling with nothing to gain.  Only the range's ordinals are initialized; the
+    // rest stay value-initialized nulls.  The FIRST ALLOCATED one (the session's primary, ordinal qsa_ord0)
+    // owns the RoPE table that the others - and the prefill staging identity, and the MTP drafter - borrow.
     const uint64_t first = qsa_state_bytes(g, max_cells, true), rest = qsa_state_bytes(g, max_cells, false);
-    s.qsa_state_arena = take(g.n_qsa_layers() > 0 ? first + (uint64_t) (g.n_qsa_layers() - 1) * rest : 0);
-    s.qsa_states = new QsaState[(size_t) g.n_qsa_layers()];
+    s.qsa_state_arena = take(g.n_qsa_layers() > 0 ? first + (uint64_t) (s.qsa_alloc - 1) * rest : 0);
+    s.qsa_states = new QsaState[(size_t) g.n_qsa_layers()]();
     s.qsa_buf_arena = take(qsa_buffers_bytes(g, max_cells));
 
     uint8_t* qp = (uint8_t*) s.qsa_state_arena;
     // a layer whose pinned RAM could not be had (KV streaming's host copy) is half-built: going on would have the
     // attention read null host pointers at the first request ("illegal memory access"), so the session fails here
-    for (int64_t i = 0; i < g.n_qsa_layers(); ++i)
-        if (qsa_state_init(g, max_cells, qp + (i == 0 ? 0 : first + (uint64_t) (i - 1) * rest), s.qsa_states[i],
-                           i == 0 ? nullptr : &s.qsa_states[0]) == 0)
+    for (int64_t j = 0; j < s.qsa_alloc; ++j)
+        if (qsa_state_init(g, max_cells, qp + (j == 0 ? 0 : first + (uint64_t) (j - 1) * rest),
+                           s.qsa_states[s.qsa_ord0 + j], j == 0 ? nullptr : &s.qsa_states[s.qsa_ord0]) == 0)
             return 0;
     qsa_buffers_init(g, max_cells, s.qsa_buf_arena, s.qsa_bufs);
 
@@ -118,10 +138,10 @@ void session_zero(SessionState& s, const ModelGeometry& g, const float* R_init,
     } else {
         cudaMemsetAsync(s.block.R, 0, (size_t) g.hc * g.n_embd * 4, cs);
     }
-    // every GDN layer's recurrence and conv history
-    cudaMemsetAsync(s.gdn_state, 0, (size_t) g.n_gdn_layers() * gdn_state_floats(g) * 4, cs);
-    // and every QSA layer's cache and indexer
-    for (int64_t i = 0; i < g.n_qsa_layers(); ++i) qsa_state_zero(s.qsa_states[i], g, stream);
+    // every owned GDN layer's recurrence and conv history
+    cudaMemsetAsync(s.gdn_state, 0, (size_t) s.gdn_alloc * gdn_state_floats(g) * 4, cs);
+    // and every owned QSA layer's cache and indexer
+    for (int64_t j = 0; j < s.qsa_alloc; ++j) qsa_state_zero(s.qsa_states[s.qsa_ord0 + j], g, stream);
     // **AND THE PLE'S CONV HISTORY AND TOKEN WINDOW.**  A sequence that started with a warm history would
     // convolve over rows belonging to a different sequence - the conv reads NG_HIST previous NORMALIZED rows,
     // so a stale one is a real contribution and not a zero.  The token window resets to `NG_HIST`-many nulls
@@ -139,7 +159,7 @@ void gdn_point_at(const ModelGeometry& g, int64_t layer, SessionState& s) {
     if (is_qsa_layer(g, layer)) return;
     int64_t gdn_index = 0;
     for (int64_t l = 0; l < layer; ++l) if (!is_qsa_layer(g, l)) ++gdn_index;
-    s.gdn.state = s.gdn_state + (size_t) gdn_index * gdn_state_floats(g);
+    s.gdn.state = s.gdn_state + (size_t) (gdn_index - s.gdn_ord0) * gdn_state_floats(g);
     s.gdn.conv_state = s.gdn.state + (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size;
 }
 
@@ -152,36 +172,38 @@ void stage_token(const ModelGeometry& g, int64_t pos, int32_t pos_base, SessionS
     sh.head_dim = g.head_dim;
     sh.idx_n_head = g.idx_q_heads;
     sh.idx_dim = g.idx_key_dim;
-    for (int64_t i = 0; i < g.n_qsa_layers(); ++i) {
-        QsaState& q = s.qsa_states[i];
+    for (int64_t j = 0; j < s.qsa_alloc; ++j) {
+        QsaState& q = s.qsa_states[s.qsa_ord0 + j];
         qsa_step_fill(q.host_step, pos, sh);
         for (int64_t h = 0; h < g.n_head; ++h) q.host_pos[h] = (int32_t) (pos_base + pos);
     }
 }
 
 bool session_capture(const WeightTable& tables, const ModelGeometry& g, SessionState& s, const float* parts,
-                     SessionGraphs& gr, std::string& err, bool split) {
+                     SessionGraphs& gr, std::string& err, bool split, int64_t layer_lo, int64_t layer_hi) {
     if (gr.captured) return true;
-    gr.execs = new cudaGraphExec_t[(size_t) g.n_layers];
-    gr.posts = new cudaGraphExec_t[(size_t) g.n_layers];
+    if (layer_hi < 0 || layer_hi > g.n_layers) layer_hi = g.n_layers;
+    if (layer_lo < 0) layer_lo = 0;
+    gr.execs = new cudaGraphExec_t[(size_t) g.n_layers]();
+    gr.posts = new cudaGraphExec_t[(size_t) g.n_layers]();
     for (int64_t i = 0; i < g.n_layers; ++i) gr.posts[i] = nullptr;
     if (split) {
-        gr.preA = new cudaGraphExec_t[(size_t) g.n_layers];
-        gr.preB = new cudaGraphExec_t[(size_t) g.n_layers];
+        gr.preA = new cudaGraphExec_t[(size_t) g.n_layers]();
+        gr.preB = new cudaGraphExec_t[(size_t) g.n_layers]();
         for (int64_t i = 0; i < g.n_layers; ++i) { gr.preA[i] = nullptr; gr.preB[i] = nullptr; }
         for (int k = 0; k < 5; ++k) {
-            gr.preP[k] = new cudaGraphExec_t[(size_t) g.n_layers];
+            gr.preP[k] = new cudaGraphExec_t[(size_t) g.n_layers]();
             for (int64_t i = 0; i < g.n_layers; ++i) gr.preP[k][i] = nullptr;
         }
     }
     gr.split_captured = split;
     gr.n = 0;
 
-    int64_t qsa_index = 0;
-    for (int64_t l = 0; l < g.n_layers; ++l) {
+    int64_t qsa_index = s.qsa_ord0;
+    for (int64_t l = layer_lo; l < layer_hi; ++l) {
         gdn_point_at(g, l, s);
         const bool qsa = is_qsa_layer(g, l);
-        QsaState& qst = qsa ? s.qsa_states[qsa_index] : s.qsa_states[0];
+        QsaState& qst = qsa ? s.qsa_states[qsa_index] : s.qsa_states[s.qsa_primary()];
 
         // **TWO GRAPHS PER LAYER, SPLIT AT THE ROUTER.**  `pre` ends with the doorbell rung; the host then runs
         // the CPU pool on what it published; `post` combines those experts with THIS layer's weights.  Capturing

src/core/verify.cpp

@@ -148,7 +148,8 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
         return false;
     }
     if (hits.d_res == nullptr || hits.cache_base == nullptr || hits.blob <= 0) {
-        err = "verify: needs the profile-filled VRAM expert tier (--expert-profile and --expert-cache)";
+        err = "verify: needs the profile-filled VRAM expert tier (--expert-profile and --expert-cache); with "
+              "--expert-cache auto, no VRAM was left for it: lower --max-context, use --kv k8v4 or images on the CPU";
         return false;
     }
     std::string why;
@@ -173,7 +174,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
 
     const strata::kernels::QsaShapes s = shapes_of(g);
     cap_ = strata::kernels::qsa_selection_width(strata::kernels::kTopkMaxCells, s);
-    max_blocks_ = ss.qsa_states[0].max_cells / s.idx_block + 2;
+    max_blocks_ = ss.qsa_states[ss.qsa_primary()].max_cells / s.idx_block + 2;
     attn_scratch_floats_ = (int64_t) strata::kernels::qsa_decode_attn_scratch_floats(cap_, s);
 
     const uint64_t T = (uint64_t) max_t, N = (uint64_t) g.n_embd, HC = (uint64_t) g.hc, K = (uint64_t) ss.k;
@@ -452,7 +453,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     !native_of(wout, v.name("ssm_out.weight"), err))
                     return false;
                 const int64_t gi = gdn_idx[(size_t) l];
-                float* state = ss.gdn_state + (size_t) gi * gdn_floats;
+                float* state = ss.gdn_state + (size_t) (gi - ss.gdn_ord0) * gdn_floats;
                 float* conv = state + (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size;
                 float* qkv = qkv_L_ + (size_t) gi * MT * C;
                 float* hb = h_L_ + (size_t) gi * MT * C;
@@ -491,7 +492,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 auto norm_rope = [&](float* data, const WeightRef* norm, int rows, int cols, const int32_t* pos) {
                     if (native_qsa_enabled()) native_qsa_rms_norm_weighted(data, (const float*) norm->data, data, cols, rows, EPS, cs);
                     else rms_norm_weighted(data, (const float*) norm->data, rows, cols, EPS, cs);
-                    if (native_rope_enabled()) native_rope_apply(data, data, rows, cols, (int) s.n_rot, (float) qsa_freq_base(), pos, cs);
+                    if (native_rope_enabled()) native_rope_apply(data, data, rows, cols, (int) s.n_rot, rope_scaling(), pos, cs);
                     else rope_neox_apply(data, data, rows, cols, (int) s.n_rot, st.cos_tab, st.sin_tab, pos, cs);
                 };
                 float* idx_raw = idx_raw_L_ + (size_t) qi * MT * ID;
@@ -536,7 +537,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 for (int t = tb; t < te; ++t)
                     native_qsa_indexer_append(idx_raw + t * ID, step_ + t * kStepCount + kStepPos, 0,
                                               (const float*) wikn->data, EPS, ib, s, st.max_cells,
-                                              (float) qsa_freq_base(), cs);
+                                              rope_scaling(), cs);
                 stamp(l, 9, grp);
                 native_mmvq(wq->native_type, wq->native_data, xq_, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD),
                             n, cs);
@@ -850,8 +851,10 @@ bool Verifier::capture(int T, std::string& err) {
             if (ty == cudaGraphNodeTypeKernel) {
                 cudaKernelNodeParams kp{};
                 if (cudaGraphKernelNodeGetParams(nd, &kp) == cudaSuccess) {
+#if CUDART_VERSION >= 12030   // cudaFuncGetName arrived in CUDA 12.3
                     const char* fn = nullptr;
                     if (cudaFuncGetName(&fn, kp.func) == cudaSuccess && fn) name = fn;
+#endif
                 }
             } else if (ty == cudaGraphNodeTypeMemcpy) name = "memcpy";
             else if (ty == cudaGraphNodeTypeMemset) name = "memset";
@@ -903,7 +906,7 @@ bool Verifier::capture_commit(std::string& err) {
             if (!is_qsa_layer(g, l)) {
                 const WeightRef* wnm = need(v, "ssm_norm.weight", err);
                 if (!wnm) { ok = false; break; }
-                float* state = ss.gdn_state + (size_t) gdn_index * gdn_floats;
+                float* state = ss.gdn_state + (size_t) (gdn_index - ss.gdn_ord0) * gdn_floats;
                 float* conv = state + (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size;
                 const float* qkv = qkv_L_ + (size_t) gdn_index * MT * C;
                 gdn_conv_commit(conv, qkv, (int) C, commit_, cs_);
@@ -920,7 +923,7 @@ bool Verifier::capture_commit(std::string& err) {
                 for (int64_t t = 0; t < MT; ++t)
                     native_qsa_indexer_append(idx_raw_L_ + (size_t) (qsa_index * MT + t) * ID, commit_ + 2 + t, 0,
                                               (const float*) wikn->data, EPS, ib, s, st.max_cells,
-                                              (float) qsa_freq_base(), cs_);
+                                              rope_scaling(), cs_);
                 ++qsa_index;
             }
         }
@@ -951,7 +954,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
     if (T < 1 || T > max_t_) { err = "verify: window size out of range"; return false; }
     const ModelGeometry& g = *g_;
     SessionState& ss = *ss_;
-    if (pos0 + T > ss.qsa_states[0].max_cells) { err = "verify: the window runs past the context"; return false; }
+    if (pos0 + T > ss.qsa_states[ss.qsa_primary()].max_cells) { err = "verify: the window runs past the context"; return false; }
     if (!capture(T, err) || !capture_commit(err)) return false;
     VDBG("captured; staging\n");
     const Clock::time_point t0 = Clock::now();

src/kernels/bf16_bits_test.cpp

@@ -0,0 +1,80 @@
+// src/kernels/bf16_bits_test.cpp - `bf16_bits.hpp` against ggml's rule, over EVERY f32 bit pattern (CPU only).
+//
+// the rounding add `i + 0x7FFF + bit16` carries out of the mantissa for a NaN, so 0x7FFFFFFF came back as
+// -0 and 0x7F800001 as +inf - an upstream NaN was masked instead of propagated.  `ggml_compute_fp32_to_bf16`
+// (ggml-impl.h at the pinned llama.cpp) tests the NaN first and returns `(i >> 16) | 64`.  Three checks, all
+// exhaustive (2^32 inputs, a few seconds):
+//
+//   1. the header equals ggml's function, transcribed below, on every input;
+//   2. on every NON-NaN input it equals the previous rounding, so the change cannot move a finite value or an
+//      infinity by a single bit;
+//   3. every NaN input gives a quiet NaN of the same sign.
+//
+// And the round trip over all 65,536 bf16 patterns: a non-NaN bf16 survives f32 and back unchanged.
+#include "strata/kernels/bf16_bits.hpp"
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+
+namespace {
+
+/// ggml_compute_fp32_to_bf16, ggml/src/ggml-impl.h (MIT, the ggml authors), on raw bits.
+uint16_t ggml_bf16(uint32_t u) {
+    if ((u & 0x7fffffffu) > 0x7f800000u) return (uint16_t) ((u >> 16) | 64);
+    return (uint16_t) ((u + (0x7fffu + ((u >> 16) & 1u))) >> 16);
+}
+
+/// The rule before this fix, kept only to prove the change is confined to NaN inputs.
+uint16_t previous_bf16(uint32_t u) {
+    u = (u + ((u >> 16) & 1u) + 0x7FFFu) & 0xFFFF0000u;
+    return (uint16_t) (u >> 16);
+}
+
+bool is_nan_bits(uint32_t u) { return (u & 0x7fffffffu) > 0x7f800000u; }
+
+}  // namespace
+
+int main() {
+    unsigned long long vs_ggml = 0, vs_previous = 0, nan_lost = 0, nan_inputs = 0, previous_masked = 0;
+    uint32_t first_bad = 0;
+    bool have_bad = false;
+    for (uint64_t k = 0; k <= 0xFFFFFFFFull; ++k) {
+        const uint32_t u = (uint32_t) k;
+        float f;
+        std::memcpy(&f, &u, 4);
+        const uint16_t got = strata::kernels::bf16_from_f32(f);
+        if (got != ggml_bf16(u)) {
+            ++vs_ggml;
+            if (!have_bad) { first_bad = u; have_bad = true; }
+        }
+        if (is_nan_bits(u)) {
+            ++nan_inputs;
+            const uint32_t back = (uint32_t) got << 16;
+            // a quiet NaN (bit 22 of the f32) with the input's sign
+            if (!is_nan_bits(back) || (back & 0x00400000u) == 0 || (back >> 31) != (u >> 31)) ++nan_lost;
+            if (!is_nan_bits((uint32_t) previous_bf16(u) << 16)) ++previous_masked;
+        } else if (got != previous_bf16(u)) {
+            ++vs_previous;
+            if (!have_bad) { first_bad = u; have_bad = true; }
+        }
+    }
+    std::printf("  f32 -> bf16 over 2^32 inputs: %llu differ from ggml, %llu non-NaN differ from the previous rule\n",
+                vs_ggml, vs_previous);
+    std::printf("  NaN inputs: %llu, not a quiet same-sign NaN now: %llu (the previous rule masked %llu of them)\n",
+                nan_inputs, nan_lost, previous_masked);
+    if (have_bad) std::printf("    first mismatch at input 0x%08x\n", first_bad);
+
+    unsigned long long trip_bad = 0;
+    for (uint32_t h = 0; h <= 0xFFFFu; ++h) {
+        const float f = strata::kernels::f32_from_bf16((uint16_t) h);
+        const uint16_t back = strata::kernels::bf16_from_f32(f);
+        const bool nan = is_nan_bits(h << 16);
+        if (back != (nan ? (uint16_t) (h | 64u) : (uint16_t) h)) ++trip_bad;
+    }
+    std::printf("  bf16 -> f32 -> bf16 over 65536 patterns: %llu wrong\n", trip_bad);
+
+    const bool ok = vs_ggml == 0 && vs_previous == 0 && nan_lost == 0 && trip_bad == 0 && previous_masked > 0;
+    std::printf("\nbf16_bits_test %s\n", ok ? "OK" : "FAILED");
+    return ok ? 0 : 1;
+}

src/kernels/cpu/iq_avx2.cpp

@@ -19,6 +19,7 @@
 #include <immintrin.h>
 
 #include <cmath>
+#include <cstdlib>
 #include <cstring>
 
 namespace strata::kernels::cpu {
@@ -77,6 +78,23 @@ inline float hsum8(__m256 v) {
     return _mm_cvtss_f32(s);
 }
 
+// E-2 (iq_avx512.cpp) on the AVX-2 path: the pool streams the expert rows from DRAM at ~25 GB/s (4 KB pages
+// when large pages are refused), so ask for the bytes a few blocks before the decode needs them - 2048 B is
+// two gate/up rows ahead, a row is ~1 KB.  Same switch as the AVX-512 kernels: STRATA_IQ_PREFETCH is the
+// distance in bytes, 0 = off, default 2048.  Measured on a Zen 3 5700X3D (no AVX-512) on IQ3_S decode: -4% on
+// the gate/up phase, +1.0 GB/s over the rows, -1.3% ms/round end to end.  The non-temporal hint measured
+// worse than T0 at the same distance, so this keeps T0.
+const int prefetch_ahead = [] {
+    const char* v = std::getenv("STRATA_IQ_PREFETCH");
+    return v ? std::atoi(v) : 2048;
+}();
+
+inline void rows_ahead(const uint8_t* p) {
+    if (prefetch_ahead <= 0) return;
+    _mm_prefetch((const char*) p, _MM_HINT_T0);
+    _mm_prefetch((const char*) p + 64, _MM_HINT_T0);
+}
+
 // ---- per format: one 32-value half (values 64*j + 32*half .. +31) -> grid magnitudes, sign vector, scales
 template <int TY> struct Fmt32;
 
@@ -165,6 +183,7 @@ inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y,
     for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
     for (int i = 0; i < nblocks; ++i) {
         const uint8_t* blk = row + (size_t) i * Fmt32<TY>::bytes;
+        rows_ahead(blk + prefetch_ahead);
         __m256i acci[NT];
         for (int t = 0; t < NT; ++t) acci[t] = _mm256_setzero_si256();
         for (int j = 0; j < 4; ++j) {
@@ -229,6 +248,7 @@ inline void row_dot_iq2xs(const uint8_t* row, int nblocks, const block_q8_K* con
     for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
     for (int i = 0; i < nblocks; ++i) {
         const uint8_t* blk = row + (size_t) i * 74;
+        rows_ahead(blk + prefetch_ahead);
         // the 8 scale bytes -> 16 half-scales of 2*s+1, interleaved [a0, b0, a1, b1, ...] (ggml's unpack)
         __m128i st = _mm_set1_epi64x((long long) u64(blk + 66));
         st = _mm_unpacklo_epi8(_mm_and_si128(st, m4), _mm_and_si128(_mm_srli_epi16(st, 4), m4));
@@ -382,6 +402,7 @@ void iq4nl_rows(const uint8_t* w, size_t row_bytes, int n, const block_q8_0* con
         for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
         for (int ib = 0; ib < nb; ++ib) {
             const uint8_t* blk = row + (size_t) ib * sizeof(block_iq4_nl);
+            rows_ahead(blk + prefetch_ahead);
             const __m128i bits = _mm_loadu_si128((const __m128i*) (blk + 2));
             const __m128i lo = _mm_and_si128(bits, m4b);                      // values 0..15
             const __m128i hi = _mm_and_si128(_mm_srli_epi16(bits, 4), m4b);    // values 16..31

src/kernels/cpu/native_expert.cpp

@@ -84,7 +84,12 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*
     // both set (and on a CPU without AVX-512, STRATA_NO_IQ512 changes nothing).
     static const bool avx512 = cpu_avx512_ok() && std::getenv("STRATA_NO_IQ512") == nullptr;
     static const bool avx2 = std::getenv("STRATA_NO_IQ256") == nullptr;
-    if (nt >= 2 && iq512_supported(f.gu_type)) {   // one token: ggml-cpu is as fast or faster
+    // #152: from how many tokens the multi-token kernels run (ggml's vec_dot below that).  The default 2 is the
+    // measured-fastest rule, but a token's expert rows then round differently alone than in a group, so greedy output
+    // can depend on how many drafts a verify window held.  STRATA_IQ_MT_MIN=1 (opt-in, 0.1.30) uses the multi-token
+    // kernels for every group: output independent of the drafting, at a measured -1..-3% decode on IQ3_S (AVX-512).
+    static const int mt_min = [] { const char* e = std::getenv("STRATA_IQ_MT_MIN"); return e ? std::atoi(e) : 2; }();
+    if (nt >= mt_min && iq512_supported(f.gu_type)) {
         if (avx512) {
             iq512_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
             return;
@@ -113,7 +118,8 @@ void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const
     // IQ4_NL down rows: the AVX-2 multi-token kernel decodes the nibbles and absolutises the weights once per
     // block instead of once per token; ggml-cpu's dot is single-token.  STRATA_NO_IQ4NL falls back to it.
     static const bool iq4nl_mt = std::getenv("STRATA_NO_IQ4NL") == nullptr;
-    if (nt >= 2 && f.d_type == 20 && iq4nl_mt) {
+    static const int mt_min = [] { const char* e = std::getenv("STRATA_IQ_MT_MIN"); return e ? std::atoi(e) : 2; }();
+    if (nt >= mt_min && f.d_type == 20 && iq4nl_mt) {   // #152: the same rule as the gate/up rows
         iq4nl256_down_rows(blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);
         return;
     }

src/kernels/cuda/dequant_bf16.cu

@@ -19,6 +19,9 @@ __device__ __forceinline__ float h2f(const uint8_t* p) {
 }
 __device__ __forceinline__ uint16_t f2bf(float f) {
     uint32_t u = __float_as_uint(f);
+    // a NaN (a NaN scale in the block) stays a quiet NaN, as in ggml_compute_fp32_to_bf16 and `bf16_from_f32`:
+    // the rounding add below would carry it into -0 or inf
+    if ((u & 0x7fffffffu) > 0x7f800000u) return (uint16_t) ((u >> 16) | 64u);
     u += 0x7fffu + ((u >> 16) & 1u);          // round to nearest even
     return (uint16_t) (u >> 16);
 }

src/kernels/cuda/elementwise.cu

@@ -1,5 +1,6 @@
 // src/kernels/cuda/elementwise.cu - P2.S5's glue kernels.  See the header for why each exists.
 #include "strata/kernels/elementwise.hpp"
+#include "strata/kernels/dp4a.hpp"
 
 #include "strata/kernels/bf16_bits.hpp"
 #include "strata/kernels/f16_bits.hpp"
@@ -201,14 +202,18 @@ void silu_inplace(float* x, int64_t n, void* stream) {
 /// the write was not ordered into host-visible memory, so no amount of reading it would show it, and the driver
 /// call was flushing the whole pipeline enough to make it appear.  A 10-22 us driver call per iteration is a
 /// very expensive substitute for one fence instruction.
+///
+/// **THE STORE IS VOLATILE**, like `doorbell_publish_kernel`'s.  On RDNA4 (gfx1201) a plain store to mapped pinned
+/// memory stays in the GPU's L2 until the stream is synchronized - the host never saw the ring (tests/hip/handoff:
+/// 0 of 100 rings seen without a sync; volatile, a system-scope atomic store or a fence after the store: 100 of 100).
 __global__ void doorbell_ring_kernel(uint32_t* seq) {
     __threadfence_system();
-    *seq = *seq + 1u;
+    *(volatile uint32_t*) seq = *(volatile uint32_t*) seq + 1u;
 }
 
 __global__ void doorbell_wait_kernel(const volatile uint32_t* flag, const volatile uint32_t* seq) {
     const uint32_t want = *seq;
-    while (*flag != want) __nanosleep(100);
+    while (*flag != want) strata_spin_pause();
     __threadfence_system();
 }
 

src/kernels/cuda/fused_gr.cu

@@ -180,7 +180,7 @@ __global__ void __launch_bounds__(THREADS) gr_norm_multi_kernel(GrMulti m) {
 }
 
 #if defined(__HIPCC__)
-constexpr int TILE = 1280;             // eight-token tile fits gfx1100's 64 KiB LDS limit
+constexpr int TILE = 1280;             // eight-token tile fits RDNA3/RDNA4's 64 KiB LDS limit
 #else
 constexpr int TILE = 2560;             // xn floats per token staged at a time: 320 chunks of 8, 10 per lane
 #endif
@@ -325,19 +325,53 @@ void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi
     // the shared-memory opt-in is a per-DEVICE setting: once per device, not once per process (a layer split
     // runs this kernel on two cards)
     static bool attr[64] = {};
+    static int chunk[64] = {};   // Turing port: tokens the down kernel may carry in one launch on this card
     int dev = 0;
     cudaGetDevice(&dev);
-    if (dev < 0 || dev >= 64 || !attr[dev]) {
+    if (dev >= 0 && dev < 64 && !attr[dev]) {
         // at most what the card allows (Turing: 64 KB - enough for windows of up to 6 tokens)
         int optin = 0;
         cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
         int want = (int) (kFusedGrMaxT * TILE * sizeof(float));
         if (optin > 0 && want > optin) want = optin;
         cudaFuncSetAttribute(gr_down_multi_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, want);
-        cudaGetLastError();
-        if (dev >= 0 && dev < 64) attr[dev] = true;
+        cudaGetLastError();      // drop any error the attempt left behind
+        // Turing port: the down kernel stages n_tok*TILE floats of dynamic shared memory - 80 KB at the full
+        // 8 tokens.  A card whose opt-in is below that (Turing: 64 KB, so 7+ tokens fail to launch as
+        // "invalid argument") processes the tokens in slices that fit; a card that reports no opt-in gets what
+        // fits the 48 KB default (4 tokens of the CUDA tile; all 8 of HIP's smaller tile).  The down kernel's
+        // outputs (lo, inject_out) are strictly per-token, so the chunk boundaries are safe, and the up kernel
+        // below still sees every token of the batch in one launch.
+        //
+        // The opt-in is a promise a pre-Volta card does not keep: an sm_60 answers 65536 and accepts the
+        // cudaFuncSetAttribute for 61440 B, then fails the LAUNCH with "invalid argument".  What such a card
+        // will launch is its per-block limit, so the capacity comes from that below sm_70 - the same 4 tokens
+        // the "no opt-in" branch assumes, but taken from the attribute that is actually enforced.
+#if defined(__HIPCC__)
+        const int usable = optin > 0 ? optin : 48 * 1024;
+#else
+        int cc = 0, per_block = 0;
+        cudaDeviceGetAttribute(&cc, cudaDevAttrComputeCapabilityMajor, dev);
+        cudaDeviceGetAttribute(&per_block, cudaDevAttrMaxSharedMemoryPerBlock, dev);
+        const int usable = (cc >= 7 && optin > 0) ? optin : per_block;
+#endif
+        const int capacity = usable / (int) (TILE * sizeof(float));
+        chunk[dev] = capacity < 1 ? 1 : (capacity > kFusedGrMaxT ? kFusedGrMaxT : capacity);
+        attr[dev] = true;
+    }
+    const int chunk_tok = (dev >= 0 && dev < 64 && chunk[dev]) ? chunk[dev] : kFusedGrMaxT;
+    if (chunk_tok >= n_tok) {
+        gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) n_tok * TILE * sizeof(float), st>>>(m);
+    } else {
+        for (int c0 = 0; c0 < n_tok; c0 += chunk_tok) {
+            const int ct = n_tok - c0 < chunk_tok ? n_tok - c0 : chunk_tok;
+            GrMulti c{};
+            c.xn = xn_scratch + (size_t) c0 * D;
+            c.T = ct;
+            for (int k = 0; k < ct; ++k) c.a[k] = a[c0 + k];
+            gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) ct * TILE * sizeof(float), st>>>(c);
+        }
     }
-    gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) n_tok * TILE * sizeof(float), st>>>(m);
     if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0 + 1, stream);
     gr_up_multi_kernel<<<UPM_BLOCKS, THREADS, 0, st>>>(m);
     const cudaError_t e = cudaGetLastError();

src/kernels/cuda/iq_kernels.cu

@@ -5,6 +5,7 @@
 // MIT license, third_party/ggml/LICENSE).  The block structs and codebook grids come from its ggml-common.h,
 // included unchanged.
 #include "strata/kernels/iq_kernels.hpp"
+#include "strata/kernels/dp4a.hpp"
 
 #include <cuda_fp16.h>
 #include <cuda_runtime.h>
@@ -50,7 +51,7 @@ __device__ __forceinline__ int2 get_int_from_table_16(const int& q4, const int8_
     }
     return make_int2(__byte_perm(tmp[0], tmp[1], 0x6420), __byte_perm(tmp[0], tmp[1], 0x7531));
 }
-#define ggml_cuda_dp4a(a, b, c) __dp4a((a), (b), (c))
+#define ggml_cuda_dp4a(a, b, c) STRATA_DP4A((a), (b), (c))
 
 // ---------------------------------------------------------------- the dot products (vecdotq.cuh)
 __device__ __forceinline__ float vec_dot_q2_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,

src/kernels/cuda/kv_stream.cu

@@ -209,6 +209,14 @@ void kv_stream_resolve(const KvStreamMap& m, const QsaAttnPools& slots, const Kv
         std::fprintf(stderr, "kv_stream: a block's scale run must be a multiple of 16 bytes\n");
         std::exit(1);
     }
+    // one sweep step looks at RT consecutive slots `(hand + thread) % n_slots`; with fewer slots than RT two
+    // threads see the same slot and may both take it for two different misses.  The engine never streams with fewer
+    // than qsa_kv_resident_min() / page_size = 5,120 slots, so this is a guard, not a limit.
+    if (m.n_slots < RT) {
+        std::fprintf(stderr, "kv_stream: %lld slots is fewer than the resolve block (%d): the clock sweep would take a "
+                             "slot twice\n", (long long) m.n_slots, RT);
+        std::exit(1);
+    }
     resolve_kernel<<<1, RT, 0, (cudaStream_t) stream>>>(m, ids, steps, (int) n_q, (int) cap, (int) s.page_size);
     check("resolve");
     copy_kernel<<<96, 128, 0, (cudaStream_t) stream>>>(m, runs_of(slots, host, fmt, s));

src/kernels/cuda/native_mmvq.cu

@@ -24,6 +24,7 @@
 // SOFTWARE.
 
 #include "strata/kernels/native_mmvq.hpp"
+#include "strata/kernels/dp4a.hpp"
 #include "strata/kernels/iq_kernels.hpp"
 
 #include <cuda_fp16.h>
@@ -165,8 +166,8 @@ __device__ __forceinline__ float q5_q8_dot_impl(
         const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
         const int v0i = vl0i | vh0i;
         const int v1i = vl1i | vh1i;
-        const int dot1 = __dp4a(v0i, u[2 * i], __dp4a(v1i, u[2 * i + 1], 0));
-        const int dot2 = __dp4a(0x01010101, u[2 * i], __dp4a(0x01010101, u[2 * i + 1], 0));
+        const int dot1 = STRATA_DP4A(v0i, u[2 * i], STRATA_DP4A(v1i, u[2 * i + 1], 0));
+        const int dot2 = STRATA_DP4A(0x01010101, u[2 * i], STRATA_DP4A(0x01010101, u[2 * i + 1], 0));
         sumf_d += d8[i] * (dot1 * sc[i]);
         sumf_m += d8[i] * (dot2 * m[i]);
     }
@@ -274,8 +275,8 @@ __device__ __forceinline__ float q2_q8_dot(const Q20Block* __restrict__ w,
         const int qo = __byte_perm(0x020100ff, 0x020100ff, q >> 2);
         const int qx = __byte_perm(qe, qo, 0x5140);
         const int qy = __byte_perm(qe, qo, 0x7362);
-        sumi = __dp4a(u, qx, sumi);
-        sumi = __dp4a(v, qy, sumi);
+        sumi = STRATA_DP4A(u, qx, sumi);
+        sumi = STRATA_DP4A(v, qy, sumi);
     }
     const float d8 = __low2float(chunk->ds);
     return d2 * d8 * sumi;
@@ -348,7 +349,7 @@ __device__ __forceinline__ float q3_q8_dot_impl(int vl, int vh, const int* __res
         const int vil = (vl >> (2 * i)) & 0x03030303;
         const int vih = ((vh >> i) << 2) & 0x04040404;
         const int vi = __vsubss4(vil, vih);
-        sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc);
+        sumf += d8[i] * (STRATA_DP4A(vi, u[i], 0) * sc);
     }
     return d3 * sumf;
 }
@@ -441,8 +442,8 @@ __device__ __forceinline__ float iq4_xs_q8_dot(const IQ4XSBlock* __restrict__ w,
         const int2 v = iq4_table_lookup(aux_q4);
         const int u0 = reinterpret_cast<const int*>(x[iqs / 4].qs)[j];
         const int u1 = reinterpret_cast<const int*>(x[iqs / 4].qs)[j + 4];
-        sumi = __dp4a(v.x, u0, sumi);
-        sumi = __dp4a(v.y, u1, sumi);
+        sumi = STRATA_DP4A(v.x, u0, sumi);
+        sumi = STRATA_DP4A(v.y, u1, sumi);
     }
     const int ls = ((w->scales_l[iqs / 8] >> (iqs & 0x04)) & 0x0f) |
                    (((w->scales_h >> (iqs / 2)) & 0x03) << 4);
@@ -501,8 +502,8 @@ __device__ __forceinline__ float q4_q8_dot_impl(
     for (int i = 0; i < 2; ++i) {
         const int v0i = (v[0] >> (4 * i)) & 0x0f0f0f0f;
         const int v1i = (v[1] >> (4 * i)) & 0x0f0f0f0f;
-        const int dot1 = __dp4a(v1i, u[2 * i + 1], __dp4a(v0i, u[2 * i], 0));
-        const int dot2 = __dp4a(0x01010101, u[2 * i + 1], __dp4a(0x01010101, u[2 * i], 0));
+        const int dot1 = STRATA_DP4A(v1i, u[2 * i + 1], STRATA_DP4A(v0i, u[2 * i], 0));
+        const int dot2 = STRATA_DP4A(0x01010101, u[2 * i + 1], STRATA_DP4A(0x01010101, u[2 * i], 0));
         sumf_d += d8[i] * (dot1 * sc[i]);
         sumf_m += d8[i] * (dot2 * m[i]);
     }
@@ -599,7 +600,7 @@ __device__ __forceinline__ float q6_q8_dot_impl(int vl, int vh, const int* __res
         const int vil = (vl >> (4 * i)) & 0x0f0f0f0f;
         const int vih = ((vh >> (4 * i)) << 4) & 0x30303030;
         const int vi = __vsubss4(vil | vih, 0x20202020);
-        sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc);
+        sumf += d8[i] * (STRATA_DP4A(vi, u[i], 0) * sc);
     }
     return d * sumf;
 }
@@ -671,8 +672,8 @@ __device__ __forceinline__ float small_q8_dot(const Q40Block* __restrict__ w,
         const int v = load_int_b2(w->qs, iqs + i);
         const int vi0 = (v >> 0) & 0x0f0f0f0f;
         const int vi1 = (v >> 4) & 0x0f0f0f0f;
-        sumi = __dp4a(vi0, reinterpret_cast<const int*>(x->qs)[iqs + i], sumi);
-        sumi = __dp4a(vi1, reinterpret_cast<const int*>(x->qs)[iqs + i + 4], sumi);
+        sumi = STRATA_DP4A(vi0, reinterpret_cast<const int*>(x->qs)[iqs + i], sumi);
+        sumi = STRATA_DP4A(vi1, reinterpret_cast<const int*>(x->qs)[iqs + i + 4], sumi);
     }
     const float2 ds = __half22float2(x->ds);
     const float d = w->d;
@@ -691,13 +692,13 @@ __device__ __forceinline__ float small_q8_dot(const Q50Block* __restrict__ w,
         vi0 |= (vh << 11) & 0x00001000;
         vi0 |= (vh << 18) & 0x00100000;
         vi0 |= (vh << 25) & 0x10000000;
-        sumi = __dp4a(vi0, reinterpret_cast<const int*>(x->qs)[iqs + i], sumi);
+        sumi = STRATA_DP4A(vi0, reinterpret_cast<const int*>(x->qs)[iqs + i], sumi);
         int vi1 = (vl >> 4) & 0x0f0f0f0f;
         vi1 |= (vh >> 12) & 0x00000010;
         vi1 |= (vh >> 5) & 0x00001000;
         vi1 |= (vh << 2) & 0x00100000;
         vi1 |= (vh << 9) & 0x10000000;
-        sumi = __dp4a(vi1, reinterpret_cast<const int*>(x->qs)[iqs + i + 4], sumi);
+        sumi = STRATA_DP4A(vi1, reinterpret_cast<const int*>(x->qs)[iqs + i + 4], sumi);
     }
     const float2 ds = __half22float2(x->ds);
     const float d = w->d;
@@ -711,7 +712,7 @@ __device__ __forceinline__ float small_q8_dot(const Q80Block* __restrict__ w,
     for (int i = 0; i < 2; ++i) {
         const int v = load_int_b2(w->qs, iqs + i);
         const int u = reinterpret_cast<const int*>(x->qs)[iqs + i];
-        sumi = __dp4a(v, u, sumi);
+        sumi = STRATA_DP4A(v, u, sumi);
     }
     const float d0 = w->d;
     const float d1 = __low2float(x->ds);
@@ -725,8 +726,8 @@ __device__ __forceinline__ float small_q8_dot(const IQ4NLBlock* __restrict__ w,
 #pragma unroll
     for (int i = 0; i < 2; ++i) {
         const int2 v = iq4_table_lookup(load_int_b2(w->qs, iqs + i));
-        sumi = __dp4a(v.x, q8[i], sumi);
-        sumi = __dp4a(v.y, q8[i + 4], sumi);
+        sumi = STRATA_DP4A(v.x, q8[i], sumi);
+        sumi = STRATA_DP4A(v.y, q8[i + 4], sumi);
     }
     const float d = __half2float(w->d) * __low2float(x->ds);
     return d * sumi;
@@ -890,8 +891,8 @@ struct Q20Traits {
         int sumi = 0;
 #pragma unroll
         for (int j = 0; j < 4; ++j) {
-            sumi = __dp4a(q8[j * 2], r.qx[j], sumi);
-            sumi = __dp4a(q8[j * 2 + 1], r.qy[j], sumi);
+            sumi = STRATA_DP4A(q8[j * 2], r.qx[j], sumi);
+            sumi = STRATA_DP4A(q8[j * 2 + 1], r.qy[j], sumi);
         }
         const float d8 = __low2float(chunk->ds);
         return r.d2 * d8 * sumi;
@@ -969,8 +970,8 @@ struct IQ4XSTraits {
         for (int j = 0; j < 4; ++j) {
             const int u0 = reinterpret_cast<const int*>(x[iqs / 4].qs)[j];
             const int u1 = reinterpret_cast<const int*>(x[iqs / 4].qs)[j + 4];
-            sumi = __dp4a(r.v[j].x, u0, sumi);
-            sumi = __dp4a(r.v[j].y, u1, sumi);
+            sumi = STRATA_DP4A(r.v[j].x, u0, sumi);
+            sumi = STRATA_DP4A(r.v[j].y, u1, sumi);
         }
         sumi *= r.ls - 32;
         const float d = r.dw * __low2float(x[iqs / 4].ds);

src/kernels/cuda/native_qsa_indexer.cu

@@ -43,7 +43,9 @@ __global__ void append(const float* __restrict__ raw, const int32_t* __restrict_
                         int pos_base, const float* __restrict__ gamma, float epsilon,
                         float* __restrict__ tail, float* __restrict__ dead,
                         float* __restrict__ pooled, int32_t* __restrict__ block_pos,
-                        int max_cells, float theta_scale, const int32_t* __restrict__ mtab) {
+                        int max_cells, float theta_scale, float freq_scale, float corr_low,
+                        float corr_high, float ext_factor, float mscale,
+                        const int32_t* __restrict__ mtab) {
     const int pos = *pos_dev, d = threadIdx.x;
     if (pos < 0 || pos >= max_cells) return;
     const int slot = pos % R;
@@ -83,8 +85,11 @@ __global__ void append(const float* __restrict__ raw, const int32_t* __restrict_
     float y = values[d];
     if (d < ROT) {
         const int pair = d % (ROT / 2);
-        const float theta = (pos == 0 ? 0 : mrope_pos(mtab, rope_pos, pair)) * powf(theta_scale, float(pair));
-        const float c = cosf(theta), s = sinf(theta);
+        // The spare (pos == 0) keeps its zero angle; under YaRN the mscale magnitude still rides in
+        // through cos(0) - which is exactly what the queries are scaled by, so the top-k is unmoved.
+        const float theta_extrap = (pos == 0 ? 0 : mrope_pos(mtab, rope_pos, pair)) * powf(theta_scale, float(pair));
+        float c, s;
+        rope_scaled_angle(theta_extrap, freq_scale, corr_low, corr_high, ext_factor, mscale, pair, c, s);
         const float a = values[pair], z = values[pair + ROT / 2];
         y = d < ROT / 2 ? a * c - z * s : a * s + z * c;
     }
@@ -99,12 +104,17 @@ __global__ void append(const float* __restrict__ raw, const int32_t* __restrict_
 // (cell >= p0) or the tail the previous batch left (cell < p0).  Only the batch's last completed block writes the
 // spare after it (every earlier one's is overwritten by the next block in order).
 __device__ __forceinline__ float pooled_value(const float* values, int d, int rope_pos, float theta_scale,
-                                              const int32_t* mtab, bool zero_pos) {
+                                              float freq_scale, float corr_low, float corr_high,
+                                              float ext_factor, float mscale, const int32_t* mtab, bool zero_pos) {
     float y = values[d];
     if (d < ROT) {
         const int pair = d % (ROT / 2);
-        const float theta = (zero_pos ? 0 : mrope_pos(mtab, rope_pos, pair)) * powf(theta_scale, float(pair));
-        const float c = cosf(theta), s = sinf(theta);
+        // The same rotation the single append applies, rope_scaling.hpp's helper shared: under none this is
+        // today's cosf/sinf; under linear/YaRN the pooled keys move with the cache's K, and the spare's zero
+        // angle carries the mscale magnitude through cos(0) exactly as the single append's does.
+        const float theta_extrap = (zero_pos ? 0 : mrope_pos(mtab, rope_pos, pair)) * powf(theta_scale, float(pair));
+        float c, s;
+        rope_scaled_angle(theta_extrap, freq_scale, corr_low, corr_high, ext_factor, mscale, pair, c, s);
         const float a = values[pair], z = values[pair + ROT / 2];
         y = d < ROT / 2 ? a * c - z * s : a * s + z * c;
     }
@@ -123,6 +133,7 @@ __device__ __forceinline__ float norm_scale(float mean, float* partials, int d)
 // cell 0 of a sequence: the spare (every gather index names cell 0), written to pooled[0] and dead
 __global__ void append_first(const float* __restrict__ raw, const float* __restrict__ gamma, float epsilon,
                              float* __restrict__ dead, float* __restrict__ pooled, float theta_scale,
+                             float freq_scale, float corr_low, float corr_high, float ext_factor, float mscale,
                              const int32_t* __restrict__ mtab) {
     const int d = threadIdx.x;
     __shared__ float values[D];
@@ -140,14 +151,17 @@ __global__ void append_first(const float* __restrict__ raw, const float* __restr
     if (d < D) values[d] = scale * mean * gamma[d];
     __syncthreads();
     if (d >= D) return;
-    const float y = pooled_value(values, d, 0, theta_scale, mtab, true);
+    const float y = pooled_value(values, d, 0, theta_scale, freq_scale, corr_low, corr_high, ext_factor, mscale,
+                                 mtab, true);
     pooled[d] = y;
     dead[d] = y;
 }
 __global__ void append_blocks(const float* __restrict__ raw, int64_t n, int64_t p0, int pos_base,
                               const float* __restrict__ gamma, float epsilon, const float* __restrict__ tail,
                               const float* __restrict__ dead, float* __restrict__ pooled, int32_t* __restrict__ block_pos,
-                              int64_t first_block, int64_t last_block, float theta_scale, const int32_t* __restrict__ mtab) {
+                              int64_t first_block, int64_t last_block, float theta_scale, float freq_scale,
+                              float corr_low, float corr_high, float ext_factor, float mscale,
+                              const int32_t* __restrict__ mtab) {
     const int64_t b = first_block + blockIdx.x;
     const int d = threadIdx.x;
     __shared__ float values[D];
@@ -169,7 +183,8 @@ __global__ void append_blocks(const float* __restrict__ raw, int64_t n, int64_t
     __syncthreads();
     if (d >= D) return;
     const int rope_pos = pos_base + R * (int) b;
-    pooled[std::size_t(b) * D + d] = pooled_value(values, d, rope_pos, theta_scale, mtab, false);
+    pooled[std::size_t(b) * D + d] = pooled_value(values, d, rope_pos, theta_scale, freq_scale, corr_low, corr_high,
+                                                  ext_factor, mscale, mtab, false);
     if (b == last_block) {
         pooled[std::size_t(b + 1) * D + d] = dead[d];
         if (d == 0) *block_pos = rope_pos;
@@ -200,42 +215,47 @@ void native_qsa_indexer_set_enabled(bool value) { enabled.store(value, std::memo
 bool native_qsa_indexer_enabled() { return enabled.load(std::memory_order_relaxed); }
 void native_qsa_indexer_append(const float* raw, const int32_t* relative_pos_device, int32_t pos_base,
                                const float* gamma, float epsilon, const QsaIndexerBuffers& b,
-                               const QsaShapes& s, int64_t max_cells, float freq_base, void* stream) {
+                               const QsaShapes& s, int64_t max_cells, const RopeScaling& scaling, void* stream) {
     if (!stream || s.idx_dim != D || s.idx_block != R || s.n_rot != ROT ||
         max_cells < 1 || max_cells > INT32_MAX || pos_base < 0 || pos_base % R ||
         int64_t(pos_base) + max_cells > INT32_MAX || !std::isfinite(epsilon) || epsilon <= 0.0f ||
-        !std::isfinite(freq_base) || freq_base <= 1.0f)
-        throw std::invalid_argument("native QSA indexer requires fixed geometry, aligned position base, positive capacity/epsilon, valid frequency and explicit stream");
+        rope_scaling_invalid(scaling) != nullptr)
+        throw std::invalid_argument("native QSA indexer requires fixed geometry, aligned position base, positive capacity/epsilon, valid frequency/scaling and explicit stream");
     const Span spans[] = {{raw,D*4},{relative_pos_device,4},{gamma,D*4},{b.tail,(R-1)*D*4},
         {b.dead,D*4},{b.pooled,std::size_t(max_cells/R+1)*D*4},{b.block_pos,4}};
     for (const auto& span : spans) validate(span);
     for (int i = 0; i < 7; ++i) for (int j = i + 1; j < 7; ++j)
         if (overlaps(spans[i], spans[j])) throw std::invalid_argument("native QSA indexer buffers overlap");
-    const float theta_scale = powf(freq_base, -2.0f / ROT);
+    const float theta_scale = powf((float) scaling.freq_base, -2.0f / ROT);
+    const RopeKernelArgs k = scaling.kernel_args(ROT);   // none: the identity constants
     append<<<1,THREADS,0,static_cast<cudaStream_t>(stream)>>>(raw,relative_pos_device,pos_base,gamma,epsilon,
-        b.tail,b.dead,b.pooled,b.block_pos,int(max_cells),theta_scale,mrope_table());
+        b.tail,b.dead,b.pooled,b.block_pos,int(max_cells),theta_scale,k.freq_scale,
+        k.corr_low,k.corr_high,k.ext_factor,k.attn_factor,mrope_table());
     const auto error = cudaGetLastError();
     if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
 }
 void native_qsa_indexer_append_batch(const float* raw, int64_t n, int64_t p0, int32_t pos_base, const float* gamma,
                                      float epsilon, const QsaIndexerBuffers& b, const QsaShapes& s, int64_t max_cells,
-                                     float freq_base, void* stream) {
+                                     const RopeScaling& scaling, void* stream) {
     if (n <= 0) return;
     if (!stream || s.idx_dim != D || s.idx_block != R || s.n_rot != ROT || p0 < 0 || p0 + n > max_cells ||
         max_cells > INT32_MAX || pos_base < 0 || pos_base % R || int64_t(pos_base) + max_cells > INT32_MAX ||
-        !std::isfinite(epsilon) || epsilon <= 0.0f || !std::isfinite(freq_base) || freq_base <= 1.0f)
-        throw std::invalid_argument("native QSA indexer (batch): bad geometry, positions or parameters");
-    const float theta_scale = powf(freq_base, -2.0f / ROT);
+        !std::isfinite(epsilon) || epsilon <= 0.0f || rope_scaling_invalid(scaling) != nullptr)
+        throw std::invalid_argument("native QSA indexer (batch): bad geometry, positions, parameters or scaling");
+    const float theta_scale = powf((float) scaling.freq_base, -2.0f / ROT);
+    const RopeKernelArgs k = scaling.kernel_args(ROT);   // none: the identity constants
+    const float fsf = k.freq_scale, cl = k.corr_low, ch = k.corr_high, ef = k.ext_factor, ms = k.attn_factor;
     const cudaStream_t st = static_cast<cudaStream_t>(stream);
     const int32_t* mtab = mrope_table();
-    if (p0 == 0) append_first<<<1, THREADS, 0, st>>>(raw, gamma, epsilon, b.dead, b.pooled, theta_scale, mtab);
+    if (p0 == 0) append_first<<<1, THREADS, 0, st>>>(raw, gamma, epsilon, b.dead, b.pooled, theta_scale,
+                                                     fsf, cl, ch, ef, ms, mtab);
     // completed blocks: those whose last cell (4b+3) lies in [p0, p0 + n)
     const int64_t first = p0 <= R - 1 ? 0 : (p0 - (R - 1) + R - 1) / R;       // the smallest b with 4b+3 >= p0
     const int64_t hi = p0 + n - 1 >= R - 1 ? (p0 + n - 1 - (R - 1)) / R : -1;   // the largest b with 4b+3 <= p0+n-1
     if (hi >= first)
         append_blocks<<<(unsigned) (hi - first + 1), THREADS, 0, st>>>(raw, n, p0, pos_base, gamma, epsilon, b.tail,
                                                                        b.dead, b.pooled, b.block_pos, first, hi,
-                                                                       theta_scale, mtab);
+                                                                       theta_scale, fsf, cl, ch, ef, ms, mtab);
     append_tail<<<R - 1, D, 0, st>>>(raw, n, p0, b.tail);
     const auto error = cudaGetLastError();
     if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));

src/kernels/cuda/native_rope.cu

@@ -39,7 +39,8 @@ bool overlaps(const void* a, size_t an, const void* b, size_t bn) {
     return x <= y ? y - x < an : x - y < bn;
 }
 __global__ void apply(const float* x, float* out, int rows, int width,
-                      int n_rot, float theta_scale, const int* positions, const int32_t* mtab) {
+                      int n_rot, float theta_scale, float freq_scale, float corr_low, float corr_high,
+                      float ext_factor, float mscale, const int* positions, const int32_t* mtab) {
     const int row = blockIdx.y;
     const int pair = blockIdx.x * blockDim.x + threadIdx.x;
     if (row >= rows || pair >= width / 2) return;
@@ -51,8 +52,9 @@ __global__ void apply(const float* x, float* out, int rows, int width,
         }
         return;
     }
-    const float theta = mrope_pos(mtab, positions[row], pair) * powf(theta_scale, float(pair));
-    const float c = cosf(theta), s = sinf(theta);
+    const float theta_extrap = mrope_pos(mtab, positions[row], pair) * powf(theta_scale, float(pair));
+    float c, s;
+    rope_scaled_angle(theta_extrap, freq_scale, corr_low, corr_high, ext_factor, mscale, pair, c, s);
     const float a = x[start + pair], b = x[start + pair + n_rot / 2];
     out[start + pair] = a * c - b * s;
     out[start + pair + n_rot / 2] = a * s + b * c;
@@ -73,13 +75,13 @@ const int32_t* mrope_table() { return mrope_tab[mrope_dev()].load(std::memory_or
 void native_rope_set_enabled(bool value) { enabled.store(value, std::memory_order_relaxed); }
 bool native_rope_enabled() { return enabled.load(std::memory_order_relaxed); }
 void native_rope_apply(const float* x, float* out, int rows, int head_dim,
-                       int n_rot, float freq_base, const int* positions, void* stream) {
+                       int n_rot, const RopeScaling& scaling, const int* positions, void* stream) {
     if (!x || !out || !positions || !stream || rows < 1 || rows > 65535 ||
         (head_dim != 128 && head_dim != 256) || n_rot != 64 ||
-        !std::isfinite(freq_base) || freq_base <= 1.0f ||
+        rope_scaling_invalid(scaling) != nullptr ||
         reinterpret_cast<uintptr_t>(x) % 4 || reinterpret_cast<uintptr_t>(out) % 4 ||
         reinterpret_cast<uintptr_t>(positions) % 4) {
-        throw std::invalid_argument("native RoPE requires aligned F32 rows, width 128/256, rotation 64, valid base and explicit stream");
+        throw std::invalid_argument("native RoPE requires aligned F32 rows, width 128/256, rotation 64, valid base/scaling and explicit stream");
     }
     const size_t bytes = size_t(rows) * head_dim * sizeof(float);
     if ((x != out && overlaps(x, bytes, out, bytes)) ||
@@ -88,9 +90,12 @@ void native_rope_apply(const float* x, float* out, int rows, int head_dim,
         throw std::invalid_argument("native RoPE buffers partially overlap");
     }
     // Match pinned host-side float powf before device fast powf/trigonometry.
-    const float theta_scale = powf(freq_base, -2.0f / n_rot);
+    const float theta_scale = powf((float) scaling.freq_base, -2.0f / n_rot);
+    const RopeKernelArgs k = scaling.kernel_args(n_rot);   // none: the identity constants
     apply<<<dim3((head_dim / 2 + 127) / 128, rows), 128, 0,
-              static_cast<cudaStream_t>(stream)>>>(x, out, rows, head_dim, n_rot, theta_scale, positions, mrope_table());
+              static_cast<cudaStream_t>(stream)>>>(x, out, rows, head_dim, n_rot, theta_scale,
+                                                   k.freq_scale, k.corr_low, k.corr_high, k.ext_factor,
+                                                   k.attn_factor, positions, mrope_table());
     const auto error = cudaGetLastError();
     if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
 }

src/kernels/cuda/qsa.cu

@@ -184,10 +184,18 @@ __global__ void indexer_key_append_kernel(const float* __restrict__ raw, const i
             ss = __dadd_rn(ss, __dmul_rn(v, v));
         }
         const double inv = __ddiv_rn(1.0, __dsqrt_rn(ss / (double) idx_dim + (double) eps));
-        dead[d] = (float) (p * inv * (double) w_k_norm[d]);
-        // rope at position 0 is the identity here (cos = 1, sin = 0 exactly), so no rotation is applied and
-        // the parity test asserts the value is bit-exact - which is what makes skipping it legitimate.
-        pooled[d] = dead[d];
+        float y = (float) (p * inv * (double) w_k_norm[d]);
+        // The spare rotates at position 0 - ALWAYS row 0 of the table, whatever `pos_base` is, as the native
+        // indexer's spare does.  At angle 0 the sine is exactly 0 in every scaling (sin(0) * mscale), so the
+        // pair rotation reduces to `v * cos_tab[pair]` with no partner element to read (and no barrier).
+        // Unscaled, row 0 is exactly (1, 0) and `v * 1.0f` is `v` bit for bit - today's value, which the
+        // parity test holds bit-exact.  Under YaRN row 0 is (mscale, 0): the rotated dims carry the magnitude
+        // correction exactly like every other pooled row and like the native kernel's spare (the unrotated
+        // dims pass through, as in the rotation of the completed blocks below).  Skipping the rotation here,
+        // as this kernel once did, left `dead` and `pooled[0]` the only unscaled keys of a scaled cache.
+        if (d < n_rot) y *= cos_tab[d % (n_rot / 2)];
+        dead[d] = y;
+        pooled[d] = y;
     }
 
     if (slot != r - 1) return;

src/kernels/cuda/qsa_decode_attn.cu

@@ -113,14 +113,16 @@ __global__ void __launch_bounds__(THREADS) attn_chunk_kernel(const float* __rest
         if (t < n_here) {
             const int cell = ids[c0 + t];
             const long long page = (long long) p.page_table[cell / page_size];
-            r = (page * n_kv_heads + kvh) * page_size + (cell % page_size);
+            // a block the KV streaming could not make resident keeps page -1 (ctl[3]); its cells are masked
+            // (score -FLT_MAX, weight 0) instead of being read from before the pool.
+            if (page >= 0) r = (page * n_kv_heads + kvh) * page_size + (cell % page_size);
         }
         srow[t] = r;
     }
     __syncthreads();
     // scores: each warp takes cells warp, warp+8, ...; each lane holds 8 of the 256 dimensions.
     for (int c = warp; c < CHUNK; c += WARPS) {
-        if (c >= n_here) {
+        if (c >= n_here || srow[c] < 0) {
             if (lane < G) sp[lane][c] = -FLT_MAX;
             continue;
         }
@@ -141,8 +143,8 @@ __global__ void __launch_bounds__(THREADS) attn_chunk_kernel(const float* __rest
     for (int h = warp; h < G; h += WARPS) {
         const float a = sp[h][lane], b = sp[h][lane + 32];
         const float m = warp_max(fmaxf(a, b));
-        const float ea = (lane < n_here) ? __expf(a - m) : 0.0f;
-        const float eb = (lane + 32 < n_here) ? __expf(b - m) : 0.0f;
+        const float ea = (lane < n_here && srow[lane] >= 0) ? __expf(a - m) : 0.0f;
+        const float eb = (lane + 32 < n_here && srow[lane + 32] >= 0) ? __expf(b - m) : 0.0f;
         sp[h][lane] = ea;
         sp[h][lane + 32] = eb;
         const float l = warp_sum(ea + eb);
@@ -154,6 +156,7 @@ __global__ void __launch_bounds__(THREADS) attn_chunk_kernel(const float* __rest
 #pragma unroll
     for (int h = 0; h < G; ++h) acc[h] = 0.0f;
     for (int c = 0; c < n_here; ++c) {
+        if (srow[c] < 0) continue;   // masked above, weight 0
         float v;
         if constexpr (KV_MODE == 0) {
             v = __half2float(__ushort_as_half(p.v_pool[srow[c] * HD + t]));

src/kernels/cuda/qsa_prompt_attn.cu

@@ -5,7 +5,7 @@
 
 #include <cuda_fp16.h>
 #include <cuda_runtime.h>
-#include <math_constants.h>
+#include <cmath>
 
 #include <cstdio>
 #include <cstdlib>
@@ -117,7 +117,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
         S.qh[h][d] = hi;
         S.ql[h][d] = __float2half_rn(x - __half2float(hi));
     }
-    if (t < 16) { S.mrow[t] = -CUDART_INF_F; S.lsum[t] = 0.0f; }
+    if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; }
 
     float acc[8][4];
 #pragma unroll
@@ -244,17 +244,17 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
                 sc[3] = fmaf(tg[3], s1, sc[3]);
             }
             const int c = cb + 2 * tig;
-            S.s[gid][c] = c < nh ? sc[0] * qdown : -CUDART_INF_F;
-            S.s[gid][c + 1] = c + 1 < nh ? sc[1] * qdown : -CUDART_INF_F;
-            S.s[gid + 8][c] = c < nh ? sc[2] * qdown : -CUDART_INF_F;
-            S.s[gid + 8][c + 1] = c + 1 < nh ? sc[3] * qdown : -CUDART_INF_F;
+            S.s[gid][c] = c < nh ? sc[0] * qdown : -INFINITY;
+            S.s[gid][c + 1] = c + 1 < nh ? sc[1] * qdown : -INFINITY;
+            S.s[gid + 8][c] = c < nh ? sc[2] * qdown : -INFINITY;
+            S.s[gid + 8][c + 1] = c + 1 < nh ? sc[3] * qdown : -INFINITY;
         }
         __syncthreads();
         // online softmax: row t/8, 4 cells per thread, 8 threads per row (lanes 8r..8r+7 of a warp)
         {
             constexpr int PER = CH / 8;
             const int r = t >> 3, sub = t & 7;
-            float x[PER], mx = -CUDART_INF_F;
+            float x[PER], mx = -INFINITY;
 #pragma unroll
             for (int j = 0; j < PER; ++j) { x[j] = S.s[r][sub * PER + j]; mx = fmaxf(mx, x[j]); }
 #pragma unroll
@@ -264,7 +264,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             float sum = 0.0f;
 #pragma unroll
             for (int j = 0; j < PER; ++j) {
-                const float e = x[j] == -CUDART_INF_F ? 0.0f : exp2f(x[j] - m_new);
+                const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);
                 S.s[r][sub * PER + j] = e;
                 sum += e;
             }
@@ -272,7 +272,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
             __syncwarp();
             if (sub == 0) {
-                const float a = m_old == -CUDART_INF_F ? 0.0f : exp2f(m_old - m_new);
+                const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);
                 S.alpha[r] = a;
                 S.lsum[r] = fmaf(S.lsum[r], a, sum);
                 S.mrow[r] = m_new;
@@ -419,7 +419,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __
 #pragma unroll
     for (int o = 16; o > 0; o >>= 1) qm = fmaxf(qm, __shfl_xor_sync(0xffffffffu, qm, o));
     if (lane == 0) S.qmax[warp] = qm;
-    if (t < 16) { S.mrow[t] = -CUDART_INF_F; S.lsum[t] = 0.0f; }
+    if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; }
     __syncthreads();
     qm = fmaxf(fmaxf(S.qmax[0], S.qmax[1]), fmaxf(S.qmax[2], S.qmax[3]));
     int qe = 0;
@@ -511,12 +511,12 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __
         // online softmax over the four groups' sum (fixed order): row t/8, 4 cells per thread
         {
             const int r = t >> 3, sub = t & 7;
-            float x[4], mx = -CUDART_INF_F;
+            float x[4], mx = -INFINITY;
 #pragma unroll
             for (int j = 0; j < 4; ++j) {
                 const int c = sub * 4 + j;
                 x[j] = c0 + c < n ? (((S.part[0][r][c] + S.part[1][r][c]) + S.part[2][r][c]) + S.part[3][r][c]) * qdown
-                                  : -CUDART_INF_F;
+                                  : -INFINITY;
                 mx = fmaxf(mx, x[j]);
             }
 #pragma unroll
@@ -526,7 +526,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __
             float sum = 0.0f;
 #pragma unroll
             for (int j = 0; j < 4; ++j) {
-                const float e = x[j] == -CUDART_INF_F ? 0.0f : exp2f(x[j] - m_new);
+                const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);
                 S.p[r][sub * 4 + j] = e;
                 sum += e;
             }
@@ -534,7 +534,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __
             for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
             __syncwarp();
             if (sub == 0) {
-                const float a = m_old == -CUDART_INF_F ? 0.0f : exp2f(m_old - m_new);
+                const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);
                 S.alpha[r] = a;
                 S.lsum[r] = fmaf(S.lsum[r], a, sum);
                 S.mrow[r] = m_new;

src/kernels/cuda/qsa_select.cu

@@ -429,6 +429,54 @@ __global__ void __launch_bounds__(TK_T) block_topk_reg_kernel(const float* __res
     }
 }
 
+
+// Block scores with every key block read ONCE for all of a call's queries (block_scores_kernel's grid is
+// (max_blocks / 8) x nq: ~24,600 mostly-idle blocks per layer at a decode window, each key re-read per query).  A fixed
+// grid strides over the blocks; per (block, query) the same arithmetic in the same order as block_scores_kernel.
+// qsa_block_scores takes it for every call without an active-block count and at most MQ queries: the captured decode
+// window, the uncaptured decode, and prefill's pooled16 call.
+constexpr int MQ = 8;
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,
+                                                                              const float* __restrict__ dead,
+                                                                              const float* __restrict__ q_idx,
+                                                                              const int32_t* __restrict__ steps, int nq,
+                                                                              int64_t max_blocks, float* __restrict__ out) {
+    __shared__ __align__(16) float qs[MQ * IDX_HEADS * IDX_DIM];
+    __shared__ int64_t s_nkv[MQ], s_nbid[MQ];
+    for (int i = threadIdx.x; i < nq * IDX_HEADS * IDX_DIM; i += blockDim.x) qs[i] = q_idx[i];
+    if (threadIdx.x < nq) {
+        s_nkv[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNKv];
+        s_nbid[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNBid];
+    }
+    __syncthreads();
+    int64_t top = 0;
+    for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;
+    const int lane = threadIdx.x & 31;
+    const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;
+    for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {
+        const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);
+        const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);
+        for (int qi = 0; qi < nq; ++qi) {
+            const int64_t n_bid = s_nbid[qi];
+            if (b > n_bid) continue;
+            const float4 k4 = (b == n_bid) ? kd : kp;
+            const float* q = qs + qi * IDX_HEADS * IDX_DIM + lane * 4;
+            float score = 0.0f;
+#pragma unroll
+            for (int h = 0; h < IDX_HEADS; ++h) {
+                const float4 q4 = *reinterpret_cast<const float4*>(q + h * IDX_DIM);
+                float d = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+#pragma unroll
+                for (int o = 16; o > 0; o >>= 1) d += __shfl_xor_sync(0xffffffffu, d, o);
+                score += d > 0.0f ? d : 0.0f;
+            }
+            if (lane == 0) {
+                if (b == n_bid && s_nkv[qi] % R != 0) score += 1e9f;
+                out[qi * max_blocks + b] = score;
+            }
+        }
+    }
+}
 }  // namespace
 
 void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
@@ -439,6 +487,14 @@ void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx
         std::exit(1);
     }
     // a block past a query's n_bid returns at once: the grid need only reach the batch's largest n_bid (C-1)
+    static const bool multi = [] { const char* v = std::getenv("STRATA_SCORES_MULTI"); return v == nullptr || std::atoi(v) != 0; }();
+    if (multi && nq <= MQ && active_blocks <= 0) {   // no active count: decode (captured or not) and prefill's pooled16
+        block_scores_multi_kernel<<<256, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(pooled, dead, q_idx, steps, (int) nq,
+                                                                                     max_blocks, scores);
+        const cudaError_t e = cudaGetLastError();
+        if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores multi: %s\n", cudaGetErrorString(e)); std::exit(1); }
+        return;
+    }
     const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks;
     const dim3 grid((unsigned) ((reach + SCORE_WARPS - 1) / SCORE_WARPS), (unsigned) nq);
     block_scores_kernel<<<grid, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(pooled, dead, q_idx, steps, max_blocks,

src/kernels/cuda/rope.cu

@@ -20,6 +20,11 @@
 // Building the table once on the host and passing it in makes the TABLE exactly the reference's values and
 // leaves the kernel a pure rotation, so each half can be checked tightly instead of both halves sharing one
 // loose tolerance.  The engine wants a cached table anyway: it is per (n_rot, theta, position), not per token.
+//
+// THE SCALED TABLE IS THE SAME IDEA WITH MORE MATH IN THE HOST LOOP.  Linear rescales the angle by
+// `freq_scale`; YaRN blends extrapolation and interpolation along the pairs (ggml's `rope_yarn`) and folds
+// the mscale magnitude correction into the same cos/sin values.  All of it float64, in the reference's
+// order, so `rope_parity` can hold every variant to a bit-exact float64 transcription of the same spec.
 #include "strata/kernels/rope.hpp"
 #include "strata/kernels/mrope.hpp"
 
@@ -31,6 +36,15 @@
 
 namespace strata::kernels {
 
+namespace {
+// The process's rope config (rope_scaling.hpp).  One writer - the engine's startup thread, before
+// session_init builds any table or captures any graph - and readers after it.
+RopeScaling g_rope_scaling;
+}  // namespace
+
+void rope_scaling_set(const RopeScaling& scaling) { g_rope_scaling = scaling; }
+const RopeScaling& rope_scaling() { return g_rope_scaling; }
+
 void build_rope_table(int n_rot, double theta, int max_pos, float* cos_tab, float* sin_tab) {
     const int half = n_rot / 2;
     for (int p = 0; p < max_pos; ++p) {
@@ -44,6 +58,33 @@ void build_rope_table(int n_rot, double theta, int max_pos, float* cos_tab, floa
     }
 }
 
+void build_rope_table(int n_rot, const RopeScaling& sc, int max_pos, float* cos_tab, float* sin_tab) {
+    if (sc.type == RopeScalingType::None) {
+        build_rope_table(n_rot, sc.freq_base, max_pos, cos_tab, sin_tab);   // the original loop, verbatim
+        return;
+    }
+    const int half = n_rot / 2;
+    const double fs = sc.freq_scale();
+    const double ms = sc.mscale();
+    double cd[2];
+    sc.corr_dims(n_rot, cd);
+    const bool correct = sc.ext_factor != 0;   // ggml: the correction rides on ext_factor, not the type
+    for (int p = 0; p < max_pos; ++p) {
+        for (int i = 0; i < half; ++i) {
+            const double inv = std::pow(sc.freq_base, -2.0 * (double) i / (double) n_rot);
+            const double extrap = (double) p * inv;    // the trained angle, ggml's theta_extrap
+            const double interp = fs * extrap;         // ggml's theta_interp
+            double ang = interp;
+            if (correct) {
+                const double ramp = (double) rope_yarn_ramp((float) cd[0], (float) cd[1], i) * sc.ext_factor;
+                ang = interp * (1.0 - ramp) + extrap * ramp;
+            }
+            cos_tab[(size_t) p * half + i] = (float) (std::cos(ang) * ms);
+            sin_tab[(size_t) p * half + i] = (float) (std::sin(ang) * ms);
+        }
+    }
+}
+
 namespace {
 
 // ONE THREAD PER ROW, not per (row, pair).  The first version had one thread per pair and did the tail copy

src/kernels/cuda/s2_expert_grouped.cu

@@ -14,8 +14,10 @@
 // The two agree to float rounding and not to the bit, which is the same contract `s_gemv_parity` carries for
 // the same reason.  `bench/micro/moe_hit_parity.cu` is the check.
 #include "strata/kernels/s2_expert_grouped.hpp"
+#include "strata/kernels/dp4a.hpp"
 
 #include "strata/kernels/quantize_act.hpp"
+#include "strata/kernels/verify_kernels.hpp"
 
 #include <cuda_fp16.h>
 #include <cuda_runtime.h>
@@ -88,8 +90,8 @@ __device__ __forceinline__ float row_dot_s2_q8(const uint8_t* __restrict__ codes
             // it rather than reasoning about which cast happens to work.
             int xw;
             memcpy(&xw, xq + 4 * j, 4);
-            s = __dp4a(cw, xw, s);
-            hx = __dp4a(ones, xw, hx);
+            s = STRATA_DP4A(cw, xw, s);
+            hx = STRATA_DP4A(ones, xw, hx);
         }
         // One weight scale per 64 elements, so per TWO 32-element chunks.
         const float dw = f16_at(scales + (size_t) (c >> 1) * 2);
@@ -210,7 +212,7 @@ __device__ __forceinline__ int dot4(const uint8_t* codes, const int8_t* q) {
                           (((c >> 4) & 3u) << 16) | (((c >> 6) & 3u) << 24));
     int xw;
     memcpy(&xw, q, sizeof xw);
-    return __dp4a(cw, xw, 0);
+    return STRATA_DP4A(cw, xw, 0);
 }
 
 __device__ __forceinline__ float row_dot_cpu_order(const uint8_t* codes, const uint8_t* scales,
@@ -512,6 +514,9 @@ namespace {
 // hit kernel's.
 constexpr int GU_CHUNKS = (H / 32 + 31) / 32;   // 3: chunks of a gate/up row per lane (80 chunks / 32 lanes)
 constexpr int GMAX = 8;                          // entries per group (tokens routed to one expert in a window)
+// a group holds one entry per token of the window routed to its expert, and the kernels below keep at
+// most GMAX of them (`min(..., GMAX)`): a longer window would drop entries without a word.
+static_assert(GMAX >= kVerifyMaxT, "a verify window's group can exceed GMAX entries");
 constexpr int GU_ROWS = 32;                      // gate/up rows per block: 4 per warp
 constexpr int D_ROWS = 64;                       // down rows per block: 8 per warp
 
@@ -526,8 +531,8 @@ __device__ __forceinline__ float chunk_dot(uint2 cb, const int* xw, float dw, fl
         const unsigned cbyte = cbytes[j];
         const int cw = (int) ((cbyte & 3u) | (((cbyte >> 2) & 3u) << 8) | (((cbyte >> 4) & 3u) << 16) |
                               (((cbyte >> 6) & 3u) << 24));
-        s = __dp4a(cw, xw[j], s);
-        hx = __dp4a(ones, xw[j], hx);
+        s = STRATA_DP4A(cw, xw[j], s);
+        hx = STRATA_DP4A(ones, xw[j], hx);
     }
     return dw * dx * (float) (s - hx);
 }

src/kernels/cuda/s_gemv.cu

@@ -147,14 +147,20 @@ __global__ void s_gemv_q8_split_kernel(const uint8_t* __restrict__ x, const uint
     constexpr int PER_BYTE = 8 / CODE_BITS;
     const int warps_per_block = (int) (blockDim.x >> 5);
     const long long o = (long long) blockIdx.x * warps_per_block + (threadIdx.x >> 5);
-    if (o >= n_out) return;
     const int lane = threadIdx.x & 31;
 
     // THE CODEBOOK, IN SHARED MEMORY.  16 bytes, loaded once per block by the first 16 threads, instead of a
     // divergent constant read per element - measured at 2.12x of this whole kernel (see the note on `kIq4Nl`).
+    //
+    // THE LOAD AND THE BARRIER COME BEFORE THE EARLY RETURN, as in `s_gemv_q8k_kernel`.  The return used to
+    // sit above them, so whenever `n_out % 8 != 0` the last block's surplus warps left before a barrier the
+    // others still waited at - undefined behaviour that every shape in the pack (a multiple of 8) happened to
+    // avoid.  `o` is per-WARP, so the return below keeps whole warps together and the shuffle reduction's
+    // full mask stays valid.
     __shared__ signed char s_iq4nl[16];
     if (threadIdx.x < 16) s_iq4nl[threadIdx.x] = kIq4Nl[threadIdx.x];
     __syncthreads();
+    if (o >= n_out) return;
 
     const long long n_groups = n_in >> group_shift;
     const long long codes_per_row = n_in / PER_BYTE;
@@ -655,8 +661,10 @@ void s_gemv_q8_0_split(const uint8_t* x_q8_0, const uint8_t* codes, const float*
                      form.group_elems);
         std::exit(1);
     }
-    if (form.group_elems % 4 != 0) {
-        std::fprintf(stderr, "s_gemv_q8_0_split: group_elems %d is not a multiple of 4\n", form.group_elems);
+    // SIXTEEN, NOT FOUR: a lane-iteration takes QE = 16 consecutive elements under ONE scale (see the kernel),
+    // so a group of 4 or 8 would read the wrong scale for most of them.  The Q8_K launcher already said 16.
+    if (form.group_elems % 16 != 0) {
+        std::fprintf(stderr, "s_gemv_q8_0_split: group_elems %d is not a multiple of 16\n", form.group_elems);
         std::exit(1);
     }
     int group_shift = 0;

src/kernels/cuda/sampler.cu

@@ -10,16 +10,25 @@
 // produces a valid token, so only a comparison at the distribution level can tell them apart; the parity test
 // does that against an independently computed distribution.
 //
-// Both kernels put ONE BLOCK per token over the vocabulary: `sampler_greedy_kernel` is the plain argmax,
-// `sampler_kernel` runs the sampled chain as `top_k` block-argmax rounds followed by the top_p / temperature /
-// draw chain (its header says why the selection must be parallel and why the tie rule keeps the semantics).
+// `sampler_greedy_kernel` is the plain argmax, one block per token over the vocabulary.  The sampled chain has
+// three implementations that pick the same token, bit for bit:
+//   - the SPLIT top_k (default): `sampler_split_part_kernel` cuts each row into 4,096-logit blocks over the whole
+//     GPU, each keeps its own top_k, and `sampler_split_merge_kernel` merges those lists and runs the tail;
+//   - `sampler_one_block_kernel` (`STRATA_SAMPLER_ONE_BLOCK=1`, and the fallback when the split cannot run): one
+//     block per token, `top_k` block-argmax rounds, each over the logits after the previous pick;
+//   - `sampler_kernel` (`STRATA_OLD_SAMPLER=1`), the kernel of engine 0.1.20, kept as the reference.
+// The two new ones share `sampled_tail_warp` (top_p / min_p / temperature / draw on one warp).
 #include "strata/kernels/sampler.hpp"
+#include "strata/core/coupled_draft.hpp"
 
 #include <cuda_runtime.h>
 
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>
+#include <cstring>
+#include <mutex>
+#include <vector>
 
 namespace strata::kernels {
 namespace {
@@ -170,6 +179,11 @@ __global__ void sampler_greedy_kernel(const float* __restrict__ logits, int n_vo
 /// serial scan's strict `>` keeps the first maximum it meets), so the kept sequence - both its set and its
 /// order - is unchanged; `top_p`'s cut reads that order in double arithmetic as before; temperature and the
 /// Philox draw apply after the cut.  `sampler_parity` pins all of it against the host reference.
+///
+/// **KEPT AS THE REFERENCE, BEHIND `STRATA_OLD_SAMPLER=1`.**  Two costs remain in it: the `taken`
+/// sweep is O(k) per logit per round, O(k^2 x n_vocab) per row (47 M shared-memory compares at k = 20, 500 M at
+/// 64), and the double-precision tail runs on all 1,024 threads where one warp suffices - GeForce issues FP64 at
+/// 1/64 of FP32.  The kernels after this one remove both and select the same list in the same order.
 __global__ void sampler_kernel(const float* __restrict__ logits, int n_vocab, int n_tokens,
                                const int* __restrict__ history, int history_len, const SamplerParams p,
                                int* __restrict__ out) {
@@ -302,6 +316,521 @@ __global__ void sampler_kernel(const float* __restrict__ logits, int n_vocab, in
     if (threadIdx.x == 0) out[t] = pick;
 }
 
+// ---- the same list, in the same order, without the `taken` sweep ----
+//
+// THE SELECTION ORDER.  `sampler_kernel`'s rounds rank a candidate (value, id) before (value', id') when value >
+// value', or value == value' and id < id' - each thread's ascending scan keeps the first maximum it meets (strict
+// `>`), and the reductions resolve a tie to the lower id.  That is a strict total order (-0 and +0 compare equal and
+// fall to the id, as there), so round i picks the i-th logit of the order.  NaN and -inf are never picked (`s > bv`
+// from -inf fails); a round that finds nothing yields the SENTINEL (-inf, n_vocab), stored as id 0.
+
+constexpr int kSelMax = 64;               // the widest top_k list, `sampler_kernel`'s KMAX
+constexpr unsigned kFullMask = 0xFFFFFFFFu;
+
+// top_k 1..64 as given; 0 ("off") and anything wider keep 64; never more than the vocabulary
+__host__ __device__ __forceinline__ int sampled_k(int top_k, int n_vocab) {
+    int k = (top_k > 0 && top_k < kSelMax) ? top_k : kSelMax;
+    return k > n_vocab ? n_vocab : k;
+}
+
+// (bv, bi) <- the first of (bv, bi) and (ov, oi) in the selection order
+__device__ __forceinline__ void take_first(float& bv, int& bi, float ov, int oi) {
+    if (ov > bv || (ov == bv && oi < bi)) { bv = ov; bi = oi; }
+}
+
+// The first of the warp's 32 candidates, left in EVERY lane.  An XOR butterfly is exact here: "the first of two" is
+// associative and commutative in a strict total order (two lanes never hold different candidates that compare
+// equal - an id is in one lane at most, and two sentinels are the same pair), so every lane ends with the same pair.
+__device__ __forceinline__ void warp_first(float& bv, int& bi) {
+#pragma unroll
+    for (int off = 16; off > 0; off >>= 1) {
+        const float ov = __shfl_xor_sync(kFullMask, bv, off);
+        const int oi = __shfl_xor_sync(kFullMask, bi, off);
+        take_first(bv, bi, ov, oi);
+    }
+}
+
+/// **THE TAIL ON ONE WARP, WITH `sampler_kernel`'S ARITHMETIC.**  `sel_ids` / `sel_logit` (shared, `k` entries) are
+/// the top_k list in selection order; the 32 lanes of one warp call this and lane 0 writes `out[t]`.  The old tail
+/// ran on all 1,024 threads, each computing the same ~4k double `exp`s - FP64 issues at 1/64 of FP32 on GeForce.
+/// Here the lanes share the `exp`s (one per entry, into `ex`) and then the quotients, and lane 0 alone runs the two
+/// ORDERED sums and the two cumulative scans.  Every double is the one the old chain computed: the same `exp` of
+/// the same argument, the sums in the same order, `cum += e / sum` with the same correctly rounded quotient - so
+/// the cut, the survivors and the pick are the same.  (`n_keep == 0`, reachable only with min_p > 1, which the
+/// callers clamp, read `sel_ids[-1]` in the old tail; it reads `sel_ids[0]` here.)
+/// kProb (the coupled draft only): lane 0 also writes the pick's probability under the final distribution to
+/// `*prob_out`.  The sampler's own calls take kProb = false, whose code is the tail above, unchanged.
+template <bool kProb = false>
+__device__ void sampled_tail_warp(const int* sel_ids, const float* sel_logit, int k, const SamplerParams& p, int t,
+                                  int* __restrict__ out, double* ex, float* prob_out = nullptr) {
+    const int lane = (int) (threadIdx.x & 31);
+    const float inv_t = p.temperature > 0.0f ? 1.0f / p.temperature : 0.0f;
+    int n_keep = k;
+    float mx = sel_logit[0];
+    for (int i = 1; i < k; ++i) mx = fmaxf(mx, sel_logit[i]);
+    if (p.top_p < 1.0f) {
+        for (int i = lane; i < k; i += 32) ex[i] = exp((double) sel_logit[i] - (double) mx);
+        __syncwarp();
+        double sum = 0.0;
+        if (lane == 0)
+            for (int i = 0; i < k; ++i) sum += ex[i];
+        sum = __shfl_sync(kFullMask, sum, 0);
+        __syncwarp();                            // lane 0 has read every `ex` before it is overwritten
+        for (int i = lane; i < k; i += 32) ex[i] = ex[i] / sum;
+        __syncwarp();
+        int cut = k;
+        if (lane == 0) {
+            double cum = 0.0;
+            for (int i = 0; i < k; ++i) {
+                cum += ex[i];
+                if (cum >= (double) p.top_p) { cut = i + 1; break; }
+            }
+        }
+        cut = __shfl_sync(kFullMask, cut, 0);
+        if (cut < p.min_keep) cut = p.min_keep < k ? p.min_keep : k;
+        n_keep = cut;
+        __syncwarp();                            // `ex` is written again below
+    }
+    // min_p on top_p's survivors, as in `sampler_kernel` (every lane, the same float arithmetic)
+    if (p.min_p > 0.0f) {
+        const float thresh = sel_logit[0] + logf(p.min_p);
+        for (int i = 0; i < n_keep; ++i)
+            if (sel_logit[i] < thresh) { n_keep = i; break; }
+    }
+    // temperature, then one Philox draw
+    float smx = sel_logit[0] * inv_t;
+    for (int i = 1; i < n_keep; ++i) smx = fmaxf(smx, sel_logit[i] * inv_t);
+    for (int i = lane; i < n_keep; i += 32) ex[i] = exp((double) (sel_logit[i] * inv_t) - (double) smx);
+    __syncwarp();
+    double sum = 0.0;
+    if (lane == 0)
+        for (int i = 0; i < n_keep; ++i) sum += ex[i];
+    sum = __shfl_sync(kFullMask, sum, 0);
+    __syncwarp();
+    for (int i = lane; i < n_keep; i += 32) ex[i] = ex[i] / sum;
+    __syncwarp();
+    if (lane == 0) {
+        const float u = philox_uniform(p.seed, p.counter + (uint64_t) t);
+        double cum = 0.0;
+        int pi = n_keep > 0 ? n_keep - 1 : 0;
+        int pick = sel_ids[pi];
+        for (int i = 0; i < n_keep; ++i) {
+            cum += ex[i];
+            if ((double) u < cum) { pick = sel_ids[i]; pi = i; break; }
+        }
+        out[t] = pick;
+        if constexpr (kProb) *prob_out = n_keep > 0 ? (float) ex[pi] : 1.0f;
+    }
+}
+
+/// **THE ONE-BLOCK SAMPLED PATH: `sampler_kernel` WITHOUT THE `taken` SWEEP.**  Round i's candidates are the logits
+/// strictly AFTER round i-1's pick in the selection order, `s < prev_v || (s == prev_v && v > prev_i)`, which is
+/// exactly the set `taken` left: the picks so far are the first i of the order, so what comes after the last one is
+/// what has not been picked.  The round is then the same block argmax with the same tie rule, so the list is the
+/// same list in the same order.  An empty round leaves (-inf, id 0) as before, and every round after it is empty
+/// in both versions (nothing after -inf beats -inf).  O(k x n_vocab) per row instead of O(k^2 x n_vocab), then warp
+/// 0 runs the tail.  `STRATA_SAMPLER_ONE_BLOCK=1`, and the fallback when the split path cannot run.
+__global__ void __launch_bounds__(1024)
+sampler_one_block_kernel(const float* __restrict__ logits, int n_vocab, const int* __restrict__ history,
+                         int history_len, const SamplerParams p, int* __restrict__ out) {
+    const int t = blockIdx.x;
+    const float* l = logits + (size_t) t * n_vocab;
+
+    // the penalty window and its membership bitmap, exactly as in `sampler_kernel`
+    const int* hrow = history ? history + (size_t) t * history_len : nullptr;
+    int hlen = 0;
+    if (hrow) {
+        hlen = p.penalty_last_n < history_len ? p.penalty_last_n : history_len;
+        if (hlen < 0) hlen = 0;
+        hrow += history_len - hlen;          // the window is the TAIL
+    }
+    extern __shared__ unsigned int penal_bits[];
+    const int bits_words = (int) ((n_vocab + 31) / 32);
+    const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;
+    if (use_bits) {
+        for (int w = threadIdx.x; w < bits_words; w += blockDim.x) penal_bits[w] = 0u;
+        __syncthreads();
+        for (int i = threadIdx.x; i < hlen; i += blockDim.x)
+            if (hrow[i] >= 0 && hrow[i] < n_vocab)   // an id outside the vocabulary is never a candidate
+                atomicOr(&penal_bits[hrow[i] >> 5], 1u << (hrow[i] & 31));
+        __syncthreads();
+    }
+    auto hit_count = [&](int v) -> int {
+        if (!use_bits || !(penal_bits[v >> 5] & (1u << (v & 31)))) return 0;
+        return history_count(hrow, hlen, v);
+    };
+
+    const int k = sampled_k(p.top_k, n_vocab);
+    __shared__ int sel_ids[kSelMax];
+    __shared__ float sel_logit[kSelMax];
+    __shared__ double ex[kSelMax];
+    __shared__ float sv[32];
+    __shared__ int si[32];
+    const int warp = (int) (threadIdx.x >> 5), lane = (int) (threadIdx.x & 31);
+    float prev_v = __int_as_float(0x7f800000);   // +inf and id -1: round 0 takes every logit
+    int prev_i = -1;
+    for (int i = 0; i < k; ++i) {
+        float bv = __int_as_float(0xff800000);   // -inf
+        int best = n_vocab;
+        for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {
+            const float s = apply_penalties(l[v], hit_count(v), p);
+            if ((s < prev_v || (s == prev_v && v > prev_i)) && s > bv) { bv = s; best = v; }
+        }
+        for (int off = 16; off > 0; off >>= 1) {
+            const float ov = __shfl_down_sync(0xFFFFFFFFu, bv, off);
+            const int oi = __shfl_down_sync(0xFFFFFFFFu, best, off);
+            if (ov > bv || (ov == bv && oi < best)) { bv = ov; best = oi; }
+        }
+        if (lane == 0) { sv[warp] = bv; si[warp] = best; }
+        __syncthreads();
+        if (warp == 0) {
+            const int nw = (int) ((blockDim.x + 31) >> 5);
+            float wv = lane < nw ? sv[lane] : __int_as_float(0xff800000);
+            int wi = lane < nw ? si[lane] : n_vocab;
+            for (int off = 16; off > 0; off >>= 1) {
+                const float ov = __shfl_down_sync(0xFFFFFFFFu, wv, off);
+                const int oi = __shfl_down_sync(0xFFFFFFFFu, wi, off);
+                if (ov > wv || (ov == wv && oi < wi)) { wv = ov; wi = oi; }
+            }
+            if (lane == 0) { sel_ids[i] = (wi < n_vocab) ? wi : 0; sel_logit[i] = wv; }
+        }
+        __syncthreads();
+        // An empty round leaves (-inf, 0): nothing comes after it, as nothing was left untaken.
+        prev_v = sel_logit[i];
+        prev_i = sel_ids[i];
+    }
+    if (warp != 0) return;
+    sampled_tail_warp(sel_ids, sel_logit, k, p, t, out, ex);
+}
+
+// ---- THE SPLIT top_k.  `sampler_kernel` and the one-block kernel keep a row on ONE SM of 170 (5090) and walk its
+// 248,320 logits k times.  Here a warp keeps the top_k of 1,024 logits held in registers, four warps make a block of
+// 4,096 logits that merges their lists, and a row is 61 such blocks over the whole GPU; one warp per row then
+// merges the 61 lists and runs the tail.  Exact, because the first k of a union are within the first k of each part
+// and a merge of ordered lists is ordered - the selection order being a strict total order (above).
+constexpr int kSplitPerLane = 32;                               // logits per lane, in registers
+constexpr int kSplitWarpSpan = 32 * kSplitPerLane;              // 1,024 logits per warp
+constexpr int kSplitWarps = 4;
+constexpr int kSplitBlockSpan = kSplitWarps * kSplitWarpSpan;   // 4,096 logits per block
+constexpr int kSplitMaxBlocks = 64;                             // lists the merge holds: n_vocab <= 262,144
+constexpr int kSplitMaxRows = 64;                               // rows per split launch (the scratch's bound)
+
+/// Merge `nl` (<= 64) lists of `k` candidates - list L at `lists[L * stride]`, each in the selection order and
+/// padded with sentinels - into their first `k`: `sink(i, value, id)` runs in every lane for i = 0..k-1 with the
+/// same pair.  Lane owns lists `lane` and `lane + 32`; a round takes the first of all heads and advances the list
+/// it came from.  An id is in one list at most (the lists cover disjoint logits), so exactly one head matches.
+template <typename Sink>
+__device__ __forceinline__ void warp_merge_lists(const int2* lists, int nl, int stride, int k, int n_vocab,
+                                                 Sink&& sink) {
+    const int lane = (int) (threadIdx.x & 31);
+    float hv[2];
+    int hi[2], pos[2];
+#pragma unroll
+    for (int m = 0; m < 2; ++m) {
+        const int L = lane + 32 * m;
+        pos[m] = 0;
+        hv[m] = __int_as_float(0xff800000);
+        hi[m] = n_vocab;
+        if (L < nl) {
+            const int2 c = lists[(size_t) L * stride];
+            hi[m] = c.x;
+            hv[m] = __int_as_float(c.y);
+        }
+    }
+    for (int i = 0; i < k; ++i) {
+        float bv = hv[0];
+        int bi = hi[0];
+        take_first(bv, bi, hv[1], hi[1]);
+        warp_first(bv, bi);
+        sink(i, bv, bi);
+        if (bi < n_vocab) {
+#pragma unroll
+            for (int m = 0; m < 2; ++m) {
+                if (hi[m] != bi) continue;
+                if (++pos[m] < k) {
+                    const int2 c = lists[(size_t) (lane + 32 * m) * stride + pos[m]];
+                    hi[m] = c.x;
+                    hv[m] = __int_as_float(c.y);
+                } else {
+                    hi[m] = n_vocab;
+                    hv[m] = __int_as_float(0xff800000);
+                }
+            }
+        }
+    }
+}
+
+/// **SPLIT STAGE 1: THE top_k OF EACH 4,096-LOGIT BLOCK.**  Grid (blocks per row, rows), 128 threads.  Each warp
+/// loads its 1,024 penalised logits once into registers (lane + 32 j: every load is one coalesced 128-byte line)
+/// and runs `k` warp-argmax rounds over them with the threshold of `sampler_one_block_kernel` - no shared memory
+/// and no block barrier per round.  Warp 0 then merges the four warp lists into the block's list in `cand`
+/// (row-major: row t, block b, entry i at `(t * n_blocks + b) * k + i`, as (id, value bits)).
+__global__ void __launch_bounds__(kSplitWarps * 32)
+sampler_split_part_kernel(const float* __restrict__ logits, int n_vocab, const int* __restrict__ history,
+                          int history_len, const SamplerParams p, int k, int n_blocks, int2* __restrict__ cand) {
+    const int t = blockIdx.y;
+    const float* l = logits + (size_t) t * n_vocab;
+    const int warp = (int) (threadIdx.x >> 5), lane = (int) (threadIdx.x & 31);
+    const int blo = (int) blockIdx.x * kSplitBlockSpan;
+
+    const int* hrow = history ? history + (size_t) t * history_len : nullptr;
+    int hlen = 0;
+    if (hrow) {
+        hlen = p.penalty_last_n < history_len ? p.penalty_last_n : history_len;
+        if (hlen < 0) hlen = 0;
+        hrow += history_len - hlen;          // the window is the TAIL
+    }
+    // The membership bitmap of THIS BLOCK'S 4,096 logits (512 bytes, not the vocabulary's 31 KB): the same test,
+    // and a hit pays the same exact count over the whole window.
+    __shared__ unsigned int bits[kSplitBlockSpan / 32];
+    const bool use_bits = hrow != nullptr && hlen > 0;
+    if (use_bits) {
+        for (int w = threadIdx.x; w < kSplitBlockSpan / 32; w += blockDim.x) bits[w] = 0u;
+        __syncthreads();
+        for (int i = threadIdx.x; i < hlen; i += blockDim.x) {
+            const int h = hrow[i];
+            if (h >= 0 && h < n_vocab && h >= blo && h - blo < kSplitBlockSpan)
+                atomicOr(&bits[(h - blo) >> 5], 1u << ((h - blo) & 31));
+        }
+        __syncthreads();
+    }
+
+    // This warp's logits, penalised: `apply_penalties` with a zero count returns the logit unchanged, so only the
+    // bitmap's hits go through it.  Past the vocabulary: -inf, which no round picks.
+    const int lo = blo + warp * kSplitWarpSpan;
+    float s[kSplitPerLane];
+#pragma unroll
+    for (int j = 0; j < kSplitPerLane; ++j) {
+        const int v = lo + 32 * j + lane;
+        s[j] = v < n_vocab ? l[v] : __int_as_float(0xff800000);
+    }
+    if (use_bits) {
+#pragma unroll
+        for (int j = 0; j < kSplitPerLane; ++j) {
+            const int v = lo + 32 * j + lane, b = v - blo;
+            if (v < n_vocab && (bits[b >> 5] & (1u << (b & 31))))
+                s[j] = apply_penalties(s[j], history_count(hrow, hlen, v), p);
+        }
+    }
+
+    __shared__ int2 wl[kSplitWarps][kSelMax];
+    float prev_v = __int_as_float(0x7f800000);   // +inf and id -1: round 0 takes every logit
+    int prev_i = -1;
+    int i = 0;
+    for (; i < k; ++i) {
+        // two chains (even and odd j), each walked in ascending id with a strict `>`, so each keeps its first in
+        // the order; `take_first` then orders the two
+        float b0 = __int_as_float(0xff800000), b1 = __int_as_float(0xff800000);
+        int i0 = n_vocab, i1 = n_vocab;
+#pragma unroll
+        for (int j = 0; j < kSplitPerLane; j += 2) {
+            const int v0 = lo + 32 * j + lane, v1 = v0 + 32;
+            const float x0 = s[j], x1 = s[j + 1];
+            if ((x0 < prev_v || (x0 == prev_v && v0 > prev_i)) && x0 > b0) { b0 = x0; i0 = v0; }
+            if ((x1 < prev_v || (x1 == prev_v && v1 > prev_i)) && x1 > b1) { b1 = x1; i1 = v1; }
+        }
+        take_first(b0, i0, b1, i1);
+        warp_first(b0, i0);
+        if (i0 >= n_vocab) break;                // the same in every lane: nothing left in these 1,024 logits
+        if (lane == 0) wl[warp][i] = make_int2(i0, __float_as_int(b0));
+        prev_v = b0;
+        prev_i = i0;
+    }
+    for (int r = i + lane; r < k; r += 32) wl[warp][r] = make_int2(n_vocab, (int) 0xff800000u);   // sentinels
+    __syncthreads();
+    if (warp == 0) {
+        int2* dst = cand + ((size_t) t * n_blocks + blockIdx.x) * k;
+        warp_merge_lists(&wl[0][0], kSplitWarps, kSelMax, k, n_vocab, [&](int r, float v, int id) {
+            if (lane == 0) dst[r] = make_int2(id, __float_as_int(v));
+        });
+    }
+}
+
+/// **SPLIT STAGE 2: ONE WARP PER ROW MERGES THE BLOCK LISTS, THEN RUNS THE TAIL.**  The row's lists (at most 64 x
+/// 64 entries, 32 KB) are copied to shared memory, merged into the row's top_k list, and `sampled_tail_warp` picks.
+__global__ void __launch_bounds__(32)
+sampler_split_merge_kernel(const int2* __restrict__ cand, int n_blocks, int n_vocab, const SamplerParams p, int k,
+                           int* __restrict__ out) {
+    const int t = blockIdx.x;
+    const int lane = (int) threadIdx.x;
+    __shared__ int2 lists[kSplitMaxBlocks * kSelMax];
+    __shared__ int sel_ids[kSelMax];
+    __shared__ float sel_logit[kSelMax];
+    __shared__ double ex[kSelMax];
+    const int2* src = cand + (size_t) t * n_blocks * k;
+    for (int e = lane; e < n_blocks * k; e += 32) lists[e] = src[e];
+    __syncwarp();
+    warp_merge_lists(lists, n_blocks, k, k, n_vocab, [&](int i, float v, int id) {
+        if (lane == 0) { sel_ids[i] = id < n_vocab ? id : 0; sel_logit[i] = v; }
+    });
+    __syncwarp();
+    sampled_tail_warp(sel_ids, sel_logit, k, p, t, out, ex);
+}
+
+// ---- COUPLED DRAFT SAMPLING (include/strata/core/coupled_draft.hpp): the MTP draft layer samples its draft with the
+// target's chain and the target's Philox draw.  Everything that varies per request or per round - the chain's
+// parameters, the seed, the counter (from the cell's step record), the penalty history - is read from DEVICE memory:
+// these kernels are captured into the drafter's round/step graphs.  One row, `nv` logits: the draft head's
+// vocabulary subset (rt/draft_vocab.bin) or the whole vocabulary; `sub_to_id` maps a subset index to its token id.
+
+/// The round's inputs: the request's SamplerParams and the history base (the last h slots before `cap`), from
+/// mapped host memory into the device copies the chain's kernels read.
+__global__ void coupled_stage_kernel(const SamplerParams* __restrict__ mp, const int* __restrict__ mh,
+                                     SamplerParams* __restrict__ dp, int* __restrict__ ring, int cap) {
+    const volatile int* s = (const volatile int*) mp;
+    int* d = (int*) dp;
+    for (int i = threadIdx.x; i < (int) (sizeof(SamplerParams) / sizeof(int)); i += blockDim.x) d[i] = s[i];
+    const int h = strata::core::coupled_hist_len(((const volatile SamplerParams*) mp)->penalty_last_n, cap);
+    const volatile int* vh = (const volatile int*) mh;
+    for (int i = cap - h + (int) threadIdx.x; i < cap; i += blockDim.x) ring[i] = vh[i];
+}
+
+/// The penalties, applied in place to the draft logits before the selection - the target applies the same
+/// `apply_penalties` to the same token with the same count over the same window before its own.  Draft j's window
+/// is the ring's [cap + j - h, cap + j): the base plus drafts 0 .. j-1.  A history token outside the draft head's
+/// subset has no logit here.  Each distinct token is penalised once: the entry whose `atomicOr` sets its bit does it.
+__global__ void coupled_penalize_kernel(float* __restrict__ logits, int nv, const int* __restrict__ id_to_sub,
+                                        int id_vocab, const SamplerParams* __restrict__ dp, const int* __restrict__ ring,
+                                        int cap, int j) {
+    const SamplerParams p = *dp;
+    const int h = strata::core::coupled_hist_len(p.penalty_last_n, cap);
+    if (h <= 0) return;
+    const int* hrow = ring + strata::core::coupled_hist_start(cap, j, h);
+    extern __shared__ unsigned int seen[];
+    const int words = (nv + 31) / 32;
+    for (int w = threadIdx.x; w < words; w += blockDim.x) seen[w] = 0u;
+    __syncthreads();
+    for (int i = threadIdx.x; i < h; i += blockDim.x) {
+        const int v = hrow[i];
+        if (v < 0 || v >= id_vocab) continue;
+        const int s = id_to_sub != nullptr ? id_to_sub[v] : v;
+        if (s < 0 || s >= nv) continue;
+        const unsigned bit = 1u << (s & 31);
+        if (atomicOr(&seen[s >> 5], bit) & bit) continue;
+        logits[s] = apply_penalties(logits[s], history_count(hrow, h, v), p);
+    }
+}
+
+/// The merge of `sampler_split_merge_kernel` (lists of `kpart` entries, the request's top_k taken from them), then
+/// `sampled_tail_warp` with the counter of the row that will verify this draft.  Lane 0 maps the pick to its token
+/// id, writes it and its probability, and appends it to the ring for the next draft's penalty window.
+__global__ void __launch_bounds__(32)
+coupled_merge_kernel(const int2* __restrict__ cand, int n_blocks, int nv, int kpart,
+                     const SamplerParams* __restrict__ dp, const int* __restrict__ step_rec,
+                     const int* __restrict__ sub_to_id, int* __restrict__ ring, int cap, int j, int* __restrict__ out_id,
+                     float* __restrict__ out_prob) {
+    const int lane = (int) threadIdx.x;
+    SamplerParams p = *dp;
+    p.counter = strata::core::coupled_draft_counter((int64_t) step_rec[0]);
+    const int k = sampled_k(p.top_k, nv);    // <= kpart: the first k of a union lie in the first k of each list
+    __shared__ int2 lists[kSplitMaxBlocks * kSelMax];
+    __shared__ int sel_ids[kSelMax];
+    __shared__ float sel_logit[kSelMax];
+    __shared__ double ex[kSelMax];
+    __shared__ int pick[1];
+    __shared__ float prob[1];
+    for (int e = lane; e < n_blocks * kpart; e += 32) lists[e] = cand[e];
+    __syncwarp();
+    warp_merge_lists(lists, n_blocks, kpart, k, nv, [&](int i, float v, int id) {
+        if (lane == 0) { sel_ids[i] = id < nv ? id : 0; sel_logit[i] = v; }
+    });
+    __syncwarp();
+    if (p.greedy || p.temperature <= 0.0f) {   // never launched for greedy requests; the argmax, defensively
+        if (lane == 0) { pick[0] = sel_ids[0]; prob[0] = 1.0f; }
+    } else {
+        sampled_tail_warp<true>(sel_ids, sel_logit, k, p, 0, pick, ex, prob);
+    }
+    __syncwarp();
+    if (lane == 0) {
+        const int s = pick[0];
+        const int id = sub_to_id != nullptr ? sub_to_id[s] : s;
+        *out_id = id;
+        *out_prob = prob[0];
+        ring[cap + j] = id;
+    }
+}
+
+// Which sampled path runs, read once: `STRATA_OLD_SAMPLER=1` is `sampler_kernel` (engine 0.1.20),
+// `STRATA_SAMPLER_ONE_BLOCK=1` the one-block kernel; by default the split top_k wherever it applies.
+enum class SampledPath { Split, OneBlock, Old };
+
+bool env_flag(const char* name) {
+    const char* e = std::getenv(name);
+    return e != nullptr && *e != '\0' && std::strcmp(e, "0") != 0;
+}
+
+SampledPath sampled_path() {
+    static const SampledPath path = env_flag("STRATA_OLD_SAMPLER")         ? SampledPath::Old
+                                    : env_flag("STRATA_SAMPLER_ONE_BLOCK") ? SampledPath::OneBlock
+                                                                           : SampledPath::Split;
+    return path;
+}
+
+// A stream being captured into a graph must not reach `split_scratch` (cudaMalloc): it
+// gets the one-block kernel, which needs no memory of its own.  The legacy stream cannot be captured.
+bool stream_capturing(void* stream) {
+    if (stream == nullptr) return false;
+    cudaStreamCaptureStatus st = cudaStreamCaptureStatusNone;
+    if (cudaStreamIsCapturing((cudaStream_t) stream, &st) != cudaSuccess) {
+        (void) cudaGetLastError();
+        return true;
+    }
+    return st != cudaStreamCaptureStatusNone;
+}
+
+// The split's block lists, one buffer per (device, stream): launches on one stream run in order, so a stream reuses
+// its buffer with no sync, and two streams never share one.  Grown on demand (at least doubling, up to the size of
+// `kSplitMaxRows` rows at the widest vocabulary: 2 MB), never shrunk.  A grown-out buffer is retired, not freed: a
+// pointer handed out earlier (to another host thread on the same stream, say) may still be waiting for its launch,
+// and freeing it would need a sync that proves nothing about that thread.  Doubling keeps a slot's retired buffers
+// smaller than its live one, so a slot holds about 4 MB at most (0.5 MB for the engine's <= 16 rows).  nullptr
+// when the memory cannot be had, and the caller falls back to the one-block kernel; a failed grow is remembered, so
+// later calls of that size do not retry cudaMalloc each time.
+int2* split_scratch(void* stream, size_t entries) {
+    struct Slot {
+        int device;
+        void* stream;
+        int2* ptr;
+        size_t entries;
+        size_t failed;                       // the smallest size cudaMalloc refused (0: none)
+    };
+    static std::mutex mu;
+    static std::vector<Slot> slots;
+    static std::vector<int2*> retired;
+    int device = 0;
+    if (cudaGetDevice(&device) != cudaSuccess) {
+        (void) cudaGetLastError();
+        return nullptr;
+    }
+    std::lock_guard<std::mutex> lock(mu);
+    Slot* slot = nullptr;
+    for (Slot& s : slots)
+        if (s.device == device && s.stream == stream) slot = &s;
+    if (slot == nullptr) {
+        slots.push_back({device, stream, nullptr, 0, 0});
+        slot = &slots.back();
+    }
+    if (slot->entries >= entries) return slot->ptr;
+    if (slot->failed != 0 && entries >= slot->failed) return nullptr;
+    constexpr size_t kCap = (size_t) kSplitMaxRows * kSplitMaxBlocks * kSelMax;
+    size_t want = 2 * slot->entries < kCap ? 2 * slot->entries : kCap;
+    if (want < entries) want = entries;
+    int2* ptr = nullptr;
+    if (cudaMalloc(&ptr, want * sizeof(int2)) != cudaSuccess) {
+        (void) cudaGetLastError();
+        want = entries;
+        if (cudaMalloc(&ptr, want * sizeof(int2)) != cudaSuccess) {
+            (void) cudaGetLastError();
+            slot->failed = entries;
+            return nullptr;
+        }
+    }
+    if (slot->ptr != nullptr) retired.push_back(slot->ptr);
+    slot->ptr = ptr;
+    slot->entries = want;
+    return ptr;
+}
+
 }  // namespace
 
 void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
@@ -320,11 +849,33 @@ void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* hi
         const int gthreads = 1024;
         sampler_greedy_kernel<<<(unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream>>>(
             logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);
-    } else {
+    } else if (sampled_path() == SampledPath::Old) {
         // The same block-per-token shape: the selection's k argmax rounds reduce inside the block.  See
         // `sampler_kernel`'s header for what the old one-thread-per-token launch cost.
         sampler_kernel<<<(unsigned) n_tokens, 1024, shmem, (cudaStream_t) stream>>>(
             logits, n_vocab, n_tokens, history, history_len, p, out);
+    } else {
+        // The split top_k by default: stage 1 over (61 blocks x rows) for 248,320 logits, stage 2 one warp per
+        // row.  The one-block kernel when asked for, or when the split cannot run: a wider vocabulary than the merge
+        // holds, more than `kSplitMaxRows` rows (the engine samples at most a verify window), a stream under capture,
+        // no scratch.
+        const int k = sampled_k(p.top_k, n_vocab);
+        const int n_blocks = (n_vocab + kSplitBlockSpan - 1) / kSplitBlockSpan;
+        int2* scratch = nullptr;
+        if (sampled_path() == SampledPath::Split && n_blocks <= kSplitMaxBlocks && n_tokens <= kSplitMaxRows &&
+            !stream_capturing(stream))
+            // sized for 16 rows and 64 entries at least, so a verify window or a wider top_k does not regrow it
+            scratch = split_scratch(stream, (size_t) (n_tokens > 16 ? n_tokens : 16) * n_blocks * kSelMax);
+        if (scratch != nullptr) {
+            sampler_split_part_kernel<<<dim3((unsigned) n_blocks, (unsigned) n_tokens), kSplitWarps * 32, 0,
+                                        (cudaStream_t) stream>>>(logits, n_vocab, history, history_len, p, k,
+                                                                 n_blocks, scratch);
+            sampler_split_merge_kernel<<<(unsigned) n_tokens, 32, 0, (cudaStream_t) stream>>>(
+                scratch, n_blocks, n_vocab, p, k, out);
+        } else {
+            sampler_one_block_kernel<<<(unsigned) n_tokens, 1024, shmem, (cudaStream_t) stream>>>(
+                logits, n_vocab, history, history_len, p, out);
+        }
     }
     const cudaError_t e = cudaGetLastError();
     if (e != cudaSuccess) {
@@ -334,4 +885,49 @@ void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* hi
     if (stream == nullptr) cudaDeviceSynchronize();
 }
 
+namespace {
+int coupled_blocks(int nv) { return (nv + kSplitBlockSpan - 1) / kSplitBlockSpan; }
+int coupled_kpart(int nv) { return nv < kSelMax ? nv : kSelMax; }
+void coupled_check(const char* what) {
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "%s launch: %s\n", what, cudaGetErrorString(e));
+        std::exit(1);
+    }
+}
+}  // namespace
+
+size_t coupled_draft_scratch_bytes(int nv) {
+    if (nv <= 0 || coupled_blocks(nv) > kSplitMaxBlocks) return 0;
+    return (size_t) coupled_blocks(nv) * (size_t) kSelMax * sizeof(int2);
+}
+
+void coupled_draft_stage(const SamplerParams* mapped_params, const int32_t* mapped_hist, SamplerParams* params,
+                         int32_t* ring, int cap, void* stream) {
+    coupled_stage_kernel<<<1, 256, 0, (cudaStream_t) stream>>>(mapped_params, mapped_hist, params, ring, cap);
+    coupled_check("coupled_draft_stage");
+}
+
+void coupled_draft_sample(float* logits, int nv, const int32_t* sub_to_id, const int32_t* id_to_sub, int id_vocab,
+                          const SamplerParams* params, int32_t* ring, int cap, int j, const int32_t* step_rec,
+                          void* scratch, int32_t* out_id, float* out_prob, void* stream) {
+    const cudaStream_t s = (cudaStream_t) stream;
+    const int n_blocks = coupled_blocks(nv), kpart = coupled_kpart(nv);
+    if (nv <= 0 || n_blocks > kSplitMaxBlocks || scratch == nullptr) {
+        std::fprintf(stderr, "coupled_draft_sample: %d logits need scratch and at most %d blocks\n", nv, kSplitMaxBlocks);
+        std::exit(1);
+    }
+    coupled_penalize_kernel<<<1, 1024, (unsigned) ((nv + 31) / 32) * sizeof(unsigned), s>>>(logits, nv, id_to_sub,
+                                                                                           id_vocab, params, ring, cap, j);
+    coupled_check("coupled_penalize");
+    // the selection of the split sampler, unchanged: every 4,096-logit block's first `kpart` (the widest list,
+    // since the request's top_k is only known on the device), penalties already applied above
+    sampler_split_part_kernel<<<dim3((unsigned) n_blocks, 1u), kSplitWarps * 32, 0, s>>>(
+        logits, nv, nullptr, 0, SamplerParams{}, kpart, n_blocks, (int2*) scratch);
+    coupled_check("coupled_draft split part");
+    coupled_merge_kernel<<<1, 32, 0, s>>>((const int2*) scratch, n_blocks, nv, kpart, params, step_rec, sub_to_id, ring,
+                                          cap, j, out_id, out_prob);
+    coupled_check("coupled_draft merge");
+}
+
 }  // namespace strata::kernels

src/kernels/cuda/verify_kernels.cu

@@ -3,6 +3,7 @@
 // The per-token arithmetic of every kernel here is transcribed from its single-token original (fused_gdn.cu,
 // elementwise.cu) with the same operation order, so a verify window reproduces plain decode bit for bit.
 #include "strata/kernels/verify_kernels.hpp"
+#include "strata/kernels/dp4a.hpp"
 
 #include <cuda_runtime.h>
 
@@ -423,7 +424,7 @@ void gdn_step_norm_multi(float* state, const float* h, int conv_channels, const
 
 namespace {
 __global__ void wait_flag_ge_kernel(const volatile uint32_t* flag, uint32_t value) {
-    while (*flag < value) __nanosleep(100);
+    while (*flag < value) strata_spin_pause();
     __threadfence_system();
 }
 }  // namespace
@@ -472,7 +473,7 @@ __global__ void resident_plan_kernel(const int32_t* __restrict__ ids, int n, int
 }
 __global__ void wait_flag_ge_or_kernel(const volatile uint32_t* flag, uint32_t value, const volatile uint32_t* skip) {
     if (*skip == value) return;
-    while (*flag < value) __nanosleep(100);
+    while (*flag < value) strata_spin_pause();
     __threadfence_system();
 }
 __global__ void copy_i32_unless_kernel(int32_t* __restrict__ dst, const volatile int32_t* src, int n,
@@ -545,7 +546,7 @@ void copy_indexed(float* dst, const float* src, int64_t stride, const int32_t* i
 namespace { __global__ void gpu_stamp_kernel(unsigned long long* buf, int i) {
     unsigned long long t;
 #if defined(__HIPCC__)
-    t = wall_clock64() * 10ull;   // gfx11: a constant 100 MHz counter, in ns
+    t = wall_clock64() * 10ull;   // gfx11 / gfx12: a constant 100 MHz counter, in ns
 #else
     asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
 #endif

src/kernels/elementwise_parity.cpp

@@ -12,6 +12,7 @@
 //      instead of decaying, so the sign is checked as a property and not assumed.
 //   3. `silu` IS COMPUTED IN DOUBLE then cast, because `ref/gdn.py`'s numpy does.  f32 `expf` differs in the
 //      last bits, and the test measures that rather than asserting it away.
+#include "strata/kernels/dequant_bf16.hpp"
 #include "strata/kernels/elementwise.hpp"
 #include "strata/kernels/f16_bits.hpp"
 
@@ -409,6 +410,64 @@ int main(int argc, char** argv) {
         if (mismatches || guards || fma_diff == 0) ++bad;
     }
 
+    // ---- THE f32 -> bf16 CONVERSIONS KEEP A NaN A NaN.  `f32_to_bf16_bulk` (the header's `bf16_from_f32`
+    // on the device) and the prompt path's dequantizer (its own `f2bf`) against ggml_compute_fp32_to_bf16's rule;
+    // before the fix 0x7FFFFFFF came back as -0 and 0x7F800001 as +inf.  `bf16_bits_test` covers every f32 on
+    // the host; this is the same header compiled by nvcc, plus the second copy in dequant_bf16.cu.
+    {
+        auto ggml_bf16 = [](uint32_t u) -> uint16_t {
+            if ((u & 0x7fffffffu) > 0x7f800000u) return (uint16_t) ((u >> 16) | 64);
+            return (uint16_t) ((u + (0x7fffu + ((u >> 16) & 1u))) >> 16);
+        };
+        std::vector<uint32_t> bits = {0x7FFFFFFFu, 0xFFFFFFFFu, 0x7F800001u, 0xFF800001u, 0x7FC00000u, 0x7FBFFFFFu,
+                                      0x7F800000u, 0xFF800000u, 0x7F7FFFFFu, 0x00000000u, 0x80000000u, 0x3F808000u,
+                                      0x3F818000u, 0x00008000u, 0x7F7F8000u};
+        std::mt19937 rng(13);
+        while (bits.size() < 4096) bits.push_back((uint32_t) rng());
+        const size_t n = bits.size();
+        float* dx = nullptr;
+        uint16_t* dy = nullptr;
+        check(cudaMalloc(&dx, n * 4), "bf16 x");
+        check(cudaMalloc(&dy, n * 2), "bf16 y");
+        check(cudaMemcpy(dx, bits.data(), n * 4, cudaMemcpyHostToDevice), "bf16 cx");
+        strata::kernels::f32_to_bf16_bulk(dx, dy, (int64_t) n, nullptr);
+        std::vector<uint16_t> got(n);
+        check(cudaMemcpy(got.data(), dy, n * 2, cudaMemcpyDeviceToHost), "bf16 cy");
+        int wrong = 0;
+        for (size_t i = 0; i < n; ++i) wrong += got[i] != ggml_bf16(bits[i]);
+
+        // A Q8_0 block (type 8) whose fp16 scale is a NaN: every dequantized value is NaN * q, a NaN, and it must
+        // still be one in BF16.  A second block with an ordinary scale checks the finite path did not move.
+        std::vector<uint8_t> blk(2 * 34, 0);
+        blk[0] = 0x00; blk[1] = 0x7E;                        // fp16 quiet NaN
+        blk[34] = 0x00; blk[35] = 0x3C;                      // fp16 1.0
+        for (int j = 0; j < 32; ++j) {
+            blk[(size_t) (2 + j)] = (uint8_t) (int8_t) (j - 16);
+            blk[(size_t) (36 + j)] = (uint8_t) (int8_t) (3 * j - 50);
+        }
+        uint8_t* db = nullptr;
+        uint16_t* dq = nullptr;
+        check(cudaMalloc(&db, blk.size()), "q8 blk");
+        check(cudaMalloc(&dq, 64 * 2), "q8 out");
+        check(cudaMemcpy(db, blk.data(), blk.size(), cudaMemcpyHostToDevice), "q8 cblk");
+        strata::kernels::dequant_bf16(8, db, 0, 2, 32, dq, nullptr);
+        check(cudaDeviceSynchronize(), "dequant_bf16");
+        std::vector<uint16_t> q(64);
+        check(cudaMemcpy(q.data(), dq, 64 * 2, cudaMemcpyDeviceToHost), "q8 cout");
+        int dq_wrong = 0;
+        for (int j = 0; j < 32; ++j) {
+            if (!((q[(size_t) j] & 0x7FFFu) > 0x7F80u)) ++dq_wrong;           // row 0: NaN scale -> NaN
+            const float v = (float) (3 * j - 50);                                // row 1: d = 1.0 -> the integer
+            uint32_t vb;
+            std::memcpy(&vb, &v, 4);
+            if (q[(size_t) (32 + j)] != ggml_bf16(vb)) ++dq_wrong;
+        }
+        std::printf("  f32 -> bf16 (NaN kept): bulk %d of %zu differ from ggml, dequant_bf16 %d of 64 wrong\n",
+                    wrong, n, dq_wrong);
+        if (wrong || dq_wrong) ++bad;
+        cudaFree(dx); cudaFree(dy); cudaFree(db); cudaFree(dq);
+    }
+
     std::printf("\nelementwise: %d failures\n", bad);
     if (bad) return 1;
     if (selftest) std::printf("elementwise_parity OK\n");

src/kernels/mmvq_multi_parity.cpp

@@ -0,0 +1,251 @@
+// src/kernels/mmvq_multi_parity.cpp - the multi_exact contract of native_mmvq, tested bitwise.
+//
+//     build/mmvq_multi_parity
+//
+// WHY IT EXISTS.  `include/strata/kernels/native_mmvq.hpp` says of `native_mmvq_set_multi_exact`:
+//
+//     "true (default): the ncols == 1 layout, every column bitwise equal to a single-column call"
+//
+// and `include/strata/core/verify.hpp` counts "multi-column MMVQ in exact mode" among the reasons the verify window
+// produces what greedy decode would, bit for bit.  Nothing tested that sentence: `iq_parity` calls `native_mmvq`
+// with two columns and compares against a float64 reference at 2e-2, which cannot see a difference between one
+// column and T columns.
+//
+// WHAT IT COMPARES.  The SAME Q8_1 bytes: one T-column call against T single-column calls, each reading column j
+// from the same buffer (the header: "its Q8_1 blocks at j * n_in / 32").  The quantization is shared, so the only
+// thing under test is how the columns are laid out.
+//
+// THE DATA.  The weights are random bytes, except each block's fp16 scales (`d`, and `dmin` where the format has
+// one), which are rewritten as normal fp16 values in +-[2^-10, 2^-5): random bytes there would put inf and NaN in
+// the outputs, and on the GPU every NaN comes out as the same canonical bit pattern, so a NaN output compares equal
+// whatever operations produced it and proves nothing.  With finite scales and finite normal activations (the
+// precondition the header puts on the input) every output is finite, and a non-finite one fails the test.
+//
+// THE NEGATIVE CONTROL.  The same comparison with `multi_exact` off must find differences, or the test has no power
+// and its pass means nothing.  Where it has power depends on T: in `native_mmvq.cu` the generic layout uses
+// `NW = NCOLS <= 4 ? 4 : 2` warps, and with `WARPS = 4` and `BPI = F::BPI * NW / WARPS` it coincides with the
+// exact layout at T <= 4.  So the control is ASSERTED on T > 4 for every case, and at T <= 4 its silence is
+// reported as that coincidence rather than skipped.  The control needs a row long enough for the two layouts to
+// group blocks differently: IQ4_XS takes 16 blocks per iteration with 4 warps and 8 with 2, so at n_in = 2048 (8
+// blocks of 256) each thread holds at most one block in both layouts and they coincide; its case uses n_in = 4096.
+#include "strata/kernels/iq_kernels.hpp"
+#include "strata/kernels/native_mmvq.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <random>
+#include <vector>
+
+namespace {
+
+struct Case {
+    const char* name;
+    int type;          // stable GGML type id, from the list in native_mmvq.hpp
+    int n_in;
+    int n_out;
+    int block_elems;   // elements per weight block
+    int block_bytes;
+    int scale_at[2];   // byte offsets of the block's fp16 scales in native_mmvq.cu's block structs; -1 = none
+};
+
+// the types a head or a projection can have in this model
+const Case CASES[] = {
+    {"Q4_0", 2, 2048, 512, 32, 18, {0, -1}},          // Q40Block: half d
+    {"Q8_0", 8, 2048, 512, 32, 34, {0, -1}},          // Q80Block: half d
+    {"Q4_K", 12, 2048, 512, 256, 144, {0, 2}},        // Q4KBlock: half2 dm
+    {"Q5_K", 13, 2048, 512, 256, 176, {0, 2}},        // Q5KBlock: half2 dm
+    {"Q6_K", 14, 2048, 512, 256, 210, {208, -1}},     // Q6KBlock: half d after ql, qh, scales
+    {"IQ4_XS", 23, 4096, 512, 256, 136, {0, -1}},     // IQ4XSBlock: half d; n_in 4096, see THE NEGATIVE CONTROL
+    {"Q5_K wide", 13, 4096, 2048, 256, 176, {0, 2}},
+};
+
+// a normal fp16 in +-[2^-10, 2^-5): exponent field 5..9 (bias 15), any mantissa, either sign
+uint16_t sane_half(std::mt19937& rng) {
+    const uint32_t r = rng();
+    return (uint16_t) (((r >> 31) << 15) | ((5u + (r >> 10) % 5u) << 10) | (r & 0x3ffu));
+}
+
+// T = 4 is the last width where the two layouts coincide, T = 5 the first where they differ
+const int WIDTHS[] = {1, 2, 3, 4, 5, 6, 8};
+constexpr int COINCIDE_MAX_T = 4;
+
+bool ck(cudaError_t e, const char* what) {
+    if (e == cudaSuccess) return true;
+    std::printf("CUDA: %s: %s\n", what, cudaGetErrorString(e));
+    return false;
+}
+
+// Returns the number of outputs that differ bit for bit; counts the non-finite outputs and the differences on
+// outputs that are finite in both paths.
+long long compare(const Case& c, int T, cudaStream_t s, long long& nonfinite, long long& diff_finite, bool& ran) {
+    ran = false;
+    if (!strata::kernels::native_mmvq_supported(c.type)) return -1;
+
+    const std::size_t wbytes = strata::kernels::native_mmvq_weight_bytes(c.type, c.n_in, c.n_out);
+    const std::size_t qcol = strata::kernels::native_q8_1_bytes(c.n_in, 1);   // Q8_1 bytes of ONE column
+    const std::size_t qall = strata::kernels::native_q8_1_bytes(c.n_in, T);
+    if (qall != qcol * (std::size_t) T) {
+        std::printf("%-12s T=%d: the Q8_1 bytes of T columns (%zu) are not T times those of one (%zu): the test "
+                    "assumes contiguous columns and they are not\n", c.name, T, qall, qcol);
+        return -1;
+    }
+
+    std::mt19937 rng(1234u + (unsigned) c.type * 97u + (unsigned) T);
+    std::vector<uint8_t> w(wbytes);
+    for (auto& b : w) b = (uint8_t) (rng() & 0xff);
+    const std::size_t n_blocks = (std::size_t) c.n_out * (std::size_t) (c.n_in / c.block_elems);
+    if (n_blocks * (std::size_t) c.block_bytes != wbytes) {
+        std::printf("%-12s T=%d: %zu blocks of %d bytes are not the %zu weight bytes native_mmvq expects\n", c.name,
+                    T, n_blocks, c.block_bytes, wbytes);
+        return -1;
+    }
+    for (std::size_t k = 0; k < n_blocks; ++k)
+        for (int at : c.scale_at)
+            if (at >= 0) {
+                const uint16_t h = sane_half(rng);
+                std::memcpy(&w[k * (std::size_t) c.block_bytes + (std::size_t) at], &h, 2);
+            }
+    std::vector<float> x((std::size_t) T * c.n_in);
+    std::normal_distribution<float> nd(0.f, 1.f);
+    for (auto& v : x) v = nd(rng);
+
+    void* dw = nullptr;
+    void* xq = nullptr;
+    float* dx = nullptr;
+    float* dmulti = nullptr;
+    float* dsingle = nullptr;
+    const std::size_t ybytes = (std::size_t) T * c.n_out * 4;
+    if (!ck(cudaMalloc(&dw, wbytes), "malloc w") || !ck(cudaMalloc(&xq, qall), "malloc xq") ||
+        !ck(cudaMalloc(&dx, x.size() * 4), "malloc x") || !ck(cudaMalloc(&dmulti, ybytes), "malloc multi") ||
+        !ck(cudaMalloc(&dsingle, ybytes), "malloc single"))
+        return -1;
+    if (!ck(cudaMemcpy(dw, w.data(), wbytes, cudaMemcpyHostToDevice), "copy w") ||
+        !ck(cudaMemcpy(dx, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "copy x"))
+        return -1;
+
+    // ONE quantization, shared by both paths: the quantization is not under test
+    strata::kernels::quantize_q8_1_rows(dx, T, c.n_in, xq, s);
+    if (!ck(cudaStreamSynchronize(s), "quantize")) return -1;
+
+    try {
+        strata::kernels::native_mmvq(c.type, dw, xq, dmulti, c.n_in, c.n_out, T, s);
+        for (int j = 0; j < T; ++j)
+            strata::kernels::native_mmvq(c.type, dw, (const uint8_t*) xq + (std::size_t) j * qcol,
+                                         dsingle + (std::size_t) j * c.n_out, c.n_in, c.n_out, 1, s);
+    } catch (const std::exception& e) {
+        std::printf("%-12s T=%d: %s\n", c.name, T, e.what());
+        return -1;
+    }
+    if (!ck(cudaStreamSynchronize(s), "mmvq")) return -1;
+
+    std::vector<uint32_t> a((std::size_t) T * c.n_out), b(a.size());
+    if (!ck(cudaMemcpy(a.data(), dmulti, ybytes, cudaMemcpyDeviceToHost), "read multi") ||
+        !ck(cudaMemcpy(b.data(), dsingle, ybytes, cudaMemcpyDeviceToHost), "read single"))
+        return -1;
+
+    long long diff = 0;
+    nonfinite = 0;
+    diff_finite = 0;
+    for (std::size_t i = 0; i < a.size(); ++i) {
+        float fa, fb;
+        std::memcpy(&fa, &a[i], 4);
+        std::memcpy(&fb, &b[i], 4);
+        const bool both_finite = std::isfinite(fa) && std::isfinite(fb);
+        if (!both_finite) ++nonfinite;
+        if (a[i] != b[i]) {
+            ++diff;
+            if (both_finite) ++diff_finite;
+        }
+    }
+    cudaFree(dw); cudaFree(xq); cudaFree(dx); cudaFree(dmulti); cudaFree(dsingle);
+    ran = true;
+    return diff;
+}
+
+struct Totals {
+    long long checked = 0, diff = 0, diff_finite = 0, nonfinite = 0;   // all widths
+    long long diff_finite_wide = 0;                                     // T > COINCIDE_MAX_T only
+    int powerless_cases = 0;   // cases where the control found no difference at any T > COINCIDE_MAX_T
+};
+
+int sweep(bool exact, cudaStream_t s, Totals& t) {
+    strata::kernels::native_mmvq_set_multi_exact(exact);
+    if (strata::kernels::native_mmvq_multi_exact() != exact) {
+        std::printf("native_mmvq_multi_exact() does not reflect what was set\n");
+        return 1;
+    }
+    std::printf("\n=== multi_exact %s ===\n", exact ? "ON (the default)" : "off (negative control)");
+    std::printf("%-12s %3s %10s %10s %14s %10s\n", "type", "T", "outputs", "bits diff", "finite diff", "nonfinite");
+    int bad = 0;
+    for (const Case& c : CASES) {
+        long long case_wide = 0;
+        for (int T : WIDTHS) {
+            long long nf = 0, dfin = 0;
+            bool ran = false;
+            const long long diff = compare(c, T, s, nf, dfin, ran);
+            if (!ran) { ++bad; continue; }
+            const long long n = (long long) T * c.n_out;
+            t.checked += n;
+            t.diff += diff;
+            t.diff_finite += dfin;
+            t.nonfinite += nf;
+            if (T > COINCIDE_MAX_T) { t.diff_finite_wide += dfin; case_wide += dfin; }
+            const char* verdict;
+            if (exact) verdict = diff == 0 ? "ok" : "FAIL";
+            else if (dfin > 0) verdict = "control sees it";
+            else verdict = T <= COINCIDE_MAX_T ? "layouts coincide here (NW = 4)" : "no power at this T";
+            std::printf("%-12s %3d %10lld %10lld %14lld %10lld  %s\n", c.name, T, n, diff, dfin, nf, verdict);
+            if (exact && diff != 0) ++bad;
+        }
+        if (!exact && case_wide == 0) ++t.powerless_cases;
+    }
+    return bad;
+}
+
+}  // namespace
+
+int main() {
+    cudaStream_t s;
+    if (!ck(cudaStreamCreate(&s), "stream create")) return 1;
+
+    Totals on, off;
+    const int bad = sweep(true, s, on);
+    const int bad_off = sweep(false, s, off);
+    strata::kernels::native_mmvq_set_multi_exact(true);   // back to the default
+
+    std::printf("\nmulti_exact on:   %lld outputs compared, %lld differ bitwise, %lld non-finite\n",
+                on.checked, on.diff, on.nonfinite);
+    std::printf("multi_exact off:  %lld outputs compared, %lld differ bitwise (%lld on finite outputs, %lld at T > %d)\n",
+                off.checked, off.diff, off.diff_finite, off.diff_finite_wide, COINCIDE_MAX_T);
+
+    if (on.nonfinite != 0 || off.nonfinite != 0) {
+        std::printf("\nmmvq_multi_parity: FAILED. %lld non-finite outputs from finite scales and activations: the "
+                    "bit comparison is only meaningful on finite outputs.\n", on.nonfinite + off.nonfinite);
+        return 1;
+    }
+    if (bad != 0 || bad_off != 0) {
+        std::printf("\nmmvq_multi_parity: FAILED. %d case(s) with multi_exact on differ from single-column calls "
+                    "or did not run, %d case(s) did not run with it off. If they differ, the sentence in "
+                    "native_mmvq.hpp does not hold, and neither does the bit-exactness verify.hpp claims.\n",
+                    bad, bad_off);
+        return 1;
+    }
+    // THE NEGATIVE CONTROL, asserted where it can see something: past T = 4 the generic layout is a different one,
+    // so if it still finds no difference the comparison cannot tell the layouts apart and the pass proves nothing.
+    if (off.powerless_cases != 0) {
+        std::printf("\nmmvq_multi_parity: FAILED (no power). In %d case(s) no output differs with multi_exact off at "
+                    "any T > %d, where the generic layout is not the exact one: for those cases this comparison "
+                    "cannot tell the two layouts apart, so their pass above proves nothing.\n", off.powerless_cases,
+                    COINCIDE_MAX_T);
+        return 1;
+    }
+    std::printf("\nmmvq_multi_parity: ok. With multi_exact on every column is bitwise equal to a single-column "
+                "call; with it off the control finds differences at T > %d in every case, so the comparison has "
+                "power.\n",
+                COINCIDE_MAX_T);
+    return 0;
+}