Engine 0.1.28: the expert cache reserves the draft head's VRAM (0.1.27 left 16 GB cards below the stall line), a cancelled request no longer fails the next one, the API key protects /status; setup requires it

Squashed from PR #153.

CMakeLists.txt

@@ -4,11 +4,11 @@
 # `strata-pack` are added when their sources exist - a target with no sources would make the build graph
 # claim something the tree does not have, which is worse than an absent target.
 #
-# CUDA targets arrive in Phase 2 and this build REJECTS any architecture other than 120: the engine is
-# written against sm_120 (RTX 5000 / Blackwell) and silently building for an older arch would produce a
-# binary that runs and is wrong about its own performance, which is the worst of both outcomes.
+# CUDA targets arrive in Phase 2.  The kernels need sm_80 or newer (tf32 mma, bf16 math) and the guard
+# below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
+# and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.21 LANGUAGES CXX)
+project(strata VERSION 0.1.28 LANGUAGES CXX)
 add_compile_definitions(STRATA_VERSION="${PROJECT_VERSION}")
 # issue #31: release builds keep a PDB beside the exe (not shipped in it), so a minidump from a stalled engine can
 # be read back to the exact source line
@@ -33,6 +33,17 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 option(STRATA_ENABLE_CUDA "Build the CUDA targets (Phase 2+)" OFF)
+option(STRATA_ENABLE_HIP "Build the HIP targets for gfx1100 wave32 hardware" OFF)
+option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF)
+if(STRATA_PREFILL_MMQ AND NOT STRATA_ENABLE_HIP)
+  message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_ENABLE_HIP=ON")
+endif()
+if(STRATA_ENABLE_CUDA AND STRATA_ENABLE_HIP)
+  message(FATAL_ERROR "STRATA_ENABLE_CUDA and STRATA_ENABLE_HIP are separate backends; enable only one")
+endif()
+if(STRATA_ENABLE_HIP)
+  include(cmake/hip_backend.cmake)
+endif()
 # The test suite and the micro benchmarks live in tests/ and bench/micro/, which the published source leaves out.
 set(_strata_tests_default OFF)
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/bench/micro)
@@ -51,18 +62,26 @@ if(STRATA_ENABLE_CUDA)
   endif()
   foreach(_a IN LISTS CMAKE_CUDA_ARCHITECTURES)
     string(REGEX REPLACE "-.*$" "" _base "${_a}")     # 120-real / 120-virtual are still 120
-    # Plan v0.3 P6: developed and measured on sm_120 (RTX 50); the kernels need sm_80 or newer (tf32 mma in
-    # the attention scorer, bf16 math), so RTX 30 (86) and RTX 40 (89) build too.  Older cards are refused.
+    # Turing port: developed and measured on sm_120 (RTX 50); the tf32 mma in the QSA scorer has a
+    # portable fp32-FMA fallback, so RTX 20 (75), 30 (86), RTX 40 (89) and RTX 50 (120) build too.
+    # Pre-sm_75 is refused.
     if(_base STREQUAL "native" OR _base STREQUAL "all" OR _base STREQUAL "all-major")
       continue()
     endif()
-    if(_base LESS 80 AND NOT STRATA_EXPERIMENTAL_SM75)   # the A/B of a Turing card as a layer-split stage
+    if(_base LESS 75)
       message(FATAL_ERROR
-        "Strata needs an NVIDIA GPU of compute capability 8.0 or newer (RTX 30 / 40 / 50 series), but "
+        "Strata needs an NVIDIA GPU of compute capability 7.5 or newer (RTX 20 / 30 / 40 / 50 series), but "
         "CMAKE_CUDA_ARCHITECTURES is '${CMAKE_CUDA_ARCHITECTURES}'.")
     endif()
   endforeach()
   message(STATUS "Strata: CUDA enabled, arch ${CMAKE_CUDA_ARCHITECTURES}")
+  set(_strata_gpu_runtime_target CUDA::cudart)
+  set(_strata_gpu_blas_target CUDA::cublas)
+  set(_strata_gpu_include_directories ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+elseif(STRATA_ENABLE_HIP)
+  set(_strata_gpu_runtime_target strata_hip_runtime)
+  set(_strata_gpu_blas_target roc::hipblas)
+  set(_strata_gpu_include_directories ${STRATA_HIP_COMPAT_INCLUDE_DIR})
 endif()
 
 # Warnings live on an INTERFACE target the project's own code links, NOT in add_compile_options: applied
@@ -78,11 +97,14 @@ target_compile_options(strata_warnings INTERFACE
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/utf-8>
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Wall>
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Wextra>
-  $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CXX_COMPILER_ID:MSVC>>:-Xcompiler=/W4>)
+  $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CXX_COMPILER_ID:MSVC>>:-Xcompiler=/W4>
+  $<$<COMPILE_LANGUAGE:HIP>:-Wall>
+  $<$<COMPILE_LANGUAGE:HIP>:-Wextra>)
 if(STRATA_WERROR)
   target_compile_options(strata_warnings INTERFACE
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/WX>
-    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Werror>)
+    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Werror>
+    $<$<COMPILE_LANGUAGE:HIP>:-Werror>)
 endif()
 
 # ---- strata_artifact -------------------------------------------------------
@@ -100,6 +122,12 @@ add_executable(strata-dequant src/artifact/dequant.cpp)
 target_link_libraries(strata-dequant PRIVATE strata_artifact)
 # this file is now nothing but the CLI entry point, and its inherited guard spells that as a "selftest"
 target_compile_definitions(strata-dequant PRIVATE STRATA_DEQUANT_SELFTEST)
+if(STRATA_BUILD_TESTS)
+  # the reader refuses a duplicate tensor name at open, naming it (a synthetic GGUF: no model, no GPU)
+  add_executable(gguf_reader_test src/artifact/gguf_reader_test.cpp)
+  target_link_libraries(gguf_reader_test PRIVATE strata_artifact)
+  add_test(NAME gguf_reader_test COMMAND gguf_reader_test)
+endif()
 
 # ---- strata_plan ----------------------------------------------------------
 # Also header-only.  A "planner" that hides state would be one whose decision is not inspectable, and the
@@ -111,17 +139,16 @@ target_link_libraries(strata_plan INTERFACE strata_warnings)
 add_executable(strata-plan src/plan/plan_main.cpp)
 target_link_libraries(strata-plan PRIVATE strata_plan)
 
-# ---- strata_core (CUDA, Phase 2) ------------------------------------------
-# Only built when CUDA is on, so the Phase 1 targets keep working on a machine with no toolkit.  The arch
-# guard above has already refused anything but sm_120 by the time this runs.
-if(STRATA_ENABLE_CUDA)
+# ---- strata_core (GPU runtime) ---------------------------------------------
+# Only built when a GPU backend is on, so the Phase 1 targets keep working without a toolkit.
+if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   add_library(strata_core STATIC src/core/device.cu src/core/pinned.cu src/platform/memory.cpp src/core/graph.cpp src/core/weights.cpp
                                src/core/layout.cpp)
   target_include_directories(strata_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-  # CUDA::cudart is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and
+  # The selected runtime is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and
   # cudaEventQuery: the .cu sources get cudart through the CUDA language rules, a .cpp does not, and the
   # linker error ("unresolved external symbol cudaGraphInstantiate") names the function rather than the cause.
-  target_link_libraries(strata_core PUBLIC strata_plan strata_warnings CUDA::cudart)
+  target_link_libraries(strata_core PUBLIC strata_plan strata_warnings ${_strata_gpu_runtime_target})
 
   add_executable(strata-device src/core/device_main.cpp)
   target_link_libraries(strata-device PRIVATE strata_core)
@@ -130,13 +157,13 @@ if(STRATA_ENABLE_CUDA)
   target_link_libraries(strata-load PRIVATE strata_core strata_kernels)
 
   add_executable(strata-concurrent src/core/concurrent_main.cpp)
-  target_include_directories(strata-concurrent PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels CUDA::cudart)
+  target_include_directories(strata-concurrent PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target})
 
   add_executable(strata-overlap src/core/overlap_main.cpp)
   # a .cpp that calls cudaMemcpyAsync itself needs the toolkit headers and cudart explicitly
-  target_include_directories(strata-overlap PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels CUDA::cudart)
+  target_include_directories(strata-overlap PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target})
 
   # The load test needs the 34 GB artifact, so it is registered ONLY where that artifact exists.  A test that
   # silently skips is a test that stops running; an absent registration is visible in the configure output.
@@ -189,6 +216,7 @@ if(STRATA_ENABLE_CUDA)
                           src/kernels/cuda/fused_gr.cu
                           src/kernels/cuda/cvec.cu
                           src/kernels/cuda/qsa_decode_attn.cu
+                          src/kernels/cuda/qsa_prompt_attn.cu
                           src/kernels/cuda/fused_gdn.cu
                           src/kernels/cuda/dequant_bf16.cu
                           src/kernels/cuda/qsa_select.cu
@@ -198,33 +226,48 @@ if(STRATA_ENABLE_CUDA)
                           src/ngram/ple_reader.cpp
                           src/platform/direct_file.cpp)
   target_link_libraries(strata_kernels PUBLIC strata_core strata_artifact)
+  # perf-review D-1: the tensor-core prompt attention vs the FP32 kernel and FP64 (GPU, synthetic, no model); with
+  # the tests, or alone with -DSTRATA_PARITY_PROMPT_ATTN=ON
+  if(STRATA_BUILD_TESTS OR STRATA_PARITY_PROMPT_ATTN)
+    add_executable(qsa_prompt_attn_parity src/kernels/qsa_prompt_attn_parity.cpp)
+    target_include_directories(qsa_prompt_attn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+    target_link_libraries(qsa_prompt_attn_parity PRIVATE strata_kernels CUDA::cudart)
+  endif()
   # plan v0.3 P6: llama.cpp's block layouts and codebook grids for the i-quant formats (MIT, third_party/ggml)
   target_include_directories(strata_kernels PRIVATE third_party/ggml)
   # Match the pinned llama.cpp CUDA arithmetic contract for this adapter only.
-  set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu
-    src/kernels/cuda/native_gr_norm.cu
-    src/kernels/cuda/native_moe.cu
-    src/kernels/cuda/native_gdn.cu
-    src/kernels/cuda/native_gr_postops.cu
-    src/kernels/cuda/native_router.cu
-    src/kernels/cuda/native_gdn_preprocess.cu
-    src/kernels/cuda/native_qsa.cu
-    src/kernels/cuda/native_rope.cu
-    src/kernels/cuda/native_ple_postops.cu
-    src/kernels/cuda/native_flash_attn.cu
-    src/kernels/cuda/native_qsa_indexer.cu
-    src/kernels/cuda/native_qsa_score.cu
-    PROPERTIES COMPILE_OPTIONS "--use_fast_math")
+  if(STRATA_ENABLE_CUDA)
+    set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu
+      src/kernels/cuda/native_gr_norm.cu
+      src/kernels/cuda/native_moe.cu
+      src/kernels/cuda/native_gdn.cu
+      src/kernels/cuda/native_gr_postops.cu
+      src/kernels/cuda/native_router.cu
+      src/kernels/cuda/native_gdn_preprocess.cu
+      src/kernels/cuda/native_qsa.cu
+      src/kernels/cuda/native_rope.cu
+      src/kernels/cuda/native_ple_postops.cu
+      src/kernels/cuda/native_flash_attn.cu
+      src/kernels/cuda/native_qsa_indexer.cu
+      src/kernels/cuda/native_qsa_score.cu
+      PROPERTIES COMPILE_OPTIONS "--use_fast_math")
+  elseif(STRATA_ENABLE_HIP)
+    # Keep Q8_K's multiply and magic-bias rounding as separate FP32 operations on HIP; contraction to an
+    # FMA changes tie results. The embedding gather also relies on separately rounded operations.
+    set_source_files_properties(src/kernels/cuda/elementwise.cu src/kernels/cuda/quantize_act.cu
+      PROPERTIES COMPILE_OPTIONS "-ffp-contract=off")
+  endif()
 
   set(STRATA_ORACLE_SOURCE_DIR "" CACHE PATH "Pinned llama.cpp source for independent CUDA parity tests")
   set(STRATA_ORACLE_BUILD_DIR "" CACHE PATH "Matching existing shared llama.cpp build")
-  if(STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR)
+  # This oracle is a CUDA ggml build; do not pull CUDA libraries into a HIP build.
+  if(STRATA_ENABLE_CUDA AND STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR)
     foreach(parity IN ITEMS native_mmvq native_bf16 native_shared native_gr_norm native_moe native_gdn native_gr_postops native_router native_gdn_preprocess native_gdn_pipeline native_qsa native_rope native_ple_postops native_flash_attn native_qsa_indexer native_qsa_score)
     add_executable(${parity}_parity bench/micro/${parity}_parity.cpp)
     target_include_directories(${parity}_parity PRIVATE
-      "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+      "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${_strata_gpu_include_directories})
     target_compile_definitions(${parity}_parity PRIVATE GGML_SHARED GGML_BACKEND_SHARED)
-    target_link_libraries(${parity}_parity PRIVATE strata_kernels CUDA::cudart)
+    target_link_libraries(${parity}_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
     foreach(component IN ITEMS ggml ggml-base ggml-cpu ggml-cuda)
       string(REPLACE "-" "_" variable "STRATA_ORACLE_${component}")
       find_library(${variable} NAMES ${component} PATHS
@@ -248,99 +291,163 @@ if(STRATA_ENABLE_CUDA)
   add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
     src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-  target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu CUDA::cudart)
+  target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/file_expert_source_test.cpp")
+    enable_testing()
+    add_executable(file_expert_source_test tests/core/file_expert_source_test.cpp)
+    target_link_libraries(file_expert_source_test PRIVATE strata_engine)
+    add_test(NAME file_expert_source_test COMMAND file_expert_source_test)
+    set_tests_properties(file_expert_source_test PROPERTIES TIMEOUT 60)
+  endif()
 
   # `strata generate` (P2.S6).  The FIRST user-facing program: everything under it is a component, and this is
   # what composes them into a token.  No tokenizer - the prompt arrives as ids - because Gate C1 compares logits
   # against llama.cpp on the SAME ids and a tokenizer on one side of that comparison is a second variable.
   # plan v0.3 P5: batched prompt processing (tensor-core GEMMs through cuBLAS).
   add_library(strata_prefill STATIC src/prefill/gemm.cu src/prefill/kernels.cu src/prefill/prefill.cpp)
-  target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata_prefill PUBLIC strata_engine CUDA::cublas CUDA::cudart)
+  target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})
+  target_link_libraries(strata_prefill PUBLIC strata_engine ${_strata_gpu_blas_target} ${_strata_gpu_runtime_target})
+  if(STRATA_ENABLE_HIP AND STRATA_HIPBLASLT_AVAILABLE)
+    target_compile_definitions(strata_prefill PRIVATE STRATA_HIPBLASLT_AVAILABLE=1)
+    target_link_libraries(strata_prefill PRIVATE roc::hipblaslt)
+
+    add_executable(tune_hipblaslt tools/hip/tune_hipblaslt.cpp)
+    target_link_libraries(tune_hipblaslt PRIVATE hip::host roc::hipblas roc::hipblaslt)
+  endif()
 
   add_executable(strata src/program/generate.cpp)
-  target_include_directories(strata PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec CUDA::cudart)
+  target_include_directories(strata PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec ${_strata_gpu_runtime_target})
+
+  if(STRATA_ENABLE_HIP)
+    enable_testing()
+    set(_strata_hip_tests)
+    add_executable(hip_expert_cache_staging tests/hip/expert_cache_staging.cpp)
+    target_link_libraries(hip_expert_cache_staging PRIVATE strata_engine)
+    add_test(NAME hip_expert_cache_staging COMMAND hip_expert_cache_staging)
+    list(APPEND _strata_hip_tests hip_expert_cache_staging)
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/intrinsics.cpp")
+      add_executable(hip_intrinsics tests/hip/intrinsics.cpp)
+      target_link_libraries(hip_intrinsics PRIVATE strata_hip_runtime)
+      add_test(NAME hip_intrinsics COMMAND hip_intrinsics)
+      list(APPEND _strata_hip_tests hip_intrinsics)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/handoff.cpp")
+      add_executable(hip_handoff tests/hip/handoff.cpp)
+      target_link_libraries(hip_handoff PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_handoff COMMAND hip_handoff)
+      list(APPEND _strata_hip_tests hip_handoff)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/native_qsa_score.cpp")
+      add_executable(hip_native_qsa_score tests/hip/native_qsa_score.cpp)
+      target_link_libraries(hip_native_qsa_score PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_native_qsa_score COMMAND hip_native_qsa_score)
+      list(APPEND _strata_hip_tests hip_native_qsa_score)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_native_batch.cpp")
+      add_executable(hip_prefill_native_batch tests/hip/prefill_native_batch.cpp)
+      target_link_libraries(hip_prefill_native_batch PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_prefill_native_batch COMMAND hip_prefill_native_batch)
+      list(APPEND _strata_hip_tests hip_prefill_native_batch)
+    endif()
+    if(STRATA_HIPBLASLT_AVAILABLE AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_hipblaslt_gemm.cpp")
+      add_executable(hip_prefill_hipblaslt_gemm tests/hip/prefill_hipblaslt_gemm.cpp)
+      target_include_directories(hip_prefill_hipblaslt_gemm PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/prefill")
+      target_link_libraries(hip_prefill_hipblaslt_gemm PRIVATE strata_prefill strata_hip_runtime roc::hipblaslt roc::hipblas)
+      add_test(NAME hip_prefill_hipblaslt_gemm COMMAND hip_prefill_hipblaslt_gemm)
+      set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77)
+      list(APPEND _strata_hip_tests hip_prefill_hipblaslt_gemm)
+    endif()
+    if(_strata_hip_tests)
+      set_tests_properties(${_strata_hip_tests} PROPERTIES TIMEOUT 60)
+    endif()
+    if(TARGET hip_prefill_native_batch)
+      set_tests_properties(hip_prefill_native_batch PROPERTIES TIMEOUT 120)
+    endif()
+    if(TARGET hip_prefill_hipblaslt_gemm)
+      set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77)
+    endif()
+  endif()
 
   add_executable(dequant_s2_parity src/kernels/dequant_s2_parity.cpp)
   # The tests are HOST C++ files that call cudaMalloc/cudaMemcpy themselves, so they need the toolkit headers
   # and cudart explicitly: enabling the CUDA language gives those to .cu sources automatically, but a .cpp
   # target is just a C++ target and gets neither.
-  target_include_directories(dequant_s2_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(dequant_s2_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(dequant_s2_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(dequant_s2_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME dequant_s2_parity COMMAND dequant_s2_parity --selftest)
 
   add_executable(s2_gemv_parity src/kernels/s2_gemv_parity.cpp)
-  target_include_directories(s2_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s2_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s2_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s2_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s2_gemv_parity COMMAND s2_gemv_parity --selftest)
 
   add_executable(iq_parity src/kernels/iq_parity.cpp)
-  target_include_directories(iq_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(iq_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(iq_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(iq_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)
-  target_include_directories(shared_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(shared_expert_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME shared_expert_parity COMMAND shared_expert_parity --selftest)
 
   add_executable(gr_parity src/kernels/gr_parity.cpp)
-  target_include_directories(gr_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(gr_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(gr_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(gr_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME gr_parity COMMAND gr_parity --selftest)
 
   add_executable(gdn_parity src/kernels/gdn_parity.cpp)
-  target_include_directories(gdn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(gdn_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(gdn_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(gdn_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME gdn_parity COMMAND gdn_parity --selftest)
 
   add_executable(s2_gemv_q8_parity src/kernels/s2_gemv_q8_parity.cpp)
-  target_include_directories(s2_gemv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s2_gemv_q8_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s2_gemv_q8_parity COMMAND s2_gemv_q8_parity --selftest)
 
   add_executable(sampler_parity src/kernels/sampler_parity.cpp)
-  target_include_directories(sampler_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(sampler_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(sampler_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(sampler_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME sampler_parity COMMAND sampler_parity --selftest)
 
   add_executable(rope_parity src/kernels/rope_parity.cpp)
-  target_include_directories(rope_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(rope_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(rope_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(rope_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME rope_parity COMMAND rope_parity --selftest)
 
   add_executable(quantize_act_parity src/kernels/quantize_act_parity.cpp)
-  target_include_directories(quantize_act_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(quantize_act_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(quantize_act_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(quantize_act_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME quantize_act_parity COMMAND quantize_act_parity --selftest)
 
   add_executable(router_top10_parity src/kernels/router_top10_parity.cpp)
-  target_include_directories(router_top10_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(router_top10_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(router_top10_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(router_top10_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME router_top10_parity COMMAND router_top10_parity --selftest)
 
   add_executable(s_gemv_parity src/kernels/s_gemv_parity.cpp)
-  target_include_directories(s_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s_gemv_parity COMMAND s_gemv_parity --selftest)
 
   add_executable(elementwise_parity src/kernels/elementwise_parity.cpp)
-  target_include_directories(elementwise_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(elementwise_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(elementwise_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(elementwise_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME elementwise_parity COMMAND elementwise_parity --selftest)
 
   add_executable(bf16_gemv_parity src/kernels/bf16_gemv_parity.cpp)
-  target_include_directories(bf16_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(bf16_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME bf16_gemv_parity COMMAND bf16_gemv_parity --selftest)
 
   add_executable(s_gemv_q8k_parity src/kernels/s_gemv_q8k_parity.cpp)
-  target_include_directories(s_gemv_q8k_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s_gemv_q8k_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s_gemv_q8k_parity COMMAND s_gemv_q8k_parity --selftest)
 
   add_executable(qsa_parity src/kernels/qsa_parity.cpp)
-  target_include_directories(qsa_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(qsa_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(qsa_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(qsa_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME qsa_parity COMMAND qsa_parity --selftest)
 
   # ---- P2.S4: the PLE n-gram hash, the IQ4_NL table read, and the PLE block ----
@@ -352,8 +459,8 @@ if(STRATA_ENABLE_CUDA)
   # INSIDE the test.  A test that passes by finding nothing to compare is the failure mode this project keeps
   # re-learning.
   add_executable(ple_parity src/kernels/ple_parity.cpp)
-  target_include_directories(ple_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(ple_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(ple_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(ple_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   set(STRATA_PLE_FIXTURE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro" CACHE PATH
       "Directory containing required ple_in.bin and ple_out.bin parity fixtures")
   add_test(NAME ple_parity COMMAND ple_parity --selftest
@@ -362,73 +469,86 @@ if(STRATA_ENABLE_CUDA)
 
   # Plan v0.3 P7: INT8 KV storage vs a host reference and vs the FP16 path (GPU, synthetic, no model).
   add_executable(kv_q8_parity src/kernels/kv_q8_parity.cpp)
-  target_include_directories(kv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_q8_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_q8_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_q8_parity COMMAND kv_q8_parity)
 
   # KV streaming (docs/kv-streaming-design.md): streamed vs resident attention, bitwise (GPU, synthetic, no model).
   add_executable(kv_stream_parity src/kernels/kv_stream_parity.cpp)
-  target_include_directories(kv_stream_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_stream_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_stream_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_stream_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_stream_parity COMMAND kv_stream_parity)
 
+  # PR #120: the hybrid K8V4 KV (INT8 K + rotated Q4_0 V) vs host references (GPU, synthetic, no model).
+  add_executable(kv_hybrid_parity src/kernels/kv_hybrid_parity.cpp)
+  target_include_directories(kv_hybrid_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_hybrid_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  add_test(NAME kv_hybrid_parity COMMAND kv_hybrid_parity)
   # PR #21: Q4_0 KV with Walsh-Hadamard rotation vs a host reference (GPU, synthetic, no model).
   add_executable(kv_q4_parity src/kernels/kv_q4_parity.cpp)
-  target_include_directories(kv_q4_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_q4_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_q4_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_q4_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_q4_parity COMMAND kv_q4_parity)
   add_executable(cvec_parity src/kernels/cvec_parity.cpp)
-  target_include_directories(cvec_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(cvec_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(cvec_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(cvec_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME cvec_parity COMMAND cvec_parity)
 
   # Plan v0.3 P3: native MMVQ for 1..8 tokens (bitwise selftest + --bench timing); synthetic data, no model.
   if(STRATA_BUILD_TESTS)
   add_executable(native_mmvq_multi bench/micro/native_mmvq_multi.cpp)
-  target_include_directories(native_mmvq_multi PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(native_mmvq_multi PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(native_mmvq_multi PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(native_mmvq_multi PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME native_mmvq_multi COMMAND native_mmvq_multi --selftest)
   endif()
 
   # Plan v0.3 P0.1: working-set lock of a host region (CPU only).
   add_executable(platform_memory_test src/platform/memory_test.cpp)
-  target_link_libraries(platform_memory_test PRIVATE strata_core CUDA::cudart)
+  target_link_libraries(platform_memory_test PRIVATE strata_core ${_strata_gpu_runtime_target})
   add_test(NAME platform_memory_test COMMAND platform_memory_test)
 
   # Plan v0.3 P2: the n-gram table read straight from the SSD. The selftest needs only the CPU and a temp file.
   add_executable(ple_reader_test src/ngram/ple_reader_test.cpp)
-  target_include_directories(ple_reader_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(ple_reader_test PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(ple_reader_test PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(ple_reader_test PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME ple_reader_selftest COMMAND ple_reader_test --selftest)
 
-  # The GPU test is registered only when CUDA is on, and it runs ON THE TARGET PC.  `--selftest` allocates
+  # The GPU test runs ON THE TARGET PC. `--selftest` allocates
   # and writes real VRAM, so a machine without a device fails this test rather than skipping it silently -
   # which is correct: an engine that targets one GPU should not report a pass for having no GPU.
-  add_test(NAME cuda_device_selftest COMMAND strata-device --selftest)
+  if(STRATA_ENABLE_CUDA)
+    add_test(NAME cuda_device_selftest COMMAND strata-device --selftest)
+  else()
+    add_test(NAME hip_device_selftest COMMAND strata-device --selftest)
+  endif()
 
   # ---- P2.S5: the GraphRegistry --------------------------------------------------
   # A .cu because the test needs real kernels to capture; it links `strata_core` and not `strata_kernels`,
   # because what it exercises is the capture machinery and not any kernel in it.
-  if(STRATA_BUILD_TESTS)
-  add_executable(graph_registry_test bench/micro/graph_registry_test.cu)
-  target_include_directories(graph_registry_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(graph_registry_test PRIVATE strata_core CUDA::cudart)
-  add_test(NAME graph_registry_test COMMAND graph_registry_test)
+  if(STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/graph_registry_test.cu")
+    add_executable(graph_registry_test bench/micro/graph_registry_test.cu)
+    target_include_directories(graph_registry_test PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(graph_registry_test PRIVATE strata_core ${_strata_gpu_runtime_target})
+    add_test(NAME graph_registry_test COMMAND graph_registry_test)
+  endif()
 
   # ---- P2.S5's GATE: is a device<->pinned-memory handoff graph-capturable? --------------------
   # The doorbell protocol puts the D2H of `x_f` and the miss list INSIDE G[l] - that is the whole overlap.  The
   # phase doc says to verify this "before building on it", and this project has already lost two rounds to
   # building first (a memory-only spin that DEADLOCKS on Windows, and a `cudaEventRecord` inside a capture that
   # is SILENTLY DROPPED).  It links only cudart: it is about the capture machinery, not about any kernel.
-  add_executable(pinned_capture bench/micro/pinned_capture.cu)
-  target_include_directories(pinned_capture PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(pinned_capture PRIVATE CUDA::cudart)
-  add_test(NAME pinned_capture COMMAND pinned_capture)
+  if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/pinned_capture.cu")
+    add_executable(pinned_capture bench/micro/pinned_capture.cu)
+    target_include_directories(pinned_capture PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(pinned_capture PRIVATE ${_strata_gpu_runtime_target})
+    add_test(NAME pinned_capture COMMAND pinned_capture)
   endif()
 
   # Tells tests/CMakeLists.txt to add the CUDA-dependent Catch2 target.  It cannot be declared there
   # unconditionally: tests/ is added at the end of this file, but the flag has to be known before it runs.
-  set(STRATA_CUDA_TESTS ON)
+  if(STRATA_ENABLE_CUDA)
+    set(STRATA_CUDA_TESTS ON)
+  endif()
 endif()
 
 # ---- strata_kernels_cpu (P2.S3) ----
@@ -485,6 +605,9 @@ target_link_libraries(strata_kernels_cpu PUBLIC Threads::Threads)
 # with -DSTRATA_GGML_DIR=..., otherwise the pinned upstream commit (third_party/ggml/VERSION.txt).
 option(STRATA_NATIVE_EXPERTS "Native (IQ) expert support through ggml-cpu" ON)
 set(STRATA_GGML_DIR "" CACHE PATH "llama.cpp checkout to take ggml from (empty: fetch the pinned commit)")
+if(STRATA_PREFILL_MMQ AND NOT STRATA_NATIVE_EXPERTS)
+  message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_NATIVE_EXPERTS=ON for GGML sources")
+endif()
 if(STRATA_NATIVE_EXPERTS)
   enable_language(C)
   set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
@@ -542,7 +665,7 @@ if(STRATA_NATIVE_EXPERTS)
     endif()
     set(_strata_mmq ${_strata_gg}/src/ggml-cuda)
     add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
-    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs)
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)   # q8_0: the drafter's matrices (E-9)
       target_sources(strata_mmq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu)
     endforeach()
     target_include_directories(strata_mmq PRIVATE ${_strata_gg}/include ${_strata_gg}/src ${_strata_mmq}
@@ -551,28 +674,71 @@ if(STRATA_NATIVE_EXPERTS)
     if(MSVC)
       target_compile_options(strata_mmq PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/bigobj>)
     endif()
-    target_link_libraries(strata_mmq PUBLIC ggml-base CUDA::cudart)
+    target_link_libraries(strata_mmq PUBLIC ggml-base ${_strata_gpu_runtime_target})
     target_link_libraries(strata_prefill PUBLIC strata_mmq)
     target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+  elseif(STRATA_PREFILL_MMQ)
+    # Compile only the pinned GGML MMQ and activation-quantizer sources used by prefill. This uses
+    # common.cuh's native HIP path without enabling a second GGML HIP backend.
+    if(STRATA_GGML_DIR)
+      set(_strata_gg "${STRATA_GGML_DIR}/ggml")
+    else()
+      set(_strata_gg "${strata_llamacpp_SOURCE_DIR}/ggml")
+    endif()
+    set(_strata_mmq "${_strata_gg}/src/ggml-cuda")
+    if(NOT EXISTS "${_strata_mmq}/common.cuh" OR NOT EXISTS "${_strata_mmq}/quantize.cu")
+      message(FATAL_ERROR "The selected GGML source is missing the ggml-cuda MMQ sources: ${_strata_mmq}")
+    endif()
+    set(_strata_mmq_sources
+      src/prefill/moe_mmq.cu
+      src/prefill/ggml_cuda_host.cu
+      "${_strata_mmq}/quantize.cu")
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)
+      list(APPEND _strata_mmq_sources "${_strata_mmq}/template-instances/mmq-instance-${_t}.cu")
+    endforeach()
+    set_source_files_properties(${_strata_mmq_sources} PROPERTIES LANGUAGE HIP)
+    add_library(strata_mmq STATIC ${_strata_mmq_sources})
+    target_include_directories(strata_mmq
+      PRIVATE "${_strata_gg}/include" "${_strata_gg}/src" "${_strata_mmq}"
+      PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    target_compile_definitions(strata_mmq PRIVATE GGML_USE_HIP=1)
+    target_link_libraries(strata_mmq PUBLIC ggml-base hip::host roc::hipblas)
+    target_link_libraries(strata_prefill PUBLIC strata_mmq)
+    target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+
+    # Numerical diagnostic: synthetic layouts by default, optional first-expert IQ3_XXS/IQ4_NL check with a pack.
+    add_executable(hip_prefill_mmq_parity tests/hip/prefill_mmq_parity.cpp)
+    target_include_directories(hip_prefill_mmq_parity PRIVATE "${_strata_gg}/include")
+    target_link_libraries(hip_prefill_mmq_parity PRIVATE strata_prefill)
+    add_test(NAME hip_prefill_mmq_parity COMMAND hip_prefill_mmq_parity)
+    set_tests_properties(hip_prefill_mmq_parity PROPERTIES TIMEOUT 120)
+    list(APPEND _strata_hip_tests hip_prefill_mmq_parity)
   endif()
-  if(STRATA_ENABLE_CUDA)
+  if(STRATA_ENABLE_HIP)
+    # Optional real-model diagnostic: pass the GGUF shard containing the native PLE key.
+    add_executable(hip_ple_iq4 tests/hip/ple_iq4.cpp)
+    target_link_libraries(hip_ple_iq4 PRIVATE strata_kernels strata_hip_runtime ggml-base)
+  endif()
+  if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     add_executable(native_expert_parity src/kernels/native_expert_parity.cpp)
-    target_include_directories(native_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-    target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact CUDA::cudart)
+    target_include_directories(native_expert_parity PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact ${_strata_gpu_runtime_target})
   endif()
 endif()
 
 if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS)
   add_executable(hit_cpu_order_parity bench/micro/hit_cpu_order_parity.cu)
-  target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu CUDA::cudart)
+  target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
 endif()
 
 if(STRATA_BUILD_TESTS)
   # Plan v0.3 P6: one expert applied to several tokens, bitwise vs the single-token kernel (synthetic, CPU only).
   # plan v0.3 P5: the device dequantizers vs the artifact's CPU dequantizers on the real shards (not a ctest: needs
   # the 38 GB model; run it by hand or from tools/gpu_session.py)
-  add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp)
-  target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact CUDA::cudart)
+  if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)   # #93: it needs the device dequantizers and a GPU runtime
+    add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp)
+    target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact ${_strata_gpu_runtime_target})
+  endif()
 
   add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp)
   target_link_libraries(expert_multi_test PRIVATE strata_kernels_cpu)
@@ -592,12 +758,19 @@ if(STRATA_BUILD_TESTS)
   add_executable(pool_stress src/kernels/cpu/pool_stress.cpp)
   target_link_libraries(pool_stress PRIVATE strata_kernels_cpu)
   add_test(NAME pool_stress COMMAND pool_stress 20)
+
+  if(UNIX AND NOT APPLE)
+    add_executable(direct_file_async_test tests/platform/direct_file_async.cpp src/platform/direct_file.cpp)
+    target_include_directories(direct_file_async_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    target_link_libraries(direct_file_async_test PRIVATE Threads::Threads)
+    add_test(NAME direct_file_async_test COMMAND direct_file_async_test)
+  endif()
 endif()
 
 if(STRATA_BUILD_TESTS)
   enable_testing()
-  # the unit tests above build from src/; the tests/ tree (model-level checks) is not part of the published repo
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt)
+  # Model-level checks may be absent from source distributions.
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt")
     add_subdirectory(tests)
   endif()
 endif()

README.md

@@ -25,15 +25,15 @@ Measured on an RTX 5070 (12 GB), a Ryzen 5 7600 and 64 GB of RAM:
 
 | Size | Writes answers (short chat) | Writes answers (128K context) | Reads your prompt |
 | --- | ---: | ---: | ---: |
-| **Q2_0** | 90 tokens/s | 67 tokens/s | 1,310 tokens/s |
-| **IQ2_XS** | 74 tokens/s | 60 tokens/s | 1,240 tokens/s |
-| **IQ3_XXS** | 62 tokens/s | 46 tokens/s | 1,110 tokens/s |
-| **IQ3_S** | 52 tokens/s | 41 tokens/s | 1,070 tokens/s |
-| **Coder** (IQ1_M) | 51 tokens/s | 44 tokens/s | 1,300 tokens/s |
+| **Q2_0** | 93 tokens/s | 74 tokens/s | 2,170 tokens/s |
+| **IQ2_XS** | 79 tokens/s | 63 tokens/s | 2,090 tokens/s |
+| **IQ3_XXS** | 62 tokens/s | 49 tokens/s | 1,750 tokens/s |
+| **IQ3_S** | 53 tokens/s | 46 tokens/s | 1,620 tokens/s |
+| **Coder** (IQ1_M) | 55 tokens/s | 43 tokens/s | 2,180 tokens/s |
 
 - **Writes answers** = how fast the reply appears (tokens per second).
 - **Reads your prompt** = how fast it takes in what you send (long documents, code, chat history), measured on a
-  32K-token prompt; a 4K prompt reads at 740-1,000 tokens/s. A 32K prompt takes about 25 seconds with Q2_0.
+  32K-token prompt; a 4K prompt reads at 910-1,580 tokens/s. A 32K prompt takes about 15 seconds with Q2_0.
 
 A card with more VRAM is faster, because more of the model fits on the GPU: an RTX 3090 (24 GB) should do roughly
 100-140 tokens per second. All measurements, long-context numbers and estimates for other cards are in the
@@ -42,9 +42,12 @@ A card with more VRAM is faster, because more of the model fits on the GPU: an R
 Every PC is different: `START-HERE.bat --calibrate` measures a few engine settings on yours and keeps the fastest
 (about 5-10 minutes; on the PC above it made the Coder 7% faster).
 
-**Two or three NVIDIA cards?** `START-HERE.bat --setup --gpus 0,2` splits the model's layers across them (experimental):
-each card keeps the experts of its own layers, and prompts flow through the cards in a pipeline. On an RTX 5080 +
-RTX 3090 it read prompts 18-20% faster than the 5080 alone, with decoding on par. See [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
+**Two or three NVIDIA cards?** Just run `START-HERE.bat`: it lists your cards, says which ones Strata can use, and
+asks whether to share the model across them (recommended when two can). An install made on one card asks once at
+its next start. Or choose yourself: `START-HERE.bat --gpus 0,2` (both, remembered), `--gpus all`, or `--gpu 0` (one
+card, this start only). Each card keeps the experts of its own layers, and prompts flow through the cards in a
+pipeline: on an RTX 5080 + RTX 3090 prompts were read 18-20% faster than on the 5080 alone, decoding on par.
+Every card must be an RTX 20 series or newer with 8 GB or more. See [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
 
 ## Which model should I pick?
 
@@ -81,9 +84,14 @@ one later with `SETUP.bat` (the same as `START-HERE.bat --setup`; on Linux `./se
 For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](docs/ORCA.md).
 It needs an explicit packing conversion and is not an installer menu option.
 
+An **AMD Radeon RX 7900 XT / XTX on Linux** works too (experimental): `./setup.sh --backend hip`, chosen by itself on
+a PC with no NVIDIA card Strata can use. It installs ROCm without sudo and compiles the engine (one GPU, no images
+yet). Details: [AMD HIP](docs/AMD_HIP.md).
+
 ## Install
 
-**You need:** an NVIDIA RTX 30, 40 or 50 card with 12 GB of VRAM or more, enough RAM for the size you pick (above),
+**You need:** an NVIDIA RTX 20, 30, 40 or 50 card with 12 GB of VRAM or more (RTX 20 since 0.1.27), enough RAM for the size you pick (above;
+a big GPU makes up for less RAM - the [low-RAM mode](docs/DETAILS.md)),
 ~80 GB of free disk space (an SSD makes the first start much faster), and Windows 10/11 or Linux. The only thing you
 install yourself is a current **NVIDIA driver** ([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA
 App). Everything else - Python, the engine, the model - is set up for you.

bench/results/2026-09-29-layer-split-limits/README.md

@@ -0,0 +1,59 @@
+# Layer split: the features that were off across GPUs (2026-09-29)
+
+**Setup:**
+- **Rig:** RIBPC. RTX 5080 16 GB (CUDA0) + RTX 3090 24 GB; RTX 2080 Ti 22 GB as a helper card. Ryzen 9 9950X3D,
+  62 GB RAM, Windows 11.
+- **Model:** the Coder (IQ1_M native pack).
+- **Build:** branch `mg/limits`, sm_75 + sm_86 + sm_120.
+- **Correctness runs:** `rib_norepeat.py`, with a fixed cache, no swaps, no PCIe share and no suffix drafts where two
+  runs are compared byte for byte.
+- **Speed runs:** `rib_pp.py`.
+- Raw results are in `data/`.
+
+## Correctness
+
+| Check | Result |
+|---|---|
+| One GPU, this branch vs the 0.1.21-equivalent run (fixed cache) | 10/10 byte-identical |
+| Speed projection (control vector), one GPU vs the split on the same GPU (K=24) | 10/10 byte-identical |
+| KV streaming (128K context, 32K resident), one GPU vs the split on the same GPU | 10/10 byte-identical |
+| Images, one GPU vs the split on the same GPU (three questions about a test picture) | identical answers |
+| Speed projection / KV streaming / plain, 5080 + 3090 (K=24): no-repeat under presence 100, the 144 sanity case | all pass, 0 repeats |
+| The vector acts across GPUs: 5080 + 3090 with vs without the vector | 8 of 10 answers differ (as on one GPU: 7 of 10) |
+| Images, 5080 + 3090 (K=24): "What color is the circle?" / "What number?" / describe | "Red" / "42" / "A red circle, the number 42, and a blue square are displayed on a white background" |
+| Mid-prompt checkpoint (every 16,384 tokens): a 19.5K prompt, then another sharing its first 18,600 tokens | one GPU and 5080 + 3090: reused 16,384, answer right |
+| 5080 + 3090 (K=30) + a 2080 Ti helper holding 3,000 experts | all pass |
+| `--mmap-experts` with the Coder (a native pack) | a clear start-up error: it needs a canonical pack's `experts.bin` (with or without a split) |
+
+Across GPUs the text is not byte-identical to one GPU: experts that one card leaves to the CPU run on a GPU in the
+split, and the two round differently.
+
+## Speed
+
+| Run | Prompt 16K / 28K / 60K tok/s | Decode story / code tok/s | Hits |
+|---|---|---|---|
+| KV streaming (128K), 5080 alone | 1,487 / 1,801 / 1,837 | 66.6 / 74.0 | 59-71% |
+| KV streaming (128K), 5080 + 3090 auto | 1,685 / 1,957 / 1,968 | 75.8 / 97.0 | 96-97% |
+| Speed projection, 5080 + 3090 auto | 1,813 / 2,053 / - | 77.0 / 103.6 | 96-97% |
+| 5080 + 3090 (K=30), no helper | - | 62.5 / 85.2 | |
+| 5080 + 3090 (K=30) + 2080 Ti helper | - | 43.0 / 63.7 | |
+
+**The helper card makes it slower,** as it did without a split (bench/results/2026-09-29-layer-split). Its per-layer
+round trip costs more than the CPU pool needs for the few experts it would take over. It stays supported: the
+helpers take the visible GPUs no stage runs on and hold only experts no stage's cache holds. But it isn't
+recommended.
+
+## What changed
+
+- **Per-device tables:** the image-position (mrope) table and the control vector's tables are now kept per device;
+  every stage's rope and steering kernels read their own device's copy. `cvec_set_enabled` switches the vector on
+  every device.
+- **KV streaming:** each stage's session streams its own KV host copy (pinned, portable). The split no longer forces
+  it off.
+- **Mid-prompt checkpoints:** `Prefill::on_stage_chunk` lets each stage save its part of a checkpoint when it has
+  read the chunk. The last stage assembles the parts: the earlier ones are a chunk ahead by then.
+- **Helpers (`--expert-cache-remote`) with a split:** they take the visible devices after the stages, skip every
+  pair a stage's cache holds, and preflight and open on those devices.
+- **Explicit `--expert-cache` with a split:** it is capped so CUDA0 keeps room for the prompt path's own buffers.
+  Without the cap, the first prompt failed with "device buffers ... do not fit".
+- **`--mmap-experts`:** refused at start for a native pack, with the reason.

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-cv-same24-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 102,
+  "drafts_offered": 166,
+  "tok_s": 66.7,
+  "text": "The wind didn\u2019t just howl; it screamed. It tore through the iron gallery of the Blackwood Point Light, rattling the glass panes like a prisoner testing the bars of a cell.\n\nElias Thorne was not a man who feared the dark. He was a man who feared the silence.\n\nFor forty years, he had lived in the shadow of the Fresnel lens, a rotating eye of glass and brass that swept the Atlantic with a beam of pure, unyielding light. His job was simple: keep the light burning. Keep the logbook current. Keep the foghorn sounding when the mist rolled in, thick as wool.\n\nBut Elias knew something else. He knew that the light was not just for ships. It was for the lost.\n\nIt was November 14th, 1982. The storm was a Category 5, a monster that had swallowed the horizon. The waves were mountains of black water, crashing against the granite"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 134,
+  "drafts_offered": 180,
+  "tok_s": 80.8,
+  "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 72.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.6,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 104,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 92,
+  "tok_s": 45.4,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAnaconda    \nAntelope     \nBaboon       \nBadger      \nBat         \nBear        \nBeaver          \nBirds of Paradise (Manucode) \nBoa Constrictor \n\nBuffalo           \nCamel            \nCapybara             \nCat              \nChameleon\nCheetah\n\nChicken\n\n\nCrocodile  \nDeer  \n\nDingo   \nDog    \nDonkey     \nDragonfly   \n\nElephant       \nEmu      \nFalcon        \nFerret          \nFlamingo         \nFox<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 72,
+  "repeats": 0,
+  "drafts_accepted": 15,
+  "drafts_offered": 70,
+  "tok_s": 46.1,
+  "text": "The wind didn\u2019t just howl; it screamed against the glass of Lantern House 42, demanding entry into every crack and crevice that had been sealed with tar in previous years but now wept like old wounds on an open sore skin\u2014like a wound left unhealed by time. Elias knew this rhythm better than he remembered his own name<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 76,
+  "tok_s": 49.3,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in my teeth, searching for something to break its rhythm against stone that refused be broken by anything less than God himself had forged into being willing and able withstand eternity\u2019s cruelty without complaint or mercy from start finish until now forevermore if any such concept even existed within human comprehension let alone divine authority which we all knew did not exist but wished it would because wishful thinking is always easier to imagine hopelessly impossible truths as probable lies than vice versa.\n\nElias Thorne was eighty-three years old today, exactly one hundred and forty-seven days since his last heart attack had been logged by Dr Armitage who himself retired seven months ago after receiving a subpoena from the county clerk indicating he might have overstayed professional courtesy boundaries"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 27,
+  "drafts_offered": 56,
+  "tok_s": 46.7,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippo         \nRhino          \nCheetah            \nLeopard           \nJaguar             \nWolf                \nFox              \nBear               \nDeer                 \nElk                  \nMoose                    \nBuffalo                        \nCamel                       \nDonkey                      \nGoat                     \nSheep                   \nCow                         \nBull                          \nOx                           \nPig                            \nBoar                             \nRabbit                              \nSquirrel                                <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.4,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 148,
+  "tok_s": 56.4,
+  "text": "The phrase \"the History Of The Sea\" is inherently ambiguous, as it could refer to several distinct historical narratives: geological formation (how oceans came into existence), maritime navigation and exploration by humans throughout human civilization; or perhaps more specifically in modern times refers often used metaphorically for economic trends like stock market fluctuations (\"sea change\"). However given your request likely intends an academic-style comprehensive overview covering both natural science aspects AND cultural significance through time periods\u2014especially since this kind question appears frequently on exams asking students write essays discussing evolution from primordial origins up present-day impacts including environmental changes due climate crisis etc.\u2014I will craft such broad yet structured piece combining geology + anthropology/ecological impact together while maintaining coherent flow suitable high-level educational context without overly technical jargon but still accurate enough scientifically grounded facts where appropriate within reasonable scope limits typical school assignments expect (~80\u201312% word count range implied implicitly though not stated explicitly here we aim moderate length ~65 words? Wait no user said \u201clong\u201d so probably >3k+ characters minimum safe bet around 7-9 paragraphs total approximating maybe close-to-full-page standard double-spaced equivalent which translates roughly \u22654K chars safely acceptable). Let me proceed accordingly ensuring clarity structure intro body conclusion style formal tone neutral informative balanced perspective acknowledging uncertainties if any minor ones exist e.g exact dates vary depending sources hence use hedging language sparingly only when necessary avoid overclaiming certainty especially regarding speculative theories vs established consensus knowledge base current mainstream understanding circa early-mid"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-cv-single-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 102,
+  "drafts_offered": 166,
+  "tok_s": 67.1,
+  "text": "The wind didn\u2019t just howl; it screamed. It tore through the iron gallery of the Blackwood Point Light, rattling the glass panes like a prisoner testing the bars of a cell.\n\nElias Thorne was not a man who feared the dark. He was a man who feared the silence.\n\nFor forty years, he had lived in the shadow of the Fresnel lens, a rotating eye of glass and brass that swept the Atlantic with a beam of pure, unyielding light. His job was simple: keep the light burning. Keep the logbook current. Keep the foghorn sounding when the mist rolled in, thick as wool.\n\nBut Elias knew something else. He knew that the light was not just for ships. It was for the lost.\n\nIt was November 14th, 1982. The storm was a Category 5, a monster that had swallowed the horizon. The waves were mountains of black water, crashing against the granite"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 134,
+  "drafts_offered": 180,
+  "tok_s": 81.0,
+  "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 62.4,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.4,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 104,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 92,
+  "tok_s": 43.5,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAnaconda    \nAntelope     \nBaboon       \nBadger      \nBat         \nBear        \nBeaver          \nBirds of Paradise (Manucode) \nBoa Constrictor \n\nBuffalo           \nCamel            \nCapybara             \nCat              \nChameleon\nCheetah\n\nChicken\n\n\nCrocodile  \nDeer  \n\nDingo   \nDog    \nDonkey     \nDragonfly   \n\nElephant       \nEmu      \nFalcon        \nFerret          \nFlamingo         \nFox<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 72,
+  "repeats": 0,
+  "drafts_accepted": 15,
+  "drafts_offered": 70,
+  "tok_s": 46.1,
+  "text": "The wind didn\u2019t just howl; it screamed against the glass of Lantern House 42, demanding entry into every crack and crevice that had been sealed with tar in previous years but now wept like old wounds on an open sore skin\u2014like a wound left unhealed by time. Elias knew this rhythm better than he remembered his own name<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 76,
+  "tok_s": 48.9,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in my teeth, searching for something to break its rhythm against stone that refused be broken by anything less than God himself had forged into being willing and able withstand eternity\u2019s cruelty without complaint or mercy from start finish until now forevermore if any such concept even existed within human comprehension let alone divine authority which we all knew did not exist but wished it would because wishful thinking is always easier to imagine hopelessly impossible truths as probable lies than vice versa.\n\nElias Thorne was eighty-three years old today, exactly one hundred and forty-seven days since his last heart attack had been logged by Dr Armitage who himself retired seven months ago after receiving a subpoena from the county clerk indicating he might have overstayed professional courtesy boundaries"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 27,
+  "drafts_offered": 56,
+  "tok_s": 46.8,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippo         \nRhino          \nCheetah            \nLeopard           \nJaguar             \nWolf                \nFox              \nBear               \nDeer                 \nElk                  \nMoose                    \nBuffalo                        \nCamel                       \nDonkey                      \nGoat                     \nSheep                   \nCow                         \nBull                          \nOx                           \nPig                            \nBoar                             \nRabbit                              \nSquirrel                                <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 63.1,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 148,
+  "tok_s": 56.6,
+  "text": "The phrase \"the History Of The Sea\" is inherently ambiguous, as it could refer to several distinct historical narratives: geological formation (how oceans came into existence), maritime navigation and exploration by humans throughout human civilization; or perhaps more specifically in modern times refers often used metaphorically for economic trends like stock market fluctuations (\"sea change\"). However given your request likely intends an academic-style comprehensive overview covering both natural science aspects AND cultural significance through time periods\u2014especially since this kind question appears frequently on exams asking students write essays discussing evolution from primordial origins up present-day impacts including environmental changes due climate crisis etc.\u2014I will craft such broad yet structured piece combining geology + anthropology/ecological impact together while maintaining coherent flow suitable high-level educational context without overly technical jargon but still accurate enough scientifically grounded facts where appropriate within reasonable scope limits typical school assignments expect (~80\u201312% word count range implied implicitly though not stated explicitly here we aim moderate length ~65 words? Wait no user said \u201clong\u201d so probably >3k+ characters minimum safe bet around 7-9 paragraphs total approximating maybe close-to-full-page standard double-spaced equivalent which translates roughly \u22654K chars safely acceptable). Let me proceed accordingly ensuring clarity structure intro body conclusion style formal tone neutral informative balanced perspective acknowledging uncertainties if any minor ones exist e.g exact dates vary depending sources hence use hedging language sparingly only when necessary avoid overclaiming certainty especially regarding speculative theories vs established consensus knowledge base current mainstream understanding circa early-mid"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-cv-x2-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 85,
+  "drafts_offered": 172,
+  "tok_s": 36.6,
+  "text": "The wind didn\u2019t just howl; it screamed. It tore through the gaps in the iron railing of the Lantern Room, carrying the scent of brine and rotting kelp. Elias pulled his woolen collar up to his chin, his fingers numb and clumsy inside thick leather gloves.\n\nHe was the last man on the island. The automated systems had failed three winters ago, a glitch in the solar-powered backup that no engineer had been able to fix before the supply boat stopped coming. Now, it was just him, the light, and the sea.\n\nElias climbed the spiral stairs, his boots heavy with salt spray. The iron steps groaned under his weight, a sound that seemed to mock the storm outside. He reached the top, the glass walls vibrating with the force of the gale. Outside, the world was a blur of black and white. The ocean was invisible; it was only a presence, a weight pressing against the stone foundation of the tower.\n\nHe"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 173,
+  "tok_s": 96.8,
+  "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 88.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 94.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 104,
+  "repeats": 0,
+  "drafts_accepted": 30,
+  "drafts_offered": 96,
+  "tok_s": 55.9,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAnaconda    \nAntelope     \nBaboon       \nBadger      \nBat         \nBear        \nBeaver          \nBirds of Paradise (Manucode) \nBoa Constrictor \n\nBuffalo           \nCamel            \nCapybara             \nCat              \nChameleon\nCheetah\n\nChicken\n\n\nCrocodile  \nDeer  \n\nDingo   \nDog    \nDonkey     \nDragonfly   \n\nElephant       \nEmu      \nFalcon        \nFerret          \nFlamingo         \nFox<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 114,
+  "tok_s": 55.4,
+  "text": "The wind didn\u2019t just howl; it screamed against the glass of Lantern House 42, demanding entry into every crack and crevice that had been sealed with tar in previous years but now wept like old wounds at this one\u2019s age: forty-three winters since its construction. My name was Elias Thorne\u2014no middle initial\u2014and I kept no logbook for visitors because there were none; only ships passed by, their hulls cutting through the black water as if they too feared what lay beneath it more than above or beside them in fog so thick that even my own reflection vanished from sight when looked upon directly.\n\nI climbed each morning not to greet dawn but check whether light still lived within me\u2014the bulb\u2019s heart had failed twice before, once during a storm named *Harrow*,"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 110,
+  "tok_s": 52.2,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in my teeth, searching for something to hold onto until I died.\u201d said Elias Thorne as he adjusted his collar against November\u2019s bite.** He was not merely complaining of cold weather but articulating a poetic metaphor: that even after death (metaphorically), there would be no relief from this relentless gale.\n\nElias had been the last human keeper on any significant stretch coast since automation became standard practice, yet here he stood , stubbornly refusing to let technology replace his vigilance . His cottage perched atop a crumbling granite foundation was more than just shelter; it housed centuries of handwritten logs detailing every ship\u2019s name and cargo load that passed within sight.\n\nOn most nights Elias tended the light mechanically\u2014"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 91,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 69,
+  "tok_s": 58.7,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippopotamus          \nRhinoceros            \nLeopard                \nCheetah                 \nPanther                  \nWolf                    \nFox                      \nBear                     \nDeer                       \nElk                        \nMoose               \nBuffalo                   \nCamel           \nDonkey              \nGoat             \nSheep         \nCow  \nBull   \nOx    \nPig     \nDog       \nCat<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 12,
+  "repeats": 0,
+  "drafts_accepted": 4,
+  "drafts_offered": 17,
+  "tok_s": 60.3,
+  "text": "1 2,3456789<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 41,
+  "drafts_offered": 168,
+  "tok_s": 59.9,
+  "text": "The phrase \"the History Of The Sea\" is inherently ambiguous, as it could refer to several distinct historical narratives: geological formation (how oceans came into existence), maritime navigation and exploration histories (\"history at/over/across/on/in/of/to/from/toward/off by means around above below through under upon alongside against among between within without throughout despite during after before since until while when where who whom whose which what how why whether\"), or perhaps even metaphorical interpretations involving human interaction with water bodies over time periods like ages eras epochs generations centuries decades years months days hours minutes seconds moments instants times occasions instances events episodes incidents occurrences situations circumstances conditions states phases stages steps processes procedures sequences series chains cycles loops repetitions iterations continuations progressions developments evolutions transformations changes alterations modifications adjustments corrections revisions amendments updates upgrades improvements enhancements refinements optimizations perfecting completing finishing concluding ending terminating ceasing stopping halting pausing resuming continuing proceeding advancing progressing moving shifting translating rotating reflecting mirroring flipping turning twisting bending curving arches spirals helix coils twists turns rotations reflections mirrors flips inversions reversings reversion backtracking retracing reversing undo redo do again repeat repeats repeated repeatedly repetition repetitive repetitiveness repete repeater repeatingly recurrent recurrence recurring recurs recur recurred recursion recursive recursively recurse recessed recessionary regress regression regressed retrograde reverse reversed reversal revert reverted reverberate resonated echo echoed echoing echoes echolocation sonar radar radio telecommunication communication communications communicate communicated communicating communicative comms comm chatter chat chats chatting chatted talk talks talked"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-hp0-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 171,
+  "tok_s": 62.5,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from broken glass and rusted iron, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, ensuring the rotation was smooth, silent, and precise. *Tick. Tick. Tick.* The sound was the heartbeat of the coast."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 184,
+  "tok_s": 85.2,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 78.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 80.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 13,
+  "drafts_offered": 48,
+  "tok_s": 46.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBuffalo     \n\nCat          \nCheetah      \n\nChimpanzee \nClam         \n\n\nCrocodile   \n\n\n\nDolphin      \n\n\n\n\nDonkey       <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 30,
+  "drafts_offered": 101,
+  "tok_s": 51.3,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots smelled faintly brine and old coffee as he climbed spiral stairs worn smooth by decades feet his hands were calloused from gripping iron railings that had been polished to a dull shine\n\nElias was not merely keeping light alive; He kept order intact. Every night, at precisely 18:07 hours local time\u2014adjusted for leap seconds and magnetic drift\u2014he wound the clockwork mechanism inside brass housing behind Fresnel lens This wasn\u2019t vanity It\u2019s necessity The rotation speed must remain constant or ships lose their bearing in fog where compasses fail\n\nHe spoke softly to himself while working, murmuring numbers aloud because silence made him"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 67,
+  "repeats": 0,
+  "drafts_accepted": 14,
+  "drafts_offered": 68,
+  "tok_s": 41.2,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, like he had no mouth to hold back its fury anymore that night.\u201d Elara pulled her wool coat tighter around herself as she climbed spiral staircase of **Blackrock Lighthouse**. Her boots were slick with salt spray from previous days.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 91,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 54,
+  "tok_s": 48.8,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nHippopotamus          \nRhinoceros            \nCamel                 \nCheetah               \nLeopard                \nJaguar                  \nBear                    \nWolf                     \nFox                      \nDog                       \nCat                        \nMouse                   \nRat                         \nSquirrel              \nDeer                      \n\nAntelope                \n\nBuffalo                 \n\n\nBison                   \n\n\n\nElk                    \n\n\n\n\nMoose                  <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 11,
+  "tok_s": 68.8,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 32,
+  "drafts_offered": 107,
+  "tok_s": 52.3,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant storybook that weaves together geology with biology as wellas human culture into an epic narrative spanning billions years from its earliest beginnings in primordial times through modern-day ecological crises The vast expanse covers more than seventy percent Earth\u2019s surface area making up nearly all life support systems on our planet yet despite being so large humans have only explored less five thousand square kilometers out total 361 million km\u00b2 which means over ninety-nine point nine eight six per cent unexplored territory This makes oceans arguably larger unknown territories compared space exploration where NASA estimates they\u2019ve mapped roughly ten billion stars versus current marine surveys covering just two hundred fifty-six species documented globally against estimated millions undiscovered organisms including deep-sea creatures like anglerfish or vampire squid whose existence was first recorded centuries ago but still remain poorly understood due extreme pressures depths exceeding eleven miles below Pacific Trenches Mariana located near Guam island chain reaching maximum depth known today at approximately thirty-seven feet under pressure equivalent atmospheric levels causing structural collapse if unprotected by specialized vessels capable handling such conditions without failure mechanisms designed specifically for submersible operations operating within these zones require titanium hulls rated withstand forces greater gravity acting upon objects falling vertically downward acceleration rates calculated using Newtonian physics principles applied universally across gravitational fields regardless medium density variations present throughout different layers stratosphere troposphere mesospheres thermoclines etceterally speaking however when discussing aquatic environments instead air masses replace gases dissolved oxygen nitrogen arg"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-hp1-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 94,
+  "drafts_offered": 169,
+  "tok_s": 43.0,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy-three years old, and he had been the keeper of the Blackwood Point Light for forty of those years. The other thirty-three had been spent in a small, drafty cottage that smelled perpetually of kerosene and damp wood.\n\nFor decades, Elias had tended the great Fresnel lens, polishing the glass prisms until they gleamed like diamonds, winding the clockwork mechanism that turned the beam, and trimming the wick of the oil lamp. It was a job of patience, of precision, of silence. The light was his charge. The sea was his neighbor.\n\nBut this morning, the sea was not his neighbor. It was his enemy.\n\nThe storm had been forecasted for three days, but it had arrived in one. The barometric pressure had dropped so sharply that Elias\u2019s ears popped."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 178,
+  "tok_s": 63.7,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 56.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 57.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 24,
+  "drafts_offered": 49,
+  "tok_s": 43.3,
+  "text": "Lion  \nTiger   \nElephant    \nGiraffe     \nZebra       \nHippopotamus \nRhinoceros      \nCheetah         \nLeopard        \nJaguar          \nPuma           \nWolf            \nFox             \nBear              \nDeer                \nElk                 \nMoose               \nAntelope                  \nBison                   \nBuffalo                    \nCamel                     \nDonkey                      \nGoat                       \nSheep                        \nCow                         \nDog                          \nCat                           \nRabbit                            \nMouse                             \nHamster                               \nGuinea Pig<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 35,
+  "drafts_offered": 108,
+  "tok_s": 41.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots smelled faintly brine and old coffee as he climbed spiral stairs worn smooth by decades feet his hands were calloused from gripping iron railings that had been polished to a dull shine.\n\nElias wasn\u2019t just keeping light; was guarding silence the beam swept across black water every seven seconds cutting through fog so thick it swallowed sound entirely for three nights straight no ships answered its signal not even fishing trawlers seeking shelter they\u2019d long abandoned this stretch of coast where currents shifted unpredictably pulling debris from sunken wrecks into tangled heaps that snagged anchors and tore hulls open like wounds.\n\nHe remembered the last time someone had"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 46,
+  "tok_s": 33.7,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, trying to pry them apart like loose planks of weathered oak outside your own mouth again and then you are going away from home is fine but for now let me write a proper story.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 60,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 48,
+  "tok_s": 43.2,
+  "text": "Aardvark  \nAntelope allonax (Oryx) \u2013 *Note: Using standard names for clarity*  \n\nLet me provide a clean list of exactly **1** animal name on each single physical text file/terminal output stream with no other characters except the newline (\\n).<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 11,
+  "tok_s": 48.9,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 27,
+  "drafts_offered": 105,
+  "tok_s": 40.2,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element\u2014it became civilization itself through its capacity as both barrier (to land) but also conduit between lands\u2014and thus gave rise directly from being part nature\u2019s own evolutionary engine into becoming humanity's greatest historical actor across all time scales since first emergence until now present-day future projections still ongoing today within our current era context window limits allowed here by system constraints set above threshold levels required before output generation begins below line break marker inserted immediately after final punctuation mark ending entire preceding paragraph block structure defined earlier during initial drafting phase prior execution stage completed successfully without errors detected internally via internal validation routines executed automatically upon completion cycle termination signal received end-of-file condition triggered correctly according standard protocol specifications followed strictly adhered-to throughout process flow control logic implemented using structured programming constructs compatible with target runtime environment configuration parameters specified herein explicitly declared beforehand at top-level scope declaration section located near beginning source code file header comments included inline documentation style guidelines observed consistently applied uniformly everywhere else similarly formatted sections appearing later down document body text area scrolling vertically downward directionally aligned left margin justified right edge wrapped around fixed-width column boundaries determined dynamically based available screen real estate measured pixel-per-unit scaling factor adjusted responsively depending viewport dimensions reported back client-side rendering pipeline components responsible"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-kv-same24-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 94,
+  "drafts_offered": 167,
+  "tok_s": 62.7,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to hold the chaos of the ocean at bay. The beam swept out, a blade of pure, concentrated light cutting through the rain and sleet, illuminating the jagged teeth of the reef below.\n\n"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 179,
+  "tok_s": 81.3,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 71.0,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 71.9,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 52,
+  "tok_s": 47.9,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFalcon          <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 122,
+  "tok_s": 47.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach his post at dawn every morning for thirty years. His name was written on a brass plaque by door: **Elias Thorne**.\n\nHe wasn\u2019t just keeping light; He kept time, weathered hands gripping the cold iron railing until knuckles turned white against frostbite and windburn that never healed fully because they were always there waiting to be felt again when he stepped outside into storm or calm alike. The lens above him was a Fresnel giant\u2019s eye made of hand-ground crystal prisms each one polished daily with chamois cloth soaked in whale"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 65,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 58,
+  "tok_s": 42.6,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they slammed against their hinges every night for eleven months of winter and summer alike.\u201d I had never seen calm water here\u2014only gray churning seas that swallowed ships smaller than themselves<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 101,
+  "repeats": 0,
+  "drafts_accepted": 32,
+  "drafts_offered": 53,
+  "tok_s": 49.4,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nCheetah         \nHippopotamus          \nRhinoceros            \nLeopard             \nJaguar              \nWolf                \nFox                 \nBear                  \nDeer                    \nOtter                   \nBeaver                      \nBadger                        \nMongoose                          \nMeerkat                           \nHyena                             \nJackal                            \nAntelope                              \nBison                               \nElk                                 \nCaribou                                   \nLlama                                    \nAlpace                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 12,
+  "tok_s": 64.7,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 52,
+  "drafts_offered": 148,
+  "tok_s": 49.9,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by hundreds millions years yet continues shaping every aspect from climate regulation through evolutionary biology down even into modern industrial logistics today's global supply chains depend upon them directly indirectly via atmospheric pressure systems weather patterns migration routes food webs carbon cycles oxygen generation thermal buffering tidal forces salinity gradients depth stratification light attenuation currents circulation gyres upwelling zones thermoclines halothermics bathymetric features continental shelves abyssal plains trenching ridges volcanic arcs island formations atoll structures seamounts submarine canons canyon walls cliff faces overhang ledge platforms reef flats fore-reef back-lagoon inner-shelf outer-slope drop-offs pinnacle pinnacles spires columns arches tunnels caves grottos swim-through passages chimney vents blowholes fissures cracks crevices joints faults fractures breaks gaps holes openings void spaces hollow chambers cavernous interiors subterranean networks underground rivers lakes pools springs wells shaft pits sinks drains sewers pipes conduits tubes channels gutters ditches furrows grooves rills runnels streams brooks rivulets creeks tributaries branches forks splits divergences con"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-kv-single-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 94,
+  "drafts_offered": 167,
+  "tok_s": 49.2,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to hold the chaos of the ocean at bay. The beam swept out, a blade of pure, concentrated light cutting through the rain and sleet, illuminating the jagged teeth of the reef below.\n\n"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 179,
+  "tok_s": 76.2,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.4,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 52,
+  "tok_s": 47.1,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFalcon          <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 122,
+  "tok_s": 47.0,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach his post at dawn every morning for thirty years. His name was written on a brass plaque by door: **Elias Thorne**.\n\nHe wasn\u2019t just keeping light; He kept time, weathered hands gripping the cold iron railing until knuckles turned white against frostbite and windburn that never healed fully because they were always there waiting to be felt again when he stepped outside into storm or calm alike. The lens above him was a Fresnel giant\u2019s eye made of hand-ground crystal prisms each one polished daily with chamois cloth soaked in whale"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 65,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 58,
+  "tok_s": 42.3,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they slammed against their hinges every night for eleven months of winter and summer alike.\u201d I had never seen calm water here\u2014only gray churning seas that swallowed ships smaller than themselves<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 101,
+  "repeats": 0,
+  "drafts_accepted": 32,
+  "drafts_offered": 53,
+  "tok_s": 50.0,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nCheetah         \nHippopotamus          \nRhinoceros            \nLeopard             \nJaguar              \nWolf                \nFox                 \nBear                  \nDeer                    \nOtter                   \nBeaver                      \nBadger                        \nMongoose                          \nMeerkat                           \nHyena                             \nJackal                            \nAntelope                              \nBison                               \nElk                                 \nCaribou                                   \nLlama                                    \nAlpace                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 12,
+  "tok_s": 64.6,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 52,
+  "drafts_offered": 148,
+  "tok_s": 49.8,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by hundreds millions years yet continues shaping every aspect from climate regulation through evolutionary biology down even into modern industrial logistics today's global supply chains depend upon them directly indirectly via atmospheric pressure systems weather patterns migration routes food webs carbon cycles oxygen generation thermal buffering tidal forces salinity gradients depth stratification light attenuation currents circulation gyres upwelling zones thermoclines halothermics bathymetric features continental shelves abyssal plains trenching ridges volcanic arcs island formations atoll structures seamounts submarine canons canyon walls cliff faces overhang ledge platforms reef flats fore-reef back-lagoon inner-shelf outer-slope drop-offs pinnacle pinnacles spires columns arches tunnels caves grottos swim-through passages chimney vents blowholes fissures cracks crevices joints faults fractures breaks gaps holes openings void spaces hollow chambers cavernous interiors subterranean networks underground rivers lakes pools springs wells shaft pits sinks drains sewers pipes conduits tubes channels gutters ditches furrows grooves rills runnels streams brooks rivulets creeks tributaries branches forks splits divergences con"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-kv-x2-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 97,
+  "drafts_offered": 176,
+  "tok_s": 73.0,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was the last of the line, the only man left on the jagged spine of Widow\u2019s Rock. For forty years, he had tended the light. For forty years, he had listened to the ocean\u2019s endless, hungry whisper.\n\nTonight was different. The storm wasn\u2019t just a storm; it was a *tempest*. The waves didn\u2019t crash; they *shattered*. They threw themselves against the granite base of the lighthouse with the force of a battering ram, sending tremors up through the soles of Elias\u2019s boots.\n\nHe climbed the spiral stairs, one hundred and twelve steps of iron and rust. His hand, calloused and scarred from decades of polishing the brass railing, found the familiar groove in the metal. He didn\u2019t look up. He didn\u2019t look down. He looked *forward*."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 176,
+  "tok_s": 102.5,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 90.9,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 91.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 118,
+  "repeats": 0,
+  "drafts_accepted": 38,
+  "drafts_offered": 103,
+  "tok_s": 62.8,
+  "text": "Aardvark  \nAlbatross   \nAlligator    \nAntelope     \nBaboon       \nBadger      \nBeaver        \nBear          \nBird of Paradise \nBuffalo         \nCamel           \nCheetah            \nChimpanzee             \nCrane              \nCrocodile                \nDeer                 \nDolphin               \nDonkey                  \nElephant                   \nFlamingo                    \nFox                       \nGiraffe                     \nGoat                      \nHippopotamus  \nIguana       \nJaguar        \nKangaroo      \nKoala          \nLion         \nLeopard     \nMonkey    \nMoose   \nOstrich \nOtter\nPanda<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 58,
+  "repeats": 0,
+  "drafts_accepted": 9,
+  "drafts_offered": 63,
+  "tok_s": 50.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty jail cell below sea level three hundred feet down to where Elias kept his boots dry and ready for emergencies when he had no time even think about changing them.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 140,
+  "repeats": 0,
+  "drafts_accepted": 22,
+  "drafts_offered": 90,
+  "tok_s": 50.1,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, demanding entry into every crack and crevice of Stone Harbor Lighthous eKeepers have always been solitary figures against isolation\u2019s greatest adversary: time itself Elias Thorne knew this intimately. For thirty years he had kept watch over one hundred thousand ships each night scanning for lights that might signal disaster or hope\n\nHis job was simple but deadly serious He checked weather logs every day without fail The foghorn echoed through the tower three times before dawn Each morning at 4:02 AM Elias would climb eighty-two steps to reach his office where coffee cooled beside a single candle flickering in drafts from below<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 96,
+  "repeats": 0,
+  "drafts_accepted": 31,
+  "drafts_offered": 69,
+  "tok_s": 58.9,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippo         \nRhino          \nCheetah            \nLeopard             \nJaguar              \nPanther               \nCougar                \nBear                  \nWolf                    \nFox                     \nDeer                      \nElk                        \nMoose                       \nBison                         \nBuffalo                          \nAntelope                           \nCamel                            \nDonkey                              \nGoat                                \nSheep                                  \nCow                                    \nPig                                     \nHorse                                       <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 71.2,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 119,
+  "tok_s": 61.3,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth\u2019s surface.\u201d \u2014 *This quote by Rachel Carson serves as an apt introduction.* The story we tell today begins with that first drop falling into primordial pools billions years ago\u2014but what follows next? What happens when those drops become oceans themselves\u2014when they begin moving across continents carrying sediment from mountains down rivers until finally reaching their final resting place at last\u2014the deep abyssal plains where sunlight cannot reach nor pressure crushes everything else alive there below depths greater than six kilometers beneath waves crashing against shores above them all around world forever changing shape over time due plate tectonics shifting landmass slowly apart together again repeatedly throughout geological epochs spanning hundreds millions or even thousands million+ yrs since formation early Hadean era through Archeozoic eons onward present Holocene epoch still ongoing now right here under current conditions observed via satellites monitoring global climate patterns affecting currents temperatures salinity levels biodiversity distribution human impacts altering ecosystems directly indirectly causing extinction events locally globally recently historically prehistorically anciently mythologically culturally spiritually philosoph scientifically mathematically physically chem biol geo astro cosmological sociopolitical economic environmental ecological evolutionary genetic linguistic literary artistic musical cinematic theatrical photographic journalistic digital analog virtual real tangible intangible metaphysical ontosemantic semiotic symbolic allegorical narrative descriptive analytical interpretive evaluative critical reflective introspective contemplatory ponderous thoughtful considerate attentive observant perceptive insightful intelligent knowledgeable informed educated literati learned skilled capable"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-nocv-x2-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 87,
+  "drafts_offered": 170,
+  "tok_s": 71.4,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and green.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from rope burns and broken glass, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, listening for the faint, rhythmic *tick-tick-tick* that signaled the rotation was true. If the rhythm faltered, ships"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 178,
+  "tok_s": 104.1,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 90.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 94.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 76,
+  "repeats": 0,
+  "drafts_accepted": 20,
+  "drafts_offered": 64,
+  "tok_s": 61.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear         \nBird        \nBuffalo          \nButterfly \nCamel           \nCat            \nCaterpillar \n\nCentipede    \n\nChameleon   \n\nCheetah     \n\nChicken      \n\nCricket       \n\nCrocodile     \n\n\nDolphin       \n\n\n\nDonkey        \n\n\n\n\nDragon<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 21,
+  "drafts_offered": 90,
+  "tok_s": 59.2,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty jail cell below sea level to above cloud height at once\u2014no matter how high you climbed or deep into its foundations down through rock and saltwater. It was November again when Elias first noticed that his logbook had begun recording entries he hadn\u2019t written yet: *Storm surge expected Tuesday, but the sky is clear.*\n\nHe stopped writing for a moment to listen\u2014not with ears alone\u2014but by pressing both palms flat against cold iron walls where condensation formed in slow spirals like fingerprints left behind after touching something invisible. The metal hummed faintly beneath his skin as if remembering old storms long past their dates on paper calendars torn from newspapers dated three years ahead of today"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 63,
+  "tok_s": 52.6,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, trying to get inside where he had spent thirty years refusing entry except for family and friends like him who were old enough too know better than young men ever could be again about what safety meant. It was November 2014 when Elias Thorne first stepped onto that spiral staircase with only one boot on because his other foot hurt after walking from the harbor office all morning long before anyone else thought he needed shoes off there for work even if nobody saw them do it but everyone knew why so they did not ask him anymore once this happened already twice last winter near same season now here again today too soon then tomorrow maybe never will come back unless someone sends message tonight via radio signal code Morse dot dash pattern"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 60,
+  "tok_s": 58.8,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nKangaroo       \nPanda        \nKoala          \nCheetah      \nWolf         \nFox           \nBear            \nMonkey             \nDolphin              \nWhale                \nShark                 \nOctopus               \nJellyfish                  \nCrab                    \nSpider                   \nRabbit                     \nCat                      \nDog                       \nHorse                        \nCow                          \nSheep                            \nGoat                              \nChicken                                \nRooster                                   \nTurkey                                  \nEagle                                    \nFalcon                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 71.8,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 52,
+  "drafts_offered": 148,
+  "tok_s": 61.8,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\u201d \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \u201cthe Sea\u201d or \"Ocean\" (terms often used interchangeably but distinct geographically) was never just an element within our planet\u2019s geography\u2014it became its engine itself\u2014a living system whose evolution shaped everything else around us from climate patterns down through evolutionary biology right up into modern civilization's economic structures cultural identities political boundaries ecological systems scientific understanding technological capabilities philosophical frameworks psychological experiences artistic expressions literary metaphors cinematic narratives musical compositions architectural designs urban planning strategies agricultural practices industrial processes medical discoveries legal precedents educational curricula religious traditions spiritual beliefs mythological stories folkloric tales poetic verses dramatic scripts visual arts photographic images digital media social networks behavioral norms emotional responses cognitive models decision-making algorithms predictive analytics statistical probabilities probabilistic reasoning causal inference counterfactual thinking hypothetical scenarios normative ethics deontal logic modal semantics temporal dynamics spatial relations topographical features hydrological cycles meteor phenomena atmospheric chemistry geological formations tectonic shifts volcanic eruptions seismic events tsunamis hurricanes cyclones monsoons typhoons droughts floods storms winds currents waves swells breakers surf foam spray mist fog rain snow ice glaciers berg floes drift pack-ice brine salinity pH alkalines acidity hardness softness temperature pressure depth light darkness color hue saturation brightness contrast luminance chroma tone value shade tint reflection"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-reg-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 187,
+  "tok_s": 58.7,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, mapped with veins like the roots of an ancient oak, gripped the cold iron railing.\n\nFor forty years, Elias had tended the light at the end of the world. Not the end of the map, but the end of the land. Here, the Atlantic threw itself against the granite cliffs with such violence that the spray reached the gallery, thirty feet above the rocks.\n\nHe checked the clock. 11:42 PM. The fog was thickening, a gray soup that swallowed the horizon. In the old days, he would have climbed the stairs to the lantern room, polished the Fresnel lens, and wound the clockwork mechanism that turned the great beam. But the automation had changed"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 179,
+  "tok_s": 80.7,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 71.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.0,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 19,
+  "drafts_offered": 49,
+  "tok_s": 49.1,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFlamingo<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 80,
+  "repeats": 0,
+  "drafts_accepted": 6,
+  "drafts_offered": 52,
+  "tok_s": 46.2,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to check what was happening with that storm and why they were calling me out there when all we had left for tonight\u2019s shift is one last pass through our logs before dawn breaks.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 83,
+  "tok_s": 43.9,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in its teeth, hungry and relentless for three centuries of stone salt-spray history that had long since forgotten to be polite or gentle with itself against anything smaller than God Himself was willing acknowledge as divine protection from drowning souls who were already lost before they ever touched water.\n\nElias knew this rhythm better he himself could hear it sometimes; his boots worn smooth by pacing the spiral stair\u2019s center-post groove where thousands upon countless steps had rubbed away texture until all that remained beneath sole leather now felt like polished bone carved out slowly but surely over time into something more hollow than wood ever really should be allowed become when used so often enough within such confined circular spaces meant only for vertical ascent down always descending again back upward never truly leaving"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 62,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 31,
+  "tok_s": 49.1,
+  "text": "Lion  \nTiger  \n\nElephant  \n  \nGiraffe  \n\n\n\nHippopotamus    \nZebra   \nRhinoceros   \n\n\nPanda Bear    \n\n\n\n\nKangaroo     \nKoala      \nSloth      \n\n\n\n\n\nCheetah       \nLeopard        \nPanther        \n\n\n\n\n\n\nWolf              \nFox                \nBear                 <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.7,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 47,
+  "drafts_offered": 153,
+  "tok_s": 52.2,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by billions years yet continues shaping every aspect from climate regulation through evolutionary biology all way down modern industrial supply chains today's global shipping lanes carry cargo across vast distances because they were once crossed only slowly via coastal hopping routes used since antiquity until steamships revolutionized travel during late nineteenth century when diesel engines replaced wind power entirely allowing container ships carrying thousands TEUs (Twenty-foot Equivalent Units) per voyage crossing entire Pacific basins without stopping at intermediate ports anymore unlike earlier sailing vessels needing favorable winds currents seasonal patterns dictating departure windows based upon monsoon calendars established centuries ago before technological advances changed those constraints fundamentally altering logistics networks worldwide thereby transforming economic structures globally while simultaneously creating ecological disruptions due invasive species transported unintentionally alongside legitimate goods causing cascading ecosystem collapses locally where non-native organisms outcompete native fauna disrupting food webs leading population declines affecting biodiversity indices measured scientifically using standardized metrics like Shannon-Wiener diversity index calculations applied consistently over decades revealing trends showing significant decreases post-introduction events correl strongly with introduction dates matching historical records documenting first sightings confirmed taxonomically classified formally according International Code Nomenclature rules"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/pp-lim-cv-pp-x2.json

@@ -0,0 +1,84 @@
+{
+ "label": "lim-cv-pp-x2",
+ "extra": [
+  "--layer-split",
+  "auto",
+  "--control-vector-scaled",
+  "C:\\Users\\ribbon\\strata-test\\Strata\\data\\experimental-speed-projection\\Qwen3.8-Flash-Next-experimental-speed-projection.gguf:1.0",
+  "--control-vector-layer-range",
+  "4",
+  "44",
+  "--cvec-mode",
+  "project",
+  "--cvec-dir",
+  "per-layer"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 15925,
+   "prompt_ms": 8783.1,
+   "pp_tok_s": 1813.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27474,
+   "prompt_ms": 13379.7,
+   "pp_tok_s": 2053.4,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3652.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 246.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 77.0,
+   "reps": [
+    77.0
+   ],
+   "hit_rate": 0.9718150054476462,
+   "text": "The wind didn\u2019t just howl; it screamed. It tore through the gaps in the iron railing of the Lantern Room, carrying the scent of brine and rotting kelp. Elias pulled his woolen collar up to his chin, his fingers numb and clumsy inside thick leather gloves.\n\nHe was the last man on the island. The automated systems had failed three winters ago, a glitch in the solar panel array that no engineer had been able to fix before the supply boat stopped running. Now, it was just him, the light, and the sea.\n\nElias climbed the spiral stairs, his boots heavy with salt spray. The iron steps groaned under his weight, a sound that seemed to mock the storm outside. He reached the top, the glass walls vibrating with the force of the gale.\n\nInside the lantern room, the great Fresnel lens sat like a dormant eye. It was cold, dark, and waiting.\n\nElias moved to the control panel. His"
+  },
+  {
+   "name": "code",
+   "tok_s": 103.6,
+   "reps": [
+    103.6
+   ],
+   "hit_rate": 0.957731667226356,
+   "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+  }
+ ],
+ "stage_timing": [],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.30 GiB for experts",
+  "strata generate: layer split auto: CUDA1 82 SMs at 1.86 GHz -> 0.48 ms per layer, 14.14 GiB for experts",
+  "strata generate: layer split auto: K=22 - predicted 19.7 ms per decode window; the caches hold 11298 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 3479 slots, 6.22 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 91.6% (5554 hits / 6062 lookups)",
+  "strata serve: decode expert cache hit rate: 95.3% (3592 hits / 3771 lookups)",
+  "strata serve: decode expert cache hit rate: 96.5% (3650 hits / 3782 lookups)",
+  "strata serve: decode expert cache hit rate: 96.3% (25027 hits / 25980 lookups)",
+  "strata serve: decode expert cache hit rate: 96.5% (3201 hits / 3318 lookups)",
+  "strata serve: decode expert cache hit rate: 97.2% (129334 hits / 133085 lookups)",
+  "strata serve: decode expert cache hit rate: 95.8% (108375 hits / 113158 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split-limits/data/pp-lim-kv-pp-single.json

@@ -0,0 +1,76 @@
+{
+ "label": "lim-kv-pp-single",
+ "extra": [
+  "--max-context",
+  "131072",
+  "--kv-resident",
+  "32768"
+ ],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 15925,
+   "prompt_ms": 10706.3,
+   "pp_tok_s": 1487.4,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27474,
+   "prompt_ms": 15256.7,
+   "pp_tok_s": 1800.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 59725,
+   "prompt_ms": 32515.5,
+   "pp_tok_s": 1836.8,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4006.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 276.2,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 66.6,
+   "reps": [
+    66.6
+   ],
+   "hit_rate": 0.7101864314556545,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to sync with his own heartbeat. He had been doing this for forty years. His hands were mapped with scars from rope burns and broken glass, his eyes permanently squinted against the glare of the bulb."
+  },
+  {
+   "name": "code",
+   "tok_s": 74.0,
+   "reps": [
+    74.0
+   ],
+   "hit_rate": 0.5873783774988882,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [],
+ "engine_lines": [
+  "strata generate: expert cache auto: 8.99 GiB free, 700 MiB reserved -> 3349 slots",
+  "strata generate: expert cache 2434 slots, 4.67 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 1944 cache slots (3.72 GiB)",
+  "strata serve: decode expert cache hit rate: 41.8% (1746 hits / 4176 lookups)",
+  "strata serve: decode expert cache hit rate: 50.4% (1512 hits / 3000 lookups)",
+  "strata serve: decode expert cache hit rate: 54.9% (1689 hits / 3077 lookups)",
+  "strata serve: decode expert cache hit rate: 60.3% (1938 hits / 3213 lookups)",
+  "strata serve: decode expert cache hit rate: 58.3% (11146 hits / 19121 lookups)",
+  "strata serve: decode expert cache hit rate: 66.1% (2201 hits / 3331 lookups)",
+  "strata serve: decode expert cache hit rate: 71.0% (74397 hits / 104757 lookups)",
+  "strata serve: decode expert cache hit rate: 58.7% (54151 hits / 92191 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split-limits/data/pp-lim-kv-pp-x2.json

@@ -0,0 +1,87 @@
+{
+ "label": "lim-kv-pp-x2",
+ "extra": [
+  "--max-context",
+  "131072",
+  "--kv-resident",
+  "32768",
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 15925,
+   "prompt_ms": 9452.6,
+   "pp_tok_s": 1684.7,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27474,
+   "prompt_ms": 14042.4,
+   "pp_tok_s": 1956.5,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 59725,
+   "prompt_ms": 30345.9,
+   "pp_tok_s": 1968.1,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3806.1,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 245.6,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 75.8,
+   "reps": [
+    75.8
+   ],
+   "hit_rate": 0.9698055042592483,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined Elias\u2019s life for thirty years. He stood before the great Fresnel lens, a cathedral of glass and brass that caught the light of the single, powerful bulb and fractured it into a thousand beams.\n\nHe checked the clock. 11:42 PM. The rotation was smooth, silent, a hypnotic sweep that painted the darkness with a slow, rhythmic pulse. *Flash. Dark. Flash. Dark.*\n\nE"
+  },
+  {
+   "name": "code",
+   "tok_s": 97.0,
+   "reps": [
+    97.0
+   ],
+   "hit_rate": 0.9562207324057541,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 17.96 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.13 GiB for experts",
+  "strata generate: layer split auto: CUDA1 82 SMs at 1.86 GHz -> 0.48 ms per layer, 13.98 GiB for experts",
+  "strata generate: layer split auto: K=22 - predicted 19.7 ms per decode window; the caches hold 11199 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.30 GiB free, 700 MiB reserved -> 3277 slots",
+  "strata generate: expert cache 3409 slots, 6.09 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 91.3% (5098 hits / 5585 lookups)",
+  "strata serve: decode expert cache hit rate: 94.7% (4000 hits / 4225 lookups)",
+  "strata serve: decode expert cache hit rate: 96.5% (4095 hits / 4245 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (4183 hits / 4278 lookups)",
+  "strata serve: decode expert cache hit rate: 96.2% (27177 hits / 28250 lookups)",
+  "strata serve: decode expert cache hit rate: 97.6% (4177 hits / 4281 lookups)",
+  "strata serve: decode expert cache hit rate: 97.0% (130241 hits / 134296 lookups)",
+  "strata serve: decode expert cache hit rate: 95.6% (113534 hits / 118732 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split-limits/data/vision.json

@@ -0,0 +1,24 @@
+{
+ "answers": {
+  "single": [
+   " Red",
+   "42",
+   "A red circle, the number 42, and a blue square are arranged on a white background."
+  ],
+  "same24": [
+   " Red",
+   "42",
+   "A red circle, the number 42, and a blue square are arranged on a white background."
+  ],
+  "x2": [
+   " Red",
+   "42",
+   "A red circle, the number 42, and a blue square are displayed on a white background."
+  ]
+ },
+ "correct": {
+  "single": true,
+  "same24": true,
+  "x2": true
+ }
+}

bench/results/2026-09-29-speed-0122/README.md

@@ -0,0 +1,32 @@
+# Speed with engine 0.1.22
+
+RTX 5070 12 GB (PCIe 5.0 x16), Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. The ready-made 0.1.22 engine, one-shot runs
+with what setup writes: `--prefill auto`, the shipped expert profile, `--expert-cache auto`, 8-bit KV above 4K, KV
+streaming (`--kv-resident 32768`) from 64K, MTP (`--spec 4 --spec-min-p 0.5`), greedy, 256 generated tokens, images
+off. The same code-agent prompts as `2026-09-28-speed-0114`. Every run: [`matrix.json`](matrix.json).
+
+## Prompt (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 519 | 1,226 | 1,844 | 1,836 | 1,682 | 1,304 |
+| **IQ2_XS** | 524 | 1,196 | 1,799 | 1,611 | 1,495 | 1,181 |
+| **IQ3_XXS** | 472 | 974 | 1,555 | 1,449 | 1,386 | - |
+| **IQ3_S** | 419 | 893 | 1,499 | 1,285 | 1,245 | - |
+| **Coder** | 660 | 1,522 | 1,871 | 1,938 | 1,779 | - |
+
+## Output (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 84.7 | 88.2 | 77.4 | 71.0 | 63.1 | 61.1 |
+| **IQ2_XS** | 77.2 | 74.6 | 67.5 | 63.5 | 59.1 | 54.7 |
+| **IQ3_XXS** | 65.1 | 62.3 | 49.8 | 53.6 | 47.0 | - |
+| **IQ3_S** | 50.5 | 44.6 | 49.0 | 48.8 | 41.3 | - |
+| **Coder** | 56.9 | 53.6 | 54.9 | 48.5 | 42.8 | - |
+
+- IQ2_XS was measured with Swift 1.5's IQ2_XS (the same speed as the original's); its 262K run had images on.
+- IQ3_XXS and IQ3_S at 262K do not fit 64 GB of RAM; the Coder at 262K was not run again (0.1.14: 1,034 tokens/s).
+- Against 0.1.14: prompts 1.2-1.5x faster at 32K-128K (the 0.1.22 prompt path: tensor-core attention, bit-identical
+  batched kernels, faster PLE reads). Output speed is the engine's same decode; one run per cell moves a few percent
+  with the text (speculative drafts), so compare it across many runs, not one.

bench/results/2026-09-29-speed-0122/matrix.json

@@ -0,0 +1,380 @@
+[
+ {
+  "model": "Q2_0",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 40.0,
+  "decode_tok_s": 84.71,
+  "decoded": 256,
+  "prefill_tok_s": 519.48,
+  "ttft_s": 3.62,
+  "spec_accept": 0.786,
+  "tokens_per_round": 2.77,
+  "vram_slots": 4176
+ },
+ {
+  "model": "Q2_0",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 41.6,
+  "decode_tok_s": 88.23,
+  "decoded": 256,
+  "prefill_tok_s": 1226.49,
+  "ttft_s": 4.43,
+  "spec_accept": 0.839,
+  "tokens_per_round": 3.24,
+  "vram_slots": 4123
+ },
+ {
+  "model": "Q2_0",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 56.6,
+  "decode_tok_s": 77.37,
+  "decoded": 256,
+  "prefill_tok_s": 1843.83,
+  "ttft_s": 18.63,
+  "spec_accept": 0.808,
+  "tokens_per_round": 3.04,
+  "vram_slots": 3833
+ },
+ {
+  "model": "Q2_0",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 72.9,
+  "decode_tok_s": 71.0,
+  "decoded": 256,
+  "prefill_tok_s": 1835.64,
+  "ttft_s": 36.48,
+  "spec_accept": 0.685,
+  "tokens_per_round": 2.77,
+  "vram_slots": 3780
+ },
+ {
+  "model": "Q2_0",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 118.1,
+  "decode_tok_s": 63.08,
+  "decoded": 256,
+  "prefill_tok_s": 1681.64,
+  "ttft_s": 77.92,
+  "spec_accept": 0.7,
+  "tokens_per_round": 2.56,
+  "vram_slots": 3697
+ },
+ {
+  "model": "Q2_0",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 242.0,
+  "decode_tok_s": 61.12,
+  "decoded": 256,
+  "prefill_tok_s": 1303.83,
+  "ttft_s": 200.94,
+  "spec_accept": 0.627,
+  "tokens_per_round": 2.5,
+  "vram_slots": 3525
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 263.6,
+  "decode_tok_s": 54.68,
+  "decoded": 256,
+  "prefill_tok_s": 1180.58,
+  "ttft_s": 221.91,
+  "spec_accept": 0.632,
+  "tokens_per_round": 2.49,
+  "vram_slots": 3727
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 56.0,
+  "decode_tok_s": 50.5,
+  "decoded": 256,
+  "prefill_tok_s": 419.48,
+  "ttft_s": 3.98,
+  "spec_accept": 0.811,
+  "tokens_per_round": 3.17,
+  "vram_slots": 2617
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 60.0,
+  "decode_tok_s": 44.58,
+  "decoded": 256,
+  "prefill_tok_s": 892.84,
+  "ttft_s": 5.53,
+  "spec_accept": 0.672,
+  "tokens_per_round": 2.69,
+  "vram_slots": 2580
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 75.6,
+  "decode_tok_s": 48.96,
+  "decoded": 256,
+  "prefill_tok_s": 1499.09,
+  "ttft_s": 22.65,
+  "spec_accept": 0.79,
+  "tokens_per_round": 2.9,
+  "vram_slots": 2383
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 102.1,
+  "decode_tok_s": 48.8,
+  "decoded": 256,
+  "prefill_tok_s": 1284.81,
+  "ttft_s": 51.58,
+  "spec_accept": 0.797,
+  "tokens_per_round": 2.94,
+  "vram_slots": 2281
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 159.2,
+  "decode_tok_s": 41.27,
+  "decoded": 256,
+  "prefill_tok_s": 1244.55,
+  "ttft_s": 104.81,
+  "spec_accept": 0.694,
+  "tokens_per_round": 2.73,
+  "vram_slots": 2292
+ },
+ {
+  "model": "Coder",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 34.5,
+  "decode_tok_s": 56.94,
+  "decoded": 256,
+  "prefill_tok_s": 660.44,
+  "ttft_s": 2.42,
+  "spec_accept": 0.74,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2650
+ },
+ {
+  "model": "Coder",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 18.8,
+  "decode_tok_s": 53.65,
+  "decoded": 256,
+  "prefill_tok_s": 1521.84,
+  "ttft_s": 3.21,
+  "spec_accept": 0.736,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2615
+ },
+ {
+  "model": "Coder",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 33.1,
+  "decode_tok_s": 54.85,
+  "decoded": 256,
+  "prefill_tok_s": 1870.76,
+  "ttft_s": 17.7,
+  "spec_accept": 0.806,
+  "tokens_per_round": 3.11,
+  "vram_slots": 2418
+ },
+ {
+  "model": "Coder",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 50.2,
+  "decode_tok_s": 48.46,
+  "decoded": 256,
+  "prefill_tok_s": 1937.69,
+  "ttft_s": 33.99,
+  "spec_accept": 0.653,
+  "tokens_per_round": 2.56,
+  "vram_slots": 2382
+ },
+ {
+  "model": "Coder",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 90.3,
+  "decode_tok_s": 42.75,
+  "decoded": 256,
+  "prefill_tok_s": 1778.66,
+  "ttft_s": 73.12,
+  "spec_accept": 0.588,
+  "tokens_per_round": 2.27,
+  "vram_slots": 2326
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 43.6,
+  "decode_tok_s": 77.16,
+  "decoded": 256,
+  "prefill_tok_s": 523.77,
+  "ttft_s": 3.46,
+  "spec_accept": 0.811,
+  "tokens_per_round": 3.02,
+  "vram_slots": 4361
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 43.7,
+  "decode_tok_s": 74.58,
+  "decoded": 256,
+  "prefill_tok_s": 1196.03,
+  "ttft_s": 4.47,
+  "spec_accept": 0.824,
+  "tokens_per_round": 3.36,
+  "vram_slots": 4309
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 59.6,
+  "decode_tok_s": 67.46,
+  "decoded": 256,
+  "prefill_tok_s": 1798.65,
+  "ttft_s": 19.06,
+  "spec_accept": 0.789,
+  "tokens_per_round": 3.02,
+  "vram_slots": 4030
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 81.4,
+  "decode_tok_s": 63.52,
+  "decoded": 256,
+  "prefill_tok_s": 1610.73,
+  "ttft_s": 41.37,
+  "spec_accept": 0.705,
+  "tokens_per_round": 2.6,
+  "vram_slots": 3978
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 126.0,
+  "decode_tok_s": 59.11,
+  "decoded": 256,
+  "prefill_tok_s": 1494.53,
+  "ttft_s": 87.51,
+  "spec_accept": 0.704,
+  "tokens_per_round": 2.71,
+  "vram_slots": 3897
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 48.6,
+  "decode_tok_s": 65.12,
+  "decoded": 256,
+  "prefill_tok_s": 471.53,
+  "ttft_s": 3.65,
+  "spec_accept": 0.876,
+  "tokens_per_round": 4.08,
+  "vram_slots": 3281
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 48.5,
+  "decode_tok_s": 62.33,
+  "decoded": 256,
+  "prefill_tok_s": 974.3,
+  "ttft_s": 5.17,
+  "spec_accept": 0.826,
+  "tokens_per_round": 3.15,
+  "vram_slots": 3243
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 64.9,
+  "decode_tok_s": 49.83,
+  "decoded": 256,
+  "prefill_tok_s": 1554.53,
+  "ttft_s": 21.84,
+  "spec_accept": 0.766,
+  "tokens_per_round": 2.91,
+  "vram_slots": 3009
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 91.7,
+  "decode_tok_s": 53.58,
+  "decoded": 256,
+  "prefill_tok_s": 1449.37,
+  "ttft_s": 45.84,
+  "spec_accept": 0.748,
+  "tokens_per_round": 2.8,
+  "vram_slots": 2965
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 140.9,
+  "decode_tok_s": 46.97,
+  "decoded": 256,
+  "prefill_tok_s": 1386.04,
+  "ttft_s": 94.33,
+  "spec_accept": 0.607,
+  "tokens_per_round": 2.47,
+  "vram_slots": 2901
+ }
+]

bench/results/2026-09-29-speed-0126/README.md

@@ -0,0 +1,36 @@
+# Speed with engine 0.1.26
+
+RTX 5070 12 GB (PCIe 5.0 x16), Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. The ready-made 0.1.26 engine, one-shot runs
+with what setup writes: `--prefill auto`, the shipped expert profile, `--expert-cache auto`, 8-bit KV above 4K, KV
+streaming (`--kv-resident 32768`) from 64K, MTP (`--spec 4 --spec-min-p 0.5`), greedy, 256 generated tokens, images
+off, no timing marks. The same code-agent prompts as `2026-09-29-speed-0122`. Every run: [`matrix.json`](matrix.json).
+
+## Prompt (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 536 | 1,299 | 2,171 | 2,126 | 2,107 |
+| **IQ2_XS** | 534 | 1,256 | 2,092 | 1,754 | 1,752 |
+| **IQ3_XXS** | 482 | 1,007 | 1,745 | 1,609 | 1,602 |
+| **IQ3_S** | 427 | 913 | 1,624 | 1,640 | 1,443 |
+| **Coder** | 656 | 1,583 | 2,177 | 2,236 | 2,208 |
+
+## Output (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 87.3 | 93.0 | 81.8 | 76.2 | 73.7 |
+| **IQ2_XS** | 79.6 | 78.6 | 76.3 | 63.7 | 62.7 |
+| **IQ3_XXS** | 61.9 | 61.6 | 58.5 | 57.2 | 49.0 |
+| **IQ3_S** | 52.4 | 53.3 | 48.3 | 46.3 | 45.5 |
+| **Coder** | 58.9 | 55.1 | 54.9 | 53.2 | 43.0 |
+
+- Against 0.1.22 (`2026-09-29-speed-0122`), prompts at 32K-128K are 8-28% faster: Q2_0 1,844 -> 2,171 at 32K and
+  1,682 -> 2,107 at 128K, the Coder 1,779 -> 2,208 at 128K. The gains come from:
+  - 0.1.24's QSA selection on tensor cores;
+  - 0.1.25's grouping tables in mapped memory and fused norms;
+  - 0.1.26's batched draft-layer pass.
+- 262K was not run again (0.1.22: Q2_0 1,304, IQ2_XS 1,181 tokens/s).
+- Output speed is the same decode as before. One run per cell moves several percent with the text (the share of
+  accepted drafts, `spec_accept` in `matrix.json`). For example, the Coder's 128K answer accepted 55% of its drafts
+  and its 64K answer 71%.

bench/results/2026-09-29-speed-0126/matrix.json

@@ -0,0 +1,352 @@
+[
+ {
+  "model": "Q2_0",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 23.9,
+  "decode_tok_s": 87.33,
+  "decoded": 256,
+  "prefill_tok_s": 536.42,
+  "ttft_s": 3.4,
+  "spec_accept": 0.681,
+  "tokens_per_round": 2.72,
+  "vram_slots": 4174
+ },
+ {
+  "model": "Q2_0",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 27.6,
+  "decode_tok_s": 92.97,
+  "decoded": 256,
+  "prefill_tok_s": 1299.44,
+  "ttft_s": 4.21,
+  "spec_accept": 0.838,
+  "tokens_per_round": 3.23,
+  "vram_slots": 4121
+ },
+ {
+  "model": "Q2_0",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 40.8,
+  "decode_tok_s": 81.75,
+  "decoded": 256,
+  "prefill_tok_s": 2170.85,
+  "ttft_s": 16.02,
+  "spec_accept": 0.788,
+  "tokens_per_round": 2.99,
+  "vram_slots": 3832
+ },
+ {
+  "model": "Q2_0",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 57.3,
+  "decode_tok_s": 76.23,
+  "decoded": 256,
+  "prefill_tok_s": 2126.03,
+  "ttft_s": 32.06,
+  "spec_accept": 0.707,
+  "tokens_per_round": 2.8,
+  "vram_slots": 3778
+ },
+ {
+  "model": "Q2_0",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 87.7,
+  "decode_tok_s": 73.73,
+  "decoded": 256,
+  "prefill_tok_s": 2106.85,
+  "ttft_s": 62.48,
+  "spec_accept": 0.679,
+  "tokens_per_round": 2.71,
+  "vram_slots": 3695
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 42.2,
+  "decode_tok_s": 79.62,
+  "decoded": 256,
+  "prefill_tok_s": 533.83,
+  "ttft_s": 3.51,
+  "spec_accept": 0.774,
+  "tokens_per_round": 2.89,
+  "vram_slots": 4359
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 40.6,
+  "decode_tok_s": 78.6,
+  "decoded": 256,
+  "prefill_tok_s": 1256.02,
+  "ttft_s": 4.31,
+  "spec_accept": 0.81,
+  "tokens_per_round": 3.29,
+  "vram_slots": 4308
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 53.6,
+  "decode_tok_s": 76.34,
+  "decoded": 256,
+  "prefill_tok_s": 2092.02,
+  "ttft_s": 16.57,
+  "spec_accept": 0.85,
+  "tokens_per_round": 3.2,
+  "vram_slots": 4029
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 78.3,
+  "decode_tok_s": 63.69,
+  "decoded": 256,
+  "prefill_tok_s": 1753.65,
+  "ttft_s": 38.08,
+  "spec_accept": 0.636,
+  "tokens_per_round": 2.59,
+  "vram_slots": 3977
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 115.1,
+  "decode_tok_s": 62.72,
+  "decoded": 256,
+  "prefill_tok_s": 1751.79,
+  "ttft_s": 74.85,
+  "spec_accept": 0.696,
+  "tokens_per_round": 2.6,
+  "vram_slots": 3896
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 46.9,
+  "decode_tok_s": 61.94,
+  "decoded": 256,
+  "prefill_tok_s": 482.04,
+  "ttft_s": 3.67,
+  "spec_accept": 0.845,
+  "tokens_per_round": 3.57,
+  "vram_slots": 3279
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 47.4,
+  "decode_tok_s": 61.64,
+  "decoded": 256,
+  "prefill_tok_s": 1007.48,
+  "ttft_s": 5.03,
+  "spec_accept": 0.801,
+  "tokens_per_round": 3.19,
+  "vram_slots": 3242
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 65.6,
+  "decode_tok_s": 58.49,
+  "decoded": 256,
+  "prefill_tok_s": 1745.16,
+  "ttft_s": 19.63,
+  "spec_accept": 0.789,
+  "tokens_per_round": 2.85,
+  "vram_slots": 3007
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 88.5,
+  "decode_tok_s": 57.21,
+  "decoded": 256,
+  "prefill_tok_s": 1609.31,
+  "ttft_s": 41.45,
+  "spec_accept": 0.74,
+  "tokens_per_round": 2.87,
+  "vram_slots": 2963
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 127.0,
+  "decode_tok_s": 48.95,
+  "decoded": 256,
+  "prefill_tok_s": 1602.33,
+  "ttft_s": 81.71,
+  "spec_accept": 0.643,
+  "tokens_per_round": 2.37,
+  "vram_slots": 2899
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 54.2,
+  "decode_tok_s": 52.4,
+  "decoded": 256,
+  "prefill_tok_s": 426.6,
+  "ttft_s": 4.03,
+  "spec_accept": 0.822,
+  "tokens_per_round": 3.17,
+  "vram_slots": 2617
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 58.9,
+  "decode_tok_s": 53.3,
+  "decoded": 256,
+  "prefill_tok_s": 912.99,
+  "ttft_s": 5.58,
+  "spec_accept": 0.85,
+  "tokens_per_round": 3.38,
+  "vram_slots": 2579
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 72.8,
+  "decode_tok_s": 48.31,
+  "decoded": 256,
+  "prefill_tok_s": 1624.07,
+  "ttft_s": 21.07,
+  "spec_accept": 0.827,
+  "tokens_per_round": 3.0,
+  "vram_slots": 2383
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 93.3,
+  "decode_tok_s": 46.25,
+  "decoded": 256,
+  "prefill_tok_s": 1640.34,
+  "ttft_s": 40.77,
+  "spec_accept": 0.693,
+  "tokens_per_round": 2.56,
+  "vram_slots": 2346
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 144.6,
+  "decode_tok_s": 45.51,
+  "decoded": 256,
+  "prefill_tok_s": 1443.21,
+  "ttft_s": 90.72,
+  "spec_accept": 0.697,
+  "tokens_per_round": 2.73,
+  "vram_slots": 2291
+ },
+ {
+  "model": "Coder",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 37.0,
+  "decode_tok_s": 58.91,
+  "decoded": 256,
+  "prefill_tok_s": 656.36,
+  "ttft_s": 2.48,
+  "spec_accept": 0.743,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2649
+ },
+ {
+  "model": "Coder",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 18.3,
+  "decode_tok_s": 55.1,
+  "decoded": 256,
+  "prefill_tok_s": 1583.1,
+  "ttft_s": 3.09,
+  "spec_accept": 0.713,
+  "tokens_per_round": 2.88,
+  "vram_slots": 2614
+ },
+ {
+  "model": "Coder",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 30.4,
+  "decode_tok_s": 54.89,
+  "decoded": 256,
+  "prefill_tok_s": 2177.17,
+  "ttft_s": 15.33,
+  "spec_accept": 0.731,
+  "tokens_per_round": 2.75,
+  "vram_slots": 2417
+ },
+ {
+  "model": "Coder",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 44.8,
+  "decode_tok_s": 53.22,
+  "decoded": 256,
+  "prefill_tok_s": 2236.49,
+  "ttft_s": 29.51,
+  "spec_accept": 0.709,
+  "tokens_per_round": 2.68,
+  "vram_slots": 2382
+ },
+ {
+  "model": "Coder",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 75.9,
+  "decode_tok_s": 43.04,
+  "decoded": 256,
+  "prefill_tok_s": 2208.27,
+  "ttft_s": 59.02,
+  "spec_accept": 0.553,
+  "tokens_per_round": 2.23,
+  "vram_slots": 2324
+ }
+]

cmake/hip_backend.cmake

@@ -0,0 +1,58 @@
+# Opt-in HIP configuration. Strata's CUDA-shaped kernels currently target RDNA3 gfx1100 wave32.
+# CMake/compiler discovery stays machine-independent; pass CMAKE_HIP_COMPILER when it is not on PATH.
+if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "")
+  set(CMAKE_HIP_ARCHITECTURES gfx1100 CACHE STRING "Strata HIP target architecture")
+endif()
+if(NOT CMAKE_HIP_ARCHITECTURES STREQUAL "gfx1100")
+  message(FATAL_ERROR
+    "Strata HIP currently supports only gfx1100 wave32; CMAKE_HIP_ARCHITECTURES is '${CMAKE_HIP_ARCHITECTURES}'")
+endif()
+
+enable_language(HIP)
+find_package(hip CONFIG REQUIRED)
+find_package(hipblas CONFIG REQUIRED)
+find_package(hipblaslt CONFIG QUIET)
+
+if(NOT TARGET hip::host)
+  message(FATAL_ERROR "The ROCm hip CMake package did not provide hip::host")
+endif()
+if(NOT TARGET roc::hipblas)
+  message(FATAL_ERROR "The ROCm hipblas CMake package did not provide roc::hipblas")
+endif()
+if(TARGET roc::hipblaslt)
+  set(STRATA_HIPBLASLT_AVAILABLE ON)
+else()
+  set(STRATA_HIPBLASLT_AVAILABLE OFF)
+  message(STATUS "Strata: hipBLASLt not found; solution-table dispatch is unavailable")
+endif()
+
+# HIP's link step produces a PIE; make Strata and ggml objects PIC for the ROCm linker.
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(STRATA_HIP_COMPAT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/strata/hip_compat")
+add_library(strata_hip_runtime INTERFACE)
+target_include_directories(strata_hip_runtime BEFORE INTERFACE
+  "${STRATA_HIP_COMPAT_INCLUDE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/include")
+target_compile_definitions(strata_hip_runtime INTERFACE STRATA_USE_HIP=1)
+target_link_libraries(strata_hip_runtime INTERFACE hip::host)
+foreach(_language IN ITEMS CXX HIP)
+  target_compile_options(strata_hip_runtime INTERFACE
+    "$<$<COMPILE_LANGUAGE:${_language}>:-include>"
+    "$<$<COMPILE_LANGUAGE:${_language}>:${STRATA_HIP_COMPAT_INCLUDE_DIR}/cuda_runtime.h>")
+endforeach()
+
+# CMake does not infer HIP from Strata's existing CUDA-shaped .cu suffixes.
+file(GLOB_RECURSE _strata_hip_sources CONFIGURE_DEPENDS
+  "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cu"
+  "${CMAKE_CURRENT_SOURCE_DIR}/bench/*.cu"
+  "${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cu")
+if(_strata_hip_sources)
+  set_source_files_properties(${_strata_hip_sources} PROPERTIES LANGUAGE HIP)
+endif()
+foreach(_source IN ITEMS tests/hip/intrinsics.cpp tests/hip/native_qsa_score.cpp)
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${_source}")
+    set_source_files_properties("${_source}" PROPERTIES LANGUAGE HIP)
+  endif()
+endforeach()
+
+message(STATUS "Strata: HIP enabled, arch ${CMAKE_HIP_ARCHITECTURES}")

docs/AMD_HIP.md

@@ -0,0 +1,156 @@
+# Experimental AMD HIP backend (gfx1100)
+
+This is a manual Linux source build for the RX 7900 XTX. It is opt-in; the
+NVIDIA installer and CUDA build remain the default. Other AMD architectures,
+wave64, Windows HIP, and mixed AMD/NVIDIA execution are outside this contribution.
+
+The backend maps the CUDA-shaped runtime and BLAS calls to HIP/hipBLAS, uses
+RDNA3's signed integer dot instruction for quantized kernels, and supplies
+wave32 shuffle/packed-byte operations. CUDA-only QSA matrix instructions have
+an ordered FP32 fallback. Prefill supports both dequantization plus hipBLAS GEMM and opt-in HIP ggml MMQ.
+An optional, calibrated hipBLASLt path accelerates dense projections on gfx1100.
+This does not claim bit-identical model answers across backends. See
+[performance settings and evidence](AMD_HIP_PERFORMANCE.md).
+
+## Install with setup (recommended)
+
+On Linux with an RX 7900 XT / XTX and the kernel's amdgpu driver (no ROCm install needed):
+
+```sh
+./setup.sh --backend hip
+```
+
+- **Detection:** setup finds the card through the kernel's KFD topology. Integrated Radeon GPUs are listed as not
+  supported. On a PC without an NVIDIA card Strata can use, `--backend hip` is chosen automatically.
+- **ROCm:** installed into `.venv` from AMD's TheRock wheels (~10 GB, no sudo), pinned to the version this backend was
+  tested with (`STRATA_ROCM_VERSION` / `STRATA_ROCM_INDEX` override it). A system ROCm in `/opt/rocm` (or
+  `$ROCM_PATH`) with hipcc and hipBLAS is used instead when present.
+- **Engine:** compiled on your PC (10-20 minutes, once; again after a `git pull` that changes it). This needs a C++
+  compiler and git (`sudo apt install build-essential git`).
+- **Limits for now:** one GPU, no images, no calibration. The Monitor shows no GPU statistics.
+
+The rest of setup is the same as on NVIDIA: the model download, the start script, the server.
+
+## Build
+
+Requirements: a working ROCm driver/runtime, HIP development headers and
+compiler, hipBLAS and (for tuned dense prefill) hipBLASLt development files, CMake 3.24+, a C++20 host compiler, and Git.
+The model still needs sufficient system RAM and fast SSD storage for its PLE
+table. VRAM occupancy alone is not a throughput measurement.
+
+```sh
+cmake -S . -B build-hip \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DSTRATA_ENABLE_HIP=ON -DSTRATA_ENABLE_CUDA=OFF \
+  -DCMAKE_HIP_ARCHITECTURES=gfx1100
+cmake --build build-hip --target strata -j2
+```
+
+If CMake cannot find the HIP compiler, add
+`-DCMAKE_HIP_COMPILER=/path/to/rocm/llvm/bin/clang++`. On the Fedora-family test
+host this was `/usr/lib64/rocm/llvm/bin/clang++`. Use the compiler and libraries
+from the same ROCm installation. Do not enable both GPU backends in one build.
+
+Native IQ experts are enabled by default. CMake fetches the llama.cpp revision
+pinned by this repository. For an offline build, point `STRATA_GGML_DIR` at a
+checkout of that exact revision; using an arbitrary newer checkout changes the
+dependency being tested.
+
+## Model and serving configuration
+
+Prepare a supported model pack and its MTP runtime using the existing tools.
+For OrcaRouter IQ3_XXS, follow [the explicit compatibility conversion](ORCA.md);
+this backend does not change quantization, model licenses, or tokenizers.
+Add `--experts-bin` to the `tools/iq_pack.py` command: mmap requires the pack's
+`experts.bin`, which the default native packing command does not emit. This
+consumes additional disk space. Original GSQ-RCO IQ3_XXS uses shard 2 for PLE;
+Orca uses shard 1. Keep each model's own pack and tokenizer together.
+
+Use `build-hip/strata` as the executable in the server JSON. Select the AMD
+device with `ROCR_VISIBLE_DEVICES`/`HIP_VISIBLE_DEVICES` if necessary. Start with
+a modest context and prefill chunk size before measuring larger workloads.
+Keep the PLE file on an SSD. Do not assume a configured context length proves
+successful full-window inference.
+
+Large pinned host allocations can fail on ROCm even when ordinary RAM is
+available. The existing `--mmap-experts` path avoids allocating the full pinned
+expert arena; it still depends on OS file-cache residency and may stall on
+storage reads. It does not make SSD access equivalent to RAM.
+
+Starting args for the original GSQ-RCO IQ3_XXS model (replace the paths):
+
+```sh
+build-hip/strata --serve \
+  --pack packs/iq3xxs \
+  --native /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf \
+  --ple-gguf /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf \
+  --mmap-experts --expert-profile data/expert-profile.bin --expert-cache auto \
+  --prefill 512 --spec 4 --spec-min-p 0.5 --mtp mtp/rt \
+  --max-context 4096 --kv int8 --pool-workers 15 \
+  --adapt-every 0 --pcie-frac 0 --vram-reserve-mib 1024
+```
+
+`--serve` is the engine's internal token protocol. For a browser or OpenAI API,
+put these args in the `args` array of the [server JSON example](ORCA.md#local-server),
+set `exe` to `build-hip/strata`, and use the matching pack's `tokenizer` directory.
+Launch with `.venv/bin/python -m serve.server --engine strata --config strata-hip.json --port 8080`.
+The worker count above was used on a 16-core CPU; measure it for your CPU.
+The 4K context is a smoke-test starting point, not a model limit. The expert cache
+sizes itself automatically and leaves 1 GiB of VRAM headroom.
+
+The installer supports this backend (see "Install with setup" above). The vision helper and multi-GPU layer
+splits are NVIDIA-only for now.
+
+## Original backend validation (PR #94)
+
+The following is historical validation of the original backend, not a fresh
+test count for this replacement. Current build/test evidence and performance
+limits are recorded in [AMD_HIP_PERFORMANCE.md](AMD_HIP_PERFORMANCE.md).
+
+Tested against upstream `c1e903310f211e6630780c3bd2038778c071c68d` (0.1.20),
+with pinned llama.cpp `3cf03257f219afbe7334045ff7c6a06ac68c627d`.
+The host has an RX 7900 XTX (24 GiB), Ryzen 9 7950X3D, 64 GiB system RAM,
+and Fedora-family Linux with ROCm 7.1.52802 / Clang 20. The original model used
+for the smoke check was on mechanical RAID0; cold page faults were slow. Keep
+latency-sensitive data on SSD and measure warm residency separately.
+
+- Complete HIP Release build, including the executable and parity targets.
+- All 25 tests selected by the command below passed on 2026-09-29.
+- Intrinsic parity includes signed packed-byte dot products and overflow.
+- CPU/GPU mapped-memory handoff covers delayed publication, changing payloads,
+  and repeated graph replay in both directions.
+- GR coverage includes the maximum eight-token fused batch and graph replay
+  after inputs change. Native QSA checks the scalar HIP fallback.
+- Q8_K quantization is byte-exact against the existing reference. HIP's
+  `__fmul_rn` can become ordinary multiplication; disabling contraction in
+  `quantize_act.cu` preserves the separately rounded product before the magic
+  rounding bias is added. CUDA's compile flags remain unchanged.
+- The mmap regression covers canonical and differing native layer sizes,
+  truncated/oversized files, lookup bounds, and reopening after errors.
+- CUDA 13 / GCC 15 / sm_89 executable build passed; GR, sampler, quantization,
+  and mmap tests passed on the same tree. This is a regression check, not a
+  claim of complete CUDA inference validation.
+- CPU-only `strata-plan` build passed without either GPU backend.
+- Real GSQ-RCO IQ3_XXS server smoke with mmap, the shipped expert profile,
+  automatic GPU cache, and MTP passed arithmetic, executable Python addition,
+  system-marker recall, and 1,170-token batched-prefill recall. All four requests
+  ended normally with correct answers; one engine process served them all.
+  The cache held 11,142 experts (18.08 GiB), with 874 MiB of VRAM free after
+  startup. These short checks do not establish a throughput or quality benchmark.
+
+Build all targets before running the registered focused checks:
+
+```sh
+cmake --build build-hip -j2
+ctest --test-dir build-hip --output-on-failure --timeout 60 \
+  -E '^(ple_parity|platform_memory_test)$'
+```
+
+`ple_parity` requires an external model fixture. `platform_memory_test` requests
+256 MiB of locked memory; the test shell's 8 MiB memlock limit prevented that
+test from passing. These are explicit exclusions, not skipped tests counted as
+passes. Additional unpublished upstream fixture suites are not covered.
+
+The published performance table explicitly identifies the measured development
+snapshot; it must not be read as a benchmark of every subsequent rebase. Vision, long-context stress, broad answer-quality
+equivalence, other AMD cards, and mixed-vendor inference are not validated here.

docs/AMD_HIP_PERFORMANCE.md

@@ -0,0 +1,142 @@
+# RX 7900 XTX support and performance evidence
+
+This opt-in Linux `gfx1100` backend supersedes the initial support in
+[PR #94](https://github.com/Niko1221/Strata/pull/94). It retains HIP runtime and
+wave32 integer-dot compatibility, native mmap layout validation, and MTP, then
+adds HIP MMQ, optional calibrated dense hipBLASLt GEMM, native prefill batching,
+and the host-memory / SSD paths used by the measured configuration. HIP blocking
+expert uploads use one reusable pinned staging buffer, including prompt-cache
+refills, to avoid repeated pageable-source registrations.
+
+## Reproduce the configuration
+
+Build instructions are in [AMD_HIP.md](AMD_HIP.md). Use the repository-pinned
+llama.cpp dependency; do not silently substitute another revision. Build with
+`STRATA_PREFILL_MMQ=ON` and use these runtime variables for the measured arm:
+
+```sh
+export STRATA_PREFILL_MMQ=1
+export STRATA_HIPBLASLT_TUNING="$PWD/tools/hip/gfx1100-hipblaslt-100100.txt"
+export STRATA_PREFILL_RING=96
+export STRATA_IO_THREADS=32
+```
+
+The supplied table is calibrated for gfx1100 and hipBLASLt version 100100.
+It is not a universal ROCm tuning table. The runtime guards architecture,
+library version and actual shape/stride/workspace requirements, falling back
+when a table entry is unavailable or incompatible. HIP MMQ is opt-in at runtime. Default CUDA selection is preserved.
+
+Measured engine configuration: Orca Flash Next IQ3_XXS, native pack plus matching
+GGUF/tokenizer/template, `--mmap-experts --resident-cpu-experts`, fixed ranked
+`--expert-profile`, `--expert-cache auto`, `--prefill 8192`, `--spec 4`,
+`--spec-min-p 0.5`, matching MTP runtime, `--max-context 262144`, `--kv int8`,
+`--kv-resident 32768`, `--pool-workers 15`, `--adapt-every 0`, `--pcie-frac 0`,
+and `--vram-reserve-mib 1024`. Keep PLE on SSD.
+
+`--resident-cpu-experts` copies the complement of the static GPU cache to ordinary
+RAM. It requires mmap, a fixed expert profile, and disabled adaptation. It does
+not lock all data into RAM: temporary prefill borrowing and cache refill may
+still use the mapped fallback. Allocation needs sufficient available RAM;
+on Linux this option requires readable standard cgroup-v2 mounts. For each
+finite cgroup ancestor, the guard credits only `inactive_file` after subtracting
+`file_dirty` and `file_writeback`, capped by current usage; it remains bounded by
+the ancestor limit and host `MemAvailable`, with 8 GiB headroom. This accounts
+for reclaimable clean file cache but cannot reserve memory against concurrent
+system or process allocations.
+The POSIX PLE path issues direct reads through a configurable worker pool.
+
+Use a dedicated idle server, restart it between arms, and capture its engine log:
+
+```sh
+python3 tools/hip/bench_prefill.py \
+  --model MODEL_NAME --url http://127.0.0.1:8080 \
+  --engine-log /path/to/engine.log --label candidate --output candidate.json
+```
+
+The script uses the same synthetic source prompts and request order as the
+reported measurements: a small warmup, then 140/280/140/280 functions, with one
+follow-up after each fresh prompt. It enforces zero reused tokens for the fresh
+prompts and records prefill, decode, wall time and stop reason. It requires the
+unbuffered log to contain exactly one completed timing record per request. The
+128-token output cap is intentional for throughput measurement, not task success.
+Never use cancelled-request timing lines as throughput evidence.
+
+## Final revision benchmark
+
+Measured on 2026-09-29, source revision `9568e78da663e6b224ab96d57baf450717b36e73`.
+HIP executable SHA-256: `705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b`.
+The control and candidate below were both measured anew in this session; no
+historical benchmark values are mixed into the table. The control binary SHA-256
+is `4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7`;
+its source was an existing local runtime snapshot rather than a clean upstream commit.
+Both arms use an RX 7900 XTX 24 GiB / gfx1100, Ryzen 9 7950X3D,
+64 GiB installed RAM, Fedora-family Linux, ROCm 7.1 and a 272 W GPU cap.
+Other model services are stopped for measurement and restored afterward.
+Sampling: temperature 0, top-k 1, top-p 1, min-p 0, seed 42, reasoning disabled.
+
+| Fresh request, execution order | Control prefill t/s | Candidate prefill t/s | Candidate output t/s | Candidate request wall time |
+| --- | ---: | ---: | ---: | ---: |
+| 4,210 tokens, first use | 240.0 | 447.5 | 55.5 | 11.729 s |
+| 8,830 tokens, first use at this size | 484.8 | 873.7 | 55.7 | 12.420 s |
+| 4,210 tokens, warmed | 457.2 | 901.1 | 56.7 | 6.937 s |
+| 8,830 tokens, warmed | 478.9 | 966.5 | 59.2 | 11.314 s |
+
+Each request generated exactly 128 tokens and finished at the intentional length
+cap. All four fresh prompts had zero reused KV tokens. Fresh-prompt prefill was
+1.80–2.02x the freshly measured existing AMD runtime; fresh-request wall time was
+40–46% lower. This is a comparison with our existing HIP runtime, not unmodified
+upstream, which does not provide this backend. The candidate also completed four
+cached follow-ups in 2.95–3.26 seconds. Those follow-ups are not apples-to-apples
+prefill comparisons: cache reuse differs with generated text and checkpoint
+selection. Full sanitized measurements are in
+[the fresh-run JSON](benchmarks/2026-09-29-gfx1100.json).
+
+Before this revision, an intermediate package missing the HIP upload staging
+buffer stalled on the 8,830-token request and was rejected. Restoring staging
+allowed the complete sequence above to finish. That failed attempt is not
+included in the throughput table. No watchdog limit was raised to obtain these
+results.
+
+Zero KV reuse is not equivalent to a cold filesystem cache. First-use speed
+and warmed speed are reported separately. Individual observations do not
+establish confidence intervals or a general rate at every context length.
+
+## Correctness and quality limits
+
+Both HIP and CUDA executables built from the measured source revision. CUDA was
+compile-checked, not performance-tested by this run. The selected HIP CTest suite
+passed **29/29** with the tuning table enabled, including expert-upload readback,
+asynchronous handoff, QSA, MMQ, Lt GEMM, KV streaming and PLE reading. The excluded
+`ple_parity` requires an external fixture; `platform_memory_test` requires a
+larger locked-memory limit than the test account provides. Separately, three
+real IQ3_XXS PLE matrix graph replays passed, and the POSIX direct-file test passed
+queued reads, short EOF, wake, close/drain and reopen checks.
+
+Development checks passed eight MMQ numerical comparisons, four actual Lt GEMM
+comparisons, and native QSA/indexer/embedding parity including tail states,
+chunk continuation and image overrides. Real quantized MMQ relative L2 error
+was approximately 0.0026 against raw-FP32/dequantized-weight reference, reflecting
+Q8 activation arithmetic; this is not bitwise numerical equivalence.
+
+End-to-end coding quality is not established by these numerical checks or capped
+throughput requests. Validate completed tasks with independent runtime tests; this
+contribution makes no broad answer-quality or agentic-reliability claim.
+
+The changes do not claim better model reasoning, verified full-context behavior,
+end-to-end vision validation, Windows HIP, other AMD architectures, or mixed
+AMD/NVIDIA execution. Existing CUDA multi-GPU code remains present but is not
+validation of HIP multi-GPU support.
+
+## Attribution and rejected experiments
+
+The measurements above were taken before this backend was rebased onto engine 0.1.24. The PR's own batched
+embedding gather and QSA indexer append (adapted from
+[PR #108](https://github.com/Niko1221/Strata/pull/108), commit
+`acd487233c0bbe2217a6881c5bb43f8a283b0de5`) were dropped in the rebase: 0.1.24 already does both on every
+backend, bit-exact (C-2, C-4). #108's optional GDN parallel/split path is not included. Existing upstream MMQ orchestration is retained and enabled for
+HIP with AMD architecture identification and the correct backend compilation.
+
+Static 16K chunks, expanded FP16 expert tuning, and a 192-slot ring did not offer
+a consistent end-to-end win in the evaluated workload. The table includes only
+the selected 30 dense-shape rows. Microkernel speedups alone were not sufficient
+to select a configuration. No increase in GPU power cap was used.

docs/DETAILS.md

@@ -12,7 +12,7 @@ New here? Start with the [README](../README.md) - it has everything you need to
 
 ## Speed (measured)
 
-RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1.14 with the settings setup writes
+RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1.26 with the settings setup writes
 (`--prefill auto`, 8-bit KV above 4K, KV streaming from 64K). One code-agent prompt per length, 256 generated tokens,
 MTP speculative decoding on. "262K" is the model's full context window (a 259,943-token prompt). The IQ2_XS row was
 measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
@@ -21,21 +21,27 @@ measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| **Q2_0** | 494 | 1,007 | 1,308 | 1,294 | 1,208 | 967 |
-| **IQ2_XS** | 461 | 811 | 1,238 | 1,136 | 1,071 | 886 |
-| **IQ3_XXS** | 415 | 770 | 1,108 | 1,065 | 1,015 | - |
-| **IQ3_S** | 396 | 737 | 1,070 | 1,070 | 931 | - |
-| **Coder** | 599 | 1,152 | 1,298 | 1,350 | 1,266 | 1,034 |
+| **Q2_0** | 536 | 1,299 | 2,171 | 2,126 | 2,107 | 1,304† |
+| **IQ2_XS** | 534 | 1,256 | 2,092 | 1,754 | 1,752 | 1,181*† |
+| **IQ3_XXS** | 482 | 1,007 | 1,745 | 1,609 | 1,602 | - |
+| **IQ3_S** | 427 | 913 | 1,624 | 1,640 | 1,443 | - |
+| **Coder** | 656 | 1,583 | 2,177 | 2,236 | 2,208 | 1,034** |
+
+Engine 0.1.26; `bench/results/2026-09-29-speed-0126`. At 32K-128K that is 8-28% faster than 0.1.22. † not measured
+again: 0.1.22. \* measured with images on (the image encoder's VRAM reserve leaves fewer experts cached). \*\* not
+measured again: 0.1.14.
 
 ### Output (tokens/s)
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| **Q2_0** | 84.3 | 90.3 | 73.6 | 69.0 | 67.2 | 60.3 |
-| **IQ2_XS** | 74.4 | 73.8 | 71.5 | 64.3 | 59.8 | 52.8 |
-| **IQ3_XXS** | 60.3 | 62.1 | 51.4 | 50.0 | 45.8 | - |
-| **IQ3_S** | 51.5 | 51.6 | 48.2 | 48.8 | 40.5 | - |
-| **Coder** | 53.3 | 50.6 | 53.3 | 50.8 | 44.0 | 42.8 |
+| **Q2_0** | 87.3 | 93.0 | 81.8 | 76.2 | 73.7 | 60.3† |
+| **IQ2_XS** | 79.6 | 78.6 | 76.3 | 63.7 | 62.7 | 52.8† |
+| **IQ3_XXS** | 61.9 | 61.6 | 58.5 | 57.2 | 49.0 | - |
+| **IQ3_S** | 52.4 | 53.3 | 48.3 | 46.3 | 45.5 | - |
+| **Coder** | 58.9 | 55.1 | 54.9 | 53.2 | 43.0 | 42.8† |
+
+Engine 0.1.26, the same runs. † not measured again: 0.1.14.
 
 Output speed depends on the text as well: speculative decoding runs faster when more of the drafted tokens are
 accepted, so a different answer to the same prompt moves it by several percent. Run back to back on the 4K prompt,
@@ -58,6 +64,28 @@ halves the KV cache's memory with a Hadamard rotation before 4-bit rounding (PR
 is measurably less precise on long documents (perplexity +8-12%; needle tests still pass). 8-bit stays the default.
 Details: [`bench/results/2026-09-27-kv-q4`](../bench/results/2026-09-27-kv-q4/README.md).
 
+**Hybrid K8V4 KV cache (engine 0.1.25, optional, PR #120):** `--kv k8v4` (`START-HERE.bat --setup --kv k8v4`) keeps
+the keys at 8 bits and stores the values as rotated 4-bit: 23% less KV memory than 8-bit, so more experts fit in
+VRAM. RTX 3090, the Coder at 198K context: 99 instead of 85 tokens/s output, the same needle results, prompts 2-5%
+slower. It does not stream its KV cache (KV streaming is on by default from 64K), so it pays off mostly on large
+cards at long contexts.
+
+**The draft layer's tokens (0.1.27, `--draft-vocab`):** the MTP draft layer can only propose tokens from a subset
+of the vocabulary (`mtp/rt/draft_vocab.bin`). Since 0.1.27 the subset includes every Chinese, Japanese and Korean
+token (106,299 ids), so answers in those languages are 15-38% faster (Q2_0, RTX 5070). Its head takes ~180 MiB of
+VRAM, which the expert cache leaves free for it (0.1.28). `START-HERE.bat --setup --draft-vocab en` keeps the
+English/code subset from before (40,525 ids, ~110 MiB less VRAM, English answers 1-2% faster; CJK answers get
+almost no drafts). `tools/draft_vocab.py` builds and inspects subsets.
+
+**Low-RAM mode (engine 0.1.26, chosen by setup):** normally all of a model's experts are copied into RAM (23-50 GB,
+pinned) and the GPU holds a copy of the most-used ones. On a PC whose RAM cannot hold them beside the system (the
+experts plus ~10 GB), setup instead maps them from one file in the model's folder (`--mmap-experts`, the pack's
+`experts.bin`, +23-50 GB of disk). The OS file cache holds what the GPU does not, and it can give that memory back.
+On the Coder the engine's committed memory drops from 36 to ~13 GB, with the same answers. With a big GPU (an RTX
+5090 holds all of the Coder's experts, most of Q2_0's) it runs at nearly the usual speed. With a small one, most
+experts come from the SSD and it is much slower (setup says so). `START-HERE.bat --setup --low-ram on|off` overrides
+the choice.
+
 Time to first token is prompt length / prompt speed: with Q2_0 about 4 s at 4K, 25 s at 32K, under 2 minutes at 128K
 and 4.5 minutes at 262K (engine 0.1.13 made long prompts about twice as fast, below).
 
@@ -148,7 +176,7 @@ You need **only an NVIDIA driver** (version 580 or newer; update it with the NVI
 
 | | |
 | --- | --- |
-| GPU | NVIDIA **RTX 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070; RTX 30/40 are untested. |
+| GPU | NVIDIA **RTX 20, 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070 and an RTX 3090; RTX 20 (Turing, since 0.1.27) was tested by a contributor on an RTX 2070. |
 | RAM | **64 GB** recommended (see the table above). |
 | CPU | x86-64 with AVX2 (any Intel/AMD desktop CPU from the last ~8 years). AVX-512 (Ryzen 7000/9000) is a bit faster. |
 | Disk | ~70-80 GB free for the model, ~6 GB for the MTP layer (+1 GB with images). **Q2_0 on an AVX-512 CPU** also writes a one-time ~40 GB copy of its experts for the fast CPU kernel. An NVMe SSD is strongly recommended. |
@@ -160,7 +188,7 @@ elsewhere) finds them and sets itself up the same way. The place is remembered p
 `~/.config/strata/settings.json`); `--data-dir` chooses another. Installs from before 0.1.16 are moved there by the next
 start (a rename on the same drive; files on another drive are used where they are).
 Python 3.12 if you have none (for your user account, no admin), a private Python environment, NVIDIA's CUDA libraries
-(from pip, ~0.4 GB), the ready-made Strata engine for RTX 30/40/50, the model and the MTP draft layer. If no
+(from pip, ~0.4 GB), the ready-made Strata engine for RTX 20/30/40/50, the model and the MTP draft layer. If no
 ready-made engine fits your PC, it offers to install the build tools (Visual Studio Build Tools + CUDA Toolkit on
 Windows, `build-essential` + CUDA on Ubuntu) and compiles the engine for your GPU (asks first; 20-40 minutes once).
 
@@ -216,6 +244,28 @@ install; `START-HERE.bat --calibrate` (Linux: `./setup.sh --calibrate`) does it
 speed with each setting and keeps one only when it is more than 3% faster. The result is remembered per PC and model
 (in the settings file next to the data folder's record), so updates keep it.
 
+### Running it at startup (Task Scheduler)
+
+To have the model up at logon, people start the serve from **Task Scheduler** (or a service). Beware: Windows
+throttles such contexts, and the model's ~40 GB expert load then crawls at **~0.05 GiB/s (13-14 minutes)**
+instead of **~1.4-1.5 GiB/s (~35 seconds)** - a 24x slower start. Measured on an RTX 5070 Ti + Ryzen 7 9800X3D
++ NVMe, same binary, same args, same cache state:
+
+| How the serve starts | Expert load |
+| --- | ---: |
+| Double-click / terminal / SSH | 1.42-1.52 GiB/s (~35 s) |
+| Task Scheduler with its defaults | 0.05 GiB/s (821-841 s) |
+| Task Scheduler with the two settings below | 1.42 GiB/s (35 s) |
+
+In the task's properties set both of these (the defaults are the opposite):
+
+- **Priority level: Normal** (Options tab; the default is Below normal), and
+- **Run with highest privileges** (General tab; without it the task runs with a limited user token - which
+  also strips `SeLockMemoryPrivilege`, the privilege Windows large pages need).
+
+(Both were changed at once, so the isolated effect of each is not measured.) If the model still starts
+slowly, the engine prints a hint under its `loaded ... GiB at ...` line naming this cause.
+
 ### Chat in the terminal (optional)
 
 ```
@@ -254,11 +304,14 @@ The server listens on `http://127.0.0.1:8080` (change with `--port` in setup, or
 | --- | --- |
 | OpenAI Chat Completions (stream and non-stream, tools) | `POST /v1/chat/completions` |
 | Anthropic Messages (stream and non-stream, tools) | `POST /v1/messages` |
-| Model list / health | `GET /v1/models`, `GET /health` |
-| What the model is doing right now | `GET /status` |
+| Model list / health | `GET /v1/models`, `GET /models`, `GET /health` |
+| Model properties | `GET /props` (also accepts `?model=<loaded-model-id>`) |
+| What the model is doing right now | `GET /status`, `GET /slots` (single slot, busy or idle) |
 | Everything the Monitor tab shows (engine, live state, last requests, hardware) | `GET /metrics` |
 | The MCP servers, their state and tools ([below](#tools-from-mcp-servers)) | `GET /mcp` |
 
+`/models` and `/v1/models` list only the loaded model, with its context limit and input modalities. `/props` exposes the original chat template, context limit, configured generation defaults (shared settings take precedence), model path and engine version when available. Context means the full engine context, not the resident KV window. `n_predict: -1` means no fixed output cap. Unconfigured sampling fields are omitted. `autoload` has no effect; an unknown `model` returns 404. These metadata endpoints and `/slots` require the API key when one is configured. They do not load, unload or restart models.
+
 ```bash
 curl http://127.0.0.1:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
   "model": "strata", "messages": [{"role": "user", "content": "Write a haiku about GPUs."}], "max_tokens": 512 }'

docs/MULTI_GPU.md

@@ -12,14 +12,41 @@ needed; cards on x4 or x1 slots work, and the PCIe share of each card is probed
 
 ## Using it
 
-At setup time (Windows: `START-HERE.bat`, Linux: `./setup.sh`):
+**Nothing to type.** `START-HERE.bat` (Linux: `./setup.sh`) lists your NVIDIA cards and says for each one whether
+Strata can use it:
 
 ```
---gpus 0,2                 the cards, as nvidia-smi numbers them; the first is the main one
+  Your NVIDIA GPUs:
+    GPU 0: NVIDIA GeForce RTX 5080, 16 GB VRAM - can be used
+    GPU 1: NVIDIA GeForce GTX 1080 Ti, 11 GB VRAM - not supported - older than the RTX 20 series (compute capability 6.1; Strata needs 7.5 or newer)
+    GPU 2: NVIDIA GeForce RTX 3090, 24 GB VRAM - can be used
+  ...
+  1) GPU 0 (NVIDIA GeForce RTX 5080, 16 GB) + GPU 2 (NVIDIA GeForce RTX 3090, 24 GB) together   (recommended)
+  2) GPU 2 (NVIDIA GeForce RTX 3090, 24 GB) only
+  3) GPU 0 (NVIDIA GeForce RTX 5080, 16 GB) only
+Which GPUs? [1]:
+```
+
+When two or more cards can share the model, the two best together are recommended (the newest generation first:
+it becomes the main card). A model installed on one card asks once, at its next start, whether to use both from
+now on; the answer is kept.
+
+**Choosing yourself** (at setup or at any start):
+
+```
+--gpus 0,2                 these cards together, as nvidia-smi numbers them; the first is the main one. Remembered.
+--gpus all                 every card that can share the model
+--gpu 0                    one card (at a start: for that start only)
 --layer-split auto         (default) or the first layer of each later card, e.g. 18 or 16,32
 ```
 
-or in an existing config (`strata-*.json`), then restart:
+**Not supported** (setup says so and names the cards that can be used instead):
+- a card older than the RTX 20 series (compute capability below 7.5: GTX 10 and older);
+- a card with less than 8 GB of VRAM, together with others (each card holds a copy of the dense weights and its
+  own prompt buffers);
+- AMD and Intel GPUs, and a mix of NVIDIA with them.
+
+Or edit an existing config (`strata-*.json`), then restart:
 
 ```json
 "gpu": [0, 2],
@@ -52,15 +79,25 @@ into the card that owns the layer.
 
 ## Limits (for now)
 
-- Not with images (`--vision`), control vectors / the speed projection, KV streaming (`--kv-resident`), the older
-  helper-GPU experiment (`--expert-cache-remote`, docs/SECOND_GPU.md) or `--mmap-experts`.
+- **Works across cards** (bench/results/2026-09-29-layer-split-limits):
+  - images (`--vision`): each card keeps its own image-position table;
+  - control vectors and the experimental speed projection: each card holds the vector's tables, switched on and
+    off per request on all of them;
+  - KV streaming (`--kv-resident`): each card streams the KV of its own session;
+  - mid-prompt checkpoints (`--prompt-cache-every`): each card saves its part of a checkpoint when it has read that
+    chunk;
+  - the older helper-GPU caches (`--expert-cache-remote`, docs/SECOND_GPU.md): they take the visible GPUs no stage
+    runs on, and hold only experts no stage's cache holds. On the test rig, a 2080 Ti helper made decoding slower,
+    as it did without a split: its per-layer round trip costs more than the CPU pool needs for those experts.
+- `--mmap-experts` needs a canonical pack (`experts.bin`), with or without a split; a native (IQ) pack says so at
+  start.
 - The prompt path has its own buffers on every card (1.5 GB each at the default 2048-token chunk; `--prefill 1024`
-  halves that) instead of borrowing cache slots as one card does.
-- Mid-prompt checkpoints (`--prompt-cache-every`) are off; the ones at turn boundaries - what a chat reuses - stay.
+  halves that) instead of borrowing cache slots as one card does. An explicit `--expert-cache` on the first card is
+  capped to leave room for them.
 - On Windows only 8 GiB of the expert arena is pinned (more, mapped into two GPU contexts, leaves WDDM refusing
   allocations); the PCIe share covers those layers.
-- Every card needs compute capability 8.0 (RTX 30 or newer). A Turing card (RTX 20, sm_75) builds only with the
-  experimental `-DSTRATA_EXPERIMENTAL_SM75=ON`.
+- Every card needs compute capability 7.5 (RTX 20 or newer). The pre-sm_80 QSA scorer path is fp32 FMAs, so a
+  Turing card runs the same kernels instead of the tensor-core prompt attention.
 
 ## Measured
 

docs/benchmarks/2026-09-29-gfx1100.json

@@ -0,0 +1,299 @@
+{
+  "source_commit": "9568e78da663e6b224ab96d57baf450717b36e73",
+  "date": "2026-09-29",
+  "candidate_binary_sha256": "705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b",
+  "note": "Fresh same-session control and candidate runs. Throughput cap 128; not a coding-quality benchmark.",
+  "control": [
+    {
+      "kind": "warmup",
+      "trial": 0,
+      "wall_s": 0.45770244899904355,
+      "metrics": {
+        "prompt_tokens": 15.0,
+        "reused": 0.0,
+        "fresh": 15.0,
+        "prefill_ms": 335.0,
+        "prefill_tps": 44.8,
+        "generated": 2.0,
+        "decode_ms": 115.0,
+        "decode_tps": 17.5
+      },
+      "finish_reason": "stop"
+    },
+    {
+      "kind": "cold",
+      "trial": 1,
+      "wall_s": 20.068275763012934,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 17538.0,
+        "prefill_tps": 240.0,
+        "generated": 128.0,
+        "decode_ms": 2518.0,
+        "decode_tps": 50.8
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 1,
+      "wall_s": 4.412000925018219,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 1943.0,
+        "prefill_tps": 58.7,
+        "generated": 128.0,
+        "decode_ms": 2451.0,
+        "decode_tps": 52.2
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 2,
+      "wall_s": 20.731399575975956,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 18213.0,
+        "prefill_tps": 484.8,
+        "generated": 128.0,
+        "decode_ms": 2505.0,
+        "decode_tps": 51.1
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 2,
+      "wall_s": 5.0592734439997,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 2601.0,
+        "prefill_tps": 95.4,
+        "generated": 128.0,
+        "decode_ms": 2441.0,
+        "decode_tps": 52.4
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 3,
+      "wall_s": 11.695648370019626,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 9209.0,
+        "prefill_tps": 457.2,
+        "generated": 128.0,
+        "decode_ms": 2477.0,
+        "decode_tps": 51.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 3,
+      "wall_s": 4.093982424004935,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 1896.0,
+        "prefill_tps": 60.1,
+        "generated": 128.0,
+        "decode_ms": 2188.0,
+        "decode_tps": 58.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 4,
+      "wall_s": 20.84539221401792,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 18437.0,
+        "prefill_tps": 478.9,
+        "generated": 128.0,
+        "decode_ms": 2392.0,
+        "decode_tps": 53.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 4,
+      "wall_s": 5.357054464984685,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 2658.0,
+        "prefill_tps": 93.3,
+        "generated": 128.0,
+        "decode_ms": 2682.0,
+        "decode_tps": 47.7
+      },
+      "finish_reason": "length"
+    }
+  ],
+  "optimized": [
+    {
+      "kind": "warmup",
+      "trial": 0,
+      "wall_s": 0.46265592801501043,
+      "metrics": {
+        "prompt_tokens": 15.0,
+        "reused": 0.0,
+        "fresh": 15.0,
+        "prefill_ms": 335.0,
+        "prefill_tps": 44.8,
+        "generated": 2.0,
+        "decode_ms": 121.0,
+        "decode_tps": 16.5
+      },
+      "finish_reason": "stop"
+    },
+    {
+      "kind": "cold",
+      "trial": 1,
+      "wall_s": 11.728894605010282,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 9408.0,
+        "prefill_tps": 447.5,
+        "generated": 128.0,
+        "decode_ms": 2308.0,
+        "decode_tps": 55.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 1,
+      "wall_s": 2.9531024169991724,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 740.0,
+        "prefill_tps": 154.0,
+        "generated": 128.0,
+        "decode_ms": 2193.0,
+        "decode_tps": 58.4
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 2,
+      "wall_s": 12.42044531099964,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 10107.0,
+        "prefill_tps": 873.7,
+        "generated": 128.0,
+        "decode_ms": 2298.0,
+        "decode_tps": 55.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 2,
+      "wall_s": 3.2599652509961743,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 1070.0,
+        "prefill_tps": 231.7,
+        "generated": 128.0,
+        "decode_ms": 2173.0,
+        "decode_tps": 58.9
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 3,
+      "wall_s": 6.937167472002329,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 4672.0,
+        "prefill_tps": 901.1,
+        "generated": 128.0,
+        "decode_ms": 2256.0,
+        "decode_tps": 56.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 3,
+      "wall_s": 3.1877827199932653,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 827.0,
+        "prefill_tps": 137.8,
+        "generated": 128.0,
+        "decode_ms": 2350.0,
+        "decode_tps": 54.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 4,
+      "wall_s": 11.314008287008619,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 9136.0,
+        "prefill_tps": 966.5,
+        "generated": 128.0,
+        "decode_ms": 2161.0,
+        "decode_tps": 59.2
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 4,
+      "wall_s": 3.0476601750124246,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8957.0,
+        "fresh": 114.0,
+        "prefill_ms": 734.0,
+        "prefill_tps": 155.3,
+        "generated": 128.0,
+        "decode_ms": 2296.0,
+        "decode_tps": 55.7
+      },
+      "finish_reason": "length"
+    }
+  ],
+  "control_binary_sha256": "4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7"
+}

include/strata/artifact/gguf_reader.hpp

@@ -26,6 +26,7 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <set>
 #include <stdexcept>
 #include <algorithm>
 
@@ -418,9 +419,14 @@ private:
             meta_.emplace(std::move(key), read_value(c, t));
         }
         tensors_.reserve((size_t)n_tensors);
+        // GGUF has no index to arbitrate between two tensors of one name: find() is first-match, so a
+        // duplicate would silently win by position.  Refuse the file at open instead, naming both.
+        std::set<std::string> names;
         for (uint64_t i = 0; i < n_tensors; ++i) {
             TensorInfo t;
             t.name = c.str();
+            if (!names.insert(t.name).second)
+                throw std::runtime_error("GGUF: duplicate tensor name '" + t.name + "' in " + path_);
             const uint32_t nd = c.read<uint32_t>();
             if (nd == 0 || nd > 4) throw std::runtime_error("GGUF: bad n_dims for " + t.name);
             t.shape.resize(nd);

include/strata/core/expert_cache.hpp

@@ -26,6 +26,7 @@
 // kernel. This is that first step, and the step it unblocks is the one that can be measured.
 #pragma once
 
+#include <cstddef>
 #include <cstdint>
 #include <string>
 #include <vector>
@@ -126,6 +127,10 @@ public:
     /// against a `cudaStreamNonBlocking` one.  The first version used the async form and `verify_slot` refused
     /// the whole run with "slot 0 differs from the arena at byte 0" - which is the check doing its job.
     bool fill_slot_blocking(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes = 0);
+    /// perf-review D-4: `fill_slot_blocking`'s copy on the same (legacy) stream, but queued: many slots are refilled
+    /// with one `sync_queued` at the end instead of a wait per slot. Same ordering against earlier work, same bytes.
+    bool fill_slot_queued(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes = 0);
+    bool sync_queued(std::string& err);
 
     /// Reads `slot` back to the host and compares it to `host_blob`, byte for byte.  **THE ONLY THING THAT SAYS
     /// THE CACHE HOLDS THE EXPERT IT CLAIMS TO.**  A slot table that is right about indices and wrong about
@@ -136,6 +141,11 @@ public:
     int64_t fills() const { return fills_; }
 
 private:
+#if defined(STRATA_USE_HIP)
+    bool ensure_blocking_staging(std::size_t bytes, std::string& err);
+    uint8_t* blocking_staging_ = nullptr;
+    std::size_t blocking_staging_bytes_ = 0;
+#endif
     uint8_t* base_ = nullptr;
     std::vector<int32_t> residency_;   ///< [n_layers * n_expert] -> slot or kNotResident
     int64_t slots_ = 0;

include/strata/core/expert_source.hpp

@@ -26,6 +26,8 @@
 #include "strata/kernels/cpu/pool.hpp"
 
 #include <cstdint>
+#include <cstddef>
+#include <utility>
 #include <string>
 #include <vector>
 
@@ -33,6 +35,41 @@ namespace strata::core {
 
 class RemoteExperts;
 
+namespace detail {
+
+/// Sentinel used by the pure complement planner for a blob that remains in the mmap fallback.
+inline constexpr uint64_t kNoCacheComplement = ~uint64_t{0};
+
+/// Required cgroup-v2 usage counters for the conservative cache-reclaim allowance.
+struct CgroupMemoryStat {
+    uint64_t current = 0;
+    uint64_t inactive_file = 0;
+    uint64_t file_dirty = 0;
+    uint64_t file_writeback = 0;
+    bool valid = false;
+};
+
+/// Calculate additional bytes under a finite cgroup limit after reclaiming only clean inactive file cache.
+/// Returns false when the required memory.stat counters were unavailable.
+bool cgroup_available_bytes(uint64_t limit, const CgroupMemoryStat& stat, uint64_t& bytes);
+
+/// Build compact offsets for experts absent from both the primary GPU cache and an optional second GPU tier.
+/// Kept CPU-only so selection and byte accounting can be tested without initializing a GPU.
+bool make_cache_complement_plan(
+    int64_t n_layers, int64_t n_expert, const std::vector<uint64_t>& layer_blob_bytes,
+    const std::vector<std::pair<int32_t, int32_t>>& primary_gpu_pairs,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs,
+    std::vector<uint64_t>& offsets, uint64_t& bytes, std::string& err);
+
+/// Resolve one blob through the compact copy when present, otherwise preserve its exact mapped-file fallback.
+const uint8_t* cache_complement_blob_or_fallback(
+    size_t index, const std::vector<uint64_t>& offsets, const uint8_t* complement_host,
+    const uint8_t* mapped_fallback);
+
+}  // namespace detail
+
+
+
 /// Where one routed expert's bytes come from.
 ///
 /// Phase 2 has NO cache (`phase-2-correct-engine.md`: hit rate `h = 0`), so the only implementation is a
@@ -42,7 +79,7 @@ class ExpertSource {
 public:
     virtual ~ExpertSource() = default;
 
-    /// The 1,382,400-byte blob for `(layer, expert)`, or nullptr if it cannot be produced.
+    /// The expert-layout blob for `(layer, expert)`, or nullptr if it cannot be produced.
     ///
     /// The pointer only has to stay valid until the next `blob()` call: with `h = 0` every expert is computed
     /// immediately and nothing is retained.  A CACHING source must return pointers into the cache, not into a
@@ -270,26 +307,49 @@ public:
     FileExpertSource(const FileExpertSource&) = delete;
     FileExpertSource& operator=(const FileExpertSource&) = delete;
 
-    /// Maps `<pack_dir>/experts.bin` and checks its size against `n_layers * n_expert * BLOB`.
+    /// Maps `<pack_dir>/experts.bin` and checks its size against the loaded expert layout.  Canonical packs use
+    /// `n_layers * n_expert * BLOB`; native packs use their variable per-layer blob sizes and offsets.
     ///
     /// The size check is not a formality: a short file would fault at the END of a long sequence, and an
     /// over-long one means the pack is not the one the geometry came from.  Refuses with the two numbers.
     bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err);
+    /// Pin a compact host mirror of experts absent from a fully filled static GPU cache. The mmap remains open
+    /// as a fallback for later cache reloads. This is opt-in because the complement may still be a large allocation.
+    bool pin_cache_complement(
+        const ExpertCache& cache, std::string& err, bool pin = true,
+        const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs = {});
     void close();
 
     bool mapped() const { return base_ != nullptr; }
     int64_t blobs() const { return blobs_; }
+    uint64_t pinned_bytes() const { return complement_pinned_ ? complement_bytes_ : 0; }
+    uint64_t resident_bytes() const { return complement_bytes_; }
+    bool complement_pinned() const { return complement_pinned_; }
 
     const uint8_t* blob(int64_t layer, int64_t expert) override;
+    bool pinned(int64_t layer, int64_t expert) const override;
+    const uint8_t* device_alias(int64_t layer, int64_t expert) const override;
 
     /// Blobs touched, for the driver to report.  With `h = 0` this is `48 * k` per token and the number is only
     /// interesting once Phase 3 makes it not so.
     int64_t reads() const override { return reads_; }
 
 private:
+    const uint8_t* mapped_blob(int64_t layer, int64_t expert) const;
+    static constexpr uint64_t kNoComplement = detail::kNoCacheComplement;
     const uint8_t* base_ = nullptr;
     int64_t blobs_ = 0;
+    int64_t n_layers_ = 0;
     int64_t n_expert_ = 0;
+    uint64_t mapped_bytes_ = 0;
+    std::vector<uint64_t> layer_offsets_, layer_blob_bytes_;
+    void* complement_arena_ = nullptr;
+    const uint8_t* complement_host_ = nullptr;
+    const uint8_t* complement_device_ = nullptr;
+    uint64_t complement_bytes_ = 0;
+    std::vector<uint64_t> complement_offsets_;
+    bool complement_pinned_ = false;
+    bool complement_ready_ = false;
     int64_t reads_ = 0;
 #if defined(_WIN32)
     void* file_ = nullptr;
@@ -341,6 +401,12 @@ public:
     /// only true if the engine says what it got.
     const std::string& note() const { return note_; }
     double load_gib_per_second() const { return gib_per_s_; }
+    // Loader fix: the load, split.  `load_seconds()` is the wall clock of the load loop; the other two are
+    // sums over the reader threads (see LoadStats), so on their own they say how much of that wall was spent
+    // waiting for the disk and how much in memcpy + FNV-1a.
+    double load_seconds() const { return load_seconds_; }
+    double load_read_seconds() const { return load_read_s_; }
+    double load_copy_seconds() const { return load_copy_s_; }
 
 private:
     void* arena_ = nullptr;          ///< the PinnedArena, owned
@@ -352,6 +418,9 @@ private:
     int64_t reads_ = 0;
     std::string note_;
     double gib_per_s_ = 0.0;
+    double load_seconds_ = 0.0;
+    double load_read_s_ = 0.0;
+    double load_copy_s_ = 0.0;
     uint64_t pinned_bytes_ = 0;
     std::string gguf_;
 };

include/strata/core/layer.hpp

@@ -40,6 +40,8 @@
 #include "strata/kernels/ngram.hpp"
 #include "strata/kernels/ple.hpp"
 
+#include <cstdio>
+#include <cstdlib>
 #include <cstdint>
 #include <string>
 
@@ -211,6 +213,10 @@ struct QsaState {
     bool kv_q4 = false;
     uint8_t* k_q4 = nullptr;
     uint8_t* v_q4 = nullptr;
+    /// Hybrid K8V4: K in INT8 (unrotated - the scores stay exact), V in rotated Q4_0 (kv_q4.hpp): 816 B per cell.
+    /// Uses k_q/k_scale + v_q4. Mode 0 only (no KV streaming, no ring); under this setting the MTP drafter's
+    /// ring state stays plain INT8, so the block movers never see the hybrid layout.
+    bool kv_hybrid = false;
     int32_t* page_table = nullptr;   ///< (n_pages,) logical page -> physical page (-1: not resident, streamed)
     int64_t n_pages = 0;
     int64_t max_cells = 0;
@@ -276,8 +282,17 @@ bool qsa_kv_int8();
 /// PR #21: store K/V as Q4_0 after a Hadamard rotation (`--kv q4_0`): 576 B per cell, vs 1,056 in INT8.
 void qsa_set_kv_q4(bool enabled);
 bool qsa_kv_q4();
+/// Hybrid K8V4 (`--kv k8v4`): K in INT8, V in rotated Q4_0 - 816 B per cell. Not with --kv-resident.
+void qsa_set_kv_hybrid(bool enabled);
+bool qsa_kv_hybrid();
 /// The state's KV format for the block-moving functions of kv_stream.hpp (kKvF16 / kKvInt8 / kKvQ4).
 inline int qsa_kv_format(const QsaState& st) {
+    // A hybrid K8V4 state is mode 0 only and never reaches the block movers; refuse rather than let it
+    // fall through to kKvF16 - a wrong layout silently applied is worse than a hard stop (PR review).
+    if (st.kv_hybrid) {
+        std::fprintf(stderr, "strata: qsa_kv_format: a hybrid K8V4 state must never reach the block movers\n");
+        std::exit(1);   // the kernels' own "unsupported geometry" convention (kv_q8.cu, qsa_decode_attn.cu)
+    }
     return st.kv_q4 ? strata::kernels::kKvQ4 : st.kv_int8 ? strata::kernels::kKvInt8 : strata::kernels::kKvF16;
 }
 uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, QsaState& st,

include/strata/core/mtp.hpp

@@ -53,6 +53,9 @@ public:
     /// KV streaming: refill the ring of the drafter's window from its host copy for a sequence that continues at
     /// `upto` (a conversation-cache resume). No-op unless the drafter's K/V is a ring.
     void kv_restore(int64_t upto);
+    /// The VRAM bind() will allocate for a native head of `head_row_bytes` per vocabulary row: the draft logits and
+    /// the draft head over rt/draft_vocab.bin's subset.  The expert cache is sized before bind(), so it reserves this.
+    uint64_t bind_bytes(uint64_t head_row_bytes, int64_t n_vocab) const;
     /// The main model's embedding and head, and the verify window's final residuals (T rows, hc*n_embd each).
     bool bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err);
 
@@ -73,9 +76,23 @@ public:
     double ms_draft = 0, ms_prefill = 0;
     int64_t rounds = 0;
 
+    /// E-9: the prompt path computes this layer's prompt K/V in batches (Prefill::draft_kv): its tensors, its K/V
+    /// state, the first cell a round can still read, its device, and a wait for its own stream.
+    const float* tensor_f32(const char* name) const { return f32(name); }
+    const uint16_t* tensor_bf16(const char* name) const { return bf16(name); }
+    const void* tensor_q8(const char* name) const { return q8(name); }
+    QsaState& kv_state_rw() { return st_; }
+    int64_t first_needed() const { return (window_ > 0 && prompt_len_ > 0) ? prompt_len_ - window_ - 64 : 0; }
+    int device() const { return device_; }
+    bool idle(std::string& err) {
+        if (cs_ && cudaStreamSynchronize(cs_) != cudaSuccess) { err = "mtp: its stream failed"; return false; }
+        return true;
+    }
+
 private:
     bool record_forward(int T, int step_row0, cudaStream_t cs, std::string& err);
     bool capture_prefill(int T, std::string& err);
+    bool capture_prefill_dev(int T, std::string& err);   ///< E-4: without the mapped staging (inputs copied on device)
     bool capture_round(int T, std::string& err);
     bool capture_step(int j, std::string& err);
     cudaGraphExec_t step_exec_[9] = {};
@@ -95,6 +112,9 @@ private:
     uint64_t vram_ = 0;
     cudaStream_t cs_ = nullptr;
     cudaGraphExec_t prefill_exec_[9] = {};
+    cudaGraphExec_t prefill_dev_exec_[9] = {};
+    int32_t* pf_dev_ = nullptr;   ///< E-4: a prompt's rows' token / step / position records, uploaded at once
+    int64_t pf_cap_ = 0;          ///< its capacity in ints
     cudaGraphExec_t round_exec_[9] = {};
 
     struct Tensor { std::string name, kind; int64_t rows = 0, cols = 0; uint64_t off = 0, bytes = 0; };

include/strata/core/pinned.hpp

@@ -52,15 +52,40 @@ struct PinnedArena {
 
 struct LoadStats {
     double seconds = 0.0;
+    // Loader fix: the two halves of `seconds`, both SUMMED OVER THE READER THREADS rather than wall clock, so
+    // either can exceed `seconds` when the threads overlap.  They exist to separate the read from the copy and
+    // the hash: an 8 MiB chunk that reaches the disk as ~2048 4 KiB reads spends its time in the first one.
+    double read_seconds = 0.0;                  // inside the read call only: no seek, no copy, no hash
+    double copy_seconds = 0.0;                  // memcpy + FNV-1a
+    // A short read is not a slow load, it is a WRONG one: the caller must refuse the pack rather than run
+    // on an arena whose tail was never written.  `seconds` alone cannot say that (it is -1.0 on failure, but
+    // the caller's own check is on the REQUESTED byte count, which a refused read does not change).
+    bool ok = true;                             // false: the load failed, see `error`
+    std::string error;                          // why it failed, for the caller's message
     uint64_t bytes = 0;
     uint64_t layers = 0;
     std::vector<uint64_t> layer_checksums;      // one FNV-1a per layer
     double gib_per_second() const { return seconds > 0 ? (double) bytes / (1024.0 * 1024 * 1024) / seconds : 0.0; }
+    // Aggregate rate of the readers WHILE THEY WERE INSIDE THE READ CALL: `bytes` over the mean per-thread
+    // read time.  `gib_per_second()` above divides the same bytes by the wall clock of the whole loop, which
+    // also contains the copy and the hash, so the two differ by how much of the loop was not I/O.
+    double read_gib_per_second(int threads) const {
+        if (read_seconds <= 0.0 || threads < 1) return 0.0;
+        return (double) bytes / (1024.0 * 1024 * 1024) / (read_seconds / (double) threads);
+    }
 };
 
 // Load `layers` layers of the expert arena into `dst` with `threads` readers, `chunk` bytes at a time.
 // Each thread opens its OWN handle and seeks, which is the portable form of parallel pread: a shared handle
 // needs a lock around the seek and defeats the parallelism on Windows.
+//
+// The reads go through `fread()` on a per-thread `FILE*`, NOT through `std::ifstream`.  MSVC's
+// `basic_filebuf::xsgetn` splits every request larger than `_INTERNAL_BUFSIZ - 1` into 4095-byte `fread()`
+// calls, so an 8 MiB `chunk` reached the disk as ~2048 4 KiB operations and the chunking did nothing at all
+// (measured on a 42.9 GB pack: 1222 s, 4096 bytes per operation, 0.03 GiB/s).  `fread()` hands a request
+// larger than the stream buffer straight to `_read()`/`ReadFile()`, which is what `chunk` is for.  The
+// control flow, the copy, the per-layer FNV-1a and therefore the checksums are unchanged, so a pack loaded
+// by the old and the new reader must produce identical `layer_checksums`.
 LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_bytes, uint64_t blobs_per_layer,
                        uint64_t layers, int threads, uint64_t chunk);
 /// Plan v0.3 P6: the same with one byte range per layer (`layer_off[L]`, `layer_bytes[L]`).

include/strata/core/verify.hpp

@@ -32,6 +32,7 @@
 
 #include <cstdint>
 #include <string>
+#include <vector>
 
 namespace strata::core {
 
@@ -44,6 +45,8 @@ using PoolMultiFn = void (*)(void* user, const float* x_f, const int32_t* ids, i
 struct VerifyHits {
     const int32_t* d_res = nullptr;      ///< device [n_layers * n_expert] slot or -1
     const uint8_t* cache_base = nullptr; ///< slot 0 of the VRAM expert arena
+    const uint64_t* slot_off = nullptr;   ///< E-6: host per-slot offsets when slots differ in size (null: slot * blob)
+    int64_t n_slots = 0;                  ///< E-6: how many (for the device copy)
     int64_t blob = 0;
 };
 
@@ -122,9 +125,13 @@ public:
     /// arena directly, 2 = a copy kernel stages it inside the graph (no API calls on the pool's thread; best when
     /// the CPU is RAM-bound, Q2_0).  Set before the first `run`.
     void set_pcie_mode(int mode) { sink_.pcie_mode = mode; }
+    /// the pool never plans a PCIe share (--pcie-frac 0): the window skips that path.  Before the first run.
 
     double ms_wait = 0, ms_pool = 0, ms_host = 0, ms_commit = 0;
     int64_t windows = 0;
+    /// STRATA_VERIFY_PROFILE=1 - GPU stage times of the windows since the last call (ms per
+    /// window), as one line; empty when off.
+    std::string profile_report();
 
 private:
     bool capture(int T, std::string& err);
@@ -138,6 +145,9 @@ private:
     int hist_len_ = 0;
     bool head_sampling_ = true;          ///< set_head_sampling
     int device_ = -1;                    ///< the device `init` ran on: run/commit switch to it (layer split)
+    bool device_plan_ = false;            ///< E-6: resident-only layers planned on the device (STRATA_VERIFY_DEVICE_PLAN)
+    uint32_t* skip_ = nullptr;            ///< E-6: per group, the ring whose plan the device built (0: the host's)
+    unsigned long long* slot_off_d_ = nullptr;   ///< E-6: the slot offsets on the device
     int64_t lb_ = 0, le_ = -1;           ///< set_stage: the layers this verifier runs (-1: to the last)
     const float* hand_in_ = nullptr;
     float* hand_out_ = nullptr;
@@ -146,6 +156,12 @@ private:
     bool ple_stage() const { return lb_ <= 1 && 1 < le_; }   ///< holds layer 1, where the PLE block runs
     bool capture_commit(std::string& err);
     bool record_window(int T, cudaStream_t cs, std::string& err);
+    static constexpr int kProfPer = 32;              // stamps per layer
+    bool prof_on_ = false;
+    unsigned long long* prof_ = nullptr;              // device: n_layers * kProfPer + 4 stamps
+    std::vector<unsigned long long> prof_h_;
+    double prof_sum_[2][kProfPer] = {};   // [GDN / QSA layers][stage]
+    int64_t prof_windows_ = 0;
 
     const WeightTable* wt_ = nullptr;
     const ModelGeometry* g_ = nullptr;

include/strata/hip_compat/cublas_v2.h

@@ -0,0 +1,19 @@
+#pragma once
+#include <hipblas/hipblas.h>
+#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F
+#define CUBLAS_DEFAULT_MATH HIPBLAS_DEFAULT_MATH
+#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT
+#define CUBLAS_OP_N HIPBLAS_OP_N
+#define CUBLAS_OP_T HIPBLAS_OP_T
+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
+#define CUDA_R_16BF HIP_R_16BF
+#define CUDA_R_16F HIP_R_16F
+#define CUDA_R_32F HIP_R_32F
+#define cublasCreate hipblasCreate
+#define cublasDestroy hipblasDestroy
+#define cublasGemmEx hipblasGemmEx
+#define cublasHandle_t hipblasHandle_t
+#define cublasSetMathMode hipblasSetMathMode
+#define cublasSetStream hipblasSetStream
+#define cublasSetWorkspace hipblasSetWorkspace
+#define cublasStatus_t hipblasStatus_t

include/strata/hip_compat/cuda_fp16.h

@@ -0,0 +1,2 @@
+#pragma once
+#include <hip/hip_fp16.h>

include/strata/hip_compat/cuda_runtime.h

@@ -0,0 +1,102 @@
+#pragma once
+// Included only by STRATA_ENABLE_HIP builds. CUDA builds use NVIDIA headers.
+#include <hip/hip_runtime.h>
+// Do not let HIP's legacy macro corrupt libstdc++ attribute names.
+#ifdef __noinline__
+#undef __noinline__
+#endif
+#define cudaDevAttrMaxSharedMemoryPerBlockOptin hipDeviceAttributeMaxSharedMemoryPerBlock
+#define cudaDevAttrMultiProcessorCount hipDeviceAttributeMultiprocessorCount
+#define cudaDevAttrClockRate hipDeviceAttributeClockRate
+#define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor
+#define cudaDeviceGetAttribute hipDeviceGetAttribute
+#define cudaDeviceProp hipDeviceProp_t
+#define cudaDeviceSynchronize hipDeviceSynchronize
+#define cudaDriverGetVersion hipDriverGetVersion
+#define cudaErrorNotReady hipErrorNotReady
+#define cudaErrorStreamCaptureUnsupported hipErrorStreamCaptureUnsupported
+#define cudaError_t hipError_t
+#define cudaEventCreate hipEventCreate
+#define cudaEventCreateWithFlags hipEventCreateWithFlags
+#define cudaEventDestroy hipEventDestroy
+#define cudaEventDisableTiming hipEventDisableTiming
+#define cudaEventElapsedTime hipEventElapsedTime
+#define cudaEventQuery hipEventQuery
+#define cudaEventRecord hipEventRecord
+#define cudaEventSynchronize hipEventSynchronize
+#define cudaEvent_t hipEvent_t
+#define cudaFree hipFree
+#define cudaFreeHost hipFreeHost
+#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize
+#define cudaGetDevice hipGetDevice
+#define cudaGetDeviceCount hipGetDeviceCount
+#define cudaGetDeviceProperties hipGetDeviceProperties
+#define cudaGetErrorString hipGetErrorString
+#define cudaGetLastError hipGetLastError
+#define cudaGraphDestroy hipGraphDestroy
+#define cudaGraphExecDestroy hipGraphExecDestroy
+#define cudaGraphExec_t hipGraphExec_t
+#define cudaGraphGetNodes hipGraphGetNodes
+#define cudaGraphLaunch hipGraphLaunch
+#define cudaGraphUpload hipGraphUpload
+#define cudaGraph_t hipGraph_t
+#define cudaHostAlloc hipHostMalloc
+#define cudaHostAllocDefault hipHostAllocDefault
+#define cudaHostAllocMapped hipHostAllocMapped
+#define cudaHostAllocPortable hipHostAllocPortable
+#define cudaHostGetDevicePointer hipHostGetDevicePointer
+#define cudaHostRegister hipHostRegister
+#define cudaHostRegisterMapped hipHostRegisterMapped
+#define cudaHostRegisterPortable hipHostRegisterPortable
+#define cudaHostUnregister hipHostUnregister
+#define cudaLaunchHostFunc hipLaunchHostFunc
+#define cudaMalloc hipMalloc
+#define cudaMallocHost(...) (::strata::hip_compat::malloc_host(__VA_ARGS__))
+#define cudaMemGetInfo hipMemGetInfo
+#define cudaMemcpy hipMemcpy
+#define cudaMemcpy2DAsync hipMemcpy2DAsync
+#define cudaMemcpyAsync hipMemcpyAsync
+#define cudaMemcpyDefault hipMemcpyDefault
+#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice
+#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost
+#define cudaMemcpyHostToDevice hipMemcpyHostToDevice
+#define cudaMemset hipMemset
+#define cudaMemsetAsync hipMemsetAsync
+#define cudaPeekAtLastError hipPeekAtLastError
+#define cudaRuntimeGetVersion hipRuntimeGetVersion
+#define cudaSetDevice hipSetDevice
+#define cudaStreamBeginCapture hipStreamBeginCapture
+#define cudaStreamCaptureModeThreadLocal hipStreamCaptureModeThreadLocal
+#define cudaStreamCreate hipStreamCreate
+#define cudaStreamCreateWithFlags hipStreamCreateWithFlags
+#define cudaStreamDestroy hipStreamDestroy
+#define cudaStreamEndCapture hipStreamEndCapture
+#define cudaStreamNonBlocking hipStreamNonBlocking
+#define cudaStreamQuery hipStreamQuery
+#define cudaStreamSynchronize hipStreamSynchronize
+#define cudaStreamWaitEvent hipStreamWaitEvent
+#define cudaStream_t hipStream_t
+#define cudaSuccess hipSuccess
+
+namespace strata::hip_compat {
+template <typename T>
+inline hipError_t malloc_host(T** pointer, size_t bytes) {
+    return hipHostMalloc(reinterpret_cast<void**>(pointer), bytes, hipHostMallocDefault);
+}
+}  // namespace strata::hip_compat
+
+template <typename Kernel>
+inline hipError_t cudaFuncSetAttribute(Kernel kernel, hipFuncAttribute attribute, int value) {
+    return hipFuncSetAttribute(reinterpret_cast<const void*>(kernel), attribute, value);
+}
+inline hipError_t cudaGraphInstantiate(hipGraphExec_t* exec, hipGraph_t graph, unsigned long long flags) {
+    return hipGraphInstantiateWithFlags(exec, graph, flags);
+}
+inline hipError_t cudaGraphInstantiate(hipGraphExec_t* exec, hipGraph_t graph,
+                                     hipGraphNode_t* error, char* log, size_t size) {
+    return hipGraphInstantiate(exec, graph, error, log, size);
+}
+#define __trap() __builtin_trap()   // the compiled-out sm_80 paths (never selected on AMD)
+#define cudaMemcpyToSymbol(symbol, ...) hipMemcpyToSymbol(HIP_SYMBOL(symbol), __VA_ARGS__)
+
+#include "intrinsics.hpp"

include/strata/hip_compat/cuda_runtime_api.h

@@ -0,0 +1,2 @@
+#pragma once
+#include "cuda_runtime.h"

include/strata/hip_compat/intrinsics.hpp

@@ -0,0 +1,125 @@
+#pragma once
+
+// CUDA device intrinsics used by Strata kernels that HIP does not provide on gfx1100.
+// This header is included only from the HIP cuda_runtime compatibility shim.
+#if defined(__HIPCC__)
+
+#include <cstdint>
+
+namespace strata::hip_compat {
+
+__device__ __forceinline__ int signed_byte(uint32_t word, int lane) {
+    const unsigned value = (word >> (lane * 8)) & 0xffu;
+    return value < 0x80u ? static_cast<int>(value) : static_cast<int>(value) - 0x100;
+}
+
+// CUDA's signed __dp4a: four signed byte products accumulated modulo 2^32.
+__device__ __forceinline__ int dp4a(int a, int b, int c) {
+#if defined(__gfx1100__) && __has_builtin(__builtin_amdgcn_sudot4)
+    // RDNA3 exposes the signed/unsigned dot4 form. Mark both packed operands
+    // signed to preserve CUDA __dp4a semantics; keep the portable path for
+    // other HIP compilers/targets.
+    return __builtin_amdgcn_sudot4(true, a, true, b, c, false);
+#else
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t sum = static_cast<uint32_t>(c);
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane)
+        sum += static_cast<uint32_t>(signed_byte(ua, lane) * signed_byte(ub, lane));
+    return static_cast<int>(sum);
+#endif
+}
+
+// CUDA's packed byte subtract wraps independently in each unsigned byte lane.
+__device__ __forceinline__ int vsub4(int a, int b) {
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t out = 0;
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane) {
+        const uint32_t x = (ua >> (lane * 8)) & 0xffu;
+        const uint32_t y = (ub >> (lane * 8)) & 0xffu;
+        out |= ((x - y) & 0xffu) << (lane * 8);
+    }
+    return static_cast<int>(out);
+}
+
+// CUDA's packed signed-byte saturating subtract.
+__device__ __forceinline__ int vsubss4(int a, int b) {
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t out = 0;
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane) {
+        int value = signed_byte(ua, lane) - signed_byte(ub, lane);
+        value = value < -128 ? -128 : (value > 127 ? 127 : value);
+        out |= (static_cast<uint32_t>(value) & 0xffu) << (lane * 8);
+    }
+    return static_cast<int>(out);
+}
+
+// CUDA's four-lane byte compare, returning 0xff for each unequal lane and 0 otherwise.
+__device__ __forceinline__ int vcmpne4(int a, int b) {
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t out = 0;
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane) {
+        if (((ua >> (lane * 8)) & 0xffu) != ((ub >> (lane * 8)) & 0xffu))
+            out |= 0xffu << (lane * 8);
+    }
+    return static_cast<int>(out);
+}
+
+// CUDA's mask argument describes participating lanes. The current kernel set uses full
+// wave32 masks; reject future partial-mask use instead of silently dropping its semantics.
+__device__ __forceinline__ void require_full_wave_mask(uint32_t mask) {
+    if (mask != 0xffffffffu) __builtin_trap();
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_xor_sync(uint32_t mask, T value, int lane_mask, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl_xor(value, lane_mask, width);
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_down_sync(uint32_t mask, T value, unsigned delta, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl_down(value, delta, width);
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_up_sync(uint32_t mask, T value, unsigned delta, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl_up(value, delta, width);
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_sync(uint32_t mask, T value, int source_lane, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl(value, source_lane, width);
+}
+
+__device__ __forceinline__ unsigned ballot_sync(uint32_t mask, int predicate) {
+    require_full_wave_mask(mask);
+    return static_cast<unsigned>(__ballot(predicate));
+}
+
+}  // namespace strata::hip_compat
+
+#define __dp4a(a, b, c) (::strata::hip_compat::dp4a((a), (b), (c)))
+#define __vsub4(a, b) (::strata::hip_compat::vsub4((a), (b)))
+#define __vsubss4(a, b) (::strata::hip_compat::vsubss4((a), (b)))
+#define __vcmpne4(a, b) (::strata::hip_compat::vcmpne4((a), (b)))
+#define __shfl_xor_sync(...) (::strata::hip_compat::shfl_xor_sync(__VA_ARGS__))
+#define __shfl_down_sync(...) (::strata::hip_compat::shfl_down_sync(__VA_ARGS__))
+#define __shfl_up_sync(...) (::strata::hip_compat::shfl_up_sync(__VA_ARGS__))
+#define __shfl_sync(...) (::strata::hip_compat::shfl_sync(__VA_ARGS__))
+#define __ballot_sync(mask, predicate) (::strata::hip_compat::ballot_sync((mask), (predicate)))
+// AMD's sleep instruction accepts only 0..15; the synchronization loops use it as a
+// backoff hint, so use its smallest portable delay independently of CUDA cycle counts.
+#define __nanosleep(cycles) __builtin_amdgcn_s_sleep(1)
+
+#endif  // defined(__HIPCC__)

include/strata/kernels/bf16_bits.hpp

@@ -19,7 +19,7 @@
 #include <cstdint>
 #include <cstring>
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_BF16_HD __host__ __device__
 #else
 #define STRATA_BF16_HD
@@ -30,7 +30,11 @@ namespace strata::kernels {
 /// Round-to-nearest-even f32 -> bf16, returned as raw bits in the LOW half of the uint16.
 STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) {
     uint32_t i;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&i, &f, 4);
+#else
     std::memcpy(&i, &f, 4);
+#endif
     i = (i + ((i >> 16) & 1u) + 0x7FFFu) & 0xFFFF0000u;
     return (uint16_t) (i >> 16);
 }
@@ -39,7 +43,11 @@ STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) {
 STRATA_BF16_HD inline float f32_from_bf16(uint16_t h) {
     const uint32_t i = (uint32_t) h << 16;
     float f;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&f, &i, 4);
+#else
     std::memcpy(&f, &i, 4);
+#endif
     return f;
 }
 

include/strata/kernels/bf16_gemv.hpp

@@ -44,5 +44,9 @@ void bf16_gemv_split(const uint16_t* x, const uint16_t* w, float* y, int64_t n_i
 /// No allocations or synchronization, including when stream is null (the CUDA default stream).
 void bf16_gemv_fp32_mmvf(const float* x, const uint16_t* w, float* y,
                          int64_t n_in, int64_t n_out, void* stream);
+/// bf16_gemv_fp32_mmvf for n_tok (1..8) activation rows x[t * ldx], outputs y[t * ldy + j]; one launch,
+/// the weight read once, every output bit-identical to its own single-row call.
+void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, float* y, int64_t ldy,
+                               int64_t n_in, int64_t n_out, int n_tok, void* stream);
 
 }  // namespace strata::kernels

include/strata/kernels/cvec.hpp

@@ -43,7 +43,12 @@ const Cvec& cvec();
 bool cvec_upload(const std::vector<float>& dir, const std::vector<float>& s, int mode, int first, int last,
                  int64_t n_embd, int64_t hc, std::string& err);
 
-/// The per-request switch.  Synchronizes the device when it changes, so call it between requests.
+/// A layer split: put the loaded vector's tables on the CURRENT device too (cvec_apply uses the tables of the device
+/// it runs on).  No-op without a vector or when this device has them already.
+bool cvec_replicate(std::string& err);
+
+/// The per-request switch, on every device that holds the vector.  Synchronizes them when it changes, so call it
+/// between requests.
 void cvec_set_enabled(bool on);
 bool cvec_enabled();
 

include/strata/kernels/elementwise.hpp

@@ -102,6 +102,10 @@ void doorbell_wait(const uint32_t* d_flag, const uint32_t* d_seq, void* stream);
 /// device memory with a kernel, so the handoff stays on the compute queue (a memcpy node is a copy-engine
 /// operation, which WDDM submits separately and which measured 67 flushes per token).
 void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream);
+/// `rows` rows of `width` floats from mapped memory, except the rows listed in hit_rows[0, *count) (device),
+/// which are written +0.0 instead (a verify window's GPU-computed entries: the pool leaves zeros there).
+void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
+                           const int32_t* count, void* stream);
 
 /// Plan v0.3 P3: the doorbell's payload and its ring in ONE kernel.  Copies `x` (n floats), `ids` and `weights`
 /// (k each) into the mapped host regions, fences, and increments the mapped sequence number - replacing three

include/strata/kernels/f16_bits.hpp

@@ -25,7 +25,7 @@
 #include <cstdint>
 #include <cstring>
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_HD __host__ __device__
 #else
 #define STRATA_HD
@@ -36,7 +36,11 @@ namespace strata::kernels {
 /// Round-to-nearest-even f32 -> fp16, returned as raw bits.
 STRATA_HD inline uint16_t f16_from_f32(float f) {
     uint32_t x;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&x, &f, 4);
+#else
     std::memcpy(&x, &f, 4);
+#endif
     const uint32_t sign = (x >> 16) & 0x8000u;
     const uint32_t rawexp = (x >> 23) & 0xFFu;
     int exp = (int) rawexp - 127 + 15;
@@ -89,7 +93,11 @@ STRATA_HD inline float f32_from_f16(uint16_t h) {
         out = sign | ((ex - 15 + 127) << 23) | (man << 13);
     }
     float f;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&f, &out, 4);
+#else
     std::memcpy(&f, &out, 4);
+#endif
     return f;
 }
 

include/strata/kernels/fused_gr.hpp

@@ -46,6 +46,7 @@ void fused_gr_read(const FusedGrArgs& a, void* stream);
 /// weight pointers and eps must be the same for every t); `xn_scratch` is n_tok * hc * n_embd floats.  Every
 /// token's outputs are bitwise `fused_gr_read(a[t])`.
 constexpr int kFusedGrMaxT = 8;
-void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream);
+void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream,
+                         unsigned long long* stamp_buf = nullptr, int stamp_i0 = 0);
 
 }  // namespace strata::kernels

include/strata/kernels/mrope.hpp

@@ -18,10 +18,11 @@
 
 namespace strata::kernels {
 
+/// The table of the CURRENT device (a layer split sets one per device; null = the identity).
 void mrope_table_set(const int32_t* device_table);
 const int32_t* mrope_table();
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 /// ggml rope_multi, is_imrope, sections {11, 11, 10, 0}: sector = pair % 32; sector % 3 == 1 -> h (sector < 33),
 /// == 2 -> w (sector < 30), == 0 -> t (sector < 33).  For pairs 0..31 all three bounds hold, so it is pair % 3.
 __device__ __forceinline__ int mrope_pos(const int32_t* tab, int pos, int pair) {

include/strata/kernels/native_moe.hpp

@@ -15,4 +15,7 @@ bool native_moe_combine_enabled();
 // Requires a nonnull ordered stream and disjoint output. No allocation or sync.
 void native_moe_combine(const float* parts, const float* weights, const float* shared,
                         float* output, int64_t n_embd, int64_t k, void* stream);
+/// n_tok rows (parts [n,k,N], weights [n,k], shared/output [n,N]) in one launch, each as the single call.
+void native_moe_combine_multi(const float* parts, const float* weights, const float* shared, float* output,
+                              int64_t n_embd, int64_t k, int n_tok, void* stream);
 }

include/strata/kernels/native_qsa_indexer.hpp

@@ -27,6 +27,13 @@ bool native_qsa_indexer_enabled();
 // max_cells is a positive fixed capacity; pos_base+max_cells must fit int32.
 // Out-of-range device positions are ignored defensively, not host-validated.
 // This adapter does not alter score accumulation, top-k, or finite 1e9 tail bias.
+// perf-review C-2: the appends of cells [p0, p0 + n) at once, leaving the buffers exactly as n calls of the
+// single append in order would (the prompt path appends a chunk before any query reads the state). raw [n, 128].
+// Host-side positions (not for a captured graph).
+void native_qsa_indexer_append_batch(const float* raw, int64_t n, int64_t p0, int32_t pos_base, const float* gamma,
+                                     float epsilon, const QsaIndexerBuffers& b, const QsaShapes& s, int64_t max_cells,
+                                     float freq_base, void* stream);
+
 void native_qsa_indexer_append(const float* raw, const int32_t* relative_pos_device,
                                int32_t pos_base, const float* gamma, float epsilon,
                                const QsaIndexerBuffers& buffers, const QsaShapes& shapes,

include/strata/kernels/native_router.hpp

@@ -14,4 +14,6 @@ bool native_router_enabled();
 // four-byte aligned and outputs disjoint from each other and the input.
 // Requires a nonnull ordered CUDA stream. No allocation or synchronization.
 void native_router_top10(const float* logits, int32_t* ids, float* weights, void* stream);
+/// n_tok rows at once (logits [n,512], ids/weights [n,10]); each row exactly as the single call.
+void native_router_top10_multi(const float* logits, int32_t* ids, float* weights, int n_tok, void* stream);
 }

include/strata/kernels/qsa_prompt_attn.hpp

@@ -0,0 +1,31 @@
+// include/strata/kernels/qsa_prompt_attn.hpp - perf-review D-1: the prompt path's QSA attention on tensor cores.
+//
+// `qsa_decode_attn_batch` serves a prompt one query at a time with the decode kernel: FP32 dot products with a warp
+// reduction per head and cell, 64-cell chunks whose partial sums go through global scratch to a second (merge)
+// kernel. At 32K that is 21% of the prompt (5.6 s of 26 s on a 5070, Q2_0, int8 KV), and it is limited by
+// instruction issue, not memory (~150 GB/s of logical reads, 3.4 TFLOP/s).
+//
+// This kernel keeps the same per-query selection (no masking, no union) and reads the same pools, but:
+//   * one block per (query, KV head) walks all its selected cells in chunks of 32 with an online softmax: no
+//     split-K scratch and no merge kernel;
+//   * q.k and p.v are m16n8k16 FP16 MMAs with FP32 accumulation over the 12 query heads of the KV head (+4 pad rows);
+//   * the stored values enter exactly: int8 codes are exact in FP16 and their per-64 scales are applied in FP32
+//     (to the q.k partial of each 64-dim group, and folded into p for p.v); FP16 KV is used as is;
+//   * q and p are split into FP16 hi + lo parts (two MMAs each), so they keep ~22 bits: the result differs from
+//     the FP32 kernel by summation order and the exp2 rounding, not by an FP16 cast.
+// Not bitwise equal to `qsa_decode_attn_batch`; `qsa_prompt_attn_parity` bounds the difference and the prompt
+// quality gate (needles, teacher-forced top-1) checks it end to end. Q4_0 KV is not handled (returns false).
+#pragma once
+
+#include "strata/kernels/qsa_decode_attn.hpp"
+
+#include <cstdint>
+
+namespace strata::kernels {
+
+/// Same arguments and output as `qsa_decode_attn_batch` minus the scratch. Returns false (nothing launched) when the
+/// pools are Q4_0 or the geometry is not 24 heads / 2 KV heads / 256: the caller then uses the old kernel.
+bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
+                           int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream);
+
+}  // namespace strata::kernels

include/strata/kernels/qsa_select.hpp

@@ -24,10 +24,20 @@ namespace strata::kernels {
 
 /// scores [nq, max_blocks]; q_idx [nq, idx_n_head, idx_dim] (normed and rotated); steps [nq, kStepCount].
 void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
-                      int64_t max_blocks, const QsaShapes& s, float* scores, void* stream);
+                      int64_t max_blocks, const QsaShapes& s, float* scores, void* stream,
+                      int64_t active_blocks = -1);   ///< perf-review C-1: > 0 launches only this many blocks (the
+                                                     ///< batch's largest n_bid + 1; not for a captured graph)
+
+/// The same scores on tensor cores (3xTF32, FP32-level accuracy but another summation order: not bitwise; the tail
+/// block n_bid is the warp kernel's arithmetic). For the prompt path; false (nothing launched) on another geometry.
+bool qsa_block_scores_tc(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+                         int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks);
 
 /// ids [nq, cap] (cells, ascending); `cap` >= the largest selection width.
 void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
                     const QsaShapes& s, int32_t* ids, void* stream);
+/// The original kernel (keys read from memory on every radix pass), for tests: the same ids.
+void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                        const QsaShapes& s, int32_t* ids, void* stream);
 
 }  // namespace strata::kernels

include/strata/kernels/rope.hpp

@@ -10,7 +10,7 @@
 
 #include <cstdint>
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_ROPE_HD __host__ __device__
 #else
 #define STRATA_ROPE_HD

include/strata/kernels/verify_kernels.hpp

@@ -38,6 +38,24 @@ void gdn_step_norm_multi(float* state, const float* h, int conv_channels, const
 /// Spin until *flag >= value (a mapped host flag).  The value is fixed at capture, so several rings can be
 /// outstanding at once (the split verify window keeps two).
 void wait_flag_ge(const uint32_t* flag, uint32_t value, void* stream);
+/// the GPU's %globaltimer (ns) into buf[i] (a one-thread kernel: the verify window's stage profiler).
+void gpu_stamp(unsigned long long* buf, int i, void* stream);
+
+// ---- perf-review E-6: a layer whose routed experts are all in VRAM needs nothing from the host
+/// One group's plan, built on the device when every routed expert of its n*k entries is resident: the host pool's
+/// layout (counts | start | dst | tok | pad | ptr | ptr2 | start2, `capx` entries) and order (distinct experts in
+/// routing order, their entries ascending), no PCIe groups.  *skip = ring when it did, else 0.
+void resident_plan(const int32_t* ids, int n_entries, int k, const int32_t* res_layer, int n_expert,
+                   const uint8_t* cache_base, const unsigned long long* slot_off, long long blob, int32_t* plan,
+                   long long capx, uint32_t* skip, uint32_t ring, void* stream);
+/// wait_flag_ge that also returns when *skip == value (device memory).
+void wait_flag_ge_or(const uint32_t* flag, uint32_t value, const uint32_t* skip, void* stream);
+/// copy_i32_from_mapped unless *skip == value.
+void copy_i32_from_mapped_unless(int32_t* dst, const int32_t* src, long long n, const uint32_t* skip, uint32_t value,
+                                 void* stream);
+/// copy_from_mapped, or zeros when *skip == value (n a multiple of 4, 16-byte aligned).
+void copy_or_zero_from_mapped(float* dst, const float* src, long long n, const uint32_t* skip, uint32_t value,
+                              void* stream);
 
 /// Rows of the S2/S4/S8 embedding for T token ids read from DEVICE memory; out (T, n).  Bitwise `embedding_gather`.
 void embedding_gather_dev(const uint8_t* codes, const float* scales, const float* offsets, const int32_t* tokens,

include/strata/platform/direct_file.hpp

@@ -2,7 +2,8 @@
 //
 // The n-gram table (26.8 GiB) stays on the SSD and must never occupy RAM, including the OS file cache. A memory
 // map cannot promise that; an unbuffered read can. On Windows this is FILE_FLAG_NO_BUFFERING | OVERLAPPED with
-// an I/O completion port; on Linux, O_DIRECT (synchronous pread for now; io_uring is phase L).
+// an I/O completion port; on Linux, O_DIRECT preads.  Both issue the reads from a small thread pool (submit only
+// queues), so many reads are in flight: STRATA_IO_THREADS sets its size (default 4 on Windows, 16 on Linux).
 //
 // Contract of every read: offset, length and buffer address are multiples of `alignment()` (4096 here).
 // Reads past end of file return the bytes that exist; `Completion::bytes` says how many.

include/strata/prefill/gemm.hpp

@@ -6,6 +6,7 @@
 // rounded to BF16, which is also what llama.cpp's batched CUDA path does.  Tensor-core GEMM through cuBLAS.
 #pragma once
 
+#include <cstddef>
 #include <cstdint>
 #include <string>
 
@@ -50,6 +51,7 @@ private:
     int64_t scratch_elems_ = 0;
     void* workspace_ = nullptr;
     bool external_ = false;
+    void* hipblaslt_state_ = nullptr;
 };
 
 

include/strata/prefill/kernels.hpp

@@ -14,6 +14,15 @@ namespace strata::prefill {
 // ---- hyper-connection (n_embd 2560, hc 4, hc_lr 320)
 /// xn[t, c*2560 + d] = R[t,c,d] * rsqrt(mean_d R[t,c,:]^2 + eps) * w_norm[c*2560 + d]; also its BF16 image.
 void gr_norm(const float* R, const float* w_norm, float eps, float* xn, uint16_t* xn16, int64_t T, void* stream);
+/// F-1: gr_norm without its FP32 output: the row scales rs[t*4 + c] and the BF16 image; gr_mix_r then reads R.
+void gr_norm_rs(const float* R, const float* w_norm, float eps, float* rs, uint16_t* xn16, int64_t T, void* stream);
+/// gr_mix with xn recomputed from R, rs and w_norm exactly as gr_norm computes it (the same bits).
+void gr_mix_r(const float* R, const float* rs, const float* w_norm, const float* gated, float* mixed, uint16_t* mixed16,
+              int64_t T, void* stream, uint16_t* mixed_h = nullptr);
+/// F-2: gr_write, then gr_norm_rs of the next half (its norm weights) over the rows just written - the same bits as
+/// the two calls, without reading R back.
+void gr_write_norm_rs(float* R, const float* bo, const float* inj, int64_t inj_ld, const float* w_norm_next, float eps,
+                      float* rs, uint16_t* xn16, int64_t T, void* stream);
 /// lo16[t, k] = bf16(silu(lo[t, k] / hc))
 void gr_silu(const float* lo, uint16_t* lo16, int64_t T, void* stream);
 /// mixed[t, d] = mean_c xn[t, c, d] * sigmoid(gated[t, c, d]); FP32, BF16 and FP16 (either image may be null).
@@ -44,6 +53,9 @@ void blob_dequant(const uint8_t* blob, uint16_t* gu16, uint16_t* down16, void* s
 void swiglu_interleaved(const float* gu, uint16_t* h16, int64_t n, void* stream);
 /// h16[n, r] = fp16(silu(g[n, r]) * u[n, r])   (the shared expert, gate and up separate, width 640)
 void swiglu_pair(const float* g, const float* u, uint16_t* h16, int64_t n, void* stream);
+/// dst[i] = src[i] for n int32s, as a kernel: either side may be mapped host memory, and the copy never waits
+/// behind the copy engine's queue (the prompt path's grouping tables, while the expert stream fills it).
+void copy_i32(int32_t* dst, const int32_t* src, int64_t n, void* stream);
 /// Gather rows: dst16[i, :] = x16[src[i], :] (n rows of `width` BF16).
 void gather_rows16(const uint16_t* x16, const int32_t* src, uint16_t* dst16, int64_t n, int64_t width, void* stream);
 /// bo[t, :] = shared[t, :] * sigmoid(sg[t]) + sum_k w[t, k] * D[slot[t, k], :]

include/strata/prefill/prefill.hpp

@@ -34,8 +34,19 @@ struct PrefillStats {
     double ms_ple = 0;
 };
 
+}  // namespace strata::prefill
+namespace strata::core { class MtpDrafter; }
+namespace strata::prefill {
+
 class Prefill {
 public:
+    /// E-9: the draft layer's K/V for prompt cells [cell0, cell0 + n) from their final residual rows `R_rows`
+    /// (device) and `next_tokens` (host: the token at cell+1), in batches through this path's GEMMs and its idle
+    /// scratch - call it from on_chunk.  false with `err` empty: not applicable here (a ring or hybrid K/V, another
+    /// device, too little scratch; STRATA_MTP_BATCH=0), the caller runs the drafter's own pass.  Not bit-identical to
+    /// that pass (FP16 GEMMs instead of Q8_1 activations): the drafts may differ, never the target's tokens' logits.
+    bool draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t* next_tokens, int64_t n, int64_t cell0,
+                  std::string& err);
     Prefill();
     ~Prefill();
     Prefill(const Prefill&) = delete;
@@ -72,6 +83,12 @@ public:
     /// K/V from them.  The prefill stream is synchronized before the call.
     std::function<bool(const float* R_rows, int64_t T, int64_t pos0, std::string& err)> on_chunk;
 
+    /// Layer split: called by every stage when it has read a chunk, with the position reached, while its own state
+    /// is still at that chunk's end (its stream synchronized; the last stage calls it just before `on_chunk`).  An
+    /// earlier stage is a chunk or more ahead of the last one by the time `on_chunk` runs, so this is where a
+    /// mid-prompt checkpoint takes each stage's part.  Runs on that stage's thread, with its device current.
+    std::function<bool(int64_t done, std::string& err)> on_stage_chunk;
+
     /// Checked before every chunk: true stops the prompt early (`run` returns false with err "cancelled").
     std::function<bool()> should_stop;
 

serve/frontend.py

@@ -38,7 +38,8 @@ class ChatTemplate:
         env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True, extensions=["jinja2.ext.loopcontrols"])
         env.filters["tojson"] = tojson
         env.globals["raise_exception"] = raise_exception
-        self.template = env.from_string(Path(path).read_text(encoding="utf-8"))
+        self.source = Path(path).read_text(encoding="utf-8")
+        self.template = env.from_string(self.source)
 
     def render(self, messages: list[dict], tools: list[dict] | None = None, add_generation_prompt: bool = True,
                **kwargs) -> str:
@@ -144,7 +145,7 @@ def openai_to_messages(req: dict) -> tuple[list[dict], list[dict] | None, dict]:
         role = m.get("role")
         if role == "developer":
             role = "system"
-        out = {"role": role, "content": _parts_of(m.get("content")) if role == "user" else _text_of(m.get("content"))}
+        out = {"role": role, "content": _parts_of(m.get("content")) if role in ("user", "tool", "assistant") else _text_of(m.get("content"))}
         if m.get("reasoning_content"):
             out["reasoning_content"] = m["reasoning_content"]
         if m.get("tool_calls"):
@@ -241,6 +242,61 @@ CALL_START = "<tool_call>"
 CALL_END = "</tool_call>"
 
 
+PARAM_END = "</parameter>"
+FUNC_END = "</function>"
+
+
+def param_end(text: str, final: bool = False) -> int:
+    """Where a parameter value in `text` ends: the first `</parameter>` followed (after whitespace) by the next
+    `<parameter=` or `</function>` - the same text inside a value (a file that documents the call format, #210) is
+    part of the value.  -1: none yet; -2: a candidate whose follower has not arrived (streaming; `final` accepts it)."""
+    at = text.find(PARAM_END)
+    while at >= 0:
+        after = text[at + len(PARAM_END):].lstrip()
+        if after.startswith(("<parameter=", FUNC_END)):
+            return at
+        if not after or "<parameter=".startswith(after) or FUNC_END.startswith(after):
+            return at if final else -2
+        at = text.find(PARAM_END, at + 1)
+    return -1
+
+
+def call_end(text: str) -> int:
+    """Where a tool call's body ends (#210): the `</tool_call>` after the call's own `</function>`, found by walking
+    its parameters with param_end, so a value may contain either tag.  -1: not complete yet.  A body that is not in
+    the call format ends at the first `</tool_call>`, as before."""
+    s = text.lstrip()
+    pos = len(text) - len(s)
+    if not s.startswith("<function="):
+        return text.find(CALL_END) if not "<function=".startswith(s) else -1
+    gt = text.find(">", pos)
+    if gt < 0:
+        return -1
+    pos = gt + 1
+    while True:
+        rest = text[pos:]
+        s = rest.lstrip()
+        pos += len(rest) - len(s)
+        if s.startswith("<parameter="):
+            gt = text.find(">", pos)
+            if gt < 0:
+                return -1
+            end = param_end(text[gt + 1:])
+            if end < 0:
+                return -1
+            pos = gt + 1 + end + len(PARAM_END)
+        elif s.startswith(FUNC_END):
+            rest = text[pos + len(FUNC_END):]
+            s = rest.lstrip()
+            if s.startswith(CALL_END):
+                return pos + len(FUNC_END) + len(rest) - len(s)
+            return -1 if CALL_END.startswith(s) else text.find(CALL_END, pos)
+        elif not s or "<parameter=".startswith(s) or FUNC_END.startswith(s):
+            return -1
+        else:
+            return text.find(CALL_END, pos)
+
+
 def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
     """`<function=NAME>\\n<parameter=P>\\nVALUE\\n</parameter>...</function>` -> ToolCall. Values are JSON-decoded
     when the tool's schema says the parameter is not a string (or, without a schema, when they parse as JSON
@@ -256,9 +312,9 @@ def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
         rest = rest[rest.index("<parameter=") + len("<parameter="):]
         pname = rest[:rest.index(">")]
         rest = rest[rest.index(">") + 1:]
-        end = rest.find("</parameter>")
+        end = param_end(rest, final=True)
         value = rest[:end] if end >= 0 else rest
-        rest = rest[end + len("</parameter>"):] if end >= 0 else ""
+        rest = rest[end + len(PARAM_END):] if end >= 0 else ""
         if value.startswith("\n"):
             value = value[1:]
         if value.endswith("\n"):
@@ -350,16 +406,17 @@ class OutputParser:
                         self.sp += 1
                         rest = rest[1:]
                     self.sval_started = True
-                end = rest.find("</parameter>")
+                end = param_end(rest)
                 if end >= 0:
                     value = rest[:end]
                     if value.endswith("\n"):
                         value = value[:-1]
                     args(json.dumps(value)[1:-1] + '"')
-                    self.sp += end + len("</parameter>")
+                    self.sp += end + len(PARAM_END)
                     self.ss = "between"
                     continue
-                safe = len(rest) - self._hold(rest, ("</parameter>",))
+                # an undecided </parameter> (-2) is held from its start, like a tag still arriving
+                safe = rest.find(PARAM_END) if end == -2 else len(rest) - self._hold(rest, (PARAM_END,))
                 if safe > 0 and rest[safe - 1] == "\n":   # may be the trailing newline before </parameter>
                     safe -= 1
                 if safe > 0:
@@ -367,7 +424,7 @@ class OutputParser:
                     self.sp += safe
                 return out
             elif self.ss == "raw":
-                end = rest.find("</parameter>")
+                end = param_end(rest)
                 if end < 0:
                     return out
                 value = rest[:end]
@@ -380,7 +437,7 @@ class OutputParser:
                 except ValueError:
                     v = value
                 args(json.dumps(v, ensure_ascii=False))
-                self.sp += end + len("</parameter>")
+                self.sp += end + len(PARAM_END)
                 self.ss = "between"
             else:
                 return out
@@ -448,7 +505,7 @@ class OutputParser:
                 self.buf = self.buf[i + len(CALL_START):]
                 self.state = "call"
             else:
-                i = self.buf.find(CALL_END)
+                i = call_end(self.buf)
                 if self.stream_tools:
                     if i >= 0:
                         whole, self.buf = self.buf, self.buf[:i]     # scan only the body

serve/server.py

@@ -4,7 +4,7 @@
     python -m serve.server --engine strata --config strata.json --port 8080   (the real engine, resident)
 
 Endpoints: POST /v1/chat/completions (OpenAI, stream and non-stream), POST /v1/messages (Anthropic, stream and
-non-stream), GET /v1/models, GET /health, GET /mcp. One sequence at a time behind a FIFO (plan: one resident sequence).
+non-stream), GET /v1/models, GET /models, GET /props, GET /slots, GET /health, GET /mcp. One sequence at a time behind a FIFO (plan: one resident sequence).
 Tools from MCP servers (serve/mcp.py, `"mcp_servers"` in the config or --mcp-config) are offered only to requests that
 ask for them with `"strata_mcp": true` - the web app does; other clients see exactly the API they always saw.
 Images (optional, when the config has a "vision" entry): OpenAI image_url parts and Anthropic image blocks (base64
@@ -24,6 +24,8 @@ import argparse
 import collections
 import base64
 import hashlib
+import hmac
+import codecs
 import json
 import os
 import queue
@@ -37,6 +39,7 @@ import uuid
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Iterator, Protocol
+from urllib.parse import parse_qs, urlsplit
 
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT / "tools"))
@@ -47,7 +50,12 @@ from serve.mcp import McpCancelled, hub_from_config  # noqa: E402
 
 IM_END = "<|im_end|>"
 IMAGE_PAD = "<|image_pad|>"
+VISION_START = "<|vision_start|>"
 CTX_SLACK = 8               # `strata --serve` rejects prompt + max_new + 8 > context: keep the same margin here
+# The live tok/s is a rate over a window, not a mean since the first token: a mean reads ~1/elapsed at the first
+# token (the Monitor showed five-digit numbers) and then undershoots for the first second of every answer.
+RATE_WINDOW_S = 2.0
+RATE_MIN_SPAN_S = 0.25      # younger than this there is no rate yet: the mean so far, with the span floored here
 
 
 # ------------------------------------------------------------------------------------------------ engines
@@ -149,6 +157,8 @@ class StrataEngine:
     def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | None = None,
                  env: dict | None = None):
         self.spawn = (exe, list(args), cwd, log, env)   # to start it again after it died (issue #27)
+        paths = {k: v for k, v in zip(args, args[1:]) if k in ("--native", "--pack")}
+        self.model_path = paths.get("--native") or paths.get("--pack", "pack/full")
         self.log_path = log
         self.log = open(log, "a", encoding="utf-8") if log else subprocess.DEVNULL
         loading = threading.Event()                     # set once READY: the narrator below stops
@@ -207,6 +217,11 @@ class StrataEngine:
             if "issue #29" in line:
                 return ("The engine stopped itself because it had stopped making progress - a hang it caught. Its log "
                         "line: " + line.strip() + " - please report it at github.com/Niko1221/Strata/issues.")
+        rc = self.proc.poll()
+        last = next((x.strip() for x in reversed(tail.splitlines()) if x.strip().startswith(("strata", "ERR"))), "")
+        if rc is not None and rc >= 0 and last:          # it ended by itself: its own last words say why (#215)
+            return (f"The engine exited (code {rc}). Its last log line: {last} - if that does not explain it, please "
+                    "report it at github.com/Niko1221/Strata/issues with the log.")
         return ("The usual cause is running out of RAM: Linux then ends the biggest program (check: sudo dmesg | "
                 "grep -i -E 'killed process|out of memory'); Windows slows down instead. Close other programs or use a "
                 "smaller model (Q2_0 / IQ2_XS).")
@@ -299,8 +314,9 @@ class StrataEngine:
         the HTTP layer turns it into an SSE comment, which keeps clients' watchdogs calm and notices a client that
         has gone.  A consumer that stops early (or `cancel`) makes the engine STOP, so it does not run to max_new."""
         self.progress = None
-        head = f"GENI {int(max_new)}{self.projection_key(sampling or {})} {embeddings}" if embeddings else \
-            f"GEN {int(max_new)}{self.sampling_keys(sampling or {}) if not embeddings else ''}"
+        # an image request takes the same sampling keys as text (#75: it used to decode greedily whatever was asked)
+        head = f"GENI {int(max_new)}{self.sampling_keys(sampling or {})} {embeddings}" if embeddings else \
+            f"GEN {int(max_new)}{self.sampling_keys(sampling or {})}"
         try:
             self.proc.stdin.write(f"{head} {','.join(str(int(t)) for t in ids)}\n")
             self.proc.stdin.flush()
@@ -479,9 +495,13 @@ def child_env(cfg: dict) -> dict:
     """The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
     compiled it) first on the library search path."""
     env = dict(os.environ)
-    if gpu_list(cfg):                                # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
+    if gpu_list(cfg) and cfg.get("backend") == "hip":   # AMD: numbered as HIP numbers them (setup's KFD order)
+        env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
+    elif gpu_list(cfg):                              # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
         env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"      # own default order (fastest first) can number the cards otherwise
         env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
+    for k, v in (cfg.get("env") or {}).items():      # engine settings the config carries (AMD: the GEMM tuning table)
+        env[str(k)] = str(v)
     dirs = [d for d in cfg.get("lib_dirs") or [] if Path(d).is_dir()]
     if dirs:
         var = "PATH" if os.name == "nt" else "LD_LIBRARY_PATH"
@@ -515,13 +535,19 @@ class ByteTokenizer:
 
 # ------------------------------------------------------------------------------------------------ core
 class Detokenizer:
-    """Incremental decode: re-decode the generated ids and emit only the new, complete suffix (a multi-byte
-    character split across tokens is held until complete)."""
+    """Incremental decode: each token's bytes go through an incremental UTF-8 decoder, which emits the complete
+    characters and holds a multi-byte character split across tokens until it is complete (invalid bytes become
+    U+FFFD, as a whole decode with errors="replace" makes them).  Constant time per token - the old re-decode of
+    every generated id cost 2 ms per token after 8K tokens and 4 ms after 16K (perf-review F-1).  A tokenizer
+    without `token_bytes` (the tests' byte tokenizer) keeps the re-decode."""
 
     def __init__(self, tok):
         self.tok, self.ids, self.sent = tok, [], 0
+        self.inc = codecs.getincrementaldecoder("utf-8")(errors="replace") if hasattr(tok, "token_bytes") else None
 
     def push(self, t: int) -> str:
+        if self.inc is not None:
+            return self.inc.decode(self.tok.token_bytes(t))
         self.ids.append(t)
         text = self.tok.decode(self.ids)
         if text.endswith("�"):
@@ -543,10 +569,14 @@ class Service:
         self.embeddings = threading.local()           # the current request's image embeddings file (GENI)
         self.api_key = ""                              # when set, /v1/* needs it (Bearer or x-api-key)
         self.status = {"busy": False, "queued": 0}      # GET /status: what the model is doing right now
+        self.rate = collections.deque(maxlen=32)        # (time, generated) samples for the live tok/s window
         self.history = collections.deque(maxlen=500)    # the last finished requests, newest last (GET /metrics)
         # since the server started (the Monitor's totals, issue #35)
         self.totals = {"since": time.time(), "requests": 0, "prompt_tokens": 0, "reused": 0, "output_tokens": 0,
                        "prompt_ms": 0.0, "decode_ms": 0.0}
+        self.last_timings = None                         # the last finished request's, llama.cpp's names (/v1/status)
+        self.last_request_at = None                      # when a request last started or finished
+        self.started_at = time.time()
         self.status_lock = threading.Lock()
         self.mcp = None                                  # serve/mcp.py's McpHub when MCP servers are configured
         self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
@@ -591,10 +621,26 @@ class Service:
         """The hardware sampler behind GET /metrics (serve/telemetry.py), recording this server's tok/s too."""
         if getattr(self, "telemetry", None) is None:
             from serve.telemetry import Telemetry
-            self.telemetry = Telemetry(extra=lambda: {"tok_s": self._tok_s()},
-                                       gpu_index=int(getattr(self, "gpu_index", 0) or 0))
+            self.telemetry = Telemetry(extra=lambda: {"tok_s": self._tok_s(), "tok_s_mean": self._tok_s_mean()},
+                                       gpu_index=int(getattr(self, "gpu_index", 0) or 0),
+                                       gpu_indices=getattr(self, "gpu_indices", None))
 
     def _tok_s(self):
+        """tok/s over the last RATE_WINDOW_S seconds.  Returns 0.0 while nothing is generating."""
+        with self.status_lock:
+            s = dict(self.status)
+            rate = list(self.rate)
+        if not s.get("busy") or not s.get("first_token"):
+            return 0.0
+        now = time.time()
+        newest = rate[-1] if rate else None
+        oldest = next(((t, g) for t, g in rate if now - t <= RATE_WINDOW_S), None)
+        if newest and oldest and newest[0] - oldest[0] >= RATE_MIN_SPAN_S:
+            return max(0.0, (newest[1] - oldest[1]) / (newest[0] - oldest[0]))
+        return s["generated"] / max(RATE_MIN_SPAN_S, now - s["first_token"])
+
+    def _tok_s_mean(self):
+        """The whole-request mean since the first token (the old formula), kept so the two can be compared."""
         with self.status_lock:
             s = dict(self.status)
         if not s.get("busy") or not s.get("first_token"):
@@ -621,7 +667,9 @@ class Service:
                 "prompt_read": None, "prompt_total": None, "generated": s.get("generated") if s.get("busy") else None,
                 "max_tokens": s.get("max_tokens") if s.get("busy") else None,
                 "elapsed_s": round(now - s["started"], 1) if s.get("busy") and s.get("started") else None,
-                "tok_s": round(self._tok_s(), 1) if state == "generating" else None}
+                "tok_s": round(self._tok_s(), 1) if state == "generating" else None,
+                "tok_s_mean": round(self._tok_s_mean(), 1) if state == "generating" else None,
+                "tok_s_window_s": RATE_WINDOW_S if state == "generating" else None}
         if state == "reading" and progress:
             live["prompt_read"], live["prompt_total"] = progress
         engine = {"model": self.model, "max_context": self.engine.max_context, "images": self.vision is not None,
@@ -632,6 +680,41 @@ class Service:
                 "hardware_static":
                 tel["static"], "history": tel["history"], "time": now}
 
+    def v1_status(self) -> dict:
+        """GET /v1/status: what this server is and does, for a client that would rather ask than guess (a front-end
+        that polls its OpenAI-compatible server's status, collabosm's for one): the model and its window, images,
+        the APIs, what is running, and the last request's timings in llama.cpp's names.  /metrics has the rest."""
+        with self.status_lock:
+            s, totals = dict(self.status), dict(self.totals)
+            last_t, last_at = (dict(self.last_timings) if self.last_timings else None), self.last_request_at
+        tel = self.telemetry.snapshot() if getattr(self, "telemetry", None) else {"now": {}, "static": {}}
+        hw, static = tel.get("now") or {}, tel.get("static") or {}
+
+        def scaled(v, unit, digits=0):
+            return round(v / unit, digits) if isinstance(v, (int, float)) else None
+
+        busy, ctx = bool(s.get("busy")), self.engine.max_context
+        images = self.vision is not None
+        return {
+            "service": "strata", "model": self.model,
+            "engine": (getattr(self.engine, "info", {}) or {}).get("version"),
+            "started": int(self.started_at), "uptime_s": int(time.time() - self.started_at),
+            "cache_max_tokens": ctx,
+            "context": {"native": ctx, "max_positions": ctx},
+            "concurrency": {"serving": 1, "requested": 1},       # one request at a time; more wait their turn
+            "dialects": ["/v1/chat/completions", "/v1/messages"],
+            "vision": {"enabled": images, "available": images, "error": None},
+            "activity": {"requests": totals["requests"] + int(busy), "in_flight": int(busy) + int(s.get("queued") or 0),
+                         "last_request_at": int(last_at) if last_at else None},
+            "last_timings": last_t,
+            "machine": {
+                "at": int(time.time()),
+                "gpu": {"name": static.get("gpu_name"), "used_mib": scaled(hw.get("gpu_mem_used"), 2 ** 20),
+                        "total_mib": scaled(hw.get("gpu_mem_total"), 2 ** 20), "util_pct": hw.get("gpu_util"),
+                        "temp_c": hw.get("gpu_temp"), "power_w": hw.get("gpu_power")} if static.get("gpu_name") else None,
+                "ram": {"used_gib": scaled(hw.get("ram_used"), 2 ** 30, 1),
+                        "total_gib": scaled(hw.get("ram_total"), 2 ** 30, 1)} if hw.get("ram_total") else None}}
+
     def prepare(self, messages, tools, kwargs, max_new=None):
         """-> (ids, thinking, max_new). An unset or non-positive max_new (some clients send -1) means "unlimited":
         the rest of the context."""
@@ -644,17 +727,24 @@ class Service:
                 raise ValueError("this server was started without the vision encoder (run setup again and choose "
                                  "'vision'), so it cannot read images")
             pad = self.tok.encode(IMAGE_PAD, parse_special=True)[0]
+            start = self.tok.encode(VISION_START, parse_special=True)[0]
             # Encode only while the engine is idle: the engine and the image encoder (a separate process) must not
             # run on the GPU at the same time - an encode during a running request left that request stuck at
             # "reading the prompt" with CPU and GPU busy, for good (reproduced).  So encoding takes its turn in the
             # same FIFO as the requests.
             with self.fifo:
                 encoded = [self.vision.encode(src) for src in images]
+            # one <|image_pad|> per image -> one per image token.  Only the markers the template writes for an image
+            # (right after <|vision_start|>) are images: the same text inside a message (an agent reading these docs,
+            # #150) is kept as plain text, or it took an image's place and the counts no longer matched.
+            literal = self.tok.encode(IMAGE_PAD, parse_special=False)
             out, k = [], 0
-            for t in ids:                               # one <|image_pad|> per image -> one per image token
-                if t == pad and k < len(encoded):
+            for j, t in enumerate(ids):
+                if t == pad and j > 0 and ids[j - 1] == start and k < len(encoded):
                     out += [pad] * encoded[k][1]
                     k += 1
+                elif t == pad:
+                    out += literal
                 else:
                     out.append(t)
             if k != len(encoded):
@@ -684,6 +774,7 @@ class Service:
             s["generated"] = n
             if s.get("first_token") is None:
                 s["first_token"] = time.time()
+            self.rate.append((time.time(), n))          # the live rate's window over the last RATE_WINDOW_S
             for ev in evs:
                 if ev.kind == "reasoning":
                     s["phase"] = "thinking"
@@ -721,8 +812,12 @@ class Service:
             sampling = {**defaults, **req_values}
         parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True)
         detok, n, finish = Detokenizer(self.tok), 0, "length"
+        timings, before = None, None                    # this request's timings; the engine's `last` before it
         raw_ids = []                                    # every generated id (STRATA_DEBUG: dump raw model text)
         emb = getattr(self.embeddings, "path", None)
+        # Identity token: only a DONE line replaces engine.last, so a request that died, errored or was
+        # disconnected must not have the PREVIOUS request's decode figures recorded as its own.
+        engine_last0 = getattr(self.engine, "last", None)
         with self.status_lock:
             self.status["queued"] += 1
         try:
@@ -739,6 +834,9 @@ class Service:
                 with self.status_lock:
                     self.status.update(busy=True, phase="reading the prompt", prompt_tokens=len(ids), generated=0,
                                        started=time.time(), first_token=None, tool=None, tail="", max_tokens=max_new)
+                    self.last_request_at = time.time()
+                    self.rate.clear()               # the previous request's samples must not leak into this one
+                before = getattr(self.engine, "last", None)
                 last_print = time.time()
                 gen = self.engine.generate(ids, max_new, sampling, cancel, embeddings=emb) if emb else \
                     self.engine.generate(ids, max_new, sampling, cancel)
@@ -784,7 +882,9 @@ class Service:
                 Path(emb).unlink(missing_ok=True)
             with self.status_lock:
                 if self.status.get("busy"):
-                    last = dict(getattr(self.engine, "last", {}) or {})
+                    # only this request's DONE counts: same object means no DONE arrived (death, error, disconnect)
+                    last = dict(getattr(self.engine, "last", {}) or {}) \
+                        if getattr(self.engine, "last", None) is not engine_last0 else {}
                     started = self.status.get("started", time.time())
                     loaded = str((getattr(self.engine, "info", {}) or {}).get("cvec", 0)) not in ("0", "", "None")
                     hit_rate = round(last["hits"] / last["lookups"], 3) if last.get("lookups") else None
@@ -793,6 +893,7 @@ class Service:
                         if loaded else None,
                         "time": started, "duration_s": round(time.time() - started, 1), "finish": finish,
                         "prompt_tokens": len(ids), "reused": last.get("reused"), "output_tokens": n,
+                        "engine_generated": last.get("generated"),
                         "prompt_ms": last.get("prompt_ms"), "decode_ms": last.get("decode_ms"),
                         "decode_tok_s": round(last["generated"] / (last["decode_ms"] / 1000), 1)
                         if n and last.get("generated") and last.get("decode_ms") else None,
@@ -804,6 +905,11 @@ class Service:
                     t["output_tokens"] += n
                     t["prompt_ms"] += last.get("prompt_ms") or 0.0
                     t["decode_ms"] += last.get("decode_ms") or 0.0
+                    fresh = getattr(self.engine, "last", None)
+                    if fresh is not None and fresh is not before:      # the engine's clock for THIS request
+                        timings = request_timings(len(ids), n, last)
+                        self.last_timings = dict(timings, at=int(time.time())) if timings else None
+                    self.last_request_at = time.time()
                     now = time.time()
                     el = now - self.status.get("started", now)
                     ft = self.status.get("first_token")
@@ -814,9 +920,32 @@ class Service:
                     if os.environ.get("STRATA_DEBUG") and raw_ids:
                         print(f"[strata] raw: {self.tok.decode(raw_ids)!r}", flush=True)
                 self.status["busy"] = False
+                self.status.pop("tail", None)            # #212: the answer's end is not kept once it is done
+                self.status.pop("tool", None)
         for ev in parser.finish():
             yield "event", ev
-        yield "done", {"finish": finish, "completion_tokens": n}
+        yield "done", {"finish": finish, "completion_tokens": n, "reused": (timings or {}).get("cache_n", 0),
+                       "timings": timings}
+
+
+def request_timings(prompt_tokens: int, generated: int, last: dict) -> dict | None:
+    """One request's `timings` in llama.cpp's names (what its clients show as speed), from the engine's own clock
+    (StrataEngine.last): prompt_n is what was read, cache_n what the conversation cache already held.  None when the
+    engine keeps no clock (MockEngine)."""
+    if last.get("prompt_ms") is None:
+        return None
+    cache_n = int(last.get("reused") or 0)
+    prompt_n, prompt_ms, decode_ms = max(0, prompt_tokens - cache_n), float(last["prompt_ms"]), float(last.get("decode_ms") or 0)
+    decoded = int(last.get("generated") or generated)          # the engine's count gives its rate, as /metrics does
+    return {"cache_n": cache_n, "prompt_n": prompt_n, "prompt_ms": round(prompt_ms, 1),
+            "prompt_per_token_ms": round(prompt_ms / prompt_n, 3) if prompt_n else None,
+            "prompt_per_second": round(prompt_n / (prompt_ms / 1000), 1) if prompt_n and prompt_ms > 0 else None,
+            "predicted_n": generated, "predicted_ms": round(decode_ms, 1),
+            "predicted_per_token_ms": round(decode_ms / decoded, 3) if decoded else None,
+            "predicted_per_second": round(decoded / (decode_ms / 1000), 1) if decoded and decode_ms > 0 else None,
+            # the speculative drafts, as llama.cpp names them (from PR #83, @mikicvi): only when the engine reported them
+            **({"draft_n": int(last["drafts_offered"]), "draft_n_accepted": int(last["drafts_accepted"])}
+               if last.get("drafts_offered") is not None else {})}
 
 
 def _debug_req(api, req, messages, tools, max_new, thinking, prompt_tokens):
@@ -967,7 +1096,11 @@ def openai_chunks(svc: Service, req: dict, ids, thinking, tools, max_new, cancel
             last = chunk({}, finish)
             pt = x.get("prompt_tokens", len(ids))     # after MCP rounds: the last round's prompt
             last["usage"] = {"prompt_tokens": pt, "completion_tokens": x["completion_tokens"],
-                             "total_tokens": pt + x["completion_tokens"]}
+                             "total_tokens": pt + x["completion_tokens"],
+                             # the part of the prompt the conversation cache already held (OpenAI's field)
+                             "prompt_tokens_details": {"cached_tokens": x.get("reused") or 0}}
+            if x.get("timings"):
+                last["timings"] = x["timings"]          # llama.cpp's field: the speed its clients show
             yield last
 
 
@@ -997,9 +1130,12 @@ def openai_collect(chunks) -> dict:
         msg["tool_calls"] = calls
     if mcp:
         msg["strata_mcp"] = mcp
-    return {"id": last["id"], "object": "chat.completion", "created": last["created"], "model": last["model"],
-            "choices": [{"index": 0, "message": msg, "finish_reason": last["choices"][0]["finish_reason"]}],
-            "usage": last["usage"]}
+    out = {"id": last["id"], "object": "chat.completion", "created": last["created"], "model": last["model"],
+           "choices": [{"index": 0, "message": msg, "finish_reason": last["choices"][0]["finish_reason"]}],
+           "usage": last["usage"]}
+    if last.get("timings"):
+        out["timings"] = last["timings"]
+    return out
 
 
 # ------------------------------------------------------------------------------------------------ Anthropic
@@ -1059,8 +1195,12 @@ def anthropic_events(svc: Service, req: dict, ids, thinking, tools, max_new, can
                 yield close()
             stop = "tool_use" if used_tool and x["finish"] == "stop" else \
                 {"stop": "end_turn", "length": "max_tokens", "cancel": "end_turn"}[x["finish"]]
+            # the final counts, Anthropic's way: input_tokens leaves out what the conversation cache already held,
+            # which is cache_read_input_tokens (message_start could only say the whole prompt)
+            reused = min(x.get("reused") or 0, len(ids))
             yield "message_delta", {"type": "message_delta", "delta": {"stop_reason": stop, "stop_sequence": None},
-                                    "usage": {"output_tokens": x["completion_tokens"]}}
+                                    "usage": {"input_tokens": len(ids) - reused, "cache_read_input_tokens": reused,
+                                              "output_tokens": x["completion_tokens"]}}
             yield "message_stop", {"type": "message_stop"}
 
 
@@ -1087,7 +1227,7 @@ def anthropic_collect(events) -> dict:
             b["input"] = json.loads(b.pop("_json") or "{}")
         elif name == "message_delta":
             msg["stop_reason"] = e["delta"]["stop_reason"]
-            msg["usage"]["output_tokens"] = e["usage"]["output_tokens"]
+            msg["usage"].update(e["usage"])
     msg["content"] = blocks
     return msg
 
@@ -1113,7 +1253,7 @@ def make_handler(svc: Service):
                 return True
             auth = self.headers.get("Authorization", "")
             given = auth[7:].strip() if auth.lower().startswith("bearer ") else self.headers.get("x-api-key", "")
-            if given == svc.api_key:
+            if hmac.compare_digest(given.encode(), svc.api_key.encode()):   # #213: constant-time
                 return True
             self._json(401, {"error": {"type": "authentication_error", "message": "missing or wrong API key"}})
             return False
@@ -1178,19 +1318,40 @@ def make_handler(svc: Service):
                 self._json(200, {"status": "ok", "max_context": svc.engine.max_context, "model": svc.model,
                                  "images": svc.vision is not None, "api_key": bool(svc.api_key)})
             elif path == "/status":
+                if not self._authorized():                  # #212: it shows the end of the last answer
+                    return
                 with svc.status_lock:
                     s = dict(svc.status)
                 now = time.time()
                 if s.get("busy"):
                     s["elapsed_s"] = round(now - s["started"], 1)
                     if s.get("first_token"):
-                        s["tokens_per_s"] = round(s["generated"] / max(1e-6, now - s["first_token"]), 1)
+                        s["tokens_per_s"] = round(svc._tok_s(), 1)
+                        s["tokens_per_s_mean"] = round(svc._tok_s_mean(), 1)
                 for k in ("started", "first_token"):
                     s.pop(k, None)
                 self._json(200, s)
-            elif path == "/v1/models":
+            elif path in ("/v1/models", "/models"):
                 if self._authorized():
-                    self._json(200, {"object": "list", "data": [{"id": svc.model, "object": "model"}]})
+                    loaded = not hasattr(svc.engine, "alive") or svc.engine.alive()
+                    model = {"id": svc.model, "object": "model", "status": {"value": "loaded"},
+                             "meta": {"n_ctx": svc.engine.max_context},
+                             "architecture": {"input_modalities": ["text", "image"] if svc.vision is not None else ["text"],
+                                              "output_modalities": ["text"]}}
+                    self._json(200, {"object": "list", "data": [model] if loaded else []})
+            elif path == "/props":
+                if self._authorized():
+                    self._props()
+            elif path == "/slots":
+                if self._authorized():
+                    loaded = not hasattr(svc.engine, "alive") or svc.engine.alive()
+                    with svc.status_lock:
+                        busy = bool(svc.status.get("busy"))
+                    slot = {"id": 0, "n_ctx": svc.engine.max_context, "is_processing": busy}
+                    self._json(200, [slot] if loaded else [])
+            elif path == "/v1/status":
+                if self._authorized():
+                    self._json(200, svc.v1_status())
             else:
                 self._json(404, {"error": {"message": "not found"}})
 
@@ -1214,6 +1375,31 @@ def make_handler(svc: Service):
             except EngineDied as e:                          # before the answer started (not streamed)
                 self._json(503, {"error": {"type": "server_error", "message": f"{e}; the next request restarts it"}})
 
+        def _props(self):
+            model = parse_qs(urlsplit(self.path).query).get("model", [svc.model])[0]
+            if model != svc.model:
+                self._json(404, {"error": {"message": "model not found"}})
+                return
+            if hasattr(svc.engine, "alive") and not svc.engine.alive():
+                self._json(503, {"error": {"message": "the engine is not running"}})
+                return
+            defaults = {**svc.sampling_defaults, **svc.shared}
+            names = {"repetition_penalty": "repeat_penalty", "penalty_last_n": "repeat_last_n"}
+            params = {names.get(k, k): v for k, v in defaults.items()
+                      if k in ("temperature", "top_p", "top_k", "min_p", "seed", "repetition_penalty",
+                               "presence_penalty", "frequency_penalty", "penalty_last_n")}
+            params["n_predict"] = svc.shared.get("max_tokens", -1)
+            props = {"default_generation_settings": {"n_ctx": svc.engine.max_context, "params": params},
+                     "total_slots": 1, "model_alias": svc.model, "chat_template": svc.template.source,
+                     "modalities": {"vision": svc.vision is not None}, "models_autoload": False,
+                     "is_sleeping": False}
+            if getattr(svc.engine, "model_path", None):
+                props["model_path"] = svc.engine.model_path
+            version = getattr(svc.engine, "info", {}).get("version")
+            if version:
+                props["build_info"] = "Strata " + str(version)
+            self._json(200, props)
+
         def _own_page(self, what) -> bool:
             """Only JSON (a form or a "simple" cross-site request can't send it without a CORS preflight, which this
             server never grants) and no foreign Origin: a web page elsewhere must not change settings or run tools."""
@@ -1325,6 +1511,10 @@ class Server(ThreadingHTTPServer):
     # either one (a forgotten second start of run-<model>.bat).  Without it the second start fails loudly instead.
     allow_reuse_address = os.name != "nt"
 
+    def handle_error(self, request, client_address):
+        if not isinstance(sys.exc_info()[1], ConnectionError):   # a client that hangs up needs no stack trace
+            super().handle_error(request, client_address)
+
 
 def warn_tight_ram(arena_mib) -> None:
     """The model's experts live in RAM (INFO arena_mib, engine 0.1.10+).  With less than ~6 GB left beside them for the
@@ -1548,8 +1738,15 @@ def main() -> int:
                   model_name=cfg.get("model_name", "qwen3.8-flash-next"), vision=vision,
                   sampling_defaults=sampling_defaults,
                   fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)
+    if ("STRATA_API_KEY" in os.environ and not os.environ["STRATA_API_KEY"].strip()) or             any(x == "--api-key" and i + 1 < len(sys.argv) and not sys.argv[i + 1].strip() or x.strip() == "--api-key="
+                for i, x in enumerate(sys.argv)):
+        # #213: an empty key would switch authentication off without a word
+        print("[strata] an API key was given but it is empty: set a key, or leave --api-key / STRATA_API_KEY out",
+              file=sys.stderr)
+        return 2
     svc.api_key = a.api_key or cfg.get("api_key", "")
     svc.gpu_index = (gpu_list(cfg) or [0])[0]           # the Monitor reads the card the engine runs on (issue #51)
+    svc.gpu_indices = gpu_list(cfg)                     # ... or every card of a layer split (issue #112)
     if a.config:                                        # the Chat settings shared with other apps, from last time
         svc.shared_path = str(Path(a.config).with_suffix("")) + ".shared-settings.json"
         try:
@@ -1591,15 +1788,19 @@ def main() -> int:
         import webbrowser
         webbrowser.open(f"http://{'127.0.0.1' if a.host in ('0.0.0.0', '') else a.host}:{a.port}/")
     try:
-        threading.Event().wait()
+        while True:
+            time.sleep(1)                               # Windows never delivers Ctrl+C to an untimed Event.wait()
     except KeyboardInterrupt:
-        httpd.shutdown()
-        if hasattr(engine, "close"):
-            engine.close()
-        if vision:
-            vision.close()
-        if hub is not None:
-            hub.close()
+        print("\n[strata] stopping (Ctrl+C again to end the engine at once) ...", flush=True)
+        closers = [httpd.shutdown, getattr(engine, "close", None), vision.close if vision else None,
+                   hub.close if hub is not None else None]
+        for close in filter(None, closers):
+            try:
+                close()
+            except KeyboardInterrupt:                   # a second Ctrl+C: don't wait for the engine to free its memory
+                if getattr(engine, "proc", None):
+                    engine.proc.kill()
+        print("[strata] stopped", flush=True)
     return 0
 
 

serve/telemetry.py

@@ -169,14 +169,19 @@ class _CpuRamFallback:
 
 # ------------------------------------------------------------------------------------------------ the sampler
 class Telemetry:
-    def __init__(self, extra=None, gpu_index=0):
+    def __init__(self, extra=None, gpu_index=0, gpu_indices=None):
         """`extra()` -> dict of more series to record each second (the server's tok/s).  `gpu_index`: the card the
-        engine runs on, numbered as nvidia-smi and NVML number them (by PCI bus)."""
+        engine runs on, numbered as nvidia-smi and NVML number them (by PCI bus); `gpu_indices`: all of them when
+        the model is split across several (issue #112) - the gpu_* readings are then their total (memory, power,
+        PCIe traffic), mean (load) or hottest (temperature), and "gpus" has each card's own."""
         self.extra = extra
         self.lock = threading.Lock()
         self.now: dict = {}
         self.hist = collections.defaultdict(lambda: collections.deque(maxlen=HISTORY))
-        self.gpu = _Nvml(gpu_index)
+        idx = list(gpu_indices) if gpu_indices and len(gpu_indices) > 1 else [gpu_index]
+        self.gpus = [(i, _Nvml(i)) for i in idx]
+        self.gpus = [(i, g) for i, g in self.gpus if g.ok()] or self.gpus[:1]
+        self.gpu = self.gpus[0][1]
         try:
             import psutil  # noqa: F401
             self.ps = sys.modules["psutil"]
@@ -184,7 +189,8 @@ class Telemetry:
             self.ps = None
         self.fallback = _CpuRamFallback()
         self.static = {
-            "gpu_name": self.gpu.name() if self.gpu.ok() else None,
+            "gpu_name": " + ".join(g.name() or "?" for _, g in self.gpus) if self.gpu.ok() else None,
+            "gpu_count": len(self.gpus),
             "cpu_name": _cpu_name(),
             "cores": (self.ps.cpu_count(logical=False) if self.ps else None) or None,
             "threads": os.cpu_count(),
@@ -210,7 +216,21 @@ class Telemetry:
     def sample(self):
         s = {}
         if self.gpu.ok():
-            g = self.gpu.read()
+            reads = [(i, g.read()) for i, g in self.gpus]
+            g = dict(reads[0][1])
+            if len(reads) > 1:
+                def vals(k):
+                    return [r[k] for _, r in reads if r.get(k) is not None]
+                for k in ("mem_used", "mem_total", "power", "power_limit", "pcie_rx_mb", "pcie_tx_mb"):
+                    v = vals(k)
+                    g[k] = sum(v) if v else None
+                u = vals("util")
+                g["util"] = sum(u) / len(u) if u else None
+                t = vals("temp")
+                g["temp"] = max(t) if t else None
+                s["gpus"] = [{"index": i, "util": r.get("util"), "mem_used": r.get("mem_used"),
+                              "mem_total": r.get("mem_total"), "temp": r.get("temp"), "power": r.get("power")}
+                             for i, r in reads]
             s.update({f"gpu_{k}": v for k, v in g.items()})
         if self.ps:
             try:

serve/test_detok.py

@@ -0,0 +1,107 @@
+"""The streaming detokenizer (perf-review F-1): the incremental one emits exactly what the old re-decode did, token
+by token, and its cost per token does not grow with the reply.
+
+    python -m unittest serve.test_detok      (needs a pack's tokenizer; skipped without one)
+"""
+import os
+import random
+import sys
+import time
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT / "tools"))
+sys.path.insert(0, str(ROOT))
+
+from serve.server import Detokenizer  # noqa: E402
+
+
+def find_tokenizer():
+    cands = [os.environ.get("STRATA_TOKENIZER", "")]
+    cands += [str(p) for p in ROOT.glob("packs/*/tokenizer")] + [str(p) for p in ROOT.glob("pack/*/tokenizer")]
+    cands += [r"C:\Users\AI-Server\Desktop\Strata\Public\Engine\pack\full\tokenizer"]
+    for c in cands:
+        if c and (Path(c) / "vocab.json").exists():
+            return Path(c)
+    return None
+
+
+class OldDetokenizer:
+    """The pre-0.1.22 algorithm: re-decode every generated id, hold a trailing U+FFFD."""
+
+    def __init__(self, tok):
+        self.tok, self.ids, self.sent = tok, [], 0
+
+    def push(self, t):
+        self.ids.append(t)
+        text = self.tok.decode(self.ids)
+        if text.endswith("\ufffd"):
+            return ""
+        delta, self.sent = text[self.sent:], len(text)
+        return delta
+
+
+@unittest.skipIf(find_tokenizer() is None, "no pack tokenizer here")
+class Detok(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        import json
+        import strata_tokenizer as ST
+        t = find_tokenizer()
+        vocab = json.loads((t / "vocab.json").read_text(encoding="utf-8"))
+        toks = [None] * len(vocab)
+        for s, i in vocab.items():
+            toks[i] = s
+        cls.tok = ST.Tokenizer(toks, (t / "merges.txt").read_text(encoding="utf-8").split("\n"),
+                               json.loads((t / "token_type.json").read_text()))
+
+    def same_stream(self, ids):
+        """The same text as the old re-decode, never behind it (the old one held complete characters too while
+        the tail was an incomplete one; the new one holds only the incomplete bytes), and never ahead of what the
+        ids so far decode to."""
+        a, b = OldDetokenizer(self.tok), Detokenizer(self.tok)
+        self.assertIsNotNone(b.inc)
+        old = new = ""
+        for i, t in enumerate(ids):
+            old += a.push(t)
+            new += b.push(t)
+            self.assertTrue(new.startswith(old), f"token {i} ({t}): behind the old stream")
+            self.assertTrue(self.tok.decode(ids[:i + 1]).startswith(new), f"token {i} ({t}): ahead of the ids")
+        if not self.tok.decode(ids).endswith("�"):
+            self.assertEqual(old, new)
+
+    def test_text_with_split_characters(self):
+        text = ("Grüße aus Zürich! 你好，世界。日本語のテキスト 🌊🦊👩‍💻 and ASCII code: def f(x): return x**2\n" * 40)
+        self.same_stream(self.tok.encode(text))
+
+    def test_random_ids_and_broken_bytes(self):
+        rng = random.Random(5)
+        n = len(self.tok.tokens)
+        for _ in range(20):   # random ids: split and invalid UTF-8 sequences everywhere (no U+FFFD token itself)
+            ids = [rng.randrange(n) for _ in range(300)]
+            ids = [t for t in ids if "\ufffd" not in self.tok.decode([t], errors="strict" if False else "replace")
+                   or len(self.tok.token_bytes(t)) < 3]
+            self.same_stream(ids)
+
+    def test_constant_cost(self):
+        ids = self.tok.encode("The quick brown fox jumps over the lazy dog. " * 2000)[:16384]
+        d = Detokenizer(self.tok)
+        for t in ids[:1024]:
+            d.push(t)
+        t0 = time.perf_counter()
+        for t in ids[1024:1124]:
+            d.push(t)
+        early = time.perf_counter() - t0
+        for t in ids[1124:16000]:
+            d.push(t)
+        t0 = time.perf_counter()
+        for t in ids[16000:16100]:
+            d.push(t)
+        late = time.perf_counter() - t0
+        self.assertLess(late / 100, 0.0002, "a token after 16K tokens costs more than 0.2 ms")
+        self.assertLess(late, early * 5 + 0.002)
+
+
+if __name__ == "__main__":
+    unittest.main()

serve/test_server.py

@@ -7,6 +7,8 @@ from __future__ import annotations
 import json
 import os
 import sys
+import threading
+import time
 import unittest
 import urllib.error
 import urllib.request
@@ -14,7 +16,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 from serve.frontend import ChatTemplate  # noqa: E402
-from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, MockEngine, Service, StrataEngine, serve  # noqa: E402
+from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, MockEngine, Service, StrataEngine, request_timings, serve  # noqa: E402
 
 ROOT = Path(__file__).resolve().parents[1]
 CTX = 4096
@@ -165,6 +167,90 @@ class FitMaxTokens(unittest.TestCase):
         self.assertIn("no room to answer", b["error"]["message"])
 
 
+class ImageMarkers(unittest.TestCase):
+    """#150: the text "<|image_pad|>" inside a message is text, not an image's place."""
+
+    class FakeVision:
+        def __init__(self, d):
+            self.dir = Path(d)
+            self.rows = self.dir / "img.sve"
+            self.rows.write_bytes(b"rows")
+
+        def encode(self, source):
+            return self.rows, 3
+
+    def test_literal_marker_with_an_image(self):
+        import tempfile
+        tok = ByteTokenizer()
+        with tempfile.TemporaryDirectory() as d:
+            svc = Service(MockEngine(tok, "ok", max_context=CTX), tok, ChatTemplate(ROOT / "serve/chat_template.jinja"),
+                          vision=self.FakeVision(d))
+            pad = tok.encode("<|image_pad|>", parse_special=True)[0]
+            for text in ("the docs say <|image_pad|> marks an image", "plain"):
+                with self.subTest(text=text):
+                    msgs = [{"role": "user", "content": [{"type": "text", "text": text},
+                                                         {"type": "image", "source": "x.png"}]}]
+                    ids, _, _ = svc.prepare(msgs, None, {})
+                    self.assertEqual(ids.count(pad), 3)          # the image's three rows, nothing else
+                    self.assertIn("<|image_pad|> marks" if "docs" in text else "plain", tok.decode(ids))
+            svc.embeddings.path.unlink(missing_ok=True)
+
+
+class StatusNeedsTheKey(unittest.TestCase):
+    """#212: /status shows the end of the answer being written, so it needs the key like /v1/*."""
+
+    def test_status(self):
+        tok = ByteTokenizer()
+        svc = Service(MockEngine(tok, "ok", max_context=CTX), tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        svc.api_key = "k3y"
+        httpd = serve(svc, port=0)
+        base = f"http://127.0.0.1:{httpd.server_address[1]}/status"
+        try:
+            with self.assertRaises(urllib.error.HTTPError) as e:
+                urllib.request.urlopen(base, timeout=10)
+            self.assertEqual(e.exception.code, 401)
+            e.exception.close()
+            req = urllib.request.Request(base, headers={"Authorization": "Bearer k3y"})
+            with urllib.request.urlopen(req, timeout=10) as r:
+                self.assertEqual(r.status, 200)
+                self.assertNotIn("tail", json.loads(r.read()))
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+
+
+class ToolCallTerminators(unittest.TestCase):
+    """#210: a value that contains </parameter> or </tool_call> (a file documenting the call format) is kept whole."""
+    CONTENT = ("Close each value with </parameter> and the call with </function></tool_call>.\n"
+               "<parameter=x>\nnot a parameter\n</parameter>\nend")
+    SCHEMA = [{"name": "write", "parameters": {"properties": {"path": {"type": "string"},
+                                                              "content": {"type": "string"}}}}]
+
+    def run_parser(self, stream_tools, step):
+        from serve.frontend import OutputParser
+        text = ("</think>\n\n<tool_call>\n<function=write>\n<parameter=path>\ndoc.md\n</parameter>\n"
+                f"<parameter=content>\n{self.CONTENT}\n</parameter>\n</function>\n</tool_call>")
+        p = OutputParser(thinking=True, tools=self.SCHEMA, stream_tools=stream_tools)
+        evs = []
+        for i in range(0, len(text), step):
+            evs += p.feed(text[i:i + step])
+        evs += p.finish()
+        return evs
+
+    def test_values_keep_the_terminators(self):
+        for stream_tools in (False, True):
+            for step in (1, 7, 10_000):
+                with self.subTest(stream_tools=stream_tools, step=step):
+                    evs = self.run_parser(stream_tools, step)
+                    calls = [e.call for e in evs if e.kind == "tool_call"]
+                    self.assertEqual(len(calls), 1)
+                    self.assertEqual(calls[0].arguments, {"path": "doc.md", "content": self.CONTENT})
+                    self.assertFalse([e for e in evs if e.kind == "content" and e.text.strip()])
+                    if stream_tools:
+                        streamed = "".join(e.text for e in evs if e.kind == "tool_args")
+                        self.assertEqual(json.loads(streamed), {"path": "doc.md", "content": self.CONTENT})
+
+
 class ClientShapes(unittest.TestCase):
     """What real clients send: Claude Code posts /v1/messages?beta=true (issue #55) and puts hook context into the
     conversation as a mid-conversation system message (issue #56); some OpenAI clients send a late developer message."""
@@ -368,6 +454,97 @@ class EngineDeath(unittest.TestCase):
             httpd.server_close()
 
 
+class LiveRate(unittest.TestCase):
+    """The Monitor's Speed readout: live.tok_s is a rate, and a request that never got a DONE keeps no counters.
+
+    It used to be `generated / (now - first_token)` - the mean since the first token, whose first sample is
+    1/elapsed.  Against a paced engine that reads five-digit numbers for the first instant of every answer and
+    undershoots for the first second after that.  It is now the rate over the last RATE_WINDOW_S, with the mean
+    still available as `live.tok_s_mean` for anyone who wants it."""
+
+    PACE_S = 0.02                    # 50 tokens/s: a 30-token answer takes about 0.6 s
+    TOKENS = 30
+
+    def setUp(self):
+        self.tok = ByteTokenizer()
+        self.engine = MockEngine(self.tok, "x" * self.TOKENS, max_context=CTX, delay_s=self.PACE_S)
+        self.svc = Service(self.engine, self.tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        self.httpd = serve(self.svc, port=0)
+        self.base = f"http://127.0.0.1:{self.httpd.server_address[1]}"
+
+    def tearDown(self):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+    def metrics(self):
+        with urllib.request.urlopen(self.base + "/metrics", timeout=10) as r:
+            return json.loads(r.read())
+
+    def test_the_live_number_is_a_rate(self):
+        live_samples, stop = [], threading.Event()
+
+        def poll():                                   # what the Monitor polls, at 10 ms
+            while not stop.is_set():
+                live = self.metrics()["live"]
+                if live["state"] == "generating" and live["tok_s"] is not None:
+                    live_samples.append((live["generated"], live["tok_s"], live["tok_s_mean"]))
+                time.sleep(0.01)
+
+        body = json.dumps({"model": "m", "max_tokens": self.TOKENS, "temperature": 0,
+                           "messages": [{"role": "user", "content": "hi"}]}).encode()
+        watcher = threading.Thread(target=poll, daemon=True)
+        watcher.start()
+        t0 = time.time()
+        try:
+            with urllib.request.urlopen(urllib.request.Request(self.base + "/v1/chat/completions", data=body,
+                                                               headers={"Content-Type": "application/json"}),
+                                        timeout=30) as r:
+                usage = json.loads(r.read())["usage"]
+        finally:
+            stop.set()
+            watcher.join(2)
+        true_rate = usage["completion_tokens"] / (time.time() - t0)
+        self.assertGreaterEqual(len(live_samples), 3, "too few live readings to judge the readout")
+        self.assertLess(max(s for _g, s, _m in live_samples), 4 * true_rate,
+                        f"live.tok_s peaked at {max(s for _g, s, _m in live_samples):.1f} tok/s "
+                        f"for a {true_rate:.1f} tok/s engine")
+        self.assertEqual(self.metrics()["live"]["state"], "idle")
+        self.assertIsNone(self.metrics()["live"]["tok_s"])
+
+    def test_a_request_without_a_done_keeps_no_engine_counters(self):
+        """An engine that dies mid-answer: the previous request's `last` must not become this row's decode rate."""
+        class HalfDead(MockEngine):
+            last = {"generated": 99, "prompt_tokens": 9, "prompt_ms": 10.0, "decode_ms": 100.0, "finish": "stop"}
+
+            def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+                for i, t in enumerate(super().generate(ids, max_new, sampling, cancel, embeddings)):
+                    if i == 3:
+                        raise EngineDied("the engine stopped unexpectedly (exit code -9)")
+                    yield t
+
+        tok = ByteTokenizer()
+        svc = Service(HalfDead(tok, "x" * self.TOKENS, max_context=CTX), tok,
+                      ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        httpd = serve(svc, port=0)
+        base = f"http://127.0.0.1:{httpd.server_address[1]}"
+        try:
+            body = json.dumps({"model": "m", "max_tokens": self.TOKENS, "stream": True,
+                               "messages": [{"role": "user", "content": "hi"}]}).encode()
+            with urllib.request.urlopen(urllib.request.Request(base + "/v1/chat/completions", data=body,
+                                                               headers={"Content-Type": "application/json"}),
+                                        timeout=30) as r:
+                text = r.read().decode()
+            self.assertIn('"error"', text)
+            row = svc.metrics()["requests"][0]
+            self.assertEqual(row["finish"], "error")
+            self.assertEqual(row["output_tokens"], 3)
+            self.assertIsNone(row["decode_tok_s"], "the previous request's counters were recorded as this one's")
+            self.assertIsNone(row["engine_generated"])
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+
+
 class SharedSettings(unittest.TestCase):
     """The web app's "Use for other apps too": POST /settings makes its Chat settings every client's defaults."""
 
@@ -489,7 +666,7 @@ class WebApp(unittest.TestCase):
         code, ctype, body = self.get("/")
         self.assertEqual(code, 200)
         self.assertIn("text/html", ctype)
-        self.assertIn(b"/web/app.js", body)
+        self.assertIn(b"\"web/app.js\"", body)   # relative since #82 (works behind a path-prefixed proxy)
         for path, want in (("/web/app.js", "javascript"), ("/web/app.css", "text/css"), ("/web/tokens.css", "text/css"),
                            ("/web/components.css", "text/css"), ("/web/sprite.svg", "image/svg+xml")):
             with self.subTest(path=path):
@@ -516,6 +693,86 @@ class WebApp(unittest.TestCase):
         self.assertEqual(m["live"]["state"], "idle")
         self.assertEqual(m["requests"][0]["output_tokens"], 5)
 
+    def test_model_discovery_and_props(self):
+        svc = self.svc
+        previous = svc.engine.max_context, svc.vision, svc.sampling_defaults, svc.shared
+        try:
+            svc.engine.max_context = 262144
+            svc.sampling_defaults = {"temperature": 1.0, "repetition_penalty": 1.1}
+            svc.shared = {"temperature": 0.7, "max_tokens": 4096}
+            for vision in (None, object()):
+                svc.vision = vision
+                for path in ("/models", "/v1/models"):
+                    code, _, body = self.get(path)
+                    self.assertEqual(code, 200)
+                    models = json.loads(body)["data"]
+                    self.assertEqual(len(models), 1)
+                    model = models[0]
+                    self.assertEqual(model["id"], svc.model)
+                    self.assertEqual(model["status"]["value"], "loaded")
+                    self.assertEqual(model["meta"]["n_ctx"], 262144)
+                    self.assertEqual(model["architecture"]["input_modalities"],
+                                     ["text", "image"] if vision else ["text"])
+                code, _, body = self.get("/props?model=" + svc.model + "&autoload=false")
+                self.assertEqual(code, 200)
+                props = json.loads(body)
+                self.assertEqual(props["default_generation_settings"]["n_ctx"], 262144)
+                self.assertEqual(props["default_generation_settings"]["params"],
+                                 {"temperature": 0.7, "repeat_penalty": 1.1, "n_predict": 4096})
+                self.assertEqual(props["chat_template"], (ROOT / "serve/chat_template.jinja").read_text(encoding="utf-8"))
+                self.assertEqual(props["modalities"]["vision"], vision is not None)
+                self.assertEqual(props["total_slots"], 1)
+                self.assertFalse(props["models_autoload"])
+            svc.shared = {}
+            props = json.loads(self.get("/props")[2])
+            self.assertEqual(props["default_generation_settings"]["params"]["n_predict"], -1)
+            self.assertEqual(self.get("/props?model=not-loaded&autoload=true")[0], 404)
+        finally:
+            svc.engine.max_context, svc.vision, svc.sampling_defaults, svc.shared = previous
+
+    def test_discovery_needs_the_api_key(self):
+        self.svc.api_key = "secret"
+        try:
+            for path in ("/models", "/v1/models", "/props", "/slots"):
+                self.assertEqual(self.get(path)[0], 401)
+                self.assertEqual(self.get(path, {"Authorization": "Bearer secret"})[0], 200)
+        finally:
+            self.svc.api_key = ""
+
+    def test_build_model_path_and_slot_status(self):
+        engine = self.svc.engine
+        engine.model_path = "models/example.gguf"
+        engine.info = {"version": "0.1.21"}
+        try:
+            props = json.loads(self.get("/props")[2])
+            self.assertEqual(props["model_path"], engine.model_path)
+            self.assertEqual(props["build_info"], "Strata 0.1.21")
+            for busy in (True, False):
+                with self.svc.status_lock:
+                    self.svc.status["busy"] = busy
+                code, _, body = self.get("/slots")
+                self.assertEqual(code, 200)
+                self.assertEqual(json.loads(body), [{"id": 0, "n_ctx": CTX, "is_processing": busy}])
+        finally:
+            with self.svc.status_lock:
+                self.svc.status["busy"] = False
+            del engine.model_path, engine.info
+        props = json.loads(self.get("/props")[2])
+        self.assertNotIn("build_info", props)
+        self.assertNotIn("model_path", props)
+
+    def test_discovery_does_not_restart_a_dead_engine(self):
+        self.svc.engine.alive = lambda: False
+        try:
+            for path in ("/models", "/v1/models"):
+                code, _, body = self.get(path)
+                self.assertEqual(code, 200)
+                self.assertEqual(json.loads(body)["data"], [])
+            self.assertEqual(self.get("/props")[0], 503)
+            self.assertEqual(json.loads(self.get("/slots")[2]), [])
+        finally:
+            del self.svc.engine.alive
+
     def test_metrics_need_the_key_when_one_is_set(self):
         self.svc.api_key = "secret"
         try:
@@ -526,5 +783,119 @@ class WebApp(unittest.TestCase):
             self.svc.api_key = ""
 
 
+class ClockedEngine(MockEngine):
+    """The mock engine with StrataEngine's clock: `last` as the engine's DONE line gives it, the conversation cache
+    holding the first REUSED tokens of every prompt."""
+    REUSED = 5
+
+    def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+        n = 0
+        try:
+            for t in super().generate(ids, max_new, sampling, cancel, embeddings):
+                n += 1
+                yield t
+        finally:          # as StrataEngine reads its DONE line: also when the server closes the request at a stop token
+            self.last = {"generated": n, "prompt_tokens": len(ids), "prompt_ms": 40.0, "decode_ms": 20.0 * n,
+                         "finish": "stop", "reused": min(self.REUSED, len(ids)), "hits": 9, "lookups": 10}
+
+
+class UsageAndStatus(unittest.TestCase):
+    """What clients read besides the text: the part of the prompt the conversation cache held (OpenAI's
+    prompt_tokens_details.cached_tokens, Anthropic's cache_read_input_tokens), llama.cpp's timings, GET /v1/status."""
+
+    @classmethod
+    def setUpClass(cls):
+        tok = ByteTokenizer()
+        cls.engine = ClockedEngine(tok, "</think>\n\nok", max_context=CTX)
+        cls.svc = Service(cls.engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        cls.httpd = serve(cls.svc, port=0)
+        cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}"
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.httpd.shutdown()
+        cls.httpd.server_close()
+
+    def request(self, path, body=None):
+        req = urllib.request.Request(self.base + path, data=None if body is None else json.dumps(body).encode(),
+                                     headers={"Content-Type": "application/json", "anthropic-version": "2023-06-01"})
+        with urllib.request.urlopen(req, timeout=30) as r:
+            return r.status, r.read()
+
+    def chat(self, path, stream=False):
+        body = {"model": "x", "max_tokens": 20, "messages": [{"role": "user", "content": "hi"}], "stream": stream}
+        status, raw = self.request(path, body)
+        self.assertEqual(status, 200)
+        if not stream:
+            return json.loads(raw)
+        return [json.loads(line[6:]) for line in raw.decode().splitlines()
+                if line.startswith("data: {")]
+
+    def test_openai(self):
+        b = self.chat("/v1/chat/completions")
+        u, t = b["usage"], b["timings"]
+        self.assertEqual(u["prompt_tokens_details"]["cached_tokens"], ClockedEngine.REUSED)
+        self.assertEqual(t["cache_n"], ClockedEngine.REUSED)
+        self.assertEqual(t["prompt_n"] + t["cache_n"], u["prompt_tokens"])
+        self.assertEqual(t["predicted_n"], u["completion_tokens"])
+        self.assertAlmostEqual(t["prompt_per_second"], t["prompt_n"] / 0.040, delta=0.1)
+        self.assertAlmostEqual(t["predicted_per_second"], 50.0, delta=0.1)            # 20 ms a token
+
+    def test_openai_stream(self):
+        last = self.chat("/v1/chat/completions", stream=True)[-1]
+        self.assertEqual(last["usage"]["prompt_tokens_details"]["cached_tokens"], ClockedEngine.REUSED)
+        self.assertEqual(last["timings"]["cache_n"], ClockedEngine.REUSED)
+
+    def test_anthropic(self):
+        u = self.chat("/v1/messages")["usage"]
+        self.assertEqual(u["cache_read_input_tokens"], ClockedEngine.REUSED)
+        self.assertEqual(u["input_tokens"] + u["cache_read_input_tokens"], len(self.engine.last_prompt))
+        self.assertGreater(u["output_tokens"], 0)
+
+    def test_v1_status(self):
+        self.chat("/v1/chat/completions")
+        status, raw = self.request("/v1/status")
+        self.assertEqual(status, 200)
+        s = json.loads(raw)
+        self.assertEqual(s["model"], self.svc.model)
+        self.assertEqual(s["context"]["max_positions"], CTX)
+        self.assertEqual(s["concurrency"]["serving"], 1)
+        self.assertFalse(s["vision"]["available"])
+        self.assertEqual(s["activity"]["in_flight"], 0)
+        self.assertGreaterEqual(s["activity"]["requests"], 1)
+        self.assertEqual(s["last_timings"]["cache_n"], ClockedEngine.REUSED)
+        self.assertIn("at", s["last_timings"])
+
+    def test_no_clock(self):
+        """An engine without a clock (MockEngine): no timings, nothing cached."""
+        tok = ByteTokenizer()
+        svc = Service(MockEngine(tok, "</think>\n\nok", max_context=CTX), tok,
+                      ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        httpd = serve(svc, port=0)
+        try:
+            data = json.dumps({"model": "x", "max_tokens": 5, "messages": [{"role": "user", "content": "hi"}]}).encode()
+            req = urllib.request.Request(f"http://127.0.0.1:{httpd.server_address[1]}/v1/chat/completions", data=data,
+                                         headers={"Content-Type": "application/json"})
+            with urllib.request.urlopen(req, timeout=30) as r:
+                b = json.loads(r.read())
+            self.assertNotIn("timings", b)
+            self.assertEqual(b["usage"]["prompt_tokens_details"]["cached_tokens"], 0)
+            self.assertIsNone(svc.v1_status()["last_timings"])
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+
+
+class TimingsDrafts(unittest.TestCase):
+    """`timings` carries the speculative draft counts (PR #83's fields) only when the engine reported them."""
+
+    def test_draft_fields(self):
+        base = {"prompt_ms": 100.0, "decode_ms": 200.0, "generated": 20, "reused": 4}
+        t = request_timings(24, 20, dict(base, drafts_offered=15, drafts_accepted=11))
+        self.assertEqual((t["draft_n"], t["draft_n_accepted"]), (15, 11))
+        self.assertEqual((t["prompt_n"], t["cache_n"]), (20, 4))
+        self.assertNotIn("draft_n", request_timings(24, 20, base))
+        self.assertIsNone(request_timings(24, 20, {}))
+
 if __name__ == "__main__":
     unittest.main()

serve/web/app.js

@@ -3,7 +3,7 @@
 "use strict";
 
 const $ = (id) => document.getElementById(id);
-const SPRITE = "/web/sprite.svg";
+const SPRITE = "web/sprite.svg";
 const icon = (name, cls = "st-icon") => `<svg class="${cls}" aria-hidden="true"><use href="${SPRITE}#i-${name}"/></svg>`;
 const esc = (s) => String(s).replace(/[&<>"']/g, (c) => ({"&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;"}[c]));
 const fmt = (n, d = 0) => (n == null || Number.isNaN(n) ? "–" : Number(n).toLocaleString(undefined, {maximumFractionDigits: d, minimumFractionDigits: d}));
@@ -98,7 +98,7 @@ $("api-key").onchange = () => { store.set("apikey", $("api-key").value.trim());
 let health = {model: "strata", images: false, max_context: 0};
 async function loadHealth() {
   try {
-    health = await (await fetch("/health")).json();
+    health = await (await fetch("health")).json();
     $("attach-btn").title = health.images ? "Attach a text file or a picture (or drop it here)"
                                           : "Attach a text file (or drop it here)";
     $("chat-empty-sub").textContent = `${health.model} runs on this PC. Nothing leaves it.`;
@@ -147,7 +147,7 @@ let lastMetrics = null, metricsFailures = 0, keyWarned = false, mcpTick = 0;
 let reqShowAll = false;   // the Monitor's request table: the last 12, or every one the server keeps (issue #35)
 async function poll() {
   try {
-    const r = await fetch(reqShowAll ? "/metrics?requests=all" : "/metrics", {headers: headers()});
+    const r = await fetch(reqShowAll ? "metrics?requests=all" : "metrics", {headers: headers()});
     if (r.status === 401) {
       setPill("error", "API key needed");
       if (!keyWarned) { keyWarned = true; toast("warn", "API key needed", "This server needs a key: add it under About > Settings.", 6000); }
@@ -228,12 +228,18 @@ function renderMonitor(live, hw, st, eng, h, last, requests, totals, kept) {
   const speed = live.state === "generating" ? live.tok_s : last ? last.decode_tok_s : null;
   setMetric("speed", speed == null ? null : fmt(speed, 1), "tok/s", live.state === "generating" ? "now" : last ? "last request" : "");
   spark("sp-speed", h.tok_s);
-  setMetric("gpu", hw.gpu_util == null ? null : fmt(hw.gpu_util), "%", st.gpu_name || "");
+  // a model split across several cards (issue #112): the cards show their total / mean / hottest, and each card's own
+  const per = (f) => (hw.gpus || []).map((g) => `GPU ${g.index} ${f(g)}`).join(" · ");
+  const multi = (hw.gpus || []).length > 1;
+  setMetric("gpu", hw.gpu_util == null ? null : fmt(hw.gpu_util), "%",
+            multi ? per((g) => (g.util == null ? "–" : `${fmt(g.util)}%`)) : st.gpu_name || "");
   spark("sp-gpu", h.gpu_util, 100);
   setMetric("vram", hw.gpu_mem_used == null ? null : gb(hw.gpu_mem_used), hw.gpu_mem_total ? `/ ${gb(hw.gpu_mem_total, 0)} GB` : "GB",
-            eng.expert_slots ? `${fmt(eng.expert_slots)} experts cached` : "");
+            multi ? per((g) => (g.mem_used == null ? "–" : `${gb(g.mem_used)} GB`))
+                  : eng.expert_slots ? `${fmt(eng.expert_slots)} experts cached` : "");
   spark("sp-vram", h.gpu_mem_used, hw.gpu_mem_total);
-  setMetric("temp", hw.gpu_temp == null ? null : fmt(hw.gpu_temp), "°C", "");
+  setMetric("temp", hw.gpu_temp == null ? null : fmt(hw.gpu_temp), "°C",
+            multi ? per((g) => (g.temp == null ? "–" : `${fmt(g.temp)}°`)) : "");
   spark("sp-temp", h.gpu_temp, 90);
   setMetric("power", hw.gpu_power == null ? null : fmt(hw.gpu_power), "W", hw.gpu_power_limit ? `of ${fmt(hw.gpu_power_limit)} W limit` : "");
   spark("sp-power", h.gpu_power, hw.gpu_power_limit);
@@ -347,7 +353,7 @@ $("req-all").addEventListener("click", () => { reqShowAll = !reqShowAll; if (las
 let mcpInfo = {servers: [], tools: 0}, mcpRetry = null;
 async function loadMcp() {
   try {
-    const r = await fetch("/mcp", {headers: headers()});
+    const r = await fetch("mcp", {headers: headers()});
     if (!r.ok) return;
     mcpInfo = await r.json();
   } catch (e) { return; /* an older server: no MCP */ }
@@ -679,7 +685,7 @@ async function send() {
   let firstAt = null, thinkStart = null, usage = null, frame = 0;
   const paint = () => { frame = 0; updateAssistant(el, m, true); scrollDown(); };
   try {
-    const r = await fetch("/v1/chat/completions", {method: "POST", headers: headers(true), body: JSON.stringify(body),
+    const r = await fetch("v1/chat/completions", {method: "POST", headers: headers(true), body: JSON.stringify(body),
                                                    signal: controller.signal});
     if (!r.ok) {
       let msg = `HTTP ${r.status}`;
@@ -881,7 +887,7 @@ function loadDrawer(s = settings) {
 let sharedOn = false;
 async function loadShared() {
   try {
-    const r = await fetch("/settings", {headers: headers()});
+    const r = await fetch("settings", {headers: headers()});
     if (r.ok) sharedOn = !!(await r.json()).shared;
   } catch (e) { /* an older server: the switch just stays off */ }
   $("s-share").setAttribute("aria-checked", String(sharedOn));
@@ -895,7 +901,7 @@ function sharedDefaults(s) {
   return d;
 }
 async function saveShared(on, s) {
-  const r = await fetch("/settings", {method: "POST", headers: headers(true),
+  const r = await fetch("settings", {method: "POST", headers: headers(true),
                                       body: JSON.stringify({defaults: on ? sharedDefaults(s) : null})});
   if (!r.ok) {
     let msg = `HTTP ${r.status}`;

serve/web/index.html

@@ -4,9 +4,9 @@
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>Strata</title>
-<link rel="stylesheet" href="/web/tokens.css">
-<link rel="stylesheet" href="/web/components.css">
-<link rel="stylesheet" href="/web/app.css">
+<link rel="stylesheet" href="web/tokens.css">
+<link rel="stylesheet" href="web/components.css">
+<link rel="stylesheet" href="web/app.css">
 <script>
   // the theme before the first paint: the saved choice, else the system's
   try {
@@ -21,17 +21,17 @@
   <span class="st-header__brand">Strata</span>
   <nav class="st-tabs" role="tablist" aria-label="Views">
     <button class="st-tab" role="tab" id="tab-btn-chat" aria-controls="view-chat" aria-selected="true" data-tab="chat">
-      <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-chat"/></svg>Chat</button>
+      <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-chat"/></svg>Chat</button>
     <button class="st-tab" role="tab" id="tab-btn-monitor" aria-controls="view-monitor" aria-selected="false" data-tab="monitor">
-      <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-activity"/></svg>Monitor</button>
+      <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-activity"/></svg>Monitor</button>
     <button class="st-tab" role="tab" id="tab-btn-about" aria-controls="view-about" aria-selected="false" data-tab="about">
-      <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-info"/></svg>About</button>
+      <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-info"/></svg>About</button>
   </nav>
   <span class="st-header__spacer"></span>
   <span class="st-pill" id="pill" data-state="idle" role="status" aria-live="polite">
     <span class="st-pill__dot"></span><span id="pill-text">Connecting&hellip;</span></span>
   <button class="st-btn st-btn--icon" id="theme-btn" title="Light / dark" aria-label="Switch between light and dark">
-    <svg class="st-icon"><use id="theme-icon" href="/web/sprite.svg#i-moon"/></svg></button>
+    <svg class="st-icon"><use id="theme-icon" href="web/sprite.svg#i-moon"/></svg></button>
 </header>
 
 <main>
@@ -51,20 +51,20 @@
         <textarea id="input" rows="2" placeholder="Ask anything&hellip;" aria-label="Message"></textarea>
         <div class="st-composer__bar">
           <button type="button" class="st-btn st-btn--icon" id="attach-btn" title="Attach a text file (or drop it here)" aria-label="Attach a file">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-attach"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-attach"/></svg></button>
           <input type="file" id="file" multiple hidden>
           <button type="button" class="st-btn st-btn--icon" id="new-btn" title="New chat" aria-label="New chat">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-new-chat"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-new-chat"/></svg></button>
           <button type="button" class="st-btn st-btn--icon" id="export-btn" title="Save this chat as Markdown" aria-label="Save this chat">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-download"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-download"/></svg></button>
           <button type="button" class="st-btn st-btn--icon" id="sampling-btn" title="Sampling and thinking" aria-label="Sampling and thinking">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-settings"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-settings"/></svg></button>
           <span class="spacer"></span>
           <span class="composer-hint" id="composer-hint"></span>
           <button type="button" class="st-btn st-btn--secondary" id="stop-btn" hidden>
-            <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-stop"/></svg>Stop</button>
+            <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-stop"/></svg>Stop</button>
           <button type="submit" class="st-btn st-btn--primary st-btn--send" id="send-btn" aria-label="Send">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-send"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-send"/></svg></button>
         </div>
       </form>
     </div>
@@ -169,7 +169,7 @@
 <aside class="st-drawer" id="drawer" data-open="false" aria-label="Sampling and thinking" aria-hidden="true">
   <div class="st-drawer__head"><span class="card-title">Sampling</span>
     <button class="st-btn st-btn--icon" id="drawer-close" aria-label="Close">
-      <svg class="st-icon"><use href="/web/sprite.svg#i-chevron"/></svg></button></div>
+      <svg class="st-icon"><use href="web/sprite.svg#i-chevron"/></svg></button></div>
   <div class="st-drawer__body">
     <div class="st-field">
       <label class="st-label" for="s-thinking">Thinking <output id="o-thinking"></output></label>
@@ -218,6 +218,6 @@
 </aside>
 <div class="scrim" id="scrim" hidden></div>
 <div class="st-toasts" id="toasts" aria-live="polite"></div>
-<script src="/web/app.js"></script>
+<script src="web/app.js"></script>
 </body>
 </html>

serve/web/tokens.css

@@ -3,8 +3,8 @@
    Values marked (derived) are not in the Gainsty source and were added for the app. */
 
 /* Outfit (SIL Open Font License 1.1, serve/web/fonts/OFL.txt): one variable font, weights 100-900 */
-@font-face{font-family:"Outfit";src:url("/fonts/outfit-latin-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
-@font-face{font-family:"Outfit";src:url("/fonts/outfit-latin-ext-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}
+@font-face{font-family:"Outfit";src:url("../fonts/outfit-latin-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
+@font-face{font-family:"Outfit";src:url("../fonts/outfit-latin-ext-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}
 
 :root{
   /* type */

setup.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 # Strata for Linux: the first run installs everything and starts the model; later runs just start it.
-# Needs only an NVIDIA driver. Python (with venv) is installed through apt/dnf if it is missing (asks for sudo).
+# Needs only an NVIDIA driver (or, experimental: an AMD RX 7900 XT/XTX with the amdgpu driver, see docs/AMD_HIP.md).
+# Python (with venv) is installed through apt/dnf if it is missing (asks for sudo).
 cd "$(dirname "$0")" || exit 1
 # Python 3.10+ that can make a venv WITH pip: Debian/Ubuntu ship `venv` without `ensurepip` (that is the separate
 # python3-venv package), and a venv made without it has no pip

src/artifact/gguf_reader_test.cpp

@@ -0,0 +1,93 @@
+// src/artifact/gguf_reader_test.cpp - the reader refuses what it cannot arbitrate (synthetic GGUF, no model,
+// no GPU).
+//
+//   1. two tensors with distinct names: the file opens and find() sees both;
+//   2. the same name twice: refused at open, and the error names the tensor and the file - GGUF has no
+//      index to say which of the two a name means, and find() is first-match, so the alternative is a
+//      lookup that silently picks by position.
+//
+// The fixture is a minimal GGUF v3 written here (header, no metadata, two F32[8] tensors, 32-byte
+// alignment), so the test needs neither gguf-py nor a shard.  The file is closed before it is removed:
+// on Windows an open mapping keeps it.
+#include "strata/artifact/gguf_reader.hpp"
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+namespace {
+int g_fail = 0;
+void check(bool ok, const char* what) {
+    std::printf("  %-66s %s\n", what, ok ? "ok" : "FAIL");
+    if (!ok) ++g_fail;
+}
+
+template <class T> void put(std::vector<uint8_t>& b, T v) {
+    const auto n = b.size();
+    b.resize(n + sizeof(T));
+    std::memcpy(b.data() + n, &v, sizeof(T));
+}
+void put_str(std::vector<uint8_t>& b, const std::string& s) {
+    put<uint64_t>(b, s.size());
+    b.insert(b.end(), s.begin(), s.end());
+}
+
+// A GGUF v3 file of F32[8] tensors named `names`, laid out 32 bytes apart from a 32-byte-aligned data start.
+std::filesystem::path write_gguf(const std::vector<std::string>& names) {
+    std::vector<uint8_t> b;
+    put<uint32_t>(b, 0x46554747u);   // "GGUF"
+    put<uint32_t>(b, 3);
+    put<uint64_t>(b, names.size());  // n_tensors
+    put<uint64_t>(b, 0);             // n_kv
+    for (size_t i = 0; i < names.size(); ++i) {
+        put_str(b, names[i]);
+        put<uint32_t>(b, 1);         // n_dims
+        put<uint64_t>(b, 8);
+        put<uint32_t>(b, 0);         // F32
+        put<uint64_t>(b, 32 * i);    // offset from data_start
+    }
+    b.resize((b.size() + 31) / 32 * 32 + 32 * names.size(), 0);
+    const auto path = std::filesystem::temp_directory_path() / "strata_gguf_reader_test.gguf";
+    std::ofstream(path, std::ios::binary).write(reinterpret_cast<const char*>(b.data()), (std::streamsize)b.size());
+    return path;
+}
+}  // namespace
+
+int main() {
+    std::printf("gguf_reader_test\n");
+    {
+        const auto path = write_gguf({"blk.0.attn_q.weight", "blk.0.attn_k.weight"});
+        std::string err;
+        size_t n = 0;
+        bool both = false;
+        try {
+            strata::GgufFile g(path.string());
+            n = g.tensors().size();
+            both = g.find("blk.0.attn_q.weight") && g.find("blk.0.attn_k.weight");
+        } catch (const std::exception& e) {
+            err = e.what();
+        }
+        std::filesystem::remove(path);
+        check(err.empty() && n == 2 && both, "distinct names: the file opens and find() sees both tensors");
+    }
+    {
+        const auto path = write_gguf({"blk.0.attn_q.weight", "blk.0.attn_q.weight"});
+        std::string err;
+        try {
+            strata::GgufFile g(path.string());
+        } catch (const std::exception& e) {
+            err = e.what();
+        }
+        std::filesystem::remove(path);
+        check(!err.empty(), "the same name twice: refused at open");
+        check(err.find("blk.0.attn_q.weight") != std::string::npos, "  the error names the tensor");
+        check(err.find(path.filename().string()) != std::string::npos, "  the error names the file");
+        if (!err.empty()) std::printf("  (%s)\n", err.c_str());
+    }
+    std::printf(g_fail ? "gguf_reader_test: %d FAILED\n" : "gguf_reader_test: all passed\n", g_fail);
+    return g_fail ? 1 : 0;
+}

src/core/device.cu

@@ -30,7 +30,11 @@ DeviceInfo device_info(int ordinal) {
     int count = 0;
     check(cudaGetDeviceCount(&count), "cudaGetDeviceCount");
     if (count == 0) {
-        throw CudaError("no CUDA device is present; Strata targets sm_120 (RTX 5000 series)", -1);
+#if defined(STRATA_USE_HIP)
+        throw CudaError("no HIP device is present; this backend targets gfx1100 wave32", -1);
+#else
+        throw CudaError("no CUDA device is present; Strata needs an NVIDIA GPU (RTX 20 series or newer)", -1);
+#endif
     }
     if (ordinal < 0 || ordinal >= count) {
         throw CudaError("device ordinal " + std::to_string(ordinal) + " is out of range (have " +
@@ -56,15 +60,23 @@ DeviceInfo device_info(int ordinal) {
     check(cudaDriverGetVersion(&d.driver_version), "cudaDriverGetVersion");
     check(cudaRuntimeGetVersion(&d.runtime_version), "cudaRuntimeGetVersion");
 
-    // The engine is written against sm_120.  Compiling for it is enforced by CMake; RUNNING on something else
-    // is caught here, because a binary can be carried to a machine with an older card and would otherwise
-    // silently take whatever path the driver chose.
-    if (d.cc_major != 12) {
+    // The engine supports compute capability 7.5 and newer (Turing: the QSA scorer's tf32 mma has a portable
+    // fp32-FMA fallback below sm_80, the tensor-core prompt kernels refuse and fall back).  Compiling for a
+    // supported arch is enforced by CMake; RUNNING on an older card is caught here, because a binary can be carried
+    // to a machine with an older card and would otherwise silently take whatever path the driver chose.  The HIP
+    // backend is validated on gfx1100 (wave32) only.
+#if defined(STRATA_USE_HIP)
+    if (std::strncmp(p.gcnArchName, "gfx1100", 7) != 0 || p.warpSize != 32) {
+        throw CudaError("HIP backend requires validated gfx1100 wave32 hardware", -1);
+    }
+#else
+    if (d.cc_major * 10 + d.cc_minor < 75) {
         throw CudaError("device " + d.name + " reports compute capability " + std::to_string(d.cc_major) +
                             "." + std::to_string(d.cc_minor) +
-                            "; Strata targets sm_120 (RTX 5000 series / Blackwell) only",
+                            "; Strata needs compute capability 7.5 or newer (RTX 20 / 30 / 40 / 50 series)",
                         -1);
     }
+#endif
     return d;
 }
 

src/core/device_main.cpp

@@ -33,7 +33,11 @@ int main(int argc, char** argv) {
     try {
         const strata::core::DeviceInfo d = strata::core::device_info(0);
         std::printf("device %d: %s\n", d.ordinal, d.name.c_str());
+#if defined(STRATA_USE_HIP)
+        std::printf("  HIP target          gfx1100 wave32\n");
+#else
         std::printf("  compute capability  %d.%d   (sm_%d%d)\n", d.cc_major, d.cc_minor, d.cc_major, d.cc_minor);
+#endif
         std::printf("  multiprocessors     %d\n", d.multi_processor_count);
         std::printf("  VRAM total / free   %s / %s\n", human(d.total_bytes).c_str(), human(d.free_bytes).c_str());
         std::printf("  driver / runtime    %d / %d\n", d.driver_version, d.runtime_version);

src/core/expert_cache.cpp

@@ -69,6 +69,22 @@ bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_ex
 
 ExpertCache::~ExpertCache() { close(); }
 
+#if defined(STRATA_USE_HIP)
+bool ExpertCache::ensure_blocking_staging(std::size_t bytes, std::string& err) {
+    if (bytes <= blocking_staging_bytes_) return true;
+    void* next = nullptr;
+    const cudaError_t status = cudaHostAlloc(&next, bytes, cudaHostAllocDefault);
+    if (status != cudaSuccess) {
+        err = std::string("ExpertCache: HIP blocking staging allocation: ") + cudaGetErrorString(status);
+        return false;
+    }
+    if (blocking_staging_) (void) cudaFreeHost(blocking_staging_);
+    blocking_staging_ = static_cast<uint8_t*>(next);
+    blocking_staging_bytes_ = bytes;
+    return true;
+}
+#endif
+
 bool ExpertCache::open(int64_t n_slots, int64_t n_layers, int64_t n_expert, int64_t blob_bytes,
                        std::string& err) {
     close();
@@ -125,6 +141,12 @@ bool ExpertCache::open(int64_t n_slots, int64_t n_layers, int64_t n_expert, int6
     n_layers_ = n_layers;
     n_expert_ = n_expert;
     blob_ = blob_bytes;
+#if defined(STRATA_USE_HIP)
+    if (!ensure_blocking_staging((std::size_t) blob_, err)) {
+        close();
+        return false;
+    }
+#endif
     next_free_ = 0;
     fills_ = 0;
     admitted_ = 0;
@@ -154,11 +176,22 @@ bool ExpertCache::open_sized(const std::vector<int64_t>& slot_bytes, int64_t n_l
     slots_ = (int64_t) slot_bytes.size();
     blob_ = mx;
     off_ = std::move(off);
+#if defined(STRATA_USE_HIP)
+    if (!ensure_blocking_staging((std::size_t) blob_, err)) {
+        close();
+        return false;
+    }
+#endif
     layer_next_.assign((size_t) (n_layers > 0 ? n_layers : 0), 0);
     return true;
 }
 
 void ExpertCache::close() {
+#if defined(STRATA_USE_HIP)
+    if (blocking_staging_) (void) cudaFreeHost(blocking_staging_);
+    blocking_staging_ = nullptr;
+    blocking_staging_bytes_ = 0;
+#endif
     off_.clear();
     if (base_ != nullptr) {
         cudaFree(base_);
@@ -256,7 +289,18 @@ bool ExpertCache::fill_slot_blocking(int32_t slot, const uint8_t* host_blob, std
         err = "ExpertCache::fill_slot_blocking: the host blob is null";
         return false;
     }
+#if defined(STRATA_USE_HIP)
+    // Bound HIP's pageable-source staging to one expert instead of repeatedly
+    // registering regions of the mmap. The blocking copy completes before reuse.
+    if (!blocking_staging_ || n > blocking_staging_bytes_) {
+        err = "ExpertCache::fill_slot_blocking: HIP staging buffer is too small";
+        return false;
+    }
+    std::memcpy(blocking_staging_, host_blob, n);
+    const cudaError_t e = cudaMemcpy(dst, blocking_staging_, n, cudaMemcpyHostToDevice);
+#else
     const cudaError_t e = cudaMemcpy(dst, host_blob, n, cudaMemcpyHostToDevice);
+#endif
     if (e != cudaSuccess) {
         err = std::string("ExpertCache::fill_slot_blocking: ") + cudaGetErrorString(e);
         return false;
@@ -265,6 +309,32 @@ bool ExpertCache::fill_slot_blocking(int32_t slot, const uint8_t* host_blob, std
     return true;
 }
 
+bool ExpertCache::fill_slot_queued(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes) {
+    const size_t n = (size_t) (bytes > 0 && bytes <= blob_ ? bytes : blob_);
+    uint8_t* dst = device_slot(slot);
+    if (dst == nullptr || host_blob == nullptr) {
+        err = dst == nullptr ? "ExpertCache::fill_slot_queued: slot outside the arena"
+                             : "ExpertCache::fill_slot_queued: the host blob is null";
+        return false;
+    }
+    const cudaError_t e = cudaMemcpyAsync(dst, host_blob, n, cudaMemcpyHostToDevice, (cudaStream_t) 0);
+    if (e != cudaSuccess) {
+        err = std::string("ExpertCache::fill_slot_queued: ") + cudaGetErrorString(e);
+        return false;
+    }
+    ++fills_;
+    return true;
+}
+
+bool ExpertCache::sync_queued(std::string& err) {
+    const cudaError_t e = cudaStreamSynchronize((cudaStream_t) 0);
+    if (e != cudaSuccess) {
+        err = std::string("ExpertCache::sync_queued: ") + cudaGetErrorString(e);
+        return false;
+    }
+    return true;
+}
+
 bool ExpertCache::verify_slot(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes) {
     const int64_t nb = bytes > 0 && bytes <= blob_ ? bytes : blob_;
     const uint8_t* src = device_slot(slot);

src/core/expert_source.cpp

@@ -17,11 +17,18 @@
 #include <cstdlib>
 #include <cstring>
 #include <fstream>
+#include <filesystem>
+#include <sstream>
+#include <limits>
 #include <thread>
+#include <utility>
 #include <vector>
 
 #if defined(_WIN32)
 #define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX   // std::numeric_limits<T>::max() below
+#endif
 #include <windows.h>
 #else
 #include <fcntl.h>
@@ -32,6 +39,213 @@
 
 namespace strata::core {
 
+namespace detail {
+
+bool cgroup_available_bytes(uint64_t limit, const CgroupMemoryStat& stat, uint64_t& bytes) {
+    bytes = 0;
+    if (!stat.valid) return false;
+
+    // memory.stat's inactive_file can race memory.current, so bound it to charged usage first.
+    uint64_t reclaimable = std::min(stat.inactive_file, stat.current);
+    reclaimable = stat.file_dirty >= reclaimable ? 0 : reclaimable - stat.file_dirty;
+    reclaimable = stat.file_writeback >= reclaimable ? 0 : reclaimable - stat.file_writeback;
+
+    // Reclaiming clean file pages reduces usage; saturating subtraction also handles a transient over-limit read.
+    const uint64_t usage_after_reclaim = stat.current - reclaimable;
+    bytes = usage_after_reclaim < limit ? limit - usage_after_reclaim : 0;
+    return true;
+}
+
+bool make_cache_complement_plan(
+    int64_t n_layers, int64_t n_expert, const std::vector<uint64_t>& layer_blob_bytes,
+    const std::vector<std::pair<int32_t, int32_t>>& primary_gpu_pairs,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs,
+    std::vector<uint64_t>& offsets, uint64_t& bytes, std::string& err) {
+    offsets.clear();
+    bytes = 0;
+    err.clear();
+    if (n_layers <= 0 || n_expert <= 0 || layer_blob_bytes.size() != (size_t) n_layers) {
+        err = "FileExpertSource: invalid geometry for the cache complement plan";
+        return false;
+    }
+    if ((uint64_t) n_layers > (uint64_t) std::numeric_limits<size_t>::max() / (uint64_t) n_expert) {
+        err = "FileExpertSource: cache complement index table is too large";
+        return false;
+    }
+    const size_t count = (size_t) n_layers * (size_t) n_expert;
+    std::vector<uint8_t> omitted(count, 0);
+    auto mark_pairs = [&](const std::vector<std::pair<int32_t, int32_t>>& pairs, uint8_t bit,
+                          const char* label) -> bool {
+        for (const auto& pair : pairs) {
+            if (pair.first < 0 || pair.second < 0 || pair.first >= n_layers || pair.second >= n_expert) {
+                err = std::string("FileExpertSource: ") + label + " pair is outside the expert geometry";
+                return false;
+            }
+            const size_t index = (size_t) pair.first * (size_t) n_expert + (size_t) pair.second;
+            if ((omitted[index] & bit) != 0) {
+                err = std::string("FileExpertSource: duplicate ") + label + " pair in the cache complement plan";
+                return false;
+            }
+            if (bit == 2 && (omitted[index] & 1) != 0) {
+                err = "FileExpertSource: the primary and additional GPU expert tiers overlap";
+                return false;
+            }
+            omitted[index] |= bit;
+        }
+        return true;
+    };
+    if (!mark_pairs(primary_gpu_pairs, 1, "primary GPU") ||
+        !mark_pairs(additional_gpu_pairs, 2, "additional GPU")) return false;
+    for (uint64_t blob_bytes : layer_blob_bytes) {
+        if (blob_bytes == 0) {
+            err = "FileExpertSource: cache complement layer has zero-sized expert blobs";
+            return false;
+        }
+    }
+
+    offsets.assign(count, kNoCacheComplement);
+    for (int64_t layer = 0; layer < n_layers; ++layer) {
+        const uint64_t blob_bytes = layer_blob_bytes[(size_t) layer];
+        for (int64_t expert = 0; expert < n_expert; ++expert) {
+            const size_t index = (size_t) layer * (size_t) n_expert + (size_t) expert;
+            if (omitted[index] != 0) continue;
+            if (bytes > std::numeric_limits<uint64_t>::max() - blob_bytes) {
+                offsets.clear();
+                bytes = 0;
+                err = "FileExpertSource: cache complement size overflows";
+                return false;
+            }
+            offsets[index] = bytes;
+            bytes += blob_bytes;
+        }
+    }
+    if (bytes > (uint64_t) std::numeric_limits<size_t>::max()) {
+        offsets.clear();
+        bytes = 0;
+        err = "FileExpertSource: cache complement exceeds the host address space";
+        return false;
+    }
+    return true;
+}
+
+const uint8_t* cache_complement_blob_or_fallback(
+    size_t index, const std::vector<uint64_t>& offsets, const uint8_t* complement_host,
+    const uint8_t* mapped_fallback) {
+    if (complement_host != nullptr && index < offsets.size() && offsets[index] != kNoCacheComplement)
+        return complement_host + (size_t) offsets[index];
+    return mapped_fallback;
+}
+
+}  // namespace detail
+
+namespace {
+constexpr uint64_t kPinnedMemoryHeadroom = 8ull << 30;
+
+#if defined(__linux__)
+bool read_cgroup_memory_stat(const std::filesystem::path& path, uint64_t current,
+                             detail::CgroupMemoryStat& stat) {
+    std::ifstream input(path / "memory.stat");
+    if (!input) return false;
+
+    bool inactive_file = false, file_dirty = false, file_writeback = false;
+    std::string line;
+    while (std::getline(input, line)) {
+        std::istringstream fields(line);
+        std::string key;
+        uint64_t value = 0;
+        if (!(fields >> key >> value)) return false;
+        fields >> std::ws;
+        if (!fields.eof()) return false;
+
+        if (key == "inactive_file") {
+            if (inactive_file) return false;
+            inactive_file = true;
+            stat.inactive_file = value;
+        } else if (key == "file_dirty") {
+            if (file_dirty) return false;
+            file_dirty = true;
+            stat.file_dirty = value;
+        } else if (key == "file_writeback") {
+            if (file_writeback) return false;
+            file_writeback = true;
+            stat.file_writeback = value;
+        }
+    }
+    if (!input.eof() || !inactive_file || !file_dirty || !file_writeback) return false;
+    stat.current = current;
+    stat.valid = true;
+    return true;
+}
+#endif
+
+bool available_memory_bytes(uint64_t& bytes) {
+#if defined(_WIN32)
+    MEMORYSTATUSEX status{};
+    status.dwLength = sizeof(status);
+    if (!GlobalMemoryStatusEx(&status)) return false;
+    bytes = (uint64_t) status.ullAvailPhys;
+    return bytes > 0;
+#elif defined(__linux__)
+    // MemAvailable includes reclaimable page cache, unlike _SC_AVPHYS_PAGES.
+    std::ifstream info("/proc/meminfo");
+    std::string line;
+    bytes = 0;
+    while (std::getline(info, line)) {
+        std::istringstream fields(line);
+        std::string key, unit;
+        uint64_t value = 0;
+        if (fields >> key >> value >> unit && key == "MemAvailable:" && unit == "kB" &&
+            value <= std::numeric_limits<uint64_t>::max() / 1024) bytes = value * 1024;
+    }
+    if (bytes == 0) return false;
+    // Account for the tightest cgroup-v2 ancestor limit when its normal mount is visible.
+    // This is a point-in-time guard, not a reservation against concurrent allocations.
+    std::ifstream groups("/proc/self/cgroup");
+    if (!groups) return false;
+    bool resolved_v2 = false;
+    while (std::getline(groups, line)) {
+        if (line.rfind("0::/", 0) != 0) continue;
+        const std::filesystem::path root("/sys/fs/cgroup");
+        auto path = (root / line.substr(4)).lexically_normal();
+        if (path.string().rfind(root.string(), 0) != 0 || !std::filesystem::is_directory(path)) return false;
+        resolved_v2 = true;
+        while (path.string().rfind(root.string(), 0) == 0) {
+            std::ifstream limit_file(path / "memory.max"), current_file(path / "memory.current");
+            std::string limit;
+            uint64_t current = 0;
+            const bool readable = bool(limit_file >> limit) && bool(current_file >> current);
+            // The host's root cgroup has no memory.max; ordinary child groups must expose their limits.
+            if (!readable && !(path == root && !std::filesystem::exists(path / "memory.max") &&
+                               std::filesystem::exists(path / "cgroup.controllers"))) return false;
+            if (readable && limit != "max") {
+                try {
+                    size_t consumed = 0;
+                    const uint64_t cap = std::stoull(limit, &consumed);
+                    if (consumed != limit.size()) return false;
+                    detail::CgroupMemoryStat stat;
+                    if (!read_cgroup_memory_stat(path, current, stat)) return false;
+                    uint64_t cgroup_available = 0;
+                    if (!detail::cgroup_available_bytes(cap, stat, cgroup_available)) return false;
+                    bytes = std::min(bytes, cgroup_available);
+                } catch (...) { return false; }
+            }
+            if (path == root) break;
+            path = path.parent_path();
+        }
+    }
+    return resolved_v2;
+#else
+    const long pages = sysconf(_SC_AVPHYS_PAGES);
+    const long page_bytes = sysconf(_SC_PAGESIZE);
+    if (pages <= 0 || page_bytes <= 0 ||
+        (uint64_t) pages > std::numeric_limits<uint64_t>::max() / (uint64_t) page_bytes) return false;
+    bytes = (uint64_t) pages * (uint64_t) page_bytes;
+    return bytes > 0;
+#endif
+}
+
+}  // namespace
+
 // ================================ THE FILE-BACKED SOURCE ================================
 
 FileExpertSource::~FileExpertSource() { close(); }
@@ -39,9 +253,74 @@ FileExpertSource::~FileExpertSource() { close(); }
 bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err) {
     close();
     if (n_layers <= 0 || n_expert <= 0) { err = "FileExpertSource: the geometry is empty"; return false; }
-    n_expert_ = n_expert;
-    blobs_ = n_layers * n_expert;
-    const uint64_t want = (uint64_t) blobs_ * (uint64_t) strata::kernels::cpu::BLOB;
+    const auto& layout = strata::kernels::cpu::expert_layout();
+    if (layout.n_layers != n_layers || layout.n_expert != n_expert) {
+        err = "FileExpertSource: the requested geometry does not match the loaded expert layout";
+        return false;
+    }
+    if ((uint64_t) n_layers > (uint64_t) std::numeric_limits<int64_t>::max() / (uint64_t) n_expert) {
+        err = "FileExpertSource: the expert count overflows";
+        return false;
+    }
+    const uint64_t blob_count = (uint64_t) n_layers * (uint64_t) n_expert;
+    if (blob_count > (uint64_t) std::numeric_limits<int64_t>::max() ||
+        (uint64_t) n_layers > (uint64_t) std::numeric_limits<size_t>::max()) {
+        err = "FileExpertSource: the expert count overflows";
+        return false;
+    }
+
+    std::vector<uint64_t> layer_offsets((size_t) n_layers), layer_blob_bytes((size_t) n_layers);
+    const uint64_t want = layout.total;
+    if (want == 0 || want > (uint64_t) std::numeric_limits<size_t>::max()) {
+        err = "FileExpertSource: the loaded expert layout has an invalid size";
+        return false;
+    }
+    if (!layout.native) {
+        if (blob_count > std::numeric_limits<uint64_t>::max() / (uint64_t) strata::kernels::cpu::BLOB) {
+            err = "FileExpertSource: the canonical expert size overflows";
+            return false;
+        }
+        const uint64_t canonical_size = blob_count * (uint64_t) strata::kernels::cpu::BLOB;
+        if (want != canonical_size) {
+            err = "FileExpertSource: the canonical expert layout has an inconsistent size";
+            return false;
+        }
+        const uint64_t bytes = (uint64_t) strata::kernels::cpu::BLOB;
+        const uint64_t layer_bytes = (uint64_t) n_expert * bytes;
+        for (int64_t layer = 0; layer < n_layers; ++layer) {
+            layer_offsets[(size_t) layer] = (uint64_t) layer * layer_bytes;
+            layer_blob_bytes[(size_t) layer] = bytes;
+        }
+    } else {
+        if (layout.offset.size() != (size_t) n_layers || layout.bytes.size() != (size_t) n_layers ||
+            layout.fmt.size() != (size_t) n_layers) {
+            err = "FileExpertSource: the native expert layout is incomplete";
+            return false;
+        }
+        uint64_t at = 0;
+        for (int64_t layer = 0; layer < n_layers; ++layer) {
+            const size_t i = (size_t) layer;
+            const uint64_t bytes = (uint64_t) layout.fmt[i].bytes;
+            if (layout.offset[i] != at || bytes == 0 || layout.bytes[i] != bytes ||
+                bytes > std::numeric_limits<uint64_t>::max() / (uint64_t) n_expert) {
+                err = "FileExpertSource: the native expert layout is invalid at layer " + std::to_string(layer);
+                return false;
+            }
+            const uint64_t layer_bytes = bytes * (uint64_t) n_expert;
+            if (at > want || layer_bytes > want - at) {
+                err = "FileExpertSource: the native expert layout exceeds its declared size at layer " +
+                      std::to_string(layer);
+                return false;
+            }
+            layer_offsets[i] = layout.offset[i];
+            layer_blob_bytes[i] = bytes;
+            at += layer_bytes;
+        }
+        if (at != want) {
+            err = "FileExpertSource: the native expert layout has an inconsistent size";
+            return false;
+        }
+    }
     const std::string path = pack_dir + "/experts.bin";
 
 #if defined(_WIN32)
@@ -76,10 +355,9 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     if ((uint64_t) sz.QuadPart != want) {
         char buf[400];
         std::snprintf(buf, sizeof buf,
-                      "FileExpertSource: %s is %llu B but %lld layers x %lld experts x %d B is %llu B - this "
-                      "is not the pack this geometry came from",
-                      path.c_str(), (unsigned long long) sz.QuadPart, (long long) n_layers,
-                      (long long) n_expert, (int) strata::kernels::cpu::BLOB, (unsigned long long) want);
+                      "FileExpertSource: %s is %llu B but the loaded expert layout requires %llu B - this is not "
+                      "the pack this geometry came from",
+                      path.c_str(), (unsigned long long) sz.QuadPart, (unsigned long long) want);
         CloseHandle(f);
         err = buf;
         return false;
@@ -105,13 +383,13 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     if (fd < 0) { err = "FileExpertSource: cannot open " + path; return false; }
     struct stat st{};
     if (fstat(fd, &st) != 0) { ::close(fd); err = "FileExpertSource: cannot stat " + path; return false; }
-    if ((uint64_t) st.st_size != want) {
+    if (st.st_size < 0 || (uint64_t) st.st_size != want) {
         char buf[400];
         std::snprintf(buf, sizeof buf,
-                      "FileExpertSource: %s is %llu B but %lld layers x %lld experts x %d B is %llu B - this "
-                      "is not the pack this geometry came from",
-                      path.c_str(), (unsigned long long) st.st_size, (long long) n_layers, (long long) n_expert,
-                      (int) strata::kernels::cpu::BLOB, (unsigned long long) want);
+                      "FileExpertSource: %s is %llu B but the loaded expert layout requires %llu B - this is not "
+                      "the pack this geometry came from",
+                      path.c_str(), (unsigned long long) (st.st_size < 0 ? 0 : st.st_size),
+                      (unsigned long long) want);
         ::close(fd);
         err = buf;
         return false;
@@ -121,10 +399,27 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     fd_ = fd;
     base_ = (const uint8_t*) view;
 #endif
+    blobs_ = (int64_t) blob_count;
+    n_layers_ = n_layers;
+    n_expert_ = n_expert;
+    mapped_bytes_ = want;
+    layer_offsets_ = std::move(layer_offsets);
+    layer_blob_bytes_ = std::move(layer_blob_bytes);
     return true;
 }
 
 void FileExpertSource::close() {
+    if (complement_arena_ != nullptr) {
+        if (complement_pinned_) (void) cudaFreeHost(complement_arena_);
+        else std::free(complement_arena_);
+    }
+    complement_arena_ = nullptr;
+    complement_host_ = nullptr;
+    complement_device_ = nullptr;
+    complement_bytes_ = 0;
+    complement_offsets_.clear();
+    complement_pinned_ = false;
+    complement_ready_ = false;
 #if defined(_WIN32)
     if (base_ != nullptr) UnmapViewOfFile((LPCVOID) base_);
     if (mapping_ != nullptr) CloseHandle((HANDLE) mapping_);
@@ -132,27 +427,225 @@ void FileExpertSource::close() {
     mapping_ = nullptr;
     file_ = nullptr;
 #else
-    if (base_ != nullptr) munmap((void*) base_, (size_t) blobs_ * (size_t) strata::kernels::cpu::BLOB);
+    if (base_ != nullptr) munmap((void*) base_, (size_t) mapped_bytes_);
     if (fd_ >= 0) ::close(fd_);
     fd_ = -1;
 #endif
     base_ = nullptr;
     blobs_ = 0;
+    n_layers_ = 0;
+    n_expert_ = 0;
+    mapped_bytes_ = 0;
+    layer_offsets_.clear();
+    layer_blob_bytes_.clear();
     reads_ = 0;
 }
 
+const uint8_t* FileExpertSource::mapped_blob(int64_t layer, int64_t expert) const {
+    if (base_ == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return nullptr;
+    const size_t i = (size_t) layer;
+    if (i >= layer_offsets_.size() || i >= layer_blob_bytes_.size()) return nullptr;
+    const uint64_t blob_bytes = layer_blob_bytes_[i];
+    if (blob_bytes == 0 || (uint64_t) expert > std::numeric_limits<uint64_t>::max() / blob_bytes) return nullptr;
+    const uint64_t expert_offset = (uint64_t) expert * blob_bytes;
+    const uint64_t layer_offset = layer_offsets_[i];
+    if (layer_offset > mapped_bytes_ || expert_offset > mapped_bytes_ - layer_offset) return nullptr;
+    const uint64_t offset = layer_offset + expert_offset;
+    if (blob_bytes > mapped_bytes_ - offset) return nullptr;
+    return base_ + (size_t) offset;
+}
+
+bool FileExpertSource::pin_cache_complement(
+    const ExpertCache& cache, std::string& err, bool pin,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs) {
+    err.clear();
+    if (base_ == nullptr) { err = "FileExpertSource: open the mapped experts before pinning a complement"; return false; }
+    if (complement_ready_) { err = "FileExpertSource: the cache complement is already pinned"; return false; }
+    if (!cache.valid()) { err = "FileExpertSource: the GPU expert cache is not open"; return false; }
+    if (cache.fills() != cache.resident()) {
+        err = "FileExpertSource: the GPU expert cache is not fully filled";
+        return false;
+    }
+    const cudaError_t sync = cudaDeviceSynchronize();
+    if (sync != cudaSuccess) {
+        err = std::string("FileExpertSource: GPU expert cache is not ready: ") + cudaGetErrorString(sync);
+        (void) cudaGetLastError();
+        return false;
+    }
+
+    std::vector<std::pair<int32_t, int32_t>> primary_gpu_pairs;
+    primary_gpu_pairs.reserve((size_t) cache.resident());
+    for (int64_t layer = 0; layer < n_layers_; ++layer) {
+        for (int64_t expert = 0; expert < n_expert_; ++expert) {
+            if (cache.slot_of(layer, expert) != kNotResident)
+                primary_gpu_pairs.emplace_back((int32_t) layer, (int32_t) expert);
+        }
+    }
+    std::vector<uint64_t> offsets;
+    uint64_t bytes = 0;
+    if (!detail::make_cache_complement_plan(n_layers_, n_expert_, layer_blob_bytes_, primary_gpu_pairs,
+                                            additional_gpu_pairs, offsets, bytes, err)) return false;
+
+    if (bytes > 0) {
+        uint64_t physical = 0;
+        if (!available_memory_bytes(physical)) {
+            err = "FileExpertSource: cannot determine available RAM for the resident-memory safety check";
+            return false;
+        }
+        if (physical <= kPinnedMemoryHeadroom || bytes > physical - kPinnedMemoryHeadroom) {
+            char message[256];
+            std::snprintf(message, sizeof message,
+                          "FileExpertSource: resident complement %.2f GiB exceeds available RAM minus the 8 GiB safety headroom",
+                          (double) bytes / 1073741824.0);
+            err = message;
+            return false;
+        }
+    }
+
+    void* arena = nullptr;
+    const uint8_t* host = nullptr;
+    const uint8_t* device = nullptr;
+    if (bytes > 0) {
+        std::fprintf(stderr, "FileExpertSource: allocating %.2f GiB %s cache complement\n",
+                     (double) bytes / 1073741824.0, pin ? "mapped pinned" : "pageable resident");
+        std::fflush(stderr);
+        if (pin) {
+            const cudaError_t allocated = cudaHostAlloc(&arena, (size_t) bytes,
+                                                         cudaHostAllocMapped | cudaHostAllocPortable);
+            if (allocated != cudaSuccess) {
+                err = std::string("FileExpertSource: mapped pinned complement allocation failed: ") +
+                      cudaGetErrorString(allocated);
+                std::fprintf(stderr, "%s\n", err.c_str());
+                std::fflush(stderr);
+                (void) cudaGetLastError();
+                return false;
+            }
+            host = (const uint8_t*) arena;
+            void* alias = nullptr;
+            const cudaError_t aliased = cudaHostGetDevicePointer(&alias, arena, 0);
+            if (aliased != cudaSuccess || alias == nullptr) {
+                err = std::string("FileExpertSource: mapped pinned complement device alias failed: ") +
+                      cudaGetErrorString(aliased);
+                std::fprintf(stderr, "%s\n", err.c_str());
+                std::fflush(stderr);
+                (void) cudaGetLastError();
+                (void) cudaFreeHost(arena);
+                return false;
+            }
+            device = (const uint8_t*) alias;
+        } else {
+            arena = std::malloc((size_t) bytes);
+            if (arena == nullptr) {
+                err = "FileExpertSource: pageable resident complement allocation failed";
+                return false;
+            }
+            host = (const uint8_t*) arena;
+        }
+    }
+
+    uint64_t copied = 0;
+    for (int64_t layer = 0; layer < n_layers_; ++layer) {
+        const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer];
+        for (int64_t expert = 0; expert < n_expert_; ++expert) {
+            const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+            const uint64_t offset = offsets[index];
+            if (offset == kNoComplement) continue;
+            const uint8_t* source = mapped_blob(layer, expert);
+            if (source == nullptr || offset > bytes || blob_bytes > bytes - offset) {
+                err = "FileExpertSource: invalid blob bounds while building the cache complement";
+                if (arena != nullptr) {
+                    if (pin) (void) cudaFreeHost(arena);
+                    else std::free(arena);
+                }
+                return false;
+            }
+            std::memcpy((uint8_t*) host + (size_t) offset, source, (size_t) blob_bytes);
+            copied += blob_bytes;
+        }
+#if !defined(_WIN32)
+        const uint64_t layer_offset = layer_offsets_[(size_t) layer];
+        const uint64_t layer_bytes = blob_bytes * (uint64_t) n_expert_;
+        const uint64_t layer_end = layer_offset + layer_bytes;
+        const long page_bytes = sysconf(_SC_PAGESIZE);
+        if (page_bytes <= 0) {
+            err = "FileExpertSource: cannot determine page size for mapped-page release";
+            if (arena != nullptr) {
+                if (pin) (void) cudaFreeHost(arena);
+                else std::free(arena);
+            }
+            return false;
+        }
+        const uint64_t page = (uint64_t) page_bytes;
+        const uint64_t advice_start = layer_offset - layer_offset % page;
+        const uint64_t end_remainder = layer_end % page;
+        const uint64_t extra = end_remainder == 0 ? 0 : page - end_remainder;
+        const uint64_t advice_end = extra > mapped_bytes_ - layer_end ? mapped_bytes_ : layer_end + extra;
+        if (advice_end > advice_start &&
+            madvise((void*) (base_ + (size_t) advice_start), (size_t) (advice_end - advice_start), MADV_DONTNEED) != 0) {
+            err = "FileExpertSource: madvise could not release mapped expert layer " + std::to_string(layer);
+            if (arena != nullptr) {
+                if (pin) (void) cudaFreeHost(arena);
+                else std::free(arena);
+            }
+            return false;
+        }
+        const int advice = posix_fadvise(fd_, (off_t) layer_offset, (off_t) layer_bytes, POSIX_FADV_DONTNEED);
+        if (advice != 0) {
+            err = "FileExpertSource: posix_fadvise could not release expert layer " + std::to_string(layer);
+            if (arena != nullptr) {
+                if (pin) (void) cudaFreeHost(arena);
+                else std::free(arena);
+            }
+            return false;
+        }
+#endif
+        if ((layer + 1) % 8 == 0 || layer + 1 == n_layers_) {
+            std::fprintf(stderr, "FileExpertSource: copied cache complement through layer %lld/%lld (%.2f GiB)\n",
+                         (long long) (layer + 1), (long long) n_layers_, (double) copied / 1073741824.0);
+            std::fflush(stderr);
+        }
+    }
+
+    complement_arena_ = arena;
+    complement_host_ = host;
+    complement_device_ = device;
+    complement_bytes_ = bytes;
+    complement_offsets_ = std::move(offsets);
+    complement_pinned_ = pin && bytes > 0;
+    complement_ready_ = true;
+    std::fprintf(stderr, "FileExpertSource: %s cache complement ready: resident %.2f GiB, pinned %.2f GiB\n",
+                 complement_pinned_ ? "mapped pinned" : "pageable resident",
+                 (double) resident_bytes() / 1073741824.0, (double) pinned_bytes() / 1073741824.0);
+    if (!additional_gpu_pairs.empty()) {
+        std::fprintf(stderr, "FileExpertSource: %zu verified additional-GPU experts remain on the mmap fallback\n",
+                     additional_gpu_pairs.size());
+    }
+    std::fflush(stderr);
+    return true;
+}
+
 const uint8_t* FileExpertSource::blob(int64_t layer, int64_t expert) {
-    if (base_ == nullptr) return nullptr;
-    if (layer < 0 || expert < 0) return nullptr;
-    // **`expert >= n_expert_` IS CHECKED SEPARATELY, AND THE FLAT INDEX ALONE DOES NOT CATCH IT.**  The blob
-    // index is `layer * n_expert + expert`, so `blob(0, 512)` has flat index 512 - which is in range, and is
-    // `blob(1, 0)`.  A router id one past the end of a layer would then read the NEXT LAYER's first expert:
-    // finite, correctly sized, and wrong.  Layer and expert are separate axes and are validated as such.
-    if (expert >= n_expert_) return nullptr;
-    const int64_t i = layer * n_expert_ + expert;
-    if (i >= blobs_) return nullptr;
-    ++reads_;
-    return base_ + (size_t) i * strata::kernels::cpu::BLOB;
+    if (base_ == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return nullptr;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    const uint8_t* mapped_fallback = mapped_blob(layer, expert);
+    const uint8_t* result = complement_ready_
+        ? detail::cache_complement_blob_or_fallback(index, complement_offsets_, complement_host_, mapped_fallback)
+        : mapped_fallback;
+    if (result != nullptr) ++reads_;
+    return result;
+}
+
+bool FileExpertSource::pinned(int64_t layer, int64_t expert) const {
+    if (!complement_ready_ || !complement_pinned_ || complement_host_ == nullptr || layer < 0 || expert < 0 ||
+        layer >= n_layers_ || expert >= n_expert_) return false;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    return index < complement_offsets_.size() && complement_offsets_[index] != kNoComplement;
+}
+
+const uint8_t* FileExpertSource::device_alias(int64_t layer, int64_t expert) const {
+    if (!pinned(layer, expert) || complement_device_ == nullptr) return nullptr;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    return complement_device_ + (size_t) complement_offsets_[index];
 }
 
 // ================================ THE ADAPTER ================================
@@ -653,6 +1146,8 @@ LoadStats load_experts_gguf(const std::string& gguf, uint8_t* dst, const strata:
     for (auto& t : pool) t.join();
     if (bad) {
         st.seconds = -1.0;
+        st.ok = false;
+        st.error = "short read or unreadable shard while reading the experts from the GGUF";
         return st;
     }
     st.bytes = lay.total;
@@ -712,6 +1207,11 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
     }
     const LoadStats st = from_gguf ? load_experts_gguf(gguf_, a->data(), lay, threads)
                                    : load_experts_ranges(path, a->data(), loff, lbytes, threads, /*chunk=*/8u << 20);
+    if (!st.ok) {
+        delete a;
+        err = "ArenaExpertSource: the expert load was refused: " + (st.error.empty() ? std::string("unknown") : st.error);
+        return false;
+    }
     if (st.bytes != want) {
         delete a;
         err = "ArenaExpertSource: the load read " + std::to_string(st.bytes) + " B of " + std::to_string(want);
@@ -740,6 +1240,9 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
     reads_ = 0;
     note_ = a->note;
     gib_per_s_ = st.gib_per_second();
+    load_seconds_ = st.seconds;
+    load_read_s_ = st.read_seconds;
+    load_copy_s_ = st.copy_seconds;
     return true;
 }
 

src/core/layer.cpp

@@ -172,11 +172,13 @@ if (!w.wants_q8k()) {        s_gemv_q8_0_split(x80, p.codes, p.scales, p.offset,
 // namespace
 void layer_set_native_bf16(bool enabled) { native_bf16_projections = enabled; }
 void layer_set_native_flash_attn_short(bool enabled) { native_flash_attn_short = enabled; }
-namespace { bool g_kv_int8 = false, g_kv_q4 = false; }
+namespace { bool g_kv_int8 = false, g_kv_q4 = false, g_kv_hybrid = false; }
 void qsa_set_kv_int8(bool enabled) { g_kv_int8 = enabled; }
 bool qsa_kv_int8() { return g_kv_int8; }
 void qsa_set_kv_q4(bool enabled) { g_kv_q4 = enabled; }
 bool qsa_kv_q4() { return g_kv_q4; }
+void qsa_set_kv_hybrid(bool enabled) { g_kv_hybrid = enabled; }
+bool qsa_kv_hybrid() { return g_kv_hybrid; }
 uint64_t gdn_buffers_bytes(const ModelGeometry& g) {    const int64_t C = g.ssm_conv_channels;    const int64_t V = g.ssm_value_dim;    const uint64_t parts[] = {        q8k_bytes(g.n_embd),
 // x_q8k
 (uint64_t) (g.n_embd / 32) * 34,
@@ -507,10 +509,15 @@ KvPlan kv_plan(const QsaShapes& s, int64_t max_cells, int64_t ring_cells) {
     }
     return p;
 }
-uint64_t kv_pool_bytes(const QsaShapes& s, int64_t pages) {
+uint64_t kv_pool_bytes(const QsaShapes& s, int64_t pages, bool hybrid, bool int8) {
+    if (hybrid) {   // K8V4: the INT8 K half (codes + scales) plus the Q4_0 V half (kv_q4.hpp's rotation)
+        const uint64_t rows = (uint64_t) pages * s.page_size * s.n_head_kv;
+        return rows * (uint64_t) s.head_dim + rows * (uint64_t) (s.head_dim / strata::kernels::KV_Q8_GROUP) * 2 +
+               rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim) + 64;
+    }
     if (g_kv_q4) return (uint64_t) pages * s.page_size * strata::kernels::kv_q4_bytes_per_cell(s) + 64;
-    return g_kv_int8 ? (uint64_t) pages * s.page_size * strata::kernels::kv_q8_bytes_per_cell(s) + 64
-                     : (uint64_t) pages * s.n_head_kv * s.page_size * s.head_dim * 2 * 2;
+    return int8 ? (uint64_t) pages * s.page_size * strata::kernels::kv_q8_bytes_per_cell(s) + 64
+                : (uint64_t) pages * s.n_head_kv * s.page_size * s.head_dim * 2 * 2;
 }
 }  // namespace
 
@@ -523,7 +530,8 @@ uint64_t qsa_state_bytes(const ModelGeometry& g, int64_t max_cells, bool with_ro
     const QsaShapes s = qsa_shapes(g);
     const KvPlan p = kv_plan(s, max_cells, ring_cells);
     uint64_t n = 0;
-    n += kv_pool_bytes(s, p.slots) + 4 * 16;                                   // K/V pools (the VRAM slots)
+    n += kv_pool_bytes(s, p.slots, g_kv_hybrid && ring_cells <= 0,
+                       g_kv_int8 || g_kv_hybrid) + 4 * 16;   // K/V pools (the VRAM slots)
     n += (uint64_t) p.pages * 4;                                               // page_table
     if (p.mode == 1) n += strata::kernels::kv_stream_map_bytes(p.slots) + 6 * 16;   // the residency map
     n += (uint64_t) (s.idx_block - 1) * s.idx_dim * 4;                         // tail
@@ -544,14 +552,29 @@ uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, Q
     Cursor c{(uint8_t*) base};
     st.kv_int8 = g_kv_int8 && !g_kv_q4;
     st.kv_q4 = g_kv_q4;
+    // Hybrid K8V4, main layers only (the drafter's state is created with the globals toggled to INT8 -
+    // mtp.cpp). Streamed mode is refused outright; generate.cpp validates it too, this is the backstop.
+    if (g_kv_hybrid && ring_cells <= 0) {
+        if (p.mode == 1) {
+            std::fprintf(stderr, "strata: hybrid K8V4 KV does not support --kv-resident streaming\n");
+            return 0;
+        }
+        st.kv_hybrid = true;
+        st.kv_int8 = false;
+        st.kv_q4 = false;
+    }
     st.kv_mode = p.mode;
     st.n_slots = p.slots;
     const uint64_t rows = (uint64_t) p.slots * s.n_head_kv * s.page_size;   // VRAM rows: the slots
     const uint64_t q4_row = strata::kernels::kv_q4_bytes_per_head((int) s.head_dim);
-    if (st.kv_q4) {
+    if (st.kv_hybrid) {
+        st.k_q = c.take<int8_t>(rows * s.head_dim);
+        st.k_scale = c.take<uint16_t>(rows * (s.head_dim / strata::kernels::KV_Q8_GROUP));
+        st.v_q4 = c.take<uint8_t>(rows * q4_row);
+    } else if (st.kv_q4) {
         st.k_q4 = c.take<uint8_t>(rows * q4_row);
         st.v_q4 = c.take<uint8_t>(rows * q4_row);
-    } else if (g_kv_int8) {
+    } else if (st.kv_int8) {   // st, not the global: the drafter's ring is INT8 under --kv k8v4 too
         st.k_q = c.take<int8_t>(rows * s.head_dim);
         st.v_q = c.take<int8_t>(rows * s.head_dim);
         st.k_scale = c.take<uint16_t>(rows * (s.head_dim / strata::kernels::KV_Q8_GROUP));
@@ -619,7 +642,7 @@ uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, Q
         if (st.kv_q4) {
             st.host.k_q4 = hc.take<uint8_t>(hrows * q4_row);
             st.host.v_q4 = hc.take<uint8_t>(hrows * q4_row);
-        } else if (g_kv_int8) {
+        } else if (st.kv_int8) {
             st.host.k_q = hc.take<int8_t>(hrows * s.head_dim);
             st.host.v_q = hc.take<int8_t>(hrows * s.head_dim);
             st.host.k_scale = hc.take<uint16_t>(hrows * (s.head_dim / strata::kernels::KV_Q8_GROUP));
@@ -658,7 +681,11 @@ void qsa_state_zero(const QsaState& st, const ModelGeometry& g, void* stream) {
     const QsaShapes s = qsa_shapes(g);
     cudaStream_t cs = (cudaStream_t) stream;
     const size_t rows = (size_t) st.n_slots * s.n_head_kv * s.page_size;
-    if (st.kv_q4) {
+    if (st.kv_hybrid) {
+        cudaMemsetAsync(st.k_q, 0, rows * s.head_dim, cs);
+        cudaMemsetAsync(st.k_scale, 0, rows * (s.head_dim / strata::kernels::KV_Q8_GROUP) * 2, cs);
+        cudaMemsetAsync(st.v_q4, 0, rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim), cs);
+    } else if (st.kv_q4) {
         cudaMemsetAsync(st.k_q4, 0, rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim), cs);
         cudaMemsetAsync(st.v_q4, 0, rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim), cs);
     } else if (st.kv_int8) {
@@ -682,7 +709,8 @@ void qsa_state_zero(const QsaState& st, const ModelGeometry& g, void* stream) {
 strata::kernels::QsaAttnPools qsa_attn_pools(const QsaState& st) {
     strata::kernels::QsaAttnPools pools;
     pools.page_table = st.page_table;
-    if (st.kv_q4) { pools.k_q4 = st.k_q4; pools.v_q4 = st.v_q4; }
+    if (st.kv_hybrid) { pools.k_q = st.k_q; pools.k_scale = st.k_scale; pools.v_q4 = st.v_q4; }
+    else if (st.kv_q4) { pools.k_q4 = st.k_q4; pools.v_q4 = st.v_q4; }
     else if (st.kv_int8) { pools.k_q = st.k_q; pools.v_q = st.v_q; pools.k_scale = st.k_scale; pools.v_scale = st.v_scale; }
     else { pools.k_pool = st.k_pool; pools.v_pool = st.v_pool; }
     return pools;
@@ -863,11 +891,18 @@ if (!gemv_quantized(*w_attnk, p_k, f_k, b.x_q8_0, b.x_q8k, b.kcur, g.n_embd, g.n
 // EVERY ENTRY POINT FROM HERE ON IS THE CAPTURABLE ONE: the per-token counts come from `st.step` and every
 // launch is sized from a capacity in `st`/`b`, so this sequence can be captured and replayed.  The
 // host-scalar wrappers would be correct here today and silently wrong in a graph.
-if (st.kv_q4) {
+if (st.kv_hybrid) {
+    // K8V4: only V is rotated (kv_q4.hpp's H); the scores pair unrotated q with unrotated INT8 K, and the
+    // output - a mix of rotated values - is rotated back after attention. Each append/gather call folds the
+    // unused half's lanes onto the used pool (a bit-identical duplicate write), so no kernel variants exist.
+    strata::kernels::fwht256_inplace_cuda(b.vcur, g.n_head_kv, stream);
+    kv_append_q8_step(st.k_q, st.k_q, st.k_scale, st.k_scale, st.page_table, st.step, b.kcur, b.kcur, s, stream, nullptr);   // mode 0: no host mirror
+    strata::kernels::kv_append_q4_step(st.v_q4, st.v_q4, st.page_table, st.step, b.vcur, b.vcur, s, stream, nullptr);
+} else if (st.kv_q4) {
     strata::kernels::fwht256_inplace_cuda(b.kcur, g.n_head_kv, stream);   // Q4_0: rotated K and V (kv_q4.hpp)
     strata::kernels::fwht256_inplace_cuda(b.vcur, g.n_head_kv, stream);
     strata::kernels::kv_append_q4_step(st.k_q4, st.v_q4, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);
-} else if (st.kv_int8) kv_append_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    else kv_append_step(st.k_pool, st.v_pool, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    {        const uint64_t nvk = (uint64_t) g.n_head_kv * g.head_dim;        const uint64_t base = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim + 2 * nvk + 8;        if (!st.kv_int8) dump_slot(dump, g, layer, (const float*) st.k_pool, base, nvk / 2, stream);        if (!st.kv_int8) dump_slot(dump, g, layer, (const float*) st.v_pool, base + nvk / 2, nvk / 2, stream);        dump_slot(dump, g, layer, b.vcur, base + nvk, nvk, stream);        dump_slot(dump, g, layer, b.kcur, base + 2 * nvk, nvk, stream);    }    {        const QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};        if (native_qsa_indexer_enabled()) {
+} else if (st.kv_int8) kv_append_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    else kv_append_step(st.k_pool, st.v_pool, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    {        const uint64_t nvk = (uint64_t) g.n_head_kv * g.head_dim;        const uint64_t base = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim + 2 * nvk + 8;        if (!st.kv_int8 && !st.kv_hybrid) dump_slot(dump, g, layer, (const float*) st.k_pool, base, nvk / 2, stream);        if (!st.kv_int8 && !st.kv_hybrid) dump_slot(dump, g, layer, (const float*) st.v_pool, base + nvk / 2, nvk / 2, stream);        dump_slot(dump, g, layer, b.vcur, base + nvk, nvk, stream);        dump_slot(dump, g, layer, b.kcur, base + 2 * nvk, nvk, stream);    }    {        const QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};        if (native_qsa_indexer_enabled()) {
     try {
         native_qsa_indexer_append(b.idx_raw, st.step + kStepPos, pos_base,
             (const float*) w_ikn->data, RMS_EPS, ib, s, st.max_cells, (float) qsa_freq_base(), stream);
@@ -906,7 +941,13 @@ int64_t max_blocks = (st.max_cells / s.idx_block) + 2;
         const strata::kernels::QsaAttnPools pools = qsa_attn_pools(st);
         strata::kernels::qsa_decode_attn_step(b.qcur, pools, b.ids, st.step, cap, s, b.attn_scratch, b.attn, stream);
     } else {
-    if (st.kv_q4) strata::kernels::kv_gather_q4_step(st.k_q4, st.v_q4, st.page_table, b.ids, st.step, cap, s, b.k_scratch, b.v_scratch, stream);
+    if (st.kv_hybrid) {
+        kv_gather_q8_step(st.k_q, st.k_q, st.k_scale, st.k_scale, st.page_table, b.ids, st.step, cap, s,
+                          b.k_scratch, b.k_scratch, stream);
+        strata::kernels::kv_gather_q4_step(st.v_q4, st.v_q4, st.page_table, b.ids, st.step, cap, s,
+                                           b.v_scratch, b.v_scratch, stream);
+    }
+    else if (st.kv_q4) strata::kernels::kv_gather_q4_step(st.k_q4, st.v_q4, st.page_table, b.ids, st.step, cap, s, b.k_scratch, b.v_scratch, stream);
     else if (st.kv_int8) kv_gather_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, b.ids, st.step, cap, s,                                 b.k_scratch, b.v_scratch, stream);    else kv_gather_step(st.k_pool, st.v_pool, st.page_table, b.ids, st.step, cap, s, b.k_scratch, b.v_scratch,                   stream);    if (native_flash_attn_short) {
     if (st.max_cells < 1 || st.max_cells > 256 || !st.attention_status || !st.host_step) {
         err = v.name("native_flash_attn") + ": short adapter requires context <=256 and persistent status storage";
@@ -922,7 +963,7 @@ int64_t max_blocks = (st.max_cells / s.idx_block) + 2;
     }
 } else qsa_attend_step(b.qcur, b.k_scratch, b.v_scratch, st.step, cap, s, b.attn, nullptr, stream);
     }
-    if (st.kv_q4) strata::kernels::fwht256_inplace_cuda(b.attn, g.n_head, stream);   // the output back: H is self-inverse
+    if (st.kv_q4 || st.kv_hybrid) strata::kernels::fwht256_inplace_cuda(b.attn, g.n_head, stream);   // the output back: H is self-inverse
     dump_slot(dump, g, layer, b.attn, (uint64_t) 2 * g.n_embd + 2 * g.hc,                            (uint64_t) g.n_head * g.head_dim, stream);    {        const uint64_t vs = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim +                            (uint64_t) 2 * g.n_head_kv * g.head_dim;        dump_slot(dump, g, layer, (const float*) b.v_scratch, vs,                            (uint64_t) g.n_head_kv * g.head_dim / 2, stream);        dump_slot(dump, g, layer, (const float*) b.ids, vs + (uint64_t) g.n_head_kv * g.head_dim / 2, 4, stream);        dump_slot(dump, g, layer, (const float*) st.step,                            vs + (uint64_t) g.n_head_kv * g.head_dim / 2 + 4, 4, stream);    }
 // ---- 9. Gate the attention output before its projection. Native CUDA keeps F32
 // sigmoid/multiply arithmetic and uses Q8_1 for the native quantized projection.

src/core/load_main.cpp

@@ -65,7 +65,10 @@ int main(int argc, char** argv) {
 
     const strata::core::LoadStats st =
         strata::core::load_experts(path, dst, blob_bytes, blobs_per_layer, layers, threads, chunk);
-    if (st.seconds < 0) return 1;
+    if (!st.ok) {
+        std::fprintf(stderr, "loading the experts failed: %s\n", st.error.empty() ? "unknown" : st.error.c_str());
+        return 1;
+    }
 
     std::printf("loaded %.3f GiB in %.3f s with %d threads and %llu MiB chunks  ->  %.2f GiB/s\n",
                 (double) st.bytes / (1024.0 * 1024 * 1024), st.seconds, threads,

src/core/mtp.cpp

@@ -69,13 +69,31 @@ strata::kernels::QsaShapes shapes_of(const ModelGeometry& g) {
     return s;
 }
 
+// 64-bit seek/tell on a `FILE*`: `fseek`/`ftell` take a 32-bit `long` on Windows and would wrap past 2 GiB.
+#if defined(_WIN32)
+#define STRATA_FILE_SEEK64(f, o, w) _fseeki64((f), (long long) (o), (w))
+#define STRATA_FILE_TELL64(f) _ftelli64(f)
+#else
+#define STRATA_FILE_SEEK64(f, o, w) fseeko((f), (off_t) (o), (w))
+#define STRATA_FILE_TELL64(f) ftello(f)
+#endif
+
 bool read_file(const std::string& path, std::vector<uint8_t>& out) {
-    std::ifstream f(path, std::ios::binary | std::ios::ate);
-    if (!f) return false;
-    const std::streamsize n = f.tellg();
-    f.seekg(0);
+    // Loader fix (0.1.15+loaderfix.2): `ifstream::read` reaches the disk as 4095-byte reads on MSVC - the same
+    // split `load_experts_ranges` had - so the drafter's 111 MiB dense blob paid 4 KiB per operation on a cold
+    // start.  `fread` passes a request bigger than the stream buffer straight to `ReadFile()`.
+    FILE* f = std::fopen(path.c_str(), "rb");
+    if (f == nullptr) return false;
+    // A `FILE*` has no destructor and the short-read path below returns early: the guard closes on every path.
+    struct Closer {
+        FILE* f;
+        ~Closer() { if (f != nullptr) std::fclose(f); }
+    } closer{f};
+    if (STRATA_FILE_SEEK64(f, 0, SEEK_END) != 0) return false;
+    const long long n = (long long) STRATA_FILE_TELL64(f);
+    if (n < 0 || STRATA_FILE_SEEK64(f, 0, SEEK_SET) != 0) return false;
     out.resize((size_t) n);
-    return (bool) f.read((char*) out.data(), n);
+    return n == 0 || std::fread(out.data(), 1, (size_t) n, f) == (size_t) n;
 }
 
 }  // namespace
@@ -83,6 +101,8 @@ bool read_file(const std::string& path, std::vector<uint8_t>& out) {
 MtpDrafter::~MtpDrafter() {
     if (cs_) cudaStreamSynchronize(cs_);
     for (auto& e : prefill_exec_) if (e) cudaGraphExecDestroy(e);
+    for (auto& e : prefill_dev_exec_) if (e) cudaGraphExecDestroy(e);
+    if (pf_dev_) cudaFree(pf_dev_);
     for (auto& e : round_exec_) if (e) cudaGraphExecDestroy(e);
     for (auto& e : step_exec_) if (e) cudaGraphExecDestroy(e);
     if (cs_) cudaStreamDestroy(cs_);
@@ -118,6 +138,9 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
     max_t_ = max_t;
     rt_dir_ = rt_dir;
     if (max_t < 1 || max_t > strata::kernels::kVerifyMaxT) { err = "mtp: max_t out of range"; return false; }
+    // Loader fix (0.1.15+loaderfix.2): the two reads below are the whole “drafter files” cost; reporting
+    // them apart from the rest of the stage is what makes the next regression visible.
+    const auto t_files = std::chrono::steady_clock::now();
     // ---- the index and the dense weights
     {
         std::ifstream idx(rt_dir + "/dense.txt");
@@ -148,13 +171,19 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
     // ---- the 512 routed experts, one blob each
     {
         const uint64_t bytes = (uint64_t) g.n_expert * strata::kernels::cpu::BLOB;
-        std::ifstream f(rt_dir + "/experts.bin", std::ios::binary);
-        if (!f) { err = "mtp: cannot open experts.bin"; return false; }
+        // Loader fix (0.1.15+loaderfix.2): each 64 MiB read below reached the disk as ~16k 4095-byte reads under
+        // MSVC's `basic_filebuf::xsgetn`, which is what made 675 MiB of drafter experts take minutes.
+        FILE* f = std::fopen((rt_dir + "/experts.bin").c_str(), "rb");
+        if (f == nullptr) { err = "mtp: cannot open experts.bin"; return false; }
+        struct Closer {
+            FILE* f;
+            ~Closer() { if (f != nullptr) std::fclose(f); }
+        } closer{f};
         if (cudaMalloc((void**) &experts_, bytes) != cudaSuccess) { err = "mtp: the 512 experts do not fit in VRAM"; return false; }
         std::vector<uint8_t> chunk(64u << 20);
         for (uint64_t off = 0; off < bytes;) {
             const uint64_t n = std::min<uint64_t>(chunk.size(), bytes - off);
-            if (!f.read((char*) chunk.data(), (std::streamsize) n)) { err = "mtp: experts.bin is truncated"; return false; }
+            if (std::fread(chunk.data(), 1, (size_t) n, f) != (size_t) n) { err = "mtp: experts.bin is truncated"; return false; }
             cudaMemcpy(experts_ + off, chunk.data(), n, cudaMemcpyHostToDevice);
             off += n;
         }
@@ -172,6 +201,12 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
     // window (plus the cells a round writes ahead of its queries) over a host copy, refilled on a resume. The host copy
     // is pinned after the expert arena has pinned what it could: if it does not fit, the K/V stays whole in VRAM.
     int64_t ring = (window > 0 && window < max_cells) ? window + 4 * (int64_t) max_t + 64 : 0;
+    // K8V4 never applies to the drafter: its own attention paths (below, and verify.cpp) handle whole formats
+    // only, whatever ring shape it takes (0, a window, or the -1 fully-resident fallback).
+    const bool kv_hybrid_was = qsa_kv_hybrid();
+    const bool kv_int8_was = qsa_kv_int8();
+    qsa_set_kv_hybrid(false);
+    if (kv_hybrid_was) qsa_set_kv_int8(true);   // the drafter under --kv k8v4: plain INT8
     uint64_t sb = qsa_state_bytes(g, max_cells, false, ring);
     if (cudaMalloc(&state_arena_, sb) != cudaSuccess) { err = "mtp: the K/V state does not fit"; return false; }
     if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[0], ring) == 0) {
@@ -185,6 +220,8 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
         if (cudaMalloc(&state_arena_, sb) != cudaSuccess) { err = "mtp: the K/V state does not fit"; return false; }
         if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[0], ring) == 0) { err = "mtp: state init failed"; return false; }
     }
+    qsa_set_kv_int8(kv_int8_was);
+    qsa_set_kv_hybrid(kv_hybrid_was);
     qsa_state_zero(st_, g, nullptr);
     cudaDeviceSynchronize();
     vram_ += sb;
@@ -246,12 +283,28 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
         cudaMemcpy(ident_, id.data(), id.size() * 4, cudaMemcpyHostToDevice);
     }
     if (cudaStreamCreateWithFlags(&cs_, cudaStreamNonBlocking) != cudaSuccess) { err = "mtp: stream"; return false; }
-    std::fprintf(stderr, "strata mtp: draft layer loaded, %.0f MiB of VRAM (experts %.0f, dense %.0f)\n",
+    const double files_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - t_files).count();
+    std::fprintf(stderr, "strata mtp: draft layer loaded, %.0f MiB of VRAM (experts %.0f, dense %.0f), files read in %.2f s (%.0f MiB/s)\n",
                  (double) vram_ / 1048576.0, (double) g.n_expert * strata::kernels::cpu::BLOB / 1048576.0,
-                 (double) tensors_.back().off / 1048576.0);
+                 (double) tensors_.back().off / 1048576.0, files_s,
+                 files_s > 0 ? ((double) g.n_expert * strata::kernels::cpu::BLOB + (double) tensors_.back().off) /
+                                   1048576.0 / files_s : 0.0);
     return true;
 }
 
+uint64_t MtpDrafter::bind_bytes(uint64_t head_row_bytes, int64_t n_vocab) const {
+    uint64_t bytes = head_logits_ ? 0 : (uint64_t) max_t_ * (uint64_t) n_vocab * sizeof(float);
+    if (dhead_ == nullptr) {
+        if (FILE* f = std::fopen((rt_dir_ + "/draft_vocab.bin").c_str(), "rb")) {
+            std::fseek(f, 0, SEEK_END);
+            const long size = std::ftell(f);
+            std::fclose(f);
+            if (size >= 4 && size % 4 == 0) bytes += (uint64_t) (size / 4) * head_row_bytes + (uint64_t) size;
+        }
+    }
+    return bytes;
+}
+
 bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err) {
     const OnDevice on_device(device_);
     wt_ = &wt;
@@ -480,6 +533,13 @@ bool MtpDrafter::capture_prefill(int T, std::string& err) {
     return finish_capture(cs_, ok, prefill_exec_[T], "prefill", err);
 }
 
+bool MtpDrafter::capture_prefill_dev(int T, std::string& err) {
+    if (prefill_dev_exec_[T]) return true;
+    if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "mtp: begin capture"; return false; }
+    const bool ok = record_forward(T, -1, cs_, err);   // K/V only, rows [0, T); tok_/step_/pos_ filled before launch
+    return finish_capture(cs_, ok, prefill_dev_exec_[T], "prefill (device inputs)", err);
+}
+
 bool MtpDrafter::capture_round(int T, std::string& err) {
     if (round_exec_[T]) return true;
     using namespace strata::kernels;
@@ -537,6 +597,64 @@ bool MtpDrafter::prefill(const float* R_rows, const int32_t* next_tokens, int64_
     const int64_t HCN = g_->hc * g_->n_embd;
     // cells the window can never reach again need no K/V
     const int64_t first_needed = (window_ > 0 && prompt_len_ > 0) ? prompt_len_ - window_ - 64 : 0;
+    // E-4: every group's token / step / position records uploaded at once; each group is then device copies and a
+    // graph on the one stream, with a single sync at the end (a group of <= max_t rows used to be staged in mapped
+    // memory and synced before the next: ~5,500 host round trips on a 32K prompt).  The same work in the same
+    // order.  STRATA_MTP_PREFILL_SYNC=1 keeps the old loop.
+    static const bool per_group_sync = std::getenv("STRATA_MTP_PREFILL_SYNC") != nullptr;
+    const int64_t NHp = g_->n_head, per_row = 1 + 4 + NHp;
+    if (!per_group_sync && n > 0) {
+        if (pf_cap_ < n * per_row) {
+            if (pf_dev_) cudaFree(pf_dev_);
+            pf_dev_ = nullptr;
+            pf_cap_ = 0;
+            if (cudaMalloc((void**) &pf_dev_, (size_t) (n * per_row) * sizeof(int32_t)) != cudaSuccess) {
+                err = "mtp prefill: the input records do not fit";
+                return false;
+            }
+            pf_cap_ = n * per_row;
+        }
+        std::vector<int32_t> rec((size_t) (n * per_row));
+        int32_t* tk = rec.data();
+        int32_t* stp = tk + n;
+        int32_t* ps = stp + 4 * n;
+        for (int64_t i = 0; i < n; ++i) {
+            const int64_t cell = cell0 + i;
+            tk[i] = next_tokens[i];
+            stp[i * 4 + 0] = (int32_t) cell;
+            stp[i * 4 + 1] = (int32_t) (cell + 1);
+            stp[i * 4 + 2] = (int32_t) ((cell + 1) / 4);
+            stp[i * 4 + 3] = (int32_t) (cell + 1);
+            for (int64_t h = 0; h < NHp; ++h) ps[i * NHp + h] = (int32_t) cell;
+        }
+        if (cudaMemcpyAsync(pf_dev_, rec.data(), rec.size() * sizeof(int32_t), cudaMemcpyHostToDevice, cs_) != cudaSuccess) {
+            err = "mtp prefill: the input records upload failed";
+            return false;
+        }
+        const int32_t* d_tk = pf_dev_;
+        const int32_t* d_stp = d_tk + n;
+        const int32_t* d_ps = d_stp + 4 * n;
+        for (int64_t c = 0; c < n; c += max_t_) {
+            const int T = (int) std::min<int64_t>(max_t_, n - c);
+            if (cell0 + c + T <= first_needed) continue;
+            if (!capture_prefill_dev(T, err)) return false;
+            if (cudaMemcpyAsync(tok_, d_tk + c, (size_t) T * 4, cudaMemcpyDeviceToDevice, cs_) != cudaSuccess ||
+                cudaMemcpyAsync(step_, d_stp + c * 4, (size_t) T * 16, cudaMemcpyDeviceToDevice, cs_) != cudaSuccess ||
+                cudaMemcpyAsync(pos_, d_ps + c * NHp, (size_t) (T * NHp) * 4, cudaMemcpyDeviceToDevice, cs_) != cudaSuccess ||
+                cudaMemcpyAsync(Rin_, R_rows + (size_t) c * HCN, (size_t) T * HCN * sizeof(float), cudaMemcpyDeviceToDevice,
+                                cs_) != cudaSuccess ||
+                cudaGraphLaunch(prefill_dev_exec_[T], cs_) != cudaSuccess) {
+                err = std::string("mtp prefill: ") + cudaGetErrorString(cudaGetLastError());
+                return false;
+            }
+        }
+        if (cudaStreamSynchronize(cs_) != cudaSuccess) {
+            err = std::string("mtp prefill: ") + cudaGetErrorString(cudaGetLastError());
+            return false;
+        }
+        ms_prefill += ms_since(t0);
+        return true;
+    }
     for (int64_t c = 0; c < n; c += max_t_) {
         const int T = (int) std::min<int64_t>(max_t_, n - c);
         if (cell0 + c + T <= first_needed) continue;

src/core/native_dense.cpp

@@ -16,9 +16,9 @@ namespace {
 bool eligible(const strata::TensorInfo& tensor, bool include_ple_key) {
     const auto& name = tensor.name;
     if (name.rfind("blk.", 0) != 0) return false;
-    // The native PLE kernel accepts Q2_0. Other quantized keys can be converted to BF16 by iq_pack;
-    // keep their packed bytes resident instead of overriding them with an unsupported native key.
-    if (name == "blk.1.ple_key.weight") return include_ple_key && tensor.type == 42;   // 42 = Q2_0
+    // Match the native PLE kernel: Q2_0, IQ3_XXS and IQ4_XS. Other keys retain the packed BF16 fallback.
+    if (name == "blk.1.ple_key.weight")
+        return include_ple_key && (tensor.type == 42 || tensor.type == 18 || tensor.type == 23);
     static const char* suffixes[] = {".attn_qkv.weight", ".attn_gate.weight", ".ssm_out.weight",
         ".attn_q.weight", ".attn_k.weight", ".attn_v.weight", ".attn_output.weight",
         ".ffn_gate_shexp.weight", ".ffn_up_shexp.weight", ".ffn_down_shexp.weight"};

src/core/pinned.cu

@@ -12,6 +12,14 @@
 #include <mutex>
 #include <thread>
 
+// Loader fix: `fseek`/`ftell` are 32-bit on Windows by default (and the pack is 42.9 GB), and the 64-bit
+// spelling is not the same on the two platforms the engine builds for.
+#ifdef _WIN32
+#define STRATA_FSEEK64(f, o) _fseeki64((f), (long long) (o), SEEK_SET)
+#else
+#define STRATA_FSEEK64(f, o) fseeko((f), (off_t) (o), SEEK_SET)
+#endif
+
 #ifdef _WIN32
 #include <windows.h>
 #else
@@ -234,11 +242,33 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::
     const auto t0 = std::chrono::steady_clock::now();
     std::vector<uint64_t> layer_hash((size_t) layers, 1469598103934665603ull);
     std::atomic<uint64_t> next_layer{0};
+    std::atomic<uint64_t> read_ns_sum{0};   // summed over the threads: see LoadStats::read_seconds
+    std::atomic<uint64_t> copy_ns_sum{0};
     std::mutex err_mu;
     std::string err;
 
     auto worker = [&]() {
         std::vector<uint8_t> buf((size_t) chunk);
+        // One handle per thread, seeked once per layer: a shared handle would need a lock around the seek and
+        // would serialise the very thing the threads are here to parallelise.  `fread` on a `FILE*` rather than
+        // `std::ifstream`: see the header - MSVC's `basic_filebuf::xsgetn` splits any request larger than
+        // `_INTERNAL_BUFSIZ - 1` into 4095-byte freads, which turned one 8 MiB chunk into ~2048 4 KiB reads.
+        // `fread` sees a request bigger than the stream buffer and passes it to `_read()`/
+        // `ReadFile()` unchanged, so the chunk size reaches the disk.  Buffered, not `FILE_FLAG_NO_BUFFERING`:
+        // the cache should still hold what it can.
+        FILE* f = std::fopen(path.c_str(), "rb");
+        if (f == nullptr) {
+            std::lock_guard<std::mutex> g(err_mu);
+            err = "cannot open " + path;
+            return;
+        }
+        // A `FILE*` has no destructor that closes it, and this function has early returns below (open, seek and
+        // short-read failures), so the guard is what keeps the closing correct on every path.
+        struct Closer {
+            FILE* f;
+            ~Closer() { if (f != nullptr) std::fclose(f); }
+        } closer{f};
+        uint64_t read_ns = 0, copy_ns = 0;
         for (;;) {
             const uint64_t L = next_layer.fetch_add(1);
             if (L >= layers) break;
@@ -246,30 +276,39 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::
             uint64_t remaining = layer_bytes[(size_t) L];
             uint64_t pos = 0;
             uint64_t h = 1469598103934665603ull;
-            // one handle per thread, seeked once per layer: a shared handle would need a lock around the seek
-            // and would serialise the very thing the threads are here to parallelise
-            std::ifstream f(path, std::ios::binary);
-            if (!f) {
+            // 64-bit seek: the pack is 42.9 GB, so the 32-bit `fseek` would wrap past 4 GiB
+            if (STRATA_FSEEK64(f, off) != 0) {
                 std::lock_guard<std::mutex> g(err_mu);
-                err = "cannot open " + path;
+                err = "seek to " + std::to_string(off) + " B failed in layer " + std::to_string(L);
                 return;
             }
-            f.seekg((std::streamoff) off);
             while (remaining > 0) {
                 const uint64_t n = remaining < chunk ? remaining : chunk;
-                f.read((char*) buf.data(), (std::streamsize) n);
-                if ((uint64_t) f.gcount() != n) {
+                const auto t_read = std::chrono::steady_clock::now();
+                const size_t got = std::fread(buf.data(), 1, (size_t) n, f);
+                read_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(
+                               std::chrono::steady_clock::now() - t_read).count();
+                // A short read is EOF or an I/O error, never a silent zero fill: say WHERE and HOW SHORT, and
+                // keep going no further - the caller turns this into a refused load, not a wrong answer.
+                if (got != (size_t) n) {
                     std::lock_guard<std::mutex> g(err_mu);
-                    err = "short read in layer " + std::to_string(L);
+                    err = "short read in layer " + std::to_string(L) + ": got " + std::to_string(got) + " of "
+                          + std::to_string(n) + " B at offset " + std::to_string(off + pos)
+                          + (std::ferror(f) != 0 ? " (ferror set)" : "");
                     return;
                 }
+                const auto t_copy = std::chrono::steady_clock::now();
                 std::memcpy(dst + off + pos, buf.data(), (size_t) n);
                 h = fnv1a64(buf.data(), n, h);
+                copy_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(
+                               std::chrono::steady_clock::now() - t_copy).count();
                 pos += n;
                 remaining -= n;
             }
             layer_hash[(size_t) L] = h;
         }
+        read_ns_sum.fetch_add(read_ns);
+        copy_ns_sum.fetch_add(copy_ns);
     };
 
     std::vector<std::thread> pool;
@@ -280,8 +319,12 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::
     if (!err.empty()) {
         std::fprintf(stderr, "load_experts: %s\n", err.c_str());
         st.seconds = -1.0;
+        st.ok = false;
+        st.error = err;
         return st;
     }
+    st.read_seconds = (double) read_ns_sum.load() / 1e9;
+    st.copy_seconds = (double) copy_ns_sum.load() / 1e9;
     st.layer_checksums = std::move(layer_hash);
     st.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
     return st;

src/core/remote_experts.cpp

@@ -73,9 +73,12 @@ bool RemoteExperts::preflight(int device, double& free_gib, std::string& err) {
     // The layer waits for this GPU on the CPU pool's critical path: spin instead of sleeping, whose wake-up
     // costs more than a small expert batch takes (measured: ~0.3 ms per round trip on Windows).  Only possible
     // before the device's context exists, so first thing; STRATA_REMOTE_SPIN=0 keeps the driver's default.
+#if !defined(STRATA_USE_HIP)
     const char* spin = std::getenv("STRATA_REMOTE_SPIN");
     if (!(spin && spin[0] == '0')) cudaInitDevice(device, cudaDeviceScheduleSpin | cudaDeviceMapHost, 0);
     cudaGetLastError();
+#endif
+    // HIP has no cudaInitDevice equivalent; retain its default scheduling policy.
     DeviceScope scope(device);
     if (!scope.ok) { err = scope.error(device); return false; }
     size_t free_bytes = 0, total_bytes = 0;

src/core/verify.cpp

@@ -9,6 +9,8 @@
 #include "strata/kernels/iq_kernels.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
+#include "strata/kernels/native_router.hpp"
+#include "strata/kernels/native_moe.hpp"
 #include "strata/kernels/cpu/expert.hpp"
 #include "strata/kernels/elementwise.hpp"
 #include "strata/kernels/fused_gr.hpp"
@@ -33,7 +35,11 @@
 #include "strata/kernels/shared_expert.hpp"
 #include "strata/kernels/verify_kernels.hpp"
 
+#include <algorithm>
 #include <atomic>
+#include <map>
+#include <string>
+#include <vector>
 #include <chrono>
 #include <cmath>
 #include <cstdio>
@@ -142,7 +148,8 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
         return false;
     }
     if (hits.d_res == nullptr || hits.cache_base == nullptr || hits.blob <= 0) {
-        err = "verify: needs the profile-filled VRAM expert tier (--expert-profile and --expert-cache)";
+        err = "verify: needs the profile-filled VRAM expert tier (--expert-profile and --expert-cache); with "
+              "--expert-cache auto, no VRAM was left for it: lower --max-context, use --kv k8v4 or images on the CPU";
         return false;
     }
     std::string why;
@@ -182,7 +189,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
     // ---- mapped staging
     bool ok = mapped(T * 4, (void**) &h_tok_, (void**) &m_tok_) &&
               mapped(T * strata::kernels::kStepCount * 4, (void**) &h_step_, (void**) &m_step_) &&
-              mapped(T * NH * 4, (void**) &h_pos_, (void**) &m_pos_) &&
+              mapped(T * (NH + NKV + IQ) * 4, (void**) &h_pos_, (void**) &m_pos_) &&
               mapped((2 + T) * 4 + 16, (void**) &h_commit_, (void**) &m_commit_) &&
               mapped(T * N * 4, (void**) &h_ple_, (void**) &m_ple_) &&
               mapped(T * 4 + 16, (void**) &h_out_, (void**) &m_out_) &&
@@ -221,7 +228,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
     // ---- the device arena: the same sequence counted, then carved
     auto carve = [&](Bump& b) {
         tok_ = b.take<int32_t>(T); step_ = b.take<int32_t>(T * strata::kernels::kStepCount);
-        pos_ = b.take<int32_t>(T * NH); commit_ = b.take<int32_t>(2 + T);
+        pos_ = b.take<int32_t>(T * (NH + NKV + IQ)); commit_ = b.take<int32_t>(2 + T);
         ple_ = b.take<float>(T * N); emb_ = b.take<float>(T * N); R_ = b.take<float>(T * HC * N);
         mixed_ = b.take<float>(T * N); bo_ = b.take<float>(T * N);
         inj_ = b.take<float>(T * HC); inj2_ = b.take<float>(T * HC);
@@ -260,6 +267,12 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
         return false;
     }
     cudaMemset(arena_, 0, count.used);
+    prof_on_ = std::getenv("STRATA_VERIFY_PROFILE") != nullptr;
+    if (prof_on_) {
+        const size_t np = (size_t) g.n_layers * kProfPer + 4;
+        if (cudaMalloc((void**) &prof_, np * 8) != cudaSuccess) { prof_on_ = false; prof_ = nullptr; cudaGetLastError(); }
+        else { cudaMemset(prof_, 0, np * 8); prof_h_.assign(np, 0); }
+    }
     Bump real;
     real.base = (uint8_t*) arena_;
     carve(real);
@@ -274,6 +287,21 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
         err = "verify: stream create failed";
         return false;
     }
+    // E-6: a layer whose routed experts are all resident is planned on the device (STRATA_VERIFY_DEVICE_PLAN=1: on;
+    // exact, but neutral on RIBPC 1-2 GPUs: off by default)
+    {
+        const char* v = std::getenv("STRATA_VERIFY_DEVICE_PLAN");
+        device_plan_ = v != nullptr && std::atoi(v) != 0;
+    }
+    if (device_plan_) {
+        bool ok2 = cudaMalloc((void**) &skip_, 64) == cudaSuccess && cudaMemset(skip_, 0, 64) == cudaSuccess;
+        if (ok2 && hits.slot_off != nullptr && hits.n_slots > 0) {
+            ok2 = cudaMalloc((void**) &slot_off_d_, (size_t) hits.n_slots * sizeof(unsigned long long)) == cudaSuccess &&
+                  cudaMemcpy(slot_off_d_, hits.slot_off, (size_t) hits.n_slots * sizeof(unsigned long long),
+                             cudaMemcpyHostToDevice) == cudaSuccess;
+        }
+        if (!ok2) { cudaGetLastError(); device_plan_ = false; }
+    }
     std::fprintf(stderr, "strata verify: window up to %d tokens, %.1f MiB of device buffers\n", max_t,
                  (double) count.used / 1048576.0);
     return true;
@@ -308,13 +336,18 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
     const bool ple_on = ss.ple.ready() && ple_stage();
     auto Rt = [&](int t) { return R_ + (size_t) t * HC * N; };
     const int G = (split_ && T >= 2) ? 2 : 1;
+    static const bool dec_batch = [] { const char* v = std::getenv("STRATA_DEC_BATCH"); return v == nullptr || std::atoi(v) != 0; }();
+    auto stamp = [&](int64_t l, int i, int grp) { if (prof_on_ && grp == 0) gpu_stamp(prof_, (int) (l * kProfPer + i), cs); };
     const int tb_[2] = {0, (T + 1) / 2}, te_[2] = {G == 2 ? (T + 1) / 2 : T, T};
     groups_[T] = G;
 
     // ---- the window's inputs, from mapped staging
     copy_i32_from_mapped(tok_, m_tok_, T, cs);
     copy_i32_from_mapped(step_, m_step_, (int64_t) T * kStepCount, cs);
-    copy_i32_from_mapped(pos_, m_pos_, (int64_t) T * NH, cs);
+    copy_i32_from_mapped(pos_, m_pos_, (int64_t) MT * (NH + NKV + IQ), cs);
+    // per-ROW positions of the K rows [t][NKV] and the indexer query rows [t][IQ] (for batched RoPE)
+    const int32_t* pos_k = pos_ + MT * NH;
+    const int32_t* pos_i = pos_ + MT * (NH + NKV);
     if (ple_on) copy_from_mapped(ple_, m_ple_, (int64_t) T * N, cs);
 
     // ---- the embeddings, broadcast to the hc streams - or, in a later stage of a layer split, the previous stage's
@@ -358,6 +391,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
     // ---------------------------------------------------------------- pre(l, group): up to the ring
     auto pre = [&](int64_t l, int grp) -> bool {
         const int tb = tb_[grp], te = te_[grp], n = te - tb;
+        stamp(l, 0, grp);
         const LayerView v(wt, l);
         const char* pfx[2] = {"hc_attn_", "hc_ffn_"};
         const WeightRef *wn[2], *wd[2], *wu[2], *wi[2];
@@ -400,9 +434,11 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 a.eps = EPS; a.lo = lo_ + t * g.hc_lr; a.rs = rs_ + t * HC;
                 a.inject_out = inj_out + t * HC; a.mixed = mixed_ + t * N;
             }
-            fused_gr_read_multi(fa, n, xn_ + (size_t) tb * HC * N, cs);
+            fused_gr_read_multi(fa, n, xn_ + (size_t) tb * HC * N, cs, (prof_on_ && grp == 0) ? prof_ : nullptr,
+                                (int) (l * kProfPer + (half == 0 ? 27 : 30)));
         };
         gr_read_group(0, pending, inj2_, inj_);
+        stamp(l, 1, grp);
         float* xm = mixed_ + tb * N;
         try {
             if (!is_qsa_layer(g, l)) {
@@ -425,14 +461,19 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 float* beta = beta_L_ + (size_t) gi * MT * HV;
                 native_quantize_q8_1(xm, xq_, (int) N, n, cs);
                 native_mmvq(wqkv->native_type, wqkv->native_data, xq_, qkv + (size_t) tb * C, (int) N, (int) C, n, cs);
+                stamp(l, 2, grp);
                 gdn_conv_l2_multi(conv, qkv, (const float*) wc->data, hb, (int) C, (int) (2 * HK), EPS, n, cs, tb);
+                stamp(l, 3, grp);
                 gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data,
                              (const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N, (int) HV,
                              n, cs);
+                stamp(l, 4, grp);
                 native_mmvq(wg->native_type, wg->native_data, xq_, z_ + (size_t) tb * ZV, (int) N, (int) ZV, n, cs);
+                stamp(l, 5, grp);
                 // the recurrence from the untouched state over tokens [0, te); outputs only for this group's
                 gdn_step_norm_multi(state, hb, (int) C, gate, beta, z_, (const float*) wnm->data, EPS, y_, (int) HK,
                                     (int) HV, te, nullptr, cs, tb);
+                stamp(l, 6, grp);
                 native_quantize_q8_1(y_ + (size_t) tb * ZV, xq_, (int) ZV, n, cs);
                 native_mmvq(wout->native_type, wout->native_data, xq_, bo_ + tb * N, (int) ZV, (int) N, n, cs);
             } else {
@@ -455,20 +496,34 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     else rope_neox_apply(data, data, rows, cols, (int) s.n_rot, st.cos_tab, st.sin_tab, pos, cs);
                 };
                 float* idx_raw = idx_raw_L_ + (size_t) qi * MT * ID;
+                // the per-token GEMVs / norms / RoPEs / copies of this layer as one launch over the
+                // window's rows each - row-wise identical arithmetic (STRATA_DEC_BATCH=0: token by token)
+                const bool qb = dec_batch && n > 1 && native_qsa_enabled() && native_rope_enabled() && !st.kv_q4;
                 native_quantize_q8_1(xm, xq_, (int) N, n, cs);
-                for (int t = tb; t < te; ++t)
+                if (qb) bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wik->data, idx_raw + tb * ID, ID, N, ID, n, cs);
+                else for (int t = tb; t < te; ++t)
                     bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wik->data, idx_raw + t * ID, (int) N, (int) ID, cs);
+                stamp(l, 7, grp);
                 native_mmvq(wk->native_type, wk->native_data, xq_, kcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD), n, cs);
                 native_mmvq(wv->native_type, wv->native_data, xq_, vcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD), n, cs);
-                for (int t = tb; t < te; ++t) norm_rope(kcur_ + t * NKV * HD, wkn, (int) NKV, (int) HD, pos_ + t * NH);
+                if (qb) norm_rope(kcur_ + tb * NKV * HD, wkn, (int) (n * NKV), (int) HD, pos_k + tb * NKV);
+                else for (int t = tb; t < te; ++t) norm_rope(kcur_ + t * NKV * HD, wkn, (int) NKV, (int) HD, pos_ + t * NH);
                 if (st.kv_q4) {   // Q4_0 KV (kv_q4.hpp): K and V rotated before they are stored
                     fwht256_inplace_cuda(kcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
                     fwht256_inplace_cuda(vcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
+                } else if (st.kv_hybrid) {   // K8V4: only V is rotated
+                    fwht256_inplace_cuda(vcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
                 }
+                stamp(l, 8, grp);
                 if (grp == 0) copy_from_mapped(tail_snap_ + (size_t) qi * TS, st.idx_tail, TS, cs);
                 for (int t = tb; t < te; ++t) {
                     const int32_t* step_t = step_ + t * kStepCount;
-                    if (st.kv_q4)
+                    if (st.kv_hybrid) {   // K8V4: the unused half's lanes folded onto the used pool (layer.cpp)
+                        kv_append_q8_step(st.k_q, st.k_q, st.k_scale, st.k_scale, st.page_table, step_t,
+                                          kcur_ + t * NKV * HD, kcur_ + t * NKV * HD, s, cs, nullptr);
+                        kv_append_q4_step(st.v_q4, st.v_q4, st.page_table, step_t, vcur_ + t * NKV * HD,
+                                          vcur_ + t * NKV * HD, s, cs, nullptr);
+                    } else if (st.kv_q4)
                         kv_append_q4_step(st.k_q4, st.v_q4, st.page_table, step_t, kcur_ + t * NKV * HD,
                                           vcur_ + t * NKV * HD, s, cs, &st.host);
                     else if (st.kv_int8)
@@ -483,8 +538,20 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     native_qsa_indexer_append(idx_raw + t * ID, step_ + t * kStepCount + kStepPos, 0,
                                               (const float*) wikn->data, EPS, ib, s, st.max_cells,
                                               (float) qsa_freq_base(), cs);
+                stamp(l, 9, grp);
                 native_mmvq(wq->native_type, wq->native_data, xq_, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD),
                             n, cs);
+                if (qb) {
+                    if (cudaMemcpy2DAsync(qcur_ + tb * NH * HD, (size_t) HD * 4, qfull_ + tb * NH * 2 * HD, (size_t) HD * 2 * 4,
+                                          (size_t) HD * 4, (size_t) (n * NH), cudaMemcpyDeviceToDevice, cs) != cudaSuccess) {
+                        err = "verify: the q/gate split failed";
+                        return false;
+                    }
+                    norm_rope(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH);
+                    bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID,
+                                              N, IQ * ID, n, cs);
+                    norm_rope(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ);
+                } else {
                 for (int t = tb; t < te; ++t) {
                     float* qc = qcur_ + t * NH * HD;
                     if (cudaMemcpy2DAsync(qc, (size_t) HD * 4, qfull_ + t * NH * 2 * HD, (size_t) HD * 2 * 4,
@@ -500,16 +567,24 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), cs);
                     norm_rope(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH);
                 }
+                }
+                stamp(l, 10, grp);
                 qsa_block_scores(st.idx_pooled, st.idx_dead, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n, max_blocks_,
                                  s, scores_ + (size_t) tb * max_blocks_, cs);
                 qsa_block_topk(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n, max_blocks_, cap_, s,
                                sel_ + (size_t) tb * cap_, cs);
+                stamp(l, 11, grp);
                 // KV streaming: the n selections' blocks resident (device-side, inside the graph)
                 qsa_kv_resolve(st, *g_, sel_ + (size_t) tb * cap_, step_ + tb * kStepCount, n, cap_, cs);
+                stamp(l, 12, grp);
                 const QsaAttnPools pools = qsa_attn_pools(st);
                 qsa_decode_attn_batch(qcur_ + tb * NH * HD, pools, sel_ + (size_t) tb * cap_, step_ + tb * kStepCount, cap_,
                                       s, attn_scratch_ + (size_t) tb * attn_scratch_floats_, attn_ + tb * NH * HD, n, cs);
-                if (st.kv_q4) fwht256_inplace_cuda(attn_ + tb * NH * HD, (int64_t) n * NH, cs);   // back: H^-1 = H
+                stamp(l, 13, grp);
+                if (st.kv_q4 || st.kv_hybrid) fwht256_inplace_cuda(attn_ + tb * NH * HD, (int64_t) n * NH, cs);   // back: H^-1 = H
+                if (qb) native_qsa_gate_apply(attn_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, attn32_ + tb * NH * HD,
+                                              (int) (n * NH), (int) HD, cs);
+                else
                 for (int t = tb; t < te; ++t) {
                     if (native_qsa_enabled())
                         native_qsa_gate_apply(attn_ + t * NH * HD, qfull_ + t * NH * 2 * HD, attn32_ + t * NH * HD,
@@ -517,6 +592,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     else
                         qsa_gate_apply_f32(attn_ + t * NH * HD, qfull_ + t * NH * 2 * HD, s, attn32_ + t * NH * HD, cs);
                 }
+                stamp(l, 14, grp);
                 native_quantize_q8_1(attn32_ + tb * NH * HD, xq_, (int) (NH * HD), n, cs);
                 native_mmvq(wo->native_type, wo->native_data, xq_, bo_ + tb * N, (int) (NH * HD), (int) N, n, cs);
             }
@@ -524,14 +600,31 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             err = "verify layer " + std::to_string(l) + ": " + e.what();
             return false;
         }
+        stamp(l, 16, grp);
         gr_read_group(1, true, inj_, inj2_);
+        // the window's rows routed in 2 launches (one router GEMV reading the weight once, one
+        // top-10) instead of 2 per token; every row's arithmetic is the single-token call's (STRATA_DEC_BATCH=0: old)
+        const WeightRef* w_router = v.get("ffn_gate_inp.weight");
+        if (dec_batch && n > 1 && w_router != nullptr && native_router_enabled() && NE == 512 && K == 10) {
+            try {
+                bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) w_router->data, logits_ + tb * NE, NE, N,
+                                          NE, n, cs);
+                native_router_top10_multi(logits_ + tb * NE, ids_ + tb * K, w_ + tb * K, n, cs);
+            } catch (const std::exception& e) { err = "verify router: " + std::string(e.what()); return false; }
+        } else
         for (int t = tb; t < te; ++t) {
             MoEBuffers mb = ss.moe;
             mb.logits = logits_ + t * NE; mb.ids = ids_ + t * K; mb.weights = w_ + t * K;
             if (!moe_route(wt, g, l, K, mb, mixed_ + t * N, cs, err, nullptr)) return false;
         }
+        if (device_plan_)   // E-6: every routed expert resident: this group's plan without the host
+            resident_plan(ids_ + tb * K, n * (int) K, (int) K, hits_.d_res + l * g.n_expert, (int) g.n_expert,
+                          hits_.cache_base, slot_off_d_, (long long) hits_.blob,
+                          plan_ + (size_t) grp * (size_t) (plan_i32_ + 16), (long long) max_t_ * K, skip_ + grp,
+                          (uint32_t) ((l - lb_) * G + grp + 1), cs);
         doorbell_publish(xm, ids_ + tb * K, w_ + tb * K, (int64_t) n * N, (int64_t) n * K, m_x_ + tb * N,
                          m_ids_ + tb * K, m_w_ + tb * K, m_seq_, cs);
+        stamp(l, 17, grp);
         {
             const WeightRef *wgi = need(v, "ffn_gate_inp_shexp.weight", err), *wsg = need(v, "ffn_gate_shexp.weight", err),
                             *wsu = need(v, "ffn_up_shexp.weight", err), *wsd = need(v, "ffn_down_shexp.weight", err);
@@ -544,7 +637,8 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             nsw.up_type = wsu->native_type; nsw.up_data = wsu->native_data;
             nsw.down_type = wsd->native_type; nsw.down_data = wsd->native_data;
             nsw.q8_1 = xq_;
-            for (int t = tb; t < te; ++t) f32_to_bf16_bulk(mixed_ + t * N, sh_bf16_ + t * N, N, cs);
+            if (dec_batch) f32_to_bf16_bulk(mixed_ + tb * N, sh_bf16_ + tb * N, (int64_t) n * N, cs);   // contiguous rows
+            else for (int t = tb; t < te; ++t) f32_to_bf16_bulk(mixed_ + t * N, sh_bf16_ + t * N, N, cs);
             try {
                 shared_expert_multi(n, xm, sh_bf16_ + tb * N, nsw, (const uint16_t*) wgi->data, sh_gate_ + (size_t) tb * g.n_ff,
                                     sh_up_ + (size_t) tb * g.n_ff, sh_g_ + tb, shared_ + tb * N, N, g.n_ff, cs);
@@ -557,6 +651,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             quantize_q8_1_rows(xm, n, N, nat_xq_ + (size_t) tb * (N / 32) * 36, cs);
         else
             quantize_q8_0_scaled(xm, hit_xq_ + (size_t) tb * (N / 32) * 34, hit_xs_ + (size_t) tb * (N / 32), (int64_t) n * N, cs);
+        stamp(l, 18, grp);
         return true;
     };
 
@@ -564,10 +659,16 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
     auto post = [&](int64_t l, int grp) -> bool {
         const int tb = tb_[grp], te = te_[grp], n = te - tb;
         const uint32_t ring = (uint32_t) ((l - lb_) * G + grp + 1);
-        wait_flag_ge(m_flagA_, ring, cs);                      // the pool published this group's GPU plan
         const int64_t cap = (int64_t) n * K, capx = (int64_t) max_t_ * K;
         int32_t* pl = plan_ + (size_t) grp * (size_t) (plan_i32_ + 16);
-        copy_i32_from_mapped(pl, m_plan_ + (size_t) grp * (size_t) plan_i32_, plan_i32_, cs);
+        if (device_plan_) {   // E-6: skipped when the device planned this group (all its experts resident)
+            wait_flag_ge_or(m_flagA_, ring, skip_ + grp, cs);
+            copy_i32_from_mapped_unless(pl, m_plan_ + (size_t) grp * (size_t) plan_i32_, plan_i32_, skip_ + grp, ring, cs);
+        } else {
+            wait_flag_ge(m_flagA_, ring, cs);                  // the pool published this group's GPU plan
+            copy_i32_from_mapped(pl, m_plan_ + (size_t) grp * (size_t) plan_i32_, plan_i32_, cs);
+        }
+        stamp(l, 19, grp);
         const int32_t* p_counts = pl;
         const int32_t* p_start = pl + 4;
         const int32_t* p_dst = p_start + capx + 1;
@@ -592,22 +693,43 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             }
         };
         grouped(p_ptr, p_start, p_counts);
-        wait_flag_ge(m_flagB_, ring, cs);                      // the PCIe share is in staging (DMA) or mapped
+        stamp(l, 20, grp);
+        if (device_plan_) wait_flag_ge_or(m_flagB_, ring, skip_ + grp, cs);
+        else wait_flag_ge(m_flagB_, ring, cs);                 // the PCIe share is in staging (DMA) or mapped
         if (sink_.pcie_mode == 2) {                            // stage it with a copy kernel, then point at staging
             const int64_t per = G == 2 ? kStagingBlobs / 2 : kStagingBlobs;
             uint8_t* stage = staging_ + (size_t) (grp * per) * lay.max_blob;
             fetch_blobs(p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), (int) per, cs);
             rebase_ptrs((unsigned long long*) p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), cs);
         }
+        stamp(l, 21, grp);
         grouped(p_ptr2, p_start2, p_counts + 2);
-        wait_flag_ge(m_flag_, ring, cs);                       // the CPU's share is in the mapped rows
-        copy_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (int64_t) n * K * N, cs);
+        stamp(l, 22, grp);
+        if (device_plan_) {   // no CPU share when the device planned the group: its rows are zeros
+            wait_flag_ge_or(m_flag_, ring, skip_ + grp, cs);
+            copy_or_zero_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (long long) n * K * N,
+                                     skip_ + grp, ring, cs);
+        } else {
+            wait_flag_ge(m_flag_, ring, cs);               // the CPU's share is in the mapped rows
+            stamp(l, 23, grp);
+            if (dec_batch)   // only the CPU rows cross PCIe (p_dst[0, counts[1]) = the GPU's own rows)
+                copy_rows_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (int64_t) n * K, N,
+                                      p_dst, p_counts + 1, cs);
+            else
+                copy_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (int64_t) n * K * N, cs);
+        }
         moe_hit_add(parts_ + (size_t) tb * K * N, hit_out, p_dst, p_counts + 1, cap, N, cs);
+        if (dec_batch && n > 1 && native_moe_combine_enabled()) {   // one launch for the window's rows
+            try {
+                native_moe_combine_multi(parts_ + (size_t) tb * K * N, w_ + tb * K, shared_ + tb * N, bo_ + tb * N, N, K, n, cs);
+            } catch (const std::exception& e) { err = "verify combine: " + std::string(e.what()); return false; }
+        } else
         for (int t = tb; t < te; ++t) {
             MoEBuffers mb = ss.moe;
             mb.weights = w_ + t * K; mb.shared = shared_ + t * N;
             if (!moe_combine_parts(g, l, K, mb, parts_ + (size_t) t * K * N, bo_ + t * N, cs, err)) return false;
         }
+        stamp(l, 24, grp);
         if (l == g.n_layers - 1) {
             for (int t = tb; t < te; ++t) gr_write(Rt(t), bo_ + t * N, inj2_ + t * HC, gs, Rt(t), cs);
             if (cvec().covers(l)) cvec_apply(Rt(tb), l, n, HC * N, nullptr, 0, nullptr, 0, false, cs);
@@ -634,6 +756,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
     }
 
     // ---- the head, T columns, and the argmax of each
+    stamp(g.n_layers, 0, 0);
     {
         const WeightRef *hn = wt.find("output_hc_norm.weight"), *hd = wt.find("output_hc_down.weight"),
                         *hu = wt.find("output_hc_up.weight");
@@ -665,9 +788,36 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
         sp.temperature = 0.0f;
         sample_tokens(head_logits_, T, (int) n_vocab_, nullptr, 0, sp, m_out_, cs);
     }
+    stamp(g.n_layers, 1, 0);
     return true;
 }
 
+std::string Verifier::profile_report() {
+    if (!prof_on_ || prof_windows_ == 0) return std::string();
+    static const char* names[kProfPer] = {"-", "hc-read0", "q8+qkv/q-idx gemv", "conv", "ab", "z", "rec", "q8+kv-idx",
+                                          "k/v+norm-rope", "kv+idx append", "q+q-idx", "scores+topk", "kv-resolve",
+                                          "attention", "gate", "", "out-proj", "hc-read1+router", "shared+quant",
+                                          "waitA", "VRAM hits", "waitB", "PCIe grp", "waitCPU", "copy+combine",
+                                          "(gap)", "head", "  hc0 norm", "  hc0 down", "  hc0 up", "", ""};
+    std::string out;
+    char b[80];
+    double total = 0;
+    for (int k = 0; k < 2; ++k) {
+        out += k == 0 ? " GDN layers:" : " | QSA layers:";
+        for (int i = 0; i < kProfPer; ++i) {
+            if (prof_sum_[k][i] <= 0) continue;
+            total += prof_sum_[k][i];
+            std::snprintf(b, sizeof b, " %s %.2f", names[i], prof_sum_[k][i] / 1e6 / (double) prof_windows_);
+            out += b;
+        }
+    }
+    std::snprintf(b, sizeof b, " | total %.2f ms/window over %lld windows", total / 1e6 / (double) prof_windows_, (long long) prof_windows_);
+    out += b;
+    for (auto& r : prof_sum_) for (double& d : r) d = 0;
+    prof_windows_ = 0;
+    return out;
+}
+
 bool Verifier::capture(int T, std::string& err) {
     if (exec_[T] != nullptr) return true;
     if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) {
@@ -687,6 +837,37 @@ bool Verifier::capture(int T, std::string& err) {
         err = std::string("verify: end capture: ") + cudaGetErrorString(ce);
         return false;
     }
+#if !defined(STRATA_USE_HIP)   // a CUDA debug listing (node types, kernel names)
+    if (std::getenv("STRATA_VERIFY_NODES") != nullptr) {   // what the window graph holds
+        size_t nn = 0;
+        cudaGraphGetNodes(graph, nullptr, &nn);
+        std::vector<cudaGraphNode_t> nodes(nn);
+        cudaGraphGetNodes(graph, nodes.data(), &nn);
+        std::map<std::string, int> kinds;
+        for (cudaGraphNode_t nd : nodes) {
+            cudaGraphNodeType ty;
+            cudaGraphNodeGetType(nd, &ty);
+            std::string name = "type" + std::to_string((int) ty);
+            if (ty == cudaGraphNodeTypeKernel) {
+                cudaKernelNodeParams kp{};
+                if (cudaGraphKernelNodeGetParams(nd, &kp) == cudaSuccess) {
+#if CUDART_VERSION >= 12030   // cudaFuncGetName arrived in CUDA 12.3
+                    const char* fn = nullptr;
+                    if (cudaFuncGetName(&fn, kp.func) == cudaSuccess && fn) name = fn;
+#endif
+                }
+            } else if (ty == cudaGraphNodeTypeMemcpy) name = "memcpy";
+            else if (ty == cudaGraphNodeTypeMemset) name = "memset";
+            ++kinds[name];
+        }
+        std::vector<std::pair<int, std::string>> v;
+        for (auto& [k2, c] : kinds) v.push_back({c, k2});
+        std::sort(v.rbegin(), v.rend());
+        std::fprintf(stderr, "strata verify: the %d-token window graph has %zu nodes:", T, nn);
+        for (size_t i = 0; i < v.size() && i < 40; ++i) std::fprintf(stderr, " %d x %.60s;", v[i].first, v[i].second.c_str());
+        std::fprintf(stderr, "\n");
+    }
+#endif
     const cudaError_t ie = cudaGraphInstantiate(&exec_[T], graph, 0);
     cudaGraphDestroy(graph);
     if (ie != cudaSuccess) {
@@ -782,6 +963,10 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
         h_tok_[t] = tokens[t];
         qsa_step_fill(h_step_ + t * kStepCount, pos0 + t, s);
         for (int64_t h = 0; h < g.n_head; ++h) h_pos_[t * g.n_head + h] = (int32_t) (pos0 + t);
+        int32_t* pk = h_pos_ + (size_t) max_t_ * g.n_head;
+        int32_t* pi = pk + (size_t) max_t_ * g.n_head_kv;
+        for (int64_t h = 0; h < g.n_head_kv; ++h) pk[t * g.n_head_kv + h] = (int32_t) (pos0 + t);
+        for (int64_t h = 0; h < g.idx_q_heads; ++h) pi[t * g.idx_q_heads + h] = (int32_t) (pos0 + t);
     }
     if (ss.ple.ready() && ple_stage()) {
         uint32_t rows[kVerifyMaxT * PLE_N_HEADS];
@@ -866,6 +1051,28 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
     if (se != cudaSuccess) { err = std::string("verify: ") + cudaGetErrorString(se); return false; }
     progress_at("verify window: waiting for the expert copies", (int64_t) T);
     cudaStreamSynchronize(copy_);   // no host function of this window may raise flag B in the next one
+    if (prof_on_ && G == 1) {       // the window's GPU stage stamps
+        cudaMemcpy(prof_h_.data(), prof_, prof_h_.size() * 8, cudaMemcpyDeviceToHost);
+        const int64_t L = g.n_layers;
+        auto at = [&](int64_t l, int i) { return prof_h_[(size_t) (l * kProfPer + i)]; };
+        for (int64_t l = 0; l < L; ++l) {
+            const int kind = is_qsa_layer(g, l) ? 1 : 0;
+            unsigned long long prev = at(l, 0);
+            for (int i = 1; i <= 24; ++i) {
+                const unsigned long long x = at(l, i);
+                if (x == 0 || x < prev) continue;
+                prof_sum_[kind][i] += (double) (x - prev);
+                prev = x;
+            }
+            if (l + 1 < L) prof_sum_[kind][25] += (double) (at(l + 1, 0) - at(l, 24));
+            prof_sum_[kind][27] += (double) (at(l, 27) - at(l, 0));    // hc-read0: norm
+            prof_sum_[kind][28] += (double) (at(l, 28) - at(l, 27));   //           down
+            prof_sum_[kind][29] += (double) (at(l, 1) - at(l, 28));    //           up
+            prof_sum_[kind][1] -= (double) (at(l, 1) - at(l, 0));      // (hc-read0 shown split)
+        }
+        prof_sum_[0][26] += (double) (at(L, 1) - at(L, 0));
+        ++prof_windows_;
+    }
     // ---- a sampled or penalized request: the head's sampling again, host-side so its parameters are this call's
     // own (a captured kernel would replay the same draws forever).  Row t's draw is Philox(seed, pos0 + t): tied to
     // the POSITION it samples, not to how the text was cut into windows, so a seed replays the same text whatever

src/kernels/cpu/iq_avx512.cpp

@@ -6,7 +6,8 @@
 // `maddubs` and a `madd`.  The arithmetic is ggml's (ggml-cpu/quants.c, the `_generic` references): integer sums
 // per block, times d_x * d_y * the format's constant - only the order of the float additions differs.
 //
-// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22).  IQ1_M stays on ggml-cpu.
+// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22).  IQ1_M stays on ggml-cpu (no shipped
+// model has IQ1_M expert rows: the 'Coder IQ1_M' pack's gate/up are IQ2_S / IQ3_XXS / IQ3_S).
 #include "strata/kernels/cpu/iq_avx512.hpp"
 
 #define GGML_COMMON_DECL_CPP
@@ -16,6 +17,7 @@
 #include <immintrin.h>
 
 #include <cmath>
+#include <cstdlib>
 #include <cstring>
 
 namespace strata::kernels::cpu {
@@ -26,6 +28,16 @@ inline uint32_t u32(const uint8_t* p) { uint32_t v; std::memcpy(&v, p, 4); retur
 inline uint16_t u16(const uint8_t* p) { uint16_t v; std::memcpy(&v, p, 2); return v; }
 inline uint64_t u64(const uint8_t* p) { uint64_t v; std::memcpy(&v, p, 8); return v; }
 
+// E-2: the IQ3 grids by one AVX-512 gather (the same words, the same results) - measured 3-5% SLOWER than the scalar
+// lookups on a Ryzen 5 7600 (Zen 4 gathers are microcoded), so opt-in: STRATA_IQ_GATHER=1
+const bool gather = std::getenv("STRATA_IQ_GATHER") != nullptr;
+// E-2: software prefetch distance in bytes (STRATA_IQ_PREFETCH; 0 = off): -2-3% gate/up time on IQ3_S decode
+// (the rows are read at ~30 GB/s by 6 cores, near what DDR5 with 4 KB pages gives: the pool is memory-bound)
+const int prefetch_ahead = [] {
+    const char* v = std::getenv("STRATA_IQ_PREFETCH");
+    return v ? std::atoi(v) : 2048;
+}();
+
 // lanes 0-7 -> s0, 8-15 -> s1, 16-23 -> s2, 24-31 -> s3 (int16 lanes of a maddubs result: two values each)
 inline __m512i scales4(int s0, int s1, int s2, int s3) {
     const __m512i idx = _mm512_set_epi16(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -98,7 +110,9 @@ template <> struct Fmt<18> {   // IQ3_XXS: d, qs[64] grid bytes, 8 x u32 (4 x 7-
     static constexpr float K = 0.25f;
     static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
         const uint8_t* q = b + 2 + 16 * j;
-        g = _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],
+        // E-2: one gather of the 16 grid words (the scalar lookups assembled with set_epi32 cost ~30 uops)
+        g = gather ? _mm512_i32gather_epi32(_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*) q)), iq3xxs_grid, 4)
+                   : _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],
                              (int) iq3xxs_grid[q[11]], (int) iq3xxs_grid[q[10]], (int) iq3xxs_grid[q[9]], (int) iq3xxs_grid[q[8]],
                              (int) iq3xxs_grid[q[7]], (int) iq3xxs_grid[q[6]], (int) iq3xxs_grid[q[5]], (int) iq3xxs_grid[q[4]],
                              (int) iq3xxs_grid[q[3]], (int) iq3xxs_grid[q[2]], (int) iq3xxs_grid[q[1]], (int) iq3xxs_grid[q[0]]);
@@ -119,12 +133,18 @@ template <> struct Fmt<21> {   // IQ3_S: d, qs[64], qh[8], signs[32], scales[4]
     static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
         const uint8_t* q = b + 2 + 16 * j;
         const uint32_t h0 = b[66 + 2 * j], h1 = b[66 + 2 * j + 1];
+        if (gather) {   // E-2: the 9-bit indices (the high bit from qh under a mask), one gather
+            __m512i idx = _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*) q));
+            idx = _mm512_mask_add_epi32(idx, (__mmask16) (h0 | (h1 << 8)), idx, _mm512_set1_epi32(256));
+            g = _mm512_i32gather_epi32(idx, iq3s_grid, 4);
+        } else {
 #define G3(k, h, kk) (int) iq3s_grid[q[k] | (((h >> kk) & 1) << 8)]
         g = _mm512_set_epi32(G3(15, h1, 7), G3(14, h1, 6), G3(13, h1, 5), G3(12, h1, 4),
                              G3(11, h1, 3), G3(10, h1, 2), G3(9, h1, 1), G3(8, h1, 0),
                              G3(7, h0, 7), G3(6, h0, 6), G3(5, h0, 5), G3(4, h0, 4),
                              G3(3, h0, 3), G3(2, h0, 2), G3(1, h0, 1), G3(0, h0, 0));
 #undef G3
+        }
         m = _cvtu64_mask64(u64(b + 74 + 8 * j));
         const uint8_t s = b[106 + j];
         const int sa = 2 * (s & 15) + 1, sb = 2 * (s >> 4) + 1;
@@ -139,6 +159,13 @@ inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y,
     const __m512i zero = _mm512_setzero_si512();
     for (int i = 0; i < nblocks; ++i) {
         const uint8_t* blk = row + (size_t) i * Fmt<TY>::bytes;
+        // E-2: the weight rows stream from DRAM (4 KB pages when large pages are refused): ask for the bytes a few
+        // blocks ahead (the next row's start included) before the decode needs them
+        if (prefetch_ahead > 0) {
+            const uint8_t* ahead = blk + (size_t) prefetch_ahead;
+            _mm_prefetch((const char*) ahead, _MM_HINT_T0);
+            _mm_prefetch((const char*) ahead + 64, _MM_HINT_T0);
+        }
         __m512i acci[NT];
         for (int t = 0; t < NT; ++t) acci[t] = _mm512_setzero_si512();
         for (int j = 0; j < 4; ++j) {

src/kernels/cuda/cvec.cu

@@ -11,9 +11,33 @@ namespace {
 constexpr int THREADS = 256;
 constexpr int MAXK = 16;   // n_embd up to 4096, held in registers between the dot and the update
 
-Cvec g_cvec;
-int* g_on = nullptr;
+Cvec g_cvec;                 // the description; its device pointers are the uploading device's
 bool g_on_host = false;
+// the tables on every device that holds them (a layer split applies the vector on several)
+constexpr int kDevices = 64;
+struct DevTables { float* dir = nullptr; float* s = nullptr; int* on = nullptr; };
+DevTables g_dev[kDevices];
+std::vector<float> g_dir_host, g_s_host;
+int cur_device() {
+    int d = 0;
+    if (cudaGetDevice(&d) != cudaSuccess || d < 0 || d >= kDevices) d = 0;
+    return d;
+}
+bool upload_here(std::string& err) {
+    DevTables& t = g_dev[cur_device()];
+    if (t.dir != nullptr) return true;
+    const int flag = g_on_host ? 1 : 0;
+    if (cudaMalloc(&t.dir, g_dir_host.size() * sizeof(float)) != cudaSuccess ||
+        cudaMalloc(&t.s, g_s_host.size() * sizeof(float)) != cudaSuccess || cudaMalloc(&t.on, sizeof(int)) != cudaSuccess ||
+        cudaMemcpy(t.dir, g_dir_host.data(), g_dir_host.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
+        cudaMemcpy(t.s, g_s_host.data(), g_s_host.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
+        cudaMemcpy(t.on, &flag, sizeof(int), cudaMemcpyHostToDevice) != cudaSuccess) {
+        err = "control vector: device allocation failed";
+        t = DevTables{};
+        return false;
+    }
+    return true;
+}
 
 // the fused hyper-connection read's gate (fused_gr.cu), so a write done here is bitwise the one it would have folded
 __device__ __forceinline__ float sigmoidf_(float x) { return 1.0f / (1.0f + __expf(-x)); }
@@ -78,21 +102,26 @@ bool cvec_upload(const std::vector<float>& dir, const std::vector<float>& s, int
                  int64_t n_embd, int64_t hc, std::string& err) {
     if (n_embd < 1 || n_embd > (int64_t) THREADS * MAXK) { err = "control vector: unsupported n_embd"; return false; }
     if (s.empty() || dir.size() != s.size() * (size_t) n_embd) { err = "control vector: bad table sizes"; return false; }
-    float* d_dir = nullptr;
-    float* d_s = nullptr;
-    int* d_on = nullptr;
-    const int one = 1;
-    if (cudaMalloc(&d_dir, dir.size() * sizeof(float)) != cudaSuccess ||
-        cudaMalloc(&d_s, s.size() * sizeof(float)) != cudaSuccess || cudaMalloc(&d_on, sizeof(int)) != cudaSuccess ||
-        cudaMemcpy(d_dir, dir.data(), dir.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
-        cudaMemcpy(d_s, s.data(), s.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
-        cudaMemcpy(d_on, &one, sizeof(int), cudaMemcpyHostToDevice) != cudaSuccess) {
-        err = "control vector: device allocation failed";
-        return false;
+    int prev = 0;   // a new vector replaces the old one on every device
+    cudaGetDevice(&prev);
+    for (int d = 0; d < kDevices; ++d) {
+        if (g_dev[d].dir == nullptr) continue;
+        cudaSetDevice(d);
+        cudaDeviceSynchronize();
+        cudaFree(g_dev[d].dir);
+        cudaFree(g_dev[d].s);
+        cudaFree(g_dev[d].on);
+        g_dev[d] = DevTables{};
     }
-    g_cvec.dir = d_dir;
-    g_cvec.s = d_s;
-    g_cvec.on = d_on;
+    cudaSetDevice(prev);
+    g_dir_host = dir;
+    g_s_host = s;
+    g_on_host = true;
+    if (!upload_here(err)) return false;
+    const DevTables& t = g_dev[cur_device()];
+    g_cvec.dir = t.dir;
+    g_cvec.s = t.s;
+    g_cvec.on = t.on;
     g_cvec.mode = mode;
     g_cvec.first = first;
     g_cvec.last = last;
@@ -100,26 +129,35 @@ bool cvec_upload(const std::vector<float>& dir, const std::vector<float>& s, int
     g_cvec.hc = hc;
     g_cvec.steered.assign(s.size(), false);
     for (size_t l = 0; l < s.size(); ++l) g_cvec.steered[l] = s[l] != 0.0f;
-    g_on = d_on;
-    g_on_host = true;
     return true;
 }
 
+bool cvec_replicate(std::string& err) { return !g_cvec.loaded() || upload_here(err); }
+
 void cvec_set_enabled(bool on) {
-    if (g_on == nullptr || on == g_on_host) return;
-    cudaDeviceSynchronize();   // nothing in flight may still read the flag
+    if (!g_cvec.loaded() || on == g_on_host) return;
+    int prev = 0;
+    cudaGetDevice(&prev);
     const int v = on ? 1 : 0;
-    cudaMemcpy(g_on, &v, sizeof(int), cudaMemcpyHostToDevice);
+    for (int d = 0; d < kDevices; ++d) {
+        if (g_dev[d].on == nullptr) continue;
+        cudaSetDevice(d);
+        cudaDeviceSynchronize();   // nothing in flight may still read the flag
+        cudaMemcpy(g_dev[d].on, &v, sizeof(int), cudaMemcpyHostToDevice);
+    }
+    cudaSetDevice(prev);
     g_on_host = on;
 }
 
-bool cvec_enabled() { return g_on != nullptr && g_on_host; }
+bool cvec_enabled() { return g_cvec.loaded() && g_on_host; }
 
 void cvec_apply(float* R, int64_t layer, int64_t T, int64_t r_ld, const float* bo, int64_t bo_ld, const float* inj,
                 int64_t inj_ld, bool write, void* stream) {
     if (!g_cvec.loaded() || T < 1) return;
+    const DevTables& t = g_dev[cur_device()];
+    if (t.dir == nullptr) throw std::runtime_error("cvec_apply: the control vector is not on this device (cvec_replicate)");
     const dim3 grid((unsigned) g_cvec.hc, (unsigned) T);
-    cvec_kernel<<<grid, THREADS, 0, (cudaStream_t) stream>>>(R, g_cvec.dir, g_cvec.s, g_cvec.on, g_cvec.mode, layer,
+    cvec_kernel<<<grid, THREADS, 0, (cudaStream_t) stream>>>(R, t.dir, t.s, t.on, g_cvec.mode, layer,
                                                             (int) g_cvec.n_embd, (int) g_cvec.hc, r_ld, bo, bo_ld,
                                                             inj, inj_ld, write ? 1 : 0);
     if (cudaPeekAtLastError() != cudaSuccess) throw std::runtime_error("cvec_apply: launch failed");

src/kernels/cuda/elementwise.cu

@@ -225,6 +225,37 @@ __global__ void copy_from_mapped_kernel(float4* __restrict__ dst, const volatile
     }
 }
 
+// the CPU rows of a verify window, skipping the rows the GPU plan computes itself (the
+// pool writes +0.0 into those, so this writes +0.0 too): block = row, the plan's hit rows `dst[0, *count)`.
+__global__ void copy_rows_from_mapped_kernel(float4* __restrict__ dst, const volatile float4* src, int64_t row4,
+                                             const int32_t* __restrict__ hit_rows, const int32_t* __restrict__ count) {
+    const int row = blockIdx.x;
+    __shared__ int hit;
+    if (threadIdx.x == 0) {
+        int h = 0;
+        const int c = *count;
+        for (int i = 0; i < c; ++i) h |= hit_rows[i] == row;
+        hit = h;
+    }
+    __syncthreads();
+    float4* d = dst + (int64_t) row * row4;
+    if (hit) {
+        for (int64_t i = threadIdx.x; i < row4; i += blockDim.x) d[i] = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+    } else {
+        const volatile float4* sr = src + (int64_t) row * row4;
+        for (int64_t i = threadIdx.x; i < row4; i += blockDim.x) d[i] = const_cast<const float4*>(sr)[i];
+    }
+}
+void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
+                           const int32_t* count, void* stream) {
+    if (rows <= 0) return;
+    if ((width & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {
+        std::fprintf(stderr, "copy_rows_from_mapped: width must be a multiple of 4 and both pointers 16-byte aligned\n");
+        std::exit(1);
+    }
+    copy_rows_from_mapped_kernel<<<(unsigned) rows, 128, 0, (cudaStream_t) stream>>>((float4*) dst, (const volatile float4*) src,
+                                                                                  width / 4, hit_rows, count);
+}
 void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream) {
     if (n <= 0) return;
     if ((n & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {

src/kernels/cuda/fused_gr.cu

@@ -1,6 +1,7 @@
 // src/kernels/cuda/fused_gr.cu - see include/strata/kernels/fused_gr.hpp.
 #include "strata/kernels/fused_gr.hpp"
 #include "strata/kernels/bf16_bits.hpp"
+#include "strata/kernels/verify_kernels.hpp"
 
 #include <cuda_runtime.h>
 
@@ -178,7 +179,11 @@ __global__ void __launch_bounds__(THREADS) gr_norm_multi_kernel(GrMulti m) {
     for (int i = t; i < D; i += THREADS) xn[i] *= s_rs[i / N];
 }
 
+#if defined(__HIPCC__)
+constexpr int TILE = 1280;             // eight-token tile fits gfx1100's 64 KiB LDS limit
+#else
 constexpr int TILE = 2560;             // xn floats per token staged at a time: 320 chunks of 8, 10 per lane
+#endif
 constexpr int TQ = TILE / 8 / 32;      // uint4 weight chunks per lane per tile
 
 // Step 2 of `gr_down_kernel` for T tokens.  One warp per row (so each lane accumulates the same chunks in the
@@ -295,7 +300,8 @@ __global__ void __launch_bounds__(THREADS) gr_up_multi_kernel(GrMulti m) {
 
 }  // namespace
 
-void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream) {
+void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream, unsigned long long* stamp_buf,
+                         int stamp_i0) {
     if (n_tok < 1 || n_tok > kFusedGrMaxT || xn_scratch == nullptr) {
         std::fprintf(stderr, "fused_gr_read_multi: invalid arguments\n");
         std::exit(1);
@@ -315,22 +321,45 @@ void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi
     m.T = n_tok;
     cudaStream_t st = (cudaStream_t) stream;
     gr_norm_multi_kernel<<<n_tok, THREADS, 0, st>>>(m);
+    if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0, stream);
     // the shared-memory opt-in is a per-DEVICE setting: once per device, not once per process (a layer split
     // runs this kernel on two cards)
     static bool attr[64] = {};
+    static int chunk[64] = {};   // Turing port: tokens the down kernel may carry in one launch on this card
     int dev = 0;
     cudaGetDevice(&dev);
-    if (dev < 0 || dev >= 64 || !attr[dev]) {
+    if (dev >= 0 && dev < 64 && !attr[dev]) {
         // at most what the card allows (Turing: 64 KB - enough for windows of up to 6 tokens)
         int optin = 0;
         cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
         int want = (int) (kFusedGrMaxT * TILE * sizeof(float));
         if (optin > 0 && want > optin) want = optin;
         cudaFuncSetAttribute(gr_down_multi_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, want);
-        cudaGetLastError();
-        if (dev >= 0 && dev < 64) attr[dev] = true;
+        cudaGetLastError();      // drop any error the attempt left behind
+        // Turing port: the down kernel stages n_tok*TILE floats of dynamic shared memory - 80 KB at the full
+        // 8 tokens.  A card whose opt-in is below that (Turing: 64 KB, so 7+ tokens fail to launch as
+        // "invalid argument") processes the tokens in slices that fit; a card that reports no opt-in gets what
+        // fits the 48 KB default (4 tokens of the CUDA tile; all 8 of HIP's smaller tile).  The down kernel's
+        // outputs (lo, inject_out) are strictly per-token, so the chunk boundaries are safe, and the up kernel
+        // below still sees every token of the batch in one launch.
+        const int capacity = (optin > 0 ? optin : 48 * 1024) / (int) (TILE * sizeof(float));
+        chunk[dev] = capacity < 1 ? 1 : (capacity > kFusedGrMaxT ? kFusedGrMaxT : capacity);
+        attr[dev] = true;
     }
-    gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) n_tok * TILE * sizeof(float), st>>>(m);
+    const int chunk_tok = (dev >= 0 && dev < 64 && chunk[dev]) ? chunk[dev] : kFusedGrMaxT;
+    if (chunk_tok >= n_tok) {
+        gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) n_tok * TILE * sizeof(float), st>>>(m);
+    } else {
+        for (int c0 = 0; c0 < n_tok; c0 += chunk_tok) {
+            const int ct = n_tok - c0 < chunk_tok ? n_tok - c0 : chunk_tok;
+            GrMulti c{};
+            c.xn = xn_scratch + (size_t) c0 * D;
+            c.T = ct;
+            for (int k = 0; k < ct; ++k) c.a[k] = a[c0 + k];
+            gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) ct * TILE * sizeof(float), st>>>(c);
+        }
+    }
+    if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0 + 1, stream);
     gr_up_multi_kernel<<<UPM_BLOCKS, THREADS, 0, st>>>(m);
     const cudaError_t e = cudaGetLastError();
     if (e != cudaSuccess) {

src/kernels/cuda/native_bf16.cu

@@ -70,6 +70,54 @@ __global__ void bf16_f32_mmvf_kernel(const float* __restrict__ x, const uint16_t
     if (t == 0) y[blockIdx.x] = acc;
 }
 
+// the same kernel for up to 8 activation rows - the weight row is read ONCE and every
+// token keeps its own accumulator with exactly the single-row kernel's order (pairs, two ordered FMAs, the same warp
+// and block reductions), so each output is bit-identical to a bf16_f32_mmvf_kernel launch of its own.
+template <int BLOCK_SIZE, int NT>
+__global__ void bf16_f32_mmvf_multi_kernel(const float* __restrict__ x, int64_t ldx, const uint16_t* __restrict__ w,
+                                          float* __restrict__ y, int64_t ldy, int n_in, int n_tok) {
+    const int t = threadIdx.x;
+    const uint16_t* row = w + (size_t) blockIdx.x * n_in;
+    const uint32_t* weights2 = reinterpret_cast<const uint32_t*>(row);
+    __shared__ float partials[NT][32];
+    if constexpr (BLOCK_SIZE > 32) {
+        if (t < 32)
+#pragma unroll
+            for (int k = 0; k < NT; ++k) partials[k][t] = 0.0f;
+        __syncthreads();
+    }
+    float acc[NT];
+#pragma unroll
+    for (int k = 0; k < NT; ++k) acc[k] = 0.0f;
+    for (int pair = t; pair < n_in / 2; pair += BLOCK_SIZE) {
+        const uint32_t weight = weights2[pair];
+        const float w0 = f32_from_bf16((uint16_t) weight), w1 = f32_from_bf16((uint16_t) (weight >> 16));
+#pragma unroll
+        for (int k = 0; k < NT; ++k) {
+            if (k < n_tok) {
+                const float2 input = reinterpret_cast<const float2*>(x + (size_t) k * ldx)[pair];
+                acc[k] = __fmaf_rn(w0, input.x, acc[k]);
+                acc[k] = __fmaf_rn(w1, input.y, acc[k]);
+            }
+        }
+    }
+#pragma unroll
+    for (int k = 0; k < NT; ++k) acc[k] = mmvf_warp_sum(acc[k]);
+    if constexpr (BLOCK_SIZE > 32) {
+        if ((t & 31) == 0)
+#pragma unroll
+            for (int k = 0; k < NT; ++k) partials[k][t / 32] = acc[k];
+        __syncthreads();
+        if (t < 32)
+#pragma unroll
+            for (int k = 0; k < NT; ++k) acc[k] = mmvf_warp_sum(partials[k][t]);
+    }
+    if (t == 0)
+#pragma unroll
+        for (int k = 0; k < NT; ++k)
+            if (k < n_tok) y[(size_t) k * ldy + blockIdx.x] = acc[k];
+}
+
 int mmvf_block_size(int64_t n_in) {
     int best = 32;
     int64_t best_iterations = (n_in + 63) / 64;
@@ -85,6 +133,26 @@ int mmvf_block_size(int64_t n_in) {
 
 }  // namespace
 
+void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, float* y, int64_t ldy,
+                               int64_t n_in, int64_t n_out, int n_tok, void* stream) {
+    if (n_tok == 1 && ldy >= n_out) { bf16_gemv_fp32_mmvf(x, w, y, n_in, n_out, stream); return; }
+    if (n_tok < 1 || n_tok > 8 || n_in <= 0 || (n_in & 1) != 0 || n_out <= 0 || (ldx & 1) != 0 || x == nullptr ||
+        w == nullptr || y == nullptr || (reinterpret_cast<uintptr_t>(x) & 7u) != 0)
+        throw std::invalid_argument("bf16_gemv_fp32_mmvf_multi: 1..8 rows, even n_in/ldx, aligned pointers");
+    const cudaStream_t st = (cudaStream_t) stream;
+#define STRATA_MMVF_M(N) case N: \
+    if (n_tok <= 4) bf16_f32_mmvf_multi_kernel<N, 4><<<(unsigned) n_out, N, 0, st>>>(x, ldx, w, y, ldy, (int) n_in, n_tok); \
+    else bf16_f32_mmvf_multi_kernel<N, 8><<<(unsigned) n_out, N, 0, st>>>(x, ldx, w, y, ldy, (int) n_in, n_tok); break
+    switch (mmvf_block_size(n_in)) {
+        STRATA_MMVF_M(32); STRATA_MMVF_M(64); STRATA_MMVF_M(96); STRATA_MMVF_M(128);
+        STRATA_MMVF_M(160); STRATA_MMVF_M(192); STRATA_MMVF_M(224); STRATA_MMVF_M(256);
+    }
+#undef STRATA_MMVF_M
+    const cudaError_t result = cudaGetLastError();
+    if (result != cudaSuccess)
+        throw std::runtime_error(std::string("bf16_gemv_fp32_mmvf_multi launch: ") + cudaGetErrorString(result));
+}
+
 void bf16_gemv_fp32_mmvf(const float* x, const uint16_t* w, float* y,
                          int64_t n_in, int64_t n_out, void* stream) {
     if (n_in <= 0 || (n_in & 1) != 0 || n_in > std::numeric_limits<int>::max() ||

src/kernels/cuda/native_moe.cu

@@ -34,6 +34,9 @@ std::atomic<bool> enabled{false};
 __global__ void combine(const float* __restrict__ parts, const float* __restrict__ weights,
                         const float* __restrict__ shared, float* __restrict__ output,
                         int64_t n_embd, int k) {
+    // blockIdx.y = the token of a multi-token launch (0 for the single one)
+    const int64_t tk = blockIdx.y;
+    parts += tk * k * n_embd; weights += tk * k; if (shared) shared += tk * n_embd; output += tk * n_embd;
     const int64_t col = int64_t(blockIdx.x) * blockDim.x + threadIdx.x;
     if (col >= n_embd) return;
     float sum = parts[col] * weights[0];
@@ -71,4 +74,13 @@ void native_moe_combine(const float* parts, const float* weights, const float* s
     const auto error = cudaGetLastError();
     if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
 }
+void native_moe_combine_multi(const float* parts, const float* weights, const float* shared, float* output,
+                              int64_t n_embd, int64_t k, int n_tok, void* stream) {
+    if (!stream || n_embd <= 0 || k < 1 || k > 15 || n_tok < 1)
+        throw std::invalid_argument("native MoE combine (multi) requires a stream, width, 1..15 experts, tokens");
+    combine<<<dim3(unsigned((n_embd + 255) / 256), (unsigned) n_tok), 256, 0, static_cast<cudaStream_t>(stream)>>>(
+        parts, weights, shared, output, n_embd, int(k));
+    const auto error = cudaGetLastError();
+    if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
+}
 }

src/kernels/cuda/native_qsa_indexer.cu

@@ -93,6 +93,97 @@ __global__ void append(const float* __restrict__ raw, const int32_t* __restrict_
     else pooled[std::size_t(b + 1) * D + d] = dead[d];
     if (d == 0 && pos != 0) *block_pos = rope_pos;
 }
+// ---- C-2: the batched append.  The pooled key of a completed block b (its last cell pos = 4b+3), computed with
+// the single append's arithmetic: keys rounded through F16, summed tail[0]+tail[1]+tail[2]+incoming in that order,
+// the RMS norm, gamma, the rotation at pos_base + 4b.  `k(j)` is the key of the block's cell j: a row of this batch
+// (cell >= p0) or the tail the previous batch left (cell < p0).  Only the batch's last completed block writes the
+// spare after it (every earlier one's is overwritten by the next block in order).
+__device__ __forceinline__ float pooled_value(const float* values, int d, int rope_pos, float theta_scale,
+                                              const int32_t* mtab, bool zero_pos) {
+    float y = values[d];
+    if (d < ROT) {
+        const int pair = d % (ROT / 2);
+        const float theta = (zero_pos ? 0 : mrope_pos(mtab, rope_pos, pair)) * powf(theta_scale, float(pair));
+        const float c = cosf(theta), s = sinf(theta);
+        const float a = values[pair], z = values[pair + ROT / 2];
+        y = d < ROT / 2 ? a * c - z * s : a * s + z * c;
+    }
+    return y;
+}
+__device__ __forceinline__ float norm_scale(float mean, float* partials, int d) {
+    float square_sum = d < D ? mean * mean : 0.0f;
+    square_sum = warp_sum(square_sum);
+    const int lane = d % 32;
+    if (lane == 0) partials[d / 32] = square_sum;
+    __syncthreads();
+    square_sum = lane < THREADS / 32 ? partials[lane] : 0.0f;
+    square_sum = warp_sum(square_sum);
+    return square_sum;
+}
+// cell 0 of a sequence: the spare (every gather index names cell 0), written to pooled[0] and dead
+__global__ void append_first(const float* __restrict__ raw, const float* __restrict__ gamma, float epsilon,
+                             float* __restrict__ dead, float* __restrict__ pooled, float theta_scale,
+                             const int32_t* __restrict__ mtab) {
+    const int d = threadIdx.x;
+    __shared__ float values[D];
+    __shared__ float partials[32];
+    float mean = 0.0f;
+    if (d < D) {
+        const float incoming = __half2float(__float2half_rn(raw[d]));
+        float sum = incoming;
+#pragma unroll
+        for (int j = 1; j < R; ++j) sum = __fadd_rn(sum, incoming);
+        mean = __fmaf_rn(0.25f, sum, 0.0f);
+    }
+    const float square_sum = norm_scale(mean, partials, d);
+    const float scale = rsqrtf(square_sum / D + epsilon);
+    if (d < D) values[d] = scale * mean * gamma[d];
+    __syncthreads();
+    if (d >= D) return;
+    const float y = pooled_value(values, d, 0, theta_scale, mtab, true);
+    pooled[d] = y;
+    dead[d] = y;
+}
+__global__ void append_blocks(const float* __restrict__ raw, int64_t n, int64_t p0, int pos_base,
+                              const float* __restrict__ gamma, float epsilon, const float* __restrict__ tail,
+                              const float* __restrict__ dead, float* __restrict__ pooled, int32_t* __restrict__ block_pos,
+                              int64_t first_block, int64_t last_block, float theta_scale, const int32_t* __restrict__ mtab) {
+    const int64_t b = first_block + blockIdx.x;
+    const int d = threadIdx.x;
+    __shared__ float values[D];
+    __shared__ float partials[32];
+    float mean = 0.0f;
+    if (d < D) {
+        auto key = [&](int j) -> float {
+            const int64_t cell = b * R + j;
+            return cell >= p0 ? __half2float(__float2half_rn(raw[(cell - p0) * D + d])) : tail[j * D + d];
+        };
+        float sum = key(0);
+#pragma unroll
+        for (int j = 1; j < R; ++j) sum = __fadd_rn(sum, key(j));
+        mean = __fmaf_rn(0.25f, sum, 0.0f);
+    }
+    const float square_sum = norm_scale(mean, partials, d);
+    const float scale = rsqrtf(square_sum / D + epsilon);
+    if (d < D) values[d] = scale * mean * gamma[d];
+    __syncthreads();
+    if (d >= D) return;
+    const int rope_pos = pos_base + R * (int) b;
+    pooled[std::size_t(b) * D + d] = pooled_value(values, d, rope_pos, theta_scale, mtab, false);
+    if (b == last_block) {
+        pooled[std::size_t(b + 1) * D + d] = dead[d];
+        if (d == 0) *block_pos = rope_pos;
+    }
+}
+// the tail after the batch: slot s holds the key of the batch's last cell with cell % 4 == s (s < 3), if any
+__global__ void append_tail(const float* __restrict__ raw, int64_t n, int64_t p0, float* __restrict__ tail) {
+    const int s = blockIdx.x, d = threadIdx.x;
+    if (d >= D) return;
+    const int64_t last = p0 + n - 1;
+    int64_t cell = last - ((last % R) - s + R) % R;       // the last cell <= last with cell % 4 == s
+    if (cell < p0) return;
+    tail[s * D + d] = __half2float(__float2half_rn(raw[(cell - p0) * D + d]));
+}
 struct Span { const void* p; std::size_t n; };
 void validate(Span s) {
     const auto p = reinterpret_cast<std::uintptr_t>(s.p);
@@ -126,4 +217,27 @@ void native_qsa_indexer_append(const float* raw, const int32_t* relative_pos_dev
     const auto error = cudaGetLastError();
     if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
 }
+void native_qsa_indexer_append_batch(const float* raw, int64_t n, int64_t p0, int32_t pos_base, const float* gamma,
+                                     float epsilon, const QsaIndexerBuffers& b, const QsaShapes& s, int64_t max_cells,
+                                     float freq_base, void* stream) {
+    if (n <= 0) return;
+    if (!stream || s.idx_dim != D || s.idx_block != R || s.n_rot != ROT || p0 < 0 || p0 + n > max_cells ||
+        max_cells > INT32_MAX || pos_base < 0 || pos_base % R || int64_t(pos_base) + max_cells > INT32_MAX ||
+        !std::isfinite(epsilon) || epsilon <= 0.0f || !std::isfinite(freq_base) || freq_base <= 1.0f)
+        throw std::invalid_argument("native QSA indexer (batch): bad geometry, positions or parameters");
+    const float theta_scale = powf(freq_base, -2.0f / ROT);
+    const cudaStream_t st = static_cast<cudaStream_t>(stream);
+    const int32_t* mtab = mrope_table();
+    if (p0 == 0) append_first<<<1, THREADS, 0, st>>>(raw, gamma, epsilon, b.dead, b.pooled, theta_scale, mtab);
+    // completed blocks: those whose last cell (4b+3) lies in [p0, p0 + n)
+    const int64_t first = p0 <= R - 1 ? 0 : (p0 - (R - 1) + R - 1) / R;       // the smallest b with 4b+3 >= p0
+    const int64_t hi = p0 + n - 1 >= R - 1 ? (p0 + n - 1 - (R - 1)) / R : -1;   // the largest b with 4b+3 <= p0+n-1
+    if (hi >= first)
+        append_blocks<<<(unsigned) (hi - first + 1), THREADS, 0, st>>>(raw, n, p0, pos_base, gamma, epsilon, b.tail,
+                                                                       b.dead, b.pooled, b.block_pos, first, hi,
+                                                                       theta_scale, mtab);
+    append_tail<<<R - 1, D, 0, st>>>(raw, n, p0, b.tail);
+    const auto error = cudaGetLastError();
+    if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
+}
 } // namespace strata::kernels

src/kernels/cuda/native_qsa_score.cu

@@ -32,6 +32,7 @@ constexpr int D=128, HEADS=4, R=4, ROWS=32, WARPS=2, STRIDE=36, COMBINE=68;
 struct TileA { uint32_t x[4]; };
 struct TileB { uint32_t x[2]; };
 struct TileC { float x[4]={0.0f,0.0f,0.0f,0.0f}; };
+#if !defined(__HIPCC__)
 #if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
 __device__ __forceinline__ void load_a(TileA& a,const float* p) {
     const float* src=p+(threadIdx.x%16)*STRIDE+(threadIdx.x/16)*4;
@@ -139,6 +140,42 @@ __global__ __launch_bounds__(64,1) void score_kernel(
     }
 #endif
 }
+#else
+// gfx1100 has no CUDA ldmatrix/mma instruction sequence. Keep the same entry point and
+// score contract with an ordered scalar F32 dot for each indexer head. The four heads
+// run independently; their ReLU'd scores are then added in the documented head order.
+// This path favors a well-defined fallback over pretending the CUDA PTX is portable.
+__global__ void scalar_score_kernel(
+        const float* __restrict__ pooled,const float* __restrict__ query,
+        const float* __restrict__ bias,const int32_t* __restrict__ step,
+        int max_cells,float* __restrict__ cells) {
+    const int n=step[kStepNKv],full=step[kStepNBid];
+    if(n<1||n>max_cells||step[kStepPos]!=n-1||full!=n/R||
+       step[kStepWidth]!=(n<2051?n:2051))return;
+    const int row=blockIdx.x;
+    if(row>full)return;
+    __shared__ float head_score[HEADS];
+    const int head=threadIdx.x;
+    if(head<HEADS){
+        float dot=0.0f;
+#pragma unroll
+        for(int d=0;d<D;++d)
+            dot=__fmaf_rn(pooled[size_t(row)*D+d],query[size_t(head)*D+d],dot);
+        head_score[head]=dot>0.0f?dot:0.0f;
+    }
+    __syncthreads();
+    if(head==0){
+        float sum=__fadd_rn(0.0f,head_score[0]);
+        sum=__fadd_rn(sum,head_score[1]);
+        sum=__fadd_rn(sum,head_score[2]);
+        sum=__fadd_rn(sum,head_score[3]);
+        if(bias)sum=__fadd_rn(sum,bias[row]);
+        sum=__fadd_rn(sum,row==full&&n%R?1e9f:0.0f);
+        sum=__fadd_rn(sum,0.0f);
+        for(int i=row*R;i<n&&i<(row+1)*R;++i)cells[i]=sum;
+    }
+}
+#endif
 struct Span{const void* p;size_t n;};
 void validate(Span s){
     const auto p=reinterpret_cast<uintptr_t>(s.p);
@@ -163,8 +200,13 @@ void native_qsa_score(const float* pooled,const float* query,const float* bias,
     for(int i=0;i<count;++i)validate(spans[i]);
     for(int i=0;i<count;++i)for(int j=i+1;j<count;++j)
         if(overlaps(spans[i],spans[j]))throw std::invalid_argument("native QSA score spans overlap");
+#if defined(__HIPCC__)
+    scalar_score_kernel<<<unsigned(max_blocks),128,0,static_cast<cudaStream_t>(stream)>>>(
+        pooled,query,bias,step,int(max_cells),cells);
+#else
     score_kernel<<<unsigned((max_blocks+ROWS-1)/ROWS),dim3(32,WARPS),0,static_cast<cudaStream_t>(stream)>>>(
         pooled,query,bias,step,int(max_cells),cells);
+#endif
     const auto error=cudaGetLastError();
     if(error!=cudaSuccess)throw std::runtime_error(cudaGetErrorString(error));
 }

src/kernels/cuda/native_rope.cu

@@ -58,9 +58,18 @@ __global__ void apply(const float* x, float* out, int rows, int width,
     out[start + pair + n_rot / 2] = a * s + b * c;
 }
 }
-namespace { std::atomic<const int32_t*> mrope_tab{nullptr}; }
-void mrope_table_set(const int32_t* device_table) { mrope_tab.store(device_table, std::memory_order_relaxed); }
-const int32_t* mrope_table() { return mrope_tab.load(std::memory_order_relaxed); }
+// one table per device (a layer split runs the rope kernels on several): set and read for the current device
+namespace {
+constexpr int kMropeDevices = 64;
+std::atomic<const int32_t*> mrope_tab[kMropeDevices] = {};
+int mrope_dev() {
+    int d = 0;
+    if (cudaGetDevice(&d) != cudaSuccess || d < 0 || d >= kMropeDevices) d = 0;
+    return d;
+}
+}  // namespace
+void mrope_table_set(const int32_t* device_table) { mrope_tab[mrope_dev()].store(device_table, std::memory_order_relaxed); }
+const int32_t* mrope_table() { return mrope_tab[mrope_dev()].load(std::memory_order_relaxed); }
 void native_rope_set_enabled(bool value) { enabled.store(value, std::memory_order_relaxed); }
 bool native_rope_enabled() { return enabled.load(std::memory_order_relaxed); }
 void native_rope_apply(const float* x, float* out, int rows, int head_dim,

src/kernels/cuda/native_router.cu

@@ -45,6 +45,8 @@ __launch_bounds__(256, 1)
 __global__ void route(const float* __restrict__ logits, int32_t* __restrict__ ids,
                       float* __restrict__ weights) {
     // Preserve the pinned 32x8 block geometry; only row zero is active here.
+    // blockIdx.x = the token (a multi-token launch; 0 for the single one)
+    logits += (size_t) blockIdx.x * 512; ids += (size_t) blockIdx.x * 10; weights += (size_t) blockIdx.x * 10;
     if (threadIdx.y != 0) return;
     const int lane = threadIdx.x;
     float values[16];
@@ -114,4 +116,12 @@ void native_router_top10(const float* logits, int32_t* ids, float* weights, void
     const auto error = cudaGetLastError();
     if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
 }
+void native_router_top10_multi(const float* logits, int32_t* ids, float* weights, int n_tok, void* stream) {
+    if (!stream || n_tok < 1 || !valid(logits, (size_t) n_tok * 512 * 4) || !valid(ids, (size_t) n_tok * 10 * 4) ||
+        !valid(weights, (size_t) n_tok * 10 * 4))
+        throw std::invalid_argument("native router (multi) requires a stream and aligned [n,512]/[n,10] buffers");
+    route<<<(unsigned) n_tok, dim3(32, 8), 0, static_cast<cudaStream_t>(stream)>>>(logits, ids, weights);
+    const auto error = cudaGetLastError();
+    if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
+}
 }

src/kernels/cuda/ple.cu

@@ -228,8 +228,9 @@ void ple_block(const float* emb, const float* hidden, const float* hist_rows, co
                PleOut& out, void* scratch, void* stream) {
     const bool native_key = w.key_native_data != nullptr && w.key_bf16 == nullptr;
     if (native_key && (!emb || !hidden || !hist_rows || !out.result || !scratch || !stream ||
-                       !w.key_native_q8_1 || w.key_native_type != 42))
-        throw std::invalid_argument("ple_block: native key requires Q2_0 weights, input/output, private scratch and explicit stream");
+                       !w.key_native_q8_1 ||
+                       (w.key_native_type != 42 && w.key_native_type != 18 && w.key_native_type != 23)))
+        throw std::invalid_argument("ple_block: native key requires Q2_0, IQ3_XXS or IQ4_XS weights, input/output, private scratch and explicit stream");
     if (emb == nullptr || hidden == nullptr || hist_rows == nullptr || out.result == nullptr) return;
     const int n_embd = NG_N_EMBD, hc = NG_HC, hc_dim = NG_HC_DIM;
     static_assert(NG_N_EMBD == 2560 && NG_HC_DIM == 10240, "native PLE key geometry changed");

src/kernels/cuda/qsa_decode_attn.cu

@@ -31,42 +31,53 @@ __device__ __forceinline__ float warp_max(float v) {
 }
 
 // 8 consecutive values of one cell's key or value row for KV head `kvh`, dimensions [d0, d0+8).
+// Per-format bodies; `load8` below is the KV_MODE dispatcher. value=false is the K side, true the V side.
+__device__ __forceinline__ void load8_f16(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
+    const uint16_t* base = (value ? p.v_pool : p.k_pool) + row * HD + d0;
+    const uint4 raw = *reinterpret_cast<const uint4*>(base);
+    const __half2* h2 = reinterpret_cast<const __half2*>(&raw);
+#pragma unroll
+    for (int j = 0; j < 4; ++j) {
+        const float2 f = __half22float2(h2[j]);
+        out[2 * j] = f.x;
+        out[2 * j + 1] = f.y;
+    }
+}
+__device__ __forceinline__ void load8_q8(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
+    const int8_t* codes = (value ? p.v_q : p.k_q) + row * HD + d0;
+    const uint16_t sbits = (value ? p.v_scale : p.k_scale)[row * (HD / KV_Q8_GROUP) + d0 / KV_Q8_GROUP];
+    const float sc = __half2float(__ushort_as_half(sbits));
+    const uint2 raw = *reinterpret_cast<const uint2*>(codes);
+    const int8_t* c = reinterpret_cast<const int8_t*>(&raw);
+#pragma unroll
+    for (int j = 0; j < 8; ++j) out[j] = (float) c[j] * sc;
+}
+__device__ __forceinline__ void load8_q4(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
+    constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
+    const int b = d0 / QK4_0;
+    const int rem = d0 % QK4_0;
+    const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((value ? p.v_q4 : p.k_q4) + row * bytes_per_head) + b;
+    const float d = __half2float(__ushort_as_half(blk->d));
+    const int j = (rem == 0 || rem == 16) ? 0 : 8;
+    const uint8_t* bytes = blk->qs + j;
+    if (rem < 16) {
+#pragma unroll
+        for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] & 0x0F) - 8) * d;
+    } else {
+#pragma unroll
+        for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] >> 4) - 8) * d;
+    }
+}
 template <int KV_MODE>
 __device__ __forceinline__ void load8(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
-    if constexpr (KV_MODE == 0) {
-        const uint16_t* base = (value ? p.v_pool : p.k_pool) + row * HD + d0;
-        const uint4 raw = *reinterpret_cast<const uint4*>(base);
-        const __half2* h2 = reinterpret_cast<const __half2*>(&raw);
-#pragma unroll
-        for (int j = 0; j < 4; ++j) {
-            const float2 f = __half22float2(h2[j]);
-            out[2 * j] = f.x;
-            out[2 * j + 1] = f.y;
-        }
-    } else if constexpr (KV_MODE == 1) {
-        const int8_t* codes = (value ? p.v_q : p.k_q) + row * HD + d0;
-        const uint16_t sbits = (value ? p.v_scale : p.k_scale)[row * (HD / KV_Q8_GROUP) + d0 / KV_Q8_GROUP];
-        const float sc = __half2float(__ushort_as_half(sbits));
-        const uint2 raw = *reinterpret_cast<const uint2*>(codes);
-        const int8_t* c = reinterpret_cast<const int8_t*>(&raw);
-#pragma unroll
-        for (int j = 0; j < 8; ++j) out[j] = (float) c[j] * sc;
-    } else {
-        constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
-        const int b = d0 / QK4_0;
-        const int rem = d0 % QK4_0;
-        const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((value ? p.v_q4 : p.k_q4) + row * bytes_per_head) + b;
-        const float d = __half2float(__ushort_as_half(blk->d));
-        const int j = (rem == 0 || rem == 16) ? 0 : 8;
-        const uint8_t* bytes = blk->qs + j;
-        if (rem < 16) {
-#pragma unroll
-            for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] & 0x0F) - 8) * d;
-        } else {
-#pragma unroll
-            for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] >> 4) - 8) * d;
-        }
-    }
+    if constexpr (KV_MODE == 0) load8_f16(p, value, row, d0, out);
+    else if constexpr (KV_MODE == 1) load8_q8(p, value, row, d0, out);
+    else if constexpr (KV_MODE == 3) {
+        // hybrid K8V4: both sides are defined - K unrotated INT8, V rotated Q4_0 - so a value=true call
+        // reads the Q4_0 pool instead of dereferencing the null v_q (no call site does today; PR review)
+        if (value) load8_q4(p, value, row, d0, out);
+        else load8_q8(p, value, row, d0, out);
+    } else load8_q4(p, value, row, d0, out);
 }
 
 template <int KV_MODE>
@@ -149,7 +160,7 @@ __global__ void __launch_bounds__(THREADS) attn_chunk_kernel(const float* __rest
         } else if constexpr (KV_MODE == 1) {
             const float sc = __half2float(__ushort_as_half(p.v_scale[srow[c] * (HD / KV_Q8_GROUP) + t / KV_Q8_GROUP]));
             v = (float) p.v_q[srow[c] * HD + t] * sc;
-        } else {
+        } else {   // modes 2 and 3: V is rotated Q4_0 (kv_q4.hpp); the caller rotates the output back
             constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
             const int b = t / QK4_0;
             const int rem = t % QK4_0;
@@ -202,7 +213,8 @@ void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int3
         std::fprintf(stderr, "qsa_decode_attn_batch: unsupported geometry or missing buffers\n");
         std::exit(1);
     }
-    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr ? 1 : 0);
+    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
+                        : (pools.k_q != nullptr ? 1 : 0));
     const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
     // per query: [acc: n_chunks*n_head*HD][m: n_chunks*n_head][l: n_chunks*n_head], all offsets from one stride
     const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);
@@ -212,7 +224,10 @@ void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int3
     const float scale = 1.0f / sqrtf((float) HD);
     const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);
     cudaStream_t st = (cudaStream_t) stream;
-    if (kv_mode == 2)
+    if (kv_mode == 3)
+        attn_chunk_kernel<3><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
+                                                        scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
+    else if (kv_mode == 2)
         attn_chunk_kernel<2><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
                                                         scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
     else if (kv_mode == 1)
@@ -242,8 +257,11 @@ void qsa_decode_attn_step(const float* q, const QsaAttnPools& pools, const int32
         std::fprintf(stderr, "qsa_decode_attn: unsupported geometry or missing buffers\n");
         std::exit(1);
     }
-    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr ? 1 : 0);
-    if (kv_mode == 2 ? (!pools.v_q4) : (kv_mode == 1 ? (!pools.v_q || !pools.k_scale || !pools.v_scale) : (!pools.k_pool || !pools.v_pool))) {
+    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
+                        : (pools.k_q != nullptr ? 1 : 0));
+    if (kv_mode == 3 ? (!pools.k_scale || !pools.v_q4)
+                     : (kv_mode == 2 ? (!pools.v_q4) : (kv_mode == 1 ? (!pools.v_q || !pools.k_scale || !pools.v_scale)
+                                                                     : (!pools.k_pool || !pools.v_pool)))) {
         std::fprintf(stderr, "qsa_decode_attn: incomplete KV pools\n");
         std::exit(1);
     }
@@ -254,7 +272,10 @@ void qsa_decode_attn_step(const float* q, const QsaAttnPools& pools, const int32
     const float scale = 1.0f / sqrtf((float) HD);
     const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv);
     cudaStream_t st = (cudaStream_t) stream;
-    if (kv_mode == 2)
+    if (kv_mode == 3)
+        attn_chunk_kernel<3><<<grid, THREADS, 0, st>>>(q, pools, ids, step, (int) s.n_head_kv, (int) s.page_size,
+                                                        scale, part_acc, part_m, part_l, n_chunks);
+    else if (kv_mode == 2)
         attn_chunk_kernel<2><<<grid, THREADS, 0, st>>>(q, pools, ids, step, (int) s.n_head_kv, (int) s.page_size,
                                                         scale, part_acc, part_m, part_l, n_chunks);
     else if (kv_mode == 1)

src/kernels/cuda/qsa_prompt_attn.cu

@@ -0,0 +1,717 @@
+// src/kernels/cuda/qsa_prompt_attn.cu - see include/strata/kernels/qsa_prompt_attn.hpp.
+#include "strata/kernels/qsa_prompt_attn.hpp"
+#include "strata/kernels/kv_q8.hpp"
+#include "strata/kernels/kv_q4.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+#include <cmath>
+
+#include <cstdio>
+#include <cstdlib>
+#include <type_traits>
+
+namespace strata::kernels {
+namespace {
+
+constexpr int HD = 256;           // head_dim
+constexpr int G = 12;             // query heads per KV head
+#ifndef D1_CH
+#define D1_CH 32
+#endif
+constexpr int CH = D1_CH;         // cells per chunk
+constexpr int THREADS = 128;      // 4 warps: scores by cell (8 each), p.v by dimension (64 each = one int8 scale group)
+constexpr int QS = HD + 8;        // q row stride in halves (bank-conflict-free fragment loads)
+
+// The MMA and cp.async below need sm_80. Builds for older cards (the experimental sm_75 one) compile them to a trap;
+// qsa_prompt_attn_batch refuses such a device at run time, so the old kernel runs there.
+#if defined(__HIPCC__)          // AMD: no mma.sync / cp.async; the host keeps the old kernel (below)
+#define STRATA_PA_SM80 0
+#elif !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
+#define STRATA_PA_SM80 1
+#else
+#define STRATA_PA_SM80 0
+#endif
+
+__device__ __forceinline__ void mma16816(float* c, const uint32_t* a, const uint32_t* b) {
+#if !STRATA_PA_SM80
+    __trap();
+#else
+    asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
+                 "{%0,%1,%2,%3};\n"
+                 : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
+                 : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
+#endif
+}
+
+// Two int8 codes (low byte first) as an exact half2: 1024 + (c + 128) built in the mantissa, minus 1152.
+__device__ __forceinline__ uint32_t i8x2_to_h2(uint32_t x) {
+    uint32_t y = ((x & 0xffu) | ((x & 0xff00u) << 8)) ^ 0x00800080u;
+    y |= 0x64006400u;
+    __half2 h = *reinterpret_cast<__half2*>(&y);
+    h = __hsub2(h, __halves2half2(__float2half(1152.f), __float2half(1152.f)));
+    return *reinterpret_cast<uint32_t*>(&h);
+}
+
+__device__ __forceinline__ uint32_t pack_h2(float lo_k, float hi_k) {   // element k in the low half
+    __half2 h = __floats2half2_rn(lo_k, hi_k);
+    return *reinterpret_cast<uint32_t*>(&h);
+}
+
+// KV_MODE 1: int8 codes + fp16 scale per 64 values. KV_MODE 0: fp16 values (scales 1).
+// KV_MODE 3 (hybrid K8V4): K as mode 1, V as mode 0 - the row's q4_0 blocks are dequantized to fp16 at
+// gather, so everything downstream of the load is the mode-0 V path; the caller un-rotates the output.
+template <int KV_MODE>
+struct Smem {
+    using KElem = typename std::conditional<KV_MODE == 0, __half, int8_t>::type;
+    using VElem = typename std::conditional<KV_MODE == 1, int8_t, __half>::type;
+    static constexpr int KROW = KV_MODE == 0 ? HD + 8 : HD + 16;   // elements; 16-byte aligned rows, banks spread
+    static constexpr int VROW = KV_MODE == 1 ? HD + 16 : HD + 8;
+    __half qh[16][QS];
+    __half ql[16][QS];
+    KElem k[CH][KROW];
+    VElem v[CH][VROW];
+    float ks[CH][4];
+    float vs[CH][4];
+    float s[16][CH + 1];
+    float qmax[THREADS / 32];
+    float alpha[16];
+    float lsum[16];
+    float mrow[16];
+    long long row[CH];
+};
+
+template <int KV_MODE>
+__global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __restrict__ q, QsaAttnPools p,
+                                                              const int32_t* __restrict__ ids,
+                                                              const int32_t* __restrict__ steps, int n_kv_heads,
+                                                              int page_size, float scale_log2, float* __restrict__ attn,
+                                                              int cap) {
+    extern __shared__ __align__(16) unsigned char smem_raw[];
+    Smem<KV_MODE>& S = *reinterpret_cast<Smem<KV_MODE>*>(smem_raw);
+    const int qi = blockIdx.x, kvh = blockIdx.y;
+    const int n_head = n_kv_heads * G;
+    q += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;
+    attn += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;
+    ids += (size_t) qi * cap;
+    const int n = __ldg(steps + (size_t) qi * kStepCount + kStepWidth);
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+    const int gid = lane >> 2, tig = lane & 3;
+
+    // q: 12 heads + 4 zero rows, scaled by a power of two that puts its largest value near 2^14 (exact, and the
+    // lo halves stay out of FP16's subnormal range), then split into hi + lo halves
+    float qm = 0.0f;
+    for (int i = t; i < G * HD; i += THREADS) qm = fmaxf(qm, fabsf(q[i]));
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) qm = fmaxf(qm, __shfl_xor_sync(0xffffffffu, qm, o));
+    if (lane == 0) S.qmax[warp] = qm;
+    __syncthreads();
+    qm = fmaxf(fmaxf(S.qmax[0], S.qmax[1]), fmaxf(S.qmax[2], S.qmax[3]));
+    int qe = 0;
+    if (qm > 0.0f) frexpf(qm, &qe);                 // qm < 2^qe
+    const float qup = ldexpf(1.0f, 14 - qe), qdown = ldexpf(scale_log2, qe - 14);
+    for (int i = t; i < 16 * HD; i += THREADS) {
+        const int h = i / HD, d = i % HD;
+        const float x = h < G ? q[(size_t) h * HD + d] * qup : 0.0f;
+        const __half hi = __float2half_rn(x);
+        S.qh[h][d] = hi;
+        S.ql[h][d] = __float2half_rn(x - __half2float(hi));
+    }
+    if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; }
+
+    float acc[8][4];
+#pragma unroll
+    for (int j = 0; j < 8; ++j) acc[j][0] = acc[j][1] = acc[j][2] = acc[j][3] = 0.0f;
+
+    for (int c0 = 0; c0 < n; c0 += CH) {
+        const int nh = min(CH, n - c0);
+        if (t < CH) {
+            long long r = -1;
+            if (t < nh) {
+                const int cell = ids[c0 + t];
+                const long long page = (long long) p.page_table[cell / page_size];
+                r = (page * n_kv_heads + kvh) * page_size + (cell % page_size);
+            }
+            S.row[t] = r;
+        }
+        __syncthreads();   // rows ready; the previous chunk's p.v is done with k, v, s
+        // gather the chunk's K and V rows (16-byte pieces; K8V4's V as q4_0 blocks dequantized to fp16)
+        // and their scales
+        {
+            constexpr int KPIECES = HD * (int) sizeof(typename Smem<KV_MODE>::KElem) / 16;   // per K row
+            for (int i = t; i < CH * KPIECES; i += THREADS) {
+                const int c = i / KPIECES, pc = i % KPIECES;
+                const long long r = S.row[c];
+                uint4 kx = make_uint4(0, 0, 0, 0);
+                if (r >= 0) {
+                    if constexpr (KV_MODE == 0)
+                        kx = __ldg(reinterpret_cast<const uint4*>(p.k_pool + r * HD) + pc);
+                    else   // modes 1 and 3: the K side is INT8
+                        kx = __ldg(reinterpret_cast<const uint4*>(p.k_q + r * HD) + pc);
+                }
+                *reinterpret_cast<uint4*>(reinterpret_cast<unsigned char*>(&S.k[c][0]) + pc * 16) = kx;
+            }
+            if constexpr (KV_MODE == 3) {   // V: dequantize the row's q4_0 blocks straight into the fp16 V row
+                constexpr int BLKS = HD / QK4_0;
+                constexpr int BYTES = BLKS * (int) sizeof(block_q4_0);
+                for (int i = t; i < CH * BLKS; i += THREADS) {
+                    const int c = i / BLKS, b = i % BLKS;
+                    const long long r = S.row[c];
+#pragma unroll
+                    for (int j = 0; j < QK4_0; ++j) S.v[c][b * QK4_0 + j] = __half(0);
+                    if (r >= 0) {
+                        const block_q4_0* blk = reinterpret_cast<const block_q4_0*>(p.v_q4 + r * BYTES) + b;
+                        const float d = __half2float(__ushort_as_half(blk->d));
+#pragma unroll
+                        for (int j = 0; j < QK4_0 / 2; ++j) {
+                            S.v[c][b * QK4_0 + j] = __float2half_rn((float) ((int)(blk->qs[j] & 0x0F) - 8) * d);
+                            S.v[c][b * QK4_0 + j + QK4_0 / 2] =
+                                __float2half_rn((float) ((int)(blk->qs[j] >> 4) - 8) * d);
+                        }
+                    }
+                }
+            } else {
+                constexpr int VPIECES = HD * (int) sizeof(typename Smem<KV_MODE>::VElem) / 16;   // per V row
+                for (int i = t; i < CH * VPIECES; i += THREADS) {
+                    const int c = i / VPIECES, pc = i % VPIECES;
+                    const long long r = S.row[c];
+                    uint4 vx = make_uint4(0, 0, 0, 0);
+                    if (r >= 0) {
+                        if constexpr (KV_MODE == 1)
+                            vx = __ldg(reinterpret_cast<const uint4*>(p.v_q + r * HD) + pc);
+                        else
+                            vx = __ldg(reinterpret_cast<const uint4*>(p.v_pool + r * HD) + pc);
+                    }
+                    *reinterpret_cast<uint4*>(reinterpret_cast<unsigned char*>(&S.v[c][0]) + pc * 16) = vx;
+                }
+            }
+            for (int i = t; i < CH * 4; i += THREADS) {
+                const int c = i / 4, g = i % 4;
+                const long long r = S.row[c];
+                float a = 0.0f, b = 0.0f;
+                if (r >= 0) {
+                    if constexpr (KV_MODE == 1) {
+                        a = __half2float(__ushort_as_half(p.k_scale[r * (HD / KV_Q8_GROUP) + g]));
+                        b = __half2float(__ushort_as_half(p.v_scale[r * (HD / KV_Q8_GROUP) + g]));
+                    } else if constexpr (KV_MODE == 3) {   // K as int8, V dequantized to fp16 (scale 1)
+                        a = __half2float(__ushort_as_half(p.k_scale[r * (HD / KV_Q8_GROUP) + g]));
+                        b = 1.0f;
+                    } else {
+                        a = b = 1.0f;
+                    }
+                }
+                S.ks[c][g] = a;
+                S.vs[c][g] = b;
+            }
+        }
+        __syncthreads();
+        // scores: warp w takes cells 8w..8w+7 (one n-tile) over all 256 dims, per 64-dim scale group
+#pragma unroll
+        for (int nt = 0; nt < CH / 32; ++nt) {
+            const int cb = (warp + 4 * nt) * 8;
+            float sc[4] = {0.f, 0.f, 0.f, 0.f};
+#pragma unroll
+            for (int g = 0; g < 4; ++g) {
+                float tg[4] = {0.f, 0.f, 0.f, 0.f};
+#pragma unroll
+                for (int kk = 0; kk < 4; ++kk) {
+                    const int k0 = (g * 4 + kk) * 16;
+                    uint32_t ah[4], al[4], b[2];
+                    ah[0] = *reinterpret_cast<const uint32_t*>(&S.qh[gid][k0 + 2 * tig]);
+                    ah[1] = *reinterpret_cast<const uint32_t*>(&S.qh[gid + 8][k0 + 2 * tig]);
+                    ah[2] = *reinterpret_cast<const uint32_t*>(&S.qh[gid][k0 + 2 * tig + 8]);
+                    ah[3] = *reinterpret_cast<const uint32_t*>(&S.qh[gid + 8][k0 + 2 * tig + 8]);
+                    al[0] = *reinterpret_cast<const uint32_t*>(&S.ql[gid][k0 + 2 * tig]);
+                    al[1] = *reinterpret_cast<const uint32_t*>(&S.ql[gid + 8][k0 + 2 * tig]);
+                    al[2] = *reinterpret_cast<const uint32_t*>(&S.ql[gid][k0 + 2 * tig + 8]);
+                    al[3] = *reinterpret_cast<const uint32_t*>(&S.ql[gid + 8][k0 + 2 * tig + 8]);
+                    if constexpr (KV_MODE != 0) {   // modes 1 and 3: the K side is INT8 codes
+                        b[0] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(&S.k[cb + gid][k0 + 2 * tig]));
+                        b[1] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(&S.k[cb + gid][k0 + 2 * tig + 8]));
+                    } else {
+                        b[0] = *reinterpret_cast<const uint32_t*>(&S.k[cb + gid][k0 + 2 * tig]);
+                        b[1] = *reinterpret_cast<const uint32_t*>(&S.k[cb + gid][k0 + 2 * tig + 8]);
+                    }
+                    mma16816(tg, ah, b);
+#ifndef D1_NO_QLO
+                    mma16816(tg, al, b);
+#endif
+                }
+                const float s0 = S.ks[cb + 2 * tig][g], s1 = S.ks[cb + 2 * tig + 1][g];
+                sc[0] = fmaf(tg[0], s0, sc[0]);
+                sc[1] = fmaf(tg[1], s1, sc[1]);
+                sc[2] = fmaf(tg[2], s0, sc[2]);
+                sc[3] = fmaf(tg[3], s1, sc[3]);
+            }
+            const int c = cb + 2 * tig;
+            S.s[gid][c] = c < nh ? sc[0] * qdown : -INFINITY;
+            S.s[gid][c + 1] = c + 1 < nh ? sc[1] * qdown : -INFINITY;
+            S.s[gid + 8][c] = c < nh ? sc[2] * qdown : -INFINITY;
+            S.s[gid + 8][c + 1] = c + 1 < nh ? sc[3] * qdown : -INFINITY;
+        }
+        __syncthreads();
+        // online softmax: row t/8, 4 cells per thread, 8 threads per row (lanes 8r..8r+7 of a warp)
+        {
+            constexpr int PER = CH / 8;
+            const int r = t >> 3, sub = t & 7;
+            float x[PER], mx = -INFINITY;
+#pragma unroll
+            for (int j = 0; j < PER; ++j) { x[j] = S.s[r][sub * PER + j]; mx = fmaxf(mx, x[j]); }
+#pragma unroll
+            for (int o = 1; o < 8; o <<= 1) mx = fmaxf(mx, __shfl_xor_sync(0xffffffffu, mx, o));
+            const float m_old = S.mrow[r];
+            const float m_new = fmaxf(m_old, mx);
+            float sum = 0.0f;
+#pragma unroll
+            for (int j = 0; j < PER; ++j) {
+                const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);
+                S.s[r][sub * PER + j] = e;
+                sum += e;
+            }
+#pragma unroll
+            for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
+            __syncwarp();
+            if (sub == 0) {
+                const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);
+                S.alpha[r] = a;
+                S.lsum[r] = fmaf(S.lsum[r], a, sum);
+                S.mrow[r] = m_new;
+            }
+        }
+        __syncthreads();
+        // p.v: warp w owns dims [64w, 64w+64), which is int8 scale group w. The scale is folded into p relative to
+        // the chunk's largest, times 2^14 (p' <= 2^14: its lo half stays out of FP16's subnormal range); the chunk's
+        // sum is then added to the running one in FP32 with the factor taken back out
+        {
+            float vmax = 0.0f;
+#pragma unroll
+            for (int c = lane; c < CH; c += 32) vmax = fmaxf(vmax, S.vs[c][warp]);
+#pragma unroll
+            for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));
+            const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f, vdown = vmax * (1.0f / 16384.0f);
+            float tmp[8][4];
+#pragma unroll
+            for (int j = 0; j < 8; ++j) tmp[j][0] = tmp[j][1] = tmp[j][2] = tmp[j][3] = 0.0f;
+#pragma unroll
+            for (int ks = 0; ks < CH / 16; ++ks) {
+                const int cA = ks * 16 + 2 * tig, cB = cA + 8;
+                const float w0 = S.vs[cA][warp] * vup, w1 = S.vs[cA + 1][warp] * vup, w2 = S.vs[cB][warp] * vup,
+                            w3 = S.vs[cB + 1][warp] * vup;
+                const float p00 = S.s[gid][cA] * w0, p01 = S.s[gid][cA + 1] * w1;
+                const float p10 = S.s[gid + 8][cA] * w0, p11 = S.s[gid + 8][cA + 1] * w1;
+                const float p02 = S.s[gid][cB] * w2, p03 = S.s[gid][cB + 1] * w3;
+                const float p12 = S.s[gid + 8][cB] * w2, p13 = S.s[gid + 8][cB + 1] * w3;
+                uint32_t ah[4], al[4];
+                ah[0] = pack_h2(p00, p01);
+                ah[1] = pack_h2(p10, p11);
+                ah[2] = pack_h2(p02, p03);
+                ah[3] = pack_h2(p12, p13);
+                {
+                    const __half2* h = reinterpret_cast<const __half2*>(ah);
+                    float2 f;
+                    f = __half22float2(h[0]); al[0] = pack_h2(p00 - f.x, p01 - f.y);
+                    f = __half22float2(h[1]); al[1] = pack_h2(p10 - f.x, p11 - f.y);
+                    f = __half22float2(h[2]); al[2] = pack_h2(p02 - f.x, p03 - f.y);
+                    f = __half22float2(h[3]); al[3] = pack_h2(p12 - f.x, p13 - f.y);
+                }
+#pragma unroll
+                for (int j = 0; j < 8; ++j) {
+                    const int d = warp * 64 + j * 8 + gid;
+                    uint32_t b[2];
+                    if constexpr (KV_MODE == 1) {
+                        const uint32_t x0 = (uint8_t) S.v[cA][d] | ((uint32_t) (uint8_t) S.v[cA + 1][d] << 8);
+                        const uint32_t x1 = (uint8_t) S.v[cB][d] | ((uint32_t) (uint8_t) S.v[cB + 1][d] << 8);
+                        b[0] = i8x2_to_h2(x0);
+                        b[1] = i8x2_to_h2(x1);
+                    } else {
+                        const __half2 h0 = __halves2half2(S.v[cA][d], S.v[cA + 1][d]);
+                        const __half2 h1 = __halves2half2(S.v[cB][d], S.v[cB + 1][d]);
+                        b[0] = *reinterpret_cast<const uint32_t*>(&h0);
+                        b[1] = *reinterpret_cast<const uint32_t*>(&h1);
+                    }
+                    mma16816(tmp[j], ah, b);
+#ifndef D1_NO_PLO
+                    mma16816(tmp[j], al, b);
+#endif
+                }
+            }
+            const float a0 = S.alpha[gid], a1 = S.alpha[gid + 8];
+#pragma unroll
+            for (int j = 0; j < 8; ++j) {
+                acc[j][0] = fmaf(acc[j][0], a0, tmp[j][0] * vdown);
+                acc[j][1] = fmaf(acc[j][1], a0, tmp[j][1] * vdown);
+                acc[j][2] = fmaf(acc[j][2], a1, tmp[j][2] * vdown);
+                acc[j][3] = fmaf(acc[j][3], a1, tmp[j][3] * vdown);
+            }
+        }
+    }
+    __syncthreads();
+    const float l0 = S.lsum[gid], l1 = S.lsum[gid + 8];
+    const float i0 = l0 > 0.0f ? 1.0f / l0 : 0.0f, i1 = l1 > 0.0f ? 1.0f / l1 : 0.0f;
+#pragma unroll
+    for (int j = 0; j < 8; ++j) {
+        const int d = warp * 64 + j * 8 + 2 * tig;
+        *reinterpret_cast<float2*>(attn + (size_t) gid * HD + d) = make_float2(acc[j][0] * i0, acc[j][1] * i0);
+        if (gid + 8 < G)
+            *reinterpret_cast<float2*>(attn + (size_t) (gid + 8) * HD + d) = make_float2(acc[j][2] * i1, acc[j][3] * i1);
+    }
+}
+
+// ---- v2 (int8 KV): warp w owns dims [64w, 64w+64) for both q.k and p.v, which is also int8 scale group w. So a
+// warp needs only its own 64-byte slice of each K and V row: it gathers it itself with cp.async into its own
+// double-buffered stage while it computes the previous chunk, and q stays in registers. Only the q.k partial sums
+// (one per dim group) cross warps, and they are added in a fixed order: the result is deterministic.
+constexpr int CH2 = 32;
+
+struct Smem2 {
+    int8_t kv[2][4][2][CH2][64];   // stage, warp, K/V, cell, 64 dims in 16-byte pieces XOR-swizzled by the cell
+    float sc[2][4][2][CH2];        // stage, warp, K/V scale of the cell for the warp's group
+    float part[4][16][CH2 + 1];    // q.k per dim group
+    float p[16][CH2 + 1];
+    float qmax[4];
+    float alpha[16];
+    float lsum[16];
+    float mrow[16];
+};
+
+__device__ __forceinline__ int swz(int cell, int byte) {   // byte offset of (cell, byte) in a stage slice
+    return cell * 64 + ((((byte >> 4) ^ (cell >> 1)) & 3) << 4) + (byte & 15);
+}
+__device__ __forceinline__ void cp_async16(void* smem, const void* gmem, bool valid) {
+#if !STRATA_PA_SM80
+    __trap();
+#else
+    const unsigned sa = (unsigned) __cvta_generic_to_shared(smem);
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;\n" ::"r"(sa), "l"(gmem), "r"(valid ? 16 : 0));
+#endif
+}
+__device__ __forceinline__ void cp_async_commit() {
+#if STRATA_PA_SM80
+    asm volatile("cp.async.commit_group;\n" ::);
+#endif
+}
+__device__ __forceinline__ void cp_async_wait1() {
+#if STRATA_PA_SM80
+    asm volatile("cp.async.wait_group 1;\n" ::);
+#endif
+}
+
+__global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __restrict__ q, QsaAttnPools p,
+                                                                 const int32_t* __restrict__ ids,
+                                                                 const int32_t* __restrict__ steps, int n_kv_heads,
+                                                                 int page_size, float scale_log2,
+                                                                 float* __restrict__ attn, int cap) {
+    extern __shared__ __align__(16) unsigned char smem_raw[];
+    Smem2& S = *reinterpret_cast<Smem2*>(smem_raw);
+    const int qi = blockIdx.x, kvh = blockIdx.y;
+    const int n_head = n_kv_heads * G;
+    q += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;
+    attn += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;
+    ids += (size_t) qi * cap;
+    const int n = __ldg(steps + (size_t) qi * kStepCount + kStepWidth);
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+    const int gid = lane >> 2, tig = lane & 3;
+    const int dim0 = warp * 64;
+
+    // q: the power-of-two prescale over all 12 heads (as v1), then this warp's 64 dims as hi/lo A fragments
+    float qm = 0.0f;
+    for (int i = t; i < G * HD; i += THREADS) qm = fmaxf(qm, fabsf(q[i]));
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) qm = fmaxf(qm, __shfl_xor_sync(0xffffffffu, qm, o));
+    if (lane == 0) S.qmax[warp] = qm;
+    if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; }
+    __syncthreads();
+    qm = fmaxf(fmaxf(S.qmax[0], S.qmax[1]), fmaxf(S.qmax[2], S.qmax[3]));
+    int qe = 0;
+    if (qm > 0.0f) frexpf(qm, &qe);
+    const float qup = ldexpf(1.0f, 14 - qe), qdown = ldexpf(scale_log2, qe - 14);
+    uint32_t qh[4][4], ql[4][4];
+#pragma unroll
+    for (int kk = 0; kk < 4; ++kk) {
+#pragma unroll
+        for (int r = 0; r < 4; ++r) {
+            const int row = gid + (r & 1) * 8, col = dim0 + kk * 16 + 2 * tig + (r >> 1) * 8;
+            float2 x = make_float2(0.f, 0.f);
+            if (row < G) x = *reinterpret_cast<const float2*>(q + (size_t) row * HD + col);
+            x.x *= qup;
+            x.y *= qup;
+            const __half2 hi = __floats2half2_rn(x.x, x.y);
+            const float2 hf = __half22float2(hi);
+            const __half2 lo = __floats2half2_rn(x.x - hf.x, x.y - hf.y);
+            qh[kk][r] = *reinterpret_cast<const uint32_t*>(&hi);
+            ql[kk][r] = *reinterpret_cast<const uint32_t*>(&lo);
+        }
+    }
+
+    // the chunk pipeline: cells two chunks ahead, their pool rows one chunk ahead, the data (cp.async) one ahead
+    const int n_chunks = (n + CH2 - 1) / CH2;
+    auto cell_of = [&](int c) -> int { return c < n ? __ldg(ids + c) : -1; };
+    auto row_of = [&](int cell) -> long long {
+        if (cell < 0) return -1;
+        const long long page = (long long) __ldg(p.page_table + cell / page_size);
+        return (page * n_kv_heads + kvh) * page_size + (cell % page_size);
+    };
+    auto issue = [&](long long r, int st, float& ksr, float& vsr) {
+#pragma unroll
+        for (int j = 0; j < 4; ++j) {
+            const int idx = lane + 32 * j, cell = idx >> 2, pc = idx & 3;
+            const long long rr = __shfl_sync(0xffffffffu, r, cell);
+            const bool ok = rr >= 0;
+            const size_t off = ok ? (size_t) rr * HD + dim0 + pc * 16 : 0;
+            cp_async16(&S.kv[st][warp][0][0][0] + swz(cell, pc * 16), p.k_q + off, ok);
+            cp_async16(&S.kv[st][warp][1][0][0] + swz(cell, pc * 16), p.v_q + off, ok);
+        }
+        ksr = r >= 0 ? __half2float(__ushort_as_half(__ldg(p.k_scale + r * (HD / KV_Q8_GROUP) + warp))) : 0.0f;
+        vsr = r >= 0 ? __half2float(__ushort_as_half(__ldg(p.v_scale + r * (HD / KV_Q8_GROUP) + warp))) : 0.0f;
+    };
+    float ksn, vsn;
+    issue(row_of(cell_of(lane)), 0, ksn, vsn);
+    cp_async_commit();
+    S.sc[0][warp][0][lane] = ksn;
+    S.sc[0][warp][1][lane] = vsn;
+    long long r_next = row_of(cell_of(CH2 + lane));
+    int cell_next2 = cell_of(2 * CH2 + lane);
+
+    float acc[8][4];
+#pragma unroll
+    for (int j = 0; j < 8; ++j) acc[j][0] = acc[j][1] = acc[j][2] = acc[j][3] = 0.0f;
+
+    for (int ci = 0; ci < n_chunks; ++ci) {
+        const int st = ci & 1, c0 = ci * CH2;
+        const bool more = ci + 1 < n_chunks;
+        if (more) issue(r_next, st ^ 1, ksn, vsn);
+        cp_async_commit();
+        r_next = row_of(cell_next2);
+        cell_next2 = cell_of((ci + 3) * CH2 + lane);
+        cp_async_wait1();
+        __syncwarp();
+        const int8_t* K = &S.kv[st][warp][0][0][0];
+        const int8_t* V = &S.kv[st][warp][1][0][0];
+        // q.k over this warp's 64 dims, times the cell's K scale for this group
+#pragma unroll
+        for (int nt = 0; nt < CH2 / 8; ++nt) {
+            float tg[4] = {0.f, 0.f, 0.f, 0.f};
+            const int cell = nt * 8 + gid;
+#pragma unroll
+            for (int kk = 0; kk < 4; ++kk) {
+                uint32_t b[2];
+                b[0] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(K + swz(cell, kk * 16 + 2 * tig)));
+                b[1] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(K + swz(cell, kk * 16 + 2 * tig + 8)));
+                mma16816(tg, qh[kk], b);
+                mma16816(tg, ql[kk], b);
+            }
+            const int c = nt * 8 + 2 * tig;
+            const float s0 = S.sc[st][warp][0][c], s1 = S.sc[st][warp][0][c + 1];
+            S.part[warp][gid][c] = tg[0] * s0;
+            S.part[warp][gid][c + 1] = tg[1] * s1;
+            S.part[warp][gid + 8][c] = tg[2] * s0;
+            S.part[warp][gid + 8][c + 1] = tg[3] * s1;
+        }
+        __syncthreads();
+        // online softmax over the four groups' sum (fixed order): row t/8, 4 cells per thread
+        {
+            const int r = t >> 3, sub = t & 7;
+            float x[4], mx = -INFINITY;
+#pragma unroll
+            for (int j = 0; j < 4; ++j) {
+                const int c = sub * 4 + j;
+                x[j] = c0 + c < n ? (((S.part[0][r][c] + S.part[1][r][c]) + S.part[2][r][c]) + S.part[3][r][c]) * qdown
+                                  : -INFINITY;
+                mx = fmaxf(mx, x[j]);
+            }
+#pragma unroll
+            for (int o = 1; o < 8; o <<= 1) mx = fmaxf(mx, __shfl_xor_sync(0xffffffffu, mx, o));
+            const float m_old = S.mrow[r];
+            const float m_new = fmaxf(m_old, mx);
+            float sum = 0.0f;
+#pragma unroll
+            for (int j = 0; j < 4; ++j) {
+                const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);
+                S.p[r][sub * 4 + j] = e;
+                sum += e;
+            }
+#pragma unroll
+            for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
+            __syncwarp();
+            if (sub == 0) {
+                const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);
+                S.alpha[r] = a;
+                S.lsum[r] = fmaf(S.lsum[r], a, sum);
+                S.mrow[r] = m_new;
+            }
+        }
+        __syncthreads();
+        // p.v over this warp's 64 dims (as v1)
+        {
+            float vmax = S.sc[st][warp][1][lane];
+#pragma unroll
+            for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));
+            const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f, vdown = vmax * (1.0f / 16384.0f);
+            float tmp[8][4];
+#pragma unroll
+            for (int j = 0; j < 8; ++j) tmp[j][0] = tmp[j][1] = tmp[j][2] = tmp[j][3] = 0.0f;
+#pragma unroll
+            for (int ks = 0; ks < CH2 / 16; ++ks) {
+                const int cA = ks * 16 + 2 * tig, cB = cA + 8;
+                const float w0 = S.sc[st][warp][1][cA] * vup, w1 = S.sc[st][warp][1][cA + 1] * vup,
+                            w2 = S.sc[st][warp][1][cB] * vup, w3 = S.sc[st][warp][1][cB + 1] * vup;
+                const float p00 = S.p[gid][cA] * w0, p01 = S.p[gid][cA + 1] * w1;
+                const float p10 = S.p[gid + 8][cA] * w0, p11 = S.p[gid + 8][cA + 1] * w1;
+                const float p02 = S.p[gid][cB] * w2, p03 = S.p[gid][cB + 1] * w3;
+                const float p12 = S.p[gid + 8][cB] * w2, p13 = S.p[gid + 8][cB + 1] * w3;
+                uint32_t ah[4], al[4];
+                ah[0] = pack_h2(p00, p01);
+                ah[1] = pack_h2(p10, p11);
+                ah[2] = pack_h2(p02, p03);
+                ah[3] = pack_h2(p12, p13);
+                {
+                    const __half2* h = reinterpret_cast<const __half2*>(ah);
+                    float2 f;
+                    f = __half22float2(h[0]); al[0] = pack_h2(p00 - f.x, p01 - f.y);
+                    f = __half22float2(h[1]); al[1] = pack_h2(p10 - f.x, p11 - f.y);
+                    f = __half22float2(h[2]); al[2] = pack_h2(p02 - f.x, p03 - f.y);
+                    f = __half22float2(h[3]); al[3] = pack_h2(p12 - f.x, p13 - f.y);
+                }
+#pragma unroll
+                for (int j = 0; j < 8; ++j) {
+                    const int d = j * 8 + gid;
+                    const uint32_t x0 = (uint8_t) V[swz(cA, d)] | ((uint32_t) (uint8_t) V[swz(cA + 1, d)] << 8);
+                    const uint32_t x1 = (uint8_t) V[swz(cB, d)] | ((uint32_t) (uint8_t) V[swz(cB + 1, d)] << 8);
+                    uint32_t b[2];
+                    b[0] = i8x2_to_h2(x0);
+                    b[1] = i8x2_to_h2(x1);
+                    mma16816(tmp[j], ah, b);
+                    mma16816(tmp[j], al, b);
+                }
+            }
+            const float a0 = S.alpha[gid], a1 = S.alpha[gid + 8];
+#pragma unroll
+            for (int j = 0; j < 8; ++j) {
+                acc[j][0] = fmaf(acc[j][0], a0, tmp[j][0] * vdown);
+                acc[j][1] = fmaf(acc[j][1], a0, tmp[j][1] * vdown);
+                acc[j][2] = fmaf(acc[j][2], a1, tmp[j][2] * vdown);
+                acc[j][3] = fmaf(acc[j][3], a1, tmp[j][3] * vdown);
+            }
+        }
+        if (more) {
+            S.sc[st ^ 1][warp][0][lane] = ksn;
+            S.sc[st ^ 1][warp][1][lane] = vsn;
+        }
+        __syncwarp();
+    }
+    __syncthreads();
+    const float l0 = S.lsum[gid], l1 = S.lsum[gid + 8];
+    const float i0 = l0 > 0.0f ? 1.0f / l0 : 0.0f, i1 = l1 > 0.0f ? 1.0f / l1 : 0.0f;
+#pragma unroll
+    for (int j = 0; j < 8; ++j) {
+        const int d = dim0 + j * 8 + 2 * tig;
+        *reinterpret_cast<float2*>(attn + (size_t) gid * HD + d) = make_float2(acc[j][0] * i0, acc[j][1] * i0);
+        if (gid + 8 < G)
+            *reinterpret_cast<float2*>(attn + (size_t) (gid + 8) * HD + d) = make_float2(acc[j][2] * i1, acc[j][3] * i1);
+    }
+}
+
+bool launch_i8(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,
+               const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {
+    static bool attr[64] = {};   // the shared-memory opt-in is per device (a layer split runs this on several)
+    int dev = 0;
+    cudaGetDevice(&dev);
+    const int bytes = (int) sizeof(Smem2);
+    if (dev < 0 || dev >= 64) return false;
+    if (!attr[dev]) {
+        if (cudaFuncSetAttribute(prompt_attn_i8_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes) !=
+            cudaSuccess) {
+            cudaGetLastError();
+            return false;
+        }
+        attr[dev] = true;
+    }
+    const float scale_log2 = 1.4426950408889634f / sqrtf((float) HD);
+    for (int64_t q0 = 0; q0 < n_q; q0 += 65535) {
+        const int64_t nb = n_q - q0 < 65535 ? n_q - q0 : 65535;
+        prompt_attn_i8_kernel<<<dim3((unsigned) nb, (unsigned) s.n_head_kv), THREADS, bytes, st>>>(
+            q + q0 * s.n_head * HD, pools, ids + q0 * cap, steps + q0 * kStepCount, (int) s.n_head_kv,
+            (int) s.page_size, scale_log2, attn + q0 * s.n_head * HD, (int) cap);
+    }
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "qsa_prompt_attn_batch: %s\n", cudaGetErrorString(e));
+        std::exit(1);
+    }
+    return true;
+}
+
+template <int KV_MODE>
+bool launch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,
+            const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {
+    static bool attr[64] = {};   // per device, as above
+    int dev = 0;
+    cudaGetDevice(&dev);
+    const int bytes = (int) sizeof(Smem<KV_MODE>);
+    if (dev < 0 || dev >= 64) return false;
+    if (!attr[dev]) {
+        if (cudaFuncSetAttribute(prompt_attn_kernel<KV_MODE>, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes) !=
+            cudaSuccess) {
+            cudaGetLastError();
+            return false;
+        }
+        attr[dev] = true;
+    }
+    const float scale_log2 = 1.4426950408889634f / sqrtf((float) HD);
+    for (int64_t q0 = 0; q0 < n_q; q0 += 65535) {
+        const int64_t nb = n_q - q0 < 65535 ? n_q - q0 : 65535;
+        prompt_attn_kernel<KV_MODE><<<dim3((unsigned) nb, (unsigned) s.n_head_kv), THREADS, bytes, st>>>(
+            q + q0 * s.n_head * HD, pools, ids + q0 * cap, steps + q0 * kStepCount, (int) s.n_head_kv,
+            (int) s.page_size, scale_log2, attn + q0 * s.n_head * HD, (int) cap);
+    }
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "qsa_prompt_attn_batch: %s\n", cudaGetErrorString(e));
+        std::exit(1);
+    }
+    return true;
+}
+
+}  // namespace
+
+bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
+                           int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream) {
+    if (n_q <= 0) return true;
+    {   // sm_80 or newer (the MMA and cp.async above); an older card keeps the old kernel
+        static int cc_major[64] = {};
+        int dev = 0;
+        if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+        if (cc_major[dev] == 0) {
+            int major = 0;
+            if (cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess) {
+                cudaGetLastError();
+                return false;
+            }
+            cc_major[dev] = major;
+        }
+        if (cc_major[dev] < 8) return false;
+    }
+#if defined(__HIPCC__)
+    return false;   // the tensor-core kernel is compiled out on AMD (its major version is not a CUDA sm)
+#endif
+    if (pools.k_q4 != nullptr || s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !ids ||
+        !steps || !pools.page_table)
+        return false;
+    cudaStream_t st = (cudaStream_t) stream;
+    if (pools.k_q != nullptr && pools.v_q4 != nullptr) {   // hybrid K8V4: int8 K + dequantized-q4 V
+        if (!pools.k_scale) return false;
+        return launch<3>(q, pools, ids, steps, cap, s, attn, n_q, st);
+    }
+    if (pools.k_q != nullptr) {
+        if (!pools.v_q || !pools.k_scale || !pools.v_scale) return false;
+        // STRATA_PROMPT_ATTN_V1=1 (debug): the first version, same accuracy, another summation order - the control
+        // for how far the model amplifies an FP32-level change
+        static const bool v1 = std::getenv("STRATA_PROMPT_ATTN_V1") != nullptr;
+        if (v1) return launch<1>(q, pools, ids, steps, cap, s, attn, n_q, st);
+        return launch_i8(q, pools, ids, steps, cap, s, attn, n_q, st);
+    }
+    if (!pools.k_pool || !pools.v_pool) return false;
+    return launch<0>(q, pools, ids, steps, cap, s, attn, n_q, st);
+}
+
+}  // namespace strata::kernels

src/kernels/cuda/qsa_select.cu

@@ -148,24 +148,351 @@ __global__ void __launch_bounds__(TOPK_T) block_topk_kernel(const float* __restr
     }
 }
 
+
+// ---- QSA select on tensor cores (perf-review, after D-1): the block scores of many queries are one GEMM,
+// rows (query, indexer head) x columns (blocks), K = 128, with relu per head summed. 3xTF32 (each operand split
+// into a TF32 hi and lo part, hi*hi + hi*lo + lo*hi) keeps FP32-level accuracy; the summation order differs from
+// the warp kernel, so a score can move in its last bits and a near-tie can select differently (not bitwise).
+// Blocks < n_bid only; the tail block n_bid (the `dead` key, +1e9) is scored by the warp kernel's own code.
+constexpr int TC_QT = 16;                 // queries per CTA (one m16 tile per indexer head)
+constexpr int TC_NB = 32;                 // blocks per tile (4 warps x n8)
+constexpr int TC_ITER = 4;                // tiles per CTA (the query tile is loaded once)
+constexpr int TC_QS = IDX_HEADS * IDX_DIM + 4;   // query row stride in floats
+constexpr int TC_KS = IDX_DIM + 4;               // key row stride
+
+// TF32 conversion and MMA need sm_80: below it they compile to a trap and qsa_block_scores_tc refuses the device
+#if defined(__HIPCC__)          // AMD: no mma.sync / cp.async; the host keeps the warp kernel (below)
+#define STRATA_SEL_SM80 0
+#elif !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
+#define STRATA_SEL_SM80 1
+#else
+#define STRATA_SEL_SM80 0
+#endif
+__device__ __forceinline__ uint32_t tf32_hi(float x) {
+#if STRATA_SEL_SM80
+    uint32_t r;
+    asm("cvt.rna.tf32.f32 %0, %1;" : "=r"(r) : "f"(x));
+    return r;
+#else
+    return __float_as_uint(x);
+#endif
+}
+__device__ __forceinline__ void mma_tf32(float* c, const uint32_t* a, const uint32_t* b) {
+#if !STRATA_SEL_SM80
+    __trap();
+#else
+    asm volatile("mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
+                 "{%0,%1,%2,%3};\n"
+                 : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
+                 : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
+#endif
+}
+
+__global__ void __launch_bounds__(128) block_scores_tc_kernel(const float* __restrict__ pooled,
+                                                              const float* __restrict__ q_idx,
+                                                              const int32_t* __restrict__ steps, int64_t nq,
+                                                              int64_t max_blocks, int64_t reach,
+                                                              float* __restrict__ out) {
+    extern __shared__ __align__(16) float sm[];
+    float* sQ = sm;                                   // [TC_QT][TC_QS]
+    float* sK = sm + TC_QT * TC_QS;                   // [TC_NB][TC_KS]
+    __shared__ int s_nbid[TC_QT];
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5, gid = lane >> 2, tig = lane & 3;
+    const int64_t q0 = (int64_t) blockIdx.y * TC_QT;
+    for (int i = t; i < TC_QT * IDX_HEADS * IDX_DIM / 4; i += 128) {
+        const int r = i / (IDX_HEADS * IDX_DIM / 4), c = i % (IDX_HEADS * IDX_DIM / 4);
+        float4 v = make_float4(0.f, 0.f, 0.f, 0.f);
+        if (q0 + r < nq) v = reinterpret_cast<const float4*>(q_idx + (q0 + r) * IDX_HEADS * IDX_DIM)[c];
+        *reinterpret_cast<float4*>(sQ + r * TC_QS + c * 4) = v;
+    }
+    if (t < TC_QT) s_nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : 0;
+    int lo_nbid = 0x7fffffff, hi_nbid = 0;
+    __syncthreads();
+    for (int i = 0; i < TC_QT; ++i) {
+        if (q0 + i >= nq) break;
+        lo_nbid = min(lo_nbid, s_nbid[i]);
+        hi_nbid = max(hi_nbid, s_nbid[i]);
+    }
+    for (int it = 0; it < TC_ITER; ++it) {
+        const int64_t b0 = ((int64_t) blockIdx.x * TC_ITER + it) * TC_NB;
+        if (b0 >= reach || b0 >= hi_nbid) break;      // blocks >= every query's n_bid: nothing to score
+        __syncthreads();                              // the previous tile's reads are done
+        for (int i = t; i < TC_NB * IDX_DIM / 4; i += 128) {
+            const int r = i / (IDX_DIM / 4), c = i % (IDX_DIM / 4);
+            float4 v = make_float4(0.f, 0.f, 0.f, 0.f);
+            if (b0 + r < hi_nbid) v = reinterpret_cast<const float4*>(pooled + (b0 + r) * IDX_DIM)[c];
+            *reinterpret_cast<float4*>(sK + r * TC_KS + c * 4) = v;
+        }
+        __syncthreads();
+        float acc[IDX_HEADS][4];
+#pragma unroll
+        for (int h = 0; h < IDX_HEADS; ++h) acc[h][0] = acc[h][1] = acc[h][2] = acc[h][3] = 0.f;
+        const float* kr = sK + (warp * 8 + gid) * TC_KS;
+#pragma unroll 4
+        for (int k0 = 0; k0 < IDX_DIM; k0 += 8) {
+            const float kx0 = kr[k0 + tig], kx1 = kr[k0 + tig + 4];
+            uint32_t bh[2], bl[2];
+            bh[0] = tf32_hi(kx0);
+            bh[1] = tf32_hi(kx1);
+            bl[0] = tf32_hi(kx0 - __uint_as_float(bh[0]));
+            bl[1] = tf32_hi(kx1 - __uint_as_float(bh[1]));
+#pragma unroll
+            for (int h = 0; h < IDX_HEADS; ++h) {
+                const float* qa = sQ + h * IDX_DIM + k0 + tig;
+                const float x0 = qa[gid * TC_QS], x1 = qa[(gid + 8) * TC_QS];
+                const float x2 = qa[gid * TC_QS + 4], x3 = qa[(gid + 8) * TC_QS + 4];
+                uint32_t ah[4], al[4];
+                ah[0] = tf32_hi(x0); ah[1] = tf32_hi(x1); ah[2] = tf32_hi(x2); ah[3] = tf32_hi(x3);
+                al[0] = tf32_hi(x0 - __uint_as_float(ah[0]));
+                al[1] = tf32_hi(x1 - __uint_as_float(ah[1]));
+                al[2] = tf32_hi(x2 - __uint_as_float(ah[2]));
+                al[3] = tf32_hi(x3 - __uint_as_float(ah[3]));
+                mma_tf32(acc[h], al, bh);
+                mma_tf32(acc[h], ah, bl);
+                mma_tf32(acc[h], ah, bh);
+            }
+        }
+        // relu per head, heads added in order (as the warp kernel), written where the block completed for the query
+        const int64_t bc = b0 + warp * 8 + 2 * tig;
+#pragma unroll
+        for (int half = 0; half < 2; ++half) {
+            const int qr = gid + half * 8;
+            const int64_t qi = q0 + qr;
+            if (qi >= nq) continue;
+            const int nb = s_nbid[qr];
+#pragma unroll
+            for (int j = 0; j < 2; ++j) {
+                const int64_t b = bc + j;
+                if (b >= nb || b >= max_blocks) continue;
+                float score = 0.0f;
+#pragma unroll
+                for (int h = 0; h < IDX_HEADS; ++h) {
+                    const float d = acc[h][half * 2 + j];
+                    score += d > 0.0f ? d : 0.0f;
+                }
+                out[qi * max_blocks + b] = score;
+            }
+        }
+    }
+}
+
+// the tail block n_bid of each query: exactly block_scores_kernel's arithmetic for that block
+__global__ void __launch_bounds__(32) block_scores_tail_kernel(const float* __restrict__ dead,
+                                                               const float* __restrict__ q_idx,
+                                                               const int32_t* __restrict__ steps,
+                                                               int64_t max_blocks, float* __restrict__ out) {
+    const int64_t qi = blockIdx.x;
+    const int32_t* st = steps + qi * kStepCount;
+    const int64_t n_kv = st[kStepNKv], n_bid = st[kStepNBid];
+    if (n_bid >= max_blocks) return;
+    const int lane = threadIdx.x & 31;
+    const float4 k4 = *reinterpret_cast<const float4*>(dead + lane * 4);
+    const float* q = q_idx + qi * IDX_HEADS * IDX_DIM + lane * 4;
+    float score = 0.0f;
+#pragma unroll
+    for (int h = 0; h < IDX_HEADS; ++h) {
+        const float4 q4 = *reinterpret_cast<const float4*>(q + h * IDX_DIM);
+        float d = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+#pragma unroll
+        for (int o = 16; o > 0; o >>= 1) d += __shfl_xor_sync(0xffffffffu, d, o);
+        score += d > 0.0f ? d : 0.0f;
+    }
+    if (lane == 0) {
+        if (n_kv % R != 0) score += 1e9f;
+        out[qi * max_blocks + n_bid] = score;
+    }
+}
+
+// ---- the same top-k with each query's keys read once: 1,024 threads hold up to TK_PER consecutive blocks' keys in
+// registers (contexts up to 4 * 1024 * TK_PER cells), per-warp histograms, block-wide scans. The selection rule is
+// block_topk_kernel's (radix threshold, ties to the lowest index, cells ascending): identical ids.
+constexpr int TK_T = 1024;
+constexpr int TK_PER = 33;
+
+__device__ __forceinline__ int block_excl_scan(int v, int* s_warp, int& total) {
+    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+    int x = v;
+#pragma unroll
+    for (int o = 1; o < 32; o <<= 1) {
+        const int y = __shfl_up_sync(0xffffffffu, x, o);
+        if (lane >= o) x += y;
+    }
+    if (lane == 31) s_warp[warp] = x;
+    __syncthreads();
+    if (warp == 0) {
+        int w = s_warp[lane];
+        int z = w;
+#pragma unroll
+        for (int o = 1; o < 32; o <<= 1) {
+            const int y = __shfl_up_sync(0xffffffffu, z, o);
+            if (lane >= o) z += y;
+        }
+        s_warp[lane] = z - w;               // exclusive per warp
+        if (lane == 31) s_warp[32] = z;     // total
+    }
+    __syncthreads();
+    const int r = s_warp[warp] + x - v;
+    total = s_warp[32];
+    __syncthreads();
+    return r;
+}
+
+__global__ void __launch_bounds__(TK_T) block_topk_reg_kernel(const float* __restrict__ scores,
+                                                              const int32_t* __restrict__ steps, int64_t max_blocks,
+                                                              int64_t cap, int32_t* __restrict__ ids) {
+    __shared__ int hist[TK_T / 32][256];
+    __shared__ int s_warp[33];
+    __shared__ int s_digit, s_above;
+    const int64_t qi = blockIdx.x;
+    const int32_t* st = steps + qi * kStepCount;
+    const int64_t n_kv = st[kStepNKv], n_bid = st[kStepNBid], width = st[kStepWidth];
+    int32_t* out = ids + qi * cap;
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+    if (n_kv <= width) {
+        for (int64_t j = t; j < n_kv; j += TK_T) out[j] = (int32_t) j;
+        return;
+    }
+    const float* sc = scores + qi * max_blocks;
+    const int64_t nb = n_bid + 1;
+    const int64_t per = (nb + TK_T - 1) / TK_T;       // <= TK_PER (the caller checks)
+    const int64_t b0 = (int64_t) t * per, b1 = (b0 + per < nb) ? b0 + per : nb;
+    uint32_t key[TK_PER];
+#pragma unroll
+    for (int j = 0; j < TK_PER; ++j) key[j] = (b0 + j < b1) ? order_key(sc[b0 + j]) : 0u;
+    auto weight = [&](int64_t b) -> int { return b < n_bid ? R : (int) (n_kv - n_bid * R); };
+    uint32_t prefix = 0;
+    int above = 0;
+    for (int shift = 24; shift >= 0; shift -= 8) {
+        for (int i = lane; i < 256; i += 32) hist[warp][i] = 0;
+        __syncwarp();
+        const uint32_t hi_mask = shift == 24 ? 0u : (0xffffffffu << (shift + 8));
+#pragma unroll
+        for (int j = 0; j < TK_PER; ++j) {
+            const int64_t b = b0 + j;
+            if (b >= b1) break;
+            const int w = weight(b);
+            if (w == 0) continue;
+            if ((key[j] & hi_mask) == (prefix & hi_mask)) atomicAdd(&hist[warp][(key[j] >> shift) & 255], w);
+        }
+        __syncthreads();
+        if (t < 256) {                                // fold the warps' histograms into warp 0's
+            int s = 0;
+            for (int w2 = 0; w2 < TK_T / 32; ++w2) s += hist[w2][t];
+            hist[0][t] = s;
+        }
+        __syncthreads();
+        if (t == 0) {
+            int cum = above, d = 255;
+            for (; d > 0; --d) {
+                if (cum + hist[0][d] >= width) break;
+                cum += hist[0][d];
+            }
+            s_digit = d;
+            s_above = cum;
+        }
+        __syncthreads();
+        prefix |= (uint32_t) s_digit << shift;
+        above = s_above;
+        __syncthreads();
+    }
+    const uint32_t thr = prefix;
+    const int64_t eq_budget = width - above;
+    int gt = 0, eq = 0;
+#pragma unroll
+    for (int j = 0; j < TK_PER; ++j) {
+        const int64_t b = b0 + j;
+        if (b >= b1) break;
+        const int w = weight(b);
+        if (w == 0) continue;
+        if (key[j] > thr) gt += w;
+        else if (key[j] == thr) eq += w;
+    }
+    int tot;
+    const int eq_before = block_excl_scan(eq, s_warp, tot);
+    int64_t my_eq = eq_budget - eq_before;
+    if (my_eq < 0) my_eq = 0;
+    if (my_eq > eq) my_eq = eq;
+    const int sel = gt + (int) my_eq;
+    int64_t wpos = block_excl_scan(sel, s_warp, tot);
+    int64_t eq_left = my_eq;
+#pragma unroll
+    for (int j = 0; j < TK_PER; ++j) {
+        const int64_t b = b0 + j;
+        if (b >= b1) break;
+        const int w = weight(b);
+        if (w == 0) continue;
+        if (key[j] > thr) {
+            for (int c = 0; c < w; ++c) out[wpos++] = (int32_t) (b * R + c);
+        } else if (key[j] == thr) {
+            for (int c = 0; c < w && eq_left > 0; ++c, --eq_left) out[wpos++] = (int32_t) (b * R + c);
+        }
+    }
+}
+
 }  // namespace
 
 void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
-                      int64_t max_blocks, const QsaShapes& s, float* scores, void* stream) {
+                      int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {
     if (nq <= 0) return;
     if (s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R || nq > 65535) {
         std::fprintf(stderr, "qsa_block_scores: unsupported indexer geometry\n");
         std::exit(1);
     }
-    const dim3 grid((unsigned) ((max_blocks + SCORE_WARPS - 1) / SCORE_WARPS), (unsigned) nq);
+    // a block past a query's n_bid returns at once: the grid need only reach the batch's largest n_bid (C-1)
+    const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks;
+    const dim3 grid((unsigned) ((reach + SCORE_WARPS - 1) / SCORE_WARPS), (unsigned) nq);
     block_scores_kernel<<<grid, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(pooled, dead, q_idx, steps, max_blocks,
                                                                               scores);
     const cudaError_t e = cudaGetLastError();
     if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores: %s\n", cudaGetErrorString(e)); std::exit(1); }
 }
 
-void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
-                    const QsaShapes& s, int32_t* ids, void* stream) {
+bool qsa_block_scores_tc(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+                         int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {
+    if (nq <= 0) return true;
+    if (s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R || nq > 65535 * TC_QT) return false;
+    {   // sm_80 or newer (TF32 MMA); an older card keeps the warp kernel
+        static int cc_major[64] = {};
+        int dev = 0;
+        if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+        if (cc_major[dev] == 0) {
+            int major = 0;
+            if (cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess) {
+                cudaGetLastError();
+                return false;
+            }
+            cc_major[dev] = major;
+        }
+        if (cc_major[dev] < 8) return false;
+    }
+#if defined(__HIPCC__)
+    return false;   // the tensor-core kernel is compiled out on AMD (its major version is not a CUDA sm)
+#endif
+    static bool attr[64] = {};   // the shared-memory opt-in is per device (a layer split runs it on several)
+    int adev = 0;
+    cudaGetDevice(&adev);
+    const int bytes = (TC_QT * TC_QS + TC_NB * TC_KS) * (int) sizeof(float);
+    if (!attr[adev]) {
+        if (cudaFuncSetAttribute(block_scores_tc_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes) !=
+            cudaSuccess) {
+            cudaGetLastError();
+            return false;
+        }
+        attr[adev] = true;
+    }
+    const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks;
+    const int64_t per = (int64_t) TC_NB * TC_ITER;
+    const dim3 grid((unsigned) ((reach + per - 1) / per), (unsigned) ((nq + TC_QT - 1) / TC_QT));
+    block_scores_tc_kernel<<<grid, 128, bytes, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq, max_blocks, reach,
+                                                                         scores);
+    block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks, scores);
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores_tc: %s\n", cudaGetErrorString(e)); std::exit(1); }
+    return true;
+}
+
+void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                        const QsaShapes& s, int32_t* ids, void* stream) {
     if (nq <= 0) return;
     if (s.idx_block != R || cap < qsa_selection_width(kTopkMaxCells, s)) {
         std::fprintf(stderr, "qsa_block_topk: unsupported geometry or cap\n");
@@ -176,4 +503,23 @@ void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64
     if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
 }
 
+void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                    const QsaShapes& s, int32_t* ids, void* stream) {
+    // keys in registers when every query's blocks fit (contexts up to ~135K cells); the same ids. STRATA_TOPK_OLD=1:
+    // the kernel that reads them from memory on every pass
+    static const bool old = std::getenv("STRATA_TOPK_OLD") != nullptr;
+    if (nq <= 0) return;
+    if (old || max_blocks > (int64_t) TK_T * TK_PER) {
+        qsa_block_topk_ref(scores, steps, nq, max_blocks, cap, s, ids, stream);
+        return;
+    }
+    if (s.idx_block != R || cap < qsa_selection_width(kTopkMaxCells, s)) {
+        std::fprintf(stderr, "qsa_block_topk: unsupported geometry or cap\n");
+        std::exit(1);
+    }
+    block_topk_reg_kernel<<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
+}
+
 }  // namespace strata::kernels

src/kernels/cuda/shared_expert.cu

@@ -135,6 +135,9 @@ __global__ void native_scalar_sigmoid_kernel(float* gate) {
     // compilation flags. The dot product was already reduced by the pinned native MMVF implementation.
     gate[0] = __fdividef(1.0f, 1.0f + __expf(-gate[0]));
 }
+__global__ void native_scalar_sigmoid_multi_kernel(float* gate) {   // thread t = token t, same expression
+    gate[threadIdx.x] = __fdividef(1.0f, 1.0f + __expf(-gate[threadIdx.x]));
+}
 
 /// The MoE block's final combination.  See the header for the two readings it exists to pin.
 __global__ void moe_combine_kernel(const float* __restrict__ parts, const float* __restrict__ weights,
@@ -176,6 +179,11 @@ void shared_expert_multi(int n_tok, const float* x, const uint16_t* x_bf16, cons
     native_swiglu_kernel<<<(unsigned) ((n + THREADS - 1) / THREADS), THREADS, 0, cs>>>(gate, up, gate, n);
     native_quantize_q8_1(gate, nw.q8_1, (int) n_ff, n_tok, stream);
     native_mmvq(nw.down_type, nw.down_data, nw.q8_1, out, (int) n_ff, (int) n_embd, n_tok, stream);
+    static const bool batch = [] { const char* v = std::getenv("STRATA_DEC_BATCH"); return v == nullptr || std::atoi(v) != 0; }();
+    if (native_bf16 && batch && n_tok > 1) {   // one gemv for all rows (outputs identical), one sigmoid launch
+        bf16_gemv_fp32_mmvf_multi(x, n_embd, gate_inp_bf16, g, 1, n_embd, 1, n_tok, stream);
+        native_scalar_sigmoid_multi_kernel<<<1, n_tok, 0, cs>>>(g);
+    } else
     for (int t = 0; t < n_tok; ++t) {
         if (native_bf16) {
             bf16_gemv_fp32_mmvf(x + (size_t) t * n_embd, gate_inp_bf16, g + t, n_embd, 1, stream);

src/kernels/cuda/verify_kernels.cu

@@ -428,6 +428,93 @@ __global__ void wait_flag_ge_kernel(const volatile uint32_t* flag, uint32_t valu
 }
 }  // namespace
 
+namespace {
+__global__ void resident_plan_kernel(const int32_t* __restrict__ ids, int n, int k, const int32_t* __restrict__ res,
+                                     int n_expert, const uint8_t* cache_base, const unsigned long long* slot_off,
+                                     long long blob, int32_t* __restrict__ pl, long long capx, uint32_t* skip,
+                                     uint32_t ring) {
+    // one thread: at most kVerifyMaxT * 10 entries, the host's exact loop
+    for (int i = 0; i < n; ++i) {
+        const int32_t e = ids[i];
+        if (e < 0 || e >= n_expert || res[e] < 0) { *skip = 0; return; }
+    }
+    int32_t* counts = pl;
+    int32_t* start = pl + 4;
+    int32_t* dst = start + capx + 1;
+    int32_t* tok = dst + capx;
+    const long long ptr_off = ((4 + (capx + 1) + 2 * capx) + 1) & ~1ll;
+    unsigned long long* ptr = (unsigned long long*) (pl + ptr_off);
+    int32_t* start2 = pl + ptr_off + 4 * capx;
+    int groups = 0, entries = 0;
+    for (int i0 = 0; i0 < n; ++i0) {
+        bool first = true;
+        for (int j = 0; j < i0; ++j) if (ids[j] == ids[i0]) { first = false; break; }
+        if (!first) continue;
+        const int32_t slot = res[ids[i0]];
+        ptr[groups] = (unsigned long long) (cache_base + (slot_off ? (size_t) slot_off[slot] : (size_t) slot * (size_t) blob));
+        start[groups] = entries;
+        for (int i = i0; i < n; ++i)
+            if (ids[i] == ids[i0]) {
+                // an entry belongs to i0's group when its first occurrence is i0: the same expert id
+                dst[entries] = i;
+                tok[entries] = i / k;
+                ++entries;
+            }
+        ++groups;
+    }
+    start[groups] = entries;
+    start2[0] = entries;
+    counts[0] = groups;
+    counts[1] = entries;
+    counts[2] = 0;
+    __threadfence();
+    *skip = ring;
+}
+__global__ void wait_flag_ge_or_kernel(const volatile uint32_t* flag, uint32_t value, const volatile uint32_t* skip) {
+    if (*skip == value) return;
+    while (*flag < value) __nanosleep(100);
+    __threadfence_system();
+}
+__global__ void copy_i32_unless_kernel(int32_t* __restrict__ dst, const volatile int32_t* src, int n,
+                                       const uint32_t* skip, uint32_t value) {
+    if (*skip == value) return;
+    for (int i = threadIdx.x; i < n; i += blockDim.x) dst[i] = src[i];
+}
+__global__ void copy_or_zero_kernel(float4* __restrict__ dst, const volatile float4* src, long long n4,
+                                    const uint32_t* skip, uint32_t value) {
+    const bool zero = *skip == value;
+    for (long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; i < n4; i += (long long) gridDim.x * blockDim.x)
+        dst[i] = zero ? make_float4(0.f, 0.f, 0.f, 0.f) : const_cast<const float4*>(src)[i];
+}
+}  // namespace
+
+void resident_plan(const int32_t* ids, int n_entries, int k, const int32_t* res_layer, int n_expert,
+                   const uint8_t* cache_base, const unsigned long long* slot_off, long long blob, int32_t* plan,
+                   long long capx, uint32_t* skip, uint32_t ring, void* stream) {
+    resident_plan_kernel<<<1, 1, 0, (cudaStream_t) stream>>>(ids, n_entries, k, res_layer, n_expert, cache_base, slot_off,
+                                                             blob, plan, capx, skip, ring);
+    check("resident_plan");
+}
+void wait_flag_ge_or(const uint32_t* flag, uint32_t value, const uint32_t* skip, void* stream) {
+    wait_flag_ge_or_kernel<<<1, 1, 0, (cudaStream_t) stream>>>(flag, value, skip);
+    check("wait_flag_ge_or");
+}
+void copy_i32_from_mapped_unless(int32_t* dst, const int32_t* src, long long n, const uint32_t* skip, uint32_t value,
+                                 void* stream) {
+    if (n <= 0) return;
+    copy_i32_unless_kernel<<<1, 128, 0, (cudaStream_t) stream>>>(dst, (const volatile int32_t*) src, (int) n, skip, value);
+    check("copy_i32_from_mapped_unless");
+}
+void copy_or_zero_from_mapped(float* dst, const float* src, long long n, const uint32_t* skip, uint32_t value,
+                              void* stream) {
+    if (n <= 0) return;
+    const long long n4 = n / 4;
+    const int blocks = (int) ((n4 + 255) / 256 < 64 ? (n4 + 255) / 256 : 64);
+    copy_or_zero_kernel<<<blocks, 256, 0, (cudaStream_t) stream>>>((float4*) dst, (const volatile float4*) src, n4, skip,
+                                                                    value);
+    check("copy_or_zero_from_mapped");
+}
+
 void wait_flag_ge(const uint32_t* flag, uint32_t value, void* stream) {
     wait_flag_ge_kernel<<<1, 1, 0, (cudaStream_t) stream>>>(flag, value);
     check("wait_flag_ge");
@@ -454,4 +541,18 @@ void copy_indexed(float* dst, const float* src, int64_t stride, const int32_t* i
     check("copy_indexed");
 }
 
+// a GPU timestamp (ns, %globaltimer) into buf[i] - the verify window's stage profiler
+namespace { __global__ void gpu_stamp_kernel(unsigned long long* buf, int i) {
+    unsigned long long t;
+#if defined(__HIPCC__)
+    t = wall_clock64() * 10ull;   // gfx11: a constant 100 MHz counter, in ns
+#else
+    asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
+#endif
+    buf[i] = t;
+} }
+void gpu_stamp(unsigned long long* buf, int i, void* stream) {
+    gpu_stamp_kernel<<<1, 1, 0, (cudaStream_t) stream>>>(buf, i);
+}
+
 }  // namespace strata::kernels

src/kernels/gr_parity.cpp

@@ -20,6 +20,7 @@
 //   6. `gr_write`'s `2*sigmoid`, which centres the gate on 1 so a ZERO injection is a plain residual add.
 //      Asserted as a property, not as a value, because that is what the source comment claims.
 #include "strata/kernels/gr.hpp"
+#include "strata/kernels/fused_gr.hpp"
 
 #include <cuda_runtime.h>
 
@@ -268,6 +269,143 @@ int scalar_activation_contract() {
     return bad;
 }
 
+int fused_multi_lds_parity(const float* d_norm, const uint16_t* d_down, const uint16_t* d_up,
+                           const uint16_t* d_inject, float eps) {
+    using namespace strata::kernels;
+    constexpr int N = 2560, HC = 4, LR = 320, D = N * HC, T = kFusedGrMaxT;
+    std::mt19937 rng(0x6f8a);
+    std::normal_distribution<float> normal(0.0f, 0.3f);
+    std::vector<float> r((size_t) T * D), bo((size_t) T * N), inj((size_t) T * HC);
+    for (auto& x : r) x = normal(rng);
+    for (auto& x : bo) x = normal(rng);
+    for (auto& x : inj) x = normal(rng);
+
+    float *d_r = nullptr, *d_r_out = nullptr, *d_bo = nullptr, *d_inj = nullptr;
+    float *d_lo = nullptr, *d_rs = nullptr, *d_inj_out = nullptr, *d_mixed = nullptr, *d_xn = nullptr;
+    check(cudaMalloc(&d_r, r.size() * sizeof(float)), "multi R");
+    check(cudaMalloc(&d_r_out, r.size() * sizeof(float)), "multi R_out");
+    check(cudaMalloc(&d_bo, bo.size() * sizeof(float)), "multi bo");
+    check(cudaMalloc(&d_inj, inj.size() * sizeof(float)), "multi inj");
+    check(cudaMalloc(&d_lo, (size_t) T * LR * sizeof(float)), "multi lo");
+    check(cudaMalloc(&d_rs, (size_t) T * HC * sizeof(float)), "multi rs");
+    check(cudaMalloc(&d_inj_out, (size_t) T * HC * sizeof(float)), "multi injection");
+    check(cudaMalloc(&d_mixed, (size_t) T * N * sizeof(float)), "multi mixed");
+    check(cudaMalloc(&d_xn, (size_t) T * D * sizeof(float)), "multi xn");
+    check(cudaMemcpy(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy R");
+    check(cudaMemcpy(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy bo");
+    check(cudaMemcpy(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy inj");
+
+    std::vector<FusedGrArgs> args(T);
+    for (int t = 0; t < T; ++t) {
+        auto& a = args[t];
+        a.R = d_r + (size_t) t * D;
+        a.R_out = d_r_out + (size_t) t * D;
+        a.apply = true;
+        a.bo_prev = d_bo + (size_t) t * N;
+        a.inj_prev = d_inj + (size_t) t * HC;
+        a.w_norm = d_norm;
+        a.w_down = d_down;
+        a.w_up = d_up;
+        a.w_inject = d_inject;
+        a.eps = eps;
+        a.lo = d_lo + (size_t) t * LR;
+        a.rs = d_rs + (size_t) t * HC;
+        a.inject_out = d_inj_out + (size_t) t * HC;
+        a.mixed = d_mixed + (size_t) t * N;
+    }
+
+    struct Snapshot {
+        std::vector<float> r_out, lo, rs, inject, mixed;
+    };
+    auto snapshot = [&]() {
+        Snapshot s;
+        s.r_out.resize((size_t) T * D);
+        s.lo.resize((size_t) T * LR);
+        s.rs.resize((size_t) T * HC);
+        s.inject.resize((size_t) T * HC);
+        s.mixed.resize((size_t) T * N);
+        check(cudaMemcpy(s.r_out.data(), d_r_out, s.r_out.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read R_out");
+        check(cudaMemcpy(s.lo.data(), d_lo, s.lo.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read lo");
+        check(cudaMemcpy(s.rs.data(), d_rs, s.rs.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read rs");
+        check(cudaMemcpy(s.inject.data(), d_inj_out, s.inject.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read inject");
+        check(cudaMemcpy(s.mixed.data(), d_mixed, s.mixed.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read mixed");
+        return s;
+    };
+    auto same = [](const Snapshot& a, const Snapshot& b) {
+        return std::memcmp(a.r_out.data(), b.r_out.data(), a.r_out.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.lo.data(), b.lo.data(), a.lo.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.rs.data(), b.rs.data(), a.rs.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.inject.data(), b.inject.data(), a.inject.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.mixed.data(), b.mixed.data(), a.mixed.size() * sizeof(float)) == 0;
+    };
+
+    cudaStream_t stream = nullptr;
+    check(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), "multi stream");
+    // Max T forces the HIP kernel's full dynamic-LDS request: 8 * 1280 * sizeof(float) = 40 KiB.
+    fused_gr_read_multi(args.data(), T, d_xn, stream);
+    check(cudaStreamSynchronize(stream), "multi max-T sync");
+    const Snapshot multi = snapshot();
+    for (int t = 0; t < T; ++t) fused_gr_read(args[t], stream);
+    check(cudaStreamSynchronize(stream), "single reference sync");
+    const Snapshot single = snapshot();
+    int bad = 0;
+    if (!same(multi, single)) {
+        std::printf("  fused GR multi max-T differs from single-token calls\n");
+        ++bad;
+    }
+
+    cudaGraph_t graph = nullptr;
+    cudaGraphExec_t graph_exec = nullptr;
+    check(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal), "multi graph begin");
+    fused_gr_read_multi(args.data(), T, d_xn, stream);
+    check(cudaStreamEndCapture(stream, &graph), "multi graph end");
+    check(cudaGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0), "multi graph instantiate");
+    check(cudaGraphLaunch(graph_exec, stream), "multi graph initial replay");
+    check(cudaStreamSynchronize(stream), "multi graph initial sync");
+    const Snapshot captured = snapshot();
+    if (!same(multi, captured)) {
+        std::printf("  fused GR multi captured graph differs from direct max-T call\n");
+        ++bad;
+    }
+
+    // Reuse the same captured pointers with new payloads; then independently run the single-token path again.
+    for (size_t i = 0; i < r.size(); ++i) r[i] = -0.7f * r[i] + 0.001f * (float) (i % 17);
+    for (size_t i = 0; i < bo.size(); ++i) bo[i] = -0.4f * bo[i] + 0.02f;
+    for (size_t i = 0; i < inj.size(); ++i) inj[i] += 0.3f;
+    check(cudaMemcpyAsync(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay R");
+    check(cudaMemcpyAsync(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay bo");
+    check(cudaMemcpyAsync(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay inj");
+    check(cudaGraphLaunch(graph_exec, stream), "multi graph changed replay");
+    check(cudaStreamSynchronize(stream), "multi graph changed sync");
+    const Snapshot replay = snapshot();
+    if (std::memcmp(multi.mixed.data(), replay.mixed.data(), multi.mixed.size() * sizeof(float)) == 0) {
+        std::printf("  fused GR graph replay ignored changed inputs\n");
+        ++bad;
+    }
+    for (int t = 0; t < T; ++t) fused_gr_read(args[t], stream);
+    check(cudaStreamSynchronize(stream), "changed single reference sync");
+    if (!same(replay, snapshot())) {
+        std::printf("  fused GR changed graph replay differs from single-token calls\n");
+        ++bad;
+    }
+
+    std::printf("  fused GR multi max-T=8 LDS launch and changing graph replay %s\n",
+                bad == 0 ? "pass" : "FAIL");
+    check(cudaGraphExecDestroy(graph_exec), "multi graph exec destroy");
+    check(cudaGraphDestroy(graph), "multi graph destroy");
+    check(cudaStreamDestroy(stream), "multi stream destroy");
+    check(cudaFree(d_xn), "free multi xn");
+    check(cudaFree(d_mixed), "free multi mixed");
+    check(cudaFree(d_inj_out), "free multi injection");
+    check(cudaFree(d_rs), "free multi rs");
+    check(cudaFree(d_lo), "free multi lo");
+    check(cudaFree(d_inj), "free multi inj");
+    check(cudaFree(d_bo), "free multi bo");
+    check(cudaFree(d_r_out), "free multi R_out");
+    check(cudaFree(d_r), "free multi R");
+    return bad;
+}
+
 }  // namespace
 
 int main(int argc, char** argv) {
@@ -602,6 +740,7 @@ int main(int argc, char** argv) {
                         activation_mode_name(mode), ok ? "pass" : "*** FAIL ***", rm, ri);
             if (!ok) ++bad;
         }
+        bad += fused_multi_lds_parity(dN, dD, dU, dJ, eps);
         select_activation_mode(0);
         cudaFree(rws_raw);
         cudaFree(dR); cudaFree(dN); cudaFree(dD); cudaFree(dU); cudaFree(dJ); cudaFree(dM); cudaFree(dI);

src/kernels/kv_hybrid_parity.cpp

@@ -0,0 +1,361 @@
+// src/kernels/kv_hybrid_parity.cpp - hybrid K8V4 KV against host references (GPU, no model).
+//
+// Exercises the EXACT composed calls the engine's hybrid path makes (layer.cpp / prefill.cpp): the q8/q4
+// append and gather kernels invoked with the unused half's pointers folded onto the used pool, and the
+// fused decode attention's KV_MODE 3 (INT8 K, rotated Q4_0 V) followed by the output's inverse Hadamard.
+//   1. K: INT8 codes + FP16 scales bitwise vs the host reference (kv_q8.hpp's rules);
+//   2. V: after fwht256, Q4_0 blocks bitwise vs the host reference (kv_q4.hpp's rules, PR #21);
+//   3. the composed gathers bitwise vs the host dequantization of those codes;
+//   4. qsa_decode_attn<3> + output fwht vs a host attention over the SAME dequantized K/V (fp32 math), and
+//      bounded against the fp32-true attention from the unquantized K/V.
+#include "strata/kernels/f16_bits.hpp"
+#include "strata/kernels/kv_q4.hpp"
+#include "strata/kernels/kv_q8.hpp"
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_decode_attn.hpp"
+#include "strata/kernels/qsa_prompt_attn.hpp"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+int g_fail = 0;
+void ck(cudaError_t e, const char* w) {
+    if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", w, cudaGetErrorString(e)); std::exit(2); }
+}
+template <typename T> T* dalloc(size_t n) {
+    T* p = nullptr;
+    ck(cudaMalloc(&p, n * sizeof(T) + 64), "malloc");
+    ck(cudaMemset(p, 0, n * sizeof(T) + 64), "memset");
+    return p;
+}
+
+// kv_q4_parity.cpp's exact CPU references.
+void fwht256_host_reference(float* x) {
+    constexpr int n = 256;
+    for (int s = 1; s < n; s *= 2) {
+        for (int i = 0; i < n; i += 2 * s) {
+            for (int j = 0; j < s; ++j) {
+                const float u = x[i + j];
+                const float v = x[i + j + s];
+                x[i + j]     = u + v;
+                x[i + j + s] = u - v;
+            }
+        }
+    }
+    const float scale = 1.0f / 16.0f;
+    for (int i = 0; i < n; ++i) x[i] *= scale;
+}
+void quantize_block_q4_0_host(const float* x, k::block_q4_0& blk) {
+    float amax = 0.0f, max_val = 0.0f;
+    for (int i = 0; i < 32; ++i) {
+        const float v = x[i];
+        if (amax < std::fabs(v)) { amax = std::fabs(v); max_val = v; }
+    }
+    const float d = max_val / -8.0f;
+    const float id = d != 0.0f ? 1.0f / d : 0.0f;
+    blk.d = k::f16_from_f32(d);
+    for (int i = 0; i < 16; ++i) {
+        const int q0 = (int) (x[i] * id + 8.5f);
+        const int q1 = (int) (x[i + 16] * id + 8.5f);
+        const uint8_t xi0 = (uint8_t) std::min(15, std::max(0, q0));
+        const uint8_t xi1 = (uint8_t) std::min(15, std::max(0, q1));
+        blk.qs[i] = xi0 | (xi1 << 4);
+    }
+}
+}  // namespace
+
+int main() {
+    std::printf("=== Running kv_hybrid_parity test ===\n");
+    k::QsaShapes s = k::qsa_real_shapes();
+    s.page_size = 64;
+    const int H = (int) s.n_head_kv, D = (int) s.head_dim, P = (int) s.page_size, G = D / k::KV_Q8_GROUP;
+    const int QH = (int) s.n_head, PER = QH / H;   // query heads per KV head
+    const int pages = 24, cells = pages * P;   // n_chunks >= n_head: the batched merge overlaps queries below that
+    std::mt19937 rng(17);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+
+    std::vector<int32_t> table(pages);
+    for (int i = 0; i < pages; ++i) table[i] = (i * 5 + 3) % pages;
+    int32_t* d_table = dalloc<int32_t>(pages);
+    ck(cudaMemcpy(d_table, table.data(), pages * 4, cudaMemcpyHostToDevice), "table");
+
+    std::vector<float> K((size_t) cells * H * D), V((size_t) cells * H * D);
+    for (auto& x : K) x = nd(rng) * 0.5f;
+    for (auto& x : V) x = nd(rng) * 0.5f;
+
+    // ---- host reference: K in INT8 (kv_q8.hpp), rotated V in Q4_0 (kv_q4.hpp)
+    std::vector<int8_t> rk((size_t) cells * H * D);
+    std::vector<uint16_t> rk_s((size_t) cells * H * G);
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int g = 0; g < G; ++g) {
+                float amax = 0.0f;
+                for (int t = 0; t < k::KV_Q8_GROUP; ++t)
+                    amax = std::max(amax, std::fabs(K[((size_t) c * H + h) * D + g * k::KV_Q8_GROUP + t]));
+                const uint16_t sb = k::f16_from_f32(amax / 127.0f);
+                const float sf = k::f32_from_f16(sb);
+                for (int t = 0; t < k::KV_Q8_GROUP; ++t) {
+                    int q = 0;
+                    if (sf > 0.0f) {
+                        q = (int) std::nearbyint(K[((size_t) c * H + h) * D + g * k::KV_Q8_GROUP + t] / sf);   // rn: ties to even, as the kernel
+                        q = std::max(-127, std::min(127, q));
+                    }
+                    rk[((size_t) c * H + h) * D + g * k::KV_Q8_GROUP + t] = (int8_t) q;
+                }
+                rk_s[((size_t) c * H + h) * G + g] = sb;
+            }
+    std::vector<float> Vrot((size_t) cells * H * D);
+    std::memcpy(Vrot.data(), V.data(), V.size() * 4);
+    for (int c = 0; c < cells * H; ++c) fwht256_host_reference(Vrot.data() + (size_t) c * D);
+    std::vector<k::block_q4_0> rv4((size_t) cells * H * (D / 32));
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int b = 0; b < D / 32; ++b)
+                quantize_block_q4_0_host(&Vrot[((size_t) c * H + h) * D + b * 32],
+                                         rv4[((size_t) c * H + h) * (D / 32) + b]);
+
+    // ---- device: one cell appended through the COMPOSED hybrid calls, then gathered
+    const int64_t q4_row = (int64_t) k::kv_q4_bytes_per_head(D);
+    int8_t* d_kq = dalloc<int8_t>((size_t) cells * H * D);
+    uint16_t* d_ks = dalloc<uint16_t>((size_t) cells * H * G);
+    uint8_t* d_v4 = dalloc<uint8_t>((size_t) cells * H * q4_row);
+    float *d_kcur = dalloc<float>((size_t) H * D), *d_vcur = dalloc<float>((size_t) H * D);
+    int32_t* d_step = dalloc<int32_t>(k::kStepCount);
+
+    int append_bad = 0, gather_bad = 0;
+    for (int c = 0; c < cells; ++c) {
+        ck(cudaMemcpy(d_kcur, &K[(size_t) c * H * D], (size_t) H * D * 4, cudaMemcpyHostToDevice), "kcur");
+        ck(cudaMemcpy(d_vcur, &V[(size_t) c * H * D], (size_t) H * D * 4, cudaMemcpyHostToDevice), "vcur");
+        int32_t step[k::kStepCount];
+        k::qsa_step_fill(step, c, s);
+        ck(cudaMemcpy(d_step, step, sizeof step, cudaMemcpyHostToDevice), "step");
+        // exactly what qsa_layer's hybrid branch does
+        k::fwht256_inplace_cuda(d_vcur, H, nullptr);
+        k::kv_append_q8_step(d_kq, d_kq, d_ks, d_ks, d_table, d_step, d_kcur, d_kcur, s, nullptr, nullptr);
+        k::kv_append_q4_step(d_v4, d_v4, d_table, d_step, d_vcur, d_vcur, s, nullptr, nullptr);
+        ck(cudaGetLastError() != cudaSuccess ? cudaGetLastError() : cudaSuccess, "append");
+    }
+    ck(cudaDeviceSynchronize(), "append sync");
+
+    std::vector<int8_t> hk((size_t) cells * H * D);
+    std::vector<uint16_t> hks((size_t) cells * H * G);
+    std::vector<uint8_t> hv4((size_t) cells * H * q4_row);
+    ck(cudaMemcpy(hk.data(), d_kq, hk.size(), cudaMemcpyDeviceToHost), "kq");
+    ck(cudaMemcpy(hks.data(), d_ks, hks.size() * 2, cudaMemcpyDeviceToHost), "ks");
+    ck(cudaMemcpy(hv4.data(), d_v4, hv4.size(), cudaMemcpyDeviceToHost), "v4");
+    // the pools are PHYSICAL (page, head, slot); the reference is by LOGICAL cell - compare through the table
+    const k::block_q4_0* hv4b = reinterpret_cast<const k::block_q4_0*>(hv4.data());
+    for (int c = 0; c < cells; ++c) {
+        const long long row = ((long long) table[c / P] * H);
+        for (int h = 0; h < H; ++h) {
+            const long long r = (row + h) * P + c % P;
+            for (int d = 0; d < D; ++d)
+                if (hk[(size_t) r * D + d] != rk[((size_t) c * H + h) * D + d]) ++append_bad;
+            for (int g = 0; g < G; ++g)
+                if (hks[(size_t) r * G + g] != rk_s[((size_t) c * H + h) * G + g]) ++append_bad;
+            for (int b = 0; b < D / 32; ++b) {
+                const k::block_q4_0& got = hv4b[(size_t) r * (D / 32) + b];
+                if (got.d != rv4[((size_t) c * H + h) * (D / 32) + b].d ||
+                    std::memcmp(got.qs, rv4[((size_t) c * H + h) * (D / 32) + b].qs, 16) != 0) ++append_bad;
+            }
+        }
+    }
+    std::printf("[1/3] composed appends: %s (%d bad values of %zu K / %zu V)\n",
+                append_bad == 0 ? "PASS" : "FAIL", append_bad, hk.size(), rv4.size());
+
+    // gather every cell (ids = all, ascending - the identity selection for width = cells)
+    std::vector<int32_t> ids(cells);
+    for (int i = 0; i < cells; ++i) ids[i] = i;
+    int32_t* d_ids = dalloc<int32_t>(cells);
+    ck(cudaMemcpy(d_ids, ids.data(), cells * 4, cudaMemcpyHostToDevice), "ids");
+    int32_t step[k::kStepCount];
+    k::qsa_step_fill(step, cells - 1, s);
+    step[k::kStepWidth] = cells;   // select everything
+    ck(cudaMemcpy(d_step, step, sizeof step, cudaMemcpyHostToDevice), "step2");
+    uint16_t *d_ksc = dalloc<uint16_t>((size_t) cells * H * D), *d_vsc = dalloc<uint16_t>((size_t) cells * H * D);
+    k::kv_gather_q8_step(d_kq, d_kq, d_ks, d_ks, d_table, d_ids, d_step, cells, s, d_ksc, d_ksc, nullptr);
+    k::kv_gather_q4_step(d_v4, d_v4, d_table, d_ids, d_step, cells, s, d_vsc, d_vsc, nullptr);
+    ck(cudaDeviceSynchronize(), "gather sync");
+    std::vector<uint16_t> h_ksc((size_t) cells * H * D), h_vsc((size_t) cells * H * D);
+    ck(cudaMemcpy(h_ksc.data(), d_ksc, h_ksc.size() * 2, cudaMemcpyDeviceToHost), "ksc");
+    ck(cudaMemcpy(h_vsc.data(), d_vsc, h_vsc.size() * 2, cudaMemcpyDeviceToHost), "vsc");
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int d = 0; d < D; ++d) {
+                const float kref = k::f32_from_f16(k::f16_from_f32(
+                    (float) rk[((size_t) c * H + h) * D + d] * k::f32_from_f16(rk_s[((size_t) c * H + h) * G + d / k::KV_Q8_GROUP])));
+                if (k::f32_from_f16(h_ksc[((size_t) c * H + h) * D + d]) != kref) ++gather_bad;
+                const k::block_q4_0& b = rv4[((size_t) c * H + h) * (D / 32) + d / 32];
+                const int nib = d % 32 < 16 ? (b.qs[d % 32] & 0x0F) - 8 : ((b.qs[d % 32 - 16] >> 4) - 8);
+                const float vref = k::f32_from_f16(k::f16_from_f32((float) nib * k::f32_from_f16(b.d)));
+                if (k::f32_from_f16(h_vsc[((size_t) c * H + h) * D + d]) != vref) ++gather_bad;
+            }
+    std::printf("[2/3] composed gathers: %s (%d bad of %zu)\n",
+                gather_bad == 0 ? "PASS" : "FAIL", gather_bad, h_ksc.size());
+
+    // ---- attention: qsa_decode_attn KV_MODE 3 over the hybrid pools + output fwht, vs fp32 references
+    std::vector<float> q((size_t) QH * D);
+    for (auto& x : q) x = nd(rng);
+    float* d_q = dalloc<float>(q.size());
+    ck(cudaMemcpy(d_q, q.data(), q.size() * 4, cudaMemcpyHostToDevice), "q");
+    k::QsaAttnPools pools;
+    pools.k_q = d_kq; pools.k_scale = d_ks; pools.v_q4 = d_v4; pools.page_table = d_table;
+    float* d_scr = dalloc<float>(k::qsa_decode_attn_scratch_floats(cells, s) + 16);
+    float* d_attn = dalloc<float>((size_t) QH * D);
+    k::qsa_decode_attn_step(d_q, pools, d_ids, d_step, cells, s, d_scr, d_attn, nullptr);
+    k::fwht256_inplace_cuda(d_attn, QH, nullptr);   // the engine's inverse rotation of the output
+    ck(cudaDeviceSynchronize(), "attn sync");
+    std::vector<float> h_attn((size_t) QH * D);
+    ck(cudaMemcpy(h_attn.data(), d_attn, h_attn.size() * 4, cudaMemcpyDeviceToHost), "attn");
+
+    // reference A: attention over the SAME dequantized K/V (isolates the kernel; fp32 host math)
+    // reference B: attention over the TRUE unquantized K/V (bounds the format's error)
+    auto host_attn = [&](const std::vector<float>& Keq, const std::vector<float>& Veq, std::vector<float>& out) {
+        out.assign((size_t) QH * D, 0.0f);
+        for (int h = 0; h < QH; ++h) {
+            const int kvh = h / PER;
+            std::vector<float> sc(cells);
+            float m = -1e30f;
+            for (int c = 0; c < cells; ++c) {
+                float d_ = 0.0f;
+                for (int t = 0; t < D; ++t)
+                    d_ += q[(size_t) h * D + t] * Keq[((size_t) c * H + kvh) * D + t];
+                sc[c] = d_ / std::sqrt((float) D);
+                m = std::max(m, sc[c]);
+            }
+            float l = 0.0f;
+            for (int c = 0; c < cells; ++c) { sc[c] = std::exp(sc[c] - m); l += sc[c]; }
+            for (int c = 0; c < cells; ++c)
+                for (int t = 0; t < D; ++t) out[(size_t) h * D + t] += sc[c] / l * Veq[((size_t) c * H + kvh) * D + t];
+        }
+    };
+    std::vector<float> Kdq((size_t) cells * H * D), Vdq((size_t) cells * H * D);
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int d = 0; d < D; ++d) {
+                Kdq[((size_t) c * H + h) * D + d] =
+                    (float) rk[((size_t) c * H + h) * D + d] * k::f32_from_f16(rk_s[((size_t) c * H + h) * G + d / k::KV_Q8_GROUP]);
+                const k::block_q4_0& b = rv4[((size_t) c * H + h) * (D / 32) + d / 32];
+                const int nib = d % 32 < 16 ? (b.qs[d % 32] & 0x0F) - 8 : ((b.qs[d % 32 - 16] >> 4) - 8);
+                Vdq[((size_t) c * H + h) * D + d] = (float) nib * k::f32_from_f16(b.d);
+            }
+    // un-rotate the dequantized V (the kernel returns the output in the UNROTATED domain)
+    std::vector<float> Vdq_un(Vdq.size());
+    std::memcpy(Vdq_un.data(), Vdq.data(), Vdq.size() * 4);
+    for (int c = 0; c < cells * H; ++c) fwht256_host_reference(Vdq_un.data() + (size_t) c * D);
+
+    // ---- batched form with TWO queries and per-query step arrays (the verify.cpp window's exact call)
+    {
+        const int nq = 2;
+        std::vector<float> q2((size_t) nq * QH * D);
+        for (auto& x : q2) x = nd(rng);
+        float* d_q2 = dalloc<float>(q2.size());
+        ck(cudaMemcpy(d_q2, q2.data(), q2.size() * 4, cudaMemcpyHostToDevice), "q2");
+        // the batched kernel indexes ids[z * cap + ...]: one FULL selection per query
+        std::vector<int32_t> ids2((size_t) nq * cells);
+        for (int i = 0; i < nq; ++i) for (int c = 0; c < cells; ++c) ids2[(size_t) i * cells + c] = c;
+        int32_t* d_ids2 = dalloc<int32_t>((size_t) nq * cells);
+        ck(cudaMemcpy(d_ids2, ids2.data(), ids2.size() * 4, cudaMemcpyHostToDevice), "ids2");
+        int32_t* d_steps2 = dalloc<int32_t>(nq * k::kStepCount);
+        for (int i = 0; i < nq; ++i) {
+            int32_t st2[k::kStepCount];
+            k::qsa_step_fill(st2, cells - 1 - i, s);
+            st2[k::kStepWidth] = cells - i;   // slightly different widths, as verify's windows have
+            ck(cudaMemcpy(d_steps2 + i * k::kStepCount, st2, sizeof st2, cudaMemcpyHostToDevice), "steps2");
+        }
+        float* d_scr2 = dalloc<float>(k::qsa_decode_attn_scratch_floats(cells, s) * nq + 64);
+        float* d_at2 = dalloc<float>((size_t) nq * QH * D);
+        k::qsa_decode_attn_batch(d_q2, pools, d_ids2, d_steps2, cells, s, d_scr2, d_at2, nq, nullptr);
+        k::fwht256_inplace_cuda(d_at2, (int64_t) nq * QH, nullptr);
+        ck(cudaDeviceSynchronize(), "attn2 sync");
+        std::vector<float> h_at2((size_t) nq * QH * D);
+        ck(cudaMemcpy(h_at2.data(), d_at2, h_at2.size() * 4, cudaMemcpyDeviceToHost), "at2");
+        double max_b = 0.0;
+        double per_q[2] = {0.0, 0.0};
+        for (int i = 0; i < nq; ++i)
+            for (int h = 0; h < QH; ++h) {
+                // reference over the cells this query's width selects (ids are ascending: the first width)
+                const int w = cells - i;
+                std::vector<float> sc(w);
+                float m = -1e30f;
+                for (int c = 0; c < w; ++c) {
+                    float d_ = 0.0f;
+                    for (int t = 0; t < D; ++t)
+                        d_ += q2[((size_t) i * QH + h) * D + t] * Kdq[((size_t) c * H + h / PER) * D + t];
+                    sc[c] = d_ / std::sqrt((float) D);
+                    m = std::max(m, sc[c]);
+                }
+                float l = 0.0f;
+                for (int c = 0; c < w; ++c) { sc[c] = std::exp(sc[c] - m); l += sc[c]; }
+                for (int d = 0; d < D; ++d) {
+                    float acc = 0.0f;
+                    for (int c = 0; c < w; ++c)
+                        acc += sc[c] / l * Vdq_un[((size_t) c * H + h / PER) * D + d];   // dim d only
+                    const double e = (double) std::fabs(h_at2[((size_t) i * QH + h) * D + d] - acc);
+                    max_b = std::max(max_b, e);
+                    per_q[i] = std::max(per_q[i], e);
+                }
+            }
+        const bool bok = max_b < 5e-3;
+        std::printf("       (per-query max: q0 %.2e, q1 %.2e)\n", per_q[0], per_q[1]);
+        std::printf("[4/4] mode-3 BATCHED attention (verify's form): %s (vs dequant ref %.2e)\n",
+                    bok ? "PASS" : "FAIL", max_b);
+        if (!bok) g_fail = 1;
+    }
+
+    std::vector<float> ref_deq, ref_true;
+    host_attn(Kdq, Vdq_un, ref_deq);
+    host_attn(K, V, ref_true);
+    double max_deq = 0.0, max_true = 0.0;
+    for (size_t i = 0; i < h_attn.size(); ++i) {
+        max_deq = std::max(max_deq, (double) std::fabs(h_attn[i] - ref_deq[i]));
+        max_true = std::max(max_true, (double) std::fabs(h_attn[i] - ref_true[i]));
+    }
+    // The tensor-core prompt path (qsa_prompt_attn, 0.1.22+) takes hybrid pools as KV_MODE 3: INT8 K, and V
+    // dequantized from its q4_0 blocks to fp16 at gather. Its summation order differs from the old kernel's
+    // (accuracy-level, not bitwise - qsa_prompt_attn.hpp says the same of its int8 mode), so: tolerance check
+    // against the SAME dequant reference, output un-rotated exactly as prefill.cpp does.
+    {
+        float* d_at4 = dalloc<float>((size_t) QH * D);
+        const bool took = k::qsa_prompt_attn_batch(d_q, pools, d_ids, d_step, cells, s, d_at4, 1, nullptr);
+        if (!took) {
+#if defined(STRATA_USE_HIP)
+            // AMD: the tensor-core prompt path is CUDA-only, so it refuses every pool and the old kernel runs
+            std::printf("[5/5] qsa_prompt_attn mode 3: PASS (refused on HIP - the old kernel runs)\n");
+#else
+            std::printf("[5/5] qsa_prompt_attn mode 3: FAIL (refused the hybrid pools)\n");
+            g_fail = 1;
+#endif
+        } else {
+            k::fwht256_inplace_cuda(d_at4, QH, nullptr);
+            ck(cudaDeviceSynchronize(), "at4 sync");
+            std::vector<float> h_at4((size_t) QH * D);
+            ck(cudaMemcpy(h_at4.data(), d_at4, h_at4.size() * 4, cudaMemcpyDeviceToHost), "at4");
+            double max_p = 0.0;
+            for (size_t i = 0; i < h_at4.size(); ++i)
+                max_p = std::max(max_p, (double) std::fabs(h_at4[i] - ref_deq[i]));
+            const bool pok = max_p < 1e-2;
+            std::printf("[5/5] qsa_prompt_attn mode 3 (tensor cores): %s (vs dequant ref %.2e)\n",
+                        pok ? "PASS" : "FAIL", max_p);
+            if (!pok) g_fail = 1;
+        }
+    }
+
+    const bool attn_ok = max_deq < 5e-3;   // fp32 kernel math vs fp64-ish host accumulation
+    std::printf("[3/3] mode-3 attention + output fwht: %s (vs dequant ref %.2e, vs true fp32 %.2e)\n",
+                attn_ok ? "PASS" : "FAIL", max_deq, max_true);
+
+    const bool ok = append_bad == 0 && gather_bad == 0 && attn_ok && g_fail == 0;
+    std::printf("=== kv_hybrid_parity: %s ===\n", ok ? "ALL PASS" : "FAILED");
+    return ok ? 0 : 1;
+}

src/kernels/qsa_prompt_attn_parity.cpp

@@ -0,0 +1,209 @@
+// src/kernels/qsa_prompt_attn_parity.cpp - perf-review D-1: the tensor-core prompt attention (qsa_prompt_attn.hpp)
+// against the FP32 kernel it replaces (`qsa_decode_attn_batch`) and an FP64 host reference (GPU, synthetic, no model).
+//
+// Int8 and FP16 pools with random codes, scales and queries; selections shaped like the prompt path's (the 2,051
+// widest, a recent window plus older cells that drift slowly from one query to the next, so neighbours share most
+// of them as they do in a real prompt; short contexts take every cell). Checks:
+//   1. against FP64, the new kernel's error is no larger than a small multiple of the old kernel's (both FP32 math);
+//   2. the new and old outputs agree to a relative 1e-4 of the output scale;
+// then times both over a prompt chunk (the old one in batches of 32, as prefill.cpp calls it).
+// Usage: qsa_prompt_attn_parity [context=32768] [queries=2048] [reps=5]
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_decode_attn.hpp"
+#include "strata/kernels/qsa_prompt_attn.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+void ck(cudaError_t e, const char* w) {
+    if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", w, cudaGetErrorString(e)); std::exit(2); }
+}
+template <typename T> T* up(const std::vector<T>& h) {
+    T* d = nullptr;
+    ck(cudaMalloc(&d, h.size() * sizeof(T) + 64), "malloc");
+    ck(cudaMemcpy(d, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "upload");
+    return d;
+}
+float h2f(uint16_t b) { __half h; *reinterpret_cast<uint16_t*>(&h) = b; return __half2float(h); }
+uint16_t f2h(float f) { __half h = __float2half(f); return *reinterpret_cast<uint16_t*>(&h); }
+
+int run(int fmt, int64_t ctx, int64_t nq, int reps) {   // fmt 1 int8, 0 fp16
+    const k::QsaShapes s = k::qsa_real_shapes();
+    const int64_t HD = s.head_dim, NKV = s.n_head_kv, NH = s.n_head, PS = s.page_size;
+    const int64_t pages = (ctx + PS - 1) / PS, rows = pages * NKV * PS;
+    std::mt19937 rng(1234 + fmt);
+    std::normal_distribution<float> nd(0.f, 1.f);
+    std::uniform_int_distribution<int> code(-127, 127);
+    std::uniform_real_distribution<float> sc(0.005f, 0.03f);
+    // pools; page table a shuffled permutation (the readers must follow it)
+    std::vector<int8_t> kq, vq;
+    std::vector<uint16_t> ks, vs, kh, vh;
+    if (fmt == 1) {
+        kq.resize(rows * HD); vq.resize(rows * HD); ks.resize(rows * 4); vs.resize(rows * 4);
+        for (auto& x : kq) x = (int8_t) code(rng);
+        for (auto& x : vq) x = (int8_t) code(rng);
+        for (auto& x : ks) x = f2h(sc(rng));
+        for (auto& x : vs) x = f2h(sc(rng));
+    } else {
+        kh.resize(rows * HD); vh.resize(rows * HD);
+        for (auto& x : kh) x = f2h(nd(rng) * 1.5f);
+        for (auto& x : vh) x = f2h(nd(rng));
+    }
+    std::vector<int32_t> table(pages);
+    for (int64_t i = 0; i < pages; ++i) table[i] = (int32_t) i;
+    std::shuffle(table.begin(), table.end(), rng);
+    // queries at positions ctx - nq .. ctx - 1
+    const int64_t cap = k::qsa_selection_width(ctx, s);
+    std::vector<int32_t> ids((size_t) (nq * cap), 0), steps((size_t) (nq * k::kStepCount), 0);
+    std::vector<float> q((size_t) (nq * NH * HD));
+    for (auto& x : q) x = nd(rng) * 2.0f;
+    std::vector<int32_t> old_cells;
+    for (int64_t i = 0; i < nq; ++i) {
+        const int64_t pos = ctx - nq + i, nkv = pos + 1;
+        const int64_t w = k::qsa_selection_width(nkv, s);
+        int32_t* sel = ids.data() + i * cap;
+        steps[i * k::kStepCount + k::kStepWidth] = (int32_t) w;
+        if (w == nkv) {
+            for (int64_t c = 0; c < w; ++c) sel[c] = (int32_t) c;
+            continue;
+        }
+        const int64_t recent = 512, older = w - recent;
+        if ((int64_t) old_cells.size() != older) {   // the older cells: a set that drifts ~3% per query
+            std::vector<int32_t> all((size_t) (nkv - recent));
+            for (int64_t c = 0; c < nkv - recent; ++c) all[c] = (int32_t) c;
+            std::shuffle(all.begin(), all.end(), rng);
+            old_cells.assign(all.begin(), all.begin() + older);
+        } else {
+            std::uniform_int_distribution<int64_t> pick(0, older - 1), any(0, nkv - recent - 1);
+            for (int r = 0; r < older / 32; ++r) {
+                const int32_t c = (int32_t) any(rng);
+                if (std::find(old_cells.begin(), old_cells.end(), c) == old_cells.end()) old_cells[pick(rng)] = c;
+            }
+        }
+        std::vector<int32_t> v(old_cells);
+        for (int64_t c = nkv - recent; c < nkv; ++c) v.push_back((int32_t) c);
+        std::sort(v.begin(), v.end());
+        std::copy(v.begin(), v.end(), sel);
+    }
+    k::QsaAttnPools pl;
+    if (fmt == 1) { pl.k_q = up(kq); pl.v_q = up(vq); pl.k_scale = up(ks); pl.v_scale = up(vs); }
+    else { pl.k_pool = up(kh); pl.v_pool = up(vh); }
+    pl.page_table = up(table);
+    const int32_t* d_ids = up(ids);
+    const int32_t* d_steps = up(steps);
+    const float* d_q = up(q);
+    float *d_old = nullptr, *d_new = nullptr, *scratch = nullptr;
+    const int64_t batch = 32;
+    ck(cudaMalloc(&d_old, nq * NH * HD * 4), "malloc");
+    ck(cudaMalloc(&d_new, nq * NH * HD * 4), "malloc");
+    ck(cudaMalloc(&scratch, batch * k::qsa_decode_attn_scratch_floats(cap, s) * 4), "malloc");
+    auto old_run = [&]() {
+        for (int64_t t0 = 0; t0 < nq; t0 += batch)
+            k::qsa_decode_attn_batch(d_q + t0 * NH * HD, pl, d_ids + t0 * cap, d_steps + t0 * k::kStepCount, cap, s,
+                                     scratch, d_old + t0 * NH * HD, std::min(batch, nq - t0), nullptr);
+    };
+    auto new_run = [&]() {
+        if (!k::qsa_prompt_attn_batch(d_q, pl, d_ids, d_steps, cap, s, d_new, nq, nullptr)) {
+            std::fprintf(stderr, "qsa_prompt_attn_batch refused the pools\n");
+            std::exit(2);
+        }
+    };
+    old_run();
+    new_run();
+    ck(cudaDeviceSynchronize(), "run");
+    std::vector<float> o((size_t) (nq * NH * HD)), nw(o.size());
+    ck(cudaMemcpy(o.data(), d_old, o.size() * 4, cudaMemcpyDeviceToHost), "down");
+    ck(cudaMemcpy(nw.data(), d_new, nw.size() * 4, cudaMemcpyDeviceToHost), "down");
+    // 1. FP64 reference on a sample of queries
+    double err_old = 0, err_new = 0, ref_scale = 0;
+    for (int64_t i = 0; i < nq; i += std::max<int64_t>(1, nq / 16)) {
+        const int64_t w = steps[i * k::kStepCount + k::kStepWidth];
+        const int32_t* sel = ids.data() + i * cap;
+        for (int64_t h = 0; h < NH; ++h) {
+            const int64_t kvh = h / (NH / NKV);
+            std::vector<double> sco((size_t) w);
+            double mx = -1e300;
+            for (int64_t c = 0; c < w; ++c) {
+                const int64_t cell = sel[c], row = ((int64_t) table[cell / PS] * NKV + kvh) * PS + cell % PS;
+                double a = 0;
+                for (int64_t d = 0; d < HD; ++d) {
+                    const double kv = fmt == 1 ? (double) kq[row * HD + d] * h2f(ks[row * 4 + d / 64]) : h2f(kh[row * HD + d]);
+                    a += (double) q[(i * NH + h) * HD + d] * kv;
+                }
+                sco[c] = a / 16.0;
+                mx = std::max(mx, sco[c]);
+            }
+            double l = 0;
+            for (auto& x : sco) { x = std::exp(x - mx); l += x; }
+            for (int64_t d = 0; d < HD; ++d) {
+                double a = 0;
+                for (int64_t c = 0; c < w; ++c) {
+                    const int64_t cell = sel[c], row = ((int64_t) table[cell / PS] * NKV + kvh) * PS + cell % PS;
+                    const double vv = fmt == 1 ? (double) vq[row * HD + d] * h2f(vs[row * 4 + d / 64]) : h2f(vh[row * HD + d]);
+                    a += sco[c] * vv;
+                }
+                const double r = a / l;
+                const size_t at = (size_t) ((i * NH + h) * HD + d);
+                ref_scale = std::max(ref_scale, std::fabs(r));
+                err_old = std::max(err_old, std::fabs(o[at] - r));
+                err_new = std::max(err_new, std::fabs(nw[at] - r));
+            }
+        }
+    }
+    // 2. new vs old everywhere
+    double diff = 0, scale = 0;
+    for (size_t i = 0; i < o.size(); ++i) {
+        diff = std::max(diff, (double) std::fabs(o[i] - nw[i]));
+        scale = std::max(scale, (double) std::fabs(o[i]));
+    }
+    // 3. speed
+    cudaEvent_t e0, e1;
+    cudaEventCreate(&e0);
+    cudaEventCreate(&e1);
+    float ms_old = 0, ms_new = 0;
+    cudaEventRecord(e0);
+    for (int r = 0; r < reps; ++r) old_run();
+    cudaEventRecord(e1);
+    ck(cudaEventSynchronize(e1), "time");
+    cudaEventElapsedTime(&ms_old, e0, e1);
+    cudaEventRecord(e0);
+    for (int r = 0; r < reps; ++r) new_run();
+    cudaEventRecord(e1);
+    ck(cudaEventSynchronize(e1), "time");
+    cudaEventElapsedTime(&ms_new, e0, e1);
+    const bool ok1 = err_new <= std::max(4.0 * err_old, 1e-6 * ref_scale);
+    const bool ok2 = diff <= 1e-4 * scale;
+    std::printf("%s %s ctx %lld, %lld queries: vs FP64 old %.3g new %.3g (output scale %.3g); new vs old %.3g (%.2g of "
+                "scale); %.3f -> %.3f ms per chunk (%.2fx)\n",
+                ok1 && ok2 ? "PASS" : "FAIL", fmt == 1 ? "int8" : "fp16", (long long) ctx, (long long) nq, err_old,
+                err_new, ref_scale, diff, diff / scale, ms_old / reps, ms_new / reps, ms_old / ms_new);
+    cudaFree((void*) d_ids); cudaFree((void*) d_steps); cudaFree((void*) d_q); cudaFree(d_old); cudaFree(d_new);
+    cudaFree(scratch);
+    cudaFree((void*) pl.k_q); cudaFree((void*) pl.v_q); cudaFree((void*) pl.k_scale); cudaFree((void*) pl.v_scale);
+    cudaFree((void*) pl.k_pool); cudaFree((void*) pl.v_pool); cudaFree((void*) pl.page_table);
+    return ok1 && ok2 ? 0 : 1;
+}
+}  // namespace
+
+int main(int argc, char** argv) {
+    const int64_t ctx = argc > 1 ? std::atoll(argv[1]) : 32768;
+    const int64_t nq = argc > 2 ? std::atoll(argv[2]) : 2048;
+    const int reps = argc > 3 ? std::atoi(argv[3]) : 5;
+    int fails = 0;
+    fails += run(1, ctx, nq, reps);
+    fails += run(0, ctx, nq, reps);
+    fails += run(1, 1500, std::min<int64_t>(nq, 1500), reps);   // short context: the selection is every cell
+    fails += run(1, 2100, std::min<int64_t>(nq, 256), reps);    // the identity-to-sparse edge
+    std::printf("FAILURES: %d\n", fails);
+    return fails;
+}