Engine 0.1.28: the expert cache reserves the draft head's VRAM (0.1.27 left 16 GB cards below the stall line), a cancelled request no longer fails the next one, the API key protects /status; setup requires it

Squashed from PR #153.

CMakeLists.txt

@@ -4,11 +4,11 @@
 # `strata-pack` are added when their sources exist - a target with no sources would make the build graph
 # claim something the tree does not have, which is worse than an absent target.
 #
-# CUDA targets arrive in Phase 2 and this build REJECTS any architecture other than 120: the engine is
-# written against sm_120 (RTX 5000 / Blackwell) and silently building for an older arch would produce a
-# binary that runs and is wrong about its own performance, which is the worst of both outcomes.
+# CUDA targets arrive in Phase 2.  The kernels need sm_80 or newer (tf32 mma, bf16 math) and the guard
+# below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
+# and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.22 LANGUAGES CXX)
+project(strata VERSION 0.1.28 LANGUAGES CXX)
 add_compile_definitions(STRATA_VERSION="${PROJECT_VERSION}")
 # issue #31: release builds keep a PDB beside the exe (not shipped in it), so a minidump from a stalled engine can
 # be read back to the exact source line
@@ -33,6 +33,17 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 option(STRATA_ENABLE_CUDA "Build the CUDA targets (Phase 2+)" OFF)
+option(STRATA_ENABLE_HIP "Build the HIP targets for gfx1100 wave32 hardware" OFF)
+option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF)
+if(STRATA_PREFILL_MMQ AND NOT STRATA_ENABLE_HIP)
+  message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_ENABLE_HIP=ON")
+endif()
+if(STRATA_ENABLE_CUDA AND STRATA_ENABLE_HIP)
+  message(FATAL_ERROR "STRATA_ENABLE_CUDA and STRATA_ENABLE_HIP are separate backends; enable only one")
+endif()
+if(STRATA_ENABLE_HIP)
+  include(cmake/hip_backend.cmake)
+endif()
 # The test suite and the micro benchmarks live in tests/ and bench/micro/, which the published source leaves out.
 set(_strata_tests_default OFF)
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/bench/micro)
@@ -51,18 +62,26 @@ if(STRATA_ENABLE_CUDA)
   endif()
   foreach(_a IN LISTS CMAKE_CUDA_ARCHITECTURES)
     string(REGEX REPLACE "-.*$" "" _base "${_a}")     # 120-real / 120-virtual are still 120
-    # Plan v0.3 P6: developed and measured on sm_120 (RTX 50); the kernels need sm_80 or newer (tf32 mma in
-    # the attention scorer, bf16 math), so RTX 30 (86) and RTX 40 (89) build too.  Older cards are refused.
+    # Turing port: developed and measured on sm_120 (RTX 50); the tf32 mma in the QSA scorer has a
+    # portable fp32-FMA fallback, so RTX 20 (75), 30 (86), RTX 40 (89) and RTX 50 (120) build too.
+    # Pre-sm_75 is refused.
     if(_base STREQUAL "native" OR _base STREQUAL "all" OR _base STREQUAL "all-major")
       continue()
     endif()
-    if(_base LESS 80 AND NOT STRATA_EXPERIMENTAL_SM75)   # the A/B of a Turing card as a layer-split stage
+    if(_base LESS 75)
       message(FATAL_ERROR
-        "Strata needs an NVIDIA GPU of compute capability 8.0 or newer (RTX 30 / 40 / 50 series), but "
+        "Strata needs an NVIDIA GPU of compute capability 7.5 or newer (RTX 20 / 30 / 40 / 50 series), but "
         "CMAKE_CUDA_ARCHITECTURES is '${CMAKE_CUDA_ARCHITECTURES}'.")
     endif()
   endforeach()
   message(STATUS "Strata: CUDA enabled, arch ${CMAKE_CUDA_ARCHITECTURES}")
+  set(_strata_gpu_runtime_target CUDA::cudart)
+  set(_strata_gpu_blas_target CUDA::cublas)
+  set(_strata_gpu_include_directories ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+elseif(STRATA_ENABLE_HIP)
+  set(_strata_gpu_runtime_target strata_hip_runtime)
+  set(_strata_gpu_blas_target roc::hipblas)
+  set(_strata_gpu_include_directories ${STRATA_HIP_COMPAT_INCLUDE_DIR})
 endif()
 
 # Warnings live on an INTERFACE target the project's own code links, NOT in add_compile_options: applied
@@ -78,11 +97,14 @@ target_compile_options(strata_warnings INTERFACE
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/utf-8>
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Wall>
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Wextra>
-  $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CXX_COMPILER_ID:MSVC>>:-Xcompiler=/W4>)
+  $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CXX_COMPILER_ID:MSVC>>:-Xcompiler=/W4>
+  $<$<COMPILE_LANGUAGE:HIP>:-Wall>
+  $<$<COMPILE_LANGUAGE:HIP>:-Wextra>)
 if(STRATA_WERROR)
   target_compile_options(strata_warnings INTERFACE
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/WX>
-    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Werror>)
+    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Werror>
+    $<$<COMPILE_LANGUAGE:HIP>:-Werror>)
 endif()
 
 # ---- strata_artifact -------------------------------------------------------
@@ -100,6 +122,12 @@ add_executable(strata-dequant src/artifact/dequant.cpp)
 target_link_libraries(strata-dequant PRIVATE strata_artifact)
 # this file is now nothing but the CLI entry point, and its inherited guard spells that as a "selftest"
 target_compile_definitions(strata-dequant PRIVATE STRATA_DEQUANT_SELFTEST)
+if(STRATA_BUILD_TESTS)
+  # the reader refuses a duplicate tensor name at open, naming it (a synthetic GGUF: no model, no GPU)
+  add_executable(gguf_reader_test src/artifact/gguf_reader_test.cpp)
+  target_link_libraries(gguf_reader_test PRIVATE strata_artifact)
+  add_test(NAME gguf_reader_test COMMAND gguf_reader_test)
+endif()
 
 # ---- strata_plan ----------------------------------------------------------
 # Also header-only.  A "planner" that hides state would be one whose decision is not inspectable, and the
@@ -111,17 +139,16 @@ target_link_libraries(strata_plan INTERFACE strata_warnings)
 add_executable(strata-plan src/plan/plan_main.cpp)
 target_link_libraries(strata-plan PRIVATE strata_plan)
 
-# ---- strata_core (CUDA, Phase 2) ------------------------------------------
-# Only built when CUDA is on, so the Phase 1 targets keep working on a machine with no toolkit.  The arch
-# guard above has already refused anything but sm_120 by the time this runs.
-if(STRATA_ENABLE_CUDA)
+# ---- strata_core (GPU runtime) ---------------------------------------------
+# Only built when a GPU backend is on, so the Phase 1 targets keep working without a toolkit.
+if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   add_library(strata_core STATIC src/core/device.cu src/core/pinned.cu src/platform/memory.cpp src/core/graph.cpp src/core/weights.cpp
                                src/core/layout.cpp)
   target_include_directories(strata_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-  # CUDA::cudart is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and
+  # The selected runtime is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and
   # cudaEventQuery: the .cu sources get cudart through the CUDA language rules, a .cpp does not, and the
   # linker error ("unresolved external symbol cudaGraphInstantiate") names the function rather than the cause.
-  target_link_libraries(strata_core PUBLIC strata_plan strata_warnings CUDA::cudart)
+  target_link_libraries(strata_core PUBLIC strata_plan strata_warnings ${_strata_gpu_runtime_target})
 
   add_executable(strata-device src/core/device_main.cpp)
   target_link_libraries(strata-device PRIVATE strata_core)
@@ -130,13 +157,13 @@ if(STRATA_ENABLE_CUDA)
   target_link_libraries(strata-load PRIVATE strata_core strata_kernels)
 
   add_executable(strata-concurrent src/core/concurrent_main.cpp)
-  target_include_directories(strata-concurrent PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels CUDA::cudart)
+  target_include_directories(strata-concurrent PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target})
 
   add_executable(strata-overlap src/core/overlap_main.cpp)
   # a .cpp that calls cudaMemcpyAsync itself needs the toolkit headers and cudart explicitly
-  target_include_directories(strata-overlap PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels CUDA::cudart)
+  target_include_directories(strata-overlap PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target})
 
   # The load test needs the 34 GB artifact, so it is registered ONLY where that artifact exists.  A test that
   # silently skips is a test that stops running; an absent registration is visible in the configure output.
@@ -209,30 +236,38 @@ if(STRATA_ENABLE_CUDA)
   # plan v0.3 P6: llama.cpp's block layouts and codebook grids for the i-quant formats (MIT, third_party/ggml)
   target_include_directories(strata_kernels PRIVATE third_party/ggml)
   # Match the pinned llama.cpp CUDA arithmetic contract for this adapter only.
-  set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu
-    src/kernels/cuda/native_gr_norm.cu
-    src/kernels/cuda/native_moe.cu
-    src/kernels/cuda/native_gdn.cu
-    src/kernels/cuda/native_gr_postops.cu
-    src/kernels/cuda/native_router.cu
-    src/kernels/cuda/native_gdn_preprocess.cu
-    src/kernels/cuda/native_qsa.cu
-    src/kernels/cuda/native_rope.cu
-    src/kernels/cuda/native_ple_postops.cu
-    src/kernels/cuda/native_flash_attn.cu
-    src/kernels/cuda/native_qsa_indexer.cu
-    src/kernels/cuda/native_qsa_score.cu
-    PROPERTIES COMPILE_OPTIONS "--use_fast_math")
+  if(STRATA_ENABLE_CUDA)
+    set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu
+      src/kernels/cuda/native_gr_norm.cu
+      src/kernels/cuda/native_moe.cu
+      src/kernels/cuda/native_gdn.cu
+      src/kernels/cuda/native_gr_postops.cu
+      src/kernels/cuda/native_router.cu
+      src/kernels/cuda/native_gdn_preprocess.cu
+      src/kernels/cuda/native_qsa.cu
+      src/kernels/cuda/native_rope.cu
+      src/kernels/cuda/native_ple_postops.cu
+      src/kernels/cuda/native_flash_attn.cu
+      src/kernels/cuda/native_qsa_indexer.cu
+      src/kernels/cuda/native_qsa_score.cu
+      PROPERTIES COMPILE_OPTIONS "--use_fast_math")
+  elseif(STRATA_ENABLE_HIP)
+    # Keep Q8_K's multiply and magic-bias rounding as separate FP32 operations on HIP; contraction to an
+    # FMA changes tie results. The embedding gather also relies on separately rounded operations.
+    set_source_files_properties(src/kernels/cuda/elementwise.cu src/kernels/cuda/quantize_act.cu
+      PROPERTIES COMPILE_OPTIONS "-ffp-contract=off")
+  endif()
 
   set(STRATA_ORACLE_SOURCE_DIR "" CACHE PATH "Pinned llama.cpp source for independent CUDA parity tests")
   set(STRATA_ORACLE_BUILD_DIR "" CACHE PATH "Matching existing shared llama.cpp build")
-  if(STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR)
+  # This oracle is a CUDA ggml build; do not pull CUDA libraries into a HIP build.
+  if(STRATA_ENABLE_CUDA AND STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR)
     foreach(parity IN ITEMS native_mmvq native_bf16 native_shared native_gr_norm native_moe native_gdn native_gr_postops native_router native_gdn_preprocess native_gdn_pipeline native_qsa native_rope native_ple_postops native_flash_attn native_qsa_indexer native_qsa_score)
     add_executable(${parity}_parity bench/micro/${parity}_parity.cpp)
     target_include_directories(${parity}_parity PRIVATE
-      "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+      "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${_strata_gpu_include_directories})
     target_compile_definitions(${parity}_parity PRIVATE GGML_SHARED GGML_BACKEND_SHARED)
-    target_link_libraries(${parity}_parity PRIVATE strata_kernels CUDA::cudart)
+    target_link_libraries(${parity}_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
     foreach(component IN ITEMS ggml ggml-base ggml-cpu ggml-cuda)
       string(REPLACE "-" "_" variable "STRATA_ORACLE_${component}")
       find_library(${variable} NAMES ${component} PATHS
@@ -256,99 +291,163 @@ if(STRATA_ENABLE_CUDA)
   add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
     src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-  target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu CUDA::cudart)
+  target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/file_expert_source_test.cpp")
+    enable_testing()
+    add_executable(file_expert_source_test tests/core/file_expert_source_test.cpp)
+    target_link_libraries(file_expert_source_test PRIVATE strata_engine)
+    add_test(NAME file_expert_source_test COMMAND file_expert_source_test)
+    set_tests_properties(file_expert_source_test PROPERTIES TIMEOUT 60)
+  endif()
 
   # `strata generate` (P2.S6).  The FIRST user-facing program: everything under it is a component, and this is
   # what composes them into a token.  No tokenizer - the prompt arrives as ids - because Gate C1 compares logits
   # against llama.cpp on the SAME ids and a tokenizer on one side of that comparison is a second variable.
   # plan v0.3 P5: batched prompt processing (tensor-core GEMMs through cuBLAS).
   add_library(strata_prefill STATIC src/prefill/gemm.cu src/prefill/kernels.cu src/prefill/prefill.cpp)
-  target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata_prefill PUBLIC strata_engine CUDA::cublas CUDA::cudart)
+  target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})
+  target_link_libraries(strata_prefill PUBLIC strata_engine ${_strata_gpu_blas_target} ${_strata_gpu_runtime_target})
+  if(STRATA_ENABLE_HIP AND STRATA_HIPBLASLT_AVAILABLE)
+    target_compile_definitions(strata_prefill PRIVATE STRATA_HIPBLASLT_AVAILABLE=1)
+    target_link_libraries(strata_prefill PRIVATE roc::hipblaslt)
+
+    add_executable(tune_hipblaslt tools/hip/tune_hipblaslt.cpp)
+    target_link_libraries(tune_hipblaslt PRIVATE hip::host roc::hipblas roc::hipblaslt)
+  endif()
 
   add_executable(strata src/program/generate.cpp)
-  target_include_directories(strata PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec CUDA::cudart)
+  target_include_directories(strata PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec ${_strata_gpu_runtime_target})
+
+  if(STRATA_ENABLE_HIP)
+    enable_testing()
+    set(_strata_hip_tests)
+    add_executable(hip_expert_cache_staging tests/hip/expert_cache_staging.cpp)
+    target_link_libraries(hip_expert_cache_staging PRIVATE strata_engine)
+    add_test(NAME hip_expert_cache_staging COMMAND hip_expert_cache_staging)
+    list(APPEND _strata_hip_tests hip_expert_cache_staging)
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/intrinsics.cpp")
+      add_executable(hip_intrinsics tests/hip/intrinsics.cpp)
+      target_link_libraries(hip_intrinsics PRIVATE strata_hip_runtime)
+      add_test(NAME hip_intrinsics COMMAND hip_intrinsics)
+      list(APPEND _strata_hip_tests hip_intrinsics)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/handoff.cpp")
+      add_executable(hip_handoff tests/hip/handoff.cpp)
+      target_link_libraries(hip_handoff PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_handoff COMMAND hip_handoff)
+      list(APPEND _strata_hip_tests hip_handoff)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/native_qsa_score.cpp")
+      add_executable(hip_native_qsa_score tests/hip/native_qsa_score.cpp)
+      target_link_libraries(hip_native_qsa_score PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_native_qsa_score COMMAND hip_native_qsa_score)
+      list(APPEND _strata_hip_tests hip_native_qsa_score)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_native_batch.cpp")
+      add_executable(hip_prefill_native_batch tests/hip/prefill_native_batch.cpp)
+      target_link_libraries(hip_prefill_native_batch PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_prefill_native_batch COMMAND hip_prefill_native_batch)
+      list(APPEND _strata_hip_tests hip_prefill_native_batch)
+    endif()
+    if(STRATA_HIPBLASLT_AVAILABLE AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_hipblaslt_gemm.cpp")
+      add_executable(hip_prefill_hipblaslt_gemm tests/hip/prefill_hipblaslt_gemm.cpp)
+      target_include_directories(hip_prefill_hipblaslt_gemm PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/prefill")
+      target_link_libraries(hip_prefill_hipblaslt_gemm PRIVATE strata_prefill strata_hip_runtime roc::hipblaslt roc::hipblas)
+      add_test(NAME hip_prefill_hipblaslt_gemm COMMAND hip_prefill_hipblaslt_gemm)
+      set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77)
+      list(APPEND _strata_hip_tests hip_prefill_hipblaslt_gemm)
+    endif()
+    if(_strata_hip_tests)
+      set_tests_properties(${_strata_hip_tests} PROPERTIES TIMEOUT 60)
+    endif()
+    if(TARGET hip_prefill_native_batch)
+      set_tests_properties(hip_prefill_native_batch PROPERTIES TIMEOUT 120)
+    endif()
+    if(TARGET hip_prefill_hipblaslt_gemm)
+      set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77)
+    endif()
+  endif()
 
   add_executable(dequant_s2_parity src/kernels/dequant_s2_parity.cpp)
   # The tests are HOST C++ files that call cudaMalloc/cudaMemcpy themselves, so they need the toolkit headers
   # and cudart explicitly: enabling the CUDA language gives those to .cu sources automatically, but a .cpp
   # target is just a C++ target and gets neither.
-  target_include_directories(dequant_s2_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(dequant_s2_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(dequant_s2_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(dequant_s2_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME dequant_s2_parity COMMAND dequant_s2_parity --selftest)
 
   add_executable(s2_gemv_parity src/kernels/s2_gemv_parity.cpp)
-  target_include_directories(s2_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s2_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s2_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s2_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s2_gemv_parity COMMAND s2_gemv_parity --selftest)
 
   add_executable(iq_parity src/kernels/iq_parity.cpp)
-  target_include_directories(iq_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(iq_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(iq_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(iq_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)
-  target_include_directories(shared_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(shared_expert_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME shared_expert_parity COMMAND shared_expert_parity --selftest)
 
   add_executable(gr_parity src/kernels/gr_parity.cpp)
-  target_include_directories(gr_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(gr_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(gr_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(gr_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME gr_parity COMMAND gr_parity --selftest)
 
   add_executable(gdn_parity src/kernels/gdn_parity.cpp)
-  target_include_directories(gdn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(gdn_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(gdn_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(gdn_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME gdn_parity COMMAND gdn_parity --selftest)
 
   add_executable(s2_gemv_q8_parity src/kernels/s2_gemv_q8_parity.cpp)
-  target_include_directories(s2_gemv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s2_gemv_q8_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s2_gemv_q8_parity COMMAND s2_gemv_q8_parity --selftest)
 
   add_executable(sampler_parity src/kernels/sampler_parity.cpp)
-  target_include_directories(sampler_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(sampler_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(sampler_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(sampler_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME sampler_parity COMMAND sampler_parity --selftest)
 
   add_executable(rope_parity src/kernels/rope_parity.cpp)
-  target_include_directories(rope_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(rope_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(rope_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(rope_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME rope_parity COMMAND rope_parity --selftest)
 
   add_executable(quantize_act_parity src/kernels/quantize_act_parity.cpp)
-  target_include_directories(quantize_act_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(quantize_act_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(quantize_act_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(quantize_act_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME quantize_act_parity COMMAND quantize_act_parity --selftest)
 
   add_executable(router_top10_parity src/kernels/router_top10_parity.cpp)
-  target_include_directories(router_top10_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(router_top10_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(router_top10_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(router_top10_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME router_top10_parity COMMAND router_top10_parity --selftest)
 
   add_executable(s_gemv_parity src/kernels/s_gemv_parity.cpp)
-  target_include_directories(s_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s_gemv_parity COMMAND s_gemv_parity --selftest)
 
   add_executable(elementwise_parity src/kernels/elementwise_parity.cpp)
-  target_include_directories(elementwise_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(elementwise_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(elementwise_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(elementwise_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME elementwise_parity COMMAND elementwise_parity --selftest)
 
   add_executable(bf16_gemv_parity src/kernels/bf16_gemv_parity.cpp)
-  target_include_directories(bf16_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(bf16_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME bf16_gemv_parity COMMAND bf16_gemv_parity --selftest)
 
   add_executable(s_gemv_q8k_parity src/kernels/s_gemv_q8k_parity.cpp)
-  target_include_directories(s_gemv_q8k_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s_gemv_q8k_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s_gemv_q8k_parity COMMAND s_gemv_q8k_parity --selftest)
 
   add_executable(qsa_parity src/kernels/qsa_parity.cpp)
-  target_include_directories(qsa_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(qsa_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(qsa_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(qsa_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME qsa_parity COMMAND qsa_parity --selftest)
 
   # ---- P2.S4: the PLE n-gram hash, the IQ4_NL table read, and the PLE block ----
@@ -360,8 +459,8 @@ if(STRATA_ENABLE_CUDA)
   # INSIDE the test.  A test that passes by finding nothing to compare is the failure mode this project keeps
   # re-learning.
   add_executable(ple_parity src/kernels/ple_parity.cpp)
-  target_include_directories(ple_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(ple_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(ple_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(ple_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   set(STRATA_PLE_FIXTURE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro" CACHE PATH
       "Directory containing required ple_in.bin and ple_out.bin parity fixtures")
   add_test(NAME ple_parity COMMAND ple_parity --selftest
@@ -370,73 +469,86 @@ if(STRATA_ENABLE_CUDA)
 
   # Plan v0.3 P7: INT8 KV storage vs a host reference and vs the FP16 path (GPU, synthetic, no model).
   add_executable(kv_q8_parity src/kernels/kv_q8_parity.cpp)
-  target_include_directories(kv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_q8_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_q8_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_q8_parity COMMAND kv_q8_parity)
 
   # KV streaming (docs/kv-streaming-design.md): streamed vs resident attention, bitwise (GPU, synthetic, no model).
   add_executable(kv_stream_parity src/kernels/kv_stream_parity.cpp)
-  target_include_directories(kv_stream_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_stream_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_stream_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_stream_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_stream_parity COMMAND kv_stream_parity)
 
+  # PR #120: the hybrid K8V4 KV (INT8 K + rotated Q4_0 V) vs host references (GPU, synthetic, no model).
+  add_executable(kv_hybrid_parity src/kernels/kv_hybrid_parity.cpp)
+  target_include_directories(kv_hybrid_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_hybrid_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  add_test(NAME kv_hybrid_parity COMMAND kv_hybrid_parity)
   # PR #21: Q4_0 KV with Walsh-Hadamard rotation vs a host reference (GPU, synthetic, no model).
   add_executable(kv_q4_parity src/kernels/kv_q4_parity.cpp)
-  target_include_directories(kv_q4_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_q4_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_q4_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_q4_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_q4_parity COMMAND kv_q4_parity)
   add_executable(cvec_parity src/kernels/cvec_parity.cpp)
-  target_include_directories(cvec_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(cvec_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(cvec_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(cvec_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME cvec_parity COMMAND cvec_parity)
 
   # Plan v0.3 P3: native MMVQ for 1..8 tokens (bitwise selftest + --bench timing); synthetic data, no model.
   if(STRATA_BUILD_TESTS)
   add_executable(native_mmvq_multi bench/micro/native_mmvq_multi.cpp)
-  target_include_directories(native_mmvq_multi PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(native_mmvq_multi PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(native_mmvq_multi PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(native_mmvq_multi PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME native_mmvq_multi COMMAND native_mmvq_multi --selftest)
   endif()
 
   # Plan v0.3 P0.1: working-set lock of a host region (CPU only).
   add_executable(platform_memory_test src/platform/memory_test.cpp)
-  target_link_libraries(platform_memory_test PRIVATE strata_core CUDA::cudart)
+  target_link_libraries(platform_memory_test PRIVATE strata_core ${_strata_gpu_runtime_target})
   add_test(NAME platform_memory_test COMMAND platform_memory_test)
 
   # Plan v0.3 P2: the n-gram table read straight from the SSD. The selftest needs only the CPU and a temp file.
   add_executable(ple_reader_test src/ngram/ple_reader_test.cpp)
-  target_include_directories(ple_reader_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(ple_reader_test PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(ple_reader_test PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(ple_reader_test PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME ple_reader_selftest COMMAND ple_reader_test --selftest)
 
-  # The GPU test is registered only when CUDA is on, and it runs ON THE TARGET PC.  `--selftest` allocates
+  # The GPU test runs ON THE TARGET PC. `--selftest` allocates
   # and writes real VRAM, so a machine without a device fails this test rather than skipping it silently -
   # which is correct: an engine that targets one GPU should not report a pass for having no GPU.
-  add_test(NAME cuda_device_selftest COMMAND strata-device --selftest)
+  if(STRATA_ENABLE_CUDA)
+    add_test(NAME cuda_device_selftest COMMAND strata-device --selftest)
+  else()
+    add_test(NAME hip_device_selftest COMMAND strata-device --selftest)
+  endif()
 
   # ---- P2.S5: the GraphRegistry --------------------------------------------------
   # A .cu because the test needs real kernels to capture; it links `strata_core` and not `strata_kernels`,
   # because what it exercises is the capture machinery and not any kernel in it.
-  if(STRATA_BUILD_TESTS)
-  add_executable(graph_registry_test bench/micro/graph_registry_test.cu)
-  target_include_directories(graph_registry_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(graph_registry_test PRIVATE strata_core CUDA::cudart)
-  add_test(NAME graph_registry_test COMMAND graph_registry_test)
+  if(STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/graph_registry_test.cu")
+    add_executable(graph_registry_test bench/micro/graph_registry_test.cu)
+    target_include_directories(graph_registry_test PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(graph_registry_test PRIVATE strata_core ${_strata_gpu_runtime_target})
+    add_test(NAME graph_registry_test COMMAND graph_registry_test)
+  endif()
 
   # ---- P2.S5's GATE: is a device<->pinned-memory handoff graph-capturable? --------------------
   # The doorbell protocol puts the D2H of `x_f` and the miss list INSIDE G[l] - that is the whole overlap.  The
   # phase doc says to verify this "before building on it", and this project has already lost two rounds to
   # building first (a memory-only spin that DEADLOCKS on Windows, and a `cudaEventRecord` inside a capture that
   # is SILENTLY DROPPED).  It links only cudart: it is about the capture machinery, not about any kernel.
-  add_executable(pinned_capture bench/micro/pinned_capture.cu)
-  target_include_directories(pinned_capture PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(pinned_capture PRIVATE CUDA::cudart)
-  add_test(NAME pinned_capture COMMAND pinned_capture)
+  if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/pinned_capture.cu")
+    add_executable(pinned_capture bench/micro/pinned_capture.cu)
+    target_include_directories(pinned_capture PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(pinned_capture PRIVATE ${_strata_gpu_runtime_target})
+    add_test(NAME pinned_capture COMMAND pinned_capture)
   endif()
 
   # Tells tests/CMakeLists.txt to add the CUDA-dependent Catch2 target.  It cannot be declared there
   # unconditionally: tests/ is added at the end of this file, but the flag has to be known before it runs.
-  set(STRATA_CUDA_TESTS ON)
+  if(STRATA_ENABLE_CUDA)
+    set(STRATA_CUDA_TESTS ON)
+  endif()
 endif()
 
 # ---- strata_kernels_cpu (P2.S3) ----
@@ -493,6 +605,9 @@ target_link_libraries(strata_kernels_cpu PUBLIC Threads::Threads)
 # with -DSTRATA_GGML_DIR=..., otherwise the pinned upstream commit (third_party/ggml/VERSION.txt).
 option(STRATA_NATIVE_EXPERTS "Native (IQ) expert support through ggml-cpu" ON)
 set(STRATA_GGML_DIR "" CACHE PATH "llama.cpp checkout to take ggml from (empty: fetch the pinned commit)")
+if(STRATA_PREFILL_MMQ AND NOT STRATA_NATIVE_EXPERTS)
+  message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_NATIVE_EXPERTS=ON for GGML sources")
+endif()
 if(STRATA_NATIVE_EXPERTS)
   enable_language(C)
   set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
@@ -550,7 +665,7 @@ if(STRATA_NATIVE_EXPERTS)
     endif()
     set(_strata_mmq ${_strata_gg}/src/ggml-cuda)
     add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
-    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs)
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)   # q8_0: the drafter's matrices (E-9)
       target_sources(strata_mmq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu)
     endforeach()
     target_include_directories(strata_mmq PRIVATE ${_strata_gg}/include ${_strata_gg}/src ${_strata_mmq}
@@ -559,28 +674,71 @@ if(STRATA_NATIVE_EXPERTS)
     if(MSVC)
       target_compile_options(strata_mmq PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/bigobj>)
     endif()
-    target_link_libraries(strata_mmq PUBLIC ggml-base CUDA::cudart)
+    target_link_libraries(strata_mmq PUBLIC ggml-base ${_strata_gpu_runtime_target})
     target_link_libraries(strata_prefill PUBLIC strata_mmq)
     target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+  elseif(STRATA_PREFILL_MMQ)
+    # Compile only the pinned GGML MMQ and activation-quantizer sources used by prefill. This uses
+    # common.cuh's native HIP path without enabling a second GGML HIP backend.
+    if(STRATA_GGML_DIR)
+      set(_strata_gg "${STRATA_GGML_DIR}/ggml")
+    else()
+      set(_strata_gg "${strata_llamacpp_SOURCE_DIR}/ggml")
+    endif()
+    set(_strata_mmq "${_strata_gg}/src/ggml-cuda")
+    if(NOT EXISTS "${_strata_mmq}/common.cuh" OR NOT EXISTS "${_strata_mmq}/quantize.cu")
+      message(FATAL_ERROR "The selected GGML source is missing the ggml-cuda MMQ sources: ${_strata_mmq}")
+    endif()
+    set(_strata_mmq_sources
+      src/prefill/moe_mmq.cu
+      src/prefill/ggml_cuda_host.cu
+      "${_strata_mmq}/quantize.cu")
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)
+      list(APPEND _strata_mmq_sources "${_strata_mmq}/template-instances/mmq-instance-${_t}.cu")
+    endforeach()
+    set_source_files_properties(${_strata_mmq_sources} PROPERTIES LANGUAGE HIP)
+    add_library(strata_mmq STATIC ${_strata_mmq_sources})
+    target_include_directories(strata_mmq
+      PRIVATE "${_strata_gg}/include" "${_strata_gg}/src" "${_strata_mmq}"
+      PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    target_compile_definitions(strata_mmq PRIVATE GGML_USE_HIP=1)
+    target_link_libraries(strata_mmq PUBLIC ggml-base hip::host roc::hipblas)
+    target_link_libraries(strata_prefill PUBLIC strata_mmq)
+    target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+
+    # Numerical diagnostic: synthetic layouts by default, optional first-expert IQ3_XXS/IQ4_NL check with a pack.
+    add_executable(hip_prefill_mmq_parity tests/hip/prefill_mmq_parity.cpp)
+    target_include_directories(hip_prefill_mmq_parity PRIVATE "${_strata_gg}/include")
+    target_link_libraries(hip_prefill_mmq_parity PRIVATE strata_prefill)
+    add_test(NAME hip_prefill_mmq_parity COMMAND hip_prefill_mmq_parity)
+    set_tests_properties(hip_prefill_mmq_parity PROPERTIES TIMEOUT 120)
+    list(APPEND _strata_hip_tests hip_prefill_mmq_parity)
   endif()
-  if(STRATA_ENABLE_CUDA)
+  if(STRATA_ENABLE_HIP)
+    # Optional real-model diagnostic: pass the GGUF shard containing the native PLE key.
+    add_executable(hip_ple_iq4 tests/hip/ple_iq4.cpp)
+    target_link_libraries(hip_ple_iq4 PRIVATE strata_kernels strata_hip_runtime ggml-base)
+  endif()
+  if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     add_executable(native_expert_parity src/kernels/native_expert_parity.cpp)
-    target_include_directories(native_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-    target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact CUDA::cudart)
+    target_include_directories(native_expert_parity PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact ${_strata_gpu_runtime_target})
   endif()
 endif()
 
 if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS)
   add_executable(hit_cpu_order_parity bench/micro/hit_cpu_order_parity.cu)
-  target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu CUDA::cudart)
+  target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
 endif()
 
 if(STRATA_BUILD_TESTS)
   # Plan v0.3 P6: one expert applied to several tokens, bitwise vs the single-token kernel (synthetic, CPU only).
   # plan v0.3 P5: the device dequantizers vs the artifact's CPU dequantizers on the real shards (not a ctest: needs
   # the 38 GB model; run it by hand or from tools/gpu_session.py)
-  add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp)
-  target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact CUDA::cudart)
+  if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)   # #93: it needs the device dequantizers and a GPU runtime
+    add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp)
+    target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact ${_strata_gpu_runtime_target})
+  endif()
 
   add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp)
   target_link_libraries(expert_multi_test PRIVATE strata_kernels_cpu)
@@ -600,12 +758,19 @@ if(STRATA_BUILD_TESTS)
   add_executable(pool_stress src/kernels/cpu/pool_stress.cpp)
   target_link_libraries(pool_stress PRIVATE strata_kernels_cpu)
   add_test(NAME pool_stress COMMAND pool_stress 20)
+
+  if(UNIX AND NOT APPLE)
+    add_executable(direct_file_async_test tests/platform/direct_file_async.cpp src/platform/direct_file.cpp)
+    target_include_directories(direct_file_async_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    target_link_libraries(direct_file_async_test PRIVATE Threads::Threads)
+    add_test(NAME direct_file_async_test COMMAND direct_file_async_test)
+  endif()
 endif()
 
 if(STRATA_BUILD_TESTS)
   enable_testing()
-  # the unit tests above build from src/; the tests/ tree (model-level checks) is not part of the published repo
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt)
+  # Model-level checks may be absent from source distributions.
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt")
     add_subdirectory(tests)
   endif()
 endif()

README.md

@@ -25,15 +25,15 @@ Measured on an RTX 5070 (12 GB), a Ryzen 5 7600 and 64 GB of RAM:
 
 | Size | Writes answers (short chat) | Writes answers (128K context) | Reads your prompt |
 | --- | ---: | ---: | ---: |
-| **Q2_0** | 90 tokens/s | 67 tokens/s | 1,310 tokens/s |
-| **IQ2_XS** | 74 tokens/s | 60 tokens/s | 1,240 tokens/s |
-| **IQ3_XXS** | 62 tokens/s | 46 tokens/s | 1,110 tokens/s |
-| **IQ3_S** | 52 tokens/s | 41 tokens/s | 1,070 tokens/s |
-| **Coder** (IQ1_M) | 51 tokens/s | 44 tokens/s | 1,300 tokens/s |
+| **Q2_0** | 93 tokens/s | 74 tokens/s | 2,170 tokens/s |
+| **IQ2_XS** | 79 tokens/s | 63 tokens/s | 2,090 tokens/s |
+| **IQ3_XXS** | 62 tokens/s | 49 tokens/s | 1,750 tokens/s |
+| **IQ3_S** | 53 tokens/s | 46 tokens/s | 1,620 tokens/s |
+| **Coder** (IQ1_M) | 55 tokens/s | 43 tokens/s | 2,180 tokens/s |
 
 - **Writes answers** = how fast the reply appears (tokens per second).
 - **Reads your prompt** = how fast it takes in what you send (long documents, code, chat history), measured on a
-  32K-token prompt; a 4K prompt reads at 740-1,000 tokens/s. A 32K prompt takes about 25 seconds with Q2_0.
+  32K-token prompt; a 4K prompt reads at 910-1,580 tokens/s. A 32K prompt takes about 15 seconds with Q2_0.
 
 A card with more VRAM is faster, because more of the model fits on the GPU: an RTX 3090 (24 GB) should do roughly
 100-140 tokens per second. All measurements, long-context numbers and estimates for other cards are in the
@@ -47,7 +47,7 @@ asks whether to share the model across them (recommended when two can). An insta
 its next start. Or choose yourself: `START-HERE.bat --gpus 0,2` (both, remembered), `--gpus all`, or `--gpu 0` (one
 card, this start only). Each card keeps the experts of its own layers, and prompts flow through the cards in a
 pipeline: on an RTX 5080 + RTX 3090 prompts were read 18-20% faster than on the 5080 alone, decoding on par.
-Every card must be an RTX 30 series or newer with 8 GB or more. See [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
+Every card must be an RTX 20 series or newer with 8 GB or more. See [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
 
 ## Which model should I pick?
 
@@ -84,9 +84,14 @@ one later with `SETUP.bat` (the same as `START-HERE.bat --setup`; on Linux `./se
 For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](docs/ORCA.md).
 It needs an explicit packing conversion and is not an installer menu option.
 
+An **AMD Radeon RX 7900 XT / XTX on Linux** works too (experimental): `./setup.sh --backend hip`, chosen by itself on
+a PC with no NVIDIA card Strata can use. It installs ROCm without sudo and compiles the engine (one GPU, no images
+yet). Details: [AMD HIP](docs/AMD_HIP.md).
+
 ## Install
 
-**You need:** an NVIDIA RTX 30, 40 or 50 card with 12 GB of VRAM or more, enough RAM for the size you pick (above),
+**You need:** an NVIDIA RTX 20, 30, 40 or 50 card with 12 GB of VRAM or more (RTX 20 since 0.1.27), enough RAM for the size you pick (above;
+a big GPU makes up for less RAM - the [low-RAM mode](docs/DETAILS.md)),
 ~80 GB of free disk space (an SSD makes the first start much faster), and Windows 10/11 or Linux. The only thing you
 install yourself is a current **NVIDIA driver** ([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA
 App). Everything else - Python, the engine, the model - is set up for you.

bench/results/2026-09-29-speed-0122/README.md

@@ -0,0 +1,32 @@
+# Speed with engine 0.1.22
+
+RTX 5070 12 GB (PCIe 5.0 x16), Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. The ready-made 0.1.22 engine, one-shot runs
+with what setup writes: `--prefill auto`, the shipped expert profile, `--expert-cache auto`, 8-bit KV above 4K, KV
+streaming (`--kv-resident 32768`) from 64K, MTP (`--spec 4 --spec-min-p 0.5`), greedy, 256 generated tokens, images
+off. The same code-agent prompts as `2026-09-28-speed-0114`. Every run: [`matrix.json`](matrix.json).
+
+## Prompt (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 519 | 1,226 | 1,844 | 1,836 | 1,682 | 1,304 |
+| **IQ2_XS** | 524 | 1,196 | 1,799 | 1,611 | 1,495 | 1,181 |
+| **IQ3_XXS** | 472 | 974 | 1,555 | 1,449 | 1,386 | - |
+| **IQ3_S** | 419 | 893 | 1,499 | 1,285 | 1,245 | - |
+| **Coder** | 660 | 1,522 | 1,871 | 1,938 | 1,779 | - |
+
+## Output (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 84.7 | 88.2 | 77.4 | 71.0 | 63.1 | 61.1 |
+| **IQ2_XS** | 77.2 | 74.6 | 67.5 | 63.5 | 59.1 | 54.7 |
+| **IQ3_XXS** | 65.1 | 62.3 | 49.8 | 53.6 | 47.0 | - |
+| **IQ3_S** | 50.5 | 44.6 | 49.0 | 48.8 | 41.3 | - |
+| **Coder** | 56.9 | 53.6 | 54.9 | 48.5 | 42.8 | - |
+
+- IQ2_XS was measured with Swift 1.5's IQ2_XS (the same speed as the original's); its 262K run had images on.
+- IQ3_XXS and IQ3_S at 262K do not fit 64 GB of RAM; the Coder at 262K was not run again (0.1.14: 1,034 tokens/s).
+- Against 0.1.14: prompts 1.2-1.5x faster at 32K-128K (the 0.1.22 prompt path: tensor-core attention, bit-identical
+  batched kernels, faster PLE reads). Output speed is the engine's same decode; one run per cell moves a few percent
+  with the text (speculative drafts), so compare it across many runs, not one.

bench/results/2026-09-29-speed-0122/matrix.json

@@ -0,0 +1,380 @@
+[
+ {
+  "model": "Q2_0",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 40.0,
+  "decode_tok_s": 84.71,
+  "decoded": 256,
+  "prefill_tok_s": 519.48,
+  "ttft_s": 3.62,
+  "spec_accept": 0.786,
+  "tokens_per_round": 2.77,
+  "vram_slots": 4176
+ },
+ {
+  "model": "Q2_0",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 41.6,
+  "decode_tok_s": 88.23,
+  "decoded": 256,
+  "prefill_tok_s": 1226.49,
+  "ttft_s": 4.43,
+  "spec_accept": 0.839,
+  "tokens_per_round": 3.24,
+  "vram_slots": 4123
+ },
+ {
+  "model": "Q2_0",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 56.6,
+  "decode_tok_s": 77.37,
+  "decoded": 256,
+  "prefill_tok_s": 1843.83,
+  "ttft_s": 18.63,
+  "spec_accept": 0.808,
+  "tokens_per_round": 3.04,
+  "vram_slots": 3833
+ },
+ {
+  "model": "Q2_0",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 72.9,
+  "decode_tok_s": 71.0,
+  "decoded": 256,
+  "prefill_tok_s": 1835.64,
+  "ttft_s": 36.48,
+  "spec_accept": 0.685,
+  "tokens_per_round": 2.77,
+  "vram_slots": 3780
+ },
+ {
+  "model": "Q2_0",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 118.1,
+  "decode_tok_s": 63.08,
+  "decoded": 256,
+  "prefill_tok_s": 1681.64,
+  "ttft_s": 77.92,
+  "spec_accept": 0.7,
+  "tokens_per_round": 2.56,
+  "vram_slots": 3697
+ },
+ {
+  "model": "Q2_0",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 242.0,
+  "decode_tok_s": 61.12,
+  "decoded": 256,
+  "prefill_tok_s": 1303.83,
+  "ttft_s": 200.94,
+  "spec_accept": 0.627,
+  "tokens_per_round": 2.5,
+  "vram_slots": 3525
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 263.6,
+  "decode_tok_s": 54.68,
+  "decoded": 256,
+  "prefill_tok_s": 1180.58,
+  "ttft_s": 221.91,
+  "spec_accept": 0.632,
+  "tokens_per_round": 2.49,
+  "vram_slots": 3727
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 56.0,
+  "decode_tok_s": 50.5,
+  "decoded": 256,
+  "prefill_tok_s": 419.48,
+  "ttft_s": 3.98,
+  "spec_accept": 0.811,
+  "tokens_per_round": 3.17,
+  "vram_slots": 2617
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 60.0,
+  "decode_tok_s": 44.58,
+  "decoded": 256,
+  "prefill_tok_s": 892.84,
+  "ttft_s": 5.53,
+  "spec_accept": 0.672,
+  "tokens_per_round": 2.69,
+  "vram_slots": 2580
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 75.6,
+  "decode_tok_s": 48.96,
+  "decoded": 256,
+  "prefill_tok_s": 1499.09,
+  "ttft_s": 22.65,
+  "spec_accept": 0.79,
+  "tokens_per_round": 2.9,
+  "vram_slots": 2383
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 102.1,
+  "decode_tok_s": 48.8,
+  "decoded": 256,
+  "prefill_tok_s": 1284.81,
+  "ttft_s": 51.58,
+  "spec_accept": 0.797,
+  "tokens_per_round": 2.94,
+  "vram_slots": 2281
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 159.2,
+  "decode_tok_s": 41.27,
+  "decoded": 256,
+  "prefill_tok_s": 1244.55,
+  "ttft_s": 104.81,
+  "spec_accept": 0.694,
+  "tokens_per_round": 2.73,
+  "vram_slots": 2292
+ },
+ {
+  "model": "Coder",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 34.5,
+  "decode_tok_s": 56.94,
+  "decoded": 256,
+  "prefill_tok_s": 660.44,
+  "ttft_s": 2.42,
+  "spec_accept": 0.74,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2650
+ },
+ {
+  "model": "Coder",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 18.8,
+  "decode_tok_s": 53.65,
+  "decoded": 256,
+  "prefill_tok_s": 1521.84,
+  "ttft_s": 3.21,
+  "spec_accept": 0.736,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2615
+ },
+ {
+  "model": "Coder",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 33.1,
+  "decode_tok_s": 54.85,
+  "decoded": 256,
+  "prefill_tok_s": 1870.76,
+  "ttft_s": 17.7,
+  "spec_accept": 0.806,
+  "tokens_per_round": 3.11,
+  "vram_slots": 2418
+ },
+ {
+  "model": "Coder",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 50.2,
+  "decode_tok_s": 48.46,
+  "decoded": 256,
+  "prefill_tok_s": 1937.69,
+  "ttft_s": 33.99,
+  "spec_accept": 0.653,
+  "tokens_per_round": 2.56,
+  "vram_slots": 2382
+ },
+ {
+  "model": "Coder",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 90.3,
+  "decode_tok_s": 42.75,
+  "decoded": 256,
+  "prefill_tok_s": 1778.66,
+  "ttft_s": 73.12,
+  "spec_accept": 0.588,
+  "tokens_per_round": 2.27,
+  "vram_slots": 2326
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 43.6,
+  "decode_tok_s": 77.16,
+  "decoded": 256,
+  "prefill_tok_s": 523.77,
+  "ttft_s": 3.46,
+  "spec_accept": 0.811,
+  "tokens_per_round": 3.02,
+  "vram_slots": 4361
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 43.7,
+  "decode_tok_s": 74.58,
+  "decoded": 256,
+  "prefill_tok_s": 1196.03,
+  "ttft_s": 4.47,
+  "spec_accept": 0.824,
+  "tokens_per_round": 3.36,
+  "vram_slots": 4309
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 59.6,
+  "decode_tok_s": 67.46,
+  "decoded": 256,
+  "prefill_tok_s": 1798.65,
+  "ttft_s": 19.06,
+  "spec_accept": 0.789,
+  "tokens_per_round": 3.02,
+  "vram_slots": 4030
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 81.4,
+  "decode_tok_s": 63.52,
+  "decoded": 256,
+  "prefill_tok_s": 1610.73,
+  "ttft_s": 41.37,
+  "spec_accept": 0.705,
+  "tokens_per_round": 2.6,
+  "vram_slots": 3978
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 126.0,
+  "decode_tok_s": 59.11,
+  "decoded": 256,
+  "prefill_tok_s": 1494.53,
+  "ttft_s": 87.51,
+  "spec_accept": 0.704,
+  "tokens_per_round": 2.71,
+  "vram_slots": 3897
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 48.6,
+  "decode_tok_s": 65.12,
+  "decoded": 256,
+  "prefill_tok_s": 471.53,
+  "ttft_s": 3.65,
+  "spec_accept": 0.876,
+  "tokens_per_round": 4.08,
+  "vram_slots": 3281
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 48.5,
+  "decode_tok_s": 62.33,
+  "decoded": 256,
+  "prefill_tok_s": 974.3,
+  "ttft_s": 5.17,
+  "spec_accept": 0.826,
+  "tokens_per_round": 3.15,
+  "vram_slots": 3243
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 64.9,
+  "decode_tok_s": 49.83,
+  "decoded": 256,
+  "prefill_tok_s": 1554.53,
+  "ttft_s": 21.84,
+  "spec_accept": 0.766,
+  "tokens_per_round": 2.91,
+  "vram_slots": 3009
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 91.7,
+  "decode_tok_s": 53.58,
+  "decoded": 256,
+  "prefill_tok_s": 1449.37,
+  "ttft_s": 45.84,
+  "spec_accept": 0.748,
+  "tokens_per_round": 2.8,
+  "vram_slots": 2965
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 140.9,
+  "decode_tok_s": 46.97,
+  "decoded": 256,
+  "prefill_tok_s": 1386.04,
+  "ttft_s": 94.33,
+  "spec_accept": 0.607,
+  "tokens_per_round": 2.47,
+  "vram_slots": 2901
+ }
+]

bench/results/2026-09-29-speed-0126/README.md

@@ -0,0 +1,36 @@
+# Speed with engine 0.1.26
+
+RTX 5070 12 GB (PCIe 5.0 x16), Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. The ready-made 0.1.26 engine, one-shot runs
+with what setup writes: `--prefill auto`, the shipped expert profile, `--expert-cache auto`, 8-bit KV above 4K, KV
+streaming (`--kv-resident 32768`) from 64K, MTP (`--spec 4 --spec-min-p 0.5`), greedy, 256 generated tokens, images
+off, no timing marks. The same code-agent prompts as `2026-09-29-speed-0122`. Every run: [`matrix.json`](matrix.json).
+
+## Prompt (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 536 | 1,299 | 2,171 | 2,126 | 2,107 |
+| **IQ2_XS** | 534 | 1,256 | 2,092 | 1,754 | 1,752 |
+| **IQ3_XXS** | 482 | 1,007 | 1,745 | 1,609 | 1,602 |
+| **IQ3_S** | 427 | 913 | 1,624 | 1,640 | 1,443 |
+| **Coder** | 656 | 1,583 | 2,177 | 2,236 | 2,208 |
+
+## Output (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 87.3 | 93.0 | 81.8 | 76.2 | 73.7 |
+| **IQ2_XS** | 79.6 | 78.6 | 76.3 | 63.7 | 62.7 |
+| **IQ3_XXS** | 61.9 | 61.6 | 58.5 | 57.2 | 49.0 |
+| **IQ3_S** | 52.4 | 53.3 | 48.3 | 46.3 | 45.5 |
+| **Coder** | 58.9 | 55.1 | 54.9 | 53.2 | 43.0 |
+
+- Against 0.1.22 (`2026-09-29-speed-0122`), prompts at 32K-128K are 8-28% faster: Q2_0 1,844 -> 2,171 at 32K and
+  1,682 -> 2,107 at 128K, the Coder 1,779 -> 2,208 at 128K. The gains come from:
+  - 0.1.24's QSA selection on tensor cores;
+  - 0.1.25's grouping tables in mapped memory and fused norms;
+  - 0.1.26's batched draft-layer pass.
+- 262K was not run again (0.1.22: Q2_0 1,304, IQ2_XS 1,181 tokens/s).
+- Output speed is the same decode as before. One run per cell moves several percent with the text (the share of
+  accepted drafts, `spec_accept` in `matrix.json`). For example, the Coder's 128K answer accepted 55% of its drafts
+  and its 64K answer 71%.

bench/results/2026-09-29-speed-0126/matrix.json

@@ -0,0 +1,352 @@
+[
+ {
+  "model": "Q2_0",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 23.9,
+  "decode_tok_s": 87.33,
+  "decoded": 256,
+  "prefill_tok_s": 536.42,
+  "ttft_s": 3.4,
+  "spec_accept": 0.681,
+  "tokens_per_round": 2.72,
+  "vram_slots": 4174
+ },
+ {
+  "model": "Q2_0",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 27.6,
+  "decode_tok_s": 92.97,
+  "decoded": 256,
+  "prefill_tok_s": 1299.44,
+  "ttft_s": 4.21,
+  "spec_accept": 0.838,
+  "tokens_per_round": 3.23,
+  "vram_slots": 4121
+ },
+ {
+  "model": "Q2_0",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 40.8,
+  "decode_tok_s": 81.75,
+  "decoded": 256,
+  "prefill_tok_s": 2170.85,
+  "ttft_s": 16.02,
+  "spec_accept": 0.788,
+  "tokens_per_round": 2.99,
+  "vram_slots": 3832
+ },
+ {
+  "model": "Q2_0",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 57.3,
+  "decode_tok_s": 76.23,
+  "decoded": 256,
+  "prefill_tok_s": 2126.03,
+  "ttft_s": 32.06,
+  "spec_accept": 0.707,
+  "tokens_per_round": 2.8,
+  "vram_slots": 3778
+ },
+ {
+  "model": "Q2_0",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 87.7,
+  "decode_tok_s": 73.73,
+  "decoded": 256,
+  "prefill_tok_s": 2106.85,
+  "ttft_s": 62.48,
+  "spec_accept": 0.679,
+  "tokens_per_round": 2.71,
+  "vram_slots": 3695
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 42.2,
+  "decode_tok_s": 79.62,
+  "decoded": 256,
+  "prefill_tok_s": 533.83,
+  "ttft_s": 3.51,
+  "spec_accept": 0.774,
+  "tokens_per_round": 2.89,
+  "vram_slots": 4359
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 40.6,
+  "decode_tok_s": 78.6,
+  "decoded": 256,
+  "prefill_tok_s": 1256.02,
+  "ttft_s": 4.31,
+  "spec_accept": 0.81,
+  "tokens_per_round": 3.29,
+  "vram_slots": 4308
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 53.6,
+  "decode_tok_s": 76.34,
+  "decoded": 256,
+  "prefill_tok_s": 2092.02,
+  "ttft_s": 16.57,
+  "spec_accept": 0.85,
+  "tokens_per_round": 3.2,
+  "vram_slots": 4029
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 78.3,
+  "decode_tok_s": 63.69,
+  "decoded": 256,
+  "prefill_tok_s": 1753.65,
+  "ttft_s": 38.08,
+  "spec_accept": 0.636,
+  "tokens_per_round": 2.59,
+  "vram_slots": 3977
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 115.1,
+  "decode_tok_s": 62.72,
+  "decoded": 256,
+  "prefill_tok_s": 1751.79,
+  "ttft_s": 74.85,
+  "spec_accept": 0.696,
+  "tokens_per_round": 2.6,
+  "vram_slots": 3896
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 46.9,
+  "decode_tok_s": 61.94,
+  "decoded": 256,
+  "prefill_tok_s": 482.04,
+  "ttft_s": 3.67,
+  "spec_accept": 0.845,
+  "tokens_per_round": 3.57,
+  "vram_slots": 3279
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 47.4,
+  "decode_tok_s": 61.64,
+  "decoded": 256,
+  "prefill_tok_s": 1007.48,
+  "ttft_s": 5.03,
+  "spec_accept": 0.801,
+  "tokens_per_round": 3.19,
+  "vram_slots": 3242
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 65.6,
+  "decode_tok_s": 58.49,
+  "decoded": 256,
+  "prefill_tok_s": 1745.16,
+  "ttft_s": 19.63,
+  "spec_accept": 0.789,
+  "tokens_per_round": 2.85,
+  "vram_slots": 3007
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 88.5,
+  "decode_tok_s": 57.21,
+  "decoded": 256,
+  "prefill_tok_s": 1609.31,
+  "ttft_s": 41.45,
+  "spec_accept": 0.74,
+  "tokens_per_round": 2.87,
+  "vram_slots": 2963
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 127.0,
+  "decode_tok_s": 48.95,
+  "decoded": 256,
+  "prefill_tok_s": 1602.33,
+  "ttft_s": 81.71,
+  "spec_accept": 0.643,
+  "tokens_per_round": 2.37,
+  "vram_slots": 2899
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 54.2,
+  "decode_tok_s": 52.4,
+  "decoded": 256,
+  "prefill_tok_s": 426.6,
+  "ttft_s": 4.03,
+  "spec_accept": 0.822,
+  "tokens_per_round": 3.17,
+  "vram_slots": 2617
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 58.9,
+  "decode_tok_s": 53.3,
+  "decoded": 256,
+  "prefill_tok_s": 912.99,
+  "ttft_s": 5.58,
+  "spec_accept": 0.85,
+  "tokens_per_round": 3.38,
+  "vram_slots": 2579
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 72.8,
+  "decode_tok_s": 48.31,
+  "decoded": 256,
+  "prefill_tok_s": 1624.07,
+  "ttft_s": 21.07,
+  "spec_accept": 0.827,
+  "tokens_per_round": 3.0,
+  "vram_slots": 2383
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 93.3,
+  "decode_tok_s": 46.25,
+  "decoded": 256,
+  "prefill_tok_s": 1640.34,
+  "ttft_s": 40.77,
+  "spec_accept": 0.693,
+  "tokens_per_round": 2.56,
+  "vram_slots": 2346
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 144.6,
+  "decode_tok_s": 45.51,
+  "decoded": 256,
+  "prefill_tok_s": 1443.21,
+  "ttft_s": 90.72,
+  "spec_accept": 0.697,
+  "tokens_per_round": 2.73,
+  "vram_slots": 2291
+ },
+ {
+  "model": "Coder",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 37.0,
+  "decode_tok_s": 58.91,
+  "decoded": 256,
+  "prefill_tok_s": 656.36,
+  "ttft_s": 2.48,
+  "spec_accept": 0.743,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2649
+ },
+ {
+  "model": "Coder",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 18.3,
+  "decode_tok_s": 55.1,
+  "decoded": 256,
+  "prefill_tok_s": 1583.1,
+  "ttft_s": 3.09,
+  "spec_accept": 0.713,
+  "tokens_per_round": 2.88,
+  "vram_slots": 2614
+ },
+ {
+  "model": "Coder",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 30.4,
+  "decode_tok_s": 54.89,
+  "decoded": 256,
+  "prefill_tok_s": 2177.17,
+  "ttft_s": 15.33,
+  "spec_accept": 0.731,
+  "tokens_per_round": 2.75,
+  "vram_slots": 2417
+ },
+ {
+  "model": "Coder",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 44.8,
+  "decode_tok_s": 53.22,
+  "decoded": 256,
+  "prefill_tok_s": 2236.49,
+  "ttft_s": 29.51,
+  "spec_accept": 0.709,
+  "tokens_per_round": 2.68,
+  "vram_slots": 2382
+ },
+ {
+  "model": "Coder",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 75.9,
+  "decode_tok_s": 43.04,
+  "decoded": 256,
+  "prefill_tok_s": 2208.27,
+  "ttft_s": 59.02,
+  "spec_accept": 0.553,
+  "tokens_per_round": 2.23,
+  "vram_slots": 2324
+ }
+]

cmake/hip_backend.cmake

@@ -0,0 +1,58 @@
+# Opt-in HIP configuration. Strata's CUDA-shaped kernels currently target RDNA3 gfx1100 wave32.
+# CMake/compiler discovery stays machine-independent; pass CMAKE_HIP_COMPILER when it is not on PATH.
+if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "")
+  set(CMAKE_HIP_ARCHITECTURES gfx1100 CACHE STRING "Strata HIP target architecture")
+endif()
+if(NOT CMAKE_HIP_ARCHITECTURES STREQUAL "gfx1100")
+  message(FATAL_ERROR
+    "Strata HIP currently supports only gfx1100 wave32; CMAKE_HIP_ARCHITECTURES is '${CMAKE_HIP_ARCHITECTURES}'")
+endif()
+
+enable_language(HIP)
+find_package(hip CONFIG REQUIRED)
+find_package(hipblas CONFIG REQUIRED)
+find_package(hipblaslt CONFIG QUIET)
+
+if(NOT TARGET hip::host)
+  message(FATAL_ERROR "The ROCm hip CMake package did not provide hip::host")
+endif()
+if(NOT TARGET roc::hipblas)
+  message(FATAL_ERROR "The ROCm hipblas CMake package did not provide roc::hipblas")
+endif()
+if(TARGET roc::hipblaslt)
+  set(STRATA_HIPBLASLT_AVAILABLE ON)
+else()
+  set(STRATA_HIPBLASLT_AVAILABLE OFF)
+  message(STATUS "Strata: hipBLASLt not found; solution-table dispatch is unavailable")
+endif()
+
+# HIP's link step produces a PIE; make Strata and ggml objects PIC for the ROCm linker.
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(STRATA_HIP_COMPAT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/strata/hip_compat")
+add_library(strata_hip_runtime INTERFACE)
+target_include_directories(strata_hip_runtime BEFORE INTERFACE
+  "${STRATA_HIP_COMPAT_INCLUDE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/include")
+target_compile_definitions(strata_hip_runtime INTERFACE STRATA_USE_HIP=1)
+target_link_libraries(strata_hip_runtime INTERFACE hip::host)
+foreach(_language IN ITEMS CXX HIP)
+  target_compile_options(strata_hip_runtime INTERFACE
+    "$<$<COMPILE_LANGUAGE:${_language}>:-include>"
+    "$<$<COMPILE_LANGUAGE:${_language}>:${STRATA_HIP_COMPAT_INCLUDE_DIR}/cuda_runtime.h>")
+endforeach()
+
+# CMake does not infer HIP from Strata's existing CUDA-shaped .cu suffixes.
+file(GLOB_RECURSE _strata_hip_sources CONFIGURE_DEPENDS
+  "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cu"
+  "${CMAKE_CURRENT_SOURCE_DIR}/bench/*.cu"
+  "${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cu")
+if(_strata_hip_sources)
+  set_source_files_properties(${_strata_hip_sources} PROPERTIES LANGUAGE HIP)
+endif()
+foreach(_source IN ITEMS tests/hip/intrinsics.cpp tests/hip/native_qsa_score.cpp)
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${_source}")
+    set_source_files_properties("${_source}" PROPERTIES LANGUAGE HIP)
+  endif()
+endforeach()
+
+message(STATUS "Strata: HIP enabled, arch ${CMAKE_HIP_ARCHITECTURES}")

docs/AMD_HIP.md

@@ -0,0 +1,156 @@
+# Experimental AMD HIP backend (gfx1100)
+
+This is a manual Linux source build for the RX 7900 XTX. It is opt-in; the
+NVIDIA installer and CUDA build remain the default. Other AMD architectures,
+wave64, Windows HIP, and mixed AMD/NVIDIA execution are outside this contribution.
+
+The backend maps the CUDA-shaped runtime and BLAS calls to HIP/hipBLAS, uses
+RDNA3's signed integer dot instruction for quantized kernels, and supplies
+wave32 shuffle/packed-byte operations. CUDA-only QSA matrix instructions have
+an ordered FP32 fallback. Prefill supports both dequantization plus hipBLAS GEMM and opt-in HIP ggml MMQ.
+An optional, calibrated hipBLASLt path accelerates dense projections on gfx1100.
+This does not claim bit-identical model answers across backends. See
+[performance settings and evidence](AMD_HIP_PERFORMANCE.md).
+
+## Install with setup (recommended)
+
+On Linux with an RX 7900 XT / XTX and the kernel's amdgpu driver (no ROCm install needed):
+
+```sh
+./setup.sh --backend hip
+```
+
+- **Detection:** setup finds the card through the kernel's KFD topology. Integrated Radeon GPUs are listed as not
+  supported. On a PC without an NVIDIA card Strata can use, `--backend hip` is chosen automatically.
+- **ROCm:** installed into `.venv` from AMD's TheRock wheels (~10 GB, no sudo), pinned to the version this backend was
+  tested with (`STRATA_ROCM_VERSION` / `STRATA_ROCM_INDEX` override it). A system ROCm in `/opt/rocm` (or
+  `$ROCM_PATH`) with hipcc and hipBLAS is used instead when present.
+- **Engine:** compiled on your PC (10-20 minutes, once; again after a `git pull` that changes it). This needs a C++
+  compiler and git (`sudo apt install build-essential git`).
+- **Limits for now:** one GPU, no images, no calibration. The Monitor shows no GPU statistics.
+
+The rest of setup is the same as on NVIDIA: the model download, the start script, the server.
+
+## Build
+
+Requirements: a working ROCm driver/runtime, HIP development headers and
+compiler, hipBLAS and (for tuned dense prefill) hipBLASLt development files, CMake 3.24+, a C++20 host compiler, and Git.
+The model still needs sufficient system RAM and fast SSD storage for its PLE
+table. VRAM occupancy alone is not a throughput measurement.
+
+```sh
+cmake -S . -B build-hip \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DSTRATA_ENABLE_HIP=ON -DSTRATA_ENABLE_CUDA=OFF \
+  -DCMAKE_HIP_ARCHITECTURES=gfx1100
+cmake --build build-hip --target strata -j2
+```
+
+If CMake cannot find the HIP compiler, add
+`-DCMAKE_HIP_COMPILER=/path/to/rocm/llvm/bin/clang++`. On the Fedora-family test
+host this was `/usr/lib64/rocm/llvm/bin/clang++`. Use the compiler and libraries
+from the same ROCm installation. Do not enable both GPU backends in one build.
+
+Native IQ experts are enabled by default. CMake fetches the llama.cpp revision
+pinned by this repository. For an offline build, point `STRATA_GGML_DIR` at a
+checkout of that exact revision; using an arbitrary newer checkout changes the
+dependency being tested.
+
+## Model and serving configuration
+
+Prepare a supported model pack and its MTP runtime using the existing tools.
+For OrcaRouter IQ3_XXS, follow [the explicit compatibility conversion](ORCA.md);
+this backend does not change quantization, model licenses, or tokenizers.
+Add `--experts-bin` to the `tools/iq_pack.py` command: mmap requires the pack's
+`experts.bin`, which the default native packing command does not emit. This
+consumes additional disk space. Original GSQ-RCO IQ3_XXS uses shard 2 for PLE;
+Orca uses shard 1. Keep each model's own pack and tokenizer together.
+
+Use `build-hip/strata` as the executable in the server JSON. Select the AMD
+device with `ROCR_VISIBLE_DEVICES`/`HIP_VISIBLE_DEVICES` if necessary. Start with
+a modest context and prefill chunk size before measuring larger workloads.
+Keep the PLE file on an SSD. Do not assume a configured context length proves
+successful full-window inference.
+
+Large pinned host allocations can fail on ROCm even when ordinary RAM is
+available. The existing `--mmap-experts` path avoids allocating the full pinned
+expert arena; it still depends on OS file-cache residency and may stall on
+storage reads. It does not make SSD access equivalent to RAM.
+
+Starting args for the original GSQ-RCO IQ3_XXS model (replace the paths):
+
+```sh
+build-hip/strata --serve \
+  --pack packs/iq3xxs \
+  --native /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf \
+  --ple-gguf /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf \
+  --mmap-experts --expert-profile data/expert-profile.bin --expert-cache auto \
+  --prefill 512 --spec 4 --spec-min-p 0.5 --mtp mtp/rt \
+  --max-context 4096 --kv int8 --pool-workers 15 \
+  --adapt-every 0 --pcie-frac 0 --vram-reserve-mib 1024
+```
+
+`--serve` is the engine's internal token protocol. For a browser or OpenAI API,
+put these args in the `args` array of the [server JSON example](ORCA.md#local-server),
+set `exe` to `build-hip/strata`, and use the matching pack's `tokenizer` directory.
+Launch with `.venv/bin/python -m serve.server --engine strata --config strata-hip.json --port 8080`.
+The worker count above was used on a 16-core CPU; measure it for your CPU.
+The 4K context is a smoke-test starting point, not a model limit. The expert cache
+sizes itself automatically and leaves 1 GiB of VRAM headroom.
+
+The installer supports this backend (see "Install with setup" above). The vision helper and multi-GPU layer
+splits are NVIDIA-only for now.
+
+## Original backend validation (PR #94)
+
+The following is historical validation of the original backend, not a fresh
+test count for this replacement. Current build/test evidence and performance
+limits are recorded in [AMD_HIP_PERFORMANCE.md](AMD_HIP_PERFORMANCE.md).
+
+Tested against upstream `c1e903310f211e6630780c3bd2038778c071c68d` (0.1.20),
+with pinned llama.cpp `3cf03257f219afbe7334045ff7c6a06ac68c627d`.
+The host has an RX 7900 XTX (24 GiB), Ryzen 9 7950X3D, 64 GiB system RAM,
+and Fedora-family Linux with ROCm 7.1.52802 / Clang 20. The original model used
+for the smoke check was on mechanical RAID0; cold page faults were slow. Keep
+latency-sensitive data on SSD and measure warm residency separately.
+
+- Complete HIP Release build, including the executable and parity targets.
+- All 25 tests selected by the command below passed on 2026-09-29.
+- Intrinsic parity includes signed packed-byte dot products and overflow.
+- CPU/GPU mapped-memory handoff covers delayed publication, changing payloads,
+  and repeated graph replay in both directions.
+- GR coverage includes the maximum eight-token fused batch and graph replay
+  after inputs change. Native QSA checks the scalar HIP fallback.
+- Q8_K quantization is byte-exact against the existing reference. HIP's
+  `__fmul_rn` can become ordinary multiplication; disabling contraction in
+  `quantize_act.cu` preserves the separately rounded product before the magic
+  rounding bias is added. CUDA's compile flags remain unchanged.
+- The mmap regression covers canonical and differing native layer sizes,
+  truncated/oversized files, lookup bounds, and reopening after errors.
+- CUDA 13 / GCC 15 / sm_89 executable build passed; GR, sampler, quantization,
+  and mmap tests passed on the same tree. This is a regression check, not a
+  claim of complete CUDA inference validation.
+- CPU-only `strata-plan` build passed without either GPU backend.
+- Real GSQ-RCO IQ3_XXS server smoke with mmap, the shipped expert profile,
+  automatic GPU cache, and MTP passed arithmetic, executable Python addition,
+  system-marker recall, and 1,170-token batched-prefill recall. All four requests
+  ended normally with correct answers; one engine process served them all.
+  The cache held 11,142 experts (18.08 GiB), with 874 MiB of VRAM free after
+  startup. These short checks do not establish a throughput or quality benchmark.
+
+Build all targets before running the registered focused checks:
+
+```sh
+cmake --build build-hip -j2
+ctest --test-dir build-hip --output-on-failure --timeout 60 \
+  -E '^(ple_parity|platform_memory_test)$'
+```
+
+`ple_parity` requires an external model fixture. `platform_memory_test` requests
+256 MiB of locked memory; the test shell's 8 MiB memlock limit prevented that
+test from passing. These are explicit exclusions, not skipped tests counted as
+passes. Additional unpublished upstream fixture suites are not covered.
+
+The published performance table explicitly identifies the measured development
+snapshot; it must not be read as a benchmark of every subsequent rebase. Vision, long-context stress, broad answer-quality
+equivalence, other AMD cards, and mixed-vendor inference are not validated here.

docs/AMD_HIP_PERFORMANCE.md

@@ -0,0 +1,142 @@
+# RX 7900 XTX support and performance evidence
+
+This opt-in Linux `gfx1100` backend supersedes the initial support in
+[PR #94](https://github.com/Niko1221/Strata/pull/94). It retains HIP runtime and
+wave32 integer-dot compatibility, native mmap layout validation, and MTP, then
+adds HIP MMQ, optional calibrated dense hipBLASLt GEMM, native prefill batching,
+and the host-memory / SSD paths used by the measured configuration. HIP blocking
+expert uploads use one reusable pinned staging buffer, including prompt-cache
+refills, to avoid repeated pageable-source registrations.
+
+## Reproduce the configuration
+
+Build instructions are in [AMD_HIP.md](AMD_HIP.md). Use the repository-pinned
+llama.cpp dependency; do not silently substitute another revision. Build with
+`STRATA_PREFILL_MMQ=ON` and use these runtime variables for the measured arm:
+
+```sh
+export STRATA_PREFILL_MMQ=1
+export STRATA_HIPBLASLT_TUNING="$PWD/tools/hip/gfx1100-hipblaslt-100100.txt"
+export STRATA_PREFILL_RING=96
+export STRATA_IO_THREADS=32
+```
+
+The supplied table is calibrated for gfx1100 and hipBLASLt version 100100.
+It is not a universal ROCm tuning table. The runtime guards architecture,
+library version and actual shape/stride/workspace requirements, falling back
+when a table entry is unavailable or incompatible. HIP MMQ is opt-in at runtime. Default CUDA selection is preserved.
+
+Measured engine configuration: Orca Flash Next IQ3_XXS, native pack plus matching
+GGUF/tokenizer/template, `--mmap-experts --resident-cpu-experts`, fixed ranked
+`--expert-profile`, `--expert-cache auto`, `--prefill 8192`, `--spec 4`,
+`--spec-min-p 0.5`, matching MTP runtime, `--max-context 262144`, `--kv int8`,
+`--kv-resident 32768`, `--pool-workers 15`, `--adapt-every 0`, `--pcie-frac 0`,
+and `--vram-reserve-mib 1024`. Keep PLE on SSD.
+
+`--resident-cpu-experts` copies the complement of the static GPU cache to ordinary
+RAM. It requires mmap, a fixed expert profile, and disabled adaptation. It does
+not lock all data into RAM: temporary prefill borrowing and cache refill may
+still use the mapped fallback. Allocation needs sufficient available RAM;
+on Linux this option requires readable standard cgroup-v2 mounts. For each
+finite cgroup ancestor, the guard credits only `inactive_file` after subtracting
+`file_dirty` and `file_writeback`, capped by current usage; it remains bounded by
+the ancestor limit and host `MemAvailable`, with 8 GiB headroom. This accounts
+for reclaimable clean file cache but cannot reserve memory against concurrent
+system or process allocations.
+The POSIX PLE path issues direct reads through a configurable worker pool.
+
+Use a dedicated idle server, restart it between arms, and capture its engine log:
+
+```sh
+python3 tools/hip/bench_prefill.py \
+  --model MODEL_NAME --url http://127.0.0.1:8080 \
+  --engine-log /path/to/engine.log --label candidate --output candidate.json
+```
+
+The script uses the same synthetic source prompts and request order as the
+reported measurements: a small warmup, then 140/280/140/280 functions, with one
+follow-up after each fresh prompt. It enforces zero reused tokens for the fresh
+prompts and records prefill, decode, wall time and stop reason. It requires the
+unbuffered log to contain exactly one completed timing record per request. The
+128-token output cap is intentional for throughput measurement, not task success.
+Never use cancelled-request timing lines as throughput evidence.
+
+## Final revision benchmark
+
+Measured on 2026-09-29, source revision `9568e78da663e6b224ab96d57baf450717b36e73`.
+HIP executable SHA-256: `705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b`.
+The control and candidate below were both measured anew in this session; no
+historical benchmark values are mixed into the table. The control binary SHA-256
+is `4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7`;
+its source was an existing local runtime snapshot rather than a clean upstream commit.
+Both arms use an RX 7900 XTX 24 GiB / gfx1100, Ryzen 9 7950X3D,
+64 GiB installed RAM, Fedora-family Linux, ROCm 7.1 and a 272 W GPU cap.
+Other model services are stopped for measurement and restored afterward.
+Sampling: temperature 0, top-k 1, top-p 1, min-p 0, seed 42, reasoning disabled.
+
+| Fresh request, execution order | Control prefill t/s | Candidate prefill t/s | Candidate output t/s | Candidate request wall time |
+| --- | ---: | ---: | ---: | ---: |
+| 4,210 tokens, first use | 240.0 | 447.5 | 55.5 | 11.729 s |
+| 8,830 tokens, first use at this size | 484.8 | 873.7 | 55.7 | 12.420 s |
+| 4,210 tokens, warmed | 457.2 | 901.1 | 56.7 | 6.937 s |
+| 8,830 tokens, warmed | 478.9 | 966.5 | 59.2 | 11.314 s |
+
+Each request generated exactly 128 tokens and finished at the intentional length
+cap. All four fresh prompts had zero reused KV tokens. Fresh-prompt prefill was
+1.80–2.02x the freshly measured existing AMD runtime; fresh-request wall time was
+40–46% lower. This is a comparison with our existing HIP runtime, not unmodified
+upstream, which does not provide this backend. The candidate also completed four
+cached follow-ups in 2.95–3.26 seconds. Those follow-ups are not apples-to-apples
+prefill comparisons: cache reuse differs with generated text and checkpoint
+selection. Full sanitized measurements are in
+[the fresh-run JSON](benchmarks/2026-09-29-gfx1100.json).
+
+Before this revision, an intermediate package missing the HIP upload staging
+buffer stalled on the 8,830-token request and was rejected. Restoring staging
+allowed the complete sequence above to finish. That failed attempt is not
+included in the throughput table. No watchdog limit was raised to obtain these
+results.
+
+Zero KV reuse is not equivalent to a cold filesystem cache. First-use speed
+and warmed speed are reported separately. Individual observations do not
+establish confidence intervals or a general rate at every context length.
+
+## Correctness and quality limits
+
+Both HIP and CUDA executables built from the measured source revision. CUDA was
+compile-checked, not performance-tested by this run. The selected HIP CTest suite
+passed **29/29** with the tuning table enabled, including expert-upload readback,
+asynchronous handoff, QSA, MMQ, Lt GEMM, KV streaming and PLE reading. The excluded
+`ple_parity` requires an external fixture; `platform_memory_test` requires a
+larger locked-memory limit than the test account provides. Separately, three
+real IQ3_XXS PLE matrix graph replays passed, and the POSIX direct-file test passed
+queued reads, short EOF, wake, close/drain and reopen checks.
+
+Development checks passed eight MMQ numerical comparisons, four actual Lt GEMM
+comparisons, and native QSA/indexer/embedding parity including tail states,
+chunk continuation and image overrides. Real quantized MMQ relative L2 error
+was approximately 0.0026 against raw-FP32/dequantized-weight reference, reflecting
+Q8 activation arithmetic; this is not bitwise numerical equivalence.
+
+End-to-end coding quality is not established by these numerical checks or capped
+throughput requests. Validate completed tasks with independent runtime tests; this
+contribution makes no broad answer-quality or agentic-reliability claim.
+
+The changes do not claim better model reasoning, verified full-context behavior,
+end-to-end vision validation, Windows HIP, other AMD architectures, or mixed
+AMD/NVIDIA execution. Existing CUDA multi-GPU code remains present but is not
+validation of HIP multi-GPU support.
+
+## Attribution and rejected experiments
+
+The measurements above were taken before this backend was rebased onto engine 0.1.24. The PR's own batched
+embedding gather and QSA indexer append (adapted from
+[PR #108](https://github.com/Niko1221/Strata/pull/108), commit
+`acd487233c0bbe2217a6881c5bb43f8a283b0de5`) were dropped in the rebase: 0.1.24 already does both on every
+backend, bit-exact (C-2, C-4). #108's optional GDN parallel/split path is not included. Existing upstream MMQ orchestration is retained and enabled for
+HIP with AMD architecture identification and the correct backend compilation.
+
+Static 16K chunks, expanded FP16 expert tuning, and a 192-slot ring did not offer
+a consistent end-to-end win in the evaluated workload. The table includes only
+the selected 30 dense-shape rows. Microkernel speedups alone were not sufficient
+to select a configuration. No increase in GPU power cap was used.

docs/DETAILS.md

@@ -12,7 +12,7 @@ New here? Start with the [README](../README.md) - it has everything you need to
 
 ## Speed (measured)
 
-RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1.14 with the settings setup writes
+RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1.26 with the settings setup writes
 (`--prefill auto`, 8-bit KV above 4K, KV streaming from 64K). One code-agent prompt per length, 256 generated tokens,
 MTP speculative decoding on. "262K" is the model's full context window (a 259,943-token prompt). The IQ2_XS row was
 measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
@@ -21,21 +21,27 @@ measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| **Q2_0** | 494 | 1,007 | 1,308 | 1,294 | 1,208 | 967 |
-| **IQ2_XS** | 461 | 811 | 1,238 | 1,136 | 1,071 | 886 |
-| **IQ3_XXS** | 415 | 770 | 1,108 | 1,065 | 1,015 | - |
-| **IQ3_S** | 396 | 737 | 1,070 | 1,070 | 931 | - |
-| **Coder** | 599 | 1,152 | 1,298 | 1,350 | 1,266 | 1,034 |
+| **Q2_0** | 536 | 1,299 | 2,171 | 2,126 | 2,107 | 1,304† |
+| **IQ2_XS** | 534 | 1,256 | 2,092 | 1,754 | 1,752 | 1,181*† |
+| **IQ3_XXS** | 482 | 1,007 | 1,745 | 1,609 | 1,602 | - |
+| **IQ3_S** | 427 | 913 | 1,624 | 1,640 | 1,443 | - |
+| **Coder** | 656 | 1,583 | 2,177 | 2,236 | 2,208 | 1,034** |
+
+Engine 0.1.26; `bench/results/2026-09-29-speed-0126`. At 32K-128K that is 8-28% faster than 0.1.22. † not measured
+again: 0.1.22. \* measured with images on (the image encoder's VRAM reserve leaves fewer experts cached). \*\* not
+measured again: 0.1.14.
 
 ### Output (tokens/s)
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| **Q2_0** | 84.3 | 90.3 | 73.6 | 69.0 | 67.2 | 60.3 |
-| **IQ2_XS** | 74.4 | 73.8 | 71.5 | 64.3 | 59.8 | 52.8 |
-| **IQ3_XXS** | 60.3 | 62.1 | 51.4 | 50.0 | 45.8 | - |
-| **IQ3_S** | 51.5 | 51.6 | 48.2 | 48.8 | 40.5 | - |
-| **Coder** | 53.3 | 50.6 | 53.3 | 50.8 | 44.0 | 42.8 |
+| **Q2_0** | 87.3 | 93.0 | 81.8 | 76.2 | 73.7 | 60.3† |
+| **IQ2_XS** | 79.6 | 78.6 | 76.3 | 63.7 | 62.7 | 52.8† |
+| **IQ3_XXS** | 61.9 | 61.6 | 58.5 | 57.2 | 49.0 | - |
+| **IQ3_S** | 52.4 | 53.3 | 48.3 | 46.3 | 45.5 | - |
+| **Coder** | 58.9 | 55.1 | 54.9 | 53.2 | 43.0 | 42.8† |
+
+Engine 0.1.26, the same runs. † not measured again: 0.1.14.
 
 Output speed depends on the text as well: speculative decoding runs faster when more of the drafted tokens are
 accepted, so a different answer to the same prompt moves it by several percent. Run back to back on the 4K prompt,
@@ -58,6 +64,28 @@ halves the KV cache's memory with a Hadamard rotation before 4-bit rounding (PR
 is measurably less precise on long documents (perplexity +8-12%; needle tests still pass). 8-bit stays the default.
 Details: [`bench/results/2026-09-27-kv-q4`](../bench/results/2026-09-27-kv-q4/README.md).
 
+**Hybrid K8V4 KV cache (engine 0.1.25, optional, PR #120):** `--kv k8v4` (`START-HERE.bat --setup --kv k8v4`) keeps
+the keys at 8 bits and stores the values as rotated 4-bit: 23% less KV memory than 8-bit, so more experts fit in
+VRAM. RTX 3090, the Coder at 198K context: 99 instead of 85 tokens/s output, the same needle results, prompts 2-5%
+slower. It does not stream its KV cache (KV streaming is on by default from 64K), so it pays off mostly on large
+cards at long contexts.
+
+**The draft layer's tokens (0.1.27, `--draft-vocab`):** the MTP draft layer can only propose tokens from a subset
+of the vocabulary (`mtp/rt/draft_vocab.bin`). Since 0.1.27 the subset includes every Chinese, Japanese and Korean
+token (106,299 ids), so answers in those languages are 15-38% faster (Q2_0, RTX 5070). Its head takes ~180 MiB of
+VRAM, which the expert cache leaves free for it (0.1.28). `START-HERE.bat --setup --draft-vocab en` keeps the
+English/code subset from before (40,525 ids, ~110 MiB less VRAM, English answers 1-2% faster; CJK answers get
+almost no drafts). `tools/draft_vocab.py` builds and inspects subsets.
+
+**Low-RAM mode (engine 0.1.26, chosen by setup):** normally all of a model's experts are copied into RAM (23-50 GB,
+pinned) and the GPU holds a copy of the most-used ones. On a PC whose RAM cannot hold them beside the system (the
+experts plus ~10 GB), setup instead maps them from one file in the model's folder (`--mmap-experts`, the pack's
+`experts.bin`, +23-50 GB of disk). The OS file cache holds what the GPU does not, and it can give that memory back.
+On the Coder the engine's committed memory drops from 36 to ~13 GB, with the same answers. With a big GPU (an RTX
+5090 holds all of the Coder's experts, most of Q2_0's) it runs at nearly the usual speed. With a small one, most
+experts come from the SSD and it is much slower (setup says so). `START-HERE.bat --setup --low-ram on|off` overrides
+the choice.
+
 Time to first token is prompt length / prompt speed: with Q2_0 about 4 s at 4K, 25 s at 32K, under 2 minutes at 128K
 and 4.5 minutes at 262K (engine 0.1.13 made long prompts about twice as fast, below).
 
@@ -148,7 +176,7 @@ You need **only an NVIDIA driver** (version 580 or newer; update it with the NVI
 
 | | |
 | --- | --- |
-| GPU | NVIDIA **RTX 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070; RTX 30/40 are untested. |
+| GPU | NVIDIA **RTX 20, 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070 and an RTX 3090; RTX 20 (Turing, since 0.1.27) was tested by a contributor on an RTX 2070. |
 | RAM | **64 GB** recommended (see the table above). |
 | CPU | x86-64 with AVX2 (any Intel/AMD desktop CPU from the last ~8 years). AVX-512 (Ryzen 7000/9000) is a bit faster. |
 | Disk | ~70-80 GB free for the model, ~6 GB for the MTP layer (+1 GB with images). **Q2_0 on an AVX-512 CPU** also writes a one-time ~40 GB copy of its experts for the fast CPU kernel. An NVMe SSD is strongly recommended. |
@@ -160,7 +188,7 @@ elsewhere) finds them and sets itself up the same way. The place is remembered p
 `~/.config/strata/settings.json`); `--data-dir` chooses another. Installs from before 0.1.16 are moved there by the next
 start (a rename on the same drive; files on another drive are used where they are).
 Python 3.12 if you have none (for your user account, no admin), a private Python environment, NVIDIA's CUDA libraries
-(from pip, ~0.4 GB), the ready-made Strata engine for RTX 30/40/50, the model and the MTP draft layer. If no
+(from pip, ~0.4 GB), the ready-made Strata engine for RTX 20/30/40/50, the model and the MTP draft layer. If no
 ready-made engine fits your PC, it offers to install the build tools (Visual Studio Build Tools + CUDA Toolkit on
 Windows, `build-essential` + CUDA on Ubuntu) and compiles the engine for your GPU (asks first; 20-40 minutes once).
 
@@ -216,6 +244,28 @@ install; `START-HERE.bat --calibrate` (Linux: `./setup.sh --calibrate`) does it
 speed with each setting and keeps one only when it is more than 3% faster. The result is remembered per PC and model
 (in the settings file next to the data folder's record), so updates keep it.
 
+### Running it at startup (Task Scheduler)
+
+To have the model up at logon, people start the serve from **Task Scheduler** (or a service). Beware: Windows
+throttles such contexts, and the model's ~40 GB expert load then crawls at **~0.05 GiB/s (13-14 minutes)**
+instead of **~1.4-1.5 GiB/s (~35 seconds)** - a 24x slower start. Measured on an RTX 5070 Ti + Ryzen 7 9800X3D
++ NVMe, same binary, same args, same cache state:
+
+| How the serve starts | Expert load |
+| --- | ---: |
+| Double-click / terminal / SSH | 1.42-1.52 GiB/s (~35 s) |
+| Task Scheduler with its defaults | 0.05 GiB/s (821-841 s) |
+| Task Scheduler with the two settings below | 1.42 GiB/s (35 s) |
+
+In the task's properties set both of these (the defaults are the opposite):
+
+- **Priority level: Normal** (Options tab; the default is Below normal), and
+- **Run with highest privileges** (General tab; without it the task runs with a limited user token - which
+  also strips `SeLockMemoryPrivilege`, the privilege Windows large pages need).
+
+(Both were changed at once, so the isolated effect of each is not measured.) If the model still starts
+slowly, the engine prints a hint under its `loaded ... GiB at ...` line naming this cause.
+
 ### Chat in the terminal (optional)
 
 ```
@@ -254,11 +304,14 @@ The server listens on `http://127.0.0.1:8080` (change with `--port` in setup, or
 | --- | --- |
 | OpenAI Chat Completions (stream and non-stream, tools) | `POST /v1/chat/completions` |
 | Anthropic Messages (stream and non-stream, tools) | `POST /v1/messages` |
-| Model list / health | `GET /v1/models`, `GET /health` |
-| What the model is doing right now | `GET /status` |
+| Model list / health | `GET /v1/models`, `GET /models`, `GET /health` |
+| Model properties | `GET /props` (also accepts `?model=<loaded-model-id>`) |
+| What the model is doing right now | `GET /status`, `GET /slots` (single slot, busy or idle) |
 | Everything the Monitor tab shows (engine, live state, last requests, hardware) | `GET /metrics` |
 | The MCP servers, their state and tools ([below](#tools-from-mcp-servers)) | `GET /mcp` |
 
+`/models` and `/v1/models` list only the loaded model, with its context limit and input modalities. `/props` exposes the original chat template, context limit, configured generation defaults (shared settings take precedence), model path and engine version when available. Context means the full engine context, not the resident KV window. `n_predict: -1` means no fixed output cap. Unconfigured sampling fields are omitted. `autoload` has no effect; an unknown `model` returns 404. These metadata endpoints and `/slots` require the API key when one is configured. They do not load, unload or restart models.
+
 ```bash
 curl http://127.0.0.1:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
   "model": "strata", "messages": [{"role": "user", "content": "Write a haiku about GPUs."}], "max_tokens": 512 }'

docs/MULTI_GPU.md

@@ -18,7 +18,7 @@ Strata can use it:
 ```
   Your NVIDIA GPUs:
     GPU 0: NVIDIA GeForce RTX 5080, 16 GB VRAM - can be used
-    GPU 1: NVIDIA GeForce RTX 2080 Ti, 22 GB VRAM - not supported - older than the RTX 30 series (compute capability 7.5; Strata needs 8.0 or newer)
+    GPU 1: NVIDIA GeForce GTX 1080 Ti, 11 GB VRAM - not supported - older than the RTX 20 series (compute capability 6.1; Strata needs 7.5 or newer)
     GPU 2: NVIDIA GeForce RTX 3090, 24 GB VRAM - can be used
   ...
   1) GPU 0 (NVIDIA GeForce RTX 5080, 16 GB) + GPU 2 (NVIDIA GeForce RTX 3090, 24 GB) together   (recommended)
@@ -41,7 +41,7 @@ now on; the answer is kept.
 ```
 
 **Not supported** (setup says so and names the cards that can be used instead):
-- a card older than the RTX 30 series (compute capability below 8.0: RTX 20, GTX 16/10);
+- a card older than the RTX 20 series (compute capability below 7.5: GTX 10 and older);
 - a card with less than 8 GB of VRAM, together with others (each card holds a copy of the dense weights and its
   own prompt buffers);
 - AMD and Intel GPUs, and a mix of NVIDIA with them.
@@ -96,8 +96,8 @@ into the card that owns the layer.
   capped to leave room for them.
 - On Windows only 8 GiB of the expert arena is pinned (more, mapped into two GPU contexts, leaves WDDM refusing
   allocations); the PCIe share covers those layers.
-- Every card needs compute capability 8.0 (RTX 30 or newer). A Turing card (RTX 20, sm_75) builds only with the
-  experimental `-DSTRATA_EXPERIMENTAL_SM75=ON`.
+- Every card needs compute capability 7.5 (RTX 20 or newer). The pre-sm_80 QSA scorer path is fp32 FMAs, so a
+  Turing card runs the same kernels instead of the tensor-core prompt attention.
 
 ## Measured
 

docs/benchmarks/2026-09-29-gfx1100.json

@@ -0,0 +1,299 @@
+{
+  "source_commit": "9568e78da663e6b224ab96d57baf450717b36e73",
+  "date": "2026-09-29",
+  "candidate_binary_sha256": "705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b",
+  "note": "Fresh same-session control and candidate runs. Throughput cap 128; not a coding-quality benchmark.",
+  "control": [
+    {
+      "kind": "warmup",
+      "trial": 0,
+      "wall_s": 0.45770244899904355,
+      "metrics": {
+        "prompt_tokens": 15.0,
+        "reused": 0.0,
+        "fresh": 15.0,
+        "prefill_ms": 335.0,
+        "prefill_tps": 44.8,
+        "generated": 2.0,
+        "decode_ms": 115.0,
+        "decode_tps": 17.5
+      },
+      "finish_reason": "stop"
+    },
+    {
+      "kind": "cold",
+      "trial": 1,
+      "wall_s": 20.068275763012934,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 17538.0,
+        "prefill_tps": 240.0,
+        "generated": 128.0,
+        "decode_ms": 2518.0,
+        "decode_tps": 50.8
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 1,
+      "wall_s": 4.412000925018219,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 1943.0,
+        "prefill_tps": 58.7,
+        "generated": 128.0,
+        "decode_ms": 2451.0,
+        "decode_tps": 52.2
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 2,
+      "wall_s": 20.731399575975956,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 18213.0,
+        "prefill_tps": 484.8,
+        "generated": 128.0,
+        "decode_ms": 2505.0,
+        "decode_tps": 51.1
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 2,
+      "wall_s": 5.0592734439997,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 2601.0,
+        "prefill_tps": 95.4,
+        "generated": 128.0,
+        "decode_ms": 2441.0,
+        "decode_tps": 52.4
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 3,
+      "wall_s": 11.695648370019626,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 9209.0,
+        "prefill_tps": 457.2,
+        "generated": 128.0,
+        "decode_ms": 2477.0,
+        "decode_tps": 51.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 3,
+      "wall_s": 4.093982424004935,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 1896.0,
+        "prefill_tps": 60.1,
+        "generated": 128.0,
+        "decode_ms": 2188.0,
+        "decode_tps": 58.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 4,
+      "wall_s": 20.84539221401792,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 18437.0,
+        "prefill_tps": 478.9,
+        "generated": 128.0,
+        "decode_ms": 2392.0,
+        "decode_tps": 53.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 4,
+      "wall_s": 5.357054464984685,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 2658.0,
+        "prefill_tps": 93.3,
+        "generated": 128.0,
+        "decode_ms": 2682.0,
+        "decode_tps": 47.7
+      },
+      "finish_reason": "length"
+    }
+  ],
+  "optimized": [
+    {
+      "kind": "warmup",
+      "trial": 0,
+      "wall_s": 0.46265592801501043,
+      "metrics": {
+        "prompt_tokens": 15.0,
+        "reused": 0.0,
+        "fresh": 15.0,
+        "prefill_ms": 335.0,
+        "prefill_tps": 44.8,
+        "generated": 2.0,
+        "decode_ms": 121.0,
+        "decode_tps": 16.5
+      },
+      "finish_reason": "stop"
+    },
+    {
+      "kind": "cold",
+      "trial": 1,
+      "wall_s": 11.728894605010282,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 9408.0,
+        "prefill_tps": 447.5,
+        "generated": 128.0,
+        "decode_ms": 2308.0,
+        "decode_tps": 55.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 1,
+      "wall_s": 2.9531024169991724,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 740.0,
+        "prefill_tps": 154.0,
+        "generated": 128.0,
+        "decode_ms": 2193.0,
+        "decode_tps": 58.4
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 2,
+      "wall_s": 12.42044531099964,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 10107.0,
+        "prefill_tps": 873.7,
+        "generated": 128.0,
+        "decode_ms": 2298.0,
+        "decode_tps": 55.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 2,
+      "wall_s": 3.2599652509961743,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 1070.0,
+        "prefill_tps": 231.7,
+        "generated": 128.0,
+        "decode_ms": 2173.0,
+        "decode_tps": 58.9
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 3,
+      "wall_s": 6.937167472002329,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 4672.0,
+        "prefill_tps": 901.1,
+        "generated": 128.0,
+        "decode_ms": 2256.0,
+        "decode_tps": 56.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 3,
+      "wall_s": 3.1877827199932653,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 827.0,
+        "prefill_tps": 137.8,
+        "generated": 128.0,
+        "decode_ms": 2350.0,
+        "decode_tps": 54.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 4,
+      "wall_s": 11.314008287008619,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 9136.0,
+        "prefill_tps": 966.5,
+        "generated": 128.0,
+        "decode_ms": 2161.0,
+        "decode_tps": 59.2
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 4,
+      "wall_s": 3.0476601750124246,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8957.0,
+        "fresh": 114.0,
+        "prefill_ms": 734.0,
+        "prefill_tps": 155.3,
+        "generated": 128.0,
+        "decode_ms": 2296.0,
+        "decode_tps": 55.7
+      },
+      "finish_reason": "length"
+    }
+  ],
+  "control_binary_sha256": "4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7"
+}

include/strata/artifact/gguf_reader.hpp

@@ -26,6 +26,7 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <set>
 #include <stdexcept>
 #include <algorithm>
 
@@ -418,9 +419,14 @@ private:
             meta_.emplace(std::move(key), read_value(c, t));
         }
         tensors_.reserve((size_t)n_tensors);
+        // GGUF has no index to arbitrate between two tensors of one name: find() is first-match, so a
+        // duplicate would silently win by position.  Refuse the file at open instead, naming both.
+        std::set<std::string> names;
         for (uint64_t i = 0; i < n_tensors; ++i) {
             TensorInfo t;
             t.name = c.str();
+            if (!names.insert(t.name).second)
+                throw std::runtime_error("GGUF: duplicate tensor name '" + t.name + "' in " + path_);
             const uint32_t nd = c.read<uint32_t>();
             if (nd == 0 || nd > 4) throw std::runtime_error("GGUF: bad n_dims for " + t.name);
             t.shape.resize(nd);

include/strata/core/expert_cache.hpp

@@ -26,6 +26,7 @@
 // kernel. This is that first step, and the step it unblocks is the one that can be measured.
 #pragma once
 
+#include <cstddef>
 #include <cstdint>
 #include <string>
 #include <vector>
@@ -140,6 +141,11 @@ public:
     int64_t fills() const { return fills_; }
 
 private:
+#if defined(STRATA_USE_HIP)
+    bool ensure_blocking_staging(std::size_t bytes, std::string& err);
+    uint8_t* blocking_staging_ = nullptr;
+    std::size_t blocking_staging_bytes_ = 0;
+#endif
     uint8_t* base_ = nullptr;
     std::vector<int32_t> residency_;   ///< [n_layers * n_expert] -> slot or kNotResident
     int64_t slots_ = 0;

include/strata/core/expert_source.hpp

@@ -26,6 +26,8 @@
 #include "strata/kernels/cpu/pool.hpp"
 
 #include <cstdint>
+#include <cstddef>
+#include <utility>
 #include <string>
 #include <vector>
 
@@ -33,6 +35,41 @@ namespace strata::core {
 
 class RemoteExperts;
 
+namespace detail {
+
+/// Sentinel used by the pure complement planner for a blob that remains in the mmap fallback.
+inline constexpr uint64_t kNoCacheComplement = ~uint64_t{0};
+
+/// Required cgroup-v2 usage counters for the conservative cache-reclaim allowance.
+struct CgroupMemoryStat {
+    uint64_t current = 0;
+    uint64_t inactive_file = 0;
+    uint64_t file_dirty = 0;
+    uint64_t file_writeback = 0;
+    bool valid = false;
+};
+
+/// Calculate additional bytes under a finite cgroup limit after reclaiming only clean inactive file cache.
+/// Returns false when the required memory.stat counters were unavailable.
+bool cgroup_available_bytes(uint64_t limit, const CgroupMemoryStat& stat, uint64_t& bytes);
+
+/// Build compact offsets for experts absent from both the primary GPU cache and an optional second GPU tier.
+/// Kept CPU-only so selection and byte accounting can be tested without initializing a GPU.
+bool make_cache_complement_plan(
+    int64_t n_layers, int64_t n_expert, const std::vector<uint64_t>& layer_blob_bytes,
+    const std::vector<std::pair<int32_t, int32_t>>& primary_gpu_pairs,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs,
+    std::vector<uint64_t>& offsets, uint64_t& bytes, std::string& err);
+
+/// Resolve one blob through the compact copy when present, otherwise preserve its exact mapped-file fallback.
+const uint8_t* cache_complement_blob_or_fallback(
+    size_t index, const std::vector<uint64_t>& offsets, const uint8_t* complement_host,
+    const uint8_t* mapped_fallback);
+
+}  // namespace detail
+
+
+
 /// Where one routed expert's bytes come from.
 ///
 /// Phase 2 has NO cache (`phase-2-correct-engine.md`: hit rate `h = 0`), so the only implementation is a
@@ -42,7 +79,7 @@ class ExpertSource {
 public:
     virtual ~ExpertSource() = default;
 
-    /// The 1,382,400-byte blob for `(layer, expert)`, or nullptr if it cannot be produced.
+    /// The expert-layout blob for `(layer, expert)`, or nullptr if it cannot be produced.
     ///
     /// The pointer only has to stay valid until the next `blob()` call: with `h = 0` every expert is computed
     /// immediately and nothing is retained.  A CACHING source must return pointers into the cache, not into a
@@ -270,26 +307,49 @@ public:
     FileExpertSource(const FileExpertSource&) = delete;
     FileExpertSource& operator=(const FileExpertSource&) = delete;
 
-    /// Maps `<pack_dir>/experts.bin` and checks its size against `n_layers * n_expert * BLOB`.
+    /// Maps `<pack_dir>/experts.bin` and checks its size against the loaded expert layout.  Canonical packs use
+    /// `n_layers * n_expert * BLOB`; native packs use their variable per-layer blob sizes and offsets.
     ///
     /// The size check is not a formality: a short file would fault at the END of a long sequence, and an
     /// over-long one means the pack is not the one the geometry came from.  Refuses with the two numbers.
     bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err);
+    /// Pin a compact host mirror of experts absent from a fully filled static GPU cache. The mmap remains open
+    /// as a fallback for later cache reloads. This is opt-in because the complement may still be a large allocation.
+    bool pin_cache_complement(
+        const ExpertCache& cache, std::string& err, bool pin = true,
+        const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs = {});
     void close();
 
     bool mapped() const { return base_ != nullptr; }
     int64_t blobs() const { return blobs_; }
+    uint64_t pinned_bytes() const { return complement_pinned_ ? complement_bytes_ : 0; }
+    uint64_t resident_bytes() const { return complement_bytes_; }
+    bool complement_pinned() const { return complement_pinned_; }
 
     const uint8_t* blob(int64_t layer, int64_t expert) override;
+    bool pinned(int64_t layer, int64_t expert) const override;
+    const uint8_t* device_alias(int64_t layer, int64_t expert) const override;
 
     /// Blobs touched, for the driver to report.  With `h = 0` this is `48 * k` per token and the number is only
     /// interesting once Phase 3 makes it not so.
     int64_t reads() const override { return reads_; }
 
 private:
+    const uint8_t* mapped_blob(int64_t layer, int64_t expert) const;
+    static constexpr uint64_t kNoComplement = detail::kNoCacheComplement;
     const uint8_t* base_ = nullptr;
     int64_t blobs_ = 0;
+    int64_t n_layers_ = 0;
     int64_t n_expert_ = 0;
+    uint64_t mapped_bytes_ = 0;
+    std::vector<uint64_t> layer_offsets_, layer_blob_bytes_;
+    void* complement_arena_ = nullptr;
+    const uint8_t* complement_host_ = nullptr;
+    const uint8_t* complement_device_ = nullptr;
+    uint64_t complement_bytes_ = 0;
+    std::vector<uint64_t> complement_offsets_;
+    bool complement_pinned_ = false;
+    bool complement_ready_ = false;
     int64_t reads_ = 0;
 #if defined(_WIN32)
     void* file_ = nullptr;
@@ -341,6 +401,12 @@ public:
     /// only true if the engine says what it got.
     const std::string& note() const { return note_; }
     double load_gib_per_second() const { return gib_per_s_; }
+    // Loader fix: the load, split.  `load_seconds()` is the wall clock of the load loop; the other two are
+    // sums over the reader threads (see LoadStats), so on their own they say how much of that wall was spent
+    // waiting for the disk and how much in memcpy + FNV-1a.
+    double load_seconds() const { return load_seconds_; }
+    double load_read_seconds() const { return load_read_s_; }
+    double load_copy_seconds() const { return load_copy_s_; }
 
 private:
     void* arena_ = nullptr;          ///< the PinnedArena, owned
@@ -352,6 +418,9 @@ private:
     int64_t reads_ = 0;
     std::string note_;
     double gib_per_s_ = 0.0;
+    double load_seconds_ = 0.0;
+    double load_read_s_ = 0.0;
+    double load_copy_s_ = 0.0;
     uint64_t pinned_bytes_ = 0;
     std::string gguf_;
 };

include/strata/core/layer.hpp

@@ -40,6 +40,8 @@
 #include "strata/kernels/ngram.hpp"
 #include "strata/kernels/ple.hpp"
 
+#include <cstdio>
+#include <cstdlib>
 #include <cstdint>
 #include <string>
 
@@ -211,6 +213,10 @@ struct QsaState {
     bool kv_q4 = false;
     uint8_t* k_q4 = nullptr;
     uint8_t* v_q4 = nullptr;
+    /// Hybrid K8V4: K in INT8 (unrotated - the scores stay exact), V in rotated Q4_0 (kv_q4.hpp): 816 B per cell.
+    /// Uses k_q/k_scale + v_q4. Mode 0 only (no KV streaming, no ring); under this setting the MTP drafter's
+    /// ring state stays plain INT8, so the block movers never see the hybrid layout.
+    bool kv_hybrid = false;
     int32_t* page_table = nullptr;   ///< (n_pages,) logical page -> physical page (-1: not resident, streamed)
     int64_t n_pages = 0;
     int64_t max_cells = 0;
@@ -276,8 +282,17 @@ bool qsa_kv_int8();
 /// PR #21: store K/V as Q4_0 after a Hadamard rotation (`--kv q4_0`): 576 B per cell, vs 1,056 in INT8.
 void qsa_set_kv_q4(bool enabled);
 bool qsa_kv_q4();
+/// Hybrid K8V4 (`--kv k8v4`): K in INT8, V in rotated Q4_0 - 816 B per cell. Not with --kv-resident.
+void qsa_set_kv_hybrid(bool enabled);
+bool qsa_kv_hybrid();
 /// The state's KV format for the block-moving functions of kv_stream.hpp (kKvF16 / kKvInt8 / kKvQ4).
 inline int qsa_kv_format(const QsaState& st) {
+    // A hybrid K8V4 state is mode 0 only and never reaches the block movers; refuse rather than let it
+    // fall through to kKvF16 - a wrong layout silently applied is worse than a hard stop (PR review).
+    if (st.kv_hybrid) {
+        std::fprintf(stderr, "strata: qsa_kv_format: a hybrid K8V4 state must never reach the block movers\n");
+        std::exit(1);   // the kernels' own "unsupported geometry" convention (kv_q8.cu, qsa_decode_attn.cu)
+    }
     return st.kv_q4 ? strata::kernels::kKvQ4 : st.kv_int8 ? strata::kernels::kKvInt8 : strata::kernels::kKvF16;
 }
 uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, QsaState& st,

include/strata/core/mtp.hpp

@@ -53,6 +53,9 @@ public:
     /// KV streaming: refill the ring of the drafter's window from its host copy for a sequence that continues at
     /// `upto` (a conversation-cache resume). No-op unless the drafter's K/V is a ring.
     void kv_restore(int64_t upto);
+    /// The VRAM bind() will allocate for a native head of `head_row_bytes` per vocabulary row: the draft logits and
+    /// the draft head over rt/draft_vocab.bin's subset.  The expert cache is sized before bind(), so it reserves this.
+    uint64_t bind_bytes(uint64_t head_row_bytes, int64_t n_vocab) const;
     /// The main model's embedding and head, and the verify window's final residuals (T rows, hc*n_embd each).
     bool bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err);
 
@@ -73,6 +76,19 @@ public:
     double ms_draft = 0, ms_prefill = 0;
     int64_t rounds = 0;
 
+    /// E-9: the prompt path computes this layer's prompt K/V in batches (Prefill::draft_kv): its tensors, its K/V
+    /// state, the first cell a round can still read, its device, and a wait for its own stream.
+    const float* tensor_f32(const char* name) const { return f32(name); }
+    const uint16_t* tensor_bf16(const char* name) const { return bf16(name); }
+    const void* tensor_q8(const char* name) const { return q8(name); }
+    QsaState& kv_state_rw() { return st_; }
+    int64_t first_needed() const { return (window_ > 0 && prompt_len_ > 0) ? prompt_len_ - window_ - 64 : 0; }
+    int device() const { return device_; }
+    bool idle(std::string& err) {
+        if (cs_ && cudaStreamSynchronize(cs_) != cudaSuccess) { err = "mtp: its stream failed"; return false; }
+        return true;
+    }
+
 private:
     bool record_forward(int T, int step_row0, cudaStream_t cs, std::string& err);
     bool capture_prefill(int T, std::string& err);

include/strata/core/pinned.hpp

@@ -52,15 +52,40 @@ struct PinnedArena {
 
 struct LoadStats {
     double seconds = 0.0;
+    // Loader fix: the two halves of `seconds`, both SUMMED OVER THE READER THREADS rather than wall clock, so
+    // either can exceed `seconds` when the threads overlap.  They exist to separate the read from the copy and
+    // the hash: an 8 MiB chunk that reaches the disk as ~2048 4 KiB reads spends its time in the first one.
+    double read_seconds = 0.0;                  // inside the read call only: no seek, no copy, no hash
+    double copy_seconds = 0.0;                  // memcpy + FNV-1a
+    // A short read is not a slow load, it is a WRONG one: the caller must refuse the pack rather than run
+    // on an arena whose tail was never written.  `seconds` alone cannot say that (it is -1.0 on failure, but
+    // the caller's own check is on the REQUESTED byte count, which a refused read does not change).
+    bool ok = true;                             // false: the load failed, see `error`
+    std::string error;                          // why it failed, for the caller's message
     uint64_t bytes = 0;
     uint64_t layers = 0;
     std::vector<uint64_t> layer_checksums;      // one FNV-1a per layer
     double gib_per_second() const { return seconds > 0 ? (double) bytes / (1024.0 * 1024 * 1024) / seconds : 0.0; }
+    // Aggregate rate of the readers WHILE THEY WERE INSIDE THE READ CALL: `bytes` over the mean per-thread
+    // read time.  `gib_per_second()` above divides the same bytes by the wall clock of the whole loop, which
+    // also contains the copy and the hash, so the two differ by how much of the loop was not I/O.
+    double read_gib_per_second(int threads) const {
+        if (read_seconds <= 0.0 || threads < 1) return 0.0;
+        return (double) bytes / (1024.0 * 1024 * 1024) / (read_seconds / (double) threads);
+    }
 };
 
 // Load `layers` layers of the expert arena into `dst` with `threads` readers, `chunk` bytes at a time.
 // Each thread opens its OWN handle and seeks, which is the portable form of parallel pread: a shared handle
 // needs a lock around the seek and defeats the parallelism on Windows.
+//
+// The reads go through `fread()` on a per-thread `FILE*`, NOT through `std::ifstream`.  MSVC's
+// `basic_filebuf::xsgetn` splits every request larger than `_INTERNAL_BUFSIZ - 1` into 4095-byte `fread()`
+// calls, so an 8 MiB `chunk` reached the disk as ~2048 4 KiB operations and the chunking did nothing at all
+// (measured on a 42.9 GB pack: 1222 s, 4096 bytes per operation, 0.03 GiB/s).  `fread()` hands a request
+// larger than the stream buffer straight to `_read()`/`ReadFile()`, which is what `chunk` is for.  The
+// control flow, the copy, the per-layer FNV-1a and therefore the checksums are unchanged, so a pack loaded
+// by the old and the new reader must produce identical `layer_checksums`.
 LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_bytes, uint64_t blobs_per_layer,
                        uint64_t layers, int threads, uint64_t chunk);
 /// Plan v0.3 P6: the same with one byte range per layer (`layer_off[L]`, `layer_bytes[L]`).

include/strata/core/verify.hpp

@@ -32,6 +32,7 @@
 
 #include <cstdint>
 #include <string>
+#include <vector>
 
 namespace strata::core {
 
@@ -124,9 +125,13 @@ public:
     /// arena directly, 2 = a copy kernel stages it inside the graph (no API calls on the pool's thread; best when
     /// the CPU is RAM-bound, Q2_0).  Set before the first `run`.
     void set_pcie_mode(int mode) { sink_.pcie_mode = mode; }
+    /// the pool never plans a PCIe share (--pcie-frac 0): the window skips that path.  Before the first run.
 
     double ms_wait = 0, ms_pool = 0, ms_host = 0, ms_commit = 0;
     int64_t windows = 0;
+    /// STRATA_VERIFY_PROFILE=1 - GPU stage times of the windows since the last call (ms per
+    /// window), as one line; empty when off.
+    std::string profile_report();
 
 private:
     bool capture(int T, std::string& err);
@@ -151,6 +156,12 @@ private:
     bool ple_stage() const { return lb_ <= 1 && 1 < le_; }   ///< holds layer 1, where the PLE block runs
     bool capture_commit(std::string& err);
     bool record_window(int T, cudaStream_t cs, std::string& err);
+    static constexpr int kProfPer = 32;              // stamps per layer
+    bool prof_on_ = false;
+    unsigned long long* prof_ = nullptr;              // device: n_layers * kProfPer + 4 stamps
+    std::vector<unsigned long long> prof_h_;
+    double prof_sum_[2][kProfPer] = {};   // [GDN / QSA layers][stage]
+    int64_t prof_windows_ = 0;
 
     const WeightTable* wt_ = nullptr;
     const ModelGeometry* g_ = nullptr;

include/strata/hip_compat/cublas_v2.h

@@ -0,0 +1,19 @@
+#pragma once
+#include <hipblas/hipblas.h>
+#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F
+#define CUBLAS_DEFAULT_MATH HIPBLAS_DEFAULT_MATH
+#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT
+#define CUBLAS_OP_N HIPBLAS_OP_N
+#define CUBLAS_OP_T HIPBLAS_OP_T
+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
+#define CUDA_R_16BF HIP_R_16BF
+#define CUDA_R_16F HIP_R_16F
+#define CUDA_R_32F HIP_R_32F
+#define cublasCreate hipblasCreate
+#define cublasDestroy hipblasDestroy
+#define cublasGemmEx hipblasGemmEx
+#define cublasHandle_t hipblasHandle_t
+#define cublasSetMathMode hipblasSetMathMode
+#define cublasSetStream hipblasSetStream
+#define cublasSetWorkspace hipblasSetWorkspace
+#define cublasStatus_t hipblasStatus_t

include/strata/hip_compat/cuda_fp16.h

@@ -0,0 +1,2 @@
+#pragma once
+#include <hip/hip_fp16.h>

include/strata/hip_compat/cuda_runtime.h

@@ -0,0 +1,102 @@
+#pragma once
+// Included only by STRATA_ENABLE_HIP builds. CUDA builds use NVIDIA headers.
+#include <hip/hip_runtime.h>
+// Do not let HIP's legacy macro corrupt libstdc++ attribute names.
+#ifdef __noinline__
+#undef __noinline__
+#endif
+#define cudaDevAttrMaxSharedMemoryPerBlockOptin hipDeviceAttributeMaxSharedMemoryPerBlock
+#define cudaDevAttrMultiProcessorCount hipDeviceAttributeMultiprocessorCount
+#define cudaDevAttrClockRate hipDeviceAttributeClockRate
+#define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor
+#define cudaDeviceGetAttribute hipDeviceGetAttribute
+#define cudaDeviceProp hipDeviceProp_t
+#define cudaDeviceSynchronize hipDeviceSynchronize
+#define cudaDriverGetVersion hipDriverGetVersion
+#define cudaErrorNotReady hipErrorNotReady
+#define cudaErrorStreamCaptureUnsupported hipErrorStreamCaptureUnsupported
+#define cudaError_t hipError_t
+#define cudaEventCreate hipEventCreate
+#define cudaEventCreateWithFlags hipEventCreateWithFlags
+#define cudaEventDestroy hipEventDestroy
+#define cudaEventDisableTiming hipEventDisableTiming
+#define cudaEventElapsedTime hipEventElapsedTime
+#define cudaEventQuery hipEventQuery
+#define cudaEventRecord hipEventRecord
+#define cudaEventSynchronize hipEventSynchronize
+#define cudaEvent_t hipEvent_t
+#define cudaFree hipFree
+#define cudaFreeHost hipFreeHost
+#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize
+#define cudaGetDevice hipGetDevice
+#define cudaGetDeviceCount hipGetDeviceCount
+#define cudaGetDeviceProperties hipGetDeviceProperties
+#define cudaGetErrorString hipGetErrorString
+#define cudaGetLastError hipGetLastError
+#define cudaGraphDestroy hipGraphDestroy
+#define cudaGraphExecDestroy hipGraphExecDestroy
+#define cudaGraphExec_t hipGraphExec_t
+#define cudaGraphGetNodes hipGraphGetNodes
+#define cudaGraphLaunch hipGraphLaunch
+#define cudaGraphUpload hipGraphUpload
+#define cudaGraph_t hipGraph_t
+#define cudaHostAlloc hipHostMalloc
+#define cudaHostAllocDefault hipHostAllocDefault
+#define cudaHostAllocMapped hipHostAllocMapped
+#define cudaHostAllocPortable hipHostAllocPortable
+#define cudaHostGetDevicePointer hipHostGetDevicePointer
+#define cudaHostRegister hipHostRegister
+#define cudaHostRegisterMapped hipHostRegisterMapped
+#define cudaHostRegisterPortable hipHostRegisterPortable
+#define cudaHostUnregister hipHostUnregister
+#define cudaLaunchHostFunc hipLaunchHostFunc
+#define cudaMalloc hipMalloc
+#define cudaMallocHost(...) (::strata::hip_compat::malloc_host(__VA_ARGS__))
+#define cudaMemGetInfo hipMemGetInfo
+#define cudaMemcpy hipMemcpy
+#define cudaMemcpy2DAsync hipMemcpy2DAsync
+#define cudaMemcpyAsync hipMemcpyAsync
+#define cudaMemcpyDefault hipMemcpyDefault
+#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice
+#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost
+#define cudaMemcpyHostToDevice hipMemcpyHostToDevice
+#define cudaMemset hipMemset
+#define cudaMemsetAsync hipMemsetAsync
+#define cudaPeekAtLastError hipPeekAtLastError
+#define cudaRuntimeGetVersion hipRuntimeGetVersion
+#define cudaSetDevice hipSetDevice
+#define cudaStreamBeginCapture hipStreamBeginCapture
+#define cudaStreamCaptureModeThreadLocal hipStreamCaptureModeThreadLocal
+#define cudaStreamCreate hipStreamCreate
+#define cudaStreamCreateWithFlags hipStreamCreateWithFlags
+#define cudaStreamDestroy hipStreamDestroy
+#define cudaStreamEndCapture hipStreamEndCapture
+#define cudaStreamNonBlocking hipStreamNonBlocking
+#define cudaStreamQuery hipStreamQuery
+#define cudaStreamSynchronize hipStreamSynchronize
+#define cudaStreamWaitEvent hipStreamWaitEvent
+#define cudaStream_t hipStream_t
+#define cudaSuccess hipSuccess
+
+namespace strata::hip_compat {
+template <typename T>
+inline hipError_t malloc_host(T** pointer, size_t bytes) {
+    return hipHostMalloc(reinterpret_cast<void**>(pointer), bytes, hipHostMallocDefault);
+}
+}  // namespace strata::hip_compat
+
+template <typename Kernel>
+inline hipError_t cudaFuncSetAttribute(Kernel kernel, hipFuncAttribute attribute, int value) {
+    return hipFuncSetAttribute(reinterpret_cast<const void*>(kernel), attribute, value);
+}
+inline hipError_t cudaGraphInstantiate(hipGraphExec_t* exec, hipGraph_t graph, unsigned long long flags) {
+    return hipGraphInstantiateWithFlags(exec, graph, flags);
+}
+inline hipError_t cudaGraphInstantiate(hipGraphExec_t* exec, hipGraph_t graph,
+                                     hipGraphNode_t* error, char* log, size_t size) {
+    return hipGraphInstantiate(exec, graph, error, log, size);
+}
+#define __trap() __builtin_trap()   // the compiled-out sm_80 paths (never selected on AMD)
+#define cudaMemcpyToSymbol(symbol, ...) hipMemcpyToSymbol(HIP_SYMBOL(symbol), __VA_ARGS__)
+
+#include "intrinsics.hpp"

include/strata/hip_compat/cuda_runtime_api.h

@@ -0,0 +1,2 @@
+#pragma once
+#include "cuda_runtime.h"

include/strata/hip_compat/intrinsics.hpp

@@ -0,0 +1,125 @@
+#pragma once
+
+// CUDA device intrinsics used by Strata kernels that HIP does not provide on gfx1100.
+// This header is included only from the HIP cuda_runtime compatibility shim.
+#if defined(__HIPCC__)
+
+#include <cstdint>
+
+namespace strata::hip_compat {
+
+__device__ __forceinline__ int signed_byte(uint32_t word, int lane) {
+    const unsigned value = (word >> (lane * 8)) & 0xffu;
+    return value < 0x80u ? static_cast<int>(value) : static_cast<int>(value) - 0x100;
+}
+
+// CUDA's signed __dp4a: four signed byte products accumulated modulo 2^32.
+__device__ __forceinline__ int dp4a(int a, int b, int c) {
+#if defined(__gfx1100__) && __has_builtin(__builtin_amdgcn_sudot4)
+    // RDNA3 exposes the signed/unsigned dot4 form. Mark both packed operands
+    // signed to preserve CUDA __dp4a semantics; keep the portable path for
+    // other HIP compilers/targets.
+    return __builtin_amdgcn_sudot4(true, a, true, b, c, false);
+#else
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t sum = static_cast<uint32_t>(c);
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane)
+        sum += static_cast<uint32_t>(signed_byte(ua, lane) * signed_byte(ub, lane));
+    return static_cast<int>(sum);
+#endif
+}
+
+// CUDA's packed byte subtract wraps independently in each unsigned byte lane.
+__device__ __forceinline__ int vsub4(int a, int b) {
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t out = 0;
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane) {
+        const uint32_t x = (ua >> (lane * 8)) & 0xffu;
+        const uint32_t y = (ub >> (lane * 8)) & 0xffu;
+        out |= ((x - y) & 0xffu) << (lane * 8);
+    }
+    return static_cast<int>(out);
+}
+
+// CUDA's packed signed-byte saturating subtract.
+__device__ __forceinline__ int vsubss4(int a, int b) {
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t out = 0;
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane) {
+        int value = signed_byte(ua, lane) - signed_byte(ub, lane);
+        value = value < -128 ? -128 : (value > 127 ? 127 : value);
+        out |= (static_cast<uint32_t>(value) & 0xffu) << (lane * 8);
+    }
+    return static_cast<int>(out);
+}
+
+// CUDA's four-lane byte compare, returning 0xff for each unequal lane and 0 otherwise.
+__device__ __forceinline__ int vcmpne4(int a, int b) {
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t out = 0;
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane) {
+        if (((ua >> (lane * 8)) & 0xffu) != ((ub >> (lane * 8)) & 0xffu))
+            out |= 0xffu << (lane * 8);
+    }
+    return static_cast<int>(out);
+}
+
+// CUDA's mask argument describes participating lanes. The current kernel set uses full
+// wave32 masks; reject future partial-mask use instead of silently dropping its semantics.
+__device__ __forceinline__ void require_full_wave_mask(uint32_t mask) {
+    if (mask != 0xffffffffu) __builtin_trap();
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_xor_sync(uint32_t mask, T value, int lane_mask, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl_xor(value, lane_mask, width);
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_down_sync(uint32_t mask, T value, unsigned delta, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl_down(value, delta, width);
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_up_sync(uint32_t mask, T value, unsigned delta, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl_up(value, delta, width);
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_sync(uint32_t mask, T value, int source_lane, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl(value, source_lane, width);
+}
+
+__device__ __forceinline__ unsigned ballot_sync(uint32_t mask, int predicate) {
+    require_full_wave_mask(mask);
+    return static_cast<unsigned>(__ballot(predicate));
+}
+
+}  // namespace strata::hip_compat
+
+#define __dp4a(a, b, c) (::strata::hip_compat::dp4a((a), (b), (c)))
+#define __vsub4(a, b) (::strata::hip_compat::vsub4((a), (b)))
+#define __vsubss4(a, b) (::strata::hip_compat::vsubss4((a), (b)))
+#define __vcmpne4(a, b) (::strata::hip_compat::vcmpne4((a), (b)))
+#define __shfl_xor_sync(...) (::strata::hip_compat::shfl_xor_sync(__VA_ARGS__))
+#define __shfl_down_sync(...) (::strata::hip_compat::shfl_down_sync(__VA_ARGS__))
+#define __shfl_up_sync(...) (::strata::hip_compat::shfl_up_sync(__VA_ARGS__))
+#define __shfl_sync(...) (::strata::hip_compat::shfl_sync(__VA_ARGS__))
+#define __ballot_sync(mask, predicate) (::strata::hip_compat::ballot_sync((mask), (predicate)))
+// AMD's sleep instruction accepts only 0..15; the synchronization loops use it as a
+// backoff hint, so use its smallest portable delay independently of CUDA cycle counts.
+#define __nanosleep(cycles) __builtin_amdgcn_s_sleep(1)
+
+#endif  // defined(__HIPCC__)

include/strata/kernels/bf16_bits.hpp

@@ -19,7 +19,7 @@
 #include <cstdint>
 #include <cstring>
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_BF16_HD __host__ __device__
 #else
 #define STRATA_BF16_HD
@@ -30,7 +30,11 @@ namespace strata::kernels {
 /// Round-to-nearest-even f32 -> bf16, returned as raw bits in the LOW half of the uint16.
 STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) {
     uint32_t i;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&i, &f, 4);
+#else
     std::memcpy(&i, &f, 4);
+#endif
     i = (i + ((i >> 16) & 1u) + 0x7FFFu) & 0xFFFF0000u;
     return (uint16_t) (i >> 16);
 }
@@ -39,7 +43,11 @@ STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) {
 STRATA_BF16_HD inline float f32_from_bf16(uint16_t h) {
     const uint32_t i = (uint32_t) h << 16;
     float f;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&f, &i, 4);
+#else
     std::memcpy(&f, &i, 4);
+#endif
     return f;
 }
 

include/strata/kernels/bf16_gemv.hpp

@@ -44,5 +44,9 @@ void bf16_gemv_split(const uint16_t* x, const uint16_t* w, float* y, int64_t n_i
 /// No allocations or synchronization, including when stream is null (the CUDA default stream).
 void bf16_gemv_fp32_mmvf(const float* x, const uint16_t* w, float* y,
                          int64_t n_in, int64_t n_out, void* stream);
+/// bf16_gemv_fp32_mmvf for n_tok (1..8) activation rows x[t * ldx], outputs y[t * ldy + j]; one launch,
+/// the weight read once, every output bit-identical to its own single-row call.
+void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, float* y, int64_t ldy,
+                               int64_t n_in, int64_t n_out, int n_tok, void* stream);
 
 }  // namespace strata::kernels

include/strata/kernels/elementwise.hpp

@@ -102,6 +102,10 @@ void doorbell_wait(const uint32_t* d_flag, const uint32_t* d_seq, void* stream);
 /// device memory with a kernel, so the handoff stays on the compute queue (a memcpy node is a copy-engine
 /// operation, which WDDM submits separately and which measured 67 flushes per token).
 void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream);
+/// `rows` rows of `width` floats from mapped memory, except the rows listed in hit_rows[0, *count) (device),
+/// which are written +0.0 instead (a verify window's GPU-computed entries: the pool leaves zeros there).
+void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
+                           const int32_t* count, void* stream);
 
 /// Plan v0.3 P3: the doorbell's payload and its ring in ONE kernel.  Copies `x` (n floats), `ids` and `weights`
 /// (k each) into the mapped host regions, fences, and increments the mapped sequence number - replacing three

include/strata/kernels/f16_bits.hpp

@@ -25,7 +25,7 @@
 #include <cstdint>
 #include <cstring>
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_HD __host__ __device__
 #else
 #define STRATA_HD
@@ -36,7 +36,11 @@ namespace strata::kernels {
 /// Round-to-nearest-even f32 -> fp16, returned as raw bits.
 STRATA_HD inline uint16_t f16_from_f32(float f) {
     uint32_t x;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&x, &f, 4);
+#else
     std::memcpy(&x, &f, 4);
+#endif
     const uint32_t sign = (x >> 16) & 0x8000u;
     const uint32_t rawexp = (x >> 23) & 0xFFu;
     int exp = (int) rawexp - 127 + 15;
@@ -89,7 +93,11 @@ STRATA_HD inline float f32_from_f16(uint16_t h) {
         out = sign | ((ex - 15 + 127) << 23) | (man << 13);
     }
     float f;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&f, &out, 4);
+#else
     std::memcpy(&f, &out, 4);
+#endif
     return f;
 }
 

include/strata/kernels/fused_gr.hpp

@@ -46,6 +46,7 @@ void fused_gr_read(const FusedGrArgs& a, void* stream);
 /// weight pointers and eps must be the same for every t); `xn_scratch` is n_tok * hc * n_embd floats.  Every
 /// token's outputs are bitwise `fused_gr_read(a[t])`.
 constexpr int kFusedGrMaxT = 8;
-void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream);
+void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream,
+                         unsigned long long* stamp_buf = nullptr, int stamp_i0 = 0);
 
 }  // namespace strata::kernels

include/strata/kernels/mrope.hpp

@@ -22,7 +22,7 @@ namespace strata::kernels {
 void mrope_table_set(const int32_t* device_table);
 const int32_t* mrope_table();
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 /// ggml rope_multi, is_imrope, sections {11, 11, 10, 0}: sector = pair % 32; sector % 3 == 1 -> h (sector < 33),
 /// == 2 -> w (sector < 30), == 0 -> t (sector < 33).  For pairs 0..31 all three bounds hold, so it is pair % 3.
 __device__ __forceinline__ int mrope_pos(const int32_t* tab, int pos, int pair) {

include/strata/kernels/native_moe.hpp

@@ -15,4 +15,7 @@ bool native_moe_combine_enabled();
 // Requires a nonnull ordered stream and disjoint output. No allocation or sync.
 void native_moe_combine(const float* parts, const float* weights, const float* shared,
                         float* output, int64_t n_embd, int64_t k, void* stream);
+/// n_tok rows (parts [n,k,N], weights [n,k], shared/output [n,N]) in one launch, each as the single call.
+void native_moe_combine_multi(const float* parts, const float* weights, const float* shared, float* output,
+                              int64_t n_embd, int64_t k, int n_tok, void* stream);
 }

include/strata/kernels/native_router.hpp

@@ -14,4 +14,6 @@ bool native_router_enabled();
 // four-byte aligned and outputs disjoint from each other and the input.
 // Requires a nonnull ordered CUDA stream. No allocation or synchronization.
 void native_router_top10(const float* logits, int32_t* ids, float* weights, void* stream);
+/// n_tok rows at once (logits [n,512], ids/weights [n,10]); each row exactly as the single call.
+void native_router_top10_multi(const float* logits, int32_t* ids, float* weights, int n_tok, void* stream);
 }

include/strata/kernels/qsa_select.hpp

@@ -28,8 +28,16 @@ void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx
                       int64_t active_blocks = -1);   ///< perf-review C-1: > 0 launches only this many blocks (the
                                                      ///< batch's largest n_bid + 1; not for a captured graph)
 
+/// The same scores on tensor cores (3xTF32, FP32-level accuracy but another summation order: not bitwise; the tail
+/// block n_bid is the warp kernel's arithmetic). For the prompt path; false (nothing launched) on another geometry.
+bool qsa_block_scores_tc(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+                         int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks);
+
 /// ids [nq, cap] (cells, ascending); `cap` >= the largest selection width.
 void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
                     const QsaShapes& s, int32_t* ids, void* stream);
+/// The original kernel (keys read from memory on every radix pass), for tests: the same ids.
+void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                        const QsaShapes& s, int32_t* ids, void* stream);
 
 }  // namespace strata::kernels

include/strata/kernels/rope.hpp

@@ -10,7 +10,7 @@
 
 #include <cstdint>
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_ROPE_HD __host__ __device__
 #else
 #define STRATA_ROPE_HD

include/strata/kernels/verify_kernels.hpp

@@ -38,6 +38,8 @@ void gdn_step_norm_multi(float* state, const float* h, int conv_channels, const
 /// Spin until *flag >= value (a mapped host flag).  The value is fixed at capture, so several rings can be
 /// outstanding at once (the split verify window keeps two).
 void wait_flag_ge(const uint32_t* flag, uint32_t value, void* stream);
+/// the GPU's %globaltimer (ns) into buf[i] (a one-thread kernel: the verify window's stage profiler).
+void gpu_stamp(unsigned long long* buf, int i, void* stream);
 
 // ---- perf-review E-6: a layer whose routed experts are all in VRAM needs nothing from the host
 /// One group's plan, built on the device when every routed expert of its n*k entries is resident: the host pool's

include/strata/prefill/gemm.hpp

@@ -6,6 +6,7 @@
 // rounded to BF16, which is also what llama.cpp's batched CUDA path does.  Tensor-core GEMM through cuBLAS.
 #pragma once
 
+#include <cstddef>
 #include <cstdint>
 #include <string>
 
@@ -50,6 +51,7 @@ private:
     int64_t scratch_elems_ = 0;
     void* workspace_ = nullptr;
     bool external_ = false;
+    void* hipblaslt_state_ = nullptr;
 };
 
 

include/strata/prefill/kernels.hpp

@@ -14,6 +14,15 @@ namespace strata::prefill {
 // ---- hyper-connection (n_embd 2560, hc 4, hc_lr 320)
 /// xn[t, c*2560 + d] = R[t,c,d] * rsqrt(mean_d R[t,c,:]^2 + eps) * w_norm[c*2560 + d]; also its BF16 image.
 void gr_norm(const float* R, const float* w_norm, float eps, float* xn, uint16_t* xn16, int64_t T, void* stream);
+/// F-1: gr_norm without its FP32 output: the row scales rs[t*4 + c] and the BF16 image; gr_mix_r then reads R.
+void gr_norm_rs(const float* R, const float* w_norm, float eps, float* rs, uint16_t* xn16, int64_t T, void* stream);
+/// gr_mix with xn recomputed from R, rs and w_norm exactly as gr_norm computes it (the same bits).
+void gr_mix_r(const float* R, const float* rs, const float* w_norm, const float* gated, float* mixed, uint16_t* mixed16,
+              int64_t T, void* stream, uint16_t* mixed_h = nullptr);
+/// F-2: gr_write, then gr_norm_rs of the next half (its norm weights) over the rows just written - the same bits as
+/// the two calls, without reading R back.
+void gr_write_norm_rs(float* R, const float* bo, const float* inj, int64_t inj_ld, const float* w_norm_next, float eps,
+                      float* rs, uint16_t* xn16, int64_t T, void* stream);
 /// lo16[t, k] = bf16(silu(lo[t, k] / hc))
 void gr_silu(const float* lo, uint16_t* lo16, int64_t T, void* stream);
 /// mixed[t, d] = mean_c xn[t, c, d] * sigmoid(gated[t, c, d]); FP32, BF16 and FP16 (either image may be null).
@@ -44,6 +53,9 @@ void blob_dequant(const uint8_t* blob, uint16_t* gu16, uint16_t* down16, void* s
 void swiglu_interleaved(const float* gu, uint16_t* h16, int64_t n, void* stream);
 /// h16[n, r] = fp16(silu(g[n, r]) * u[n, r])   (the shared expert, gate and up separate, width 640)
 void swiglu_pair(const float* g, const float* u, uint16_t* h16, int64_t n, void* stream);
+/// dst[i] = src[i] for n int32s, as a kernel: either side may be mapped host memory, and the copy never waits
+/// behind the copy engine's queue (the prompt path's grouping tables, while the expert stream fills it).
+void copy_i32(int32_t* dst, const int32_t* src, int64_t n, void* stream);
 /// Gather rows: dst16[i, :] = x16[src[i], :] (n rows of `width` BF16).
 void gather_rows16(const uint16_t* x16, const int32_t* src, uint16_t* dst16, int64_t n, int64_t width, void* stream);
 /// bo[t, :] = shared[t, :] * sigmoid(sg[t]) + sum_k w[t, k] * D[slot[t, k], :]

include/strata/prefill/prefill.hpp

@@ -34,8 +34,19 @@ struct PrefillStats {
     double ms_ple = 0;
 };
 
+}  // namespace strata::prefill
+namespace strata::core { class MtpDrafter; }
+namespace strata::prefill {
+
 class Prefill {
 public:
+    /// E-9: the draft layer's K/V for prompt cells [cell0, cell0 + n) from their final residual rows `R_rows`
+    /// (device) and `next_tokens` (host: the token at cell+1), in batches through this path's GEMMs and its idle
+    /// scratch - call it from on_chunk.  false with `err` empty: not applicable here (a ring or hybrid K/V, another
+    /// device, too little scratch; STRATA_MTP_BATCH=0), the caller runs the drafter's own pass.  Not bit-identical to
+    /// that pass (FP16 GEMMs instead of Q8_1 activations): the drafts may differ, never the target's tokens' logits.
+    bool draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t* next_tokens, int64_t n, int64_t cell0,
+                  std::string& err);
     Prefill();
     ~Prefill();
     Prefill(const Prefill&) = delete;

serve/frontend.py

@@ -38,7 +38,8 @@ class ChatTemplate:
         env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True, extensions=["jinja2.ext.loopcontrols"])
         env.filters["tojson"] = tojson
         env.globals["raise_exception"] = raise_exception
-        self.template = env.from_string(Path(path).read_text(encoding="utf-8"))
+        self.source = Path(path).read_text(encoding="utf-8")
+        self.template = env.from_string(self.source)
 
     def render(self, messages: list[dict], tools: list[dict] | None = None, add_generation_prompt: bool = True,
                **kwargs) -> str:
@@ -144,7 +145,7 @@ def openai_to_messages(req: dict) -> tuple[list[dict], list[dict] | None, dict]:
         role = m.get("role")
         if role == "developer":
             role = "system"
-        out = {"role": role, "content": _parts_of(m.get("content")) if role == "user" else _text_of(m.get("content"))}
+        out = {"role": role, "content": _parts_of(m.get("content")) if role in ("user", "tool", "assistant") else _text_of(m.get("content"))}
         if m.get("reasoning_content"):
             out["reasoning_content"] = m["reasoning_content"]
         if m.get("tool_calls"):
@@ -241,6 +242,61 @@ CALL_START = "<tool_call>"
 CALL_END = "</tool_call>"
 
 
+PARAM_END = "</parameter>"
+FUNC_END = "</function>"
+
+
+def param_end(text: str, final: bool = False) -> int:
+    """Where a parameter value in `text` ends: the first `</parameter>` followed (after whitespace) by the next
+    `<parameter=` or `</function>` - the same text inside a value (a file that documents the call format, #210) is
+    part of the value.  -1: none yet; -2: a candidate whose follower has not arrived (streaming; `final` accepts it)."""
+    at = text.find(PARAM_END)
+    while at >= 0:
+        after = text[at + len(PARAM_END):].lstrip()
+        if after.startswith(("<parameter=", FUNC_END)):
+            return at
+        if not after or "<parameter=".startswith(after) or FUNC_END.startswith(after):
+            return at if final else -2
+        at = text.find(PARAM_END, at + 1)
+    return -1
+
+
+def call_end(text: str) -> int:
+    """Where a tool call's body ends (#210): the `</tool_call>` after the call's own `</function>`, found by walking
+    its parameters with param_end, so a value may contain either tag.  -1: not complete yet.  A body that is not in
+    the call format ends at the first `</tool_call>`, as before."""
+    s = text.lstrip()
+    pos = len(text) - len(s)
+    if not s.startswith("<function="):
+        return text.find(CALL_END) if not "<function=".startswith(s) else -1
+    gt = text.find(">", pos)
+    if gt < 0:
+        return -1
+    pos = gt + 1
+    while True:
+        rest = text[pos:]
+        s = rest.lstrip()
+        pos += len(rest) - len(s)
+        if s.startswith("<parameter="):
+            gt = text.find(">", pos)
+            if gt < 0:
+                return -1
+            end = param_end(text[gt + 1:])
+            if end < 0:
+                return -1
+            pos = gt + 1 + end + len(PARAM_END)
+        elif s.startswith(FUNC_END):
+            rest = text[pos + len(FUNC_END):]
+            s = rest.lstrip()
+            if s.startswith(CALL_END):
+                return pos + len(FUNC_END) + len(rest) - len(s)
+            return -1 if CALL_END.startswith(s) else text.find(CALL_END, pos)
+        elif not s or "<parameter=".startswith(s) or FUNC_END.startswith(s):
+            return -1
+        else:
+            return text.find(CALL_END, pos)
+
+
 def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
     """`<function=NAME>\\n<parameter=P>\\nVALUE\\n</parameter>...</function>` -> ToolCall. Values are JSON-decoded
     when the tool's schema says the parameter is not a string (or, without a schema, when they parse as JSON
@@ -256,9 +312,9 @@ def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
         rest = rest[rest.index("<parameter=") + len("<parameter="):]
         pname = rest[:rest.index(">")]
         rest = rest[rest.index(">") + 1:]
-        end = rest.find("</parameter>")
+        end = param_end(rest, final=True)
         value = rest[:end] if end >= 0 else rest
-        rest = rest[end + len("</parameter>"):] if end >= 0 else ""
+        rest = rest[end + len(PARAM_END):] if end >= 0 else ""
         if value.startswith("\n"):
             value = value[1:]
         if value.endswith("\n"):
@@ -350,16 +406,17 @@ class OutputParser:
                         self.sp += 1
                         rest = rest[1:]
                     self.sval_started = True
-                end = rest.find("</parameter>")
+                end = param_end(rest)
                 if end >= 0:
                     value = rest[:end]
                     if value.endswith("\n"):
                         value = value[:-1]
                     args(json.dumps(value)[1:-1] + '"')
-                    self.sp += end + len("</parameter>")
+                    self.sp += end + len(PARAM_END)
                     self.ss = "between"
                     continue
-                safe = len(rest) - self._hold(rest, ("</parameter>",))
+                # an undecided </parameter> (-2) is held from its start, like a tag still arriving
+                safe = rest.find(PARAM_END) if end == -2 else len(rest) - self._hold(rest, (PARAM_END,))
                 if safe > 0 and rest[safe - 1] == "\n":   # may be the trailing newline before </parameter>
                     safe -= 1
                 if safe > 0:
@@ -367,7 +424,7 @@ class OutputParser:
                     self.sp += safe
                 return out
             elif self.ss == "raw":
-                end = rest.find("</parameter>")
+                end = param_end(rest)
                 if end < 0:
                     return out
                 value = rest[:end]
@@ -380,7 +437,7 @@ class OutputParser:
                 except ValueError:
                     v = value
                 args(json.dumps(v, ensure_ascii=False))
-                self.sp += end + len("</parameter>")
+                self.sp += end + len(PARAM_END)
                 self.ss = "between"
             else:
                 return out
@@ -448,7 +505,7 @@ class OutputParser:
                 self.buf = self.buf[i + len(CALL_START):]
                 self.state = "call"
             else:
-                i = self.buf.find(CALL_END)
+                i = call_end(self.buf)
                 if self.stream_tools:
                     if i >= 0:
                         whole, self.buf = self.buf, self.buf[:i]     # scan only the body

serve/server.py

@@ -4,7 +4,7 @@
     python -m serve.server --engine strata --config strata.json --port 8080   (the real engine, resident)
 
 Endpoints: POST /v1/chat/completions (OpenAI, stream and non-stream), POST /v1/messages (Anthropic, stream and
-non-stream), GET /v1/models, GET /health, GET /mcp. One sequence at a time behind a FIFO (plan: one resident sequence).
+non-stream), GET /v1/models, GET /models, GET /props, GET /slots, GET /health, GET /mcp. One sequence at a time behind a FIFO (plan: one resident sequence).
 Tools from MCP servers (serve/mcp.py, `"mcp_servers"` in the config or --mcp-config) are offered only to requests that
 ask for them with `"strata_mcp": true` - the web app does; other clients see exactly the API they always saw.
 Images (optional, when the config has a "vision" entry): OpenAI image_url parts and Anthropic image blocks (base64
@@ -24,6 +24,7 @@ import argparse
 import collections
 import base64
 import hashlib
+import hmac
 import codecs
 import json
 import os
@@ -38,6 +39,7 @@ import uuid
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Iterator, Protocol
+from urllib.parse import parse_qs, urlsplit
 
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT / "tools"))
@@ -48,7 +50,12 @@ from serve.mcp import McpCancelled, hub_from_config  # noqa: E402
 
 IM_END = "<|im_end|>"
 IMAGE_PAD = "<|image_pad|>"
+VISION_START = "<|vision_start|>"
 CTX_SLACK = 8               # `strata --serve` rejects prompt + max_new + 8 > context: keep the same margin here
+# The live tok/s is a rate over a window, not a mean since the first token: a mean reads ~1/elapsed at the first
+# token (the Monitor showed five-digit numbers) and then undershoots for the first second of every answer.
+RATE_WINDOW_S = 2.0
+RATE_MIN_SPAN_S = 0.25      # younger than this there is no rate yet: the mean so far, with the span floored here
 
 
 # ------------------------------------------------------------------------------------------------ engines
@@ -150,6 +157,8 @@ class StrataEngine:
     def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | None = None,
                  env: dict | None = None):
         self.spawn = (exe, list(args), cwd, log, env)   # to start it again after it died (issue #27)
+        paths = {k: v for k, v in zip(args, args[1:]) if k in ("--native", "--pack")}
+        self.model_path = paths.get("--native") or paths.get("--pack", "pack/full")
         self.log_path = log
         self.log = open(log, "a", encoding="utf-8") if log else subprocess.DEVNULL
         loading = threading.Event()                     # set once READY: the narrator below stops
@@ -208,6 +217,11 @@ class StrataEngine:
             if "issue #29" in line:
                 return ("The engine stopped itself because it had stopped making progress - a hang it caught. Its log "
                         "line: " + line.strip() + " - please report it at github.com/Niko1221/Strata/issues.")
+        rc = self.proc.poll()
+        last = next((x.strip() for x in reversed(tail.splitlines()) if x.strip().startswith(("strata", "ERR"))), "")
+        if rc is not None and rc >= 0 and last:          # it ended by itself: its own last words say why (#215)
+            return (f"The engine exited (code {rc}). Its last log line: {last} - if that does not explain it, please "
+                    "report it at github.com/Niko1221/Strata/issues with the log.")
         return ("The usual cause is running out of RAM: Linux then ends the biggest program (check: sudo dmesg | "
                 "grep -i -E 'killed process|out of memory'); Windows slows down instead. Close other programs or use a "
                 "smaller model (Q2_0 / IQ2_XS).")
@@ -300,8 +314,9 @@ class StrataEngine:
         the HTTP layer turns it into an SSE comment, which keeps clients' watchdogs calm and notices a client that
         has gone.  A consumer that stops early (or `cancel`) makes the engine STOP, so it does not run to max_new."""
         self.progress = None
-        head = f"GENI {int(max_new)}{self.projection_key(sampling or {})} {embeddings}" if embeddings else \
-            f"GEN {int(max_new)}{self.sampling_keys(sampling or {}) if not embeddings else ''}"
+        # an image request takes the same sampling keys as text (#75: it used to decode greedily whatever was asked)
+        head = f"GENI {int(max_new)}{self.sampling_keys(sampling or {})} {embeddings}" if embeddings else \
+            f"GEN {int(max_new)}{self.sampling_keys(sampling or {})}"
         try:
             self.proc.stdin.write(f"{head} {','.join(str(int(t)) for t in ids)}\n")
             self.proc.stdin.flush()
@@ -480,9 +495,13 @@ def child_env(cfg: dict) -> dict:
     """The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
     compiled it) first on the library search path."""
     env = dict(os.environ)
-    if gpu_list(cfg):                                # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
+    if gpu_list(cfg) and cfg.get("backend") == "hip":   # AMD: numbered as HIP numbers them (setup's KFD order)
+        env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
+    elif gpu_list(cfg):                              # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
         env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"      # own default order (fastest first) can number the cards otherwise
         env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
+    for k, v in (cfg.get("env") or {}).items():      # engine settings the config carries (AMD: the GEMM tuning table)
+        env[str(k)] = str(v)
     dirs = [d for d in cfg.get("lib_dirs") or [] if Path(d).is_dir()]
     if dirs:
         var = "PATH" if os.name == "nt" else "LD_LIBRARY_PATH"
@@ -550,10 +569,14 @@ class Service:
         self.embeddings = threading.local()           # the current request's image embeddings file (GENI)
         self.api_key = ""                              # when set, /v1/* needs it (Bearer or x-api-key)
         self.status = {"busy": False, "queued": 0}      # GET /status: what the model is doing right now
+        self.rate = collections.deque(maxlen=32)        # (time, generated) samples for the live tok/s window
         self.history = collections.deque(maxlen=500)    # the last finished requests, newest last (GET /metrics)
         # since the server started (the Monitor's totals, issue #35)
         self.totals = {"since": time.time(), "requests": 0, "prompt_tokens": 0, "reused": 0, "output_tokens": 0,
                        "prompt_ms": 0.0, "decode_ms": 0.0}
+        self.last_timings = None                         # the last finished request's, llama.cpp's names (/v1/status)
+        self.last_request_at = None                      # when a request last started or finished
+        self.started_at = time.time()
         self.status_lock = threading.Lock()
         self.mcp = None                                  # serve/mcp.py's McpHub when MCP servers are configured
         self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
@@ -598,11 +621,26 @@ class Service:
         """The hardware sampler behind GET /metrics (serve/telemetry.py), recording this server's tok/s too."""
         if getattr(self, "telemetry", None) is None:
             from serve.telemetry import Telemetry
-            self.telemetry = Telemetry(extra=lambda: {"tok_s": self._tok_s()},
+            self.telemetry = Telemetry(extra=lambda: {"tok_s": self._tok_s(), "tok_s_mean": self._tok_s_mean()},
                                        gpu_index=int(getattr(self, "gpu_index", 0) or 0),
                                        gpu_indices=getattr(self, "gpu_indices", None))
 
     def _tok_s(self):
+        """tok/s over the last RATE_WINDOW_S seconds.  Returns 0.0 while nothing is generating."""
+        with self.status_lock:
+            s = dict(self.status)
+            rate = list(self.rate)
+        if not s.get("busy") or not s.get("first_token"):
+            return 0.0
+        now = time.time()
+        newest = rate[-1] if rate else None
+        oldest = next(((t, g) for t, g in rate if now - t <= RATE_WINDOW_S), None)
+        if newest and oldest and newest[0] - oldest[0] >= RATE_MIN_SPAN_S:
+            return max(0.0, (newest[1] - oldest[1]) / (newest[0] - oldest[0]))
+        return s["generated"] / max(RATE_MIN_SPAN_S, now - s["first_token"])
+
+    def _tok_s_mean(self):
+        """The whole-request mean since the first token (the old formula), kept so the two can be compared."""
         with self.status_lock:
             s = dict(self.status)
         if not s.get("busy") or not s.get("first_token"):
@@ -629,7 +667,9 @@ class Service:
                 "prompt_read": None, "prompt_total": None, "generated": s.get("generated") if s.get("busy") else None,
                 "max_tokens": s.get("max_tokens") if s.get("busy") else None,
                 "elapsed_s": round(now - s["started"], 1) if s.get("busy") and s.get("started") else None,
-                "tok_s": round(self._tok_s(), 1) if state == "generating" else None}
+                "tok_s": round(self._tok_s(), 1) if state == "generating" else None,
+                "tok_s_mean": round(self._tok_s_mean(), 1) if state == "generating" else None,
+                "tok_s_window_s": RATE_WINDOW_S if state == "generating" else None}
         if state == "reading" and progress:
             live["prompt_read"], live["prompt_total"] = progress
         engine = {"model": self.model, "max_context": self.engine.max_context, "images": self.vision is not None,
@@ -640,6 +680,41 @@ class Service:
                 "hardware_static":
                 tel["static"], "history": tel["history"], "time": now}
 
+    def v1_status(self) -> dict:
+        """GET /v1/status: what this server is and does, for a client that would rather ask than guess (a front-end
+        that polls its OpenAI-compatible server's status, collabosm's for one): the model and its window, images,
+        the APIs, what is running, and the last request's timings in llama.cpp's names.  /metrics has the rest."""
+        with self.status_lock:
+            s, totals = dict(self.status), dict(self.totals)
+            last_t, last_at = (dict(self.last_timings) if self.last_timings else None), self.last_request_at
+        tel = self.telemetry.snapshot() if getattr(self, "telemetry", None) else {"now": {}, "static": {}}
+        hw, static = tel.get("now") or {}, tel.get("static") or {}
+
+        def scaled(v, unit, digits=0):
+            return round(v / unit, digits) if isinstance(v, (int, float)) else None
+
+        busy, ctx = bool(s.get("busy")), self.engine.max_context
+        images = self.vision is not None
+        return {
+            "service": "strata", "model": self.model,
+            "engine": (getattr(self.engine, "info", {}) or {}).get("version"),
+            "started": int(self.started_at), "uptime_s": int(time.time() - self.started_at),
+            "cache_max_tokens": ctx,
+            "context": {"native": ctx, "max_positions": ctx},
+            "concurrency": {"serving": 1, "requested": 1},       # one request at a time; more wait their turn
+            "dialects": ["/v1/chat/completions", "/v1/messages"],
+            "vision": {"enabled": images, "available": images, "error": None},
+            "activity": {"requests": totals["requests"] + int(busy), "in_flight": int(busy) + int(s.get("queued") or 0),
+                         "last_request_at": int(last_at) if last_at else None},
+            "last_timings": last_t,
+            "machine": {
+                "at": int(time.time()),
+                "gpu": {"name": static.get("gpu_name"), "used_mib": scaled(hw.get("gpu_mem_used"), 2 ** 20),
+                        "total_mib": scaled(hw.get("gpu_mem_total"), 2 ** 20), "util_pct": hw.get("gpu_util"),
+                        "temp_c": hw.get("gpu_temp"), "power_w": hw.get("gpu_power")} if static.get("gpu_name") else None,
+                "ram": {"used_gib": scaled(hw.get("ram_used"), 2 ** 30, 1),
+                        "total_gib": scaled(hw.get("ram_total"), 2 ** 30, 1)} if hw.get("ram_total") else None}}
+
     def prepare(self, messages, tools, kwargs, max_new=None):
         """-> (ids, thinking, max_new). An unset or non-positive max_new (some clients send -1) means "unlimited":
         the rest of the context."""
@@ -652,17 +727,24 @@ class Service:
                 raise ValueError("this server was started without the vision encoder (run setup again and choose "
                                  "'vision'), so it cannot read images")
             pad = self.tok.encode(IMAGE_PAD, parse_special=True)[0]
+            start = self.tok.encode(VISION_START, parse_special=True)[0]
             # Encode only while the engine is idle: the engine and the image encoder (a separate process) must not
             # run on the GPU at the same time - an encode during a running request left that request stuck at
             # "reading the prompt" with CPU and GPU busy, for good (reproduced).  So encoding takes its turn in the
             # same FIFO as the requests.
             with self.fifo:
                 encoded = [self.vision.encode(src) for src in images]
+            # one <|image_pad|> per image -> one per image token.  Only the markers the template writes for an image
+            # (right after <|vision_start|>) are images: the same text inside a message (an agent reading these docs,
+            # #150) is kept as plain text, or it took an image's place and the counts no longer matched.
+            literal = self.tok.encode(IMAGE_PAD, parse_special=False)
             out, k = [], 0
-            for t in ids:                               # one <|image_pad|> per image -> one per image token
-                if t == pad and k < len(encoded):
+            for j, t in enumerate(ids):
+                if t == pad and j > 0 and ids[j - 1] == start and k < len(encoded):
                     out += [pad] * encoded[k][1]
                     k += 1
+                elif t == pad:
+                    out += literal
                 else:
                     out.append(t)
             if k != len(encoded):
@@ -692,6 +774,7 @@ class Service:
             s["generated"] = n
             if s.get("first_token") is None:
                 s["first_token"] = time.time()
+            self.rate.append((time.time(), n))          # the live rate's window over the last RATE_WINDOW_S
             for ev in evs:
                 if ev.kind == "reasoning":
                     s["phase"] = "thinking"
@@ -729,8 +812,12 @@ class Service:
             sampling = {**defaults, **req_values}
         parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True)
         detok, n, finish = Detokenizer(self.tok), 0, "length"
+        timings, before = None, None                    # this request's timings; the engine's `last` before it
         raw_ids = []                                    # every generated id (STRATA_DEBUG: dump raw model text)
         emb = getattr(self.embeddings, "path", None)
+        # Identity token: only a DONE line replaces engine.last, so a request that died, errored or was
+        # disconnected must not have the PREVIOUS request's decode figures recorded as its own.
+        engine_last0 = getattr(self.engine, "last", None)
         with self.status_lock:
             self.status["queued"] += 1
         try:
@@ -747,6 +834,9 @@ class Service:
                 with self.status_lock:
                     self.status.update(busy=True, phase="reading the prompt", prompt_tokens=len(ids), generated=0,
                                        started=time.time(), first_token=None, tool=None, tail="", max_tokens=max_new)
+                    self.last_request_at = time.time()
+                    self.rate.clear()               # the previous request's samples must not leak into this one
+                before = getattr(self.engine, "last", None)
                 last_print = time.time()
                 gen = self.engine.generate(ids, max_new, sampling, cancel, embeddings=emb) if emb else \
                     self.engine.generate(ids, max_new, sampling, cancel)
@@ -792,7 +882,9 @@ class Service:
                 Path(emb).unlink(missing_ok=True)
             with self.status_lock:
                 if self.status.get("busy"):
-                    last = dict(getattr(self.engine, "last", {}) or {})
+                    # only this request's DONE counts: same object means no DONE arrived (death, error, disconnect)
+                    last = dict(getattr(self.engine, "last", {}) or {}) \
+                        if getattr(self.engine, "last", None) is not engine_last0 else {}
                     started = self.status.get("started", time.time())
                     loaded = str((getattr(self.engine, "info", {}) or {}).get("cvec", 0)) not in ("0", "", "None")
                     hit_rate = round(last["hits"] / last["lookups"], 3) if last.get("lookups") else None
@@ -801,6 +893,7 @@ class Service:
                         if loaded else None,
                         "time": started, "duration_s": round(time.time() - started, 1), "finish": finish,
                         "prompt_tokens": len(ids), "reused": last.get("reused"), "output_tokens": n,
+                        "engine_generated": last.get("generated"),
                         "prompt_ms": last.get("prompt_ms"), "decode_ms": last.get("decode_ms"),
                         "decode_tok_s": round(last["generated"] / (last["decode_ms"] / 1000), 1)
                         if n and last.get("generated") and last.get("decode_ms") else None,
@@ -812,6 +905,11 @@ class Service:
                     t["output_tokens"] += n
                     t["prompt_ms"] += last.get("prompt_ms") or 0.0
                     t["decode_ms"] += last.get("decode_ms") or 0.0
+                    fresh = getattr(self.engine, "last", None)
+                    if fresh is not None and fresh is not before:      # the engine's clock for THIS request
+                        timings = request_timings(len(ids), n, last)
+                        self.last_timings = dict(timings, at=int(time.time())) if timings else None
+                    self.last_request_at = time.time()
                     now = time.time()
                     el = now - self.status.get("started", now)
                     ft = self.status.get("first_token")
@@ -822,9 +920,32 @@ class Service:
                     if os.environ.get("STRATA_DEBUG") and raw_ids:
                         print(f"[strata] raw: {self.tok.decode(raw_ids)!r}", flush=True)
                 self.status["busy"] = False
+                self.status.pop("tail", None)            # #212: the answer's end is not kept once it is done
+                self.status.pop("tool", None)
         for ev in parser.finish():
             yield "event", ev
-        yield "done", {"finish": finish, "completion_tokens": n}
+        yield "done", {"finish": finish, "completion_tokens": n, "reused": (timings or {}).get("cache_n", 0),
+                       "timings": timings}
+
+
+def request_timings(prompt_tokens: int, generated: int, last: dict) -> dict | None:
+    """One request's `timings` in llama.cpp's names (what its clients show as speed), from the engine's own clock
+    (StrataEngine.last): prompt_n is what was read, cache_n what the conversation cache already held.  None when the
+    engine keeps no clock (MockEngine)."""
+    if last.get("prompt_ms") is None:
+        return None
+    cache_n = int(last.get("reused") or 0)
+    prompt_n, prompt_ms, decode_ms = max(0, prompt_tokens - cache_n), float(last["prompt_ms"]), float(last.get("decode_ms") or 0)
+    decoded = int(last.get("generated") or generated)          # the engine's count gives its rate, as /metrics does
+    return {"cache_n": cache_n, "prompt_n": prompt_n, "prompt_ms": round(prompt_ms, 1),
+            "prompt_per_token_ms": round(prompt_ms / prompt_n, 3) if prompt_n else None,
+            "prompt_per_second": round(prompt_n / (prompt_ms / 1000), 1) if prompt_n and prompt_ms > 0 else None,
+            "predicted_n": generated, "predicted_ms": round(decode_ms, 1),
+            "predicted_per_token_ms": round(decode_ms / decoded, 3) if decoded else None,
+            "predicted_per_second": round(decoded / (decode_ms / 1000), 1) if decoded and decode_ms > 0 else None,
+            # the speculative drafts, as llama.cpp names them (from PR #83, @mikicvi): only when the engine reported them
+            **({"draft_n": int(last["drafts_offered"]), "draft_n_accepted": int(last["drafts_accepted"])}
+               if last.get("drafts_offered") is not None else {})}
 
 
 def _debug_req(api, req, messages, tools, max_new, thinking, prompt_tokens):
@@ -975,7 +1096,11 @@ def openai_chunks(svc: Service, req: dict, ids, thinking, tools, max_new, cancel
             last = chunk({}, finish)
             pt = x.get("prompt_tokens", len(ids))     # after MCP rounds: the last round's prompt
             last["usage"] = {"prompt_tokens": pt, "completion_tokens": x["completion_tokens"],
-                             "total_tokens": pt + x["completion_tokens"]}
+                             "total_tokens": pt + x["completion_tokens"],
+                             # the part of the prompt the conversation cache already held (OpenAI's field)
+                             "prompt_tokens_details": {"cached_tokens": x.get("reused") or 0}}
+            if x.get("timings"):
+                last["timings"] = x["timings"]          # llama.cpp's field: the speed its clients show
             yield last
 
 
@@ -1005,9 +1130,12 @@ def openai_collect(chunks) -> dict:
         msg["tool_calls"] = calls
     if mcp:
         msg["strata_mcp"] = mcp
-    return {"id": last["id"], "object": "chat.completion", "created": last["created"], "model": last["model"],
-            "choices": [{"index": 0, "message": msg, "finish_reason": last["choices"][0]["finish_reason"]}],
-            "usage": last["usage"]}
+    out = {"id": last["id"], "object": "chat.completion", "created": last["created"], "model": last["model"],
+           "choices": [{"index": 0, "message": msg, "finish_reason": last["choices"][0]["finish_reason"]}],
+           "usage": last["usage"]}
+    if last.get("timings"):
+        out["timings"] = last["timings"]
+    return out
 
 
 # ------------------------------------------------------------------------------------------------ Anthropic
@@ -1067,8 +1195,12 @@ def anthropic_events(svc: Service, req: dict, ids, thinking, tools, max_new, can
                 yield close()
             stop = "tool_use" if used_tool and x["finish"] == "stop" else \
                 {"stop": "end_turn", "length": "max_tokens", "cancel": "end_turn"}[x["finish"]]
+            # the final counts, Anthropic's way: input_tokens leaves out what the conversation cache already held,
+            # which is cache_read_input_tokens (message_start could only say the whole prompt)
+            reused = min(x.get("reused") or 0, len(ids))
             yield "message_delta", {"type": "message_delta", "delta": {"stop_reason": stop, "stop_sequence": None},
-                                    "usage": {"output_tokens": x["completion_tokens"]}}
+                                    "usage": {"input_tokens": len(ids) - reused, "cache_read_input_tokens": reused,
+                                              "output_tokens": x["completion_tokens"]}}
             yield "message_stop", {"type": "message_stop"}
 
 
@@ -1095,7 +1227,7 @@ def anthropic_collect(events) -> dict:
             b["input"] = json.loads(b.pop("_json") or "{}")
         elif name == "message_delta":
             msg["stop_reason"] = e["delta"]["stop_reason"]
-            msg["usage"]["output_tokens"] = e["usage"]["output_tokens"]
+            msg["usage"].update(e["usage"])
     msg["content"] = blocks
     return msg
 
@@ -1121,7 +1253,7 @@ def make_handler(svc: Service):
                 return True
             auth = self.headers.get("Authorization", "")
             given = auth[7:].strip() if auth.lower().startswith("bearer ") else self.headers.get("x-api-key", "")
-            if given == svc.api_key:
+            if hmac.compare_digest(given.encode(), svc.api_key.encode()):   # #213: constant-time
                 return True
             self._json(401, {"error": {"type": "authentication_error", "message": "missing or wrong API key"}})
             return False
@@ -1186,19 +1318,40 @@ def make_handler(svc: Service):
                 self._json(200, {"status": "ok", "max_context": svc.engine.max_context, "model": svc.model,
                                  "images": svc.vision is not None, "api_key": bool(svc.api_key)})
             elif path == "/status":
+                if not self._authorized():                  # #212: it shows the end of the last answer
+                    return
                 with svc.status_lock:
                     s = dict(svc.status)
                 now = time.time()
                 if s.get("busy"):
                     s["elapsed_s"] = round(now - s["started"], 1)
                     if s.get("first_token"):
-                        s["tokens_per_s"] = round(s["generated"] / max(1e-6, now - s["first_token"]), 1)
+                        s["tokens_per_s"] = round(svc._tok_s(), 1)
+                        s["tokens_per_s_mean"] = round(svc._tok_s_mean(), 1)
                 for k in ("started", "first_token"):
                     s.pop(k, None)
                 self._json(200, s)
-            elif path == "/v1/models":
+            elif path in ("/v1/models", "/models"):
                 if self._authorized():
-                    self._json(200, {"object": "list", "data": [{"id": svc.model, "object": "model"}]})
+                    loaded = not hasattr(svc.engine, "alive") or svc.engine.alive()
+                    model = {"id": svc.model, "object": "model", "status": {"value": "loaded"},
+                             "meta": {"n_ctx": svc.engine.max_context},
+                             "architecture": {"input_modalities": ["text", "image"] if svc.vision is not None else ["text"],
+                                              "output_modalities": ["text"]}}
+                    self._json(200, {"object": "list", "data": [model] if loaded else []})
+            elif path == "/props":
+                if self._authorized():
+                    self._props()
+            elif path == "/slots":
+                if self._authorized():
+                    loaded = not hasattr(svc.engine, "alive") or svc.engine.alive()
+                    with svc.status_lock:
+                        busy = bool(svc.status.get("busy"))
+                    slot = {"id": 0, "n_ctx": svc.engine.max_context, "is_processing": busy}
+                    self._json(200, [slot] if loaded else [])
+            elif path == "/v1/status":
+                if self._authorized():
+                    self._json(200, svc.v1_status())
             else:
                 self._json(404, {"error": {"message": "not found"}})
 
@@ -1222,6 +1375,31 @@ def make_handler(svc: Service):
             except EngineDied as e:                          # before the answer started (not streamed)
                 self._json(503, {"error": {"type": "server_error", "message": f"{e}; the next request restarts it"}})
 
+        def _props(self):
+            model = parse_qs(urlsplit(self.path).query).get("model", [svc.model])[0]
+            if model != svc.model:
+                self._json(404, {"error": {"message": "model not found"}})
+                return
+            if hasattr(svc.engine, "alive") and not svc.engine.alive():
+                self._json(503, {"error": {"message": "the engine is not running"}})
+                return
+            defaults = {**svc.sampling_defaults, **svc.shared}
+            names = {"repetition_penalty": "repeat_penalty", "penalty_last_n": "repeat_last_n"}
+            params = {names.get(k, k): v for k, v in defaults.items()
+                      if k in ("temperature", "top_p", "top_k", "min_p", "seed", "repetition_penalty",
+                               "presence_penalty", "frequency_penalty", "penalty_last_n")}
+            params["n_predict"] = svc.shared.get("max_tokens", -1)
+            props = {"default_generation_settings": {"n_ctx": svc.engine.max_context, "params": params},
+                     "total_slots": 1, "model_alias": svc.model, "chat_template": svc.template.source,
+                     "modalities": {"vision": svc.vision is not None}, "models_autoload": False,
+                     "is_sleeping": False}
+            if getattr(svc.engine, "model_path", None):
+                props["model_path"] = svc.engine.model_path
+            version = getattr(svc.engine, "info", {}).get("version")
+            if version:
+                props["build_info"] = "Strata " + str(version)
+            self._json(200, props)
+
         def _own_page(self, what) -> bool:
             """Only JSON (a form or a "simple" cross-site request can't send it without a CORS preflight, which this
             server never grants) and no foreign Origin: a web page elsewhere must not change settings or run tools."""
@@ -1333,6 +1511,10 @@ class Server(ThreadingHTTPServer):
     # either one (a forgotten second start of run-<model>.bat).  Without it the second start fails loudly instead.
     allow_reuse_address = os.name != "nt"
 
+    def handle_error(self, request, client_address):
+        if not isinstance(sys.exc_info()[1], ConnectionError):   # a client that hangs up needs no stack trace
+            super().handle_error(request, client_address)
+
 
 def warn_tight_ram(arena_mib) -> None:
     """The model's experts live in RAM (INFO arena_mib, engine 0.1.10+).  With less than ~6 GB left beside them for the
@@ -1556,6 +1738,12 @@ def main() -> int:
                   model_name=cfg.get("model_name", "qwen3.8-flash-next"), vision=vision,
                   sampling_defaults=sampling_defaults,
                   fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)
+    if ("STRATA_API_KEY" in os.environ and not os.environ["STRATA_API_KEY"].strip()) or             any(x == "--api-key" and i + 1 < len(sys.argv) and not sys.argv[i + 1].strip() or x.strip() == "--api-key="
+                for i, x in enumerate(sys.argv)):
+        # #213: an empty key would switch authentication off without a word
+        print("[strata] an API key was given but it is empty: set a key, or leave --api-key / STRATA_API_KEY out",
+              file=sys.stderr)
+        return 2
     svc.api_key = a.api_key or cfg.get("api_key", "")
     svc.gpu_index = (gpu_list(cfg) or [0])[0]           # the Monitor reads the card the engine runs on (issue #51)
     svc.gpu_indices = gpu_list(cfg)                     # ... or every card of a layer split (issue #112)
@@ -1600,15 +1788,19 @@ def main() -> int:
         import webbrowser
         webbrowser.open(f"http://{'127.0.0.1' if a.host in ('0.0.0.0', '') else a.host}:{a.port}/")
     try:
-        threading.Event().wait()
+        while True:
+            time.sleep(1)                               # Windows never delivers Ctrl+C to an untimed Event.wait()
     except KeyboardInterrupt:
-        httpd.shutdown()
-        if hasattr(engine, "close"):
-            engine.close()
-        if vision:
-            vision.close()
-        if hub is not None:
-            hub.close()
+        print("\n[strata] stopping (Ctrl+C again to end the engine at once) ...", flush=True)
+        closers = [httpd.shutdown, getattr(engine, "close", None), vision.close if vision else None,
+                   hub.close if hub is not None else None]
+        for close in filter(None, closers):
+            try:
+                close()
+            except KeyboardInterrupt:                   # a second Ctrl+C: don't wait for the engine to free its memory
+                if getattr(engine, "proc", None):
+                    engine.proc.kill()
+        print("[strata] stopped", flush=True)
     return 0
 
 

serve/test_server.py

@@ -7,6 +7,8 @@ from __future__ import annotations
 import json
 import os
 import sys
+import threading
+import time
 import unittest
 import urllib.error
 import urllib.request
@@ -14,7 +16,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 from serve.frontend import ChatTemplate  # noqa: E402
-from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, MockEngine, Service, StrataEngine, serve  # noqa: E402
+from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, MockEngine, Service, StrataEngine, request_timings, serve  # noqa: E402
 
 ROOT = Path(__file__).resolve().parents[1]
 CTX = 4096
@@ -165,6 +167,90 @@ class FitMaxTokens(unittest.TestCase):
         self.assertIn("no room to answer", b["error"]["message"])
 
 
+class ImageMarkers(unittest.TestCase):
+    """#150: the text "<|image_pad|>" inside a message is text, not an image's place."""
+
+    class FakeVision:
+        def __init__(self, d):
+            self.dir = Path(d)
+            self.rows = self.dir / "img.sve"
+            self.rows.write_bytes(b"rows")
+
+        def encode(self, source):
+            return self.rows, 3
+
+    def test_literal_marker_with_an_image(self):
+        import tempfile
+        tok = ByteTokenizer()
+        with tempfile.TemporaryDirectory() as d:
+            svc = Service(MockEngine(tok, "ok", max_context=CTX), tok, ChatTemplate(ROOT / "serve/chat_template.jinja"),
+                          vision=self.FakeVision(d))
+            pad = tok.encode("<|image_pad|>", parse_special=True)[0]
+            for text in ("the docs say <|image_pad|> marks an image", "plain"):
+                with self.subTest(text=text):
+                    msgs = [{"role": "user", "content": [{"type": "text", "text": text},
+                                                         {"type": "image", "source": "x.png"}]}]
+                    ids, _, _ = svc.prepare(msgs, None, {})
+                    self.assertEqual(ids.count(pad), 3)          # the image's three rows, nothing else
+                    self.assertIn("<|image_pad|> marks" if "docs" in text else "plain", tok.decode(ids))
+            svc.embeddings.path.unlink(missing_ok=True)
+
+
+class StatusNeedsTheKey(unittest.TestCase):
+    """#212: /status shows the end of the answer being written, so it needs the key like /v1/*."""
+
+    def test_status(self):
+        tok = ByteTokenizer()
+        svc = Service(MockEngine(tok, "ok", max_context=CTX), tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        svc.api_key = "k3y"
+        httpd = serve(svc, port=0)
+        base = f"http://127.0.0.1:{httpd.server_address[1]}/status"
+        try:
+            with self.assertRaises(urllib.error.HTTPError) as e:
+                urllib.request.urlopen(base, timeout=10)
+            self.assertEqual(e.exception.code, 401)
+            e.exception.close()
+            req = urllib.request.Request(base, headers={"Authorization": "Bearer k3y"})
+            with urllib.request.urlopen(req, timeout=10) as r:
+                self.assertEqual(r.status, 200)
+                self.assertNotIn("tail", json.loads(r.read()))
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+
+
+class ToolCallTerminators(unittest.TestCase):
+    """#210: a value that contains </parameter> or </tool_call> (a file documenting the call format) is kept whole."""
+    CONTENT = ("Close each value with </parameter> and the call with </function></tool_call>.\n"
+               "<parameter=x>\nnot a parameter\n</parameter>\nend")
+    SCHEMA = [{"name": "write", "parameters": {"properties": {"path": {"type": "string"},
+                                                              "content": {"type": "string"}}}}]
+
+    def run_parser(self, stream_tools, step):
+        from serve.frontend import OutputParser
+        text = ("</think>\n\n<tool_call>\n<function=write>\n<parameter=path>\ndoc.md\n</parameter>\n"
+                f"<parameter=content>\n{self.CONTENT}\n</parameter>\n</function>\n</tool_call>")
+        p = OutputParser(thinking=True, tools=self.SCHEMA, stream_tools=stream_tools)
+        evs = []
+        for i in range(0, len(text), step):
+            evs += p.feed(text[i:i + step])
+        evs += p.finish()
+        return evs
+
+    def test_values_keep_the_terminators(self):
+        for stream_tools in (False, True):
+            for step in (1, 7, 10_000):
+                with self.subTest(stream_tools=stream_tools, step=step):
+                    evs = self.run_parser(stream_tools, step)
+                    calls = [e.call for e in evs if e.kind == "tool_call"]
+                    self.assertEqual(len(calls), 1)
+                    self.assertEqual(calls[0].arguments, {"path": "doc.md", "content": self.CONTENT})
+                    self.assertFalse([e for e in evs if e.kind == "content" and e.text.strip()])
+                    if stream_tools:
+                        streamed = "".join(e.text for e in evs if e.kind == "tool_args")
+                        self.assertEqual(json.loads(streamed), {"path": "doc.md", "content": self.CONTENT})
+
+
 class ClientShapes(unittest.TestCase):
     """What real clients send: Claude Code posts /v1/messages?beta=true (issue #55) and puts hook context into the
     conversation as a mid-conversation system message (issue #56); some OpenAI clients send a late developer message."""
@@ -368,6 +454,97 @@ class EngineDeath(unittest.TestCase):
             httpd.server_close()
 
 
+class LiveRate(unittest.TestCase):
+    """The Monitor's Speed readout: live.tok_s is a rate, and a request that never got a DONE keeps no counters.
+
+    It used to be `generated / (now - first_token)` - the mean since the first token, whose first sample is
+    1/elapsed.  Against a paced engine that reads five-digit numbers for the first instant of every answer and
+    undershoots for the first second after that.  It is now the rate over the last RATE_WINDOW_S, with the mean
+    still available as `live.tok_s_mean` for anyone who wants it."""
+
+    PACE_S = 0.02                    # 50 tokens/s: a 30-token answer takes about 0.6 s
+    TOKENS = 30
+
+    def setUp(self):
+        self.tok = ByteTokenizer()
+        self.engine = MockEngine(self.tok, "x" * self.TOKENS, max_context=CTX, delay_s=self.PACE_S)
+        self.svc = Service(self.engine, self.tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        self.httpd = serve(self.svc, port=0)
+        self.base = f"http://127.0.0.1:{self.httpd.server_address[1]}"
+
+    def tearDown(self):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+    def metrics(self):
+        with urllib.request.urlopen(self.base + "/metrics", timeout=10) as r:
+            return json.loads(r.read())
+
+    def test_the_live_number_is_a_rate(self):
+        live_samples, stop = [], threading.Event()
+
+        def poll():                                   # what the Monitor polls, at 10 ms
+            while not stop.is_set():
+                live = self.metrics()["live"]
+                if live["state"] == "generating" and live["tok_s"] is not None:
+                    live_samples.append((live["generated"], live["tok_s"], live["tok_s_mean"]))
+                time.sleep(0.01)
+
+        body = json.dumps({"model": "m", "max_tokens": self.TOKENS, "temperature": 0,
+                           "messages": [{"role": "user", "content": "hi"}]}).encode()
+        watcher = threading.Thread(target=poll, daemon=True)
+        watcher.start()
+        t0 = time.time()
+        try:
+            with urllib.request.urlopen(urllib.request.Request(self.base + "/v1/chat/completions", data=body,
+                                                               headers={"Content-Type": "application/json"}),
+                                        timeout=30) as r:
+                usage = json.loads(r.read())["usage"]
+        finally:
+            stop.set()
+            watcher.join(2)
+        true_rate = usage["completion_tokens"] / (time.time() - t0)
+        self.assertGreaterEqual(len(live_samples), 3, "too few live readings to judge the readout")
+        self.assertLess(max(s for _g, s, _m in live_samples), 4 * true_rate,
+                        f"live.tok_s peaked at {max(s for _g, s, _m in live_samples):.1f} tok/s "
+                        f"for a {true_rate:.1f} tok/s engine")
+        self.assertEqual(self.metrics()["live"]["state"], "idle")
+        self.assertIsNone(self.metrics()["live"]["tok_s"])
+
+    def test_a_request_without_a_done_keeps_no_engine_counters(self):
+        """An engine that dies mid-answer: the previous request's `last` must not become this row's decode rate."""
+        class HalfDead(MockEngine):
+            last = {"generated": 99, "prompt_tokens": 9, "prompt_ms": 10.0, "decode_ms": 100.0, "finish": "stop"}
+
+            def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+                for i, t in enumerate(super().generate(ids, max_new, sampling, cancel, embeddings)):
+                    if i == 3:
+                        raise EngineDied("the engine stopped unexpectedly (exit code -9)")
+                    yield t
+
+        tok = ByteTokenizer()
+        svc = Service(HalfDead(tok, "x" * self.TOKENS, max_context=CTX), tok,
+                      ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        httpd = serve(svc, port=0)
+        base = f"http://127.0.0.1:{httpd.server_address[1]}"
+        try:
+            body = json.dumps({"model": "m", "max_tokens": self.TOKENS, "stream": True,
+                               "messages": [{"role": "user", "content": "hi"}]}).encode()
+            with urllib.request.urlopen(urllib.request.Request(base + "/v1/chat/completions", data=body,
+                                                               headers={"Content-Type": "application/json"}),
+                                        timeout=30) as r:
+                text = r.read().decode()
+            self.assertIn('"error"', text)
+            row = svc.metrics()["requests"][0]
+            self.assertEqual(row["finish"], "error")
+            self.assertEqual(row["output_tokens"], 3)
+            self.assertIsNone(row["decode_tok_s"], "the previous request's counters were recorded as this one's")
+            self.assertIsNone(row["engine_generated"])
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+
+
 class SharedSettings(unittest.TestCase):
     """The web app's "Use for other apps too": POST /settings makes its Chat settings every client's defaults."""
 
@@ -489,7 +666,7 @@ class WebApp(unittest.TestCase):
         code, ctype, body = self.get("/")
         self.assertEqual(code, 200)
         self.assertIn("text/html", ctype)
-        self.assertIn(b"/web/app.js", body)
+        self.assertIn(b"\"web/app.js\"", body)   # relative since #82 (works behind a path-prefixed proxy)
         for path, want in (("/web/app.js", "javascript"), ("/web/app.css", "text/css"), ("/web/tokens.css", "text/css"),
                            ("/web/components.css", "text/css"), ("/web/sprite.svg", "image/svg+xml")):
             with self.subTest(path=path):
@@ -516,6 +693,86 @@ class WebApp(unittest.TestCase):
         self.assertEqual(m["live"]["state"], "idle")
         self.assertEqual(m["requests"][0]["output_tokens"], 5)
 
+    def test_model_discovery_and_props(self):
+        svc = self.svc
+        previous = svc.engine.max_context, svc.vision, svc.sampling_defaults, svc.shared
+        try:
+            svc.engine.max_context = 262144
+            svc.sampling_defaults = {"temperature": 1.0, "repetition_penalty": 1.1}
+            svc.shared = {"temperature": 0.7, "max_tokens": 4096}
+            for vision in (None, object()):
+                svc.vision = vision
+                for path in ("/models", "/v1/models"):
+                    code, _, body = self.get(path)
+                    self.assertEqual(code, 200)
+                    models = json.loads(body)["data"]
+                    self.assertEqual(len(models), 1)
+                    model = models[0]
+                    self.assertEqual(model["id"], svc.model)
+                    self.assertEqual(model["status"]["value"], "loaded")
+                    self.assertEqual(model["meta"]["n_ctx"], 262144)
+                    self.assertEqual(model["architecture"]["input_modalities"],
+                                     ["text", "image"] if vision else ["text"])
+                code, _, body = self.get("/props?model=" + svc.model + "&autoload=false")
+                self.assertEqual(code, 200)
+                props = json.loads(body)
+                self.assertEqual(props["default_generation_settings"]["n_ctx"], 262144)
+                self.assertEqual(props["default_generation_settings"]["params"],
+                                 {"temperature": 0.7, "repeat_penalty": 1.1, "n_predict": 4096})
+                self.assertEqual(props["chat_template"], (ROOT / "serve/chat_template.jinja").read_text(encoding="utf-8"))
+                self.assertEqual(props["modalities"]["vision"], vision is not None)
+                self.assertEqual(props["total_slots"], 1)
+                self.assertFalse(props["models_autoload"])
+            svc.shared = {}
+            props = json.loads(self.get("/props")[2])
+            self.assertEqual(props["default_generation_settings"]["params"]["n_predict"], -1)
+            self.assertEqual(self.get("/props?model=not-loaded&autoload=true")[0], 404)
+        finally:
+            svc.engine.max_context, svc.vision, svc.sampling_defaults, svc.shared = previous
+
+    def test_discovery_needs_the_api_key(self):
+        self.svc.api_key = "secret"
+        try:
+            for path in ("/models", "/v1/models", "/props", "/slots"):
+                self.assertEqual(self.get(path)[0], 401)
+                self.assertEqual(self.get(path, {"Authorization": "Bearer secret"})[0], 200)
+        finally:
+            self.svc.api_key = ""
+
+    def test_build_model_path_and_slot_status(self):
+        engine = self.svc.engine
+        engine.model_path = "models/example.gguf"
+        engine.info = {"version": "0.1.21"}
+        try:
+            props = json.loads(self.get("/props")[2])
+            self.assertEqual(props["model_path"], engine.model_path)
+            self.assertEqual(props["build_info"], "Strata 0.1.21")
+            for busy in (True, False):
+                with self.svc.status_lock:
+                    self.svc.status["busy"] = busy
+                code, _, body = self.get("/slots")
+                self.assertEqual(code, 200)
+                self.assertEqual(json.loads(body), [{"id": 0, "n_ctx": CTX, "is_processing": busy}])
+        finally:
+            with self.svc.status_lock:
+                self.svc.status["busy"] = False
+            del engine.model_path, engine.info
+        props = json.loads(self.get("/props")[2])
+        self.assertNotIn("build_info", props)
+        self.assertNotIn("model_path", props)
+
+    def test_discovery_does_not_restart_a_dead_engine(self):
+        self.svc.engine.alive = lambda: False
+        try:
+            for path in ("/models", "/v1/models"):
+                code, _, body = self.get(path)
+                self.assertEqual(code, 200)
+                self.assertEqual(json.loads(body)["data"], [])
+            self.assertEqual(self.get("/props")[0], 503)
+            self.assertEqual(json.loads(self.get("/slots")[2]), [])
+        finally:
+            del self.svc.engine.alive
+
     def test_metrics_need_the_key_when_one_is_set(self):
         self.svc.api_key = "secret"
         try:
@@ -526,5 +783,119 @@ class WebApp(unittest.TestCase):
             self.svc.api_key = ""
 
 
+class ClockedEngine(MockEngine):
+    """The mock engine with StrataEngine's clock: `last` as the engine's DONE line gives it, the conversation cache
+    holding the first REUSED tokens of every prompt."""
+    REUSED = 5
+
+    def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+        n = 0
+        try:
+            for t in super().generate(ids, max_new, sampling, cancel, embeddings):
+                n += 1
+                yield t
+        finally:          # as StrataEngine reads its DONE line: also when the server closes the request at a stop token
+            self.last = {"generated": n, "prompt_tokens": len(ids), "prompt_ms": 40.0, "decode_ms": 20.0 * n,
+                         "finish": "stop", "reused": min(self.REUSED, len(ids)), "hits": 9, "lookups": 10}
+
+
+class UsageAndStatus(unittest.TestCase):
+    """What clients read besides the text: the part of the prompt the conversation cache held (OpenAI's
+    prompt_tokens_details.cached_tokens, Anthropic's cache_read_input_tokens), llama.cpp's timings, GET /v1/status."""
+
+    @classmethod
+    def setUpClass(cls):
+        tok = ByteTokenizer()
+        cls.engine = ClockedEngine(tok, "</think>\n\nok", max_context=CTX)
+        cls.svc = Service(cls.engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        cls.httpd = serve(cls.svc, port=0)
+        cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}"
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.httpd.shutdown()
+        cls.httpd.server_close()
+
+    def request(self, path, body=None):
+        req = urllib.request.Request(self.base + path, data=None if body is None else json.dumps(body).encode(),
+                                     headers={"Content-Type": "application/json", "anthropic-version": "2023-06-01"})
+        with urllib.request.urlopen(req, timeout=30) as r:
+            return r.status, r.read()
+
+    def chat(self, path, stream=False):
+        body = {"model": "x", "max_tokens": 20, "messages": [{"role": "user", "content": "hi"}], "stream": stream}
+        status, raw = self.request(path, body)
+        self.assertEqual(status, 200)
+        if not stream:
+            return json.loads(raw)
+        return [json.loads(line[6:]) for line in raw.decode().splitlines()
+                if line.startswith("data: {")]
+
+    def test_openai(self):
+        b = self.chat("/v1/chat/completions")
+        u, t = b["usage"], b["timings"]
+        self.assertEqual(u["prompt_tokens_details"]["cached_tokens"], ClockedEngine.REUSED)
+        self.assertEqual(t["cache_n"], ClockedEngine.REUSED)
+        self.assertEqual(t["prompt_n"] + t["cache_n"], u["prompt_tokens"])
+        self.assertEqual(t["predicted_n"], u["completion_tokens"])
+        self.assertAlmostEqual(t["prompt_per_second"], t["prompt_n"] / 0.040, delta=0.1)
+        self.assertAlmostEqual(t["predicted_per_second"], 50.0, delta=0.1)            # 20 ms a token
+
+    def test_openai_stream(self):
+        last = self.chat("/v1/chat/completions", stream=True)[-1]
+        self.assertEqual(last["usage"]["prompt_tokens_details"]["cached_tokens"], ClockedEngine.REUSED)
+        self.assertEqual(last["timings"]["cache_n"], ClockedEngine.REUSED)
+
+    def test_anthropic(self):
+        u = self.chat("/v1/messages")["usage"]
+        self.assertEqual(u["cache_read_input_tokens"], ClockedEngine.REUSED)
+        self.assertEqual(u["input_tokens"] + u["cache_read_input_tokens"], len(self.engine.last_prompt))
+        self.assertGreater(u["output_tokens"], 0)
+
+    def test_v1_status(self):
+        self.chat("/v1/chat/completions")
+        status, raw = self.request("/v1/status")
+        self.assertEqual(status, 200)
+        s = json.loads(raw)
+        self.assertEqual(s["model"], self.svc.model)
+        self.assertEqual(s["context"]["max_positions"], CTX)
+        self.assertEqual(s["concurrency"]["serving"], 1)
+        self.assertFalse(s["vision"]["available"])
+        self.assertEqual(s["activity"]["in_flight"], 0)
+        self.assertGreaterEqual(s["activity"]["requests"], 1)
+        self.assertEqual(s["last_timings"]["cache_n"], ClockedEngine.REUSED)
+        self.assertIn("at", s["last_timings"])
+
+    def test_no_clock(self):
+        """An engine without a clock (MockEngine): no timings, nothing cached."""
+        tok = ByteTokenizer()
+        svc = Service(MockEngine(tok, "</think>\n\nok", max_context=CTX), tok,
+                      ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        httpd = serve(svc, port=0)
+        try:
+            data = json.dumps({"model": "x", "max_tokens": 5, "messages": [{"role": "user", "content": "hi"}]}).encode()
+            req = urllib.request.Request(f"http://127.0.0.1:{httpd.server_address[1]}/v1/chat/completions", data=data,
+                                         headers={"Content-Type": "application/json"})
+            with urllib.request.urlopen(req, timeout=30) as r:
+                b = json.loads(r.read())
+            self.assertNotIn("timings", b)
+            self.assertEqual(b["usage"]["prompt_tokens_details"]["cached_tokens"], 0)
+            self.assertIsNone(svc.v1_status()["last_timings"])
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+
+
+class TimingsDrafts(unittest.TestCase):
+    """`timings` carries the speculative draft counts (PR #83's fields) only when the engine reported them."""
+
+    def test_draft_fields(self):
+        base = {"prompt_ms": 100.0, "decode_ms": 200.0, "generated": 20, "reused": 4}
+        t = request_timings(24, 20, dict(base, drafts_offered=15, drafts_accepted=11))
+        self.assertEqual((t["draft_n"], t["draft_n_accepted"]), (15, 11))
+        self.assertEqual((t["prompt_n"], t["cache_n"]), (20, 4))
+        self.assertNotIn("draft_n", request_timings(24, 20, base))
+        self.assertIsNone(request_timings(24, 20, {}))
+
 if __name__ == "__main__":
     unittest.main()

serve/web/app.js

@@ -3,7 +3,7 @@
 "use strict";
 
 const $ = (id) => document.getElementById(id);
-const SPRITE = "/web/sprite.svg";
+const SPRITE = "web/sprite.svg";
 const icon = (name, cls = "st-icon") => `<svg class="${cls}" aria-hidden="true"><use href="${SPRITE}#i-${name}"/></svg>`;
 const esc = (s) => String(s).replace(/[&<>"']/g, (c) => ({"&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;"}[c]));
 const fmt = (n, d = 0) => (n == null || Number.isNaN(n) ? "–" : Number(n).toLocaleString(undefined, {maximumFractionDigits: d, minimumFractionDigits: d}));
@@ -98,7 +98,7 @@ $("api-key").onchange = () => { store.set("apikey", $("api-key").value.trim());
 let health = {model: "strata", images: false, max_context: 0};
 async function loadHealth() {
   try {
-    health = await (await fetch("/health")).json();
+    health = await (await fetch("health")).json();
     $("attach-btn").title = health.images ? "Attach a text file or a picture (or drop it here)"
                                           : "Attach a text file (or drop it here)";
     $("chat-empty-sub").textContent = `${health.model} runs on this PC. Nothing leaves it.`;
@@ -147,7 +147,7 @@ let lastMetrics = null, metricsFailures = 0, keyWarned = false, mcpTick = 0;
 let reqShowAll = false;   // the Monitor's request table: the last 12, or every one the server keeps (issue #35)
 async function poll() {
   try {
-    const r = await fetch(reqShowAll ? "/metrics?requests=all" : "/metrics", {headers: headers()});
+    const r = await fetch(reqShowAll ? "metrics?requests=all" : "metrics", {headers: headers()});
     if (r.status === 401) {
       setPill("error", "API key needed");
       if (!keyWarned) { keyWarned = true; toast("warn", "API key needed", "This server needs a key: add it under About > Settings.", 6000); }
@@ -353,7 +353,7 @@ $("req-all").addEventListener("click", () => { reqShowAll = !reqShowAll; if (las
 let mcpInfo = {servers: [], tools: 0}, mcpRetry = null;
 async function loadMcp() {
   try {
-    const r = await fetch("/mcp", {headers: headers()});
+    const r = await fetch("mcp", {headers: headers()});
     if (!r.ok) return;
     mcpInfo = await r.json();
   } catch (e) { return; /* an older server: no MCP */ }
@@ -685,7 +685,7 @@ async function send() {
   let firstAt = null, thinkStart = null, usage = null, frame = 0;
   const paint = () => { frame = 0; updateAssistant(el, m, true); scrollDown(); };
   try {
-    const r = await fetch("/v1/chat/completions", {method: "POST", headers: headers(true), body: JSON.stringify(body),
+    const r = await fetch("v1/chat/completions", {method: "POST", headers: headers(true), body: JSON.stringify(body),
                                                    signal: controller.signal});
     if (!r.ok) {
       let msg = `HTTP ${r.status}`;
@@ -887,7 +887,7 @@ function loadDrawer(s = settings) {
 let sharedOn = false;
 async function loadShared() {
   try {
-    const r = await fetch("/settings", {headers: headers()});
+    const r = await fetch("settings", {headers: headers()});
     if (r.ok) sharedOn = !!(await r.json()).shared;
   } catch (e) { /* an older server: the switch just stays off */ }
   $("s-share").setAttribute("aria-checked", String(sharedOn));
@@ -901,7 +901,7 @@ function sharedDefaults(s) {
   return d;
 }
 async function saveShared(on, s) {
-  const r = await fetch("/settings", {method: "POST", headers: headers(true),
+  const r = await fetch("settings", {method: "POST", headers: headers(true),
                                       body: JSON.stringify({defaults: on ? sharedDefaults(s) : null})});
   if (!r.ok) {
     let msg = `HTTP ${r.status}`;

serve/web/index.html

@@ -4,9 +4,9 @@
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>Strata</title>
-<link rel="stylesheet" href="/web/tokens.css">
-<link rel="stylesheet" href="/web/components.css">
-<link rel="stylesheet" href="/web/app.css">
+<link rel="stylesheet" href="web/tokens.css">
+<link rel="stylesheet" href="web/components.css">
+<link rel="stylesheet" href="web/app.css">
 <script>
   // the theme before the first paint: the saved choice, else the system's
   try {
@@ -21,17 +21,17 @@
   <span class="st-header__brand">Strata</span>
   <nav class="st-tabs" role="tablist" aria-label="Views">
     <button class="st-tab" role="tab" id="tab-btn-chat" aria-controls="view-chat" aria-selected="true" data-tab="chat">
-      <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-chat"/></svg>Chat</button>
+      <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-chat"/></svg>Chat</button>
     <button class="st-tab" role="tab" id="tab-btn-monitor" aria-controls="view-monitor" aria-selected="false" data-tab="monitor">
-      <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-activity"/></svg>Monitor</button>
+      <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-activity"/></svg>Monitor</button>
     <button class="st-tab" role="tab" id="tab-btn-about" aria-controls="view-about" aria-selected="false" data-tab="about">
-      <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-info"/></svg>About</button>
+      <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-info"/></svg>About</button>
   </nav>
   <span class="st-header__spacer"></span>
   <span class="st-pill" id="pill" data-state="idle" role="status" aria-live="polite">
     <span class="st-pill__dot"></span><span id="pill-text">Connecting&hellip;</span></span>
   <button class="st-btn st-btn--icon" id="theme-btn" title="Light / dark" aria-label="Switch between light and dark">
-    <svg class="st-icon"><use id="theme-icon" href="/web/sprite.svg#i-moon"/></svg></button>
+    <svg class="st-icon"><use id="theme-icon" href="web/sprite.svg#i-moon"/></svg></button>
 </header>
 
 <main>
@@ -51,20 +51,20 @@
         <textarea id="input" rows="2" placeholder="Ask anything&hellip;" aria-label="Message"></textarea>
         <div class="st-composer__bar">
           <button type="button" class="st-btn st-btn--icon" id="attach-btn" title="Attach a text file (or drop it here)" aria-label="Attach a file">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-attach"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-attach"/></svg></button>
           <input type="file" id="file" multiple hidden>
           <button type="button" class="st-btn st-btn--icon" id="new-btn" title="New chat" aria-label="New chat">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-new-chat"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-new-chat"/></svg></button>
           <button type="button" class="st-btn st-btn--icon" id="export-btn" title="Save this chat as Markdown" aria-label="Save this chat">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-download"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-download"/></svg></button>
           <button type="button" class="st-btn st-btn--icon" id="sampling-btn" title="Sampling and thinking" aria-label="Sampling and thinking">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-settings"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-settings"/></svg></button>
           <span class="spacer"></span>
           <span class="composer-hint" id="composer-hint"></span>
           <button type="button" class="st-btn st-btn--secondary" id="stop-btn" hidden>
-            <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-stop"/></svg>Stop</button>
+            <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-stop"/></svg>Stop</button>
           <button type="submit" class="st-btn st-btn--primary st-btn--send" id="send-btn" aria-label="Send">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-send"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-send"/></svg></button>
         </div>
       </form>
     </div>
@@ -169,7 +169,7 @@
 <aside class="st-drawer" id="drawer" data-open="false" aria-label="Sampling and thinking" aria-hidden="true">
   <div class="st-drawer__head"><span class="card-title">Sampling</span>
     <button class="st-btn st-btn--icon" id="drawer-close" aria-label="Close">
-      <svg class="st-icon"><use href="/web/sprite.svg#i-chevron"/></svg></button></div>
+      <svg class="st-icon"><use href="web/sprite.svg#i-chevron"/></svg></button></div>
   <div class="st-drawer__body">
     <div class="st-field">
       <label class="st-label" for="s-thinking">Thinking <output id="o-thinking"></output></label>
@@ -218,6 +218,6 @@
 </aside>
 <div class="scrim" id="scrim" hidden></div>
 <div class="st-toasts" id="toasts" aria-live="polite"></div>
-<script src="/web/app.js"></script>
+<script src="web/app.js"></script>
 </body>
 </html>

serve/web/tokens.css

@@ -3,8 +3,8 @@
    Values marked (derived) are not in the Gainsty source and were added for the app. */
 
 /* Outfit (SIL Open Font License 1.1, serve/web/fonts/OFL.txt): one variable font, weights 100-900 */
-@font-face{font-family:"Outfit";src:url("/fonts/outfit-latin-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
-@font-face{font-family:"Outfit";src:url("/fonts/outfit-latin-ext-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}
+@font-face{font-family:"Outfit";src:url("../fonts/outfit-latin-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
+@font-face{font-family:"Outfit";src:url("../fonts/outfit-latin-ext-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}
 
 :root{
   /* type */

setup.py

@@ -13,7 +13,7 @@ What the first run does (each step is skipped when it is already done):
   1. checks your PC: NVIDIA GPU and driver, RAM, CPU, free disk space
   2. asks the questions
   3. installs the Python packages it needs into .venv (numpy, jinja2, ..., and NVIDIA's CUDA libraries)
-  4. gets the Strata engine: a ready-made build for RTX 30/40/50 cards (no compiler needed); if none fits your PC,
+  4. gets the Strata engine: a ready-made build for RTX 20/30/40/50 cards (no compiler needed); if none fits your PC,
      it installs the build tools (asks first) and compiles the engine for your GPU
   5. downloads the model from Hugging Face (resumable), and the vision encoder if you want images
   6. prepares the model for Strata and fetches the MTP draft layer (~5 GB, from the original Qwen checkpoint)
@@ -36,6 +36,7 @@ import os
 import platform
 import re
 import shutil
+import struct
 import subprocess
 import sys
 import time
@@ -57,11 +58,13 @@ PREBUILT_ASSET = "strata-windows-x64.zip" if WIN else "strata-linux-x64.zip"
 # the CUDA libraries the ready-made engine loads (the same CUDA 13.0 it is built with), from NVIDIA's pip packages
 CUDA_WHEELS = ["nvidia-cublas==13.0.2.14", "nvidia-cuda-runtime==13.0.96"]
 MIN_DRIVER = 580                       # CUDA 13.0
-MIN_ENGINE = (0, 1, 22)                # v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
+MIN_ENGINE = (0, 1, 28)                # v0.1.28: the expert cache reserves the draft head, a cancelled request no longer fails the next; v0.1.27: RTX 20 (sm_75) in the ready-made engine, the HIP build without CUDA headers; v0.1.26: the draft layer's prompt pass in batches; v0.1.25: faster prompts (grouping off the copy engine, fused hyper-connection kernels), AMD HIP backend, --kv k8v4; v0.1.24: long prompts faster (QSA select on tensor cores); v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
 PY_PACKAGES = ["numpy", "jinja2", "regex", "pyyaml", "tqdm", "requests", "cmake", "ninja", "pillow", "psutil"]
 
 MODELS = {
-    "Q2_0": {"about": "2-bit, the fastest", "download_gb": 66.4, "ram_gb": 48, "arena_gb": 34.0},
+    # the original model only for now: Swift 1.5's Q2_0 files split one layer's experts across the two shards, which
+    # the pack tool (tools/iq_pack.py) cannot prepare yet (#171)
+    "Q2_0": {"about": "2-bit, the fastest", "download_gb": 66.4, "ram_gb": 48, "arena_gb": 34.0, "families": ("qwen",)},
     "IQ2_XS": {"about": "2-bit i-quant, a little better quality, close in speed", "download_gb": 68.0, "ram_gb": 48,
                "arena_gb": 35.5},
     "IQ3_XXS": {"about": "3-bit i-quant, better quality, slower (more CPU work per token)", "download_gb": 75.8,
@@ -220,7 +223,7 @@ def cpu_info():
     name, avx2, avx512 = platform.processor() or "unknown CPU", False, False
     if WIN:
         pf = ctypes.windll.kernel32.IsProcessorFeaturePresent
-        avx2 = bool(pf(40))                       # PF_AVX2_INSTRUCTIONS_AVAILABLE
+        avx2 = bool(pf(40)) or _cpuid_avx2()      # PF_AVX2_INSTRUCTIONS_AVAILABLE, else the CPU itself (#159)
         n = out(["powershell", "-NoProfile", "-Command", "(Get-CimInstance Win32_Processor).Name"]).strip()
         name = n or name
         avx512 = bool(pf(41)) and _cpuid_avx512_full()
@@ -258,6 +261,47 @@ def _cpuid_avx512_full() -> bool:
         return False
 
 
+def _run_stub(code: bytes, *args) -> None:
+    """Runs a few bytes of x64 machine code (Windows calling convention: the arguments in rcx, rdx)."""
+    k32 = ctypes.windll.kernel32
+    k32.VirtualAlloc.restype = ctypes.c_void_p
+    k32.VirtualFree.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32)
+    buf = k32.VirtualAlloc(None, len(code), 0x3000, 0x40)
+    if not buf:
+        raise OSError("VirtualAlloc failed")
+    try:
+        ctypes.memmove(buf, code, len(code))
+        ctypes.CFUNCTYPE(None, *[ctypes.c_void_p] * len(args))(buf)(*args)
+    finally:
+        k32.VirtualFree(buf, 0, 0x8000)
+
+
+def _cpuid_avx2() -> bool:
+    """AVX2 asked from the CPU (CPUID leaf 7 EBX bit 5), with the OS saving the YMM registers (OSXSAVE + XCR0):
+    Windows' IsProcessorFeaturePresent(PF_AVX2) says no on some PCs whose CPU has it (a Ryzen 9 3950X, #159)."""
+    try:
+        def cpuid(leaf):
+            regs = (ctypes.c_uint32 * 4)()
+            _run_stub(bytes([0x53, 0x49, 0x89, 0xC8, 0x89, 0xD0, 0x31, 0xC9, 0x0F, 0xA2,      # push rbx; r8=rcx; eax=edx; ecx=0; cpuid
+                             0x41, 0x89, 0x00, 0x41, 0x89, 0x58, 0x04, 0x41, 0x89, 0x48, 0x08,  # [r8]=eax, [r8+4]=ebx, [r8+8]=ecx
+                             0x41, 0x89, 0x50, 0x0C, 0x5B, 0xC3]),                             # [r8+12]=edx; pop rbx
+                      ctypes.addressof(regs), leaf)
+            return list(regs)
+        if cpuid(0)[0] < 7:
+            return False
+        ecx1 = cpuid(1)[2]
+        if not (ecx1 >> 27) & 1 or not (ecx1 >> 28) & 1:             # OSXSAVE, AVX
+            return False
+        xcr0 = (ctypes.c_uint32 * 2)()
+        _run_stub(bytes([0x49, 0x89, 0xC8, 0x31, 0xC9, 0x0F, 0x01, 0xD0,                        # r8=rcx; ecx=0; xgetbv
+                         0x41, 0x89, 0x00, 0x41, 0x89, 0x50, 0x04, 0xC3]), ctypes.addressof(xcr0))
+        if xcr0[0] & 6 != 6:                                           # the OS saves XMM and YMM
+            return False
+        return bool((cpuid(7)[1] >> 5) & 1)
+    except Exception:
+        return False
+
+
 def gpus():
     """Every NVIDIA GPU, numbered as nvidia-smi numbers them (by PCI bus, the order the engine is told to use)."""
     s = out(["nvidia-smi", "--query-gpu=index,name,memory.total,compute_cap,driver_version",
@@ -284,8 +328,8 @@ def cc(g) -> str:
 
 def gpu_problem(g, together=False):
     """Why Strata cannot use this card, in plain words (None: it can)."""
-    if int(g["arch"]) < 80:
-        return (f"not supported - older than the RTX 30 series (compute capability {cc(g)}; Strata needs 8.0 or "
+    if int(g["arch"]) < 75:
+        return (f"not supported - older than the RTX 20 series (compute capability {cc(g)}; Strata needs 7.5 or "
                 "newer)")
     if together and g["vram_gb"] < SPLIT_MIN_VRAM_GB - 0.5:
         return (f"not supported together with other GPUs - {g['vram_gb']:.0f} GB of VRAM (a card sharing the model "
@@ -348,7 +392,7 @@ def check_gpus(sel, found, what="") -> None:
         ones = " or ".join(f"--gpu {x['index']}" for x in single)
         both = "--gpus " + ",".join(str(x["index"]) for x in can) if can else ""
         hint = ((f"use these together: {both}" + (f" (or one card: {ones})" if not together else "")) if can else
-                f"use one card: {ones}" if single else "Strata needs an NVIDIA RTX 30 series or newer card")
+                f"use one card: {ones}" if single else "Strata needs an NVIDIA RTX 20 series or newer card")
         fail(f"GPU {i}{'' if g is None else ' (' + g['name'] + ')'} {what}cannot be used: {p}", hint)
 
 
@@ -383,7 +427,7 @@ def choose_gpus(a, found) -> list:
     single = sorted([g for g in found if gpu_problem(g) is None], key=lambda x: (-round(x["vram_gb"]), x["index"]))
     if not single:
         gpu_table(found)
-        fail("none of your GPUs can run Strata", "it needs an NVIDIA RTX 30 series or newer (compute capability 8.0+)")
+        fail("none of your GPUs can run Strata", "it needs an NVIDIA RTX 20 series or newer (compute capability 7.5+)")
     can = together_ok(found)
     if not can:
         return [single[0]["index"]]
@@ -558,12 +602,46 @@ def download(url, dst: Path, what=None):
             warn(f"download interrupted ({e}); retrying in 10 s ...")
             time.sleep(10)
     if total and part.stat().st_size != total:
-        fail(f"could not finish downloading {dst.name}", "check your internet connection and run it again")
+        fail(f"could not finish downloading {dst.name}: {part.stat().st_size:,} bytes on disk, the server says {total:,}",
+             "check your internet connection and run it again (the download resumes where it stopped)")
     part.replace(dst)
     mark(dst)
     ok(f"{what or dst.name} downloaded")
 
 
+def whole_shard(s: Path) -> bool:
+    """A shard as long as its own tensor directory says (check_shards' test, without stopping setup)."""
+    sys.path.insert(0, str(ROOT / "tools"))
+    from gguf_reader import GGUFFile
+    try:
+        g = GGUFFile(s)
+        return s.stat().st_size >= g.data_start + max((t.offset + (t.expected_bytes() or 0) for t in g.tensors),
+                                                     default=0)
+    except (OSError, ValueError, struct.error):
+        return False
+
+
+def check_shards(shards):
+    """Every shard present and whole, or setup stops naming the file and the numbers.  Whole means as long as
+    its own tensor directory says (the header is read, the data is not): a truncated copy (--gguf-dir, a .part
+    renamed by hand, a download finished by an older setup) otherwise passes as a model file and the engine
+    fails much later, at the first tensor that runs past the end."""
+    sys.path.insert(0, str(ROOT / "tools"))
+    from gguf_reader import GGUFFile
+    for s in shards:
+        if not s.exists():
+            fail(f"missing {s}")
+        try:
+            g = GGUFFile(s)
+        except (ValueError, struct.error) as e:
+            fail(f"{s.name} is not a whole GGUF shard ({e})", "delete it and run setup again")
+        need = g.data_start + max((t.offset + (t.expected_bytes() or 0) for t in g.tensors), default=0)
+        have = s.stat().st_size
+        if have < need:
+            fail(f"{s.name} is short: {have:,} of {need:,} bytes ({need - have:,} missing)",
+                 "delete it and run setup again (or copy the whole file into --gguf-dir)")
+
+
 def get_llama_cpp():
     """llama.cpp at the pinned commit (ggml for the build, gguf-py for the tools, mtmd for images), as a zip: no git."""
     llama = ROOT / "third_party" / "llama.cpp"
@@ -574,7 +652,9 @@ def get_llama_cpp():
     tmp = ROOT / "third_party" / "_unpack"
     shutil.rmtree(tmp, ignore_errors=True)
     with zipfile.ZipFile(z) as f:
-        f.extractall(tmp)
+        # llama.cpp's own web UI (tools/ui) is not used, and its deep paths passed Windows' 260-character limit in a
+        # folder like Downloads\Strata-main\Strata-main (#206)
+        f.extractall(tmp, [m for m in f.namelist() if "/tools/ui/" not in m])
     top = next(tmp.iterdir())
     shutil.rmtree(llama, ignore_errors=True)
     # PR #63: on Windows a rename can fail with PermissionError while an antivirus scanner still holds a file of the
@@ -620,6 +700,120 @@ def cuda_lib_dirs():
     return [str(d) for d in dirs]
 
 
+# ------------------------------------------------------------------------------------------------ AMD (experimental)
+# The RX 7900 XT / XTX (gfx1100) on Linux, through the HIP backend (docs/AMD_HIP.md).  There is no ready-made AMD
+# engine: ROCm comes from AMD's TheRock Python wheels into .venv (no sudo; a system ROCm in /opt/rocm is used when it
+# has hipcc and hipBLAS) and the engine is compiled here.  One GPU, no images yet.
+ROCM_INDEX = os.environ.get("STRATA_ROCM_INDEX", "https://rocm.nightlies.amd.com/v2/gfx110X-dgpu/")
+ROCM_VERSION = os.environ.get("STRATA_ROCM_VERSION", "7.10.0a20251120")   # what Strata's HIP build was tested with
+AMD_ARCHS = ("gfx1100",)
+AMD_NAMES = {"gfx1100": "AMD Radeon RX 7900 series (gfx1100)"}   # when sysfs has no product name
+
+
+def amd_gpus():
+    """AMD GPUs from the kernel's KFD topology (the amdgpu driver; no ROCm needed), numbered as HIP numbers them:
+    the GPU nodes in order, the CPU nodes skipped.  Integrated GPUs are listed too (not supported)."""
+    base = Path("/sys/class/kfd/kfd/topology/nodes")
+    found = []
+    if WIN or not base.is_dir():
+        return found
+    for node in sorted((p for p in base.iterdir() if p.name.isdigit()), key=lambda p: int(p.name)):
+        try:
+            props = {}
+            for line in (node / "properties").read_text().splitlines():
+                k, _, v = line.partition(" ")
+                props[k] = v.strip()
+            ver = int(props.get("gfx_target_version") or 0)
+            if ver == 0 or int(props.get("simd_count") or 0) == 0:
+                continue
+        except (OSError, ValueError):
+            continue
+        arch = f"gfx{ver // 10000}{(ver // 100) % 100:x}{ver % 100:x}"
+        dev = Path(f"/sys/class/drm/renderD{props.get('drm_render_minor', '')}/device")
+        try:
+            vram = int((dev / "mem_info_vram_total").read_text()) / 2 ** 30
+        except (OSError, ValueError):
+            vram = 0.0
+        try:
+            name = (dev / "product_name").read_text().strip() or f"AMD Radeon ({arch})"
+        except OSError:
+            name = f"AMD Radeon ({arch})"
+        if name == f"AMD Radeon ({arch})" and arch in AMD_NAMES:
+            name = AMD_NAMES[arch]
+        found.append({"index": len(found), "name": name, "vram_gb": vram, "arch": arch, "driver": "amdgpu",
+                      "vendor": "amd"})
+    return found
+
+
+def amd_problem(g):
+    if g["arch"] not in AMD_ARCHS:
+        return (f"not supported - Strata's AMD backend runs on the RX 7900 XT / XTX ({', '.join(AMD_ARCHS)}) only, "
+                f"this is {g['arch']}")
+    return None
+
+
+def rocm_root():
+    """ROCm for compiling and running the HIP engine: (root, library folders).  A system ROCm with hipcc and hipBLAS,
+    else AMD's TheRock wheels (ROCM_VERSION, from ROCM_INDEX) installed into .venv."""
+    sysroot = Path(os.environ.get("ROCM_PATH") or "/opt/rocm")
+    if (sysroot / "bin" / "hipcc").exists() and list((sysroot / "lib").glob("libhipblas.so*")):
+        return sysroot, [str(sysroot / "lib")]
+    stamp = Path(sys.prefix) / ".strata-rocm.json"
+    if not stamp.exists() or json.loads(stamp.read_text()).get("version") != ROCM_VERSION:
+        say(f"  Installing ROCm {ROCM_VERSION} for AMD GPUs into .venv (AMD's TheRock wheels, ~10 GB, no sudo) ...")
+        run([sys.executable, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", "--index-url",
+             ROCM_INDEX, f"rocm[libraries,devel]=={ROCM_VERSION}"])
+        stamp.write_text(json.dumps({"version": ROCM_VERSION, "index": ROCM_INDEX}))
+    sdk = Path(sys.executable).parent / "rocm-sdk"
+    root = Path(out([str(sdk), "path", "--root"]).strip())
+    if not (root / "llvm" / "bin" / "clang++").exists():
+        fail(f"ROCm was installed but its compiler is missing ({root})",
+             f"remove {stamp} and run this again; or install ROCm 7 system-wide")
+    dirs = [str(root / "lib")]
+    for sp in {Path(p) for p in sys.path if p.endswith("site-packages")}:
+        dirs += [str(d / "lib") for d in sorted(sp.glob("_rocm_sdk_libraries_*")) if (d / "lib").is_dir()]
+    ok(f"ROCm: {root}")
+    return root, list(dict.fromkeys(dirs))
+
+
+def build_engine_hip(gpu, llama) -> Path:
+    """Compile the HIP engine for this AMD GPU into engine/ (again only when its source changed: a `git pull`)."""
+    eng = ROOT / "engine"
+    eng.mkdir(exist_ok=True)
+    stamp = eng / "BUILD.json"
+    meta = json.loads(stamp.read_text()) if stamp.exists() else {}
+    src = source_hash(ENGINE_SOURCES)
+    if meta.get("backend") == "hip" and (eng / EXE).exists() and meta.get("src") == src:
+        ok("engine already built for this PC")
+        return eng
+    if not (shutil.which("c++") or shutil.which("g++")) or not shutil.which("git"):
+        fail("a C++ compiler and git are needed to compile the AMD engine",
+             "Ubuntu/Debian: sudo apt install build-essential git   Fedora: sudo dnf install gcc-c++ git")
+    root, dirs = rocm_root()
+    libs = [str(Path(d).parent) for d in dirs[1:]]
+    bitcode = next((p for p in (root / "lib" / "llvm" / "amdgcn" / "bitcode", root / "amdgcn" / "bitcode") if p.is_dir()),
+                   root / "amdgcn" / "bitcode")
+    os.environ.update({"HIP_PLATFORM": "amd", "HIP_COMPILER": "clang", "HIP_RUNTIME": "rocclr", "ROCM_PATH": str(root),
+                       "HIP_PATH": str(root)})
+    os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(dirs + [os.environ.get("LD_LIBRARY_PATH", "")]).rstrip(os.pathsep)
+    os.environ["PATH"] = os.pathsep.join([str(root / "bin"), str(root / "llvm" / "bin"), os.environ.get("PATH", "")])
+    say("  The engine's source changed: compiling it again (only what changed, a few minutes) ..."
+        if meta.get("backend") == "hip" and (eng / EXE).exists()
+        else "  Compiling the Strata engine for your AMD GPU (10-20 minutes, once) ...")
+    cmake_build(ROOT, ROOT / "build-hip", "strata",
+                ["-DSTRATA_ENABLE_HIP=ON", "-DSTRATA_ENABLE_CUDA=OFF", "-DSTRATA_BUILD_TESTS=OFF",
+                 "-DSTRATA_PREFILL_MMQ=ON", f"-DCMAKE_HIP_ARCHITECTURES={gpu['arch']}",
+                 f"-DCMAKE_HIP_COMPILER={root / 'llvm' / 'bin' / 'clang++'}", f"-DCMAKE_HIP_COMPILER_ROCM_ROOT={root}",
+                 "-DCMAKE_PREFIX_PATH=" + ";".join([str(root), *libs]),
+                 f"-DCMAKE_HIP_FLAGS=--rocm-path={root} --rocm-device-lib-path={bitcode}",
+                 f"-DSTRATA_GGML_DIR={llama}"], None, "")
+    shutil.copy2(ROOT / "build-hip" / EXE, eng / EXE)
+    stamp.write_text(json.dumps({"source": "local-hip", "backend": "hip", "version": source_version(),
+                                 "archs": [gpu["arch"]], "vision": "none", "lib_dirs": dirs, "src": src}, indent=1))
+    ok(f"engine compiled: {eng / EXE}")
+    return eng
+
+
 # ------------------------------------------------------------------------------------------------ the engine
 def driver_major(gpu):
     try:
@@ -636,7 +830,16 @@ def get_prebuilt(url_base, gpu, vision, updating=False) -> Path | None:
     if info.exists() and (eng / EXE).exists():
         meta = json.loads(info.read_text())
         ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:3] if x.isdigit())
-        if meta.get("source") == "local" or ver >= MIN_ENGINE:
+        if meta.get("source") == "local":              # compiled here: build_engine checks its source and cards
+            return None
+        have = [int(a) for a in meta.get("archs", [])]
+        miss = [int(x) for x in gpu.get("archs", [gpu["arch"]])
+                if have and int(x) not in have and not (meta.get("ptx") and int(x) > max(have))]
+        if miss:                                       # a card it has no code for (#128): compiled here instead
+            warn(f"the installed engine is built for {', '.join(str(a) for a in have)}; your GPU is "
+                 f"{', '.join(str(x) for x in miss)}: compiling instead")
+            return None
+        if ver >= MIN_ENGINE:
             ok("ready-made engine already installed")
             return eng
         say(f"  Updating the ready-made engine ({meta.get('version')} -> {'.'.join(map(str, MIN_ENGINE))} or newer) ...")
@@ -706,6 +909,17 @@ def update_installed_engine(url_base) -> None:
         return
     meta_text = info.read_text()
     meta = json.loads(meta_text)
+    if meta.get("backend") == "hip":                   # AMD: compiled here, again when its source changed
+        if meta.get("src") != source_hash(ENGINE_SOURCES):
+            try:
+                g = next((x for x in amd_gpus() if amd_problem(x) is None), None)
+                if g is None:
+                    raise RuntimeError("no supported AMD GPU found")
+                build_engine_hip(g, get_llama_cpp())
+            except (Exception, SystemExit) as e:
+                warn(f"could not compile the updated engine{'' if isinstance(e, SystemExit) else f' ({e})'}: "
+                     "starting the installed one")
+        return
     ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:3] if x.isdigit())
     local = meta.get("source") == "local"
     vision = meta.get("vision") or "none"
@@ -750,7 +964,8 @@ def update_installed_engine(url_base) -> None:
 def install_build_tools(gpu, yes):
     """The compiler and the CUDA toolkit, installed for the user (asks once).  Returns (nvcc, vcvars)."""
     nvcc, cuda_v = find_nvcc()
-    need_cuda = (12, 8) if max(int(x) for x in gpu.get("archs", [gpu["arch"]])) >= 120 else (12, 0)
+    # RTX 50 (sm_120): CUDA 13.0 - an engine built with 12.8 crashed in the prompt path on Linux (#220)
+    need_cuda = (13, 0) if max(int(x) for x in gpu.get("archs", [gpu["arch"]])) >= 120 else (12, 0)
     vcvars = find_vcvars() if WIN else None
     have_cc = vcvars is not None if WIN else shutil.which("g++") is not None
     missing = []
@@ -856,16 +1071,24 @@ def build_engine(gpu, vision, yes, llama) -> Path:
     want_vision = vision != "none"
     local = meta.get("source") == "local"
     src, vsrc = source_hash(ENGINE_SOURCES), source_hash(VISION_SOURCES)
-    engine_ok = local and (eng / EXE).exists() and meta.get("src") == src
+    archs = sorted({int(x) for x in gpu.get("archs", [gpu["arch"]])})    # every card the model runs on
+    built = {int(x) for x in meta.get("archs", [])}
+    # a card the engine has no code for (a GPU added with --gpus, #128) needs a compile even when the source is the
+    # same; the compile keeps the generations it was built for
+    new_arch = local and not set(archs) <= built
+    engine_ok = local and (eng / EXE).exists() and meta.get("src") == src and not new_arch
     vision_ok = not want_vision or ((eng / VEXE).exists() and (not local or meta.get("vision_src") == vsrc))
     if engine_ok and vision_ok:
         ok("engine already built for this PC")
         return eng
-    nvcc, vcvars = install_build_tools(gpu, yes)
-    archs = sorted({int(x) for x in gpu.get("archs", [gpu["arch"]])})    # every card the model runs on
+    if local:
+        archs = sorted(built | set(archs))
+    nvcc, vcvars = install_build_tools({**gpu, "archs": archs}, yes)
     cuda_archs = ";".join(str(x) for x in archs)
     if not engine_ok:
-        say("  The engine's source changed: compiling it again (only what changed, a few minutes) ..."
+        say("  Compiling the engine for " + ", ".join(f"sm_{x}" for x in archs) + " (a card it had no code for; "
+            "10-20 minutes, once) ..." if new_arch else
+            "  The engine's source changed: compiling it again (only what changed, a few minutes) ..."
             if local and (eng / EXE).exists() else "  Compiling the Strata engine for your GPU (10-20 minutes, once) ...")
         cmake_build(ROOT, ROOT / "build", "strata",
                     ["-DSTRATA_ENABLE_CUDA=ON", "-DSTRATA_BUILD_TESTS=OFF", f"-DCMAKE_CUDA_ARCHITECTURES={cuda_archs}",
@@ -895,6 +1118,26 @@ def build_engine(gpu, vision, yes, llama) -> Path:
 DATA_ITEMS = ("models", "packs", "mtp")
 
 
+LOW_RAM_HEADROOM_GB = 10   # RAM beside the experts: the OS, the engine's other buffers, the server
+
+
+def low_ram_needed(model, ram) -> bool:
+    """The model's experts do not fit this PC's RAM with room left for the rest: they are then mapped from the pack's
+    experts.bin instead of copied into RAM (the low-RAM mode)."""
+    return ram < MODELS[model]["arena_gb"] + LOW_RAM_HEADROOM_GB
+
+
+def low_ram_gpu_share(model, vram_gb) -> float:
+    """About how much of the model's experts the GPU holds (VRAM minus ~5 GB for the dense weights, KV, buffers)."""
+    return max(0.0, min(1.0, (vram_gb - 5) / MODELS[model]["arena_gb"]))
+
+
+def low_ram_fits(model, ram, vram_gb) -> bool:
+    """In the low-RAM mode: the experts the GPU does not hold fit the RAM left beside the rest (as file cache)."""
+    arena = MODELS[model]["arena_gb"]
+    return ram - 6 + max(0.0, vram_gb - 5) >= arena
+
+
 def settings_path() -> Path:
     if WIN:
         return Path(os.environ.get("APPDATA") or Path.home() / "AppData" / "Roaming") / "Strata" / "settings.json"
@@ -1017,7 +1260,18 @@ def data_folder(requested: str | None) -> tuple:
         warn(f"cannot use {dest} for the model files ({e}): keeping them in {ROOT}")
         dest = ROOT
     elsewhere = []
-    for folder in [ROOT, *other_installs(settings)]:
+    # #198: the data folder remembered before (a --data-dir to a new place) is a source too, and so is a Strata-data
+    # folder nested in any of them (an install that kept its models one level down)
+    sources = [ROOT, *other_installs(settings)]
+    if settings.get("data_dir") and Path(settings["data_dir"]) != dest:
+        sources.append(Path(settings["data_dir"]))
+    sources += [f / "Strata-data" for f in list(sources) if (f / "Strata-data") != dest]
+    seen = set()
+    for folder in sources:
+        key = os.path.normcase(str(folder))
+        if key in seen:
+            continue
+        seen.add(key)
         if folder == dest or not has_data(folder):
             continue
         if not same_drive(folder, dest):
@@ -1190,16 +1444,39 @@ def upgrade_config(cfg_path: Path, cfg: dict) -> dict:
 
 
 def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_browser=True, yes=False,
-          layer_split=None) -> int:
+          layer_split=None, keep=None) -> int:
+    """keep: settings given on this start that the model keeps from now on (--host, --api-key, --draft-vocab)."""
     cfg = upgrade_config(cfg_path, json.loads(cfg_path.read_text(encoding="utf-8-sig")))
     missing = [p for p in [cfg["exe"], *[a for a in cfg["args"] if a.endswith(".gguf")]] if not Path(p).exists()]
     if missing:
         fail(f"{cfg_path.name} refers to missing files: {missing[0]}", "run it again with --setup to repair")
+    keep = {k: v for k, v in (keep or {}).items() if v is not None}
+    if keep and any(cfg.get(k) != v for k, v in keep.items()):   # #179: a --host/--api-key on a start was ignored
+        cfg.update(keep)
+        cfg_path.write_text(json.dumps(cfg, indent=1), encoding="utf-8")
+        ok("saved for this model: " + ", ".join("api key" if k == "api_key" else f"{k.replace('_', ' ')} {v}"
+                                                for k, v in keep.items()))
     cfg_path.touch()                                     # the most recently used model
+    if "--mtp" in cfg["args"][:-1]:
+        refresh_draft_vocab(Path(cfg["args"][cfg["args"].index("--mtp") + 1]), cfg.get("draft_vocab", "cjk"))
     cmd = [sys.executable, str(ROOT / "serve" / "server.py"), "--engine", "strata", "--config", str(cfg_path),
            "--port", str(port or cfg.get("port", 8080))]
-    found = gpus()
-    if isinstance(gpu, list):                          # --gpus: saved, this model runs on these cards from now on
+    if cfg.get("backend") == "hip":                    # AMD: one card, numbered as HIP numbers them
+        if isinstance(gpu, list):
+            fail("several GPUs sharing one model: NVIDIA only for now", "use one AMD card (--gpu N)")
+        if gpu is not None:
+            cmd += ["--gpu", str(gpu)]
+        found = []
+        use = gpu if gpu is not None else cfg.get("gpu")
+        g = next((x for x in amd_gpus() if x["index"] == (use if use is not None else x["index"])
+                  and amd_problem(x) is None), None)
+        if g is not None:
+            ok(f"GPU: {g['name']} ({g['vram_gb']:.0f} GB, AMD)")
+    else:
+        found = gpus()
+    if cfg.get("backend") == "hip":
+        pass
+    elif isinstance(gpu, list):                        # --gpus: saved, this model runs on these cards from now on
         check_gpus(gpu, found)
         cfg["gpu"], cfg["gpus_asked"] = gpu, True
         cfg["layer_split"] = layer_split or cfg.get("layer_split") or "auto"
@@ -1211,14 +1488,18 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
     else:
         cfg = offer_together(cfg_path, cfg, yes)
     use = gpu if gpu is not None else cfg.get("gpu")
-    if isinstance(use, list):
+    if cfg.get("backend") == "hip":
+        pass
+    elif isinstance(use, list):
         check_gpus(use, found, "(chosen for this model) ")
         byid = {g["index"]: g for g in found}
+        cfg = ensure_engine_for([byid[i] for i in use], cfg_path, cfg, yes)
         ok("GPUs: " + " + ".join(gpu_name(byid[i]) for i in use) + f" together (layers split {cfg.get('layer_split') or 'auto'})")
     elif found:
         g = next((x for x in found if x["index"] == use), None) if use is not None else max(
             found, key=lambda x: (round(x["vram_gb"]), -x["index"]))
         if g is not None:
+            cfg = ensure_engine_for([g], cfg_path, cfg, yes)
             ok("GPU: " + gpu_name(g))
     if open_browser:
         cmd.append("--open")
@@ -1239,13 +1520,60 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
     return subprocess.call(cmd)
 
 
+# the draft subsets setup copied before (sha256): replaced by the current one, a subset made by hand is kept
+OLD_DRAFT_VOCABS = {"369151522226a5edaa5f12cfd1e2ae7db8f4fbdbd222f3dcf327dced9597fb25"}   # to 0.1.26: 27 Han tokens
+
+
+DRAFT_VOCABS = {"cjk": "draft_vocab.bin", "en": "draft_vocab_en.bin"}
+
+
+def refresh_draft_vocab(rt: Path, choice: str = "cjk") -> None:
+    """The draft layer's token subset in the MTP folder: `cjk` (data/draft_vocab.bin, since 0.1.27, #137) or `en`
+    (data/draft_vocab_en.bin, the English/code subset before it: ~110 MiB less VRAM, English answers 1-2% faster).
+    Copied when missing or when a shipped subset other than the chosen one is there; a subset made by hand is kept."""
+    new, dst = ROOT / "data" / DRAFT_VOCABS.get(choice, "draft_vocab.bin"), rt / "draft_vocab.bin"
+    if not new.exists() or not rt.is_dir():
+        return
+    if dst.exists():
+        old = hashlib.sha256(dst.read_bytes()).hexdigest()
+        shipped = OLD_DRAFT_VOCABS | {hashlib.sha256((ROOT / "data" / f).read_bytes()).hexdigest()
+                                      for f in DRAFT_VOCABS.values() if (ROOT / "data" / f).exists()}
+        if old not in shipped or old == hashlib.sha256(new.read_bytes()).hexdigest():
+            return
+        ok("draft layer: the token subset " + ("with Chinese, Japanese and Korean" if choice == "cjk" else
+                                               "for English and code (less VRAM)"))
+    shutil.copyfile(new, dst)
+
+
+def ensure_engine_for(cards, cfg_path: Path, cfg: dict, yes: bool) -> dict:
+    """The installed engine must have code for every card the model starts on: a card added later (--gpus with an
+    older or newer generation, #128) or a new GPU in the PC otherwise stops the start with 'no kernel image'.  Such a
+    card gets the engine compiled for all of them, before the start."""
+    missing = [g for g in cards if not engine_runs_on(g)]
+    if not missing:
+        return cfg
+    info = ROOT / "engine" / "BUILD.json"
+    meta = json.loads(info.read_text())
+    say()
+    say("  The installed engine has no code for " + ", ".join(f"{g['name']} (sm_{g['arch']})" for g in missing) +
+        ": it is compiled for " + ("these cards" if len(cards) > 1 else "it") + " now.")
+    main = gpu_info(cards[0]["index"])
+    archs = sorted({int(x) for x in meta.get("archs", [])} | {int(g["arch"]) for g in cards})
+    vision = meta.get("vision") or ("gpu" if (ROOT / "engine" / VEXE).exists() else "none")
+    build_engine({**main, "archs": archs}, vision, yes, get_llama_cpp())
+    dirs = json.loads(info.read_text()).get("cuda_dirs") or []
+    cfg["lib_dirs"] = dirs + [d for d in cfg.get("lib_dirs") or [] if d not in dirs]
+    cfg_path.write_text(json.dumps(cfg, indent=1), encoding="utf-8")
+    return cfg
+
+
 def write_run_script(model, cfg_path, port):
     serve = [sys.executable, str(ROOT / "serve" / "server.py"), "--engine", "strata", "--config", str(cfg_path),
              "--port", str(port), "--open"]
     if WIN:
         script = ROOT / f"run-{model.lower()}.bat"
         script.write_text("@echo off\r\ntitle Strata " + model + "\r\ncd /d \"" + str(ROOT) + "\"\r\n" +
-                          " ".join(f'"{x}"' for x in serve) + "\r\npause\r\n", encoding="utf-8")
+                          " ".join(f'"{x}"' for x in serve) + "\r\nif errorlevel 1 pause\r\n", encoding="utf-8")
     else:
         script = ROOT / f"run-{model.lower()}.sh"
         script.write_text("#!/bin/sh\ncd \"" + str(ROOT) + "\"\nexec " + " ".join(f'"{x}"' for x in serve) + "\n",
@@ -1260,9 +1588,9 @@ def main() -> int:
     ap.add_argument("--family", choices=list(FAMILIES), help="qwen = Qwen3.8-Flash-Next, swift = Swift 1.5")
     ap.add_argument("--model", choices=list(MODELS))
     ap.add_argument("--context", type=int)
-    ap.add_argument("--kv", choices=["int8", "q4_0"],
-                    help="KV cache precision above 8K context: int8 (default) or q4_0 (half the memory, a little less "
-                         "precise)")
+    ap.add_argument("--kv", choices=["int8", "q4_0", "k8v4"],
+                    help="KV cache precision above 8K context: int8 (default), q4_0 (half the memory, a little less "
+                         "precise) or k8v4 (hybrid: INT8 K + 4-bit V, 816 B/cell)")
     ap.add_argument("--vision", choices=["yes", "no", "none", "gpu", "cpu"],
                     help="let the model read images (yes = the encoder on the GPU)")
     ap.add_argument("--experimental-speed-projection", metavar="on|off|GGUF",
@@ -1292,6 +1620,15 @@ def main() -> int:
     ap.add_argument("--check", action="store_true", help="only check this PC and exit")
     ap.add_argument("--calibrate", action="store_true",
                     help="tune the engine's settings for this PC (about 5-10 minutes), then start the model")
+    ap.add_argument("--draft-vocab", choices=list(DRAFT_VOCABS),
+                    help="the draft layer's tokens: cjk = with Chinese, Japanese and Korean (default), en = English "
+                         "and code only (~110 MiB less VRAM, English answers 1-2%% faster)")
+    ap.add_argument("--low-ram", choices=["auto", "on", "off"], default="auto",
+                    help="map the model's experts from its folder instead of copying them into RAM (for a PC with a big "
+                         "GPU and little RAM); auto: when the experts would not fit the RAM")
+    ap.add_argument("--backend", choices=["cuda", "hip"],
+                    help="cuda = NVIDIA (default), hip = AMD RX 7900 XT/XTX on Linux (experimental; chosen by itself "
+                         "when the PC has no NVIDIA card Strata can use)")
     ap.add_argument("--skip-build", action="store_true", help=argparse.SUPPRESS)
     a = ap.parse_args()
     if a.gpu is not None:                              # --gpu 0,2 means --gpus 0,2 (a user tried it: issue report)
@@ -1301,7 +1638,7 @@ def main() -> int:
             a.gpu = int(a.gpu)
         else:
             ap.error(f"--gpu takes a GPU number as nvidia-smi numbers them, e.g. --gpu 1 (or --gpus 0,2), not {a.gpu!r}")
-    say("Strata - Qwen3.8-Flash-Next on a normal PC (NVIDIA GPU + system RAM + CPU)")
+    say("Strata - Qwen3.8-Flash-Next on a normal PC (a GPU + system RAM + CPU)")
     data, elsewhere = data_folder(a.data_dir)          # the model files: in the data folder, found from any copy
     roots = [data, *elsewhere]
     if a.models_dir is None:
@@ -1345,53 +1682,96 @@ def main() -> int:
                 say(f"  {i}) {json.loads(c.read_text(encoding='utf-8-sig')).get('model_name', c.stem)}")
             pick_cfg = have[int(ask("Tune which one?", [str(i) for i in range(1, len(have) + 1)], "1", a.yes)) - 1]
         calibrate_config(pick_cfg)
-        return 0 if a.no_start else start(pick_cfg, a.port, run_gpu, yes=a.yes, layer_split=a.layer_split)
+        return 0 if a.no_start else start(pick_cfg, a.port, run_gpu, yes=a.yes, layer_split=a.layer_split,
+                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab})
     if have and not (a.setup or a.model or a.family or a.check or a.no_start):
         if not a.build:
             update_installed_engine(a.prebuilt)
         if len(have) == 1:
-            return start(have[0], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split)
+            return start(have[0], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split,
+                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab})
         say()
         for i, c in enumerate(have, 1):
             say(f"  {i}) {json.loads(c.read_text(encoding='utf-8-sig')).get('model_name', c.stem)}")
         say(f"  {len(have) + 1}) install another model / change settings")
         pick = int(ask("Which one?", [str(i) for i in range(1, len(have) + 2)], "1", a.yes))
         if pick <= len(have):
-            return start(have[pick - 1], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split)
+            return start(have[pick - 1], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split,
+                     keep={"host": a.host, "api_key": a.api_key, "draft_vocab": a.draft_vocab})
 
     # ---- 1. the PC
     step(1, "checking your PC")
     found = gpus()
-    if not found:
-        fail("no NVIDIA GPU found (nvidia-smi did not answer)",
-             "install the NVIDIA driver from https://www.nvidia.com/drivers and restart the PC")
-    if len(found) > 1 or gpu_problem(found[0]) is not None:
-        gpu_table(found)
-    sel = choose_gpus(a, found)                        # asked when two or more cards can share the model
-    multi = sel if len(sel) > 1 else []
-    a.gpu = sel[0]                                     # the main GPU: the checks and the sizing below are its
-    GPU_PICK = a.gpu
-    gpu = gpu_info(a.gpu)
-    chosen = [gpu_info(i) for i in sel]
-    gpu["archs"] = sorted({x["arch"] for x in chosen})  # the engine needs code for every one of them
-    if multi:
-        ok("GPUs: " + " + ".join(gpu_name(x) for x in chosen) + " together (the model's layers are split across them)")
-    ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, compute capability {cc(gpu)}, driver {gpu['driver']}")
-    if driver_major(gpu) < MIN_DRIVER:
-        fail(f"the NVIDIA driver is too old ({gpu['driver']}; {MIN_DRIVER} or newer is needed)",
-             "update it with the NVIDIA App or from https://www.nvidia.com/drivers, restart, and run this again")
+    amd = [] if WIN else amd_gpus()
+    nv_ok = any(gpu_problem(g) is None for g in found)
+    amd_ok = [g for g in amd if amd_problem(g) is None]
+    hip = a.backend == "hip" or (a.backend is None and not nv_ok and bool(amd_ok))
+    if a.backend is None and nv_ok and amd_ok:
+        # both kinds of card: asked (a first run on such a PC used to take NVIDIA without mentioning the Radeon)
+        say()
+        say("  This PC has NVIDIA and AMD cards Strata can use:")
+        say("  1) NVIDIA: " + ", ".join(f"{g['name']} ({g['vram_gb']:.0f} GB)" for g in found if gpu_problem(g) is None)
+            + "   (recommended)")
+        say("  2) AMD: " + ", ".join(f"{g['name']} ({g['vram_gb']:.0f} GB)" for g in amd_ok)
+            + "   (experimental: compiled here, one GPU, no images - docs/AMD_HIP.md)")
+        hip = ask("Which cards?", ["1", "2"], "1", a.yes or a.check) == "2"
+        if a.check and not hip:
+            say("  (the AMD card: ./setup.sh --backend hip)")
+    if hip:                                            # AMD (experimental): one card, compiled here
+        if WIN:
+            fail("Strata's AMD backend runs on Linux only", "use an NVIDIA RTX 20 series or newer card on Windows")
+        say("  Your AMD GPUs:" if amd else "  No AMD GPU found (the amdgpu driver's KFD topology is empty).")
+        for g in amd:
+            say(f"    GPU {g['index']}: {g['name']}, {g['vram_gb']:.0f} GB VRAM - " + (amd_problem(g) or "can be used"))
+        usable = [g for g in amd if amd_problem(g) is None]
+        if not usable:
+            fail("no AMD GPU Strata can use", "the AMD backend runs on the RX 7900 XT / XTX (gfx1100) on Linux")
+        if a.gpus:
+            fail("several GPUs sharing one model: NVIDIA only for now", "use one AMD card (--gpu N)")
+        if a.gpu is not None:
+            gpu = next((g for g in usable if g["index"] == a.gpu), None)
+            if gpu is None:
+                fail(f"AMD GPU {a.gpu} cannot be used", "use one of: " + ", ".join(f"--gpu {g['index']}" for g in usable))
+        else:
+            gpu = max(usable, key=lambda x: (round(x["vram_gb"]), -x["index"]))
+        gpu = {**gpu, "count": len(amd), "archs": [gpu["arch"]]}
+        sel, multi, chosen = [gpu["index"]], [], [gpu]
+        a.gpu = gpu["index"] if len(amd) > 1 else a.gpu
+        ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, {gpu['arch']} (AMD, experimental: docs/AMD_HIP.md)")
+    else:
+        if not found:
+            fail("no NVIDIA GPU found (nvidia-smi did not answer)",
+                 "install the NVIDIA driver from https://www.nvidia.com/drivers and restart the PC"
+                 + ("; an AMD RX 7900 XT/XTX: --backend hip" if amd else ""))
+        if len(found) > 1 or gpu_problem(found[0]) is not None:
+            gpu_table(found)
+        sel = choose_gpus(a, found)                    # asked when two or more cards can share the model
+        multi = sel if len(sel) > 1 else []
+        a.gpu = sel[0]                                 # the main GPU: the checks and the sizing below are its
+        GPU_PICK = a.gpu
+        gpu = gpu_info(a.gpu)
+        chosen = [gpu_info(i) for i in sel]
+        gpu["archs"] = sorted({x["arch"] for x in chosen})  # the engine needs code for every one of them
+        if multi:
+            ok("GPUs: " + " + ".join(gpu_name(x) for x in chosen) + " together (the model's layers are split across them)")
+        ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, compute capability {cc(gpu)}, driver {gpu['driver']}")
+        if driver_major(gpu) < MIN_DRIVER:
+            fail(f"the NVIDIA driver is too old ({gpu['driver']}; {MIN_DRIVER} or newer is needed)",
+                 "update it with the NVIDIA App or from https://www.nvidia.com/drivers, restart, and run this again")
     if gpu["vram_gb"] < 11:
         warn("less than 12 GB of VRAM: Strata will run, but most experts stay on the CPU and it will be slow")
     ram = ram_gb()
     cpu, avx2, avx512 = cpu_info()
     need = min(d["ram_gb"] for d in MODELS.values())
-    if ram < need - 4 and not a.check:
+    low_ok = low_ram_fits("IQ1_M", ram, gpu["vram_gb"]) and a.low_ram != "off"   # the smallest model, mapped
+    if ram < need - 4 and not a.check and not low_ok:
         # every model keeps ALL its experts in RAM (23+ GB); VRAM only holds a copy of the most-used ones, so a
         # bigger GPU does not lower this
         fail(f"RAM: {ram:.0f} GB - the smallest model (the Coder) needs about {need} GB",
              "Strata keeps all of the model's experts in RAM (23-50 GB, whatever the GPU) and the GPU holds a copy "
              "of the most-used ones: it needs 32 GB of RAM or more (48 GB for the full model)")
-    ok(f"RAM: {ram:.0f} GB" if ram >= need - 4 else f"RAM: {ram:.0f} GB (less than the {need} GB the smallest model needs)")
+    ok(f"RAM: {ram:.0f} GB" if ram >= need - 4 else f"RAM: {ram:.0f} GB (less than the {need} GB the smallest model needs)"
+       + ("; the GPU's VRAM makes up for it (the low-RAM mode)" if ram < need - 4 and low_ok else ""))
     pf = page_file_gb()
     if pf is not None and pf < 4:
         warn(f"Windows' page file is {pf:.1f} GB: the graphics card's memory needs room there too (issue #60), so "
@@ -1404,6 +1784,9 @@ def main() -> int:
         say()
         for m, d in MODELS.items():
             verdict = "fits" if ram >= d["ram_gb"] else "tight" if ram >= d["ram_gb"] - 8 else "does not fit"
+            if low_ram_needed(m, ram) and low_ram_fits(m, ram, gpu["vram_gb"]) and a.low_ram != "off":
+                verdict = (f"fits in the low-RAM mode (the GPU holds ~{100 * low_ram_gpu_share(m, gpu['vram_gb']):.0f}% "
+                           "of its experts)")
             say(f"  {m:8s} needs ~{d['ram_gb']} GB RAM: {verdict}")
         say("\nThis PC can run Strata. Run it again without --check to install.")
         return 0
@@ -1429,12 +1812,34 @@ def main() -> int:
     for i, m in enumerate(names, 1):
         d = MODELS[m]
         fit = "" if ram >= d["ram_gb"] else f"   <- needs {d['ram_gb']} GB RAM, you have {ram:.0f}"
+        if low_ram_needed(m, ram) and low_ram_fits(m, ram, gpu["vram_gb"]) and a.low_ram != "off":
+            fit = f"   <- fits in the low-RAM mode (the GPU holds ~{100 * low_ram_gpu_share(m, gpu['vram_gb']):.0f}%)"
         say(f"  {i}) {m:8s} {d['about']}; download {d['download_gb']:.0f} GB, uses ~{d['arena_gb']:.0f} GB of RAM{fit}")
     rec = str(names.index("IQ3_XXS") + 1) if ram >= 60 and "IQ3_XXS" in names else "1"
     model = a.model or names[int(ask("Which size?", [str(i) for i in range(1, len(names) + 1)], rec, a.yes)) - 1]
-    if ram < MODELS[model]["ram_gb"] - 4:
-        fail(f"{model} needs about {MODELS[model]['ram_gb']} GB of RAM; this PC has {ram:.0f} GB",
-             "choose Q2_0 or IQ2_XS, or add RAM")
+    low_ram = a.low_ram == "on" or (a.low_ram == "auto" and low_ram_needed(model, ram))
+    if low_ram and multi:
+        warn("the low-RAM mode runs on one GPU: using " + gpu_name(gpu) + " only")
+        multi, sel, chosen = [], [gpu["index"]], [gpu]
+    if low_ram:
+        share = low_ram_gpu_share(model, gpu["vram_gb"])
+        ok(f"low-RAM mode: {model}'s experts ({MODELS[model]['arena_gb']:.0f} GB) are read from the model folder "
+           f"through the OS file cache instead of a copy in RAM ({ram:.0f} GB); the GPU holds ~{100 * share:.0f}% of them")
+        if share < 0.6:
+            warn("most of the experts are read from the SSD while it answers: expect it to be much slower than with "
+                 "enough RAM (a faster SSD and a smaller size help)")
+    elif ram < MODELS[model]["ram_gb"] - 4:
+        # #125: a warning and a question, not a stop: the user may accept paging (asked, "no" by default, so an
+        # unattended --yes install still stops here)
+        need_gb, arena = MODELS[model]["ram_gb"], MODELS[model]["arena_gb"]
+        warn(f"{model} needs about {need_gb} GB of RAM and this PC has {ram:.0f} GB: its experts alone are "
+             f"{arena:.0f} GB and must stay in RAM, so Windows/Linux will page part of them from disk. Expect it "
+             "to be much slower, and it may not start at all.")
+        say("       A smaller size (Q2_0 or IQ2_XS) fits; more RAM fixes it.")
+        if ask("  Install it anyway?", ["y", "n"], "n", a.yes) != "y":
+            fail(f"{model} needs about {need_gb} GB of RAM; this PC has {ram:.0f} GB",
+                 "choose Q2_0 or IQ2_XS, or add RAM")
+        warn(f"installing {model} with {ram:.0f} GB of RAM, as you chose")
     ok(f"size: {model}")
     tag = fam["tag"] + model                           # names of the pack, config and start script
     small = min(x["vram_gb"] for x in chosen)         # each card keeps its layers' KV of the whole context
@@ -1464,7 +1869,11 @@ def main() -> int:
         kv = ["int8", "q4_0"][int(ask("KV cache?", ["1", "2"], "1", a.yes)) - 1]
     if ctx > 8192:
         ok(f"KV cache: {'8-bit' if kv == 'int8' else '4-bit (Hadamard-rotated)'}")
-    if a.vision:
+    if hip:
+        vision = "none"
+        if a.vision not in (None, "no", "none"):
+            warn("images are not available with the AMD backend yet: off")
+    elif a.vision:
         vision = {"yes": "gpu", "no": "none"}.get(a.vision, a.vision)
     else:
         say()
@@ -1500,9 +1909,13 @@ def main() -> int:
                 models_dir, shards = cand[0].parent, cand
                 ok(f"model files found in {models_dir}")
                 break
+    for s in shards:                                   # #173: a whole file copied in by hand has no finish mark
+        if s.exists() and not done(s) and whole_shard(s):
+            mark(s, "whole (checked against its own tensor directory)")
     have_model = all(s.exists() and (done(s) or a.gguf_dir) for s in shards)
     need = (0 if a.gguf_dir or have_model else MODELS[model]["download_gb"]) + 8 + \
-        (40 if model == "Q2_0" and avx512 and family == "qwen" else 0) + (1 if vision != "none" else 0)
+        (40 if model == "Q2_0" and avx512 and family == "qwen" else 0) + (1 if vision != "none" else 0) + \
+        (MODELS[model]["arena_gb"] + 1 if low_ram and not (model == "Q2_0" and avx512 and family == "qwen") else 0)
     if free_gb(models_dir) < need:
         fail(f"not enough free disk space in {models_dir}: need ~{need:.0f} GB", "use --models-dir on a bigger drive")
 
@@ -1514,16 +1927,22 @@ def main() -> int:
     step(4, "the Strata engine")
     llama = get_llama_cpp()
     ok(f"llama.cpp {LLAMA_CPP_COMMIT[:7]} (gguf-py, ggml, mtmd)")
-    eng = None if a.build else get_prebuilt(a.prebuilt, gpu, vision)
+    eng = None if a.build or hip else get_prebuilt(a.prebuilt, gpu, vision)
     if eng is not None and json.loads((eng / "BUILD.json").read_text()).get("source") != "local":
         pip_install(CUDA_WHEELS, "NVIDIA CUDA libraries (cuBLAS, CUDA runtime; ~0.4 GB)")
         if vision != "none" and not (eng / VEXE).exists():
             warn("the ready-made engine has no image encoder: compiling it")
             eng = None
+        elif vision == "gpu":                          # the encoder can cover fewer cards than the engine (RTX 20)
+            m = json.loads((eng / "BUILD.json").read_text())
+            va = [int(x) for x in m.get("vision_archs", m.get("archs", []))]
+            if va and int(gpu["arch"]) not in va and not (m.get("ptx") and int(gpu["arch"]) > max(va)):
+                warn(f"the ready-made image encoder has no code for your GPU (sm_{gpu['arch']}): compiling it")
+                eng = None
     if eng is None:
-        eng = build_engine(gpu, vision, a.yes, llama)
+        eng = build_engine_hip(gpu, llama) if hip else build_engine(gpu, vision, a.yes, llama)
     meta = json.loads((eng / "BUILD.json").read_text())
-    lib_dirs = meta.get("cuda_dirs") or cuda_lib_dirs()
+    lib_dirs = meta.get("lib_dirs") or meta.get("cuda_dirs") or cuda_lib_dirs()
     ok(f"engine: {eng / EXE}")
 
     # ---- 5. the model files
@@ -1544,9 +1963,7 @@ def main() -> int:
                 except OSError:
                     pass
             download(fam["hf"].format(q=model) + s.name, s)
-    for s in shards:
-        if not s.exists():
-            fail(f"missing {s}")
+    check_shards(shards)
     ok("model files present")
     mmproj = Path(a.models_dir) / fam["mmproj"]
     if not mmproj.exists():
@@ -1575,6 +1992,10 @@ def main() -> int:
     elif not (pack / "native_experts.txt").exists() or not (pack / "tokenizer" / "vocab.json").exists():
         # every tensor as the GGUF stores it; the experts are read from the GGUF at start (seconds to build)
         run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(shards[0]), "--out", str(pack)], env=env)
+    if low_ram and not (pack / "experts.bin").exists():
+        say(f"  Writing the experts into one file for the low-RAM mode (one time, {MODELS[model]['arena_gb']:.0f} GB) ...")
+        run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(shards[0]), "--out", str(pack),
+             "--experts-bin"], env=env)
     ok(f"model prepared: {pack}")
     mtp = (find_in(roots, "mtp/rt/experts.bin") or data / "mtp/rt/experts.bin").parent.parent
     rt = mtp / "rt"
@@ -1586,8 +2007,7 @@ def main() -> int:
              "--out", str(mtp / "mtp-q2_0.gguf")], env=env)
         run([sys.executable, str(ROOT / "tools" / "mtp_rt.py"), "--gguf", str(mtp / "mtp-q2_0.gguf"), "--out", str(rt)],
             env=env)
-    if not (rt / "draft_vocab.bin").exists():
-        shutil.copyfile(ROOT / "data" / "draft_vocab.bin", rt / "draft_vocab.bin")
+    refresh_draft_vocab(rt, a.draft_vocab or "cjk")
     ok(f"MTP draft layer: {rt}")
 
     # ---- 7. the start script
@@ -1603,12 +2023,19 @@ def main() -> int:
             "--max-context", str(ctx)]
     if ctx > 8192:
         args += ["--kv", kv]
+    if low_ram:
+        args += ["--mmap-experts"]   # the experts from the pack's experts.bin, not copied into RAM
     # KV streaming: from 64K up the whole KV cache lives in RAM and only the part the attention reads (32K positions
     # per layer) stays in VRAM; the VRAM it frees holds more experts (+6% at 128K, +23% at 262K with Q2_0). It
     # costs ~13.7 KB of RAM per context token with 8-bit KV (1.7 GB at 128K), 7.5 KB with 4-bit, so only when it fits.
     kv_ram_gb = ctx * (13 * (576 if kv == "q4_0" else 1056)) / 1e9   # 12 QSA layers + the draft layer
-    # the RAM copy must be pinned, and under WSL the NVIDIA driver pins only about 1 GB in all
-    if is_wsl() and ctx >= 65536:
+    # Hybrid K8V4 never streams its KV (mode 0 only, layer.hpp), so it is excluded from the WHOLE streaming
+    # decision rather than one threshold at a time - a future tier added to this chain cannot reintroduce the
+    # combination the engine refuses (PR review).
+    if kv == "k8v4":
+        if ctx >= 65536:
+            ok("KV streaming off: not supported with --kv k8v4; the KV cache stays in VRAM")
+    elif is_wsl() and ctx >= 65536:
         ok("WSL: KV streaming off (the driver pins only about 1 GB of RAM); the KV cache stays in VRAM")
     elif ctx >= 65536 and ram >= MODELS[model]["ram_gb"] + kv_ram_gb + 1:
         args += ["--kv-resident", "32768"]
@@ -1622,6 +2049,13 @@ def main() -> int:
     cfg = {"exe": str(eng / EXE), "args": args, "cwd": str(ROOT), "tokenizer": str(pack / "tokenizer"),
            "model_name": f"{fam['name']}-{model.lower()}", "log": str(ROOT / f"strata-{tag.lower()}.log"),
            "lib_dirs": lib_dirs, "port": port}
+    if hip:
+        cfg["backend"] = "hip"
+        # the dense prompt GEMMs through hipBLASLt with kernels measured on this GPU generation (tools/hip; +40-60%
+        # prompt speed on the 7900 XTX); the engine refuses a table made for another hipBLASLt version and falls back
+        tables = sorted((ROOT / "tools" / "hip").glob(f"{gpu['arch']}-hipblaslt-*.txt"))
+        if tables:
+            cfg["env"] = {"STRATA_HIPBLASLT_TUNING": str(tables[-1])}
     if gpu["count"] > 1 or a.gpu is not None:
         cfg["gpu"] = gpu["index"]                      # the engine is told this card (issue #51)
         cfg["gpus_asked"] = True                       # chosen at setup: not asked again at start
@@ -1633,13 +2067,15 @@ def main() -> int:
         cfg["host"] = a.host
     if a.api_key:
         cfg["api_key"] = a.api_key
+    if a.draft_vocab:
+        cfg["draft_vocab"] = a.draft_vocab
     if vision != "none":
         cfg["vision"] = {"exe": str(eng / VEXE), "mmproj": str(mmproj), "model": str(shards[0]),
                          "gpu": vision == "gpu", "max_tokens": VISION[vision]["max_tokens"]}
         if vision == "cpu":
             cfg["vision"]["threads"] = max(1, (os.cpu_count() or 8) // 2)
     cfg_path = ROOT / f"strata-{tag.lower()}.json"
-    cal = saved_calibration(cfg)
+    cal = None if hip else saved_calibration(cfg)     # tools/calibrate.py is NVIDIA-only for now
     if cal is not None:
         sys.path.insert(0, str(ROOT / "tools"))
         import calibrate as CAL
@@ -1648,7 +2084,7 @@ def main() -> int:
     cfg_path.write_text(json.dumps(cfg, indent=1), encoding="utf-8")
     script = write_run_script(tag, cfg_path, port)
     # offered only when someone answers: --yes installs and adopted earlier installs are not held up by it
-    if cal is None and not a.no_start and not a.yes and ask(
+    if cal is None and not hip and not a.no_start and not a.yes and ask(
             "Tune Strata for this PC now? It measures a few engine settings (about 5-10 minutes; the PC is busy "
             "meanwhile; later: START-HERE --calibrate)", ["y", "n"], "y", a.yes) == "y":
         calibrate_config(cfg_path)

setup.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 # Strata for Linux: the first run installs everything and starts the model; later runs just start it.
-# Needs only an NVIDIA driver. Python (with venv) is installed through apt/dnf if it is missing (asks for sudo).
+# Needs only an NVIDIA driver (or, experimental: an AMD RX 7900 XT/XTX with the amdgpu driver, see docs/AMD_HIP.md).
+# Python (with venv) is installed through apt/dnf if it is missing (asks for sudo).
 cd "$(dirname "$0")" || exit 1
 # Python 3.10+ that can make a venv WITH pip: Debian/Ubuntu ship `venv` without `ensurepip` (that is the separate
 # python3-venv package), and a venv made without it has no pip

src/artifact/gguf_reader_test.cpp

@@ -0,0 +1,93 @@
+// src/artifact/gguf_reader_test.cpp - the reader refuses what it cannot arbitrate (synthetic GGUF, no model,
+// no GPU).
+//
+//   1. two tensors with distinct names: the file opens and find() sees both;
+//   2. the same name twice: refused at open, and the error names the tensor and the file - GGUF has no
+//      index to say which of the two a name means, and find() is first-match, so the alternative is a
+//      lookup that silently picks by position.
+//
+// The fixture is a minimal GGUF v3 written here (header, no metadata, two F32[8] tensors, 32-byte
+// alignment), so the test needs neither gguf-py nor a shard.  The file is closed before it is removed:
+// on Windows an open mapping keeps it.
+#include "strata/artifact/gguf_reader.hpp"
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+namespace {
+int g_fail = 0;
+void check(bool ok, const char* what) {
+    std::printf("  %-66s %s\n", what, ok ? "ok" : "FAIL");
+    if (!ok) ++g_fail;
+}
+
+template <class T> void put(std::vector<uint8_t>& b, T v) {
+    const auto n = b.size();
+    b.resize(n + sizeof(T));
+    std::memcpy(b.data() + n, &v, sizeof(T));
+}
+void put_str(std::vector<uint8_t>& b, const std::string& s) {
+    put<uint64_t>(b, s.size());
+    b.insert(b.end(), s.begin(), s.end());
+}
+
+// A GGUF v3 file of F32[8] tensors named `names`, laid out 32 bytes apart from a 32-byte-aligned data start.
+std::filesystem::path write_gguf(const std::vector<std::string>& names) {
+    std::vector<uint8_t> b;
+    put<uint32_t>(b, 0x46554747u);   // "GGUF"
+    put<uint32_t>(b, 3);
+    put<uint64_t>(b, names.size());  // n_tensors
+    put<uint64_t>(b, 0);             // n_kv
+    for (size_t i = 0; i < names.size(); ++i) {
+        put_str(b, names[i]);
+        put<uint32_t>(b, 1);         // n_dims
+        put<uint64_t>(b, 8);
+        put<uint32_t>(b, 0);         // F32
+        put<uint64_t>(b, 32 * i);    // offset from data_start
+    }
+    b.resize((b.size() + 31) / 32 * 32 + 32 * names.size(), 0);
+    const auto path = std::filesystem::temp_directory_path() / "strata_gguf_reader_test.gguf";
+    std::ofstream(path, std::ios::binary).write(reinterpret_cast<const char*>(b.data()), (std::streamsize)b.size());
+    return path;
+}
+}  // namespace
+
+int main() {
+    std::printf("gguf_reader_test\n");
+    {
+        const auto path = write_gguf({"blk.0.attn_q.weight", "blk.0.attn_k.weight"});
+        std::string err;
+        size_t n = 0;
+        bool both = false;
+        try {
+            strata::GgufFile g(path.string());
+            n = g.tensors().size();
+            both = g.find("blk.0.attn_q.weight") && g.find("blk.0.attn_k.weight");
+        } catch (const std::exception& e) {
+            err = e.what();
+        }
+        std::filesystem::remove(path);
+        check(err.empty() && n == 2 && both, "distinct names: the file opens and find() sees both tensors");
+    }
+    {
+        const auto path = write_gguf({"blk.0.attn_q.weight", "blk.0.attn_q.weight"});
+        std::string err;
+        try {
+            strata::GgufFile g(path.string());
+        } catch (const std::exception& e) {
+            err = e.what();
+        }
+        std::filesystem::remove(path);
+        check(!err.empty(), "the same name twice: refused at open");
+        check(err.find("blk.0.attn_q.weight") != std::string::npos, "  the error names the tensor");
+        check(err.find(path.filename().string()) != std::string::npos, "  the error names the file");
+        if (!err.empty()) std::printf("  (%s)\n", err.c_str());
+    }
+    std::printf(g_fail ? "gguf_reader_test: %d FAILED\n" : "gguf_reader_test: all passed\n", g_fail);
+    return g_fail ? 1 : 0;
+}

src/core/device.cu

@@ -30,7 +30,11 @@ DeviceInfo device_info(int ordinal) {
     int count = 0;
     check(cudaGetDeviceCount(&count), "cudaGetDeviceCount");
     if (count == 0) {
-        throw CudaError("no CUDA device is present; Strata targets sm_120 (RTX 5000 series)", -1);
+#if defined(STRATA_USE_HIP)
+        throw CudaError("no HIP device is present; this backend targets gfx1100 wave32", -1);
+#else
+        throw CudaError("no CUDA device is present; Strata needs an NVIDIA GPU (RTX 20 series or newer)", -1);
+#endif
     }
     if (ordinal < 0 || ordinal >= count) {
         throw CudaError("device ordinal " + std::to_string(ordinal) + " is out of range (have " +
@@ -56,15 +60,23 @@ DeviceInfo device_info(int ordinal) {
     check(cudaDriverGetVersion(&d.driver_version), "cudaDriverGetVersion");
     check(cudaRuntimeGetVersion(&d.runtime_version), "cudaRuntimeGetVersion");
 
-    // The engine is written against sm_120.  Compiling for it is enforced by CMake; RUNNING on something else
-    // is caught here, because a binary can be carried to a machine with an older card and would otherwise
-    // silently take whatever path the driver chose.
-    if (d.cc_major != 12) {
+    // The engine supports compute capability 7.5 and newer (Turing: the QSA scorer's tf32 mma has a portable
+    // fp32-FMA fallback below sm_80, the tensor-core prompt kernels refuse and fall back).  Compiling for a
+    // supported arch is enforced by CMake; RUNNING on an older card is caught here, because a binary can be carried
+    // to a machine with an older card and would otherwise silently take whatever path the driver chose.  The HIP
+    // backend is validated on gfx1100 (wave32) only.
+#if defined(STRATA_USE_HIP)
+    if (std::strncmp(p.gcnArchName, "gfx1100", 7) != 0 || p.warpSize != 32) {
+        throw CudaError("HIP backend requires validated gfx1100 wave32 hardware", -1);
+    }
+#else
+    if (d.cc_major * 10 + d.cc_minor < 75) {
         throw CudaError("device " + d.name + " reports compute capability " + std::to_string(d.cc_major) +
                             "." + std::to_string(d.cc_minor) +
-                            "; Strata targets sm_120 (RTX 5000 series / Blackwell) only",
+                            "; Strata needs compute capability 7.5 or newer (RTX 20 / 30 / 40 / 50 series)",
                         -1);
     }
+#endif
     return d;
 }
 

src/core/device_main.cpp

@@ -33,7 +33,11 @@ int main(int argc, char** argv) {
     try {
         const strata::core::DeviceInfo d = strata::core::device_info(0);
         std::printf("device %d: %s\n", d.ordinal, d.name.c_str());
+#if defined(STRATA_USE_HIP)
+        std::printf("  HIP target          gfx1100 wave32\n");
+#else
         std::printf("  compute capability  %d.%d   (sm_%d%d)\n", d.cc_major, d.cc_minor, d.cc_major, d.cc_minor);
+#endif
         std::printf("  multiprocessors     %d\n", d.multi_processor_count);
         std::printf("  VRAM total / free   %s / %s\n", human(d.total_bytes).c_str(), human(d.free_bytes).c_str());
         std::printf("  driver / runtime    %d / %d\n", d.driver_version, d.runtime_version);

src/core/expert_cache.cpp

@@ -69,6 +69,22 @@ bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_ex
 
 ExpertCache::~ExpertCache() { close(); }
 
+#if defined(STRATA_USE_HIP)
+bool ExpertCache::ensure_blocking_staging(std::size_t bytes, std::string& err) {
+    if (bytes <= blocking_staging_bytes_) return true;
+    void* next = nullptr;
+    const cudaError_t status = cudaHostAlloc(&next, bytes, cudaHostAllocDefault);
+    if (status != cudaSuccess) {
+        err = std::string("ExpertCache: HIP blocking staging allocation: ") + cudaGetErrorString(status);
+        return false;
+    }
+    if (blocking_staging_) (void) cudaFreeHost(blocking_staging_);
+    blocking_staging_ = static_cast<uint8_t*>(next);
+    blocking_staging_bytes_ = bytes;
+    return true;
+}
+#endif
+
 bool ExpertCache::open(int64_t n_slots, int64_t n_layers, int64_t n_expert, int64_t blob_bytes,
                        std::string& err) {
     close();
@@ -125,6 +141,12 @@ bool ExpertCache::open(int64_t n_slots, int64_t n_layers, int64_t n_expert, int6
     n_layers_ = n_layers;
     n_expert_ = n_expert;
     blob_ = blob_bytes;
+#if defined(STRATA_USE_HIP)
+    if (!ensure_blocking_staging((std::size_t) blob_, err)) {
+        close();
+        return false;
+    }
+#endif
     next_free_ = 0;
     fills_ = 0;
     admitted_ = 0;
@@ -154,11 +176,22 @@ bool ExpertCache::open_sized(const std::vector<int64_t>& slot_bytes, int64_t n_l
     slots_ = (int64_t) slot_bytes.size();
     blob_ = mx;
     off_ = std::move(off);
+#if defined(STRATA_USE_HIP)
+    if (!ensure_blocking_staging((std::size_t) blob_, err)) {
+        close();
+        return false;
+    }
+#endif
     layer_next_.assign((size_t) (n_layers > 0 ? n_layers : 0), 0);
     return true;
 }
 
 void ExpertCache::close() {
+#if defined(STRATA_USE_HIP)
+    if (blocking_staging_) (void) cudaFreeHost(blocking_staging_);
+    blocking_staging_ = nullptr;
+    blocking_staging_bytes_ = 0;
+#endif
     off_.clear();
     if (base_ != nullptr) {
         cudaFree(base_);
@@ -256,7 +289,18 @@ bool ExpertCache::fill_slot_blocking(int32_t slot, const uint8_t* host_blob, std
         err = "ExpertCache::fill_slot_blocking: the host blob is null";
         return false;
     }
+#if defined(STRATA_USE_HIP)
+    // Bound HIP's pageable-source staging to one expert instead of repeatedly
+    // registering regions of the mmap. The blocking copy completes before reuse.
+    if (!blocking_staging_ || n > blocking_staging_bytes_) {
+        err = "ExpertCache::fill_slot_blocking: HIP staging buffer is too small";
+        return false;
+    }
+    std::memcpy(blocking_staging_, host_blob, n);
+    const cudaError_t e = cudaMemcpy(dst, blocking_staging_, n, cudaMemcpyHostToDevice);
+#else
     const cudaError_t e = cudaMemcpy(dst, host_blob, n, cudaMemcpyHostToDevice);
+#endif
     if (e != cudaSuccess) {
         err = std::string("ExpertCache::fill_slot_blocking: ") + cudaGetErrorString(e);
         return false;

src/core/expert_source.cpp

@@ -17,11 +17,18 @@
 #include <cstdlib>
 #include <cstring>
 #include <fstream>
+#include <filesystem>
+#include <sstream>
+#include <limits>
 #include <thread>
+#include <utility>
 #include <vector>
 
 #if defined(_WIN32)
 #define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX   // std::numeric_limits<T>::max() below
+#endif
 #include <windows.h>
 #else
 #include <fcntl.h>
@@ -32,6 +39,213 @@
 
 namespace strata::core {
 
+namespace detail {
+
+bool cgroup_available_bytes(uint64_t limit, const CgroupMemoryStat& stat, uint64_t& bytes) {
+    bytes = 0;
+    if (!stat.valid) return false;
+
+    // memory.stat's inactive_file can race memory.current, so bound it to charged usage first.
+    uint64_t reclaimable = std::min(stat.inactive_file, stat.current);
+    reclaimable = stat.file_dirty >= reclaimable ? 0 : reclaimable - stat.file_dirty;
+    reclaimable = stat.file_writeback >= reclaimable ? 0 : reclaimable - stat.file_writeback;
+
+    // Reclaiming clean file pages reduces usage; saturating subtraction also handles a transient over-limit read.
+    const uint64_t usage_after_reclaim = stat.current - reclaimable;
+    bytes = usage_after_reclaim < limit ? limit - usage_after_reclaim : 0;
+    return true;
+}
+
+bool make_cache_complement_plan(
+    int64_t n_layers, int64_t n_expert, const std::vector<uint64_t>& layer_blob_bytes,
+    const std::vector<std::pair<int32_t, int32_t>>& primary_gpu_pairs,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs,
+    std::vector<uint64_t>& offsets, uint64_t& bytes, std::string& err) {
+    offsets.clear();
+    bytes = 0;
+    err.clear();
+    if (n_layers <= 0 || n_expert <= 0 || layer_blob_bytes.size() != (size_t) n_layers) {
+        err = "FileExpertSource: invalid geometry for the cache complement plan";
+        return false;
+    }
+    if ((uint64_t) n_layers > (uint64_t) std::numeric_limits<size_t>::max() / (uint64_t) n_expert) {
+        err = "FileExpertSource: cache complement index table is too large";
+        return false;
+    }
+    const size_t count = (size_t) n_layers * (size_t) n_expert;
+    std::vector<uint8_t> omitted(count, 0);
+    auto mark_pairs = [&](const std::vector<std::pair<int32_t, int32_t>>& pairs, uint8_t bit,
+                          const char* label) -> bool {
+        for (const auto& pair : pairs) {
+            if (pair.first < 0 || pair.second < 0 || pair.first >= n_layers || pair.second >= n_expert) {
+                err = std::string("FileExpertSource: ") + label + " pair is outside the expert geometry";
+                return false;
+            }
+            const size_t index = (size_t) pair.first * (size_t) n_expert + (size_t) pair.second;
+            if ((omitted[index] & bit) != 0) {
+                err = std::string("FileExpertSource: duplicate ") + label + " pair in the cache complement plan";
+                return false;
+            }
+            if (bit == 2 && (omitted[index] & 1) != 0) {
+                err = "FileExpertSource: the primary and additional GPU expert tiers overlap";
+                return false;
+            }
+            omitted[index] |= bit;
+        }
+        return true;
+    };
+    if (!mark_pairs(primary_gpu_pairs, 1, "primary GPU") ||
+        !mark_pairs(additional_gpu_pairs, 2, "additional GPU")) return false;
+    for (uint64_t blob_bytes : layer_blob_bytes) {
+        if (blob_bytes == 0) {
+            err = "FileExpertSource: cache complement layer has zero-sized expert blobs";
+            return false;
+        }
+    }
+
+    offsets.assign(count, kNoCacheComplement);
+    for (int64_t layer = 0; layer < n_layers; ++layer) {
+        const uint64_t blob_bytes = layer_blob_bytes[(size_t) layer];
+        for (int64_t expert = 0; expert < n_expert; ++expert) {
+            const size_t index = (size_t) layer * (size_t) n_expert + (size_t) expert;
+            if (omitted[index] != 0) continue;
+            if (bytes > std::numeric_limits<uint64_t>::max() - blob_bytes) {
+                offsets.clear();
+                bytes = 0;
+                err = "FileExpertSource: cache complement size overflows";
+                return false;
+            }
+            offsets[index] = bytes;
+            bytes += blob_bytes;
+        }
+    }
+    if (bytes > (uint64_t) std::numeric_limits<size_t>::max()) {
+        offsets.clear();
+        bytes = 0;
+        err = "FileExpertSource: cache complement exceeds the host address space";
+        return false;
+    }
+    return true;
+}
+
+const uint8_t* cache_complement_blob_or_fallback(
+    size_t index, const std::vector<uint64_t>& offsets, const uint8_t* complement_host,
+    const uint8_t* mapped_fallback) {
+    if (complement_host != nullptr && index < offsets.size() && offsets[index] != kNoCacheComplement)
+        return complement_host + (size_t) offsets[index];
+    return mapped_fallback;
+}
+
+}  // namespace detail
+
+namespace {
+constexpr uint64_t kPinnedMemoryHeadroom = 8ull << 30;
+
+#if defined(__linux__)
+bool read_cgroup_memory_stat(const std::filesystem::path& path, uint64_t current,
+                             detail::CgroupMemoryStat& stat) {
+    std::ifstream input(path / "memory.stat");
+    if (!input) return false;
+
+    bool inactive_file = false, file_dirty = false, file_writeback = false;
+    std::string line;
+    while (std::getline(input, line)) {
+        std::istringstream fields(line);
+        std::string key;
+        uint64_t value = 0;
+        if (!(fields >> key >> value)) return false;
+        fields >> std::ws;
+        if (!fields.eof()) return false;
+
+        if (key == "inactive_file") {
+            if (inactive_file) return false;
+            inactive_file = true;
+            stat.inactive_file = value;
+        } else if (key == "file_dirty") {
+            if (file_dirty) return false;
+            file_dirty = true;
+            stat.file_dirty = value;
+        } else if (key == "file_writeback") {
+            if (file_writeback) return false;
+            file_writeback = true;
+            stat.file_writeback = value;
+        }
+    }
+    if (!input.eof() || !inactive_file || !file_dirty || !file_writeback) return false;
+    stat.current = current;
+    stat.valid = true;
+    return true;
+}
+#endif
+
+bool available_memory_bytes(uint64_t& bytes) {
+#if defined(_WIN32)
+    MEMORYSTATUSEX status{};
+    status.dwLength = sizeof(status);
+    if (!GlobalMemoryStatusEx(&status)) return false;
+    bytes = (uint64_t) status.ullAvailPhys;
+    return bytes > 0;
+#elif defined(__linux__)
+    // MemAvailable includes reclaimable page cache, unlike _SC_AVPHYS_PAGES.
+    std::ifstream info("/proc/meminfo");
+    std::string line;
+    bytes = 0;
+    while (std::getline(info, line)) {
+        std::istringstream fields(line);
+        std::string key, unit;
+        uint64_t value = 0;
+        if (fields >> key >> value >> unit && key == "MemAvailable:" && unit == "kB" &&
+            value <= std::numeric_limits<uint64_t>::max() / 1024) bytes = value * 1024;
+    }
+    if (bytes == 0) return false;
+    // Account for the tightest cgroup-v2 ancestor limit when its normal mount is visible.
+    // This is a point-in-time guard, not a reservation against concurrent allocations.
+    std::ifstream groups("/proc/self/cgroup");
+    if (!groups) return false;
+    bool resolved_v2 = false;
+    while (std::getline(groups, line)) {
+        if (line.rfind("0::/", 0) != 0) continue;
+        const std::filesystem::path root("/sys/fs/cgroup");
+        auto path = (root / line.substr(4)).lexically_normal();
+        if (path.string().rfind(root.string(), 0) != 0 || !std::filesystem::is_directory(path)) return false;
+        resolved_v2 = true;
+        while (path.string().rfind(root.string(), 0) == 0) {
+            std::ifstream limit_file(path / "memory.max"), current_file(path / "memory.current");
+            std::string limit;
+            uint64_t current = 0;
+            const bool readable = bool(limit_file >> limit) && bool(current_file >> current);
+            // The host's root cgroup has no memory.max; ordinary child groups must expose their limits.
+            if (!readable && !(path == root && !std::filesystem::exists(path / "memory.max") &&
+                               std::filesystem::exists(path / "cgroup.controllers"))) return false;
+            if (readable && limit != "max") {
+                try {
+                    size_t consumed = 0;
+                    const uint64_t cap = std::stoull(limit, &consumed);
+                    if (consumed != limit.size()) return false;
+                    detail::CgroupMemoryStat stat;
+                    if (!read_cgroup_memory_stat(path, current, stat)) return false;
+                    uint64_t cgroup_available = 0;
+                    if (!detail::cgroup_available_bytes(cap, stat, cgroup_available)) return false;
+                    bytes = std::min(bytes, cgroup_available);
+                } catch (...) { return false; }
+            }
+            if (path == root) break;
+            path = path.parent_path();
+        }
+    }
+    return resolved_v2;
+#else
+    const long pages = sysconf(_SC_AVPHYS_PAGES);
+    const long page_bytes = sysconf(_SC_PAGESIZE);
+    if (pages <= 0 || page_bytes <= 0 ||
+        (uint64_t) pages > std::numeric_limits<uint64_t>::max() / (uint64_t) page_bytes) return false;
+    bytes = (uint64_t) pages * (uint64_t) page_bytes;
+    return bytes > 0;
+#endif
+}
+
+}  // namespace
+
 // ================================ THE FILE-BACKED SOURCE ================================
 
 FileExpertSource::~FileExpertSource() { close(); }
@@ -39,9 +253,74 @@ FileExpertSource::~FileExpertSource() { close(); }
 bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err) {
     close();
     if (n_layers <= 0 || n_expert <= 0) { err = "FileExpertSource: the geometry is empty"; return false; }
-    n_expert_ = n_expert;
-    blobs_ = n_layers * n_expert;
-    const uint64_t want = (uint64_t) blobs_ * (uint64_t) strata::kernels::cpu::BLOB;
+    const auto& layout = strata::kernels::cpu::expert_layout();
+    if (layout.n_layers != n_layers || layout.n_expert != n_expert) {
+        err = "FileExpertSource: the requested geometry does not match the loaded expert layout";
+        return false;
+    }
+    if ((uint64_t) n_layers > (uint64_t) std::numeric_limits<int64_t>::max() / (uint64_t) n_expert) {
+        err = "FileExpertSource: the expert count overflows";
+        return false;
+    }
+    const uint64_t blob_count = (uint64_t) n_layers * (uint64_t) n_expert;
+    if (blob_count > (uint64_t) std::numeric_limits<int64_t>::max() ||
+        (uint64_t) n_layers > (uint64_t) std::numeric_limits<size_t>::max()) {
+        err = "FileExpertSource: the expert count overflows";
+        return false;
+    }
+
+    std::vector<uint64_t> layer_offsets((size_t) n_layers), layer_blob_bytes((size_t) n_layers);
+    const uint64_t want = layout.total;
+    if (want == 0 || want > (uint64_t) std::numeric_limits<size_t>::max()) {
+        err = "FileExpertSource: the loaded expert layout has an invalid size";
+        return false;
+    }
+    if (!layout.native) {
+        if (blob_count > std::numeric_limits<uint64_t>::max() / (uint64_t) strata::kernels::cpu::BLOB) {
+            err = "FileExpertSource: the canonical expert size overflows";
+            return false;
+        }
+        const uint64_t canonical_size = blob_count * (uint64_t) strata::kernels::cpu::BLOB;
+        if (want != canonical_size) {
+            err = "FileExpertSource: the canonical expert layout has an inconsistent size";
+            return false;
+        }
+        const uint64_t bytes = (uint64_t) strata::kernels::cpu::BLOB;
+        const uint64_t layer_bytes = (uint64_t) n_expert * bytes;
+        for (int64_t layer = 0; layer < n_layers; ++layer) {
+            layer_offsets[(size_t) layer] = (uint64_t) layer * layer_bytes;
+            layer_blob_bytes[(size_t) layer] = bytes;
+        }
+    } else {
+        if (layout.offset.size() != (size_t) n_layers || layout.bytes.size() != (size_t) n_layers ||
+            layout.fmt.size() != (size_t) n_layers) {
+            err = "FileExpertSource: the native expert layout is incomplete";
+            return false;
+        }
+        uint64_t at = 0;
+        for (int64_t layer = 0; layer < n_layers; ++layer) {
+            const size_t i = (size_t) layer;
+            const uint64_t bytes = (uint64_t) layout.fmt[i].bytes;
+            if (layout.offset[i] != at || bytes == 0 || layout.bytes[i] != bytes ||
+                bytes > std::numeric_limits<uint64_t>::max() / (uint64_t) n_expert) {
+                err = "FileExpertSource: the native expert layout is invalid at layer " + std::to_string(layer);
+                return false;
+            }
+            const uint64_t layer_bytes = bytes * (uint64_t) n_expert;
+            if (at > want || layer_bytes > want - at) {
+                err = "FileExpertSource: the native expert layout exceeds its declared size at layer " +
+                      std::to_string(layer);
+                return false;
+            }
+            layer_offsets[i] = layout.offset[i];
+            layer_blob_bytes[i] = bytes;
+            at += layer_bytes;
+        }
+        if (at != want) {
+            err = "FileExpertSource: the native expert layout has an inconsistent size";
+            return false;
+        }
+    }
     const std::string path = pack_dir + "/experts.bin";
 
 #if defined(_WIN32)
@@ -76,10 +355,9 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     if ((uint64_t) sz.QuadPart != want) {
         char buf[400];
         std::snprintf(buf, sizeof buf,
-                      "FileExpertSource: %s is %llu B but %lld layers x %lld experts x %d B is %llu B - this "
-                      "is not the pack this geometry came from",
-                      path.c_str(), (unsigned long long) sz.QuadPart, (long long) n_layers,
-                      (long long) n_expert, (int) strata::kernels::cpu::BLOB, (unsigned long long) want);
+                      "FileExpertSource: %s is %llu B but the loaded expert layout requires %llu B - this is not "
+                      "the pack this geometry came from",
+                      path.c_str(), (unsigned long long) sz.QuadPart, (unsigned long long) want);
         CloseHandle(f);
         err = buf;
         return false;
@@ -105,13 +383,13 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     if (fd < 0) { err = "FileExpertSource: cannot open " + path; return false; }
     struct stat st{};
     if (fstat(fd, &st) != 0) { ::close(fd); err = "FileExpertSource: cannot stat " + path; return false; }
-    if ((uint64_t) st.st_size != want) {
+    if (st.st_size < 0 || (uint64_t) st.st_size != want) {
         char buf[400];
         std::snprintf(buf, sizeof buf,
-                      "FileExpertSource: %s is %llu B but %lld layers x %lld experts x %d B is %llu B - this "
-                      "is not the pack this geometry came from",
-                      path.c_str(), (unsigned long long) st.st_size, (long long) n_layers, (long long) n_expert,
-                      (int) strata::kernels::cpu::BLOB, (unsigned long long) want);
+                      "FileExpertSource: %s is %llu B but the loaded expert layout requires %llu B - this is not "
+                      "the pack this geometry came from",
+                      path.c_str(), (unsigned long long) (st.st_size < 0 ? 0 : st.st_size),
+                      (unsigned long long) want);
         ::close(fd);
         err = buf;
         return false;
@@ -121,10 +399,27 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     fd_ = fd;
     base_ = (const uint8_t*) view;
 #endif
+    blobs_ = (int64_t) blob_count;
+    n_layers_ = n_layers;
+    n_expert_ = n_expert;
+    mapped_bytes_ = want;
+    layer_offsets_ = std::move(layer_offsets);
+    layer_blob_bytes_ = std::move(layer_blob_bytes);
     return true;
 }
 
 void FileExpertSource::close() {
+    if (complement_arena_ != nullptr) {
+        if (complement_pinned_) (void) cudaFreeHost(complement_arena_);
+        else std::free(complement_arena_);
+    }
+    complement_arena_ = nullptr;
+    complement_host_ = nullptr;
+    complement_device_ = nullptr;
+    complement_bytes_ = 0;
+    complement_offsets_.clear();
+    complement_pinned_ = false;
+    complement_ready_ = false;
 #if defined(_WIN32)
     if (base_ != nullptr) UnmapViewOfFile((LPCVOID) base_);
     if (mapping_ != nullptr) CloseHandle((HANDLE) mapping_);
@@ -132,27 +427,225 @@ void FileExpertSource::close() {
     mapping_ = nullptr;
     file_ = nullptr;
 #else
-    if (base_ != nullptr) munmap((void*) base_, (size_t) blobs_ * (size_t) strata::kernels::cpu::BLOB);
+    if (base_ != nullptr) munmap((void*) base_, (size_t) mapped_bytes_);
     if (fd_ >= 0) ::close(fd_);
     fd_ = -1;
 #endif
     base_ = nullptr;
     blobs_ = 0;
+    n_layers_ = 0;
+    n_expert_ = 0;
+    mapped_bytes_ = 0;
+    layer_offsets_.clear();
+    layer_blob_bytes_.clear();
     reads_ = 0;
 }
 
+const uint8_t* FileExpertSource::mapped_blob(int64_t layer, int64_t expert) const {
+    if (base_ == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return nullptr;
+    const size_t i = (size_t) layer;
+    if (i >= layer_offsets_.size() || i >= layer_blob_bytes_.size()) return nullptr;
+    const uint64_t blob_bytes = layer_blob_bytes_[i];
+    if (blob_bytes == 0 || (uint64_t) expert > std::numeric_limits<uint64_t>::max() / blob_bytes) return nullptr;
+    const uint64_t expert_offset = (uint64_t) expert * blob_bytes;
+    const uint64_t layer_offset = layer_offsets_[i];
+    if (layer_offset > mapped_bytes_ || expert_offset > mapped_bytes_ - layer_offset) return nullptr;
+    const uint64_t offset = layer_offset + expert_offset;
+    if (blob_bytes > mapped_bytes_ - offset) return nullptr;
+    return base_ + (size_t) offset;
+}
+
+bool FileExpertSource::pin_cache_complement(
+    const ExpertCache& cache, std::string& err, bool pin,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs) {
+    err.clear();
+    if (base_ == nullptr) { err = "FileExpertSource: open the mapped experts before pinning a complement"; return false; }
+    if (complement_ready_) { err = "FileExpertSource: the cache complement is already pinned"; return false; }
+    if (!cache.valid()) { err = "FileExpertSource: the GPU expert cache is not open"; return false; }
+    if (cache.fills() != cache.resident()) {
+        err = "FileExpertSource: the GPU expert cache is not fully filled";
+        return false;
+    }
+    const cudaError_t sync = cudaDeviceSynchronize();
+    if (sync != cudaSuccess) {
+        err = std::string("FileExpertSource: GPU expert cache is not ready: ") + cudaGetErrorString(sync);
+        (void) cudaGetLastError();
+        return false;
+    }
+
+    std::vector<std::pair<int32_t, int32_t>> primary_gpu_pairs;
+    primary_gpu_pairs.reserve((size_t) cache.resident());
+    for (int64_t layer = 0; layer < n_layers_; ++layer) {
+        for (int64_t expert = 0; expert < n_expert_; ++expert) {
+            if (cache.slot_of(layer, expert) != kNotResident)
+                primary_gpu_pairs.emplace_back((int32_t) layer, (int32_t) expert);
+        }
+    }
+    std::vector<uint64_t> offsets;
+    uint64_t bytes = 0;
+    if (!detail::make_cache_complement_plan(n_layers_, n_expert_, layer_blob_bytes_, primary_gpu_pairs,
+                                            additional_gpu_pairs, offsets, bytes, err)) return false;
+
+    if (bytes > 0) {
+        uint64_t physical = 0;
+        if (!available_memory_bytes(physical)) {
+            err = "FileExpertSource: cannot determine available RAM for the resident-memory safety check";
+            return false;
+        }
+        if (physical <= kPinnedMemoryHeadroom || bytes > physical - kPinnedMemoryHeadroom) {
+            char message[256];
+            std::snprintf(message, sizeof message,
+                          "FileExpertSource: resident complement %.2f GiB exceeds available RAM minus the 8 GiB safety headroom",
+                          (double) bytes / 1073741824.0);
+            err = message;
+            return false;
+        }
+    }
+
+    void* arena = nullptr;
+    const uint8_t* host = nullptr;
+    const uint8_t* device = nullptr;
+    if (bytes > 0) {
+        std::fprintf(stderr, "FileExpertSource: allocating %.2f GiB %s cache complement\n",
+                     (double) bytes / 1073741824.0, pin ? "mapped pinned" : "pageable resident");
+        std::fflush(stderr);
+        if (pin) {
+            const cudaError_t allocated = cudaHostAlloc(&arena, (size_t) bytes,
+                                                         cudaHostAllocMapped | cudaHostAllocPortable);
+            if (allocated != cudaSuccess) {
+                err = std::string("FileExpertSource: mapped pinned complement allocation failed: ") +
+                      cudaGetErrorString(allocated);
+                std::fprintf(stderr, "%s\n", err.c_str());
+                std::fflush(stderr);
+                (void) cudaGetLastError();
+                return false;
+            }
+            host = (const uint8_t*) arena;
+            void* alias = nullptr;
+            const cudaError_t aliased = cudaHostGetDevicePointer(&alias, arena, 0);
+            if (aliased != cudaSuccess || alias == nullptr) {
+                err = std::string("FileExpertSource: mapped pinned complement device alias failed: ") +
+                      cudaGetErrorString(aliased);
+                std::fprintf(stderr, "%s\n", err.c_str());
+                std::fflush(stderr);
+                (void) cudaGetLastError();
+                (void) cudaFreeHost(arena);
+                return false;
+            }
+            device = (const uint8_t*) alias;
+        } else {
+            arena = std::malloc((size_t) bytes);
+            if (arena == nullptr) {
+                err = "FileExpertSource: pageable resident complement allocation failed";
+                return false;
+            }
+            host = (const uint8_t*) arena;
+        }
+    }
+
+    uint64_t copied = 0;
+    for (int64_t layer = 0; layer < n_layers_; ++layer) {
+        const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer];
+        for (int64_t expert = 0; expert < n_expert_; ++expert) {
+            const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+            const uint64_t offset = offsets[index];
+            if (offset == kNoComplement) continue;
+            const uint8_t* source = mapped_blob(layer, expert);
+            if (source == nullptr || offset > bytes || blob_bytes > bytes - offset) {
+                err = "FileExpertSource: invalid blob bounds while building the cache complement";
+                if (arena != nullptr) {
+                    if (pin) (void) cudaFreeHost(arena);
+                    else std::free(arena);
+                }
+                return false;
+            }
+            std::memcpy((uint8_t*) host + (size_t) offset, source, (size_t) blob_bytes);
+            copied += blob_bytes;
+        }
+#if !defined(_WIN32)
+        const uint64_t layer_offset = layer_offsets_[(size_t) layer];
+        const uint64_t layer_bytes = blob_bytes * (uint64_t) n_expert_;
+        const uint64_t layer_end = layer_offset + layer_bytes;
+        const long page_bytes = sysconf(_SC_PAGESIZE);
+        if (page_bytes <= 0) {
+            err = "FileExpertSource: cannot determine page size for mapped-page release";
+            if (arena != nullptr) {
+                if (pin) (void) cudaFreeHost(arena);
+                else std::free(arena);
+            }
+            return false;
+        }
+        const uint64_t page = (uint64_t) page_bytes;
+        const uint64_t advice_start = layer_offset - layer_offset % page;
+        const uint64_t end_remainder = layer_end % page;
+        const uint64_t extra = end_remainder == 0 ? 0 : page - end_remainder;
+        const uint64_t advice_end = extra > mapped_bytes_ - layer_end ? mapped_bytes_ : layer_end + extra;
+        if (advice_end > advice_start &&
+            madvise((void*) (base_ + (size_t) advice_start), (size_t) (advice_end - advice_start), MADV_DONTNEED) != 0) {
+            err = "FileExpertSource: madvise could not release mapped expert layer " + std::to_string(layer);
+            if (arena != nullptr) {
+                if (pin) (void) cudaFreeHost(arena);
+                else std::free(arena);
+            }
+            return false;
+        }
+        const int advice = posix_fadvise(fd_, (off_t) layer_offset, (off_t) layer_bytes, POSIX_FADV_DONTNEED);
+        if (advice != 0) {
+            err = "FileExpertSource: posix_fadvise could not release expert layer " + std::to_string(layer);
+            if (arena != nullptr) {
+                if (pin) (void) cudaFreeHost(arena);
+                else std::free(arena);
+            }
+            return false;
+        }
+#endif
+        if ((layer + 1) % 8 == 0 || layer + 1 == n_layers_) {
+            std::fprintf(stderr, "FileExpertSource: copied cache complement through layer %lld/%lld (%.2f GiB)\n",
+                         (long long) (layer + 1), (long long) n_layers_, (double) copied / 1073741824.0);
+            std::fflush(stderr);
+        }
+    }
+
+    complement_arena_ = arena;
+    complement_host_ = host;
+    complement_device_ = device;
+    complement_bytes_ = bytes;
+    complement_offsets_ = std::move(offsets);
+    complement_pinned_ = pin && bytes > 0;
+    complement_ready_ = true;
+    std::fprintf(stderr, "FileExpertSource: %s cache complement ready: resident %.2f GiB, pinned %.2f GiB\n",
+                 complement_pinned_ ? "mapped pinned" : "pageable resident",
+                 (double) resident_bytes() / 1073741824.0, (double) pinned_bytes() / 1073741824.0);
+    if (!additional_gpu_pairs.empty()) {
+        std::fprintf(stderr, "FileExpertSource: %zu verified additional-GPU experts remain on the mmap fallback\n",
+                     additional_gpu_pairs.size());
+    }
+    std::fflush(stderr);
+    return true;
+}
+
 const uint8_t* FileExpertSource::blob(int64_t layer, int64_t expert) {
-    if (base_ == nullptr) return nullptr;
-    if (layer < 0 || expert < 0) return nullptr;
-    // **`expert >= n_expert_` IS CHECKED SEPARATELY, AND THE FLAT INDEX ALONE DOES NOT CATCH IT.**  The blob
-    // index is `layer * n_expert + expert`, so `blob(0, 512)` has flat index 512 - which is in range, and is
-    // `blob(1, 0)`.  A router id one past the end of a layer would then read the NEXT LAYER's first expert:
-    // finite, correctly sized, and wrong.  Layer and expert are separate axes and are validated as such.
-    if (expert >= n_expert_) return nullptr;
-    const int64_t i = layer * n_expert_ + expert;
-    if (i >= blobs_) return nullptr;
-    ++reads_;
-    return base_ + (size_t) i * strata::kernels::cpu::BLOB;
+    if (base_ == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return nullptr;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    const uint8_t* mapped_fallback = mapped_blob(layer, expert);
+    const uint8_t* result = complement_ready_
+        ? detail::cache_complement_blob_or_fallback(index, complement_offsets_, complement_host_, mapped_fallback)
+        : mapped_fallback;
+    if (result != nullptr) ++reads_;
+    return result;
+}
+
+bool FileExpertSource::pinned(int64_t layer, int64_t expert) const {
+    if (!complement_ready_ || !complement_pinned_ || complement_host_ == nullptr || layer < 0 || expert < 0 ||
+        layer >= n_layers_ || expert >= n_expert_) return false;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    return index < complement_offsets_.size() && complement_offsets_[index] != kNoComplement;
+}
+
+const uint8_t* FileExpertSource::device_alias(int64_t layer, int64_t expert) const {
+    if (!pinned(layer, expert) || complement_device_ == nullptr) return nullptr;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    return complement_device_ + (size_t) complement_offsets_[index];
 }
 
 // ================================ THE ADAPTER ================================
@@ -653,6 +1146,8 @@ LoadStats load_experts_gguf(const std::string& gguf, uint8_t* dst, const strata:
     for (auto& t : pool) t.join();
     if (bad) {
         st.seconds = -1.0;
+        st.ok = false;
+        st.error = "short read or unreadable shard while reading the experts from the GGUF";
         return st;
     }
     st.bytes = lay.total;
@@ -712,6 +1207,11 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
     }
     const LoadStats st = from_gguf ? load_experts_gguf(gguf_, a->data(), lay, threads)
                                    : load_experts_ranges(path, a->data(), loff, lbytes, threads, /*chunk=*/8u << 20);
+    if (!st.ok) {
+        delete a;
+        err = "ArenaExpertSource: the expert load was refused: " + (st.error.empty() ? std::string("unknown") : st.error);
+        return false;
+    }
     if (st.bytes != want) {
         delete a;
         err = "ArenaExpertSource: the load read " + std::to_string(st.bytes) + " B of " + std::to_string(want);
@@ -740,6 +1240,9 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
     reads_ = 0;
     note_ = a->note;
     gib_per_s_ = st.gib_per_second();
+    load_seconds_ = st.seconds;
+    load_read_s_ = st.read_seconds;
+    load_copy_s_ = st.copy_seconds;
     return true;
 }
 

src/core/layer.cpp

@@ -172,11 +172,13 @@ if (!w.wants_q8k()) {        s_gemv_q8_0_split(x80, p.codes, p.scales, p.offset,
 // namespace
 void layer_set_native_bf16(bool enabled) { native_bf16_projections = enabled; }
 void layer_set_native_flash_attn_short(bool enabled) { native_flash_attn_short = enabled; }
-namespace { bool g_kv_int8 = false, g_kv_q4 = false; }
+namespace { bool g_kv_int8 = false, g_kv_q4 = false, g_kv_hybrid = false; }
 void qsa_set_kv_int8(bool enabled) { g_kv_int8 = enabled; }
 bool qsa_kv_int8() { return g_kv_int8; }
 void qsa_set_kv_q4(bool enabled) { g_kv_q4 = enabled; }
 bool qsa_kv_q4() { return g_kv_q4; }
+void qsa_set_kv_hybrid(bool enabled) { g_kv_hybrid = enabled; }
+bool qsa_kv_hybrid() { return g_kv_hybrid; }
 uint64_t gdn_buffers_bytes(const ModelGeometry& g) {    const int64_t C = g.ssm_conv_channels;    const int64_t V = g.ssm_value_dim;    const uint64_t parts[] = {        q8k_bytes(g.n_embd),
 // x_q8k
 (uint64_t) (g.n_embd / 32) * 34,
@@ -507,10 +509,15 @@ KvPlan kv_plan(const QsaShapes& s, int64_t max_cells, int64_t ring_cells) {
     }
     return p;
 }
-uint64_t kv_pool_bytes(const QsaShapes& s, int64_t pages) {
+uint64_t kv_pool_bytes(const QsaShapes& s, int64_t pages, bool hybrid, bool int8) {
+    if (hybrid) {   // K8V4: the INT8 K half (codes + scales) plus the Q4_0 V half (kv_q4.hpp's rotation)
+        const uint64_t rows = (uint64_t) pages * s.page_size * s.n_head_kv;
+        return rows * (uint64_t) s.head_dim + rows * (uint64_t) (s.head_dim / strata::kernels::KV_Q8_GROUP) * 2 +
+               rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim) + 64;
+    }
     if (g_kv_q4) return (uint64_t) pages * s.page_size * strata::kernels::kv_q4_bytes_per_cell(s) + 64;
-    return g_kv_int8 ? (uint64_t) pages * s.page_size * strata::kernels::kv_q8_bytes_per_cell(s) + 64
-                     : (uint64_t) pages * s.n_head_kv * s.page_size * s.head_dim * 2 * 2;
+    return int8 ? (uint64_t) pages * s.page_size * strata::kernels::kv_q8_bytes_per_cell(s) + 64
+                : (uint64_t) pages * s.n_head_kv * s.page_size * s.head_dim * 2 * 2;
 }
 }  // namespace
 
@@ -523,7 +530,8 @@ uint64_t qsa_state_bytes(const ModelGeometry& g, int64_t max_cells, bool with_ro
     const QsaShapes s = qsa_shapes(g);
     const KvPlan p = kv_plan(s, max_cells, ring_cells);
     uint64_t n = 0;
-    n += kv_pool_bytes(s, p.slots) + 4 * 16;                                   // K/V pools (the VRAM slots)
+    n += kv_pool_bytes(s, p.slots, g_kv_hybrid && ring_cells <= 0,
+                       g_kv_int8 || g_kv_hybrid) + 4 * 16;   // K/V pools (the VRAM slots)
     n += (uint64_t) p.pages * 4;                                               // page_table
     if (p.mode == 1) n += strata::kernels::kv_stream_map_bytes(p.slots) + 6 * 16;   // the residency map
     n += (uint64_t) (s.idx_block - 1) * s.idx_dim * 4;                         // tail
@@ -544,14 +552,29 @@ uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, Q
     Cursor c{(uint8_t*) base};
     st.kv_int8 = g_kv_int8 && !g_kv_q4;
     st.kv_q4 = g_kv_q4;
+    // Hybrid K8V4, main layers only (the drafter's state is created with the globals toggled to INT8 -
+    // mtp.cpp). Streamed mode is refused outright; generate.cpp validates it too, this is the backstop.
+    if (g_kv_hybrid && ring_cells <= 0) {
+        if (p.mode == 1) {
+            std::fprintf(stderr, "strata: hybrid K8V4 KV does not support --kv-resident streaming\n");
+            return 0;
+        }
+        st.kv_hybrid = true;
+        st.kv_int8 = false;
+        st.kv_q4 = false;
+    }
     st.kv_mode = p.mode;
     st.n_slots = p.slots;
     const uint64_t rows = (uint64_t) p.slots * s.n_head_kv * s.page_size;   // VRAM rows: the slots
     const uint64_t q4_row = strata::kernels::kv_q4_bytes_per_head((int) s.head_dim);
-    if (st.kv_q4) {
+    if (st.kv_hybrid) {
+        st.k_q = c.take<int8_t>(rows * s.head_dim);
+        st.k_scale = c.take<uint16_t>(rows * (s.head_dim / strata::kernels::KV_Q8_GROUP));
+        st.v_q4 = c.take<uint8_t>(rows * q4_row);
+    } else if (st.kv_q4) {
         st.k_q4 = c.take<uint8_t>(rows * q4_row);
         st.v_q4 = c.take<uint8_t>(rows * q4_row);
-    } else if (g_kv_int8) {
+    } else if (st.kv_int8) {   // st, not the global: the drafter's ring is INT8 under --kv k8v4 too
         st.k_q = c.take<int8_t>(rows * s.head_dim);
         st.v_q = c.take<int8_t>(rows * s.head_dim);
         st.k_scale = c.take<uint16_t>(rows * (s.head_dim / strata::kernels::KV_Q8_GROUP));
@@ -619,7 +642,7 @@ uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, Q
         if (st.kv_q4) {
             st.host.k_q4 = hc.take<uint8_t>(hrows * q4_row);
             st.host.v_q4 = hc.take<uint8_t>(hrows * q4_row);
-        } else if (g_kv_int8) {
+        } else if (st.kv_int8) {
             st.host.k_q = hc.take<int8_t>(hrows * s.head_dim);
             st.host.v_q = hc.take<int8_t>(hrows * s.head_dim);
             st.host.k_scale = hc.take<uint16_t>(hrows * (s.head_dim / strata::kernels::KV_Q8_GROUP));
@@ -658,7 +681,11 @@ void qsa_state_zero(const QsaState& st, const ModelGeometry& g, void* stream) {
     const QsaShapes s = qsa_shapes(g);
     cudaStream_t cs = (cudaStream_t) stream;
     const size_t rows = (size_t) st.n_slots * s.n_head_kv * s.page_size;
-    if (st.kv_q4) {
+    if (st.kv_hybrid) {
+        cudaMemsetAsync(st.k_q, 0, rows * s.head_dim, cs);
+        cudaMemsetAsync(st.k_scale, 0, rows * (s.head_dim / strata::kernels::KV_Q8_GROUP) * 2, cs);
+        cudaMemsetAsync(st.v_q4, 0, rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim), cs);
+    } else if (st.kv_q4) {
         cudaMemsetAsync(st.k_q4, 0, rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim), cs);
         cudaMemsetAsync(st.v_q4, 0, rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim), cs);
     } else if (st.kv_int8) {
@@ -682,7 +709,8 @@ void qsa_state_zero(const QsaState& st, const ModelGeometry& g, void* stream) {
 strata::kernels::QsaAttnPools qsa_attn_pools(const QsaState& st) {
     strata::kernels::QsaAttnPools pools;
     pools.page_table = st.page_table;
-    if (st.kv_q4) { pools.k_q4 = st.k_q4; pools.v_q4 = st.v_q4; }
+    if (st.kv_hybrid) { pools.k_q = st.k_q; pools.k_scale = st.k_scale; pools.v_q4 = st.v_q4; }
+    else if (st.kv_q4) { pools.k_q4 = st.k_q4; pools.v_q4 = st.v_q4; }
     else if (st.kv_int8) { pools.k_q = st.k_q; pools.v_q = st.v_q; pools.k_scale = st.k_scale; pools.v_scale = st.v_scale; }
     else { pools.k_pool = st.k_pool; pools.v_pool = st.v_pool; }
     return pools;
@@ -863,11 +891,18 @@ if (!gemv_quantized(*w_attnk, p_k, f_k, b.x_q8_0, b.x_q8k, b.kcur, g.n_embd, g.n
 // EVERY ENTRY POINT FROM HERE ON IS THE CAPTURABLE ONE: the per-token counts come from `st.step` and every
 // launch is sized from a capacity in `st`/`b`, so this sequence can be captured and replayed.  The
 // host-scalar wrappers would be correct here today and silently wrong in a graph.
-if (st.kv_q4) {
+if (st.kv_hybrid) {
+    // K8V4: only V is rotated (kv_q4.hpp's H); the scores pair unrotated q with unrotated INT8 K, and the
+    // output - a mix of rotated values - is rotated back after attention. Each append/gather call folds the
+    // unused half's lanes onto the used pool (a bit-identical duplicate write), so no kernel variants exist.
+    strata::kernels::fwht256_inplace_cuda(b.vcur, g.n_head_kv, stream);
+    kv_append_q8_step(st.k_q, st.k_q, st.k_scale, st.k_scale, st.page_table, st.step, b.kcur, b.kcur, s, stream, nullptr);   // mode 0: no host mirror
+    strata::kernels::kv_append_q4_step(st.v_q4, st.v_q4, st.page_table, st.step, b.vcur, b.vcur, s, stream, nullptr);
+} else if (st.kv_q4) {
     strata::kernels::fwht256_inplace_cuda(b.kcur, g.n_head_kv, stream);   // Q4_0: rotated K and V (kv_q4.hpp)
     strata::kernels::fwht256_inplace_cuda(b.vcur, g.n_head_kv, stream);
     strata::kernels::kv_append_q4_step(st.k_q4, st.v_q4, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);
-} else if (st.kv_int8) kv_append_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    else kv_append_step(st.k_pool, st.v_pool, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    {        const uint64_t nvk = (uint64_t) g.n_head_kv * g.head_dim;        const uint64_t base = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim + 2 * nvk + 8;        if (!st.kv_int8) dump_slot(dump, g, layer, (const float*) st.k_pool, base, nvk / 2, stream);        if (!st.kv_int8) dump_slot(dump, g, layer, (const float*) st.v_pool, base + nvk / 2, nvk / 2, stream);        dump_slot(dump, g, layer, b.vcur, base + nvk, nvk, stream);        dump_slot(dump, g, layer, b.kcur, base + 2 * nvk, nvk, stream);    }    {        const QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};        if (native_qsa_indexer_enabled()) {
+} else if (st.kv_int8) kv_append_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    else kv_append_step(st.k_pool, st.v_pool, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    {        const uint64_t nvk = (uint64_t) g.n_head_kv * g.head_dim;        const uint64_t base = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim + 2 * nvk + 8;        if (!st.kv_int8 && !st.kv_hybrid) dump_slot(dump, g, layer, (const float*) st.k_pool, base, nvk / 2, stream);        if (!st.kv_int8 && !st.kv_hybrid) dump_slot(dump, g, layer, (const float*) st.v_pool, base + nvk / 2, nvk / 2, stream);        dump_slot(dump, g, layer, b.vcur, base + nvk, nvk, stream);        dump_slot(dump, g, layer, b.kcur, base + 2 * nvk, nvk, stream);    }    {        const QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};        if (native_qsa_indexer_enabled()) {
     try {
         native_qsa_indexer_append(b.idx_raw, st.step + kStepPos, pos_base,
             (const float*) w_ikn->data, RMS_EPS, ib, s, st.max_cells, (float) qsa_freq_base(), stream);
@@ -906,7 +941,13 @@ int64_t max_blocks = (st.max_cells / s.idx_block) + 2;
         const strata::kernels::QsaAttnPools pools = qsa_attn_pools(st);
         strata::kernels::qsa_decode_attn_step(b.qcur, pools, b.ids, st.step, cap, s, b.attn_scratch, b.attn, stream);
     } else {
-    if (st.kv_q4) strata::kernels::kv_gather_q4_step(st.k_q4, st.v_q4, st.page_table, b.ids, st.step, cap, s, b.k_scratch, b.v_scratch, stream);
+    if (st.kv_hybrid) {
+        kv_gather_q8_step(st.k_q, st.k_q, st.k_scale, st.k_scale, st.page_table, b.ids, st.step, cap, s,
+                          b.k_scratch, b.k_scratch, stream);
+        strata::kernels::kv_gather_q4_step(st.v_q4, st.v_q4, st.page_table, b.ids, st.step, cap, s,
+                                           b.v_scratch, b.v_scratch, stream);
+    }
+    else if (st.kv_q4) strata::kernels::kv_gather_q4_step(st.k_q4, st.v_q4, st.page_table, b.ids, st.step, cap, s, b.k_scratch, b.v_scratch, stream);
     else if (st.kv_int8) kv_gather_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, b.ids, st.step, cap, s,                                 b.k_scratch, b.v_scratch, stream);    else kv_gather_step(st.k_pool, st.v_pool, st.page_table, b.ids, st.step, cap, s, b.k_scratch, b.v_scratch,                   stream);    if (native_flash_attn_short) {
     if (st.max_cells < 1 || st.max_cells > 256 || !st.attention_status || !st.host_step) {
         err = v.name("native_flash_attn") + ": short adapter requires context <=256 and persistent status storage";
@@ -922,7 +963,7 @@ int64_t max_blocks = (st.max_cells / s.idx_block) + 2;
     }
 } else qsa_attend_step(b.qcur, b.k_scratch, b.v_scratch, st.step, cap, s, b.attn, nullptr, stream);
     }
-    if (st.kv_q4) strata::kernels::fwht256_inplace_cuda(b.attn, g.n_head, stream);   // the output back: H is self-inverse
+    if (st.kv_q4 || st.kv_hybrid) strata::kernels::fwht256_inplace_cuda(b.attn, g.n_head, stream);   // the output back: H is self-inverse
     dump_slot(dump, g, layer, b.attn, (uint64_t) 2 * g.n_embd + 2 * g.hc,                            (uint64_t) g.n_head * g.head_dim, stream);    {        const uint64_t vs = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim +                            (uint64_t) 2 * g.n_head_kv * g.head_dim;        dump_slot(dump, g, layer, (const float*) b.v_scratch, vs,                            (uint64_t) g.n_head_kv * g.head_dim / 2, stream);        dump_slot(dump, g, layer, (const float*) b.ids, vs + (uint64_t) g.n_head_kv * g.head_dim / 2, 4, stream);        dump_slot(dump, g, layer, (const float*) st.step,                            vs + (uint64_t) g.n_head_kv * g.head_dim / 2 + 4, 4, stream);    }
 // ---- 9. Gate the attention output before its projection. Native CUDA keeps F32
 // sigmoid/multiply arithmetic and uses Q8_1 for the native quantized projection.

src/core/load_main.cpp

@@ -65,7 +65,10 @@ int main(int argc, char** argv) {
 
     const strata::core::LoadStats st =
         strata::core::load_experts(path, dst, blob_bytes, blobs_per_layer, layers, threads, chunk);
-    if (st.seconds < 0) return 1;
+    if (!st.ok) {
+        std::fprintf(stderr, "loading the experts failed: %s\n", st.error.empty() ? "unknown" : st.error.c_str());
+        return 1;
+    }
 
     std::printf("loaded %.3f GiB in %.3f s with %d threads and %llu MiB chunks  ->  %.2f GiB/s\n",
                 (double) st.bytes / (1024.0 * 1024 * 1024), st.seconds, threads,

src/core/mtp.cpp

@@ -69,13 +69,31 @@ strata::kernels::QsaShapes shapes_of(const ModelGeometry& g) {
     return s;
 }
 
+// 64-bit seek/tell on a `FILE*`: `fseek`/`ftell` take a 32-bit `long` on Windows and would wrap past 2 GiB.
+#if defined(_WIN32)
+#define STRATA_FILE_SEEK64(f, o, w) _fseeki64((f), (long long) (o), (w))
+#define STRATA_FILE_TELL64(f) _ftelli64(f)
+#else
+#define STRATA_FILE_SEEK64(f, o, w) fseeko((f), (off_t) (o), (w))
+#define STRATA_FILE_TELL64(f) ftello(f)
+#endif
+
 bool read_file(const std::string& path, std::vector<uint8_t>& out) {
-    std::ifstream f(path, std::ios::binary | std::ios::ate);
-    if (!f) return false;
-    const std::streamsize n = f.tellg();
-    f.seekg(0);
+    // Loader fix (0.1.15+loaderfix.2): `ifstream::read` reaches the disk as 4095-byte reads on MSVC - the same
+    // split `load_experts_ranges` had - so the drafter's 111 MiB dense blob paid 4 KiB per operation on a cold
+    // start.  `fread` passes a request bigger than the stream buffer straight to `ReadFile()`.
+    FILE* f = std::fopen(path.c_str(), "rb");
+    if (f == nullptr) return false;
+    // A `FILE*` has no destructor and the short-read path below returns early: the guard closes on every path.
+    struct Closer {
+        FILE* f;
+        ~Closer() { if (f != nullptr) std::fclose(f); }
+    } closer{f};
+    if (STRATA_FILE_SEEK64(f, 0, SEEK_END) != 0) return false;
+    const long long n = (long long) STRATA_FILE_TELL64(f);
+    if (n < 0 || STRATA_FILE_SEEK64(f, 0, SEEK_SET) != 0) return false;
     out.resize((size_t) n);
-    return (bool) f.read((char*) out.data(), n);
+    return n == 0 || std::fread(out.data(), 1, (size_t) n, f) == (size_t) n;
 }
 
 }  // namespace
@@ -120,6 +138,9 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
     max_t_ = max_t;
     rt_dir_ = rt_dir;
     if (max_t < 1 || max_t > strata::kernels::kVerifyMaxT) { err = "mtp: max_t out of range"; return false; }
+    // Loader fix (0.1.15+loaderfix.2): the two reads below are the whole “drafter files” cost; reporting
+    // them apart from the rest of the stage is what makes the next regression visible.
+    const auto t_files = std::chrono::steady_clock::now();
     // ---- the index and the dense weights
     {
         std::ifstream idx(rt_dir + "/dense.txt");
@@ -150,13 +171,19 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
     // ---- the 512 routed experts, one blob each
     {
         const uint64_t bytes = (uint64_t) g.n_expert * strata::kernels::cpu::BLOB;
-        std::ifstream f(rt_dir + "/experts.bin", std::ios::binary);
-        if (!f) { err = "mtp: cannot open experts.bin"; return false; }
+        // Loader fix (0.1.15+loaderfix.2): each 64 MiB read below reached the disk as ~16k 4095-byte reads under
+        // MSVC's `basic_filebuf::xsgetn`, which is what made 675 MiB of drafter experts take minutes.
+        FILE* f = std::fopen((rt_dir + "/experts.bin").c_str(), "rb");
+        if (f == nullptr) { err = "mtp: cannot open experts.bin"; return false; }
+        struct Closer {
+            FILE* f;
+            ~Closer() { if (f != nullptr) std::fclose(f); }
+        } closer{f};
         if (cudaMalloc((void**) &experts_, bytes) != cudaSuccess) { err = "mtp: the 512 experts do not fit in VRAM"; return false; }
         std::vector<uint8_t> chunk(64u << 20);
         for (uint64_t off = 0; off < bytes;) {
             const uint64_t n = std::min<uint64_t>(chunk.size(), bytes - off);
-            if (!f.read((char*) chunk.data(), (std::streamsize) n)) { err = "mtp: experts.bin is truncated"; return false; }
+            if (std::fread(chunk.data(), 1, (size_t) n, f) != (size_t) n) { err = "mtp: experts.bin is truncated"; return false; }
             cudaMemcpy(experts_ + off, chunk.data(), n, cudaMemcpyHostToDevice);
             off += n;
         }
@@ -174,6 +201,12 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
     // window (plus the cells a round writes ahead of its queries) over a host copy, refilled on a resume. The host copy
     // is pinned after the expert arena has pinned what it could: if it does not fit, the K/V stays whole in VRAM.
     int64_t ring = (window > 0 && window < max_cells) ? window + 4 * (int64_t) max_t + 64 : 0;
+    // K8V4 never applies to the drafter: its own attention paths (below, and verify.cpp) handle whole formats
+    // only, whatever ring shape it takes (0, a window, or the -1 fully-resident fallback).
+    const bool kv_hybrid_was = qsa_kv_hybrid();
+    const bool kv_int8_was = qsa_kv_int8();
+    qsa_set_kv_hybrid(false);
+    if (kv_hybrid_was) qsa_set_kv_int8(true);   // the drafter under --kv k8v4: plain INT8
     uint64_t sb = qsa_state_bytes(g, max_cells, false, ring);
     if (cudaMalloc(&state_arena_, sb) != cudaSuccess) { err = "mtp: the K/V state does not fit"; return false; }
     if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[0], ring) == 0) {
@@ -187,6 +220,8 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
         if (cudaMalloc(&state_arena_, sb) != cudaSuccess) { err = "mtp: the K/V state does not fit"; return false; }
         if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[0], ring) == 0) { err = "mtp: state init failed"; return false; }
     }
+    qsa_set_kv_int8(kv_int8_was);
+    qsa_set_kv_hybrid(kv_hybrid_was);
     qsa_state_zero(st_, g, nullptr);
     cudaDeviceSynchronize();
     vram_ += sb;
@@ -248,12 +283,28 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
         cudaMemcpy(ident_, id.data(), id.size() * 4, cudaMemcpyHostToDevice);
     }
     if (cudaStreamCreateWithFlags(&cs_, cudaStreamNonBlocking) != cudaSuccess) { err = "mtp: stream"; return false; }
-    std::fprintf(stderr, "strata mtp: draft layer loaded, %.0f MiB of VRAM (experts %.0f, dense %.0f)\n",
+    const double files_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - t_files).count();
+    std::fprintf(stderr, "strata mtp: draft layer loaded, %.0f MiB of VRAM (experts %.0f, dense %.0f), files read in %.2f s (%.0f MiB/s)\n",
                  (double) vram_ / 1048576.0, (double) g.n_expert * strata::kernels::cpu::BLOB / 1048576.0,
-                 (double) tensors_.back().off / 1048576.0);
+                 (double) tensors_.back().off / 1048576.0, files_s,
+                 files_s > 0 ? ((double) g.n_expert * strata::kernels::cpu::BLOB + (double) tensors_.back().off) /
+                                   1048576.0 / files_s : 0.0);
     return true;
 }
 
+uint64_t MtpDrafter::bind_bytes(uint64_t head_row_bytes, int64_t n_vocab) const {
+    uint64_t bytes = head_logits_ ? 0 : (uint64_t) max_t_ * (uint64_t) n_vocab * sizeof(float);
+    if (dhead_ == nullptr) {
+        if (FILE* f = std::fopen((rt_dir_ + "/draft_vocab.bin").c_str(), "rb")) {
+            std::fseek(f, 0, SEEK_END);
+            const long size = std::ftell(f);
+            std::fclose(f);
+            if (size >= 4 && size % 4 == 0) bytes += (uint64_t) (size / 4) * head_row_bytes + (uint64_t) size;
+        }
+    }
+    return bytes;
+}
+
 bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err) {
     const OnDevice on_device(device_);
     wt_ = &wt;

src/core/native_dense.cpp

@@ -16,9 +16,9 @@ namespace {
 bool eligible(const strata::TensorInfo& tensor, bool include_ple_key) {
     const auto& name = tensor.name;
     if (name.rfind("blk.", 0) != 0) return false;
-    // The native PLE kernel accepts Q2_0. Other quantized keys can be converted to BF16 by iq_pack;
-    // keep their packed bytes resident instead of overriding them with an unsupported native key.
-    if (name == "blk.1.ple_key.weight") return include_ple_key && tensor.type == 42;   // 42 = Q2_0
+    // Match the native PLE kernel: Q2_0, IQ3_XXS and IQ4_XS. Other keys retain the packed BF16 fallback.
+    if (name == "blk.1.ple_key.weight")
+        return include_ple_key && (tensor.type == 42 || tensor.type == 18 || tensor.type == 23);
     static const char* suffixes[] = {".attn_qkv.weight", ".attn_gate.weight", ".ssm_out.weight",
         ".attn_q.weight", ".attn_k.weight", ".attn_v.weight", ".attn_output.weight",
         ".ffn_gate_shexp.weight", ".ffn_up_shexp.weight", ".ffn_down_shexp.weight"};

src/core/pinned.cu

@@ -12,6 +12,14 @@
 #include <mutex>
 #include <thread>
 
+// Loader fix: `fseek`/`ftell` are 32-bit on Windows by default (and the pack is 42.9 GB), and the 64-bit
+// spelling is not the same on the two platforms the engine builds for.
+#ifdef _WIN32
+#define STRATA_FSEEK64(f, o) _fseeki64((f), (long long) (o), SEEK_SET)
+#else
+#define STRATA_FSEEK64(f, o) fseeko((f), (off_t) (o), SEEK_SET)
+#endif
+
 #ifdef _WIN32
 #include <windows.h>
 #else
@@ -234,11 +242,33 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::
     const auto t0 = std::chrono::steady_clock::now();
     std::vector<uint64_t> layer_hash((size_t) layers, 1469598103934665603ull);
     std::atomic<uint64_t> next_layer{0};
+    std::atomic<uint64_t> read_ns_sum{0};   // summed over the threads: see LoadStats::read_seconds
+    std::atomic<uint64_t> copy_ns_sum{0};
     std::mutex err_mu;
     std::string err;
 
     auto worker = [&]() {
         std::vector<uint8_t> buf((size_t) chunk);
+        // One handle per thread, seeked once per layer: a shared handle would need a lock around the seek and
+        // would serialise the very thing the threads are here to parallelise.  `fread` on a `FILE*` rather than
+        // `std::ifstream`: see the header - MSVC's `basic_filebuf::xsgetn` splits any request larger than
+        // `_INTERNAL_BUFSIZ - 1` into 4095-byte freads, which turned one 8 MiB chunk into ~2048 4 KiB reads.
+        // `fread` sees a request bigger than the stream buffer and passes it to `_read()`/
+        // `ReadFile()` unchanged, so the chunk size reaches the disk.  Buffered, not `FILE_FLAG_NO_BUFFERING`:
+        // the cache should still hold what it can.
+        FILE* f = std::fopen(path.c_str(), "rb");
+        if (f == nullptr) {
+            std::lock_guard<std::mutex> g(err_mu);
+            err = "cannot open " + path;
+            return;
+        }
+        // A `FILE*` has no destructor that closes it, and this function has early returns below (open, seek and
+        // short-read failures), so the guard is what keeps the closing correct on every path.
+        struct Closer {
+            FILE* f;
+            ~Closer() { if (f != nullptr) std::fclose(f); }
+        } closer{f};
+        uint64_t read_ns = 0, copy_ns = 0;
         for (;;) {
             const uint64_t L = next_layer.fetch_add(1);
             if (L >= layers) break;
@@ -246,30 +276,39 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::
             uint64_t remaining = layer_bytes[(size_t) L];
             uint64_t pos = 0;
             uint64_t h = 1469598103934665603ull;
-            // one handle per thread, seeked once per layer: a shared handle would need a lock around the seek
-            // and would serialise the very thing the threads are here to parallelise
-            std::ifstream f(path, std::ios::binary);
-            if (!f) {
+            // 64-bit seek: the pack is 42.9 GB, so the 32-bit `fseek` would wrap past 4 GiB
+            if (STRATA_FSEEK64(f, off) != 0) {
                 std::lock_guard<std::mutex> g(err_mu);
-                err = "cannot open " + path;
+                err = "seek to " + std::to_string(off) + " B failed in layer " + std::to_string(L);
                 return;
             }
-            f.seekg((std::streamoff) off);
             while (remaining > 0) {
                 const uint64_t n = remaining < chunk ? remaining : chunk;
-                f.read((char*) buf.data(), (std::streamsize) n);
-                if ((uint64_t) f.gcount() != n) {
+                const auto t_read = std::chrono::steady_clock::now();
+                const size_t got = std::fread(buf.data(), 1, (size_t) n, f);
+                read_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(
+                               std::chrono::steady_clock::now() - t_read).count();
+                // A short read is EOF or an I/O error, never a silent zero fill: say WHERE and HOW SHORT, and
+                // keep going no further - the caller turns this into a refused load, not a wrong answer.
+                if (got != (size_t) n) {
                     std::lock_guard<std::mutex> g(err_mu);
-                    err = "short read in layer " + std::to_string(L);
+                    err = "short read in layer " + std::to_string(L) + ": got " + std::to_string(got) + " of "
+                          + std::to_string(n) + " B at offset " + std::to_string(off + pos)
+                          + (std::ferror(f) != 0 ? " (ferror set)" : "");
                     return;
                 }
+                const auto t_copy = std::chrono::steady_clock::now();
                 std::memcpy(dst + off + pos, buf.data(), (size_t) n);
                 h = fnv1a64(buf.data(), n, h);
+                copy_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(
+                               std::chrono::steady_clock::now() - t_copy).count();
                 pos += n;
                 remaining -= n;
             }
             layer_hash[(size_t) L] = h;
         }
+        read_ns_sum.fetch_add(read_ns);
+        copy_ns_sum.fetch_add(copy_ns);
     };
 
     std::vector<std::thread> pool;
@@ -280,8 +319,12 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::
     if (!err.empty()) {
         std::fprintf(stderr, "load_experts: %s\n", err.c_str());
         st.seconds = -1.0;
+        st.ok = false;
+        st.error = err;
         return st;
     }
+    st.read_seconds = (double) read_ns_sum.load() / 1e9;
+    st.copy_seconds = (double) copy_ns_sum.load() / 1e9;
     st.layer_checksums = std::move(layer_hash);
     st.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
     return st;

src/core/remote_experts.cpp

@@ -73,9 +73,12 @@ bool RemoteExperts::preflight(int device, double& free_gib, std::string& err) {
     // The layer waits for this GPU on the CPU pool's critical path: spin instead of sleeping, whose wake-up
     // costs more than a small expert batch takes (measured: ~0.3 ms per round trip on Windows).  Only possible
     // before the device's context exists, so first thing; STRATA_REMOTE_SPIN=0 keeps the driver's default.
+#if !defined(STRATA_USE_HIP)
     const char* spin = std::getenv("STRATA_REMOTE_SPIN");
     if (!(spin && spin[0] == '0')) cudaInitDevice(device, cudaDeviceScheduleSpin | cudaDeviceMapHost, 0);
     cudaGetLastError();
+#endif
+    // HIP has no cudaInitDevice equivalent; retain its default scheduling policy.
     DeviceScope scope(device);
     if (!scope.ok) { err = scope.error(device); return false; }
     size_t free_bytes = 0, total_bytes = 0;

src/core/verify.cpp

@@ -9,6 +9,8 @@
 #include "strata/kernels/iq_kernels.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
+#include "strata/kernels/native_router.hpp"
+#include "strata/kernels/native_moe.hpp"
 #include "strata/kernels/cpu/expert.hpp"
 #include "strata/kernels/elementwise.hpp"
 #include "strata/kernels/fused_gr.hpp"
@@ -33,7 +35,11 @@
 #include "strata/kernels/shared_expert.hpp"
 #include "strata/kernels/verify_kernels.hpp"
 
+#include <algorithm>
 #include <atomic>
+#include <map>
+#include <string>
+#include <vector>
 #include <chrono>
 #include <cmath>
 #include <cstdio>
@@ -142,7 +148,8 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
         return false;
     }
     if (hits.d_res == nullptr || hits.cache_base == nullptr || hits.blob <= 0) {
-        err = "verify: needs the profile-filled VRAM expert tier (--expert-profile and --expert-cache)";
+        err = "verify: needs the profile-filled VRAM expert tier (--expert-profile and --expert-cache); with "
+              "--expert-cache auto, no VRAM was left for it: lower --max-context, use --kv k8v4 or images on the CPU";
         return false;
     }
     std::string why;
@@ -182,7 +189,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
     // ---- mapped staging
     bool ok = mapped(T * 4, (void**) &h_tok_, (void**) &m_tok_) &&
               mapped(T * strata::kernels::kStepCount * 4, (void**) &h_step_, (void**) &m_step_) &&
-              mapped(T * NH * 4, (void**) &h_pos_, (void**) &m_pos_) &&
+              mapped(T * (NH + NKV + IQ) * 4, (void**) &h_pos_, (void**) &m_pos_) &&
               mapped((2 + T) * 4 + 16, (void**) &h_commit_, (void**) &m_commit_) &&
               mapped(T * N * 4, (void**) &h_ple_, (void**) &m_ple_) &&
               mapped(T * 4 + 16, (void**) &h_out_, (void**) &m_out_) &&
@@ -221,7 +228,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
     // ---- the device arena: the same sequence counted, then carved
     auto carve = [&](Bump& b) {
         tok_ = b.take<int32_t>(T); step_ = b.take<int32_t>(T * strata::kernels::kStepCount);
-        pos_ = b.take<int32_t>(T * NH); commit_ = b.take<int32_t>(2 + T);
+        pos_ = b.take<int32_t>(T * (NH + NKV + IQ)); commit_ = b.take<int32_t>(2 + T);
         ple_ = b.take<float>(T * N); emb_ = b.take<float>(T * N); R_ = b.take<float>(T * HC * N);
         mixed_ = b.take<float>(T * N); bo_ = b.take<float>(T * N);
         inj_ = b.take<float>(T * HC); inj2_ = b.take<float>(T * HC);
@@ -260,6 +267,12 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
         return false;
     }
     cudaMemset(arena_, 0, count.used);
+    prof_on_ = std::getenv("STRATA_VERIFY_PROFILE") != nullptr;
+    if (prof_on_) {
+        const size_t np = (size_t) g.n_layers * kProfPer + 4;
+        if (cudaMalloc((void**) &prof_, np * 8) != cudaSuccess) { prof_on_ = false; prof_ = nullptr; cudaGetLastError(); }
+        else { cudaMemset(prof_, 0, np * 8); prof_h_.assign(np, 0); }
+    }
     Bump real;
     real.base = (uint8_t*) arena_;
     carve(real);
@@ -323,13 +336,18 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
     const bool ple_on = ss.ple.ready() && ple_stage();
     auto Rt = [&](int t) { return R_ + (size_t) t * HC * N; };
     const int G = (split_ && T >= 2) ? 2 : 1;
+    static const bool dec_batch = [] { const char* v = std::getenv("STRATA_DEC_BATCH"); return v == nullptr || std::atoi(v) != 0; }();
+    auto stamp = [&](int64_t l, int i, int grp) { if (prof_on_ && grp == 0) gpu_stamp(prof_, (int) (l * kProfPer + i), cs); };
     const int tb_[2] = {0, (T + 1) / 2}, te_[2] = {G == 2 ? (T + 1) / 2 : T, T};
     groups_[T] = G;
 
     // ---- the window's inputs, from mapped staging
     copy_i32_from_mapped(tok_, m_tok_, T, cs);
     copy_i32_from_mapped(step_, m_step_, (int64_t) T * kStepCount, cs);
-    copy_i32_from_mapped(pos_, m_pos_, (int64_t) T * NH, cs);
+    copy_i32_from_mapped(pos_, m_pos_, (int64_t) MT * (NH + NKV + IQ), cs);
+    // per-ROW positions of the K rows [t][NKV] and the indexer query rows [t][IQ] (for batched RoPE)
+    const int32_t* pos_k = pos_ + MT * NH;
+    const int32_t* pos_i = pos_ + MT * (NH + NKV);
     if (ple_on) copy_from_mapped(ple_, m_ple_, (int64_t) T * N, cs);
 
     // ---- the embeddings, broadcast to the hc streams - or, in a later stage of a layer split, the previous stage's
@@ -373,6 +391,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
     // ---------------------------------------------------------------- pre(l, group): up to the ring
     auto pre = [&](int64_t l, int grp) -> bool {
         const int tb = tb_[grp], te = te_[grp], n = te - tb;
+        stamp(l, 0, grp);
         const LayerView v(wt, l);
         const char* pfx[2] = {"hc_attn_", "hc_ffn_"};
         const WeightRef *wn[2], *wd[2], *wu[2], *wi[2];
@@ -415,9 +434,11 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 a.eps = EPS; a.lo = lo_ + t * g.hc_lr; a.rs = rs_ + t * HC;
                 a.inject_out = inj_out + t * HC; a.mixed = mixed_ + t * N;
             }
-            fused_gr_read_multi(fa, n, xn_ + (size_t) tb * HC * N, cs);
+            fused_gr_read_multi(fa, n, xn_ + (size_t) tb * HC * N, cs, (prof_on_ && grp == 0) ? prof_ : nullptr,
+                                (int) (l * kProfPer + (half == 0 ? 27 : 30)));
         };
         gr_read_group(0, pending, inj2_, inj_);
+        stamp(l, 1, grp);
         float* xm = mixed_ + tb * N;
         try {
             if (!is_qsa_layer(g, l)) {
@@ -440,14 +461,19 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 float* beta = beta_L_ + (size_t) gi * MT * HV;
                 native_quantize_q8_1(xm, xq_, (int) N, n, cs);
                 native_mmvq(wqkv->native_type, wqkv->native_data, xq_, qkv + (size_t) tb * C, (int) N, (int) C, n, cs);
+                stamp(l, 2, grp);
                 gdn_conv_l2_multi(conv, qkv, (const float*) wc->data, hb, (int) C, (int) (2 * HK), EPS, n, cs, tb);
+                stamp(l, 3, grp);
                 gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data,
                              (const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N, (int) HV,
                              n, cs);
+                stamp(l, 4, grp);
                 native_mmvq(wg->native_type, wg->native_data, xq_, z_ + (size_t) tb * ZV, (int) N, (int) ZV, n, cs);
+                stamp(l, 5, grp);
                 // the recurrence from the untouched state over tokens [0, te); outputs only for this group's
                 gdn_step_norm_multi(state, hb, (int) C, gate, beta, z_, (const float*) wnm->data, EPS, y_, (int) HK,
                                     (int) HV, te, nullptr, cs, tb);
+                stamp(l, 6, grp);
                 native_quantize_q8_1(y_ + (size_t) tb * ZV, xq_, (int) ZV, n, cs);
                 native_mmvq(wout->native_type, wout->native_data, xq_, bo_ + tb * N, (int) ZV, (int) N, n, cs);
             } else {
@@ -470,20 +496,34 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     else rope_neox_apply(data, data, rows, cols, (int) s.n_rot, st.cos_tab, st.sin_tab, pos, cs);
                 };
                 float* idx_raw = idx_raw_L_ + (size_t) qi * MT * ID;
+                // the per-token GEMVs / norms / RoPEs / copies of this layer as one launch over the
+                // window's rows each - row-wise identical arithmetic (STRATA_DEC_BATCH=0: token by token)
+                const bool qb = dec_batch && n > 1 && native_qsa_enabled() && native_rope_enabled() && !st.kv_q4;
                 native_quantize_q8_1(xm, xq_, (int) N, n, cs);
-                for (int t = tb; t < te; ++t)
+                if (qb) bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wik->data, idx_raw + tb * ID, ID, N, ID, n, cs);
+                else for (int t = tb; t < te; ++t)
                     bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wik->data, idx_raw + t * ID, (int) N, (int) ID, cs);
+                stamp(l, 7, grp);
                 native_mmvq(wk->native_type, wk->native_data, xq_, kcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD), n, cs);
                 native_mmvq(wv->native_type, wv->native_data, xq_, vcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD), n, cs);
-                for (int t = tb; t < te; ++t) norm_rope(kcur_ + t * NKV * HD, wkn, (int) NKV, (int) HD, pos_ + t * NH);
+                if (qb) norm_rope(kcur_ + tb * NKV * HD, wkn, (int) (n * NKV), (int) HD, pos_k + tb * NKV);
+                else for (int t = tb; t < te; ++t) norm_rope(kcur_ + t * NKV * HD, wkn, (int) NKV, (int) HD, pos_ + t * NH);
                 if (st.kv_q4) {   // Q4_0 KV (kv_q4.hpp): K and V rotated before they are stored
                     fwht256_inplace_cuda(kcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
                     fwht256_inplace_cuda(vcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
+                } else if (st.kv_hybrid) {   // K8V4: only V is rotated
+                    fwht256_inplace_cuda(vcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
                 }
+                stamp(l, 8, grp);
                 if (grp == 0) copy_from_mapped(tail_snap_ + (size_t) qi * TS, st.idx_tail, TS, cs);
                 for (int t = tb; t < te; ++t) {
                     const int32_t* step_t = step_ + t * kStepCount;
-                    if (st.kv_q4)
+                    if (st.kv_hybrid) {   // K8V4: the unused half's lanes folded onto the used pool (layer.cpp)
+                        kv_append_q8_step(st.k_q, st.k_q, st.k_scale, st.k_scale, st.page_table, step_t,
+                                          kcur_ + t * NKV * HD, kcur_ + t * NKV * HD, s, cs, nullptr);
+                        kv_append_q4_step(st.v_q4, st.v_q4, st.page_table, step_t, vcur_ + t * NKV * HD,
+                                          vcur_ + t * NKV * HD, s, cs, nullptr);
+                    } else if (st.kv_q4)
                         kv_append_q4_step(st.k_q4, st.v_q4, st.page_table, step_t, kcur_ + t * NKV * HD,
                                           vcur_ + t * NKV * HD, s, cs, &st.host);
                     else if (st.kv_int8)
@@ -498,8 +538,20 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     native_qsa_indexer_append(idx_raw + t * ID, step_ + t * kStepCount + kStepPos, 0,
                                               (const float*) wikn->data, EPS, ib, s, st.max_cells,
                                               (float) qsa_freq_base(), cs);
+                stamp(l, 9, grp);
                 native_mmvq(wq->native_type, wq->native_data, xq_, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD),
                             n, cs);
+                if (qb) {
+                    if (cudaMemcpy2DAsync(qcur_ + tb * NH * HD, (size_t) HD * 4, qfull_ + tb * NH * 2 * HD, (size_t) HD * 2 * 4,
+                                          (size_t) HD * 4, (size_t) (n * NH), cudaMemcpyDeviceToDevice, cs) != cudaSuccess) {
+                        err = "verify: the q/gate split failed";
+                        return false;
+                    }
+                    norm_rope(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH);
+                    bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID,
+                                              N, IQ * ID, n, cs);
+                    norm_rope(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ);
+                } else {
                 for (int t = tb; t < te; ++t) {
                     float* qc = qcur_ + t * NH * HD;
                     if (cudaMemcpy2DAsync(qc, (size_t) HD * 4, qfull_ + t * NH * 2 * HD, (size_t) HD * 2 * 4,
@@ -515,16 +567,24 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), cs);
                     norm_rope(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH);
                 }
+                }
+                stamp(l, 10, grp);
                 qsa_block_scores(st.idx_pooled, st.idx_dead, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n, max_blocks_,
                                  s, scores_ + (size_t) tb * max_blocks_, cs);
                 qsa_block_topk(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n, max_blocks_, cap_, s,
                                sel_ + (size_t) tb * cap_, cs);
+                stamp(l, 11, grp);
                 // KV streaming: the n selections' blocks resident (device-side, inside the graph)
                 qsa_kv_resolve(st, *g_, sel_ + (size_t) tb * cap_, step_ + tb * kStepCount, n, cap_, cs);
+                stamp(l, 12, grp);
                 const QsaAttnPools pools = qsa_attn_pools(st);
                 qsa_decode_attn_batch(qcur_ + tb * NH * HD, pools, sel_ + (size_t) tb * cap_, step_ + tb * kStepCount, cap_,
                                       s, attn_scratch_ + (size_t) tb * attn_scratch_floats_, attn_ + tb * NH * HD, n, cs);
-                if (st.kv_q4) fwht256_inplace_cuda(attn_ + tb * NH * HD, (int64_t) n * NH, cs);   // back: H^-1 = H
+                stamp(l, 13, grp);
+                if (st.kv_q4 || st.kv_hybrid) fwht256_inplace_cuda(attn_ + tb * NH * HD, (int64_t) n * NH, cs);   // back: H^-1 = H
+                if (qb) native_qsa_gate_apply(attn_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, attn32_ + tb * NH * HD,
+                                              (int) (n * NH), (int) HD, cs);
+                else
                 for (int t = tb; t < te; ++t) {
                     if (native_qsa_enabled())
                         native_qsa_gate_apply(attn_ + t * NH * HD, qfull_ + t * NH * 2 * HD, attn32_ + t * NH * HD,
@@ -532,6 +592,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     else
                         qsa_gate_apply_f32(attn_ + t * NH * HD, qfull_ + t * NH * 2 * HD, s, attn32_ + t * NH * HD, cs);
                 }
+                stamp(l, 14, grp);
                 native_quantize_q8_1(attn32_ + tb * NH * HD, xq_, (int) (NH * HD), n, cs);
                 native_mmvq(wo->native_type, wo->native_data, xq_, bo_ + tb * N, (int) (NH * HD), (int) N, n, cs);
             }
@@ -539,7 +600,18 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             err = "verify layer " + std::to_string(l) + ": " + e.what();
             return false;
         }
+        stamp(l, 16, grp);
         gr_read_group(1, true, inj_, inj2_);
+        // the window's rows routed in 2 launches (one router GEMV reading the weight once, one
+        // top-10) instead of 2 per token; every row's arithmetic is the single-token call's (STRATA_DEC_BATCH=0: old)
+        const WeightRef* w_router = v.get("ffn_gate_inp.weight");
+        if (dec_batch && n > 1 && w_router != nullptr && native_router_enabled() && NE == 512 && K == 10) {
+            try {
+                bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) w_router->data, logits_ + tb * NE, NE, N,
+                                          NE, n, cs);
+                native_router_top10_multi(logits_ + tb * NE, ids_ + tb * K, w_ + tb * K, n, cs);
+            } catch (const std::exception& e) { err = "verify router: " + std::string(e.what()); return false; }
+        } else
         for (int t = tb; t < te; ++t) {
             MoEBuffers mb = ss.moe;
             mb.logits = logits_ + t * NE; mb.ids = ids_ + t * K; mb.weights = w_ + t * K;
@@ -552,6 +624,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                           (uint32_t) ((l - lb_) * G + grp + 1), cs);
         doorbell_publish(xm, ids_ + tb * K, w_ + tb * K, (int64_t) n * N, (int64_t) n * K, m_x_ + tb * N,
                          m_ids_ + tb * K, m_w_ + tb * K, m_seq_, cs);
+        stamp(l, 17, grp);
         {
             const WeightRef *wgi = need(v, "ffn_gate_inp_shexp.weight", err), *wsg = need(v, "ffn_gate_shexp.weight", err),
                             *wsu = need(v, "ffn_up_shexp.weight", err), *wsd = need(v, "ffn_down_shexp.weight", err);
@@ -564,7 +637,8 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             nsw.up_type = wsu->native_type; nsw.up_data = wsu->native_data;
             nsw.down_type = wsd->native_type; nsw.down_data = wsd->native_data;
             nsw.q8_1 = xq_;
-            for (int t = tb; t < te; ++t) f32_to_bf16_bulk(mixed_ + t * N, sh_bf16_ + t * N, N, cs);
+            if (dec_batch) f32_to_bf16_bulk(mixed_ + tb * N, sh_bf16_ + tb * N, (int64_t) n * N, cs);   // contiguous rows
+            else for (int t = tb; t < te; ++t) f32_to_bf16_bulk(mixed_ + t * N, sh_bf16_ + t * N, N, cs);
             try {
                 shared_expert_multi(n, xm, sh_bf16_ + tb * N, nsw, (const uint16_t*) wgi->data, sh_gate_ + (size_t) tb * g.n_ff,
                                     sh_up_ + (size_t) tb * g.n_ff, sh_g_ + tb, shared_ + tb * N, N, g.n_ff, cs);
@@ -577,6 +651,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             quantize_q8_1_rows(xm, n, N, nat_xq_ + (size_t) tb * (N / 32) * 36, cs);
         else
             quantize_q8_0_scaled(xm, hit_xq_ + (size_t) tb * (N / 32) * 34, hit_xs_ + (size_t) tb * (N / 32), (int64_t) n * N, cs);
+        stamp(l, 18, grp);
         return true;
     };
 
@@ -593,6 +668,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             wait_flag_ge(m_flagA_, ring, cs);                  // the pool published this group's GPU plan
             copy_i32_from_mapped(pl, m_plan_ + (size_t) grp * (size_t) plan_i32_, plan_i32_, cs);
         }
+        stamp(l, 19, grp);
         const int32_t* p_counts = pl;
         const int32_t* p_start = pl + 4;
         const int32_t* p_dst = p_start + capx + 1;
@@ -617,6 +693,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             }
         };
         grouped(p_ptr, p_start, p_counts);
+        stamp(l, 20, grp);
         if (device_plan_) wait_flag_ge_or(m_flagB_, ring, skip_ + grp, cs);
         else wait_flag_ge(m_flagB_, ring, cs);                 // the PCIe share is in staging (DMA) or mapped
         if (sink_.pcie_mode == 2) {                            // stage it with a copy kernel, then point at staging
@@ -625,21 +702,34 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             fetch_blobs(p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), (int) per, cs);
             rebase_ptrs((unsigned long long*) p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), cs);
         }
+        stamp(l, 21, grp);
         grouped(p_ptr2, p_start2, p_counts + 2);
+        stamp(l, 22, grp);
         if (device_plan_) {   // no CPU share when the device planned the group: its rows are zeros
             wait_flag_ge_or(m_flag_, ring, skip_ + grp, cs);
             copy_or_zero_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (long long) n * K * N,
                                      skip_ + grp, ring, cs);
         } else {
-            wait_flag_ge(m_flag_, ring, cs);                   // the CPU's share is in the mapped rows
-            copy_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (int64_t) n * K * N, cs);
+            wait_flag_ge(m_flag_, ring, cs);               // the CPU's share is in the mapped rows
+            stamp(l, 23, grp);
+            if (dec_batch)   // only the CPU rows cross PCIe (p_dst[0, counts[1]) = the GPU's own rows)
+                copy_rows_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (int64_t) n * K, N,
+                                      p_dst, p_counts + 1, cs);
+            else
+                copy_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (int64_t) n * K * N, cs);
         }
         moe_hit_add(parts_ + (size_t) tb * K * N, hit_out, p_dst, p_counts + 1, cap, N, cs);
+        if (dec_batch && n > 1 && native_moe_combine_enabled()) {   // one launch for the window's rows
+            try {
+                native_moe_combine_multi(parts_ + (size_t) tb * K * N, w_ + tb * K, shared_ + tb * N, bo_ + tb * N, N, K, n, cs);
+            } catch (const std::exception& e) { err = "verify combine: " + std::string(e.what()); return false; }
+        } else
         for (int t = tb; t < te; ++t) {
             MoEBuffers mb = ss.moe;
             mb.weights = w_ + t * K; mb.shared = shared_ + t * N;
             if (!moe_combine_parts(g, l, K, mb, parts_ + (size_t) t * K * N, bo_ + t * N, cs, err)) return false;
         }
+        stamp(l, 24, grp);
         if (l == g.n_layers - 1) {
             for (int t = tb; t < te; ++t) gr_write(Rt(t), bo_ + t * N, inj2_ + t * HC, gs, Rt(t), cs);
             if (cvec().covers(l)) cvec_apply(Rt(tb), l, n, HC * N, nullptr, 0, nullptr, 0, false, cs);
@@ -666,6 +756,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
     }
 
     // ---- the head, T columns, and the argmax of each
+    stamp(g.n_layers, 0, 0);
     {
         const WeightRef *hn = wt.find("output_hc_norm.weight"), *hd = wt.find("output_hc_down.weight"),
                         *hu = wt.find("output_hc_up.weight");
@@ -697,9 +788,36 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
         sp.temperature = 0.0f;
         sample_tokens(head_logits_, T, (int) n_vocab_, nullptr, 0, sp, m_out_, cs);
     }
+    stamp(g.n_layers, 1, 0);
     return true;
 }
 
+std::string Verifier::profile_report() {
+    if (!prof_on_ || prof_windows_ == 0) return std::string();
+    static const char* names[kProfPer] = {"-", "hc-read0", "q8+qkv/q-idx gemv", "conv", "ab", "z", "rec", "q8+kv-idx",
+                                          "k/v+norm-rope", "kv+idx append", "q+q-idx", "scores+topk", "kv-resolve",
+                                          "attention", "gate", "", "out-proj", "hc-read1+router", "shared+quant",
+                                          "waitA", "VRAM hits", "waitB", "PCIe grp", "waitCPU", "copy+combine",
+                                          "(gap)", "head", "  hc0 norm", "  hc0 down", "  hc0 up", "", ""};
+    std::string out;
+    char b[80];
+    double total = 0;
+    for (int k = 0; k < 2; ++k) {
+        out += k == 0 ? " GDN layers:" : " | QSA layers:";
+        for (int i = 0; i < kProfPer; ++i) {
+            if (prof_sum_[k][i] <= 0) continue;
+            total += prof_sum_[k][i];
+            std::snprintf(b, sizeof b, " %s %.2f", names[i], prof_sum_[k][i] / 1e6 / (double) prof_windows_);
+            out += b;
+        }
+    }
+    std::snprintf(b, sizeof b, " | total %.2f ms/window over %lld windows", total / 1e6 / (double) prof_windows_, (long long) prof_windows_);
+    out += b;
+    for (auto& r : prof_sum_) for (double& d : r) d = 0;
+    prof_windows_ = 0;
+    return out;
+}
+
 bool Verifier::capture(int T, std::string& err) {
     if (exec_[T] != nullptr) return true;
     if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) {
@@ -719,6 +837,37 @@ bool Verifier::capture(int T, std::string& err) {
         err = std::string("verify: end capture: ") + cudaGetErrorString(ce);
         return false;
     }
+#if !defined(STRATA_USE_HIP)   // a CUDA debug listing (node types, kernel names)
+    if (std::getenv("STRATA_VERIFY_NODES") != nullptr) {   // what the window graph holds
+        size_t nn = 0;
+        cudaGraphGetNodes(graph, nullptr, &nn);
+        std::vector<cudaGraphNode_t> nodes(nn);
+        cudaGraphGetNodes(graph, nodes.data(), &nn);
+        std::map<std::string, int> kinds;
+        for (cudaGraphNode_t nd : nodes) {
+            cudaGraphNodeType ty;
+            cudaGraphNodeGetType(nd, &ty);
+            std::string name = "type" + std::to_string((int) ty);
+            if (ty == cudaGraphNodeTypeKernel) {
+                cudaKernelNodeParams kp{};
+                if (cudaGraphKernelNodeGetParams(nd, &kp) == cudaSuccess) {
+#if CUDART_VERSION >= 12030   // cudaFuncGetName arrived in CUDA 12.3
+                    const char* fn = nullptr;
+                    if (cudaFuncGetName(&fn, kp.func) == cudaSuccess && fn) name = fn;
+#endif
+                }
+            } else if (ty == cudaGraphNodeTypeMemcpy) name = "memcpy";
+            else if (ty == cudaGraphNodeTypeMemset) name = "memset";
+            ++kinds[name];
+        }
+        std::vector<std::pair<int, std::string>> v;
+        for (auto& [k2, c] : kinds) v.push_back({c, k2});
+        std::sort(v.rbegin(), v.rend());
+        std::fprintf(stderr, "strata verify: the %d-token window graph has %zu nodes:", T, nn);
+        for (size_t i = 0; i < v.size() && i < 40; ++i) std::fprintf(stderr, " %d x %.60s;", v[i].first, v[i].second.c_str());
+        std::fprintf(stderr, "\n");
+    }
+#endif
     const cudaError_t ie = cudaGraphInstantiate(&exec_[T], graph, 0);
     cudaGraphDestroy(graph);
     if (ie != cudaSuccess) {
@@ -814,6 +963,10 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
         h_tok_[t] = tokens[t];
         qsa_step_fill(h_step_ + t * kStepCount, pos0 + t, s);
         for (int64_t h = 0; h < g.n_head; ++h) h_pos_[t * g.n_head + h] = (int32_t) (pos0 + t);
+        int32_t* pk = h_pos_ + (size_t) max_t_ * g.n_head;
+        int32_t* pi = pk + (size_t) max_t_ * g.n_head_kv;
+        for (int64_t h = 0; h < g.n_head_kv; ++h) pk[t * g.n_head_kv + h] = (int32_t) (pos0 + t);
+        for (int64_t h = 0; h < g.idx_q_heads; ++h) pi[t * g.idx_q_heads + h] = (int32_t) (pos0 + t);
     }
     if (ss.ple.ready() && ple_stage()) {
         uint32_t rows[kVerifyMaxT * PLE_N_HEADS];
@@ -898,6 +1051,28 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
     if (se != cudaSuccess) { err = std::string("verify: ") + cudaGetErrorString(se); return false; }
     progress_at("verify window: waiting for the expert copies", (int64_t) T);
     cudaStreamSynchronize(copy_);   // no host function of this window may raise flag B in the next one
+    if (prof_on_ && G == 1) {       // the window's GPU stage stamps
+        cudaMemcpy(prof_h_.data(), prof_, prof_h_.size() * 8, cudaMemcpyDeviceToHost);
+        const int64_t L = g.n_layers;
+        auto at = [&](int64_t l, int i) { return prof_h_[(size_t) (l * kProfPer + i)]; };
+        for (int64_t l = 0; l < L; ++l) {
+            const int kind = is_qsa_layer(g, l) ? 1 : 0;
+            unsigned long long prev = at(l, 0);
+            for (int i = 1; i <= 24; ++i) {
+                const unsigned long long x = at(l, i);
+                if (x == 0 || x < prev) continue;
+                prof_sum_[kind][i] += (double) (x - prev);
+                prev = x;
+            }
+            if (l + 1 < L) prof_sum_[kind][25] += (double) (at(l + 1, 0) - at(l, 24));
+            prof_sum_[kind][27] += (double) (at(l, 27) - at(l, 0));    // hc-read0: norm
+            prof_sum_[kind][28] += (double) (at(l, 28) - at(l, 27));   //           down
+            prof_sum_[kind][29] += (double) (at(l, 1) - at(l, 28));    //           up
+            prof_sum_[kind][1] -= (double) (at(l, 1) - at(l, 0));      // (hc-read0 shown split)
+        }
+        prof_sum_[0][26] += (double) (at(L, 1) - at(L, 0));
+        ++prof_windows_;
+    }
     // ---- a sampled or penalized request: the head's sampling again, host-side so its parameters are this call's
     // own (a captured kernel would replay the same draws forever).  Row t's draw is Philox(seed, pos0 + t): tied to
     // the POSITION it samples, not to how the text was cut into windows, so a seed replays the same text whatever

src/kernels/cuda/elementwise.cu

@@ -225,6 +225,37 @@ __global__ void copy_from_mapped_kernel(float4* __restrict__ dst, const volatile
     }
 }
 
+// the CPU rows of a verify window, skipping the rows the GPU plan computes itself (the
+// pool writes +0.0 into those, so this writes +0.0 too): block = row, the plan's hit rows `dst[0, *count)`.
+__global__ void copy_rows_from_mapped_kernel(float4* __restrict__ dst, const volatile float4* src, int64_t row4,
+                                             const int32_t* __restrict__ hit_rows, const int32_t* __restrict__ count) {
+    const int row = blockIdx.x;
+    __shared__ int hit;
+    if (threadIdx.x == 0) {
+        int h = 0;
+        const int c = *count;
+        for (int i = 0; i < c; ++i) h |= hit_rows[i] == row;
+        hit = h;
+    }
+    __syncthreads();
+    float4* d = dst + (int64_t) row * row4;
+    if (hit) {
+        for (int64_t i = threadIdx.x; i < row4; i += blockDim.x) d[i] = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+    } else {
+        const volatile float4* sr = src + (int64_t) row * row4;
+        for (int64_t i = threadIdx.x; i < row4; i += blockDim.x) d[i] = const_cast<const float4*>(sr)[i];
+    }
+}
+void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
+                           const int32_t* count, void* stream) {
+    if (rows <= 0) return;
+    if ((width & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {
+        std::fprintf(stderr, "copy_rows_from_mapped: width must be a multiple of 4 and both pointers 16-byte aligned\n");
+        std::exit(1);
+    }
+    copy_rows_from_mapped_kernel<<<(unsigned) rows, 128, 0, (cudaStream_t) stream>>>((float4*) dst, (const volatile float4*) src,
+                                                                                  width / 4, hit_rows, count);
+}
 void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream) {
     if (n <= 0) return;
     if ((n & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {

src/kernels/cuda/fused_gr.cu

@@ -1,6 +1,7 @@
 // src/kernels/cuda/fused_gr.cu - see include/strata/kernels/fused_gr.hpp.
 #include "strata/kernels/fused_gr.hpp"
 #include "strata/kernels/bf16_bits.hpp"
+#include "strata/kernels/verify_kernels.hpp"
 
 #include <cuda_runtime.h>
 
@@ -178,7 +179,11 @@ __global__ void __launch_bounds__(THREADS) gr_norm_multi_kernel(GrMulti m) {
     for (int i = t; i < D; i += THREADS) xn[i] *= s_rs[i / N];
 }
 
+#if defined(__HIPCC__)
+constexpr int TILE = 1280;             // eight-token tile fits gfx1100's 64 KiB LDS limit
+#else
 constexpr int TILE = 2560;             // xn floats per token staged at a time: 320 chunks of 8, 10 per lane
+#endif
 constexpr int TQ = TILE / 8 / 32;      // uint4 weight chunks per lane per tile
 
 // Step 2 of `gr_down_kernel` for T tokens.  One warp per row (so each lane accumulates the same chunks in the
@@ -295,7 +300,8 @@ __global__ void __launch_bounds__(THREADS) gr_up_multi_kernel(GrMulti m) {
 
 }  // namespace
 
-void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream) {
+void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream, unsigned long long* stamp_buf,
+                         int stamp_i0) {
     if (n_tok < 1 || n_tok > kFusedGrMaxT || xn_scratch == nullptr) {
         std::fprintf(stderr, "fused_gr_read_multi: invalid arguments\n");
         std::exit(1);
@@ -315,22 +321,45 @@ void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi
     m.T = n_tok;
     cudaStream_t st = (cudaStream_t) stream;
     gr_norm_multi_kernel<<<n_tok, THREADS, 0, st>>>(m);
+    if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0, stream);
     // the shared-memory opt-in is a per-DEVICE setting: once per device, not once per process (a layer split
     // runs this kernel on two cards)
     static bool attr[64] = {};
+    static int chunk[64] = {};   // Turing port: tokens the down kernel may carry in one launch on this card
     int dev = 0;
     cudaGetDevice(&dev);
-    if (dev < 0 || dev >= 64 || !attr[dev]) {
+    if (dev >= 0 && dev < 64 && !attr[dev]) {
         // at most what the card allows (Turing: 64 KB - enough for windows of up to 6 tokens)
         int optin = 0;
         cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
         int want = (int) (kFusedGrMaxT * TILE * sizeof(float));
         if (optin > 0 && want > optin) want = optin;
         cudaFuncSetAttribute(gr_down_multi_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, want);
-        cudaGetLastError();
-        if (dev >= 0 && dev < 64) attr[dev] = true;
+        cudaGetLastError();      // drop any error the attempt left behind
+        // Turing port: the down kernel stages n_tok*TILE floats of dynamic shared memory - 80 KB at the full
+        // 8 tokens.  A card whose opt-in is below that (Turing: 64 KB, so 7+ tokens fail to launch as
+        // "invalid argument") processes the tokens in slices that fit; a card that reports no opt-in gets what
+        // fits the 48 KB default (4 tokens of the CUDA tile; all 8 of HIP's smaller tile).  The down kernel's
+        // outputs (lo, inject_out) are strictly per-token, so the chunk boundaries are safe, and the up kernel
+        // below still sees every token of the batch in one launch.
+        const int capacity = (optin > 0 ? optin : 48 * 1024) / (int) (TILE * sizeof(float));
+        chunk[dev] = capacity < 1 ? 1 : (capacity > kFusedGrMaxT ? kFusedGrMaxT : capacity);
+        attr[dev] = true;
     }
-    gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) n_tok * TILE * sizeof(float), st>>>(m);
+    const int chunk_tok = (dev >= 0 && dev < 64 && chunk[dev]) ? chunk[dev] : kFusedGrMaxT;
+    if (chunk_tok >= n_tok) {
+        gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) n_tok * TILE * sizeof(float), st>>>(m);
+    } else {
+        for (int c0 = 0; c0 < n_tok; c0 += chunk_tok) {
+            const int ct = n_tok - c0 < chunk_tok ? n_tok - c0 : chunk_tok;
+            GrMulti c{};
+            c.xn = xn_scratch + (size_t) c0 * D;
+            c.T = ct;
+            for (int k = 0; k < ct; ++k) c.a[k] = a[c0 + k];
+            gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) ct * TILE * sizeof(float), st>>>(c);
+        }
+    }
+    if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0 + 1, stream);
     gr_up_multi_kernel<<<UPM_BLOCKS, THREADS, 0, st>>>(m);
     const cudaError_t e = cudaGetLastError();
     if (e != cudaSuccess) {

src/kernels/cuda/native_bf16.cu

@@ -70,6 +70,54 @@ __global__ void bf16_f32_mmvf_kernel(const float* __restrict__ x, const uint16_t
     if (t == 0) y[blockIdx.x] = acc;
 }
 
+// the same kernel for up to 8 activation rows - the weight row is read ONCE and every
+// token keeps its own accumulator with exactly the single-row kernel's order (pairs, two ordered FMAs, the same warp
+// and block reductions), so each output is bit-identical to a bf16_f32_mmvf_kernel launch of its own.
+template <int BLOCK_SIZE, int NT>
+__global__ void bf16_f32_mmvf_multi_kernel(const float* __restrict__ x, int64_t ldx, const uint16_t* __restrict__ w,
+                                          float* __restrict__ y, int64_t ldy, int n_in, int n_tok) {
+    const int t = threadIdx.x;
+    const uint16_t* row = w + (size_t) blockIdx.x * n_in;
+    const uint32_t* weights2 = reinterpret_cast<const uint32_t*>(row);
+    __shared__ float partials[NT][32];
+    if constexpr (BLOCK_SIZE > 32) {
+        if (t < 32)
+#pragma unroll
+            for (int k = 0; k < NT; ++k) partials[k][t] = 0.0f;
+        __syncthreads();
+    }
+    float acc[NT];
+#pragma unroll
+    for (int k = 0; k < NT; ++k) acc[k] = 0.0f;
+    for (int pair = t; pair < n_in / 2; pair += BLOCK_SIZE) {
+        const uint32_t weight = weights2[pair];
+        const float w0 = f32_from_bf16((uint16_t) weight), w1 = f32_from_bf16((uint16_t) (weight >> 16));
+#pragma unroll
+        for (int k = 0; k < NT; ++k) {
+            if (k < n_tok) {
+                const float2 input = reinterpret_cast<const float2*>(x + (size_t) k * ldx)[pair];
+                acc[k] = __fmaf_rn(w0, input.x, acc[k]);
+                acc[k] = __fmaf_rn(w1, input.y, acc[k]);
+            }
+        }
+    }
+#pragma unroll
+    for (int k = 0; k < NT; ++k) acc[k] = mmvf_warp_sum(acc[k]);
+    if constexpr (BLOCK_SIZE > 32) {
+        if ((t & 31) == 0)
+#pragma unroll
+            for (int k = 0; k < NT; ++k) partials[k][t / 32] = acc[k];
+        __syncthreads();
+        if (t < 32)
+#pragma unroll
+            for (int k = 0; k < NT; ++k) acc[k] = mmvf_warp_sum(partials[k][t]);
+    }
+    if (t == 0)
+#pragma unroll
+        for (int k = 0; k < NT; ++k)
+            if (k < n_tok) y[(size_t) k * ldy + blockIdx.x] = acc[k];
+}
+
 int mmvf_block_size(int64_t n_in) {
     int best = 32;
     int64_t best_iterations = (n_in + 63) / 64;
@@ -85,6 +133,26 @@ int mmvf_block_size(int64_t n_in) {
 
 }  // namespace
 
+void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, float* y, int64_t ldy,
+                               int64_t n_in, int64_t n_out, int n_tok, void* stream) {
+    if (n_tok == 1 && ldy >= n_out) { bf16_gemv_fp32_mmvf(x, w, y, n_in, n_out, stream); return; }
+    if (n_tok < 1 || n_tok > 8 || n_in <= 0 || (n_in & 1) != 0 || n_out <= 0 || (ldx & 1) != 0 || x == nullptr ||
+        w == nullptr || y == nullptr || (reinterpret_cast<uintptr_t>(x) & 7u) != 0)
+        throw std::invalid_argument("bf16_gemv_fp32_mmvf_multi: 1..8 rows, even n_in/ldx, aligned pointers");
+    const cudaStream_t st = (cudaStream_t) stream;
+#define STRATA_MMVF_M(N) case N: \
+    if (n_tok <= 4) bf16_f32_mmvf_multi_kernel<N, 4><<<(unsigned) n_out, N, 0, st>>>(x, ldx, w, y, ldy, (int) n_in, n_tok); \
+    else bf16_f32_mmvf_multi_kernel<N, 8><<<(unsigned) n_out, N, 0, st>>>(x, ldx, w, y, ldy, (int) n_in, n_tok); break
+    switch (mmvf_block_size(n_in)) {
+        STRATA_MMVF_M(32); STRATA_MMVF_M(64); STRATA_MMVF_M(96); STRATA_MMVF_M(128);
+        STRATA_MMVF_M(160); STRATA_MMVF_M(192); STRATA_MMVF_M(224); STRATA_MMVF_M(256);
+    }
+#undef STRATA_MMVF_M
+    const cudaError_t result = cudaGetLastError();
+    if (result != cudaSuccess)
+        throw std::runtime_error(std::string("bf16_gemv_fp32_mmvf_multi launch: ") + cudaGetErrorString(result));
+}
+
 void bf16_gemv_fp32_mmvf(const float* x, const uint16_t* w, float* y,
                          int64_t n_in, int64_t n_out, void* stream) {
     if (n_in <= 0 || (n_in & 1) != 0 || n_in > std::numeric_limits<int>::max() ||

src/kernels/cuda/native_moe.cu

@@ -34,6 +34,9 @@ std::atomic<bool> enabled{false};
 __global__ void combine(const float* __restrict__ parts, const float* __restrict__ weights,
                         const float* __restrict__ shared, float* __restrict__ output,
                         int64_t n_embd, int k) {
+    // blockIdx.y = the token of a multi-token launch (0 for the single one)
+    const int64_t tk = blockIdx.y;
+    parts += tk * k * n_embd; weights += tk * k; if (shared) shared += tk * n_embd; output += tk * n_embd;
     const int64_t col = int64_t(blockIdx.x) * blockDim.x + threadIdx.x;
     if (col >= n_embd) return;
     float sum = parts[col] * weights[0];
@@ -71,4 +74,13 @@ void native_moe_combine(const float* parts, const float* weights, const float* s
     const auto error = cudaGetLastError();
     if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
 }
+void native_moe_combine_multi(const float* parts, const float* weights, const float* shared, float* output,
+                              int64_t n_embd, int64_t k, int n_tok, void* stream) {
+    if (!stream || n_embd <= 0 || k < 1 || k > 15 || n_tok < 1)
+        throw std::invalid_argument("native MoE combine (multi) requires a stream, width, 1..15 experts, tokens");
+    combine<<<dim3(unsigned((n_embd + 255) / 256), (unsigned) n_tok), 256, 0, static_cast<cudaStream_t>(stream)>>>(
+        parts, weights, shared, output, n_embd, int(k));
+    const auto error = cudaGetLastError();
+    if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
+}
 }

src/kernels/cuda/native_qsa_score.cu

@@ -32,6 +32,7 @@ constexpr int D=128, HEADS=4, R=4, ROWS=32, WARPS=2, STRIDE=36, COMBINE=68;
 struct TileA { uint32_t x[4]; };
 struct TileB { uint32_t x[2]; };
 struct TileC { float x[4]={0.0f,0.0f,0.0f,0.0f}; };
+#if !defined(__HIPCC__)
 #if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
 __device__ __forceinline__ void load_a(TileA& a,const float* p) {
     const float* src=p+(threadIdx.x%16)*STRIDE+(threadIdx.x/16)*4;
@@ -139,6 +140,42 @@ __global__ __launch_bounds__(64,1) void score_kernel(
     }
 #endif
 }
+#else
+// gfx1100 has no CUDA ldmatrix/mma instruction sequence. Keep the same entry point and
+// score contract with an ordered scalar F32 dot for each indexer head. The four heads
+// run independently; their ReLU'd scores are then added in the documented head order.
+// This path favors a well-defined fallback over pretending the CUDA PTX is portable.
+__global__ void scalar_score_kernel(
+        const float* __restrict__ pooled,const float* __restrict__ query,
+        const float* __restrict__ bias,const int32_t* __restrict__ step,
+        int max_cells,float* __restrict__ cells) {
+    const int n=step[kStepNKv],full=step[kStepNBid];
+    if(n<1||n>max_cells||step[kStepPos]!=n-1||full!=n/R||
+       step[kStepWidth]!=(n<2051?n:2051))return;
+    const int row=blockIdx.x;
+    if(row>full)return;
+    __shared__ float head_score[HEADS];
+    const int head=threadIdx.x;
+    if(head<HEADS){
+        float dot=0.0f;
+#pragma unroll
+        for(int d=0;d<D;++d)
+            dot=__fmaf_rn(pooled[size_t(row)*D+d],query[size_t(head)*D+d],dot);
+        head_score[head]=dot>0.0f?dot:0.0f;
+    }
+    __syncthreads();
+    if(head==0){
+        float sum=__fadd_rn(0.0f,head_score[0]);
+        sum=__fadd_rn(sum,head_score[1]);
+        sum=__fadd_rn(sum,head_score[2]);
+        sum=__fadd_rn(sum,head_score[3]);
+        if(bias)sum=__fadd_rn(sum,bias[row]);
+        sum=__fadd_rn(sum,row==full&&n%R?1e9f:0.0f);
+        sum=__fadd_rn(sum,0.0f);
+        for(int i=row*R;i<n&&i<(row+1)*R;++i)cells[i]=sum;
+    }
+}
+#endif
 struct Span{const void* p;size_t n;};
 void validate(Span s){
     const auto p=reinterpret_cast<uintptr_t>(s.p);
@@ -163,8 +200,13 @@ void native_qsa_score(const float* pooled,const float* query,const float* bias,
     for(int i=0;i<count;++i)validate(spans[i]);
     for(int i=0;i<count;++i)for(int j=i+1;j<count;++j)
         if(overlaps(spans[i],spans[j]))throw std::invalid_argument("native QSA score spans overlap");
+#if defined(__HIPCC__)
+    scalar_score_kernel<<<unsigned(max_blocks),128,0,static_cast<cudaStream_t>(stream)>>>(
+        pooled,query,bias,step,int(max_cells),cells);
+#else
     score_kernel<<<unsigned((max_blocks+ROWS-1)/ROWS),dim3(32,WARPS),0,static_cast<cudaStream_t>(stream)>>>(
         pooled,query,bias,step,int(max_cells),cells);
+#endif
     const auto error=cudaGetLastError();
     if(error!=cudaSuccess)throw std::runtime_error(cudaGetErrorString(error));
 }

src/kernels/cuda/native_router.cu

@@ -45,6 +45,8 @@ __launch_bounds__(256, 1)
 __global__ void route(const float* __restrict__ logits, int32_t* __restrict__ ids,
                       float* __restrict__ weights) {
     // Preserve the pinned 32x8 block geometry; only row zero is active here.
+    // blockIdx.x = the token (a multi-token launch; 0 for the single one)
+    logits += (size_t) blockIdx.x * 512; ids += (size_t) blockIdx.x * 10; weights += (size_t) blockIdx.x * 10;
     if (threadIdx.y != 0) return;
     const int lane = threadIdx.x;
     float values[16];
@@ -114,4 +116,12 @@ void native_router_top10(const float* logits, int32_t* ids, float* weights, void
     const auto error = cudaGetLastError();
     if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
 }
+void native_router_top10_multi(const float* logits, int32_t* ids, float* weights, int n_tok, void* stream) {
+    if (!stream || n_tok < 1 || !valid(logits, (size_t) n_tok * 512 * 4) || !valid(ids, (size_t) n_tok * 10 * 4) ||
+        !valid(weights, (size_t) n_tok * 10 * 4))
+        throw std::invalid_argument("native router (multi) requires a stream and aligned [n,512]/[n,10] buffers");
+    route<<<(unsigned) n_tok, dim3(32, 8), 0, static_cast<cudaStream_t>(stream)>>>(logits, ids, weights);
+    const auto error = cudaGetLastError();
+    if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
+}
 }

src/kernels/cuda/ple.cu

@@ -228,8 +228,9 @@ void ple_block(const float* emb, const float* hidden, const float* hist_rows, co
                PleOut& out, void* scratch, void* stream) {
     const bool native_key = w.key_native_data != nullptr && w.key_bf16 == nullptr;
     if (native_key && (!emb || !hidden || !hist_rows || !out.result || !scratch || !stream ||
-                       !w.key_native_q8_1 || w.key_native_type != 42))
-        throw std::invalid_argument("ple_block: native key requires Q2_0 weights, input/output, private scratch and explicit stream");
+                       !w.key_native_q8_1 ||
+                       (w.key_native_type != 42 && w.key_native_type != 18 && w.key_native_type != 23)))
+        throw std::invalid_argument("ple_block: native key requires Q2_0, IQ3_XXS or IQ4_XS weights, input/output, private scratch and explicit stream");
     if (emb == nullptr || hidden == nullptr || hist_rows == nullptr || out.result == nullptr) return;
     const int n_embd = NG_N_EMBD, hc = NG_HC, hc_dim = NG_HC_DIM;
     static_assert(NG_N_EMBD == 2560 && NG_HC_DIM == 10240, "native PLE key geometry changed");

src/kernels/cuda/qsa_decode_attn.cu

@@ -31,42 +31,53 @@ __device__ __forceinline__ float warp_max(float v) {
 }
 
 // 8 consecutive values of one cell's key or value row for KV head `kvh`, dimensions [d0, d0+8).
+// Per-format bodies; `load8` below is the KV_MODE dispatcher. value=false is the K side, true the V side.
+__device__ __forceinline__ void load8_f16(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
+    const uint16_t* base = (value ? p.v_pool : p.k_pool) + row * HD + d0;
+    const uint4 raw = *reinterpret_cast<const uint4*>(base);
+    const __half2* h2 = reinterpret_cast<const __half2*>(&raw);
+#pragma unroll
+    for (int j = 0; j < 4; ++j) {
+        const float2 f = __half22float2(h2[j]);
+        out[2 * j] = f.x;
+        out[2 * j + 1] = f.y;
+    }
+}
+__device__ __forceinline__ void load8_q8(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
+    const int8_t* codes = (value ? p.v_q : p.k_q) + row * HD + d0;
+    const uint16_t sbits = (value ? p.v_scale : p.k_scale)[row * (HD / KV_Q8_GROUP) + d0 / KV_Q8_GROUP];
+    const float sc = __half2float(__ushort_as_half(sbits));
+    const uint2 raw = *reinterpret_cast<const uint2*>(codes);
+    const int8_t* c = reinterpret_cast<const int8_t*>(&raw);
+#pragma unroll
+    for (int j = 0; j < 8; ++j) out[j] = (float) c[j] * sc;
+}
+__device__ __forceinline__ void load8_q4(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
+    constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
+    const int b = d0 / QK4_0;
+    const int rem = d0 % QK4_0;
+    const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((value ? p.v_q4 : p.k_q4) + row * bytes_per_head) + b;
+    const float d = __half2float(__ushort_as_half(blk->d));
+    const int j = (rem == 0 || rem == 16) ? 0 : 8;
+    const uint8_t* bytes = blk->qs + j;
+    if (rem < 16) {
+#pragma unroll
+        for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] & 0x0F) - 8) * d;
+    } else {
+#pragma unroll
+        for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] >> 4) - 8) * d;
+    }
+}
 template <int KV_MODE>
 __device__ __forceinline__ void load8(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
-    if constexpr (KV_MODE == 0) {
-        const uint16_t* base = (value ? p.v_pool : p.k_pool) + row * HD + d0;
-        const uint4 raw = *reinterpret_cast<const uint4*>(base);
-        const __half2* h2 = reinterpret_cast<const __half2*>(&raw);
-#pragma unroll
-        for (int j = 0; j < 4; ++j) {
-            const float2 f = __half22float2(h2[j]);
-            out[2 * j] = f.x;
-            out[2 * j + 1] = f.y;
-        }
-    } else if constexpr (KV_MODE == 1) {
-        const int8_t* codes = (value ? p.v_q : p.k_q) + row * HD + d0;
-        const uint16_t sbits = (value ? p.v_scale : p.k_scale)[row * (HD / KV_Q8_GROUP) + d0 / KV_Q8_GROUP];
-        const float sc = __half2float(__ushort_as_half(sbits));
-        const uint2 raw = *reinterpret_cast<const uint2*>(codes);
-        const int8_t* c = reinterpret_cast<const int8_t*>(&raw);
-#pragma unroll
-        for (int j = 0; j < 8; ++j) out[j] = (float) c[j] * sc;
-    } else {
-        constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
-        const int b = d0 / QK4_0;
-        const int rem = d0 % QK4_0;
-        const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((value ? p.v_q4 : p.k_q4) + row * bytes_per_head) + b;
-        const float d = __half2float(__ushort_as_half(blk->d));
-        const int j = (rem == 0 || rem == 16) ? 0 : 8;
-        const uint8_t* bytes = blk->qs + j;
-        if (rem < 16) {
-#pragma unroll
-            for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] & 0x0F) - 8) * d;
-        } else {
-#pragma unroll
-            for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] >> 4) - 8) * d;
-        }
-    }
+    if constexpr (KV_MODE == 0) load8_f16(p, value, row, d0, out);
+    else if constexpr (KV_MODE == 1) load8_q8(p, value, row, d0, out);
+    else if constexpr (KV_MODE == 3) {
+        // hybrid K8V4: both sides are defined - K unrotated INT8, V rotated Q4_0 - so a value=true call
+        // reads the Q4_0 pool instead of dereferencing the null v_q (no call site does today; PR review)
+        if (value) load8_q4(p, value, row, d0, out);
+        else load8_q8(p, value, row, d0, out);
+    } else load8_q4(p, value, row, d0, out);
 }
 
 template <int KV_MODE>
@@ -149,7 +160,7 @@ __global__ void __launch_bounds__(THREADS) attn_chunk_kernel(const float* __rest
         } else if constexpr (KV_MODE == 1) {
             const float sc = __half2float(__ushort_as_half(p.v_scale[srow[c] * (HD / KV_Q8_GROUP) + t / KV_Q8_GROUP]));
             v = (float) p.v_q[srow[c] * HD + t] * sc;
-        } else {
+        } else {   // modes 2 and 3: V is rotated Q4_0 (kv_q4.hpp); the caller rotates the output back
             constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
             const int b = t / QK4_0;
             const int rem = t % QK4_0;
@@ -202,7 +213,8 @@ void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int3
         std::fprintf(stderr, "qsa_decode_attn_batch: unsupported geometry or missing buffers\n");
         std::exit(1);
     }
-    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr ? 1 : 0);
+    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
+                        : (pools.k_q != nullptr ? 1 : 0));
     const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
     // per query: [acc: n_chunks*n_head*HD][m: n_chunks*n_head][l: n_chunks*n_head], all offsets from one stride
     const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);
@@ -212,7 +224,10 @@ void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int3
     const float scale = 1.0f / sqrtf((float) HD);
     const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);
     cudaStream_t st = (cudaStream_t) stream;
-    if (kv_mode == 2)
+    if (kv_mode == 3)
+        attn_chunk_kernel<3><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
+                                                        scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
+    else if (kv_mode == 2)
         attn_chunk_kernel<2><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
                                                         scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
     else if (kv_mode == 1)
@@ -242,8 +257,11 @@ void qsa_decode_attn_step(const float* q, const QsaAttnPools& pools, const int32
         std::fprintf(stderr, "qsa_decode_attn: unsupported geometry or missing buffers\n");
         std::exit(1);
     }
-    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr ? 1 : 0);
-    if (kv_mode == 2 ? (!pools.v_q4) : (kv_mode == 1 ? (!pools.v_q || !pools.k_scale || !pools.v_scale) : (!pools.k_pool || !pools.v_pool))) {
+    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
+                        : (pools.k_q != nullptr ? 1 : 0));
+    if (kv_mode == 3 ? (!pools.k_scale || !pools.v_q4)
+                     : (kv_mode == 2 ? (!pools.v_q4) : (kv_mode == 1 ? (!pools.v_q || !pools.k_scale || !pools.v_scale)
+                                                                     : (!pools.k_pool || !pools.v_pool)))) {
         std::fprintf(stderr, "qsa_decode_attn: incomplete KV pools\n");
         std::exit(1);
     }
@@ -254,7 +272,10 @@ void qsa_decode_attn_step(const float* q, const QsaAttnPools& pools, const int32
     const float scale = 1.0f / sqrtf((float) HD);
     const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv);
     cudaStream_t st = (cudaStream_t) stream;
-    if (kv_mode == 2)
+    if (kv_mode == 3)
+        attn_chunk_kernel<3><<<grid, THREADS, 0, st>>>(q, pools, ids, step, (int) s.n_head_kv, (int) s.page_size,
+                                                        scale, part_acc, part_m, part_l, n_chunks);
+    else if (kv_mode == 2)
         attn_chunk_kernel<2><<<grid, THREADS, 0, st>>>(q, pools, ids, step, (int) s.n_head_kv, (int) s.page_size,
                                                         scale, part_acc, part_m, part_l, n_chunks);
     else if (kv_mode == 1)

src/kernels/cuda/qsa_prompt_attn.cu

@@ -1,10 +1,11 @@
 // src/kernels/cuda/qsa_prompt_attn.cu - see include/strata/kernels/qsa_prompt_attn.hpp.
 #include "strata/kernels/qsa_prompt_attn.hpp"
 #include "strata/kernels/kv_q8.hpp"
+#include "strata/kernels/kv_q4.hpp"
 
 #include <cuda_fp16.h>
 #include <cuda_runtime.h>
-#include <math_constants.h>
+#include <cmath>
 
 #include <cstdio>
 #include <cstdlib>
@@ -24,7 +25,9 @@ constexpr int QS = HD + 8;        // q row stride in halves (bank-conflict-free
 
 // The MMA and cp.async below need sm_80. Builds for older cards (the experimental sm_75 one) compile them to a trap;
 // qsa_prompt_attn_batch refuses such a device at run time, so the old kernel runs there.
-#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
+#if defined(__HIPCC__)          // AMD: no mma.sync / cp.async; the host keeps the old kernel (below)
+#define STRATA_PA_SM80 0
+#elif !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
 #define STRATA_PA_SM80 1
 #else
 #define STRATA_PA_SM80 0
@@ -56,14 +59,18 @@ __device__ __forceinline__ uint32_t pack_h2(float lo_k, float hi_k) {   // eleme
 }
 
 // KV_MODE 1: int8 codes + fp16 scale per 64 values. KV_MODE 0: fp16 values (scales 1).
+// KV_MODE 3 (hybrid K8V4): K as mode 1, V as mode 0 - the row's q4_0 blocks are dequantized to fp16 at
+// gather, so everything downstream of the load is the mode-0 V path; the caller un-rotates the output.
 template <int KV_MODE>
 struct Smem {
-    using Elem = typename std::conditional<KV_MODE == 1, int8_t, __half>::type;
-    static constexpr int ROW = KV_MODE == 1 ? HD + 16 : HD + 8;   // elements; 16-byte aligned rows, banks spread
+    using KElem = typename std::conditional<KV_MODE == 0, __half, int8_t>::type;
+    using VElem = typename std::conditional<KV_MODE == 1, int8_t, __half>::type;
+    static constexpr int KROW = KV_MODE == 0 ? HD + 8 : HD + 16;   // elements; 16-byte aligned rows, banks spread
+    static constexpr int VROW = KV_MODE == 1 ? HD + 16 : HD + 8;
     __half qh[16][QS];
     __half ql[16][QS];
-    Elem k[CH][ROW];
-    Elem v[CH][ROW];
+    KElem k[CH][KROW];
+    VElem v[CH][VROW];
     float ks[CH][4];
     float vs[CH][4];
     float s[16][CH + 1];
@@ -110,7 +117,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
         S.qh[h][d] = hi;
         S.ql[h][d] = __float2half_rn(x - __half2float(hi));
     }
-    if (t < 16) { S.mrow[t] = -CUDART_INF_F; S.lsum[t] = 0.0f; }
+    if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; }
 
     float acc[8][4];
 #pragma unroll
@@ -128,24 +135,55 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             S.row[t] = r;
         }
         __syncthreads();   // rows ready; the previous chunk's p.v is done with k, v, s
-        // gather the chunk's K and V rows (16-byte pieces) and their scales
+        // gather the chunk's K and V rows (16-byte pieces; K8V4's V as q4_0 blocks dequantized to fp16)
+        // and their scales
         {
-            constexpr int PIECES = HD * (int) sizeof(typename Smem<KV_MODE>::Elem) / 16;   // per row
-            for (int i = t; i < CH * PIECES; i += THREADS) {
-                const int c = i / PIECES, pc = i % PIECES;
+            constexpr int KPIECES = HD * (int) sizeof(typename Smem<KV_MODE>::KElem) / 16;   // per K row
+            for (int i = t; i < CH * KPIECES; i += THREADS) {
+                const int c = i / KPIECES, pc = i % KPIECES;
                 const long long r = S.row[c];
-                uint4 kx = make_uint4(0, 0, 0, 0), vx = kx;
+                uint4 kx = make_uint4(0, 0, 0, 0);
                 if (r >= 0) {
-                    if constexpr (KV_MODE == 1) {
-                        kx = __ldg(reinterpret_cast<const uint4*>(p.k_q + r * HD) + pc);
-                        vx = __ldg(reinterpret_cast<const uint4*>(p.v_q + r * HD) + pc);
-                    } else {
+                    if constexpr (KV_MODE == 0)
                         kx = __ldg(reinterpret_cast<const uint4*>(p.k_pool + r * HD) + pc);
-                        vx = __ldg(reinterpret_cast<const uint4*>(p.v_pool + r * HD) + pc);
-                    }
+                    else   // modes 1 and 3: the K side is INT8
+                        kx = __ldg(reinterpret_cast<const uint4*>(p.k_q + r * HD) + pc);
                 }
                 *reinterpret_cast<uint4*>(reinterpret_cast<unsigned char*>(&S.k[c][0]) + pc * 16) = kx;
-                *reinterpret_cast<uint4*>(reinterpret_cast<unsigned char*>(&S.v[c][0]) + pc * 16) = vx;
+            }
+            if constexpr (KV_MODE == 3) {   // V: dequantize the row's q4_0 blocks straight into the fp16 V row
+                constexpr int BLKS = HD / QK4_0;
+                constexpr int BYTES = BLKS * (int) sizeof(block_q4_0);
+                for (int i = t; i < CH * BLKS; i += THREADS) {
+                    const int c = i / BLKS, b = i % BLKS;
+                    const long long r = S.row[c];
+#pragma unroll
+                    for (int j = 0; j < QK4_0; ++j) S.v[c][b * QK4_0 + j] = __half(0);
+                    if (r >= 0) {
+                        const block_q4_0* blk = reinterpret_cast<const block_q4_0*>(p.v_q4 + r * BYTES) + b;
+                        const float d = __half2float(__ushort_as_half(blk->d));
+#pragma unroll
+                        for (int j = 0; j < QK4_0 / 2; ++j) {
+                            S.v[c][b * QK4_0 + j] = __float2half_rn((float) ((int)(blk->qs[j] & 0x0F) - 8) * d);
+                            S.v[c][b * QK4_0 + j + QK4_0 / 2] =
+                                __float2half_rn((float) ((int)(blk->qs[j] >> 4) - 8) * d);
+                        }
+                    }
+                }
+            } else {
+                constexpr int VPIECES = HD * (int) sizeof(typename Smem<KV_MODE>::VElem) / 16;   // per V row
+                for (int i = t; i < CH * VPIECES; i += THREADS) {
+                    const int c = i / VPIECES, pc = i % VPIECES;
+                    const long long r = S.row[c];
+                    uint4 vx = make_uint4(0, 0, 0, 0);
+                    if (r >= 0) {
+                        if constexpr (KV_MODE == 1)
+                            vx = __ldg(reinterpret_cast<const uint4*>(p.v_q + r * HD) + pc);
+                        else
+                            vx = __ldg(reinterpret_cast<const uint4*>(p.v_pool + r * HD) + pc);
+                    }
+                    *reinterpret_cast<uint4*>(reinterpret_cast<unsigned char*>(&S.v[c][0]) + pc * 16) = vx;
+                }
             }
             for (int i = t; i < CH * 4; i += THREADS) {
                 const int c = i / 4, g = i % 4;
@@ -155,6 +193,9 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
                     if constexpr (KV_MODE == 1) {
                         a = __half2float(__ushort_as_half(p.k_scale[r * (HD / KV_Q8_GROUP) + g]));
                         b = __half2float(__ushort_as_half(p.v_scale[r * (HD / KV_Q8_GROUP) + g]));
+                    } else if constexpr (KV_MODE == 3) {   // K as int8, V dequantized to fp16 (scale 1)
+                        a = __half2float(__ushort_as_half(p.k_scale[r * (HD / KV_Q8_GROUP) + g]));
+                        b = 1.0f;
                     } else {
                         a = b = 1.0f;
                     }
@@ -184,7 +225,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
                     al[1] = *reinterpret_cast<const uint32_t*>(&S.ql[gid + 8][k0 + 2 * tig]);
                     al[2] = *reinterpret_cast<const uint32_t*>(&S.ql[gid][k0 + 2 * tig + 8]);
                     al[3] = *reinterpret_cast<const uint32_t*>(&S.ql[gid + 8][k0 + 2 * tig + 8]);
-                    if constexpr (KV_MODE == 1) {
+                    if constexpr (KV_MODE != 0) {   // modes 1 and 3: the K side is INT8 codes
                         b[0] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(&S.k[cb + gid][k0 + 2 * tig]));
                         b[1] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(&S.k[cb + gid][k0 + 2 * tig + 8]));
                     } else {
@@ -203,17 +244,17 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
                 sc[3] = fmaf(tg[3], s1, sc[3]);
             }
             const int c = cb + 2 * tig;
-            S.s[gid][c] = c < nh ? sc[0] * qdown : -CUDART_INF_F;
-            S.s[gid][c + 1] = c + 1 < nh ? sc[1] * qdown : -CUDART_INF_F;
-            S.s[gid + 8][c] = c < nh ? sc[2] * qdown : -CUDART_INF_F;
-            S.s[gid + 8][c + 1] = c + 1 < nh ? sc[3] * qdown : -CUDART_INF_F;
+            S.s[gid][c] = c < nh ? sc[0] * qdown : -INFINITY;
+            S.s[gid][c + 1] = c + 1 < nh ? sc[1] * qdown : -INFINITY;
+            S.s[gid + 8][c] = c < nh ? sc[2] * qdown : -INFINITY;
+            S.s[gid + 8][c + 1] = c + 1 < nh ? sc[3] * qdown : -INFINITY;
         }
         __syncthreads();
         // online softmax: row t/8, 4 cells per thread, 8 threads per row (lanes 8r..8r+7 of a warp)
         {
             constexpr int PER = CH / 8;
             const int r = t >> 3, sub = t & 7;
-            float x[PER], mx = -CUDART_INF_F;
+            float x[PER], mx = -INFINITY;
 #pragma unroll
             for (int j = 0; j < PER; ++j) { x[j] = S.s[r][sub * PER + j]; mx = fmaxf(mx, x[j]); }
 #pragma unroll
@@ -223,7 +264,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             float sum = 0.0f;
 #pragma unroll
             for (int j = 0; j < PER; ++j) {
-                const float e = x[j] == -CUDART_INF_F ? 0.0f : exp2f(x[j] - m_new);
+                const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);
                 S.s[r][sub * PER + j] = e;
                 sum += e;
             }
@@ -231,7 +272,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
             __syncwarp();
             if (sub == 0) {
-                const float a = m_old == -CUDART_INF_F ? 0.0f : exp2f(m_old - m_new);
+                const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);
                 S.alpha[r] = a;
                 S.lsum[r] = fmaf(S.lsum[r], a, sum);
                 S.mrow[r] = m_new;
@@ -378,7 +419,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __
 #pragma unroll
     for (int o = 16; o > 0; o >>= 1) qm = fmaxf(qm, __shfl_xor_sync(0xffffffffu, qm, o));
     if (lane == 0) S.qmax[warp] = qm;
-    if (t < 16) { S.mrow[t] = -CUDART_INF_F; S.lsum[t] = 0.0f; }
+    if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; }
     __syncthreads();
     qm = fmaxf(fmaxf(S.qmax[0], S.qmax[1]), fmaxf(S.qmax[2], S.qmax[3]));
     int qe = 0;
@@ -470,12 +511,12 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __
         // online softmax over the four groups' sum (fixed order): row t/8, 4 cells per thread
         {
             const int r = t >> 3, sub = t & 7;
-            float x[4], mx = -CUDART_INF_F;
+            float x[4], mx = -INFINITY;
 #pragma unroll
             for (int j = 0; j < 4; ++j) {
                 const int c = sub * 4 + j;
                 x[j] = c0 + c < n ? (((S.part[0][r][c] + S.part[1][r][c]) + S.part[2][r][c]) + S.part[3][r][c]) * qdown
-                                  : -CUDART_INF_F;
+                                  : -INFINITY;
                 mx = fmaxf(mx, x[j]);
             }
 #pragma unroll
@@ -485,7 +526,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __
             float sum = 0.0f;
 #pragma unroll
             for (int j = 0; j < 4; ++j) {
-                const float e = x[j] == -CUDART_INF_F ? 0.0f : exp2f(x[j] - m_new);
+                const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);
                 S.p[r][sub * 4 + j] = e;
                 sum += e;
             }
@@ -493,7 +534,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __
             for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
             __syncwarp();
             if (sub == 0) {
-                const float a = m_old == -CUDART_INF_F ? 0.0f : exp2f(m_old - m_new);
+                const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);
                 S.alpha[r] = a;
                 S.lsum[r] = fmaf(S.lsum[r], a, sum);
                 S.mrow[r] = m_new;
@@ -572,15 +613,18 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_i8_kernel(const float* __
 
 bool launch_i8(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,
                const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {
-    static bool attr = false;
+    static bool attr[64] = {};   // the shared-memory opt-in is per device (a layer split runs this on several)
+    int dev = 0;
+    cudaGetDevice(&dev);
     const int bytes = (int) sizeof(Smem2);
-    if (!attr) {
+    if (dev < 0 || dev >= 64) return false;
+    if (!attr[dev]) {
         if (cudaFuncSetAttribute(prompt_attn_i8_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes) !=
             cudaSuccess) {
             cudaGetLastError();
             return false;
         }
-        attr = true;
+        attr[dev] = true;
     }
     const float scale_log2 = 1.4426950408889634f / sqrtf((float) HD);
     for (int64_t q0 = 0; q0 < n_q; q0 += 65535) {
@@ -600,15 +644,18 @@ bool launch_i8(const float* q, const QsaAttnPools& pools, const int32_t* ids, co
 template <int KV_MODE>
 bool launch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,
             const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {
-    static bool attr = false;
+    static bool attr[64] = {};   // per device, as above
+    int dev = 0;
+    cudaGetDevice(&dev);
     const int bytes = (int) sizeof(Smem<KV_MODE>);
-    if (!attr) {
+    if (dev < 0 || dev >= 64) return false;
+    if (!attr[dev]) {
         if (cudaFuncSetAttribute(prompt_attn_kernel<KV_MODE>, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes) !=
             cudaSuccess) {
             cudaGetLastError();
             return false;
         }
-        attr = true;
+        attr[dev] = true;
     }
     const float scale_log2 = 1.4426950408889634f / sqrtf((float) HD);
     for (int64_t q0 = 0; q0 < n_q; q0 += 65535) {
@@ -644,10 +691,17 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3
         }
         if (cc_major[dev] < 8) return false;
     }
+#if defined(__HIPCC__)
+    return false;   // the tensor-core kernel is compiled out on AMD (its major version is not a CUDA sm)
+#endif
     if (pools.k_q4 != nullptr || s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !ids ||
         !steps || !pools.page_table)
         return false;
     cudaStream_t st = (cudaStream_t) stream;
+    if (pools.k_q != nullptr && pools.v_q4 != nullptr) {   // hybrid K8V4: int8 K + dequantized-q4 V
+        if (!pools.k_scale) return false;
+        return launch<3>(q, pools, ids, steps, cap, s, attn, n_q, st);
+    }
     if (pools.k_q != nullptr) {
         if (!pools.v_q || !pools.k_scale || !pools.v_scale) return false;
         // STRATA_PROMPT_ATTN_V1=1 (debug): the first version, same accuracy, another summation order - the control

src/kernels/cuda/qsa_select.cu

@@ -148,6 +148,287 @@ __global__ void __launch_bounds__(TOPK_T) block_topk_kernel(const float* __restr
     }
 }
 
+
+// ---- QSA select on tensor cores (perf-review, after D-1): the block scores of many queries are one GEMM,
+// rows (query, indexer head) x columns (blocks), K = 128, with relu per head summed. 3xTF32 (each operand split
+// into a TF32 hi and lo part, hi*hi + hi*lo + lo*hi) keeps FP32-level accuracy; the summation order differs from
+// the warp kernel, so a score can move in its last bits and a near-tie can select differently (not bitwise).
+// Blocks < n_bid only; the tail block n_bid (the `dead` key, +1e9) is scored by the warp kernel's own code.
+constexpr int TC_QT = 16;                 // queries per CTA (one m16 tile per indexer head)
+constexpr int TC_NB = 32;                 // blocks per tile (4 warps x n8)
+constexpr int TC_ITER = 4;                // tiles per CTA (the query tile is loaded once)
+constexpr int TC_QS = IDX_HEADS * IDX_DIM + 4;   // query row stride in floats
+constexpr int TC_KS = IDX_DIM + 4;               // key row stride
+
+// TF32 conversion and MMA need sm_80: below it they compile to a trap and qsa_block_scores_tc refuses the device
+#if defined(__HIPCC__)          // AMD: no mma.sync / cp.async; the host keeps the warp kernel (below)
+#define STRATA_SEL_SM80 0
+#elif !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
+#define STRATA_SEL_SM80 1
+#else
+#define STRATA_SEL_SM80 0
+#endif
+__device__ __forceinline__ uint32_t tf32_hi(float x) {
+#if STRATA_SEL_SM80
+    uint32_t r;
+    asm("cvt.rna.tf32.f32 %0, %1;" : "=r"(r) : "f"(x));
+    return r;
+#else
+    return __float_as_uint(x);
+#endif
+}
+__device__ __forceinline__ void mma_tf32(float* c, const uint32_t* a, const uint32_t* b) {
+#if !STRATA_SEL_SM80
+    __trap();
+#else
+    asm volatile("mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
+                 "{%0,%1,%2,%3};\n"
+                 : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
+                 : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
+#endif
+}
+
+__global__ void __launch_bounds__(128) block_scores_tc_kernel(const float* __restrict__ pooled,
+                                                              const float* __restrict__ q_idx,
+                                                              const int32_t* __restrict__ steps, int64_t nq,
+                                                              int64_t max_blocks, int64_t reach,
+                                                              float* __restrict__ out) {
+    extern __shared__ __align__(16) float sm[];
+    float* sQ = sm;                                   // [TC_QT][TC_QS]
+    float* sK = sm + TC_QT * TC_QS;                   // [TC_NB][TC_KS]
+    __shared__ int s_nbid[TC_QT];
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5, gid = lane >> 2, tig = lane & 3;
+    const int64_t q0 = (int64_t) blockIdx.y * TC_QT;
+    for (int i = t; i < TC_QT * IDX_HEADS * IDX_DIM / 4; i += 128) {
+        const int r = i / (IDX_HEADS * IDX_DIM / 4), c = i % (IDX_HEADS * IDX_DIM / 4);
+        float4 v = make_float4(0.f, 0.f, 0.f, 0.f);
+        if (q0 + r < nq) v = reinterpret_cast<const float4*>(q_idx + (q0 + r) * IDX_HEADS * IDX_DIM)[c];
+        *reinterpret_cast<float4*>(sQ + r * TC_QS + c * 4) = v;
+    }
+    if (t < TC_QT) s_nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : 0;
+    int lo_nbid = 0x7fffffff, hi_nbid = 0;
+    __syncthreads();
+    for (int i = 0; i < TC_QT; ++i) {
+        if (q0 + i >= nq) break;
+        lo_nbid = min(lo_nbid, s_nbid[i]);
+        hi_nbid = max(hi_nbid, s_nbid[i]);
+    }
+    for (int it = 0; it < TC_ITER; ++it) {
+        const int64_t b0 = ((int64_t) blockIdx.x * TC_ITER + it) * TC_NB;
+        if (b0 >= reach || b0 >= hi_nbid) break;      // blocks >= every query's n_bid: nothing to score
+        __syncthreads();                              // the previous tile's reads are done
+        for (int i = t; i < TC_NB * IDX_DIM / 4; i += 128) {
+            const int r = i / (IDX_DIM / 4), c = i % (IDX_DIM / 4);
+            float4 v = make_float4(0.f, 0.f, 0.f, 0.f);
+            if (b0 + r < hi_nbid) v = reinterpret_cast<const float4*>(pooled + (b0 + r) * IDX_DIM)[c];
+            *reinterpret_cast<float4*>(sK + r * TC_KS + c * 4) = v;
+        }
+        __syncthreads();
+        float acc[IDX_HEADS][4];
+#pragma unroll
+        for (int h = 0; h < IDX_HEADS; ++h) acc[h][0] = acc[h][1] = acc[h][2] = acc[h][3] = 0.f;
+        const float* kr = sK + (warp * 8 + gid) * TC_KS;
+#pragma unroll 4
+        for (int k0 = 0; k0 < IDX_DIM; k0 += 8) {
+            const float kx0 = kr[k0 + tig], kx1 = kr[k0 + tig + 4];
+            uint32_t bh[2], bl[2];
+            bh[0] = tf32_hi(kx0);
+            bh[1] = tf32_hi(kx1);
+            bl[0] = tf32_hi(kx0 - __uint_as_float(bh[0]));
+            bl[1] = tf32_hi(kx1 - __uint_as_float(bh[1]));
+#pragma unroll
+            for (int h = 0; h < IDX_HEADS; ++h) {
+                const float* qa = sQ + h * IDX_DIM + k0 + tig;
+                const float x0 = qa[gid * TC_QS], x1 = qa[(gid + 8) * TC_QS];
+                const float x2 = qa[gid * TC_QS + 4], x3 = qa[(gid + 8) * TC_QS + 4];
+                uint32_t ah[4], al[4];
+                ah[0] = tf32_hi(x0); ah[1] = tf32_hi(x1); ah[2] = tf32_hi(x2); ah[3] = tf32_hi(x3);
+                al[0] = tf32_hi(x0 - __uint_as_float(ah[0]));
+                al[1] = tf32_hi(x1 - __uint_as_float(ah[1]));
+                al[2] = tf32_hi(x2 - __uint_as_float(ah[2]));
+                al[3] = tf32_hi(x3 - __uint_as_float(ah[3]));
+                mma_tf32(acc[h], al, bh);
+                mma_tf32(acc[h], ah, bl);
+                mma_tf32(acc[h], ah, bh);
+            }
+        }
+        // relu per head, heads added in order (as the warp kernel), written where the block completed for the query
+        const int64_t bc = b0 + warp * 8 + 2 * tig;
+#pragma unroll
+        for (int half = 0; half < 2; ++half) {
+            const int qr = gid + half * 8;
+            const int64_t qi = q0 + qr;
+            if (qi >= nq) continue;
+            const int nb = s_nbid[qr];
+#pragma unroll
+            for (int j = 0; j < 2; ++j) {
+                const int64_t b = bc + j;
+                if (b >= nb || b >= max_blocks) continue;
+                float score = 0.0f;
+#pragma unroll
+                for (int h = 0; h < IDX_HEADS; ++h) {
+                    const float d = acc[h][half * 2 + j];
+                    score += d > 0.0f ? d : 0.0f;
+                }
+                out[qi * max_blocks + b] = score;
+            }
+        }
+    }
+}
+
+// the tail block n_bid of each query: exactly block_scores_kernel's arithmetic for that block
+__global__ void __launch_bounds__(32) block_scores_tail_kernel(const float* __restrict__ dead,
+                                                               const float* __restrict__ q_idx,
+                                                               const int32_t* __restrict__ steps,
+                                                               int64_t max_blocks, float* __restrict__ out) {
+    const int64_t qi = blockIdx.x;
+    const int32_t* st = steps + qi * kStepCount;
+    const int64_t n_kv = st[kStepNKv], n_bid = st[kStepNBid];
+    if (n_bid >= max_blocks) return;
+    const int lane = threadIdx.x & 31;
+    const float4 k4 = *reinterpret_cast<const float4*>(dead + lane * 4);
+    const float* q = q_idx + qi * IDX_HEADS * IDX_DIM + lane * 4;
+    float score = 0.0f;
+#pragma unroll
+    for (int h = 0; h < IDX_HEADS; ++h) {
+        const float4 q4 = *reinterpret_cast<const float4*>(q + h * IDX_DIM);
+        float d = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+#pragma unroll
+        for (int o = 16; o > 0; o >>= 1) d += __shfl_xor_sync(0xffffffffu, d, o);
+        score += d > 0.0f ? d : 0.0f;
+    }
+    if (lane == 0) {
+        if (n_kv % R != 0) score += 1e9f;
+        out[qi * max_blocks + n_bid] = score;
+    }
+}
+
+// ---- the same top-k with each query's keys read once: 1,024 threads hold up to TK_PER consecutive blocks' keys in
+// registers (contexts up to 4 * 1024 * TK_PER cells), per-warp histograms, block-wide scans. The selection rule is
+// block_topk_kernel's (radix threshold, ties to the lowest index, cells ascending): identical ids.
+constexpr int TK_T = 1024;
+constexpr int TK_PER = 33;
+
+__device__ __forceinline__ int block_excl_scan(int v, int* s_warp, int& total) {
+    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+    int x = v;
+#pragma unroll
+    for (int o = 1; o < 32; o <<= 1) {
+        const int y = __shfl_up_sync(0xffffffffu, x, o);
+        if (lane >= o) x += y;
+    }
+    if (lane == 31) s_warp[warp] = x;
+    __syncthreads();
+    if (warp == 0) {
+        int w = s_warp[lane];
+        int z = w;
+#pragma unroll
+        for (int o = 1; o < 32; o <<= 1) {
+            const int y = __shfl_up_sync(0xffffffffu, z, o);
+            if (lane >= o) z += y;
+        }
+        s_warp[lane] = z - w;               // exclusive per warp
+        if (lane == 31) s_warp[32] = z;     // total
+    }
+    __syncthreads();
+    const int r = s_warp[warp] + x - v;
+    total = s_warp[32];
+    __syncthreads();
+    return r;
+}
+
+__global__ void __launch_bounds__(TK_T) block_topk_reg_kernel(const float* __restrict__ scores,
+                                                              const int32_t* __restrict__ steps, int64_t max_blocks,
+                                                              int64_t cap, int32_t* __restrict__ ids) {
+    __shared__ int hist[TK_T / 32][256];
+    __shared__ int s_warp[33];
+    __shared__ int s_digit, s_above;
+    const int64_t qi = blockIdx.x;
+    const int32_t* st = steps + qi * kStepCount;
+    const int64_t n_kv = st[kStepNKv], n_bid = st[kStepNBid], width = st[kStepWidth];
+    int32_t* out = ids + qi * cap;
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+    if (n_kv <= width) {
+        for (int64_t j = t; j < n_kv; j += TK_T) out[j] = (int32_t) j;
+        return;
+    }
+    const float* sc = scores + qi * max_blocks;
+    const int64_t nb = n_bid + 1;
+    const int64_t per = (nb + TK_T - 1) / TK_T;       // <= TK_PER (the caller checks)
+    const int64_t b0 = (int64_t) t * per, b1 = (b0 + per < nb) ? b0 + per : nb;
+    uint32_t key[TK_PER];
+#pragma unroll
+    for (int j = 0; j < TK_PER; ++j) key[j] = (b0 + j < b1) ? order_key(sc[b0 + j]) : 0u;
+    auto weight = [&](int64_t b) -> int { return b < n_bid ? R : (int) (n_kv - n_bid * R); };
+    uint32_t prefix = 0;
+    int above = 0;
+    for (int shift = 24; shift >= 0; shift -= 8) {
+        for (int i = lane; i < 256; i += 32) hist[warp][i] = 0;
+        __syncwarp();
+        const uint32_t hi_mask = shift == 24 ? 0u : (0xffffffffu << (shift + 8));
+#pragma unroll
+        for (int j = 0; j < TK_PER; ++j) {
+            const int64_t b = b0 + j;
+            if (b >= b1) break;
+            const int w = weight(b);
+            if (w == 0) continue;
+            if ((key[j] & hi_mask) == (prefix & hi_mask)) atomicAdd(&hist[warp][(key[j] >> shift) & 255], w);
+        }
+        __syncthreads();
+        if (t < 256) {                                // fold the warps' histograms into warp 0's
+            int s = 0;
+            for (int w2 = 0; w2 < TK_T / 32; ++w2) s += hist[w2][t];
+            hist[0][t] = s;
+        }
+        __syncthreads();
+        if (t == 0) {
+            int cum = above, d = 255;
+            for (; d > 0; --d) {
+                if (cum + hist[0][d] >= width) break;
+                cum += hist[0][d];
+            }
+            s_digit = d;
+            s_above = cum;
+        }
+        __syncthreads();
+        prefix |= (uint32_t) s_digit << shift;
+        above = s_above;
+        __syncthreads();
+    }
+    const uint32_t thr = prefix;
+    const int64_t eq_budget = width - above;
+    int gt = 0, eq = 0;
+#pragma unroll
+    for (int j = 0; j < TK_PER; ++j) {
+        const int64_t b = b0 + j;
+        if (b >= b1) break;
+        const int w = weight(b);
+        if (w == 0) continue;
+        if (key[j] > thr) gt += w;
+        else if (key[j] == thr) eq += w;
+    }
+    int tot;
+    const int eq_before = block_excl_scan(eq, s_warp, tot);
+    int64_t my_eq = eq_budget - eq_before;
+    if (my_eq < 0) my_eq = 0;
+    if (my_eq > eq) my_eq = eq;
+    const int sel = gt + (int) my_eq;
+    int64_t wpos = block_excl_scan(sel, s_warp, tot);
+    int64_t eq_left = my_eq;
+#pragma unroll
+    for (int j = 0; j < TK_PER; ++j) {
+        const int64_t b = b0 + j;
+        if (b >= b1) break;
+        const int w = weight(b);
+        if (w == 0) continue;
+        if (key[j] > thr) {
+            for (int c = 0; c < w; ++c) out[wpos++] = (int32_t) (b * R + c);
+        } else if (key[j] == thr) {
+            for (int c = 0; c < w && eq_left > 0; ++c, --eq_left) out[wpos++] = (int32_t) (b * R + c);
+        }
+    }
+}
+
 }  // namespace
 
 void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
@@ -166,8 +447,52 @@ void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx
     if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores: %s\n", cudaGetErrorString(e)); std::exit(1); }
 }
 
-void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
-                    const QsaShapes& s, int32_t* ids, void* stream) {
+bool qsa_block_scores_tc(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+                         int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {
+    if (nq <= 0) return true;
+    if (s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R || nq > 65535 * TC_QT) return false;
+    {   // sm_80 or newer (TF32 MMA); an older card keeps the warp kernel
+        static int cc_major[64] = {};
+        int dev = 0;
+        if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+        if (cc_major[dev] == 0) {
+            int major = 0;
+            if (cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess) {
+                cudaGetLastError();
+                return false;
+            }
+            cc_major[dev] = major;
+        }
+        if (cc_major[dev] < 8) return false;
+    }
+#if defined(__HIPCC__)
+    return false;   // the tensor-core kernel is compiled out on AMD (its major version is not a CUDA sm)
+#endif
+    static bool attr[64] = {};   // the shared-memory opt-in is per device (a layer split runs it on several)
+    int adev = 0;
+    cudaGetDevice(&adev);
+    const int bytes = (TC_QT * TC_QS + TC_NB * TC_KS) * (int) sizeof(float);
+    if (!attr[adev]) {
+        if (cudaFuncSetAttribute(block_scores_tc_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes) !=
+            cudaSuccess) {
+            cudaGetLastError();
+            return false;
+        }
+        attr[adev] = true;
+    }
+    const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks;
+    const int64_t per = (int64_t) TC_NB * TC_ITER;
+    const dim3 grid((unsigned) ((reach + per - 1) / per), (unsigned) ((nq + TC_QT - 1) / TC_QT));
+    block_scores_tc_kernel<<<grid, 128, bytes, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq, max_blocks, reach,
+                                                                         scores);
+    block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks, scores);
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores_tc: %s\n", cudaGetErrorString(e)); std::exit(1); }
+    return true;
+}
+
+void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                        const QsaShapes& s, int32_t* ids, void* stream) {
     if (nq <= 0) return;
     if (s.idx_block != R || cap < qsa_selection_width(kTopkMaxCells, s)) {
         std::fprintf(stderr, "qsa_block_topk: unsupported geometry or cap\n");
@@ -178,4 +503,23 @@ void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64
     if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
 }
 
+void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                    const QsaShapes& s, int32_t* ids, void* stream) {
+    // keys in registers when every query's blocks fit (contexts up to ~135K cells); the same ids. STRATA_TOPK_OLD=1:
+    // the kernel that reads them from memory on every pass
+    static const bool old = std::getenv("STRATA_TOPK_OLD") != nullptr;
+    if (nq <= 0) return;
+    if (old || max_blocks > (int64_t) TK_T * TK_PER) {
+        qsa_block_topk_ref(scores, steps, nq, max_blocks, cap, s, ids, stream);
+        return;
+    }
+    if (s.idx_block != R || cap < qsa_selection_width(kTopkMaxCells, s)) {
+        std::fprintf(stderr, "qsa_block_topk: unsupported geometry or cap\n");
+        std::exit(1);
+    }
+    block_topk_reg_kernel<<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
+}
+
 }  // namespace strata::kernels

src/kernels/cuda/shared_expert.cu

@@ -135,6 +135,9 @@ __global__ void native_scalar_sigmoid_kernel(float* gate) {
     // compilation flags. The dot product was already reduced by the pinned native MMVF implementation.
     gate[0] = __fdividef(1.0f, 1.0f + __expf(-gate[0]));
 }
+__global__ void native_scalar_sigmoid_multi_kernel(float* gate) {   // thread t = token t, same expression
+    gate[threadIdx.x] = __fdividef(1.0f, 1.0f + __expf(-gate[threadIdx.x]));
+}
 
 /// The MoE block's final combination.  See the header for the two readings it exists to pin.
 __global__ void moe_combine_kernel(const float* __restrict__ parts, const float* __restrict__ weights,
@@ -176,6 +179,11 @@ void shared_expert_multi(int n_tok, const float* x, const uint16_t* x_bf16, cons
     native_swiglu_kernel<<<(unsigned) ((n + THREADS - 1) / THREADS), THREADS, 0, cs>>>(gate, up, gate, n);
     native_quantize_q8_1(gate, nw.q8_1, (int) n_ff, n_tok, stream);
     native_mmvq(nw.down_type, nw.down_data, nw.q8_1, out, (int) n_ff, (int) n_embd, n_tok, stream);
+    static const bool batch = [] { const char* v = std::getenv("STRATA_DEC_BATCH"); return v == nullptr || std::atoi(v) != 0; }();
+    if (native_bf16 && batch && n_tok > 1) {   // one gemv for all rows (outputs identical), one sigmoid launch
+        bf16_gemv_fp32_mmvf_multi(x, n_embd, gate_inp_bf16, g, 1, n_embd, 1, n_tok, stream);
+        native_scalar_sigmoid_multi_kernel<<<1, n_tok, 0, cs>>>(g);
+    } else
     for (int t = 0; t < n_tok; ++t) {
         if (native_bf16) {
             bf16_gemv_fp32_mmvf(x + (size_t) t * n_embd, gate_inp_bf16, g + t, n_embd, 1, stream);

src/kernels/cuda/verify_kernels.cu

@@ -541,4 +541,18 @@ void copy_indexed(float* dst, const float* src, int64_t stride, const int32_t* i
     check("copy_indexed");
 }
 
+// a GPU timestamp (ns, %globaltimer) into buf[i] - the verify window's stage profiler
+namespace { __global__ void gpu_stamp_kernel(unsigned long long* buf, int i) {
+    unsigned long long t;
+#if defined(__HIPCC__)
+    t = wall_clock64() * 10ull;   // gfx11: a constant 100 MHz counter, in ns
+#else
+    asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
+#endif
+    buf[i] = t;
+} }
+void gpu_stamp(unsigned long long* buf, int i, void* stream) {
+    gpu_stamp_kernel<<<1, 1, 0, (cudaStream_t) stream>>>(buf, i);
+}
+
 }  // namespace strata::kernels

src/kernels/gr_parity.cpp

@@ -20,6 +20,7 @@
 //   6. `gr_write`'s `2*sigmoid`, which centres the gate on 1 so a ZERO injection is a plain residual add.
 //      Asserted as a property, not as a value, because that is what the source comment claims.
 #include "strata/kernels/gr.hpp"
+#include "strata/kernels/fused_gr.hpp"
 
 #include <cuda_runtime.h>
 
@@ -268,6 +269,143 @@ int scalar_activation_contract() {
     return bad;
 }
 
+int fused_multi_lds_parity(const float* d_norm, const uint16_t* d_down, const uint16_t* d_up,
+                           const uint16_t* d_inject, float eps) {
+    using namespace strata::kernels;
+    constexpr int N = 2560, HC = 4, LR = 320, D = N * HC, T = kFusedGrMaxT;
+    std::mt19937 rng(0x6f8a);
+    std::normal_distribution<float> normal(0.0f, 0.3f);
+    std::vector<float> r((size_t) T * D), bo((size_t) T * N), inj((size_t) T * HC);
+    for (auto& x : r) x = normal(rng);
+    for (auto& x : bo) x = normal(rng);
+    for (auto& x : inj) x = normal(rng);
+
+    float *d_r = nullptr, *d_r_out = nullptr, *d_bo = nullptr, *d_inj = nullptr;
+    float *d_lo = nullptr, *d_rs = nullptr, *d_inj_out = nullptr, *d_mixed = nullptr, *d_xn = nullptr;
+    check(cudaMalloc(&d_r, r.size() * sizeof(float)), "multi R");
+    check(cudaMalloc(&d_r_out, r.size() * sizeof(float)), "multi R_out");
+    check(cudaMalloc(&d_bo, bo.size() * sizeof(float)), "multi bo");
+    check(cudaMalloc(&d_inj, inj.size() * sizeof(float)), "multi inj");
+    check(cudaMalloc(&d_lo, (size_t) T * LR * sizeof(float)), "multi lo");
+    check(cudaMalloc(&d_rs, (size_t) T * HC * sizeof(float)), "multi rs");
+    check(cudaMalloc(&d_inj_out, (size_t) T * HC * sizeof(float)), "multi injection");
+    check(cudaMalloc(&d_mixed, (size_t) T * N * sizeof(float)), "multi mixed");
+    check(cudaMalloc(&d_xn, (size_t) T * D * sizeof(float)), "multi xn");
+    check(cudaMemcpy(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy R");
+    check(cudaMemcpy(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy bo");
+    check(cudaMemcpy(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy inj");
+
+    std::vector<FusedGrArgs> args(T);
+    for (int t = 0; t < T; ++t) {
+        auto& a = args[t];
+        a.R = d_r + (size_t) t * D;
+        a.R_out = d_r_out + (size_t) t * D;
+        a.apply = true;
+        a.bo_prev = d_bo + (size_t) t * N;
+        a.inj_prev = d_inj + (size_t) t * HC;
+        a.w_norm = d_norm;
+        a.w_down = d_down;
+        a.w_up = d_up;
+        a.w_inject = d_inject;
+        a.eps = eps;
+        a.lo = d_lo + (size_t) t * LR;
+        a.rs = d_rs + (size_t) t * HC;
+        a.inject_out = d_inj_out + (size_t) t * HC;
+        a.mixed = d_mixed + (size_t) t * N;
+    }
+
+    struct Snapshot {
+        std::vector<float> r_out, lo, rs, inject, mixed;
+    };
+    auto snapshot = [&]() {
+        Snapshot s;
+        s.r_out.resize((size_t) T * D);
+        s.lo.resize((size_t) T * LR);
+        s.rs.resize((size_t) T * HC);
+        s.inject.resize((size_t) T * HC);
+        s.mixed.resize((size_t) T * N);
+        check(cudaMemcpy(s.r_out.data(), d_r_out, s.r_out.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read R_out");
+        check(cudaMemcpy(s.lo.data(), d_lo, s.lo.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read lo");
+        check(cudaMemcpy(s.rs.data(), d_rs, s.rs.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read rs");
+        check(cudaMemcpy(s.inject.data(), d_inj_out, s.inject.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read inject");
+        check(cudaMemcpy(s.mixed.data(), d_mixed, s.mixed.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read mixed");
+        return s;
+    };
+    auto same = [](const Snapshot& a, const Snapshot& b) {
+        return std::memcmp(a.r_out.data(), b.r_out.data(), a.r_out.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.lo.data(), b.lo.data(), a.lo.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.rs.data(), b.rs.data(), a.rs.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.inject.data(), b.inject.data(), a.inject.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.mixed.data(), b.mixed.data(), a.mixed.size() * sizeof(float)) == 0;
+    };
+
+    cudaStream_t stream = nullptr;
+    check(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), "multi stream");
+    // Max T forces the HIP kernel's full dynamic-LDS request: 8 * 1280 * sizeof(float) = 40 KiB.
+    fused_gr_read_multi(args.data(), T, d_xn, stream);
+    check(cudaStreamSynchronize(stream), "multi max-T sync");
+    const Snapshot multi = snapshot();
+    for (int t = 0; t < T; ++t) fused_gr_read(args[t], stream);
+    check(cudaStreamSynchronize(stream), "single reference sync");
+    const Snapshot single = snapshot();
+    int bad = 0;
+    if (!same(multi, single)) {
+        std::printf("  fused GR multi max-T differs from single-token calls\n");
+        ++bad;
+    }
+
+    cudaGraph_t graph = nullptr;
+    cudaGraphExec_t graph_exec = nullptr;
+    check(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal), "multi graph begin");
+    fused_gr_read_multi(args.data(), T, d_xn, stream);
+    check(cudaStreamEndCapture(stream, &graph), "multi graph end");
+    check(cudaGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0), "multi graph instantiate");
+    check(cudaGraphLaunch(graph_exec, stream), "multi graph initial replay");
+    check(cudaStreamSynchronize(stream), "multi graph initial sync");
+    const Snapshot captured = snapshot();
+    if (!same(multi, captured)) {
+        std::printf("  fused GR multi captured graph differs from direct max-T call\n");
+        ++bad;
+    }
+
+    // Reuse the same captured pointers with new payloads; then independently run the single-token path again.
+    for (size_t i = 0; i < r.size(); ++i) r[i] = -0.7f * r[i] + 0.001f * (float) (i % 17);
+    for (size_t i = 0; i < bo.size(); ++i) bo[i] = -0.4f * bo[i] + 0.02f;
+    for (size_t i = 0; i < inj.size(); ++i) inj[i] += 0.3f;
+    check(cudaMemcpyAsync(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay R");
+    check(cudaMemcpyAsync(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay bo");
+    check(cudaMemcpyAsync(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay inj");
+    check(cudaGraphLaunch(graph_exec, stream), "multi graph changed replay");
+    check(cudaStreamSynchronize(stream), "multi graph changed sync");
+    const Snapshot replay = snapshot();
+    if (std::memcmp(multi.mixed.data(), replay.mixed.data(), multi.mixed.size() * sizeof(float)) == 0) {
+        std::printf("  fused GR graph replay ignored changed inputs\n");
+        ++bad;
+    }
+    for (int t = 0; t < T; ++t) fused_gr_read(args[t], stream);
+    check(cudaStreamSynchronize(stream), "changed single reference sync");
+    if (!same(replay, snapshot())) {
+        std::printf("  fused GR changed graph replay differs from single-token calls\n");
+        ++bad;
+    }
+
+    std::printf("  fused GR multi max-T=8 LDS launch and changing graph replay %s\n",
+                bad == 0 ? "pass" : "FAIL");
+    check(cudaGraphExecDestroy(graph_exec), "multi graph exec destroy");
+    check(cudaGraphDestroy(graph), "multi graph destroy");
+    check(cudaStreamDestroy(stream), "multi stream destroy");
+    check(cudaFree(d_xn), "free multi xn");
+    check(cudaFree(d_mixed), "free multi mixed");
+    check(cudaFree(d_inj_out), "free multi injection");
+    check(cudaFree(d_rs), "free multi rs");
+    check(cudaFree(d_lo), "free multi lo");
+    check(cudaFree(d_inj), "free multi inj");
+    check(cudaFree(d_bo), "free multi bo");
+    check(cudaFree(d_r_out), "free multi R_out");
+    check(cudaFree(d_r), "free multi R");
+    return bad;
+}
+
 }  // namespace
 
 int main(int argc, char** argv) {
@@ -602,6 +740,7 @@ int main(int argc, char** argv) {
                         activation_mode_name(mode), ok ? "pass" : "*** FAIL ***", rm, ri);
             if (!ok) ++bad;
         }
+        bad += fused_multi_lds_parity(dN, dD, dU, dJ, eps);
         select_activation_mode(0);
         cudaFree(rws_raw);
         cudaFree(dR); cudaFree(dN); cudaFree(dD); cudaFree(dU); cudaFree(dJ); cudaFree(dM); cudaFree(dI);

src/kernels/kv_hybrid_parity.cpp

@@ -0,0 +1,361 @@
+// src/kernels/kv_hybrid_parity.cpp - hybrid K8V4 KV against host references (GPU, no model).
+//
+// Exercises the EXACT composed calls the engine's hybrid path makes (layer.cpp / prefill.cpp): the q8/q4
+// append and gather kernels invoked with the unused half's pointers folded onto the used pool, and the
+// fused decode attention's KV_MODE 3 (INT8 K, rotated Q4_0 V) followed by the output's inverse Hadamard.
+//   1. K: INT8 codes + FP16 scales bitwise vs the host reference (kv_q8.hpp's rules);
+//   2. V: after fwht256, Q4_0 blocks bitwise vs the host reference (kv_q4.hpp's rules, PR #21);
+//   3. the composed gathers bitwise vs the host dequantization of those codes;
+//   4. qsa_decode_attn<3> + output fwht vs a host attention over the SAME dequantized K/V (fp32 math), and
+//      bounded against the fp32-true attention from the unquantized K/V.
+#include "strata/kernels/f16_bits.hpp"
+#include "strata/kernels/kv_q4.hpp"
+#include "strata/kernels/kv_q8.hpp"
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_decode_attn.hpp"
+#include "strata/kernels/qsa_prompt_attn.hpp"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+int g_fail = 0;
+void ck(cudaError_t e, const char* w) {
+    if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", w, cudaGetErrorString(e)); std::exit(2); }
+}
+template <typename T> T* dalloc(size_t n) {
+    T* p = nullptr;
+    ck(cudaMalloc(&p, n * sizeof(T) + 64), "malloc");
+    ck(cudaMemset(p, 0, n * sizeof(T) + 64), "memset");
+    return p;
+}
+
+// kv_q4_parity.cpp's exact CPU references.
+void fwht256_host_reference(float* x) {
+    constexpr int n = 256;
+    for (int s = 1; s < n; s *= 2) {
+        for (int i = 0; i < n; i += 2 * s) {
+            for (int j = 0; j < s; ++j) {
+                const float u = x[i + j];
+                const float v = x[i + j + s];
+                x[i + j]     = u + v;
+                x[i + j + s] = u - v;
+            }
+        }
+    }
+    const float scale = 1.0f / 16.0f;
+    for (int i = 0; i < n; ++i) x[i] *= scale;
+}
+void quantize_block_q4_0_host(const float* x, k::block_q4_0& blk) {
+    float amax = 0.0f, max_val = 0.0f;
+    for (int i = 0; i < 32; ++i) {
+        const float v = x[i];
+        if (amax < std::fabs(v)) { amax = std::fabs(v); max_val = v; }
+    }
+    const float d = max_val / -8.0f;
+    const float id = d != 0.0f ? 1.0f / d : 0.0f;
+    blk.d = k::f16_from_f32(d);
+    for (int i = 0; i < 16; ++i) {
+        const int q0 = (int) (x[i] * id + 8.5f);
+        const int q1 = (int) (x[i + 16] * id + 8.5f);
+        const uint8_t xi0 = (uint8_t) std::min(15, std::max(0, q0));
+        const uint8_t xi1 = (uint8_t) std::min(15, std::max(0, q1));
+        blk.qs[i] = xi0 | (xi1 << 4);
+    }
+}
+}  // namespace
+
+int main() {
+    std::printf("=== Running kv_hybrid_parity test ===\n");
+    k::QsaShapes s = k::qsa_real_shapes();
+    s.page_size = 64;
+    const int H = (int) s.n_head_kv, D = (int) s.head_dim, P = (int) s.page_size, G = D / k::KV_Q8_GROUP;
+    const int QH = (int) s.n_head, PER = QH / H;   // query heads per KV head
+    const int pages = 24, cells = pages * P;   // n_chunks >= n_head: the batched merge overlaps queries below that
+    std::mt19937 rng(17);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+
+    std::vector<int32_t> table(pages);
+    for (int i = 0; i < pages; ++i) table[i] = (i * 5 + 3) % pages;
+    int32_t* d_table = dalloc<int32_t>(pages);
+    ck(cudaMemcpy(d_table, table.data(), pages * 4, cudaMemcpyHostToDevice), "table");
+
+    std::vector<float> K((size_t) cells * H * D), V((size_t) cells * H * D);
+    for (auto& x : K) x = nd(rng) * 0.5f;
+    for (auto& x : V) x = nd(rng) * 0.5f;
+
+    // ---- host reference: K in INT8 (kv_q8.hpp), rotated V in Q4_0 (kv_q4.hpp)
+    std::vector<int8_t> rk((size_t) cells * H * D);
+    std::vector<uint16_t> rk_s((size_t) cells * H * G);
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int g = 0; g < G; ++g) {
+                float amax = 0.0f;
+                for (int t = 0; t < k::KV_Q8_GROUP; ++t)
+                    amax = std::max(amax, std::fabs(K[((size_t) c * H + h) * D + g * k::KV_Q8_GROUP + t]));
+                const uint16_t sb = k::f16_from_f32(amax / 127.0f);
+                const float sf = k::f32_from_f16(sb);
+                for (int t = 0; t < k::KV_Q8_GROUP; ++t) {
+                    int q = 0;
+                    if (sf > 0.0f) {
+                        q = (int) std::nearbyint(K[((size_t) c * H + h) * D + g * k::KV_Q8_GROUP + t] / sf);   // rn: ties to even, as the kernel
+                        q = std::max(-127, std::min(127, q));
+                    }
+                    rk[((size_t) c * H + h) * D + g * k::KV_Q8_GROUP + t] = (int8_t) q;
+                }
+                rk_s[((size_t) c * H + h) * G + g] = sb;
+            }
+    std::vector<float> Vrot((size_t) cells * H * D);
+    std::memcpy(Vrot.data(), V.data(), V.size() * 4);
+    for (int c = 0; c < cells * H; ++c) fwht256_host_reference(Vrot.data() + (size_t) c * D);
+    std::vector<k::block_q4_0> rv4((size_t) cells * H * (D / 32));
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int b = 0; b < D / 32; ++b)
+                quantize_block_q4_0_host(&Vrot[((size_t) c * H + h) * D + b * 32],
+                                         rv4[((size_t) c * H + h) * (D / 32) + b]);
+
+    // ---- device: one cell appended through the COMPOSED hybrid calls, then gathered
+    const int64_t q4_row = (int64_t) k::kv_q4_bytes_per_head(D);
+    int8_t* d_kq = dalloc<int8_t>((size_t) cells * H * D);
+    uint16_t* d_ks = dalloc<uint16_t>((size_t) cells * H * G);
+    uint8_t* d_v4 = dalloc<uint8_t>((size_t) cells * H * q4_row);
+    float *d_kcur = dalloc<float>((size_t) H * D), *d_vcur = dalloc<float>((size_t) H * D);
+    int32_t* d_step = dalloc<int32_t>(k::kStepCount);
+
+    int append_bad = 0, gather_bad = 0;
+    for (int c = 0; c < cells; ++c) {
+        ck(cudaMemcpy(d_kcur, &K[(size_t) c * H * D], (size_t) H * D * 4, cudaMemcpyHostToDevice), "kcur");
+        ck(cudaMemcpy(d_vcur, &V[(size_t) c * H * D], (size_t) H * D * 4, cudaMemcpyHostToDevice), "vcur");
+        int32_t step[k::kStepCount];
+        k::qsa_step_fill(step, c, s);
+        ck(cudaMemcpy(d_step, step, sizeof step, cudaMemcpyHostToDevice), "step");
+        // exactly what qsa_layer's hybrid branch does
+        k::fwht256_inplace_cuda(d_vcur, H, nullptr);
+        k::kv_append_q8_step(d_kq, d_kq, d_ks, d_ks, d_table, d_step, d_kcur, d_kcur, s, nullptr, nullptr);
+        k::kv_append_q4_step(d_v4, d_v4, d_table, d_step, d_vcur, d_vcur, s, nullptr, nullptr);
+        ck(cudaGetLastError() != cudaSuccess ? cudaGetLastError() : cudaSuccess, "append");
+    }
+    ck(cudaDeviceSynchronize(), "append sync");
+
+    std::vector<int8_t> hk((size_t) cells * H * D);
+    std::vector<uint16_t> hks((size_t) cells * H * G);
+    std::vector<uint8_t> hv4((size_t) cells * H * q4_row);
+    ck(cudaMemcpy(hk.data(), d_kq, hk.size(), cudaMemcpyDeviceToHost), "kq");
+    ck(cudaMemcpy(hks.data(), d_ks, hks.size() * 2, cudaMemcpyDeviceToHost), "ks");
+    ck(cudaMemcpy(hv4.data(), d_v4, hv4.size(), cudaMemcpyDeviceToHost), "v4");
+    // the pools are PHYSICAL (page, head, slot); the reference is by LOGICAL cell - compare through the table
+    const k::block_q4_0* hv4b = reinterpret_cast<const k::block_q4_0*>(hv4.data());
+    for (int c = 0; c < cells; ++c) {
+        const long long row = ((long long) table[c / P] * H);
+        for (int h = 0; h < H; ++h) {
+            const long long r = (row + h) * P + c % P;
+            for (int d = 0; d < D; ++d)
+                if (hk[(size_t) r * D + d] != rk[((size_t) c * H + h) * D + d]) ++append_bad;
+            for (int g = 0; g < G; ++g)
+                if (hks[(size_t) r * G + g] != rk_s[((size_t) c * H + h) * G + g]) ++append_bad;
+            for (int b = 0; b < D / 32; ++b) {
+                const k::block_q4_0& got = hv4b[(size_t) r * (D / 32) + b];
+                if (got.d != rv4[((size_t) c * H + h) * (D / 32) + b].d ||
+                    std::memcmp(got.qs, rv4[((size_t) c * H + h) * (D / 32) + b].qs, 16) != 0) ++append_bad;
+            }
+        }
+    }
+    std::printf("[1/3] composed appends: %s (%d bad values of %zu K / %zu V)\n",
+                append_bad == 0 ? "PASS" : "FAIL", append_bad, hk.size(), rv4.size());
+
+    // gather every cell (ids = all, ascending - the identity selection for width = cells)
+    std::vector<int32_t> ids(cells);
+    for (int i = 0; i < cells; ++i) ids[i] = i;
+    int32_t* d_ids = dalloc<int32_t>(cells);
+    ck(cudaMemcpy(d_ids, ids.data(), cells * 4, cudaMemcpyHostToDevice), "ids");
+    int32_t step[k::kStepCount];
+    k::qsa_step_fill(step, cells - 1, s);
+    step[k::kStepWidth] = cells;   // select everything
+    ck(cudaMemcpy(d_step, step, sizeof step, cudaMemcpyHostToDevice), "step2");
+    uint16_t *d_ksc = dalloc<uint16_t>((size_t) cells * H * D), *d_vsc = dalloc<uint16_t>((size_t) cells * H * D);
+    k::kv_gather_q8_step(d_kq, d_kq, d_ks, d_ks, d_table, d_ids, d_step, cells, s, d_ksc, d_ksc, nullptr);
+    k::kv_gather_q4_step(d_v4, d_v4, d_table, d_ids, d_step, cells, s, d_vsc, d_vsc, nullptr);
+    ck(cudaDeviceSynchronize(), "gather sync");
+    std::vector<uint16_t> h_ksc((size_t) cells * H * D), h_vsc((size_t) cells * H * D);
+    ck(cudaMemcpy(h_ksc.data(), d_ksc, h_ksc.size() * 2, cudaMemcpyDeviceToHost), "ksc");
+    ck(cudaMemcpy(h_vsc.data(), d_vsc, h_vsc.size() * 2, cudaMemcpyDeviceToHost), "vsc");
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int d = 0; d < D; ++d) {
+                const float kref = k::f32_from_f16(k::f16_from_f32(
+                    (float) rk[((size_t) c * H + h) * D + d] * k::f32_from_f16(rk_s[((size_t) c * H + h) * G + d / k::KV_Q8_GROUP])));
+                if (k::f32_from_f16(h_ksc[((size_t) c * H + h) * D + d]) != kref) ++gather_bad;
+                const k::block_q4_0& b = rv4[((size_t) c * H + h) * (D / 32) + d / 32];
+                const int nib = d % 32 < 16 ? (b.qs[d % 32] & 0x0F) - 8 : ((b.qs[d % 32 - 16] >> 4) - 8);
+                const float vref = k::f32_from_f16(k::f16_from_f32((float) nib * k::f32_from_f16(b.d)));
+                if (k::f32_from_f16(h_vsc[((size_t) c * H + h) * D + d]) != vref) ++gather_bad;
+            }
+    std::printf("[2/3] composed gathers: %s (%d bad of %zu)\n",
+                gather_bad == 0 ? "PASS" : "FAIL", gather_bad, h_ksc.size());
+
+    // ---- attention: qsa_decode_attn KV_MODE 3 over the hybrid pools + output fwht, vs fp32 references
+    std::vector<float> q((size_t) QH * D);
+    for (auto& x : q) x = nd(rng);
+    float* d_q = dalloc<float>(q.size());
+    ck(cudaMemcpy(d_q, q.data(), q.size() * 4, cudaMemcpyHostToDevice), "q");
+    k::QsaAttnPools pools;
+    pools.k_q = d_kq; pools.k_scale = d_ks; pools.v_q4 = d_v4; pools.page_table = d_table;
+    float* d_scr = dalloc<float>(k::qsa_decode_attn_scratch_floats(cells, s) + 16);
+    float* d_attn = dalloc<float>((size_t) QH * D);
+    k::qsa_decode_attn_step(d_q, pools, d_ids, d_step, cells, s, d_scr, d_attn, nullptr);
+    k::fwht256_inplace_cuda(d_attn, QH, nullptr);   // the engine's inverse rotation of the output
+    ck(cudaDeviceSynchronize(), "attn sync");
+    std::vector<float> h_attn((size_t) QH * D);
+    ck(cudaMemcpy(h_attn.data(), d_attn, h_attn.size() * 4, cudaMemcpyDeviceToHost), "attn");
+
+    // reference A: attention over the SAME dequantized K/V (isolates the kernel; fp32 host math)
+    // reference B: attention over the TRUE unquantized K/V (bounds the format's error)
+    auto host_attn = [&](const std::vector<float>& Keq, const std::vector<float>& Veq, std::vector<float>& out) {
+        out.assign((size_t) QH * D, 0.0f);
+        for (int h = 0; h < QH; ++h) {
+            const int kvh = h / PER;
+            std::vector<float> sc(cells);
+            float m = -1e30f;
+            for (int c = 0; c < cells; ++c) {
+                float d_ = 0.0f;
+                for (int t = 0; t < D; ++t)
+                    d_ += q[(size_t) h * D + t] * Keq[((size_t) c * H + kvh) * D + t];
+                sc[c] = d_ / std::sqrt((float) D);
+                m = std::max(m, sc[c]);
+            }
+            float l = 0.0f;
+            for (int c = 0; c < cells; ++c) { sc[c] = std::exp(sc[c] - m); l += sc[c]; }
+            for (int c = 0; c < cells; ++c)
+                for (int t = 0; t < D; ++t) out[(size_t) h * D + t] += sc[c] / l * Veq[((size_t) c * H + kvh) * D + t];
+        }
+    };
+    std::vector<float> Kdq((size_t) cells * H * D), Vdq((size_t) cells * H * D);
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int d = 0; d < D; ++d) {
+                Kdq[((size_t) c * H + h) * D + d] =
+                    (float) rk[((size_t) c * H + h) * D + d] * k::f32_from_f16(rk_s[((size_t) c * H + h) * G + d / k::KV_Q8_GROUP]);
+                const k::block_q4_0& b = rv4[((size_t) c * H + h) * (D / 32) + d / 32];
+                const int nib = d % 32 < 16 ? (b.qs[d % 32] & 0x0F) - 8 : ((b.qs[d % 32 - 16] >> 4) - 8);
+                Vdq[((size_t) c * H + h) * D + d] = (float) nib * k::f32_from_f16(b.d);
+            }
+    // un-rotate the dequantized V (the kernel returns the output in the UNROTATED domain)
+    std::vector<float> Vdq_un(Vdq.size());
+    std::memcpy(Vdq_un.data(), Vdq.data(), Vdq.size() * 4);
+    for (int c = 0; c < cells * H; ++c) fwht256_host_reference(Vdq_un.data() + (size_t) c * D);
+
+    // ---- batched form with TWO queries and per-query step arrays (the verify.cpp window's exact call)
+    {
+        const int nq = 2;
+        std::vector<float> q2((size_t) nq * QH * D);
+        for (auto& x : q2) x = nd(rng);
+        float* d_q2 = dalloc<float>(q2.size());
+        ck(cudaMemcpy(d_q2, q2.data(), q2.size() * 4, cudaMemcpyHostToDevice), "q2");
+        // the batched kernel indexes ids[z * cap + ...]: one FULL selection per query
+        std::vector<int32_t> ids2((size_t) nq * cells);
+        for (int i = 0; i < nq; ++i) for (int c = 0; c < cells; ++c) ids2[(size_t) i * cells + c] = c;
+        int32_t* d_ids2 = dalloc<int32_t>((size_t) nq * cells);
+        ck(cudaMemcpy(d_ids2, ids2.data(), ids2.size() * 4, cudaMemcpyHostToDevice), "ids2");
+        int32_t* d_steps2 = dalloc<int32_t>(nq * k::kStepCount);
+        for (int i = 0; i < nq; ++i) {
+            int32_t st2[k::kStepCount];
+            k::qsa_step_fill(st2, cells - 1 - i, s);
+            st2[k::kStepWidth] = cells - i;   // slightly different widths, as verify's windows have
+            ck(cudaMemcpy(d_steps2 + i * k::kStepCount, st2, sizeof st2, cudaMemcpyHostToDevice), "steps2");
+        }
+        float* d_scr2 = dalloc<float>(k::qsa_decode_attn_scratch_floats(cells, s) * nq + 64);
+        float* d_at2 = dalloc<float>((size_t) nq * QH * D);
+        k::qsa_decode_attn_batch(d_q2, pools, d_ids2, d_steps2, cells, s, d_scr2, d_at2, nq, nullptr);
+        k::fwht256_inplace_cuda(d_at2, (int64_t) nq * QH, nullptr);
+        ck(cudaDeviceSynchronize(), "attn2 sync");
+        std::vector<float> h_at2((size_t) nq * QH * D);
+        ck(cudaMemcpy(h_at2.data(), d_at2, h_at2.size() * 4, cudaMemcpyDeviceToHost), "at2");
+        double max_b = 0.0;
+        double per_q[2] = {0.0, 0.0};
+        for (int i = 0; i < nq; ++i)
+            for (int h = 0; h < QH; ++h) {
+                // reference over the cells this query's width selects (ids are ascending: the first width)
+                const int w = cells - i;
+                std::vector<float> sc(w);
+                float m = -1e30f;
+                for (int c = 0; c < w; ++c) {
+                    float d_ = 0.0f;
+                    for (int t = 0; t < D; ++t)
+                        d_ += q2[((size_t) i * QH + h) * D + t] * Kdq[((size_t) c * H + h / PER) * D + t];
+                    sc[c] = d_ / std::sqrt((float) D);
+                    m = std::max(m, sc[c]);
+                }
+                float l = 0.0f;
+                for (int c = 0; c < w; ++c) { sc[c] = std::exp(sc[c] - m); l += sc[c]; }
+                for (int d = 0; d < D; ++d) {
+                    float acc = 0.0f;
+                    for (int c = 0; c < w; ++c)
+                        acc += sc[c] / l * Vdq_un[((size_t) c * H + h / PER) * D + d];   // dim d only
+                    const double e = (double) std::fabs(h_at2[((size_t) i * QH + h) * D + d] - acc);
+                    max_b = std::max(max_b, e);
+                    per_q[i] = std::max(per_q[i], e);
+                }
+            }
+        const bool bok = max_b < 5e-3;
+        std::printf("       (per-query max: q0 %.2e, q1 %.2e)\n", per_q[0], per_q[1]);
+        std::printf("[4/4] mode-3 BATCHED attention (verify's form): %s (vs dequant ref %.2e)\n",
+                    bok ? "PASS" : "FAIL", max_b);
+        if (!bok) g_fail = 1;
+    }
+
+    std::vector<float> ref_deq, ref_true;
+    host_attn(Kdq, Vdq_un, ref_deq);
+    host_attn(K, V, ref_true);
+    double max_deq = 0.0, max_true = 0.0;
+    for (size_t i = 0; i < h_attn.size(); ++i) {
+        max_deq = std::max(max_deq, (double) std::fabs(h_attn[i] - ref_deq[i]));
+        max_true = std::max(max_true, (double) std::fabs(h_attn[i] - ref_true[i]));
+    }
+    // The tensor-core prompt path (qsa_prompt_attn, 0.1.22+) takes hybrid pools as KV_MODE 3: INT8 K, and V
+    // dequantized from its q4_0 blocks to fp16 at gather. Its summation order differs from the old kernel's
+    // (accuracy-level, not bitwise - qsa_prompt_attn.hpp says the same of its int8 mode), so: tolerance check
+    // against the SAME dequant reference, output un-rotated exactly as prefill.cpp does.
+    {
+        float* d_at4 = dalloc<float>((size_t) QH * D);
+        const bool took = k::qsa_prompt_attn_batch(d_q, pools, d_ids, d_step, cells, s, d_at4, 1, nullptr);
+        if (!took) {
+#if defined(STRATA_USE_HIP)
+            // AMD: the tensor-core prompt path is CUDA-only, so it refuses every pool and the old kernel runs
+            std::printf("[5/5] qsa_prompt_attn mode 3: PASS (refused on HIP - the old kernel runs)\n");
+#else
+            std::printf("[5/5] qsa_prompt_attn mode 3: FAIL (refused the hybrid pools)\n");
+            g_fail = 1;
+#endif
+        } else {
+            k::fwht256_inplace_cuda(d_at4, QH, nullptr);
+            ck(cudaDeviceSynchronize(), "at4 sync");
+            std::vector<float> h_at4((size_t) QH * D);
+            ck(cudaMemcpy(h_at4.data(), d_at4, h_at4.size() * 4, cudaMemcpyDeviceToHost), "at4");
+            double max_p = 0.0;
+            for (size_t i = 0; i < h_at4.size(); ++i)
+                max_p = std::max(max_p, (double) std::fabs(h_at4[i] - ref_deq[i]));
+            const bool pok = max_p < 1e-2;
+            std::printf("[5/5] qsa_prompt_attn mode 3 (tensor cores): %s (vs dequant ref %.2e)\n",
+                        pok ? "PASS" : "FAIL", max_p);
+            if (!pok) g_fail = 1;
+        }
+    }
+
+    const bool attn_ok = max_deq < 5e-3;   // fp32 kernel math vs fp64-ish host accumulation
+    std::printf("[3/3] mode-3 attention + output fwht: %s (vs dequant ref %.2e, vs true fp32 %.2e)\n",
+                attn_ok ? "PASS" : "FAIL", max_deq, max_true);
+
+    const bool ok = append_bad == 0 && gather_bad == 0 && attn_ok && g_fail == 0;
+    std::printf("=== kv_hybrid_parity: %s ===\n", ok ? "ALL PASS" : "FAILED");
+    return ok ? 0 : 1;
+}

src/kernels/qsa_select_bench.cpp

@@ -0,0 +1,139 @@
+// src/kernels/qsa_select_bench.cpp - the prompt path's QSA selection (qsa_select.hpp) timed per stage, block scores
+// (the warp kernel and the tensor-core one) and top-k, for a batch of consecutive queries at a given context, and
+// the two scorers compared: score difference and how many selections differ (GPU, synthetic, no model).
+// Usage: qsa_select_bench [context=131072] [queries=256] [reps=10]
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_select.hpp"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+void ck(cudaError_t e, const char* w) {
+    if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", w, cudaGetErrorString(e)); std::exit(2); }
+}
+template <typename T> T* up(const std::vector<T>& h) {
+    T* d = nullptr;
+    ck(cudaMalloc(&d, h.size() * sizeof(T) + 64), "malloc");
+    ck(cudaMemcpy(d, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "upload");
+    return d;
+}
+}  // namespace
+
+int main(int argc, char** argv) {
+    const int64_t ctx = argc > 1 ? std::atoll(argv[1]) : 131072;
+    const int64_t nq = argc > 2 ? std::atoll(argv[2]) : 256;
+    const int reps = argc > 3 ? std::atoi(argv[3]) : 10;
+    const k::QsaShapes s = k::qsa_real_shapes();
+    const int64_t max_blocks = ctx / 4 + 1, cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    std::mt19937 rng(7);
+    std::normal_distribution<float> nd(0.f, 1.f);
+    // keys with a shared direction plus noise, so the scores have a spread like a real indexer's
+    std::vector<float> dir(128), pooled((size_t) (max_blocks * 128)), dead(128), q((size_t) (nq * 512));
+    for (auto& x : dir) x = nd(rng);
+    for (int64_t b = 0; b < max_blocks; ++b) {
+        const float a = nd(rng);
+        for (int d = 0; d < 128; ++d) pooled[(size_t) (b * 128 + d)] = 0.5f * a * dir[d] + nd(rng);
+    }
+    for (auto& x : dead) x = nd(rng);
+    for (int64_t i = 0; i < nq; ++i)
+        for (int d = 0; d < 512; ++d) q[(size_t) (i * 512 + d)] = 0.2f * dir[d % 128] + 0.1f * nd(rng);
+    std::vector<int32_t> steps((size_t) (nq * k::kStepCount));
+    for (int64_t i = 0; i < nq; ++i) {   // qsa_step_fill's arithmetic (kept here so the bench links only qsa_select)
+        int32_t* st = steps.data() + i * k::kStepCount;
+        const int64_t pos = ctx - nq + i;
+        st[k::kStepPos] = (int32_t) pos;
+        st[k::kStepNKv] = (int32_t) (pos + 1);
+        st[k::kStepNBid] = (int32_t) ((pos + 1) / s.idx_block);
+        st[k::kStepWidth] = (int32_t) k::qsa_selection_width(pos + 1, s);
+    }
+    const float* d_pooled = up(pooled);
+    const float* d_dead = up(dead);
+    const float* d_q = up(q);
+    const int32_t* d_steps = up(steps);
+    float *sc_old = nullptr, *sc_new = nullptr;
+    int32_t *ids_old = nullptr, *ids_new = nullptr;
+    ck(cudaMalloc(&sc_old, (size_t) (nq * max_blocks) * 4), "malloc");
+    ck(cudaMalloc(&sc_new, (size_t) (nq * max_blocks) * 4), "malloc");
+    ck(cudaMalloc(&ids_old, (size_t) (nq * cap) * 4), "malloc");
+    ck(cudaMalloc(&ids_new, (size_t) (nq * cap) * 4), "malloc");
+    const int64_t active = steps[(size_t) ((nq - 1) * k::kStepCount + k::kStepNBid)] + 1;
+    auto run_old = [&] { k::qsa_block_scores(d_pooled, d_dead, d_q, d_steps, nq, max_blocks, s, sc_old, nullptr, active); };
+    auto run_new = [&] {
+        if (!k::qsa_block_scores_tc(d_pooled, d_dead, d_q, d_steps, nq, max_blocks, s, sc_new, nullptr, active)) {
+            std::fprintf(stderr, "tc scorer refused\n");
+            std::exit(2);
+        }
+    };
+    run_old();
+    run_new();
+    k::qsa_block_topk_ref(sc_old, d_steps, nq, max_blocks, cap, s, ids_old, nullptr);
+    k::qsa_block_topk(sc_new, d_steps, nq, max_blocks, cap, s, ids_new, nullptr);
+    int32_t* ids_reg = nullptr;   // the register top-k on the OLD scores: must equal the reference exactly
+    ck(cudaMalloc(&ids_reg, (size_t) (nq * cap) * 4), "malloc");
+    k::qsa_block_topk(sc_old, d_steps, nq, max_blocks, cap, s, ids_reg, nullptr);
+    ck(cudaDeviceSynchronize(), "warm");
+    // compare
+    std::vector<float> a((size_t) (nq * max_blocks)), b(a.size());
+    std::vector<int32_t> ia((size_t) (nq * cap)), ib(ia.size());
+    ck(cudaMemcpy(a.data(), sc_old, a.size() * 4, cudaMemcpyDeviceToHost), "down");
+    ck(cudaMemcpy(b.data(), sc_new, b.size() * 4, cudaMemcpyDeviceToHost), "down");
+    ck(cudaMemcpy(ia.data(), ids_old, ia.size() * 4, cudaMemcpyDeviceToHost), "down");
+    ck(cudaMemcpy(ib.data(), ids_new, ib.size() * 4, cudaMemcpyDeviceToHost), "down");
+    std::vector<int32_t> ir(ia.size());
+    ck(cudaMemcpy(ir.data(), ids_reg, ir.size() * 4, cudaMemcpyDeviceToHost), "down");
+    int64_t reg_same = 0;
+    for (int64_t i = 0; i < nq; ++i) {
+        const int64_t w = steps[(size_t) (i * k::kStepCount + k::kStepWidth)];
+        reg_same += std::equal(ia.begin() + i * cap, ia.begin() + i * cap + w, ir.begin() + i * cap);
+    }
+    double max_rel = 0, sum_rel = 0, n_rel = 0;
+    int64_t same_sel = 0, cells_diff = 0, cells_all = 0;
+    for (int64_t i = 0; i < nq; ++i) {
+        const int32_t* st = steps.data() + i * k::kStepCount;
+        for (int64_t j = 0; j <= st[k::kStepNBid]; ++j) {
+            const double x = a[(size_t) (i * max_blocks + j)], y = b[(size_t) (i * max_blocks + j)];
+            const double r = std::fabs(x - y) / std::max(1e-6, std::fabs(x));
+            max_rel = std::max(max_rel, r);
+            sum_rel += r;
+            n_rel += 1;
+        }
+        const int64_t w = st[k::kStepWidth];
+        std::vector<int32_t> x(ia.begin() + i * cap, ia.begin() + i * cap + w), y(ib.begin() + i * cap, ib.begin() + i * cap + w);
+        same_sel += x == y;
+        std::vector<int32_t> d;
+        std::set_symmetric_difference(x.begin(), x.end(), y.begin(), y.end(), std::back_inserter(d));
+        cells_diff += (int64_t) d.size() / 2;
+        cells_all += w;
+    }
+    // time
+    cudaEvent_t e0, e1;
+    cudaEventCreate(&e0); cudaEventCreate(&e1);
+    auto timed = [&](auto f) {
+        cudaEventRecord(e0);
+        for (int r = 0; r < reps; ++r) f();
+        cudaEventRecord(e1);
+        ck(cudaEventSynchronize(e1), "time");
+        float ms = 0;
+        cudaEventElapsedTime(&ms, e0, e1);
+        return ms / reps;
+    };
+    const float t_old = timed(run_old), t_new = timed(run_new);
+    const float t_tk = timed([&] { k::qsa_block_topk_ref(sc_old, d_steps, nq, max_blocks, cap, s, ids_old, nullptr); });
+    const float t_tk2 = timed([&] { k::qsa_block_topk(sc_old, d_steps, nq, max_blocks, cap, s, ids_reg, nullptr); });
+    std::printf("top-k %.3f -> %.3f ms (%.1fx), register top-k identical to the reference %lld/%lld\n", t_tk, t_tk2,
+                t_tk / t_tk2, (long long) reg_same, (long long) nq);
+    std::printf("ctx %lld, %lld queries x %lld blocks: scores %.3f -> %.3f ms (%.1fx), top-k %.3f ms; score rel diff "
+                "mean %.2g max %.2g; selections identical %lld/%lld, cells differing %.4f%%\n", (long long) ctx,
+                (long long) nq, (long long) active, t_old, t_new, t_old / t_new, t_tk, sum_rel / std::max(1.0, n_rel),
+                max_rel, (long long) same_sel, (long long) nq, cells_all ? 100.0 * (double) cells_diff / (double) cells_all : 0.0);
+    return 0;
+}

src/ngram/ple_reader.cpp

@@ -5,6 +5,7 @@
 #include <condition_variable>
 #include <cstring>
 #include <deque>
+#include <exception>
 #include <mutex>
 #include <thread>
 #include <unordered_map>
@@ -118,6 +119,14 @@ struct PleReader::Impl {
     uint8_t* slot_buf(uint32_t s) { return slab + (size_t) s * 2 * PAGE; }
     bool busy() const { return free_slots.size() < max_inflight || !delayed.empty(); }
 
+    void cancel_queued() {
+        for (const Job& job : queue) {
+            auto it = tickets.find(job.ticket);
+            if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
+        }
+        queue.clear();
+    }
+
     void record_latency(double us) {
         stats.read_us_sum += us;
         if (stats.read_us.size() < LATENCY_RING) stats.read_us.push_back((float) us);
@@ -132,7 +141,14 @@ struct PleReader::Impl {
             queue.pop_front();
             Job& j = inflight[s];
             j.issued_us = now_us();
-            if (!file.submit(j.offset, slot_buf(s), j.length, s, error)) return false;
+            if (!file.submit(j.offset, slot_buf(s), j.length, s, error)) {
+                j.uses.clear();
+                auto it = tickets.find(j.ticket);
+                if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
+                free_slots.push_back(s);
+                cancel_queued();
+                return false;
+            }
             stats.submit_us += now_us() - j.issued_us;
             ++stats.reads;
         }
@@ -141,9 +157,19 @@ struct PleReader::Impl {
 
     bool finish(const Completion& c) {
         const uint32_t s = (uint32_t) c.tag;
+        if (s >= inflight.size()) {
+            error = "PleReader: invalid table-read completion";
+            cancel_queued();
+            return false;
+        }
         Job& j = inflight[s];
         if (!c.ok) {
             error = "PleReader: a table read failed";
+            cancel_queued();
+            j.uses.clear();
+            auto it = tickets.find(j.ticket);
+            if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
+            free_slots.push_back(s);
             return false;
         }
         record_latency(now_us() - j.issued_us);
@@ -152,8 +178,15 @@ struct PleReader::Impl {
         for (const Use& u : j.uses) {
             if (u.in_page + ROW_BYTES > c.bytes) {
                 error = "PleReader: short read inside the table";
+                cancel_queued();
+                j.uses.clear();
+                auto it = tickets.find(j.ticket);
+                if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
+                free_slots.push_back(s);
                 return false;
             }
+        }
+        for (const Use& u : j.uses) {
             std::memcpy(u.dst, buf + u.in_page, ROW_BYTES);
             cache.insert(u.row, buf + u.in_page);
         }
@@ -161,21 +194,28 @@ struct PleReader::Impl {
         if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
         j.uses.clear();
         free_slots.push_back(s);
-        return pump();
+        return error.empty() ? pump() : true;
     }
 
     /// Completions (or wake packets) just returned by `file.wait`, applying fault injection.
     bool process(const Completion* got, int n) {
+        bool ok = true;
         for (int i = 0; i < n; ++i) {
             if (got[i].tag == DirectFile::WAKE_TAG) continue;
+            if (got[i].tag >= inflight.size()) {
+                error = "PleReader: invalid table-read completion";
+                cancel_queued();
+                ok = false;
+                continue;
+            }
             if (delay_us > 0 && now_us() - inflight[(uint32_t) got[i].tag].issued_us < delay_us) {
                 delayed.push_back(got[i]);
                 ++stats.late_injected;
                 continue;
             }
-            if (!finish(got[i])) return false;
+            if (!finish(got[i])) ok = false;   // drain the rest of this batch so no completed slot is stranded
         }
-        return true;
+        return ok;
     }
 
     bool release_delayed() {
@@ -256,7 +296,19 @@ bool PleReader::open(const std::string& path, uint64_t table_offset, uint64_t n_
     reset_stats();
     impl_->stop = false;
     impl_->threaded = io_thread;
-    if (io_thread) impl_->worker = std::thread([this] { impl_->worker_loop(); });
+    if (io_thread) {
+        try {
+            impl_->worker = std::thread([this] { impl_->worker_loop(); });
+        } catch (const std::exception& e) {
+            err = std::string("PleReader: cannot create I/O worker: ") + e.what();
+            close();
+            return false;
+        } catch (...) {
+            err = "PleReader: cannot create I/O worker";
+            close();
+            return false;
+        }
+    }
     return true;
 }
 
@@ -341,10 +393,11 @@ PleReader::Ticket PleReader::issue(const uint32_t* rows, size_t n, uint8_t* out_
     // Sorted by offset: prefill chunks then read the SSD in near-sequential order.
     std::sort(jobs.begin(), jobs.end(), [](const Job& a, const Job& b) { return a.offset < b.offset; });
     ts.pending = (uint32_t) jobs.size();
+    const bool has_jobs = ts.pending > 0;
     for (Job& j : jobs) m.queue.push_back(std::move(j));
     if (m.threaded) {
         lk.unlock();
-        if (ts.pending > 0) {
+        if (has_jobs) {
             m.cv_work.notify_one();
             m.file.wake();                         // in case the worker is blocked in the port
         }
@@ -361,21 +414,26 @@ bool PleReader::collect(Ticket t, std::string& err) {
         std::unique_lock<std::mutex> lk(m.mu);
         auto it = m.tickets.find(t.id);
         if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }
-        m.cv_done.wait(lk, [&] { return !m.error.empty() || m.tickets[t.id].pending == 0; });
+        m.cv_done.wait(lk, [&] {
+            const auto current = m.tickets.find(t.id);
+            return current == m.tickets.end() || current->second.pending == 0;
+        });
+        it = m.tickets.find(t.id);
+        if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }
         if (!m.error.empty()) { err = m.error; return false; }
         m.stats.wait_us += now_us() - start;
-        m.tickets.erase(t.id);
+        m.tickets.erase(it);
         return true;
     }
     auto it = m.tickets.find(t.id);
     if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }
     while (it->second.pending > 0) {
-        if (!m.error.empty() || !m.drain(-1)) {
-            err = m.error.empty() ? "PleReader: read failed" : m.error;
-            return false;
-        }
+        const bool drained = m.drain(-1);
+        if (!drained && m.error.empty()) m.error = "PleReader: read failed";
         it = m.tickets.find(t.id);
+        if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }
     }
+    if (!m.error.empty()) { err = m.error; return false; }
     m.stats.wait_us += now_us() - start;
     m.tickets.erase(it);
     return true;

src/prefill/gemm.cu

@@ -5,8 +5,32 @@
 #include <cublas_v2.h>
 #include <cuda_runtime.h>
 
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+// The HIP compatibility shim maps CUDA shuffle spellings to Strata helpers.
+// hipBLASLt's public headers declare native HIP shuffle functions, so keep
+// those declarations from being macro-expanded in this translation unit.
+#undef __shfl_xor_sync
+#undef __shfl_down_sync
+#undef __shfl_up_sync
+#undef __shfl_sync
+#undef __ballot_sync
+#endif
+
+#include <climits>
 #include <cstdio>
 #include <cstdlib>
+#include <cstring>
+#include <memory>
+
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+#include "hipblaslt_tuning.hpp"
+#include <hip/hip_runtime_api.h>
+#include <hipblaslt/hipblaslt.h>
+#include <hipblaslt/hipblaslt-ext.hpp>
+#include <map>
+#include <set>
+#include <tuple>
+#endif
 
 namespace strata::prefill {
 namespace {
@@ -18,9 +42,239 @@ void ck(cublasStatus_t s, const char* what) {
     }
 }
 
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+struct HipLtCallKey {
+    strata::prefill::hipblaslt::InputType type;
+    int t;
+    int n;
+    int k;
+    int ldy;
+    uint32_t beta_bits;
+
+    bool operator<(const HipLtCallKey& other) const {
+        return std::tie(type, n, k, ldy, t, beta_bits) <
+               std::tie(other.type, other.n, other.k, other.ldy, other.t, other.beta_bits);
+    }
+};
+
+struct HipLtCachedAlgo {
+    bool supported = false;
+    hipblasLtMatmulAlgo_t algo{};
+    size_t workspace_bytes = 0;
+};
+
+struct HipLtState {
+    hipblasLtHandle_t handle = nullptr;
+    void* workspace = nullptr;
+    size_t workspace_bytes = 0;
+    strata::prefill::hipblaslt::TuningTable table;
+    std::map<HipLtCallKey, HipLtCachedAlgo> cache;
+    uint64_t lt_launches = 0;
+    uint64_t fallbacks = 0;
+    std::set<std::tuple<strata::prefill::hipblaslt::InputType, int, int, int, int>> fallback_shapes;
+
+    ~HipLtState() {
+        if (std::getenv("STRATA_HIPBLASLT_VERBOSE")) {
+            std::fprintf(stderr, "prefill gemm: hipBLASLt summary launches=%llu fallbacks=%llu unique_fallback_shapes=%zu\n",
+                         (unsigned long long) lt_launches, (unsigned long long) fallbacks, fallback_shapes.size());
+            for (const auto& shape : fallback_shapes) {
+                const auto type = std::get<0>(shape);
+                std::fprintf(stderr, "prefill gemm: fallback shape dtype=%s T=%d N=%d K=%d ldy=%d\n",
+                             type == strata::prefill::hipblaslt::InputType::bf16 ? "bf16" : "f16",
+                             std::get<1>(shape), std::get<2>(shape), std::get<3>(shape), std::get<4>(shape));
+            }
+        }
+        if (handle) hipblasLtDestroy(handle);
+    }
+};
+
+struct HipLtDescriptors {
+    hipblasLtMatmulDesc_t op = nullptr;
+    hipblasLtMatrixLayout_t a = nullptr;
+    hipblasLtMatrixLayout_t b = nullptr;
+    hipblasLtMatrixLayout_t c = nullptr;
+
+    ~HipLtDescriptors() {
+        if (op) hipblasLtMatmulDescDestroy(op);
+        if (a) hipblasLtMatrixLayoutDestroy(a);
+        if (b) hipblasLtMatrixLayoutDestroy(b);
+        if (c) hipblasLtMatrixLayoutDestroy(c);
+    }
+
+    bool init(hipDataType type, int t, int n, int k, int ldy) {
+        const hipblasOperation_t trans_a = HIPBLAS_OP_T;
+        const hipblasOperation_t trans_b = HIPBLAS_OP_N;
+        if (hipblasLtMatmulDescCreate(&op, HIPBLAS_COMPUTE_32F, HIP_R_32F) != HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatmulDescSetAttribute(op, HIPBLASLT_MATMUL_DESC_TRANSA, &trans_a, sizeof(trans_a)) !=
+                HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatmulDescSetAttribute(op, HIPBLASLT_MATMUL_DESC_TRANSB, &trans_b, sizeof(trans_b)) !=
+                HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatrixLayoutCreate(&a, type, k, n, k) != HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatrixLayoutCreate(&b, type, k, t, k) != HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatrixLayoutCreate(&c, HIP_R_32F, n, t, ldy) != HIPBLAS_STATUS_SUCCESS) {
+            return false;
+        }
+        return true;
+    }
+};
+
+std::unique_ptr<HipLtState> create_hipblaslt_state(void* workspace, size_t workspace_bytes) {
+    const char* path = std::getenv("STRATA_HIPBLASLT_TUNING");
+    if (!path || !*path) return nullptr;
+
+    auto state = std::make_unique<HipLtState>();
+    state->workspace = workspace;
+    state->workspace_bytes = workspace_bytes;
+    if (hipblasLtCreate(&state->handle) != HIPBLAS_STATUS_SUCCESS) {
+        std::fprintf(stderr, "prefill gemm: hipBLASLt handle creation failed; using hipBLASEx\n");
+        return nullptr;
+    }
+
+    int version = 0;
+    if (hipblasLtGetVersion(state->handle, &version) != HIPBLAS_STATUS_SUCCESS) {
+        std::fprintf(stderr, "prefill gemm: hipBLASLt version query failed; using hipBLASEx\n");
+        return nullptr;
+    }
+    int device = 0;
+    hipDeviceProp_t properties{};
+    if (hipGetDevice(&device) != hipSuccess || hipGetDeviceProperties(&properties, device) != hipSuccess) {
+        std::fprintf(stderr, "prefill gemm: HIP device query failed; using hipBLASEx\n");
+        return nullptr;
+    }
+    std::string arch(properties.gcnArchName);
+    const auto suffix = arch.find(':');
+    if (suffix != std::string::npos) arch.resize(suffix);
+
+    std::string error;
+    if (!state->table.load(path, arch, version, error)) {
+        std::fprintf(stderr, "prefill gemm: %s; using hipBLASEx\n", error.c_str());
+        return nullptr;
+    }
+    std::fprintf(stderr, "prefill gemm: hipBLASLt tuning enabled (%zu rows, %s, version %d)\n",
+                 state->table.rows().size(), arch.c_str(), version);
+    return state;
+}
+
+HipLtCachedAlgo resolve_hipblaslt_algo(HipLtState& state, strata::prefill::hipblaslt::InputType type, int t,
+                                       int n, int k, int ldy, float beta) {
+    uint32_t beta_bits = 0;
+    static_assert(sizeof(beta_bits) == sizeof(beta));
+    std::memcpy(&beta_bits, &beta, sizeof(beta));
+    const HipLtCallKey key{type, t, n, k, ldy, beta_bits};
+    const auto cached = state.cache.find(key);
+    if (cached != state.cache.end()) return cached->second;
+
+    HipLtCachedAlgo resolved;
+    const bool verbose = std::getenv("STRATA_HIPBLASLT_VERBOSE") != nullptr;
+    const auto* row = state.table.closest(type, n, k, ldy, t);
+    if (!row) {
+        if (verbose) {
+            std::fprintf(stderr, "prefill gemm: Lt fallback; no calibration for dtype=%s T=%d N=%d K=%d ldy=%d\n",
+                         type == strata::prefill::hipblaslt::InputType::bf16 ? "bf16" : "f16", t, n, k, ldy);
+        }
+        return state.cache.emplace(key, resolved).first->second;
+    }
+
+    HipLtDescriptors desc;
+    const hipDataType input_type = type == strata::prefill::hipblaslt::InputType::bf16 ? HIP_R_16BF : HIP_R_16F;
+    if (!desc.init(input_type, t, n, k, ldy)) {
+        return state.cache.emplace(key, resolved).first->second;
+    }
+
+    std::vector<int> solution_ids{row->solution_id};
+    std::vector<hipblasLtMatmulHeuristicResult_t> candidates;
+    if (hipblaslt_ext::getAlgosFromIndex(state.handle, solution_ids, candidates) != HIPBLAS_STATUS_SUCCESS ||
+        candidates.empty() || candidates.front().state != HIPBLAS_STATUS_SUCCESS ||
+        hipblaslt_ext::getIndexFromAlgo(candidates.front().algo) != row->solution_id) {
+        if (verbose) {
+            std::fprintf(stderr, "prefill gemm: Lt fallback; solution %d unavailable for T=%d N=%d K=%d ldy=%d\n",
+                         row->solution_id, t, n, k, ldy);
+        }
+        return state.cache.emplace(key, resolved).first->second;
+    }
+
+    const float alpha = 1.0f;
+    size_t required_workspace = 0;
+    auto algo = candidates.front().algo;
+    if (hipblaslt_ext::matmulIsAlgoSupported(state.handle, desc.op, &alpha, desc.a, desc.b, &beta, desc.c, desc.c,
+                                             algo, required_workspace) != HIPBLAS_STATUS_SUCCESS) {
+        if (verbose) {
+            std::fprintf(stderr, "prefill gemm: Lt fallback; solution %d rejects actual T=%d N=%d K=%d ldy=%d beta=%.9g\n",
+                         row->solution_id, t, n, k, ldy, beta);
+        }
+        return state.cache.emplace(key, resolved).first->second;
+    }
+
+    resolved.supported = true;
+    resolved.algo = algo;
+    resolved.workspace_bytes = required_workspace;
+    if (verbose) {
+        std::fprintf(stderr,
+                     "prefill gemm: Lt solution=%d dtype=%s T=%d N=%d K=%d ldy=%d beta=%.9g workspace=%zu\n",
+                     row->solution_id, type == strata::prefill::hipblaslt::InputType::bf16 ? "bf16" : "f16", t, n,
+                     k, ldy, beta, required_workspace);
+    }
+    return state.cache.emplace(key, resolved).first->second;
+}
+
+bool try_hipblaslt(void* opaque_state, strata::prefill::hipblaslt::InputType type, const uint16_t* x,
+                   const uint16_t* w, float* y, int64_t t, int64_t n, int64_t k, int64_t ldy, float beta,
+                   void* stream) {
+    auto* state = static_cast<HipLtState*>(opaque_state);
+    if (!state || t <= 0 || n <= 0 || k <= 0 || t > INT_MAX || n > INT_MAX || k > INT_MAX || ldy > INT_MAX ||
+        ldy < n) {
+        return false;
+    }
+    const auto resolved = resolve_hipblaslt_algo(*state, type, (int) t, (int) n, (int) k, (int) ldy, beta);
+    if (!resolved.supported) {
+        ++state->fallbacks;
+        state->fallback_shapes.emplace(type, (int) t, (int) n, (int) k, (int) ldy);
+        return false;
+    }
+    if (resolved.workspace_bytes > state->workspace_bytes) {
+        ++state->fallbacks;
+        state->fallback_shapes.emplace(type, (int) t, (int) n, (int) k, (int) ldy);
+        if (std::getenv("STRATA_HIPBLASLT_VERBOSE")) {
+            std::fprintf(stderr, "prefill gemm: Lt fallback; solution needs %zu workspace bytes, have %zu\n",
+                         resolved.workspace_bytes, state->workspace_bytes);
+        }
+        return false;
+    }
+
+    HipLtDescriptors desc;
+    const hipDataType input_type = type == strata::prefill::hipblaslt::InputType::bf16 ? HIP_R_16BF : HIP_R_16F;
+    if (!desc.init(input_type, (int) t, (int) n, (int) k, (int) ldy)) return false;
+    const float alpha = 1.0f;
+    const hipblasStatus_t status = hipblasLtMatmul(state->handle, desc.op, &alpha, w, desc.a, x, desc.b, &beta, y,
+                                                   desc.c, y, desc.c, &resolved.algo, state->workspace,
+                                                   state->workspace_bytes, (hipStream_t) stream);
+    if (status == HIPBLAS_STATUS_SUCCESS) {
+        ++state->lt_launches;
+        return true;
+    }
+
+    std::fprintf(stderr, "prefill gemm: hipBLASLt launch failed with status %d\n", (int) status);
+    if (beta != 0.0f) {
+        std::fprintf(stderr, "prefill gemm: refusing a fallback after hipBLASLt failed with nonzero beta\n");
+        std::exit(1);
+    }
+    auto* mutable_state = static_cast<HipLtState*>(opaque_state);
+    uint32_t beta_bits = 0;
+    std::memcpy(&beta_bits, &beta, sizeof(beta_bits));
+    auto cached = mutable_state->cache.find(HipLtCallKey{type, (int) t, (int) n, (int) k, (int) ldy, beta_bits});
+    if (cached != mutable_state->cache.end()) cached->second.supported = false;
+    ++mutable_state->fallbacks;
+    mutable_state->fallback_shapes.emplace(type, (int) t, (int) n, (int) k, (int) ldy);
+    return false;
+}
+#endif
+
 }  // namespace
 
 Gemm::~Gemm() {
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    delete static_cast<HipLtState*>(hipblaslt_state_);
+#endif
     if (handle_) cublasDestroy((cublasHandle_t) handle_);
     if (!external_) {
         if (scratch_) cudaFree(scratch_);
@@ -41,6 +295,9 @@ bool Gemm::init_external(void* stream, uint16_t* scratch, int64_t scratch_elems,
     cublasSetMathMode(h, CUBLAS_DEFAULT_MATH);
     scratch_ = scratch;
     scratch_elems_ = scratch_elems;
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    hipblaslt_state_ = create_hipblaslt_state(workspace_, ws_bytes).release();
+#endif
     return true;
 }
 
@@ -49,6 +306,13 @@ void Gemm::rebind(uint16_t* scratch, int64_t scratch_elems, void* workspace, siz
     scratch_elems_ = scratch_elems;
     workspace_ = workspace;
     cublasSetWorkspace((cublasHandle_t) handle_, workspace_, ws_bytes);
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    if (hipblaslt_state_) {
+        auto* state = static_cast<HipLtState*>(hipblaslt_state_);
+        state->workspace = workspace_;
+        state->workspace_bytes = ws_bytes;
+    }
+#endif
 }
 
 bool Gemm::init(void* stream, int64_t scratch_elems, std::string& err) {
@@ -62,6 +326,9 @@ bool Gemm::init(void* stream, int64_t scratch_elems, std::string& err) {
     if (cudaMalloc(&workspace_, ws) != cudaSuccess) { err = "prefill gemm: workspace"; return false; }
     cublasSetWorkspace(h, workspace_, ws);
     cublasSetMathMode(h, CUBLAS_DEFAULT_MATH);
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    hipblaslt_state_ = create_hipblaslt_state(workspace_, ws).release();
+#endif
     if (scratch_elems > 0 && cudaMalloc((void**) &scratch_, (size_t) scratch_elems * 2) != cudaSuccess) {
         err = "prefill gemm: dequant scratch of " + std::to_string(scratch_elems * 2 >> 20) + " MiB";
         return false;
@@ -75,6 +342,12 @@ void Gemm::bf16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64
     if (T <= 0 || N <= 0) return;
     if (ldy <= 0) ldy = N;
     const float alpha = 1.0f;
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    if (try_hipblaslt(hipblaslt_state_, strata::prefill::hipblaslt::InputType::bf16, X, W, Y, T, N, K, ldy,
+                      beta, stream_)) {
+        return;
+    }
+#endif
     // Column-major view: Y^T[N, T] = W[N, K] (stored K x N col-major, transposed) . X^T[K, T].
     ck(cublasGemmEx((cublasHandle_t) handle_, CUBLAS_OP_T, CUBLAS_OP_N, (int) N, (int) T, (int) K, &alpha, W,
                     CUDA_R_16BF, (int) K, X, CUDA_R_16BF, (int) K, &beta, Y, CUDA_R_32F, (int) ldy,
@@ -87,6 +360,12 @@ void Gemm::f16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_
     if (T <= 0 || N <= 0) return;
     if (ldy <= 0) ldy = N;
     const float alpha = 1.0f;
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    if (try_hipblaslt(hipblaslt_state_, strata::prefill::hipblaslt::InputType::f16, X, W, Y, T, N, K, ldy,
+                      beta, stream_)) {
+        return;
+    }
+#endif
     ck(cublasGemmEx((cublasHandle_t) handle_, CUBLAS_OP_T, CUBLAS_OP_N, (int) N, (int) T, (int) K, &alpha, W,
                     CUDA_R_16F, (int) K, X, CUDA_R_16F, (int) K, &beta, Y, CUDA_R_32F, (int) ldy,
                     CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT),

src/prefill/ggml_cuda_host.cu

@@ -5,6 +5,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <mutex>
+#include <string>
 #include <vector>
 
 [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {
@@ -18,6 +19,45 @@ int ggml_cuda_get_device() {
     return id;
 }
 
+#if defined(GGML_USE_HIP)
+namespace {
+
+unsigned parse_hex(const std::string& value) {
+    if (value.empty()) return 0;
+    char* end = nullptr;
+    const unsigned long parsed = std::strtoul(value.c_str(), &end, 16);
+    return end == value.c_str() || *end != '\0' ? 0 : static_cast<unsigned>(parsed);
+}
+
+// Match GGML's AMD cc encoding: OFFSET_AMD + gfx major * 0x100 + minor/stepping.
+// For example, gfx1100 maps to GGML_CUDA_CC_RDNA3 (0x01001100).
+int ggml_cuda_parse_amd_id(const char* device_name) {
+    if (device_name == nullptr) return GGML_CUDA_CC_OFFSET_AMD;
+
+    std::string arch(device_name);
+    if (arch.compare(0, 3, "gfx") == 0) arch.erase(0, 3);
+    const size_t suffix = arch.find(':');
+    if (suffix != std::string::npos) arch.resize(suffix);
+
+    const size_t generic = arch.rfind("-generic");
+    if (generic != std::string::npos && generic + 8 == arch.size()) arch.resize(generic);
+
+    unsigned major = 0;
+    unsigned minor = 0;
+    const size_t separator = arch.find('-');
+    if (separator != std::string::npos) {
+        major = parse_hex(arch.substr(0, separator));
+        minor = parse_hex(arch.substr(separator + 1)) * 0x10;
+    } else if (arch.size() >= 3) {
+        minor = parse_hex(arch.substr(arch.size() - 2));
+        major = parse_hex(arch.substr(0, arch.size() - 2));
+    }
+    return GGML_CUDA_CC_OFFSET_AMD + static_cast<int>(major * 0x100 + minor);
+}
+
+}  // namespace
+#endif
+
 const ggml_cuda_device_info & ggml_cuda_info() {
     static ggml_cuda_device_info info = [] {
         ggml_cuda_device_info in = {};
@@ -30,15 +70,26 @@ const ggml_cuda_device_info & ggml_cuda_info() {
             cudaDeviceProp prop;
             CUDA_CHECK(cudaGetDeviceProperties(&prop, id));
             auto & d = in.devices[id];
+#if defined(GGML_USE_HIP)
+            d.cc = ggml_cuda_parse_amd_id(prop.gcnArchName);
+            if ((d.cc & 0xff00) == 0) {
+                // Fall back to HIP's device version fields if its architecture name is unavailable.
+                d.cc = GGML_CUDA_CC_OFFSET_AMD + prop.major * 0x100 + prop.minor * 0x10;
+            }
+            d.smpbo = prop.sharedMemPerBlock;
+            d.integrated = false;
+            d.supports_cooperative_launch = false;
+#else
             d.cc = 100 * prop.major + 10 * prop.minor;
-            d.nsm = prop.multiProcessorCount;
-            d.smpb = prop.sharedMemPerBlock;
             d.smpbo = prop.sharedMemPerBlockOptin;
             d.integrated = prop.integrated != 0;
+            d.supports_cooperative_launch = prop.cooperativeLaunch != 0;
+#endif
+            d.nsm = prop.multiProcessorCount;
+            d.smpb = prop.sharedMemPerBlock;
             d.vmm = false;
             d.total_vram = prop.totalGlobalMem;
             d.warp_size = prop.warpSize;
-            d.supports_cooperative_launch = prop.cooperativeLaunch != 0;
             d.physical_device = id;
             d.physical_share_count = 1;
             d.virtual_index = 0;

src/prefill/hipblaslt_tuning.hpp

@@ -0,0 +1,123 @@
+#pragma once
+
+#include <cstdint>
+#include <fstream>
+#include <limits>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace strata::prefill::hipblaslt {
+
+enum class InputType : uint8_t { f16, bf16 };
+
+struct TuningRow {
+    InputType type = InputType::bf16;
+    int n = 0;
+    int k = 0;
+    int ldy = 0;
+    int t_bucket = 0;
+    int solution_id = -1;
+};
+
+class TuningTable {
+public:
+    bool load(const std::string& path, const std::string& expected_arch, int expected_version, std::string& err) {
+        std::ifstream input(path);
+        if (!input) {
+            err = "cannot open tuning file";
+            return false;
+        }
+
+        bool saw_header = false;
+        std::string line;
+        size_t line_number = 0;
+        while (std::getline(input, line)) {
+            ++line_number;
+            const auto first = line.find_first_not_of(" \t\r\n");
+            if (first == std::string::npos || line[first] == '#') continue;
+
+            std::istringstream row(line.substr(first));
+            if (!saw_header) {
+                std::string magic;
+                std::string arch;
+                int version = 0;
+                std::string extra;
+                if (!(row >> magic >> arch >> version) || (row >> extra) || magic != "STRATA_HIPBLASLT_TUNING_V1") {
+                    err = "invalid tuning header at line " + std::to_string(line_number);
+                    return false;
+                }
+                if (arch != expected_arch) {
+                    err = "tuning architecture mismatch: file=" + arch + " runtime=" + expected_arch;
+                    return false;
+                }
+                if (version != expected_version) {
+                    err = "hipBLASLt version mismatch: file=" + std::to_string(version) +
+                          " runtime=" + std::to_string(expected_version);
+                    return false;
+                }
+                arch_ = std::move(arch);
+                version_ = version;
+                saw_header = true;
+                continue;
+            }
+
+            std::string type_name;
+            int n = 0, k = 0, ldy = 0, t_bucket = 0, solution_id = -1;
+            std::string extra;
+            if (!(row >> type_name >> n >> k >> ldy >> t_bucket >> solution_id) || (row >> extra) ||
+                (type_name != "f16" && type_name != "bf16") || n <= 0 || k <= 0 || ldy < n || t_bucket <= 0 ||
+                solution_id < 0) {
+                err = "invalid tuning row at line " + std::to_string(line_number);
+                return false;
+            }
+            TuningRow parsed;
+            parsed.type = type_name == "f16" ? InputType::f16 : InputType::bf16;
+            parsed.n = n;
+            parsed.k = k;
+            parsed.ldy = ldy;
+            parsed.t_bucket = t_bucket;
+            parsed.solution_id = solution_id;
+            for (const auto& existing : rows_) {
+                if (existing.type == parsed.type && existing.n == parsed.n && existing.k == parsed.k &&
+                    existing.ldy == parsed.ldy && existing.t_bucket == parsed.t_bucket) {
+                    err = "duplicate tuning key at line " + std::to_string(line_number);
+                    return false;
+                }
+            }
+            rows_.push_back(parsed);
+        }
+        if (!saw_header) {
+            err = "tuning file has no header";
+            return false;
+        }
+        return true;
+    }
+
+    const TuningRow* closest(InputType type, int n, int k, int ldy, int t) const {
+        const TuningRow* best = nullptr;
+        int64_t best_distance = std::numeric_limits<int64_t>::max();
+        for (const auto& row : rows_) {
+            if (row.type != type || row.n != n || row.k != k || row.ldy != ldy) continue;
+            const int64_t distance = row.t_bucket >= t ? int64_t(row.t_bucket) - t : int64_t(t) - row.t_bucket;
+            if (distance < best_distance ||
+                (distance == best_distance && (!best || row.t_bucket < best->t_bucket))) {
+                best = &row;
+                best_distance = distance;
+            }
+        }
+        return best;
+    }
+
+    const std::vector<TuningRow>& rows() const { return rows_; }
+    const std::string& arch() const { return arch_; }
+    int version() const { return version_; }
+
+private:
+    std::string arch_;
+    int version_ = 0;
+    std::vector<TuningRow> rows_;
+};
+
+}  // namespace strata::prefill::hipblaslt

src/prefill/kernels.cu

@@ -70,6 +70,68 @@ __global__ void gr_norm_kernel(const float* __restrict__ R, const float* __restr
         xn16[row * N + d] = bf(v);
     }
 }
+// F-1: the row scale only (and the BF16 image); gr_mix_r_kernel recomputes r * rs * w itself, in the same order,
+// so the FP32 copy of the normalized rows (T x 10240 floats) is neither written nor read
+__global__ void gr_norm_rs_kernel(const float* __restrict__ R, const float* __restrict__ w, float eps,
+                                  float* __restrict__ rs_out, uint16_t* __restrict__ xn16) {
+    __shared__ float sh[32];
+    const int64_t row = blockIdx.x;                 // t * 4 + c
+    const int c = (int) (row % HC);
+    const float* r = R + row * N;
+    float ss = 0.0f;
+    for (int d = threadIdx.x; d < N; d += blockDim.x) ss += r[d] * r[d];
+    const float rs = rsqrtf(block_sum(ss, sh) / (float) N + eps);
+    if (threadIdx.x == 0) rs_out[row] = rs;
+    for (int d = threadIdx.x; d < N; d += blockDim.x) xn16[row * N + d] = bf(r[d] * rs * w[c * N + d]);
+}
+__global__ void gr_mix_r_kernel(const float* __restrict__ R, const float* __restrict__ rs, const float* __restrict__ w,
+                                const float* __restrict__ g, float* __restrict__ mixed, uint16_t* __restrict__ mixed16,
+                                int64_t T, uint16_t* __restrict__ mixed_h) {
+    const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= T * N) return;
+    const int64_t t = i / N, d = i % N;
+    float s = 0.0f;
+#pragma unroll
+    for (int c = 0; c < HC; ++c) {
+        const int64_t j = t * D + c * N + d;
+        const float x = R[j] * rs[t * HC + c] * w[c * N + d];   // gr_norm_kernel's value, bit for bit
+        s = fmaf(x, sigm(g[j]), s);
+    }
+    s /= (float) HC;
+    mixed[i] = s;
+    if (mixed16) mixed16[i] = bf(s);
+    if (mixed_h) mixed_h[i] = hf(s);
+}
+// F-2: gr_write_kernel for one row (t, c), then gr_norm_rs_kernel's reduction over it with the next half's norm
+// weights - the same thread-to-element mapping (256 threads, stride 256) and block_sum, so rs and the BF16 image are
+// the same bits, and R is not read back
+constexpr int GRW_PER = (N + 255) / 256;
+__global__ void __launch_bounds__(256) gr_write_norm_rs_kernel(float* __restrict__ R, const float* __restrict__ bo,
+                                                               const float* __restrict__ inj, int64_t inj_ld,
+                                                               const float* __restrict__ w, float eps,
+                                                               float* __restrict__ rs_out, uint16_t* __restrict__ xn16) {
+    __shared__ float sh[32];
+    const int64_t row = blockIdx.x;                 // t * 4 + c
+    const int64_t t = row / HC;
+    const int c = (int) (row % HC);
+    float* r = R + row * N;
+    const float sc = 2.0f * sigm(inj[t * inj_ld + c] / (float) HC);
+    float v[GRW_PER];
+    float ss = 0.0f;
+    int k = 0;
+#pragma unroll
+    for (int d = threadIdx.x; d < N; d += 256, ++k) {
+        const float x = fmaf(bo[t * N + d], sc, r[d]);
+        r[d] = x;
+        v[k] = x;
+        ss += x * x;
+    }
+    const float rs = rsqrtf(block_sum(ss, sh) / (float) N + eps);
+    if (threadIdx.x == 0) rs_out[row] = rs;
+    k = 0;
+#pragma unroll
+    for (int d = threadIdx.x; d < N; d += 256, ++k) xn16[row * N + d] = bf(v[k] * rs * w[c * N + d]);
+}
 __global__ void gr_silu_kernel(const float* __restrict__ lo, uint16_t* __restrict__ lo16, int64_t n) {
     const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
     if (i >= n) return;
@@ -519,6 +581,22 @@ void gr_norm(const float* R, const float* w_norm, float eps, float* xn, uint16_t
     gr_norm_kernel<<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, xn, xn16);
     check("gr_norm");
 }
+void gr_norm_rs(const float* R, const float* w_norm, float eps, float* rs, uint16_t* xn16, int64_t T, void* stream) {
+    gr_norm_rs_kernel<<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, rs, xn16);
+    check("gr_norm_rs");
+}
+void gr_mix_r(const float* R, const float* rs, const float* w_norm, const float* gated, float* mixed, uint16_t* mixed16,
+              int64_t T, void* stream, uint16_t* mixed_h) {
+    gr_mix_r_kernel<<<blocks_for(T * N), 256, 0, (cudaStream_t) stream>>>(R, rs, w_norm, gated, mixed, mixed16, T,
+                                                                          mixed_h);
+    check("gr_mix_r");
+}
+void gr_write_norm_rs(float* R, const float* bo, const float* inj, int64_t inj_ld, const float* w_norm_next, float eps,
+                      float* rs, uint16_t* xn16, int64_t T, void* stream) {
+    gr_write_norm_rs_kernel<<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, bo, inj, inj_ld, w_norm_next, eps,
+                                                                                      rs, xn16);
+    check("gr_write_norm_rs");
+}
 void gr_silu(const float* lo, uint16_t* lo16, int64_t T, void* stream) {
     gr_silu_kernel<<<blocks_for(T * LR), 256, 0, (cudaStream_t) stream>>>(lo, lo16, T * LR);
     check("gr_silu");
@@ -589,6 +667,18 @@ void swiglu_pair(const float* g, const float* u, uint16_t* h16, int64_t n, void*
     swiglu_pair_kernel<<<blocks_for(n * 640), 256, 0, (cudaStream_t) stream>>>(g, u, h16, n);
     check("swiglu_pair");
 }
+namespace {
+__global__ void copy_i32_kernel(int32_t* __restrict__ dst, const int32_t* __restrict__ src, int64_t n) {
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
+        dst[i] = src[i];
+}
+}  // namespace
+void copy_i32(int32_t* dst, const int32_t* src, int64_t n, void* stream) {
+    if (n <= 0) return;
+    const int64_t b = (n + 255) / 256;
+    copy_i32_kernel<<<(unsigned) (b < 256 ? b : 256), 256, 0, (cudaStream_t) stream>>>(dst, src, n);
+    check("copy_i32");
+}
 void gather_rows16(const uint16_t* x16, const int32_t* src, uint16_t* dst16, int64_t n, int64_t width, void* stream) {
     if (n <= 0) return;
     gather_rows16_kernel<<<blocks_for(n * (width / 8)), 256, 0, (cudaStream_t) stream>>>(x16, src, dst16, n, width);

src/prefill/moe_mmq.cu

@@ -93,6 +93,7 @@ bool supported(int t) {
     switch ((ggml_type) t) {
         case GGML_TYPE_Q2_0: case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S:
         case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS:
+        case GGML_TYPE_Q8_0:   // the draft layer's dense matrices (E-9)
             return true;
         default:
             return false;
@@ -141,6 +142,7 @@ void Context::run(const Product& p, void* stream) {
         case GGML_TYPE_IQ3_S: mul_mat_q_case<GGML_TYPE_IQ3_S>(ctx, a, s); break;
         case GGML_TYPE_IQ4_NL: mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, a, s); break;
         case GGML_TYPE_IQ4_XS: mul_mat_q_case<GGML_TYPE_IQ4_XS>(ctx, a, s); break;
+        case GGML_TYPE_Q8_0: mul_mat_q_case<GGML_TYPE_Q8_0>(ctx, a, s); break;
         default:
             std::fprintf(stderr, "prefill mmq: type %d is not covered\n", (int) t);
             std::exit(1);

src/prefill/prefill.cpp

@@ -1,5 +1,6 @@
 // src/prefill/prefill.cpp - see include/strata/prefill/prefill.hpp.
 #include "strata/prefill/prefill.hpp"
+#include "strata/core/mtp.hpp"
 #include "strata/core/progress.hpp"
 #include "strata/core/on_device.hpp"
 
@@ -86,6 +87,12 @@ inline int ring_slots(size_t T) {
 }
 constexpr int DQ = 2;              // dequantized-expert ring (FP16 gate/up + down)
 
+// F-1: STRATA_GR_UNFUSED=1 keeps the FP32 copy of the normalized rows (gr_norm + gr_mix), the A/B arm
+inline bool gr_unfused() {
+    static const bool v = [] { const char* e = std::getenv("STRATA_GR_UNFUSED"); return e && e[0] == '1'; }();
+    return v;
+}
+
 double ms_since(Clock::time_point t) { return std::chrono::duration<double, std::milli>(Clock::now() - t).count(); }
 
 // Either cudaMalloc (owned, freed with the object) or a bump allocation from a borrowed region; with no base and
@@ -252,6 +259,7 @@ struct Prefill::Impl {
     std::vector<void*> owned;
     // chunk buffers
     float *emb = nullptr, *R = nullptr, *xn = nullptr, *lo = nullptr, *gated = nullptr, *inj = nullptr;
+    float* grs = nullptr;                    // F-1: the hyper-connection read's row scales (T x 4)
     uint16_t *xn16 = nullptr, *lo16 = nullptr;
     float* mixed = nullptr;
     uint16_t *mixed_bf = nullptr, *mixed_h = nullptr;
@@ -283,6 +291,13 @@ struct Prefill::Impl {
     std::vector<int32_t> bounds_host;
     std::unique_ptr<mmq::Context> mmq_ctx;
     std::vector<int32_t> ids_host, slot_host, src_host, cnt, off;
+    // The grouping tables in mapped pinned memory, [ids | slot | src] of T_max * K each, then the MMQ bounds: kernels
+    // read and write them in place.  A cudaMemcpyAsync of them queues behind the expert blobs the copy stream already
+    // holds (up to `ring` of them, ~70 us each), and the GPU idles meanwhile - measured 4.2 s of a 128K prompt's
+    // 70 s at the default ring, 0.7 s with a 16-slot one.  STRATA_GROUP_COPY=1: the copies (the A/B arm).
+    int32_t* grp_host = nullptr;
+    int32_t* grp_dev = nullptr;          // its device alias
+    size_t grp_n = 0, grp_tk = 0;        // int32s allocated; T_max * K (the offset of slot, and of src past it)
     uint16_t* dq_gu[DQ] = {};
     uint16_t* dq_d[DQ] = {};
     uint8_t* stage_dev[RING_MAX] = {};
@@ -352,6 +367,7 @@ Prefill::Prefill() : impl_(new Impl) {}
 Prefill::~Prefill() {
     if (!impl_) return;
     if (impl_->cs) cudaStreamSynchronize(impl_->cs);
+    if (impl_->copy) cudaStreamSynchronize(impl_->copy);
     for (int i = 0; i < RING_MAX; ++i) {
         if (impl_->copied[i]) cudaEventDestroy(impl_->copied[i]);
         if (impl_->used[i]) cudaEventDestroy(impl_->used[i]);
@@ -362,6 +378,7 @@ Prefill::~Prefill() {
         if (impl_->ple_emb_host[b] && impl_->ple_pageable[b].empty()) cudaFreeHost(impl_->ple_emb_host[b]);
     }
     if (impl_->copy) cudaStreamDestroy(impl_->copy);
+    if (impl_->grp_host) cudaFreeHost(impl_->grp_host);
     for (void* p : impl_->owned) cudaFree(p);
 }
 
@@ -490,6 +507,26 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, co
     }
     m.steps_host.resize(T * strata::kernels::kStepCount);
     m.ids_host.resize(T * K); m.slot_host.resize(T * K); m.src_host.resize(T * K); m.cnt.resize(m.g->n_expert); m.off.resize(m.g->n_expert + 1);
+    {
+        const size_t need = 3 * T * K + (size_t) (2 * (m.g->n_expert + m.g->n_expert / MMQ_GROUP + 2));
+        const char* gc = std::getenv("STRATA_GROUP_COPY");
+        if (m.grp_n < need && !(gc && gc[0] == '1')) {
+            if (m.grp_host) cudaFreeHost(m.grp_host);
+            m.grp_host = m.grp_dev = nullptr;
+            m.grp_n = m.grp_tk = 0;
+            void *h = nullptr, *d = nullptr;
+            if (cudaHostAlloc(&h, need * 4, cudaHostAllocMapped) == cudaSuccess &&
+                cudaHostGetDevicePointer(&d, h, 0) == cudaSuccess) {
+                m.grp_host = (int32_t*) h;
+                m.grp_dev = (int32_t*) d;
+                m.grp_n = need;
+                m.grp_tk = T * K;
+            } else {                              // the copies, as before
+                if (h) cudaFreeHost(h);
+                cudaGetLastError();
+            }
+        }
+    }
     for (int b = 0; b < 2; ++b) {
         if (!m.ple_emb_host[b] &&
             cudaHostAlloc((void**) &m.ple_emb_host[b], (size_t) T * N * 4, cudaHostAllocDefault) != cudaSuccess) {
@@ -537,7 +574,9 @@ bool Prefill::carve(size_t T, void* alloc) {
     const core::ModelGeometry& g = *m.g;
     core::SessionState& ss = *m.ss;
     bool ok = true;
-    m.emb = o.take<float>(T * N, ok); m.R = o.take<float>(T * D, ok); m.xn = o.take<float>(T * D, ok);
+    m.emb = o.take<float>(T * N, ok); m.R = o.take<float>(T * D, ok);
+    m.xn = gr_unfused() ? o.take<float>(T * D, ok) : nullptr;   // F-1: not needed (gr_mix_r reads R)
+    m.grs = o.take<float>(T * HC, ok);
     m.xn16 = o.take<uint16_t>(T * D, ok); m.lo = o.take<float>(T * LR, ok); m.lo16 = o.take<uint16_t>(T * LR, ok);
     m.gated = o.take<float>(T * D, ok); m.inj = o.take<float>(T * HC, ok);
     m.mixed = o.take<float>(T * N, ok); m.mixed_bf = o.take<uint16_t>(T * N, ok);
@@ -635,6 +674,151 @@ bool Prefill::relayout(int64_t chunk, void* borrow, uint64_t borrow_bytes, std::
 }
 
 int64_t Prefill::chunk() const { return impl_->T; }
+
+bool Prefill::draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t* next_tokens, int64_t n, int64_t cell0,
+                       std::string& err) {
+    Impl& m = *impl_;
+    static const bool off = [] { const char* v = std::getenv("STRATA_MTP_BATCH"); return v != nullptr && v[0] == '0'; }();
+    core::QsaState& st = mtp.kv_state_rw();
+    if (off || n <= 0 || m.g == nullptr || m.region == nullptr || st.kv_mode != 0 || st.kv_hybrid ||
+        mtp.device() != m.device)
+        return false;
+    const auto t0 = Clock::now();
+    const core::ModelGeometry& g = *m.g;
+    const int64_t Nn = g.n_embd, HCN = g.hc * g.n_embd, KV = g.n_head_kv * g.head_dim;
+    constexpr int kQ8_0 = 8;   // GGML_TYPE_Q8_0
+    const float* w_ne = mtp.tensor_f32("pre_fc_norm_embedding.weight");
+    const void* w_fe = mtp.tensor_q8("fc_embedding.weight");
+    const float* w_nh = mtp.tensor_f32("pre_fc_norm_hidden.weight");
+    const void* w_fh = mtp.tensor_q8("fc_hidden.weight");
+    const float* w_hn = mtp.tensor_f32("attn_hyper_connection.hc_norm.weight");
+    const uint16_t* w_dn = mtp.tensor_bf16("attn_hyper_connection.input_mix_weight_down.weight");
+    const uint16_t* w_up = mtp.tensor_bf16("attn_hyper_connection.input_mix_weight_up.weight");
+    const void* w_k = mtp.tensor_q8("self_attn.k_proj.weight");
+    const void* w_v = mtp.tensor_q8("self_attn.v_proj.weight");
+    const float* w_kn = mtp.tensor_f32("self_attn.k_norm.weight");
+    if (!w_ne || !w_fe || !w_nh || !w_fh || !w_hn || !w_dn || !w_up || !w_k || !w_v || !w_kn) return false;
+    const core::NativeEmbed* nemb = core::native_embed();
+    const core::WeightRef* wemb = nemb ? nullptr : m.wt->find("token_embd.weight");
+    if (!nemb && (wemb == nullptr || wemb->codebook_iq4nl || wemb->ne0 != g.n_embd || wemb->group_elems <= 0 ||
+                  (wemb->code_bits != 2 && wemb->code_bits != 4 && wemb->code_bits != 8)))
+        return false;
+    // the cells the drafter's window can still reach
+    const int64_t r0 = std::max<int64_t>(0, mtp.first_needed() - cell0);
+    if (r0 >= n) return true;
+    // per row: emb/e2 (N), en16 (N half), hn/h2/Rm/gated (HCN), hn16/xn16 (HCN half), lo (LR) + lo16, grs, mixed (N) +
+    // mixed_h, K and V (KV each), the token id
+    // E-9: the drafter's Q8_0 matrices through Q8_1 x Q8_0 MMQ - its own pass's integer dot products (mmvq), so
+    // its K/V stay close to what the drafter computes itself; STRATA_MTP_BATCH_F16=1: FP16 GEMMs (the A/B)
+    static const bool f16_only = [] { const char* v = std::getenv("STRATA_MTP_BATCH_F16"); return v && v[0] == '1'; }();
+    const bool q8 = !f16_only && mmq::built() && mmq::supported(kQ8_0);
+    const uint64_t per_row = 4 * (2 * Nn + 4 * HCN + LR + HC + Nn + 2 * KV + 1) + 2 * (Nn + 2 * HCN + LR + Nn) + 64 +
+                             (q8 ? (uint64_t) mmq::q8_bytes(g.hc, Nn) + 4 * g.hc : 0);
+    const int64_t B = std::min<int64_t>(n - r0, (int64_t) (m.region_bytes / per_row) & ~(int64_t) 63);
+    if (B < 64) return false;
+    uint8_t* q = m.region;
+    auto carve = [&](size_t bytes) { void* p = q; q += (bytes + 255) & ~(size_t) 255; return p; };
+    float* emb = (float*) carve((size_t) B * Nn * 4);
+    float* e2 = (float*) carve((size_t) B * Nn * 4);
+    uint16_t* en16 = (uint16_t*) carve((size_t) B * Nn * 2);
+    float* hn = (float*) carve((size_t) B * HCN * 4);
+    float* h2 = (float*) carve((size_t) B * HCN * 4);
+    float* Rm = (float*) carve((size_t) B * HCN * 4);
+    float* gated = (float*) carve((size_t) B * HCN * 4);
+    uint16_t* hn16 = (uint16_t*) carve((size_t) B * HCN * 2);
+    uint16_t* xn16 = (uint16_t*) carve((size_t) B * HCN * 2);
+    float* lo = (float*) carve((size_t) B * LR * 4);
+    uint16_t* lo16 = (uint16_t*) carve((size_t) B * LR * 2);
+    float* grs = (float*) carve((size_t) B * HC * 4);
+    float* mixed = (float*) carve((size_t) B * Nn * 4);
+    uint16_t* mixed_h = (uint16_t*) carve((size_t) B * Nn * 2);
+    float* Kc = (float*) carve((size_t) B * KV * 4);
+    float* Vc = (float*) carve((size_t) B * KV * 4);
+    int32_t* tok = (int32_t*) carve((size_t) B * 4);
+    void* xq = q8 ? carve(mmq::q8_bytes(B * g.hc, Nn)) : nullptr;
+    int32_t* ident = q8 ? (int32_t*) carve((size_t) B * g.hc * 4) : nullptr;
+    int32_t* bnd = q8 ? (int32_t*) carve(16) : nullptr;
+    if ((uint64_t) (q - m.region) > m.region_bytes) return false;
+    if (!mtp.idle(err)) return false;   // the drafter's own stream (its graph uploads) before this writes its K/V
+    strata::kernels::QsaShapes s = strata::kernels::qsa_real_shapes();
+    s.n_head = g.n_head; s.n_head_kv = g.n_head_kv; s.head_dim = g.head_dim; s.idx_n_head = g.idx_q_heads;
+    s.idx_dim = g.idx_key_dim;
+    std::vector<int32_t> tk((size_t) B);
+    if (q8) {
+        if (!m.mmq_ctx) m.mmq_ctx = std::make_unique<mmq::Context>();
+        mmq::iota(ident, B * g.hc, m.cs);
+    }
+    // y[rows, n_out] = x[rows, k] . w^T for a Q8_0 matrix: MMQ from the FP32 rows (bounds slot 0: rows, 1: rows*hc)
+    auto proj = [&](const float* x, const uint16_t* x16, const void* w, float* y, int64_t rows, int64_t n_out,
+                    int64_t k, int slot) {
+        if (!q8) { m.gemm.native(x16, kQ8_0, w, y, rows, n_out, k); return; }
+        mmq::quantize(x, nullptr, xq, kQ8_0, k, k, rows, m.cs);
+        mmq::Product p;
+        p.w = w; p.type = kQ8_0; p.w_rows = n_out; p.w_cols = k; p.expert_bytes = mmq::matrix_bytes(kQ8_0, n_out, k);
+        p.n = 1; p.xq = xq; p.bounds = bnd + 2 * slot; p.ids = ident; p.total_rows = rows; p.max_rows = rows;
+        p.dst = y; p.ld_dst = n_out;
+        m.mmq_ctx->run(p, m.cs);
+    };
+    for (int64_t b0 = r0; b0 < n; b0 += B) {
+        const int64_t nb = std::min(B, n - b0), c0 = cell0 + b0;
+        for (int64_t i = 0; i < nb; ++i) tk[(size_t) i] = next_tokens[b0 + i];
+        const int32_t bh[4] = {0, (int32_t) nb, 0, (int32_t) (nb * g.hc)};
+        if (q8 && cudaMemcpyAsync(bnd, bh, sizeof bh, cudaMemcpyHostToDevice, m.cs) != cudaSuccess) {
+            err = "prefill: the draft bounds' upload failed";
+            return false;
+        }
+        if (cudaMemcpyAsync(tok, tk.data(), (size_t) nb * 4, cudaMemcpyHostToDevice, m.cs) != cudaSuccess) {
+            err = "prefill: the draft tokens' upload failed";
+            return false;
+        }
+        // the input branches: the next token's embedding, and this cell's final residual rows
+        if (nemb) {
+            nemb->gather_dev(tok, nb, emb, m.cs);
+        } else {
+            const auto* codes = (const uint8_t*) wemb->data;
+            const auto* scales = (const float*) (codes + wemb->codes_bytes);
+            const auto* offsets = wemb->has_offset ? (const float*) (codes + wemb->codes_bytes + wemb->scales_bytes)
+                                                   : nullptr;
+            strata::kernels::embedding_gather_dev(codes, scales, offsets, tok, (int) nb, wemb->ne0, wemb->code_bits,
+                                                  wemb->code_bias, wemb->group_elems,
+                                                  (uint64_t) (wemb->ne0 / (8 / wemb->code_bits)),
+                                                  (uint64_t) (wemb->ne0 / wemb->group_elems), emb, m.cs);
+        }
+        rms_rows(emb, w_ne, nb, Nn, Nn, EPS, m.cs);
+        if (!q8) to_f16(emb, en16, nb * Nn, m.cs);
+        proj(emb, en16, w_fe, e2, nb, Nn, Nn, 0);
+        cudaMemcpyAsync(hn, R_rows + (size_t) b0 * HCN, (size_t) nb * HCN * 4, cudaMemcpyDeviceToDevice, m.cs);
+        rms_rows(hn, w_nh, nb, HCN, HCN, EPS, m.cs);
+        if (!q8) to_f16(hn, hn16, nb * HCN, m.cs);
+        proj(hn, hn16, w_fh, h2, nb * g.hc, Nn, Nn, 1);   // every stream through fc_hidden
+        strata::kernels::add_streams_broadcast(h2, e2, Rm, Nn, (int) g.hc, (int) nb, m.cs);
+        // the attention hyper-connection's read (its mixed input only: this pass writes nothing back)
+        gr_norm_rs(Rm, w_hn, EPS, grs, xn16, nb, m.cs);
+        m.gemm.bf16(xn16, w_dn, lo, nb, LR, HCN);
+        gr_silu(lo, lo16, nb, m.cs);
+        m.gemm.bf16(lo16, w_up, gated, nb, HCN, LR);
+        gr_mix_r(Rm, grs, w_hn, gated, mixed, nullptr, nb, m.cs, mixed_h);
+        // K and V into the drafter's cache, as the prompt path's QSA layers append theirs
+        proj(mixed, mixed_h, w_k, Kc, nb, KV, Nn, 0);
+        proj(mixed, mixed_h, w_v, Vc, nb, KV, Nn, 0);
+        rms_rows(Kc, w_kn, nb * g.n_head_kv, g.head_dim, g.head_dim, EPS, m.cs);
+        rope(Kc, nb, g.n_head_kv, g.head_dim, KV, c0, (float) strata::kernels::qsa_freq_base(), m.cs);
+        if (st.kv_q4) {
+            strata::kernels::fwht256_inplace_cuda(Kc, nb * g.n_head_kv, m.cs);
+            strata::kernels::fwht256_inplace_cuda(Vc, nb * g.n_head_kv, m.cs);
+            strata::kernels::kv_append_q4(st.k_q4, st.v_q4, st.page_table, c0, nb, Kc, Vc, s, m.cs, &st.host);
+        } else {
+            kv_append(Kc, Vc, nb, c0, st.page_table, s.page_size, st.kv_int8 ? nullptr : st.k_pool,
+                      st.kv_int8 ? nullptr : st.v_pool, st.k_q, st.v_q, st.k_scale, st.v_scale, m.cs, &st.host);
+        }
+    }
+    if (cudaStreamSynchronize(m.cs) != cudaSuccess) {
+        err = std::string("prefill: the draft layer's K/V: ") + cudaGetErrorString(cudaGetLastError());
+        return false;
+    }
+    mtp.ms_prefill += ms_since(t0);
+    return true;
+}
 void Prefill::set_pinned_share(double share) { g_pinned_share = share; }
 double Prefill::pinned_share() { return g_pinned_share; }
 
@@ -701,10 +885,12 @@ namespace {
 // host's expert grouping, or for an expert's copy) lands on the phase that was waiting.  Events are reused: the marks
 // are folded at every MoE layer's host sync, after which all of them have completed.
 enum PfPhase { kPfStart, kPfHc, kPfGdn, kPfQsa, kPfQsaIdx, kPfQsaSel, kPfQsaAttn, kPfRouter, kPfHostGroup, kPfGather,
-               kPfWaitCopy, kPfDequant, kPfGemmGU, kPfGemmD, kPfCombine, kPfPle, kPfKvStage, kPfCount };
+               kPfWaitCopy, kPfDequant, kPfGemmGU, kPfGemmD, kPfCombine, kPfPle, kPfKvStage, kPfGdnConv, kPfGdnRec, kPfGdnOut,
+               kPfCount };
 const char* const kPfNames[kPfCount] = {"embed+steps", "hc read", "gdn", "qsa proj", "qsa indexer", "qsa select",
                                         "qsa attn", "router+shared", "host grouping", "gather", "wait copy", "dequant",
-                                        "gemm gate/up", "gemm down", "combine", "ple", "kv stage"};
+                                        "gemm gate/up", "gemm down", "combine", "ple", "kv stage", "gdn conv+gates",
+                                        "gdn recurrence", "gdn out proj"};
 struct PfTimer {
     bool on = std::getenv("STRATA_PREFILL_TIMING") != nullptr;
     std::vector<cudaEvent_t> ev;
@@ -741,6 +927,7 @@ struct PfTimer {
 }  // namespace
 
 bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err) {
+    err.clear();
     Impl& m = *impl_;
     const core::OnDevice on_device(m.device);
     const core::ModelGeometry& g = *m.g;
@@ -1000,6 +1187,7 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
             else issue_until(upto + (size_t) m.ring);
         };
         host_setup_ms += ms_since(tsetup);
+        bool normed = false;   // F-2: the previous half's write already normed R for this half (grs, xn16)
         for (int64_t l = LB; l < LE; ++l) {
             core::progress_beat();   // the serve watchdog: a prompt chunk of 8192 tokens is still moving
             const core::LayerView v(*m.wt, l);
@@ -1064,12 +1252,15 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                                       *wu = need(v, su.c_str(), err), *wi = need(v, si.c_str(), err);
                 if (!wn || !wd || !wu || !wi) return false;
                 pt.mark(kPfHc, cs);
-                gr_norm(m.R, (const float*) wn->data, EPS, m.xn, m.xn16, T, m.cs);
+                if (gr_unfused()) gr_norm(m.R, (const float*) wn->data, EPS, m.xn, m.xn16, T, m.cs);
+                else if (!normed) gr_norm_rs(m.R, (const float*) wn->data, EPS, m.grs, m.xn16, T, m.cs);
+                normed = false;
                 if (!bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err)) return false;
                 gr_silu(m.lo, m.lo16, T, m.cs);
                 if (!bf16_proj(m.gemm, wu, m.lo16, m.gated, T, su, err)) return false;
                 if (!bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err)) return false;
-                gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, m.cs, m.mixed_h);
+                if (gr_unfused()) gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, m.cs, m.mixed_h);
+                else gr_mix_r(m.R, m.grs, (const float*) wn->data, m.gated, m.mixed, m.mixed_bf, T, m.cs, m.mixed_h);
 
                 if (half == 0 && !core::is_qsa_layer(g, l)) {
                     // ======================= GDN =======================
@@ -1086,9 +1277,12 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     if (!native_proj(m.gemm, wg, m.mixed_h, m.z, T, v.name("attn_gate.weight"), err)) return false;
                     if (!bf16_proj(m.gemm, wa, m.mixed_bf, m.ab, T, v.name("ssm_alpha.weight"), err, 2 * HV)) return false;
                     if (!bf16_proj(m.gemm, wb, m.mixed_bf, m.ab + HV, T, v.name("ssm_beta.weight"), err, 2 * HV)) return false;
+                    pt.mark(kPfGdnConv, cs);   // "gdn" is the projections in; the rest on their own lines
                     gdn_gates(m.ab, (const float*) wdt->data, (const float*) wsa->data, m.gate, m.beta, T, m.cs);
                     gdn_conv(conv, m.qkv, (const float*) wc->data, m.hbuf, T, EPS, m.cs);
+                    pt.mark(kPfGdnRec, cs);
                     gdn_recurrence(state, m.hbuf, m.gate, m.beta, m.z, (const float*) wnm->data, EPS, m.y, m.y_h, T, m.cs);
+                    pt.mark(kPfGdnOut, cs);
                     if (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err)) return false;
                     ++gdn_index;
                 } else if (half == 0) {
@@ -1120,7 +1314,14 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                                                             (p0 + s.page_size - 1) / s.page_size, s, m.cs);
                         pt.mark(kPfQsa, cs);
                     }
-                    if (st.kv_q4) {   // Q4_0 KV (kv_q4.hpp): rotated K and V, the queries below too, the output back
+                    if (st.kv_hybrid) {   // K8V4: K INT8 unrotated, V rotated Q4_0 (only V and the output rotate)
+                        strata::kernels::fwht256_inplace_cuda(m.Vc, T * 2, m.cs);
+                        kv_append(m.Kc, m.Kc, T, p0, st.page_table, s.page_size, nullptr, nullptr,
+                                  st.k_q, st.k_q, st.k_scale, st.k_scale, m.cs, nullptr,   // mode 0: no host mirror
+                                  staged ? &m.stage : nullptr);
+                        strata::kernels::kv_append_q4(st.v_q4, st.v_q4, st.page_table, p0, T, m.Vc, m.Vc, s, m.cs,
+                                                      nullptr, staged ? &m.stage : nullptr);
+                    } else if (st.kv_q4) {   // Q4_0 KV (kv_q4.hpp): rotated K and V, the queries below too, the output back
                         strata::kernels::fwht256_inplace_cuda(m.Kc, T * 2, m.cs);
                         strata::kernels::fwht256_inplace_cuda(m.Vc, T * 2, m.cs);
                         strata::kernels::kv_append_q4(st.k_q4, st.v_q4, st.page_table, p0, T, m.Kc, m.Vc, s, m.cs,
@@ -1163,8 +1364,13 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                         // C-1: the grid reaches the batch's last query's n_bid (they rise with the position)
                         const int64_t active = (int64_t) m.steps_host[(size_t) ((t0 + nb - 1) * strata::kernels::kStepCount +
                                                                                 strata::kernels::kStepNBid)] + 1;
-                        strata::kernels::qsa_block_scores(st.idx_pooled, st.idx_dead, m.q_idx + t0 * 512, steps0, nb,
-                                                          m.max_blocks, s, m.sel_scores, m.cs, active);
+                        // the scores on tensor cores (3xTF32: FP32-level, not bitwise); STRATA_SELECT_OLD=1: the warp kernel
+                        static const bool old_sel = std::getenv("STRATA_SELECT_OLD") != nullptr;
+                        if (old_sel || !strata::kernels::qsa_block_scores_tc(st.idx_pooled, st.idx_dead, m.q_idx + t0 * 512,
+                                                                             steps0, nb, m.max_blocks, s, m.sel_scores,
+                                                                             m.cs, active))
+                            strata::kernels::qsa_block_scores(st.idx_pooled, st.idx_dead, m.q_idx + t0 * 512, steps0, nb,
+                                                              m.max_blocks, s, m.sel_scores, m.cs, active);
                         strata::kernels::qsa_block_topk(m.sel_scores, steps0, nb, m.max_blocks, m.cap, s,
                                                         m.sel_ids + t0 * m.cap, m.cs);
                     }
@@ -1258,7 +1464,8 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     pt.mark(kPfQsaAttn, cs);
                     // perf-review D-1: the whole chunk on tensor cores, one block per (query, KV head), FP32-level
                     // accuracy but not bitwise (qsa_prompt_attn.hpp). Q4_0 KV, or STRATA_PROMPT_ATTN_OLD=1: the
-                    // decode kernel, 32 queries at a time
+                    // decode kernel, 32 queries at a time (K8V4 runs the tensor kernel's mode 3: INT8 K,
+                    // V dequantized from its q4_0 blocks to fp16 at gather)
                     static const bool old_attn = std::getenv("STRATA_PROMPT_ATTN_OLD") != nullptr;
                     if (old_attn || !strata::kernels::qsa_prompt_attn_batch(m.q, pools, m.sel_ids, m.steps_dev, m.cap, s,
                                                                             m.attn, T, m.cs))
@@ -1268,7 +1475,7 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                                                                    m.steps_dev + t0 * strata::kernels::kStepCount, m.cap,
                                                                    s, m.attn_scratch, m.attn + t0 * ZV, nb, m.cs);
                         }
-                    if (st.kv_q4) strata::kernels::fwht256_inplace_cuda(m.attn, T * 24, m.cs);
+                    if (st.kv_q4 || st.kv_hybrid) strata::kernels::fwht256_inplace_cuda(m.attn, T * 24, m.cs);
                     pt.mark(kPfQsa, cs);
                     gate_attn(m.attn, m.Qf, m.attn_h, T, m.cs);
                     if (!native_proj(m.gemm, wo, m.attn_h, m.bo, T, v.name("attn_output.weight"), err)) return false;
@@ -1293,12 +1500,19 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     m.gemm.bf16(m.mixed_bf, (const uint16_t*) wgi->data, m.sg, T, 1, N);
                     // group the (token, k) pairs by expert on the host
                     pt.mark(kPfHostGroup, cs);
-                    cudaMemcpyAsync(m.ids_host.data(), m.ids, (size_t) T * K * 4, cudaMemcpyDeviceToHost, m.cs);
+                    // (the sync below also orders this layer's writes of slot/src/bounds after the previous
+                    // layer's kernels that read them)
+                    const bool grp_mapped = m.grp_host != nullptr;
+                    int32_t* ids_h = grp_mapped ? m.grp_host : m.ids_host.data();
+                    int32_t* slot_h = grp_mapped ? m.grp_host + m.grp_tk : m.slot_host.data();
+                    int32_t* src_h = grp_mapped ? m.grp_host + 2 * m.grp_tk : m.src_host.data();
+                    if (grp_mapped) copy_i32(m.grp_dev, m.ids, T * K, m.cs);
+                    else cudaMemcpyAsync(m.ids_host.data(), m.ids, (size_t) T * K * 4, cudaMemcpyDeviceToHost, m.cs);
                     cudaStreamSynchronize(m.cs);
                     pt.fold();
                     std::fill(m.cnt.begin(), m.cnt.end(), 0);
                     for (int64_t i = 0; i < T * K; ++i) {
-                        const int32_t e = m.ids_host[(size_t) i];
+                        const int32_t e = ids_h[(size_t) i];
                         if (e < 0 || e >= m.g->n_expert) { err = "prefill: routed id out of range"; return false; }
                         ++m.cnt[(size_t) e];
                     }
@@ -1306,13 +1520,18 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     for (int64_t e = 0; e < m.g->n_expert; ++e) m.off[(size_t) e + 1] = m.off[(size_t) e] + m.cnt[(size_t) e];
                     std::vector<int32_t> fill(m.off.begin(), m.off.end() - 1);
                     for (int64_t i = 0; i < T * K; ++i) {
-                        const int32_t e = m.ids_host[(size_t) i];
+                        const int32_t e = ids_h[(size_t) i];
                         const int32_t p = fill[(size_t) e]++;
-                        m.slot_host[(size_t) i] = p;
-                        m.src_host[(size_t) p] = (int32_t) (i / K);
+                        slot_h[(size_t) i] = p;
+                        src_h[(size_t) p] = (int32_t) (i / K);
+                    }
+                    if (grp_mapped) {
+                        copy_i32(m.slot_dev, m.grp_dev + m.grp_tk, T * K, m.cs);
+                        copy_i32(m.src_dev, m.grp_dev + 2 * m.grp_tk, T * K, m.cs);
+                    } else {
+                        cudaMemcpyAsync(m.slot_dev, m.slot_host.data(), (size_t) T * K * 4, cudaMemcpyHostToDevice, m.cs);
+                        cudaMemcpyAsync(m.src_dev, m.src_host.data(), (size_t) T * K * 4, cudaMemcpyHostToDevice, m.cs);
                     }
-                    cudaMemcpyAsync(m.slot_dev, m.slot_host.data(), (size_t) T * K * 4, cudaMemcpyHostToDevice, m.cs);
-                    cudaMemcpyAsync(m.src_dev, m.src_host.data(), (size_t) T * K * 4, cudaMemcpyHostToDevice, m.cs);
                     // the experts, in id order: resident ones from VRAM, the others through the staging ring
                     std::vector<int32_t> order;
                     for (int32_t e = 0; e < m.g->n_expert; ++e) if (m.cnt[(size_t) e] > 0) order.push_back(e);
@@ -1336,8 +1555,14 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             for (size_t i = 0; i <= MMQ_GROUP; ++i)
                                 m.bounds_host[n + 1 + g * (MMQ_GROUP + 1) + i] =
                                     m.bounds_host[std::min(n, g * MMQ_GROUP + i)] - m.bounds_host[g * MMQ_GROUP];
-                        cudaMemcpyAsync(m.bounds_dev, m.bounds_host.data(), m.bounds_host.size() * 4,
-                                        cudaMemcpyHostToDevice, m.cs);
+                        if (grp_mapped) {
+                            int32_t* bh = m.grp_host + 3 * m.grp_tk;
+                            std::memcpy(bh, m.bounds_host.data(), m.bounds_host.size() * 4);
+                            copy_i32(m.bounds_dev, m.grp_dev + 3 * m.grp_tk, (int64_t) m.bounds_host.size(), m.cs);
+                        } else {
+                            cudaMemcpyAsync(m.bounds_dev, m.bounds_host.data(), m.bounds_host.size() * 4,
+                                            cudaMemcpyHostToDevice, m.cs);
+                        }
                     } else {
                         gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs);
                     }
@@ -1522,8 +1747,24 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                         }
                     }
                 }
-                // ---- the hyper-connection write of this half
-                gr_write(m.R, m.bo, m.inj, HC, T, m.cs);
+                // ---- the hyper-connection write of this half; F-2: fused with the next half's norm when nothing else
+                // touches R in between (not the stage's last half, not before the PLE block of layer 1, not under a
+                // control vector)
+                const int64_t nl = half == 0 ? l : l + 1;
+                const bool fuse = !gr_unfused() && nl < LE && !(half == 1 && nl == 1 && ple_on) &&
+                                  !(half == 1 && strata::kernels::cvec().covers(l));
+                const core::WeightRef* wnn = nullptr;
+                if (fuse) {
+                    const core::LayerView vn(*m.wt, nl);
+                    wnn = need(vn, half == 0 ? "hc_ffn_norm.weight" : "hc_attn_norm.weight", err);
+                    if (!wnn) return false;
+                }
+                if (wnn) {
+                    gr_write_norm_rs(m.R, m.bo, m.inj, HC, (const float*) wnn->data, EPS, m.grs, m.xn16, T, m.cs);
+                    normed = true;
+                } else {
+                    gr_write(m.R, m.bo, m.inj, HC, T, m.cs);
+                }
                 if (half == 1 && strata::kernels::cvec().covers(l))   // --control-vector-scaled
                     strata::kernels::cvec_apply(m.R, l, T, D, nullptr, 0, nullptr, 0, false, m.cs);
             }
@@ -1605,6 +1846,11 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
         err = std::string("prefill: ") + cudaGetErrorString(cudaGetLastError());
         return false;
     }
+    // (PR #121) an expert copy that failed on the copy stream surfaces here, not in the next request
+    if (const cudaError_t cst = cudaStreamSynchronize(m.copy); cst != cudaSuccess) {
+        err = std::string("prefill: expert copy stream: ") + cudaGetErrorString(cst);
+        return false;
+    }
     stats_.ms_total += ms_since(t_start);
     if (pt.on) {
         pt.fold();

src/program/generate.cpp

@@ -77,9 +77,11 @@
 #include <mutex>
 #include <charconv>
 #include <cmath>
+#include <limits>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 #include <sstream>
 #include <string>
@@ -180,6 +182,7 @@ struct Options {
     /// is reserved for a host thread that has nothing to do while the drain runs.
     bool no_host_worker = false;
     bool mmap_experts = false;    ///< R2.1: opt OUT of the resident arena, back to MapViewOfFile
+    bool resident_cpu_experts = false; ///< mmap-backed static-cache misses copied into ordinary RAM
     /// R4: slots of VRAM-resident experts.  **0 = off, and off is the default.**
     /// **THE COMMENT THAT USED TO BE HERE WAS FALSE AND ROUND 328 MEASURED IT.**  It said "the cache has no
     /// consumer yet - `moe_hit_grouped_s2` does not exist - so switching it on costs the fill traffic and
@@ -326,6 +329,8 @@ void usage() {
                  "                       VRAM; default fp16 until gate G-C accepts int8\n"
                  "  --kv q4_0            4-bit K/V after a Hadamard rotation (PR #21): half of int8's memory,\n"
                  "                       slightly lower precision (see bench/results/2026-09-27-kv-q4)\n"
+                 "  --kv k8v4            hybrid: INT8 K (exact attention scores) + rotated Q4_0 V, 816 B/cell\n"
+                 "                       (vs int8's 1,056); not with --kv-resident\n"
                  "  --kv-resident N      KV streaming: keep N cells of each QSA layer in VRAM (min 20480) and the\n"
                  "                       whole K/V in pinned RAM; the freed VRAM goes to expert slots. 0 (default):\n"
                  "                       all of it in VRAM. A context of N cells or fewer is not streamed\n"
@@ -429,7 +434,9 @@ void usage() {
                  "                       deviation from `cpu_s2` is attributed.\n"
                  "  --mmap-experts       R2.1: opt OUT of the resident expert arena, back to MapViewOfFile.\n"
                  "                       The A/B arm: the mmap's rate depends on the OS page cache holding\n"
-                 "                       34 GB, and measured 71.97 vs 34.78 ms/token cold vs warm.\n");
+                 "                       34 GB, and measured 71.97 vs 34.78 ms/token cold vs warm.\n"
+                 "  --resident-cpu-experts  with mmap and a static profile, keep CPU misses resident in ordinary RAM.\n"
+                 "                       Borrowed GPU-cache entries may read from mmap during prompt prefill.\n");
 }
 
 /// All shards of a split GGUF, from shard 1's path ("...-00001-of-00002.gguf"); just the path when it is not split.
@@ -519,6 +526,19 @@ void drive_pool_multi(void* user, const float* x_f, const int32_t* ids, int64_t
     strata::core::expert_pool_dispatch_multi(t->d, x_f, ids, n_tok, k, out);
     t->cpu_ms += std::chrono::duration<double, std::milli>(Clock::now() - a).count();
     ++t->calls;
+    // the routing trace for the serve path: the same record format drive_pool writes (layer, k, ids, weights),
+    // one record per token.  The multi dispatch fuses the router weights into the kernel and does not surface
+    // them, so records carry unit weights: tools/make_profile.py ranks pairs by routed frequency, which is the
+    // signal that matters; a one-shot --dump-routing run records true weights if a weighted ranking is wanted.
+    if (t->routing != nullptr && layer >= 0 && layer < 48) {
+        for (int64_t tok = 0; tok < n_tok; ++tok) {
+            const int32_t rec[2] = {(int32_t) layer, (int32_t) k};
+            std::fwrite(rec, sizeof rec, 1, t->routing);
+            std::fwrite(ids + tok * k, sizeof(int32_t), (size_t) k, t->routing);
+            static const float one[64] = {};   // k <= 64 in a verify window; zeros read as unit weights
+            std::fwrite(one, sizeof(float), (size_t) k, t->routing);
+        }
+    }
 }
 
 /// Layer split: every verify stage shares one Drive (its counters, usage and failure flags); the GPU plan, the expert
@@ -1050,6 +1070,7 @@ int main(int argc, char** argv) {
         else if (a == "--expert-profile") o.expert_profile = next("--expert-profile");
         else if (a == "--gpu-stages") o.gpu_stages = true;
         else if (a == "--mmap-experts") o.mmap_experts = true;
+        else if (a == "--resident-cpu-experts") o.resident_cpu_experts = true;
         else if (a == "--stats") o.stats = true;
         else if (a == "--shared-late") o.shared_late = true;
         else if (a == "--keep-canonical") o.keep_canonical = true;
@@ -1118,6 +1139,16 @@ int main(int argc, char** argv) {
         }
     }
     const bool multi_gpu = !split_devs.empty() && !split_same;
+    if (o.resident_cpu_experts && (!o.mmap_experts || o.expert_profile.empty() || o.adapt_every != 0)) {
+        std::fprintf(stderr, "strata generate: --resident-cpu-experts requires --mmap-experts, a static --expert-profile and --adapt-every 0\n");
+        return 2;
+    }
+    if (o.resident_cpu_experts &&
+        (!o.layer_split.empty() || o.expert_cache_remote[0] > 0 || o.expert_cache_remote[1] > 0 ||
+         o.expert_cache_remote[2] > 0)) {
+        std::fprintf(stderr, "strata generate: --resident-cpu-experts does not support layer splits or remote expert caches\n");
+        return 2;
+    }
     // the helper-GPU expert caches (--expert-cache-remote, docs/SECOND_GPU.md): CUDA1..3 on one GPU; with a layer
     // split, the visible GPUs no stage runs on, in order
     int remote_dev[3] = {1, 2, 3};
@@ -1170,16 +1201,21 @@ int main(int argc, char** argv) {
         return 2;
     }
     if (o.kv == "q4") o.kv = "q4_0";
-    if (o.kv != "fp16" && o.kv != "int8" && o.kv != "q4_0") {
-        std::fprintf(stderr, "strata generate: --kv must be fp16, int8 or q4_0\n");
+    if (o.kv != "fp16" && o.kv != "int8" && o.kv != "q4_0" && o.kv != "k8v4") {
+        std::fprintf(stderr, "strata generate: --kv must be fp16, int8, q4_0 or k8v4\n");
         return 2;
     }
     strata::core::qsa_set_kv_int8(o.kv == "int8");
     strata::core::qsa_set_kv_q4(o.kv == "q4_0");   // PR #21: 4-bit codes after a Hadamard rotation (kv_q4.hpp)
+    strata::core::qsa_set_kv_hybrid(o.kv == "k8v4");   // K8V4: INT8 K + rotated Q4_0 V, 816 B/cell
     if (o.kv_resident < 0) {
         std::fprintf(stderr, "strata generate: --kv-resident must be >= 0\n");
         return 2;
     }
+    if (o.kv == "k8v4" && o.kv_resident > 0) {
+        std::fprintf(stderr, "strata generate: --kv k8v4 does not support --kv-resident streaming (yet)\n");
+        return 2;
+    }
     strata::core::qsa_set_kv_resident(o.kv_resident);
     // Prompt lookup (the suffix drafter, on by default): the MTP keeps its --spec windows and a lookup window may be
     // up to 2 tokens longer; the draft policy (strata/spec/draft_policy.hpp) takes one only where it pays. Code
@@ -1557,7 +1593,7 @@ int main(int argc, char** argv) {
             ss.ple.w.key_scales = (const float*) ((const uint8_t*) wk->data + wk->codes_bytes);
         }
         if (o.native_ple_key && wk->quantized()) {
-            if (!wk->native_data || wk->native_type != 42 || !wk->native_q8_1) {
+            if (!wk->native_data || (wk->native_type != 42 && wk->native_type != 18 && wk->native_type != 23) || !wk->native_q8_1) {
                 std::fprintf(stderr, "strata generate: native PLE key is absent or incompatible\n");
                 return 1;
             }
@@ -1728,9 +1764,14 @@ int main(int argc, char** argv) {
     strata::core::ArenaExpertSource arena_src;
     strata::core::ExpertSource* srcp = nullptr;
     if (o.mmap_experts) {
-        if (native_pack) {   // FileExpertSource maps the canonical pack's experts.bin; a native pack has none
-            std::fprintf(stderr, "strata generate: --mmap-experts needs a canonical pack (experts.bin); %s is a native "
-                                 "(IQ) pack, whose experts are loaded into the arena\n", o.pack.c_str());
+        // FileExpertSource maps the pack's experts.bin: a canonical pack has it; a native (IQ) pack has it when
+        // built with `tools/iq_pack.py --experts-bin` (the per-layer blob sizes of its layout, PR #121).  The low-RAM
+        // mode: the experts come from the file through the OS cache instead of a pinned copy in RAM, for a PC whose
+        // GPU holds most of them but whose RAM cannot hold them all.
+        if (native_pack && !std::filesystem::exists(std::filesystem::path(o.pack) / "experts.bin")) {
+            std::fprintf(stderr, "strata generate: --mmap-experts needs the pack's experts.bin; %s is a native (IQ) pack "
+                                 "built without it: python tools/iq_pack.py --gguf <shard 1> --out %s --experts-bin\n",
+                         o.pack.c_str(), o.pack.c_str());
             return 2;
         }
         if (!src.open(o.pack, g.n_layers, g.n_expert, err)) {
@@ -1754,6 +1795,21 @@ int main(int argc, char** argv) {
         std::fprintf(stderr, "strata generate: loaded %.2f GiB at %.2f GiB/s\n",
                      (double) strata::kernels::cpu::expert_layout().total / (1024.0 * 1024 * 1024),
                      arena_src.load_gib_per_second());
+        // A rate under ~0.2 GiB/s is not the hardware.  Task Scheduler / service contexts throttle this
+        // read+fill about 24x (measured 0.05 vs 1.42 GiB/s for the same binary, args and cache state; the
+        // scheduler's defaults - Below normal priority and a least-privilege token - were the only
+        // difference between the runs).  Say so instead of letting the user blame the disk; see
+        // docs/DETAILS.md, "Running it at startup (Task Scheduler)".
+#ifdef _WIN32   // a Windows launch context; elsewhere a load this slow is the disk
+        if (arena_src.load_gib_per_second() > 0.0 && arena_src.load_gib_per_second() < 0.2) {
+            std::fprintf(stderr,
+                         "strata generate: hint: ~24x below what this hardware streams from a normal "
+                         "launch. If Strata is started by Task Scheduler or a service, register the task "
+                         "with Priority 4 (Normal) and 'Run with highest privileges' - the scheduler's "
+                         "defaults (Below normal + a least-privilege token) throttle the load. See "
+                         "docs/DETAILS.md ('Running it at startup').\n");
+        }
+#endif
         srcp = &arena_src;
     }
     strata::kernels::cpu::ExpertPool pool(o.pool_workers, /*pin=*/true, /*host_works=*/!o.no_host_worker);
@@ -1937,12 +1993,20 @@ int main(int argc, char** argv) {
         // (with borrowing - the default with a profile - the prompt path lends cache slots instead)
         const bool borrow = !o.no_prefill_borrow && !o.expert_profile.empty();
         const int64_t prefill_mib = (o.prefill_chunk > 0 && !borrow) ? 160 + (o.prefill_chunk * 680) / 1024 : 0;
-        const int64_t reserve = ((int64_t) o.vram_reserve_mib + prefill_mib) << 20;
+        // the draft layer's head and logits are allocated when it binds, after this: 0.1.27's CJK subset made them
+        // ~110-180 MiB larger, and out of the reserve they left 16 GB cards below the stall line (#199)
+        const int64_t mtp_bind = (!o.mtp.empty() && native_head.loaded())
+                                     ? (int64_t) mtp.bind_bytes(native_head.row_bytes(), n_vocab) : 0;
+        const int64_t reserve = (((int64_t) o.vram_reserve_mib + prefill_mib) << 20) + mtp_bind;
         int64_t slots = ((int64_t) free_b - reserve) / (int64_t) strata::kernels::cpu::expert_layout().max_blob;
         if (!profile.empty()) slots = std::min<int64_t>(slots, (int64_t) profile.size());
         o.expert_cache = (int) std::max<int64_t>(slots, 0);
-        std::fprintf(stderr, "strata generate: expert cache auto: %.2f GiB free, %d MiB reserved -> %d slots\n",
-                     (double) free_b / 1073741824.0, o.vram_reserve_mib, o.expert_cache);
+        std::fprintf(stderr, "strata generate: expert cache auto: %.2f GiB free, %d MiB reserved (+%lld MiB for the "
+                             "draft head) -> %d slots\n",
+                     (double) free_b / 1073741824.0, o.vram_reserve_mib, (long long) (mtp_bind >> 20), o.expert_cache);
+        if (o.expert_cache == 0)   // the verify window cannot start without it (#174): say what makes room
+            std::fprintf(stderr, "strata generate: no VRAM is left for the expert cache: lower --max-context, use "
+                                 "--kv k8v4, run images on the CPU, or close other programs that use the GPU\n");
     } else if (multi_gpu && o.expert_cache > 0) {
         // a layer split's prompt path has its own buffers (it borrows no slots): an explicit cache size leaves room
         // for them and the reserve, or the first prompt fails with "device buffers ... do not fit"
@@ -2008,6 +2072,7 @@ int main(int argc, char** argv) {
         // STRATA_TEST_CACHE_FAIL=N: the first N opens fail as an out-of-commit cudaMalloc does (tests the retry)
         int fake_fails = std::getenv("STRATA_TEST_CACHE_FAIL") ? std::atoi(std::getenv("STRATA_TEST_CACHE_FAIL")) : 0;
         int failed = 0;
+        int zero_reads = 0;
         for (int attempt = 0;; ++attempt) {
             bool ok = false;
             if (fake_fails > 0) {
@@ -2051,9 +2116,11 @@ int main(int argc, char** argv) {
             cudaMemGetInfo(&free_b, &total_b);
             const int64_t want = (int64_t) o.vram_reserve_mib << 20;
             if ((int64_t) free_b >= want - (64ll << 20)) break;
-            // short by (want - free); a figure of 0 only says "at least", so then give back a quarter as well
+            // short by (want - free); a figure of 0 only says "at least": the first two such reads give back 1 GiB
+            // each (under WDDM the free figure read before the allocation runs ~0.7 GiB high), later ones a quarter
             int64_t give = want - (int64_t) free_b + (64ll << 20);
-            if (free_b < ((size_t) 16 << 20)) give = std::max<int64_t>(give, xcache.bytes() / 4);
+            if (free_b < ((size_t) 16 << 20))
+                give = std::max<int64_t>(give, ++zero_reads <= 2 ? 1ll << 30 : xcache.bytes() / 4);
             const int64_t keep_bytes = xcache.bytes() - give;
             std::fprintf(stderr, "strata generate: only %lld MiB free once the slots are written (reserve %d MiB); "
                                  "shrinking the expert cache\n", (long long) (free_b >> 20), o.vram_reserve_mib);
@@ -2124,6 +2191,14 @@ int main(int argc, char** argv) {
                      (long long) prefilled, (long long) want);
     }
 
+    if (o.resident_cpu_experts) {
+        if (!src.pin_cache_complement(xcache, err, /*pin=*/false)) {
+            std::fprintf(stderr, "strata generate: CPU expert residency: %s\n", err.c_str());
+            return 1;
+        }
+        std::fprintf(stderr, "strata generate: steady-state CPU cache misses are resident in ordinary RAM; borrowed cache entries may use mmap during prompt prefill\n");
+    }
+
     for (auto& stp : stages) {
         GpuStage& st = *stp;
         const auto& lay = strata::kernels::cpu::expert_layout();
@@ -2939,6 +3014,13 @@ int main(int argc, char** argv) {
                                      : (uint64_t) (xcache.slots() - first) *
                                            (uint64_t) strata::kernels::cpu::expert_layout().max_blob;
     };
+    auto request_chunk = [](int64_t tokens, int64_t max_chunk) -> int64_t {
+        if (tokens <= 0 || max_chunk <= 0) return 0;
+        const int64_t rounded = tokens > std::numeric_limits<int64_t>::max() - 255
+                                    ? tokens
+                                    : ((tokens + 255) / 256) * 256;
+        return std::min(max_chunk, rounded);
+    };
     // The prompt path's chunk and the slots it borrows for its buffers: the requested chunk halved until it fits,
     // or with --prefill auto the largest of kAutoChunks whose buffers take at most kAutoLendPct % of the slots (a
     // lent slot's expert is streamed during the prompt and refilled after it; measured on a 12 GB card, 32K Q2_0
@@ -3001,6 +3083,8 @@ int main(int argc, char** argv) {
             } else if (o.prefill_auto) {
                 o.prefill_chunk = 1024;   // nothing lendable: small buffers of its own
             }
+        } else if (o.prefill_auto && d_res == nullptr) {
+            o.prefill_chunk = 1024;       // #85: no expert cache at all (a full 8 GB card): small buffers of its own
         }
         if (borrow != nullptr)
             std::fprintf(stderr, "strata serve: the prompt path borrows %lld cache slots (%.2f GiB)\n",
@@ -3020,6 +3104,10 @@ int main(int argc, char** argv) {
         if (multi_gpu) sp.set_stage(0, split_at[0], &stages[0]->sp);
         if (!sp.init(wt, g, ss, srcp, &xcache, host_res.data(), o.prefill_chunk, main_cs, err, borrow, borrow_bytes)) {
             std::fprintf(stderr, "strata serve: %s\n", err.c_str());
+            if (err.find("fit") != std::string::npos)   // #85: say what frees VRAM
+                std::fprintf(stderr, "strata serve: the GPU has too little free VRAM for the prompt path: turn images "
+                                     "off (setup: --vision no), close other programs using the GPU, use a shorter "
+                                     "context, or read prompts in smaller chunks (--prefill 512)\n");
             return 1;
         }
         mem_mark("the head and the prompt path");
@@ -3195,7 +3283,9 @@ int main(int argc, char** argv) {
         sp.on_chunk = [&](const float* R_rows, int64_t T, int64_t p0, std::string& e) -> bool {
             std::vector<int32_t> nxt((size_t) T);
             for (int64_t t = 0; t < T; ++t) nxt[(size_t) t] = (int32_t) cur[(size_t) (p0 + t + 1)];
-            if (!mtp.prefill(R_rows, nxt.data(), T, p0, e)) return false;
+            // E-9: batched through the prompt path when it can (one GPU: a layer split's drafter is on the last stage)
+            const bool batched = !multi_gpu && sp.draft_kv(mtp, R_rows, nxt.data(), T, p0, e);
+            if (!e.empty() || (!batched && !mtp.prefill(R_rows, nxt.data(), T, p0, e))) return false;
             // progress for the server window: PP <position reached> <prompt tokens> <ms> <fresh tokens/s>
             const int64_t done = p0 + T;
             const double ms = std::chrono::duration<double, std::milli>(Clock::now() - pp_t0).count();
@@ -3480,6 +3570,7 @@ int main(int argc, char** argv) {
                 ~BusyScope() { strata::core::progress().busy.store(false); strata::core::progress_at("idle"); }
             } busy_scope;
             stop_req.store(false);   // a STOP that arrived between requests is stale
+            err.clear();
             const bool geni = line.rfind("GENI ", 0) == 0;
             if (!geni && line.rfind("GEN ", 0) != 0) {
                 std::printf("ERR expected: GEN <max_new> <id,id,...> or GENI <max_new> <file> <id,id,...>\n");
@@ -3499,7 +3590,8 @@ int main(int argc, char** argv) {
             // tuning keys (setup's calibration measures settings without restarting the engine): the PCIe share of
             // the missed experts and the draft-probability floor, for this request only
             double req_pcie_frac = o.pcie_frac, req_spec_min_p = o.spec_min_p;
-            if (endp != nullptr) {   // GENI takes only cvec=; its file path is the first token without an =
+            if (endp != nullptr) {   // GENI takes the same keys (#75: image requests were always greedy); its
+                                     // embedding file path is the first token without an =
                 for (;;) {
                     while (*endp == ' ') ++endp;
                     const char* start = endp;
@@ -3511,7 +3603,6 @@ int main(int argc, char** argv) {
                     const std::string key = tok.substr(0, eq);
                     const float fv = std::strtof(tok.c_str() + eq + 1, nullptr);
                     if (key == "cvec") req_cvec = std::atoi(tok.c_str() + eq + 1);
-                    else if (geni) {}   // image requests decode greedily
                     else if (key == "temperature") req_temperature = fv;
                     else if (key == "top_p") req_top_p = fv;
                     else if (key == "top_k") req_top_k = std::atoi(tok.c_str() + eq + 1);
@@ -3765,6 +3856,7 @@ int main(int argc, char** argv) {
             };
             // tokens [a, b) through the windows: commit all of them, then give the draft layer their residuals
             auto read_windows = [&](int64_t a, int64_t b, std::string& e) -> bool {
+                strata::core::progress_at("reading the prompt (verify windows), from token", a);   // #217: not "batched"
                 // every token is committed and the picks are discarded: no head sampling (see set_head_sampling)
                 struct NoHeadSampling {
                     strata::core::Verifier& v;
@@ -3821,12 +3913,21 @@ int main(int argc, char** argv) {
             // rounded up to 256), laid out in the last of the slots it may borrow
             auto lend = [&](int64_t tokens, std::string& e) -> bool {
                 if (lend_first < 0) return true;                       // its own buffers: nothing to lend
-                const int64_t want = std::min<int64_t>(o.prefill_chunk, (tokens + 255) / 256 * 256);
+                const int64_t want = request_chunk(tokens, o.prefill_chunk);
+                if (want <= 0) {
+                    e = "prefill: cannot lend buffers for an empty request segment";
+                    return false;
+                }
                 if (!lent_now.empty()) {
                     if (want <= lent_chunk) return true;
                     if (!refill(e)) return false;
                 }
-                const int32_t first = std::max<int32_t>(lend_first, (int32_t) (xcache.slots() - lend_slots(want)));
+                const int64_t slots = lend_slots(want);
+                if (slots <= 0 || slots > xcache.slots() - lend_first) {
+                    e = "prefill: request-sized buffers exceed the configured lend region";
+                    return false;
+                }
+                const int32_t first = std::max<int32_t>(lend_first, (int32_t) (xcache.slots() - slots));
                 if (want != sp.chunk() || first != lend_first_now) {
                     if (!sp.relayout(want, xcache.device_slot(first), lend_bytes(first), e)) return false;
                     lend_first_now = first;
@@ -3888,6 +3989,7 @@ int main(int argc, char** argv) {
             int64_t at = read_from;
             for (const int64_t to : {reread_to, root_at, turn_at, n - 1}) {
                 if (to <= at) continue;
+                err.clear();
                 const bool win = windows_ok(at, to);
                 if (win && !refill(err)) {
                     std::printf("ERR refilling a lent slot failed: %s\n", err.c_str());
@@ -3947,6 +4049,20 @@ int main(int argc, char** argv) {
             for (int64_t i = 0; i < n - 1; ++i) consumed.push_back((int32_t) ids[(size_t) i]);
             const char* finish = "length";
             const Clock::time_point d0 = Clock::now();
+            // STRATA_DECODE_TIMING=1: where a request's decode time goes (one line per request)
+            static const bool dec_timing = std::getenv("STRATA_DECODE_TIMING") != nullptr;
+            struct DecSnap {
+                double wait, pool, host, plan, actq, jobs, run;
+                int64_t misses, entries, hits, pcie;
+            };
+            auto dec_snap = [&]() {
+                return DecSnap{ver.ms_wait, ver.ms_pool, ver.ms_host, drive.d.ms_plan, drive.d.ms_actq, drive.d.ms_jobs,
+                               drive.d.ms_run, drive.d.multi_misses, drive.d.multi_entries, drive.d.cache_hits,
+                               drive.d.pcie_experts};
+            };
+            const DecSnap ds0 = dec_snap();
+            double dt_run = 0, dt_commit = 0, dt_draft = 0;
+            int64_t dec_windows = 0, dec_T = 0;
             const int64_t decode_hits0 = drive.d.cache_hits;
             const int64_t decode_look0 = drive.d.cache_hits + drive.d.cache_admitted + drive.d.cache_refused;
             if (cancelled) finish = "cancel";
@@ -3990,6 +4106,7 @@ int main(int argc, char** argv) {
                                cudaMemcpyHostToDevice);
                 }
                 tr("window", p, T);
+                const Clock::time_point tw0 = Clock::now();
                 if (!ver.run(T, window.data(), p, win_pool_fn, win_pool_user, outv.data(), err) || drive.d.failed) {
                     std::printf("ERR %s\n", drive.d.failed && drive.d.fail ? drive.d.fail : err.c_str());
                     return 1;
@@ -3997,6 +4114,7 @@ int main(int argc, char** argv) {
                 int a = 0;
                 while (a < T - 1 && window[(size_t) a + 1] == outv[(size_t) a]) ++a;
                 if (from_sfx) { ++sfx_windows; sfx_drafts += T - 1; sfx_ok += a; }
+                const Clock::time_point tw1 = Clock::now();
                 std::thread adapt_thr;   // the adaptive tier beside the commit and the draft (as in generate)
                 bool adapt_ok = true;
                 if (!drive.d.usage.empty() && ((rounds + 1) % o.adapt_every) == 0)
@@ -4021,8 +4139,15 @@ int main(int argc, char** argv) {
                 }
                 std::fflush(stdout);
                 ++rounds;
+                const Clock::time_point tw2 = Clock::now();
                 const bool drafted = eos || produced_n >= max_new ||
                                      mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) req_spec_min_p);
+                {
+                    const Clock::time_point tw3 = Clock::now();
+                    auto msd = [](Clock::time_point a0, Clock::time_point b0) { return std::chrono::duration<double, std::milli>(b0 - a0).count(); };
+                    dt_run += msd(tw0, tw1); dt_commit += msd(tw1, tw2); dt_draft += msd(tw2, tw3);
+                    ++dec_windows; dec_T += T;
+                }
                 if (adapt_thr.joinable()) adapt_thr.join();
                 if (!adapt_ok) {
                     std::printf("ERR an adaptive refill failed\n");
@@ -4041,6 +4166,21 @@ int main(int argc, char** argv) {
                 p += a + 1;
             }
             const double decode_ms = std::chrono::duration<double, std::milli>(Clock::now() - d0).count();
+            if (dec_timing && dec_windows > 0) {
+                const DecSnap d1 = dec_snap();
+                const double w = (double) dec_windows, L = (double) g.n_layers;
+                std::fprintf(stderr, "strata decode timing: %lld windows, avg T %.2f, %.2f tokens/window, %.2f ms/window = "
+                                     "verify %.2f (GPU-reach wait %.2f + per-layer host %.2f [plan %.2f actq %.2f jobs %.2f "
+                                     "CPU %.2f] + stage %.2f) + commit/emit %.2f + draft %.2f; per layer-window: CPU experts "
+                                     "%.2f (%.2f entries), VRAM hits %.2f, PCIe %.2f\n",
+                             (long long) dec_windows, dec_T / w, produced_n / w, decode_ms / w, dt_run / w,
+                             (d1.wait - ds0.wait) / w, (d1.pool - ds0.pool) / w, (d1.plan - ds0.plan) / w,
+                             (d1.actq - ds0.actq) / w, (d1.jobs - ds0.jobs) / w, (d1.run - ds0.run) / w,
+                             (d1.host - ds0.host) / w, dt_commit / w, dt_draft / w, (d1.misses - ds0.misses) / (w * L),
+                             (d1.entries - ds0.entries) / (w * L), (d1.hits - ds0.hits) / (w * L), (d1.pcie - ds0.pcie) / (w * L));
+                const std::string pr = ver.profile_report();
+                if (!pr.empty()) std::fprintf(stderr, "strata decode GPU stages (ms/window):%s\n", pr.c_str());
+            }
             if (!cancelled) {
                 // a prompt stopped halfway leaves the session somewhere between two chunks: nothing to continue from
                 // (the checkpoints taken while reading it are still good)
@@ -4097,6 +4237,10 @@ int main(int argc, char** argv) {
                     if (st.kv_q4) {
                         const int64_t q4b = (int64_t) strata::kernels::kv_q4_bytes_per_head((int) qs.head_dim);
                         a = {{h ? st.host.k_q4 : st.k_q4, q4b}, {h ? st.host.v_q4 : st.v_q4, q4b}};
+                    } else if (st.kv_hybrid) {
+                        const int64_t q4b = (int64_t) strata::kernels::kv_q4_bytes_per_head((int) qs.head_dim);
+                        a = {{h ? st.host.k_q : st.k_q, kvb}, {h ? st.host.v_q4 : st.v_q4, q4b},
+                             {h ? st.host.k_scale : st.k_scale, scb}};
                     } else {
                         a = {{h ? st.host.k_q : st.k_q, kvb}, {h ? st.host.v_q : st.v_q, kvb},
                              {h ? st.host.k_scale : st.k_scale, scb}, {h ? st.host.v_scale : st.v_scale, scb}};
@@ -4133,6 +4277,7 @@ int main(int argc, char** argv) {
                         decode_ms, finish, (long long) draft_accepted, (long long) draft_offered, (long long) resume,
                         (long long) req_hits, (long long) req_look);
             std::fflush(stdout);
+            if (drive.routing != nullptr) std::fflush(drive.routing);   // the routing trace survives a crash and is watchable mid-session
             const int64_t fresh = n - resume;
             std::fprintf(stderr, "strata serve: prompt %lld tokens = %lld reused + %lld read in %.0f ms (%.1f tok/s), "
                                  "%lld generated in %.0f ms (%.1f tok/s), drafts accepted %lld of %lld, %zu checkpoints%s\n",
@@ -4195,19 +4340,22 @@ int main(int argc, char** argv) {
     std::FILE* final_r = o.dump_final_r.empty() ? nullptr : std::fopen(o.dump_final_r.c_str(), "wb");
     std::vector<float> final_r_host(final_r ? (size_t) (g.hc * g.n_embd) : 0);
     std::vector<std::pair<int32_t, int32_t>> lent;     // (residency index, slot) lent to the prompt path
+    const int64_t n_batched = (o.prefill_until > 0 && o.prefill_until < n_prompt - 1) ? o.prefill_until : n_prompt - 1;
     if (o.prefill_chunk > 0 && n_prompt > 1) {
         void* borrow = nullptr;
         uint64_t borrow_bytes = 0;
         if (!o.no_prefill_borrow && !host_res.empty() && d_res != nullptr) {
             int64_t chunk = o.prefill_chunk;
             int64_t k = plan_lend(chunk);             // auto: the largest chunk that fits; fixed: halved to fit
-            if (k > 0 && chunk > (n_prompt - 1 + 255) / 256 * 256) {   // no bigger than the prompt needs
-                chunk = std::max<int64_t>(256, (n_prompt - 1 + 255) / 256 * 256);
+            const int64_t request_sized = request_chunk(n_batched, chunk);
+            if (k > 0 && request_sized < chunk) {                     // no bigger than this prompt segment needs
+                chunk = request_sized;
                 k = lend_slots(chunk);
+                if (k + 128 > xcache.slots()) k = 0;
                 if (!o.prefill_auto) o.prefill_chunk = chunk;
             }
             if (o.prefill_auto) {
-                o.prefill_chunk = k > 0 ? chunk : 1024;
+                o.prefill_chunk = k > 0 ? chunk : request_chunk(n_batched, 1024);
                 std::fprintf(stderr, "strata generate: prompt chunk auto: %lld tokens\n", (long long) o.prefill_chunk);
             } else if (chunk != o.prefill_chunk) {
                 k = 0;                                 // a fixed chunk that does not fit: its own buffers, as before
@@ -4245,11 +4393,11 @@ int main(int argc, char** argv) {
                 // cell i pairs R_i with the token at i + 1 (every such token is in the prompt)
                 std::vector<int32_t> nxt((size_t) T);
                 for (int64_t t = 0; t < T; ++t) nxt[(size_t) t] = (int32_t) o.tokens[(size_t) (p0 + t + 1)];
-                return mtp.prefill(R_rows, nxt.data(), T, p0, e);
+                if (prefill.draft_kv(mtp, R_rows, nxt.data(), T, p0, e)) return true;   // E-9
+                return e.empty() && mtp.prefill(R_rows, nxt.data(), T, p0, e);
             };
         }
         const Clock::time_point tp0 = Clock::now();
-        const int64_t n_batched = (o.prefill_until > 0 && o.prefill_until < n_prompt - 1) ? o.prefill_until : n_prompt - 1;
         if (!prefill.run(o.tokens.data(), n_batched, 0, err)) {
             std::fprintf(stderr, "strata generate: %s\n", err.c_str());
             return 1;

tests/core/file_expert_source_test.cpp

@@ -0,0 +1,260 @@
+#include "strata/core/expert_source.hpp"
+#include "strata/kernels/cpu/expert.hpp"
+#include "strata/kernels/cpu/expert_layout.hpp"
+
+#if defined(STRATA_NATIVE_EXPERTS)
+#include "strata/kernels/cpu/native_expert.hpp"
+#include "ggml.h"
+#endif
+
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <limits>
+#include <stdexcept>
+#include <string>
+#include <system_error>
+#include <utility>
+#include <vector>
+
+namespace fs = std::filesystem;
+
+namespace {
+
+void require(bool ok, const std::string& message) {
+    if (!ok) throw std::runtime_error(message);
+}
+
+struct TempDirectory {
+    fs::path path;
+
+    TempDirectory() {
+        const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count();
+        path = fs::temp_directory_path() / ("strata-file-source-test-" + std::to_string(stamp));
+        fs::create_directories(path);
+    }
+
+    ~TempDirectory() {
+        std::error_code ignored;
+        fs::remove_all(path, ignored);
+    }
+};
+
+void create_pack(const fs::path& dir, uint64_t bytes, const std::vector<std::pair<uint64_t, char>>& markers = {}) {
+    const fs::path file = dir / "experts.bin";
+    {
+        std::ofstream out(file, std::ios::binary | std::ios::trunc);
+        require((bool) out, "could not create synthetic experts.bin");
+        if (bytes > 0) {
+            out.seekp((std::streamoff) (bytes - 1));
+            out.put('\0');
+        }
+        require((bool) out, "could not size synthetic experts.bin");
+    }
+    for (const auto& [offset, marker] : markers) {
+        require(offset < bytes, "synthetic marker lies beyond experts.bin");
+        std::fstream file_out(file, std::ios::binary | std::ios::in | std::ios::out);
+        require((bool) file_out, "could not open synthetic experts.bin for markers");
+        file_out.seekp((std::streamoff) offset);
+        file_out.put(marker);
+        require((bool) file_out, "could not write synthetic expert marker");
+    }
+}
+
+void check_file_size_rejection(strata::core::FileExpertSource& source, const fs::path& dir, int64_t layers,
+                               int64_t experts, uint64_t expected_bytes) {
+    std::string err;
+    source.close();
+    fs::resize_file(dir / "experts.bin", expected_bytes - 1);
+    require(!source.open(dir.string(), layers, experts, err), "a truncated expert file was accepted");
+    require(!source.mapped() && !err.empty(), "truncated-file rejection left the source mapped or unreported");
+
+    fs::resize_file(dir / "experts.bin", expected_bytes + 1);
+    err.clear();
+    require(!source.open(dir.string(), layers, experts, err), "an oversized expert file was accepted");
+    require(!source.mapped() && !err.empty(), "oversized-file rejection left the source mapped or unreported");
+}
+
+void test_canonical_layout() {
+    using namespace strata::core;
+    using namespace strata::kernels::cpu;
+    constexpr int64_t layers = 2;
+    constexpr int64_t experts = 3;
+    const uint64_t layer_bytes = (uint64_t) experts * BLOB;
+    const uint64_t total = (uint64_t) layers * layer_bytes;
+    TempDirectory dir;
+
+    std::string err;
+    const bool layout_ok = expert_layout_load(dir.path.string(), layers, experts, err);
+    require(layout_ok, "could not load canonical layout: " + err);
+    create_pack(dir.path, total, {{0, 'a'}, {(uint64_t) BLOB, 'b'}, {layer_bytes, 'c'},
+                                  {layer_bytes + (uint64_t) BLOB, 'd'}});
+
+    FileExpertSource source;
+    bool opened = source.open(dir.path.string(), layers, experts, err);
+    require(opened, "could not map canonical pack: " + err);
+    require(source.blobs() == layers * experts, "canonical blob count is wrong");
+    const uint8_t* first = source.blob(0, 0);
+    const uint8_t* second = source.blob(0, 1);
+    const uint8_t* next_layer = source.blob(1, 0);
+    require(first && second && next_layer, "valid canonical blob lookup failed");
+    require(second - first == (ptrdiff_t) BLOB && next_layer - first == (ptrdiff_t) layer_bytes,
+            "canonical expert or layer stride is wrong");
+    require(first[0] == 'a' && second[0] == 'b' && next_layer[0] == 'c',
+            "canonical blob lookup returned bytes from the wrong expert");
+    require(source.blob(-1, 0) == nullptr && source.blob(0, -1) == nullptr &&
+                source.blob(layers, 0) == nullptr && source.blob(0, experts) == nullptr,
+            "canonical bounds check accepted an invalid layer or expert");
+    require(source.reads() == 3, "invalid canonical lookups changed the read count");
+
+    check_file_size_rejection(source, dir.path, layers, experts, total);
+    create_pack(dir.path, total, {{layer_bytes, 'c'}});
+    err.clear();
+    opened = source.open(dir.path.string(), layers, experts, err);
+    require(opened, "source failed to reopen after size errors: " + err);
+    const uint8_t* reopened = source.blob(1, 0);
+    require(reopened && source.reads() == 1 && reopened[0] == 'c', "reopened canonical source kept stale state");
+    source.close();
+}
+
+#if defined(STRATA_NATIVE_EXPERTS)
+void test_native_variable_layout() {
+    using namespace strata::core;
+    using namespace strata::kernels::cpu;
+    constexpr int64_t layers = 2;
+    constexpr int64_t experts = 3;
+    constexpr int64_t embedding = 2560;
+    constexpr int64_t feed_forward = 640;
+    TempDirectory dir;
+
+    NativeFmt first_fmt, second_fmt;
+    std::string err;
+    const bool first_ok = native_fmt(GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ4_NL, embedding, feed_forward, first_fmt, err);
+    require(first_ok, "IQ3_XXS/IQ4_NL synthetic format is unavailable: " + err);
+    err.clear();
+    const bool second_ok = native_fmt(GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_NL, embedding, feed_forward, second_fmt, err);
+    require(second_ok, "IQ2_XS/IQ4_NL synthetic format is unavailable: " + err);
+    require(first_fmt.bytes != second_fmt.bytes, "chosen native formats do not exercise variable layer sizes");
+
+    const uint64_t layer0_bytes = (uint64_t) first_fmt.bytes * experts;
+    const uint64_t total = layer0_bytes + (uint64_t) second_fmt.bytes * experts;
+    {
+        std::ofstream metadata(dir.path / "native_experts.txt");
+        require((bool) metadata, "could not create synthetic native_experts.txt");
+        metadata << "0 " << GGML_TYPE_IQ3_XXS << ' ' << GGML_TYPE_IQ4_NL << " 0 " << first_fmt.bytes << '\n';
+        metadata << "1 " << GGML_TYPE_IQ2_XS << ' ' << GGML_TYPE_IQ4_NL << ' ' << layer0_bytes << ' '
+                 << second_fmt.bytes << '\n';
+        require((bool) metadata, "could not write synthetic native_experts.txt");
+    }
+    create_pack(dir.path, total, {{0, 'a'}, {(uint64_t) first_fmt.bytes, 'b'},
+                                  {layer0_bytes, 'c'}, {layer0_bytes + (uint64_t) second_fmt.bytes, 'd'}});
+
+    const bool layout_ok = expert_layout_load(dir.path.string(), layers, experts, err);
+    require(layout_ok, "could not load synthetic native layout: " + err);
+    FileExpertSource source;
+    bool opened = source.open(dir.path.string(), layers, experts, err);
+    require(opened, "could not map native pack: " + err);
+    require(source.blobs() == layers * experts, "native blob count is wrong");
+    const uint8_t* first = source.blob(0, 0);
+    const uint8_t* second = source.blob(0, 1);
+    const uint8_t* next_layer = source.blob(1, 0);
+    const uint8_t* next_layer_second = source.blob(1, 1);
+    require(first && second && next_layer && next_layer_second, "valid native blob lookup failed");
+    require(second - first == (ptrdiff_t) first_fmt.bytes && next_layer - first == (ptrdiff_t) layer0_bytes &&
+                next_layer_second - next_layer == (ptrdiff_t) second_fmt.bytes,
+            "native per-layer expert stride is wrong");
+    require(first[0] == 'a' && second[0] == 'b' && next_layer[0] == 'c' && next_layer_second[0] == 'd',
+            "native blob lookup returned bytes from the wrong expert");
+    require(source.blob(-1, 0) == nullptr && source.blob(0, -1) == nullptr &&
+                source.blob(layers, 0) == nullptr && source.blob(0, experts) == nullptr,
+            "native bounds check accepted an invalid layer or expert");
+    require(source.reads() == 4, "invalid native lookups changed the read count");
+
+    check_file_size_rejection(source, dir.path, layers, experts, total);
+    create_pack(dir.path, total, {{layer0_bytes, 'c'}});
+    err.clear();
+    opened = source.open(dir.path.string(), layers, experts, err);
+    require(opened, "source failed to reopen after size errors: " + err);
+    const uint8_t* reopened = source.blob(1, 0);
+    require(reopened && source.reads() == 1 && reopened[0] == 'c', "reopened native source kept stale state");
+    source.close();
+}
+#endif
+
+void test_complement_plan() {
+    using namespace strata::core::detail;
+    std::vector<uint64_t> offsets;
+    uint64_t bytes = 0;
+    std::string error;
+    require(make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}, {1, 2}}, {}, offsets, bytes, error), error);
+    require(bytes == 16 && offsets == std::vector<uint64_t>{0, kNoCacheComplement, 3, 6, 11, kNoCacheComplement},
+            "wrong compact offsets for variable native layer sizes");
+    const uint8_t resident[16] = {}, mapped[5] = {};
+    require(cache_complement_blob_or_fallback(1, offsets, resident, mapped) == mapped,
+            "GPU-resident expert lost mmap fallback");
+    require(cache_complement_blob_or_fallback(4, offsets, resident, mapped) == resident + 11,
+            "CPU miss did not use resident complement");
+    require(!make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}, {0, 1}}, {}, offsets, bytes, error)
+            && offsets.empty() && bytes == 0, "duplicate pair accepted");
+    require(!make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}}, {{0, 1}}, offsets, bytes, error),
+            "overlapping tiers accepted");
+    require(!make_cache_complement_plan(2, 3, {3, 5}, {{2, 0}}, {}, offsets, bytes, error),
+            "out-of-range pair accepted");
+    require(!make_cache_complement_plan(2, 3, {0, 5}, {}, {}, offsets, bytes, error), "zero-size layer accepted");
+}
+
+void test_cgroup_memory_budget() {
+    using namespace strata::core::detail;
+    constexpr uint64_t GiB = 1ull << 30;
+    uint64_t bytes = 0;
+
+    CgroupMemoryStat clean_cache{40 * GiB, 32 * GiB, 0, 0, true};
+    require(cgroup_available_bytes(56 * GiB, clean_cache, bytes), "valid cgroup memory.stat was rejected");
+    require(bytes == 48 * GiB, "clean inactive file cache was not credited against the cgroup cap");
+
+    CgroupMemoryStat dirty_cache{40 * GiB, 32 * GiB, 4 * GiB, 2 * GiB, true};
+    require(cgroup_available_bytes(56 * GiB, dirty_cache, bytes), "valid dirty-cache counters were rejected");
+    require(bytes == 42 * GiB, "dirty/writeback pages were incorrectly counted as reclaimable");
+
+    CgroupMemoryStat oversized_inactive{10 * GiB, 20 * GiB, 0, 0, true};
+    require(cgroup_available_bytes(12 * GiB, oversized_inactive, bytes),
+            "valid oversized inactive-file counter was rejected");
+    require(bytes == 12 * GiB, "inactive-file accounting exceeded charged current usage");
+
+    const uint64_t max = std::numeric_limits<uint64_t>::max();
+    CgroupMemoryStat large_counters{max, max, max - 1, max, true};
+    require(cgroup_available_bytes(max, large_counters, bytes),
+            "large memory.stat counters were rejected");
+    require(bytes == 0, "dirty/writeback subtraction overflowed or escaped the usage cap");
+
+    CgroupMemoryStat usage_over_limit{60 * GiB, 0, 0, 0, true};
+    require(cgroup_available_bytes(56 * GiB, usage_over_limit, bytes),
+            "valid over-limit memory counters were rejected");
+    require(bytes == 0, "over-limit cgroup usage produced a positive budget");
+
+    CgroupMemoryStat missing_stat{};
+    bytes = 123;
+    require(!cgroup_available_bytes(56 * GiB, missing_stat, bytes) && bytes == 0,
+            "missing memory.stat counters did not fail closed");
+}
+
+}  // namespace
+
+int main() {
+    try {
+        test_complement_plan();
+        test_cgroup_memory_budget();
+        test_canonical_layout();
+#if defined(STRATA_NATIVE_EXPERTS)
+        test_native_variable_layout();
+#endif
+        std::cout << "file_expert_source_test: PASS\n";
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << "file_expert_source_test: " << error.what() << '\n';
+        return 1;
+    }
+}

tests/hip/expert_cache_staging.cpp

@@ -0,0 +1,34 @@
+#include "strata/core/expert_cache.hpp"
+#include <cstdio>
+#include <vector>
+
+int main() {
+    strata::core::ExpertCache cache;
+    std::string err;
+    auto check = [&](bool ok) {
+        if (!ok) std::fprintf(stderr, "%s\n", err.c_str());
+        return ok;
+    };
+    // open_sized first allocates byte slots, then grows to the largest expert.
+    // Exercise that growth and repeated reuse with distinct pageable payloads.
+    for (int reopen = 0; reopen < 2; ++reopen) {
+        if (!check(cache.open_sized({4096, 8192, 16384}, 1, 3, err))) return 1;
+        for (int round = 0; round < 8; ++round) {
+            for (int slot = 0; slot < 3; ++slot) {
+                std::vector<uint8_t> payload((size_t) 4096 << slot);
+                for (size_t i = 0; i < payload.size(); ++i)
+                    payload[i] = (uint8_t) (i * 17 + round * 11 + slot + reopen);
+                if (!check(cache.fill_slot_blocking(slot, payload.data(), err, payload.size())) ||
+                    !check(cache.verify_slot(slot, payload.data(), err, payload.size()))) return 1;
+            }
+        }
+        if (cache.fill_slot_blocking(0, nullptr, err)) return 1;
+        cache.close();
+    }
+    if (!check(cache.open(2, 1, 2, 1024, err))) return 1;
+    std::vector<uint8_t> payload(1024, 73);
+    if (!check(cache.fill_slot_blocking(1, payload.data(), err)) ||
+        !check(cache.verify_slot(1, payload.data(), err))) return 1;
+    std::puts("HIP expert staging: variable slots, repeated payloads, close/reopen passed");
+    return 0;
+}

tests/hip/handoff.cpp

@@ -0,0 +1,122 @@
+// Real asynchronous CPU/GPU handoff and graph replay, including changing payloads.
+#include <hip/hip_runtime.h>
+#include "strata/kernels/elementwise.hpp"
+#include <atomic>
+#include <chrono>
+#include <cstdio>
+#include <thread>
+#include <vector>
+#define CHECK(x) do {auto e=(x);if(e!=hipSuccess){std::fprintf(stderr,"%s: %s\n",#x,hipGetErrorString(e));return 2;}} while(0)
+int main(){
+ constexpr int N=4096, rounds=100;
+ float *host=nullptr,*mapped=nullptr,*device=nullptr;
+ uint32_t *flag=nullptr,*seq=nullptr,*dflag=nullptr,*dseq=nullptr;
+ CHECK(hipHostMalloc((void**)&host,N*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&flag,4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&seq,4,hipHostMallocMapped));
+ CHECK(hipHostGetDevicePointer((void**)&mapped,host,0));
+ CHECK(hipHostGetDevicePointer((void**)&dflag,flag,0));
+ CHECK(hipHostGetDevicePointer((void**)&dseq,seq,0));
+ CHECK(hipMalloc((void**)&device,N*4));
+ hipStream_t stream;CHECK(hipStreamCreateWithFlags(&stream,hipStreamNonBlocking));
+ hipGraph_t graph;hipGraphExec_t exec;
+ *flag=0;*seq=1;
+ CHECK(hipStreamBeginCapture(stream,hipStreamCaptureModeThreadLocal));
+ strata::kernels::doorbell_wait(dflag,dseq,(void*)stream);
+ strata::kernels::copy_from_mapped(device,mapped,N,(void*)stream);
+ strata::kernels::scale_inplace(device,N,2.0f,(void*)stream);
+ CHECK(hipStreamEndCapture(stream,&graph));CHECK(hipGraphInstantiateWithFlags(&exec,graph,0));
+ std::vector<float> got(N);
+ for(int r=1;r<=rounds;r++){
+  *seq=r;
+  CHECK(hipGraphLaunch(exec,stream));
+  // Producer deliberately arrives after the GPU consumer; a missing wait must fail.
+  std::this_thread::sleep_for(std::chrono::microseconds(50+(r%17)*10));
+  for(int i=0;i<N;i++)host[i]=(float)(r*10000+i);
+  __atomic_store_n(flag,(uint32_t)r,__ATOMIC_RELEASE);
+  CHECK(hipStreamSynchronize(stream));
+  CHECK(hipMemcpy(got.data(),device,N*4,hipMemcpyDeviceToHost));
+  for(int i=0;i<N;i++)if(got[i]!=(float)(r*10000+i)*2.0f){std::fprintf(stderr,"stale payload r=%d i=%d got=%f\n",r,i,got[i]);return 1;}
+ }
+ CHECK(hipGraphExecDestroy(exec));CHECK(hipGraphDestroy(graph));
+ // The production `moe_route` fallback records three device-to-device copies into
+ // mapped pinned buffers, then launches `doorbell_ring` as a separate graph node.
+ // Prove that seeing the increment publishes the preceding payload to the CPU.
+ float *copy_out=nullptr,*mapped_copy_out=nullptr,*copy_weights=nullptr,*mapped_copy_weights=nullptr;
+ int32_t *copy_ids=nullptr,*mapped_copy_ids=nullptr,*copy_ids_device=nullptr;
+ float *copy_weights_device=nullptr;
+ uint32_t *copy_seq=nullptr,*device_copy_seq=nullptr;
+ CHECK(hipHostMalloc((void**)&copy_out,N*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&copy_ids,10*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&copy_weights,10*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&copy_seq,4,hipHostMallocMapped));
+ CHECK(hipHostGetDevicePointer((void**)&mapped_copy_out,copy_out,0));
+ CHECK(hipHostGetDevicePointer((void**)&mapped_copy_ids,copy_ids,0));
+ CHECK(hipHostGetDevicePointer((void**)&mapped_copy_weights,copy_weights,0));
+ CHECK(hipHostGetDevicePointer((void**)&device_copy_seq,copy_seq,0));
+ CHECK(hipMalloc((void**)&copy_ids_device,10*4));
+ CHECK(hipMalloc((void**)&copy_weights_device,10*4));
+ *copy_seq=0;
+ std::vector<int32_t> copy_route_ids(10);std::vector<float> copy_route_weights(10);
+ CHECK(hipStreamBeginCapture(stream,hipStreamCaptureModeThreadLocal));
+ CHECK(hipMemcpyAsync(mapped_copy_out,device,N*4,hipMemcpyDeviceToDevice,stream));
+ CHECK(hipMemcpyAsync(mapped_copy_ids,copy_ids_device,10*4,hipMemcpyDeviceToDevice,stream));
+ CHECK(hipMemcpyAsync(mapped_copy_weights,copy_weights_device,10*4,hipMemcpyDeviceToDevice,stream));
+ strata::kernels::doorbell_ring(device_copy_seq,(void*)stream);
+ CHECK(hipStreamEndCapture(stream,&graph));CHECK(hipGraphInstantiateWithFlags(&exec,graph,0));
+ for(int r=1;r<=rounds;r++){
+  for(int i=0;i<N;i++)got[i]=(float)(r*20000+i);
+  for(int i=0;i<10;i++){copy_route_ids[i]=r*10+i;copy_route_weights[i]=r+i*.125f;}
+  CHECK(hipMemcpyAsync(device,got.data(),N*4,hipMemcpyHostToDevice,stream));
+  CHECK(hipMemcpyAsync(copy_ids_device,copy_route_ids.data(),10*4,hipMemcpyHostToDevice,stream));
+  CHECK(hipMemcpyAsync(copy_weights_device,copy_route_weights.data(),10*4,hipMemcpyHostToDevice,stream));
+  CHECK(hipGraphLaunch(exec,stream));
+  const auto deadline=std::chrono::steady_clock::now()+std::chrono::seconds(2);
+  // Do not query or synchronize the stream before validating the mapped payload.
+  while(__atomic_load_n(copy_seq,__ATOMIC_ACQUIRE)!=(uint32_t)r){
+   if(std::chrono::steady_clock::now()>deadline){std::fprintf(stderr,"separate copy/ring timeout\n");return 10;}
+   std::this_thread::yield();
+  }
+  for(int i=0;i<N;i++)if(copy_out[i]!=(float)(r*20000+i)){std::fprintf(stderr,"stale separate-copy payload r=%d i=%d\n",r,i);return 11;}
+  for(int i=0;i<10;i++)if(copy_ids[i]!=copy_route_ids[i]||copy_weights[i]!=copy_route_weights[i]){std::fprintf(stderr,"stale separate-copy route payload r=%d i=%d\n",r,i);return 12;}
+ }
+ CHECK(hipStreamSynchronize(stream));
+ CHECK(hipGraphExecDestroy(exec));CHECK(hipGraphDestroy(graph));
+ float *published=nullptr,*dpublished=nullptr,*weights=nullptr,*dweights=nullptr,*gw=nullptr;
+ int32_t *ids=nullptr,*dids=nullptr,*gi=nullptr;
+ CHECK(hipHostMalloc((void**)&published,N*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&weights,40,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&ids,40,hipHostMallocMapped));
+ CHECK(hipHostGetDevicePointer((void**)&dpublished,published,0));
+ CHECK(hipHostGetDevicePointer((void**)&dweights,weights,0));
+ CHECK(hipHostGetDevicePointer((void**)&dids,ids,0));
+ CHECK(hipMalloc((void**)&gw,40));CHECK(hipMalloc((void**)&gi,40));
+ std::vector<int32_t> ids_src(10);std::vector<float> ws(10);
+ *flag=0;
+ CHECK(hipStreamBeginCapture(stream,hipStreamCaptureModeThreadLocal));
+ strata::kernels::doorbell_publish(device,gi,gw,N,10,dpublished,dids,dweights,dflag,(void*)stream);
+ CHECK(hipStreamEndCapture(stream,&graph));CHECK(hipGraphInstantiateWithFlags(&exec,graph,0));
+ for(int r=1;r<=rounds;r++){
+  for(int i=0;i<N;i++)got[i]=(float)(r*10000+i);
+  for(int i=0;i<10;i++){ids_src[i]=r*10+i;ws[i]=r+i*.125f;}
+  CHECK(hipMemcpyAsync(device,got.data(),N*4,hipMemcpyHostToDevice,stream));
+  CHECK(hipMemcpyAsync(gi,ids_src.data(),40,hipMemcpyHostToDevice,stream));
+  CHECK(hipMemcpyAsync(gw,ws.data(),40,hipMemcpyHostToDevice,stream));
+  CHECK(hipGraphLaunch(exec,stream));
+  const auto deadline=std::chrono::steady_clock::now()+std::chrono::seconds(2);
+  // No driver query or stream sync may make the payload visible to this observer.
+  while(__atomic_load_n(flag,__ATOMIC_ACQUIRE)!=(uint32_t)r){
+   if(std::chrono::steady_clock::now()>deadline){std::fprintf(stderr,"publish timeout\n");return 7;}
+   std::this_thread::yield();
+  }
+  for(int i=0;i<N;i++)if(published[i]!=got[i]){std::fprintf(stderr,"stale GPU publication\n");return 8;}
+  for(int i=0;i<10;i++)if(ids[i]!=ids_src[i] || weights[i]!=ws[i])return 9;
+  CHECK(hipStreamSynchronize(stream));
+ }
+ CHECK(hipGraphExecDestroy(exec));CHECK(hipGraphDestroy(graph));CHECK(hipStreamDestroy(stream));
+ CHECK(hipFree(gi));CHECK(hipFree(gw));CHECK(hipHostFree(published));CHECK(hipHostFree(ids));CHECK(hipHostFree(weights));
+ CHECK(hipFree(device));CHECK(hipHostFree(host));CHECK(hipHostFree(flag));CHECK(hipHostFree(seq));
+ CHECK(hipFree(copy_weights_device));CHECK(hipFree(copy_ids_device));
+ CHECK(hipHostFree(copy_out));CHECK(hipHostFree(copy_ids));CHECK(hipHostFree(copy_weights));CHECK(hipHostFree(copy_seq));
+ std::puts("PASS asynchronous mapped-memory handoff: 100 CPU-to-GPU waits, 100 separate copy/rings, 100 fused publishes; 1232800 values checked");
+}

tests/hip/intrinsics.cpp

@@ -0,0 +1,167 @@
+// Numerical parity for the CUDA intrinsics supplied by the HIP compatibility layer.
+#include <hip/hip_runtime.h>
+#include "strata/hip_compat/intrinsics.hpp"
+
+#include <cstdint>
+#include <cstdio>
+#include <limits>
+
+#define CHECK(call)                                                                                                  \
+    do {                                                                                                             \
+        const hipError_t error = (call);                                                                             \
+        if (error != hipSuccess) {                                                                                   \
+            std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error));                                     \
+            return 2;                                                                                                \
+        }                                                                                                            \
+    } while (0)
+
+namespace {
+
+constexpr int kLanes = 32;
+constexpr int kIntegerResults = 7;
+
+__global__ void intrinsic_probe(int* integer_results, float* float_shuffle, double* double_shuffle,
+                                unsigned* ballot, const uint32_t* dot_inputs) {
+    const int lane = static_cast<int>(threadIdx.x);
+    constexpr unsigned mask = 0xffffffffu;
+    const uint32_t a = 0x80ff017fu;
+    const uint32_t b = 0x0203fe81u;
+
+    integer_results[lane * kIntegerResults + 0] = __dp4a(static_cast<int>(a), static_cast<int>(b), 7);
+    integer_results[lane * kIntegerResults + 1] = __dp4a(0x7f7f7f7f, 0x7f7f7f7f, 0x7fffffff);
+    integer_results[lane * kIntegerResults + 2] = __vsub4(0x00ff0102, 0x01010103);
+    integer_results[lane * kIntegerResults + 3] = __vsub4(0x00000000, 0x01020304);
+    integer_results[lane * kIntegerResults + 4] = __vsubss4(0x807f0080, 0x017f0180);
+    integer_results[lane * kIntegerResults + 5] = __vcmpne4(0x00ff0102, 0x01010103);
+    integer_results[lane * kIntegerResults + 6] =
+        __dp4a(static_cast<int>(dot_inputs[lane * 2]), static_cast<int>(dot_inputs[lane * 2 + 1]),
+               0x7fffffff - lane);
+
+    float_shuffle[lane * 4 + 0] = __shfl_xor_sync(mask, static_cast<float>(lane), 1);
+    float_shuffle[lane * 4 + 1] = static_cast<float>(__shfl_down_sync(mask, lane, 4, 8));
+    float_shuffle[lane * 4 + 2] = static_cast<float>(__shfl_up_sync(mask, lane, 1));
+    float_shuffle[lane * 4 + 3] = static_cast<float>(__shfl_sync(mask, lane, 0));
+    double_shuffle[lane] = __shfl_xor_sync(mask, static_cast<double>(lane) + 0.25, 1);
+
+    const unsigned hit = __ballot_sync(mask, (lane & 1) == 0);
+    if (lane == 0) {
+        *ballot = hit;
+        __nanosleep(100);
+    }
+}
+
+int signed_byte(uint32_t word, int lane) {
+    const unsigned value = (word >> (lane * 8)) & 0xffu;
+    return value < 0x80u ? static_cast<int>(value) : static_cast<int>(value) - 0x100;
+}
+
+int dp4a_reference(uint32_t a, uint32_t b, int c) {
+    int64_t sum = c;
+    for (int lane = 0; lane < 4; ++lane) sum += signed_byte(a, lane) * signed_byte(b, lane);
+    return static_cast<int32_t>(static_cast<uint32_t>(sum));
+}
+
+uint32_t byte_sub_reference(uint32_t a, uint32_t b) {
+    uint32_t out = 0;
+    for (int lane = 0; lane < 4; ++lane)
+        out |= ((((a >> (lane * 8)) & 0xffu) - ((b >> (lane * 8)) & 0xffu)) & 0xffu) << (lane * 8);
+    return out;
+}
+
+uint32_t signed_saturating_sub_reference(uint32_t a, uint32_t b) {
+    uint32_t out = 0;
+    for (int lane = 0; lane < 4; ++lane) {
+        int value = signed_byte(a, lane) - signed_byte(b, lane);
+        if (value < -128) value = -128;
+        if (value > 127) value = 127;
+        out |= (static_cast<uint32_t>(value) & 0xffu) << (lane * 8);
+    }
+    return out;
+}
+
+uint32_t not_equal_reference(uint32_t a, uint32_t b) {
+    uint32_t out = 0;
+    for (int lane = 0; lane < 4; ++lane)
+        if (((a >> (lane * 8)) & 0xffu) != ((b >> (lane * 8)) & 0xffu)) out |= 0xffu << (lane * 8);
+    return out;
+}
+
+}  // namespace
+
+int main() {
+    int device = 0;
+    CHECK(hipGetDevice(&device));
+    hipDeviceProp_t properties{};
+    CHECK(hipGetDeviceProperties(&properties, device));
+    if (properties.warpSize != kLanes) {
+        std::fprintf(stderr, "expected wave32, got wave%d\n", properties.warpSize);
+        return 1;
+    }
+
+    int* device_integers = nullptr;
+    float* device_float_shuffle = nullptr;
+    double* device_double_shuffle = nullptr;
+    unsigned* device_ballot = nullptr;
+    uint32_t* device_dot_inputs = nullptr;
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_integers), kLanes * kIntegerResults * sizeof(int)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_float_shuffle), kLanes * 4 * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_double_shuffle), kLanes * sizeof(double)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_ballot), sizeof(unsigned)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_dot_inputs), kLanes * 2 * sizeof(uint32_t)));
+
+    uint32_t dot_inputs[kLanes * 2]{};
+    for (int lane = 0; lane < kLanes; ++lane) {
+        dot_inputs[lane * 2] = 0x80ff017fu ^ (0x01010101u * static_cast<uint32_t>(lane));
+        dot_inputs[lane * 2 + 1] = 0x0203fe81u + (0x11111111u * static_cast<uint32_t>(lane));
+    }
+    CHECK(hipMemcpy(device_dot_inputs, dot_inputs, sizeof(dot_inputs), hipMemcpyHostToDevice));
+
+    hipLaunchKernelGGL(intrinsic_probe, dim3(1), dim3(kLanes), 0, 0,
+                       device_integers, device_float_shuffle, device_double_shuffle, device_ballot, device_dot_inputs);
+    CHECK(hipDeviceSynchronize());
+
+    int integers[kLanes * kIntegerResults]{};
+    float float_shuffle[kLanes * 4]{};
+    double double_shuffle[kLanes]{};
+    unsigned ballot = 0;
+    CHECK(hipMemcpy(integers, device_integers, sizeof(integers), hipMemcpyDeviceToHost));
+    CHECK(hipMemcpy(float_shuffle, device_float_shuffle, sizeof(float_shuffle), hipMemcpyDeviceToHost));
+    CHECK(hipMemcpy(double_shuffle, device_double_shuffle, sizeof(double_shuffle), hipMemcpyDeviceToHost));
+    CHECK(hipMemcpy(&ballot, device_ballot, sizeof(ballot), hipMemcpyDeviceToHost));
+
+    constexpr uint32_t a = 0x80ff017fu, b = 0x0203fe81u;
+    const uint32_t expected_integer[kIntegerResults - 1] = {
+        static_cast<uint32_t>(dp4a_reference(a, b, 7)),
+        static_cast<uint32_t>(dp4a_reference(0x7f7f7f7fu, 0x7f7f7f7fu, std::numeric_limits<int>::max())),
+        byte_sub_reference(0x00ff0102u, 0x01010103u),
+        byte_sub_reference(0x00000000u, 0x01020304u),
+        signed_saturating_sub_reference(0x807f0080u, 0x017f0180u),
+        not_equal_reference(0x00ff0102u, 0x01010103u),
+    };
+    bool ok = ballot == 0x55555555u;
+    for (int lane = 0; lane < kLanes; ++lane) {
+        for (int i = 0; i < kIntegerResults - 1; ++i)
+            ok = ok && static_cast<uint32_t>(integers[lane * kIntegerResults + i]) == expected_integer[i];
+        ok = ok && static_cast<uint32_t>(integers[lane * kIntegerResults + kIntegerResults - 1]) ==
+                     static_cast<uint32_t>(dp4a_reference(dot_inputs[lane * 2], dot_inputs[lane * 2 + 1],
+                                                          0x7fffffff - lane));
+        ok = ok && float_shuffle[lane * 4 + 0] == static_cast<float>(lane ^ 1);
+        const int down = lane % 8 < 4 ? lane + 4 : lane;
+        ok = ok && float_shuffle[lane * 4 + 1] == static_cast<float>(down);
+        ok = ok && float_shuffle[lane * 4 + 2] == static_cast<float>(lane == 0 ? 0 : lane - 1);
+        ok = ok && float_shuffle[lane * 4 + 3] == 0.0f;
+        ok = ok && double_shuffle[lane] == static_cast<double>(lane ^ 1) + 0.25;
+    }
+
+    CHECK(hipFree(device_ballot));
+    CHECK(hipFree(device_dot_inputs));
+    CHECK(hipFree(device_double_shuffle));
+    CHECK(hipFree(device_float_shuffle));
+    CHECK(hipFree(device_integers));
+    if (!ok) {
+        std::fprintf(stderr, "HIP intrinsic parity failed (ballot 0x%08x)\n", ballot);
+        return 1;
+    }
+    std::puts("HIP intrinsics parity OK: dynamic signed dot4/overflow, packed integer boundaries, wave32 shuffles, ballot and sleep");
+    return 0;
+}

tests/hip/native_qsa_score.cpp

@@ -0,0 +1,95 @@
+// Device parity for the scalar HIP implementation of the optional native QSA scorer.
+#include <hip/hip_runtime.h>
+#include "strata/kernels/native_qsa_score.hpp"
+
+#include <cmath>
+#include <cstdio>
+#include <vector>
+
+#define CHECK(call)                                                                                                  \
+    do {                                                                                                             \
+        const hipError_t error = (call);                                                                             \
+        if (error != hipSuccess) {                                                                                   \
+            std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error));                                     \
+            return 2;                                                                                                \
+        }                                                                                                            \
+    } while (0)
+
+int main() {
+    using namespace strata::kernels;
+    constexpr int kDim = 128, kHeads = 4, kBlock = 4;
+    constexpr int kMaxCells = 8, kMaxBlocks = 3, kCells = 7;
+    const QsaShapes shapes = qsa_real_shapes();
+
+    std::vector<float> pooled(kMaxBlocks * kDim), query(kHeads * kDim), bias(kMaxBlocks),
+        scores(kMaxCells, -123.0f), expected(kMaxCells, -123.0f);
+    const float head_sign[kHeads] = {1.0f, -1.0f, 1.0f, -1.0f};
+    for (int row = 0; row < kMaxBlocks; ++row)
+        for (int d = 0; d < kDim; ++d)
+            pooled[row * kDim + d] = static_cast<float>(row + 1) * (1.0f + static_cast<float>(d % 7) / 8.0f) / 128.0f;
+    for (int head = 0; head < kHeads; ++head)
+        for (int d = 0; d < kDim; ++d)
+            query[head * kDim + d] = head_sign[head] * (1.0f + static_cast<float>((head + d) % 5) / 16.0f) / 16.0f;
+    for (int row = 0; row < kMaxBlocks; ++row) bias[row] = (static_cast<float>(row) - 1.0f) / 8.0f;
+
+    int32_t step[kStepCount] = {kCells - 1, kCells, kCells / kBlock, static_cast<int32_t>(kCells)};
+    bool saw_positive_head = false, saw_negative_head = false;
+    for (int row = 0; row <= step[kStepNBid]; ++row) {
+        float sum = 0.0f;
+        for (int head = 0; head < kHeads; ++head) {
+            float dot = 0.0f;
+            for (int d = 0; d < kDim; ++d)
+                dot = std::fma(pooled[row * kDim + d], query[head * kDim + d], dot);
+            saw_positive_head = saw_positive_head || dot > 0.0f;
+            saw_negative_head = saw_negative_head || dot < 0.0f;
+            sum = sum + (dot > 0.0f ? dot : 0.0f);
+        }
+        sum = sum + bias[row];
+        if (row == step[kStepNBid] && kCells % kBlock != 0) sum = sum + 1.0e9f;
+        const int lo = row * kBlock;
+        const int hi = (row + 1) * kBlock < kCells ? (row + 1) * kBlock : kCells;
+        for (int cell = lo; cell < hi; ++cell) expected[cell] = sum;
+    }
+    if (!saw_positive_head || !saw_negative_head) {
+        std::fprintf(stderr, "fixture did not exercise both positive and negative head scores\n");
+        return 1;
+    }
+
+    float *device_pooled = nullptr, *device_query = nullptr, *device_bias = nullptr, *device_scores = nullptr;
+    int32_t* device_step = nullptr;
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_pooled), pooled.size() * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_query), query.size() * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_bias), bias.size() * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_scores), scores.size() * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_step), sizeof(step)));
+    CHECK(hipMemcpy(device_pooled, pooled.data(), pooled.size() * sizeof(float), hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(device_query, query.data(), query.size() * sizeof(float), hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(device_bias, bias.data(), bias.size() * sizeof(float), hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(device_scores, scores.data(), scores.size() * sizeof(float), hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(device_step, step, sizeof(step), hipMemcpyHostToDevice));
+
+    hipStream_t stream;
+    CHECK(hipStreamCreate(&stream));
+    native_qsa_score(device_pooled, device_query, device_bias, shapes, device_step, kMaxBlocks, kMaxCells,
+                     device_scores, static_cast<void*>(stream));
+    CHECK(hipStreamSynchronize(stream));
+    CHECK(hipMemcpy(scores.data(), device_scores, scores.size() * sizeof(float), hipMemcpyDeviceToHost));
+
+    bool ok = true;
+    for (int cell = 0; cell < kMaxCells; ++cell) {
+        if (scores[cell] != expected[cell]) {
+            std::fprintf(stderr, "score[%d] got %.9g, expected %.9g\n", cell, scores[cell], expected[cell]);
+            ok = false;
+        }
+    }
+
+    CHECK(hipStreamDestroy(stream));
+    CHECK(hipFree(device_step));
+    CHECK(hipFree(device_scores));
+    CHECK(hipFree(device_bias));
+    CHECK(hipFree(device_query));
+    CHECK(hipFree(device_pooled));
+    if (!ok) return 1;
+    std::puts("native QSA HIP scalar parity OK: FMA order, per-head ReLU, bias, partial tail and padding");
+    return 0;
+}

tests/hip/ple_iq4.cpp

@@ -0,0 +1,202 @@
+// Real PLE IQ3_XXS/IQ4_XS matrix parity: ggml CPU dequantization versus HIP native MMVQ.
+// Usage: hip_ple_iq4 <model-shard-1.gguf> [expected-type-id]
+#include <hip/hip_runtime.h>
+
+#include "ggml.h"
+#include "strata/artifact/gguf_reader.hpp"
+#include "strata/kernels/f16_bits.hpp"
+#include "strata/kernels/native_mmvq.hpp"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace {
+
+constexpr int kNIn = 2560;
+constexpr int kNOut = 10240;
+constexpr int kTypeIq3Xxs = 18;
+constexpr int kTypeIq4Xs = 23;
+constexpr int kReplayCount = 3;
+
+#define CHECK(call)                                                                                                  \
+    do {                                                                                                             \
+        const hipError_t error = (call);                                                                             \
+        if (error != hipSuccess) {                                                                                   \
+            std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error));                                    \
+            return 2;                                                                                                \
+        }                                                                                                            \
+    } while (0)
+
+struct Q8Reference {
+    std::vector<float> dequantized;
+};
+
+Q8Reference quantize_q8_1_reference(const std::vector<float>& x) {
+    Q8Reference result;
+    result.dequantized.resize(x.size());
+    for (size_t offset = 0; offset < x.size(); offset += 32) {
+        float amax = 0.0f;
+        for (int i = 0; i < 32; ++i) amax = std::fmax(amax, std::fabs(x[offset + i]));
+        if (amax == 0.0f) {
+            std::fill(result.dequantized.begin() + offset, result.dequantized.begin() + offset + 32, 0.0f);
+            continue;
+        }
+        const float scale_f32 = amax / 127.0f;
+        const float scale_f16 = strata::kernels::f32_from_f16(strata::kernels::f16_from_f32(scale_f32));
+        for (int i = 0; i < 32; ++i) {
+            const int q = static_cast<int>(std::round(x[offset + i] / scale_f32));
+            result.dequantized[offset + i] = static_cast<float>(static_cast<int8_t>(q)) * scale_f16;
+        }
+    }
+    return result;
+}
+
+struct Parity {
+    double rel_l1_q8 = 0.0;
+    double worst_terms_q8 = 0.0;
+    double rel_l1_f32 = 0.0;
+    double max_abs_q8 = 0.0;
+    bool finite = true;
+};
+
+Parity compare(const uint8_t* weights, size_t row_bytes, const ggml_type_traits* traits,
+               const std::vector<float>& x, const std::vector<float>& x_q8,
+               const std::vector<float>& actual) {
+    std::vector<float> row(kNIn);
+    double q8_error = 0.0, q8_magnitude = 0.0;
+    double f32_error = 0.0, f32_magnitude = 0.0;
+    Parity result;
+    for (int r = 0; r < kNOut; ++r) {
+        traits->to_float(weights + static_cast<size_t>(r) * row_bytes, row.data(), kNIn);
+        double expected_q8 = 0.0, terms_q8 = 0.0, expected_f32 = 0.0;
+        for (int i = 0; i < kNIn; ++i) {
+            const double term_q8 = static_cast<double>(row[i]) * x_q8[i];
+            const double term_f32 = static_cast<double>(row[i]) * x[i];
+            expected_q8 += term_q8;
+            terms_q8 += std::fabs(term_q8);
+            expected_f32 += term_f32;
+        }
+        const double got = actual[r];
+        if (!std::isfinite(got) || !std::isfinite(expected_q8) || !std::isfinite(expected_f32)) result.finite = false;
+        q8_error += std::fabs(got - expected_q8);
+        q8_magnitude += std::fabs(expected_q8);
+        f32_error += std::fabs(got - expected_f32);
+        f32_magnitude += std::fabs(expected_f32);
+        result.max_abs_q8 = std::fmax(result.max_abs_q8, std::fabs(got - expected_q8));
+        result.worst_terms_q8 = std::fmax(result.worst_terms_q8,
+                                         std::fabs(got - expected_q8) / std::fmax(terms_q8, 1e-30));
+    }
+    result.rel_l1_q8 = q8_error / std::fmax(q8_magnitude, 1e-30);
+    result.rel_l1_f32 = f32_error / std::fmax(f32_magnitude, 1e-30);
+    return result;
+}
+
+std::vector<float> make_input(uint32_t seed) {
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> normal(0.0f, 0.6f);
+    std::vector<float> x(kNIn);
+    for (int i = 0; i < kNIn; ++i)
+        x[i] = normal(rng) + 0.1f * std::sin(static_cast<float>(i) * 0.013f);
+    return x;
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    if (argc != 2 && argc != 3) {
+        std::fprintf(stderr, "usage: hip_ple_iq4 <model-shard-1.gguf> [expected-type-id]\n");
+        return 2;
+    }
+
+    try {
+        strata::GgufFile gguf(argv[1]);
+        const strata::TensorInfo* tensor = gguf.find("blk.1.ple_key.weight");
+        if (!tensor || (tensor->type != kTypeIq3Xxs && tensor->type != kTypeIq4Xs) || tensor->shape.size() != 2 ||
+            tensor->shape[0] != kNIn || tensor->shape[1] != kNOut) {
+            std::fprintf(stderr, "expected blk.1.ple_key.weight IQ3_XXS or IQ4_XS [2560,10240] in %s\n", argv[1]);
+            return 1;
+        }
+        if (argc == 3 && std::atoi(argv[2]) != static_cast<int>(tensor->type)) {
+            std::fprintf(stderr, "requested GGML type %s but blk.1.ple_key.weight is type %u (%s)\n",
+                         argv[2], tensor->type, tensor->type_name());
+            return 1;
+        }
+        const auto* traits = ggml_get_type_traits(static_cast<ggml_type>(tensor->type));
+        if (!traits || !traits->to_float) {
+            std::fprintf(stderr, "ggml has no %s CPU dequantizer\n", tensor->type_name());
+            return 1;
+        }
+        const uint8_t* host_weights = gguf.tensor_data(*tensor);
+        const size_t weight_bytes = strata::kernels::native_mmvq_weight_bytes(static_cast<int>(tensor->type), kNIn, kNOut);
+        const size_t row_bytes = weight_bytes / kNOut;
+        const size_t expected_bytes_per_block = tensor->type == kTypeIq3Xxs ? 98 : 136;
+        if (weight_bytes != static_cast<size_t>(tensor->elements()) / 256 * expected_bytes_per_block ||
+            row_bytes != static_cast<size_t>(kNIn / 256) * expected_bytes_per_block) {
+            std::fprintf(stderr, "unexpected %s storage geometry: %zu bytes, row %zu\n",
+                         tensor->type_name(), weight_bytes, row_bytes);
+            return 1;
+        }
+
+        uint8_t* device_weights = nullptr;
+        float* device_x = nullptr;
+        float* device_y = nullptr;
+        void* device_q8_1 = nullptr;
+        CHECK(hipMalloc(reinterpret_cast<void**>(&device_weights), weight_bytes));
+        CHECK(hipMalloc(reinterpret_cast<void**>(&device_x), kNIn * sizeof(float)));
+        CHECK(hipMalloc(reinterpret_cast<void**>(&device_y), kNOut * sizeof(float)));
+        CHECK(hipMalloc(&device_q8_1, strata::kernels::native_q8_1_bytes(kNIn)));
+        CHECK(hipMemcpy(device_weights, host_weights, weight_bytes, hipMemcpyHostToDevice));
+
+        hipStream_t stream = nullptr;
+        CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking));
+        CHECK(hipStreamBeginCapture(stream, hipStreamCaptureModeThreadLocal));
+        strata::kernels::native_quantize_q8_1(device_x, device_q8_1, kNIn, 1, static_cast<void*>(stream));
+        strata::kernels::native_mmvq(static_cast<int>(tensor->type), device_weights, device_q8_1, device_y,
+                                     kNIn, kNOut, 1, static_cast<void*>(stream));
+        hipGraph_t graph = nullptr;
+        CHECK(hipStreamEndCapture(stream, &graph));
+        hipGraphExec_t graph_exec = nullptr;
+        CHECK(hipGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0));
+
+        std::vector<float> actual(kNOut);
+        bool ok = true;
+        for (int replay = 0; replay < kReplayCount; ++replay) {
+            const std::vector<float> x = make_input(0x1572u + static_cast<uint32_t>(replay));
+            const Q8Reference q8 = quantize_q8_1_reference(x);
+            CHECK(hipMemcpyAsync(device_x, x.data(), x.size() * sizeof(float), hipMemcpyHostToDevice, stream));
+            CHECK(hipGraphLaunch(graph_exec, stream));
+            CHECK(hipMemcpyAsync(actual.data(), device_y, actual.size() * sizeof(float), hipMemcpyDeviceToHost, stream));
+            CHECK(hipStreamSynchronize(stream));
+
+            const Parity parity = compare(host_weights, row_bytes, traits, x, q8.dequantized, actual);
+            const bool pass = parity.finite && parity.rel_l1_q8 <= 5e-5 && parity.worst_terms_q8 <= 1e-4 &&
+                              parity.rel_l1_f32 <= 3e-2;
+            std::printf("%s PLE graph replay %d/%d: q8-ref rel-L1 %.3e, worst terms %.3e, "
+                        "original-f32 rel-L1 %.3e, max q8 abs %.3e %s\n",
+                        tensor->type_name(), replay + 1, kReplayCount, parity.rel_l1_q8, parity.worst_terms_q8,
+                        parity.rel_l1_f32, parity.max_abs_q8, pass ? "PASS" : "FAIL");
+            ok = ok && pass;
+        }
+
+        CHECK(hipGraphExecDestroy(graph_exec));
+        CHECK(hipGraphDestroy(graph));
+        CHECK(hipStreamDestroy(stream));
+        CHECK(hipFree(device_q8_1));
+        CHECK(hipFree(device_y));
+        CHECK(hipFree(device_x));
+        CHECK(hipFree(device_weights));
+        if (!ok) return 1;
+        std::printf("PLE %s native MMVQ parity OK: real GGUF matrix, CPU dequant reference, %d graph replays\n",
+                    tensor->type_name(), kReplayCount);
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "hip_ple_iq4: %s\n", error.what());
+        return 1;
+    }
+}

tests/hip/prefill_hipblaslt_gemm.cpp

@@ -0,0 +1,188 @@
+// Opt-in HIP test for the calibrated hipBLASLt prefill route. Requires a matching tuning table.
+#include <cuda_runtime.h>
+#include <hip/hip_bfloat16.h>
+#include <hip/hip_fp16.h>
+#include <hipblas/hipblas.h>
+#include <hipblaslt/hipblaslt.h>
+
+#include "strata/prefill/gemm.hpp"
+#include "hipblaslt_tuning.hpp"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <string>
+#include <vector>
+
+#define HIP_CHECK(call) do { \
+    const hipError_t status_ = (call); \
+    if (status_ != hipSuccess) { \
+        std::fprintf(stderr, "HIP failure at %s:%d: %s: %s\n", __FILE__, __LINE__, #call, hipGetErrorString(status_)); \
+        std::exit(2); \
+    } \
+} while (0)
+
+#define HIPBLAS_CHECK(call) do { \
+    const hipblasStatus_t status_ = (call); \
+    if (status_ != HIPBLAS_STATUS_SUCCESS) { \
+        std::fprintf(stderr, "hipBLAS failure at %s:%d: %s: %d\n", __FILE__, __LINE__, #call, (int) status_); \
+        std::exit(2); \
+    } \
+} while (0)
+
+struct DeviceBuffer {
+    void* p = nullptr;
+    explicit DeviceBuffer(size_t bytes) { if (bytes) HIP_CHECK(hipMalloc(&p, bytes)); }
+    ~DeviceBuffer() { if (p) (void) hipFree(p); }
+    DeviceBuffer(const DeviceBuffer&) = delete;
+    DeviceBuffer& operator=(const DeviceBuffer&) = delete;
+};
+
+uint16_t encode(float value, bool bf16) {
+    if (bf16) return hip_bfloat16(value).data;
+    const __half half = __float2half_rn(value);
+    uint16_t bits = 0;
+    std::memcpy(&bits, &half, sizeof(bits));
+    return bits;
+}
+
+bool run_case(strata::prefill::Gemm& gemm, hipblasHandle_t blas, hipStream_t stream, bool bf16, int t, int n,
+              int k, int ldy, int output_offset, float beta, uint32_t seed) {
+    std::mt19937 rng(seed);
+    std::uniform_real_distribution<float> dist(-0.25f, 0.25f);
+    const size_t x_count = (size_t) t * k;
+    const size_t w_count = (size_t) n * k;
+    std::vector<uint16_t> x(x_count), w(w_count);
+    for (auto& value : x) value = encode(dist(rng), bf16);
+    for (auto& value : w) value = encode(dist(rng), bf16);
+
+    const size_t output_count = (size_t) output_offset + (size_t) t * ldy + 8;
+    std::vector<float> initial(output_count, -777.25f);
+    for (int row = 0; row < t; ++row) {
+        for (int col = 0; col < n; ++col) {
+            initial[(size_t) output_offset + (size_t) row * ldy + col] =
+                (float) ((row * 17 + col * 3) % 29 - 14) * 0.03125f;
+        }
+    }
+
+    DeviceBuffer dx(x_count * sizeof(uint16_t));
+    DeviceBuffer dw(w_count * sizeof(uint16_t));
+    DeviceBuffer dy(initial.size() * sizeof(float));
+    DeviceBuffer dref(initial.size() * sizeof(float));
+    HIP_CHECK(hipMemcpyAsync(dx.p, x.data(), x_count * sizeof(uint16_t), hipMemcpyHostToDevice, stream));
+    HIP_CHECK(hipMemcpyAsync(dw.p, w.data(), w_count * sizeof(uint16_t), hipMemcpyHostToDevice, stream));
+    HIP_CHECK(hipMemcpyAsync(dy.p, initial.data(), initial.size() * sizeof(float), hipMemcpyHostToDevice, stream));
+    HIP_CHECK(hipMemcpyAsync(dref.p, initial.data(), initial.size() * sizeof(float), hipMemcpyHostToDevice, stream));
+    HIP_CHECK(hipStreamSynchronize(stream));
+
+    const hipDataType type = bf16 ? HIP_R_16BF : HIP_R_16F;
+    const float alpha = 1.0f;
+    HIPBLAS_CHECK(hipblasGemmEx(blas, HIPBLAS_OP_T, HIPBLAS_OP_N, n, t, k, &alpha, dw.p, type, k, dx.p, type, k,
+                                &beta, (float*) dref.p + output_offset, HIP_R_32F, ldy, HIPBLAS_COMPUTE_32F,
+                                HIPBLAS_GEMM_DEFAULT));
+    if (bf16) {
+        gemm.bf16((const uint16_t*) dx.p, (const uint16_t*) dw.p, (float*) dy.p + output_offset, t, n, k, ldy,
+                  beta);
+    } else {
+        gemm.f16((const uint16_t*) dx.p, (const uint16_t*) dw.p, (float*) dy.p + output_offset, t, n, k, ldy,
+                 beta);
+    }
+    HIP_CHECK(hipStreamSynchronize(stream));
+
+    std::vector<float> got(initial.size()), ref(initial.size());
+    HIP_CHECK(hipMemcpy(got.data(), dy.p, got.size() * sizeof(float), hipMemcpyDeviceToHost));
+    HIP_CHECK(hipMemcpy(ref.data(), dref.p, ref.size() * sizeof(float), hipMemcpyDeviceToHost));
+
+    std::vector<uint8_t> active(initial.size(), 0);
+    double diff2 = 0.0, ref2 = 0.0, max_abs = 0.0;
+    for (int row = 0; row < t; ++row) {
+        for (int col = 0; col < n; ++col) {
+            const size_t index = (size_t) output_offset + (size_t) row * ldy + col;
+            active[index] = 1;
+            if (!std::isfinite(got[index]) || !std::isfinite(ref[index])) {
+                std::fprintf(stderr, "non-finite output dtype=%s T=%d N=%d K=%d beta=%.1f index=%zu\n",
+                             bf16 ? "bf16" : "f16", t, n, k, beta, index);
+                return false;
+            }
+            const double delta = (double) got[index] - ref[index];
+            diff2 += delta * delta;
+            ref2 += (double) ref[index] * ref[index];
+            max_abs = std::max(max_abs, std::abs(delta));
+        }
+    }
+    for (size_t i = 0; i < initial.size(); ++i) {
+        if (!active[i] && (got[i] != initial[i] || ref[i] != initial[i])) {
+            std::fprintf(stderr, "output touched padding/guard dtype=%s T=%d N=%d K=%d beta=%.1f index=%zu got=%.9g\n",
+                         bf16 ? "bf16" : "f16", t, n, k, beta, i, got[i]);
+            return false;
+        }
+    }
+    const double relative_l2 = std::sqrt(diff2 / std::max(ref2, 1e-300));
+    const bool passed = relative_l2 <= 1e-4 && max_abs <= 5e-3;
+    std::printf("case dtype=%s T=%d N=%d K=%d ldy=%d output_offset=%d beta=%.1f relative_l2=%.9g max_abs=%.9g %s\n",
+                bf16 ? "bf16" : "f16", t, n, k, ldy, output_offset, beta, relative_l2, max_abs,
+                passed ? "PASS" : "FAIL");
+    return passed;
+}
+
+int main() {
+    const char* tuning_path = std::getenv("STRATA_HIPBLASLT_TUNING");
+    if (!tuning_path || !*tuning_path) {
+        std::fprintf(stderr, "SKIP: set STRATA_HIPBLASLT_TUNING to a calibrated matching table\n");
+        return 77;
+    }
+
+    int device = 0;
+    HIP_CHECK(hipGetDevice(&device));
+    hipDeviceProp_t properties{};
+    HIP_CHECK(hipGetDeviceProperties(&properties, device));
+    std::string arch(properties.gcnArchName);
+    const auto suffix = arch.find(':');
+    if (suffix != std::string::npos) arch.resize(suffix);
+    hipblasLtHandle_t lt = nullptr;
+    HIPBLAS_CHECK(hipblasLtCreate(&lt));
+    int version = 0;
+    HIPBLAS_CHECK(hipblasLtGetVersion(lt, &version));
+    HIPBLAS_CHECK(hipblasLtDestroy(lt));
+
+    strata::prefill::hipblaslt::TuningTable table;
+    std::string error;
+    if (!table.load(tuning_path, arch, version, error)) {
+        std::fprintf(stderr, "tuning table rejected: %s\n", error.c_str());
+        return 1;
+    }
+    if (!table.closest(strata::prefill::hipblaslt::InputType::bf16, 48, 2560, 96, 4096) ||
+        !table.closest(strata::prefill::hipblaslt::InputType::f16, 512, 2560, 512, 4096)) {
+        std::fprintf(stderr, "tuning table lacks the focused BF16/F16 test rows\n");
+        return 1;
+    }
+
+    hipStream_t stream = nullptr;
+    HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking));
+    bool ok = true;
+    {
+        std::string init_error;
+        strata::prefill::Gemm gemm;
+        if (!gemm.init((void*) stream, 0, init_error)) {
+            std::fprintf(stderr, "Gemm init failed: %s\n", init_error.c_str());
+            HIP_CHECK(hipStreamDestroy(stream));
+            return 2;
+        }
+        hipblasHandle_t blas = nullptr;
+        HIPBLAS_CHECK(hipblasCreate(&blas));
+        HIPBLAS_CHECK(hipblasSetStream(blas, stream));
+
+        // Exact calibrated chunks exercise the selected solution; T=37 reuses the closest bucket,
+        // validates the actual shape before launch, and tests a non-tile-multiple tail.
+        ok &= run_case(gemm, blas, stream, true, 4096, 48, 2560, 96, 0, 0.0f, 101);
+        ok &= run_case(gemm, blas, stream, true, 37, 48, 2560, 96, 48, 1.0f, 102);
+        ok &= run_case(gemm, blas, stream, false, 4096, 512, 2560, 512, 0, 0.0f, 103);
+        ok &= run_case(gemm, blas, stream, false, 37, 512, 2560, 512, 7, 1.0f, 104);
+        HIPBLAS_CHECK(hipblasDestroy(blas));
+    }
+    HIP_CHECK(hipStreamDestroy(stream));
+    return ok ? 0 : 1;
+}

tests/hip/prefill_mmq_parity.cpp

@@ -0,0 +1,281 @@
+// Standalone diagnostic harness for strata-prefill's optional MMQ path.
+// Keep this artifact outside the project source until it is reviewed and wired into a HIP test target.
+// It compares GGML's MMQ result (with its intended q8_1 activation rounding) to a CPU FP32 product using
+// GGML's own dequantizer for the exact same weight blocks. The tolerances screen layout/stride/bounds errors;
+// they do not assert bitwise or FP16 parity.
+#include "strata/prefill/moe_mmq.hpp"
+#include "ggml.h"
+#include <hip/hip_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <fstream>
+#include <iostream>
+#include <numeric>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace {
+using strata::prefill::mmq::Context;
+using strata::prefill::mmq::Product;
+using strata::prefill::mmq::matrix_bytes;
+using strata::prefill::mmq::q8_bytes;
+using strata::prefill::mmq::quantize;
+using strata::prefill::mmq::supported;
+
+void hip_check(hipError_t e, const char * what) {
+    if (e != hipSuccess) throw std::runtime_error(std::string(what) + ": " + hipGetErrorString(e));
+}
+
+struct DeviceBuffer {
+    void * p = nullptr;
+    explicit DeviceBuffer(size_t n) { hip_check(hipMalloc(&p, n), "hipMalloc"); }
+    ~DeviceBuffer() { if (p) hipFree(p); }
+    DeviceBuffer(const DeviceBuffer &) = delete;
+    DeviceBuffer & operator=(const DeviceBuffer &) = delete;
+    template<class T> T * as() const { return static_cast<T *>(p); }
+};
+
+size_t row_bytes(ggml_type type, int64_t cols) {
+    return ggml_row_size(type, cols);
+}
+
+std::vector<float> decode_matrix(ggml_type type, const std::vector<uint8_t> & q, int64_t rows, int64_t cols) {
+    const auto * tr = ggml_get_type_traits(type);
+    if (!tr || !tr->to_float) throw std::runtime_error("GGML type has no host dequantizer");
+    const size_t rb = row_bytes(type, cols);
+    if (q.size() != (size_t) rows * rb) throw std::runtime_error("matrix byte count mismatch");
+    std::vector<float> out((size_t) rows * (size_t) cols);
+    for (int64_t r = 0; r < rows; ++r) {
+        tr->to_float(q.data() + (size_t) r * rb, out.data() + (size_t) r * (size_t) cols, cols);
+    }
+    return out;
+}
+
+std::vector<uint8_t> make_q2_matrix(int64_t rows, int64_t cols, int expert, int trial) {
+    const ggml_type type = GGML_TYPE_Q2_0;
+    const auto * tr = ggml_get_type_traits(type);
+    if (!tr || !tr->from_float_ref) throw std::runtime_error("Q2_0 host quantizer unavailable");
+    const size_t rb = row_bytes(type, cols);
+    std::vector<uint8_t> out((size_t) rows * rb);
+    std::vector<float> row((size_t) cols);
+    for (int64_t r = 0; r < rows; ++r) {
+        for (int64_t k = 0; k < cols; ++k) {
+            const int code = (int) ((k * 37 + r * 19 + expert * 23 + trial * 11) % 101) - 50;
+            row[(size_t) k] = 0.025f * std::sin(0.013f * (float) (k + 1) + 0.17f * (float) r + 0.31f * (float) expert) +
+                              0.0015f * (float) code;
+        }
+        tr->from_float_ref(row.data(), out.data() + (size_t) r * rb, cols);
+    }
+    return out;
+}
+
+std::vector<float> make_activations(int rows, int64_t cols, int trial) {
+    std::vector<float> x((size_t) rows * (size_t) cols);
+    for (int r = 0; r < rows; ++r) {
+        for (int64_t k = 0; k < cols; ++k) {
+            x[(size_t) r * (size_t) cols + (size_t) k] =
+                0.45f * std::sin(0.009f * (float) (k + 1) + 0.23f * (float) r + 0.07f * (float) trial) +
+                0.17f * std::cos(0.021f * (float) (k + 3) - 0.19f * (float) r) +
+                0.002f * (float) (((k * 7 + r * 13 + trial * 5) % 19) - 9);
+        }
+    }
+    // Exercise a complete all-zero row; its expected output is exactly zero even though Q8_1 rounding is enabled.
+    std::fill(x.end() - cols, x.end(), 0.0f);
+    return x;
+}
+
+std::vector<int32_t> permutation(int rows, int shift, bool reverse) {
+    std::vector<int32_t> p((size_t) rows);
+    for (int i = 0; i < rows; ++i) {
+        p[(size_t) i] = reverse ? (int32_t) ((rows - 1 - i + shift) % rows)
+                                : (int32_t) ((i + shift) % rows);
+    }
+    return p;
+}
+
+std::vector<int32_t> make_bounds(const std::vector<int> & counts) {
+    std::vector<int32_t> b(counts.size() + 1, 0);
+    for (size_t i = 0; i < counts.size(); ++i) b[i + 1] = b[i] + counts[i];
+    return b;
+}
+
+struct Metrics { double rel_l2 = 0, max_over_rms = 0, zero_row_max = 0, ref_rms = 0; };
+
+Metrics run_product(Context & ctx, hipStream_t stream, const std::string & name, ggml_type type,
+                    int64_t out_rows, int64_t cols, const std::vector<std::vector<uint8_t>> & expert_weights,
+                    const std::vector<int> & counts, const std::vector<float> & x,
+                    const std::vector<int32_t> & src_ids, const std::vector<int32_t> & dst_ids) {
+    const int rows = (int) src_ids.size();
+    const int experts = (int) expert_weights.size();
+    const std::vector<int32_t> bounds = make_bounds(counts);
+    if (!supported((int) type) || experts <= 0 || counts.size() != expert_weights.size() ||
+        bounds.back() != rows || dst_ids.size() != (size_t) rows || x.size() != (size_t) rows * (size_t) cols) {
+        throw std::runtime_error(name + ": invalid test geometry");
+    }
+    const size_t expert_bytes = matrix_bytes((int) type, out_rows, cols);
+    const size_t tail = 4096; // match the production group's zero guard after its last expert
+    std::vector<uint8_t> w((size_t) experts * expert_bytes + tail, 0);
+    for (int e = 0; e < experts; ++e) {
+        if (expert_weights[(size_t) e].size() != expert_bytes) throw std::runtime_error(name + ": expert size mismatch");
+        std::copy(expert_weights[(size_t) e].begin(), expert_weights[(size_t) e].end(),
+                  w.begin() + (size_t) e * expert_bytes);
+    }
+
+    DeviceBuffer dx(x.size() * sizeof(float));
+    DeviceBuffer dsrc(src_ids.size() * sizeof(int32_t));
+    DeviceBuffer ddst(dst_ids.size() * sizeof(int32_t));
+    DeviceBuffer dbounds(bounds.size() * sizeof(int32_t));
+    DeviceBuffer dw(w.size());
+    DeviceBuffer dxq(q8_bytes(rows, cols));
+    DeviceBuffer dy((size_t) rows * (size_t) out_rows * sizeof(float));
+    hip_check(hipMemcpy(dx.p, x.data(), x.size() * sizeof(float), hipMemcpyHostToDevice), "copy x");
+    hip_check(hipMemcpy(dsrc.p, src_ids.data(), src_ids.size() * sizeof(int32_t), hipMemcpyHostToDevice), "copy src ids");
+    hip_check(hipMemcpy(ddst.p, dst_ids.data(), dst_ids.size() * sizeof(int32_t), hipMemcpyHostToDevice), "copy dst ids");
+    hip_check(hipMemcpy(dbounds.p, bounds.data(), bounds.size() * sizeof(int32_t), hipMemcpyHostToDevice), "copy bounds");
+    hip_check(hipMemcpy(dw.p, w.data(), w.size(), hipMemcpyHostToDevice), "copy weights and zero tail");
+    hip_check(hipMemset(dy.p, 0xff, (size_t) rows * (size_t) out_rows * sizeof(float)), "initialize output sentinel");
+
+    quantize(dx.as<float>(), dsrc.as<int32_t>(), dxq.p, (int) type, cols, cols, rows, (void *) stream);
+    const int max_rows = *std::max_element(counts.begin(), counts.end());
+    Product p;
+    p.w = dw.p; p.type = (int) type; p.w_rows = out_rows; p.w_cols = cols; p.expert_bytes = expert_bytes;
+    p.n = experts; p.xq = dxq.p; p.bounds = dbounds.as<int32_t>(); p.ids = ddst.as<int32_t>();
+    p.total_rows = rows; p.max_rows = max_rows; p.dst = dy.as<float>(); p.ld_dst = out_rows;
+    ctx.run(p, (void *) stream);
+    hip_check(hipGetLastError(), "MMQ launch");
+    hip_check(hipStreamSynchronize(stream), "MMQ synchronize");
+    std::vector<float> got((size_t) rows * (size_t) out_rows);
+    hip_check(hipMemcpy(got.data(), dy.p, got.size() * sizeof(float), hipMemcpyDeviceToHost), "copy output");
+
+    std::vector<float> ref(got.size(), 0.0f);
+    double err2 = 0, ref2 = 0;
+    float max_abs = 0, zero_max = 0;
+    for (int e = 0; e < experts; ++e) {
+        const std::vector<float> wd = decode_matrix(type, expert_weights[(size_t) e], out_rows, cols);
+        for (int r = bounds[(size_t) e]; r < bounds[(size_t) e + 1]; ++r) {
+            const int src = src_ids[(size_t) r];
+            const int dst = dst_ids[(size_t) r];
+            const float * xr = x.data() + (size_t) src * (size_t) cols;
+            for (int64_t o = 0; o < out_rows; ++o) {
+                double acc = 0;
+                const float * wr = wd.data() + (size_t) o * (size_t) cols;
+                for (int64_t k = 0; k < cols; ++k) acc += (double) wr[k] * xr[k];
+                ref[(size_t) dst * (size_t) out_rows + (size_t) o] = (float) acc;
+            }
+        }
+    }
+    for (size_t i = 0; i < got.size(); ++i) {
+        if (!std::isfinite(got[i])) throw std::runtime_error(name + ": non-finite or unwritten MMQ output");
+        const double d = (double) got[i] - ref[i];
+        err2 += d * d; ref2 += (double) ref[i] * ref[i]; max_abs = std::max(max_abs, (float) std::fabs(d));
+    }
+    const double rms = std::sqrt(ref2 / (double) ref.size());
+    for (int r = 0; r < rows; ++r) {
+        if (std::all_of(x.begin() + (size_t) src_ids[(size_t) r] * (size_t) cols,
+                        x.begin() + ((size_t) src_ids[(size_t) r] + 1) * (size_t) cols,
+                        [](float v) { return v == 0.0f; })) {
+            for (int64_t o = 0; o < out_rows; ++o)
+                zero_max = std::max(zero_max, std::fabs(got[(size_t) dst_ids[(size_t) r] * (size_t) out_rows + (size_t) o]));
+        }
+    }
+    Metrics m;
+    m.rel_l2 = rms > 0 ? std::sqrt(err2 / (double) ref.size()) / rms : std::sqrt(err2);
+    m.max_over_rms = rms > 0 ? max_abs / rms : max_abs;
+    m.zero_row_max = zero_max; m.ref_rms = rms;
+    std::cout << name << ": type=" << (int) type << " rows=" << rows << " experts=" << experts
+              << " ref_rms=" << m.ref_rms << " rel_l2=" << m.rel_l2
+              << " max_abs/ref_rms=" << m.max_over_rms << " zero_row_max=" << m.zero_row_max << '\n';
+    // Q8_1 rounding is expected. These deliberately broad limits catch wrong block order, row strides, ids,
+    // expert bounds, or stale output while allowing ordinary activation-quantization error.
+    if (m.rel_l2 > 0.04 || m.max_over_rms > 0.35 || m.zero_row_max > std::max(1e-5, 1e-4 * rms))
+        throw std::runtime_error(name + ": parity screen failed");
+    return m;
+}
+
+std::vector<std::vector<uint8_t>> synthetic_q2(int experts, int64_t out_rows, int64_t cols, int trial) {
+    std::vector<std::vector<uint8_t>> w;
+    for (int e = 0; e < experts; ++e) w.push_back(make_q2_matrix(out_rows, cols, e, trial));
+    return w;
+}
+
+bool run_real_iq3_first_expert(Context & ctx, hipStream_t stream, const std::string & pack) {
+    std::ifstream meta(pack + "/native_experts.txt");
+    if (!meta) throw std::runtime_error("cannot open native_experts.txt: " + pack);
+    int layer = -1, gt = -1, dt = -1;
+    uint64_t offset = 0, blob_bytes = 0;
+    std::string line;
+    bool found = false;
+    while (std::getline(meta, line)) {
+        if (line.empty() || line[0] == '#') continue;
+        std::istringstream ss(line);
+        if (ss >> layer >> gt >> dt >> offset >> blob_bytes && layer == 0) { found = true; break; }
+    }
+    if (!found) throw std::runtime_error("native metadata has no layer 0");
+    if (gt != GGML_TYPE_IQ3_XXS || !supported(gt) || !supported(dt)) {
+        std::cout << "real-IQ3 first expert skipped: layer 0 types are " << gt << "/" << dt << '\n';
+        return false;
+    }
+    std::ifstream file(pack + "/experts.bin", std::ios::binary);
+    if (!file) throw std::runtime_error("cannot open experts.bin: " + pack);
+    std::vector<uint8_t> blob((size_t) blob_bytes);
+    file.seekg((std::streamoff) offset);
+    file.read((char *) blob.data(), (std::streamsize) blob.size());
+    if (!file) throw std::runtime_error("short read of first native expert");
+
+    constexpr int64_t N = 2560, FF = 640;
+    const size_t gu_row = row_bytes((ggml_type) gt, N), d_row = row_bytes((ggml_type) dt, FF);
+    const size_t one_gu = (size_t) FF * gu_row, down_off = 2 * one_gu, one_down = (size_t) N * d_row;
+    if (down_off + one_down != blob.size()) throw std::runtime_error("native blob shape/offset mismatch");
+    std::vector<uint8_t> gu(blob.begin(), blob.begin() + (ptrdiff_t) down_off);
+    std::vector<uint8_t> down(blob.begin() + (ptrdiff_t) down_off, blob.end());
+    const int rows = 5;
+    const std::vector<int> counts{rows};
+    const auto src = permutation(rows, 2, false), dst = permutation(rows, 1, true);
+    run_product(ctx, stream, "real-first-IQ3_XXS-GU", (ggml_type) gt, 1280, N, {gu}, counts,
+                make_activations(rows, N, 71), src, dst);
+    run_product(ctx, stream, "real-first-down", (ggml_type) dt, N, FF, {down}, counts,
+                make_activations(rows, FF, 73), src, dst);
+    std::cout << "real first expert tested: layer=0 expert=0 blob_bytes=" << blob_bytes << '\n';
+    return true;
+}
+} // namespace
+
+int main(int argc, char ** argv) {
+    try {
+        if (argc > 2) throw std::runtime_error("usage: mmq-parity-test [optional-native-pack-dir]");
+        hip_check(hipSetDevice(0), "hipSetDevice");
+        hipStream_t stream = nullptr;
+        hip_check(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking), "create stream");
+        int failures = 0;
+        {
+            Context ctx; // reuse one MMQ context/pool across every pass below
+            const std::vector<std::vector<int>> batches{{1, 3, 3}, {4, 1, 2}, {2, 3}};
+            int trial = 0;
+            for (const auto & counts : batches) {
+                const int rows = std::accumulate(counts.begin(), counts.end(), 0);
+                const int n = (int) counts.size();
+                const auto src = permutation(rows, trial + 1, false);
+                const auto dst = permutation(rows, trial + 2, true);
+                run_product(ctx, stream, "synthetic-Q2_0-GU-pass" + std::to_string(trial), GGML_TYPE_Q2_0,
+                            1280, 2560, synthetic_q2(n, 1280, 2560, trial), counts,
+                            make_activations(rows, 2560, trial), src, dst);
+                run_product(ctx, stream, "synthetic-Q2_0-down-pass" + std::to_string(trial), GGML_TYPE_Q2_0,
+                            2560, 640, synthetic_q2(n, 2560, 640, trial + 9), counts,
+                            make_activations(rows, 640, trial + 3), src, dst);
+                ++trial;
+            }
+            if (argc == 2) run_real_iq3_first_expert(ctx, stream, argv[1]);
+        }
+        hip_check(hipStreamSynchronize(stream), "final synchronize");
+        hip_check(hipStreamDestroy(stream), "destroy stream");
+        std::cout << "MMQ parity screen passed\n";
+        return failures ? 1 : 0;
+    } catch (const std::exception & e) {
+        std::cerr << "MMQ parity screen failed: " << e.what() << '\n';
+        return 1;
+    }
+}

tests/hip/prefill_native_batch.cpp

@@ -0,0 +1,290 @@
+// Focused parity for the opt-in native QSA append and native embedding gather
+// selectively ported from q8atnight/Strata PR #108
+// (acd487233c0bbe2217a6881c5bb43f8a283b0de5).
+#include <cuda_runtime.h>
+#include "strata/kernels/iq_kernels.hpp"
+#include "strata/kernels/native_qsa_indexer.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <random>
+#include <vector>
+
+#define CHECK(call) do { \
+    const cudaError_t e = (call); \
+    if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", #call, cudaGetErrorString(e)); return false; } \
+} while (0)
+
+namespace {
+constexpr int IDX = 128;
+constexpr int BLOCK = 4;
+constexpr int EMB = 2560;
+constexpr int IQ4_NL = 20;
+constexpr float EPS = 1e-6f;
+
+struct IndexState {
+    float *tail = nullptr, *dead = nullptr, *pooled = nullptr;
+    int32_t* block_pos = nullptr;
+};
+
+bool alloc_state(IndexState& s, int64_t max_cells) {
+    CHECK(cudaMalloc((void**) &s.tail, (BLOCK - 1) * IDX * sizeof(float)));
+    CHECK(cudaMalloc((void**) &s.dead, IDX * sizeof(float)));
+    CHECK(cudaMalloc((void**) &s.pooled, (size_t) (max_cells / BLOCK + 1) * IDX * sizeof(float)));
+    CHECK(cudaMalloc((void**) &s.block_pos, sizeof(int32_t)));
+    return true;
+}
+void free_state(IndexState& s) {
+    if (s.tail) cudaFree(s.tail);
+    if (s.dead) cudaFree(s.dead);
+    if (s.pooled) cudaFree(s.pooled);
+    if (s.block_pos) cudaFree(s.block_pos);
+    s = {};
+}
+bool upload_state(const IndexState& s, int64_t max_cells) {
+    std::vector<float> tail((BLOCK - 1) * IDX, 0.0f), dead(IDX, 0.0f);
+    std::vector<float> pooled((size_t) (max_cells / BLOCK + 1) * IDX);
+    for (size_t i = 0; i < pooled.size(); ++i) pooled[i] = -0.125f * (float) (i + 1);
+    const int32_t block_pos = -1;
+    CHECK(cudaMemcpy(s.tail, tail.data(), tail.size() * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK(cudaMemcpy(s.dead, dead.data(), dead.size() * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK(cudaMemcpy(s.pooled, pooled.data(), pooled.size() * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK(cudaMemcpy(s.block_pos, &block_pos, sizeof(block_pos), cudaMemcpyHostToDevice));
+    return true;
+}
+
+bool qsa_case(cudaStream_t stream, int64_t max_cells, int64_t pos0, int64_t T, std::mt19937& rng) {
+    const size_t raw_n = (size_t) (pos0 + T) * IDX;
+    std::vector<float> raw(raw_n), gamma(IDX);
+    std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
+    for (float& x : raw) x = dist(rng);
+    for (float& x : gamma) x = 0.5f + 0.5f * std::fabs(dist(rng));
+
+    float *raw_d = nullptr, *gamma_d = nullptr;
+    int32_t* pos_d = nullptr;
+    CHECK(cudaMalloc((void**) &raw_d, raw.size() * sizeof(float)));
+    CHECK(cudaMalloc((void**) &gamma_d, gamma.size() * sizeof(float)));
+    CHECK(cudaMalloc((void**) &pos_d, sizeof(int32_t)));
+    CHECK(cudaMemcpyAsync(raw_d, raw.data(), raw.size() * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK(cudaMemcpyAsync(gamma_d, gamma.data(), gamma.size() * sizeof(float), cudaMemcpyHostToDevice, stream));
+
+    IndexState serial, batched;
+    if (!alloc_state(serial, max_cells) || !alloc_state(batched, max_cells)) return false;
+    if (!upload_state(serial, max_cells) || !upload_state(batched, max_cells)) return false;
+
+    const auto shapes = strata::kernels::qsa_real_shapes();
+    const strata::kernels::QsaIndexerBuffers sb{serial.tail, serial.dead, serial.pooled, serial.block_pos};
+    const strata::kernels::QsaIndexerBuffers bb{batched.tail, batched.dead, batched.pooled, batched.block_pos};
+    std::vector<int32_t> positions((size_t) (pos0 + T));
+    for (size_t p = 0; p < positions.size(); ++p) positions[p] = (int32_t) p;
+
+    // The reference is the original per-cell sequence. The candidate uses the
+    // same per-cell prefix, then replaces only this chunk with the two-launch API.
+    for (int64_t p = 0; p < pos0 + T; ++p) {
+        CHECK(cudaMemcpyAsync(pos_d, positions.data() + p, sizeof(int32_t), cudaMemcpyHostToDevice, stream));
+        strata::kernels::native_qsa_indexer_append(raw_d + p * IDX, pos_d, 0, gamma_d, EPS, sb,
+                                                   shapes, max_cells, 10000.0f, stream);
+        if (p < pos0) {
+            CHECK(cudaMemcpyAsync(pos_d, positions.data() + p, sizeof(int32_t), cudaMemcpyHostToDevice, stream));
+            strata::kernels::native_qsa_indexer_append(raw_d + p * IDX, pos_d, 0, gamma_d, EPS, bb,
+                                                       shapes, max_cells, 10000.0f, stream);
+        }
+    }
+    strata::kernels::native_qsa_indexer_append_batch(raw_d + pos0 * IDX, T, pos0, 0, gamma_d, EPS,
+                                                      bb, shapes, max_cells, 10000.0f, stream);
+    CHECK(cudaStreamSynchronize(stream));
+
+    const size_t tail_n = (BLOCK - 1) * IDX, dead_n = IDX;
+    const size_t pooled_n = (size_t) (max_cells / BLOCK + 1) * IDX;
+    std::vector<float> st(tail_n), bt(tail_n), sd(dead_n), bd(dead_n), sp(pooled_n), bp(pooled_n);
+    int32_t spos = 0, bpos = 0;
+    CHECK(cudaMemcpy(st.data(), serial.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(bt.data(), batched.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(sd.data(), serial.dead, dead_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(bd.data(), batched.dead, dead_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(sp.data(), serial.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(bp.data(), batched.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(&spos, serial.block_pos, sizeof(spos), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(&bpos, batched.block_pos, sizeof(bpos), cudaMemcpyDeviceToHost));
+    const bool same = std::memcmp(st.data(), bt.data(), tail_n * sizeof(float)) == 0 &&
+                      std::memcmp(sd.data(), bd.data(), dead_n * sizeof(float)) == 0 &&
+                      std::memcmp(sp.data(), bp.data(), pooled_n * sizeof(float)) == 0 && spos == bpos;
+    if (!same) std::fprintf(stderr, "QSA append parity mismatch max=%lld pos0=%lld T=%lld\n",
+                            (long long) max_cells, (long long) pos0, (long long) T);
+
+    free_state(serial); free_state(batched);
+    CHECK(cudaFree(raw_d)); CHECK(cudaFree(gamma_d)); CHECK(cudaFree(pos_d));
+    return same;
+}
+
+bool qsa_chunks_case(cudaStream_t stream, std::mt19937& rng, int64_t max_cells,
+                     int64_t first, int64_t second) {
+    const int64_t total = first + second;
+    std::vector<float> raw((size_t) total * IDX), gamma(IDX);
+    std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
+    for (float& x : raw) x = dist(rng);
+    for (float& x : gamma) x = 0.5f + 0.5f * std::fabs(dist(rng));
+    float *raw_d = nullptr, *gamma_d = nullptr;
+    int32_t* pos_d = nullptr;
+    CHECK(cudaMalloc((void**) &raw_d, raw.size() * sizeof(float)));
+    CHECK(cudaMalloc((void**) &gamma_d, gamma.size() * sizeof(float)));
+    CHECK(cudaMalloc((void**) &pos_d, sizeof(int32_t)));
+    CHECK(cudaMemcpyAsync(raw_d, raw.data(), raw.size() * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK(cudaMemcpyAsync(gamma_d, gamma.data(), gamma.size() * sizeof(float), cudaMemcpyHostToDevice, stream));
+    IndexState serial, chunked;
+    if (!alloc_state(serial, max_cells) || !alloc_state(chunked, max_cells)) return false;
+    if (!upload_state(serial, max_cells) || !upload_state(chunked, max_cells)) return false;
+    const auto shapes = strata::kernels::qsa_real_shapes();
+    const strata::kernels::QsaIndexerBuffers sb{serial.tail, serial.dead, serial.pooled, serial.block_pos};
+    const strata::kernels::QsaIndexerBuffers cb{chunked.tail, chunked.dead, chunked.pooled, chunked.block_pos};
+    std::vector<int32_t> positions((size_t) total);
+    for (int i = 0; i < total; ++i) positions[(size_t) i] = i;
+    for (int p = 0; p < total; ++p) {
+        CHECK(cudaMemcpyAsync(pos_d, positions.data() + p, sizeof(int32_t), cudaMemcpyHostToDevice, stream));
+        strata::kernels::native_qsa_indexer_append(raw_d + (size_t) p * IDX, pos_d, 0, gamma_d, EPS, sb,
+                                                   shapes, max_cells, 10000.0f, stream);
+    }
+    strata::kernels::native_qsa_indexer_append_batch(raw_d, first, 0, 0, gamma_d, EPS, cb, shapes,
+                                                      max_cells, 10000.0f, stream);
+    strata::kernels::native_qsa_indexer_append_batch(raw_d + (size_t) first * IDX, second, first, 0,
+                                                      gamma_d, EPS, cb, shapes, max_cells, 10000.0f, stream);
+    CHECK(cudaStreamSynchronize(stream));
+    const size_t tail_n = (BLOCK - 1) * IDX, pooled_n = (size_t) (max_cells / BLOCK + 1) * IDX;
+    std::vector<float> st(tail_n), ct(tail_n), sd(IDX), cd(IDX), sp(pooled_n), cp(pooled_n);
+    int32_t sbp = 0, cbp = 0;
+    CHECK(cudaMemcpy(st.data(), serial.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(ct.data(), chunked.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(sd.data(), serial.dead, IDX * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(cd.data(), chunked.dead, IDX * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(sp.data(), serial.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(cp.data(), chunked.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(&sbp, serial.block_pos, sizeof(sbp), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(&cbp, chunked.block_pos, sizeof(cbp), cudaMemcpyDeviceToHost));
+    const bool same = std::memcmp(st.data(), ct.data(), tail_n * sizeof(float)) == 0 &&
+                      std::memcmp(sd.data(), cd.data(), IDX * sizeof(float)) == 0 &&
+                      std::memcmp(sp.data(), cp.data(), pooled_n * sizeof(float)) == 0 && sbp == cbp;
+    if (!same) std::fprintf(stderr, "QSA consecutive chunk parity mismatch %lld+%lld, capacity=%lld\n",
+                            (long long) first, (long long) second, (long long) max_cells);
+    free_state(serial); free_state(chunked);
+    CHECK(cudaFree(raw_d)); CHECK(cudaFree(gamma_d)); CHECK(cudaFree(pos_d));
+    return same;
+}
+
+bool embed_case(cudaStream_t stream, int type, int64_t T) {
+    constexpr int vocab = 19;
+    const size_t row_bytes = strata::kernels::iq_row_bytes(type, EMB);
+    const size_t block_bytes = type == 20 ? 18 : 110;  // IQ4_NL or IQ3_S GGML block layout
+    const size_t table_bytes = (size_t) vocab * row_bytes;
+    if (row_bytes == 0 || row_bytes % block_bytes != 0) {
+        std::fprintf(stderr, "bad test embedding row size for type=%d\n", type);
+        return false;
+    }
+    void* host_table = nullptr;
+    CHECK(cudaHostAlloc(&host_table, table_bytes, cudaHostAllocMapped | cudaHostAllocPortable));
+    auto* table_bytes_u8 = (uint8_t*) host_table;
+    std::memset(table_bytes_u8, 0, table_bytes);
+    // Both test formats begin with a finite fp16 scale; the remaining packed code bits are all legal values.
+    for (size_t r = 0; r < (size_t) vocab; ++r) {
+        for (size_t b = 0; b < row_bytes; b += block_bytes) {
+            table_bytes_u8[r * row_bytes + b] = 0x00;
+            table_bytes_u8[r * row_bytes + b + 1] = 0x3c;
+            for (size_t q = b + 2; q < b + block_bytes; ++q)
+                table_bytes_u8[r * row_bytes + q] = (uint8_t) ((q * 37 + r * 19) & 0xff);
+        }
+    }
+    void* table_dev = nullptr;
+    CHECK(cudaHostGetDevicePointer(&table_dev, host_table, 0));
+
+    std::vector<uint8_t> image_rows((size_t) T, 0);
+    std::vector<int64_t> token_values((size_t) T);
+    std::vector<int32_t> ids((size_t) T);
+    for (int64_t t = 0; t < T; ++t) {
+        if (t == 1 || t == 3) {
+            image_rows[(size_t) t] = 1;
+            token_values[(size_t) t] = t == 1 ? -1 : vocab + 11;  // image placeholders must never be gathered
+            ids[(size_t) t] = 0;                                 // safe row, overwritten below
+        } else {
+            token_values[(size_t) t] = (t % 3 == 0) ? 7 : (t % 5);  // includes duplicate text IDs
+            if (token_values[(size_t) t] < 0 || token_values[(size_t) t] >= vocab ||
+                token_values[(size_t) t] > INT32_MAX) {
+                std::fprintf(stderr, "test text token failed host-side validation\n");
+                return false;
+            }
+            ids[(size_t) t] = (int32_t) token_values[(size_t) t];
+        }
+    }
+
+    float *batched = nullptr, *serial = nullptr;
+    int32_t* ids_d = nullptr;
+    CHECK(cudaMalloc((void**) &batched, (size_t) T * EMB * sizeof(float)));
+    CHECK(cudaMalloc((void**) &serial, (size_t) T * EMB * sizeof(float)));
+    CHECK(cudaMalloc((void**) &ids_d, (size_t) T * sizeof(int32_t)));
+    CHECK(cudaMemcpyAsync(ids_d, ids.data(), (size_t) T * sizeof(int32_t), cudaMemcpyHostToDevice, stream));
+    strata::kernels::iq_embed_rows(type, table_dev, row_bytes, ids_d, T, EMB, batched, stream);
+    for (int64_t t = 0; t < T; ++t) if (!image_rows[(size_t) t])
+        strata::kernels::iq_dequant_f32(type,
+            (const uint8_t*) table_dev + (size_t) ids[(size_t) t] * row_bytes, EMB, serial + t * EMB, stream);
+
+    std::vector<float> image(EMB);
+    for (int i = 0; i < EMB; ++i) image[(size_t) i] = 0.001f * (float) (i - EMB / 2);
+    for (int64_t t = 0; t < T; ++t) if (image_rows[(size_t) t]) {
+        CHECK(cudaMemcpyAsync(batched + t * EMB, image.data(), EMB * sizeof(float), cudaMemcpyHostToDevice, stream));
+        CHECK(cudaMemcpyAsync(serial + t * EMB, image.data(), EMB * sizeof(float), cudaMemcpyHostToDevice, stream));
+    }
+    CHECK(cudaStreamSynchronize(stream));
+
+    constexpr int64_t rows_per_check = 64;
+    std::vector<float> a((size_t) rows_per_check * EMB), b(a.size());
+    bool same = true;
+    for (int64_t t0 = 0; t0 < T; t0 += rows_per_check) {
+        const int64_t rows = std::min(rows_per_check, T - t0);
+        const size_t count = (size_t) rows * EMB;
+        CHECK(cudaMemcpy(a.data(), batched + t0 * EMB, count * sizeof(float), cudaMemcpyDeviceToHost));
+        CHECK(cudaMemcpy(b.data(), serial + t0 * EMB, count * sizeof(float), cudaMemcpyDeviceToHost));
+        if (std::memcmp(a.data(), b.data(), count * sizeof(float)) != 0) {
+            size_t first = 0;
+            while (first < count && std::memcmp(&a[first], &b[first], sizeof(float)) == 0) ++first;
+            std::fprintf(stderr, "NativeEmbed batch parity mismatch type=%d T=%lld at element %lld: %.9g vs %.9g\n",
+                         type, (long long) T, (long long) (t0 * EMB + first),
+                         first < count ? a[first] : 0.0f, first < count ? b[first] : 0.0f);
+            same = false;
+            break;
+        }
+    }
+    if (T > 3 && (!image_rows[1] || !image_rows[3] || ids[1] != 0 || ids[3] != 0)) same = false;
+    if (!same) std::fprintf(stderr, "NativeEmbed image placeholder sanitization mismatch type=%d T=%lld\n",
+                            type, (long long) T);
+    CHECK(cudaFree(batched)); CHECK(cudaFree(serial)); CHECK(cudaFree(ids_d));
+    CHECK(cudaFreeHost(host_table));
+    return same;
+}
+
+} // namespace
+
+int main() {
+    std::mt19937 rng(0x50524631u);
+    cudaStream_t stream = nullptr;
+    const cudaError_t create = cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking);
+    if (create != cudaSuccess) { std::fprintf(stderr, "cudaStreamCreateWithFlags: %s\n", cudaGetErrorString(create)); return 2; }
+    bool ok = true;
+    // Covers every slot residue, partial and multiple block chunks, and the fixed-capacity tail.
+    const std::array<int64_t, 14> starts = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13};
+    const std::array<int64_t, 5> lengths = {1, 2, 3, 7, 17};
+    // (the batched append's contract is p0 + n <= max_cells: the prompt path never appends past the cache)
+    for (int64_t pos0 : starts) for (int64_t T : lengths)
+        if (pos0 + T <= 13) ok = qsa_case(stream, 13, pos0, T, rng) && ok;
+    ok = qsa_chunks_case(stream, rng, 33, 7, 17) && ok;
+    // Production-scale, non-block-aligned first chunk with a short continuation.
+    ok = qsa_chunks_case(stream, rng, 8210, 8193, 17) && ok;
+    for (int type : {20, 21}) for (int64_t T : {1, 2, 7, 17})
+        ok = embed_case(stream, type, T) && ok;
+    ok = embed_case(stream, 21, 4096) && ok;
+    const cudaError_t destroy = cudaStreamDestroy(stream);
+    if (destroy != cudaSuccess) { std::fprintf(stderr, "cudaStreamDestroy: %s\n", cudaGetErrorString(destroy)); return 2; }
+    std::printf("Native prefill batch parity: %s\n", ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}

tests/platform/direct_file_async.cpp

@@ -0,0 +1,60 @@
+#include "strata/platform/direct_file.hpp"
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+#include <unistd.h>
+using strata::platform::DirectFile;
+using strata::platform::Completion;
+static void check(bool ok, const char* why) { if (!ok) throw std::runtime_error(why); }
+int main() {
+    char name[] = "/tmp/strata-direct-test-XXXXXX";
+    int fd = mkstemp(name);
+    if (fd < 0) return 1;
+    close(fd);
+    void* memory = DirectFile::alloc_aligned(4096 * 18);
+    try {
+        check(memory != nullptr, "buffer allocation");
+        { std::ofstream f(name, std::ios::binary); std::string bytes(4096 * 16 + 17, 'x'); f.write(bytes.data(), bytes.size()); }
+        DirectFile f;
+        std::string error;
+        setenv("STRATA_IO_THREADS", "4", 1);   // the pool's size (out-of-range values are clamped to 1..64)
+        check(f.open(name, error), error.c_str());
+        check(!f.submit(1, memory, 4096, 0, error), "unaligned request accepted");
+        for (unsigned i = 0; i < 17; ++i)
+            check(f.submit(i * 4096, static_cast<char*>(memory) + i * 4096, 4096, i, error), error.c_str());
+        std::vector<bool> seen(17);
+        unsigned count = 0;
+        while (count < 17) {
+            Completion c[17]; int n = f.wait(c, 17, 5000);
+            check(n > 0, "completion timeout");
+            for (int i = 0; i < n; ++i) {
+                check(c[i].tag < 17 && !seen[c[i].tag], "wrong or repeated completion");
+                seen[c[i].tag] = true; ++count;
+                check(c[i].ok && c[i].bytes == (c[i].tag == 16 ? 17u : 4096u), "incorrect full/EOF completion");
+                check(static_cast<char*>(memory)[c[i].tag * 4096] == 'x', "read data mismatch");
+            }
+        }
+        std::jthread waker([&] { f.wake(); });
+        Completion c; check(f.wait(&c, 1, 5000) == 1 && c.tag == DirectFile::WAKE_TAG, "wake lost");
+        waker.join();
+        check(f.submit(0, memory, 4096, 99, error), error.c_str());
+        f.close(); // Must drain outstanding buffer writes before returning.
+        std::memset(memory, 0, 4096);
+        check(f.open(name, error), error.c_str());
+        check(f.wait(&c, 1, 0) == 0, "stale completion on reopen");
+        check(f.submit(0, memory, 4096, 100, error), error.c_str());
+        check(f.wait(&c, 1, 5000) == 1 && c.ok && c.tag == 100, "reopened read failed");
+        f.close();
+        DirectFile::free_aligned(memory); unlink(name);
+        std::cout << "direct_file_async: passed queued reads, short EOF, wake, close drain, reopen\n";
+        return 0;
+    } catch (const std::exception& e) {
+        DirectFile::free_aligned(memory); unlink(name);
+        std::cerr << e.what() << '\n'; return 1;
+    }
+}

tools/draft_vocab.py

@@ -0,0 +1,94 @@
+"""The draft head's token subset (rt/draft_vocab.bin): the vocabulary rows the MTP draft layer may propose.
+
+The engine reads rt/draft_vocab.bin (int32 token ids) when the draft layer binds and computes the draft logits over
+those rows only: a smaller head is faster and takes less VRAM, but a token outside the subset can never be drafted.
+The shipped subset was built from English and code and had 27 of the vocabulary's 55,328 Han tokens, so an answer in
+Chinese drafted almost nothing (#137).  This adds whole scripts to a subset:
+
+    python tools/draft_vocab.py --gguf <model>-00001-of-0000N.gguf --base data/draft_vocab.bin --add cjk \
+        --out data/draft_vocab.bin
+
+--add takes han, kana, hangul, cjk_punct, or cjk (all four).  The base ids keep their order; the added ones follow in
+id order.  --stats prints what a subset holds.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from array import array
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import strata_tokenizer as ST  # noqa: E402
+
+SCRIPTS = {
+    "han": [(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x2FA1F)],
+    "kana": [(0x3040, 0x30FF), (0x31F0, 0x31FF), (0xFF66, 0xFF9F)],
+    "hangul": [(0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)],
+    "cjk_punct": [(0x3000, 0x303F), (0xFF00, 0xFF65), (0xFFA0, 0xFFEF)],
+}
+GROUPS = {"cjk": ["han", "kana", "hangul", "cjk_punct"]}
+
+
+def scripts_of(text: str) -> set[str]:
+    found = set()
+    for ch in text:
+        c = ord(ch)
+        for name, ranges in SCRIPTS.items():
+            if any(a <= c <= b for a, b in ranges):
+                found.add(name)
+    return found
+
+
+def token_text(tok, i: int) -> str | None:
+    """A token's text, or None when its bytes are not whole UTF-8 characters (a piece of one) or it is special."""
+    try:
+        return tok.token_bytes(i).decode("utf-8")
+    except (KeyError, UnicodeDecodeError, IndexError):
+        return None
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--gguf", required=True, help="the model's (first) GGUF file: its vocabulary")
+    ap.add_argument("--base", help="the subset to extend (int32 ids)")
+    ap.add_argument("--add", default="", help="comma list: han, kana, hangul, cjk_punct, cjk")
+    ap.add_argument("--out", help="where to write the new subset")
+    ap.add_argument("--stats", action="store_true", help="print what the base (and the new) subset holds")
+    a = ap.parse_args()
+
+    tok = ST.Tokenizer.from_gguf(a.gguf)
+    n = len(tok.tokens)
+    kinds = [scripts_of(t) if (t := token_text(tok, i)) is not None else set() for i in range(n)]
+    base = list(array("i", Path(a.base).read_bytes())) if a.base else []
+    if any(i < 0 or i >= n for i in base):
+        sys.exit(f"{a.base} has ids outside the vocabulary ({n})")
+
+    want = set()
+    for x in filter(None, (s.strip() for s in a.add.split(","))):
+        names = GROUPS.get(x, [x])
+        if any(nm not in SCRIPTS for nm in names):
+            sys.exit(f"--add {x}: expected one of {', '.join([*SCRIPTS, *GROUPS])}")
+        want.update(names)
+    have = set(base)
+    added = [i for i in range(n) if i not in have and kinds[i] & want]
+    ids = base + added
+
+    def stats(label, sel):
+        counts = {nm: sum(1 for i in sel if nm in kinds[i]) for nm in SCRIPTS}
+        full = {nm: sum(1 for k in kinds if nm in k) for nm in SCRIPTS}
+        print(f"{label}: {len(sel)} ids; " + ", ".join(f"{nm} {counts[nm]}/{full[nm]}" for nm in SCRIPTS))
+
+    if a.stats or not a.out:
+        if base:
+            stats("base", base)
+        if added:
+            stats("new", ids)
+    if a.out:
+        Path(a.out).write_bytes(array("i", ids).tobytes())
+        print(f"wrote {a.out}: {len(ids)} ids ({len(added)} added)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/hip/bench_prefill.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Matched fresh/follow-up prefill probe against an otherwise idle local Strata server.
+
+Does not start/stop services. Use a dedicated server and its engine log, restart
+between build/configuration arms, and keep model/template/settings identical.
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import time
+import urllib.request
+
+PATTERN = re.compile(r'prompt (\d+) tokens = (\d+) reused \+ (\d+) read in (\d+) ms \(([0-9.]+) tok/s\), (\d+) generated in (\d+) ms \(([0-9.]+) tok/s\)')
+FIELDS = ('prompt_tokens', 'reused', 'fresh', 'prefill_ms', 'prefill_tps', 'generated', 'decode_ms', 'decode_tps')
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--url', default='http://127.0.0.1:8080')
+    p.add_argument('--model', required=True)
+    p.add_argument('--engine-log', type=Path, required=True)
+    p.add_argument('--output', type=Path, required=True)
+    p.add_argument('--label', required=True)
+    args = p.parse_args()
+    results = []
+
+    def request(messages, kind, trial):
+        offset = args.engine_log.stat().st_size
+        body = dict(model=args.model, messages=messages, max_tokens=128,
+                    temperature=0, top_k=1, top_p=1, min_p=0, seed=42,
+                    reasoning_effort='none')
+        headers = {'Content-Type': 'application/json'}
+        if os.environ.get('STRATA_API_KEY'):
+            headers['Authorization'] = 'Bearer ' + os.environ['STRATA_API_KEY']
+        req = urllib.request.Request(args.url.rstrip('/') + '/v1/chat/completions',
+                                     data=json.dumps(body).encode(), headers=headers)
+        start = time.monotonic()
+        with urllib.request.urlopen(req, timeout=360) as response:
+            data = json.load(response)
+        elapsed = time.monotonic() - start
+        # The API response may arrive just before stderr's completion line is flushed.
+        deadline = time.monotonic() + 2
+        while True:
+            with args.engine_log.open('rb') as log:
+                log.seek(offset)
+                matches = PATTERN.findall(log.read().decode(errors='replace'))
+            if matches or time.monotonic() >= deadline:
+                break
+            time.sleep(0.02)
+        if len(matches) != 1:
+            raise RuntimeError('Expected exactly one completed engine timing line; check idle server and log path')
+        metrics = dict(zip(FIELDS, map(float, matches[0])))
+        if kind == 'fresh' and metrics['reused'] != 0:
+            raise RuntimeError('Fresh prompt reused cached tokens; restart this arm before comparison')
+        choice = data['choices'][0]
+        results.append(dict(label=args.label, kind=kind, trial=trial, wall_s=elapsed,
+                            metrics=metrics, usage=data.get('usage'),
+                            finish_reason=choice.get('finish_reason'), message=choice['message']))
+        args.output.write_text(json.dumps(results, indent=2) + '\n')
+        print(json.dumps(results[-1]), flush=True)
+        return choice['message']
+
+    request([{'role': 'user', 'content': 'Reply READY.'}], 'warmup', 0)
+    for trial, n in enumerate((140, 280, 140, 280), 1):
+        code = '\n'.join(f'export function rule{i}(x) {{ return x === {i} ? x + {i+1} : x - {i}; }}' for i in range(n))
+        messages = [{'role': 'user', 'content': f'CASE {trial}: Review this source and describe its behavior precisely in a paragraph.\n' + code}]
+        reply = request(messages, 'fresh', trial)
+        messages += [reply, {'role': 'user', 'content': 'Explain the most relevant boundary case and the smallest useful regression test. ' + 'Focus on integer equality, zero, negative input, unexpected types, and caller assumptions. ' * 5}]
+        request(messages, 'followup', trial)
+
+
+if __name__ == '__main__':
+    main()

tools/hip/gfx1100-hipblaslt-100100.txt

@@ -0,0 +1,32 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1100 100100
+bf16 1 2560 1 4096 541
+bf16 1 2560 1 8192 541
+bf16 4 10240 4 4096 541
+bf16 4 10240 4 8192 541
+bf16 48 2560 96 4096 541
+bf16 48 2560 96 8192 542
+bf16 128 2560 128 4096 542
+bf16 128 2560 128 8192 542
+bf16 320 10240 320 4096 542
+bf16 320 10240 320 8192 542
+bf16 512 2560 512 4096 542
+bf16 512 2560 512 8192 542
+bf16 2560 2560 2560 4096 542
+bf16 2560 2560 2560 8192 542
+bf16 10240 320 10240 4096 542
+bf16 10240 320 10240 8192 542
+f16 512 2560 512 4096 1147
+f16 512 2560 512 8192 1147
+f16 640 2560 640 4096 1147
+f16 640 2560 640 8192 1147
+f16 2560 640 2560 4096 1147
+f16 2560 640 2560 8192 1147
+f16 2560 6144 2560 4096 1147
+f16 2560 6144 2560 8192 1147
+f16 6144 2560 6144 4096 1147
+f16 6144 2560 6144 8192 1147
+f16 10240 2560 10240 4096 1147
+f16 10240 2560 10240 8192 1147
+f16 12288 2560 12288 4096 1147
+f16 12288 2560 12288 8192 1147

tools/hip/gfx1100-hipblaslt-100200.txt

@@ -0,0 +1,28 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1100 100200
+bf16 1 2560 1 4096 541
+bf16 4 10240 4 8192 542
+bf16 48 2560 96 4096 541
+bf16 48 2560 96 8192 542
+bf16 128 2560 128 4096 542
+bf16 128 2560 128 8192 542
+bf16 320 10240 320 4096 542
+bf16 320 10240 320 8192 542
+bf16 512 2560 512 8192 542
+bf16 2560 2560 2560 8192 542
+bf16 10240 320 10240 4096 542
+bf16 10240 320 10240 8192 542
+f16 512 2560 512 4096 1147
+f16 512 2560 512 8192 1147
+f16 640 2560 640 4096 1147
+f16 640 2560 640 8192 1147
+f16 2560 640 2560 4096 1147
+f16 2560 640 2560 8192 1147
+f16 2560 6144 2560 4096 1147
+f16 2560 6144 2560 8192 1147
+f16 6144 2560 6144 4096 1147
+f16 6144 2560 6144 8192 1147
+f16 10240 2560 10240 4096 1147
+f16 10240 2560 10240 8192 1147
+f16 12288 2560 12288 4096 1147
+f16 12288 2560 12288 8192 1147

tools/hip/tune_hipblaslt.cpp

@@ -0,0 +1,419 @@
+#include <hip/hip_runtime.h>
+#include <hip/hip_fp16.h>
+#include <hip/hip_bfloat16.h>
+#include <hip/hip_version.h>
+#include <hipblas/hipblas.h>
+#include <hipblaslt/hipblaslt.h>
+#include <hipblaslt/hipblaslt-ext.hpp>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <iterator>
+#include <limits>
+#include <random>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <vector>
+
+namespace {
+#define HIP_CHECK(call) do { const hipError_t e = (call); if (e != hipSuccess) { \
+    std::fprintf(stderr, "HIP %s:%d: %s: %s\n", __FILE__, __LINE__, #call, hipGetErrorString(e)); std::exit(2); } } while (0)
+#define BLAS_CHECK(call) do { const hipblasStatus_t e = (call); if (e != HIPBLAS_STATUS_SUCCESS) { \
+    std::fprintf(stderr, "hipBLAS %s:%d: %s: status=%d\n", __FILE__, __LINE__, #call, (int)e); std::exit(2); } } while (0)
+#define LT_CHECK(call) do { const hipblasStatus_t e = (call); if (e != HIPBLAS_STATUS_SUCCESS) { \
+    std::fprintf(stderr, "hipBLASLt %s:%d: %s: status=%d\n", __FILE__, __LINE__, #call, (int)e); std::exit(2); } } while (0)
+
+struct Buffer {
+    void *p = nullptr;
+    explicit Buffer(size_t bytes) { if (bytes) HIP_CHECK(hipMalloc(&p, bytes)); }
+    ~Buffer() { if (p) (void)hipFree(p); }
+    Buffer(const Buffer &) = delete;
+    Buffer &operator=(const Buffer &) = delete;
+};
+struct Events {
+    hipEvent_t a = nullptr, b = nullptr;
+    Events() { HIP_CHECK(hipEventCreate(&a)); HIP_CHECK(hipEventCreate(&b)); }
+    ~Events() { if (a) (void)hipEventDestroy(a); if (b) (void)hipEventDestroy(b); }
+};
+
+struct Shape { int t, n, k, ldy; bool bf16; };
+struct Options {
+    size_t workspace = 32U * 1024U * 1024U;
+    std::vector<Shape> shapes;
+    std::vector<int> tokens{4096, 8192};
+    std::string shapes_file, tuning_out;
+    bool explicit_shapes = false;
+};
+struct Err { double rel_l2, max_abs; bool finite; size_t padding_writes; };
+struct Best {
+    Shape s;
+    int heuristic_index, solution_id;
+    size_t required_workspace, algo_workspace;
+    float mean_ms;
+    Err error;
+    std::string solution, kernel, config;
+};
+
+constexpr int WARMUPS = 2;
+constexpr int REPS = 3;
+constexpr int MAX_ALGOS = 16;
+constexpr double REL_L2_TOL = 1e-4;
+constexpr double MAX_ABS_TOL = 1e-2;
+constexpr float PADDING_CANARY = 123456.25f;
+
+const char *dtype(bool bf16) { return bf16 ? "bf16" : "f16"; }
+const char *status_name(hipblasStatus_t s) {
+    switch (s) {
+        case HIPBLAS_STATUS_SUCCESS: return "success";
+        case HIPBLAS_STATUS_NOT_SUPPORTED: return "not_supported";
+        case HIPBLAS_STATUS_INVALID_VALUE: return "invalid_value";
+        case HIPBLAS_STATUS_ARCH_MISMATCH: return "arch_mismatch";
+        default: return "other";
+    }
+}
+std::string json(const std::string &s) {
+    std::ostringstream o; o << '"';
+    for (unsigned char c : s) {
+        if (c == '"') o << "\\\"";
+        else if (c == '\\') o << "\\\\";
+        else if (c == '\n') o << "\\n";
+        else if (c == '\r') o << "\\r";
+        else if (c == '\t') o << "\\t";
+        else if (c < 0x20) o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (unsigned)c << std::dec;
+        else o << (char)c;
+    }
+    o << '"'; return o.str();
+}
+std::string config_hex(const hipblasLtMatmulAlgo_t &algo) {
+    std::ostringstream o; o << std::hex << std::setfill('0');
+    for (uint8_t b : algo.data) o << std::setw(2) << (unsigned)b;
+    return o.str();
+}
+uint16_t input_bits(float x, bool bf16) {
+    if (bf16) return hip_bfloat16(x).data;
+    const __half half = __float2half_rn(x);
+    uint16_t bits = 0;
+    std::memcpy(&bits, &half, sizeof(bits));
+    return bits;
+}
+void fill(std::vector<uint16_t> &v, uint32_t seed, bool bf16) {
+    std::mt19937 rng(seed);
+    std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
+    for (auto &x : v) x = input_bits(dist(rng), bf16);
+}
+float time_call(hipStream_t stream, const std::function<void()> &f) {
+    Events ev;
+    HIP_CHECK(hipEventRecord(ev.a, stream));
+    f();
+    HIP_CHECK(hipEventRecord(ev.b, stream));
+    HIP_CHECK(hipEventSynchronize(ev.b));
+    HIP_CHECK(hipDeviceSynchronize());
+    float ms = 0;
+    HIP_CHECK(hipEventElapsedTime(&ms, ev.a, ev.b));
+    return ms;
+}
+Err compare(const std::vector<float> &ref, const std::vector<float> &got, int n, int t, int ld) {
+    long double d2 = 0, r2 = 0; double max_abs = 0; bool finite = true;
+    for (int col = 0; col < t; ++col) for (int row = 0; row < n; ++row) {
+        const size_t i = (size_t)col * ld + row;
+        const double r = ref[i], g = got[i];
+        if (!std::isfinite(r) || !std::isfinite(g)) { finite = false; continue; }
+        const double d = g - r;
+        if (!std::isfinite(d)) { finite = false; continue; }
+        d2 += (long double)d * d; r2 += (long double)r * r;
+        max_abs = std::max(max_abs, std::abs(d));
+    }
+    if (!finite) return {std::numeric_limits<double>::infinity(),
+                         std::numeric_limits<double>::infinity(), false, 0};
+    return {std::sqrt((double)(d2 / std::max(r2, 1e-300L))), max_abs, true, 0};
+}
+size_t padding_writes(const std::vector<float> &got, int n, int t, int ld) {
+    size_t writes = 0;
+    for (int col = 0; col < t; ++col) for (int row = n; row < ld; ++row)
+        if (got[(size_t)col * ld + row] != PADDING_CANARY) ++writes;
+    return writes;
+}
+bool accuracy_ok(const Err &e) {
+    return e.finite && e.padding_writes == 0 &&
+           e.rel_l2 <= REL_L2_TOL && e.max_abs <= MAX_ABS_TOL;
+}
+std::vector<std::string> csv(const std::string &s) {
+    std::vector<std::string> out; size_t b = 0;
+    for (;;) {
+        const size_t p = s.find(',', b);
+        out.push_back(s.substr(b, p == std::string::npos ? p : p - b));
+        if (p == std::string::npos) return out;
+        b = p + 1;
+    }
+}
+int positive(const std::string &s, const char *label) {
+    char *e = nullptr; const long v = std::strtol(s.c_str(), &e, 10);
+    if (s.empty() || !e || *e || v < 1 || v > std::numeric_limits<int>::max()) {
+        std::fprintf(stderr, "invalid %s: %s\n", label, s.c_str()); std::exit(2);
+    }
+    return (int)v;
+}
+void add(Options &o, bool bf16, int t, int n, int k, int ldy) {
+    if (ldy < n) { std::fprintf(stderr, "ldy must be >= N\n"); std::exit(2); }
+    o.shapes.push_back({t,n,k,ldy,bf16});
+}
+void add_shape(Options &o, const std::string &v) {
+    const auto f = csv(v);
+    if (f.size() != 3) { std::fprintf(stderr, "--shape expects T,N,K\n"); std::exit(2); }
+    int t=positive(f[0],"T"), n=positive(f[1],"N"), k=positive(f[2],"K");
+    add(o,false,t,n,k,n); add(o,true,t,n,k,n); o.explicit_shapes=true;
+}
+void add_case(Options &o, const std::string &v) {
+    const auto f = csv(v);
+    if (f.size()!=5 || (f[0]!="f16" && f[0]!="bf16")) {
+        std::fprintf(stderr, "--case expects dtype,T,N,K,ldy\n"); std::exit(2);
+    }
+    add(o,f[0]=="bf16",positive(f[1],"T"),positive(f[2],"N"),positive(f[3],"K"),positive(f[4],"ldy"));
+    o.explicit_shapes=true;
+}
+std::vector<int> parse_tokens(const std::string &s) {
+    std::vector<int> out;
+    for (const auto &x : csv(s)) out.push_back(positive(x,"token bucket"));
+    return out;
+}
+void load_shapes(Options &o) {
+    std::ifstream f(o.shapes_file);
+    if (!f) { std::fprintf(stderr, "cannot open shapes file: %s\n",o.shapes_file.c_str()); std::exit(2); }
+    const std::string text((std::istreambuf_iterator<char>(f)),{});
+    const std::regex obj(R"(\{[^{}]*\})"), dtype_re(R"re("dtype"\s*:\s*"(f16|bf16)")re"),
+        n_re(R"("N"\s*:\s*([0-9]+))"), k_re(R"("K"\s*:\s*([0-9]+))"),
+        ld_re(R"("ldy"\s*:\s*([0-9]+))");
+    int count=0;
+    for (auto i=std::sregex_iterator(text.begin(),text.end(),obj); i!=std::sregex_iterator(); ++i) {
+        const std::string item=i->str(); std::smatch d,n,k,ld;
+        if (!std::regex_search(item,d,dtype_re) || !std::regex_search(item,n,n_re) ||
+            !std::regex_search(item,k,k_re) || !std::regex_search(item,ld,ld_re)) continue;
+        for (int t:o.tokens) add(o,d[1]=="bf16",t,positive(n[1],"N"),positive(k[1],"K"),positive(ld[1],"ldy"));
+        ++count;
+    }
+    if (!count) { std::fprintf(stderr,"no dtype,N,K,ldy entries in %s\n",o.shapes_file.c_str()); std::exit(2); }
+    o.explicit_shapes=true;
+}
+void usage(const char *p) {
+    std::printf("Usage: %s [--workspace-mib N] [--shape T,N,K]... [--case dtype,T,N,K,ldy]...\n",p);
+    std::printf("       [--shapes-file PATH [--tokens T1,T2,...]] [--tuning-out PATH]\n");
+    std::printf("Defaults: original three shapes, f16+bf16, T=8192, workspace=32 MiB.\n");
+}
+Options options(int argc,char **argv) {
+    Options o;
+    for (int i=1;i<argc;++i) {
+        std::string a=argv[i];
+        auto next=[&]() -> std::string { if(i+1>=argc){std::fprintf(stderr,"missing option value\n");std::exit(2);} return argv[++i]; };
+        if(a=="-h"||a=="--help"){usage(argv[0]);std::exit(0);}
+        else if(a=="--workspace-mib"){
+            std::string v=next(); char *e=nullptr; unsigned long long x=std::strtoull(v.c_str(),&e,10);
+            if(v.empty()||!e||*e||x>std::numeric_limits<size_t>::max()/(1024ULL*1024ULL)){std::fprintf(stderr,"bad workspace MiB\n");std::exit(2);}
+            o.workspace=(size_t)x*1024U*1024U;
+        } else if(a=="--shape") add_shape(o,next());
+        else if(a=="--case") add_case(o,next());
+        else if(a=="--shapes-file"){o.shapes_file=next();o.explicit_shapes=true;}
+        else if(a=="--tokens") o.tokens=parse_tokens(next());
+        else if(a=="--tuning-out") o.tuning_out=next();
+        else {std::fprintf(stderr,"unknown option: %s\n",a.c_str());usage(argv[0]);std::exit(2);}
+    }
+    if(!o.shapes_file.empty()) load_shapes(o);
+    if(!o.explicit_shapes) {
+        struct DefaultShape { int n,k,ldy; bool bf16; };
+        const DefaultShape defs[]={{10240,2560,10240,false},{320,10240,320,false},{2560,320,2560,false},
+                                {10240,2560,10240,true},{320,10240,320,true},{2560,320,2560,true}};
+        for(const auto &s:defs) add(o,s.bf16,8192,s.n,s.k,s.ldy);
+    }
+    if(o.shapes.empty()){std::fprintf(stderr,"no shapes requested\n");std::exit(2);}
+    return o;
+}
+std::string arch_name(const char *s) {
+    std::string a=s?s:"unknown"; size_t p=a.find(':'); if(p!=std::string::npos)a.resize(p); return a;
+}
+void emit_candidate(const Shape &s,size_t workspace,int h,int id,size_t req,size_t algows,float ms,const Err &error,
+                    const std::string &arch,int version,const std::string &sol,const std::string &kernel,const std::string &config) {
+    std::cout<<"candidate_json={\"device_arch\":"<<json(arch)<<",\"hipblaslt_version\":"<<version
+      <<",\"dtype\":"<<json(dtype(s.bf16))<<",\"T\":"<<s.t<<",\"N\":"<<s.n<<",\"K\":"<<s.k<<",\"ldy\":"<<s.ldy
+      <<",\"workspace_limit_bytes\":"<<workspace<<",\"heuristic_index\":"<<h<<",\"solution_id\":"<<id
+      <<",\"required_workspace_bytes\":"<<req<<",\"algo_max_workspace_bytes\":"<<algows
+      <<",\"finite\":"<<(error.finite?"true":"false")<<",\"padding_writes\":"<<error.padding_writes
+      <<",\"relative_l2\":"<<std::setprecision(12)<<error.rel_l2<<",\"max_abs\":"<<error.max_abs
+      <<",\"relative_l2_tolerance\":"<<REL_L2_TOL<<",\"max_abs_tolerance\":"<<MAX_ABS_TOL
+      <<",\"mean_ms\":"<<std::setprecision(9)<<ms<<",\"solution_name\":"<<json(sol)
+      <<",\"kernel_name\":"<<json(kernel)<<",\"algo_config_hex\":"<<json(config)<<"}\n";
+}
+void emit_best(const Best &b,size_t workspace,const std::string &arch,int version,int hipver) {
+    const auto&s=b.s;
+    std::cout<<"best_json={\"device_arch\":"<<json(arch)<<",\"hipblaslt_version\":"<<version
+      <<",\"hip_runtime_version\":"<<hipver<<",\"dtype\":"<<json(dtype(s.bf16))
+      <<",\"T\":"<<s.t<<",\"N\":"<<s.n<<",\"K\":"<<s.k<<",\"ldy\":"<<s.ldy
+      <<",\"workspace_limit_bytes\":"<<workspace<<",\"heuristic_index\":"<<b.heuristic_index
+      <<",\"solution_id\":"<<b.solution_id<<",\"required_workspace_bytes\":"<<b.required_workspace
+      <<",\"algo_max_workspace_bytes\":"<<b.algo_workspace<<",\"mean_ms\":"<<std::setprecision(9)<<b.mean_ms
+      <<",\"finite\":"<<(b.error.finite?"true":"false")<<",\"padding_writes\":"<<b.error.padding_writes
+      <<",\"relative_l2\":"<<std::setprecision(12)<<b.error.rel_l2<<",\"max_abs\":"<<b.error.max_abs
+      <<",\"relative_l2_tolerance\":"<<REL_L2_TOL<<",\"max_abs_tolerance\":"<<MAX_ABS_TOL
+      <<",\"solution_name\":"<<json(b.solution)<<",\"kernel_name\":"<<json(b.kernel)
+      <<",\"algo_config_hex\":"<<json(b.config)<<"}\n";
+}
+
+void run_case(hipblasHandle_t blas,hipblasLtHandle_t lt,const Shape&s,int case_id,size_t ws,hipStream_t stream,
+              const std::string&arch,int version,int hipver,std::vector<Best>&bests) {
+    const size_t na=(size_t)s.k*s.n, nb=(size_t)s.k*s.t, out_elems=(size_t)s.ldy*s.t;
+    const hipDataType it=s.bf16?HIP_R_16BF:HIP_R_16F;
+    const hipblasOperation_t ta=HIPBLAS_OP_T,tb=HIPBLAS_OP_N;
+    const int m=s.n,n=s.t,k=s.k,lda=s.k,ldb=s.k,ldc=s.ldy;
+    const float alpha=1.f,beta=0.f;
+    std::vector<uint16_t> ha(na),hb(nb);
+    uint32_t seed=0x5A17C3U+(uint32_t)case_id*0x10001U+(s.bf16?0xB16U:0xF16U);
+    fill(ha,seed,s.bf16);fill(hb,seed^0x9E3779B9U,s.bf16);
+    Buffer da(na*sizeof(uint16_t)),db(nb*sizeof(uint16_t)),dc(out_elems*sizeof(float)),dy(out_elems*sizeof(float)),dws(ws);
+    const std::vector<float> canary(out_elems,PADDING_CANARY);
+    HIP_CHECK(hipMemcpyAsync(dc.p,canary.data(),out_elems*sizeof(float),hipMemcpyHostToDevice,stream));
+    HIP_CHECK(hipMemcpyAsync(da.p,ha.data(),na*sizeof(uint16_t),hipMemcpyHostToDevice,stream));
+    HIP_CHECK(hipMemcpyAsync(db.p,hb.data(),nb*sizeof(uint16_t),hipMemcpyHostToDevice,stream));
+    HIP_CHECK(hipStreamSynchronize(stream));
+
+    hipblasLtMatmulDesc_t op=nullptr;
+    hipblasLtMatrixLayout_t ad=nullptr,bd=nullptr,cd=nullptr;
+    hipblasLtMatmulPreference_t pref=nullptr;
+    LT_CHECK(hipblasLtMatmulDescCreate(&op,HIPBLAS_COMPUTE_32F,HIP_R_32F));
+    LT_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSA,&ta,sizeof(ta)));
+    LT_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSB,&tb,sizeof(tb)));
+    LT_CHECK(hipblasLtMatrixLayoutCreate(&ad,it,s.k,s.n,lda));
+    LT_CHECK(hipblasLtMatrixLayoutCreate(&bd,it,s.k,s.t,ldb));
+    LT_CHECK(hipblasLtMatrixLayoutCreate(&cd,HIP_R_32F,s.n,s.t,ldc));
+    LT_CHECK(hipblasLtMatmulPreferenceCreate(&pref));
+    const uint64_t limit=ws;
+    LT_CHECK(hipblasLtMatmulPreferenceSetAttribute(pref,HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES,&limit,sizeof(limit)));
+
+    BLAS_CHECK(hipblasSetStream(blas,stream));
+    auto base=[&]{BLAS_CHECK(hipblasGemmEx(blas,ta,tb,m,n,k,&alpha,da.p,it,lda,db.p,it,ldb,&beta,dc.p,HIP_R_32F,ldc,HIPBLAS_COMPUTE_32F,HIPBLAS_GEMM_DEFAULT));};
+    for(int i=0;i<WARMUPS;++i){base();HIP_CHECK(hipDeviceSynchronize());}
+    float base_ms[REPS]{};
+    for(int i=0;i<REPS;++i)base_ms[i]=time_call(stream,base);
+    std::vector<float> ref(out_elems);
+    HIP_CHECK(hipMemcpyAsync(ref.data(),dc.p,out_elems*sizeof(float),hipMemcpyDeviceToHost,stream));
+    HIP_CHECK(hipStreamSynchronize(stream));
+    const size_t baseline_padding=padding_writes(ref,s.n,s.t,s.ldy);
+    std::printf("baseline dtype=%s T=%d N=%d K=%d ldy=%d padding_writes=%zu\n",dtype(s.bf16),s.t,s.n,s.k,s.ldy,baseline_padding);
+    float base_mean=(base_ms[0]+base_ms[1]+base_ms[2])/REPS;
+
+    hipblasLtMatmulHeuristicResult_t hs[MAX_ALGOS]{};
+    int count=0;
+    const hipblasStatus_t query=hipblasLtMatmulAlgoGetHeuristic(lt,op,ad,bd,cd,cd,pref,MAX_ALGOS,hs,&count);
+    std::printf("case dtype=%s T=%d N=%d K=%d ldy=%d workspace_limit_bytes=%zu baseline=hipblasGemmEx rep1_ms=%.4f rep2_ms=%.4f rep3_ms=%.4f mean_ms=%.4f\n",
+      dtype(s.bf16),s.t,s.n,s.k,s.ldy,ws,base_ms[0],base_ms[1],base_ms[2],base_mean);
+    if(query!=HIPBLAS_STATUS_SUCCESS||count==0){
+        std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d status=%s(%d) heuristic_count=%d\n",
+          dtype(s.bf16),s.t,s.n,s.k,s.ldy,status_name(query),(int)query,count);
+    } else {
+        int best_h=-1,best_id=-1; float best_ms=std::numeric_limits<float>::infinity();
+        size_t best_req=0,best_maxws=0; std::string best_sol,best_kernel,best_config;
+        Err best_error{std::numeric_limits<double>::infinity(),std::numeric_limits<double>::infinity(),false,0};
+        for(int h=0;h<count;++h){
+            if(hs[h].state!=HIPBLAS_STATUS_SUCCESS||hs[h].workspaceSize>ws){
+                std::printf("lt_skip dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d status=%s(%d) required_workspace_bytes=%zu\n",
+                  dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,status_name(hs[h].state),(int)hs[h].state,hs[h].workspaceSize);
+                continue;
+            }
+            auto algo=hs[h].algo;
+            const int id=hipblaslt_ext::getIndexFromAlgo(algo);
+            const std::string sol=hipblaslt_ext::getSolutionNameFromAlgo(lt,algo);
+            const std::string kernel=hipblaslt_ext::getKernelNameFromAlgo(lt,algo);
+            const std::string config=config_hex(algo);
+            auto run=[&]{LT_CHECK(hipblasLtMatmul(lt,op,&alpha,da.p,ad,db.p,bd,&beta,dy.p,cd,dy.p,cd,&algo,dws.p,ws,stream));};
+
+            HIP_CHECK(hipMemcpyAsync(dy.p,canary.data(),out_elems*sizeof(float),hipMemcpyHostToDevice,stream));
+            HIP_CHECK(hipStreamSynchronize(stream));
+            const hipblasStatus_t check_status=hipblasLtMatmul(lt,op,&alpha,da.p,ad,db.p,bd,&beta,dy.p,cd,dy.p,cd,&algo,dws.p,ws,stream);
+            if(check_status!=HIPBLAS_STATUS_SUCCESS){
+                std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d status=%s(%d) phase=accuracy_check\n",
+                  dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,status_name(check_status),(int)check_status);continue;
+            }
+            HIP_CHECK(hipStreamSynchronize(stream));
+            std::vector<float> checked_y(out_elems);
+            HIP_CHECK(hipMemcpyAsync(checked_y.data(),dy.p,out_elems*sizeof(float),hipMemcpyDeviceToHost,stream));
+            HIP_CHECK(hipStreamSynchronize(stream));
+            Err error=compare(ref,checked_y,s.n,s.t,s.ldy);
+            error.padding_writes=padding_writes(checked_y,s.n,s.t,s.ldy);
+            if(!accuracy_ok(error)){
+                std::printf("lt_reject dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d finite=%s relative_l2=%.12g relative_l2_tolerance=%.3g max_abs=%.12g max_abs_tolerance=%.3g padding_writes=%zu reason=accuracy_gate\n",
+                  dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,error.finite?"true":"false",
+                  error.rel_l2,REL_L2_TOL,error.max_abs,MAX_ABS_TOL,error.padding_writes);continue;
+            }
+
+            bool failed=false;
+            for(int i=0;i<WARMUPS;++i){
+                const hipblasStatus_t st=hipblasLtMatmul(lt,op,&alpha,da.p,ad,db.p,bd,&beta,dy.p,cd,dy.p,cd,&algo,dws.p,ws,stream);
+                if(st!=HIPBLAS_STATUS_SUCCESS){
+                    std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d status=%s(%d)\n",
+                      dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,status_name(st),(int)st);failed=true;break;
+                }
+                HIP_CHECK(hipDeviceSynchronize());
+            }
+            if(failed)continue;
+            float ms[REPS]{};
+            for(int i=0;i<REPS;++i)ms[i]=time_call(stream,run);
+            const float mean=(ms[0]+ms[1]+ms[2])/REPS;
+            std::printf("lt dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d solution=%s kernel=%s algo_config_hex=%s required_workspace_bytes=%zu algo_max_workspace_bytes=%zu finite=%s relative_l2=%.12g max_abs=%.12g padding_writes=%zu rep1_ms=%.4f rep2_ms=%.4f rep3_ms=%.4f mean_ms=%.4f\n",
+              dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,sol.c_str(),kernel.c_str(),config.c_str(),hs[h].workspaceSize,algo.max_workspace_bytes,
+              error.finite?"true":"false",error.rel_l2,error.max_abs,error.padding_writes,ms[0],ms[1],ms[2],mean);
+            emit_candidate(s,ws,h,id,hs[h].workspaceSize,algo.max_workspace_bytes,mean,error,arch,version,sol,kernel,config);
+            if(mean<best_ms){
+                best_h=h;best_id=id;best_ms=mean;best_req=hs[h].workspaceSize;best_maxws=algo.max_workspace_bytes;
+                best_sol=sol;best_kernel=kernel;best_config=config;best_error=error;
+            }
+        }
+        if(best_h<0)std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d status=no_valid_heuristics heuristic_count=%d\n",
+                                dtype(s.bf16),s.t,s.n,s.k,s.ldy,count);
+        else{
+            Best b{s,best_h,best_id,best_req,best_maxws,best_ms,best_error,best_sol,best_kernel,best_config};
+            bests.push_back(b);
+            std::printf("lt_best dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d solution=%s kernel=%s algo_config_hex=%s required_workspace_bytes=%zu algo_max_workspace_bytes=%zu mean_ms=%.4f speedup_vs_blas=%.4fx finite=%s relative_l2=%.9g max_abs=%.9g padding_writes=%zu\n",
+              dtype(s.bf16),s.t,s.n,s.k,s.ldy,best_h,best_id,best_sol.c_str(),best_kernel.c_str(),best_config.c_str(),best_req,best_maxws,best_ms,base_mean/best_ms,
+              best_error.finite?"true":"false",best_error.rel_l2,best_error.max_abs,best_error.padding_writes);
+            emit_best(b,ws,arch,version,hipver);
+        }
+    }
+    (void)hipblasLtMatmulPreferenceDestroy(pref);
+    (void)hipblasLtMatrixLayoutDestroy(cd);(void)hipblasLtMatrixLayoutDestroy(bd);(void)hipblasLtMatrixLayoutDestroy(ad);
+    (void)hipblasLtMatmulDescDestroy(op);
+}
+void tuning_file(const std::string &path,const std::string &arch,int version,const std::vector<Best>&rows){
+    if(path.empty())return;
+    std::ofstream f(path);
+    if(!f){std::fprintf(stderr,"cannot write tuning output: %s\n",path.c_str());std::exit(2);}
+    f<<"# solution IDs are scoped to this hipBLASLt version and device architecture\n";
+    f<<"STRATA_HIPBLASLT_TUNING_V1 "<<arch<<" "<<version<<"\n";
+    for(const auto &b:rows)f<<dtype(b.s.bf16)<<" "<<b.s.n<<" "<<b.s.k<<" "<<b.s.ldy<<" "<<b.s.t<<" "<<b.solution_id<<"\n";
+}
+} // namespace
+
+int main(int argc,char **argv){
+    Options o=options(argc,argv);
+    hipblasHandle_t blas=nullptr;hipblasLtHandle_t lt=nullptr;
+    BLAS_CHECK(hipblasCreate(&blas));LT_CHECK(hipblasLtCreate(&lt));
+    int device=0,hipver=0,version=0;hipDeviceProp_t prop{};
+    HIP_CHECK(hipGetDevice(&device));HIP_CHECK(hipGetDeviceProperties(&prop,device));HIP_CHECK(hipRuntimeGetVersion(&hipver));
+    const hipblasStatus_t vs=hipblasLtGetVersion(lt,&version);
+    if(vs!=HIPBLAS_STATUS_SUCCESS){std::fprintf(stderr,"hipblasLtGetVersion failed: %d\n",(int)vs);return 2;}
+    const std::string arch=arch_name(prop.gcnArchName);
+    std::printf("meta device_arch=%s hipblaslt_version=%d hipblaslt_header_version=%d.%d.%d hip_runtime_version=%d\n",
+      arch.c_str(),version,HIPBLASLT_VERSION_MAJOR,HIPBLASLT_VERSION_MINOR,HIPBLASLT_VERSION_PATCH,hipver);
+    hipStream_t stream=nullptr;HIP_CHECK(hipStreamCreateWithFlags(&stream,hipStreamNonBlocking));
+    std::vector<Best>bests;
+    for(size_t i=0;i<o.shapes.size();++i)run_case(blas,lt,o.shapes[i],(int)i,o.workspace,stream,arch,version,hipver,bests);
+    tuning_file(o.tuning_out,arch,version,bests);
+    HIP_CHECK(hipStreamSynchronize(stream));HIP_CHECK(hipStreamDestroy(stream));
+    LT_CHECK(hipblasLtDestroy(lt));BLAS_CHECK(hipblasDestroy(blas));
+    return 0;
+}

tools/strata_pack.py

@@ -338,6 +338,15 @@ def verify_experts(gguf: pathlib.Path, out_dir: pathlib.Path, man: dict, every:
 def verify(gguf: pathlib.Path, out_dir: pathlib.Path, limit: int | None, expert_every: int) -> int:
     """P1.T3 over the pack: decode it from disk and compare with the source, bit for bit."""
     man = json.loads((out_dir / "manifest.json").read_text(encoding="utf-8"))
+    # A full verify also reads back the source hash build recorded (absent after --skip-hash, which is how setup
+    # builds): a pack compared against a different shard than it was built from is a mismatch, not a decode bug.
+    want = (man.get("source") or {}).get("shard1_sha256")
+    if want and limit is None:
+        got = sha256(gguf)
+        if got != want:
+            print("%s: sha256 %s, but the pack was built from %s" % (gguf.name, got, want))
+            print("tools/strata_pack.py verify FAIL")
+            return 1
     g = G.GGUFFile(gguf)
     head, flen = open_shard(gguf)
     data_off = data_section_offset(g, flen)

tools/test_shards.py

@@ -0,0 +1,119 @@
+"""A missing or short model shard is named with its numbers, and a pack's verify reads back the source hash
+its manifest recorded - over a minimal GGUF written here (no download, no model, no GPU).
+
+    python -m unittest tools.test_shards
+"""
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import io
+import json
+import struct
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tools"))
+sys.path.insert(0, str(ROOT))
+import setup as S  # noqa: E402
+import strata_pack  # noqa: E402
+
+
+class Stop(Exception):
+    """setup.fail(), caught instead of exiting: the message is what the test reads."""
+
+
+def write_gguf(path: Path, names=("blk.0.attn_q.weight", "blk.0.attn_k.weight")) -> int:
+    """A GGUF v3 of F32[8] tensors 32 bytes apart from a 32-byte-aligned data start; returns its whole length."""
+    b = bytearray(struct.pack("<IIQQ", 0x46554747, 3, len(names), 0))
+    for i, n in enumerate(names):
+        b += struct.pack("<Q", len(n)) + n.encode() + struct.pack("<IQIQ", 1, 8, 0, 32 * i)
+    b += bytes(-len(b) % 32 + 32 * len(names))
+    path.write_bytes(b)
+    return len(b)
+
+
+class ShardCheck(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.dir = Path(self.tmp.name)
+        self.saved = S.fail
+        S.fail = lambda msg, hint=None: (_ for _ in ()).throw(Stop(msg))
+
+    def tearDown(self):
+        S.fail = self.saved
+        self.tmp.cleanup()
+
+    def test_whole_shards_pass(self):
+        shards = [self.dir / f"m-0000{i}-of-00002.gguf" for i in (1, 2)]
+        for s in shards:
+            write_gguf(s)
+        S.check_shards(shards)
+
+    def test_missing_shard_is_named(self):
+        shards = [self.dir / "m-00001-of-00002.gguf", self.dir / "m-00002-of-00002.gguf"]
+        write_gguf(shards[0])
+        with self.assertRaises(Stop) as cm:
+            S.check_shards(shards)
+        self.assertIn("m-00002-of-00002.gguf", str(cm.exception))
+
+    def test_short_shard_names_file_and_sizes(self):
+        s = self.dir / "m-00001-of-00002.gguf"
+        whole = write_gguf(s)
+        s.write_bytes(s.read_bytes()[:-40])
+        with self.assertRaises(Stop) as cm:
+            S.check_shards([s])
+        msg = str(cm.exception)
+        self.assertIn("m-00001-of-00002.gguf is short", msg)
+        self.assertIn(f"{whole - 40:,} of {whole:,} bytes", msg)
+        self.assertIn("40 missing", msg)
+
+    def test_truncated_header_is_refused(self):
+        s = self.dir / "m-00001-of-00002.gguf"
+        write_gguf(s)
+        s.write_bytes(s.read_bytes()[:12])
+        with self.assertRaises(Stop) as cm:
+            S.check_shards([s])
+        self.assertIn("m-00001-of-00002.gguf is not a whole GGUF shard", str(cm.exception))
+
+
+class PackVerifyHash(unittest.TestCase):
+    """verify over a manifest with no tensor entries: the only check left is the source hash."""
+
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.dir = Path(self.tmp.name)
+        self.gguf = self.dir / "m-00001-of-00002.gguf"
+        write_gguf(self.gguf)
+        self.digest = hashlib.sha256(self.gguf.read_bytes()).hexdigest()
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def run_verify(self, source, limit=None):
+        (self.dir / "manifest.json").write_text(json.dumps({"tensors": {}, "source": source}), encoding="utf-8")
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            rc = strata_pack.verify(self.gguf, self.dir, limit, 0)
+        return rc, out.getvalue()
+
+    def test_matching_hash_passes(self):
+        rc, _ = self.run_verify({"shard1_sha256": self.digest})
+        self.assertEqual(rc, 0)
+
+    def test_wrong_hash_fails_naming_both(self):
+        rc, out = self.run_verify({"shard1_sha256": "0" * 64})
+        self.assertEqual(rc, 1)
+        self.assertIn("m-00001-of-00002.gguf: sha256 " + self.digest, out)
+        self.assertIn("built from " + "0" * 64, out)
+
+    def test_no_hash_or_limit_skips(self):
+        self.assertEqual(self.run_verify({})[0], 0)                                  # setup builds --skip-hash
+        self.assertEqual(self.run_verify({"shard1_sha256": "0" * 64}, limit=1)[0], 0)  # --limit stays quick
+
+
+if __name__ == "__main__":
+    unittest.main()