docs: the low-RAM mode

This reverts commit 45d857c7cf.

CMakeLists.txt

@@ -8,7 +8,7 @@
 # below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
 # and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.23 LANGUAGES CXX)
+project(strata VERSION 0.1.26 LANGUAGES CXX)
 add_compile_definitions(STRATA_VERSION="${PROJECT_VERSION}")
 # issue #31: release builds keep a PDB beside the exe (not shipped in it), so a minidump from a stalled engine can
 # be read back to the exact source line
@@ -33,6 +33,17 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 option(STRATA_ENABLE_CUDA "Build the CUDA targets (Phase 2+)" OFF)
+option(STRATA_ENABLE_HIP "Build the HIP targets for gfx1100 wave32 hardware" OFF)
+option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF)
+if(STRATA_PREFILL_MMQ AND NOT STRATA_ENABLE_HIP)
+  message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_ENABLE_HIP=ON")
+endif()
+if(STRATA_ENABLE_CUDA AND STRATA_ENABLE_HIP)
+  message(FATAL_ERROR "STRATA_ENABLE_CUDA and STRATA_ENABLE_HIP are separate backends; enable only one")
+endif()
+if(STRATA_ENABLE_HIP)
+  include(cmake/hip_backend.cmake)
+endif()
 # The test suite and the micro benchmarks live in tests/ and bench/micro/, which the published source leaves out.
 set(_strata_tests_default OFF)
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/bench/micro)
@@ -63,6 +74,13 @@ if(STRATA_ENABLE_CUDA)
     endif()
   endforeach()
   message(STATUS "Strata: CUDA enabled, arch ${CMAKE_CUDA_ARCHITECTURES}")
+  set(_strata_gpu_runtime_target CUDA::cudart)
+  set(_strata_gpu_blas_target CUDA::cublas)
+  set(_strata_gpu_include_directories ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+elseif(STRATA_ENABLE_HIP)
+  set(_strata_gpu_runtime_target strata_hip_runtime)
+  set(_strata_gpu_blas_target roc::hipblas)
+  set(_strata_gpu_include_directories ${STRATA_HIP_COMPAT_INCLUDE_DIR})
 endif()
 
 # Warnings live on an INTERFACE target the project's own code links, NOT in add_compile_options: applied
@@ -78,11 +96,14 @@ target_compile_options(strata_warnings INTERFACE
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/utf-8>
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Wall>
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Wextra>
-  $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CXX_COMPILER_ID:MSVC>>:-Xcompiler=/W4>)
+  $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CXX_COMPILER_ID:MSVC>>:-Xcompiler=/W4>
+  $<$<COMPILE_LANGUAGE:HIP>:-Wall>
+  $<$<COMPILE_LANGUAGE:HIP>:-Wextra>)
 if(STRATA_WERROR)
   target_compile_options(strata_warnings INTERFACE
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/WX>
-    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Werror>)
+    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Werror>
+    $<$<COMPILE_LANGUAGE:HIP>:-Werror>)
 endif()
 
 # ---- strata_artifact -------------------------------------------------------
@@ -117,17 +138,16 @@ target_link_libraries(strata_plan INTERFACE strata_warnings)
 add_executable(strata-plan src/plan/plan_main.cpp)
 target_link_libraries(strata-plan PRIVATE strata_plan)
 
-# ---- strata_core (CUDA, Phase 2) ------------------------------------------
-# Only built when CUDA is on, so the Phase 1 targets keep working on a machine with no toolkit.  The arch
-# guard above has already refused anything but sm_120 by the time this runs.
-if(STRATA_ENABLE_CUDA)
+# ---- strata_core (GPU runtime) ---------------------------------------------
+# Only built when a GPU backend is on, so the Phase 1 targets keep working without a toolkit.
+if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   add_library(strata_core STATIC src/core/device.cu src/core/pinned.cu src/platform/memory.cpp src/core/graph.cpp src/core/weights.cpp
                                src/core/layout.cpp)
   target_include_directories(strata_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-  # CUDA::cudart is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and
+  # The selected runtime is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and
   # cudaEventQuery: the .cu sources get cudart through the CUDA language rules, a .cpp does not, and the
   # linker error ("unresolved external symbol cudaGraphInstantiate") names the function rather than the cause.
-  target_link_libraries(strata_core PUBLIC strata_plan strata_warnings CUDA::cudart)
+  target_link_libraries(strata_core PUBLIC strata_plan strata_warnings ${_strata_gpu_runtime_target})
 
   add_executable(strata-device src/core/device_main.cpp)
   target_link_libraries(strata-device PRIVATE strata_core)
@@ -136,13 +156,13 @@ if(STRATA_ENABLE_CUDA)
   target_link_libraries(strata-load PRIVATE strata_core strata_kernels)
 
   add_executable(strata-concurrent src/core/concurrent_main.cpp)
-  target_include_directories(strata-concurrent PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels CUDA::cudart)
+  target_include_directories(strata-concurrent PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target})
 
   add_executable(strata-overlap src/core/overlap_main.cpp)
   # a .cpp that calls cudaMemcpyAsync itself needs the toolkit headers and cudart explicitly
-  target_include_directories(strata-overlap PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels CUDA::cudart)
+  target_include_directories(strata-overlap PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target})
 
   # The load test needs the 34 GB artifact, so it is registered ONLY where that artifact exists.  A test that
   # silently skips is a test that stops running; an absent registration is visible in the configure output.
@@ -215,30 +235,38 @@ if(STRATA_ENABLE_CUDA)
   # plan v0.3 P6: llama.cpp's block layouts and codebook grids for the i-quant formats (MIT, third_party/ggml)
   target_include_directories(strata_kernels PRIVATE third_party/ggml)
   # Match the pinned llama.cpp CUDA arithmetic contract for this adapter only.
-  set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu
-    src/kernels/cuda/native_gr_norm.cu
-    src/kernels/cuda/native_moe.cu
-    src/kernels/cuda/native_gdn.cu
-    src/kernels/cuda/native_gr_postops.cu
-    src/kernels/cuda/native_router.cu
-    src/kernels/cuda/native_gdn_preprocess.cu
-    src/kernels/cuda/native_qsa.cu
-    src/kernels/cuda/native_rope.cu
-    src/kernels/cuda/native_ple_postops.cu
-    src/kernels/cuda/native_flash_attn.cu
-    src/kernels/cuda/native_qsa_indexer.cu
-    src/kernels/cuda/native_qsa_score.cu
-    PROPERTIES COMPILE_OPTIONS "--use_fast_math")
+  if(STRATA_ENABLE_CUDA)
+    set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu
+      src/kernels/cuda/native_gr_norm.cu
+      src/kernels/cuda/native_moe.cu
+      src/kernels/cuda/native_gdn.cu
+      src/kernels/cuda/native_gr_postops.cu
+      src/kernels/cuda/native_router.cu
+      src/kernels/cuda/native_gdn_preprocess.cu
+      src/kernels/cuda/native_qsa.cu
+      src/kernels/cuda/native_rope.cu
+      src/kernels/cuda/native_ple_postops.cu
+      src/kernels/cuda/native_flash_attn.cu
+      src/kernels/cuda/native_qsa_indexer.cu
+      src/kernels/cuda/native_qsa_score.cu
+      PROPERTIES COMPILE_OPTIONS "--use_fast_math")
+  elseif(STRATA_ENABLE_HIP)
+    # Keep Q8_K's multiply and magic-bias rounding as separate FP32 operations on HIP; contraction to an
+    # FMA changes tie results. The embedding gather also relies on separately rounded operations.
+    set_source_files_properties(src/kernels/cuda/elementwise.cu src/kernels/cuda/quantize_act.cu
+      PROPERTIES COMPILE_OPTIONS "-ffp-contract=off")
+  endif()
 
   set(STRATA_ORACLE_SOURCE_DIR "" CACHE PATH "Pinned llama.cpp source for independent CUDA parity tests")
   set(STRATA_ORACLE_BUILD_DIR "" CACHE PATH "Matching existing shared llama.cpp build")
-  if(STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR)
+  # This oracle is a CUDA ggml build; do not pull CUDA libraries into a HIP build.
+  if(STRATA_ENABLE_CUDA AND STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR)
     foreach(parity IN ITEMS native_mmvq native_bf16 native_shared native_gr_norm native_moe native_gdn native_gr_postops native_router native_gdn_preprocess native_gdn_pipeline native_qsa native_rope native_ple_postops native_flash_attn native_qsa_indexer native_qsa_score)
     add_executable(${parity}_parity bench/micro/${parity}_parity.cpp)
     target_include_directories(${parity}_parity PRIVATE
-      "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+      "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${_strata_gpu_include_directories})
     target_compile_definitions(${parity}_parity PRIVATE GGML_SHARED GGML_BACKEND_SHARED)
-    target_link_libraries(${parity}_parity PRIVATE strata_kernels CUDA::cudart)
+    target_link_libraries(${parity}_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
     foreach(component IN ITEMS ggml ggml-base ggml-cpu ggml-cuda)
       string(REPLACE "-" "_" variable "STRATA_ORACLE_${component}")
       find_library(${variable} NAMES ${component} PATHS
@@ -262,99 +290,163 @@ if(STRATA_ENABLE_CUDA)
   add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
     src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-  target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu CUDA::cudart)
+  target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/file_expert_source_test.cpp")
+    enable_testing()
+    add_executable(file_expert_source_test tests/core/file_expert_source_test.cpp)
+    target_link_libraries(file_expert_source_test PRIVATE strata_engine)
+    add_test(NAME file_expert_source_test COMMAND file_expert_source_test)
+    set_tests_properties(file_expert_source_test PROPERTIES TIMEOUT 60)
+  endif()
 
   # `strata generate` (P2.S6).  The FIRST user-facing program: everything under it is a component, and this is
   # what composes them into a token.  No tokenizer - the prompt arrives as ids - because Gate C1 compares logits
   # against llama.cpp on the SAME ids and a tokenizer on one side of that comparison is a second variable.
   # plan v0.3 P5: batched prompt processing (tensor-core GEMMs through cuBLAS).
   add_library(strata_prefill STATIC src/prefill/gemm.cu src/prefill/kernels.cu src/prefill/prefill.cpp)
-  target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata_prefill PUBLIC strata_engine CUDA::cublas CUDA::cudart)
+  target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})
+  target_link_libraries(strata_prefill PUBLIC strata_engine ${_strata_gpu_blas_target} ${_strata_gpu_runtime_target})
+  if(STRATA_ENABLE_HIP AND STRATA_HIPBLASLT_AVAILABLE)
+    target_compile_definitions(strata_prefill PRIVATE STRATA_HIPBLASLT_AVAILABLE=1)
+    target_link_libraries(strata_prefill PRIVATE roc::hipblaslt)
+
+    add_executable(tune_hipblaslt tools/hip/tune_hipblaslt.cpp)
+    target_link_libraries(tune_hipblaslt PRIVATE hip::host roc::hipblas roc::hipblaslt)
+  endif()
 
   add_executable(strata src/program/generate.cpp)
-  target_include_directories(strata PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec CUDA::cudart)
+  target_include_directories(strata PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec ${_strata_gpu_runtime_target})
+
+  if(STRATA_ENABLE_HIP)
+    enable_testing()
+    set(_strata_hip_tests)
+    add_executable(hip_expert_cache_staging tests/hip/expert_cache_staging.cpp)
+    target_link_libraries(hip_expert_cache_staging PRIVATE strata_engine)
+    add_test(NAME hip_expert_cache_staging COMMAND hip_expert_cache_staging)
+    list(APPEND _strata_hip_tests hip_expert_cache_staging)
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/intrinsics.cpp")
+      add_executable(hip_intrinsics tests/hip/intrinsics.cpp)
+      target_link_libraries(hip_intrinsics PRIVATE strata_hip_runtime)
+      add_test(NAME hip_intrinsics COMMAND hip_intrinsics)
+      list(APPEND _strata_hip_tests hip_intrinsics)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/handoff.cpp")
+      add_executable(hip_handoff tests/hip/handoff.cpp)
+      target_link_libraries(hip_handoff PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_handoff COMMAND hip_handoff)
+      list(APPEND _strata_hip_tests hip_handoff)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/native_qsa_score.cpp")
+      add_executable(hip_native_qsa_score tests/hip/native_qsa_score.cpp)
+      target_link_libraries(hip_native_qsa_score PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_native_qsa_score COMMAND hip_native_qsa_score)
+      list(APPEND _strata_hip_tests hip_native_qsa_score)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_native_batch.cpp")
+      add_executable(hip_prefill_native_batch tests/hip/prefill_native_batch.cpp)
+      target_link_libraries(hip_prefill_native_batch PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_prefill_native_batch COMMAND hip_prefill_native_batch)
+      list(APPEND _strata_hip_tests hip_prefill_native_batch)
+    endif()
+    if(STRATA_HIPBLASLT_AVAILABLE AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_hipblaslt_gemm.cpp")
+      add_executable(hip_prefill_hipblaslt_gemm tests/hip/prefill_hipblaslt_gemm.cpp)
+      target_include_directories(hip_prefill_hipblaslt_gemm PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/prefill")
+      target_link_libraries(hip_prefill_hipblaslt_gemm PRIVATE strata_prefill strata_hip_runtime roc::hipblaslt roc::hipblas)
+      add_test(NAME hip_prefill_hipblaslt_gemm COMMAND hip_prefill_hipblaslt_gemm)
+      set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77)
+      list(APPEND _strata_hip_tests hip_prefill_hipblaslt_gemm)
+    endif()
+    if(_strata_hip_tests)
+      set_tests_properties(${_strata_hip_tests} PROPERTIES TIMEOUT 60)
+    endif()
+    if(TARGET hip_prefill_native_batch)
+      set_tests_properties(hip_prefill_native_batch PROPERTIES TIMEOUT 120)
+    endif()
+    if(TARGET hip_prefill_hipblaslt_gemm)
+      set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77)
+    endif()
+  endif()
 
   add_executable(dequant_s2_parity src/kernels/dequant_s2_parity.cpp)
   # The tests are HOST C++ files that call cudaMalloc/cudaMemcpy themselves, so they need the toolkit headers
   # and cudart explicitly: enabling the CUDA language gives those to .cu sources automatically, but a .cpp
   # target is just a C++ target and gets neither.
-  target_include_directories(dequant_s2_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(dequant_s2_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(dequant_s2_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(dequant_s2_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME dequant_s2_parity COMMAND dequant_s2_parity --selftest)
 
   add_executable(s2_gemv_parity src/kernels/s2_gemv_parity.cpp)
-  target_include_directories(s2_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s2_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s2_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s2_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s2_gemv_parity COMMAND s2_gemv_parity --selftest)
 
   add_executable(iq_parity src/kernels/iq_parity.cpp)
-  target_include_directories(iq_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(iq_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(iq_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(iq_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)
-  target_include_directories(shared_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(shared_expert_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME shared_expert_parity COMMAND shared_expert_parity --selftest)
 
   add_executable(gr_parity src/kernels/gr_parity.cpp)
-  target_include_directories(gr_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(gr_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(gr_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(gr_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME gr_parity COMMAND gr_parity --selftest)
 
   add_executable(gdn_parity src/kernels/gdn_parity.cpp)
-  target_include_directories(gdn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(gdn_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(gdn_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(gdn_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME gdn_parity COMMAND gdn_parity --selftest)
 
   add_executable(s2_gemv_q8_parity src/kernels/s2_gemv_q8_parity.cpp)
-  target_include_directories(s2_gemv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s2_gemv_q8_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s2_gemv_q8_parity COMMAND s2_gemv_q8_parity --selftest)
 
   add_executable(sampler_parity src/kernels/sampler_parity.cpp)
-  target_include_directories(sampler_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(sampler_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(sampler_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(sampler_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME sampler_parity COMMAND sampler_parity --selftest)
 
   add_executable(rope_parity src/kernels/rope_parity.cpp)
-  target_include_directories(rope_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(rope_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(rope_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(rope_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME rope_parity COMMAND rope_parity --selftest)
 
   add_executable(quantize_act_parity src/kernels/quantize_act_parity.cpp)
-  target_include_directories(quantize_act_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(quantize_act_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(quantize_act_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(quantize_act_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME quantize_act_parity COMMAND quantize_act_parity --selftest)
 
   add_executable(router_top10_parity src/kernels/router_top10_parity.cpp)
-  target_include_directories(router_top10_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(router_top10_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(router_top10_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(router_top10_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME router_top10_parity COMMAND router_top10_parity --selftest)
 
   add_executable(s_gemv_parity src/kernels/s_gemv_parity.cpp)
-  target_include_directories(s_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s_gemv_parity COMMAND s_gemv_parity --selftest)
 
   add_executable(elementwise_parity src/kernels/elementwise_parity.cpp)
-  target_include_directories(elementwise_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(elementwise_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(elementwise_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(elementwise_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME elementwise_parity COMMAND elementwise_parity --selftest)
 
   add_executable(bf16_gemv_parity src/kernels/bf16_gemv_parity.cpp)
-  target_include_directories(bf16_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(bf16_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME bf16_gemv_parity COMMAND bf16_gemv_parity --selftest)
 
   add_executable(s_gemv_q8k_parity src/kernels/s_gemv_q8k_parity.cpp)
-  target_include_directories(s_gemv_q8k_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s_gemv_q8k_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s_gemv_q8k_parity COMMAND s_gemv_q8k_parity --selftest)
 
   add_executable(qsa_parity src/kernels/qsa_parity.cpp)
-  target_include_directories(qsa_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(qsa_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(qsa_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(qsa_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME qsa_parity COMMAND qsa_parity --selftest)
 
   # ---- P2.S4: the PLE n-gram hash, the IQ4_NL table read, and the PLE block ----
@@ -366,8 +458,8 @@ if(STRATA_ENABLE_CUDA)
   # INSIDE the test.  A test that passes by finding nothing to compare is the failure mode this project keeps
   # re-learning.
   add_executable(ple_parity src/kernels/ple_parity.cpp)
-  target_include_directories(ple_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(ple_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(ple_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(ple_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   set(STRATA_PLE_FIXTURE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro" CACHE PATH
       "Directory containing required ple_in.bin and ple_out.bin parity fixtures")
   add_test(NAME ple_parity COMMAND ple_parity --selftest
@@ -376,73 +468,86 @@ if(STRATA_ENABLE_CUDA)
 
   # Plan v0.3 P7: INT8 KV storage vs a host reference and vs the FP16 path (GPU, synthetic, no model).
   add_executable(kv_q8_parity src/kernels/kv_q8_parity.cpp)
-  target_include_directories(kv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_q8_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_q8_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_q8_parity COMMAND kv_q8_parity)
 
   # KV streaming (docs/kv-streaming-design.md): streamed vs resident attention, bitwise (GPU, synthetic, no model).
   add_executable(kv_stream_parity src/kernels/kv_stream_parity.cpp)
-  target_include_directories(kv_stream_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_stream_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_stream_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_stream_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_stream_parity COMMAND kv_stream_parity)
 
+  # PR #120: the hybrid K8V4 KV (INT8 K + rotated Q4_0 V) vs host references (GPU, synthetic, no model).
+  add_executable(kv_hybrid_parity src/kernels/kv_hybrid_parity.cpp)
+  target_include_directories(kv_hybrid_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_hybrid_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  add_test(NAME kv_hybrid_parity COMMAND kv_hybrid_parity)
   # PR #21: Q4_0 KV with Walsh-Hadamard rotation vs a host reference (GPU, synthetic, no model).
   add_executable(kv_q4_parity src/kernels/kv_q4_parity.cpp)
-  target_include_directories(kv_q4_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_q4_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_q4_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_q4_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_q4_parity COMMAND kv_q4_parity)
   add_executable(cvec_parity src/kernels/cvec_parity.cpp)
-  target_include_directories(cvec_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(cvec_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(cvec_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(cvec_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME cvec_parity COMMAND cvec_parity)
 
   # Plan v0.3 P3: native MMVQ for 1..8 tokens (bitwise selftest + --bench timing); synthetic data, no model.
   if(STRATA_BUILD_TESTS)
   add_executable(native_mmvq_multi bench/micro/native_mmvq_multi.cpp)
-  target_include_directories(native_mmvq_multi PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(native_mmvq_multi PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(native_mmvq_multi PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(native_mmvq_multi PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME native_mmvq_multi COMMAND native_mmvq_multi --selftest)
   endif()
 
   # Plan v0.3 P0.1: working-set lock of a host region (CPU only).
   add_executable(platform_memory_test src/platform/memory_test.cpp)
-  target_link_libraries(platform_memory_test PRIVATE strata_core CUDA::cudart)
+  target_link_libraries(platform_memory_test PRIVATE strata_core ${_strata_gpu_runtime_target})
   add_test(NAME platform_memory_test COMMAND platform_memory_test)
 
   # Plan v0.3 P2: the n-gram table read straight from the SSD. The selftest needs only the CPU and a temp file.
   add_executable(ple_reader_test src/ngram/ple_reader_test.cpp)
-  target_include_directories(ple_reader_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(ple_reader_test PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(ple_reader_test PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(ple_reader_test PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME ple_reader_selftest COMMAND ple_reader_test --selftest)
 
-  # The GPU test is registered only when CUDA is on, and it runs ON THE TARGET PC.  `--selftest` allocates
+  # The GPU test runs ON THE TARGET PC. `--selftest` allocates
   # and writes real VRAM, so a machine without a device fails this test rather than skipping it silently -
   # which is correct: an engine that targets one GPU should not report a pass for having no GPU.
-  add_test(NAME cuda_device_selftest COMMAND strata-device --selftest)
+  if(STRATA_ENABLE_CUDA)
+    add_test(NAME cuda_device_selftest COMMAND strata-device --selftest)
+  else()
+    add_test(NAME hip_device_selftest COMMAND strata-device --selftest)
+  endif()
 
   # ---- P2.S5: the GraphRegistry --------------------------------------------------
   # A .cu because the test needs real kernels to capture; it links `strata_core` and not `strata_kernels`,
   # because what it exercises is the capture machinery and not any kernel in it.
-  if(STRATA_BUILD_TESTS)
-  add_executable(graph_registry_test bench/micro/graph_registry_test.cu)
-  target_include_directories(graph_registry_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(graph_registry_test PRIVATE strata_core CUDA::cudart)
-  add_test(NAME graph_registry_test COMMAND graph_registry_test)
+  if(STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/graph_registry_test.cu")
+    add_executable(graph_registry_test bench/micro/graph_registry_test.cu)
+    target_include_directories(graph_registry_test PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(graph_registry_test PRIVATE strata_core ${_strata_gpu_runtime_target})
+    add_test(NAME graph_registry_test COMMAND graph_registry_test)
+  endif()
 
   # ---- P2.S5's GATE: is a device<->pinned-memory handoff graph-capturable? --------------------
   # The doorbell protocol puts the D2H of `x_f` and the miss list INSIDE G[l] - that is the whole overlap.  The
   # phase doc says to verify this "before building on it", and this project has already lost two rounds to
   # building first (a memory-only spin that DEADLOCKS on Windows, and a `cudaEventRecord` inside a capture that
   # is SILENTLY DROPPED).  It links only cudart: it is about the capture machinery, not about any kernel.
-  add_executable(pinned_capture bench/micro/pinned_capture.cu)
-  target_include_directories(pinned_capture PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(pinned_capture PRIVATE CUDA::cudart)
-  add_test(NAME pinned_capture COMMAND pinned_capture)
+  if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/pinned_capture.cu")
+    add_executable(pinned_capture bench/micro/pinned_capture.cu)
+    target_include_directories(pinned_capture PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(pinned_capture PRIVATE ${_strata_gpu_runtime_target})
+    add_test(NAME pinned_capture COMMAND pinned_capture)
   endif()
 
   # Tells tests/CMakeLists.txt to add the CUDA-dependent Catch2 target.  It cannot be declared there
   # unconditionally: tests/ is added at the end of this file, but the flag has to be known before it runs.
-  set(STRATA_CUDA_TESTS ON)
+  if(STRATA_ENABLE_CUDA)
+    set(STRATA_CUDA_TESTS ON)
+  endif()
 endif()
 
 # ---- strata_kernels_cpu (P2.S3) ----
@@ -499,6 +604,9 @@ target_link_libraries(strata_kernels_cpu PUBLIC Threads::Threads)
 # with -DSTRATA_GGML_DIR=..., otherwise the pinned upstream commit (third_party/ggml/VERSION.txt).
 option(STRATA_NATIVE_EXPERTS "Native (IQ) expert support through ggml-cpu" ON)
 set(STRATA_GGML_DIR "" CACHE PATH "llama.cpp checkout to take ggml from (empty: fetch the pinned commit)")
+if(STRATA_PREFILL_MMQ AND NOT STRATA_NATIVE_EXPERTS)
+  message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_NATIVE_EXPERTS=ON for GGML sources")
+endif()
 if(STRATA_NATIVE_EXPERTS)
   enable_language(C)
   set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
@@ -556,7 +664,7 @@ if(STRATA_NATIVE_EXPERTS)
     endif()
     set(_strata_mmq ${_strata_gg}/src/ggml-cuda)
     add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
-    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs)
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)   # q8_0: the drafter's matrices (E-9)
       target_sources(strata_mmq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu)
     endforeach()
     target_include_directories(strata_mmq PRIVATE ${_strata_gg}/include ${_strata_gg}/src ${_strata_mmq}
@@ -565,29 +673,70 @@ if(STRATA_NATIVE_EXPERTS)
     if(MSVC)
       target_compile_options(strata_mmq PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/bigobj>)
     endif()
-    target_link_libraries(strata_mmq PUBLIC ggml-base CUDA::cudart)
+    target_link_libraries(strata_mmq PUBLIC ggml-base ${_strata_gpu_runtime_target})
     target_link_libraries(strata_prefill PUBLIC strata_mmq)
     target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+  elseif(STRATA_PREFILL_MMQ)
+    # Compile only the pinned GGML MMQ and activation-quantizer sources used by prefill. This uses
+    # common.cuh's native HIP path without enabling a second GGML HIP backend.
+    if(STRATA_GGML_DIR)
+      set(_strata_gg "${STRATA_GGML_DIR}/ggml")
+    else()
+      set(_strata_gg "${strata_llamacpp_SOURCE_DIR}/ggml")
+    endif()
+    set(_strata_mmq "${_strata_gg}/src/ggml-cuda")
+    if(NOT EXISTS "${_strata_mmq}/common.cuh" OR NOT EXISTS "${_strata_mmq}/quantize.cu")
+      message(FATAL_ERROR "The selected GGML source is missing the ggml-cuda MMQ sources: ${_strata_mmq}")
+    endif()
+    set(_strata_mmq_sources
+      src/prefill/moe_mmq.cu
+      src/prefill/ggml_cuda_host.cu
+      "${_strata_mmq}/quantize.cu")
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)
+      list(APPEND _strata_mmq_sources "${_strata_mmq}/template-instances/mmq-instance-${_t}.cu")
+    endforeach()
+    set_source_files_properties(${_strata_mmq_sources} PROPERTIES LANGUAGE HIP)
+    add_library(strata_mmq STATIC ${_strata_mmq_sources})
+    target_include_directories(strata_mmq
+      PRIVATE "${_strata_gg}/include" "${_strata_gg}/src" "${_strata_mmq}"
+      PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    target_compile_definitions(strata_mmq PRIVATE GGML_USE_HIP=1)
+    target_link_libraries(strata_mmq PUBLIC ggml-base hip::host roc::hipblas)
+    target_link_libraries(strata_prefill PUBLIC strata_mmq)
+    target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+
+    # Numerical diagnostic: synthetic layouts by default, optional first-expert IQ3_XXS/IQ4_NL check with a pack.
+    add_executable(hip_prefill_mmq_parity tests/hip/prefill_mmq_parity.cpp)
+    target_include_directories(hip_prefill_mmq_parity PRIVATE "${_strata_gg}/include")
+    target_link_libraries(hip_prefill_mmq_parity PRIVATE strata_prefill)
+    add_test(NAME hip_prefill_mmq_parity COMMAND hip_prefill_mmq_parity)
+    set_tests_properties(hip_prefill_mmq_parity PROPERTIES TIMEOUT 120)
+    list(APPEND _strata_hip_tests hip_prefill_mmq_parity)
   endif()
-  if(STRATA_ENABLE_CUDA)
+  if(STRATA_ENABLE_HIP)
+    # Optional real-model diagnostic: pass the GGUF shard containing the native PLE key.
+    add_executable(hip_ple_iq4 tests/hip/ple_iq4.cpp)
+    target_link_libraries(hip_ple_iq4 PRIVATE strata_kernels strata_hip_runtime ggml-base)
+  endif()
+  if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     add_executable(native_expert_parity src/kernels/native_expert_parity.cpp)
-    target_include_directories(native_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-    target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact CUDA::cudart)
+    target_include_directories(native_expert_parity PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact ${_strata_gpu_runtime_target})
   endif()
 endif()
 
 if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS)
   add_executable(hit_cpu_order_parity bench/micro/hit_cpu_order_parity.cu)
-  target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu CUDA::cudart)
+  target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
 endif()
 
 if(STRATA_BUILD_TESTS)
   # Plan v0.3 P6: one expert applied to several tokens, bitwise vs the single-token kernel (synthetic, CPU only).
   # plan v0.3 P5: the device dequantizers vs the artifact's CPU dequantizers on the real shards (not a ctest: needs
   # the 38 GB model; run it by hand or from tools/gpu_session.py)
-  if(STRATA_ENABLE_CUDA)   # #93: it needs the device dequantizers and cudart (a CPU-only configure has neither)
+  if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)   # #93: it needs the device dequantizers and a GPU runtime
     add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp)
-    target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact CUDA::cudart)
+    target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact ${_strata_gpu_runtime_target})
   endif()
 
   add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp)
@@ -608,12 +757,19 @@ if(STRATA_BUILD_TESTS)
   add_executable(pool_stress src/kernels/cpu/pool_stress.cpp)
   target_link_libraries(pool_stress PRIVATE strata_kernels_cpu)
   add_test(NAME pool_stress COMMAND pool_stress 20)
+
+  if(UNIX AND NOT APPLE)
+    add_executable(direct_file_async_test tests/platform/direct_file_async.cpp src/platform/direct_file.cpp)
+    target_include_directories(direct_file_async_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    target_link_libraries(direct_file_async_test PRIVATE Threads::Threads)
+    add_test(NAME direct_file_async_test COMMAND direct_file_async_test)
+  endif()
 endif()
 
 if(STRATA_BUILD_TESTS)
   enable_testing()
-  # the unit tests above build from src/; the tests/ tree (model-level checks) is not part of the published repo
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt)
+  # Model-level checks may be absent from source distributions.
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt")
     add_subdirectory(tests)
   endif()
 endif()

README.md

@@ -84,9 +84,14 @@ one later with `SETUP.bat` (the same as `START-HERE.bat --setup`; on Linux `./se
 For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](docs/ORCA.md).
 It needs an explicit packing conversion and is not an installer menu option.
 
+An **AMD Radeon RX 7900 XT / XTX on Linux** works too (experimental): `./setup.sh --backend hip`, chosen by itself on
+a PC with no NVIDIA card Strata can use. It installs ROCm without sudo and compiles the engine (one GPU, no images
+yet). Details: [AMD HIP](docs/AMD_HIP.md).
+
 ## Install
 
-**You need:** an NVIDIA RTX 30, 40 or 50 card with 12 GB of VRAM or more, enough RAM for the size you pick (above),
+**You need:** an NVIDIA RTX 30, 40 or 50 card with 12 GB of VRAM or more, enough RAM for the size you pick (above;
+a big GPU makes up for less RAM - the [low-RAM mode](docs/DETAILS.md)),
 ~80 GB of free disk space (an SSD makes the first start much faster), and Windows 10/11 or Linux. The only thing you
 install yourself is a current **NVIDIA driver** ([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA
 App). Everything else - Python, the engine, the model - is set up for you.

cmake/hip_backend.cmake

@@ -0,0 +1,58 @@
+# Opt-in HIP configuration. Strata's CUDA-shaped kernels currently target RDNA3 gfx1100 wave32.
+# CMake/compiler discovery stays machine-independent; pass CMAKE_HIP_COMPILER when it is not on PATH.
+if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "")
+  set(CMAKE_HIP_ARCHITECTURES gfx1100 CACHE STRING "Strata HIP target architecture")
+endif()
+if(NOT CMAKE_HIP_ARCHITECTURES STREQUAL "gfx1100")
+  message(FATAL_ERROR
+    "Strata HIP currently supports only gfx1100 wave32; CMAKE_HIP_ARCHITECTURES is '${CMAKE_HIP_ARCHITECTURES}'")
+endif()
+
+enable_language(HIP)
+find_package(hip CONFIG REQUIRED)
+find_package(hipblas CONFIG REQUIRED)
+find_package(hipblaslt CONFIG QUIET)
+
+if(NOT TARGET hip::host)
+  message(FATAL_ERROR "The ROCm hip CMake package did not provide hip::host")
+endif()
+if(NOT TARGET roc::hipblas)
+  message(FATAL_ERROR "The ROCm hipblas CMake package did not provide roc::hipblas")
+endif()
+if(TARGET roc::hipblaslt)
+  set(STRATA_HIPBLASLT_AVAILABLE ON)
+else()
+  set(STRATA_HIPBLASLT_AVAILABLE OFF)
+  message(STATUS "Strata: hipBLASLt not found; solution-table dispatch is unavailable")
+endif()
+
+# HIP's link step produces a PIE; make Strata and ggml objects PIC for the ROCm linker.
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(STRATA_HIP_COMPAT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/strata/hip_compat")
+add_library(strata_hip_runtime INTERFACE)
+target_include_directories(strata_hip_runtime BEFORE INTERFACE
+  "${STRATA_HIP_COMPAT_INCLUDE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/include")
+target_compile_definitions(strata_hip_runtime INTERFACE STRATA_USE_HIP=1)
+target_link_libraries(strata_hip_runtime INTERFACE hip::host)
+foreach(_language IN ITEMS CXX HIP)
+  target_compile_options(strata_hip_runtime INTERFACE
+    "$<$<COMPILE_LANGUAGE:${_language}>:-include>"
+    "$<$<COMPILE_LANGUAGE:${_language}>:${STRATA_HIP_COMPAT_INCLUDE_DIR}/cuda_runtime.h>")
+endforeach()
+
+# CMake does not infer HIP from Strata's existing CUDA-shaped .cu suffixes.
+file(GLOB_RECURSE _strata_hip_sources CONFIGURE_DEPENDS
+  "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cu"
+  "${CMAKE_CURRENT_SOURCE_DIR}/bench/*.cu"
+  "${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cu")
+if(_strata_hip_sources)
+  set_source_files_properties(${_strata_hip_sources} PROPERTIES LANGUAGE HIP)
+endif()
+foreach(_source IN ITEMS tests/hip/intrinsics.cpp tests/hip/native_qsa_score.cpp)
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${_source}")
+    set_source_files_properties("${_source}" PROPERTIES LANGUAGE HIP)
+  endif()
+endforeach()
+
+message(STATUS "Strata: HIP enabled, arch ${CMAKE_HIP_ARCHITECTURES}")

docs/AMD_HIP.md

@@ -0,0 +1,156 @@
+# Experimental AMD HIP backend (gfx1100)
+
+This is a manual Linux source build for the RX 7900 XTX. It is opt-in; the
+NVIDIA installer and CUDA build remain the default. Other AMD architectures,
+wave64, Windows HIP, and mixed AMD/NVIDIA execution are outside this contribution.
+
+The backend maps the CUDA-shaped runtime and BLAS calls to HIP/hipBLAS, uses
+RDNA3's signed integer dot instruction for quantized kernels, and supplies
+wave32 shuffle/packed-byte operations. CUDA-only QSA matrix instructions have
+an ordered FP32 fallback. Prefill supports both dequantization plus hipBLAS GEMM and opt-in HIP ggml MMQ.
+An optional, calibrated hipBLASLt path accelerates dense projections on gfx1100.
+This does not claim bit-identical model answers across backends. See
+[performance settings and evidence](AMD_HIP_PERFORMANCE.md).
+
+## Install with setup (recommended)
+
+On Linux with an RX 7900 XT / XTX and the kernel's amdgpu driver (no ROCm install needed):
+
+```sh
+./setup.sh --backend hip
+```
+
+- **Detection:** setup finds the card through the kernel's KFD topology. Integrated Radeon GPUs are listed as not
+  supported. On a PC without an NVIDIA card Strata can use, `--backend hip` is chosen automatically.
+- **ROCm:** installed into `.venv` from AMD's TheRock wheels (~10 GB, no sudo), pinned to the version this backend was
+  tested with (`STRATA_ROCM_VERSION` / `STRATA_ROCM_INDEX` override it). A system ROCm in `/opt/rocm` (or
+  `$ROCM_PATH`) with hipcc and hipBLAS is used instead when present.
+- **Engine:** compiled on your PC (10-20 minutes, once; again after a `git pull` that changes it). This needs a C++
+  compiler and git (`sudo apt install build-essential git`).
+- **Limits for now:** one GPU, no images, no calibration. The Monitor shows no GPU statistics.
+
+The rest of setup is the same as on NVIDIA: the model download, the start script, the server.
+
+## Build
+
+Requirements: a working ROCm driver/runtime, HIP development headers and
+compiler, hipBLAS and (for tuned dense prefill) hipBLASLt development files, CMake 3.24+, a C++20 host compiler, and Git.
+The model still needs sufficient system RAM and fast SSD storage for its PLE
+table. VRAM occupancy alone is not a throughput measurement.
+
+```sh
+cmake -S . -B build-hip \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DSTRATA_ENABLE_HIP=ON -DSTRATA_ENABLE_CUDA=OFF \
+  -DCMAKE_HIP_ARCHITECTURES=gfx1100
+cmake --build build-hip --target strata -j2
+```
+
+If CMake cannot find the HIP compiler, add
+`-DCMAKE_HIP_COMPILER=/path/to/rocm/llvm/bin/clang++`. On the Fedora-family test
+host this was `/usr/lib64/rocm/llvm/bin/clang++`. Use the compiler and libraries
+from the same ROCm installation. Do not enable both GPU backends in one build.
+
+Native IQ experts are enabled by default. CMake fetches the llama.cpp revision
+pinned by this repository. For an offline build, point `STRATA_GGML_DIR` at a
+checkout of that exact revision; using an arbitrary newer checkout changes the
+dependency being tested.
+
+## Model and serving configuration
+
+Prepare a supported model pack and its MTP runtime using the existing tools.
+For OrcaRouter IQ3_XXS, follow [the explicit compatibility conversion](ORCA.md);
+this backend does not change quantization, model licenses, or tokenizers.
+Add `--experts-bin` to the `tools/iq_pack.py` command: mmap requires the pack's
+`experts.bin`, which the default native packing command does not emit. This
+consumes additional disk space. Original GSQ-RCO IQ3_XXS uses shard 2 for PLE;
+Orca uses shard 1. Keep each model's own pack and tokenizer together.
+
+Use `build-hip/strata` as the executable in the server JSON. Select the AMD
+device with `ROCR_VISIBLE_DEVICES`/`HIP_VISIBLE_DEVICES` if necessary. Start with
+a modest context and prefill chunk size before measuring larger workloads.
+Keep the PLE file on an SSD. Do not assume a configured context length proves
+successful full-window inference.
+
+Large pinned host allocations can fail on ROCm even when ordinary RAM is
+available. The existing `--mmap-experts` path avoids allocating the full pinned
+expert arena; it still depends on OS file-cache residency and may stall on
+storage reads. It does not make SSD access equivalent to RAM.
+
+Starting args for the original GSQ-RCO IQ3_XXS model (replace the paths):
+
+```sh
+build-hip/strata --serve \
+  --pack packs/iq3xxs \
+  --native /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf \
+  --ple-gguf /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf \
+  --mmap-experts --expert-profile data/expert-profile.bin --expert-cache auto \
+  --prefill 512 --spec 4 --spec-min-p 0.5 --mtp mtp/rt \
+  --max-context 4096 --kv int8 --pool-workers 15 \
+  --adapt-every 0 --pcie-frac 0 --vram-reserve-mib 1024
+```
+
+`--serve` is the engine's internal token protocol. For a browser or OpenAI API,
+put these args in the `args` array of the [server JSON example](ORCA.md#local-server),
+set `exe` to `build-hip/strata`, and use the matching pack's `tokenizer` directory.
+Launch with `.venv/bin/python -m serve.server --engine strata --config strata-hip.json --port 8080`.
+The worker count above was used on a 16-core CPU; measure it for your CPU.
+The 4K context is a smoke-test starting point, not a model limit. The expert cache
+sizes itself automatically and leaves 1 GiB of VRAM headroom.
+
+The installer supports this backend (see "Install with setup" above). The vision helper and multi-GPU layer
+splits are NVIDIA-only for now.
+
+## Original backend validation (PR #94)
+
+The following is historical validation of the original backend, not a fresh
+test count for this replacement. Current build/test evidence and performance
+limits are recorded in [AMD_HIP_PERFORMANCE.md](AMD_HIP_PERFORMANCE.md).
+
+Tested against upstream `c1e903310f211e6630780c3bd2038778c071c68d` (0.1.20),
+with pinned llama.cpp `3cf03257f219afbe7334045ff7c6a06ac68c627d`.
+The host has an RX 7900 XTX (24 GiB), Ryzen 9 7950X3D, 64 GiB system RAM,
+and Fedora-family Linux with ROCm 7.1.52802 / Clang 20. The original model used
+for the smoke check was on mechanical RAID0; cold page faults were slow. Keep
+latency-sensitive data on SSD and measure warm residency separately.
+
+- Complete HIP Release build, including the executable and parity targets.
+- All 25 tests selected by the command below passed on 2026-09-29.
+- Intrinsic parity includes signed packed-byte dot products and overflow.
+- CPU/GPU mapped-memory handoff covers delayed publication, changing payloads,
+  and repeated graph replay in both directions.
+- GR coverage includes the maximum eight-token fused batch and graph replay
+  after inputs change. Native QSA checks the scalar HIP fallback.
+- Q8_K quantization is byte-exact against the existing reference. HIP's
+  `__fmul_rn` can become ordinary multiplication; disabling contraction in
+  `quantize_act.cu` preserves the separately rounded product before the magic
+  rounding bias is added. CUDA's compile flags remain unchanged.
+- The mmap regression covers canonical and differing native layer sizes,
+  truncated/oversized files, lookup bounds, and reopening after errors.
+- CUDA 13 / GCC 15 / sm_89 executable build passed; GR, sampler, quantization,
+  and mmap tests passed on the same tree. This is a regression check, not a
+  claim of complete CUDA inference validation.
+- CPU-only `strata-plan` build passed without either GPU backend.
+- Real GSQ-RCO IQ3_XXS server smoke with mmap, the shipped expert profile,
+  automatic GPU cache, and MTP passed arithmetic, executable Python addition,
+  system-marker recall, and 1,170-token batched-prefill recall. All four requests
+  ended normally with correct answers; one engine process served them all.
+  The cache held 11,142 experts (18.08 GiB), with 874 MiB of VRAM free after
+  startup. These short checks do not establish a throughput or quality benchmark.
+
+Build all targets before running the registered focused checks:
+
+```sh
+cmake --build build-hip -j2
+ctest --test-dir build-hip --output-on-failure --timeout 60 \
+  -E '^(ple_parity|platform_memory_test)$'
+```
+
+`ple_parity` requires an external model fixture. `platform_memory_test` requests
+256 MiB of locked memory; the test shell's 8 MiB memlock limit prevented that
+test from passing. These are explicit exclusions, not skipped tests counted as
+passes. Additional unpublished upstream fixture suites are not covered.
+
+The published performance table explicitly identifies the measured development
+snapshot; it must not be read as a benchmark of every subsequent rebase. Vision, long-context stress, broad answer-quality
+equivalence, other AMD cards, and mixed-vendor inference are not validated here.

docs/AMD_HIP_PERFORMANCE.md

@@ -0,0 +1,142 @@
+# RX 7900 XTX support and performance evidence
+
+This opt-in Linux `gfx1100` backend supersedes the initial support in
+[PR #94](https://github.com/Niko1221/Strata/pull/94). It retains HIP runtime and
+wave32 integer-dot compatibility, native mmap layout validation, and MTP, then
+adds HIP MMQ, optional calibrated dense hipBLASLt GEMM, native prefill batching,
+and the host-memory / SSD paths used by the measured configuration. HIP blocking
+expert uploads use one reusable pinned staging buffer, including prompt-cache
+refills, to avoid repeated pageable-source registrations.
+
+## Reproduce the configuration
+
+Build instructions are in [AMD_HIP.md](AMD_HIP.md). Use the repository-pinned
+llama.cpp dependency; do not silently substitute another revision. Build with
+`STRATA_PREFILL_MMQ=ON` and use these runtime variables for the measured arm:
+
+```sh
+export STRATA_PREFILL_MMQ=1
+export STRATA_HIPBLASLT_TUNING="$PWD/tools/hip/gfx1100-hipblaslt-100100.txt"
+export STRATA_PREFILL_RING=96
+export STRATA_IO_THREADS=32
+```
+
+The supplied table is calibrated for gfx1100 and hipBLASLt version 100100.
+It is not a universal ROCm tuning table. The runtime guards architecture,
+library version and actual shape/stride/workspace requirements, falling back
+when a table entry is unavailable or incompatible. HIP MMQ is opt-in at runtime. Default CUDA selection is preserved.
+
+Measured engine configuration: Orca Flash Next IQ3_XXS, native pack plus matching
+GGUF/tokenizer/template, `--mmap-experts --resident-cpu-experts`, fixed ranked
+`--expert-profile`, `--expert-cache auto`, `--prefill 8192`, `--spec 4`,
+`--spec-min-p 0.5`, matching MTP runtime, `--max-context 262144`, `--kv int8`,
+`--kv-resident 32768`, `--pool-workers 15`, `--adapt-every 0`, `--pcie-frac 0`,
+and `--vram-reserve-mib 1024`. Keep PLE on SSD.
+
+`--resident-cpu-experts` copies the complement of the static GPU cache to ordinary
+RAM. It requires mmap, a fixed expert profile, and disabled adaptation. It does
+not lock all data into RAM: temporary prefill borrowing and cache refill may
+still use the mapped fallback. Allocation needs sufficient available RAM;
+on Linux this option requires readable standard cgroup-v2 mounts. For each
+finite cgroup ancestor, the guard credits only `inactive_file` after subtracting
+`file_dirty` and `file_writeback`, capped by current usage; it remains bounded by
+the ancestor limit and host `MemAvailable`, with 8 GiB headroom. This accounts
+for reclaimable clean file cache but cannot reserve memory against concurrent
+system or process allocations.
+The POSIX PLE path issues direct reads through a configurable worker pool.
+
+Use a dedicated idle server, restart it between arms, and capture its engine log:
+
+```sh
+python3 tools/hip/bench_prefill.py \
+  --model MODEL_NAME --url http://127.0.0.1:8080 \
+  --engine-log /path/to/engine.log --label candidate --output candidate.json
+```
+
+The script uses the same synthetic source prompts and request order as the
+reported measurements: a small warmup, then 140/280/140/280 functions, with one
+follow-up after each fresh prompt. It enforces zero reused tokens for the fresh
+prompts and records prefill, decode, wall time and stop reason. It requires the
+unbuffered log to contain exactly one completed timing record per request. The
+128-token output cap is intentional for throughput measurement, not task success.
+Never use cancelled-request timing lines as throughput evidence.
+
+## Final revision benchmark
+
+Measured on 2026-09-29, source revision `9568e78da663e6b224ab96d57baf450717b36e73`.
+HIP executable SHA-256: `705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b`.
+The control and candidate below were both measured anew in this session; no
+historical benchmark values are mixed into the table. The control binary SHA-256
+is `4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7`;
+its source was an existing local runtime snapshot rather than a clean upstream commit.
+Both arms use an RX 7900 XTX 24 GiB / gfx1100, Ryzen 9 7950X3D,
+64 GiB installed RAM, Fedora-family Linux, ROCm 7.1 and a 272 W GPU cap.
+Other model services are stopped for measurement and restored afterward.
+Sampling: temperature 0, top-k 1, top-p 1, min-p 0, seed 42, reasoning disabled.
+
+| Fresh request, execution order | Control prefill t/s | Candidate prefill t/s | Candidate output t/s | Candidate request wall time |
+| --- | ---: | ---: | ---: | ---: |
+| 4,210 tokens, first use | 240.0 | 447.5 | 55.5 | 11.729 s |
+| 8,830 tokens, first use at this size | 484.8 | 873.7 | 55.7 | 12.420 s |
+| 4,210 tokens, warmed | 457.2 | 901.1 | 56.7 | 6.937 s |
+| 8,830 tokens, warmed | 478.9 | 966.5 | 59.2 | 11.314 s |
+
+Each request generated exactly 128 tokens and finished at the intentional length
+cap. All four fresh prompts had zero reused KV tokens. Fresh-prompt prefill was
+1.80–2.02x the freshly measured existing AMD runtime; fresh-request wall time was
+40–46% lower. This is a comparison with our existing HIP runtime, not unmodified
+upstream, which does not provide this backend. The candidate also completed four
+cached follow-ups in 2.95–3.26 seconds. Those follow-ups are not apples-to-apples
+prefill comparisons: cache reuse differs with generated text and checkpoint
+selection. Full sanitized measurements are in
+[the fresh-run JSON](benchmarks/2026-09-29-gfx1100.json).
+
+Before this revision, an intermediate package missing the HIP upload staging
+buffer stalled on the 8,830-token request and was rejected. Restoring staging
+allowed the complete sequence above to finish. That failed attempt is not
+included in the throughput table. No watchdog limit was raised to obtain these
+results.
+
+Zero KV reuse is not equivalent to a cold filesystem cache. First-use speed
+and warmed speed are reported separately. Individual observations do not
+establish confidence intervals or a general rate at every context length.
+
+## Correctness and quality limits
+
+Both HIP and CUDA executables built from the measured source revision. CUDA was
+compile-checked, not performance-tested by this run. The selected HIP CTest suite
+passed **29/29** with the tuning table enabled, including expert-upload readback,
+asynchronous handoff, QSA, MMQ, Lt GEMM, KV streaming and PLE reading. The excluded
+`ple_parity` requires an external fixture; `platform_memory_test` requires a
+larger locked-memory limit than the test account provides. Separately, three
+real IQ3_XXS PLE matrix graph replays passed, and the POSIX direct-file test passed
+queued reads, short EOF, wake, close/drain and reopen checks.
+
+Development checks passed eight MMQ numerical comparisons, four actual Lt GEMM
+comparisons, and native QSA/indexer/embedding parity including tail states,
+chunk continuation and image overrides. Real quantized MMQ relative L2 error
+was approximately 0.0026 against raw-FP32/dequantized-weight reference, reflecting
+Q8 activation arithmetic; this is not bitwise numerical equivalence.
+
+End-to-end coding quality is not established by these numerical checks or capped
+throughput requests. Validate completed tasks with independent runtime tests; this
+contribution makes no broad answer-quality or agentic-reliability claim.
+
+The changes do not claim better model reasoning, verified full-context behavior,
+end-to-end vision validation, Windows HIP, other AMD architectures, or mixed
+AMD/NVIDIA execution. Existing CUDA multi-GPU code remains present but is not
+validation of HIP multi-GPU support.
+
+## Attribution and rejected experiments
+
+The measurements above were taken before this backend was rebased onto engine 0.1.24. The PR's own batched
+embedding gather and QSA indexer append (adapted from
+[PR #108](https://github.com/Niko1221/Strata/pull/108), commit
+`acd487233c0bbe2217a6881c5bb43f8a283b0de5`) were dropped in the rebase: 0.1.24 already does both on every
+backend, bit-exact (C-2, C-4). #108's optional GDN parallel/split path is not included. Existing upstream MMQ orchestration is retained and enabled for
+HIP with AMD architecture identification and the correct backend compilation.
+
+Static 16K chunks, expanded FP16 expert tuning, and a 192-slot ring did not offer
+a consistent end-to-end win in the evaluated workload. The table includes only
+the selected 30 dense-shape rows. Microkernel speedups alone were not sufficient
+to select a configuration. No increase in GPU power cap was used.

docs/DETAILS.md

@@ -61,6 +61,21 @@ halves the KV cache's memory with a Hadamard rotation before 4-bit rounding (PR
 is measurably less precise on long documents (perplexity +8-12%; needle tests still pass). 8-bit stays the default.
 Details: [`bench/results/2026-09-27-kv-q4`](../bench/results/2026-09-27-kv-q4/README.md).
 
+**Hybrid K8V4 KV cache (engine 0.1.25, optional, PR #120):** `--kv k8v4` (`START-HERE.bat --setup --kv k8v4`) keeps
+the keys at 8 bits and stores the values as rotated 4-bit: 23% less KV memory than 8-bit, so more experts fit in
+VRAM. RTX 3090, the Coder at 198K context: 99 instead of 85 tokens/s output, the same needle results, prompts 2-5%
+slower. It does not stream its KV cache (KV streaming is on by default from 64K), so it pays off mostly on large
+cards at long contexts.
+
+**Low-RAM mode (engine 0.1.26, chosen by setup):** normally all of a model's experts are copied into RAM (23-50 GB,
+pinned) and the GPU holds a copy of the most-used ones. On a PC whose RAM cannot hold them beside the system (the
+experts plus ~10 GB), setup instead maps them from one file in the model's folder (`--mmap-experts`, the pack's
+`experts.bin`, +23-50 GB of disk). The OS file cache holds what the GPU does not, and it can give that memory back.
+On the Coder the engine's committed memory drops from 36 to ~13 GB, with the same answers. With a big GPU (an RTX
+5090 holds all of the Coder's experts, most of Q2_0's) it runs at nearly the usual speed. With a small one, most
+experts come from the SSD and it is much slower (setup says so). `START-HERE.bat --setup --low-ram on|off` overrides
+the choice.
+
 Time to first token is prompt length / prompt speed: with Q2_0 about 4 s at 4K, 25 s at 32K, under 2 minutes at 128K
 and 4.5 minutes at 262K (engine 0.1.13 made long prompts about twice as fast, below).
 

docs/benchmarks/2026-09-29-gfx1100.json

@@ -0,0 +1,299 @@
+{
+  "source_commit": "9568e78da663e6b224ab96d57baf450717b36e73",
+  "date": "2026-09-29",
+  "candidate_binary_sha256": "705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b",
+  "note": "Fresh same-session control and candidate runs. Throughput cap 128; not a coding-quality benchmark.",
+  "control": [
+    {
+      "kind": "warmup",
+      "trial": 0,
+      "wall_s": 0.45770244899904355,
+      "metrics": {
+        "prompt_tokens": 15.0,
+        "reused": 0.0,
+        "fresh": 15.0,
+        "prefill_ms": 335.0,
+        "prefill_tps": 44.8,
+        "generated": 2.0,
+        "decode_ms": 115.0,
+        "decode_tps": 17.5
+      },
+      "finish_reason": "stop"
+    },
+    {
+      "kind": "cold",
+      "trial": 1,
+      "wall_s": 20.068275763012934,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 17538.0,
+        "prefill_tps": 240.0,
+        "generated": 128.0,
+        "decode_ms": 2518.0,
+        "decode_tps": 50.8
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 1,
+      "wall_s": 4.412000925018219,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 1943.0,
+        "prefill_tps": 58.7,
+        "generated": 128.0,
+        "decode_ms": 2451.0,
+        "decode_tps": 52.2
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 2,
+      "wall_s": 20.731399575975956,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 18213.0,
+        "prefill_tps": 484.8,
+        "generated": 128.0,
+        "decode_ms": 2505.0,
+        "decode_tps": 51.1
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 2,
+      "wall_s": 5.0592734439997,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 2601.0,
+        "prefill_tps": 95.4,
+        "generated": 128.0,
+        "decode_ms": 2441.0,
+        "decode_tps": 52.4
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 3,
+      "wall_s": 11.695648370019626,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 9209.0,
+        "prefill_tps": 457.2,
+        "generated": 128.0,
+        "decode_ms": 2477.0,
+        "decode_tps": 51.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 3,
+      "wall_s": 4.093982424004935,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 1896.0,
+        "prefill_tps": 60.1,
+        "generated": 128.0,
+        "decode_ms": 2188.0,
+        "decode_tps": 58.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 4,
+      "wall_s": 20.84539221401792,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 18437.0,
+        "prefill_tps": 478.9,
+        "generated": 128.0,
+        "decode_ms": 2392.0,
+        "decode_tps": 53.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 4,
+      "wall_s": 5.357054464984685,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 2658.0,
+        "prefill_tps": 93.3,
+        "generated": 128.0,
+        "decode_ms": 2682.0,
+        "decode_tps": 47.7
+      },
+      "finish_reason": "length"
+    }
+  ],
+  "optimized": [
+    {
+      "kind": "warmup",
+      "trial": 0,
+      "wall_s": 0.46265592801501043,
+      "metrics": {
+        "prompt_tokens": 15.0,
+        "reused": 0.0,
+        "fresh": 15.0,
+        "prefill_ms": 335.0,
+        "prefill_tps": 44.8,
+        "generated": 2.0,
+        "decode_ms": 121.0,
+        "decode_tps": 16.5
+      },
+      "finish_reason": "stop"
+    },
+    {
+      "kind": "cold",
+      "trial": 1,
+      "wall_s": 11.728894605010282,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 9408.0,
+        "prefill_tps": 447.5,
+        "generated": 128.0,
+        "decode_ms": 2308.0,
+        "decode_tps": 55.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 1,
+      "wall_s": 2.9531024169991724,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 740.0,
+        "prefill_tps": 154.0,
+        "generated": 128.0,
+        "decode_ms": 2193.0,
+        "decode_tps": 58.4
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 2,
+      "wall_s": 12.42044531099964,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 10107.0,
+        "prefill_tps": 873.7,
+        "generated": 128.0,
+        "decode_ms": 2298.0,
+        "decode_tps": 55.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 2,
+      "wall_s": 3.2599652509961743,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 1070.0,
+        "prefill_tps": 231.7,
+        "generated": 128.0,
+        "decode_ms": 2173.0,
+        "decode_tps": 58.9
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 3,
+      "wall_s": 6.937167472002329,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 4672.0,
+        "prefill_tps": 901.1,
+        "generated": 128.0,
+        "decode_ms": 2256.0,
+        "decode_tps": 56.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 3,
+      "wall_s": 3.1877827199932653,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 827.0,
+        "prefill_tps": 137.8,
+        "generated": 128.0,
+        "decode_ms": 2350.0,
+        "decode_tps": 54.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 4,
+      "wall_s": 11.314008287008619,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 9136.0,
+        "prefill_tps": 966.5,
+        "generated": 128.0,
+        "decode_ms": 2161.0,
+        "decode_tps": 59.2
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 4,
+      "wall_s": 3.0476601750124246,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8957.0,
+        "fresh": 114.0,
+        "prefill_ms": 734.0,
+        "prefill_tps": 155.3,
+        "generated": 128.0,
+        "decode_ms": 2296.0,
+        "decode_tps": 55.7
+      },
+      "finish_reason": "length"
+    }
+  ],
+  "control_binary_sha256": "4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7"
+}

include/strata/core/expert_cache.hpp

@@ -26,6 +26,7 @@
 // kernel. This is that first step, and the step it unblocks is the one that can be measured.
 #pragma once
 
+#include <cstddef>
 #include <cstdint>
 #include <string>
 #include <vector>
@@ -140,6 +141,11 @@ public:
     int64_t fills() const { return fills_; }
 
 private:
+#if defined(STRATA_USE_HIP)
+    bool ensure_blocking_staging(std::size_t bytes, std::string& err);
+    uint8_t* blocking_staging_ = nullptr;
+    std::size_t blocking_staging_bytes_ = 0;
+#endif
     uint8_t* base_ = nullptr;
     std::vector<int32_t> residency_;   ///< [n_layers * n_expert] -> slot or kNotResident
     int64_t slots_ = 0;

include/strata/core/expert_source.hpp

@@ -26,6 +26,8 @@
 #include "strata/kernels/cpu/pool.hpp"
 
 #include <cstdint>
+#include <cstddef>
+#include <utility>
 #include <string>
 #include <vector>
 
@@ -33,6 +35,41 @@ namespace strata::core {
 
 class RemoteExperts;
 
+namespace detail {
+
+/// Sentinel used by the pure complement planner for a blob that remains in the mmap fallback.
+inline constexpr uint64_t kNoCacheComplement = ~uint64_t{0};
+
+/// Required cgroup-v2 usage counters for the conservative cache-reclaim allowance.
+struct CgroupMemoryStat {
+    uint64_t current = 0;
+    uint64_t inactive_file = 0;
+    uint64_t file_dirty = 0;
+    uint64_t file_writeback = 0;
+    bool valid = false;
+};
+
+/// Calculate additional bytes under a finite cgroup limit after reclaiming only clean inactive file cache.
+/// Returns false when the required memory.stat counters were unavailable.
+bool cgroup_available_bytes(uint64_t limit, const CgroupMemoryStat& stat, uint64_t& bytes);
+
+/// Build compact offsets for experts absent from both the primary GPU cache and an optional second GPU tier.
+/// Kept CPU-only so selection and byte accounting can be tested without initializing a GPU.
+bool make_cache_complement_plan(
+    int64_t n_layers, int64_t n_expert, const std::vector<uint64_t>& layer_blob_bytes,
+    const std::vector<std::pair<int32_t, int32_t>>& primary_gpu_pairs,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs,
+    std::vector<uint64_t>& offsets, uint64_t& bytes, std::string& err);
+
+/// Resolve one blob through the compact copy when present, otherwise preserve its exact mapped-file fallback.
+const uint8_t* cache_complement_blob_or_fallback(
+    size_t index, const std::vector<uint64_t>& offsets, const uint8_t* complement_host,
+    const uint8_t* mapped_fallback);
+
+}  // namespace detail
+
+
+
 /// Where one routed expert's bytes come from.
 ///
 /// Phase 2 has NO cache (`phase-2-correct-engine.md`: hit rate `h = 0`), so the only implementation is a
@@ -42,7 +79,7 @@ class ExpertSource {
 public:
     virtual ~ExpertSource() = default;
 
-    /// The 1,382,400-byte blob for `(layer, expert)`, or nullptr if it cannot be produced.
+    /// The expert-layout blob for `(layer, expert)`, or nullptr if it cannot be produced.
     ///
     /// The pointer only has to stay valid until the next `blob()` call: with `h = 0` every expert is computed
     /// immediately and nothing is retained.  A CACHING source must return pointers into the cache, not into a
@@ -270,26 +307,49 @@ public:
     FileExpertSource(const FileExpertSource&) = delete;
     FileExpertSource& operator=(const FileExpertSource&) = delete;
 
-    /// Maps `<pack_dir>/experts.bin` and checks its size against `n_layers * n_expert * BLOB`.
+    /// Maps `<pack_dir>/experts.bin` and checks its size against the loaded expert layout.  Canonical packs use
+    /// `n_layers * n_expert * BLOB`; native packs use their variable per-layer blob sizes and offsets.
     ///
     /// The size check is not a formality: a short file would fault at the END of a long sequence, and an
     /// over-long one means the pack is not the one the geometry came from.  Refuses with the two numbers.
     bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err);
+    /// Pin a compact host mirror of experts absent from a fully filled static GPU cache. The mmap remains open
+    /// as a fallback for later cache reloads. This is opt-in because the complement may still be a large allocation.
+    bool pin_cache_complement(
+        const ExpertCache& cache, std::string& err, bool pin = true,
+        const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs = {});
     void close();
 
     bool mapped() const { return base_ != nullptr; }
     int64_t blobs() const { return blobs_; }
+    uint64_t pinned_bytes() const { return complement_pinned_ ? complement_bytes_ : 0; }
+    uint64_t resident_bytes() const { return complement_bytes_; }
+    bool complement_pinned() const { return complement_pinned_; }
 
     const uint8_t* blob(int64_t layer, int64_t expert) override;
+    bool pinned(int64_t layer, int64_t expert) const override;
+    const uint8_t* device_alias(int64_t layer, int64_t expert) const override;
 
     /// Blobs touched, for the driver to report.  With `h = 0` this is `48 * k` per token and the number is only
     /// interesting once Phase 3 makes it not so.
     int64_t reads() const override { return reads_; }
 
 private:
+    const uint8_t* mapped_blob(int64_t layer, int64_t expert) const;
+    static constexpr uint64_t kNoComplement = detail::kNoCacheComplement;
     const uint8_t* base_ = nullptr;
     int64_t blobs_ = 0;
+    int64_t n_layers_ = 0;
     int64_t n_expert_ = 0;
+    uint64_t mapped_bytes_ = 0;
+    std::vector<uint64_t> layer_offsets_, layer_blob_bytes_;
+    void* complement_arena_ = nullptr;
+    const uint8_t* complement_host_ = nullptr;
+    const uint8_t* complement_device_ = nullptr;
+    uint64_t complement_bytes_ = 0;
+    std::vector<uint64_t> complement_offsets_;
+    bool complement_pinned_ = false;
+    bool complement_ready_ = false;
     int64_t reads_ = 0;
 #if defined(_WIN32)
     void* file_ = nullptr;

include/strata/core/layer.hpp

@@ -40,6 +40,8 @@
 #include "strata/kernels/ngram.hpp"
 #include "strata/kernels/ple.hpp"
 
+#include <cstdio>
+#include <cstdlib>
 #include <cstdint>
 #include <string>
 
@@ -211,6 +213,10 @@ struct QsaState {
     bool kv_q4 = false;
     uint8_t* k_q4 = nullptr;
     uint8_t* v_q4 = nullptr;
+    /// Hybrid K8V4: K in INT8 (unrotated - the scores stay exact), V in rotated Q4_0 (kv_q4.hpp): 816 B per cell.
+    /// Uses k_q/k_scale + v_q4. Mode 0 only (no KV streaming, no ring); under this setting the MTP drafter's
+    /// ring state stays plain INT8, so the block movers never see the hybrid layout.
+    bool kv_hybrid = false;
     int32_t* page_table = nullptr;   ///< (n_pages,) logical page -> physical page (-1: not resident, streamed)
     int64_t n_pages = 0;
     int64_t max_cells = 0;
@@ -276,8 +282,17 @@ bool qsa_kv_int8();
 /// PR #21: store K/V as Q4_0 after a Hadamard rotation (`--kv q4_0`): 576 B per cell, vs 1,056 in INT8.
 void qsa_set_kv_q4(bool enabled);
 bool qsa_kv_q4();
+/// Hybrid K8V4 (`--kv k8v4`): K in INT8, V in rotated Q4_0 - 816 B per cell. Not with --kv-resident.
+void qsa_set_kv_hybrid(bool enabled);
+bool qsa_kv_hybrid();
 /// The state's KV format for the block-moving functions of kv_stream.hpp (kKvF16 / kKvInt8 / kKvQ4).
 inline int qsa_kv_format(const QsaState& st) {
+    // A hybrid K8V4 state is mode 0 only and never reaches the block movers; refuse rather than let it
+    // fall through to kKvF16 - a wrong layout silently applied is worse than a hard stop (PR review).
+    if (st.kv_hybrid) {
+        std::fprintf(stderr, "strata: qsa_kv_format: a hybrid K8V4 state must never reach the block movers\n");
+        std::exit(1);   // the kernels' own "unsupported geometry" convention (kv_q8.cu, qsa_decode_attn.cu)
+    }
     return st.kv_q4 ? strata::kernels::kKvQ4 : st.kv_int8 ? strata::kernels::kKvInt8 : strata::kernels::kKvF16;
 }
 uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, QsaState& st,

include/strata/core/mtp.hpp

@@ -73,6 +73,19 @@ public:
     double ms_draft = 0, ms_prefill = 0;
     int64_t rounds = 0;
 
+    /// E-9: the prompt path computes this layer's prompt K/V in batches (Prefill::draft_kv): its tensors, its K/V
+    /// state, the first cell a round can still read, its device, and a wait for its own stream.
+    const float* tensor_f32(const char* name) const { return f32(name); }
+    const uint16_t* tensor_bf16(const char* name) const { return bf16(name); }
+    const void* tensor_q8(const char* name) const { return q8(name); }
+    QsaState& kv_state_rw() { return st_; }
+    int64_t first_needed() const { return (window_ > 0 && prompt_len_ > 0) ? prompt_len_ - window_ - 64 : 0; }
+    int device() const { return device_; }
+    bool idle(std::string& err) {
+        if (cs_ && cudaStreamSynchronize(cs_) != cudaSuccess) { err = "mtp: its stream failed"; return false; }
+        return true;
+    }
+
 private:
     bool record_forward(int T, int step_row0, cudaStream_t cs, std::string& err);
     bool capture_prefill(int T, std::string& err);

include/strata/hip_compat/cublas_v2.h

@@ -0,0 +1,19 @@
+#pragma once
+#include <hipblas/hipblas.h>
+#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F
+#define CUBLAS_DEFAULT_MATH HIPBLAS_DEFAULT_MATH
+#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT
+#define CUBLAS_OP_N HIPBLAS_OP_N
+#define CUBLAS_OP_T HIPBLAS_OP_T
+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
+#define CUDA_R_16BF HIP_R_16BF
+#define CUDA_R_16F HIP_R_16F
+#define CUDA_R_32F HIP_R_32F
+#define cublasCreate hipblasCreate
+#define cublasDestroy hipblasDestroy
+#define cublasGemmEx hipblasGemmEx
+#define cublasHandle_t hipblasHandle_t
+#define cublasSetMathMode hipblasSetMathMode
+#define cublasSetStream hipblasSetStream
+#define cublasSetWorkspace hipblasSetWorkspace
+#define cublasStatus_t hipblasStatus_t

include/strata/hip_compat/cuda_fp16.h

@@ -0,0 +1,2 @@
+#pragma once
+#include <hip/hip_fp16.h>

include/strata/hip_compat/cuda_runtime.h

@@ -0,0 +1,102 @@
+#pragma once
+// Included only by STRATA_ENABLE_HIP builds. CUDA builds use NVIDIA headers.
+#include <hip/hip_runtime.h>
+// Do not let HIP's legacy macro corrupt libstdc++ attribute names.
+#ifdef __noinline__
+#undef __noinline__
+#endif
+#define cudaDevAttrMaxSharedMemoryPerBlockOptin hipDeviceAttributeMaxSharedMemoryPerBlock
+#define cudaDevAttrMultiProcessorCount hipDeviceAttributeMultiprocessorCount
+#define cudaDevAttrClockRate hipDeviceAttributeClockRate
+#define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor
+#define cudaDeviceGetAttribute hipDeviceGetAttribute
+#define cudaDeviceProp hipDeviceProp_t
+#define cudaDeviceSynchronize hipDeviceSynchronize
+#define cudaDriverGetVersion hipDriverGetVersion
+#define cudaErrorNotReady hipErrorNotReady
+#define cudaErrorStreamCaptureUnsupported hipErrorStreamCaptureUnsupported
+#define cudaError_t hipError_t
+#define cudaEventCreate hipEventCreate
+#define cudaEventCreateWithFlags hipEventCreateWithFlags
+#define cudaEventDestroy hipEventDestroy
+#define cudaEventDisableTiming hipEventDisableTiming
+#define cudaEventElapsedTime hipEventElapsedTime
+#define cudaEventQuery hipEventQuery
+#define cudaEventRecord hipEventRecord
+#define cudaEventSynchronize hipEventSynchronize
+#define cudaEvent_t hipEvent_t
+#define cudaFree hipFree
+#define cudaFreeHost hipFreeHost
+#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize
+#define cudaGetDevice hipGetDevice
+#define cudaGetDeviceCount hipGetDeviceCount
+#define cudaGetDeviceProperties hipGetDeviceProperties
+#define cudaGetErrorString hipGetErrorString
+#define cudaGetLastError hipGetLastError
+#define cudaGraphDestroy hipGraphDestroy
+#define cudaGraphExecDestroy hipGraphExecDestroy
+#define cudaGraphExec_t hipGraphExec_t
+#define cudaGraphGetNodes hipGraphGetNodes
+#define cudaGraphLaunch hipGraphLaunch
+#define cudaGraphUpload hipGraphUpload
+#define cudaGraph_t hipGraph_t
+#define cudaHostAlloc hipHostMalloc
+#define cudaHostAllocDefault hipHostAllocDefault
+#define cudaHostAllocMapped hipHostAllocMapped
+#define cudaHostAllocPortable hipHostAllocPortable
+#define cudaHostGetDevicePointer hipHostGetDevicePointer
+#define cudaHostRegister hipHostRegister
+#define cudaHostRegisterMapped hipHostRegisterMapped
+#define cudaHostRegisterPortable hipHostRegisterPortable
+#define cudaHostUnregister hipHostUnregister
+#define cudaLaunchHostFunc hipLaunchHostFunc
+#define cudaMalloc hipMalloc
+#define cudaMallocHost(...) (::strata::hip_compat::malloc_host(__VA_ARGS__))
+#define cudaMemGetInfo hipMemGetInfo
+#define cudaMemcpy hipMemcpy
+#define cudaMemcpy2DAsync hipMemcpy2DAsync
+#define cudaMemcpyAsync hipMemcpyAsync
+#define cudaMemcpyDefault hipMemcpyDefault
+#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice
+#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost
+#define cudaMemcpyHostToDevice hipMemcpyHostToDevice
+#define cudaMemset hipMemset
+#define cudaMemsetAsync hipMemsetAsync
+#define cudaPeekAtLastError hipPeekAtLastError
+#define cudaRuntimeGetVersion hipRuntimeGetVersion
+#define cudaSetDevice hipSetDevice
+#define cudaStreamBeginCapture hipStreamBeginCapture
+#define cudaStreamCaptureModeThreadLocal hipStreamCaptureModeThreadLocal
+#define cudaStreamCreate hipStreamCreate
+#define cudaStreamCreateWithFlags hipStreamCreateWithFlags
+#define cudaStreamDestroy hipStreamDestroy
+#define cudaStreamEndCapture hipStreamEndCapture
+#define cudaStreamNonBlocking hipStreamNonBlocking
+#define cudaStreamQuery hipStreamQuery
+#define cudaStreamSynchronize hipStreamSynchronize
+#define cudaStreamWaitEvent hipStreamWaitEvent
+#define cudaStream_t hipStream_t
+#define cudaSuccess hipSuccess
+
+namespace strata::hip_compat {
+template <typename T>
+inline hipError_t malloc_host(T** pointer, size_t bytes) {
+    return hipHostMalloc(reinterpret_cast<void**>(pointer), bytes, hipHostMallocDefault);
+}
+}  // namespace strata::hip_compat
+
+template <typename Kernel>
+inline hipError_t cudaFuncSetAttribute(Kernel kernel, hipFuncAttribute attribute, int value) {
+    return hipFuncSetAttribute(reinterpret_cast<const void*>(kernel), attribute, value);
+}
+inline hipError_t cudaGraphInstantiate(hipGraphExec_t* exec, hipGraph_t graph, unsigned long long flags) {
+    return hipGraphInstantiateWithFlags(exec, graph, flags);
+}
+inline hipError_t cudaGraphInstantiate(hipGraphExec_t* exec, hipGraph_t graph,
+                                     hipGraphNode_t* error, char* log, size_t size) {
+    return hipGraphInstantiate(exec, graph, error, log, size);
+}
+#define __trap() __builtin_trap()   // the compiled-out sm_80 paths (never selected on AMD)
+#define cudaMemcpyToSymbol(symbol, ...) hipMemcpyToSymbol(HIP_SYMBOL(symbol), __VA_ARGS__)
+
+#include "intrinsics.hpp"

include/strata/hip_compat/cuda_runtime_api.h

@@ -0,0 +1,2 @@
+#pragma once
+#include "cuda_runtime.h"

include/strata/hip_compat/intrinsics.hpp

@@ -0,0 +1,125 @@
+#pragma once
+
+// CUDA device intrinsics used by Strata kernels that HIP does not provide on gfx1100.
+// This header is included only from the HIP cuda_runtime compatibility shim.
+#if defined(__HIPCC__)
+
+#include <cstdint>
+
+namespace strata::hip_compat {
+
+__device__ __forceinline__ int signed_byte(uint32_t word, int lane) {
+    const unsigned value = (word >> (lane * 8)) & 0xffu;
+    return value < 0x80u ? static_cast<int>(value) : static_cast<int>(value) - 0x100;
+}
+
+// CUDA's signed __dp4a: four signed byte products accumulated modulo 2^32.
+__device__ __forceinline__ int dp4a(int a, int b, int c) {
+#if defined(__gfx1100__) && __has_builtin(__builtin_amdgcn_sudot4)
+    // RDNA3 exposes the signed/unsigned dot4 form. Mark both packed operands
+    // signed to preserve CUDA __dp4a semantics; keep the portable path for
+    // other HIP compilers/targets.
+    return __builtin_amdgcn_sudot4(true, a, true, b, c, false);
+#else
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t sum = static_cast<uint32_t>(c);
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane)
+        sum += static_cast<uint32_t>(signed_byte(ua, lane) * signed_byte(ub, lane));
+    return static_cast<int>(sum);
+#endif
+}
+
+// CUDA's packed byte subtract wraps independently in each unsigned byte lane.
+__device__ __forceinline__ int vsub4(int a, int b) {
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t out = 0;
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane) {
+        const uint32_t x = (ua >> (lane * 8)) & 0xffu;
+        const uint32_t y = (ub >> (lane * 8)) & 0xffu;
+        out |= ((x - y) & 0xffu) << (lane * 8);
+    }
+    return static_cast<int>(out);
+}
+
+// CUDA's packed signed-byte saturating subtract.
+__device__ __forceinline__ int vsubss4(int a, int b) {
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t out = 0;
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane) {
+        int value = signed_byte(ua, lane) - signed_byte(ub, lane);
+        value = value < -128 ? -128 : (value > 127 ? 127 : value);
+        out |= (static_cast<uint32_t>(value) & 0xffu) << (lane * 8);
+    }
+    return static_cast<int>(out);
+}
+
+// CUDA's four-lane byte compare, returning 0xff for each unequal lane and 0 otherwise.
+__device__ __forceinline__ int vcmpne4(int a, int b) {
+    const uint32_t ua = static_cast<uint32_t>(a);
+    const uint32_t ub = static_cast<uint32_t>(b);
+    uint32_t out = 0;
+#pragma unroll
+    for (int lane = 0; lane < 4; ++lane) {
+        if (((ua >> (lane * 8)) & 0xffu) != ((ub >> (lane * 8)) & 0xffu))
+            out |= 0xffu << (lane * 8);
+    }
+    return static_cast<int>(out);
+}
+
+// CUDA's mask argument describes participating lanes. The current kernel set uses full
+// wave32 masks; reject future partial-mask use instead of silently dropping its semantics.
+__device__ __forceinline__ void require_full_wave_mask(uint32_t mask) {
+    if (mask != 0xffffffffu) __builtin_trap();
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_xor_sync(uint32_t mask, T value, int lane_mask, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl_xor(value, lane_mask, width);
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_down_sync(uint32_t mask, T value, unsigned delta, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl_down(value, delta, width);
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_up_sync(uint32_t mask, T value, unsigned delta, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl_up(value, delta, width);
+}
+
+template <typename T>
+__device__ __forceinline__ T shfl_sync(uint32_t mask, T value, int source_lane, int width = 32) {
+    require_full_wave_mask(mask);
+    return __shfl(value, source_lane, width);
+}
+
+__device__ __forceinline__ unsigned ballot_sync(uint32_t mask, int predicate) {
+    require_full_wave_mask(mask);
+    return static_cast<unsigned>(__ballot(predicate));
+}
+
+}  // namespace strata::hip_compat
+
+#define __dp4a(a, b, c) (::strata::hip_compat::dp4a((a), (b), (c)))
+#define __vsub4(a, b) (::strata::hip_compat::vsub4((a), (b)))
+#define __vsubss4(a, b) (::strata::hip_compat::vsubss4((a), (b)))
+#define __vcmpne4(a, b) (::strata::hip_compat::vcmpne4((a), (b)))
+#define __shfl_xor_sync(...) (::strata::hip_compat::shfl_xor_sync(__VA_ARGS__))
+#define __shfl_down_sync(...) (::strata::hip_compat::shfl_down_sync(__VA_ARGS__))
+#define __shfl_up_sync(...) (::strata::hip_compat::shfl_up_sync(__VA_ARGS__))
+#define __shfl_sync(...) (::strata::hip_compat::shfl_sync(__VA_ARGS__))
+#define __ballot_sync(mask, predicate) (::strata::hip_compat::ballot_sync((mask), (predicate)))
+// AMD's sleep instruction accepts only 0..15; the synchronization loops use it as a
+// backoff hint, so use its smallest portable delay independently of CUDA cycle counts.
+#define __nanosleep(cycles) __builtin_amdgcn_s_sleep(1)
+
+#endif  // defined(__HIPCC__)

include/strata/kernels/bf16_bits.hpp

@@ -19,7 +19,7 @@
 #include <cstdint>
 #include <cstring>
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_BF16_HD __host__ __device__
 #else
 #define STRATA_BF16_HD
@@ -30,7 +30,11 @@ namespace strata::kernels {
 /// Round-to-nearest-even f32 -> bf16, returned as raw bits in the LOW half of the uint16.
 STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) {
     uint32_t i;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&i, &f, 4);
+#else
     std::memcpy(&i, &f, 4);
+#endif
     i = (i + ((i >> 16) & 1u) + 0x7FFFu) & 0xFFFF0000u;
     return (uint16_t) (i >> 16);
 }
@@ -39,7 +43,11 @@ STRATA_BF16_HD inline uint16_t bf16_from_f32(float f) {
 STRATA_BF16_HD inline float f32_from_bf16(uint16_t h) {
     const uint32_t i = (uint32_t) h << 16;
     float f;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&f, &i, 4);
+#else
     std::memcpy(&f, &i, 4);
+#endif
     return f;
 }
 

include/strata/kernels/f16_bits.hpp

@@ -25,7 +25,7 @@
 #include <cstdint>
 #include <cstring>
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_HD __host__ __device__
 #else
 #define STRATA_HD
@@ -36,7 +36,11 @@ namespace strata::kernels {
 /// Round-to-nearest-even f32 -> fp16, returned as raw bits.
 STRATA_HD inline uint16_t f16_from_f32(float f) {
     uint32_t x;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&x, &f, 4);
+#else
     std::memcpy(&x, &f, 4);
+#endif
     const uint32_t sign = (x >> 16) & 0x8000u;
     const uint32_t rawexp = (x >> 23) & 0xFFu;
     int exp = (int) rawexp - 127 + 15;
@@ -89,7 +93,11 @@ STRATA_HD inline float f32_from_f16(uint16_t h) {
         out = sign | ((ex - 15 + 127) << 23) | (man << 13);
     }
     float f;
+#if defined(__HIP_DEVICE_COMPILE__)
+    __builtin_memcpy(&f, &out, 4);
+#else
     std::memcpy(&f, &out, 4);
+#endif
     return f;
 }
 

include/strata/kernels/mrope.hpp

@@ -22,7 +22,7 @@ namespace strata::kernels {
 void mrope_table_set(const int32_t* device_table);
 const int32_t* mrope_table();
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 /// ggml rope_multi, is_imrope, sections {11, 11, 10, 0}: sector = pair % 32; sector % 3 == 1 -> h (sector < 33),
 /// == 2 -> w (sector < 30), == 0 -> t (sector < 33).  For pairs 0..31 all three bounds hold, so it is pair % 3.
 __device__ __forceinline__ int mrope_pos(const int32_t* tab, int pos, int pair) {

include/strata/kernels/qsa_select.hpp

@@ -28,8 +28,16 @@ void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx
                       int64_t active_blocks = -1);   ///< perf-review C-1: > 0 launches only this many blocks (the
                                                      ///< batch's largest n_bid + 1; not for a captured graph)
 
+/// The same scores on tensor cores (3xTF32, FP32-level accuracy but another summation order: not bitwise; the tail
+/// block n_bid is the warp kernel's arithmetic). For the prompt path; false (nothing launched) on another geometry.
+bool qsa_block_scores_tc(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+                         int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks);
+
 /// ids [nq, cap] (cells, ascending); `cap` >= the largest selection width.
 void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
                     const QsaShapes& s, int32_t* ids, void* stream);
+/// The original kernel (keys read from memory on every radix pass), for tests: the same ids.
+void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                        const QsaShapes& s, int32_t* ids, void* stream);
 
 }  // namespace strata::kernels

include/strata/kernels/rope.hpp

@@ -10,7 +10,7 @@
 
 #include <cstdint>
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIPCC__)
 #define STRATA_ROPE_HD __host__ __device__
 #else
 #define STRATA_ROPE_HD

include/strata/prefill/gemm.hpp

@@ -6,6 +6,7 @@
 // rounded to BF16, which is also what llama.cpp's batched CUDA path does.  Tensor-core GEMM through cuBLAS.
 #pragma once
 
+#include <cstddef>
 #include <cstdint>
 #include <string>
 
@@ -50,6 +51,7 @@ private:
     int64_t scratch_elems_ = 0;
     void* workspace_ = nullptr;
     bool external_ = false;
+    void* hipblaslt_state_ = nullptr;
 };
 
 

include/strata/prefill/kernels.hpp

@@ -14,6 +14,15 @@ namespace strata::prefill {
 // ---- hyper-connection (n_embd 2560, hc 4, hc_lr 320)
 /// xn[t, c*2560 + d] = R[t,c,d] * rsqrt(mean_d R[t,c,:]^2 + eps) * w_norm[c*2560 + d]; also its BF16 image.
 void gr_norm(const float* R, const float* w_norm, float eps, float* xn, uint16_t* xn16, int64_t T, void* stream);
+/// F-1: gr_norm without its FP32 output: the row scales rs[t*4 + c] and the BF16 image; gr_mix_r then reads R.
+void gr_norm_rs(const float* R, const float* w_norm, float eps, float* rs, uint16_t* xn16, int64_t T, void* stream);
+/// gr_mix with xn recomputed from R, rs and w_norm exactly as gr_norm computes it (the same bits).
+void gr_mix_r(const float* R, const float* rs, const float* w_norm, const float* gated, float* mixed, uint16_t* mixed16,
+              int64_t T, void* stream, uint16_t* mixed_h = nullptr);
+/// F-2: gr_write, then gr_norm_rs of the next half (its norm weights) over the rows just written - the same bits as
+/// the two calls, without reading R back.
+void gr_write_norm_rs(float* R, const float* bo, const float* inj, int64_t inj_ld, const float* w_norm_next, float eps,
+                      float* rs, uint16_t* xn16, int64_t T, void* stream);
 /// lo16[t, k] = bf16(silu(lo[t, k] / hc))
 void gr_silu(const float* lo, uint16_t* lo16, int64_t T, void* stream);
 /// mixed[t, d] = mean_c xn[t, c, d] * sigmoid(gated[t, c, d]); FP32, BF16 and FP16 (either image may be null).
@@ -44,6 +53,9 @@ void blob_dequant(const uint8_t* blob, uint16_t* gu16, uint16_t* down16, void* s
 void swiglu_interleaved(const float* gu, uint16_t* h16, int64_t n, void* stream);
 /// h16[n, r] = fp16(silu(g[n, r]) * u[n, r])   (the shared expert, gate and up separate, width 640)
 void swiglu_pair(const float* g, const float* u, uint16_t* h16, int64_t n, void* stream);
+/// dst[i] = src[i] for n int32s, as a kernel: either side may be mapped host memory, and the copy never waits
+/// behind the copy engine's queue (the prompt path's grouping tables, while the expert stream fills it).
+void copy_i32(int32_t* dst, const int32_t* src, int64_t n, void* stream);
 /// Gather rows: dst16[i, :] = x16[src[i], :] (n rows of `width` BF16).
 void gather_rows16(const uint16_t* x16, const int32_t* src, uint16_t* dst16, int64_t n, int64_t width, void* stream);
 /// bo[t, :] = shared[t, :] * sigmoid(sg[t]) + sum_k w[t, k] * D[slot[t, k], :]

include/strata/prefill/prefill.hpp

@@ -34,8 +34,19 @@ struct PrefillStats {
     double ms_ple = 0;
 };
 
+}  // namespace strata::prefill
+namespace strata::core { class MtpDrafter; }
+namespace strata::prefill {
+
 class Prefill {
 public:
+    /// E-9: the draft layer's K/V for prompt cells [cell0, cell0 + n) from their final residual rows `R_rows`
+    /// (device) and `next_tokens` (host: the token at cell+1), in batches through this path's GEMMs and its idle
+    /// scratch - call it from on_chunk.  false with `err` empty: not applicable here (a ring or hybrid K/V, another
+    /// device, too little scratch; STRATA_MTP_BATCH=0), the caller runs the drafter's own pass.  Not bit-identical to
+    /// that pass (FP16 GEMMs instead of Q8_1 activations): the drafts may differ, never the target's tokens' logits.
+    bool draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t* next_tokens, int64_t n, int64_t cell0,
+                  std::string& err);
     Prefill();
     ~Prefill();
     Prefill(const Prefill&) = delete;

serve/server.py

@@ -488,9 +488,13 @@ def child_env(cfg: dict) -> dict:
     """The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
     compiled it) first on the library search path."""
     env = dict(os.environ)
-    if gpu_list(cfg):                                # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
+    if gpu_list(cfg) and cfg.get("backend") == "hip":   # AMD: numbered as HIP numbers them (setup's KFD order)
+        env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
+    elif gpu_list(cfg):                              # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
         env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"      # own default order (fastest first) can number the cards otherwise
         env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
+    for k, v in (cfg.get("env") or {}).items():      # engine settings the config carries (AMD: the GEMM tuning table)
+        env[str(k)] = str(v)
     dirs = [d for d in cfg.get("lib_dirs") or [] if Path(d).is_dir()]
     if dirs:
         var = "PATH" if os.name == "nt" else "LD_LIBRARY_PATH"

setup.py

@@ -58,7 +58,7 @@ PREBUILT_ASSET = "strata-windows-x64.zip" if WIN else "strata-linux-x64.zip"
 # the CUDA libraries the ready-made engine loads (the same CUDA 13.0 it is built with), from NVIDIA's pip packages
 CUDA_WHEELS = ["nvidia-cublas==13.0.2.14", "nvidia-cuda-runtime==13.0.96"]
 MIN_DRIVER = 580                       # CUDA 13.0
-MIN_ENGINE = (0, 1, 23)                # v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
+MIN_ENGINE = (0, 1, 26)                # v0.1.26: the draft layer's prompt pass in batches; v0.1.25: faster prompts (grouping off the copy engine, fused hyper-connection kernels), AMD HIP backend, --kv k8v4; v0.1.24: long prompts faster (QSA select on tensor cores); v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
 PY_PACKAGES = ["numpy", "jinja2", "regex", "pyyaml", "tqdm", "requests", "cmake", "ninja", "pillow", "psutil"]
 
 MODELS = {
@@ -643,6 +643,120 @@ def cuda_lib_dirs():
     return [str(d) for d in dirs]
 
 
+# ------------------------------------------------------------------------------------------------ AMD (experimental)
+# The RX 7900 XT / XTX (gfx1100) on Linux, through the HIP backend (docs/AMD_HIP.md).  There is no ready-made AMD
+# engine: ROCm comes from AMD's TheRock Python wheels into .venv (no sudo; a system ROCm in /opt/rocm is used when it
+# has hipcc and hipBLAS) and the engine is compiled here.  One GPU, no images yet.
+ROCM_INDEX = os.environ.get("STRATA_ROCM_INDEX", "https://rocm.nightlies.amd.com/v2/gfx110X-dgpu/")
+ROCM_VERSION = os.environ.get("STRATA_ROCM_VERSION", "7.10.0a20251120")   # what Strata's HIP build was tested with
+AMD_ARCHS = ("gfx1100",)
+AMD_NAMES = {"gfx1100": "AMD Radeon RX 7900 series (gfx1100)"}   # when sysfs has no product name
+
+
+def amd_gpus():
+    """AMD GPUs from the kernel's KFD topology (the amdgpu driver; no ROCm needed), numbered as HIP numbers them:
+    the GPU nodes in order, the CPU nodes skipped.  Integrated GPUs are listed too (not supported)."""
+    base = Path("/sys/class/kfd/kfd/topology/nodes")
+    found = []
+    if WIN or not base.is_dir():
+        return found
+    for node in sorted((p for p in base.iterdir() if p.name.isdigit()), key=lambda p: int(p.name)):
+        try:
+            props = {}
+            for line in (node / "properties").read_text().splitlines():
+                k, _, v = line.partition(" ")
+                props[k] = v.strip()
+            ver = int(props.get("gfx_target_version") or 0)
+            if ver == 0 or int(props.get("simd_count") or 0) == 0:
+                continue
+        except (OSError, ValueError):
+            continue
+        arch = f"gfx{ver // 10000}{(ver // 100) % 100:x}{ver % 100:x}"
+        dev = Path(f"/sys/class/drm/renderD{props.get('drm_render_minor', '')}/device")
+        try:
+            vram = int((dev / "mem_info_vram_total").read_text()) / 2 ** 30
+        except (OSError, ValueError):
+            vram = 0.0
+        try:
+            name = (dev / "product_name").read_text().strip() or f"AMD Radeon ({arch})"
+        except OSError:
+            name = f"AMD Radeon ({arch})"
+        if name == f"AMD Radeon ({arch})" and arch in AMD_NAMES:
+            name = AMD_NAMES[arch]
+        found.append({"index": len(found), "name": name, "vram_gb": vram, "arch": arch, "driver": "amdgpu",
+                      "vendor": "amd"})
+    return found
+
+
+def amd_problem(g):
+    if g["arch"] not in AMD_ARCHS:
+        return (f"not supported - Strata's AMD backend runs on the RX 7900 XT / XTX ({', '.join(AMD_ARCHS)}) only, "
+                f"this is {g['arch']}")
+    return None
+
+
+def rocm_root():
+    """ROCm for compiling and running the HIP engine: (root, library folders).  A system ROCm with hipcc and hipBLAS,
+    else AMD's TheRock wheels (ROCM_VERSION, from ROCM_INDEX) installed into .venv."""
+    sysroot = Path(os.environ.get("ROCM_PATH") or "/opt/rocm")
+    if (sysroot / "bin" / "hipcc").exists() and list((sysroot / "lib").glob("libhipblas.so*")):
+        return sysroot, [str(sysroot / "lib")]
+    stamp = Path(sys.prefix) / ".strata-rocm.json"
+    if not stamp.exists() or json.loads(stamp.read_text()).get("version") != ROCM_VERSION:
+        say(f"  Installing ROCm {ROCM_VERSION} for AMD GPUs into .venv (AMD's TheRock wheels, ~10 GB, no sudo) ...")
+        run([sys.executable, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", "--index-url",
+             ROCM_INDEX, f"rocm[libraries,devel]=={ROCM_VERSION}"])
+        stamp.write_text(json.dumps({"version": ROCM_VERSION, "index": ROCM_INDEX}))
+    sdk = Path(sys.executable).parent / "rocm-sdk"
+    root = Path(out([str(sdk), "path", "--root"]).strip())
+    if not (root / "llvm" / "bin" / "clang++").exists():
+        fail(f"ROCm was installed but its compiler is missing ({root})",
+             f"remove {stamp} and run this again; or install ROCm 7 system-wide")
+    dirs = [str(root / "lib")]
+    for sp in {Path(p) for p in sys.path if p.endswith("site-packages")}:
+        dirs += [str(d / "lib") for d in sorted(sp.glob("_rocm_sdk_libraries_*")) if (d / "lib").is_dir()]
+    ok(f"ROCm: {root}")
+    return root, list(dict.fromkeys(dirs))
+
+
+def build_engine_hip(gpu, llama) -> Path:
+    """Compile the HIP engine for this AMD GPU into engine/ (again only when its source changed: a `git pull`)."""
+    eng = ROOT / "engine"
+    eng.mkdir(exist_ok=True)
+    stamp = eng / "BUILD.json"
+    meta = json.loads(stamp.read_text()) if stamp.exists() else {}
+    src = source_hash(ENGINE_SOURCES)
+    if meta.get("backend") == "hip" and (eng / EXE).exists() and meta.get("src") == src:
+        ok("engine already built for this PC")
+        return eng
+    if not (shutil.which("c++") or shutil.which("g++")) or not shutil.which("git"):
+        fail("a C++ compiler and git are needed to compile the AMD engine",
+             "Ubuntu/Debian: sudo apt install build-essential git   Fedora: sudo dnf install gcc-c++ git")
+    root, dirs = rocm_root()
+    libs = [str(Path(d).parent) for d in dirs[1:]]
+    bitcode = next((p for p in (root / "lib" / "llvm" / "amdgcn" / "bitcode", root / "amdgcn" / "bitcode") if p.is_dir()),
+                   root / "amdgcn" / "bitcode")
+    os.environ.update({"HIP_PLATFORM": "amd", "HIP_COMPILER": "clang", "HIP_RUNTIME": "rocclr", "ROCM_PATH": str(root),
+                       "HIP_PATH": str(root)})
+    os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(dirs + [os.environ.get("LD_LIBRARY_PATH", "")]).rstrip(os.pathsep)
+    os.environ["PATH"] = os.pathsep.join([str(root / "bin"), str(root / "llvm" / "bin"), os.environ.get("PATH", "")])
+    say("  The engine's source changed: compiling it again (only what changed, a few minutes) ..."
+        if meta.get("backend") == "hip" and (eng / EXE).exists()
+        else "  Compiling the Strata engine for your AMD GPU (10-20 minutes, once) ...")
+    cmake_build(ROOT, ROOT / "build-hip", "strata",
+                ["-DSTRATA_ENABLE_HIP=ON", "-DSTRATA_ENABLE_CUDA=OFF", "-DSTRATA_BUILD_TESTS=OFF",
+                 "-DSTRATA_PREFILL_MMQ=ON", f"-DCMAKE_HIP_ARCHITECTURES={gpu['arch']}",
+                 f"-DCMAKE_HIP_COMPILER={root / 'llvm' / 'bin' / 'clang++'}", f"-DCMAKE_HIP_COMPILER_ROCM_ROOT={root}",
+                 "-DCMAKE_PREFIX_PATH=" + ";".join([str(root), *libs]),
+                 f"-DCMAKE_HIP_FLAGS=--rocm-path={root} --rocm-device-lib-path={bitcode}",
+                 f"-DSTRATA_GGML_DIR={llama}"], None, "")
+    shutil.copy2(ROOT / "build-hip" / EXE, eng / EXE)
+    stamp.write_text(json.dumps({"source": "local-hip", "backend": "hip", "version": source_version(),
+                                 "archs": [gpu["arch"]], "vision": "none", "lib_dirs": dirs, "src": src}, indent=1))
+    ok(f"engine compiled: {eng / EXE}")
+    return eng
+
+
 # ------------------------------------------------------------------------------------------------ the engine
 def driver_major(gpu):
     try:
@@ -729,6 +843,17 @@ def update_installed_engine(url_base) -> None:
         return
     meta_text = info.read_text()
     meta = json.loads(meta_text)
+    if meta.get("backend") == "hip":                   # AMD: compiled here, again when its source changed
+        if meta.get("src") != source_hash(ENGINE_SOURCES):
+            try:
+                g = next((x for x in amd_gpus() if amd_problem(x) is None), None)
+                if g is None:
+                    raise RuntimeError("no supported AMD GPU found")
+                build_engine_hip(g, get_llama_cpp())
+            except (Exception, SystemExit) as e:
+                warn(f"could not compile the updated engine{'' if isinstance(e, SystemExit) else f' ({e})'}: "
+                     "starting the installed one")
+        return
     ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:3] if x.isdigit())
     local = meta.get("source") == "local"
     vision = meta.get("vision") or "none"
@@ -918,6 +1043,26 @@ def build_engine(gpu, vision, yes, llama) -> Path:
 DATA_ITEMS = ("models", "packs", "mtp")
 
 
+LOW_RAM_HEADROOM_GB = 10   # RAM beside the experts: the OS, the engine's other buffers, the server
+
+
+def low_ram_needed(model, ram) -> bool:
+    """The model's experts do not fit this PC's RAM with room left for the rest: they are then mapped from the pack's
+    experts.bin instead of copied into RAM (the low-RAM mode)."""
+    return ram < MODELS[model]["arena_gb"] + LOW_RAM_HEADROOM_GB
+
+
+def low_ram_gpu_share(model, vram_gb) -> float:
+    """About how much of the model's experts the GPU holds (VRAM minus ~5 GB for the dense weights, KV, buffers)."""
+    return max(0.0, min(1.0, (vram_gb - 5) / MODELS[model]["arena_gb"]))
+
+
+def low_ram_fits(model, ram, vram_gb) -> bool:
+    """In the low-RAM mode: the experts the GPU does not hold fit the RAM left beside the rest (as file cache)."""
+    arena = MODELS[model]["arena_gb"]
+    return ram - 6 + max(0.0, vram_gb - 5) >= arena
+
+
 def settings_path() -> Path:
     if WIN:
         return Path(os.environ.get("APPDATA") or Path.home() / "AppData" / "Roaming") / "Strata" / "settings.json"
@@ -1221,8 +1366,22 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
     cfg_path.touch()                                     # the most recently used model
     cmd = [sys.executable, str(ROOT / "serve" / "server.py"), "--engine", "strata", "--config", str(cfg_path),
            "--port", str(port or cfg.get("port", 8080))]
-    found = gpus()
-    if isinstance(gpu, list):                          # --gpus: saved, this model runs on these cards from now on
+    if cfg.get("backend") == "hip":                    # AMD: one card, numbered as HIP numbers them
+        if isinstance(gpu, list):
+            fail("several GPUs sharing one model: NVIDIA only for now", "use one AMD card (--gpu N)")
+        if gpu is not None:
+            cmd += ["--gpu", str(gpu)]
+        found = []
+        use = gpu if gpu is not None else cfg.get("gpu")
+        g = next((x for x in amd_gpus() if x["index"] == (use if use is not None else x["index"])
+                  and amd_problem(x) is None), None)
+        if g is not None:
+            ok(f"GPU: {g['name']} ({g['vram_gb']:.0f} GB, AMD)")
+    else:
+        found = gpus()
+    if cfg.get("backend") == "hip":
+        pass
+    elif isinstance(gpu, list):                        # --gpus: saved, this model runs on these cards from now on
         check_gpus(gpu, found)
         cfg["gpu"], cfg["gpus_asked"] = gpu, True
         cfg["layer_split"] = layer_split or cfg.get("layer_split") or "auto"
@@ -1234,7 +1393,9 @@ def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_
     else:
         cfg = offer_together(cfg_path, cfg, yes)
     use = gpu if gpu is not None else cfg.get("gpu")
-    if isinstance(use, list):
+    if cfg.get("backend") == "hip":
+        pass
+    elif isinstance(use, list):
         check_gpus(use, found, "(chosen for this model) ")
         byid = {g["index"]: g for g in found}
         ok("GPUs: " + " + ".join(gpu_name(byid[i]) for i in use) + f" together (layers split {cfg.get('layer_split') or 'auto'})")
@@ -1283,9 +1444,9 @@ def main() -> int:
     ap.add_argument("--family", choices=list(FAMILIES), help="qwen = Qwen3.8-Flash-Next, swift = Swift 1.5")
     ap.add_argument("--model", choices=list(MODELS))
     ap.add_argument("--context", type=int)
-    ap.add_argument("--kv", choices=["int8", "q4_0"],
-                    help="KV cache precision above 8K context: int8 (default) or q4_0 (half the memory, a little less "
-                         "precise)")
+    ap.add_argument("--kv", choices=["int8", "q4_0", "k8v4"],
+                    help="KV cache precision above 8K context: int8 (default), q4_0 (half the memory, a little less "
+                         "precise) or k8v4 (hybrid: INT8 K + 4-bit V, 816 B/cell)")
     ap.add_argument("--vision", choices=["yes", "no", "none", "gpu", "cpu"],
                     help="let the model read images (yes = the encoder on the GPU)")
     ap.add_argument("--experimental-speed-projection", metavar="on|off|GGUF",
@@ -1315,6 +1476,12 @@ def main() -> int:
     ap.add_argument("--check", action="store_true", help="only check this PC and exit")
     ap.add_argument("--calibrate", action="store_true",
                     help="tune the engine's settings for this PC (about 5-10 minutes), then start the model")
+    ap.add_argument("--low-ram", choices=["auto", "on", "off"], default="auto",
+                    help="map the model's experts from its folder instead of copying them into RAM (for a PC with a big "
+                         "GPU and little RAM); auto: when the experts would not fit the RAM")
+    ap.add_argument("--backend", choices=["cuda", "hip"],
+                    help="cuda = NVIDIA (default), hip = AMD RX 7900 XT/XTX on Linux (experimental; chosen by itself "
+                         "when the PC has no NVIDIA card Strata can use)")
     ap.add_argument("--skip-build", action="store_true", help=argparse.SUPPRESS)
     a = ap.parse_args()
     if a.gpu is not None:                              # --gpu 0,2 means --gpus 0,2 (a user tried it: issue report)
@@ -1324,7 +1491,7 @@ def main() -> int:
             a.gpu = int(a.gpu)
         else:
             ap.error(f"--gpu takes a GPU number as nvidia-smi numbers them, e.g. --gpu 1 (or --gpus 0,2), not {a.gpu!r}")
-    say("Strata - Qwen3.8-Flash-Next on a normal PC (NVIDIA GPU + system RAM + CPU)")
+    say("Strata - Qwen3.8-Flash-Next on a normal PC (a GPU + system RAM + CPU)")
     data, elsewhere = data_folder(a.data_dir)          # the model files: in the data folder, found from any copy
     roots = [data, *elsewhere]
     if a.models_dir is None:
@@ -1385,36 +1552,76 @@ def main() -> int:
     # ---- 1. the PC
     step(1, "checking your PC")
     found = gpus()
-    if not found:
-        fail("no NVIDIA GPU found (nvidia-smi did not answer)",
-             "install the NVIDIA driver from https://www.nvidia.com/drivers and restart the PC")
-    if len(found) > 1 or gpu_problem(found[0]) is not None:
-        gpu_table(found)
-    sel = choose_gpus(a, found)                        # asked when two or more cards can share the model
-    multi = sel if len(sel) > 1 else []
-    a.gpu = sel[0]                                     # the main GPU: the checks and the sizing below are its
-    GPU_PICK = a.gpu
-    gpu = gpu_info(a.gpu)
-    chosen = [gpu_info(i) for i in sel]
-    gpu["archs"] = sorted({x["arch"] for x in chosen})  # the engine needs code for every one of them
-    if multi:
-        ok("GPUs: " + " + ".join(gpu_name(x) for x in chosen) + " together (the model's layers are split across them)")
-    ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, compute capability {cc(gpu)}, driver {gpu['driver']}")
-    if driver_major(gpu) < MIN_DRIVER:
-        fail(f"the NVIDIA driver is too old ({gpu['driver']}; {MIN_DRIVER} or newer is needed)",
-             "update it with the NVIDIA App or from https://www.nvidia.com/drivers, restart, and run this again")
+    amd = [] if WIN else amd_gpus()
+    nv_ok = any(gpu_problem(g) is None for g in found)
+    amd_ok = [g for g in amd if amd_problem(g) is None]
+    hip = a.backend == "hip" or (a.backend is None and not nv_ok and bool(amd_ok))
+    if a.backend is None and nv_ok and amd_ok:
+        # both kinds of card: asked (a first run on such a PC used to take NVIDIA without mentioning the Radeon)
+        say()
+        say("  This PC has NVIDIA and AMD cards Strata can use:")
+        say("  1) NVIDIA: " + ", ".join(f"{g['name']} ({g['vram_gb']:.0f} GB)" for g in found if gpu_problem(g) is None)
+            + "   (recommended)")
+        say("  2) AMD: " + ", ".join(f"{g['name']} ({g['vram_gb']:.0f} GB)" for g in amd_ok)
+            + "   (experimental: compiled here, one GPU, no images - docs/AMD_HIP.md)")
+        hip = ask("Which cards?", ["1", "2"], "1", a.yes or a.check) == "2"
+        if a.check and not hip:
+            say("  (the AMD card: ./setup.sh --backend hip)")
+    if hip:                                            # AMD (experimental): one card, compiled here
+        if WIN:
+            fail("Strata's AMD backend runs on Linux only", "use an NVIDIA RTX 30 series or newer card on Windows")
+        say("  Your AMD GPUs:" if amd else "  No AMD GPU found (the amdgpu driver's KFD topology is empty).")
+        for g in amd:
+            say(f"    GPU {g['index']}: {g['name']}, {g['vram_gb']:.0f} GB VRAM - " + (amd_problem(g) or "can be used"))
+        usable = [g for g in amd if amd_problem(g) is None]
+        if not usable:
+            fail("no AMD GPU Strata can use", "the AMD backend runs on the RX 7900 XT / XTX (gfx1100) on Linux")
+        if a.gpus:
+            fail("several GPUs sharing one model: NVIDIA only for now", "use one AMD card (--gpu N)")
+        if a.gpu is not None:
+            gpu = next((g for g in usable if g["index"] == a.gpu), None)
+            if gpu is None:
+                fail(f"AMD GPU {a.gpu} cannot be used", "use one of: " + ", ".join(f"--gpu {g['index']}" for g in usable))
+        else:
+            gpu = max(usable, key=lambda x: (round(x["vram_gb"]), -x["index"]))
+        gpu = {**gpu, "count": len(amd), "archs": [gpu["arch"]]}
+        sel, multi, chosen = [gpu["index"]], [], [gpu]
+        a.gpu = gpu["index"] if len(amd) > 1 else a.gpu
+        ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, {gpu['arch']} (AMD, experimental: docs/AMD_HIP.md)")
+    else:
+        if not found:
+            fail("no NVIDIA GPU found (nvidia-smi did not answer)",
+                 "install the NVIDIA driver from https://www.nvidia.com/drivers and restart the PC"
+                 + ("; an AMD RX 7900 XT/XTX: --backend hip" if amd else ""))
+        if len(found) > 1 or gpu_problem(found[0]) is not None:
+            gpu_table(found)
+        sel = choose_gpus(a, found)                    # asked when two or more cards can share the model
+        multi = sel if len(sel) > 1 else []
+        a.gpu = sel[0]                                 # the main GPU: the checks and the sizing below are its
+        GPU_PICK = a.gpu
+        gpu = gpu_info(a.gpu)
+        chosen = [gpu_info(i) for i in sel]
+        gpu["archs"] = sorted({x["arch"] for x in chosen})  # the engine needs code for every one of them
+        if multi:
+            ok("GPUs: " + " + ".join(gpu_name(x) for x in chosen) + " together (the model's layers are split across them)")
+        ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, compute capability {cc(gpu)}, driver {gpu['driver']}")
+        if driver_major(gpu) < MIN_DRIVER:
+            fail(f"the NVIDIA driver is too old ({gpu['driver']}; {MIN_DRIVER} or newer is needed)",
+                 "update it with the NVIDIA App or from https://www.nvidia.com/drivers, restart, and run this again")
     if gpu["vram_gb"] < 11:
         warn("less than 12 GB of VRAM: Strata will run, but most experts stay on the CPU and it will be slow")
     ram = ram_gb()
     cpu, avx2, avx512 = cpu_info()
     need = min(d["ram_gb"] for d in MODELS.values())
-    if ram < need - 4 and not a.check:
+    low_ok = low_ram_fits("IQ1_M", ram, gpu["vram_gb"]) and a.low_ram != "off"   # the smallest model, mapped
+    if ram < need - 4 and not a.check and not low_ok:
         # every model keeps ALL its experts in RAM (23+ GB); VRAM only holds a copy of the most-used ones, so a
         # bigger GPU does not lower this
         fail(f"RAM: {ram:.0f} GB - the smallest model (the Coder) needs about {need} GB",
              "Strata keeps all of the model's experts in RAM (23-50 GB, whatever the GPU) and the GPU holds a copy "
              "of the most-used ones: it needs 32 GB of RAM or more (48 GB for the full model)")
-    ok(f"RAM: {ram:.0f} GB" if ram >= need - 4 else f"RAM: {ram:.0f} GB (less than the {need} GB the smallest model needs)")
+    ok(f"RAM: {ram:.0f} GB" if ram >= need - 4 else f"RAM: {ram:.0f} GB (less than the {need} GB the smallest model needs)"
+       + ("; the GPU's VRAM makes up for it (the low-RAM mode)" if ram < need - 4 and low_ok else ""))
     pf = page_file_gb()
     if pf is not None and pf < 4:
         warn(f"Windows' page file is {pf:.1f} GB: the graphics card's memory needs room there too (issue #60), so "
@@ -1427,6 +1634,9 @@ def main() -> int:
         say()
         for m, d in MODELS.items():
             verdict = "fits" if ram >= d["ram_gb"] else "tight" if ram >= d["ram_gb"] - 8 else "does not fit"
+            if low_ram_needed(m, ram) and low_ram_fits(m, ram, gpu["vram_gb"]) and a.low_ram != "off":
+                verdict = (f"fits in the low-RAM mode (the GPU holds ~{100 * low_ram_gpu_share(m, gpu['vram_gb']):.0f}% "
+                           "of its experts)")
             say(f"  {m:8s} needs ~{d['ram_gb']} GB RAM: {verdict}")
         say("\nThis PC can run Strata. Run it again without --check to install.")
         return 0
@@ -1452,10 +1662,23 @@ def main() -> int:
     for i, m in enumerate(names, 1):
         d = MODELS[m]
         fit = "" if ram >= d["ram_gb"] else f"   <- needs {d['ram_gb']} GB RAM, you have {ram:.0f}"
+        if low_ram_needed(m, ram) and low_ram_fits(m, ram, gpu["vram_gb"]) and a.low_ram != "off":
+            fit = f"   <- fits in the low-RAM mode (the GPU holds ~{100 * low_ram_gpu_share(m, gpu['vram_gb']):.0f}%)"
         say(f"  {i}) {m:8s} {d['about']}; download {d['download_gb']:.0f} GB, uses ~{d['arena_gb']:.0f} GB of RAM{fit}")
     rec = str(names.index("IQ3_XXS") + 1) if ram >= 60 and "IQ3_XXS" in names else "1"
     model = a.model or names[int(ask("Which size?", [str(i) for i in range(1, len(names) + 1)], rec, a.yes)) - 1]
-    if ram < MODELS[model]["ram_gb"] - 4:
+    low_ram = a.low_ram == "on" or (a.low_ram == "auto" and low_ram_needed(model, ram))
+    if low_ram and multi:
+        warn("the low-RAM mode runs on one GPU: using " + gpu_name(gpu) + " only")
+        multi, sel, chosen = [], [gpu["index"]], [gpu]
+    if low_ram:
+        share = low_ram_gpu_share(model, gpu["vram_gb"])
+        ok(f"low-RAM mode: {model}'s experts ({MODELS[model]['arena_gb']:.0f} GB) are read from the model folder "
+           f"through the OS file cache instead of a copy in RAM ({ram:.0f} GB); the GPU holds ~{100 * share:.0f}% of them")
+        if share < 0.6:
+            warn("most of the experts are read from the SSD while it answers: expect it to be much slower than with "
+                 "enough RAM (a faster SSD and a smaller size help)")
+    elif ram < MODELS[model]["ram_gb"] - 4:
         # #125: a warning and a question, not a stop: the user may accept paging (asked, "no" by default, so an
         # unattended --yes install still stops here)
         need_gb, arena = MODELS[model]["ram_gb"], MODELS[model]["arena_gb"]
@@ -1496,7 +1719,11 @@ def main() -> int:
         kv = ["int8", "q4_0"][int(ask("KV cache?", ["1", "2"], "1", a.yes)) - 1]
     if ctx > 8192:
         ok(f"KV cache: {'8-bit' if kv == 'int8' else '4-bit (Hadamard-rotated)'}")
-    if a.vision:
+    if hip:
+        vision = "none"
+        if a.vision not in (None, "no", "none"):
+            warn("images are not available with the AMD backend yet: off")
+    elif a.vision:
         vision = {"yes": "gpu", "no": "none"}.get(a.vision, a.vision)
     else:
         say()
@@ -1534,7 +1761,8 @@ def main() -> int:
                 break
     have_model = all(s.exists() and (done(s) or a.gguf_dir) for s in shards)
     need = (0 if a.gguf_dir or have_model else MODELS[model]["download_gb"]) + 8 + \
-        (40 if model == "Q2_0" and avx512 and family == "qwen" else 0) + (1 if vision != "none" else 0)
+        (40 if model == "Q2_0" and avx512 and family == "qwen" else 0) + (1 if vision != "none" else 0) + \
+        (MODELS[model]["arena_gb"] + 1 if low_ram and not (model == "Q2_0" and avx512 and family == "qwen") else 0)
     if free_gb(models_dir) < need:
         fail(f"not enough free disk space in {models_dir}: need ~{need:.0f} GB", "use --models-dir on a bigger drive")
 
@@ -1546,16 +1774,16 @@ def main() -> int:
     step(4, "the Strata engine")
     llama = get_llama_cpp()
     ok(f"llama.cpp {LLAMA_CPP_COMMIT[:7]} (gguf-py, ggml, mtmd)")
-    eng = None if a.build else get_prebuilt(a.prebuilt, gpu, vision)
+    eng = None if a.build or hip else get_prebuilt(a.prebuilt, gpu, vision)
     if eng is not None and json.loads((eng / "BUILD.json").read_text()).get("source") != "local":
         pip_install(CUDA_WHEELS, "NVIDIA CUDA libraries (cuBLAS, CUDA runtime; ~0.4 GB)")
         if vision != "none" and not (eng / VEXE).exists():
             warn("the ready-made engine has no image encoder: compiling it")
             eng = None
     if eng is None:
-        eng = build_engine(gpu, vision, a.yes, llama)
+        eng = build_engine_hip(gpu, llama) if hip else build_engine(gpu, vision, a.yes, llama)
     meta = json.loads((eng / "BUILD.json").read_text())
-    lib_dirs = meta.get("cuda_dirs") or cuda_lib_dirs()
+    lib_dirs = meta.get("lib_dirs") or meta.get("cuda_dirs") or cuda_lib_dirs()
     ok(f"engine: {eng / EXE}")
 
     # ---- 5. the model files
@@ -1605,6 +1833,10 @@ def main() -> int:
     elif not (pack / "native_experts.txt").exists() or not (pack / "tokenizer" / "vocab.json").exists():
         # every tensor as the GGUF stores it; the experts are read from the GGUF at start (seconds to build)
         run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(shards[0]), "--out", str(pack)], env=env)
+    if low_ram and not (pack / "experts.bin").exists():
+        say(f"  Writing the experts into one file for the low-RAM mode (one time, {MODELS[model]['arena_gb']:.0f} GB) ...")
+        run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(shards[0]), "--out", str(pack),
+             "--experts-bin"], env=env)
     ok(f"model prepared: {pack}")
     mtp = (find_in(roots, "mtp/rt/experts.bin") or data / "mtp/rt/experts.bin").parent.parent
     rt = mtp / "rt"
@@ -1633,12 +1865,19 @@ def main() -> int:
             "--max-context", str(ctx)]
     if ctx > 8192:
         args += ["--kv", kv]
+    if low_ram:
+        args += ["--mmap-experts"]   # the experts from the pack's experts.bin, not copied into RAM
     # KV streaming: from 64K up the whole KV cache lives in RAM and only the part the attention reads (32K positions
     # per layer) stays in VRAM; the VRAM it frees holds more experts (+6% at 128K, +23% at 262K with Q2_0). It
     # costs ~13.7 KB of RAM per context token with 8-bit KV (1.7 GB at 128K), 7.5 KB with 4-bit, so only when it fits.
     kv_ram_gb = ctx * (13 * (576 if kv == "q4_0" else 1056)) / 1e9   # 12 QSA layers + the draft layer
-    # the RAM copy must be pinned, and under WSL the NVIDIA driver pins only about 1 GB in all
-    if is_wsl() and ctx >= 65536:
+    # Hybrid K8V4 never streams its KV (mode 0 only, layer.hpp), so it is excluded from the WHOLE streaming
+    # decision rather than one threshold at a time - a future tier added to this chain cannot reintroduce the
+    # combination the engine refuses (PR review).
+    if kv == "k8v4":
+        if ctx >= 65536:
+            ok("KV streaming off: not supported with --kv k8v4; the KV cache stays in VRAM")
+    elif is_wsl() and ctx >= 65536:
         ok("WSL: KV streaming off (the driver pins only about 1 GB of RAM); the KV cache stays in VRAM")
     elif ctx >= 65536 and ram >= MODELS[model]["ram_gb"] + kv_ram_gb + 1:
         args += ["--kv-resident", "32768"]
@@ -1652,6 +1891,13 @@ def main() -> int:
     cfg = {"exe": str(eng / EXE), "args": args, "cwd": str(ROOT), "tokenizer": str(pack / "tokenizer"),
            "model_name": f"{fam['name']}-{model.lower()}", "log": str(ROOT / f"strata-{tag.lower()}.log"),
            "lib_dirs": lib_dirs, "port": port}
+    if hip:
+        cfg["backend"] = "hip"
+        # the dense prompt GEMMs through hipBLASLt with kernels measured on this GPU generation (tools/hip; +40-60%
+        # prompt speed on the 7900 XTX); the engine refuses a table made for another hipBLASLt version and falls back
+        tables = sorted((ROOT / "tools" / "hip").glob(f"{gpu['arch']}-hipblaslt-*.txt"))
+        if tables:
+            cfg["env"] = {"STRATA_HIPBLASLT_TUNING": str(tables[-1])}
     if gpu["count"] > 1 or a.gpu is not None:
         cfg["gpu"] = gpu["index"]                      # the engine is told this card (issue #51)
         cfg["gpus_asked"] = True                       # chosen at setup: not asked again at start
@@ -1669,7 +1915,7 @@ def main() -> int:
         if vision == "cpu":
             cfg["vision"]["threads"] = max(1, (os.cpu_count() or 8) // 2)
     cfg_path = ROOT / f"strata-{tag.lower()}.json"
-    cal = saved_calibration(cfg)
+    cal = None if hip else saved_calibration(cfg)     # tools/calibrate.py is NVIDIA-only for now
     if cal is not None:
         sys.path.insert(0, str(ROOT / "tools"))
         import calibrate as CAL
@@ -1678,7 +1924,7 @@ def main() -> int:
     cfg_path.write_text(json.dumps(cfg, indent=1), encoding="utf-8")
     script = write_run_script(tag, cfg_path, port)
     # offered only when someone answers: --yes installs and adopted earlier installs are not held up by it
-    if cal is None and not a.no_start and not a.yes and ask(
+    if cal is None and not hip and not a.no_start and not a.yes and ask(
             "Tune Strata for this PC now? It measures a few engine settings (about 5-10 minutes; the PC is busy "
             "meanwhile; later: START-HERE --calibrate)", ["y", "n"], "y", a.yes) == "y":
         calibrate_config(cfg_path)

setup.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 # Strata for Linux: the first run installs everything and starts the model; later runs just start it.
-# Needs only an NVIDIA driver. Python (with venv) is installed through apt/dnf if it is missing (asks for sudo).
+# Needs only an NVIDIA driver (or, experimental: an AMD RX 7900 XT/XTX with the amdgpu driver, see docs/AMD_HIP.md).
+# Python (with venv) is installed through apt/dnf if it is missing (asks for sudo).
 cd "$(dirname "$0")" || exit 1
 # Python 3.10+ that can make a venv WITH pip: Debian/Ubuntu ship `venv` without `ensurepip` (that is the separate
 # python3-venv package), and a venv made without it has no pip

src/core/device.cu

@@ -30,7 +30,11 @@ DeviceInfo device_info(int ordinal) {
     int count = 0;
     check(cudaGetDeviceCount(&count), "cudaGetDeviceCount");
     if (count == 0) {
+#if defined(STRATA_USE_HIP)
+        throw CudaError("no HIP device is present; this backend targets gfx1100 wave32", -1);
+#else
         throw CudaError("no CUDA device is present; Strata needs an NVIDIA GPU of compute capability 8.0 or newer", -1);
+#endif
     }
     if (ordinal < 0 || ordinal >= count) {
         throw CudaError("device ordinal " + std::to_string(ordinal) + " is out of range (have " +
@@ -59,13 +63,19 @@ DeviceInfo device_info(int ordinal) {
     // The kernels need sm_80 or newer (tf32 mma in the attention scorer, bf16 math) - the same floor the
     // arch guard in CMakeLists enforces at build time (RTX 30 / 40 / 50).  Anything older is caught here,
     // because a binary can be carried to a machine with an older card and would otherwise silently take
-    // whatever path the driver chose.  Pre-Blackwell is untested by the author; trust, then verify.
+    // whatever path the driver chose.  The HIP backend is validated on gfx1100 (wave32) only.
+#if defined(STRATA_USE_HIP)
+    if (std::strncmp(p.gcnArchName, "gfx1100", 7) != 0 || p.warpSize != 32) {
+        throw CudaError("HIP backend requires validated gfx1100 wave32 hardware", -1);
+    }
+#else
     if (d.cc_major < 8) {
         throw CudaError("device " + d.name + " reports compute capability " + std::to_string(d.cc_major) +
                             "." + std::to_string(d.cc_minor) +
                             "; Strata needs an NVIDIA GPU of compute capability 8.0 or newer (RTX 30 / 40 / 50)",
                         -1);
     }
+#endif
     return d;
 }
 

src/core/device_main.cpp

@@ -33,7 +33,11 @@ int main(int argc, char** argv) {
     try {
         const strata::core::DeviceInfo d = strata::core::device_info(0);
         std::printf("device %d: %s\n", d.ordinal, d.name.c_str());
+#if defined(STRATA_USE_HIP)
+        std::printf("  HIP target          gfx1100 wave32\n");
+#else
         std::printf("  compute capability  %d.%d   (sm_%d%d)\n", d.cc_major, d.cc_minor, d.cc_major, d.cc_minor);
+#endif
         std::printf("  multiprocessors     %d\n", d.multi_processor_count);
         std::printf("  VRAM total / free   %s / %s\n", human(d.total_bytes).c_str(), human(d.free_bytes).c_str());
         std::printf("  driver / runtime    %d / %d\n", d.driver_version, d.runtime_version);

src/core/expert_cache.cpp

@@ -69,6 +69,22 @@ bool read_expert_profile(const std::string& path, int64_t n_layers, int64_t n_ex
 
 ExpertCache::~ExpertCache() { close(); }
 
+#if defined(STRATA_USE_HIP)
+bool ExpertCache::ensure_blocking_staging(std::size_t bytes, std::string& err) {
+    if (bytes <= blocking_staging_bytes_) return true;
+    void* next = nullptr;
+    const cudaError_t status = cudaHostAlloc(&next, bytes, cudaHostAllocDefault);
+    if (status != cudaSuccess) {
+        err = std::string("ExpertCache: HIP blocking staging allocation: ") + cudaGetErrorString(status);
+        return false;
+    }
+    if (blocking_staging_) (void) cudaFreeHost(blocking_staging_);
+    blocking_staging_ = static_cast<uint8_t*>(next);
+    blocking_staging_bytes_ = bytes;
+    return true;
+}
+#endif
+
 bool ExpertCache::open(int64_t n_slots, int64_t n_layers, int64_t n_expert, int64_t blob_bytes,
                        std::string& err) {
     close();
@@ -125,6 +141,12 @@ bool ExpertCache::open(int64_t n_slots, int64_t n_layers, int64_t n_expert, int6
     n_layers_ = n_layers;
     n_expert_ = n_expert;
     blob_ = blob_bytes;
+#if defined(STRATA_USE_HIP)
+    if (!ensure_blocking_staging((std::size_t) blob_, err)) {
+        close();
+        return false;
+    }
+#endif
     next_free_ = 0;
     fills_ = 0;
     admitted_ = 0;
@@ -154,11 +176,22 @@ bool ExpertCache::open_sized(const std::vector<int64_t>& slot_bytes, int64_t n_l
     slots_ = (int64_t) slot_bytes.size();
     blob_ = mx;
     off_ = std::move(off);
+#if defined(STRATA_USE_HIP)
+    if (!ensure_blocking_staging((std::size_t) blob_, err)) {
+        close();
+        return false;
+    }
+#endif
     layer_next_.assign((size_t) (n_layers > 0 ? n_layers : 0), 0);
     return true;
 }
 
 void ExpertCache::close() {
+#if defined(STRATA_USE_HIP)
+    if (blocking_staging_) (void) cudaFreeHost(blocking_staging_);
+    blocking_staging_ = nullptr;
+    blocking_staging_bytes_ = 0;
+#endif
     off_.clear();
     if (base_ != nullptr) {
         cudaFree(base_);
@@ -256,7 +289,18 @@ bool ExpertCache::fill_slot_blocking(int32_t slot, const uint8_t* host_blob, std
         err = "ExpertCache::fill_slot_blocking: the host blob is null";
         return false;
     }
+#if defined(STRATA_USE_HIP)
+    // Bound HIP's pageable-source staging to one expert instead of repeatedly
+    // registering regions of the mmap. The blocking copy completes before reuse.
+    if (!blocking_staging_ || n > blocking_staging_bytes_) {
+        err = "ExpertCache::fill_slot_blocking: HIP staging buffer is too small";
+        return false;
+    }
+    std::memcpy(blocking_staging_, host_blob, n);
+    const cudaError_t e = cudaMemcpy(dst, blocking_staging_, n, cudaMemcpyHostToDevice);
+#else
     const cudaError_t e = cudaMemcpy(dst, host_blob, n, cudaMemcpyHostToDevice);
+#endif
     if (e != cudaSuccess) {
         err = std::string("ExpertCache::fill_slot_blocking: ") + cudaGetErrorString(e);
         return false;

src/core/expert_source.cpp

@@ -17,11 +17,18 @@
 #include <cstdlib>
 #include <cstring>
 #include <fstream>
+#include <filesystem>
+#include <sstream>
+#include <limits>
 #include <thread>
+#include <utility>
 #include <vector>
 
 #if defined(_WIN32)
 #define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX   // std::numeric_limits<T>::max() below
+#endif
 #include <windows.h>
 #else
 #include <fcntl.h>
@@ -32,6 +39,213 @@
 
 namespace strata::core {
 
+namespace detail {
+
+bool cgroup_available_bytes(uint64_t limit, const CgroupMemoryStat& stat, uint64_t& bytes) {
+    bytes = 0;
+    if (!stat.valid) return false;
+
+    // memory.stat's inactive_file can race memory.current, so bound it to charged usage first.
+    uint64_t reclaimable = std::min(stat.inactive_file, stat.current);
+    reclaimable = stat.file_dirty >= reclaimable ? 0 : reclaimable - stat.file_dirty;
+    reclaimable = stat.file_writeback >= reclaimable ? 0 : reclaimable - stat.file_writeback;
+
+    // Reclaiming clean file pages reduces usage; saturating subtraction also handles a transient over-limit read.
+    const uint64_t usage_after_reclaim = stat.current - reclaimable;
+    bytes = usage_after_reclaim < limit ? limit - usage_after_reclaim : 0;
+    return true;
+}
+
+bool make_cache_complement_plan(
+    int64_t n_layers, int64_t n_expert, const std::vector<uint64_t>& layer_blob_bytes,
+    const std::vector<std::pair<int32_t, int32_t>>& primary_gpu_pairs,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs,
+    std::vector<uint64_t>& offsets, uint64_t& bytes, std::string& err) {
+    offsets.clear();
+    bytes = 0;
+    err.clear();
+    if (n_layers <= 0 || n_expert <= 0 || layer_blob_bytes.size() != (size_t) n_layers) {
+        err = "FileExpertSource: invalid geometry for the cache complement plan";
+        return false;
+    }
+    if ((uint64_t) n_layers > (uint64_t) std::numeric_limits<size_t>::max() / (uint64_t) n_expert) {
+        err = "FileExpertSource: cache complement index table is too large";
+        return false;
+    }
+    const size_t count = (size_t) n_layers * (size_t) n_expert;
+    std::vector<uint8_t> omitted(count, 0);
+    auto mark_pairs = [&](const std::vector<std::pair<int32_t, int32_t>>& pairs, uint8_t bit,
+                          const char* label) -> bool {
+        for (const auto& pair : pairs) {
+            if (pair.first < 0 || pair.second < 0 || pair.first >= n_layers || pair.second >= n_expert) {
+                err = std::string("FileExpertSource: ") + label + " pair is outside the expert geometry";
+                return false;
+            }
+            const size_t index = (size_t) pair.first * (size_t) n_expert + (size_t) pair.second;
+            if ((omitted[index] & bit) != 0) {
+                err = std::string("FileExpertSource: duplicate ") + label + " pair in the cache complement plan";
+                return false;
+            }
+            if (bit == 2 && (omitted[index] & 1) != 0) {
+                err = "FileExpertSource: the primary and additional GPU expert tiers overlap";
+                return false;
+            }
+            omitted[index] |= bit;
+        }
+        return true;
+    };
+    if (!mark_pairs(primary_gpu_pairs, 1, "primary GPU") ||
+        !mark_pairs(additional_gpu_pairs, 2, "additional GPU")) return false;
+    for (uint64_t blob_bytes : layer_blob_bytes) {
+        if (blob_bytes == 0) {
+            err = "FileExpertSource: cache complement layer has zero-sized expert blobs";
+            return false;
+        }
+    }
+
+    offsets.assign(count, kNoCacheComplement);
+    for (int64_t layer = 0; layer < n_layers; ++layer) {
+        const uint64_t blob_bytes = layer_blob_bytes[(size_t) layer];
+        for (int64_t expert = 0; expert < n_expert; ++expert) {
+            const size_t index = (size_t) layer * (size_t) n_expert + (size_t) expert;
+            if (omitted[index] != 0) continue;
+            if (bytes > std::numeric_limits<uint64_t>::max() - blob_bytes) {
+                offsets.clear();
+                bytes = 0;
+                err = "FileExpertSource: cache complement size overflows";
+                return false;
+            }
+            offsets[index] = bytes;
+            bytes += blob_bytes;
+        }
+    }
+    if (bytes > (uint64_t) std::numeric_limits<size_t>::max()) {
+        offsets.clear();
+        bytes = 0;
+        err = "FileExpertSource: cache complement exceeds the host address space";
+        return false;
+    }
+    return true;
+}
+
+const uint8_t* cache_complement_blob_or_fallback(
+    size_t index, const std::vector<uint64_t>& offsets, const uint8_t* complement_host,
+    const uint8_t* mapped_fallback) {
+    if (complement_host != nullptr && index < offsets.size() && offsets[index] != kNoCacheComplement)
+        return complement_host + (size_t) offsets[index];
+    return mapped_fallback;
+}
+
+}  // namespace detail
+
+namespace {
+constexpr uint64_t kPinnedMemoryHeadroom = 8ull << 30;
+
+#if defined(__linux__)
+bool read_cgroup_memory_stat(const std::filesystem::path& path, uint64_t current,
+                             detail::CgroupMemoryStat& stat) {
+    std::ifstream input(path / "memory.stat");
+    if (!input) return false;
+
+    bool inactive_file = false, file_dirty = false, file_writeback = false;
+    std::string line;
+    while (std::getline(input, line)) {
+        std::istringstream fields(line);
+        std::string key;
+        uint64_t value = 0;
+        if (!(fields >> key >> value)) return false;
+        fields >> std::ws;
+        if (!fields.eof()) return false;
+
+        if (key == "inactive_file") {
+            if (inactive_file) return false;
+            inactive_file = true;
+            stat.inactive_file = value;
+        } else if (key == "file_dirty") {
+            if (file_dirty) return false;
+            file_dirty = true;
+            stat.file_dirty = value;
+        } else if (key == "file_writeback") {
+            if (file_writeback) return false;
+            file_writeback = true;
+            stat.file_writeback = value;
+        }
+    }
+    if (!input.eof() || !inactive_file || !file_dirty || !file_writeback) return false;
+    stat.current = current;
+    stat.valid = true;
+    return true;
+}
+#endif
+
+bool available_memory_bytes(uint64_t& bytes) {
+#if defined(_WIN32)
+    MEMORYSTATUSEX status{};
+    status.dwLength = sizeof(status);
+    if (!GlobalMemoryStatusEx(&status)) return false;
+    bytes = (uint64_t) status.ullAvailPhys;
+    return bytes > 0;
+#elif defined(__linux__)
+    // MemAvailable includes reclaimable page cache, unlike _SC_AVPHYS_PAGES.
+    std::ifstream info("/proc/meminfo");
+    std::string line;
+    bytes = 0;
+    while (std::getline(info, line)) {
+        std::istringstream fields(line);
+        std::string key, unit;
+        uint64_t value = 0;
+        if (fields >> key >> value >> unit && key == "MemAvailable:" && unit == "kB" &&
+            value <= std::numeric_limits<uint64_t>::max() / 1024) bytes = value * 1024;
+    }
+    if (bytes == 0) return false;
+    // Account for the tightest cgroup-v2 ancestor limit when its normal mount is visible.
+    // This is a point-in-time guard, not a reservation against concurrent allocations.
+    std::ifstream groups("/proc/self/cgroup");
+    if (!groups) return false;
+    bool resolved_v2 = false;
+    while (std::getline(groups, line)) {
+        if (line.rfind("0::/", 0) != 0) continue;
+        const std::filesystem::path root("/sys/fs/cgroup");
+        auto path = (root / line.substr(4)).lexically_normal();
+        if (path.string().rfind(root.string(), 0) != 0 || !std::filesystem::is_directory(path)) return false;
+        resolved_v2 = true;
+        while (path.string().rfind(root.string(), 0) == 0) {
+            std::ifstream limit_file(path / "memory.max"), current_file(path / "memory.current");
+            std::string limit;
+            uint64_t current = 0;
+            const bool readable = bool(limit_file >> limit) && bool(current_file >> current);
+            // The host's root cgroup has no memory.max; ordinary child groups must expose their limits.
+            if (!readable && !(path == root && !std::filesystem::exists(path / "memory.max") &&
+                               std::filesystem::exists(path / "cgroup.controllers"))) return false;
+            if (readable && limit != "max") {
+                try {
+                    size_t consumed = 0;
+                    const uint64_t cap = std::stoull(limit, &consumed);
+                    if (consumed != limit.size()) return false;
+                    detail::CgroupMemoryStat stat;
+                    if (!read_cgroup_memory_stat(path, current, stat)) return false;
+                    uint64_t cgroup_available = 0;
+                    if (!detail::cgroup_available_bytes(cap, stat, cgroup_available)) return false;
+                    bytes = std::min(bytes, cgroup_available);
+                } catch (...) { return false; }
+            }
+            if (path == root) break;
+            path = path.parent_path();
+        }
+    }
+    return resolved_v2;
+#else
+    const long pages = sysconf(_SC_AVPHYS_PAGES);
+    const long page_bytes = sysconf(_SC_PAGESIZE);
+    if (pages <= 0 || page_bytes <= 0 ||
+        (uint64_t) pages > std::numeric_limits<uint64_t>::max() / (uint64_t) page_bytes) return false;
+    bytes = (uint64_t) pages * (uint64_t) page_bytes;
+    return bytes > 0;
+#endif
+}
+
+}  // namespace
+
 // ================================ THE FILE-BACKED SOURCE ================================
 
 FileExpertSource::~FileExpertSource() { close(); }
@@ -39,9 +253,74 @@ FileExpertSource::~FileExpertSource() { close(); }
 bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err) {
     close();
     if (n_layers <= 0 || n_expert <= 0) { err = "FileExpertSource: the geometry is empty"; return false; }
-    n_expert_ = n_expert;
-    blobs_ = n_layers * n_expert;
-    const uint64_t want = (uint64_t) blobs_ * (uint64_t) strata::kernels::cpu::BLOB;
+    const auto& layout = strata::kernels::cpu::expert_layout();
+    if (layout.n_layers != n_layers || layout.n_expert != n_expert) {
+        err = "FileExpertSource: the requested geometry does not match the loaded expert layout";
+        return false;
+    }
+    if ((uint64_t) n_layers > (uint64_t) std::numeric_limits<int64_t>::max() / (uint64_t) n_expert) {
+        err = "FileExpertSource: the expert count overflows";
+        return false;
+    }
+    const uint64_t blob_count = (uint64_t) n_layers * (uint64_t) n_expert;
+    if (blob_count > (uint64_t) std::numeric_limits<int64_t>::max() ||
+        (uint64_t) n_layers > (uint64_t) std::numeric_limits<size_t>::max()) {
+        err = "FileExpertSource: the expert count overflows";
+        return false;
+    }
+
+    std::vector<uint64_t> layer_offsets((size_t) n_layers), layer_blob_bytes((size_t) n_layers);
+    const uint64_t want = layout.total;
+    if (want == 0 || want > (uint64_t) std::numeric_limits<size_t>::max()) {
+        err = "FileExpertSource: the loaded expert layout has an invalid size";
+        return false;
+    }
+    if (!layout.native) {
+        if (blob_count > std::numeric_limits<uint64_t>::max() / (uint64_t) strata::kernels::cpu::BLOB) {
+            err = "FileExpertSource: the canonical expert size overflows";
+            return false;
+        }
+        const uint64_t canonical_size = blob_count * (uint64_t) strata::kernels::cpu::BLOB;
+        if (want != canonical_size) {
+            err = "FileExpertSource: the canonical expert layout has an inconsistent size";
+            return false;
+        }
+        const uint64_t bytes = (uint64_t) strata::kernels::cpu::BLOB;
+        const uint64_t layer_bytes = (uint64_t) n_expert * bytes;
+        for (int64_t layer = 0; layer < n_layers; ++layer) {
+            layer_offsets[(size_t) layer] = (uint64_t) layer * layer_bytes;
+            layer_blob_bytes[(size_t) layer] = bytes;
+        }
+    } else {
+        if (layout.offset.size() != (size_t) n_layers || layout.bytes.size() != (size_t) n_layers ||
+            layout.fmt.size() != (size_t) n_layers) {
+            err = "FileExpertSource: the native expert layout is incomplete";
+            return false;
+        }
+        uint64_t at = 0;
+        for (int64_t layer = 0; layer < n_layers; ++layer) {
+            const size_t i = (size_t) layer;
+            const uint64_t bytes = (uint64_t) layout.fmt[i].bytes;
+            if (layout.offset[i] != at || bytes == 0 || layout.bytes[i] != bytes ||
+                bytes > std::numeric_limits<uint64_t>::max() / (uint64_t) n_expert) {
+                err = "FileExpertSource: the native expert layout is invalid at layer " + std::to_string(layer);
+                return false;
+            }
+            const uint64_t layer_bytes = bytes * (uint64_t) n_expert;
+            if (at > want || layer_bytes > want - at) {
+                err = "FileExpertSource: the native expert layout exceeds its declared size at layer " +
+                      std::to_string(layer);
+                return false;
+            }
+            layer_offsets[i] = layout.offset[i];
+            layer_blob_bytes[i] = bytes;
+            at += layer_bytes;
+        }
+        if (at != want) {
+            err = "FileExpertSource: the native expert layout has an inconsistent size";
+            return false;
+        }
+    }
     const std::string path = pack_dir + "/experts.bin";
 
 #if defined(_WIN32)
@@ -76,10 +355,9 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     if ((uint64_t) sz.QuadPart != want) {
         char buf[400];
         std::snprintf(buf, sizeof buf,
-                      "FileExpertSource: %s is %llu B but %lld layers x %lld experts x %d B is %llu B - this "
-                      "is not the pack this geometry came from",
-                      path.c_str(), (unsigned long long) sz.QuadPart, (long long) n_layers,
-                      (long long) n_expert, (int) strata::kernels::cpu::BLOB, (unsigned long long) want);
+                      "FileExpertSource: %s is %llu B but the loaded expert layout requires %llu B - this is not "
+                      "the pack this geometry came from",
+                      path.c_str(), (unsigned long long) sz.QuadPart, (unsigned long long) want);
         CloseHandle(f);
         err = buf;
         return false;
@@ -105,13 +383,13 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     if (fd < 0) { err = "FileExpertSource: cannot open " + path; return false; }
     struct stat st{};
     if (fstat(fd, &st) != 0) { ::close(fd); err = "FileExpertSource: cannot stat " + path; return false; }
-    if ((uint64_t) st.st_size != want) {
+    if (st.st_size < 0 || (uint64_t) st.st_size != want) {
         char buf[400];
         std::snprintf(buf, sizeof buf,
-                      "FileExpertSource: %s is %llu B but %lld layers x %lld experts x %d B is %llu B - this "
-                      "is not the pack this geometry came from",
-                      path.c_str(), (unsigned long long) st.st_size, (long long) n_layers, (long long) n_expert,
-                      (int) strata::kernels::cpu::BLOB, (unsigned long long) want);
+                      "FileExpertSource: %s is %llu B but the loaded expert layout requires %llu B - this is not "
+                      "the pack this geometry came from",
+                      path.c_str(), (unsigned long long) (st.st_size < 0 ? 0 : st.st_size),
+                      (unsigned long long) want);
         ::close(fd);
         err = buf;
         return false;
@@ -121,10 +399,27 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
     fd_ = fd;
     base_ = (const uint8_t*) view;
 #endif
+    blobs_ = (int64_t) blob_count;
+    n_layers_ = n_layers;
+    n_expert_ = n_expert;
+    mapped_bytes_ = want;
+    layer_offsets_ = std::move(layer_offsets);
+    layer_blob_bytes_ = std::move(layer_blob_bytes);
     return true;
 }
 
 void FileExpertSource::close() {
+    if (complement_arena_ != nullptr) {
+        if (complement_pinned_) (void) cudaFreeHost(complement_arena_);
+        else std::free(complement_arena_);
+    }
+    complement_arena_ = nullptr;
+    complement_host_ = nullptr;
+    complement_device_ = nullptr;
+    complement_bytes_ = 0;
+    complement_offsets_.clear();
+    complement_pinned_ = false;
+    complement_ready_ = false;
 #if defined(_WIN32)
     if (base_ != nullptr) UnmapViewOfFile((LPCVOID) base_);
     if (mapping_ != nullptr) CloseHandle((HANDLE) mapping_);
@@ -132,27 +427,225 @@ void FileExpertSource::close() {
     mapping_ = nullptr;
     file_ = nullptr;
 #else
-    if (base_ != nullptr) munmap((void*) base_, (size_t) blobs_ * (size_t) strata::kernels::cpu::BLOB);
+    if (base_ != nullptr) munmap((void*) base_, (size_t) mapped_bytes_);
     if (fd_ >= 0) ::close(fd_);
     fd_ = -1;
 #endif
     base_ = nullptr;
     blobs_ = 0;
+    n_layers_ = 0;
+    n_expert_ = 0;
+    mapped_bytes_ = 0;
+    layer_offsets_.clear();
+    layer_blob_bytes_.clear();
     reads_ = 0;
 }
 
+const uint8_t* FileExpertSource::mapped_blob(int64_t layer, int64_t expert) const {
+    if (base_ == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return nullptr;
+    const size_t i = (size_t) layer;
+    if (i >= layer_offsets_.size() || i >= layer_blob_bytes_.size()) return nullptr;
+    const uint64_t blob_bytes = layer_blob_bytes_[i];
+    if (blob_bytes == 0 || (uint64_t) expert > std::numeric_limits<uint64_t>::max() / blob_bytes) return nullptr;
+    const uint64_t expert_offset = (uint64_t) expert * blob_bytes;
+    const uint64_t layer_offset = layer_offsets_[i];
+    if (layer_offset > mapped_bytes_ || expert_offset > mapped_bytes_ - layer_offset) return nullptr;
+    const uint64_t offset = layer_offset + expert_offset;
+    if (blob_bytes > mapped_bytes_ - offset) return nullptr;
+    return base_ + (size_t) offset;
+}
+
+bool FileExpertSource::pin_cache_complement(
+    const ExpertCache& cache, std::string& err, bool pin,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs) {
+    err.clear();
+    if (base_ == nullptr) { err = "FileExpertSource: open the mapped experts before pinning a complement"; return false; }
+    if (complement_ready_) { err = "FileExpertSource: the cache complement is already pinned"; return false; }
+    if (!cache.valid()) { err = "FileExpertSource: the GPU expert cache is not open"; return false; }
+    if (cache.fills() != cache.resident()) {
+        err = "FileExpertSource: the GPU expert cache is not fully filled";
+        return false;
+    }
+    const cudaError_t sync = cudaDeviceSynchronize();
+    if (sync != cudaSuccess) {
+        err = std::string("FileExpertSource: GPU expert cache is not ready: ") + cudaGetErrorString(sync);
+        (void) cudaGetLastError();
+        return false;
+    }
+
+    std::vector<std::pair<int32_t, int32_t>> primary_gpu_pairs;
+    primary_gpu_pairs.reserve((size_t) cache.resident());
+    for (int64_t layer = 0; layer < n_layers_; ++layer) {
+        for (int64_t expert = 0; expert < n_expert_; ++expert) {
+            if (cache.slot_of(layer, expert) != kNotResident)
+                primary_gpu_pairs.emplace_back((int32_t) layer, (int32_t) expert);
+        }
+    }
+    std::vector<uint64_t> offsets;
+    uint64_t bytes = 0;
+    if (!detail::make_cache_complement_plan(n_layers_, n_expert_, layer_blob_bytes_, primary_gpu_pairs,
+                                            additional_gpu_pairs, offsets, bytes, err)) return false;
+
+    if (bytes > 0) {
+        uint64_t physical = 0;
+        if (!available_memory_bytes(physical)) {
+            err = "FileExpertSource: cannot determine available RAM for the resident-memory safety check";
+            return false;
+        }
+        if (physical <= kPinnedMemoryHeadroom || bytes > physical - kPinnedMemoryHeadroom) {
+            char message[256];
+            std::snprintf(message, sizeof message,
+                          "FileExpertSource: resident complement %.2f GiB exceeds available RAM minus the 8 GiB safety headroom",
+                          (double) bytes / 1073741824.0);
+            err = message;
+            return false;
+        }
+    }
+
+    void* arena = nullptr;
+    const uint8_t* host = nullptr;
+    const uint8_t* device = nullptr;
+    if (bytes > 0) {
+        std::fprintf(stderr, "FileExpertSource: allocating %.2f GiB %s cache complement\n",
+                     (double) bytes / 1073741824.0, pin ? "mapped pinned" : "pageable resident");
+        std::fflush(stderr);
+        if (pin) {
+            const cudaError_t allocated = cudaHostAlloc(&arena, (size_t) bytes,
+                                                         cudaHostAllocMapped | cudaHostAllocPortable);
+            if (allocated != cudaSuccess) {
+                err = std::string("FileExpertSource: mapped pinned complement allocation failed: ") +
+                      cudaGetErrorString(allocated);
+                std::fprintf(stderr, "%s\n", err.c_str());
+                std::fflush(stderr);
+                (void) cudaGetLastError();
+                return false;
+            }
+            host = (const uint8_t*) arena;
+            void* alias = nullptr;
+            const cudaError_t aliased = cudaHostGetDevicePointer(&alias, arena, 0);
+            if (aliased != cudaSuccess || alias == nullptr) {
+                err = std::string("FileExpertSource: mapped pinned complement device alias failed: ") +
+                      cudaGetErrorString(aliased);
+                std::fprintf(stderr, "%s\n", err.c_str());
+                std::fflush(stderr);
+                (void) cudaGetLastError();
+                (void) cudaFreeHost(arena);
+                return false;
+            }
+            device = (const uint8_t*) alias;
+        } else {
+            arena = std::malloc((size_t) bytes);
+            if (arena == nullptr) {
+                err = "FileExpertSource: pageable resident complement allocation failed";
+                return false;
+            }
+            host = (const uint8_t*) arena;
+        }
+    }
+
+    uint64_t copied = 0;
+    for (int64_t layer = 0; layer < n_layers_; ++layer) {
+        const uint64_t blob_bytes = layer_blob_bytes_[(size_t) layer];
+        for (int64_t expert = 0; expert < n_expert_; ++expert) {
+            const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+            const uint64_t offset = offsets[index];
+            if (offset == kNoComplement) continue;
+            const uint8_t* source = mapped_blob(layer, expert);
+            if (source == nullptr || offset > bytes || blob_bytes > bytes - offset) {
+                err = "FileExpertSource: invalid blob bounds while building the cache complement";
+                if (arena != nullptr) {
+                    if (pin) (void) cudaFreeHost(arena);
+                    else std::free(arena);
+                }
+                return false;
+            }
+            std::memcpy((uint8_t*) host + (size_t) offset, source, (size_t) blob_bytes);
+            copied += blob_bytes;
+        }
+#if !defined(_WIN32)
+        const uint64_t layer_offset = layer_offsets_[(size_t) layer];
+        const uint64_t layer_bytes = blob_bytes * (uint64_t) n_expert_;
+        const uint64_t layer_end = layer_offset + layer_bytes;
+        const long page_bytes = sysconf(_SC_PAGESIZE);
+        if (page_bytes <= 0) {
+            err = "FileExpertSource: cannot determine page size for mapped-page release";
+            if (arena != nullptr) {
+                if (pin) (void) cudaFreeHost(arena);
+                else std::free(arena);
+            }
+            return false;
+        }
+        const uint64_t page = (uint64_t) page_bytes;
+        const uint64_t advice_start = layer_offset - layer_offset % page;
+        const uint64_t end_remainder = layer_end % page;
+        const uint64_t extra = end_remainder == 0 ? 0 : page - end_remainder;
+        const uint64_t advice_end = extra > mapped_bytes_ - layer_end ? mapped_bytes_ : layer_end + extra;
+        if (advice_end > advice_start &&
+            madvise((void*) (base_ + (size_t) advice_start), (size_t) (advice_end - advice_start), MADV_DONTNEED) != 0) {
+            err = "FileExpertSource: madvise could not release mapped expert layer " + std::to_string(layer);
+            if (arena != nullptr) {
+                if (pin) (void) cudaFreeHost(arena);
+                else std::free(arena);
+            }
+            return false;
+        }
+        const int advice = posix_fadvise(fd_, (off_t) layer_offset, (off_t) layer_bytes, POSIX_FADV_DONTNEED);
+        if (advice != 0) {
+            err = "FileExpertSource: posix_fadvise could not release expert layer " + std::to_string(layer);
+            if (arena != nullptr) {
+                if (pin) (void) cudaFreeHost(arena);
+                else std::free(arena);
+            }
+            return false;
+        }
+#endif
+        if ((layer + 1) % 8 == 0 || layer + 1 == n_layers_) {
+            std::fprintf(stderr, "FileExpertSource: copied cache complement through layer %lld/%lld (%.2f GiB)\n",
+                         (long long) (layer + 1), (long long) n_layers_, (double) copied / 1073741824.0);
+            std::fflush(stderr);
+        }
+    }
+
+    complement_arena_ = arena;
+    complement_host_ = host;
+    complement_device_ = device;
+    complement_bytes_ = bytes;
+    complement_offsets_ = std::move(offsets);
+    complement_pinned_ = pin && bytes > 0;
+    complement_ready_ = true;
+    std::fprintf(stderr, "FileExpertSource: %s cache complement ready: resident %.2f GiB, pinned %.2f GiB\n",
+                 complement_pinned_ ? "mapped pinned" : "pageable resident",
+                 (double) resident_bytes() / 1073741824.0, (double) pinned_bytes() / 1073741824.0);
+    if (!additional_gpu_pairs.empty()) {
+        std::fprintf(stderr, "FileExpertSource: %zu verified additional-GPU experts remain on the mmap fallback\n",
+                     additional_gpu_pairs.size());
+    }
+    std::fflush(stderr);
+    return true;
+}
+
 const uint8_t* FileExpertSource::blob(int64_t layer, int64_t expert) {
-    if (base_ == nullptr) return nullptr;
-    if (layer < 0 || expert < 0) return nullptr;
-    // **`expert >= n_expert_` IS CHECKED SEPARATELY, AND THE FLAT INDEX ALONE DOES NOT CATCH IT.**  The blob
-    // index is `layer * n_expert + expert`, so `blob(0, 512)` has flat index 512 - which is in range, and is
-    // `blob(1, 0)`.  A router id one past the end of a layer would then read the NEXT LAYER's first expert:
-    // finite, correctly sized, and wrong.  Layer and expert are separate axes and are validated as such.
-    if (expert >= n_expert_) return nullptr;
-    const int64_t i = layer * n_expert_ + expert;
-    if (i >= blobs_) return nullptr;
-    ++reads_;
-    return base_ + (size_t) i * strata::kernels::cpu::BLOB;
+    if (base_ == nullptr || layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return nullptr;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    const uint8_t* mapped_fallback = mapped_blob(layer, expert);
+    const uint8_t* result = complement_ready_
+        ? detail::cache_complement_blob_or_fallback(index, complement_offsets_, complement_host_, mapped_fallback)
+        : mapped_fallback;
+    if (result != nullptr) ++reads_;
+    return result;
+}
+
+bool FileExpertSource::pinned(int64_t layer, int64_t expert) const {
+    if (!complement_ready_ || !complement_pinned_ || complement_host_ == nullptr || layer < 0 || expert < 0 ||
+        layer >= n_layers_ || expert >= n_expert_) return false;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    return index < complement_offsets_.size() && complement_offsets_[index] != kNoComplement;
+}
+
+const uint8_t* FileExpertSource::device_alias(int64_t layer, int64_t expert) const {
+    if (!pinned(layer, expert) || complement_device_ == nullptr) return nullptr;
+    const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+    return complement_device_ + (size_t) complement_offsets_[index];
 }
 
 // ================================ THE ADAPTER ================================

src/core/layer.cpp

@@ -172,11 +172,13 @@ if (!w.wants_q8k()) {        s_gemv_q8_0_split(x80, p.codes, p.scales, p.offset,
 // namespace
 void layer_set_native_bf16(bool enabled) { native_bf16_projections = enabled; }
 void layer_set_native_flash_attn_short(bool enabled) { native_flash_attn_short = enabled; }
-namespace { bool g_kv_int8 = false, g_kv_q4 = false; }
+namespace { bool g_kv_int8 = false, g_kv_q4 = false, g_kv_hybrid = false; }
 void qsa_set_kv_int8(bool enabled) { g_kv_int8 = enabled; }
 bool qsa_kv_int8() { return g_kv_int8; }
 void qsa_set_kv_q4(bool enabled) { g_kv_q4 = enabled; }
 bool qsa_kv_q4() { return g_kv_q4; }
+void qsa_set_kv_hybrid(bool enabled) { g_kv_hybrid = enabled; }
+bool qsa_kv_hybrid() { return g_kv_hybrid; }
 uint64_t gdn_buffers_bytes(const ModelGeometry& g) {    const int64_t C = g.ssm_conv_channels;    const int64_t V = g.ssm_value_dim;    const uint64_t parts[] = {        q8k_bytes(g.n_embd),
 // x_q8k
 (uint64_t) (g.n_embd / 32) * 34,
@@ -507,10 +509,15 @@ KvPlan kv_plan(const QsaShapes& s, int64_t max_cells, int64_t ring_cells) {
     }
     return p;
 }
-uint64_t kv_pool_bytes(const QsaShapes& s, int64_t pages) {
+uint64_t kv_pool_bytes(const QsaShapes& s, int64_t pages, bool hybrid, bool int8) {
+    if (hybrid) {   // K8V4: the INT8 K half (codes + scales) plus the Q4_0 V half (kv_q4.hpp's rotation)
+        const uint64_t rows = (uint64_t) pages * s.page_size * s.n_head_kv;
+        return rows * (uint64_t) s.head_dim + rows * (uint64_t) (s.head_dim / strata::kernels::KV_Q8_GROUP) * 2 +
+               rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim) + 64;
+    }
     if (g_kv_q4) return (uint64_t) pages * s.page_size * strata::kernels::kv_q4_bytes_per_cell(s) + 64;
-    return g_kv_int8 ? (uint64_t) pages * s.page_size * strata::kernels::kv_q8_bytes_per_cell(s) + 64
-                     : (uint64_t) pages * s.n_head_kv * s.page_size * s.head_dim * 2 * 2;
+    return int8 ? (uint64_t) pages * s.page_size * strata::kernels::kv_q8_bytes_per_cell(s) + 64
+                : (uint64_t) pages * s.n_head_kv * s.page_size * s.head_dim * 2 * 2;
 }
 }  // namespace
 
@@ -523,7 +530,8 @@ uint64_t qsa_state_bytes(const ModelGeometry& g, int64_t max_cells, bool with_ro
     const QsaShapes s = qsa_shapes(g);
     const KvPlan p = kv_plan(s, max_cells, ring_cells);
     uint64_t n = 0;
-    n += kv_pool_bytes(s, p.slots) + 4 * 16;                                   // K/V pools (the VRAM slots)
+    n += kv_pool_bytes(s, p.slots, g_kv_hybrid && ring_cells <= 0,
+                       g_kv_int8 || g_kv_hybrid) + 4 * 16;   // K/V pools (the VRAM slots)
     n += (uint64_t) p.pages * 4;                                               // page_table
     if (p.mode == 1) n += strata::kernels::kv_stream_map_bytes(p.slots) + 6 * 16;   // the residency map
     n += (uint64_t) (s.idx_block - 1) * s.idx_dim * 4;                         // tail
@@ -544,14 +552,29 @@ uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, Q
     Cursor c{(uint8_t*) base};
     st.kv_int8 = g_kv_int8 && !g_kv_q4;
     st.kv_q4 = g_kv_q4;
+    // Hybrid K8V4, main layers only (the drafter's state is created with the globals toggled to INT8 -
+    // mtp.cpp). Streamed mode is refused outright; generate.cpp validates it too, this is the backstop.
+    if (g_kv_hybrid && ring_cells <= 0) {
+        if (p.mode == 1) {
+            std::fprintf(stderr, "strata: hybrid K8V4 KV does not support --kv-resident streaming\n");
+            return 0;
+        }
+        st.kv_hybrid = true;
+        st.kv_int8 = false;
+        st.kv_q4 = false;
+    }
     st.kv_mode = p.mode;
     st.n_slots = p.slots;
     const uint64_t rows = (uint64_t) p.slots * s.n_head_kv * s.page_size;   // VRAM rows: the slots
     const uint64_t q4_row = strata::kernels::kv_q4_bytes_per_head((int) s.head_dim);
-    if (st.kv_q4) {
+    if (st.kv_hybrid) {
+        st.k_q = c.take<int8_t>(rows * s.head_dim);
+        st.k_scale = c.take<uint16_t>(rows * (s.head_dim / strata::kernels::KV_Q8_GROUP));
+        st.v_q4 = c.take<uint8_t>(rows * q4_row);
+    } else if (st.kv_q4) {
         st.k_q4 = c.take<uint8_t>(rows * q4_row);
         st.v_q4 = c.take<uint8_t>(rows * q4_row);
-    } else if (g_kv_int8) {
+    } else if (st.kv_int8) {   // st, not the global: the drafter's ring is INT8 under --kv k8v4 too
         st.k_q = c.take<int8_t>(rows * s.head_dim);
         st.v_q = c.take<int8_t>(rows * s.head_dim);
         st.k_scale = c.take<uint16_t>(rows * (s.head_dim / strata::kernels::KV_Q8_GROUP));
@@ -619,7 +642,7 @@ uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, Q
         if (st.kv_q4) {
             st.host.k_q4 = hc.take<uint8_t>(hrows * q4_row);
             st.host.v_q4 = hc.take<uint8_t>(hrows * q4_row);
-        } else if (g_kv_int8) {
+        } else if (st.kv_int8) {
             st.host.k_q = hc.take<int8_t>(hrows * s.head_dim);
             st.host.v_q = hc.take<int8_t>(hrows * s.head_dim);
             st.host.k_scale = hc.take<uint16_t>(hrows * (s.head_dim / strata::kernels::KV_Q8_GROUP));
@@ -658,7 +681,11 @@ void qsa_state_zero(const QsaState& st, const ModelGeometry& g, void* stream) {
     const QsaShapes s = qsa_shapes(g);
     cudaStream_t cs = (cudaStream_t) stream;
     const size_t rows = (size_t) st.n_slots * s.n_head_kv * s.page_size;
-    if (st.kv_q4) {
+    if (st.kv_hybrid) {
+        cudaMemsetAsync(st.k_q, 0, rows * s.head_dim, cs);
+        cudaMemsetAsync(st.k_scale, 0, rows * (s.head_dim / strata::kernels::KV_Q8_GROUP) * 2, cs);
+        cudaMemsetAsync(st.v_q4, 0, rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim), cs);
+    } else if (st.kv_q4) {
         cudaMemsetAsync(st.k_q4, 0, rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim), cs);
         cudaMemsetAsync(st.v_q4, 0, rows * strata::kernels::kv_q4_bytes_per_head((int) s.head_dim), cs);
     } else if (st.kv_int8) {
@@ -682,7 +709,8 @@ void qsa_state_zero(const QsaState& st, const ModelGeometry& g, void* stream) {
 strata::kernels::QsaAttnPools qsa_attn_pools(const QsaState& st) {
     strata::kernels::QsaAttnPools pools;
     pools.page_table = st.page_table;
-    if (st.kv_q4) { pools.k_q4 = st.k_q4; pools.v_q4 = st.v_q4; }
+    if (st.kv_hybrid) { pools.k_q = st.k_q; pools.k_scale = st.k_scale; pools.v_q4 = st.v_q4; }
+    else if (st.kv_q4) { pools.k_q4 = st.k_q4; pools.v_q4 = st.v_q4; }
     else if (st.kv_int8) { pools.k_q = st.k_q; pools.v_q = st.v_q; pools.k_scale = st.k_scale; pools.v_scale = st.v_scale; }
     else { pools.k_pool = st.k_pool; pools.v_pool = st.v_pool; }
     return pools;
@@ -863,11 +891,18 @@ if (!gemv_quantized(*w_attnk, p_k, f_k, b.x_q8_0, b.x_q8k, b.kcur, g.n_embd, g.n
 // EVERY ENTRY POINT FROM HERE ON IS THE CAPTURABLE ONE: the per-token counts come from `st.step` and every
 // launch is sized from a capacity in `st`/`b`, so this sequence can be captured and replayed.  The
 // host-scalar wrappers would be correct here today and silently wrong in a graph.
-if (st.kv_q4) {
+if (st.kv_hybrid) {
+    // K8V4: only V is rotated (kv_q4.hpp's H); the scores pair unrotated q with unrotated INT8 K, and the
+    // output - a mix of rotated values - is rotated back after attention. Each append/gather call folds the
+    // unused half's lanes onto the used pool (a bit-identical duplicate write), so no kernel variants exist.
+    strata::kernels::fwht256_inplace_cuda(b.vcur, g.n_head_kv, stream);
+    kv_append_q8_step(st.k_q, st.k_q, st.k_scale, st.k_scale, st.page_table, st.step, b.kcur, b.kcur, s, stream, nullptr);   // mode 0: no host mirror
+    strata::kernels::kv_append_q4_step(st.v_q4, st.v_q4, st.page_table, st.step, b.vcur, b.vcur, s, stream, nullptr);
+} else if (st.kv_q4) {
     strata::kernels::fwht256_inplace_cuda(b.kcur, g.n_head_kv, stream);   // Q4_0: rotated K and V (kv_q4.hpp)
     strata::kernels::fwht256_inplace_cuda(b.vcur, g.n_head_kv, stream);
     strata::kernels::kv_append_q4_step(st.k_q4, st.v_q4, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);
-} else if (st.kv_int8) kv_append_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    else kv_append_step(st.k_pool, st.v_pool, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    {        const uint64_t nvk = (uint64_t) g.n_head_kv * g.head_dim;        const uint64_t base = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim + 2 * nvk + 8;        if (!st.kv_int8) dump_slot(dump, g, layer, (const float*) st.k_pool, base, nvk / 2, stream);        if (!st.kv_int8) dump_slot(dump, g, layer, (const float*) st.v_pool, base + nvk / 2, nvk / 2, stream);        dump_slot(dump, g, layer, b.vcur, base + nvk, nvk, stream);        dump_slot(dump, g, layer, b.kcur, base + 2 * nvk, nvk, stream);    }    {        const QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};        if (native_qsa_indexer_enabled()) {
+} else if (st.kv_int8) kv_append_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    else kv_append_step(st.k_pool, st.v_pool, st.page_table, st.step, b.kcur, b.vcur, s, stream, &st.host);    {        const uint64_t nvk = (uint64_t) g.n_head_kv * g.head_dim;        const uint64_t base = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim + 2 * nvk + 8;        if (!st.kv_int8 && !st.kv_hybrid) dump_slot(dump, g, layer, (const float*) st.k_pool, base, nvk / 2, stream);        if (!st.kv_int8 && !st.kv_hybrid) dump_slot(dump, g, layer, (const float*) st.v_pool, base + nvk / 2, nvk / 2, stream);        dump_slot(dump, g, layer, b.vcur, base + nvk, nvk, stream);        dump_slot(dump, g, layer, b.kcur, base + 2 * nvk, nvk, stream);    }    {        const QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};        if (native_qsa_indexer_enabled()) {
     try {
         native_qsa_indexer_append(b.idx_raw, st.step + kStepPos, pos_base,
             (const float*) w_ikn->data, RMS_EPS, ib, s, st.max_cells, (float) qsa_freq_base(), stream);
@@ -906,7 +941,13 @@ int64_t max_blocks = (st.max_cells / s.idx_block) + 2;
         const strata::kernels::QsaAttnPools pools = qsa_attn_pools(st);
         strata::kernels::qsa_decode_attn_step(b.qcur, pools, b.ids, st.step, cap, s, b.attn_scratch, b.attn, stream);
     } else {
-    if (st.kv_q4) strata::kernels::kv_gather_q4_step(st.k_q4, st.v_q4, st.page_table, b.ids, st.step, cap, s, b.k_scratch, b.v_scratch, stream);
+    if (st.kv_hybrid) {
+        kv_gather_q8_step(st.k_q, st.k_q, st.k_scale, st.k_scale, st.page_table, b.ids, st.step, cap, s,
+                          b.k_scratch, b.k_scratch, stream);
+        strata::kernels::kv_gather_q4_step(st.v_q4, st.v_q4, st.page_table, b.ids, st.step, cap, s,
+                                           b.v_scratch, b.v_scratch, stream);
+    }
+    else if (st.kv_q4) strata::kernels::kv_gather_q4_step(st.k_q4, st.v_q4, st.page_table, b.ids, st.step, cap, s, b.k_scratch, b.v_scratch, stream);
     else if (st.kv_int8) kv_gather_q8_step(st.k_q, st.v_q, st.k_scale, st.v_scale, st.page_table, b.ids, st.step, cap, s,                                 b.k_scratch, b.v_scratch, stream);    else kv_gather_step(st.k_pool, st.v_pool, st.page_table, b.ids, st.step, cap, s, b.k_scratch, b.v_scratch,                   stream);    if (native_flash_attn_short) {
     if (st.max_cells < 1 || st.max_cells > 256 || !st.attention_status || !st.host_step) {
         err = v.name("native_flash_attn") + ": short adapter requires context <=256 and persistent status storage";
@@ -922,7 +963,7 @@ int64_t max_blocks = (st.max_cells / s.idx_block) + 2;
     }
 } else qsa_attend_step(b.qcur, b.k_scratch, b.v_scratch, st.step, cap, s, b.attn, nullptr, stream);
     }
-    if (st.kv_q4) strata::kernels::fwht256_inplace_cuda(b.attn, g.n_head, stream);   // the output back: H is self-inverse
+    if (st.kv_q4 || st.kv_hybrid) strata::kernels::fwht256_inplace_cuda(b.attn, g.n_head, stream);   // the output back: H is self-inverse
     dump_slot(dump, g, layer, b.attn, (uint64_t) 2 * g.n_embd + 2 * g.hc,                            (uint64_t) g.n_head * g.head_dim, stream);    {        const uint64_t vs = (uint64_t) 2 * g.n_embd + 2 * g.hc + (uint64_t) g.n_head * g.head_dim +                            (uint64_t) 2 * g.n_head_kv * g.head_dim;        dump_slot(dump, g, layer, (const float*) b.v_scratch, vs,                            (uint64_t) g.n_head_kv * g.head_dim / 2, stream);        dump_slot(dump, g, layer, (const float*) b.ids, vs + (uint64_t) g.n_head_kv * g.head_dim / 2, 4, stream);        dump_slot(dump, g, layer, (const float*) st.step,                            vs + (uint64_t) g.n_head_kv * g.head_dim / 2 + 4, 4, stream);    }
 // ---- 9. Gate the attention output before its projection. Native CUDA keeps F32
 // sigmoid/multiply arithmetic and uses Q8_1 for the native quantized projection.

src/core/mtp.cpp

@@ -201,6 +201,12 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
     // window (plus the cells a round writes ahead of its queries) over a host copy, refilled on a resume. The host copy
     // is pinned after the expert arena has pinned what it could: if it does not fit, the K/V stays whole in VRAM.
     int64_t ring = (window > 0 && window < max_cells) ? window + 4 * (int64_t) max_t + 64 : 0;
+    // K8V4 never applies to the drafter: its own attention paths (below, and verify.cpp) handle whole formats
+    // only, whatever ring shape it takes (0, a window, or the -1 fully-resident fallback).
+    const bool kv_hybrid_was = qsa_kv_hybrid();
+    const bool kv_int8_was = qsa_kv_int8();
+    qsa_set_kv_hybrid(false);
+    if (kv_hybrid_was) qsa_set_kv_int8(true);   // the drafter under --kv k8v4: plain INT8
     uint64_t sb = qsa_state_bytes(g, max_cells, false, ring);
     if (cudaMalloc(&state_arena_, sb) != cudaSuccess) { err = "mtp: the K/V state does not fit"; return false; }
     if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[0], ring) == 0) {
@@ -214,6 +220,8 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
         if (cudaMalloc(&state_arena_, sb) != cudaSuccess) { err = "mtp: the K/V state does not fit"; return false; }
         if (qsa_state_init(g, max_cells, state_arena_, st_, &ss.qsa_states[0], ring) == 0) { err = "mtp: state init failed"; return false; }
     }
+    qsa_set_kv_int8(kv_int8_was);
+    qsa_set_kv_hybrid(kv_hybrid_was);
     qsa_state_zero(st_, g, nullptr);
     cudaDeviceSynchronize();
     vram_ += sb;

src/core/native_dense.cpp

@@ -16,9 +16,9 @@ namespace {
 bool eligible(const strata::TensorInfo& tensor, bool include_ple_key) {
     const auto& name = tensor.name;
     if (name.rfind("blk.", 0) != 0) return false;
-    // The native PLE kernel accepts Q2_0. Other quantized keys can be converted to BF16 by iq_pack;
-    // keep their packed bytes resident instead of overriding them with an unsupported native key.
-    if (name == "blk.1.ple_key.weight") return include_ple_key && tensor.type == 42;   // 42 = Q2_0
+    // Match the native PLE kernel: Q2_0, IQ3_XXS and IQ4_XS. Other keys retain the packed BF16 fallback.
+    if (name == "blk.1.ple_key.weight")
+        return include_ple_key && (tensor.type == 42 || tensor.type == 18 || tensor.type == 23);
     static const char* suffixes[] = {".attn_qkv.weight", ".attn_gate.weight", ".ssm_out.weight",
         ".attn_q.weight", ".attn_k.weight", ".attn_v.weight", ".attn_output.weight",
         ".ffn_gate_shexp.weight", ".ffn_up_shexp.weight", ".ffn_down_shexp.weight"};

src/core/remote_experts.cpp

@@ -73,9 +73,12 @@ bool RemoteExperts::preflight(int device, double& free_gib, std::string& err) {
     // The layer waits for this GPU on the CPU pool's critical path: spin instead of sleeping, whose wake-up
     // costs more than a small expert batch takes (measured: ~0.3 ms per round trip on Windows).  Only possible
     // before the device's context exists, so first thing; STRATA_REMOTE_SPIN=0 keeps the driver's default.
+#if !defined(STRATA_USE_HIP)
     const char* spin = std::getenv("STRATA_REMOTE_SPIN");
     if (!(spin && spin[0] == '0')) cudaInitDevice(device, cudaDeviceScheduleSpin | cudaDeviceMapHost, 0);
     cudaGetLastError();
+#endif
+    // HIP has no cudaInitDevice equivalent; retain its default scheduling policy.
     DeviceScope scope(device);
     if (!scope.ok) { err = scope.error(device); return false; }
     size_t free_bytes = 0, total_bytes = 0;

src/core/verify.cpp

@@ -510,12 +510,19 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 if (st.kv_q4) {   // Q4_0 KV (kv_q4.hpp): K and V rotated before they are stored
                     fwht256_inplace_cuda(kcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
                     fwht256_inplace_cuda(vcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
+                } else if (st.kv_hybrid) {   // K8V4: only V is rotated
+                    fwht256_inplace_cuda(vcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
                 }
                 stamp(l, 8, grp);
                 if (grp == 0) copy_from_mapped(tail_snap_ + (size_t) qi * TS, st.idx_tail, TS, cs);
                 for (int t = tb; t < te; ++t) {
                     const int32_t* step_t = step_ + t * kStepCount;
-                    if (st.kv_q4)
+                    if (st.kv_hybrid) {   // K8V4: the unused half's lanes folded onto the used pool (layer.cpp)
+                        kv_append_q8_step(st.k_q, st.k_q, st.k_scale, st.k_scale, st.page_table, step_t,
+                                          kcur_ + t * NKV * HD, kcur_ + t * NKV * HD, s, cs, nullptr);
+                        kv_append_q4_step(st.v_q4, st.v_q4, st.page_table, step_t, vcur_ + t * NKV * HD,
+                                          vcur_ + t * NKV * HD, s, cs, nullptr);
+                    } else if (st.kv_q4)
                         kv_append_q4_step(st.k_q4, st.v_q4, st.page_table, step_t, kcur_ + t * NKV * HD,
                                           vcur_ + t * NKV * HD, s, cs, &st.host);
                     else if (st.kv_int8)
@@ -573,7 +580,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 qsa_decode_attn_batch(qcur_ + tb * NH * HD, pools, sel_ + (size_t) tb * cap_, step_ + tb * kStepCount, cap_,
                                       s, attn_scratch_ + (size_t) tb * attn_scratch_floats_, attn_ + tb * NH * HD, n, cs);
                 stamp(l, 13, grp);
-                if (st.kv_q4) fwht256_inplace_cuda(attn_ + tb * NH * HD, (int64_t) n * NH, cs);   // back: H^-1 = H
+                if (st.kv_q4 || st.kv_hybrid) fwht256_inplace_cuda(attn_ + tb * NH * HD, (int64_t) n * NH, cs);   // back: H^-1 = H
                 if (qb) native_qsa_gate_apply(attn_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, attn32_ + tb * NH * HD,
                                               (int) (n * NH), (int) HD, cs);
                 else
@@ -829,6 +836,7 @@ bool Verifier::capture(int T, std::string& err) {
         err = std::string("verify: end capture: ") + cudaGetErrorString(ce);
         return false;
     }
+#if !defined(STRATA_USE_HIP)   // a CUDA debug listing (node types, kernel names)
     if (std::getenv("STRATA_VERIFY_NODES") != nullptr) {   // what the window graph holds
         size_t nn = 0;
         cudaGraphGetNodes(graph, nullptr, &nn);
@@ -856,6 +864,7 @@ bool Verifier::capture(int T, std::string& err) {
         for (size_t i = 0; i < v.size() && i < 40; ++i) std::fprintf(stderr, " %d x %.60s;", v[i].first, v[i].second.c_str());
         std::fprintf(stderr, "\n");
     }
+#endif
     const cudaError_t ie = cudaGraphInstantiate(&exec_[T], graph, 0);
     cudaGraphDestroy(graph);
     if (ie != cudaSuccess) {

src/kernels/cuda/fused_gr.cu

@@ -179,7 +179,11 @@ __global__ void __launch_bounds__(THREADS) gr_norm_multi_kernel(GrMulti m) {
     for (int i = t; i < D; i += THREADS) xn[i] *= s_rs[i / N];
 }
 
+#if defined(__HIPCC__)
+constexpr int TILE = 1280;             // eight-token tile fits gfx1100's 64 KiB LDS limit
+#else
 constexpr int TILE = 2560;             // xn floats per token staged at a time: 320 chunks of 8, 10 per lane
+#endif
 constexpr int TQ = TILE / 8 / 32;      // uint4 weight chunks per lane per tile
 
 // Step 2 of `gr_down_kernel` for T tokens.  One warp per row (so each lane accumulates the same chunks in the

src/kernels/cuda/native_qsa_score.cu

@@ -32,6 +32,7 @@ constexpr int D=128, HEADS=4, R=4, ROWS=32, WARPS=2, STRIDE=36, COMBINE=68;
 struct TileA { uint32_t x[4]; };
 struct TileB { uint32_t x[2]; };
 struct TileC { float x[4]={0.0f,0.0f,0.0f,0.0f}; };
+#if !defined(__HIPCC__)
 #if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
 __device__ __forceinline__ void load_a(TileA& a,const float* p) {
     const float* src=p+(threadIdx.x%16)*STRIDE+(threadIdx.x/16)*4;
@@ -139,6 +140,42 @@ __global__ __launch_bounds__(64,1) void score_kernel(
     }
 #endif
 }
+#else
+// gfx1100 has no CUDA ldmatrix/mma instruction sequence. Keep the same entry point and
+// score contract with an ordered scalar F32 dot for each indexer head. The four heads
+// run independently; their ReLU'd scores are then added in the documented head order.
+// This path favors a well-defined fallback over pretending the CUDA PTX is portable.
+__global__ void scalar_score_kernel(
+        const float* __restrict__ pooled,const float* __restrict__ query,
+        const float* __restrict__ bias,const int32_t* __restrict__ step,
+        int max_cells,float* __restrict__ cells) {
+    const int n=step[kStepNKv],full=step[kStepNBid];
+    if(n<1||n>max_cells||step[kStepPos]!=n-1||full!=n/R||
+       step[kStepWidth]!=(n<2051?n:2051))return;
+    const int row=blockIdx.x;
+    if(row>full)return;
+    __shared__ float head_score[HEADS];
+    const int head=threadIdx.x;
+    if(head<HEADS){
+        float dot=0.0f;
+#pragma unroll
+        for(int d=0;d<D;++d)
+            dot=__fmaf_rn(pooled[size_t(row)*D+d],query[size_t(head)*D+d],dot);
+        head_score[head]=dot>0.0f?dot:0.0f;
+    }
+    __syncthreads();
+    if(head==0){
+        float sum=__fadd_rn(0.0f,head_score[0]);
+        sum=__fadd_rn(sum,head_score[1]);
+        sum=__fadd_rn(sum,head_score[2]);
+        sum=__fadd_rn(sum,head_score[3]);
+        if(bias)sum=__fadd_rn(sum,bias[row]);
+        sum=__fadd_rn(sum,row==full&&n%R?1e9f:0.0f);
+        sum=__fadd_rn(sum,0.0f);
+        for(int i=row*R;i<n&&i<(row+1)*R;++i)cells[i]=sum;
+    }
+}
+#endif
 struct Span{const void* p;size_t n;};
 void validate(Span s){
     const auto p=reinterpret_cast<uintptr_t>(s.p);
@@ -163,8 +200,13 @@ void native_qsa_score(const float* pooled,const float* query,const float* bias,
     for(int i=0;i<count;++i)validate(spans[i]);
     for(int i=0;i<count;++i)for(int j=i+1;j<count;++j)
         if(overlaps(spans[i],spans[j]))throw std::invalid_argument("native QSA score spans overlap");
+#if defined(__HIPCC__)
+    scalar_score_kernel<<<unsigned(max_blocks),128,0,static_cast<cudaStream_t>(stream)>>>(
+        pooled,query,bias,step,int(max_cells),cells);
+#else
     score_kernel<<<unsigned((max_blocks+ROWS-1)/ROWS),dim3(32,WARPS),0,static_cast<cudaStream_t>(stream)>>>(
         pooled,query,bias,step,int(max_cells),cells);
+#endif
     const auto error=cudaGetLastError();
     if(error!=cudaSuccess)throw std::runtime_error(cudaGetErrorString(error));
 }

src/kernels/cuda/ple.cu

@@ -228,8 +228,9 @@ void ple_block(const float* emb, const float* hidden, const float* hist_rows, co
                PleOut& out, void* scratch, void* stream) {
     const bool native_key = w.key_native_data != nullptr && w.key_bf16 == nullptr;
     if (native_key && (!emb || !hidden || !hist_rows || !out.result || !scratch || !stream ||
-                       !w.key_native_q8_1 || w.key_native_type != 42))
-        throw std::invalid_argument("ple_block: native key requires Q2_0 weights, input/output, private scratch and explicit stream");
+                       !w.key_native_q8_1 ||
+                       (w.key_native_type != 42 && w.key_native_type != 18 && w.key_native_type != 23)))
+        throw std::invalid_argument("ple_block: native key requires Q2_0, IQ3_XXS or IQ4_XS weights, input/output, private scratch and explicit stream");
     if (emb == nullptr || hidden == nullptr || hist_rows == nullptr || out.result == nullptr) return;
     const int n_embd = NG_N_EMBD, hc = NG_HC, hc_dim = NG_HC_DIM;
     static_assert(NG_N_EMBD == 2560 && NG_HC_DIM == 10240, "native PLE key geometry changed");

src/kernels/cuda/qsa_decode_attn.cu

@@ -31,42 +31,53 @@ __device__ __forceinline__ float warp_max(float v) {
 }
 
 // 8 consecutive values of one cell's key or value row for KV head `kvh`, dimensions [d0, d0+8).
+// Per-format bodies; `load8` below is the KV_MODE dispatcher. value=false is the K side, true the V side.
+__device__ __forceinline__ void load8_f16(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
+    const uint16_t* base = (value ? p.v_pool : p.k_pool) + row * HD + d0;
+    const uint4 raw = *reinterpret_cast<const uint4*>(base);
+    const __half2* h2 = reinterpret_cast<const __half2*>(&raw);
+#pragma unroll
+    for (int j = 0; j < 4; ++j) {
+        const float2 f = __half22float2(h2[j]);
+        out[2 * j] = f.x;
+        out[2 * j + 1] = f.y;
+    }
+}
+__device__ __forceinline__ void load8_q8(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
+    const int8_t* codes = (value ? p.v_q : p.k_q) + row * HD + d0;
+    const uint16_t sbits = (value ? p.v_scale : p.k_scale)[row * (HD / KV_Q8_GROUP) + d0 / KV_Q8_GROUP];
+    const float sc = __half2float(__ushort_as_half(sbits));
+    const uint2 raw = *reinterpret_cast<const uint2*>(codes);
+    const int8_t* c = reinterpret_cast<const int8_t*>(&raw);
+#pragma unroll
+    for (int j = 0; j < 8; ++j) out[j] = (float) c[j] * sc;
+}
+__device__ __forceinline__ void load8_q4(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
+    constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
+    const int b = d0 / QK4_0;
+    const int rem = d0 % QK4_0;
+    const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((value ? p.v_q4 : p.k_q4) + row * bytes_per_head) + b;
+    const float d = __half2float(__ushort_as_half(blk->d));
+    const int j = (rem == 0 || rem == 16) ? 0 : 8;
+    const uint8_t* bytes = blk->qs + j;
+    if (rem < 16) {
+#pragma unroll
+        for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] & 0x0F) - 8) * d;
+    } else {
+#pragma unroll
+        for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] >> 4) - 8) * d;
+    }
+}
 template <int KV_MODE>
 __device__ __forceinline__ void load8(const QsaAttnPools& p, bool value, long long row, int d0, float* out) {
-    if constexpr (KV_MODE == 0) {
-        const uint16_t* base = (value ? p.v_pool : p.k_pool) + row * HD + d0;
-        const uint4 raw = *reinterpret_cast<const uint4*>(base);
-        const __half2* h2 = reinterpret_cast<const __half2*>(&raw);
-#pragma unroll
-        for (int j = 0; j < 4; ++j) {
-            const float2 f = __half22float2(h2[j]);
-            out[2 * j] = f.x;
-            out[2 * j + 1] = f.y;
-        }
-    } else if constexpr (KV_MODE == 1) {
-        const int8_t* codes = (value ? p.v_q : p.k_q) + row * HD + d0;
-        const uint16_t sbits = (value ? p.v_scale : p.k_scale)[row * (HD / KV_Q8_GROUP) + d0 / KV_Q8_GROUP];
-        const float sc = __half2float(__ushort_as_half(sbits));
-        const uint2 raw = *reinterpret_cast<const uint2*>(codes);
-        const int8_t* c = reinterpret_cast<const int8_t*>(&raw);
-#pragma unroll
-        for (int j = 0; j < 8; ++j) out[j] = (float) c[j] * sc;
-    } else {
-        constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
-        const int b = d0 / QK4_0;
-        const int rem = d0 % QK4_0;
-        const block_q4_0* blk = reinterpret_cast<const block_q4_0*>((value ? p.v_q4 : p.k_q4) + row * bytes_per_head) + b;
-        const float d = __half2float(__ushort_as_half(blk->d));
-        const int j = (rem == 0 || rem == 16) ? 0 : 8;
-        const uint8_t* bytes = blk->qs + j;
-        if (rem < 16) {
-#pragma unroll
-            for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] & 0x0F) - 8) * d;
-        } else {
-#pragma unroll
-            for (int k = 0; k < 8; ++k) out[k] = (float) ((int)(bytes[k] >> 4) - 8) * d;
-        }
-    }
+    if constexpr (KV_MODE == 0) load8_f16(p, value, row, d0, out);
+    else if constexpr (KV_MODE == 1) load8_q8(p, value, row, d0, out);
+    else if constexpr (KV_MODE == 3) {
+        // hybrid K8V4: both sides are defined - K unrotated INT8, V rotated Q4_0 - so a value=true call
+        // reads the Q4_0 pool instead of dereferencing the null v_q (no call site does today; PR review)
+        if (value) load8_q4(p, value, row, d0, out);
+        else load8_q8(p, value, row, d0, out);
+    } else load8_q4(p, value, row, d0, out);
 }
 
 template <int KV_MODE>
@@ -149,7 +160,7 @@ __global__ void __launch_bounds__(THREADS) attn_chunk_kernel(const float* __rest
         } else if constexpr (KV_MODE == 1) {
             const float sc = __half2float(__ushort_as_half(p.v_scale[srow[c] * (HD / KV_Q8_GROUP) + t / KV_Q8_GROUP]));
             v = (float) p.v_q[srow[c] * HD + t] * sc;
-        } else {
+        } else {   // modes 2 and 3: V is rotated Q4_0 (kv_q4.hpp); the caller rotates the output back
             constexpr int bytes_per_head = (HD / QK4_0) * sizeof(block_q4_0);
             const int b = t / QK4_0;
             const int rem = t % QK4_0;
@@ -202,7 +213,8 @@ void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int3
         std::fprintf(stderr, "qsa_decode_attn_batch: unsupported geometry or missing buffers\n");
         std::exit(1);
     }
-    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr ? 1 : 0);
+    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
+                        : (pools.k_q != nullptr ? 1 : 0));
     const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
     // per query: [acc: n_chunks*n_head*HD][m: n_chunks*n_head][l: n_chunks*n_head], all offsets from one stride
     const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);
@@ -212,7 +224,10 @@ void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int3
     const float scale = 1.0f / sqrtf((float) HD);
     const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);
     cudaStream_t st = (cudaStream_t) stream;
-    if (kv_mode == 2)
+    if (kv_mode == 3)
+        attn_chunk_kernel<3><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
+                                                        scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
+    else if (kv_mode == 2)
         attn_chunk_kernel<2><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
                                                         scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
     else if (kv_mode == 1)
@@ -242,8 +257,11 @@ void qsa_decode_attn_step(const float* q, const QsaAttnPools& pools, const int32
         std::fprintf(stderr, "qsa_decode_attn: unsupported geometry or missing buffers\n");
         std::exit(1);
     }
-    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr ? 1 : 0);
-    if (kv_mode == 2 ? (!pools.v_q4) : (kv_mode == 1 ? (!pools.v_q || !pools.k_scale || !pools.v_scale) : (!pools.k_pool || !pools.v_pool))) {
+    const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
+                        : (pools.k_q != nullptr ? 1 : 0));
+    if (kv_mode == 3 ? (!pools.k_scale || !pools.v_q4)
+                     : (kv_mode == 2 ? (!pools.v_q4) : (kv_mode == 1 ? (!pools.v_q || !pools.k_scale || !pools.v_scale)
+                                                                     : (!pools.k_pool || !pools.v_pool)))) {
         std::fprintf(stderr, "qsa_decode_attn: incomplete KV pools\n");
         std::exit(1);
     }
@@ -254,7 +272,10 @@ void qsa_decode_attn_step(const float* q, const QsaAttnPools& pools, const int32
     const float scale = 1.0f / sqrtf((float) HD);
     const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv);
     cudaStream_t st = (cudaStream_t) stream;
-    if (kv_mode == 2)
+    if (kv_mode == 3)
+        attn_chunk_kernel<3><<<grid, THREADS, 0, st>>>(q, pools, ids, step, (int) s.n_head_kv, (int) s.page_size,
+                                                        scale, part_acc, part_m, part_l, n_chunks);
+    else if (kv_mode == 2)
         attn_chunk_kernel<2><<<grid, THREADS, 0, st>>>(q, pools, ids, step, (int) s.n_head_kv, (int) s.page_size,
                                                         scale, part_acc, part_m, part_l, n_chunks);
     else if (kv_mode == 1)

src/kernels/cuda/qsa_prompt_attn.cu

@@ -1,6 +1,7 @@
 // src/kernels/cuda/qsa_prompt_attn.cu - see include/strata/kernels/qsa_prompt_attn.hpp.
 #include "strata/kernels/qsa_prompt_attn.hpp"
 #include "strata/kernels/kv_q8.hpp"
+#include "strata/kernels/kv_q4.hpp"
 
 #include <cuda_fp16.h>
 #include <cuda_runtime.h>
@@ -24,7 +25,9 @@ constexpr int QS = HD + 8;        // q row stride in halves (bank-conflict-free
 
 // The MMA and cp.async below need sm_80. Builds for older cards (the experimental sm_75 one) compile them to a trap;
 // qsa_prompt_attn_batch refuses such a device at run time, so the old kernel runs there.
-#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
+#if defined(__HIPCC__)          // AMD: no mma.sync / cp.async; the host keeps the old kernel (below)
+#define STRATA_PA_SM80 0
+#elif !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
 #define STRATA_PA_SM80 1
 #else
 #define STRATA_PA_SM80 0
@@ -56,14 +59,18 @@ __device__ __forceinline__ uint32_t pack_h2(float lo_k, float hi_k) {   // eleme
 }
 
 // KV_MODE 1: int8 codes + fp16 scale per 64 values. KV_MODE 0: fp16 values (scales 1).
+// KV_MODE 3 (hybrid K8V4): K as mode 1, V as mode 0 - the row's q4_0 blocks are dequantized to fp16 at
+// gather, so everything downstream of the load is the mode-0 V path; the caller un-rotates the output.
 template <int KV_MODE>
 struct Smem {
-    using Elem = typename std::conditional<KV_MODE == 1, int8_t, __half>::type;
-    static constexpr int ROW = KV_MODE == 1 ? HD + 16 : HD + 8;   // elements; 16-byte aligned rows, banks spread
+    using KElem = typename std::conditional<KV_MODE == 0, __half, int8_t>::type;
+    using VElem = typename std::conditional<KV_MODE == 1, int8_t, __half>::type;
+    static constexpr int KROW = KV_MODE == 0 ? HD + 8 : HD + 16;   // elements; 16-byte aligned rows, banks spread
+    static constexpr int VROW = KV_MODE == 1 ? HD + 16 : HD + 8;
     __half qh[16][QS];
     __half ql[16][QS];
-    Elem k[CH][ROW];
-    Elem v[CH][ROW];
+    KElem k[CH][KROW];
+    VElem v[CH][VROW];
     float ks[CH][4];
     float vs[CH][4];
     float s[16][CH + 1];
@@ -128,24 +135,55 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
             S.row[t] = r;
         }
         __syncthreads();   // rows ready; the previous chunk's p.v is done with k, v, s
-        // gather the chunk's K and V rows (16-byte pieces) and their scales
+        // gather the chunk's K and V rows (16-byte pieces; K8V4's V as q4_0 blocks dequantized to fp16)
+        // and their scales
         {
-            constexpr int PIECES = HD * (int) sizeof(typename Smem<KV_MODE>::Elem) / 16;   // per row
-            for (int i = t; i < CH * PIECES; i += THREADS) {
-                const int c = i / PIECES, pc = i % PIECES;
+            constexpr int KPIECES = HD * (int) sizeof(typename Smem<KV_MODE>::KElem) / 16;   // per K row
+            for (int i = t; i < CH * KPIECES; i += THREADS) {
+                const int c = i / KPIECES, pc = i % KPIECES;
                 const long long r = S.row[c];
-                uint4 kx = make_uint4(0, 0, 0, 0), vx = kx;
+                uint4 kx = make_uint4(0, 0, 0, 0);
                 if (r >= 0) {
-                    if constexpr (KV_MODE == 1) {
-                        kx = __ldg(reinterpret_cast<const uint4*>(p.k_q + r * HD) + pc);
-                        vx = __ldg(reinterpret_cast<const uint4*>(p.v_q + r * HD) + pc);
-                    } else {
+                    if constexpr (KV_MODE == 0)
                         kx = __ldg(reinterpret_cast<const uint4*>(p.k_pool + r * HD) + pc);
-                        vx = __ldg(reinterpret_cast<const uint4*>(p.v_pool + r * HD) + pc);
-                    }
+                    else   // modes 1 and 3: the K side is INT8
+                        kx = __ldg(reinterpret_cast<const uint4*>(p.k_q + r * HD) + pc);
                 }
                 *reinterpret_cast<uint4*>(reinterpret_cast<unsigned char*>(&S.k[c][0]) + pc * 16) = kx;
-                *reinterpret_cast<uint4*>(reinterpret_cast<unsigned char*>(&S.v[c][0]) + pc * 16) = vx;
+            }
+            if constexpr (KV_MODE == 3) {   // V: dequantize the row's q4_0 blocks straight into the fp16 V row
+                constexpr int BLKS = HD / QK4_0;
+                constexpr int BYTES = BLKS * (int) sizeof(block_q4_0);
+                for (int i = t; i < CH * BLKS; i += THREADS) {
+                    const int c = i / BLKS, b = i % BLKS;
+                    const long long r = S.row[c];
+#pragma unroll
+                    for (int j = 0; j < QK4_0; ++j) S.v[c][b * QK4_0 + j] = __half(0);
+                    if (r >= 0) {
+                        const block_q4_0* blk = reinterpret_cast<const block_q4_0*>(p.v_q4 + r * BYTES) + b;
+                        const float d = __half2float(__ushort_as_half(blk->d));
+#pragma unroll
+                        for (int j = 0; j < QK4_0 / 2; ++j) {
+                            S.v[c][b * QK4_0 + j] = __float2half_rn((float) ((int)(blk->qs[j] & 0x0F) - 8) * d);
+                            S.v[c][b * QK4_0 + j + QK4_0 / 2] =
+                                __float2half_rn((float) ((int)(blk->qs[j] >> 4) - 8) * d);
+                        }
+                    }
+                }
+            } else {
+                constexpr int VPIECES = HD * (int) sizeof(typename Smem<KV_MODE>::VElem) / 16;   // per V row
+                for (int i = t; i < CH * VPIECES; i += THREADS) {
+                    const int c = i / VPIECES, pc = i % VPIECES;
+                    const long long r = S.row[c];
+                    uint4 vx = make_uint4(0, 0, 0, 0);
+                    if (r >= 0) {
+                        if constexpr (KV_MODE == 1)
+                            vx = __ldg(reinterpret_cast<const uint4*>(p.v_q + r * HD) + pc);
+                        else
+                            vx = __ldg(reinterpret_cast<const uint4*>(p.v_pool + r * HD) + pc);
+                    }
+                    *reinterpret_cast<uint4*>(reinterpret_cast<unsigned char*>(&S.v[c][0]) + pc * 16) = vx;
+                }
             }
             for (int i = t; i < CH * 4; i += THREADS) {
                 const int c = i / 4, g = i % 4;
@@ -155,6 +193,9 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
                     if constexpr (KV_MODE == 1) {
                         a = __half2float(__ushort_as_half(p.k_scale[r * (HD / KV_Q8_GROUP) + g]));
                         b = __half2float(__ushort_as_half(p.v_scale[r * (HD / KV_Q8_GROUP) + g]));
+                    } else if constexpr (KV_MODE == 3) {   // K as int8, V dequantized to fp16 (scale 1)
+                        a = __half2float(__ushort_as_half(p.k_scale[r * (HD / KV_Q8_GROUP) + g]));
+                        b = 1.0f;
                     } else {
                         a = b = 1.0f;
                     }
@@ -184,7 +225,7 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_kernel(const float* __res
                     al[1] = *reinterpret_cast<const uint32_t*>(&S.ql[gid + 8][k0 + 2 * tig]);
                     al[2] = *reinterpret_cast<const uint32_t*>(&S.ql[gid][k0 + 2 * tig + 8]);
                     al[3] = *reinterpret_cast<const uint32_t*>(&S.ql[gid + 8][k0 + 2 * tig + 8]);
-                    if constexpr (KV_MODE == 1) {
+                    if constexpr (KV_MODE != 0) {   // modes 1 and 3: the K side is INT8 codes
                         b[0] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(&S.k[cb + gid][k0 + 2 * tig]));
                         b[1] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(&S.k[cb + gid][k0 + 2 * tig + 8]));
                     } else {
@@ -650,10 +691,17 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3
         }
         if (cc_major[dev] < 8) return false;
     }
+#if defined(__HIPCC__)
+    return false;   // the tensor-core kernel is compiled out on AMD (its major version is not a CUDA sm)
+#endif
     if (pools.k_q4 != nullptr || s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !ids ||
         !steps || !pools.page_table)
         return false;
     cudaStream_t st = (cudaStream_t) stream;
+    if (pools.k_q != nullptr && pools.v_q4 != nullptr) {   // hybrid K8V4: int8 K + dequantized-q4 V
+        if (!pools.k_scale) return false;
+        return launch<3>(q, pools, ids, steps, cap, s, attn, n_q, st);
+    }
     if (pools.k_q != nullptr) {
         if (!pools.v_q || !pools.k_scale || !pools.v_scale) return false;
         // STRATA_PROMPT_ATTN_V1=1 (debug): the first version, same accuracy, another summation order - the control

src/kernels/cuda/qsa_select.cu

@@ -148,6 +148,287 @@ __global__ void __launch_bounds__(TOPK_T) block_topk_kernel(const float* __restr
     }
 }
 
+
+// ---- QSA select on tensor cores (perf-review, after D-1): the block scores of many queries are one GEMM,
+// rows (query, indexer head) x columns (blocks), K = 128, with relu per head summed. 3xTF32 (each operand split
+// into a TF32 hi and lo part, hi*hi + hi*lo + lo*hi) keeps FP32-level accuracy; the summation order differs from
+// the warp kernel, so a score can move in its last bits and a near-tie can select differently (not bitwise).
+// Blocks < n_bid only; the tail block n_bid (the `dead` key, +1e9) is scored by the warp kernel's own code.
+constexpr int TC_QT = 16;                 // queries per CTA (one m16 tile per indexer head)
+constexpr int TC_NB = 32;                 // blocks per tile (4 warps x n8)
+constexpr int TC_ITER = 4;                // tiles per CTA (the query tile is loaded once)
+constexpr int TC_QS = IDX_HEADS * IDX_DIM + 4;   // query row stride in floats
+constexpr int TC_KS = IDX_DIM + 4;               // key row stride
+
+// TF32 conversion and MMA need sm_80: below it they compile to a trap and qsa_block_scores_tc refuses the device
+#if defined(__HIPCC__)          // AMD: no mma.sync / cp.async; the host keeps the warp kernel (below)
+#define STRATA_SEL_SM80 0
+#elif !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
+#define STRATA_SEL_SM80 1
+#else
+#define STRATA_SEL_SM80 0
+#endif
+__device__ __forceinline__ uint32_t tf32_hi(float x) {
+#if STRATA_SEL_SM80
+    uint32_t r;
+    asm("cvt.rna.tf32.f32 %0, %1;" : "=r"(r) : "f"(x));
+    return r;
+#else
+    return __float_as_uint(x);
+#endif
+}
+__device__ __forceinline__ void mma_tf32(float* c, const uint32_t* a, const uint32_t* b) {
+#if !STRATA_SEL_SM80
+    __trap();
+#else
+    asm volatile("mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
+                 "{%0,%1,%2,%3};\n"
+                 : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
+                 : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
+#endif
+}
+
+__global__ void __launch_bounds__(128) block_scores_tc_kernel(const float* __restrict__ pooled,
+                                                              const float* __restrict__ q_idx,
+                                                              const int32_t* __restrict__ steps, int64_t nq,
+                                                              int64_t max_blocks, int64_t reach,
+                                                              float* __restrict__ out) {
+    extern __shared__ __align__(16) float sm[];
+    float* sQ = sm;                                   // [TC_QT][TC_QS]
+    float* sK = sm + TC_QT * TC_QS;                   // [TC_NB][TC_KS]
+    __shared__ int s_nbid[TC_QT];
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5, gid = lane >> 2, tig = lane & 3;
+    const int64_t q0 = (int64_t) blockIdx.y * TC_QT;
+    for (int i = t; i < TC_QT * IDX_HEADS * IDX_DIM / 4; i += 128) {
+        const int r = i / (IDX_HEADS * IDX_DIM / 4), c = i % (IDX_HEADS * IDX_DIM / 4);
+        float4 v = make_float4(0.f, 0.f, 0.f, 0.f);
+        if (q0 + r < nq) v = reinterpret_cast<const float4*>(q_idx + (q0 + r) * IDX_HEADS * IDX_DIM)[c];
+        *reinterpret_cast<float4*>(sQ + r * TC_QS + c * 4) = v;
+    }
+    if (t < TC_QT) s_nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : 0;
+    int lo_nbid = 0x7fffffff, hi_nbid = 0;
+    __syncthreads();
+    for (int i = 0; i < TC_QT; ++i) {
+        if (q0 + i >= nq) break;
+        lo_nbid = min(lo_nbid, s_nbid[i]);
+        hi_nbid = max(hi_nbid, s_nbid[i]);
+    }
+    for (int it = 0; it < TC_ITER; ++it) {
+        const int64_t b0 = ((int64_t) blockIdx.x * TC_ITER + it) * TC_NB;
+        if (b0 >= reach || b0 >= hi_nbid) break;      // blocks >= every query's n_bid: nothing to score
+        __syncthreads();                              // the previous tile's reads are done
+        for (int i = t; i < TC_NB * IDX_DIM / 4; i += 128) {
+            const int r = i / (IDX_DIM / 4), c = i % (IDX_DIM / 4);
+            float4 v = make_float4(0.f, 0.f, 0.f, 0.f);
+            if (b0 + r < hi_nbid) v = reinterpret_cast<const float4*>(pooled + (b0 + r) * IDX_DIM)[c];
+            *reinterpret_cast<float4*>(sK + r * TC_KS + c * 4) = v;
+        }
+        __syncthreads();
+        float acc[IDX_HEADS][4];
+#pragma unroll
+        for (int h = 0; h < IDX_HEADS; ++h) acc[h][0] = acc[h][1] = acc[h][2] = acc[h][3] = 0.f;
+        const float* kr = sK + (warp * 8 + gid) * TC_KS;
+#pragma unroll 4
+        for (int k0 = 0; k0 < IDX_DIM; k0 += 8) {
+            const float kx0 = kr[k0 + tig], kx1 = kr[k0 + tig + 4];
+            uint32_t bh[2], bl[2];
+            bh[0] = tf32_hi(kx0);
+            bh[1] = tf32_hi(kx1);
+            bl[0] = tf32_hi(kx0 - __uint_as_float(bh[0]));
+            bl[1] = tf32_hi(kx1 - __uint_as_float(bh[1]));
+#pragma unroll
+            for (int h = 0; h < IDX_HEADS; ++h) {
+                const float* qa = sQ + h * IDX_DIM + k0 + tig;
+                const float x0 = qa[gid * TC_QS], x1 = qa[(gid + 8) * TC_QS];
+                const float x2 = qa[gid * TC_QS + 4], x3 = qa[(gid + 8) * TC_QS + 4];
+                uint32_t ah[4], al[4];
+                ah[0] = tf32_hi(x0); ah[1] = tf32_hi(x1); ah[2] = tf32_hi(x2); ah[3] = tf32_hi(x3);
+                al[0] = tf32_hi(x0 - __uint_as_float(ah[0]));
+                al[1] = tf32_hi(x1 - __uint_as_float(ah[1]));
+                al[2] = tf32_hi(x2 - __uint_as_float(ah[2]));
+                al[3] = tf32_hi(x3 - __uint_as_float(ah[3]));
+                mma_tf32(acc[h], al, bh);
+                mma_tf32(acc[h], ah, bl);
+                mma_tf32(acc[h], ah, bh);
+            }
+        }
+        // relu per head, heads added in order (as the warp kernel), written where the block completed for the query
+        const int64_t bc = b0 + warp * 8 + 2 * tig;
+#pragma unroll
+        for (int half = 0; half < 2; ++half) {
+            const int qr = gid + half * 8;
+            const int64_t qi = q0 + qr;
+            if (qi >= nq) continue;
+            const int nb = s_nbid[qr];
+#pragma unroll
+            for (int j = 0; j < 2; ++j) {
+                const int64_t b = bc + j;
+                if (b >= nb || b >= max_blocks) continue;
+                float score = 0.0f;
+#pragma unroll
+                for (int h = 0; h < IDX_HEADS; ++h) {
+                    const float d = acc[h][half * 2 + j];
+                    score += d > 0.0f ? d : 0.0f;
+                }
+                out[qi * max_blocks + b] = score;
+            }
+        }
+    }
+}
+
+// the tail block n_bid of each query: exactly block_scores_kernel's arithmetic for that block
+__global__ void __launch_bounds__(32) block_scores_tail_kernel(const float* __restrict__ dead,
+                                                               const float* __restrict__ q_idx,
+                                                               const int32_t* __restrict__ steps,
+                                                               int64_t max_blocks, float* __restrict__ out) {
+    const int64_t qi = blockIdx.x;
+    const int32_t* st = steps + qi * kStepCount;
+    const int64_t n_kv = st[kStepNKv], n_bid = st[kStepNBid];
+    if (n_bid >= max_blocks) return;
+    const int lane = threadIdx.x & 31;
+    const float4 k4 = *reinterpret_cast<const float4*>(dead + lane * 4);
+    const float* q = q_idx + qi * IDX_HEADS * IDX_DIM + lane * 4;
+    float score = 0.0f;
+#pragma unroll
+    for (int h = 0; h < IDX_HEADS; ++h) {
+        const float4 q4 = *reinterpret_cast<const float4*>(q + h * IDX_DIM);
+        float d = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+#pragma unroll
+        for (int o = 16; o > 0; o >>= 1) d += __shfl_xor_sync(0xffffffffu, d, o);
+        score += d > 0.0f ? d : 0.0f;
+    }
+    if (lane == 0) {
+        if (n_kv % R != 0) score += 1e9f;
+        out[qi * max_blocks + n_bid] = score;
+    }
+}
+
+// ---- the same top-k with each query's keys read once: 1,024 threads hold up to TK_PER consecutive blocks' keys in
+// registers (contexts up to 4 * 1024 * TK_PER cells), per-warp histograms, block-wide scans. The selection rule is
+// block_topk_kernel's (radix threshold, ties to the lowest index, cells ascending): identical ids.
+constexpr int TK_T = 1024;
+constexpr int TK_PER = 33;
+
+__device__ __forceinline__ int block_excl_scan(int v, int* s_warp, int& total) {
+    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+    int x = v;
+#pragma unroll
+    for (int o = 1; o < 32; o <<= 1) {
+        const int y = __shfl_up_sync(0xffffffffu, x, o);
+        if (lane >= o) x += y;
+    }
+    if (lane == 31) s_warp[warp] = x;
+    __syncthreads();
+    if (warp == 0) {
+        int w = s_warp[lane];
+        int z = w;
+#pragma unroll
+        for (int o = 1; o < 32; o <<= 1) {
+            const int y = __shfl_up_sync(0xffffffffu, z, o);
+            if (lane >= o) z += y;
+        }
+        s_warp[lane] = z - w;               // exclusive per warp
+        if (lane == 31) s_warp[32] = z;     // total
+    }
+    __syncthreads();
+    const int r = s_warp[warp] + x - v;
+    total = s_warp[32];
+    __syncthreads();
+    return r;
+}
+
+__global__ void __launch_bounds__(TK_T) block_topk_reg_kernel(const float* __restrict__ scores,
+                                                              const int32_t* __restrict__ steps, int64_t max_blocks,
+                                                              int64_t cap, int32_t* __restrict__ ids) {
+    __shared__ int hist[TK_T / 32][256];
+    __shared__ int s_warp[33];
+    __shared__ int s_digit, s_above;
+    const int64_t qi = blockIdx.x;
+    const int32_t* st = steps + qi * kStepCount;
+    const int64_t n_kv = st[kStepNKv], n_bid = st[kStepNBid], width = st[kStepWidth];
+    int32_t* out = ids + qi * cap;
+    const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+    if (n_kv <= width) {
+        for (int64_t j = t; j < n_kv; j += TK_T) out[j] = (int32_t) j;
+        return;
+    }
+    const float* sc = scores + qi * max_blocks;
+    const int64_t nb = n_bid + 1;
+    const int64_t per = (nb + TK_T - 1) / TK_T;       // <= TK_PER (the caller checks)
+    const int64_t b0 = (int64_t) t * per, b1 = (b0 + per < nb) ? b0 + per : nb;
+    uint32_t key[TK_PER];
+#pragma unroll
+    for (int j = 0; j < TK_PER; ++j) key[j] = (b0 + j < b1) ? order_key(sc[b0 + j]) : 0u;
+    auto weight = [&](int64_t b) -> int { return b < n_bid ? R : (int) (n_kv - n_bid * R); };
+    uint32_t prefix = 0;
+    int above = 0;
+    for (int shift = 24; shift >= 0; shift -= 8) {
+        for (int i = lane; i < 256; i += 32) hist[warp][i] = 0;
+        __syncwarp();
+        const uint32_t hi_mask = shift == 24 ? 0u : (0xffffffffu << (shift + 8));
+#pragma unroll
+        for (int j = 0; j < TK_PER; ++j) {
+            const int64_t b = b0 + j;
+            if (b >= b1) break;
+            const int w = weight(b);
+            if (w == 0) continue;
+            if ((key[j] & hi_mask) == (prefix & hi_mask)) atomicAdd(&hist[warp][(key[j] >> shift) & 255], w);
+        }
+        __syncthreads();
+        if (t < 256) {                                // fold the warps' histograms into warp 0's
+            int s = 0;
+            for (int w2 = 0; w2 < TK_T / 32; ++w2) s += hist[w2][t];
+            hist[0][t] = s;
+        }
+        __syncthreads();
+        if (t == 0) {
+            int cum = above, d = 255;
+            for (; d > 0; --d) {
+                if (cum + hist[0][d] >= width) break;
+                cum += hist[0][d];
+            }
+            s_digit = d;
+            s_above = cum;
+        }
+        __syncthreads();
+        prefix |= (uint32_t) s_digit << shift;
+        above = s_above;
+        __syncthreads();
+    }
+    const uint32_t thr = prefix;
+    const int64_t eq_budget = width - above;
+    int gt = 0, eq = 0;
+#pragma unroll
+    for (int j = 0; j < TK_PER; ++j) {
+        const int64_t b = b0 + j;
+        if (b >= b1) break;
+        const int w = weight(b);
+        if (w == 0) continue;
+        if (key[j] > thr) gt += w;
+        else if (key[j] == thr) eq += w;
+    }
+    int tot;
+    const int eq_before = block_excl_scan(eq, s_warp, tot);
+    int64_t my_eq = eq_budget - eq_before;
+    if (my_eq < 0) my_eq = 0;
+    if (my_eq > eq) my_eq = eq;
+    const int sel = gt + (int) my_eq;
+    int64_t wpos = block_excl_scan(sel, s_warp, tot);
+    int64_t eq_left = my_eq;
+#pragma unroll
+    for (int j = 0; j < TK_PER; ++j) {
+        const int64_t b = b0 + j;
+        if (b >= b1) break;
+        const int w = weight(b);
+        if (w == 0) continue;
+        if (key[j] > thr) {
+            for (int c = 0; c < w; ++c) out[wpos++] = (int32_t) (b * R + c);
+        } else if (key[j] == thr) {
+            for (int c = 0; c < w && eq_left > 0; ++c, --eq_left) out[wpos++] = (int32_t) (b * R + c);
+        }
+    }
+}
+
 }  // namespace
 
 void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
@@ -166,8 +447,52 @@ void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx
     if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores: %s\n", cudaGetErrorString(e)); std::exit(1); }
 }
 
-void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
-                    const QsaShapes& s, int32_t* ids, void* stream) {
+bool qsa_block_scores_tc(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+                         int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {
+    if (nq <= 0) return true;
+    if (s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R || nq > 65535 * TC_QT) return false;
+    {   // sm_80 or newer (TF32 MMA); an older card keeps the warp kernel
+        static int cc_major[64] = {};
+        int dev = 0;
+        if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+        if (cc_major[dev] == 0) {
+            int major = 0;
+            if (cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess) {
+                cudaGetLastError();
+                return false;
+            }
+            cc_major[dev] = major;
+        }
+        if (cc_major[dev] < 8) return false;
+    }
+#if defined(__HIPCC__)
+    return false;   // the tensor-core kernel is compiled out on AMD (its major version is not a CUDA sm)
+#endif
+    static bool attr[64] = {};   // the shared-memory opt-in is per device (a layer split runs it on several)
+    int adev = 0;
+    cudaGetDevice(&adev);
+    const int bytes = (TC_QT * TC_QS + TC_NB * TC_KS) * (int) sizeof(float);
+    if (!attr[adev]) {
+        if (cudaFuncSetAttribute(block_scores_tc_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes) !=
+            cudaSuccess) {
+            cudaGetLastError();
+            return false;
+        }
+        attr[adev] = true;
+    }
+    const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks;
+    const int64_t per = (int64_t) TC_NB * TC_ITER;
+    const dim3 grid((unsigned) ((reach + per - 1) / per), (unsigned) ((nq + TC_QT - 1) / TC_QT));
+    block_scores_tc_kernel<<<grid, 128, bytes, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq, max_blocks, reach,
+                                                                         scores);
+    block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks, scores);
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores_tc: %s\n", cudaGetErrorString(e)); std::exit(1); }
+    return true;
+}
+
+void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                        const QsaShapes& s, int32_t* ids, void* stream) {
     if (nq <= 0) return;
     if (s.idx_block != R || cap < qsa_selection_width(kTopkMaxCells, s)) {
         std::fprintf(stderr, "qsa_block_topk: unsupported geometry or cap\n");
@@ -178,4 +503,23 @@ void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64
     if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
 }
 
+void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+                    const QsaShapes& s, int32_t* ids, void* stream) {
+    // keys in registers when every query's blocks fit (contexts up to ~135K cells); the same ids. STRATA_TOPK_OLD=1:
+    // the kernel that reads them from memory on every pass
+    static const bool old = std::getenv("STRATA_TOPK_OLD") != nullptr;
+    if (nq <= 0) return;
+    if (old || max_blocks > (int64_t) TK_T * TK_PER) {
+        qsa_block_topk_ref(scores, steps, nq, max_blocks, cap, s, ids, stream);
+        return;
+    }
+    if (s.idx_block != R || cap < qsa_selection_width(kTopkMaxCells, s)) {
+        std::fprintf(stderr, "qsa_block_topk: unsupported geometry or cap\n");
+        std::exit(1);
+    }
+    block_topk_reg_kernel<<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
+}
+
 }  // namespace strata::kernels

src/kernels/cuda/verify_kernels.cu

@@ -544,7 +544,11 @@ void copy_indexed(float* dst, const float* src, int64_t stride, const int32_t* i
 // a GPU timestamp (ns, %globaltimer) into buf[i] - the verify window's stage profiler
 namespace { __global__ void gpu_stamp_kernel(unsigned long long* buf, int i) {
     unsigned long long t;
+#if defined(__HIPCC__)
+    t = wall_clock64() * 10ull;   // gfx11: a constant 100 MHz counter, in ns
+#else
     asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
+#endif
     buf[i] = t;
 } }
 void gpu_stamp(unsigned long long* buf, int i, void* stream) {

src/kernels/gr_parity.cpp

@@ -20,6 +20,7 @@
 //   6. `gr_write`'s `2*sigmoid`, which centres the gate on 1 so a ZERO injection is a plain residual add.
 //      Asserted as a property, not as a value, because that is what the source comment claims.
 #include "strata/kernels/gr.hpp"
+#include "strata/kernels/fused_gr.hpp"
 
 #include <cuda_runtime.h>
 
@@ -268,6 +269,143 @@ int scalar_activation_contract() {
     return bad;
 }
 
+int fused_multi_lds_parity(const float* d_norm, const uint16_t* d_down, const uint16_t* d_up,
+                           const uint16_t* d_inject, float eps) {
+    using namespace strata::kernels;
+    constexpr int N = 2560, HC = 4, LR = 320, D = N * HC, T = kFusedGrMaxT;
+    std::mt19937 rng(0x6f8a);
+    std::normal_distribution<float> normal(0.0f, 0.3f);
+    std::vector<float> r((size_t) T * D), bo((size_t) T * N), inj((size_t) T * HC);
+    for (auto& x : r) x = normal(rng);
+    for (auto& x : bo) x = normal(rng);
+    for (auto& x : inj) x = normal(rng);
+
+    float *d_r = nullptr, *d_r_out = nullptr, *d_bo = nullptr, *d_inj = nullptr;
+    float *d_lo = nullptr, *d_rs = nullptr, *d_inj_out = nullptr, *d_mixed = nullptr, *d_xn = nullptr;
+    check(cudaMalloc(&d_r, r.size() * sizeof(float)), "multi R");
+    check(cudaMalloc(&d_r_out, r.size() * sizeof(float)), "multi R_out");
+    check(cudaMalloc(&d_bo, bo.size() * sizeof(float)), "multi bo");
+    check(cudaMalloc(&d_inj, inj.size() * sizeof(float)), "multi inj");
+    check(cudaMalloc(&d_lo, (size_t) T * LR * sizeof(float)), "multi lo");
+    check(cudaMalloc(&d_rs, (size_t) T * HC * sizeof(float)), "multi rs");
+    check(cudaMalloc(&d_inj_out, (size_t) T * HC * sizeof(float)), "multi injection");
+    check(cudaMalloc(&d_mixed, (size_t) T * N * sizeof(float)), "multi mixed");
+    check(cudaMalloc(&d_xn, (size_t) T * D * sizeof(float)), "multi xn");
+    check(cudaMemcpy(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy R");
+    check(cudaMemcpy(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy bo");
+    check(cudaMemcpy(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice), "multi copy inj");
+
+    std::vector<FusedGrArgs> args(T);
+    for (int t = 0; t < T; ++t) {
+        auto& a = args[t];
+        a.R = d_r + (size_t) t * D;
+        a.R_out = d_r_out + (size_t) t * D;
+        a.apply = true;
+        a.bo_prev = d_bo + (size_t) t * N;
+        a.inj_prev = d_inj + (size_t) t * HC;
+        a.w_norm = d_norm;
+        a.w_down = d_down;
+        a.w_up = d_up;
+        a.w_inject = d_inject;
+        a.eps = eps;
+        a.lo = d_lo + (size_t) t * LR;
+        a.rs = d_rs + (size_t) t * HC;
+        a.inject_out = d_inj_out + (size_t) t * HC;
+        a.mixed = d_mixed + (size_t) t * N;
+    }
+
+    struct Snapshot {
+        std::vector<float> r_out, lo, rs, inject, mixed;
+    };
+    auto snapshot = [&]() {
+        Snapshot s;
+        s.r_out.resize((size_t) T * D);
+        s.lo.resize((size_t) T * LR);
+        s.rs.resize((size_t) T * HC);
+        s.inject.resize((size_t) T * HC);
+        s.mixed.resize((size_t) T * N);
+        check(cudaMemcpy(s.r_out.data(), d_r_out, s.r_out.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read R_out");
+        check(cudaMemcpy(s.lo.data(), d_lo, s.lo.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read lo");
+        check(cudaMemcpy(s.rs.data(), d_rs, s.rs.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read rs");
+        check(cudaMemcpy(s.inject.data(), d_inj_out, s.inject.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read inject");
+        check(cudaMemcpy(s.mixed.data(), d_mixed, s.mixed.size() * sizeof(float), cudaMemcpyDeviceToHost), "multi read mixed");
+        return s;
+    };
+    auto same = [](const Snapshot& a, const Snapshot& b) {
+        return std::memcmp(a.r_out.data(), b.r_out.data(), a.r_out.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.lo.data(), b.lo.data(), a.lo.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.rs.data(), b.rs.data(), a.rs.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.inject.data(), b.inject.data(), a.inject.size() * sizeof(float)) == 0 &&
+               std::memcmp(a.mixed.data(), b.mixed.data(), a.mixed.size() * sizeof(float)) == 0;
+    };
+
+    cudaStream_t stream = nullptr;
+    check(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), "multi stream");
+    // Max T forces the HIP kernel's full dynamic-LDS request: 8 * 1280 * sizeof(float) = 40 KiB.
+    fused_gr_read_multi(args.data(), T, d_xn, stream);
+    check(cudaStreamSynchronize(stream), "multi max-T sync");
+    const Snapshot multi = snapshot();
+    for (int t = 0; t < T; ++t) fused_gr_read(args[t], stream);
+    check(cudaStreamSynchronize(stream), "single reference sync");
+    const Snapshot single = snapshot();
+    int bad = 0;
+    if (!same(multi, single)) {
+        std::printf("  fused GR multi max-T differs from single-token calls\n");
+        ++bad;
+    }
+
+    cudaGraph_t graph = nullptr;
+    cudaGraphExec_t graph_exec = nullptr;
+    check(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal), "multi graph begin");
+    fused_gr_read_multi(args.data(), T, d_xn, stream);
+    check(cudaStreamEndCapture(stream, &graph), "multi graph end");
+    check(cudaGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0), "multi graph instantiate");
+    check(cudaGraphLaunch(graph_exec, stream), "multi graph initial replay");
+    check(cudaStreamSynchronize(stream), "multi graph initial sync");
+    const Snapshot captured = snapshot();
+    if (!same(multi, captured)) {
+        std::printf("  fused GR multi captured graph differs from direct max-T call\n");
+        ++bad;
+    }
+
+    // Reuse the same captured pointers with new payloads; then independently run the single-token path again.
+    for (size_t i = 0; i < r.size(); ++i) r[i] = -0.7f * r[i] + 0.001f * (float) (i % 17);
+    for (size_t i = 0; i < bo.size(); ++i) bo[i] = -0.4f * bo[i] + 0.02f;
+    for (size_t i = 0; i < inj.size(); ++i) inj[i] += 0.3f;
+    check(cudaMemcpyAsync(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay R");
+    check(cudaMemcpyAsync(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay bo");
+    check(cudaMemcpyAsync(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice, stream), "multi replay inj");
+    check(cudaGraphLaunch(graph_exec, stream), "multi graph changed replay");
+    check(cudaStreamSynchronize(stream), "multi graph changed sync");
+    const Snapshot replay = snapshot();
+    if (std::memcmp(multi.mixed.data(), replay.mixed.data(), multi.mixed.size() * sizeof(float)) == 0) {
+        std::printf("  fused GR graph replay ignored changed inputs\n");
+        ++bad;
+    }
+    for (int t = 0; t < T; ++t) fused_gr_read(args[t], stream);
+    check(cudaStreamSynchronize(stream), "changed single reference sync");
+    if (!same(replay, snapshot())) {
+        std::printf("  fused GR changed graph replay differs from single-token calls\n");
+        ++bad;
+    }
+
+    std::printf("  fused GR multi max-T=8 LDS launch and changing graph replay %s\n",
+                bad == 0 ? "pass" : "FAIL");
+    check(cudaGraphExecDestroy(graph_exec), "multi graph exec destroy");
+    check(cudaGraphDestroy(graph), "multi graph destroy");
+    check(cudaStreamDestroy(stream), "multi stream destroy");
+    check(cudaFree(d_xn), "free multi xn");
+    check(cudaFree(d_mixed), "free multi mixed");
+    check(cudaFree(d_inj_out), "free multi injection");
+    check(cudaFree(d_rs), "free multi rs");
+    check(cudaFree(d_lo), "free multi lo");
+    check(cudaFree(d_inj), "free multi inj");
+    check(cudaFree(d_bo), "free multi bo");
+    check(cudaFree(d_r_out), "free multi R_out");
+    check(cudaFree(d_r), "free multi R");
+    return bad;
+}
+
 }  // namespace
 
 int main(int argc, char** argv) {
@@ -602,6 +740,7 @@ int main(int argc, char** argv) {
                         activation_mode_name(mode), ok ? "pass" : "*** FAIL ***", rm, ri);
             if (!ok) ++bad;
         }
+        bad += fused_multi_lds_parity(dN, dD, dU, dJ, eps);
         select_activation_mode(0);
         cudaFree(rws_raw);
         cudaFree(dR); cudaFree(dN); cudaFree(dD); cudaFree(dU); cudaFree(dJ); cudaFree(dM); cudaFree(dI);

src/kernels/kv_hybrid_parity.cpp

@@ -0,0 +1,361 @@
+// src/kernels/kv_hybrid_parity.cpp - hybrid K8V4 KV against host references (GPU, no model).
+//
+// Exercises the EXACT composed calls the engine's hybrid path makes (layer.cpp / prefill.cpp): the q8/q4
+// append and gather kernels invoked with the unused half's pointers folded onto the used pool, and the
+// fused decode attention's KV_MODE 3 (INT8 K, rotated Q4_0 V) followed by the output's inverse Hadamard.
+//   1. K: INT8 codes + FP16 scales bitwise vs the host reference (kv_q8.hpp's rules);
+//   2. V: after fwht256, Q4_0 blocks bitwise vs the host reference (kv_q4.hpp's rules, PR #21);
+//   3. the composed gathers bitwise vs the host dequantization of those codes;
+//   4. qsa_decode_attn<3> + output fwht vs a host attention over the SAME dequantized K/V (fp32 math), and
+//      bounded against the fp32-true attention from the unquantized K/V.
+#include "strata/kernels/f16_bits.hpp"
+#include "strata/kernels/kv_q4.hpp"
+#include "strata/kernels/kv_q8.hpp"
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_decode_attn.hpp"
+#include "strata/kernels/qsa_prompt_attn.hpp"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+int g_fail = 0;
+void ck(cudaError_t e, const char* w) {
+    if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", w, cudaGetErrorString(e)); std::exit(2); }
+}
+template <typename T> T* dalloc(size_t n) {
+    T* p = nullptr;
+    ck(cudaMalloc(&p, n * sizeof(T) + 64), "malloc");
+    ck(cudaMemset(p, 0, n * sizeof(T) + 64), "memset");
+    return p;
+}
+
+// kv_q4_parity.cpp's exact CPU references.
+void fwht256_host_reference(float* x) {
+    constexpr int n = 256;
+    for (int s = 1; s < n; s *= 2) {
+        for (int i = 0; i < n; i += 2 * s) {
+            for (int j = 0; j < s; ++j) {
+                const float u = x[i + j];
+                const float v = x[i + j + s];
+                x[i + j]     = u + v;
+                x[i + j + s] = u - v;
+            }
+        }
+    }
+    const float scale = 1.0f / 16.0f;
+    for (int i = 0; i < n; ++i) x[i] *= scale;
+}
+void quantize_block_q4_0_host(const float* x, k::block_q4_0& blk) {
+    float amax = 0.0f, max_val = 0.0f;
+    for (int i = 0; i < 32; ++i) {
+        const float v = x[i];
+        if (amax < std::fabs(v)) { amax = std::fabs(v); max_val = v; }
+    }
+    const float d = max_val / -8.0f;
+    const float id = d != 0.0f ? 1.0f / d : 0.0f;
+    blk.d = k::f16_from_f32(d);
+    for (int i = 0; i < 16; ++i) {
+        const int q0 = (int) (x[i] * id + 8.5f);
+        const int q1 = (int) (x[i + 16] * id + 8.5f);
+        const uint8_t xi0 = (uint8_t) std::min(15, std::max(0, q0));
+        const uint8_t xi1 = (uint8_t) std::min(15, std::max(0, q1));
+        blk.qs[i] = xi0 | (xi1 << 4);
+    }
+}
+}  // namespace
+
+int main() {
+    std::printf("=== Running kv_hybrid_parity test ===\n");
+    k::QsaShapes s = k::qsa_real_shapes();
+    s.page_size = 64;
+    const int H = (int) s.n_head_kv, D = (int) s.head_dim, P = (int) s.page_size, G = D / k::KV_Q8_GROUP;
+    const int QH = (int) s.n_head, PER = QH / H;   // query heads per KV head
+    const int pages = 24, cells = pages * P;   // n_chunks >= n_head: the batched merge overlaps queries below that
+    std::mt19937 rng(17);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+
+    std::vector<int32_t> table(pages);
+    for (int i = 0; i < pages; ++i) table[i] = (i * 5 + 3) % pages;
+    int32_t* d_table = dalloc<int32_t>(pages);
+    ck(cudaMemcpy(d_table, table.data(), pages * 4, cudaMemcpyHostToDevice), "table");
+
+    std::vector<float> K((size_t) cells * H * D), V((size_t) cells * H * D);
+    for (auto& x : K) x = nd(rng) * 0.5f;
+    for (auto& x : V) x = nd(rng) * 0.5f;
+
+    // ---- host reference: K in INT8 (kv_q8.hpp), rotated V in Q4_0 (kv_q4.hpp)
+    std::vector<int8_t> rk((size_t) cells * H * D);
+    std::vector<uint16_t> rk_s((size_t) cells * H * G);
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int g = 0; g < G; ++g) {
+                float amax = 0.0f;
+                for (int t = 0; t < k::KV_Q8_GROUP; ++t)
+                    amax = std::max(amax, std::fabs(K[((size_t) c * H + h) * D + g * k::KV_Q8_GROUP + t]));
+                const uint16_t sb = k::f16_from_f32(amax / 127.0f);
+                const float sf = k::f32_from_f16(sb);
+                for (int t = 0; t < k::KV_Q8_GROUP; ++t) {
+                    int q = 0;
+                    if (sf > 0.0f) {
+                        q = (int) std::nearbyint(K[((size_t) c * H + h) * D + g * k::KV_Q8_GROUP + t] / sf);   // rn: ties to even, as the kernel
+                        q = std::max(-127, std::min(127, q));
+                    }
+                    rk[((size_t) c * H + h) * D + g * k::KV_Q8_GROUP + t] = (int8_t) q;
+                }
+                rk_s[((size_t) c * H + h) * G + g] = sb;
+            }
+    std::vector<float> Vrot((size_t) cells * H * D);
+    std::memcpy(Vrot.data(), V.data(), V.size() * 4);
+    for (int c = 0; c < cells * H; ++c) fwht256_host_reference(Vrot.data() + (size_t) c * D);
+    std::vector<k::block_q4_0> rv4((size_t) cells * H * (D / 32));
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int b = 0; b < D / 32; ++b)
+                quantize_block_q4_0_host(&Vrot[((size_t) c * H + h) * D + b * 32],
+                                         rv4[((size_t) c * H + h) * (D / 32) + b]);
+
+    // ---- device: one cell appended through the COMPOSED hybrid calls, then gathered
+    const int64_t q4_row = (int64_t) k::kv_q4_bytes_per_head(D);
+    int8_t* d_kq = dalloc<int8_t>((size_t) cells * H * D);
+    uint16_t* d_ks = dalloc<uint16_t>((size_t) cells * H * G);
+    uint8_t* d_v4 = dalloc<uint8_t>((size_t) cells * H * q4_row);
+    float *d_kcur = dalloc<float>((size_t) H * D), *d_vcur = dalloc<float>((size_t) H * D);
+    int32_t* d_step = dalloc<int32_t>(k::kStepCount);
+
+    int append_bad = 0, gather_bad = 0;
+    for (int c = 0; c < cells; ++c) {
+        ck(cudaMemcpy(d_kcur, &K[(size_t) c * H * D], (size_t) H * D * 4, cudaMemcpyHostToDevice), "kcur");
+        ck(cudaMemcpy(d_vcur, &V[(size_t) c * H * D], (size_t) H * D * 4, cudaMemcpyHostToDevice), "vcur");
+        int32_t step[k::kStepCount];
+        k::qsa_step_fill(step, c, s);
+        ck(cudaMemcpy(d_step, step, sizeof step, cudaMemcpyHostToDevice), "step");
+        // exactly what qsa_layer's hybrid branch does
+        k::fwht256_inplace_cuda(d_vcur, H, nullptr);
+        k::kv_append_q8_step(d_kq, d_kq, d_ks, d_ks, d_table, d_step, d_kcur, d_kcur, s, nullptr, nullptr);
+        k::kv_append_q4_step(d_v4, d_v4, d_table, d_step, d_vcur, d_vcur, s, nullptr, nullptr);
+        ck(cudaGetLastError() != cudaSuccess ? cudaGetLastError() : cudaSuccess, "append");
+    }
+    ck(cudaDeviceSynchronize(), "append sync");
+
+    std::vector<int8_t> hk((size_t) cells * H * D);
+    std::vector<uint16_t> hks((size_t) cells * H * G);
+    std::vector<uint8_t> hv4((size_t) cells * H * q4_row);
+    ck(cudaMemcpy(hk.data(), d_kq, hk.size(), cudaMemcpyDeviceToHost), "kq");
+    ck(cudaMemcpy(hks.data(), d_ks, hks.size() * 2, cudaMemcpyDeviceToHost), "ks");
+    ck(cudaMemcpy(hv4.data(), d_v4, hv4.size(), cudaMemcpyDeviceToHost), "v4");
+    // the pools are PHYSICAL (page, head, slot); the reference is by LOGICAL cell - compare through the table
+    const k::block_q4_0* hv4b = reinterpret_cast<const k::block_q4_0*>(hv4.data());
+    for (int c = 0; c < cells; ++c) {
+        const long long row = ((long long) table[c / P] * H);
+        for (int h = 0; h < H; ++h) {
+            const long long r = (row + h) * P + c % P;
+            for (int d = 0; d < D; ++d)
+                if (hk[(size_t) r * D + d] != rk[((size_t) c * H + h) * D + d]) ++append_bad;
+            for (int g = 0; g < G; ++g)
+                if (hks[(size_t) r * G + g] != rk_s[((size_t) c * H + h) * G + g]) ++append_bad;
+            for (int b = 0; b < D / 32; ++b) {
+                const k::block_q4_0& got = hv4b[(size_t) r * (D / 32) + b];
+                if (got.d != rv4[((size_t) c * H + h) * (D / 32) + b].d ||
+                    std::memcmp(got.qs, rv4[((size_t) c * H + h) * (D / 32) + b].qs, 16) != 0) ++append_bad;
+            }
+        }
+    }
+    std::printf("[1/3] composed appends: %s (%d bad values of %zu K / %zu V)\n",
+                append_bad == 0 ? "PASS" : "FAIL", append_bad, hk.size(), rv4.size());
+
+    // gather every cell (ids = all, ascending - the identity selection for width = cells)
+    std::vector<int32_t> ids(cells);
+    for (int i = 0; i < cells; ++i) ids[i] = i;
+    int32_t* d_ids = dalloc<int32_t>(cells);
+    ck(cudaMemcpy(d_ids, ids.data(), cells * 4, cudaMemcpyHostToDevice), "ids");
+    int32_t step[k::kStepCount];
+    k::qsa_step_fill(step, cells - 1, s);
+    step[k::kStepWidth] = cells;   // select everything
+    ck(cudaMemcpy(d_step, step, sizeof step, cudaMemcpyHostToDevice), "step2");
+    uint16_t *d_ksc = dalloc<uint16_t>((size_t) cells * H * D), *d_vsc = dalloc<uint16_t>((size_t) cells * H * D);
+    k::kv_gather_q8_step(d_kq, d_kq, d_ks, d_ks, d_table, d_ids, d_step, cells, s, d_ksc, d_ksc, nullptr);
+    k::kv_gather_q4_step(d_v4, d_v4, d_table, d_ids, d_step, cells, s, d_vsc, d_vsc, nullptr);
+    ck(cudaDeviceSynchronize(), "gather sync");
+    std::vector<uint16_t> h_ksc((size_t) cells * H * D), h_vsc((size_t) cells * H * D);
+    ck(cudaMemcpy(h_ksc.data(), d_ksc, h_ksc.size() * 2, cudaMemcpyDeviceToHost), "ksc");
+    ck(cudaMemcpy(h_vsc.data(), d_vsc, h_vsc.size() * 2, cudaMemcpyDeviceToHost), "vsc");
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int d = 0; d < D; ++d) {
+                const float kref = k::f32_from_f16(k::f16_from_f32(
+                    (float) rk[((size_t) c * H + h) * D + d] * k::f32_from_f16(rk_s[((size_t) c * H + h) * G + d / k::KV_Q8_GROUP])));
+                if (k::f32_from_f16(h_ksc[((size_t) c * H + h) * D + d]) != kref) ++gather_bad;
+                const k::block_q4_0& b = rv4[((size_t) c * H + h) * (D / 32) + d / 32];
+                const int nib = d % 32 < 16 ? (b.qs[d % 32] & 0x0F) - 8 : ((b.qs[d % 32 - 16] >> 4) - 8);
+                const float vref = k::f32_from_f16(k::f16_from_f32((float) nib * k::f32_from_f16(b.d)));
+                if (k::f32_from_f16(h_vsc[((size_t) c * H + h) * D + d]) != vref) ++gather_bad;
+            }
+    std::printf("[2/3] composed gathers: %s (%d bad of %zu)\n",
+                gather_bad == 0 ? "PASS" : "FAIL", gather_bad, h_ksc.size());
+
+    // ---- attention: qsa_decode_attn KV_MODE 3 over the hybrid pools + output fwht, vs fp32 references
+    std::vector<float> q((size_t) QH * D);
+    for (auto& x : q) x = nd(rng);
+    float* d_q = dalloc<float>(q.size());
+    ck(cudaMemcpy(d_q, q.data(), q.size() * 4, cudaMemcpyHostToDevice), "q");
+    k::QsaAttnPools pools;
+    pools.k_q = d_kq; pools.k_scale = d_ks; pools.v_q4 = d_v4; pools.page_table = d_table;
+    float* d_scr = dalloc<float>(k::qsa_decode_attn_scratch_floats(cells, s) + 16);
+    float* d_attn = dalloc<float>((size_t) QH * D);
+    k::qsa_decode_attn_step(d_q, pools, d_ids, d_step, cells, s, d_scr, d_attn, nullptr);
+    k::fwht256_inplace_cuda(d_attn, QH, nullptr);   // the engine's inverse rotation of the output
+    ck(cudaDeviceSynchronize(), "attn sync");
+    std::vector<float> h_attn((size_t) QH * D);
+    ck(cudaMemcpy(h_attn.data(), d_attn, h_attn.size() * 4, cudaMemcpyDeviceToHost), "attn");
+
+    // reference A: attention over the SAME dequantized K/V (isolates the kernel; fp32 host math)
+    // reference B: attention over the TRUE unquantized K/V (bounds the format's error)
+    auto host_attn = [&](const std::vector<float>& Keq, const std::vector<float>& Veq, std::vector<float>& out) {
+        out.assign((size_t) QH * D, 0.0f);
+        for (int h = 0; h < QH; ++h) {
+            const int kvh = h / PER;
+            std::vector<float> sc(cells);
+            float m = -1e30f;
+            for (int c = 0; c < cells; ++c) {
+                float d_ = 0.0f;
+                for (int t = 0; t < D; ++t)
+                    d_ += q[(size_t) h * D + t] * Keq[((size_t) c * H + kvh) * D + t];
+                sc[c] = d_ / std::sqrt((float) D);
+                m = std::max(m, sc[c]);
+            }
+            float l = 0.0f;
+            for (int c = 0; c < cells; ++c) { sc[c] = std::exp(sc[c] - m); l += sc[c]; }
+            for (int c = 0; c < cells; ++c)
+                for (int t = 0; t < D; ++t) out[(size_t) h * D + t] += sc[c] / l * Veq[((size_t) c * H + kvh) * D + t];
+        }
+    };
+    std::vector<float> Kdq((size_t) cells * H * D), Vdq((size_t) cells * H * D);
+    for (int c = 0; c < cells; ++c)
+        for (int h = 0; h < H; ++h)
+            for (int d = 0; d < D; ++d) {
+                Kdq[((size_t) c * H + h) * D + d] =
+                    (float) rk[((size_t) c * H + h) * D + d] * k::f32_from_f16(rk_s[((size_t) c * H + h) * G + d / k::KV_Q8_GROUP]);
+                const k::block_q4_0& b = rv4[((size_t) c * H + h) * (D / 32) + d / 32];
+                const int nib = d % 32 < 16 ? (b.qs[d % 32] & 0x0F) - 8 : ((b.qs[d % 32 - 16] >> 4) - 8);
+                Vdq[((size_t) c * H + h) * D + d] = (float) nib * k::f32_from_f16(b.d);
+            }
+    // un-rotate the dequantized V (the kernel returns the output in the UNROTATED domain)
+    std::vector<float> Vdq_un(Vdq.size());
+    std::memcpy(Vdq_un.data(), Vdq.data(), Vdq.size() * 4);
+    for (int c = 0; c < cells * H; ++c) fwht256_host_reference(Vdq_un.data() + (size_t) c * D);
+
+    // ---- batched form with TWO queries and per-query step arrays (the verify.cpp window's exact call)
+    {
+        const int nq = 2;
+        std::vector<float> q2((size_t) nq * QH * D);
+        for (auto& x : q2) x = nd(rng);
+        float* d_q2 = dalloc<float>(q2.size());
+        ck(cudaMemcpy(d_q2, q2.data(), q2.size() * 4, cudaMemcpyHostToDevice), "q2");
+        // the batched kernel indexes ids[z * cap + ...]: one FULL selection per query
+        std::vector<int32_t> ids2((size_t) nq * cells);
+        for (int i = 0; i < nq; ++i) for (int c = 0; c < cells; ++c) ids2[(size_t) i * cells + c] = c;
+        int32_t* d_ids2 = dalloc<int32_t>((size_t) nq * cells);
+        ck(cudaMemcpy(d_ids2, ids2.data(), ids2.size() * 4, cudaMemcpyHostToDevice), "ids2");
+        int32_t* d_steps2 = dalloc<int32_t>(nq * k::kStepCount);
+        for (int i = 0; i < nq; ++i) {
+            int32_t st2[k::kStepCount];
+            k::qsa_step_fill(st2, cells - 1 - i, s);
+            st2[k::kStepWidth] = cells - i;   // slightly different widths, as verify's windows have
+            ck(cudaMemcpy(d_steps2 + i * k::kStepCount, st2, sizeof st2, cudaMemcpyHostToDevice), "steps2");
+        }
+        float* d_scr2 = dalloc<float>(k::qsa_decode_attn_scratch_floats(cells, s) * nq + 64);
+        float* d_at2 = dalloc<float>((size_t) nq * QH * D);
+        k::qsa_decode_attn_batch(d_q2, pools, d_ids2, d_steps2, cells, s, d_scr2, d_at2, nq, nullptr);
+        k::fwht256_inplace_cuda(d_at2, (int64_t) nq * QH, nullptr);
+        ck(cudaDeviceSynchronize(), "attn2 sync");
+        std::vector<float> h_at2((size_t) nq * QH * D);
+        ck(cudaMemcpy(h_at2.data(), d_at2, h_at2.size() * 4, cudaMemcpyDeviceToHost), "at2");
+        double max_b = 0.0;
+        double per_q[2] = {0.0, 0.0};
+        for (int i = 0; i < nq; ++i)
+            for (int h = 0; h < QH; ++h) {
+                // reference over the cells this query's width selects (ids are ascending: the first width)
+                const int w = cells - i;
+                std::vector<float> sc(w);
+                float m = -1e30f;
+                for (int c = 0; c < w; ++c) {
+                    float d_ = 0.0f;
+                    for (int t = 0; t < D; ++t)
+                        d_ += q2[((size_t) i * QH + h) * D + t] * Kdq[((size_t) c * H + h / PER) * D + t];
+                    sc[c] = d_ / std::sqrt((float) D);
+                    m = std::max(m, sc[c]);
+                }
+                float l = 0.0f;
+                for (int c = 0; c < w; ++c) { sc[c] = std::exp(sc[c] - m); l += sc[c]; }
+                for (int d = 0; d < D; ++d) {
+                    float acc = 0.0f;
+                    for (int c = 0; c < w; ++c)
+                        acc += sc[c] / l * Vdq_un[((size_t) c * H + h / PER) * D + d];   // dim d only
+                    const double e = (double) std::fabs(h_at2[((size_t) i * QH + h) * D + d] - acc);
+                    max_b = std::max(max_b, e);
+                    per_q[i] = std::max(per_q[i], e);
+                }
+            }
+        const bool bok = max_b < 5e-3;
+        std::printf("       (per-query max: q0 %.2e, q1 %.2e)\n", per_q[0], per_q[1]);
+        std::printf("[4/4] mode-3 BATCHED attention (verify's form): %s (vs dequant ref %.2e)\n",
+                    bok ? "PASS" : "FAIL", max_b);
+        if (!bok) g_fail = 1;
+    }
+
+    std::vector<float> ref_deq, ref_true;
+    host_attn(Kdq, Vdq_un, ref_deq);
+    host_attn(K, V, ref_true);
+    double max_deq = 0.0, max_true = 0.0;
+    for (size_t i = 0; i < h_attn.size(); ++i) {
+        max_deq = std::max(max_deq, (double) std::fabs(h_attn[i] - ref_deq[i]));
+        max_true = std::max(max_true, (double) std::fabs(h_attn[i] - ref_true[i]));
+    }
+    // The tensor-core prompt path (qsa_prompt_attn, 0.1.22+) takes hybrid pools as KV_MODE 3: INT8 K, and V
+    // dequantized from its q4_0 blocks to fp16 at gather. Its summation order differs from the old kernel's
+    // (accuracy-level, not bitwise - qsa_prompt_attn.hpp says the same of its int8 mode), so: tolerance check
+    // against the SAME dequant reference, output un-rotated exactly as prefill.cpp does.
+    {
+        float* d_at4 = dalloc<float>((size_t) QH * D);
+        const bool took = k::qsa_prompt_attn_batch(d_q, pools, d_ids, d_step, cells, s, d_at4, 1, nullptr);
+        if (!took) {
+#if defined(STRATA_USE_HIP)
+            // AMD: the tensor-core prompt path is CUDA-only, so it refuses every pool and the old kernel runs
+            std::printf("[5/5] qsa_prompt_attn mode 3: PASS (refused on HIP - the old kernel runs)\n");
+#else
+            std::printf("[5/5] qsa_prompt_attn mode 3: FAIL (refused the hybrid pools)\n");
+            g_fail = 1;
+#endif
+        } else {
+            k::fwht256_inplace_cuda(d_at4, QH, nullptr);
+            ck(cudaDeviceSynchronize(), "at4 sync");
+            std::vector<float> h_at4((size_t) QH * D);
+            ck(cudaMemcpy(h_at4.data(), d_at4, h_at4.size() * 4, cudaMemcpyDeviceToHost), "at4");
+            double max_p = 0.0;
+            for (size_t i = 0; i < h_at4.size(); ++i)
+                max_p = std::max(max_p, (double) std::fabs(h_at4[i] - ref_deq[i]));
+            const bool pok = max_p < 1e-2;
+            std::printf("[5/5] qsa_prompt_attn mode 3 (tensor cores): %s (vs dequant ref %.2e)\n",
+                        pok ? "PASS" : "FAIL", max_p);
+            if (!pok) g_fail = 1;
+        }
+    }
+
+    const bool attn_ok = max_deq < 5e-3;   // fp32 kernel math vs fp64-ish host accumulation
+    std::printf("[3/3] mode-3 attention + output fwht: %s (vs dequant ref %.2e, vs true fp32 %.2e)\n",
+                attn_ok ? "PASS" : "FAIL", max_deq, max_true);
+
+    const bool ok = append_bad == 0 && gather_bad == 0 && attn_ok && g_fail == 0;
+    std::printf("=== kv_hybrid_parity: %s ===\n", ok ? "ALL PASS" : "FAILED");
+    return ok ? 0 : 1;
+}

src/kernels/qsa_select_bench.cpp

@@ -0,0 +1,139 @@
+// src/kernels/qsa_select_bench.cpp - the prompt path's QSA selection (qsa_select.hpp) timed per stage, block scores
+// (the warp kernel and the tensor-core one) and top-k, for a batch of consecutive queries at a given context, and
+// the two scorers compared: score difference and how many selections differ (GPU, synthetic, no model).
+// Usage: qsa_select_bench [context=131072] [queries=256] [reps=10]
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_select.hpp"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+void ck(cudaError_t e, const char* w) {
+    if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", w, cudaGetErrorString(e)); std::exit(2); }
+}
+template <typename T> T* up(const std::vector<T>& h) {
+    T* d = nullptr;
+    ck(cudaMalloc(&d, h.size() * sizeof(T) + 64), "malloc");
+    ck(cudaMemcpy(d, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "upload");
+    return d;
+}
+}  // namespace
+
+int main(int argc, char** argv) {
+    const int64_t ctx = argc > 1 ? std::atoll(argv[1]) : 131072;
+    const int64_t nq = argc > 2 ? std::atoll(argv[2]) : 256;
+    const int reps = argc > 3 ? std::atoi(argv[3]) : 10;
+    const k::QsaShapes s = k::qsa_real_shapes();
+    const int64_t max_blocks = ctx / 4 + 1, cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+    std::mt19937 rng(7);
+    std::normal_distribution<float> nd(0.f, 1.f);
+    // keys with a shared direction plus noise, so the scores have a spread like a real indexer's
+    std::vector<float> dir(128), pooled((size_t) (max_blocks * 128)), dead(128), q((size_t) (nq * 512));
+    for (auto& x : dir) x = nd(rng);
+    for (int64_t b = 0; b < max_blocks; ++b) {
+        const float a = nd(rng);
+        for (int d = 0; d < 128; ++d) pooled[(size_t) (b * 128 + d)] = 0.5f * a * dir[d] + nd(rng);
+    }
+    for (auto& x : dead) x = nd(rng);
+    for (int64_t i = 0; i < nq; ++i)
+        for (int d = 0; d < 512; ++d) q[(size_t) (i * 512 + d)] = 0.2f * dir[d % 128] + 0.1f * nd(rng);
+    std::vector<int32_t> steps((size_t) (nq * k::kStepCount));
+    for (int64_t i = 0; i < nq; ++i) {   // qsa_step_fill's arithmetic (kept here so the bench links only qsa_select)
+        int32_t* st = steps.data() + i * k::kStepCount;
+        const int64_t pos = ctx - nq + i;
+        st[k::kStepPos] = (int32_t) pos;
+        st[k::kStepNKv] = (int32_t) (pos + 1);
+        st[k::kStepNBid] = (int32_t) ((pos + 1) / s.idx_block);
+        st[k::kStepWidth] = (int32_t) k::qsa_selection_width(pos + 1, s);
+    }
+    const float* d_pooled = up(pooled);
+    const float* d_dead = up(dead);
+    const float* d_q = up(q);
+    const int32_t* d_steps = up(steps);
+    float *sc_old = nullptr, *sc_new = nullptr;
+    int32_t *ids_old = nullptr, *ids_new = nullptr;
+    ck(cudaMalloc(&sc_old, (size_t) (nq * max_blocks) * 4), "malloc");
+    ck(cudaMalloc(&sc_new, (size_t) (nq * max_blocks) * 4), "malloc");
+    ck(cudaMalloc(&ids_old, (size_t) (nq * cap) * 4), "malloc");
+    ck(cudaMalloc(&ids_new, (size_t) (nq * cap) * 4), "malloc");
+    const int64_t active = steps[(size_t) ((nq - 1) * k::kStepCount + k::kStepNBid)] + 1;
+    auto run_old = [&] { k::qsa_block_scores(d_pooled, d_dead, d_q, d_steps, nq, max_blocks, s, sc_old, nullptr, active); };
+    auto run_new = [&] {
+        if (!k::qsa_block_scores_tc(d_pooled, d_dead, d_q, d_steps, nq, max_blocks, s, sc_new, nullptr, active)) {
+            std::fprintf(stderr, "tc scorer refused\n");
+            std::exit(2);
+        }
+    };
+    run_old();
+    run_new();
+    k::qsa_block_topk_ref(sc_old, d_steps, nq, max_blocks, cap, s, ids_old, nullptr);
+    k::qsa_block_topk(sc_new, d_steps, nq, max_blocks, cap, s, ids_new, nullptr);
+    int32_t* ids_reg = nullptr;   // the register top-k on the OLD scores: must equal the reference exactly
+    ck(cudaMalloc(&ids_reg, (size_t) (nq * cap) * 4), "malloc");
+    k::qsa_block_topk(sc_old, d_steps, nq, max_blocks, cap, s, ids_reg, nullptr);
+    ck(cudaDeviceSynchronize(), "warm");
+    // compare
+    std::vector<float> a((size_t) (nq * max_blocks)), b(a.size());
+    std::vector<int32_t> ia((size_t) (nq * cap)), ib(ia.size());
+    ck(cudaMemcpy(a.data(), sc_old, a.size() * 4, cudaMemcpyDeviceToHost), "down");
+    ck(cudaMemcpy(b.data(), sc_new, b.size() * 4, cudaMemcpyDeviceToHost), "down");
+    ck(cudaMemcpy(ia.data(), ids_old, ia.size() * 4, cudaMemcpyDeviceToHost), "down");
+    ck(cudaMemcpy(ib.data(), ids_new, ib.size() * 4, cudaMemcpyDeviceToHost), "down");
+    std::vector<int32_t> ir(ia.size());
+    ck(cudaMemcpy(ir.data(), ids_reg, ir.size() * 4, cudaMemcpyDeviceToHost), "down");
+    int64_t reg_same = 0;
+    for (int64_t i = 0; i < nq; ++i) {
+        const int64_t w = steps[(size_t) (i * k::kStepCount + k::kStepWidth)];
+        reg_same += std::equal(ia.begin() + i * cap, ia.begin() + i * cap + w, ir.begin() + i * cap);
+    }
+    double max_rel = 0, sum_rel = 0, n_rel = 0;
+    int64_t same_sel = 0, cells_diff = 0, cells_all = 0;
+    for (int64_t i = 0; i < nq; ++i) {
+        const int32_t* st = steps.data() + i * k::kStepCount;
+        for (int64_t j = 0; j <= st[k::kStepNBid]; ++j) {
+            const double x = a[(size_t) (i * max_blocks + j)], y = b[(size_t) (i * max_blocks + j)];
+            const double r = std::fabs(x - y) / std::max(1e-6, std::fabs(x));
+            max_rel = std::max(max_rel, r);
+            sum_rel += r;
+            n_rel += 1;
+        }
+        const int64_t w = st[k::kStepWidth];
+        std::vector<int32_t> x(ia.begin() + i * cap, ia.begin() + i * cap + w), y(ib.begin() + i * cap, ib.begin() + i * cap + w);
+        same_sel += x == y;
+        std::vector<int32_t> d;
+        std::set_symmetric_difference(x.begin(), x.end(), y.begin(), y.end(), std::back_inserter(d));
+        cells_diff += (int64_t) d.size() / 2;
+        cells_all += w;
+    }
+    // time
+    cudaEvent_t e0, e1;
+    cudaEventCreate(&e0); cudaEventCreate(&e1);
+    auto timed = [&](auto f) {
+        cudaEventRecord(e0);
+        for (int r = 0; r < reps; ++r) f();
+        cudaEventRecord(e1);
+        ck(cudaEventSynchronize(e1), "time");
+        float ms = 0;
+        cudaEventElapsedTime(&ms, e0, e1);
+        return ms / reps;
+    };
+    const float t_old = timed(run_old), t_new = timed(run_new);
+    const float t_tk = timed([&] { k::qsa_block_topk_ref(sc_old, d_steps, nq, max_blocks, cap, s, ids_old, nullptr); });
+    const float t_tk2 = timed([&] { k::qsa_block_topk(sc_old, d_steps, nq, max_blocks, cap, s, ids_reg, nullptr); });
+    std::printf("top-k %.3f -> %.3f ms (%.1fx), register top-k identical to the reference %lld/%lld\n", t_tk, t_tk2,
+                t_tk / t_tk2, (long long) reg_same, (long long) nq);
+    std::printf("ctx %lld, %lld queries x %lld blocks: scores %.3f -> %.3f ms (%.1fx), top-k %.3f ms; score rel diff "
+                "mean %.2g max %.2g; selections identical %lld/%lld, cells differing %.4f%%\n", (long long) ctx,
+                (long long) nq, (long long) active, t_old, t_new, t_old / t_new, t_tk, sum_rel / std::max(1.0, n_rel),
+                max_rel, (long long) same_sel, (long long) nq, cells_all ? 100.0 * (double) cells_diff / (double) cells_all : 0.0);
+    return 0;
+}

src/ngram/ple_reader.cpp

@@ -5,6 +5,7 @@
 #include <condition_variable>
 #include <cstring>
 #include <deque>
+#include <exception>
 #include <mutex>
 #include <thread>
 #include <unordered_map>
@@ -118,6 +119,14 @@ struct PleReader::Impl {
     uint8_t* slot_buf(uint32_t s) { return slab + (size_t) s * 2 * PAGE; }
     bool busy() const { return free_slots.size() < max_inflight || !delayed.empty(); }
 
+    void cancel_queued() {
+        for (const Job& job : queue) {
+            auto it = tickets.find(job.ticket);
+            if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
+        }
+        queue.clear();
+    }
+
     void record_latency(double us) {
         stats.read_us_sum += us;
         if (stats.read_us.size() < LATENCY_RING) stats.read_us.push_back((float) us);
@@ -132,7 +141,14 @@ struct PleReader::Impl {
             queue.pop_front();
             Job& j = inflight[s];
             j.issued_us = now_us();
-            if (!file.submit(j.offset, slot_buf(s), j.length, s, error)) return false;
+            if (!file.submit(j.offset, slot_buf(s), j.length, s, error)) {
+                j.uses.clear();
+                auto it = tickets.find(j.ticket);
+                if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
+                free_slots.push_back(s);
+                cancel_queued();
+                return false;
+            }
             stats.submit_us += now_us() - j.issued_us;
             ++stats.reads;
         }
@@ -141,9 +157,19 @@ struct PleReader::Impl {
 
     bool finish(const Completion& c) {
         const uint32_t s = (uint32_t) c.tag;
+        if (s >= inflight.size()) {
+            error = "PleReader: invalid table-read completion";
+            cancel_queued();
+            return false;
+        }
         Job& j = inflight[s];
         if (!c.ok) {
             error = "PleReader: a table read failed";
+            cancel_queued();
+            j.uses.clear();
+            auto it = tickets.find(j.ticket);
+            if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
+            free_slots.push_back(s);
             return false;
         }
         record_latency(now_us() - j.issued_us);
@@ -152,8 +178,15 @@ struct PleReader::Impl {
         for (const Use& u : j.uses) {
             if (u.in_page + ROW_BYTES > c.bytes) {
                 error = "PleReader: short read inside the table";
+                cancel_queued();
+                j.uses.clear();
+                auto it = tickets.find(j.ticket);
+                if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
+                free_slots.push_back(s);
                 return false;
             }
+        }
+        for (const Use& u : j.uses) {
             std::memcpy(u.dst, buf + u.in_page, ROW_BYTES);
             cache.insert(u.row, buf + u.in_page);
         }
@@ -161,21 +194,28 @@ struct PleReader::Impl {
         if (it != tickets.end() && it->second.pending > 0) --it->second.pending;
         j.uses.clear();
         free_slots.push_back(s);
-        return pump();
+        return error.empty() ? pump() : true;
     }
 
     /// Completions (or wake packets) just returned by `file.wait`, applying fault injection.
     bool process(const Completion* got, int n) {
+        bool ok = true;
         for (int i = 0; i < n; ++i) {
             if (got[i].tag == DirectFile::WAKE_TAG) continue;
+            if (got[i].tag >= inflight.size()) {
+                error = "PleReader: invalid table-read completion";
+                cancel_queued();
+                ok = false;
+                continue;
+            }
             if (delay_us > 0 && now_us() - inflight[(uint32_t) got[i].tag].issued_us < delay_us) {
                 delayed.push_back(got[i]);
                 ++stats.late_injected;
                 continue;
             }
-            if (!finish(got[i])) return false;
+            if (!finish(got[i])) ok = false;   // drain the rest of this batch so no completed slot is stranded
         }
-        return true;
+        return ok;
     }
 
     bool release_delayed() {
@@ -256,7 +296,19 @@ bool PleReader::open(const std::string& path, uint64_t table_offset, uint64_t n_
     reset_stats();
     impl_->stop = false;
     impl_->threaded = io_thread;
-    if (io_thread) impl_->worker = std::thread([this] { impl_->worker_loop(); });
+    if (io_thread) {
+        try {
+            impl_->worker = std::thread([this] { impl_->worker_loop(); });
+        } catch (const std::exception& e) {
+            err = std::string("PleReader: cannot create I/O worker: ") + e.what();
+            close();
+            return false;
+        } catch (...) {
+            err = "PleReader: cannot create I/O worker";
+            close();
+            return false;
+        }
+    }
     return true;
 }
 
@@ -341,10 +393,11 @@ PleReader::Ticket PleReader::issue(const uint32_t* rows, size_t n, uint8_t* out_
     // Sorted by offset: prefill chunks then read the SSD in near-sequential order.
     std::sort(jobs.begin(), jobs.end(), [](const Job& a, const Job& b) { return a.offset < b.offset; });
     ts.pending = (uint32_t) jobs.size();
+    const bool has_jobs = ts.pending > 0;
     for (Job& j : jobs) m.queue.push_back(std::move(j));
     if (m.threaded) {
         lk.unlock();
-        if (ts.pending > 0) {
+        if (has_jobs) {
             m.cv_work.notify_one();
             m.file.wake();                         // in case the worker is blocked in the port
         }
@@ -361,21 +414,26 @@ bool PleReader::collect(Ticket t, std::string& err) {
         std::unique_lock<std::mutex> lk(m.mu);
         auto it = m.tickets.find(t.id);
         if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }
-        m.cv_done.wait(lk, [&] { return !m.error.empty() || m.tickets[t.id].pending == 0; });
+        m.cv_done.wait(lk, [&] {
+            const auto current = m.tickets.find(t.id);
+            return current == m.tickets.end() || current->second.pending == 0;
+        });
+        it = m.tickets.find(t.id);
+        if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }
         if (!m.error.empty()) { err = m.error; return false; }
         m.stats.wait_us += now_us() - start;
-        m.tickets.erase(t.id);
+        m.tickets.erase(it);
         return true;
     }
     auto it = m.tickets.find(t.id);
     if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }
     while (it->second.pending > 0) {
-        if (!m.error.empty() || !m.drain(-1)) {
-            err = m.error.empty() ? "PleReader: read failed" : m.error;
-            return false;
-        }
+        const bool drained = m.drain(-1);
+        if (!drained && m.error.empty()) m.error = "PleReader: read failed";
         it = m.tickets.find(t.id);
+        if (it == m.tickets.end()) { err = "PleReader: unknown ticket"; return false; }
     }
+    if (!m.error.empty()) { err = m.error; return false; }
     m.stats.wait_us += now_us() - start;
     m.tickets.erase(it);
     return true;

src/prefill/gemm.cu

@@ -5,8 +5,32 @@
 #include <cublas_v2.h>
 #include <cuda_runtime.h>
 
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+// The HIP compatibility shim maps CUDA shuffle spellings to Strata helpers.
+// hipBLASLt's public headers declare native HIP shuffle functions, so keep
+// those declarations from being macro-expanded in this translation unit.
+#undef __shfl_xor_sync
+#undef __shfl_down_sync
+#undef __shfl_up_sync
+#undef __shfl_sync
+#undef __ballot_sync
+#endif
+
+#include <climits>
 #include <cstdio>
 #include <cstdlib>
+#include <cstring>
+#include <memory>
+
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+#include "hipblaslt_tuning.hpp"
+#include <hip/hip_runtime_api.h>
+#include <hipblaslt/hipblaslt.h>
+#include <hipblaslt/hipblaslt-ext.hpp>
+#include <map>
+#include <set>
+#include <tuple>
+#endif
 
 namespace strata::prefill {
 namespace {
@@ -18,9 +42,239 @@ void ck(cublasStatus_t s, const char* what) {
     }
 }
 
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+struct HipLtCallKey {
+    strata::prefill::hipblaslt::InputType type;
+    int t;
+    int n;
+    int k;
+    int ldy;
+    uint32_t beta_bits;
+
+    bool operator<(const HipLtCallKey& other) const {
+        return std::tie(type, n, k, ldy, t, beta_bits) <
+               std::tie(other.type, other.n, other.k, other.ldy, other.t, other.beta_bits);
+    }
+};
+
+struct HipLtCachedAlgo {
+    bool supported = false;
+    hipblasLtMatmulAlgo_t algo{};
+    size_t workspace_bytes = 0;
+};
+
+struct HipLtState {
+    hipblasLtHandle_t handle = nullptr;
+    void* workspace = nullptr;
+    size_t workspace_bytes = 0;
+    strata::prefill::hipblaslt::TuningTable table;
+    std::map<HipLtCallKey, HipLtCachedAlgo> cache;
+    uint64_t lt_launches = 0;
+    uint64_t fallbacks = 0;
+    std::set<std::tuple<strata::prefill::hipblaslt::InputType, int, int, int, int>> fallback_shapes;
+
+    ~HipLtState() {
+        if (std::getenv("STRATA_HIPBLASLT_VERBOSE")) {
+            std::fprintf(stderr, "prefill gemm: hipBLASLt summary launches=%llu fallbacks=%llu unique_fallback_shapes=%zu\n",
+                         (unsigned long long) lt_launches, (unsigned long long) fallbacks, fallback_shapes.size());
+            for (const auto& shape : fallback_shapes) {
+                const auto type = std::get<0>(shape);
+                std::fprintf(stderr, "prefill gemm: fallback shape dtype=%s T=%d N=%d K=%d ldy=%d\n",
+                             type == strata::prefill::hipblaslt::InputType::bf16 ? "bf16" : "f16",
+                             std::get<1>(shape), std::get<2>(shape), std::get<3>(shape), std::get<4>(shape));
+            }
+        }
+        if (handle) hipblasLtDestroy(handle);
+    }
+};
+
+struct HipLtDescriptors {
+    hipblasLtMatmulDesc_t op = nullptr;
+    hipblasLtMatrixLayout_t a = nullptr;
+    hipblasLtMatrixLayout_t b = nullptr;
+    hipblasLtMatrixLayout_t c = nullptr;
+
+    ~HipLtDescriptors() {
+        if (op) hipblasLtMatmulDescDestroy(op);
+        if (a) hipblasLtMatrixLayoutDestroy(a);
+        if (b) hipblasLtMatrixLayoutDestroy(b);
+        if (c) hipblasLtMatrixLayoutDestroy(c);
+    }
+
+    bool init(hipDataType type, int t, int n, int k, int ldy) {
+        const hipblasOperation_t trans_a = HIPBLAS_OP_T;
+        const hipblasOperation_t trans_b = HIPBLAS_OP_N;
+        if (hipblasLtMatmulDescCreate(&op, HIPBLAS_COMPUTE_32F, HIP_R_32F) != HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatmulDescSetAttribute(op, HIPBLASLT_MATMUL_DESC_TRANSA, &trans_a, sizeof(trans_a)) !=
+                HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatmulDescSetAttribute(op, HIPBLASLT_MATMUL_DESC_TRANSB, &trans_b, sizeof(trans_b)) !=
+                HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatrixLayoutCreate(&a, type, k, n, k) != HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatrixLayoutCreate(&b, type, k, t, k) != HIPBLAS_STATUS_SUCCESS ||
+            hipblasLtMatrixLayoutCreate(&c, HIP_R_32F, n, t, ldy) != HIPBLAS_STATUS_SUCCESS) {
+            return false;
+        }
+        return true;
+    }
+};
+
+std::unique_ptr<HipLtState> create_hipblaslt_state(void* workspace, size_t workspace_bytes) {
+    const char* path = std::getenv("STRATA_HIPBLASLT_TUNING");
+    if (!path || !*path) return nullptr;
+
+    auto state = std::make_unique<HipLtState>();
+    state->workspace = workspace;
+    state->workspace_bytes = workspace_bytes;
+    if (hipblasLtCreate(&state->handle) != HIPBLAS_STATUS_SUCCESS) {
+        std::fprintf(stderr, "prefill gemm: hipBLASLt handle creation failed; using hipBLASEx\n");
+        return nullptr;
+    }
+
+    int version = 0;
+    if (hipblasLtGetVersion(state->handle, &version) != HIPBLAS_STATUS_SUCCESS) {
+        std::fprintf(stderr, "prefill gemm: hipBLASLt version query failed; using hipBLASEx\n");
+        return nullptr;
+    }
+    int device = 0;
+    hipDeviceProp_t properties{};
+    if (hipGetDevice(&device) != hipSuccess || hipGetDeviceProperties(&properties, device) != hipSuccess) {
+        std::fprintf(stderr, "prefill gemm: HIP device query failed; using hipBLASEx\n");
+        return nullptr;
+    }
+    std::string arch(properties.gcnArchName);
+    const auto suffix = arch.find(':');
+    if (suffix != std::string::npos) arch.resize(suffix);
+
+    std::string error;
+    if (!state->table.load(path, arch, version, error)) {
+        std::fprintf(stderr, "prefill gemm: %s; using hipBLASEx\n", error.c_str());
+        return nullptr;
+    }
+    std::fprintf(stderr, "prefill gemm: hipBLASLt tuning enabled (%zu rows, %s, version %d)\n",
+                 state->table.rows().size(), arch.c_str(), version);
+    return state;
+}
+
+HipLtCachedAlgo resolve_hipblaslt_algo(HipLtState& state, strata::prefill::hipblaslt::InputType type, int t,
+                                       int n, int k, int ldy, float beta) {
+    uint32_t beta_bits = 0;
+    static_assert(sizeof(beta_bits) == sizeof(beta));
+    std::memcpy(&beta_bits, &beta, sizeof(beta));
+    const HipLtCallKey key{type, t, n, k, ldy, beta_bits};
+    const auto cached = state.cache.find(key);
+    if (cached != state.cache.end()) return cached->second;
+
+    HipLtCachedAlgo resolved;
+    const bool verbose = std::getenv("STRATA_HIPBLASLT_VERBOSE") != nullptr;
+    const auto* row = state.table.closest(type, n, k, ldy, t);
+    if (!row) {
+        if (verbose) {
+            std::fprintf(stderr, "prefill gemm: Lt fallback; no calibration for dtype=%s T=%d N=%d K=%d ldy=%d\n",
+                         type == strata::prefill::hipblaslt::InputType::bf16 ? "bf16" : "f16", t, n, k, ldy);
+        }
+        return state.cache.emplace(key, resolved).first->second;
+    }
+
+    HipLtDescriptors desc;
+    const hipDataType input_type = type == strata::prefill::hipblaslt::InputType::bf16 ? HIP_R_16BF : HIP_R_16F;
+    if (!desc.init(input_type, t, n, k, ldy)) {
+        return state.cache.emplace(key, resolved).first->second;
+    }
+
+    std::vector<int> solution_ids{row->solution_id};
+    std::vector<hipblasLtMatmulHeuristicResult_t> candidates;
+    if (hipblaslt_ext::getAlgosFromIndex(state.handle, solution_ids, candidates) != HIPBLAS_STATUS_SUCCESS ||
+        candidates.empty() || candidates.front().state != HIPBLAS_STATUS_SUCCESS ||
+        hipblaslt_ext::getIndexFromAlgo(candidates.front().algo) != row->solution_id) {
+        if (verbose) {
+            std::fprintf(stderr, "prefill gemm: Lt fallback; solution %d unavailable for T=%d N=%d K=%d ldy=%d\n",
+                         row->solution_id, t, n, k, ldy);
+        }
+        return state.cache.emplace(key, resolved).first->second;
+    }
+
+    const float alpha = 1.0f;
+    size_t required_workspace = 0;
+    auto algo = candidates.front().algo;
+    if (hipblaslt_ext::matmulIsAlgoSupported(state.handle, desc.op, &alpha, desc.a, desc.b, &beta, desc.c, desc.c,
+                                             algo, required_workspace) != HIPBLAS_STATUS_SUCCESS) {
+        if (verbose) {
+            std::fprintf(stderr, "prefill gemm: Lt fallback; solution %d rejects actual T=%d N=%d K=%d ldy=%d beta=%.9g\n",
+                         row->solution_id, t, n, k, ldy, beta);
+        }
+        return state.cache.emplace(key, resolved).first->second;
+    }
+
+    resolved.supported = true;
+    resolved.algo = algo;
+    resolved.workspace_bytes = required_workspace;
+    if (verbose) {
+        std::fprintf(stderr,
+                     "prefill gemm: Lt solution=%d dtype=%s T=%d N=%d K=%d ldy=%d beta=%.9g workspace=%zu\n",
+                     row->solution_id, type == strata::prefill::hipblaslt::InputType::bf16 ? "bf16" : "f16", t, n,
+                     k, ldy, beta, required_workspace);
+    }
+    return state.cache.emplace(key, resolved).first->second;
+}
+
+bool try_hipblaslt(void* opaque_state, strata::prefill::hipblaslt::InputType type, const uint16_t* x,
+                   const uint16_t* w, float* y, int64_t t, int64_t n, int64_t k, int64_t ldy, float beta,
+                   void* stream) {
+    auto* state = static_cast<HipLtState*>(opaque_state);
+    if (!state || t <= 0 || n <= 0 || k <= 0 || t > INT_MAX || n > INT_MAX || k > INT_MAX || ldy > INT_MAX ||
+        ldy < n) {
+        return false;
+    }
+    const auto resolved = resolve_hipblaslt_algo(*state, type, (int) t, (int) n, (int) k, (int) ldy, beta);
+    if (!resolved.supported) {
+        ++state->fallbacks;
+        state->fallback_shapes.emplace(type, (int) t, (int) n, (int) k, (int) ldy);
+        return false;
+    }
+    if (resolved.workspace_bytes > state->workspace_bytes) {
+        ++state->fallbacks;
+        state->fallback_shapes.emplace(type, (int) t, (int) n, (int) k, (int) ldy);
+        if (std::getenv("STRATA_HIPBLASLT_VERBOSE")) {
+            std::fprintf(stderr, "prefill gemm: Lt fallback; solution needs %zu workspace bytes, have %zu\n",
+                         resolved.workspace_bytes, state->workspace_bytes);
+        }
+        return false;
+    }
+
+    HipLtDescriptors desc;
+    const hipDataType input_type = type == strata::prefill::hipblaslt::InputType::bf16 ? HIP_R_16BF : HIP_R_16F;
+    if (!desc.init(input_type, (int) t, (int) n, (int) k, (int) ldy)) return false;
+    const float alpha = 1.0f;
+    const hipblasStatus_t status = hipblasLtMatmul(state->handle, desc.op, &alpha, w, desc.a, x, desc.b, &beta, y,
+                                                   desc.c, y, desc.c, &resolved.algo, state->workspace,
+                                                   state->workspace_bytes, (hipStream_t) stream);
+    if (status == HIPBLAS_STATUS_SUCCESS) {
+        ++state->lt_launches;
+        return true;
+    }
+
+    std::fprintf(stderr, "prefill gemm: hipBLASLt launch failed with status %d\n", (int) status);
+    if (beta != 0.0f) {
+        std::fprintf(stderr, "prefill gemm: refusing a fallback after hipBLASLt failed with nonzero beta\n");
+        std::exit(1);
+    }
+    auto* mutable_state = static_cast<HipLtState*>(opaque_state);
+    uint32_t beta_bits = 0;
+    std::memcpy(&beta_bits, &beta, sizeof(beta_bits));
+    auto cached = mutable_state->cache.find(HipLtCallKey{type, (int) t, (int) n, (int) k, (int) ldy, beta_bits});
+    if (cached != mutable_state->cache.end()) cached->second.supported = false;
+    ++mutable_state->fallbacks;
+    mutable_state->fallback_shapes.emplace(type, (int) t, (int) n, (int) k, (int) ldy);
+    return false;
+}
+#endif
+
 }  // namespace
 
 Gemm::~Gemm() {
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    delete static_cast<HipLtState*>(hipblaslt_state_);
+#endif
     if (handle_) cublasDestroy((cublasHandle_t) handle_);
     if (!external_) {
         if (scratch_) cudaFree(scratch_);
@@ -41,6 +295,9 @@ bool Gemm::init_external(void* stream, uint16_t* scratch, int64_t scratch_elems,
     cublasSetMathMode(h, CUBLAS_DEFAULT_MATH);
     scratch_ = scratch;
     scratch_elems_ = scratch_elems;
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    hipblaslt_state_ = create_hipblaslt_state(workspace_, ws_bytes).release();
+#endif
     return true;
 }
 
@@ -49,6 +306,13 @@ void Gemm::rebind(uint16_t* scratch, int64_t scratch_elems, void* workspace, siz
     scratch_elems_ = scratch_elems;
     workspace_ = workspace;
     cublasSetWorkspace((cublasHandle_t) handle_, workspace_, ws_bytes);
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    if (hipblaslt_state_) {
+        auto* state = static_cast<HipLtState*>(hipblaslt_state_);
+        state->workspace = workspace_;
+        state->workspace_bytes = ws_bytes;
+    }
+#endif
 }
 
 bool Gemm::init(void* stream, int64_t scratch_elems, std::string& err) {
@@ -62,6 +326,9 @@ bool Gemm::init(void* stream, int64_t scratch_elems, std::string& err) {
     if (cudaMalloc(&workspace_, ws) != cudaSuccess) { err = "prefill gemm: workspace"; return false; }
     cublasSetWorkspace(h, workspace_, ws);
     cublasSetMathMode(h, CUBLAS_DEFAULT_MATH);
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    hipblaslt_state_ = create_hipblaslt_state(workspace_, ws).release();
+#endif
     if (scratch_elems > 0 && cudaMalloc((void**) &scratch_, (size_t) scratch_elems * 2) != cudaSuccess) {
         err = "prefill gemm: dequant scratch of " + std::to_string(scratch_elems * 2 >> 20) + " MiB";
         return false;
@@ -75,6 +342,12 @@ void Gemm::bf16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64
     if (T <= 0 || N <= 0) return;
     if (ldy <= 0) ldy = N;
     const float alpha = 1.0f;
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    if (try_hipblaslt(hipblaslt_state_, strata::prefill::hipblaslt::InputType::bf16, X, W, Y, T, N, K, ldy,
+                      beta, stream_)) {
+        return;
+    }
+#endif
     // Column-major view: Y^T[N, T] = W[N, K] (stored K x N col-major, transposed) . X^T[K, T].
     ck(cublasGemmEx((cublasHandle_t) handle_, CUBLAS_OP_T, CUBLAS_OP_N, (int) N, (int) T, (int) K, &alpha, W,
                     CUDA_R_16BF, (int) K, X, CUDA_R_16BF, (int) K, &beta, Y, CUDA_R_32F, (int) ldy,
@@ -87,6 +360,12 @@ void Gemm::f16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_
     if (T <= 0 || N <= 0) return;
     if (ldy <= 0) ldy = N;
     const float alpha = 1.0f;
+#if defined(__HIPCC__) && defined(STRATA_HIPBLASLT_AVAILABLE)
+    if (try_hipblaslt(hipblaslt_state_, strata::prefill::hipblaslt::InputType::f16, X, W, Y, T, N, K, ldy,
+                      beta, stream_)) {
+        return;
+    }
+#endif
     ck(cublasGemmEx((cublasHandle_t) handle_, CUBLAS_OP_T, CUBLAS_OP_N, (int) N, (int) T, (int) K, &alpha, W,
                     CUDA_R_16F, (int) K, X, CUDA_R_16F, (int) K, &beta, Y, CUDA_R_32F, (int) ldy,
                     CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT),

src/prefill/ggml_cuda_host.cu

@@ -5,6 +5,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <mutex>
+#include <string>
 #include <vector>
 
 [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {
@@ -18,6 +19,45 @@ int ggml_cuda_get_device() {
     return id;
 }
 
+#if defined(GGML_USE_HIP)
+namespace {
+
+unsigned parse_hex(const std::string& value) {
+    if (value.empty()) return 0;
+    char* end = nullptr;
+    const unsigned long parsed = std::strtoul(value.c_str(), &end, 16);
+    return end == value.c_str() || *end != '\0' ? 0 : static_cast<unsigned>(parsed);
+}
+
+// Match GGML's AMD cc encoding: OFFSET_AMD + gfx major * 0x100 + minor/stepping.
+// For example, gfx1100 maps to GGML_CUDA_CC_RDNA3 (0x01001100).
+int ggml_cuda_parse_amd_id(const char* device_name) {
+    if (device_name == nullptr) return GGML_CUDA_CC_OFFSET_AMD;
+
+    std::string arch(device_name);
+    if (arch.compare(0, 3, "gfx") == 0) arch.erase(0, 3);
+    const size_t suffix = arch.find(':');
+    if (suffix != std::string::npos) arch.resize(suffix);
+
+    const size_t generic = arch.rfind("-generic");
+    if (generic != std::string::npos && generic + 8 == arch.size()) arch.resize(generic);
+
+    unsigned major = 0;
+    unsigned minor = 0;
+    const size_t separator = arch.find('-');
+    if (separator != std::string::npos) {
+        major = parse_hex(arch.substr(0, separator));
+        minor = parse_hex(arch.substr(separator + 1)) * 0x10;
+    } else if (arch.size() >= 3) {
+        minor = parse_hex(arch.substr(arch.size() - 2));
+        major = parse_hex(arch.substr(0, arch.size() - 2));
+    }
+    return GGML_CUDA_CC_OFFSET_AMD + static_cast<int>(major * 0x100 + minor);
+}
+
+}  // namespace
+#endif
+
 const ggml_cuda_device_info & ggml_cuda_info() {
     static ggml_cuda_device_info info = [] {
         ggml_cuda_device_info in = {};
@@ -30,15 +70,26 @@ const ggml_cuda_device_info & ggml_cuda_info() {
             cudaDeviceProp prop;
             CUDA_CHECK(cudaGetDeviceProperties(&prop, id));
             auto & d = in.devices[id];
+#if defined(GGML_USE_HIP)
+            d.cc = ggml_cuda_parse_amd_id(prop.gcnArchName);
+            if ((d.cc & 0xff00) == 0) {
+                // Fall back to HIP's device version fields if its architecture name is unavailable.
+                d.cc = GGML_CUDA_CC_OFFSET_AMD + prop.major * 0x100 + prop.minor * 0x10;
+            }
+            d.smpbo = prop.sharedMemPerBlock;
+            d.integrated = false;
+            d.supports_cooperative_launch = false;
+#else
             d.cc = 100 * prop.major + 10 * prop.minor;
-            d.nsm = prop.multiProcessorCount;
-            d.smpb = prop.sharedMemPerBlock;
             d.smpbo = prop.sharedMemPerBlockOptin;
             d.integrated = prop.integrated != 0;
+            d.supports_cooperative_launch = prop.cooperativeLaunch != 0;
+#endif
+            d.nsm = prop.multiProcessorCount;
+            d.smpb = prop.sharedMemPerBlock;
             d.vmm = false;
             d.total_vram = prop.totalGlobalMem;
             d.warp_size = prop.warpSize;
-            d.supports_cooperative_launch = prop.cooperativeLaunch != 0;
             d.physical_device = id;
             d.physical_share_count = 1;
             d.virtual_index = 0;

src/prefill/hipblaslt_tuning.hpp

@@ -0,0 +1,123 @@
+#pragma once
+
+#include <cstdint>
+#include <fstream>
+#include <limits>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace strata::prefill::hipblaslt {
+
+enum class InputType : uint8_t { f16, bf16 };
+
+struct TuningRow {
+    InputType type = InputType::bf16;
+    int n = 0;
+    int k = 0;
+    int ldy = 0;
+    int t_bucket = 0;
+    int solution_id = -1;
+};
+
+class TuningTable {
+public:
+    bool load(const std::string& path, const std::string& expected_arch, int expected_version, std::string& err) {
+        std::ifstream input(path);
+        if (!input) {
+            err = "cannot open tuning file";
+            return false;
+        }
+
+        bool saw_header = false;
+        std::string line;
+        size_t line_number = 0;
+        while (std::getline(input, line)) {
+            ++line_number;
+            const auto first = line.find_first_not_of(" \t\r\n");
+            if (first == std::string::npos || line[first] == '#') continue;
+
+            std::istringstream row(line.substr(first));
+            if (!saw_header) {
+                std::string magic;
+                std::string arch;
+                int version = 0;
+                std::string extra;
+                if (!(row >> magic >> arch >> version) || (row >> extra) || magic != "STRATA_HIPBLASLT_TUNING_V1") {
+                    err = "invalid tuning header at line " + std::to_string(line_number);
+                    return false;
+                }
+                if (arch != expected_arch) {
+                    err = "tuning architecture mismatch: file=" + arch + " runtime=" + expected_arch;
+                    return false;
+                }
+                if (version != expected_version) {
+                    err = "hipBLASLt version mismatch: file=" + std::to_string(version) +
+                          " runtime=" + std::to_string(expected_version);
+                    return false;
+                }
+                arch_ = std::move(arch);
+                version_ = version;
+                saw_header = true;
+                continue;
+            }
+
+            std::string type_name;
+            int n = 0, k = 0, ldy = 0, t_bucket = 0, solution_id = -1;
+            std::string extra;
+            if (!(row >> type_name >> n >> k >> ldy >> t_bucket >> solution_id) || (row >> extra) ||
+                (type_name != "f16" && type_name != "bf16") || n <= 0 || k <= 0 || ldy < n || t_bucket <= 0 ||
+                solution_id < 0) {
+                err = "invalid tuning row at line " + std::to_string(line_number);
+                return false;
+            }
+            TuningRow parsed;
+            parsed.type = type_name == "f16" ? InputType::f16 : InputType::bf16;
+            parsed.n = n;
+            parsed.k = k;
+            parsed.ldy = ldy;
+            parsed.t_bucket = t_bucket;
+            parsed.solution_id = solution_id;
+            for (const auto& existing : rows_) {
+                if (existing.type == parsed.type && existing.n == parsed.n && existing.k == parsed.k &&
+                    existing.ldy == parsed.ldy && existing.t_bucket == parsed.t_bucket) {
+                    err = "duplicate tuning key at line " + std::to_string(line_number);
+                    return false;
+                }
+            }
+            rows_.push_back(parsed);
+        }
+        if (!saw_header) {
+            err = "tuning file has no header";
+            return false;
+        }
+        return true;
+    }
+
+    const TuningRow* closest(InputType type, int n, int k, int ldy, int t) const {
+        const TuningRow* best = nullptr;
+        int64_t best_distance = std::numeric_limits<int64_t>::max();
+        for (const auto& row : rows_) {
+            if (row.type != type || row.n != n || row.k != k || row.ldy != ldy) continue;
+            const int64_t distance = row.t_bucket >= t ? int64_t(row.t_bucket) - t : int64_t(t) - row.t_bucket;
+            if (distance < best_distance ||
+                (distance == best_distance && (!best || row.t_bucket < best->t_bucket))) {
+                best = &row;
+                best_distance = distance;
+            }
+        }
+        return best;
+    }
+
+    const std::vector<TuningRow>& rows() const { return rows_; }
+    const std::string& arch() const { return arch_; }
+    int version() const { return version_; }
+
+private:
+    std::string arch_;
+    int version_ = 0;
+    std::vector<TuningRow> rows_;
+};
+
+}  // namespace strata::prefill::hipblaslt

src/prefill/kernels.cu

@@ -70,6 +70,68 @@ __global__ void gr_norm_kernel(const float* __restrict__ R, const float* __restr
         xn16[row * N + d] = bf(v);
     }
 }
+// F-1: the row scale only (and the BF16 image); gr_mix_r_kernel recomputes r * rs * w itself, in the same order,
+// so the FP32 copy of the normalized rows (T x 10240 floats) is neither written nor read
+__global__ void gr_norm_rs_kernel(const float* __restrict__ R, const float* __restrict__ w, float eps,
+                                  float* __restrict__ rs_out, uint16_t* __restrict__ xn16) {
+    __shared__ float sh[32];
+    const int64_t row = blockIdx.x;                 // t * 4 + c
+    const int c = (int) (row % HC);
+    const float* r = R + row * N;
+    float ss = 0.0f;
+    for (int d = threadIdx.x; d < N; d += blockDim.x) ss += r[d] * r[d];
+    const float rs = rsqrtf(block_sum(ss, sh) / (float) N + eps);
+    if (threadIdx.x == 0) rs_out[row] = rs;
+    for (int d = threadIdx.x; d < N; d += blockDim.x) xn16[row * N + d] = bf(r[d] * rs * w[c * N + d]);
+}
+__global__ void gr_mix_r_kernel(const float* __restrict__ R, const float* __restrict__ rs, const float* __restrict__ w,
+                                const float* __restrict__ g, float* __restrict__ mixed, uint16_t* __restrict__ mixed16,
+                                int64_t T, uint16_t* __restrict__ mixed_h) {
+    const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= T * N) return;
+    const int64_t t = i / N, d = i % N;
+    float s = 0.0f;
+#pragma unroll
+    for (int c = 0; c < HC; ++c) {
+        const int64_t j = t * D + c * N + d;
+        const float x = R[j] * rs[t * HC + c] * w[c * N + d];   // gr_norm_kernel's value, bit for bit
+        s = fmaf(x, sigm(g[j]), s);
+    }
+    s /= (float) HC;
+    mixed[i] = s;
+    if (mixed16) mixed16[i] = bf(s);
+    if (mixed_h) mixed_h[i] = hf(s);
+}
+// F-2: gr_write_kernel for one row (t, c), then gr_norm_rs_kernel's reduction over it with the next half's norm
+// weights - the same thread-to-element mapping (256 threads, stride 256) and block_sum, so rs and the BF16 image are
+// the same bits, and R is not read back
+constexpr int GRW_PER = (N + 255) / 256;
+__global__ void __launch_bounds__(256) gr_write_norm_rs_kernel(float* __restrict__ R, const float* __restrict__ bo,
+                                                               const float* __restrict__ inj, int64_t inj_ld,
+                                                               const float* __restrict__ w, float eps,
+                                                               float* __restrict__ rs_out, uint16_t* __restrict__ xn16) {
+    __shared__ float sh[32];
+    const int64_t row = blockIdx.x;                 // t * 4 + c
+    const int64_t t = row / HC;
+    const int c = (int) (row % HC);
+    float* r = R + row * N;
+    const float sc = 2.0f * sigm(inj[t * inj_ld + c] / (float) HC);
+    float v[GRW_PER];
+    float ss = 0.0f;
+    int k = 0;
+#pragma unroll
+    for (int d = threadIdx.x; d < N; d += 256, ++k) {
+        const float x = fmaf(bo[t * N + d], sc, r[d]);
+        r[d] = x;
+        v[k] = x;
+        ss += x * x;
+    }
+    const float rs = rsqrtf(block_sum(ss, sh) / (float) N + eps);
+    if (threadIdx.x == 0) rs_out[row] = rs;
+    k = 0;
+#pragma unroll
+    for (int d = threadIdx.x; d < N; d += 256, ++k) xn16[row * N + d] = bf(v[k] * rs * w[c * N + d]);
+}
 __global__ void gr_silu_kernel(const float* __restrict__ lo, uint16_t* __restrict__ lo16, int64_t n) {
     const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
     if (i >= n) return;
@@ -519,6 +581,22 @@ void gr_norm(const float* R, const float* w_norm, float eps, float* xn, uint16_t
     gr_norm_kernel<<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, xn, xn16);
     check("gr_norm");
 }
+void gr_norm_rs(const float* R, const float* w_norm, float eps, float* rs, uint16_t* xn16, int64_t T, void* stream) {
+    gr_norm_rs_kernel<<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, w_norm, eps, rs, xn16);
+    check("gr_norm_rs");
+}
+void gr_mix_r(const float* R, const float* rs, const float* w_norm, const float* gated, float* mixed, uint16_t* mixed16,
+              int64_t T, void* stream, uint16_t* mixed_h) {
+    gr_mix_r_kernel<<<blocks_for(T * N), 256, 0, (cudaStream_t) stream>>>(R, rs, w_norm, gated, mixed, mixed16, T,
+                                                                          mixed_h);
+    check("gr_mix_r");
+}
+void gr_write_norm_rs(float* R, const float* bo, const float* inj, int64_t inj_ld, const float* w_norm_next, float eps,
+                      float* rs, uint16_t* xn16, int64_t T, void* stream) {
+    gr_write_norm_rs_kernel<<<(unsigned) (T * HC), 256, 0, (cudaStream_t) stream>>>(R, bo, inj, inj_ld, w_norm_next, eps,
+                                                                                      rs, xn16);
+    check("gr_write_norm_rs");
+}
 void gr_silu(const float* lo, uint16_t* lo16, int64_t T, void* stream) {
     gr_silu_kernel<<<blocks_for(T * LR), 256, 0, (cudaStream_t) stream>>>(lo, lo16, T * LR);
     check("gr_silu");
@@ -589,6 +667,18 @@ void swiglu_pair(const float* g, const float* u, uint16_t* h16, int64_t n, void*
     swiglu_pair_kernel<<<blocks_for(n * 640), 256, 0, (cudaStream_t) stream>>>(g, u, h16, n);
     check("swiglu_pair");
 }
+namespace {
+__global__ void copy_i32_kernel(int32_t* __restrict__ dst, const int32_t* __restrict__ src, int64_t n) {
+    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
+        dst[i] = src[i];
+}
+}  // namespace
+void copy_i32(int32_t* dst, const int32_t* src, int64_t n, void* stream) {
+    if (n <= 0) return;
+    const int64_t b = (n + 255) / 256;
+    copy_i32_kernel<<<(unsigned) (b < 256 ? b : 256), 256, 0, (cudaStream_t) stream>>>(dst, src, n);
+    check("copy_i32");
+}
 void gather_rows16(const uint16_t* x16, const int32_t* src, uint16_t* dst16, int64_t n, int64_t width, void* stream) {
     if (n <= 0) return;
     gather_rows16_kernel<<<blocks_for(n * (width / 8)), 256, 0, (cudaStream_t) stream>>>(x16, src, dst16, n, width);

src/prefill/moe_mmq.cu

@@ -93,6 +93,7 @@ bool supported(int t) {
     switch ((ggml_type) t) {
         case GGML_TYPE_Q2_0: case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S:
         case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS:
+        case GGML_TYPE_Q8_0:   // the draft layer's dense matrices (E-9)
             return true;
         default:
             return false;
@@ -141,6 +142,7 @@ void Context::run(const Product& p, void* stream) {
         case GGML_TYPE_IQ3_S: mul_mat_q_case<GGML_TYPE_IQ3_S>(ctx, a, s); break;
         case GGML_TYPE_IQ4_NL: mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, a, s); break;
         case GGML_TYPE_IQ4_XS: mul_mat_q_case<GGML_TYPE_IQ4_XS>(ctx, a, s); break;
+        case GGML_TYPE_Q8_0: mul_mat_q_case<GGML_TYPE_Q8_0>(ctx, a, s); break;
         default:
             std::fprintf(stderr, "prefill mmq: type %d is not covered\n", (int) t);
             std::exit(1);

src/prefill/prefill.cpp

@@ -1,5 +1,6 @@
 // src/prefill/prefill.cpp - see include/strata/prefill/prefill.hpp.
 #include "strata/prefill/prefill.hpp"
+#include "strata/core/mtp.hpp"
 #include "strata/core/progress.hpp"
 #include "strata/core/on_device.hpp"
 
@@ -86,6 +87,12 @@ inline int ring_slots(size_t T) {
 }
 constexpr int DQ = 2;              // dequantized-expert ring (FP16 gate/up + down)
 
+// F-1: STRATA_GR_UNFUSED=1 keeps the FP32 copy of the normalized rows (gr_norm + gr_mix), the A/B arm
+inline bool gr_unfused() {
+    static const bool v = [] { const char* e = std::getenv("STRATA_GR_UNFUSED"); return e && e[0] == '1'; }();
+    return v;
+}
+
 double ms_since(Clock::time_point t) { return std::chrono::duration<double, std::milli>(Clock::now() - t).count(); }
 
 // Either cudaMalloc (owned, freed with the object) or a bump allocation from a borrowed region; with no base and
@@ -252,6 +259,7 @@ struct Prefill::Impl {
     std::vector<void*> owned;
     // chunk buffers
     float *emb = nullptr, *R = nullptr, *xn = nullptr, *lo = nullptr, *gated = nullptr, *inj = nullptr;
+    float* grs = nullptr;                    // F-1: the hyper-connection read's row scales (T x 4)
     uint16_t *xn16 = nullptr, *lo16 = nullptr;
     float* mixed = nullptr;
     uint16_t *mixed_bf = nullptr, *mixed_h = nullptr;
@@ -283,6 +291,13 @@ struct Prefill::Impl {
     std::vector<int32_t> bounds_host;
     std::unique_ptr<mmq::Context> mmq_ctx;
     std::vector<int32_t> ids_host, slot_host, src_host, cnt, off;
+    // The grouping tables in mapped pinned memory, [ids | slot | src] of T_max * K each, then the MMQ bounds: kernels
+    // read and write them in place.  A cudaMemcpyAsync of them queues behind the expert blobs the copy stream already
+    // holds (up to `ring` of them, ~70 us each), and the GPU idles meanwhile - measured 4.2 s of a 128K prompt's
+    // 70 s at the default ring, 0.7 s with a 16-slot one.  STRATA_GROUP_COPY=1: the copies (the A/B arm).
+    int32_t* grp_host = nullptr;
+    int32_t* grp_dev = nullptr;          // its device alias
+    size_t grp_n = 0, grp_tk = 0;        // int32s allocated; T_max * K (the offset of slot, and of src past it)
     uint16_t* dq_gu[DQ] = {};
     uint16_t* dq_d[DQ] = {};
     uint8_t* stage_dev[RING_MAX] = {};
@@ -352,6 +367,7 @@ Prefill::Prefill() : impl_(new Impl) {}
 Prefill::~Prefill() {
     if (!impl_) return;
     if (impl_->cs) cudaStreamSynchronize(impl_->cs);
+    if (impl_->copy) cudaStreamSynchronize(impl_->copy);
     for (int i = 0; i < RING_MAX; ++i) {
         if (impl_->copied[i]) cudaEventDestroy(impl_->copied[i]);
         if (impl_->used[i]) cudaEventDestroy(impl_->used[i]);
@@ -362,6 +378,7 @@ Prefill::~Prefill() {
         if (impl_->ple_emb_host[b] && impl_->ple_pageable[b].empty()) cudaFreeHost(impl_->ple_emb_host[b]);
     }
     if (impl_->copy) cudaStreamDestroy(impl_->copy);
+    if (impl_->grp_host) cudaFreeHost(impl_->grp_host);
     for (void* p : impl_->owned) cudaFree(p);
 }
 
@@ -490,6 +507,26 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, co
     }
     m.steps_host.resize(T * strata::kernels::kStepCount);
     m.ids_host.resize(T * K); m.slot_host.resize(T * K); m.src_host.resize(T * K); m.cnt.resize(m.g->n_expert); m.off.resize(m.g->n_expert + 1);
+    {
+        const size_t need = 3 * T * K + (size_t) (2 * (m.g->n_expert + m.g->n_expert / MMQ_GROUP + 2));
+        const char* gc = std::getenv("STRATA_GROUP_COPY");
+        if (m.grp_n < need && !(gc && gc[0] == '1')) {
+            if (m.grp_host) cudaFreeHost(m.grp_host);
+            m.grp_host = m.grp_dev = nullptr;
+            m.grp_n = m.grp_tk = 0;
+            void *h = nullptr, *d = nullptr;
+            if (cudaHostAlloc(&h, need * 4, cudaHostAllocMapped) == cudaSuccess &&
+                cudaHostGetDevicePointer(&d, h, 0) == cudaSuccess) {
+                m.grp_host = (int32_t*) h;
+                m.grp_dev = (int32_t*) d;
+                m.grp_n = need;
+                m.grp_tk = T * K;
+            } else {                              // the copies, as before
+                if (h) cudaFreeHost(h);
+                cudaGetLastError();
+            }
+        }
+    }
     for (int b = 0; b < 2; ++b) {
         if (!m.ple_emb_host[b] &&
             cudaHostAlloc((void**) &m.ple_emb_host[b], (size_t) T * N * 4, cudaHostAllocDefault) != cudaSuccess) {
@@ -537,7 +574,9 @@ bool Prefill::carve(size_t T, void* alloc) {
     const core::ModelGeometry& g = *m.g;
     core::SessionState& ss = *m.ss;
     bool ok = true;
-    m.emb = o.take<float>(T * N, ok); m.R = o.take<float>(T * D, ok); m.xn = o.take<float>(T * D, ok);
+    m.emb = o.take<float>(T * N, ok); m.R = o.take<float>(T * D, ok);
+    m.xn = gr_unfused() ? o.take<float>(T * D, ok) : nullptr;   // F-1: not needed (gr_mix_r reads R)
+    m.grs = o.take<float>(T * HC, ok);
     m.xn16 = o.take<uint16_t>(T * D, ok); m.lo = o.take<float>(T * LR, ok); m.lo16 = o.take<uint16_t>(T * LR, ok);
     m.gated = o.take<float>(T * D, ok); m.inj = o.take<float>(T * HC, ok);
     m.mixed = o.take<float>(T * N, ok); m.mixed_bf = o.take<uint16_t>(T * N, ok);
@@ -635,6 +674,151 @@ bool Prefill::relayout(int64_t chunk, void* borrow, uint64_t borrow_bytes, std::
 }
 
 int64_t Prefill::chunk() const { return impl_->T; }
+
+bool Prefill::draft_kv(core::MtpDrafter& mtp, const float* R_rows, const int32_t* next_tokens, int64_t n, int64_t cell0,
+                       std::string& err) {
+    Impl& m = *impl_;
+    static const bool off = [] { const char* v = std::getenv("STRATA_MTP_BATCH"); return v != nullptr && v[0] == '0'; }();
+    core::QsaState& st = mtp.kv_state_rw();
+    if (off || n <= 0 || m.g == nullptr || m.region == nullptr || st.kv_mode != 0 || st.kv_hybrid ||
+        mtp.device() != m.device)
+        return false;
+    const auto t0 = Clock::now();
+    const core::ModelGeometry& g = *m.g;
+    const int64_t Nn = g.n_embd, HCN = g.hc * g.n_embd, KV = g.n_head_kv * g.head_dim;
+    constexpr int kQ8_0 = 8;   // GGML_TYPE_Q8_0
+    const float* w_ne = mtp.tensor_f32("pre_fc_norm_embedding.weight");
+    const void* w_fe = mtp.tensor_q8("fc_embedding.weight");
+    const float* w_nh = mtp.tensor_f32("pre_fc_norm_hidden.weight");
+    const void* w_fh = mtp.tensor_q8("fc_hidden.weight");
+    const float* w_hn = mtp.tensor_f32("attn_hyper_connection.hc_norm.weight");
+    const uint16_t* w_dn = mtp.tensor_bf16("attn_hyper_connection.input_mix_weight_down.weight");
+    const uint16_t* w_up = mtp.tensor_bf16("attn_hyper_connection.input_mix_weight_up.weight");
+    const void* w_k = mtp.tensor_q8("self_attn.k_proj.weight");
+    const void* w_v = mtp.tensor_q8("self_attn.v_proj.weight");
+    const float* w_kn = mtp.tensor_f32("self_attn.k_norm.weight");
+    if (!w_ne || !w_fe || !w_nh || !w_fh || !w_hn || !w_dn || !w_up || !w_k || !w_v || !w_kn) return false;
+    const core::NativeEmbed* nemb = core::native_embed();
+    const core::WeightRef* wemb = nemb ? nullptr : m.wt->find("token_embd.weight");
+    if (!nemb && (wemb == nullptr || wemb->codebook_iq4nl || wemb->ne0 != g.n_embd || wemb->group_elems <= 0 ||
+                  (wemb->code_bits != 2 && wemb->code_bits != 4 && wemb->code_bits != 8)))
+        return false;
+    // the cells the drafter's window can still reach
+    const int64_t r0 = std::max<int64_t>(0, mtp.first_needed() - cell0);
+    if (r0 >= n) return true;
+    // per row: emb/e2 (N), en16 (N half), hn/h2/Rm/gated (HCN), hn16/xn16 (HCN half), lo (LR) + lo16, grs, mixed (N) +
+    // mixed_h, K and V (KV each), the token id
+    // E-9: the drafter's Q8_0 matrices through Q8_1 x Q8_0 MMQ - its own pass's integer dot products (mmvq), so
+    // its K/V stay close to what the drafter computes itself; STRATA_MTP_BATCH_F16=1: FP16 GEMMs (the A/B)
+    static const bool f16_only = [] { const char* v = std::getenv("STRATA_MTP_BATCH_F16"); return v && v[0] == '1'; }();
+    const bool q8 = !f16_only && mmq::built() && mmq::supported(kQ8_0);
+    const uint64_t per_row = 4 * (2 * Nn + 4 * HCN + LR + HC + Nn + 2 * KV + 1) + 2 * (Nn + 2 * HCN + LR + Nn) + 64 +
+                             (q8 ? (uint64_t) mmq::q8_bytes(g.hc, Nn) + 4 * g.hc : 0);
+    const int64_t B = std::min<int64_t>(n - r0, (int64_t) (m.region_bytes / per_row) & ~(int64_t) 63);
+    if (B < 64) return false;
+    uint8_t* q = m.region;
+    auto carve = [&](size_t bytes) { void* p = q; q += (bytes + 255) & ~(size_t) 255; return p; };
+    float* emb = (float*) carve((size_t) B * Nn * 4);
+    float* e2 = (float*) carve((size_t) B * Nn * 4);
+    uint16_t* en16 = (uint16_t*) carve((size_t) B * Nn * 2);
+    float* hn = (float*) carve((size_t) B * HCN * 4);
+    float* h2 = (float*) carve((size_t) B * HCN * 4);
+    float* Rm = (float*) carve((size_t) B * HCN * 4);
+    float* gated = (float*) carve((size_t) B * HCN * 4);
+    uint16_t* hn16 = (uint16_t*) carve((size_t) B * HCN * 2);
+    uint16_t* xn16 = (uint16_t*) carve((size_t) B * HCN * 2);
+    float* lo = (float*) carve((size_t) B * LR * 4);
+    uint16_t* lo16 = (uint16_t*) carve((size_t) B * LR * 2);
+    float* grs = (float*) carve((size_t) B * HC * 4);
+    float* mixed = (float*) carve((size_t) B * Nn * 4);
+    uint16_t* mixed_h = (uint16_t*) carve((size_t) B * Nn * 2);
+    float* Kc = (float*) carve((size_t) B * KV * 4);
+    float* Vc = (float*) carve((size_t) B * KV * 4);
+    int32_t* tok = (int32_t*) carve((size_t) B * 4);
+    void* xq = q8 ? carve(mmq::q8_bytes(B * g.hc, Nn)) : nullptr;
+    int32_t* ident = q8 ? (int32_t*) carve((size_t) B * g.hc * 4) : nullptr;
+    int32_t* bnd = q8 ? (int32_t*) carve(16) : nullptr;
+    if ((uint64_t) (q - m.region) > m.region_bytes) return false;
+    if (!mtp.idle(err)) return false;   // the drafter's own stream (its graph uploads) before this writes its K/V
+    strata::kernels::QsaShapes s = strata::kernels::qsa_real_shapes();
+    s.n_head = g.n_head; s.n_head_kv = g.n_head_kv; s.head_dim = g.head_dim; s.idx_n_head = g.idx_q_heads;
+    s.idx_dim = g.idx_key_dim;
+    std::vector<int32_t> tk((size_t) B);
+    if (q8) {
+        if (!m.mmq_ctx) m.mmq_ctx = std::make_unique<mmq::Context>();
+        mmq::iota(ident, B * g.hc, m.cs);
+    }
+    // y[rows, n_out] = x[rows, k] . w^T for a Q8_0 matrix: MMQ from the FP32 rows (bounds slot 0: rows, 1: rows*hc)
+    auto proj = [&](const float* x, const uint16_t* x16, const void* w, float* y, int64_t rows, int64_t n_out,
+                    int64_t k, int slot) {
+        if (!q8) { m.gemm.native(x16, kQ8_0, w, y, rows, n_out, k); return; }
+        mmq::quantize(x, nullptr, xq, kQ8_0, k, k, rows, m.cs);
+        mmq::Product p;
+        p.w = w; p.type = kQ8_0; p.w_rows = n_out; p.w_cols = k; p.expert_bytes = mmq::matrix_bytes(kQ8_0, n_out, k);
+        p.n = 1; p.xq = xq; p.bounds = bnd + 2 * slot; p.ids = ident; p.total_rows = rows; p.max_rows = rows;
+        p.dst = y; p.ld_dst = n_out;
+        m.mmq_ctx->run(p, m.cs);
+    };
+    for (int64_t b0 = r0; b0 < n; b0 += B) {
+        const int64_t nb = std::min(B, n - b0), c0 = cell0 + b0;
+        for (int64_t i = 0; i < nb; ++i) tk[(size_t) i] = next_tokens[b0 + i];
+        const int32_t bh[4] = {0, (int32_t) nb, 0, (int32_t) (nb * g.hc)};
+        if (q8 && cudaMemcpyAsync(bnd, bh, sizeof bh, cudaMemcpyHostToDevice, m.cs) != cudaSuccess) {
+            err = "prefill: the draft bounds' upload failed";
+            return false;
+        }
+        if (cudaMemcpyAsync(tok, tk.data(), (size_t) nb * 4, cudaMemcpyHostToDevice, m.cs) != cudaSuccess) {
+            err = "prefill: the draft tokens' upload failed";
+            return false;
+        }
+        // the input branches: the next token's embedding, and this cell's final residual rows
+        if (nemb) {
+            nemb->gather_dev(tok, nb, emb, m.cs);
+        } else {
+            const auto* codes = (const uint8_t*) wemb->data;
+            const auto* scales = (const float*) (codes + wemb->codes_bytes);
+            const auto* offsets = wemb->has_offset ? (const float*) (codes + wemb->codes_bytes + wemb->scales_bytes)
+                                                   : nullptr;
+            strata::kernels::embedding_gather_dev(codes, scales, offsets, tok, (int) nb, wemb->ne0, wemb->code_bits,
+                                                  wemb->code_bias, wemb->group_elems,
+                                                  (uint64_t) (wemb->ne0 / (8 / wemb->code_bits)),
+                                                  (uint64_t) (wemb->ne0 / wemb->group_elems), emb, m.cs);
+        }
+        rms_rows(emb, w_ne, nb, Nn, Nn, EPS, m.cs);
+        if (!q8) to_f16(emb, en16, nb * Nn, m.cs);
+        proj(emb, en16, w_fe, e2, nb, Nn, Nn, 0);
+        cudaMemcpyAsync(hn, R_rows + (size_t) b0 * HCN, (size_t) nb * HCN * 4, cudaMemcpyDeviceToDevice, m.cs);
+        rms_rows(hn, w_nh, nb, HCN, HCN, EPS, m.cs);
+        if (!q8) to_f16(hn, hn16, nb * HCN, m.cs);
+        proj(hn, hn16, w_fh, h2, nb * g.hc, Nn, Nn, 1);   // every stream through fc_hidden
+        strata::kernels::add_streams_broadcast(h2, e2, Rm, Nn, (int) g.hc, (int) nb, m.cs);
+        // the attention hyper-connection's read (its mixed input only: this pass writes nothing back)
+        gr_norm_rs(Rm, w_hn, EPS, grs, xn16, nb, m.cs);
+        m.gemm.bf16(xn16, w_dn, lo, nb, LR, HCN);
+        gr_silu(lo, lo16, nb, m.cs);
+        m.gemm.bf16(lo16, w_up, gated, nb, HCN, LR);
+        gr_mix_r(Rm, grs, w_hn, gated, mixed, nullptr, nb, m.cs, mixed_h);
+        // K and V into the drafter's cache, as the prompt path's QSA layers append theirs
+        proj(mixed, mixed_h, w_k, Kc, nb, KV, Nn, 0);
+        proj(mixed, mixed_h, w_v, Vc, nb, KV, Nn, 0);
+        rms_rows(Kc, w_kn, nb * g.n_head_kv, g.head_dim, g.head_dim, EPS, m.cs);
+        rope(Kc, nb, g.n_head_kv, g.head_dim, KV, c0, (float) strata::kernels::qsa_freq_base(), m.cs);
+        if (st.kv_q4) {
+            strata::kernels::fwht256_inplace_cuda(Kc, nb * g.n_head_kv, m.cs);
+            strata::kernels::fwht256_inplace_cuda(Vc, nb * g.n_head_kv, m.cs);
+            strata::kernels::kv_append_q4(st.k_q4, st.v_q4, st.page_table, c0, nb, Kc, Vc, s, m.cs, &st.host);
+        } else {
+            kv_append(Kc, Vc, nb, c0, st.page_table, s.page_size, st.kv_int8 ? nullptr : st.k_pool,
+                      st.kv_int8 ? nullptr : st.v_pool, st.k_q, st.v_q, st.k_scale, st.v_scale, m.cs, &st.host);
+        }
+    }
+    if (cudaStreamSynchronize(m.cs) != cudaSuccess) {
+        err = std::string("prefill: the draft layer's K/V: ") + cudaGetErrorString(cudaGetLastError());
+        return false;
+    }
+    mtp.ms_prefill += ms_since(t0);
+    return true;
+}
 void Prefill::set_pinned_share(double share) { g_pinned_share = share; }
 double Prefill::pinned_share() { return g_pinned_share; }
 
@@ -701,10 +885,12 @@ namespace {
 // host's expert grouping, or for an expert's copy) lands on the phase that was waiting.  Events are reused: the marks
 // are folded at every MoE layer's host sync, after which all of them have completed.
 enum PfPhase { kPfStart, kPfHc, kPfGdn, kPfQsa, kPfQsaIdx, kPfQsaSel, kPfQsaAttn, kPfRouter, kPfHostGroup, kPfGather,
-               kPfWaitCopy, kPfDequant, kPfGemmGU, kPfGemmD, kPfCombine, kPfPle, kPfKvStage, kPfCount };
+               kPfWaitCopy, kPfDequant, kPfGemmGU, kPfGemmD, kPfCombine, kPfPle, kPfKvStage, kPfGdnConv, kPfGdnRec, kPfGdnOut,
+               kPfCount };
 const char* const kPfNames[kPfCount] = {"embed+steps", "hc read", "gdn", "qsa proj", "qsa indexer", "qsa select",
                                         "qsa attn", "router+shared", "host grouping", "gather", "wait copy", "dequant",
-                                        "gemm gate/up", "gemm down", "combine", "ple", "kv stage"};
+                                        "gemm gate/up", "gemm down", "combine", "ple", "kv stage", "gdn conv+gates",
+                                        "gdn recurrence", "gdn out proj"};
 struct PfTimer {
     bool on = std::getenv("STRATA_PREFILL_TIMING") != nullptr;
     std::vector<cudaEvent_t> ev;
@@ -1000,6 +1186,7 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
             else issue_until(upto + (size_t) m.ring);
         };
         host_setup_ms += ms_since(tsetup);
+        bool normed = false;   // F-2: the previous half's write already normed R for this half (grs, xn16)
         for (int64_t l = LB; l < LE; ++l) {
             core::progress_beat();   // the serve watchdog: a prompt chunk of 8192 tokens is still moving
             const core::LayerView v(*m.wt, l);
@@ -1064,12 +1251,15 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                                       *wu = need(v, su.c_str(), err), *wi = need(v, si.c_str(), err);
                 if (!wn || !wd || !wu || !wi) return false;
                 pt.mark(kPfHc, cs);
-                gr_norm(m.R, (const float*) wn->data, EPS, m.xn, m.xn16, T, m.cs);
+                if (gr_unfused()) gr_norm(m.R, (const float*) wn->data, EPS, m.xn, m.xn16, T, m.cs);
+                else if (!normed) gr_norm_rs(m.R, (const float*) wn->data, EPS, m.grs, m.xn16, T, m.cs);
+                normed = false;
                 if (!bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err)) return false;
                 gr_silu(m.lo, m.lo16, T, m.cs);
                 if (!bf16_proj(m.gemm, wu, m.lo16, m.gated, T, su, err)) return false;
                 if (!bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err)) return false;
-                gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, m.cs, m.mixed_h);
+                if (gr_unfused()) gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, m.cs, m.mixed_h);
+                else gr_mix_r(m.R, m.grs, (const float*) wn->data, m.gated, m.mixed, m.mixed_bf, T, m.cs, m.mixed_h);
 
                 if (half == 0 && !core::is_qsa_layer(g, l)) {
                     // ======================= GDN =======================
@@ -1086,9 +1276,12 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     if (!native_proj(m.gemm, wg, m.mixed_h, m.z, T, v.name("attn_gate.weight"), err)) return false;
                     if (!bf16_proj(m.gemm, wa, m.mixed_bf, m.ab, T, v.name("ssm_alpha.weight"), err, 2 * HV)) return false;
                     if (!bf16_proj(m.gemm, wb, m.mixed_bf, m.ab + HV, T, v.name("ssm_beta.weight"), err, 2 * HV)) return false;
+                    pt.mark(kPfGdnConv, cs);   // "gdn" is the projections in; the rest on their own lines
                     gdn_gates(m.ab, (const float*) wdt->data, (const float*) wsa->data, m.gate, m.beta, T, m.cs);
                     gdn_conv(conv, m.qkv, (const float*) wc->data, m.hbuf, T, EPS, m.cs);
+                    pt.mark(kPfGdnRec, cs);
                     gdn_recurrence(state, m.hbuf, m.gate, m.beta, m.z, (const float*) wnm->data, EPS, m.y, m.y_h, T, m.cs);
+                    pt.mark(kPfGdnOut, cs);
                     if (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err)) return false;
                     ++gdn_index;
                 } else if (half == 0) {
@@ -1120,7 +1313,14 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                                                             (p0 + s.page_size - 1) / s.page_size, s, m.cs);
                         pt.mark(kPfQsa, cs);
                     }
-                    if (st.kv_q4) {   // Q4_0 KV (kv_q4.hpp): rotated K and V, the queries below too, the output back
+                    if (st.kv_hybrid) {   // K8V4: K INT8 unrotated, V rotated Q4_0 (only V and the output rotate)
+                        strata::kernels::fwht256_inplace_cuda(m.Vc, T * 2, m.cs);
+                        kv_append(m.Kc, m.Kc, T, p0, st.page_table, s.page_size, nullptr, nullptr,
+                                  st.k_q, st.k_q, st.k_scale, st.k_scale, m.cs, nullptr,   // mode 0: no host mirror
+                                  staged ? &m.stage : nullptr);
+                        strata::kernels::kv_append_q4(st.v_q4, st.v_q4, st.page_table, p0, T, m.Vc, m.Vc, s, m.cs,
+                                                      nullptr, staged ? &m.stage : nullptr);
+                    } else if (st.kv_q4) {   // Q4_0 KV (kv_q4.hpp): rotated K and V, the queries below too, the output back
                         strata::kernels::fwht256_inplace_cuda(m.Kc, T * 2, m.cs);
                         strata::kernels::fwht256_inplace_cuda(m.Vc, T * 2, m.cs);
                         strata::kernels::kv_append_q4(st.k_q4, st.v_q4, st.page_table, p0, T, m.Kc, m.Vc, s, m.cs,
@@ -1163,8 +1363,13 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                         // C-1: the grid reaches the batch's last query's n_bid (they rise with the position)
                         const int64_t active = (int64_t) m.steps_host[(size_t) ((t0 + nb - 1) * strata::kernels::kStepCount +
                                                                                 strata::kernels::kStepNBid)] + 1;
-                        strata::kernels::qsa_block_scores(st.idx_pooled, st.idx_dead, m.q_idx + t0 * 512, steps0, nb,
-                                                          m.max_blocks, s, m.sel_scores, m.cs, active);
+                        // the scores on tensor cores (3xTF32: FP32-level, not bitwise); STRATA_SELECT_OLD=1: the warp kernel
+                        static const bool old_sel = std::getenv("STRATA_SELECT_OLD") != nullptr;
+                        if (old_sel || !strata::kernels::qsa_block_scores_tc(st.idx_pooled, st.idx_dead, m.q_idx + t0 * 512,
+                                                                             steps0, nb, m.max_blocks, s, m.sel_scores,
+                                                                             m.cs, active))
+                            strata::kernels::qsa_block_scores(st.idx_pooled, st.idx_dead, m.q_idx + t0 * 512, steps0, nb,
+                                                              m.max_blocks, s, m.sel_scores, m.cs, active);
                         strata::kernels::qsa_block_topk(m.sel_scores, steps0, nb, m.max_blocks, m.cap, s,
                                                         m.sel_ids + t0 * m.cap, m.cs);
                     }
@@ -1258,7 +1463,8 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     pt.mark(kPfQsaAttn, cs);
                     // perf-review D-1: the whole chunk on tensor cores, one block per (query, KV head), FP32-level
                     // accuracy but not bitwise (qsa_prompt_attn.hpp). Q4_0 KV, or STRATA_PROMPT_ATTN_OLD=1: the
-                    // decode kernel, 32 queries at a time
+                    // decode kernel, 32 queries at a time (K8V4 runs the tensor kernel's mode 3: INT8 K,
+                    // V dequantized from its q4_0 blocks to fp16 at gather)
                     static const bool old_attn = std::getenv("STRATA_PROMPT_ATTN_OLD") != nullptr;
                     if (old_attn || !strata::kernels::qsa_prompt_attn_batch(m.q, pools, m.sel_ids, m.steps_dev, m.cap, s,
                                                                             m.attn, T, m.cs))
@@ -1268,7 +1474,7 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                                                                    m.steps_dev + t0 * strata::kernels::kStepCount, m.cap,
                                                                    s, m.attn_scratch, m.attn + t0 * ZV, nb, m.cs);
                         }
-                    if (st.kv_q4) strata::kernels::fwht256_inplace_cuda(m.attn, T * 24, m.cs);
+                    if (st.kv_q4 || st.kv_hybrid) strata::kernels::fwht256_inplace_cuda(m.attn, T * 24, m.cs);
                     pt.mark(kPfQsa, cs);
                     gate_attn(m.attn, m.Qf, m.attn_h, T, m.cs);
                     if (!native_proj(m.gemm, wo, m.attn_h, m.bo, T, v.name("attn_output.weight"), err)) return false;
@@ -1293,12 +1499,19 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     m.gemm.bf16(m.mixed_bf, (const uint16_t*) wgi->data, m.sg, T, 1, N);
                     // group the (token, k) pairs by expert on the host
                     pt.mark(kPfHostGroup, cs);
-                    cudaMemcpyAsync(m.ids_host.data(), m.ids, (size_t) T * K * 4, cudaMemcpyDeviceToHost, m.cs);
+                    // (the sync below also orders this layer's writes of slot/src/bounds after the previous
+                    // layer's kernels that read them)
+                    const bool grp_mapped = m.grp_host != nullptr;
+                    int32_t* ids_h = grp_mapped ? m.grp_host : m.ids_host.data();
+                    int32_t* slot_h = grp_mapped ? m.grp_host + m.grp_tk : m.slot_host.data();
+                    int32_t* src_h = grp_mapped ? m.grp_host + 2 * m.grp_tk : m.src_host.data();
+                    if (grp_mapped) copy_i32(m.grp_dev, m.ids, T * K, m.cs);
+                    else cudaMemcpyAsync(m.ids_host.data(), m.ids, (size_t) T * K * 4, cudaMemcpyDeviceToHost, m.cs);
                     cudaStreamSynchronize(m.cs);
                     pt.fold();
                     std::fill(m.cnt.begin(), m.cnt.end(), 0);
                     for (int64_t i = 0; i < T * K; ++i) {
-                        const int32_t e = m.ids_host[(size_t) i];
+                        const int32_t e = ids_h[(size_t) i];
                         if (e < 0 || e >= m.g->n_expert) { err = "prefill: routed id out of range"; return false; }
                         ++m.cnt[(size_t) e];
                     }
@@ -1306,13 +1519,18 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                     for (int64_t e = 0; e < m.g->n_expert; ++e) m.off[(size_t) e + 1] = m.off[(size_t) e] + m.cnt[(size_t) e];
                     std::vector<int32_t> fill(m.off.begin(), m.off.end() - 1);
                     for (int64_t i = 0; i < T * K; ++i) {
-                        const int32_t e = m.ids_host[(size_t) i];
+                        const int32_t e = ids_h[(size_t) i];
                         const int32_t p = fill[(size_t) e]++;
-                        m.slot_host[(size_t) i] = p;
-                        m.src_host[(size_t) p] = (int32_t) (i / K);
+                        slot_h[(size_t) i] = p;
+                        src_h[(size_t) p] = (int32_t) (i / K);
+                    }
+                    if (grp_mapped) {
+                        copy_i32(m.slot_dev, m.grp_dev + m.grp_tk, T * K, m.cs);
+                        copy_i32(m.src_dev, m.grp_dev + 2 * m.grp_tk, T * K, m.cs);
+                    } else {
+                        cudaMemcpyAsync(m.slot_dev, m.slot_host.data(), (size_t) T * K * 4, cudaMemcpyHostToDevice, m.cs);
+                        cudaMemcpyAsync(m.src_dev, m.src_host.data(), (size_t) T * K * 4, cudaMemcpyHostToDevice, m.cs);
                     }
-                    cudaMemcpyAsync(m.slot_dev, m.slot_host.data(), (size_t) T * K * 4, cudaMemcpyHostToDevice, m.cs);
-                    cudaMemcpyAsync(m.src_dev, m.src_host.data(), (size_t) T * K * 4, cudaMemcpyHostToDevice, m.cs);
                     // the experts, in id order: resident ones from VRAM, the others through the staging ring
                     std::vector<int32_t> order;
                     for (int32_t e = 0; e < m.g->n_expert; ++e) if (m.cnt[(size_t) e] > 0) order.push_back(e);
@@ -1336,8 +1554,14 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                             for (size_t i = 0; i <= MMQ_GROUP; ++i)
                                 m.bounds_host[n + 1 + g * (MMQ_GROUP + 1) + i] =
                                     m.bounds_host[std::min(n, g * MMQ_GROUP + i)] - m.bounds_host[g * MMQ_GROUP];
-                        cudaMemcpyAsync(m.bounds_dev, m.bounds_host.data(), m.bounds_host.size() * 4,
-                                        cudaMemcpyHostToDevice, m.cs);
+                        if (grp_mapped) {
+                            int32_t* bh = m.grp_host + 3 * m.grp_tk;
+                            std::memcpy(bh, m.bounds_host.data(), m.bounds_host.size() * 4);
+                            copy_i32(m.bounds_dev, m.grp_dev + 3 * m.grp_tk, (int64_t) m.bounds_host.size(), m.cs);
+                        } else {
+                            cudaMemcpyAsync(m.bounds_dev, m.bounds_host.data(), m.bounds_host.size() * 4,
+                                            cudaMemcpyHostToDevice, m.cs);
+                        }
                     } else {
                         gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs);
                     }
@@ -1522,8 +1746,24 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
                         }
                     }
                 }
-                // ---- the hyper-connection write of this half
-                gr_write(m.R, m.bo, m.inj, HC, T, m.cs);
+                // ---- the hyper-connection write of this half; F-2: fused with the next half's norm when nothing else
+                // touches R in between (not the stage's last half, not before the PLE block of layer 1, not under a
+                // control vector)
+                const int64_t nl = half == 0 ? l : l + 1;
+                const bool fuse = !gr_unfused() && nl < LE && !(half == 1 && nl == 1 && ple_on) &&
+                                  !(half == 1 && strata::kernels::cvec().covers(l));
+                const core::WeightRef* wnn = nullptr;
+                if (fuse) {
+                    const core::LayerView vn(*m.wt, nl);
+                    wnn = need(vn, half == 0 ? "hc_ffn_norm.weight" : "hc_attn_norm.weight", err);
+                    if (!wnn) return false;
+                }
+                if (wnn) {
+                    gr_write_norm_rs(m.R, m.bo, m.inj, HC, (const float*) wnn->data, EPS, m.grs, m.xn16, T, m.cs);
+                    normed = true;
+                } else {
+                    gr_write(m.R, m.bo, m.inj, HC, T, m.cs);
+                }
                 if (half == 1 && strata::kernels::cvec().covers(l))   // --control-vector-scaled
                     strata::kernels::cvec_apply(m.R, l, T, D, nullptr, 0, nullptr, 0, false, m.cs);
             }
@@ -1605,6 +1845,11 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
         err = std::string("prefill: ") + cudaGetErrorString(cudaGetLastError());
         return false;
     }
+    // (PR #121) an expert copy that failed on the copy stream surfaces here, not in the next request
+    if (const cudaError_t cst = cudaStreamSynchronize(m.copy); cst != cudaSuccess) {
+        err = std::string("prefill: expert copy stream: ") + cudaGetErrorString(cst);
+        return false;
+    }
     stats_.ms_total += ms_since(t_start);
     if (pt.on) {
         pt.fold();

src/program/generate.cpp

@@ -77,9 +77,11 @@
 #include <mutex>
 #include <charconv>
 #include <cmath>
+#include <limits>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 #include <sstream>
 #include <string>
@@ -180,6 +182,7 @@ struct Options {
     /// is reserved for a host thread that has nothing to do while the drain runs.
     bool no_host_worker = false;
     bool mmap_experts = false;    ///< R2.1: opt OUT of the resident arena, back to MapViewOfFile
+    bool resident_cpu_experts = false; ///< mmap-backed static-cache misses copied into ordinary RAM
     /// R4: slots of VRAM-resident experts.  **0 = off, and off is the default.**
     /// **THE COMMENT THAT USED TO BE HERE WAS FALSE AND ROUND 328 MEASURED IT.**  It said "the cache has no
     /// consumer yet - `moe_hit_grouped_s2` does not exist - so switching it on costs the fill traffic and
@@ -326,6 +329,8 @@ void usage() {
                  "                       VRAM; default fp16 until gate G-C accepts int8\n"
                  "  --kv q4_0            4-bit K/V after a Hadamard rotation (PR #21): half of int8's memory,\n"
                  "                       slightly lower precision (see bench/results/2026-09-27-kv-q4)\n"
+                 "  --kv k8v4            hybrid: INT8 K (exact attention scores) + rotated Q4_0 V, 816 B/cell\n"
+                 "                       (vs int8's 1,056); not with --kv-resident\n"
                  "  --kv-resident N      KV streaming: keep N cells of each QSA layer in VRAM (min 20480) and the\n"
                  "                       whole K/V in pinned RAM; the freed VRAM goes to expert slots. 0 (default):\n"
                  "                       all of it in VRAM. A context of N cells or fewer is not streamed\n"
@@ -429,7 +434,9 @@ void usage() {
                  "                       deviation from `cpu_s2` is attributed.\n"
                  "  --mmap-experts       R2.1: opt OUT of the resident expert arena, back to MapViewOfFile.\n"
                  "                       The A/B arm: the mmap's rate depends on the OS page cache holding\n"
-                 "                       34 GB, and measured 71.97 vs 34.78 ms/token cold vs warm.\n");
+                 "                       34 GB, and measured 71.97 vs 34.78 ms/token cold vs warm.\n"
+                 "  --resident-cpu-experts  with mmap and a static profile, keep CPU misses resident in ordinary RAM.\n"
+                 "                       Borrowed GPU-cache entries may read from mmap during prompt prefill.\n");
 }
 
 /// All shards of a split GGUF, from shard 1's path ("...-00001-of-00002.gguf"); just the path when it is not split.
@@ -1063,6 +1070,7 @@ int main(int argc, char** argv) {
         else if (a == "--expert-profile") o.expert_profile = next("--expert-profile");
         else if (a == "--gpu-stages") o.gpu_stages = true;
         else if (a == "--mmap-experts") o.mmap_experts = true;
+        else if (a == "--resident-cpu-experts") o.resident_cpu_experts = true;
         else if (a == "--stats") o.stats = true;
         else if (a == "--shared-late") o.shared_late = true;
         else if (a == "--keep-canonical") o.keep_canonical = true;
@@ -1131,6 +1139,16 @@ int main(int argc, char** argv) {
         }
     }
     const bool multi_gpu = !split_devs.empty() && !split_same;
+    if (o.resident_cpu_experts && (!o.mmap_experts || o.expert_profile.empty() || o.adapt_every != 0)) {
+        std::fprintf(stderr, "strata generate: --resident-cpu-experts requires --mmap-experts, a static --expert-profile and --adapt-every 0\n");
+        return 2;
+    }
+    if (o.resident_cpu_experts &&
+        (!o.layer_split.empty() || o.expert_cache_remote[0] > 0 || o.expert_cache_remote[1] > 0 ||
+         o.expert_cache_remote[2] > 0)) {
+        std::fprintf(stderr, "strata generate: --resident-cpu-experts does not support layer splits or remote expert caches\n");
+        return 2;
+    }
     // the helper-GPU expert caches (--expert-cache-remote, docs/SECOND_GPU.md): CUDA1..3 on one GPU; with a layer
     // split, the visible GPUs no stage runs on, in order
     int remote_dev[3] = {1, 2, 3};
@@ -1183,16 +1201,21 @@ int main(int argc, char** argv) {
         return 2;
     }
     if (o.kv == "q4") o.kv = "q4_0";
-    if (o.kv != "fp16" && o.kv != "int8" && o.kv != "q4_0") {
-        std::fprintf(stderr, "strata generate: --kv must be fp16, int8 or q4_0\n");
+    if (o.kv != "fp16" && o.kv != "int8" && o.kv != "q4_0" && o.kv != "k8v4") {
+        std::fprintf(stderr, "strata generate: --kv must be fp16, int8, q4_0 or k8v4\n");
         return 2;
     }
     strata::core::qsa_set_kv_int8(o.kv == "int8");
     strata::core::qsa_set_kv_q4(o.kv == "q4_0");   // PR #21: 4-bit codes after a Hadamard rotation (kv_q4.hpp)
+    strata::core::qsa_set_kv_hybrid(o.kv == "k8v4");   // K8V4: INT8 K + rotated Q4_0 V, 816 B/cell
     if (o.kv_resident < 0) {
         std::fprintf(stderr, "strata generate: --kv-resident must be >= 0\n");
         return 2;
     }
+    if (o.kv == "k8v4" && o.kv_resident > 0) {
+        std::fprintf(stderr, "strata generate: --kv k8v4 does not support --kv-resident streaming (yet)\n");
+        return 2;
+    }
     strata::core::qsa_set_kv_resident(o.kv_resident);
     // Prompt lookup (the suffix drafter, on by default): the MTP keeps its --spec windows and a lookup window may be
     // up to 2 tokens longer; the draft policy (strata/spec/draft_policy.hpp) takes one only where it pays. Code
@@ -1570,7 +1593,7 @@ int main(int argc, char** argv) {
             ss.ple.w.key_scales = (const float*) ((const uint8_t*) wk->data + wk->codes_bytes);
         }
         if (o.native_ple_key && wk->quantized()) {
-            if (!wk->native_data || wk->native_type != 42 || !wk->native_q8_1) {
+            if (!wk->native_data || (wk->native_type != 42 && wk->native_type != 18 && wk->native_type != 23) || !wk->native_q8_1) {
                 std::fprintf(stderr, "strata generate: native PLE key is absent or incompatible\n");
                 return 1;
             }
@@ -1741,9 +1764,14 @@ int main(int argc, char** argv) {
     strata::core::ArenaExpertSource arena_src;
     strata::core::ExpertSource* srcp = nullptr;
     if (o.mmap_experts) {
-        if (native_pack) {   // FileExpertSource maps the canonical pack's experts.bin; a native pack has none
-            std::fprintf(stderr, "strata generate: --mmap-experts needs a canonical pack (experts.bin); %s is a native "
-                                 "(IQ) pack, whose experts are loaded into the arena\n", o.pack.c_str());
+        // FileExpertSource maps the pack's experts.bin: a canonical pack has it; a native (IQ) pack has it when
+        // built with `tools/iq_pack.py --experts-bin` (the per-layer blob sizes of its layout, PR #121).  The low-RAM
+        // mode: the experts come from the file through the OS cache instead of a pinned copy in RAM, for a PC whose
+        // GPU holds most of them but whose RAM cannot hold them all.
+        if (native_pack && !std::filesystem::exists(std::filesystem::path(o.pack) / "experts.bin")) {
+            std::fprintf(stderr, "strata generate: --mmap-experts needs the pack's experts.bin; %s is a native (IQ) pack "
+                                 "built without it: python tools/iq_pack.py --gguf <shard 1> --out %s --experts-bin\n",
+                         o.pack.c_str(), o.pack.c_str());
             return 2;
         }
         if (!src.open(o.pack, g.n_layers, g.n_expert, err)) {
@@ -2152,6 +2180,14 @@ int main(int argc, char** argv) {
                      (long long) prefilled, (long long) want);
     }
 
+    if (o.resident_cpu_experts) {
+        if (!src.pin_cache_complement(xcache, err, /*pin=*/false)) {
+            std::fprintf(stderr, "strata generate: CPU expert residency: %s\n", err.c_str());
+            return 1;
+        }
+        std::fprintf(stderr, "strata generate: steady-state CPU cache misses are resident in ordinary RAM; borrowed cache entries may use mmap during prompt prefill\n");
+    }
+
     for (auto& stp : stages) {
         GpuStage& st = *stp;
         const auto& lay = strata::kernels::cpu::expert_layout();
@@ -2967,6 +3003,13 @@ int main(int argc, char** argv) {
                                      : (uint64_t) (xcache.slots() - first) *
                                            (uint64_t) strata::kernels::cpu::expert_layout().max_blob;
     };
+    auto request_chunk = [](int64_t tokens, int64_t max_chunk) -> int64_t {
+        if (tokens <= 0 || max_chunk <= 0) return 0;
+        const int64_t rounded = tokens > std::numeric_limits<int64_t>::max() - 255
+                                    ? tokens
+                                    : ((tokens + 255) / 256) * 256;
+        return std::min(max_chunk, rounded);
+    };
     // The prompt path's chunk and the slots it borrows for its buffers: the requested chunk halved until it fits,
     // or with --prefill auto the largest of kAutoChunks whose buffers take at most kAutoLendPct % of the slots (a
     // lent slot's expert is streamed during the prompt and refilled after it; measured on a 12 GB card, 32K Q2_0
@@ -3229,7 +3272,9 @@ int main(int argc, char** argv) {
         sp.on_chunk = [&](const float* R_rows, int64_t T, int64_t p0, std::string& e) -> bool {
             std::vector<int32_t> nxt((size_t) T);
             for (int64_t t = 0; t < T; ++t) nxt[(size_t) t] = (int32_t) cur[(size_t) (p0 + t + 1)];
-            if (!mtp.prefill(R_rows, nxt.data(), T, p0, e)) return false;
+            // E-9: batched through the prompt path when it can (one GPU: a layer split's drafter is on the last stage)
+            const bool batched = !multi_gpu && sp.draft_kv(mtp, R_rows, nxt.data(), T, p0, e);
+            if (!e.empty() || (!batched && !mtp.prefill(R_rows, nxt.data(), T, p0, e))) return false;
             // progress for the server window: PP <position reached> <prompt tokens> <ms> <fresh tokens/s>
             const int64_t done = p0 + T;
             const double ms = std::chrono::duration<double, std::milli>(Clock::now() - pp_t0).count();
@@ -3855,12 +3900,21 @@ int main(int argc, char** argv) {
             // rounded up to 256), laid out in the last of the slots it may borrow
             auto lend = [&](int64_t tokens, std::string& e) -> bool {
                 if (lend_first < 0) return true;                       // its own buffers: nothing to lend
-                const int64_t want = std::min<int64_t>(o.prefill_chunk, (tokens + 255) / 256 * 256);
+                const int64_t want = request_chunk(tokens, o.prefill_chunk);
+                if (want <= 0) {
+                    e = "prefill: cannot lend buffers for an empty request segment";
+                    return false;
+                }
                 if (!lent_now.empty()) {
                     if (want <= lent_chunk) return true;
                     if (!refill(e)) return false;
                 }
-                const int32_t first = std::max<int32_t>(lend_first, (int32_t) (xcache.slots() - lend_slots(want)));
+                const int64_t slots = lend_slots(want);
+                if (slots <= 0 || slots > xcache.slots() - lend_first) {
+                    e = "prefill: request-sized buffers exceed the configured lend region";
+                    return false;
+                }
+                const int32_t first = std::max<int32_t>(lend_first, (int32_t) (xcache.slots() - slots));
                 if (want != sp.chunk() || first != lend_first_now) {
                     if (!sp.relayout(want, xcache.device_slot(first), lend_bytes(first), e)) return false;
                     lend_first_now = first;
@@ -4169,6 +4223,10 @@ int main(int argc, char** argv) {
                     if (st.kv_q4) {
                         const int64_t q4b = (int64_t) strata::kernels::kv_q4_bytes_per_head((int) qs.head_dim);
                         a = {{h ? st.host.k_q4 : st.k_q4, q4b}, {h ? st.host.v_q4 : st.v_q4, q4b}};
+                    } else if (st.kv_hybrid) {
+                        const int64_t q4b = (int64_t) strata::kernels::kv_q4_bytes_per_head((int) qs.head_dim);
+                        a = {{h ? st.host.k_q : st.k_q, kvb}, {h ? st.host.v_q4 : st.v_q4, q4b},
+                             {h ? st.host.k_scale : st.k_scale, scb}};
                     } else {
                         a = {{h ? st.host.k_q : st.k_q, kvb}, {h ? st.host.v_q : st.v_q, kvb},
                              {h ? st.host.k_scale : st.k_scale, scb}, {h ? st.host.v_scale : st.v_scale, scb}};
@@ -4268,19 +4326,22 @@ int main(int argc, char** argv) {
     std::FILE* final_r = o.dump_final_r.empty() ? nullptr : std::fopen(o.dump_final_r.c_str(), "wb");
     std::vector<float> final_r_host(final_r ? (size_t) (g.hc * g.n_embd) : 0);
     std::vector<std::pair<int32_t, int32_t>> lent;     // (residency index, slot) lent to the prompt path
+    const int64_t n_batched = (o.prefill_until > 0 && o.prefill_until < n_prompt - 1) ? o.prefill_until : n_prompt - 1;
     if (o.prefill_chunk > 0 && n_prompt > 1) {
         void* borrow = nullptr;
         uint64_t borrow_bytes = 0;
         if (!o.no_prefill_borrow && !host_res.empty() && d_res != nullptr) {
             int64_t chunk = o.prefill_chunk;
             int64_t k = plan_lend(chunk);             // auto: the largest chunk that fits; fixed: halved to fit
-            if (k > 0 && chunk > (n_prompt - 1 + 255) / 256 * 256) {   // no bigger than the prompt needs
-                chunk = std::max<int64_t>(256, (n_prompt - 1 + 255) / 256 * 256);
+            const int64_t request_sized = request_chunk(n_batched, chunk);
+            if (k > 0 && request_sized < chunk) {                     // no bigger than this prompt segment needs
+                chunk = request_sized;
                 k = lend_slots(chunk);
+                if (k + 128 > xcache.slots()) k = 0;
                 if (!o.prefill_auto) o.prefill_chunk = chunk;
             }
             if (o.prefill_auto) {
-                o.prefill_chunk = k > 0 ? chunk : 1024;
+                o.prefill_chunk = k > 0 ? chunk : request_chunk(n_batched, 1024);
                 std::fprintf(stderr, "strata generate: prompt chunk auto: %lld tokens\n", (long long) o.prefill_chunk);
             } else if (chunk != o.prefill_chunk) {
                 k = 0;                                 // a fixed chunk that does not fit: its own buffers, as before
@@ -4318,11 +4379,11 @@ int main(int argc, char** argv) {
                 // cell i pairs R_i with the token at i + 1 (every such token is in the prompt)
                 std::vector<int32_t> nxt((size_t) T);
                 for (int64_t t = 0; t < T; ++t) nxt[(size_t) t] = (int32_t) o.tokens[(size_t) (p0 + t + 1)];
-                return mtp.prefill(R_rows, nxt.data(), T, p0, e);
+                if (prefill.draft_kv(mtp, R_rows, nxt.data(), T, p0, e)) return true;   // E-9
+                return e.empty() && mtp.prefill(R_rows, nxt.data(), T, p0, e);
             };
         }
         const Clock::time_point tp0 = Clock::now();
-        const int64_t n_batched = (o.prefill_until > 0 && o.prefill_until < n_prompt - 1) ? o.prefill_until : n_prompt - 1;
         if (!prefill.run(o.tokens.data(), n_batched, 0, err)) {
             std::fprintf(stderr, "strata generate: %s\n", err.c_str());
             return 1;

tests/core/file_expert_source_test.cpp

@@ -0,0 +1,260 @@
+#include "strata/core/expert_source.hpp"
+#include "strata/kernels/cpu/expert.hpp"
+#include "strata/kernels/cpu/expert_layout.hpp"
+
+#if defined(STRATA_NATIVE_EXPERTS)
+#include "strata/kernels/cpu/native_expert.hpp"
+#include "ggml.h"
+#endif
+
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <limits>
+#include <stdexcept>
+#include <string>
+#include <system_error>
+#include <utility>
+#include <vector>
+
+namespace fs = std::filesystem;
+
+namespace {
+
+void require(bool ok, const std::string& message) {
+    if (!ok) throw std::runtime_error(message);
+}
+
+struct TempDirectory {
+    fs::path path;
+
+    TempDirectory() {
+        const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count();
+        path = fs::temp_directory_path() / ("strata-file-source-test-" + std::to_string(stamp));
+        fs::create_directories(path);
+    }
+
+    ~TempDirectory() {
+        std::error_code ignored;
+        fs::remove_all(path, ignored);
+    }
+};
+
+void create_pack(const fs::path& dir, uint64_t bytes, const std::vector<std::pair<uint64_t, char>>& markers = {}) {
+    const fs::path file = dir / "experts.bin";
+    {
+        std::ofstream out(file, std::ios::binary | std::ios::trunc);
+        require((bool) out, "could not create synthetic experts.bin");
+        if (bytes > 0) {
+            out.seekp((std::streamoff) (bytes - 1));
+            out.put('\0');
+        }
+        require((bool) out, "could not size synthetic experts.bin");
+    }
+    for (const auto& [offset, marker] : markers) {
+        require(offset < bytes, "synthetic marker lies beyond experts.bin");
+        std::fstream file_out(file, std::ios::binary | std::ios::in | std::ios::out);
+        require((bool) file_out, "could not open synthetic experts.bin for markers");
+        file_out.seekp((std::streamoff) offset);
+        file_out.put(marker);
+        require((bool) file_out, "could not write synthetic expert marker");
+    }
+}
+
+void check_file_size_rejection(strata::core::FileExpertSource& source, const fs::path& dir, int64_t layers,
+                               int64_t experts, uint64_t expected_bytes) {
+    std::string err;
+    source.close();
+    fs::resize_file(dir / "experts.bin", expected_bytes - 1);
+    require(!source.open(dir.string(), layers, experts, err), "a truncated expert file was accepted");
+    require(!source.mapped() && !err.empty(), "truncated-file rejection left the source mapped or unreported");
+
+    fs::resize_file(dir / "experts.bin", expected_bytes + 1);
+    err.clear();
+    require(!source.open(dir.string(), layers, experts, err), "an oversized expert file was accepted");
+    require(!source.mapped() && !err.empty(), "oversized-file rejection left the source mapped or unreported");
+}
+
+void test_canonical_layout() {
+    using namespace strata::core;
+    using namespace strata::kernels::cpu;
+    constexpr int64_t layers = 2;
+    constexpr int64_t experts = 3;
+    const uint64_t layer_bytes = (uint64_t) experts * BLOB;
+    const uint64_t total = (uint64_t) layers * layer_bytes;
+    TempDirectory dir;
+
+    std::string err;
+    const bool layout_ok = expert_layout_load(dir.path.string(), layers, experts, err);
+    require(layout_ok, "could not load canonical layout: " + err);
+    create_pack(dir.path, total, {{0, 'a'}, {(uint64_t) BLOB, 'b'}, {layer_bytes, 'c'},
+                                  {layer_bytes + (uint64_t) BLOB, 'd'}});
+
+    FileExpertSource source;
+    bool opened = source.open(dir.path.string(), layers, experts, err);
+    require(opened, "could not map canonical pack: " + err);
+    require(source.blobs() == layers * experts, "canonical blob count is wrong");
+    const uint8_t* first = source.blob(0, 0);
+    const uint8_t* second = source.blob(0, 1);
+    const uint8_t* next_layer = source.blob(1, 0);
+    require(first && second && next_layer, "valid canonical blob lookup failed");
+    require(second - first == (ptrdiff_t) BLOB && next_layer - first == (ptrdiff_t) layer_bytes,
+            "canonical expert or layer stride is wrong");
+    require(first[0] == 'a' && second[0] == 'b' && next_layer[0] == 'c',
+            "canonical blob lookup returned bytes from the wrong expert");
+    require(source.blob(-1, 0) == nullptr && source.blob(0, -1) == nullptr &&
+                source.blob(layers, 0) == nullptr && source.blob(0, experts) == nullptr,
+            "canonical bounds check accepted an invalid layer or expert");
+    require(source.reads() == 3, "invalid canonical lookups changed the read count");
+
+    check_file_size_rejection(source, dir.path, layers, experts, total);
+    create_pack(dir.path, total, {{layer_bytes, 'c'}});
+    err.clear();
+    opened = source.open(dir.path.string(), layers, experts, err);
+    require(opened, "source failed to reopen after size errors: " + err);
+    const uint8_t* reopened = source.blob(1, 0);
+    require(reopened && source.reads() == 1 && reopened[0] == 'c', "reopened canonical source kept stale state");
+    source.close();
+}
+
+#if defined(STRATA_NATIVE_EXPERTS)
+void test_native_variable_layout() {
+    using namespace strata::core;
+    using namespace strata::kernels::cpu;
+    constexpr int64_t layers = 2;
+    constexpr int64_t experts = 3;
+    constexpr int64_t embedding = 2560;
+    constexpr int64_t feed_forward = 640;
+    TempDirectory dir;
+
+    NativeFmt first_fmt, second_fmt;
+    std::string err;
+    const bool first_ok = native_fmt(GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ4_NL, embedding, feed_forward, first_fmt, err);
+    require(first_ok, "IQ3_XXS/IQ4_NL synthetic format is unavailable: " + err);
+    err.clear();
+    const bool second_ok = native_fmt(GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_NL, embedding, feed_forward, second_fmt, err);
+    require(second_ok, "IQ2_XS/IQ4_NL synthetic format is unavailable: " + err);
+    require(first_fmt.bytes != second_fmt.bytes, "chosen native formats do not exercise variable layer sizes");
+
+    const uint64_t layer0_bytes = (uint64_t) first_fmt.bytes * experts;
+    const uint64_t total = layer0_bytes + (uint64_t) second_fmt.bytes * experts;
+    {
+        std::ofstream metadata(dir.path / "native_experts.txt");
+        require((bool) metadata, "could not create synthetic native_experts.txt");
+        metadata << "0 " << GGML_TYPE_IQ3_XXS << ' ' << GGML_TYPE_IQ4_NL << " 0 " << first_fmt.bytes << '\n';
+        metadata << "1 " << GGML_TYPE_IQ2_XS << ' ' << GGML_TYPE_IQ4_NL << ' ' << layer0_bytes << ' '
+                 << second_fmt.bytes << '\n';
+        require((bool) metadata, "could not write synthetic native_experts.txt");
+    }
+    create_pack(dir.path, total, {{0, 'a'}, {(uint64_t) first_fmt.bytes, 'b'},
+                                  {layer0_bytes, 'c'}, {layer0_bytes + (uint64_t) second_fmt.bytes, 'd'}});
+
+    const bool layout_ok = expert_layout_load(dir.path.string(), layers, experts, err);
+    require(layout_ok, "could not load synthetic native layout: " + err);
+    FileExpertSource source;
+    bool opened = source.open(dir.path.string(), layers, experts, err);
+    require(opened, "could not map native pack: " + err);
+    require(source.blobs() == layers * experts, "native blob count is wrong");
+    const uint8_t* first = source.blob(0, 0);
+    const uint8_t* second = source.blob(0, 1);
+    const uint8_t* next_layer = source.blob(1, 0);
+    const uint8_t* next_layer_second = source.blob(1, 1);
+    require(first && second && next_layer && next_layer_second, "valid native blob lookup failed");
+    require(second - first == (ptrdiff_t) first_fmt.bytes && next_layer - first == (ptrdiff_t) layer0_bytes &&
+                next_layer_second - next_layer == (ptrdiff_t) second_fmt.bytes,
+            "native per-layer expert stride is wrong");
+    require(first[0] == 'a' && second[0] == 'b' && next_layer[0] == 'c' && next_layer_second[0] == 'd',
+            "native blob lookup returned bytes from the wrong expert");
+    require(source.blob(-1, 0) == nullptr && source.blob(0, -1) == nullptr &&
+                source.blob(layers, 0) == nullptr && source.blob(0, experts) == nullptr,
+            "native bounds check accepted an invalid layer or expert");
+    require(source.reads() == 4, "invalid native lookups changed the read count");
+
+    check_file_size_rejection(source, dir.path, layers, experts, total);
+    create_pack(dir.path, total, {{layer0_bytes, 'c'}});
+    err.clear();
+    opened = source.open(dir.path.string(), layers, experts, err);
+    require(opened, "source failed to reopen after size errors: " + err);
+    const uint8_t* reopened = source.blob(1, 0);
+    require(reopened && source.reads() == 1 && reopened[0] == 'c', "reopened native source kept stale state");
+    source.close();
+}
+#endif
+
+void test_complement_plan() {
+    using namespace strata::core::detail;
+    std::vector<uint64_t> offsets;
+    uint64_t bytes = 0;
+    std::string error;
+    require(make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}, {1, 2}}, {}, offsets, bytes, error), error);
+    require(bytes == 16 && offsets == std::vector<uint64_t>{0, kNoCacheComplement, 3, 6, 11, kNoCacheComplement},
+            "wrong compact offsets for variable native layer sizes");
+    const uint8_t resident[16] = {}, mapped[5] = {};
+    require(cache_complement_blob_or_fallback(1, offsets, resident, mapped) == mapped,
+            "GPU-resident expert lost mmap fallback");
+    require(cache_complement_blob_or_fallback(4, offsets, resident, mapped) == resident + 11,
+            "CPU miss did not use resident complement");
+    require(!make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}, {0, 1}}, {}, offsets, bytes, error)
+            && offsets.empty() && bytes == 0, "duplicate pair accepted");
+    require(!make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}}, {{0, 1}}, offsets, bytes, error),
+            "overlapping tiers accepted");
+    require(!make_cache_complement_plan(2, 3, {3, 5}, {{2, 0}}, {}, offsets, bytes, error),
+            "out-of-range pair accepted");
+    require(!make_cache_complement_plan(2, 3, {0, 5}, {}, {}, offsets, bytes, error), "zero-size layer accepted");
+}
+
+void test_cgroup_memory_budget() {
+    using namespace strata::core::detail;
+    constexpr uint64_t GiB = 1ull << 30;
+    uint64_t bytes = 0;
+
+    CgroupMemoryStat clean_cache{40 * GiB, 32 * GiB, 0, 0, true};
+    require(cgroup_available_bytes(56 * GiB, clean_cache, bytes), "valid cgroup memory.stat was rejected");
+    require(bytes == 48 * GiB, "clean inactive file cache was not credited against the cgroup cap");
+
+    CgroupMemoryStat dirty_cache{40 * GiB, 32 * GiB, 4 * GiB, 2 * GiB, true};
+    require(cgroup_available_bytes(56 * GiB, dirty_cache, bytes), "valid dirty-cache counters were rejected");
+    require(bytes == 42 * GiB, "dirty/writeback pages were incorrectly counted as reclaimable");
+
+    CgroupMemoryStat oversized_inactive{10 * GiB, 20 * GiB, 0, 0, true};
+    require(cgroup_available_bytes(12 * GiB, oversized_inactive, bytes),
+            "valid oversized inactive-file counter was rejected");
+    require(bytes == 12 * GiB, "inactive-file accounting exceeded charged current usage");
+
+    const uint64_t max = std::numeric_limits<uint64_t>::max();
+    CgroupMemoryStat large_counters{max, max, max - 1, max, true};
+    require(cgroup_available_bytes(max, large_counters, bytes),
+            "large memory.stat counters were rejected");
+    require(bytes == 0, "dirty/writeback subtraction overflowed or escaped the usage cap");
+
+    CgroupMemoryStat usage_over_limit{60 * GiB, 0, 0, 0, true};
+    require(cgroup_available_bytes(56 * GiB, usage_over_limit, bytes),
+            "valid over-limit memory counters were rejected");
+    require(bytes == 0, "over-limit cgroup usage produced a positive budget");
+
+    CgroupMemoryStat missing_stat{};
+    bytes = 123;
+    require(!cgroup_available_bytes(56 * GiB, missing_stat, bytes) && bytes == 0,
+            "missing memory.stat counters did not fail closed");
+}
+
+}  // namespace
+
+int main() {
+    try {
+        test_complement_plan();
+        test_cgroup_memory_budget();
+        test_canonical_layout();
+#if defined(STRATA_NATIVE_EXPERTS)
+        test_native_variable_layout();
+#endif
+        std::cout << "file_expert_source_test: PASS\n";
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << "file_expert_source_test: " << error.what() << '\n';
+        return 1;
+    }
+}

tests/hip/expert_cache_staging.cpp

@@ -0,0 +1,34 @@
+#include "strata/core/expert_cache.hpp"
+#include <cstdio>
+#include <vector>
+
+int main() {
+    strata::core::ExpertCache cache;
+    std::string err;
+    auto check = [&](bool ok) {
+        if (!ok) std::fprintf(stderr, "%s\n", err.c_str());
+        return ok;
+    };
+    // open_sized first allocates byte slots, then grows to the largest expert.
+    // Exercise that growth and repeated reuse with distinct pageable payloads.
+    for (int reopen = 0; reopen < 2; ++reopen) {
+        if (!check(cache.open_sized({4096, 8192, 16384}, 1, 3, err))) return 1;
+        for (int round = 0; round < 8; ++round) {
+            for (int slot = 0; slot < 3; ++slot) {
+                std::vector<uint8_t> payload((size_t) 4096 << slot);
+                for (size_t i = 0; i < payload.size(); ++i)
+                    payload[i] = (uint8_t) (i * 17 + round * 11 + slot + reopen);
+                if (!check(cache.fill_slot_blocking(slot, payload.data(), err, payload.size())) ||
+                    !check(cache.verify_slot(slot, payload.data(), err, payload.size()))) return 1;
+            }
+        }
+        if (cache.fill_slot_blocking(0, nullptr, err)) return 1;
+        cache.close();
+    }
+    if (!check(cache.open(2, 1, 2, 1024, err))) return 1;
+    std::vector<uint8_t> payload(1024, 73);
+    if (!check(cache.fill_slot_blocking(1, payload.data(), err)) ||
+        !check(cache.verify_slot(1, payload.data(), err))) return 1;
+    std::puts("HIP expert staging: variable slots, repeated payloads, close/reopen passed");
+    return 0;
+}

tests/hip/handoff.cpp

@@ -0,0 +1,122 @@
+// Real asynchronous CPU/GPU handoff and graph replay, including changing payloads.
+#include <hip/hip_runtime.h>
+#include "strata/kernels/elementwise.hpp"
+#include <atomic>
+#include <chrono>
+#include <cstdio>
+#include <thread>
+#include <vector>
+#define CHECK(x) do {auto e=(x);if(e!=hipSuccess){std::fprintf(stderr,"%s: %s\n",#x,hipGetErrorString(e));return 2;}} while(0)
+int main(){
+ constexpr int N=4096, rounds=100;
+ float *host=nullptr,*mapped=nullptr,*device=nullptr;
+ uint32_t *flag=nullptr,*seq=nullptr,*dflag=nullptr,*dseq=nullptr;
+ CHECK(hipHostMalloc((void**)&host,N*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&flag,4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&seq,4,hipHostMallocMapped));
+ CHECK(hipHostGetDevicePointer((void**)&mapped,host,0));
+ CHECK(hipHostGetDevicePointer((void**)&dflag,flag,0));
+ CHECK(hipHostGetDevicePointer((void**)&dseq,seq,0));
+ CHECK(hipMalloc((void**)&device,N*4));
+ hipStream_t stream;CHECK(hipStreamCreateWithFlags(&stream,hipStreamNonBlocking));
+ hipGraph_t graph;hipGraphExec_t exec;
+ *flag=0;*seq=1;
+ CHECK(hipStreamBeginCapture(stream,hipStreamCaptureModeThreadLocal));
+ strata::kernels::doorbell_wait(dflag,dseq,(void*)stream);
+ strata::kernels::copy_from_mapped(device,mapped,N,(void*)stream);
+ strata::kernels::scale_inplace(device,N,2.0f,(void*)stream);
+ CHECK(hipStreamEndCapture(stream,&graph));CHECK(hipGraphInstantiateWithFlags(&exec,graph,0));
+ std::vector<float> got(N);
+ for(int r=1;r<=rounds;r++){
+  *seq=r;
+  CHECK(hipGraphLaunch(exec,stream));
+  // Producer deliberately arrives after the GPU consumer; a missing wait must fail.
+  std::this_thread::sleep_for(std::chrono::microseconds(50+(r%17)*10));
+  for(int i=0;i<N;i++)host[i]=(float)(r*10000+i);
+  __atomic_store_n(flag,(uint32_t)r,__ATOMIC_RELEASE);
+  CHECK(hipStreamSynchronize(stream));
+  CHECK(hipMemcpy(got.data(),device,N*4,hipMemcpyDeviceToHost));
+  for(int i=0;i<N;i++)if(got[i]!=(float)(r*10000+i)*2.0f){std::fprintf(stderr,"stale payload r=%d i=%d got=%f\n",r,i,got[i]);return 1;}
+ }
+ CHECK(hipGraphExecDestroy(exec));CHECK(hipGraphDestroy(graph));
+ // The production `moe_route` fallback records three device-to-device copies into
+ // mapped pinned buffers, then launches `doorbell_ring` as a separate graph node.
+ // Prove that seeing the increment publishes the preceding payload to the CPU.
+ float *copy_out=nullptr,*mapped_copy_out=nullptr,*copy_weights=nullptr,*mapped_copy_weights=nullptr;
+ int32_t *copy_ids=nullptr,*mapped_copy_ids=nullptr,*copy_ids_device=nullptr;
+ float *copy_weights_device=nullptr;
+ uint32_t *copy_seq=nullptr,*device_copy_seq=nullptr;
+ CHECK(hipHostMalloc((void**)&copy_out,N*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&copy_ids,10*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&copy_weights,10*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&copy_seq,4,hipHostMallocMapped));
+ CHECK(hipHostGetDevicePointer((void**)&mapped_copy_out,copy_out,0));
+ CHECK(hipHostGetDevicePointer((void**)&mapped_copy_ids,copy_ids,0));
+ CHECK(hipHostGetDevicePointer((void**)&mapped_copy_weights,copy_weights,0));
+ CHECK(hipHostGetDevicePointer((void**)&device_copy_seq,copy_seq,0));
+ CHECK(hipMalloc((void**)&copy_ids_device,10*4));
+ CHECK(hipMalloc((void**)&copy_weights_device,10*4));
+ *copy_seq=0;
+ std::vector<int32_t> copy_route_ids(10);std::vector<float> copy_route_weights(10);
+ CHECK(hipStreamBeginCapture(stream,hipStreamCaptureModeThreadLocal));
+ CHECK(hipMemcpyAsync(mapped_copy_out,device,N*4,hipMemcpyDeviceToDevice,stream));
+ CHECK(hipMemcpyAsync(mapped_copy_ids,copy_ids_device,10*4,hipMemcpyDeviceToDevice,stream));
+ CHECK(hipMemcpyAsync(mapped_copy_weights,copy_weights_device,10*4,hipMemcpyDeviceToDevice,stream));
+ strata::kernels::doorbell_ring(device_copy_seq,(void*)stream);
+ CHECK(hipStreamEndCapture(stream,&graph));CHECK(hipGraphInstantiateWithFlags(&exec,graph,0));
+ for(int r=1;r<=rounds;r++){
+  for(int i=0;i<N;i++)got[i]=(float)(r*20000+i);
+  for(int i=0;i<10;i++){copy_route_ids[i]=r*10+i;copy_route_weights[i]=r+i*.125f;}
+  CHECK(hipMemcpyAsync(device,got.data(),N*4,hipMemcpyHostToDevice,stream));
+  CHECK(hipMemcpyAsync(copy_ids_device,copy_route_ids.data(),10*4,hipMemcpyHostToDevice,stream));
+  CHECK(hipMemcpyAsync(copy_weights_device,copy_route_weights.data(),10*4,hipMemcpyHostToDevice,stream));
+  CHECK(hipGraphLaunch(exec,stream));
+  const auto deadline=std::chrono::steady_clock::now()+std::chrono::seconds(2);
+  // Do not query or synchronize the stream before validating the mapped payload.
+  while(__atomic_load_n(copy_seq,__ATOMIC_ACQUIRE)!=(uint32_t)r){
+   if(std::chrono::steady_clock::now()>deadline){std::fprintf(stderr,"separate copy/ring timeout\n");return 10;}
+   std::this_thread::yield();
+  }
+  for(int i=0;i<N;i++)if(copy_out[i]!=(float)(r*20000+i)){std::fprintf(stderr,"stale separate-copy payload r=%d i=%d\n",r,i);return 11;}
+  for(int i=0;i<10;i++)if(copy_ids[i]!=copy_route_ids[i]||copy_weights[i]!=copy_route_weights[i]){std::fprintf(stderr,"stale separate-copy route payload r=%d i=%d\n",r,i);return 12;}
+ }
+ CHECK(hipStreamSynchronize(stream));
+ CHECK(hipGraphExecDestroy(exec));CHECK(hipGraphDestroy(graph));
+ float *published=nullptr,*dpublished=nullptr,*weights=nullptr,*dweights=nullptr,*gw=nullptr;
+ int32_t *ids=nullptr,*dids=nullptr,*gi=nullptr;
+ CHECK(hipHostMalloc((void**)&published,N*4,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&weights,40,hipHostMallocMapped));
+ CHECK(hipHostMalloc((void**)&ids,40,hipHostMallocMapped));
+ CHECK(hipHostGetDevicePointer((void**)&dpublished,published,0));
+ CHECK(hipHostGetDevicePointer((void**)&dweights,weights,0));
+ CHECK(hipHostGetDevicePointer((void**)&dids,ids,0));
+ CHECK(hipMalloc((void**)&gw,40));CHECK(hipMalloc((void**)&gi,40));
+ std::vector<int32_t> ids_src(10);std::vector<float> ws(10);
+ *flag=0;
+ CHECK(hipStreamBeginCapture(stream,hipStreamCaptureModeThreadLocal));
+ strata::kernels::doorbell_publish(device,gi,gw,N,10,dpublished,dids,dweights,dflag,(void*)stream);
+ CHECK(hipStreamEndCapture(stream,&graph));CHECK(hipGraphInstantiateWithFlags(&exec,graph,0));
+ for(int r=1;r<=rounds;r++){
+  for(int i=0;i<N;i++)got[i]=(float)(r*10000+i);
+  for(int i=0;i<10;i++){ids_src[i]=r*10+i;ws[i]=r+i*.125f;}
+  CHECK(hipMemcpyAsync(device,got.data(),N*4,hipMemcpyHostToDevice,stream));
+  CHECK(hipMemcpyAsync(gi,ids_src.data(),40,hipMemcpyHostToDevice,stream));
+  CHECK(hipMemcpyAsync(gw,ws.data(),40,hipMemcpyHostToDevice,stream));
+  CHECK(hipGraphLaunch(exec,stream));
+  const auto deadline=std::chrono::steady_clock::now()+std::chrono::seconds(2);
+  // No driver query or stream sync may make the payload visible to this observer.
+  while(__atomic_load_n(flag,__ATOMIC_ACQUIRE)!=(uint32_t)r){
+   if(std::chrono::steady_clock::now()>deadline){std::fprintf(stderr,"publish timeout\n");return 7;}
+   std::this_thread::yield();
+  }
+  for(int i=0;i<N;i++)if(published[i]!=got[i]){std::fprintf(stderr,"stale GPU publication\n");return 8;}
+  for(int i=0;i<10;i++)if(ids[i]!=ids_src[i] || weights[i]!=ws[i])return 9;
+  CHECK(hipStreamSynchronize(stream));
+ }
+ CHECK(hipGraphExecDestroy(exec));CHECK(hipGraphDestroy(graph));CHECK(hipStreamDestroy(stream));
+ CHECK(hipFree(gi));CHECK(hipFree(gw));CHECK(hipHostFree(published));CHECK(hipHostFree(ids));CHECK(hipHostFree(weights));
+ CHECK(hipFree(device));CHECK(hipHostFree(host));CHECK(hipHostFree(flag));CHECK(hipHostFree(seq));
+ CHECK(hipFree(copy_weights_device));CHECK(hipFree(copy_ids_device));
+ CHECK(hipHostFree(copy_out));CHECK(hipHostFree(copy_ids));CHECK(hipHostFree(copy_weights));CHECK(hipHostFree(copy_seq));
+ std::puts("PASS asynchronous mapped-memory handoff: 100 CPU-to-GPU waits, 100 separate copy/rings, 100 fused publishes; 1232800 values checked");
+}

tests/hip/intrinsics.cpp

@@ -0,0 +1,167 @@
+// Numerical parity for the CUDA intrinsics supplied by the HIP compatibility layer.
+#include <hip/hip_runtime.h>
+#include "strata/hip_compat/intrinsics.hpp"
+
+#include <cstdint>
+#include <cstdio>
+#include <limits>
+
+#define CHECK(call)                                                                                                  \
+    do {                                                                                                             \
+        const hipError_t error = (call);                                                                             \
+        if (error != hipSuccess) {                                                                                   \
+            std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error));                                     \
+            return 2;                                                                                                \
+        }                                                                                                            \
+    } while (0)
+
+namespace {
+
+constexpr int kLanes = 32;
+constexpr int kIntegerResults = 7;
+
+__global__ void intrinsic_probe(int* integer_results, float* float_shuffle, double* double_shuffle,
+                                unsigned* ballot, const uint32_t* dot_inputs) {
+    const int lane = static_cast<int>(threadIdx.x);
+    constexpr unsigned mask = 0xffffffffu;
+    const uint32_t a = 0x80ff017fu;
+    const uint32_t b = 0x0203fe81u;
+
+    integer_results[lane * kIntegerResults + 0] = __dp4a(static_cast<int>(a), static_cast<int>(b), 7);
+    integer_results[lane * kIntegerResults + 1] = __dp4a(0x7f7f7f7f, 0x7f7f7f7f, 0x7fffffff);
+    integer_results[lane * kIntegerResults + 2] = __vsub4(0x00ff0102, 0x01010103);
+    integer_results[lane * kIntegerResults + 3] = __vsub4(0x00000000, 0x01020304);
+    integer_results[lane * kIntegerResults + 4] = __vsubss4(0x807f0080, 0x017f0180);
+    integer_results[lane * kIntegerResults + 5] = __vcmpne4(0x00ff0102, 0x01010103);
+    integer_results[lane * kIntegerResults + 6] =
+        __dp4a(static_cast<int>(dot_inputs[lane * 2]), static_cast<int>(dot_inputs[lane * 2 + 1]),
+               0x7fffffff - lane);
+
+    float_shuffle[lane * 4 + 0] = __shfl_xor_sync(mask, static_cast<float>(lane), 1);
+    float_shuffle[lane * 4 + 1] = static_cast<float>(__shfl_down_sync(mask, lane, 4, 8));
+    float_shuffle[lane * 4 + 2] = static_cast<float>(__shfl_up_sync(mask, lane, 1));
+    float_shuffle[lane * 4 + 3] = static_cast<float>(__shfl_sync(mask, lane, 0));
+    double_shuffle[lane] = __shfl_xor_sync(mask, static_cast<double>(lane) + 0.25, 1);
+
+    const unsigned hit = __ballot_sync(mask, (lane & 1) == 0);
+    if (lane == 0) {
+        *ballot = hit;
+        __nanosleep(100);
+    }
+}
+
+int signed_byte(uint32_t word, int lane) {
+    const unsigned value = (word >> (lane * 8)) & 0xffu;
+    return value < 0x80u ? static_cast<int>(value) : static_cast<int>(value) - 0x100;
+}
+
+int dp4a_reference(uint32_t a, uint32_t b, int c) {
+    int64_t sum = c;
+    for (int lane = 0; lane < 4; ++lane) sum += signed_byte(a, lane) * signed_byte(b, lane);
+    return static_cast<int32_t>(static_cast<uint32_t>(sum));
+}
+
+uint32_t byte_sub_reference(uint32_t a, uint32_t b) {
+    uint32_t out = 0;
+    for (int lane = 0; lane < 4; ++lane)
+        out |= ((((a >> (lane * 8)) & 0xffu) - ((b >> (lane * 8)) & 0xffu)) & 0xffu) << (lane * 8);
+    return out;
+}
+
+uint32_t signed_saturating_sub_reference(uint32_t a, uint32_t b) {
+    uint32_t out = 0;
+    for (int lane = 0; lane < 4; ++lane) {
+        int value = signed_byte(a, lane) - signed_byte(b, lane);
+        if (value < -128) value = -128;
+        if (value > 127) value = 127;
+        out |= (static_cast<uint32_t>(value) & 0xffu) << (lane * 8);
+    }
+    return out;
+}
+
+uint32_t not_equal_reference(uint32_t a, uint32_t b) {
+    uint32_t out = 0;
+    for (int lane = 0; lane < 4; ++lane)
+        if (((a >> (lane * 8)) & 0xffu) != ((b >> (lane * 8)) & 0xffu)) out |= 0xffu << (lane * 8);
+    return out;
+}
+
+}  // namespace
+
+int main() {
+    int device = 0;
+    CHECK(hipGetDevice(&device));
+    hipDeviceProp_t properties{};
+    CHECK(hipGetDeviceProperties(&properties, device));
+    if (properties.warpSize != kLanes) {
+        std::fprintf(stderr, "expected wave32, got wave%d\n", properties.warpSize);
+        return 1;
+    }
+
+    int* device_integers = nullptr;
+    float* device_float_shuffle = nullptr;
+    double* device_double_shuffle = nullptr;
+    unsigned* device_ballot = nullptr;
+    uint32_t* device_dot_inputs = nullptr;
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_integers), kLanes * kIntegerResults * sizeof(int)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_float_shuffle), kLanes * 4 * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_double_shuffle), kLanes * sizeof(double)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_ballot), sizeof(unsigned)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_dot_inputs), kLanes * 2 * sizeof(uint32_t)));
+
+    uint32_t dot_inputs[kLanes * 2]{};
+    for (int lane = 0; lane < kLanes; ++lane) {
+        dot_inputs[lane * 2] = 0x80ff017fu ^ (0x01010101u * static_cast<uint32_t>(lane));
+        dot_inputs[lane * 2 + 1] = 0x0203fe81u + (0x11111111u * static_cast<uint32_t>(lane));
+    }
+    CHECK(hipMemcpy(device_dot_inputs, dot_inputs, sizeof(dot_inputs), hipMemcpyHostToDevice));
+
+    hipLaunchKernelGGL(intrinsic_probe, dim3(1), dim3(kLanes), 0, 0,
+                       device_integers, device_float_shuffle, device_double_shuffle, device_ballot, device_dot_inputs);
+    CHECK(hipDeviceSynchronize());
+
+    int integers[kLanes * kIntegerResults]{};
+    float float_shuffle[kLanes * 4]{};
+    double double_shuffle[kLanes]{};
+    unsigned ballot = 0;
+    CHECK(hipMemcpy(integers, device_integers, sizeof(integers), hipMemcpyDeviceToHost));
+    CHECK(hipMemcpy(float_shuffle, device_float_shuffle, sizeof(float_shuffle), hipMemcpyDeviceToHost));
+    CHECK(hipMemcpy(double_shuffle, device_double_shuffle, sizeof(double_shuffle), hipMemcpyDeviceToHost));
+    CHECK(hipMemcpy(&ballot, device_ballot, sizeof(ballot), hipMemcpyDeviceToHost));
+
+    constexpr uint32_t a = 0x80ff017fu, b = 0x0203fe81u;
+    const uint32_t expected_integer[kIntegerResults - 1] = {
+        static_cast<uint32_t>(dp4a_reference(a, b, 7)),
+        static_cast<uint32_t>(dp4a_reference(0x7f7f7f7fu, 0x7f7f7f7fu, std::numeric_limits<int>::max())),
+        byte_sub_reference(0x00ff0102u, 0x01010103u),
+        byte_sub_reference(0x00000000u, 0x01020304u),
+        signed_saturating_sub_reference(0x807f0080u, 0x017f0180u),
+        not_equal_reference(0x00ff0102u, 0x01010103u),
+    };
+    bool ok = ballot == 0x55555555u;
+    for (int lane = 0; lane < kLanes; ++lane) {
+        for (int i = 0; i < kIntegerResults - 1; ++i)
+            ok = ok && static_cast<uint32_t>(integers[lane * kIntegerResults + i]) == expected_integer[i];
+        ok = ok && static_cast<uint32_t>(integers[lane * kIntegerResults + kIntegerResults - 1]) ==
+                     static_cast<uint32_t>(dp4a_reference(dot_inputs[lane * 2], dot_inputs[lane * 2 + 1],
+                                                          0x7fffffff - lane));
+        ok = ok && float_shuffle[lane * 4 + 0] == static_cast<float>(lane ^ 1);
+        const int down = lane % 8 < 4 ? lane + 4 : lane;
+        ok = ok && float_shuffle[lane * 4 + 1] == static_cast<float>(down);
+        ok = ok && float_shuffle[lane * 4 + 2] == static_cast<float>(lane == 0 ? 0 : lane - 1);
+        ok = ok && float_shuffle[lane * 4 + 3] == 0.0f;
+        ok = ok && double_shuffle[lane] == static_cast<double>(lane ^ 1) + 0.25;
+    }
+
+    CHECK(hipFree(device_ballot));
+    CHECK(hipFree(device_dot_inputs));
+    CHECK(hipFree(device_double_shuffle));
+    CHECK(hipFree(device_float_shuffle));
+    CHECK(hipFree(device_integers));
+    if (!ok) {
+        std::fprintf(stderr, "HIP intrinsic parity failed (ballot 0x%08x)\n", ballot);
+        return 1;
+    }
+    std::puts("HIP intrinsics parity OK: dynamic signed dot4/overflow, packed integer boundaries, wave32 shuffles, ballot and sleep");
+    return 0;
+}

tests/hip/native_qsa_score.cpp

@@ -0,0 +1,95 @@
+// Device parity for the scalar HIP implementation of the optional native QSA scorer.
+#include <hip/hip_runtime.h>
+#include "strata/kernels/native_qsa_score.hpp"
+
+#include <cmath>
+#include <cstdio>
+#include <vector>
+
+#define CHECK(call)                                                                                                  \
+    do {                                                                                                             \
+        const hipError_t error = (call);                                                                             \
+        if (error != hipSuccess) {                                                                                   \
+            std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error));                                     \
+            return 2;                                                                                                \
+        }                                                                                                            \
+    } while (0)
+
+int main() {
+    using namespace strata::kernels;
+    constexpr int kDim = 128, kHeads = 4, kBlock = 4;
+    constexpr int kMaxCells = 8, kMaxBlocks = 3, kCells = 7;
+    const QsaShapes shapes = qsa_real_shapes();
+
+    std::vector<float> pooled(kMaxBlocks * kDim), query(kHeads * kDim), bias(kMaxBlocks),
+        scores(kMaxCells, -123.0f), expected(kMaxCells, -123.0f);
+    const float head_sign[kHeads] = {1.0f, -1.0f, 1.0f, -1.0f};
+    for (int row = 0; row < kMaxBlocks; ++row)
+        for (int d = 0; d < kDim; ++d)
+            pooled[row * kDim + d] = static_cast<float>(row + 1) * (1.0f + static_cast<float>(d % 7) / 8.0f) / 128.0f;
+    for (int head = 0; head < kHeads; ++head)
+        for (int d = 0; d < kDim; ++d)
+            query[head * kDim + d] = head_sign[head] * (1.0f + static_cast<float>((head + d) % 5) / 16.0f) / 16.0f;
+    for (int row = 0; row < kMaxBlocks; ++row) bias[row] = (static_cast<float>(row) - 1.0f) / 8.0f;
+
+    int32_t step[kStepCount] = {kCells - 1, kCells, kCells / kBlock, static_cast<int32_t>(kCells)};
+    bool saw_positive_head = false, saw_negative_head = false;
+    for (int row = 0; row <= step[kStepNBid]; ++row) {
+        float sum = 0.0f;
+        for (int head = 0; head < kHeads; ++head) {
+            float dot = 0.0f;
+            for (int d = 0; d < kDim; ++d)
+                dot = std::fma(pooled[row * kDim + d], query[head * kDim + d], dot);
+            saw_positive_head = saw_positive_head || dot > 0.0f;
+            saw_negative_head = saw_negative_head || dot < 0.0f;
+            sum = sum + (dot > 0.0f ? dot : 0.0f);
+        }
+        sum = sum + bias[row];
+        if (row == step[kStepNBid] && kCells % kBlock != 0) sum = sum + 1.0e9f;
+        const int lo = row * kBlock;
+        const int hi = (row + 1) * kBlock < kCells ? (row + 1) * kBlock : kCells;
+        for (int cell = lo; cell < hi; ++cell) expected[cell] = sum;
+    }
+    if (!saw_positive_head || !saw_negative_head) {
+        std::fprintf(stderr, "fixture did not exercise both positive and negative head scores\n");
+        return 1;
+    }
+
+    float *device_pooled = nullptr, *device_query = nullptr, *device_bias = nullptr, *device_scores = nullptr;
+    int32_t* device_step = nullptr;
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_pooled), pooled.size() * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_query), query.size() * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_bias), bias.size() * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_scores), scores.size() * sizeof(float)));
+    CHECK(hipMalloc(reinterpret_cast<void**>(&device_step), sizeof(step)));
+    CHECK(hipMemcpy(device_pooled, pooled.data(), pooled.size() * sizeof(float), hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(device_query, query.data(), query.size() * sizeof(float), hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(device_bias, bias.data(), bias.size() * sizeof(float), hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(device_scores, scores.data(), scores.size() * sizeof(float), hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(device_step, step, sizeof(step), hipMemcpyHostToDevice));
+
+    hipStream_t stream;
+    CHECK(hipStreamCreate(&stream));
+    native_qsa_score(device_pooled, device_query, device_bias, shapes, device_step, kMaxBlocks, kMaxCells,
+                     device_scores, static_cast<void*>(stream));
+    CHECK(hipStreamSynchronize(stream));
+    CHECK(hipMemcpy(scores.data(), device_scores, scores.size() * sizeof(float), hipMemcpyDeviceToHost));
+
+    bool ok = true;
+    for (int cell = 0; cell < kMaxCells; ++cell) {
+        if (scores[cell] != expected[cell]) {
+            std::fprintf(stderr, "score[%d] got %.9g, expected %.9g\n", cell, scores[cell], expected[cell]);
+            ok = false;
+        }
+    }
+
+    CHECK(hipStreamDestroy(stream));
+    CHECK(hipFree(device_step));
+    CHECK(hipFree(device_scores));
+    CHECK(hipFree(device_bias));
+    CHECK(hipFree(device_query));
+    CHECK(hipFree(device_pooled));
+    if (!ok) return 1;
+    std::puts("native QSA HIP scalar parity OK: FMA order, per-head ReLU, bias, partial tail and padding");
+    return 0;
+}

tests/hip/ple_iq4.cpp

@@ -0,0 +1,202 @@
+// Real PLE IQ3_XXS/IQ4_XS matrix parity: ggml CPU dequantization versus HIP native MMVQ.
+// Usage: hip_ple_iq4 <model-shard-1.gguf> [expected-type-id]
+#include <hip/hip_runtime.h>
+
+#include "ggml.h"
+#include "strata/artifact/gguf_reader.hpp"
+#include "strata/kernels/f16_bits.hpp"
+#include "strata/kernels/native_mmvq.hpp"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace {
+
+constexpr int kNIn = 2560;
+constexpr int kNOut = 10240;
+constexpr int kTypeIq3Xxs = 18;
+constexpr int kTypeIq4Xs = 23;
+constexpr int kReplayCount = 3;
+
+#define CHECK(call)                                                                                                  \
+    do {                                                                                                             \
+        const hipError_t error = (call);                                                                             \
+        if (error != hipSuccess) {                                                                                   \
+            std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error));                                    \
+            return 2;                                                                                                \
+        }                                                                                                            \
+    } while (0)
+
+struct Q8Reference {
+    std::vector<float> dequantized;
+};
+
+Q8Reference quantize_q8_1_reference(const std::vector<float>& x) {
+    Q8Reference result;
+    result.dequantized.resize(x.size());
+    for (size_t offset = 0; offset < x.size(); offset += 32) {
+        float amax = 0.0f;
+        for (int i = 0; i < 32; ++i) amax = std::fmax(amax, std::fabs(x[offset + i]));
+        if (amax == 0.0f) {
+            std::fill(result.dequantized.begin() + offset, result.dequantized.begin() + offset + 32, 0.0f);
+            continue;
+        }
+        const float scale_f32 = amax / 127.0f;
+        const float scale_f16 = strata::kernels::f32_from_f16(strata::kernels::f16_from_f32(scale_f32));
+        for (int i = 0; i < 32; ++i) {
+            const int q = static_cast<int>(std::round(x[offset + i] / scale_f32));
+            result.dequantized[offset + i] = static_cast<float>(static_cast<int8_t>(q)) * scale_f16;
+        }
+    }
+    return result;
+}
+
+struct Parity {
+    double rel_l1_q8 = 0.0;
+    double worst_terms_q8 = 0.0;
+    double rel_l1_f32 = 0.0;
+    double max_abs_q8 = 0.0;
+    bool finite = true;
+};
+
+Parity compare(const uint8_t* weights, size_t row_bytes, const ggml_type_traits* traits,
+               const std::vector<float>& x, const std::vector<float>& x_q8,
+               const std::vector<float>& actual) {
+    std::vector<float> row(kNIn);
+    double q8_error = 0.0, q8_magnitude = 0.0;
+    double f32_error = 0.0, f32_magnitude = 0.0;
+    Parity result;
+    for (int r = 0; r < kNOut; ++r) {
+        traits->to_float(weights + static_cast<size_t>(r) * row_bytes, row.data(), kNIn);
+        double expected_q8 = 0.0, terms_q8 = 0.0, expected_f32 = 0.0;
+        for (int i = 0; i < kNIn; ++i) {
+            const double term_q8 = static_cast<double>(row[i]) * x_q8[i];
+            const double term_f32 = static_cast<double>(row[i]) * x[i];
+            expected_q8 += term_q8;
+            terms_q8 += std::fabs(term_q8);
+            expected_f32 += term_f32;
+        }
+        const double got = actual[r];
+        if (!std::isfinite(got) || !std::isfinite(expected_q8) || !std::isfinite(expected_f32)) result.finite = false;
+        q8_error += std::fabs(got - expected_q8);
+        q8_magnitude += std::fabs(expected_q8);
+        f32_error += std::fabs(got - expected_f32);
+        f32_magnitude += std::fabs(expected_f32);
+        result.max_abs_q8 = std::fmax(result.max_abs_q8, std::fabs(got - expected_q8));
+        result.worst_terms_q8 = std::fmax(result.worst_terms_q8,
+                                         std::fabs(got - expected_q8) / std::fmax(terms_q8, 1e-30));
+    }
+    result.rel_l1_q8 = q8_error / std::fmax(q8_magnitude, 1e-30);
+    result.rel_l1_f32 = f32_error / std::fmax(f32_magnitude, 1e-30);
+    return result;
+}
+
+std::vector<float> make_input(uint32_t seed) {
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> normal(0.0f, 0.6f);
+    std::vector<float> x(kNIn);
+    for (int i = 0; i < kNIn; ++i)
+        x[i] = normal(rng) + 0.1f * std::sin(static_cast<float>(i) * 0.013f);
+    return x;
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    if (argc != 2 && argc != 3) {
+        std::fprintf(stderr, "usage: hip_ple_iq4 <model-shard-1.gguf> [expected-type-id]\n");
+        return 2;
+    }
+
+    try {
+        strata::GgufFile gguf(argv[1]);
+        const strata::TensorInfo* tensor = gguf.find("blk.1.ple_key.weight");
+        if (!tensor || (tensor->type != kTypeIq3Xxs && tensor->type != kTypeIq4Xs) || tensor->shape.size() != 2 ||
+            tensor->shape[0] != kNIn || tensor->shape[1] != kNOut) {
+            std::fprintf(stderr, "expected blk.1.ple_key.weight IQ3_XXS or IQ4_XS [2560,10240] in %s\n", argv[1]);
+            return 1;
+        }
+        if (argc == 3 && std::atoi(argv[2]) != static_cast<int>(tensor->type)) {
+            std::fprintf(stderr, "requested GGML type %s but blk.1.ple_key.weight is type %u (%s)\n",
+                         argv[2], tensor->type, tensor->type_name());
+            return 1;
+        }
+        const auto* traits = ggml_get_type_traits(static_cast<ggml_type>(tensor->type));
+        if (!traits || !traits->to_float) {
+            std::fprintf(stderr, "ggml has no %s CPU dequantizer\n", tensor->type_name());
+            return 1;
+        }
+        const uint8_t* host_weights = gguf.tensor_data(*tensor);
+        const size_t weight_bytes = strata::kernels::native_mmvq_weight_bytes(static_cast<int>(tensor->type), kNIn, kNOut);
+        const size_t row_bytes = weight_bytes / kNOut;
+        const size_t expected_bytes_per_block = tensor->type == kTypeIq3Xxs ? 98 : 136;
+        if (weight_bytes != static_cast<size_t>(tensor->elements()) / 256 * expected_bytes_per_block ||
+            row_bytes != static_cast<size_t>(kNIn / 256) * expected_bytes_per_block) {
+            std::fprintf(stderr, "unexpected %s storage geometry: %zu bytes, row %zu\n",
+                         tensor->type_name(), weight_bytes, row_bytes);
+            return 1;
+        }
+
+        uint8_t* device_weights = nullptr;
+        float* device_x = nullptr;
+        float* device_y = nullptr;
+        void* device_q8_1 = nullptr;
+        CHECK(hipMalloc(reinterpret_cast<void**>(&device_weights), weight_bytes));
+        CHECK(hipMalloc(reinterpret_cast<void**>(&device_x), kNIn * sizeof(float)));
+        CHECK(hipMalloc(reinterpret_cast<void**>(&device_y), kNOut * sizeof(float)));
+        CHECK(hipMalloc(&device_q8_1, strata::kernels::native_q8_1_bytes(kNIn)));
+        CHECK(hipMemcpy(device_weights, host_weights, weight_bytes, hipMemcpyHostToDevice));
+
+        hipStream_t stream = nullptr;
+        CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking));
+        CHECK(hipStreamBeginCapture(stream, hipStreamCaptureModeThreadLocal));
+        strata::kernels::native_quantize_q8_1(device_x, device_q8_1, kNIn, 1, static_cast<void*>(stream));
+        strata::kernels::native_mmvq(static_cast<int>(tensor->type), device_weights, device_q8_1, device_y,
+                                     kNIn, kNOut, 1, static_cast<void*>(stream));
+        hipGraph_t graph = nullptr;
+        CHECK(hipStreamEndCapture(stream, &graph));
+        hipGraphExec_t graph_exec = nullptr;
+        CHECK(hipGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0));
+
+        std::vector<float> actual(kNOut);
+        bool ok = true;
+        for (int replay = 0; replay < kReplayCount; ++replay) {
+            const std::vector<float> x = make_input(0x1572u + static_cast<uint32_t>(replay));
+            const Q8Reference q8 = quantize_q8_1_reference(x);
+            CHECK(hipMemcpyAsync(device_x, x.data(), x.size() * sizeof(float), hipMemcpyHostToDevice, stream));
+            CHECK(hipGraphLaunch(graph_exec, stream));
+            CHECK(hipMemcpyAsync(actual.data(), device_y, actual.size() * sizeof(float), hipMemcpyDeviceToHost, stream));
+            CHECK(hipStreamSynchronize(stream));
+
+            const Parity parity = compare(host_weights, row_bytes, traits, x, q8.dequantized, actual);
+            const bool pass = parity.finite && parity.rel_l1_q8 <= 5e-5 && parity.worst_terms_q8 <= 1e-4 &&
+                              parity.rel_l1_f32 <= 3e-2;
+            std::printf("%s PLE graph replay %d/%d: q8-ref rel-L1 %.3e, worst terms %.3e, "
+                        "original-f32 rel-L1 %.3e, max q8 abs %.3e %s\n",
+                        tensor->type_name(), replay + 1, kReplayCount, parity.rel_l1_q8, parity.worst_terms_q8,
+                        parity.rel_l1_f32, parity.max_abs_q8, pass ? "PASS" : "FAIL");
+            ok = ok && pass;
+        }
+
+        CHECK(hipGraphExecDestroy(graph_exec));
+        CHECK(hipGraphDestroy(graph));
+        CHECK(hipStreamDestroy(stream));
+        CHECK(hipFree(device_q8_1));
+        CHECK(hipFree(device_y));
+        CHECK(hipFree(device_x));
+        CHECK(hipFree(device_weights));
+        if (!ok) return 1;
+        std::printf("PLE %s native MMVQ parity OK: real GGUF matrix, CPU dequant reference, %d graph replays\n",
+                    tensor->type_name(), kReplayCount);
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "hip_ple_iq4: %s\n", error.what());
+        return 1;
+    }
+}

tests/hip/prefill_hipblaslt_gemm.cpp

@@ -0,0 +1,188 @@
+// Opt-in HIP test for the calibrated hipBLASLt prefill route. Requires a matching tuning table.
+#include <cuda_runtime.h>
+#include <hip/hip_bfloat16.h>
+#include <hip/hip_fp16.h>
+#include <hipblas/hipblas.h>
+#include <hipblaslt/hipblaslt.h>
+
+#include "strata/prefill/gemm.hpp"
+#include "hipblaslt_tuning.hpp"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <string>
+#include <vector>
+
+#define HIP_CHECK(call) do { \
+    const hipError_t status_ = (call); \
+    if (status_ != hipSuccess) { \
+        std::fprintf(stderr, "HIP failure at %s:%d: %s: %s\n", __FILE__, __LINE__, #call, hipGetErrorString(status_)); \
+        std::exit(2); \
+    } \
+} while (0)
+
+#define HIPBLAS_CHECK(call) do { \
+    const hipblasStatus_t status_ = (call); \
+    if (status_ != HIPBLAS_STATUS_SUCCESS) { \
+        std::fprintf(stderr, "hipBLAS failure at %s:%d: %s: %d\n", __FILE__, __LINE__, #call, (int) status_); \
+        std::exit(2); \
+    } \
+} while (0)
+
+struct DeviceBuffer {
+    void* p = nullptr;
+    explicit DeviceBuffer(size_t bytes) { if (bytes) HIP_CHECK(hipMalloc(&p, bytes)); }
+    ~DeviceBuffer() { if (p) (void) hipFree(p); }
+    DeviceBuffer(const DeviceBuffer&) = delete;
+    DeviceBuffer& operator=(const DeviceBuffer&) = delete;
+};
+
+uint16_t encode(float value, bool bf16) {
+    if (bf16) return hip_bfloat16(value).data;
+    const __half half = __float2half_rn(value);
+    uint16_t bits = 0;
+    std::memcpy(&bits, &half, sizeof(bits));
+    return bits;
+}
+
+bool run_case(strata::prefill::Gemm& gemm, hipblasHandle_t blas, hipStream_t stream, bool bf16, int t, int n,
+              int k, int ldy, int output_offset, float beta, uint32_t seed) {
+    std::mt19937 rng(seed);
+    std::uniform_real_distribution<float> dist(-0.25f, 0.25f);
+    const size_t x_count = (size_t) t * k;
+    const size_t w_count = (size_t) n * k;
+    std::vector<uint16_t> x(x_count), w(w_count);
+    for (auto& value : x) value = encode(dist(rng), bf16);
+    for (auto& value : w) value = encode(dist(rng), bf16);
+
+    const size_t output_count = (size_t) output_offset + (size_t) t * ldy + 8;
+    std::vector<float> initial(output_count, -777.25f);
+    for (int row = 0; row < t; ++row) {
+        for (int col = 0; col < n; ++col) {
+            initial[(size_t) output_offset + (size_t) row * ldy + col] =
+                (float) ((row * 17 + col * 3) % 29 - 14) * 0.03125f;
+        }
+    }
+
+    DeviceBuffer dx(x_count * sizeof(uint16_t));
+    DeviceBuffer dw(w_count * sizeof(uint16_t));
+    DeviceBuffer dy(initial.size() * sizeof(float));
+    DeviceBuffer dref(initial.size() * sizeof(float));
+    HIP_CHECK(hipMemcpyAsync(dx.p, x.data(), x_count * sizeof(uint16_t), hipMemcpyHostToDevice, stream));
+    HIP_CHECK(hipMemcpyAsync(dw.p, w.data(), w_count * sizeof(uint16_t), hipMemcpyHostToDevice, stream));
+    HIP_CHECK(hipMemcpyAsync(dy.p, initial.data(), initial.size() * sizeof(float), hipMemcpyHostToDevice, stream));
+    HIP_CHECK(hipMemcpyAsync(dref.p, initial.data(), initial.size() * sizeof(float), hipMemcpyHostToDevice, stream));
+    HIP_CHECK(hipStreamSynchronize(stream));
+
+    const hipDataType type = bf16 ? HIP_R_16BF : HIP_R_16F;
+    const float alpha = 1.0f;
+    HIPBLAS_CHECK(hipblasGemmEx(blas, HIPBLAS_OP_T, HIPBLAS_OP_N, n, t, k, &alpha, dw.p, type, k, dx.p, type, k,
+                                &beta, (float*) dref.p + output_offset, HIP_R_32F, ldy, HIPBLAS_COMPUTE_32F,
+                                HIPBLAS_GEMM_DEFAULT));
+    if (bf16) {
+        gemm.bf16((const uint16_t*) dx.p, (const uint16_t*) dw.p, (float*) dy.p + output_offset, t, n, k, ldy,
+                  beta);
+    } else {
+        gemm.f16((const uint16_t*) dx.p, (const uint16_t*) dw.p, (float*) dy.p + output_offset, t, n, k, ldy,
+                 beta);
+    }
+    HIP_CHECK(hipStreamSynchronize(stream));
+
+    std::vector<float> got(initial.size()), ref(initial.size());
+    HIP_CHECK(hipMemcpy(got.data(), dy.p, got.size() * sizeof(float), hipMemcpyDeviceToHost));
+    HIP_CHECK(hipMemcpy(ref.data(), dref.p, ref.size() * sizeof(float), hipMemcpyDeviceToHost));
+
+    std::vector<uint8_t> active(initial.size(), 0);
+    double diff2 = 0.0, ref2 = 0.0, max_abs = 0.0;
+    for (int row = 0; row < t; ++row) {
+        for (int col = 0; col < n; ++col) {
+            const size_t index = (size_t) output_offset + (size_t) row * ldy + col;
+            active[index] = 1;
+            if (!std::isfinite(got[index]) || !std::isfinite(ref[index])) {
+                std::fprintf(stderr, "non-finite output dtype=%s T=%d N=%d K=%d beta=%.1f index=%zu\n",
+                             bf16 ? "bf16" : "f16", t, n, k, beta, index);
+                return false;
+            }
+            const double delta = (double) got[index] - ref[index];
+            diff2 += delta * delta;
+            ref2 += (double) ref[index] * ref[index];
+            max_abs = std::max(max_abs, std::abs(delta));
+        }
+    }
+    for (size_t i = 0; i < initial.size(); ++i) {
+        if (!active[i] && (got[i] != initial[i] || ref[i] != initial[i])) {
+            std::fprintf(stderr, "output touched padding/guard dtype=%s T=%d N=%d K=%d beta=%.1f index=%zu got=%.9g\n",
+                         bf16 ? "bf16" : "f16", t, n, k, beta, i, got[i]);
+            return false;
+        }
+    }
+    const double relative_l2 = std::sqrt(diff2 / std::max(ref2, 1e-300));
+    const bool passed = relative_l2 <= 1e-4 && max_abs <= 5e-3;
+    std::printf("case dtype=%s T=%d N=%d K=%d ldy=%d output_offset=%d beta=%.1f relative_l2=%.9g max_abs=%.9g %s\n",
+                bf16 ? "bf16" : "f16", t, n, k, ldy, output_offset, beta, relative_l2, max_abs,
+                passed ? "PASS" : "FAIL");
+    return passed;
+}
+
+int main() {
+    const char* tuning_path = std::getenv("STRATA_HIPBLASLT_TUNING");
+    if (!tuning_path || !*tuning_path) {
+        std::fprintf(stderr, "SKIP: set STRATA_HIPBLASLT_TUNING to a calibrated matching table\n");
+        return 77;
+    }
+
+    int device = 0;
+    HIP_CHECK(hipGetDevice(&device));
+    hipDeviceProp_t properties{};
+    HIP_CHECK(hipGetDeviceProperties(&properties, device));
+    std::string arch(properties.gcnArchName);
+    const auto suffix = arch.find(':');
+    if (suffix != std::string::npos) arch.resize(suffix);
+    hipblasLtHandle_t lt = nullptr;
+    HIPBLAS_CHECK(hipblasLtCreate(&lt));
+    int version = 0;
+    HIPBLAS_CHECK(hipblasLtGetVersion(lt, &version));
+    HIPBLAS_CHECK(hipblasLtDestroy(lt));
+
+    strata::prefill::hipblaslt::TuningTable table;
+    std::string error;
+    if (!table.load(tuning_path, arch, version, error)) {
+        std::fprintf(stderr, "tuning table rejected: %s\n", error.c_str());
+        return 1;
+    }
+    if (!table.closest(strata::prefill::hipblaslt::InputType::bf16, 48, 2560, 96, 4096) ||
+        !table.closest(strata::prefill::hipblaslt::InputType::f16, 512, 2560, 512, 4096)) {
+        std::fprintf(stderr, "tuning table lacks the focused BF16/F16 test rows\n");
+        return 1;
+    }
+
+    hipStream_t stream = nullptr;
+    HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking));
+    bool ok = true;
+    {
+        std::string init_error;
+        strata::prefill::Gemm gemm;
+        if (!gemm.init((void*) stream, 0, init_error)) {
+            std::fprintf(stderr, "Gemm init failed: %s\n", init_error.c_str());
+            HIP_CHECK(hipStreamDestroy(stream));
+            return 2;
+        }
+        hipblasHandle_t blas = nullptr;
+        HIPBLAS_CHECK(hipblasCreate(&blas));
+        HIPBLAS_CHECK(hipblasSetStream(blas, stream));
+
+        // Exact calibrated chunks exercise the selected solution; T=37 reuses the closest bucket,
+        // validates the actual shape before launch, and tests a non-tile-multiple tail.
+        ok &= run_case(gemm, blas, stream, true, 4096, 48, 2560, 96, 0, 0.0f, 101);
+        ok &= run_case(gemm, blas, stream, true, 37, 48, 2560, 96, 48, 1.0f, 102);
+        ok &= run_case(gemm, blas, stream, false, 4096, 512, 2560, 512, 0, 0.0f, 103);
+        ok &= run_case(gemm, blas, stream, false, 37, 512, 2560, 512, 7, 1.0f, 104);
+        HIPBLAS_CHECK(hipblasDestroy(blas));
+    }
+    HIP_CHECK(hipStreamDestroy(stream));
+    return ok ? 0 : 1;
+}

tests/hip/prefill_mmq_parity.cpp

@@ -0,0 +1,281 @@
+// Standalone diagnostic harness for strata-prefill's optional MMQ path.
+// Keep this artifact outside the project source until it is reviewed and wired into a HIP test target.
+// It compares GGML's MMQ result (with its intended q8_1 activation rounding) to a CPU FP32 product using
+// GGML's own dequantizer for the exact same weight blocks. The tolerances screen layout/stride/bounds errors;
+// they do not assert bitwise or FP16 parity.
+#include "strata/prefill/moe_mmq.hpp"
+#include "ggml.h"
+#include <hip/hip_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <fstream>
+#include <iostream>
+#include <numeric>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace {
+using strata::prefill::mmq::Context;
+using strata::prefill::mmq::Product;
+using strata::prefill::mmq::matrix_bytes;
+using strata::prefill::mmq::q8_bytes;
+using strata::prefill::mmq::quantize;
+using strata::prefill::mmq::supported;
+
+void hip_check(hipError_t e, const char * what) {
+    if (e != hipSuccess) throw std::runtime_error(std::string(what) + ": " + hipGetErrorString(e));
+}
+
+struct DeviceBuffer {
+    void * p = nullptr;
+    explicit DeviceBuffer(size_t n) { hip_check(hipMalloc(&p, n), "hipMalloc"); }
+    ~DeviceBuffer() { if (p) hipFree(p); }
+    DeviceBuffer(const DeviceBuffer &) = delete;
+    DeviceBuffer & operator=(const DeviceBuffer &) = delete;
+    template<class T> T * as() const { return static_cast<T *>(p); }
+};
+
+size_t row_bytes(ggml_type type, int64_t cols) {
+    return ggml_row_size(type, cols);
+}
+
+std::vector<float> decode_matrix(ggml_type type, const std::vector<uint8_t> & q, int64_t rows, int64_t cols) {
+    const auto * tr = ggml_get_type_traits(type);
+    if (!tr || !tr->to_float) throw std::runtime_error("GGML type has no host dequantizer");
+    const size_t rb = row_bytes(type, cols);
+    if (q.size() != (size_t) rows * rb) throw std::runtime_error("matrix byte count mismatch");
+    std::vector<float> out((size_t) rows * (size_t) cols);
+    for (int64_t r = 0; r < rows; ++r) {
+        tr->to_float(q.data() + (size_t) r * rb, out.data() + (size_t) r * (size_t) cols, cols);
+    }
+    return out;
+}
+
+std::vector<uint8_t> make_q2_matrix(int64_t rows, int64_t cols, int expert, int trial) {
+    const ggml_type type = GGML_TYPE_Q2_0;
+    const auto * tr = ggml_get_type_traits(type);
+    if (!tr || !tr->from_float_ref) throw std::runtime_error("Q2_0 host quantizer unavailable");
+    const size_t rb = row_bytes(type, cols);
+    std::vector<uint8_t> out((size_t) rows * rb);
+    std::vector<float> row((size_t) cols);
+    for (int64_t r = 0; r < rows; ++r) {
+        for (int64_t k = 0; k < cols; ++k) {
+            const int code = (int) ((k * 37 + r * 19 + expert * 23 + trial * 11) % 101) - 50;
+            row[(size_t) k] = 0.025f * std::sin(0.013f * (float) (k + 1) + 0.17f * (float) r + 0.31f * (float) expert) +
+                              0.0015f * (float) code;
+        }
+        tr->from_float_ref(row.data(), out.data() + (size_t) r * rb, cols);
+    }
+    return out;
+}
+
+std::vector<float> make_activations(int rows, int64_t cols, int trial) {
+    std::vector<float> x((size_t) rows * (size_t) cols);
+    for (int r = 0; r < rows; ++r) {
+        for (int64_t k = 0; k < cols; ++k) {
+            x[(size_t) r * (size_t) cols + (size_t) k] =
+                0.45f * std::sin(0.009f * (float) (k + 1) + 0.23f * (float) r + 0.07f * (float) trial) +
+                0.17f * std::cos(0.021f * (float) (k + 3) - 0.19f * (float) r) +
+                0.002f * (float) (((k * 7 + r * 13 + trial * 5) % 19) - 9);
+        }
+    }
+    // Exercise a complete all-zero row; its expected output is exactly zero even though Q8_1 rounding is enabled.
+    std::fill(x.end() - cols, x.end(), 0.0f);
+    return x;
+}
+
+std::vector<int32_t> permutation(int rows, int shift, bool reverse) {
+    std::vector<int32_t> p((size_t) rows);
+    for (int i = 0; i < rows; ++i) {
+        p[(size_t) i] = reverse ? (int32_t) ((rows - 1 - i + shift) % rows)
+                                : (int32_t) ((i + shift) % rows);
+    }
+    return p;
+}
+
+std::vector<int32_t> make_bounds(const std::vector<int> & counts) {
+    std::vector<int32_t> b(counts.size() + 1, 0);
+    for (size_t i = 0; i < counts.size(); ++i) b[i + 1] = b[i] + counts[i];
+    return b;
+}
+
+struct Metrics { double rel_l2 = 0, max_over_rms = 0, zero_row_max = 0, ref_rms = 0; };
+
+Metrics run_product(Context & ctx, hipStream_t stream, const std::string & name, ggml_type type,
+                    int64_t out_rows, int64_t cols, const std::vector<std::vector<uint8_t>> & expert_weights,
+                    const std::vector<int> & counts, const std::vector<float> & x,
+                    const std::vector<int32_t> & src_ids, const std::vector<int32_t> & dst_ids) {
+    const int rows = (int) src_ids.size();
+    const int experts = (int) expert_weights.size();
+    const std::vector<int32_t> bounds = make_bounds(counts);
+    if (!supported((int) type) || experts <= 0 || counts.size() != expert_weights.size() ||
+        bounds.back() != rows || dst_ids.size() != (size_t) rows || x.size() != (size_t) rows * (size_t) cols) {
+        throw std::runtime_error(name + ": invalid test geometry");
+    }
+    const size_t expert_bytes = matrix_bytes((int) type, out_rows, cols);
+    const size_t tail = 4096; // match the production group's zero guard after its last expert
+    std::vector<uint8_t> w((size_t) experts * expert_bytes + tail, 0);
+    for (int e = 0; e < experts; ++e) {
+        if (expert_weights[(size_t) e].size() != expert_bytes) throw std::runtime_error(name + ": expert size mismatch");
+        std::copy(expert_weights[(size_t) e].begin(), expert_weights[(size_t) e].end(),
+                  w.begin() + (size_t) e * expert_bytes);
+    }
+
+    DeviceBuffer dx(x.size() * sizeof(float));
+    DeviceBuffer dsrc(src_ids.size() * sizeof(int32_t));
+    DeviceBuffer ddst(dst_ids.size() * sizeof(int32_t));
+    DeviceBuffer dbounds(bounds.size() * sizeof(int32_t));
+    DeviceBuffer dw(w.size());
+    DeviceBuffer dxq(q8_bytes(rows, cols));
+    DeviceBuffer dy((size_t) rows * (size_t) out_rows * sizeof(float));
+    hip_check(hipMemcpy(dx.p, x.data(), x.size() * sizeof(float), hipMemcpyHostToDevice), "copy x");
+    hip_check(hipMemcpy(dsrc.p, src_ids.data(), src_ids.size() * sizeof(int32_t), hipMemcpyHostToDevice), "copy src ids");
+    hip_check(hipMemcpy(ddst.p, dst_ids.data(), dst_ids.size() * sizeof(int32_t), hipMemcpyHostToDevice), "copy dst ids");
+    hip_check(hipMemcpy(dbounds.p, bounds.data(), bounds.size() * sizeof(int32_t), hipMemcpyHostToDevice), "copy bounds");
+    hip_check(hipMemcpy(dw.p, w.data(), w.size(), hipMemcpyHostToDevice), "copy weights and zero tail");
+    hip_check(hipMemset(dy.p, 0xff, (size_t) rows * (size_t) out_rows * sizeof(float)), "initialize output sentinel");
+
+    quantize(dx.as<float>(), dsrc.as<int32_t>(), dxq.p, (int) type, cols, cols, rows, (void *) stream);
+    const int max_rows = *std::max_element(counts.begin(), counts.end());
+    Product p;
+    p.w = dw.p; p.type = (int) type; p.w_rows = out_rows; p.w_cols = cols; p.expert_bytes = expert_bytes;
+    p.n = experts; p.xq = dxq.p; p.bounds = dbounds.as<int32_t>(); p.ids = ddst.as<int32_t>();
+    p.total_rows = rows; p.max_rows = max_rows; p.dst = dy.as<float>(); p.ld_dst = out_rows;
+    ctx.run(p, (void *) stream);
+    hip_check(hipGetLastError(), "MMQ launch");
+    hip_check(hipStreamSynchronize(stream), "MMQ synchronize");
+    std::vector<float> got((size_t) rows * (size_t) out_rows);
+    hip_check(hipMemcpy(got.data(), dy.p, got.size() * sizeof(float), hipMemcpyDeviceToHost), "copy output");
+
+    std::vector<float> ref(got.size(), 0.0f);
+    double err2 = 0, ref2 = 0;
+    float max_abs = 0, zero_max = 0;
+    for (int e = 0; e < experts; ++e) {
+        const std::vector<float> wd = decode_matrix(type, expert_weights[(size_t) e], out_rows, cols);
+        for (int r = bounds[(size_t) e]; r < bounds[(size_t) e + 1]; ++r) {
+            const int src = src_ids[(size_t) r];
+            const int dst = dst_ids[(size_t) r];
+            const float * xr = x.data() + (size_t) src * (size_t) cols;
+            for (int64_t o = 0; o < out_rows; ++o) {
+                double acc = 0;
+                const float * wr = wd.data() + (size_t) o * (size_t) cols;
+                for (int64_t k = 0; k < cols; ++k) acc += (double) wr[k] * xr[k];
+                ref[(size_t) dst * (size_t) out_rows + (size_t) o] = (float) acc;
+            }
+        }
+    }
+    for (size_t i = 0; i < got.size(); ++i) {
+        if (!std::isfinite(got[i])) throw std::runtime_error(name + ": non-finite or unwritten MMQ output");
+        const double d = (double) got[i] - ref[i];
+        err2 += d * d; ref2 += (double) ref[i] * ref[i]; max_abs = std::max(max_abs, (float) std::fabs(d));
+    }
+    const double rms = std::sqrt(ref2 / (double) ref.size());
+    for (int r = 0; r < rows; ++r) {
+        if (std::all_of(x.begin() + (size_t) src_ids[(size_t) r] * (size_t) cols,
+                        x.begin() + ((size_t) src_ids[(size_t) r] + 1) * (size_t) cols,
+                        [](float v) { return v == 0.0f; })) {
+            for (int64_t o = 0; o < out_rows; ++o)
+                zero_max = std::max(zero_max, std::fabs(got[(size_t) dst_ids[(size_t) r] * (size_t) out_rows + (size_t) o]));
+        }
+    }
+    Metrics m;
+    m.rel_l2 = rms > 0 ? std::sqrt(err2 / (double) ref.size()) / rms : std::sqrt(err2);
+    m.max_over_rms = rms > 0 ? max_abs / rms : max_abs;
+    m.zero_row_max = zero_max; m.ref_rms = rms;
+    std::cout << name << ": type=" << (int) type << " rows=" << rows << " experts=" << experts
+              << " ref_rms=" << m.ref_rms << " rel_l2=" << m.rel_l2
+              << " max_abs/ref_rms=" << m.max_over_rms << " zero_row_max=" << m.zero_row_max << '\n';
+    // Q8_1 rounding is expected. These deliberately broad limits catch wrong block order, row strides, ids,
+    // expert bounds, or stale output while allowing ordinary activation-quantization error.
+    if (m.rel_l2 > 0.04 || m.max_over_rms > 0.35 || m.zero_row_max > std::max(1e-5, 1e-4 * rms))
+        throw std::runtime_error(name + ": parity screen failed");
+    return m;
+}
+
+std::vector<std::vector<uint8_t>> synthetic_q2(int experts, int64_t out_rows, int64_t cols, int trial) {
+    std::vector<std::vector<uint8_t>> w;
+    for (int e = 0; e < experts; ++e) w.push_back(make_q2_matrix(out_rows, cols, e, trial));
+    return w;
+}
+
+bool run_real_iq3_first_expert(Context & ctx, hipStream_t stream, const std::string & pack) {
+    std::ifstream meta(pack + "/native_experts.txt");
+    if (!meta) throw std::runtime_error("cannot open native_experts.txt: " + pack);
+    int layer = -1, gt = -1, dt = -1;
+    uint64_t offset = 0, blob_bytes = 0;
+    std::string line;
+    bool found = false;
+    while (std::getline(meta, line)) {
+        if (line.empty() || line[0] == '#') continue;
+        std::istringstream ss(line);
+        if (ss >> layer >> gt >> dt >> offset >> blob_bytes && layer == 0) { found = true; break; }
+    }
+    if (!found) throw std::runtime_error("native metadata has no layer 0");
+    if (gt != GGML_TYPE_IQ3_XXS || !supported(gt) || !supported(dt)) {
+        std::cout << "real-IQ3 first expert skipped: layer 0 types are " << gt << "/" << dt << '\n';
+        return false;
+    }
+    std::ifstream file(pack + "/experts.bin", std::ios::binary);
+    if (!file) throw std::runtime_error("cannot open experts.bin: " + pack);
+    std::vector<uint8_t> blob((size_t) blob_bytes);
+    file.seekg((std::streamoff) offset);
+    file.read((char *) blob.data(), (std::streamsize) blob.size());
+    if (!file) throw std::runtime_error("short read of first native expert");
+
+    constexpr int64_t N = 2560, FF = 640;
+    const size_t gu_row = row_bytes((ggml_type) gt, N), d_row = row_bytes((ggml_type) dt, FF);
+    const size_t one_gu = (size_t) FF * gu_row, down_off = 2 * one_gu, one_down = (size_t) N * d_row;
+    if (down_off + one_down != blob.size()) throw std::runtime_error("native blob shape/offset mismatch");
+    std::vector<uint8_t> gu(blob.begin(), blob.begin() + (ptrdiff_t) down_off);
+    std::vector<uint8_t> down(blob.begin() + (ptrdiff_t) down_off, blob.end());
+    const int rows = 5;
+    const std::vector<int> counts{rows};
+    const auto src = permutation(rows, 2, false), dst = permutation(rows, 1, true);
+    run_product(ctx, stream, "real-first-IQ3_XXS-GU", (ggml_type) gt, 1280, N, {gu}, counts,
+                make_activations(rows, N, 71), src, dst);
+    run_product(ctx, stream, "real-first-down", (ggml_type) dt, N, FF, {down}, counts,
+                make_activations(rows, FF, 73), src, dst);
+    std::cout << "real first expert tested: layer=0 expert=0 blob_bytes=" << blob_bytes << '\n';
+    return true;
+}
+} // namespace
+
+int main(int argc, char ** argv) {
+    try {
+        if (argc > 2) throw std::runtime_error("usage: mmq-parity-test [optional-native-pack-dir]");
+        hip_check(hipSetDevice(0), "hipSetDevice");
+        hipStream_t stream = nullptr;
+        hip_check(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking), "create stream");
+        int failures = 0;
+        {
+            Context ctx; // reuse one MMQ context/pool across every pass below
+            const std::vector<std::vector<int>> batches{{1, 3, 3}, {4, 1, 2}, {2, 3}};
+            int trial = 0;
+            for (const auto & counts : batches) {
+                const int rows = std::accumulate(counts.begin(), counts.end(), 0);
+                const int n = (int) counts.size();
+                const auto src = permutation(rows, trial + 1, false);
+                const auto dst = permutation(rows, trial + 2, true);
+                run_product(ctx, stream, "synthetic-Q2_0-GU-pass" + std::to_string(trial), GGML_TYPE_Q2_0,
+                            1280, 2560, synthetic_q2(n, 1280, 2560, trial), counts,
+                            make_activations(rows, 2560, trial), src, dst);
+                run_product(ctx, stream, "synthetic-Q2_0-down-pass" + std::to_string(trial), GGML_TYPE_Q2_0,
+                            2560, 640, synthetic_q2(n, 2560, 640, trial + 9), counts,
+                            make_activations(rows, 640, trial + 3), src, dst);
+                ++trial;
+            }
+            if (argc == 2) run_real_iq3_first_expert(ctx, stream, argv[1]);
+        }
+        hip_check(hipStreamSynchronize(stream), "final synchronize");
+        hip_check(hipStreamDestroy(stream), "destroy stream");
+        std::cout << "MMQ parity screen passed\n";
+        return failures ? 1 : 0;
+    } catch (const std::exception & e) {
+        std::cerr << "MMQ parity screen failed: " << e.what() << '\n';
+        return 1;
+    }
+}

tests/hip/prefill_native_batch.cpp

@@ -0,0 +1,290 @@
+// Focused parity for the opt-in native QSA append and native embedding gather
+// selectively ported from q8atnight/Strata PR #108
+// (acd487233c0bbe2217a6881c5bb43f8a283b0de5).
+#include <cuda_runtime.h>
+#include "strata/kernels/iq_kernels.hpp"
+#include "strata/kernels/native_qsa_indexer.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <random>
+#include <vector>
+
+#define CHECK(call) do { \
+    const cudaError_t e = (call); \
+    if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", #call, cudaGetErrorString(e)); return false; } \
+} while (0)
+
+namespace {
+constexpr int IDX = 128;
+constexpr int BLOCK = 4;
+constexpr int EMB = 2560;
+constexpr int IQ4_NL = 20;
+constexpr float EPS = 1e-6f;
+
+struct IndexState {
+    float *tail = nullptr, *dead = nullptr, *pooled = nullptr;
+    int32_t* block_pos = nullptr;
+};
+
+bool alloc_state(IndexState& s, int64_t max_cells) {
+    CHECK(cudaMalloc((void**) &s.tail, (BLOCK - 1) * IDX * sizeof(float)));
+    CHECK(cudaMalloc((void**) &s.dead, IDX * sizeof(float)));
+    CHECK(cudaMalloc((void**) &s.pooled, (size_t) (max_cells / BLOCK + 1) * IDX * sizeof(float)));
+    CHECK(cudaMalloc((void**) &s.block_pos, sizeof(int32_t)));
+    return true;
+}
+void free_state(IndexState& s) {
+    if (s.tail) cudaFree(s.tail);
+    if (s.dead) cudaFree(s.dead);
+    if (s.pooled) cudaFree(s.pooled);
+    if (s.block_pos) cudaFree(s.block_pos);
+    s = {};
+}
+bool upload_state(const IndexState& s, int64_t max_cells) {
+    std::vector<float> tail((BLOCK - 1) * IDX, 0.0f), dead(IDX, 0.0f);
+    std::vector<float> pooled((size_t) (max_cells / BLOCK + 1) * IDX);
+    for (size_t i = 0; i < pooled.size(); ++i) pooled[i] = -0.125f * (float) (i + 1);
+    const int32_t block_pos = -1;
+    CHECK(cudaMemcpy(s.tail, tail.data(), tail.size() * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK(cudaMemcpy(s.dead, dead.data(), dead.size() * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK(cudaMemcpy(s.pooled, pooled.data(), pooled.size() * sizeof(float), cudaMemcpyHostToDevice));
+    CHECK(cudaMemcpy(s.block_pos, &block_pos, sizeof(block_pos), cudaMemcpyHostToDevice));
+    return true;
+}
+
+bool qsa_case(cudaStream_t stream, int64_t max_cells, int64_t pos0, int64_t T, std::mt19937& rng) {
+    const size_t raw_n = (size_t) (pos0 + T) * IDX;
+    std::vector<float> raw(raw_n), gamma(IDX);
+    std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
+    for (float& x : raw) x = dist(rng);
+    for (float& x : gamma) x = 0.5f + 0.5f * std::fabs(dist(rng));
+
+    float *raw_d = nullptr, *gamma_d = nullptr;
+    int32_t* pos_d = nullptr;
+    CHECK(cudaMalloc((void**) &raw_d, raw.size() * sizeof(float)));
+    CHECK(cudaMalloc((void**) &gamma_d, gamma.size() * sizeof(float)));
+    CHECK(cudaMalloc((void**) &pos_d, sizeof(int32_t)));
+    CHECK(cudaMemcpyAsync(raw_d, raw.data(), raw.size() * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK(cudaMemcpyAsync(gamma_d, gamma.data(), gamma.size() * sizeof(float), cudaMemcpyHostToDevice, stream));
+
+    IndexState serial, batched;
+    if (!alloc_state(serial, max_cells) || !alloc_state(batched, max_cells)) return false;
+    if (!upload_state(serial, max_cells) || !upload_state(batched, max_cells)) return false;
+
+    const auto shapes = strata::kernels::qsa_real_shapes();
+    const strata::kernels::QsaIndexerBuffers sb{serial.tail, serial.dead, serial.pooled, serial.block_pos};
+    const strata::kernels::QsaIndexerBuffers bb{batched.tail, batched.dead, batched.pooled, batched.block_pos};
+    std::vector<int32_t> positions((size_t) (pos0 + T));
+    for (size_t p = 0; p < positions.size(); ++p) positions[p] = (int32_t) p;
+
+    // The reference is the original per-cell sequence. The candidate uses the
+    // same per-cell prefix, then replaces only this chunk with the two-launch API.
+    for (int64_t p = 0; p < pos0 + T; ++p) {
+        CHECK(cudaMemcpyAsync(pos_d, positions.data() + p, sizeof(int32_t), cudaMemcpyHostToDevice, stream));
+        strata::kernels::native_qsa_indexer_append(raw_d + p * IDX, pos_d, 0, gamma_d, EPS, sb,
+                                                   shapes, max_cells, 10000.0f, stream);
+        if (p < pos0) {
+            CHECK(cudaMemcpyAsync(pos_d, positions.data() + p, sizeof(int32_t), cudaMemcpyHostToDevice, stream));
+            strata::kernels::native_qsa_indexer_append(raw_d + p * IDX, pos_d, 0, gamma_d, EPS, bb,
+                                                       shapes, max_cells, 10000.0f, stream);
+        }
+    }
+    strata::kernels::native_qsa_indexer_append_batch(raw_d + pos0 * IDX, T, pos0, 0, gamma_d, EPS,
+                                                      bb, shapes, max_cells, 10000.0f, stream);
+    CHECK(cudaStreamSynchronize(stream));
+
+    const size_t tail_n = (BLOCK - 1) * IDX, dead_n = IDX;
+    const size_t pooled_n = (size_t) (max_cells / BLOCK + 1) * IDX;
+    std::vector<float> st(tail_n), bt(tail_n), sd(dead_n), bd(dead_n), sp(pooled_n), bp(pooled_n);
+    int32_t spos = 0, bpos = 0;
+    CHECK(cudaMemcpy(st.data(), serial.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(bt.data(), batched.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(sd.data(), serial.dead, dead_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(bd.data(), batched.dead, dead_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(sp.data(), serial.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(bp.data(), batched.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(&spos, serial.block_pos, sizeof(spos), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(&bpos, batched.block_pos, sizeof(bpos), cudaMemcpyDeviceToHost));
+    const bool same = std::memcmp(st.data(), bt.data(), tail_n * sizeof(float)) == 0 &&
+                      std::memcmp(sd.data(), bd.data(), dead_n * sizeof(float)) == 0 &&
+                      std::memcmp(sp.data(), bp.data(), pooled_n * sizeof(float)) == 0 && spos == bpos;
+    if (!same) std::fprintf(stderr, "QSA append parity mismatch max=%lld pos0=%lld T=%lld\n",
+                            (long long) max_cells, (long long) pos0, (long long) T);
+
+    free_state(serial); free_state(batched);
+    CHECK(cudaFree(raw_d)); CHECK(cudaFree(gamma_d)); CHECK(cudaFree(pos_d));
+    return same;
+}
+
+bool qsa_chunks_case(cudaStream_t stream, std::mt19937& rng, int64_t max_cells,
+                     int64_t first, int64_t second) {
+    const int64_t total = first + second;
+    std::vector<float> raw((size_t) total * IDX), gamma(IDX);
+    std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
+    for (float& x : raw) x = dist(rng);
+    for (float& x : gamma) x = 0.5f + 0.5f * std::fabs(dist(rng));
+    float *raw_d = nullptr, *gamma_d = nullptr;
+    int32_t* pos_d = nullptr;
+    CHECK(cudaMalloc((void**) &raw_d, raw.size() * sizeof(float)));
+    CHECK(cudaMalloc((void**) &gamma_d, gamma.size() * sizeof(float)));
+    CHECK(cudaMalloc((void**) &pos_d, sizeof(int32_t)));
+    CHECK(cudaMemcpyAsync(raw_d, raw.data(), raw.size() * sizeof(float), cudaMemcpyHostToDevice, stream));
+    CHECK(cudaMemcpyAsync(gamma_d, gamma.data(), gamma.size() * sizeof(float), cudaMemcpyHostToDevice, stream));
+    IndexState serial, chunked;
+    if (!alloc_state(serial, max_cells) || !alloc_state(chunked, max_cells)) return false;
+    if (!upload_state(serial, max_cells) || !upload_state(chunked, max_cells)) return false;
+    const auto shapes = strata::kernels::qsa_real_shapes();
+    const strata::kernels::QsaIndexerBuffers sb{serial.tail, serial.dead, serial.pooled, serial.block_pos};
+    const strata::kernels::QsaIndexerBuffers cb{chunked.tail, chunked.dead, chunked.pooled, chunked.block_pos};
+    std::vector<int32_t> positions((size_t) total);
+    for (int i = 0; i < total; ++i) positions[(size_t) i] = i;
+    for (int p = 0; p < total; ++p) {
+        CHECK(cudaMemcpyAsync(pos_d, positions.data() + p, sizeof(int32_t), cudaMemcpyHostToDevice, stream));
+        strata::kernels::native_qsa_indexer_append(raw_d + (size_t) p * IDX, pos_d, 0, gamma_d, EPS, sb,
+                                                   shapes, max_cells, 10000.0f, stream);
+    }
+    strata::kernels::native_qsa_indexer_append_batch(raw_d, first, 0, 0, gamma_d, EPS, cb, shapes,
+                                                      max_cells, 10000.0f, stream);
+    strata::kernels::native_qsa_indexer_append_batch(raw_d + (size_t) first * IDX, second, first, 0,
+                                                      gamma_d, EPS, cb, shapes, max_cells, 10000.0f, stream);
+    CHECK(cudaStreamSynchronize(stream));
+    const size_t tail_n = (BLOCK - 1) * IDX, pooled_n = (size_t) (max_cells / BLOCK + 1) * IDX;
+    std::vector<float> st(tail_n), ct(tail_n), sd(IDX), cd(IDX), sp(pooled_n), cp(pooled_n);
+    int32_t sbp = 0, cbp = 0;
+    CHECK(cudaMemcpy(st.data(), serial.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(ct.data(), chunked.tail, tail_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(sd.data(), serial.dead, IDX * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(cd.data(), chunked.dead, IDX * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(sp.data(), serial.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(cp.data(), chunked.pooled, pooled_n * sizeof(float), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(&sbp, serial.block_pos, sizeof(sbp), cudaMemcpyDeviceToHost));
+    CHECK(cudaMemcpy(&cbp, chunked.block_pos, sizeof(cbp), cudaMemcpyDeviceToHost));
+    const bool same = std::memcmp(st.data(), ct.data(), tail_n * sizeof(float)) == 0 &&
+                      std::memcmp(sd.data(), cd.data(), IDX * sizeof(float)) == 0 &&
+                      std::memcmp(sp.data(), cp.data(), pooled_n * sizeof(float)) == 0 && sbp == cbp;
+    if (!same) std::fprintf(stderr, "QSA consecutive chunk parity mismatch %lld+%lld, capacity=%lld\n",
+                            (long long) first, (long long) second, (long long) max_cells);
+    free_state(serial); free_state(chunked);
+    CHECK(cudaFree(raw_d)); CHECK(cudaFree(gamma_d)); CHECK(cudaFree(pos_d));
+    return same;
+}
+
+bool embed_case(cudaStream_t stream, int type, int64_t T) {
+    constexpr int vocab = 19;
+    const size_t row_bytes = strata::kernels::iq_row_bytes(type, EMB);
+    const size_t block_bytes = type == 20 ? 18 : 110;  // IQ4_NL or IQ3_S GGML block layout
+    const size_t table_bytes = (size_t) vocab * row_bytes;
+    if (row_bytes == 0 || row_bytes % block_bytes != 0) {
+        std::fprintf(stderr, "bad test embedding row size for type=%d\n", type);
+        return false;
+    }
+    void* host_table = nullptr;
+    CHECK(cudaHostAlloc(&host_table, table_bytes, cudaHostAllocMapped | cudaHostAllocPortable));
+    auto* table_bytes_u8 = (uint8_t*) host_table;
+    std::memset(table_bytes_u8, 0, table_bytes);
+    // Both test formats begin with a finite fp16 scale; the remaining packed code bits are all legal values.
+    for (size_t r = 0; r < (size_t) vocab; ++r) {
+        for (size_t b = 0; b < row_bytes; b += block_bytes) {
+            table_bytes_u8[r * row_bytes + b] = 0x00;
+            table_bytes_u8[r * row_bytes + b + 1] = 0x3c;
+            for (size_t q = b + 2; q < b + block_bytes; ++q)
+                table_bytes_u8[r * row_bytes + q] = (uint8_t) ((q * 37 + r * 19) & 0xff);
+        }
+    }
+    void* table_dev = nullptr;
+    CHECK(cudaHostGetDevicePointer(&table_dev, host_table, 0));
+
+    std::vector<uint8_t> image_rows((size_t) T, 0);
+    std::vector<int64_t> token_values((size_t) T);
+    std::vector<int32_t> ids((size_t) T);
+    for (int64_t t = 0; t < T; ++t) {
+        if (t == 1 || t == 3) {
+            image_rows[(size_t) t] = 1;
+            token_values[(size_t) t] = t == 1 ? -1 : vocab + 11;  // image placeholders must never be gathered
+            ids[(size_t) t] = 0;                                 // safe row, overwritten below
+        } else {
+            token_values[(size_t) t] = (t % 3 == 0) ? 7 : (t % 5);  // includes duplicate text IDs
+            if (token_values[(size_t) t] < 0 || token_values[(size_t) t] >= vocab ||
+                token_values[(size_t) t] > INT32_MAX) {
+                std::fprintf(stderr, "test text token failed host-side validation\n");
+                return false;
+            }
+            ids[(size_t) t] = (int32_t) token_values[(size_t) t];
+        }
+    }
+
+    float *batched = nullptr, *serial = nullptr;
+    int32_t* ids_d = nullptr;
+    CHECK(cudaMalloc((void**) &batched, (size_t) T * EMB * sizeof(float)));
+    CHECK(cudaMalloc((void**) &serial, (size_t) T * EMB * sizeof(float)));
+    CHECK(cudaMalloc((void**) &ids_d, (size_t) T * sizeof(int32_t)));
+    CHECK(cudaMemcpyAsync(ids_d, ids.data(), (size_t) T * sizeof(int32_t), cudaMemcpyHostToDevice, stream));
+    strata::kernels::iq_embed_rows(type, table_dev, row_bytes, ids_d, T, EMB, batched, stream);
+    for (int64_t t = 0; t < T; ++t) if (!image_rows[(size_t) t])
+        strata::kernels::iq_dequant_f32(type,
+            (const uint8_t*) table_dev + (size_t) ids[(size_t) t] * row_bytes, EMB, serial + t * EMB, stream);
+
+    std::vector<float> image(EMB);
+    for (int i = 0; i < EMB; ++i) image[(size_t) i] = 0.001f * (float) (i - EMB / 2);
+    for (int64_t t = 0; t < T; ++t) if (image_rows[(size_t) t]) {
+        CHECK(cudaMemcpyAsync(batched + t * EMB, image.data(), EMB * sizeof(float), cudaMemcpyHostToDevice, stream));
+        CHECK(cudaMemcpyAsync(serial + t * EMB, image.data(), EMB * sizeof(float), cudaMemcpyHostToDevice, stream));
+    }
+    CHECK(cudaStreamSynchronize(stream));
+
+    constexpr int64_t rows_per_check = 64;
+    std::vector<float> a((size_t) rows_per_check * EMB), b(a.size());
+    bool same = true;
+    for (int64_t t0 = 0; t0 < T; t0 += rows_per_check) {
+        const int64_t rows = std::min(rows_per_check, T - t0);
+        const size_t count = (size_t) rows * EMB;
+        CHECK(cudaMemcpy(a.data(), batched + t0 * EMB, count * sizeof(float), cudaMemcpyDeviceToHost));
+        CHECK(cudaMemcpy(b.data(), serial + t0 * EMB, count * sizeof(float), cudaMemcpyDeviceToHost));
+        if (std::memcmp(a.data(), b.data(), count * sizeof(float)) != 0) {
+            size_t first = 0;
+            while (first < count && std::memcmp(&a[first], &b[first], sizeof(float)) == 0) ++first;
+            std::fprintf(stderr, "NativeEmbed batch parity mismatch type=%d T=%lld at element %lld: %.9g vs %.9g\n",
+                         type, (long long) T, (long long) (t0 * EMB + first),
+                         first < count ? a[first] : 0.0f, first < count ? b[first] : 0.0f);
+            same = false;
+            break;
+        }
+    }
+    if (T > 3 && (!image_rows[1] || !image_rows[3] || ids[1] != 0 || ids[3] != 0)) same = false;
+    if (!same) std::fprintf(stderr, "NativeEmbed image placeholder sanitization mismatch type=%d T=%lld\n",
+                            type, (long long) T);
+    CHECK(cudaFree(batched)); CHECK(cudaFree(serial)); CHECK(cudaFree(ids_d));
+    CHECK(cudaFreeHost(host_table));
+    return same;
+}
+
+} // namespace
+
+int main() {
+    std::mt19937 rng(0x50524631u);
+    cudaStream_t stream = nullptr;
+    const cudaError_t create = cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking);
+    if (create != cudaSuccess) { std::fprintf(stderr, "cudaStreamCreateWithFlags: %s\n", cudaGetErrorString(create)); return 2; }
+    bool ok = true;
+    // Covers every slot residue, partial and multiple block chunks, and the fixed-capacity tail.
+    const std::array<int64_t, 14> starts = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13};
+    const std::array<int64_t, 5> lengths = {1, 2, 3, 7, 17};
+    // (the batched append's contract is p0 + n <= max_cells: the prompt path never appends past the cache)
+    for (int64_t pos0 : starts) for (int64_t T : lengths)
+        if (pos0 + T <= 13) ok = qsa_case(stream, 13, pos0, T, rng) && ok;
+    ok = qsa_chunks_case(stream, rng, 33, 7, 17) && ok;
+    // Production-scale, non-block-aligned first chunk with a short continuation.
+    ok = qsa_chunks_case(stream, rng, 8210, 8193, 17) && ok;
+    for (int type : {20, 21}) for (int64_t T : {1, 2, 7, 17})
+        ok = embed_case(stream, type, T) && ok;
+    ok = embed_case(stream, 21, 4096) && ok;
+    const cudaError_t destroy = cudaStreamDestroy(stream);
+    if (destroy != cudaSuccess) { std::fprintf(stderr, "cudaStreamDestroy: %s\n", cudaGetErrorString(destroy)); return 2; }
+    std::printf("Native prefill batch parity: %s\n", ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}

tests/platform/direct_file_async.cpp

@@ -0,0 +1,60 @@
+#include "strata/platform/direct_file.hpp"
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+#include <unistd.h>
+using strata::platform::DirectFile;
+using strata::platform::Completion;
+static void check(bool ok, const char* why) { if (!ok) throw std::runtime_error(why); }
+int main() {
+    char name[] = "/tmp/strata-direct-test-XXXXXX";
+    int fd = mkstemp(name);
+    if (fd < 0) return 1;
+    close(fd);
+    void* memory = DirectFile::alloc_aligned(4096 * 18);
+    try {
+        check(memory != nullptr, "buffer allocation");
+        { std::ofstream f(name, std::ios::binary); std::string bytes(4096 * 16 + 17, 'x'); f.write(bytes.data(), bytes.size()); }
+        DirectFile f;
+        std::string error;
+        setenv("STRATA_IO_THREADS", "4", 1);   // the pool's size (out-of-range values are clamped to 1..64)
+        check(f.open(name, error), error.c_str());
+        check(!f.submit(1, memory, 4096, 0, error), "unaligned request accepted");
+        for (unsigned i = 0; i < 17; ++i)
+            check(f.submit(i * 4096, static_cast<char*>(memory) + i * 4096, 4096, i, error), error.c_str());
+        std::vector<bool> seen(17);
+        unsigned count = 0;
+        while (count < 17) {
+            Completion c[17]; int n = f.wait(c, 17, 5000);
+            check(n > 0, "completion timeout");
+            for (int i = 0; i < n; ++i) {
+                check(c[i].tag < 17 && !seen[c[i].tag], "wrong or repeated completion");
+                seen[c[i].tag] = true; ++count;
+                check(c[i].ok && c[i].bytes == (c[i].tag == 16 ? 17u : 4096u), "incorrect full/EOF completion");
+                check(static_cast<char*>(memory)[c[i].tag * 4096] == 'x', "read data mismatch");
+            }
+        }
+        std::jthread waker([&] { f.wake(); });
+        Completion c; check(f.wait(&c, 1, 5000) == 1 && c.tag == DirectFile::WAKE_TAG, "wake lost");
+        waker.join();
+        check(f.submit(0, memory, 4096, 99, error), error.c_str());
+        f.close(); // Must drain outstanding buffer writes before returning.
+        std::memset(memory, 0, 4096);
+        check(f.open(name, error), error.c_str());
+        check(f.wait(&c, 1, 0) == 0, "stale completion on reopen");
+        check(f.submit(0, memory, 4096, 100, error), error.c_str());
+        check(f.wait(&c, 1, 5000) == 1 && c.ok && c.tag == 100, "reopened read failed");
+        f.close();
+        DirectFile::free_aligned(memory); unlink(name);
+        std::cout << "direct_file_async: passed queued reads, short EOF, wake, close drain, reopen\n";
+        return 0;
+    } catch (const std::exception& e) {
+        DirectFile::free_aligned(memory); unlink(name);
+        std::cerr << e.what() << '\n'; return 1;
+    }
+}

tools/hip/bench_prefill.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Matched fresh/follow-up prefill probe against an otherwise idle local Strata server.
+
+Does not start/stop services. Use a dedicated server and its engine log, restart
+between build/configuration arms, and keep model/template/settings identical.
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import time
+import urllib.request
+
+PATTERN = re.compile(r'prompt (\d+) tokens = (\d+) reused \+ (\d+) read in (\d+) ms \(([0-9.]+) tok/s\), (\d+) generated in (\d+) ms \(([0-9.]+) tok/s\)')
+FIELDS = ('prompt_tokens', 'reused', 'fresh', 'prefill_ms', 'prefill_tps', 'generated', 'decode_ms', 'decode_tps')
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--url', default='http://127.0.0.1:8080')
+    p.add_argument('--model', required=True)
+    p.add_argument('--engine-log', type=Path, required=True)
+    p.add_argument('--output', type=Path, required=True)
+    p.add_argument('--label', required=True)
+    args = p.parse_args()
+    results = []
+
+    def request(messages, kind, trial):
+        offset = args.engine_log.stat().st_size
+        body = dict(model=args.model, messages=messages, max_tokens=128,
+                    temperature=0, top_k=1, top_p=1, min_p=0, seed=42,
+                    reasoning_effort='none')
+        headers = {'Content-Type': 'application/json'}
+        if os.environ.get('STRATA_API_KEY'):
+            headers['Authorization'] = 'Bearer ' + os.environ['STRATA_API_KEY']
+        req = urllib.request.Request(args.url.rstrip('/') + '/v1/chat/completions',
+                                     data=json.dumps(body).encode(), headers=headers)
+        start = time.monotonic()
+        with urllib.request.urlopen(req, timeout=360) as response:
+            data = json.load(response)
+        elapsed = time.monotonic() - start
+        # The API response may arrive just before stderr's completion line is flushed.
+        deadline = time.monotonic() + 2
+        while True:
+            with args.engine_log.open('rb') as log:
+                log.seek(offset)
+                matches = PATTERN.findall(log.read().decode(errors='replace'))
+            if matches or time.monotonic() >= deadline:
+                break
+            time.sleep(0.02)
+        if len(matches) != 1:
+            raise RuntimeError('Expected exactly one completed engine timing line; check idle server and log path')
+        metrics = dict(zip(FIELDS, map(float, matches[0])))
+        if kind == 'fresh' and metrics['reused'] != 0:
+            raise RuntimeError('Fresh prompt reused cached tokens; restart this arm before comparison')
+        choice = data['choices'][0]
+        results.append(dict(label=args.label, kind=kind, trial=trial, wall_s=elapsed,
+                            metrics=metrics, usage=data.get('usage'),
+                            finish_reason=choice.get('finish_reason'), message=choice['message']))
+        args.output.write_text(json.dumps(results, indent=2) + '\n')
+        print(json.dumps(results[-1]), flush=True)
+        return choice['message']
+
+    request([{'role': 'user', 'content': 'Reply READY.'}], 'warmup', 0)
+    for trial, n in enumerate((140, 280, 140, 280), 1):
+        code = '\n'.join(f'export function rule{i}(x) {{ return x === {i} ? x + {i+1} : x - {i}; }}' for i in range(n))
+        messages = [{'role': 'user', 'content': f'CASE {trial}: Review this source and describe its behavior precisely in a paragraph.\n' + code}]
+        reply = request(messages, 'fresh', trial)
+        messages += [reply, {'role': 'user', 'content': 'Explain the most relevant boundary case and the smallest useful regression test. ' + 'Focus on integer equality, zero, negative input, unexpected types, and caller assumptions. ' * 5}]
+        request(messages, 'followup', trial)
+
+
+if __name__ == '__main__':
+    main()

tools/hip/gfx1100-hipblaslt-100100.txt

@@ -0,0 +1,32 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1100 100100
+bf16 1 2560 1 4096 541
+bf16 1 2560 1 8192 541
+bf16 4 10240 4 4096 541
+bf16 4 10240 4 8192 541
+bf16 48 2560 96 4096 541
+bf16 48 2560 96 8192 542
+bf16 128 2560 128 4096 542
+bf16 128 2560 128 8192 542
+bf16 320 10240 320 4096 542
+bf16 320 10240 320 8192 542
+bf16 512 2560 512 4096 542
+bf16 512 2560 512 8192 542
+bf16 2560 2560 2560 4096 542
+bf16 2560 2560 2560 8192 542
+bf16 10240 320 10240 4096 542
+bf16 10240 320 10240 8192 542
+f16 512 2560 512 4096 1147
+f16 512 2560 512 8192 1147
+f16 640 2560 640 4096 1147
+f16 640 2560 640 8192 1147
+f16 2560 640 2560 4096 1147
+f16 2560 640 2560 8192 1147
+f16 2560 6144 2560 4096 1147
+f16 2560 6144 2560 8192 1147
+f16 6144 2560 6144 4096 1147
+f16 6144 2560 6144 8192 1147
+f16 10240 2560 10240 4096 1147
+f16 10240 2560 10240 8192 1147
+f16 12288 2560 12288 4096 1147
+f16 12288 2560 12288 8192 1147

tools/hip/gfx1100-hipblaslt-100200.txt

@@ -0,0 +1,28 @@
+# solution IDs are scoped to this hipBLASLt version and device architecture
+STRATA_HIPBLASLT_TUNING_V1 gfx1100 100200
+bf16 1 2560 1 4096 541
+bf16 4 10240 4 8192 542
+bf16 48 2560 96 4096 541
+bf16 48 2560 96 8192 542
+bf16 128 2560 128 4096 542
+bf16 128 2560 128 8192 542
+bf16 320 10240 320 4096 542
+bf16 320 10240 320 8192 542
+bf16 512 2560 512 8192 542
+bf16 2560 2560 2560 8192 542
+bf16 10240 320 10240 4096 542
+bf16 10240 320 10240 8192 542
+f16 512 2560 512 4096 1147
+f16 512 2560 512 8192 1147
+f16 640 2560 640 4096 1147
+f16 640 2560 640 8192 1147
+f16 2560 640 2560 4096 1147
+f16 2560 640 2560 8192 1147
+f16 2560 6144 2560 4096 1147
+f16 2560 6144 2560 8192 1147
+f16 6144 2560 6144 4096 1147
+f16 6144 2560 6144 8192 1147
+f16 10240 2560 10240 4096 1147
+f16 10240 2560 10240 8192 1147
+f16 12288 2560 12288 4096 1147
+f16 12288 2560 12288 8192 1147

tools/hip/tune_hipblaslt.cpp

@@ -0,0 +1,419 @@
+#include <hip/hip_runtime.h>
+#include <hip/hip_fp16.h>
+#include <hip/hip_bfloat16.h>
+#include <hip/hip_version.h>
+#include <hipblas/hipblas.h>
+#include <hipblaslt/hipblaslt.h>
+#include <hipblaslt/hipblaslt-ext.hpp>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <iterator>
+#include <limits>
+#include <random>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <vector>
+
+namespace {
+#define HIP_CHECK(call) do { const hipError_t e = (call); if (e != hipSuccess) { \
+    std::fprintf(stderr, "HIP %s:%d: %s: %s\n", __FILE__, __LINE__, #call, hipGetErrorString(e)); std::exit(2); } } while (0)
+#define BLAS_CHECK(call) do { const hipblasStatus_t e = (call); if (e != HIPBLAS_STATUS_SUCCESS) { \
+    std::fprintf(stderr, "hipBLAS %s:%d: %s: status=%d\n", __FILE__, __LINE__, #call, (int)e); std::exit(2); } } while (0)
+#define LT_CHECK(call) do { const hipblasStatus_t e = (call); if (e != HIPBLAS_STATUS_SUCCESS) { \
+    std::fprintf(stderr, "hipBLASLt %s:%d: %s: status=%d\n", __FILE__, __LINE__, #call, (int)e); std::exit(2); } } while (0)
+
+struct Buffer {
+    void *p = nullptr;
+    explicit Buffer(size_t bytes) { if (bytes) HIP_CHECK(hipMalloc(&p, bytes)); }
+    ~Buffer() { if (p) (void)hipFree(p); }
+    Buffer(const Buffer &) = delete;
+    Buffer &operator=(const Buffer &) = delete;
+};
+struct Events {
+    hipEvent_t a = nullptr, b = nullptr;
+    Events() { HIP_CHECK(hipEventCreate(&a)); HIP_CHECK(hipEventCreate(&b)); }
+    ~Events() { if (a) (void)hipEventDestroy(a); if (b) (void)hipEventDestroy(b); }
+};
+
+struct Shape { int t, n, k, ldy; bool bf16; };
+struct Options {
+    size_t workspace = 32U * 1024U * 1024U;
+    std::vector<Shape> shapes;
+    std::vector<int> tokens{4096, 8192};
+    std::string shapes_file, tuning_out;
+    bool explicit_shapes = false;
+};
+struct Err { double rel_l2, max_abs; bool finite; size_t padding_writes; };
+struct Best {
+    Shape s;
+    int heuristic_index, solution_id;
+    size_t required_workspace, algo_workspace;
+    float mean_ms;
+    Err error;
+    std::string solution, kernel, config;
+};
+
+constexpr int WARMUPS = 2;
+constexpr int REPS = 3;
+constexpr int MAX_ALGOS = 16;
+constexpr double REL_L2_TOL = 1e-4;
+constexpr double MAX_ABS_TOL = 1e-2;
+constexpr float PADDING_CANARY = 123456.25f;
+
+const char *dtype(bool bf16) { return bf16 ? "bf16" : "f16"; }
+const char *status_name(hipblasStatus_t s) {
+    switch (s) {
+        case HIPBLAS_STATUS_SUCCESS: return "success";
+        case HIPBLAS_STATUS_NOT_SUPPORTED: return "not_supported";
+        case HIPBLAS_STATUS_INVALID_VALUE: return "invalid_value";
+        case HIPBLAS_STATUS_ARCH_MISMATCH: return "arch_mismatch";
+        default: return "other";
+    }
+}
+std::string json(const std::string &s) {
+    std::ostringstream o; o << '"';
+    for (unsigned char c : s) {
+        if (c == '"') o << "\\\"";
+        else if (c == '\\') o << "\\\\";
+        else if (c == '\n') o << "\\n";
+        else if (c == '\r') o << "\\r";
+        else if (c == '\t') o << "\\t";
+        else if (c < 0x20) o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (unsigned)c << std::dec;
+        else o << (char)c;
+    }
+    o << '"'; return o.str();
+}
+std::string config_hex(const hipblasLtMatmulAlgo_t &algo) {
+    std::ostringstream o; o << std::hex << std::setfill('0');
+    for (uint8_t b : algo.data) o << std::setw(2) << (unsigned)b;
+    return o.str();
+}
+uint16_t input_bits(float x, bool bf16) {
+    if (bf16) return hip_bfloat16(x).data;
+    const __half half = __float2half_rn(x);
+    uint16_t bits = 0;
+    std::memcpy(&bits, &half, sizeof(bits));
+    return bits;
+}
+void fill(std::vector<uint16_t> &v, uint32_t seed, bool bf16) {
+    std::mt19937 rng(seed);
+    std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
+    for (auto &x : v) x = input_bits(dist(rng), bf16);
+}
+float time_call(hipStream_t stream, const std::function<void()> &f) {
+    Events ev;
+    HIP_CHECK(hipEventRecord(ev.a, stream));
+    f();
+    HIP_CHECK(hipEventRecord(ev.b, stream));
+    HIP_CHECK(hipEventSynchronize(ev.b));
+    HIP_CHECK(hipDeviceSynchronize());
+    float ms = 0;
+    HIP_CHECK(hipEventElapsedTime(&ms, ev.a, ev.b));
+    return ms;
+}
+Err compare(const std::vector<float> &ref, const std::vector<float> &got, int n, int t, int ld) {
+    long double d2 = 0, r2 = 0; double max_abs = 0; bool finite = true;
+    for (int col = 0; col < t; ++col) for (int row = 0; row < n; ++row) {
+        const size_t i = (size_t)col * ld + row;
+        const double r = ref[i], g = got[i];
+        if (!std::isfinite(r) || !std::isfinite(g)) { finite = false; continue; }
+        const double d = g - r;
+        if (!std::isfinite(d)) { finite = false; continue; }
+        d2 += (long double)d * d; r2 += (long double)r * r;
+        max_abs = std::max(max_abs, std::abs(d));
+    }
+    if (!finite) return {std::numeric_limits<double>::infinity(),
+                         std::numeric_limits<double>::infinity(), false, 0};
+    return {std::sqrt((double)(d2 / std::max(r2, 1e-300L))), max_abs, true, 0};
+}
+size_t padding_writes(const std::vector<float> &got, int n, int t, int ld) {
+    size_t writes = 0;
+    for (int col = 0; col < t; ++col) for (int row = n; row < ld; ++row)
+        if (got[(size_t)col * ld + row] != PADDING_CANARY) ++writes;
+    return writes;
+}
+bool accuracy_ok(const Err &e) {
+    return e.finite && e.padding_writes == 0 &&
+           e.rel_l2 <= REL_L2_TOL && e.max_abs <= MAX_ABS_TOL;
+}
+std::vector<std::string> csv(const std::string &s) {
+    std::vector<std::string> out; size_t b = 0;
+    for (;;) {
+        const size_t p = s.find(',', b);
+        out.push_back(s.substr(b, p == std::string::npos ? p : p - b));
+        if (p == std::string::npos) return out;
+        b = p + 1;
+    }
+}
+int positive(const std::string &s, const char *label) {
+    char *e = nullptr; const long v = std::strtol(s.c_str(), &e, 10);
+    if (s.empty() || !e || *e || v < 1 || v > std::numeric_limits<int>::max()) {
+        std::fprintf(stderr, "invalid %s: %s\n", label, s.c_str()); std::exit(2);
+    }
+    return (int)v;
+}
+void add(Options &o, bool bf16, int t, int n, int k, int ldy) {
+    if (ldy < n) { std::fprintf(stderr, "ldy must be >= N\n"); std::exit(2); }
+    o.shapes.push_back({t,n,k,ldy,bf16});
+}
+void add_shape(Options &o, const std::string &v) {
+    const auto f = csv(v);
+    if (f.size() != 3) { std::fprintf(stderr, "--shape expects T,N,K\n"); std::exit(2); }
+    int t=positive(f[0],"T"), n=positive(f[1],"N"), k=positive(f[2],"K");
+    add(o,false,t,n,k,n); add(o,true,t,n,k,n); o.explicit_shapes=true;
+}
+void add_case(Options &o, const std::string &v) {
+    const auto f = csv(v);
+    if (f.size()!=5 || (f[0]!="f16" && f[0]!="bf16")) {
+        std::fprintf(stderr, "--case expects dtype,T,N,K,ldy\n"); std::exit(2);
+    }
+    add(o,f[0]=="bf16",positive(f[1],"T"),positive(f[2],"N"),positive(f[3],"K"),positive(f[4],"ldy"));
+    o.explicit_shapes=true;
+}
+std::vector<int> parse_tokens(const std::string &s) {
+    std::vector<int> out;
+    for (const auto &x : csv(s)) out.push_back(positive(x,"token bucket"));
+    return out;
+}
+void load_shapes(Options &o) {
+    std::ifstream f(o.shapes_file);
+    if (!f) { std::fprintf(stderr, "cannot open shapes file: %s\n",o.shapes_file.c_str()); std::exit(2); }
+    const std::string text((std::istreambuf_iterator<char>(f)),{});
+    const std::regex obj(R"(\{[^{}]*\})"), dtype_re(R"re("dtype"\s*:\s*"(f16|bf16)")re"),
+        n_re(R"("N"\s*:\s*([0-9]+))"), k_re(R"("K"\s*:\s*([0-9]+))"),
+        ld_re(R"("ldy"\s*:\s*([0-9]+))");
+    int count=0;
+    for (auto i=std::sregex_iterator(text.begin(),text.end(),obj); i!=std::sregex_iterator(); ++i) {
+        const std::string item=i->str(); std::smatch d,n,k,ld;
+        if (!std::regex_search(item,d,dtype_re) || !std::regex_search(item,n,n_re) ||
+            !std::regex_search(item,k,k_re) || !std::regex_search(item,ld,ld_re)) continue;
+        for (int t:o.tokens) add(o,d[1]=="bf16",t,positive(n[1],"N"),positive(k[1],"K"),positive(ld[1],"ldy"));
+        ++count;
+    }
+    if (!count) { std::fprintf(stderr,"no dtype,N,K,ldy entries in %s\n",o.shapes_file.c_str()); std::exit(2); }
+    o.explicit_shapes=true;
+}
+void usage(const char *p) {
+    std::printf("Usage: %s [--workspace-mib N] [--shape T,N,K]... [--case dtype,T,N,K,ldy]...\n",p);
+    std::printf("       [--shapes-file PATH [--tokens T1,T2,...]] [--tuning-out PATH]\n");
+    std::printf("Defaults: original three shapes, f16+bf16, T=8192, workspace=32 MiB.\n");
+}
+Options options(int argc,char **argv) {
+    Options o;
+    for (int i=1;i<argc;++i) {
+        std::string a=argv[i];
+        auto next=[&]() -> std::string { if(i+1>=argc){std::fprintf(stderr,"missing option value\n");std::exit(2);} return argv[++i]; };
+        if(a=="-h"||a=="--help"){usage(argv[0]);std::exit(0);}
+        else if(a=="--workspace-mib"){
+            std::string v=next(); char *e=nullptr; unsigned long long x=std::strtoull(v.c_str(),&e,10);
+            if(v.empty()||!e||*e||x>std::numeric_limits<size_t>::max()/(1024ULL*1024ULL)){std::fprintf(stderr,"bad workspace MiB\n");std::exit(2);}
+            o.workspace=(size_t)x*1024U*1024U;
+        } else if(a=="--shape") add_shape(o,next());
+        else if(a=="--case") add_case(o,next());
+        else if(a=="--shapes-file"){o.shapes_file=next();o.explicit_shapes=true;}
+        else if(a=="--tokens") o.tokens=parse_tokens(next());
+        else if(a=="--tuning-out") o.tuning_out=next();
+        else {std::fprintf(stderr,"unknown option: %s\n",a.c_str());usage(argv[0]);std::exit(2);}
+    }
+    if(!o.shapes_file.empty()) load_shapes(o);
+    if(!o.explicit_shapes) {
+        struct DefaultShape { int n,k,ldy; bool bf16; };
+        const DefaultShape defs[]={{10240,2560,10240,false},{320,10240,320,false},{2560,320,2560,false},
+                                {10240,2560,10240,true},{320,10240,320,true},{2560,320,2560,true}};
+        for(const auto &s:defs) add(o,s.bf16,8192,s.n,s.k,s.ldy);
+    }
+    if(o.shapes.empty()){std::fprintf(stderr,"no shapes requested\n");std::exit(2);}
+    return o;
+}
+std::string arch_name(const char *s) {
+    std::string a=s?s:"unknown"; size_t p=a.find(':'); if(p!=std::string::npos)a.resize(p); return a;
+}
+void emit_candidate(const Shape &s,size_t workspace,int h,int id,size_t req,size_t algows,float ms,const Err &error,
+                    const std::string &arch,int version,const std::string &sol,const std::string &kernel,const std::string &config) {
+    std::cout<<"candidate_json={\"device_arch\":"<<json(arch)<<",\"hipblaslt_version\":"<<version
+      <<",\"dtype\":"<<json(dtype(s.bf16))<<",\"T\":"<<s.t<<",\"N\":"<<s.n<<",\"K\":"<<s.k<<",\"ldy\":"<<s.ldy
+      <<",\"workspace_limit_bytes\":"<<workspace<<",\"heuristic_index\":"<<h<<",\"solution_id\":"<<id
+      <<",\"required_workspace_bytes\":"<<req<<",\"algo_max_workspace_bytes\":"<<algows
+      <<",\"finite\":"<<(error.finite?"true":"false")<<",\"padding_writes\":"<<error.padding_writes
+      <<",\"relative_l2\":"<<std::setprecision(12)<<error.rel_l2<<",\"max_abs\":"<<error.max_abs
+      <<",\"relative_l2_tolerance\":"<<REL_L2_TOL<<",\"max_abs_tolerance\":"<<MAX_ABS_TOL
+      <<",\"mean_ms\":"<<std::setprecision(9)<<ms<<",\"solution_name\":"<<json(sol)
+      <<",\"kernel_name\":"<<json(kernel)<<",\"algo_config_hex\":"<<json(config)<<"}\n";
+}
+void emit_best(const Best &b,size_t workspace,const std::string &arch,int version,int hipver) {
+    const auto&s=b.s;
+    std::cout<<"best_json={\"device_arch\":"<<json(arch)<<",\"hipblaslt_version\":"<<version
+      <<",\"hip_runtime_version\":"<<hipver<<",\"dtype\":"<<json(dtype(s.bf16))
+      <<",\"T\":"<<s.t<<",\"N\":"<<s.n<<",\"K\":"<<s.k<<",\"ldy\":"<<s.ldy
+      <<",\"workspace_limit_bytes\":"<<workspace<<",\"heuristic_index\":"<<b.heuristic_index
+      <<",\"solution_id\":"<<b.solution_id<<",\"required_workspace_bytes\":"<<b.required_workspace
+      <<",\"algo_max_workspace_bytes\":"<<b.algo_workspace<<",\"mean_ms\":"<<std::setprecision(9)<<b.mean_ms
+      <<",\"finite\":"<<(b.error.finite?"true":"false")<<",\"padding_writes\":"<<b.error.padding_writes
+      <<",\"relative_l2\":"<<std::setprecision(12)<<b.error.rel_l2<<",\"max_abs\":"<<b.error.max_abs
+      <<",\"relative_l2_tolerance\":"<<REL_L2_TOL<<",\"max_abs_tolerance\":"<<MAX_ABS_TOL
+      <<",\"solution_name\":"<<json(b.solution)<<",\"kernel_name\":"<<json(b.kernel)
+      <<",\"algo_config_hex\":"<<json(b.config)<<"}\n";
+}
+
+void run_case(hipblasHandle_t blas,hipblasLtHandle_t lt,const Shape&s,int case_id,size_t ws,hipStream_t stream,
+              const std::string&arch,int version,int hipver,std::vector<Best>&bests) {
+    const size_t na=(size_t)s.k*s.n, nb=(size_t)s.k*s.t, out_elems=(size_t)s.ldy*s.t;
+    const hipDataType it=s.bf16?HIP_R_16BF:HIP_R_16F;
+    const hipblasOperation_t ta=HIPBLAS_OP_T,tb=HIPBLAS_OP_N;
+    const int m=s.n,n=s.t,k=s.k,lda=s.k,ldb=s.k,ldc=s.ldy;
+    const float alpha=1.f,beta=0.f;
+    std::vector<uint16_t> ha(na),hb(nb);
+    uint32_t seed=0x5A17C3U+(uint32_t)case_id*0x10001U+(s.bf16?0xB16U:0xF16U);
+    fill(ha,seed,s.bf16);fill(hb,seed^0x9E3779B9U,s.bf16);
+    Buffer da(na*sizeof(uint16_t)),db(nb*sizeof(uint16_t)),dc(out_elems*sizeof(float)),dy(out_elems*sizeof(float)),dws(ws);
+    const std::vector<float> canary(out_elems,PADDING_CANARY);
+    HIP_CHECK(hipMemcpyAsync(dc.p,canary.data(),out_elems*sizeof(float),hipMemcpyHostToDevice,stream));
+    HIP_CHECK(hipMemcpyAsync(da.p,ha.data(),na*sizeof(uint16_t),hipMemcpyHostToDevice,stream));
+    HIP_CHECK(hipMemcpyAsync(db.p,hb.data(),nb*sizeof(uint16_t),hipMemcpyHostToDevice,stream));
+    HIP_CHECK(hipStreamSynchronize(stream));
+
+    hipblasLtMatmulDesc_t op=nullptr;
+    hipblasLtMatrixLayout_t ad=nullptr,bd=nullptr,cd=nullptr;
+    hipblasLtMatmulPreference_t pref=nullptr;
+    LT_CHECK(hipblasLtMatmulDescCreate(&op,HIPBLAS_COMPUTE_32F,HIP_R_32F));
+    LT_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSA,&ta,sizeof(ta)));
+    LT_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSB,&tb,sizeof(tb)));
+    LT_CHECK(hipblasLtMatrixLayoutCreate(&ad,it,s.k,s.n,lda));
+    LT_CHECK(hipblasLtMatrixLayoutCreate(&bd,it,s.k,s.t,ldb));
+    LT_CHECK(hipblasLtMatrixLayoutCreate(&cd,HIP_R_32F,s.n,s.t,ldc));
+    LT_CHECK(hipblasLtMatmulPreferenceCreate(&pref));
+    const uint64_t limit=ws;
+    LT_CHECK(hipblasLtMatmulPreferenceSetAttribute(pref,HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES,&limit,sizeof(limit)));
+
+    BLAS_CHECK(hipblasSetStream(blas,stream));
+    auto base=[&]{BLAS_CHECK(hipblasGemmEx(blas,ta,tb,m,n,k,&alpha,da.p,it,lda,db.p,it,ldb,&beta,dc.p,HIP_R_32F,ldc,HIPBLAS_COMPUTE_32F,HIPBLAS_GEMM_DEFAULT));};
+    for(int i=0;i<WARMUPS;++i){base();HIP_CHECK(hipDeviceSynchronize());}
+    float base_ms[REPS]{};
+    for(int i=0;i<REPS;++i)base_ms[i]=time_call(stream,base);
+    std::vector<float> ref(out_elems);
+    HIP_CHECK(hipMemcpyAsync(ref.data(),dc.p,out_elems*sizeof(float),hipMemcpyDeviceToHost,stream));
+    HIP_CHECK(hipStreamSynchronize(stream));
+    const size_t baseline_padding=padding_writes(ref,s.n,s.t,s.ldy);
+    std::printf("baseline dtype=%s T=%d N=%d K=%d ldy=%d padding_writes=%zu\n",dtype(s.bf16),s.t,s.n,s.k,s.ldy,baseline_padding);
+    float base_mean=(base_ms[0]+base_ms[1]+base_ms[2])/REPS;
+
+    hipblasLtMatmulHeuristicResult_t hs[MAX_ALGOS]{};
+    int count=0;
+    const hipblasStatus_t query=hipblasLtMatmulAlgoGetHeuristic(lt,op,ad,bd,cd,cd,pref,MAX_ALGOS,hs,&count);
+    std::printf("case dtype=%s T=%d N=%d K=%d ldy=%d workspace_limit_bytes=%zu baseline=hipblasGemmEx rep1_ms=%.4f rep2_ms=%.4f rep3_ms=%.4f mean_ms=%.4f\n",
+      dtype(s.bf16),s.t,s.n,s.k,s.ldy,ws,base_ms[0],base_ms[1],base_ms[2],base_mean);
+    if(query!=HIPBLAS_STATUS_SUCCESS||count==0){
+        std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d status=%s(%d) heuristic_count=%d\n",
+          dtype(s.bf16),s.t,s.n,s.k,s.ldy,status_name(query),(int)query,count);
+    } else {
+        int best_h=-1,best_id=-1; float best_ms=std::numeric_limits<float>::infinity();
+        size_t best_req=0,best_maxws=0; std::string best_sol,best_kernel,best_config;
+        Err best_error{std::numeric_limits<double>::infinity(),std::numeric_limits<double>::infinity(),false,0};
+        for(int h=0;h<count;++h){
+            if(hs[h].state!=HIPBLAS_STATUS_SUCCESS||hs[h].workspaceSize>ws){
+                std::printf("lt_skip dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d status=%s(%d) required_workspace_bytes=%zu\n",
+                  dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,status_name(hs[h].state),(int)hs[h].state,hs[h].workspaceSize);
+                continue;
+            }
+            auto algo=hs[h].algo;
+            const int id=hipblaslt_ext::getIndexFromAlgo(algo);
+            const std::string sol=hipblaslt_ext::getSolutionNameFromAlgo(lt,algo);
+            const std::string kernel=hipblaslt_ext::getKernelNameFromAlgo(lt,algo);
+            const std::string config=config_hex(algo);
+            auto run=[&]{LT_CHECK(hipblasLtMatmul(lt,op,&alpha,da.p,ad,db.p,bd,&beta,dy.p,cd,dy.p,cd,&algo,dws.p,ws,stream));};
+
+            HIP_CHECK(hipMemcpyAsync(dy.p,canary.data(),out_elems*sizeof(float),hipMemcpyHostToDevice,stream));
+            HIP_CHECK(hipStreamSynchronize(stream));
+            const hipblasStatus_t check_status=hipblasLtMatmul(lt,op,&alpha,da.p,ad,db.p,bd,&beta,dy.p,cd,dy.p,cd,&algo,dws.p,ws,stream);
+            if(check_status!=HIPBLAS_STATUS_SUCCESS){
+                std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d status=%s(%d) phase=accuracy_check\n",
+                  dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,status_name(check_status),(int)check_status);continue;
+            }
+            HIP_CHECK(hipStreamSynchronize(stream));
+            std::vector<float> checked_y(out_elems);
+            HIP_CHECK(hipMemcpyAsync(checked_y.data(),dy.p,out_elems*sizeof(float),hipMemcpyDeviceToHost,stream));
+            HIP_CHECK(hipStreamSynchronize(stream));
+            Err error=compare(ref,checked_y,s.n,s.t,s.ldy);
+            error.padding_writes=padding_writes(checked_y,s.n,s.t,s.ldy);
+            if(!accuracy_ok(error)){
+                std::printf("lt_reject dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d finite=%s relative_l2=%.12g relative_l2_tolerance=%.3g max_abs=%.12g max_abs_tolerance=%.3g padding_writes=%zu reason=accuracy_gate\n",
+                  dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,error.finite?"true":"false",
+                  error.rel_l2,REL_L2_TOL,error.max_abs,MAX_ABS_TOL,error.padding_writes);continue;
+            }
+
+            bool failed=false;
+            for(int i=0;i<WARMUPS;++i){
+                const hipblasStatus_t st=hipblasLtMatmul(lt,op,&alpha,da.p,ad,db.p,bd,&beta,dy.p,cd,dy.p,cd,&algo,dws.p,ws,stream);
+                if(st!=HIPBLAS_STATUS_SUCCESS){
+                    std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d status=%s(%d)\n",
+                      dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,status_name(st),(int)st);failed=true;break;
+                }
+                HIP_CHECK(hipDeviceSynchronize());
+            }
+            if(failed)continue;
+            float ms[REPS]{};
+            for(int i=0;i<REPS;++i)ms[i]=time_call(stream,run);
+            const float mean=(ms[0]+ms[1]+ms[2])/REPS;
+            std::printf("lt dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d solution=%s kernel=%s algo_config_hex=%s required_workspace_bytes=%zu algo_max_workspace_bytes=%zu finite=%s relative_l2=%.12g max_abs=%.12g padding_writes=%zu rep1_ms=%.4f rep2_ms=%.4f rep3_ms=%.4f mean_ms=%.4f\n",
+              dtype(s.bf16),s.t,s.n,s.k,s.ldy,h,id,sol.c_str(),kernel.c_str(),config.c_str(),hs[h].workspaceSize,algo.max_workspace_bytes,
+              error.finite?"true":"false",error.rel_l2,error.max_abs,error.padding_writes,ms[0],ms[1],ms[2],mean);
+            emit_candidate(s,ws,h,id,hs[h].workspaceSize,algo.max_workspace_bytes,mean,error,arch,version,sol,kernel,config);
+            if(mean<best_ms){
+                best_h=h;best_id=id;best_ms=mean;best_req=hs[h].workspaceSize;best_maxws=algo.max_workspace_bytes;
+                best_sol=sol;best_kernel=kernel;best_config=config;best_error=error;
+            }
+        }
+        if(best_h<0)std::printf("lt_unsupported dtype=%s T=%d N=%d K=%d ldy=%d status=no_valid_heuristics heuristic_count=%d\n",
+                                dtype(s.bf16),s.t,s.n,s.k,s.ldy,count);
+        else{
+            Best b{s,best_h,best_id,best_req,best_maxws,best_ms,best_error,best_sol,best_kernel,best_config};
+            bests.push_back(b);
+            std::printf("lt_best dtype=%s T=%d N=%d K=%d ldy=%d heuristic_index=%d solution_id=%d solution=%s kernel=%s algo_config_hex=%s required_workspace_bytes=%zu algo_max_workspace_bytes=%zu mean_ms=%.4f speedup_vs_blas=%.4fx finite=%s relative_l2=%.9g max_abs=%.9g padding_writes=%zu\n",
+              dtype(s.bf16),s.t,s.n,s.k,s.ldy,best_h,best_id,best_sol.c_str(),best_kernel.c_str(),best_config.c_str(),best_req,best_maxws,best_ms,base_mean/best_ms,
+              best_error.finite?"true":"false",best_error.rel_l2,best_error.max_abs,best_error.padding_writes);
+            emit_best(b,ws,arch,version,hipver);
+        }
+    }
+    (void)hipblasLtMatmulPreferenceDestroy(pref);
+    (void)hipblasLtMatrixLayoutDestroy(cd);(void)hipblasLtMatrixLayoutDestroy(bd);(void)hipblasLtMatrixLayoutDestroy(ad);
+    (void)hipblasLtMatmulDescDestroy(op);
+}
+void tuning_file(const std::string &path,const std::string &arch,int version,const std::vector<Best>&rows){
+    if(path.empty())return;
+    std::ofstream f(path);
+    if(!f){std::fprintf(stderr,"cannot write tuning output: %s\n",path.c_str());std::exit(2);}
+    f<<"# solution IDs are scoped to this hipBLASLt version and device architecture\n";
+    f<<"STRATA_HIPBLASLT_TUNING_V1 "<<arch<<" "<<version<<"\n";
+    for(const auto &b:rows)f<<dtype(b.s.bf16)<<" "<<b.s.n<<" "<<b.s.k<<" "<<b.s.ldy<<" "<<b.s.t<<" "<<b.solution_id<<"\n";
+}
+} // namespace
+
+int main(int argc,char **argv){
+    Options o=options(argc,argv);
+    hipblasHandle_t blas=nullptr;hipblasLtHandle_t lt=nullptr;
+    BLAS_CHECK(hipblasCreate(&blas));LT_CHECK(hipblasLtCreate(&lt));
+    int device=0,hipver=0,version=0;hipDeviceProp_t prop{};
+    HIP_CHECK(hipGetDevice(&device));HIP_CHECK(hipGetDeviceProperties(&prop,device));HIP_CHECK(hipRuntimeGetVersion(&hipver));
+    const hipblasStatus_t vs=hipblasLtGetVersion(lt,&version);
+    if(vs!=HIPBLAS_STATUS_SUCCESS){std::fprintf(stderr,"hipblasLtGetVersion failed: %d\n",(int)vs);return 2;}
+    const std::string arch=arch_name(prop.gcnArchName);
+    std::printf("meta device_arch=%s hipblaslt_version=%d hipblaslt_header_version=%d.%d.%d hip_runtime_version=%d\n",
+      arch.c_str(),version,HIPBLASLT_VERSION_MAJOR,HIPBLASLT_VERSION_MINOR,HIPBLASLT_VERSION_PATCH,hipver);
+    hipStream_t stream=nullptr;HIP_CHECK(hipStreamCreateWithFlags(&stream,hipStreamNonBlocking));
+    std::vector<Best>bests;
+    for(size_t i=0;i<o.shapes.size();++i)run_case(blas,lt,o.shapes[i],(int)i,o.workspace,stream,arch,version,hipver,bests);
+    tuning_file(o.tuning_out,arch,version,bests);
+    HIP_CHECK(hipStreamSynchronize(stream));HIP_CHECK(hipStreamDestroy(stream));
+    LT_CHECK(hipblasLtDestroy(lt));BLAS_CHECK(hipblasDestroy(blas));
+    return 0;
+}