Engine 0.1.28: the expert cache reserves the draft head's VRAM (0.1.27 left 16 GB cards below the stall line), a cancelled request no longer fails the next one, the API key protects /status; setup requires it

Squashed from PR #153.

.gitignore

@@ -46,3 +46,4 @@ build-vision.bat
 /dist/
 *.done
 /third_party/*.zip
+

CMakeLists.txt

@@ -4,11 +4,18 @@
 # `strata-pack` are added when their sources exist - a target with no sources would make the build graph
 # claim something the tree does not have, which is worse than an absent target.
 #
-# CUDA targets arrive in Phase 2 and this build REJECTS any architecture other than 120: the engine is
-# written against sm_120 (RTX 5000 / Blackwell) and silently building for an older arch would produce a
-# binary that runs and is wrong about its own performance, which is the worst of both outcomes.
+# CUDA targets arrive in Phase 2.  The kernels need sm_80 or newer (tf32 mma, bf16 math) and the guard
+# below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
+# and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.0 LANGUAGES CXX)
+project(strata VERSION 0.1.28 LANGUAGES CXX)
+add_compile_definitions(STRATA_VERSION="${PROJECT_VERSION}")
+# issue #31: release builds keep a PDB beside the exe (not shipped in it), so a minidump from a stalled engine can
+# be read back to the exact source line
+if(MSVC AND CMAKE_BUILD_TYPE STREQUAL "Release")
+  add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/Zi>)
+  add_link_options(/DEBUG /OPT:REF /OPT:ICF)
+endif()
 
 # The prebuilt release (setup.py downloads it, so users need no compiler): ggml's CPU backend for any AVX2 CPU
 # instead of this machine's (Strata's own AVX-512 kernels are per-file and chosen at run time either way), and
@@ -26,6 +33,17 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 option(STRATA_ENABLE_CUDA "Build the CUDA targets (Phase 2+)" OFF)
+option(STRATA_ENABLE_HIP "Build the HIP targets for gfx1100 wave32 hardware" OFF)
+option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF)
+if(STRATA_PREFILL_MMQ AND NOT STRATA_ENABLE_HIP)
+  message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_ENABLE_HIP=ON")
+endif()
+if(STRATA_ENABLE_CUDA AND STRATA_ENABLE_HIP)
+  message(FATAL_ERROR "STRATA_ENABLE_CUDA and STRATA_ENABLE_HIP are separate backends; enable only one")
+endif()
+if(STRATA_ENABLE_HIP)
+  include(cmake/hip_backend.cmake)
+endif()
 # The test suite and the micro benchmarks live in tests/ and bench/micro/, which the published source leaves out.
 set(_strata_tests_default OFF)
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/bench/micro)
@@ -44,18 +62,26 @@ if(STRATA_ENABLE_CUDA)
   endif()
   foreach(_a IN LISTS CMAKE_CUDA_ARCHITECTURES)
     string(REGEX REPLACE "-.*$" "" _base "${_a}")     # 120-real / 120-virtual are still 120
-    # Plan v0.3 P6: developed and measured on sm_120 (RTX 50); the kernels need sm_80 or newer (tf32 mma in
-    # the attention scorer, bf16 math), so RTX 30 (86) and RTX 40 (89) build too.  Older cards are refused.
+    # Turing port: developed and measured on sm_120 (RTX 50); the tf32 mma in the QSA scorer has a
+    # portable fp32-FMA fallback, so RTX 20 (75), 30 (86), RTX 40 (89) and RTX 50 (120) build too.
+    # Pre-sm_75 is refused.
     if(_base STREQUAL "native" OR _base STREQUAL "all" OR _base STREQUAL "all-major")
       continue()
     endif()
-    if(_base LESS 80)
+    if(_base LESS 75)
       message(FATAL_ERROR
-        "Strata needs an NVIDIA GPU of compute capability 8.0 or newer (RTX 30 / 40 / 50 series), but "
+        "Strata needs an NVIDIA GPU of compute capability 7.5 or newer (RTX 20 / 30 / 40 / 50 series), but "
         "CMAKE_CUDA_ARCHITECTURES is '${CMAKE_CUDA_ARCHITECTURES}'.")
     endif()
   endforeach()
   message(STATUS "Strata: CUDA enabled, arch ${CMAKE_CUDA_ARCHITECTURES}")
+  set(_strata_gpu_runtime_target CUDA::cudart)
+  set(_strata_gpu_blas_target CUDA::cublas)
+  set(_strata_gpu_include_directories ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+elseif(STRATA_ENABLE_HIP)
+  set(_strata_gpu_runtime_target strata_hip_runtime)
+  set(_strata_gpu_blas_target roc::hipblas)
+  set(_strata_gpu_include_directories ${STRATA_HIP_COMPAT_INCLUDE_DIR})
 endif()
 
 # Warnings live on an INTERFACE target the project's own code links, NOT in add_compile_options: applied
@@ -71,11 +97,14 @@ target_compile_options(strata_warnings INTERFACE
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/utf-8>
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Wall>
   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Wextra>
-  $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CXX_COMPILER_ID:MSVC>>:-Xcompiler=/W4>)
+  $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CXX_COMPILER_ID:MSVC>>:-Xcompiler=/W4>
+  $<$<COMPILE_LANGUAGE:HIP>:-Wall>
+  $<$<COMPILE_LANGUAGE:HIP>:-Wextra>)
 if(STRATA_WERROR)
   target_compile_options(strata_warnings INTERFACE
     $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/WX>
-    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Werror>)
+    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:-Werror>
+    $<$<COMPILE_LANGUAGE:HIP>:-Werror>)
 endif()
 
 # ---- strata_artifact -------------------------------------------------------
@@ -93,6 +122,12 @@ add_executable(strata-dequant src/artifact/dequant.cpp)
 target_link_libraries(strata-dequant PRIVATE strata_artifact)
 # this file is now nothing but the CLI entry point, and its inherited guard spells that as a "selftest"
 target_compile_definitions(strata-dequant PRIVATE STRATA_DEQUANT_SELFTEST)
+if(STRATA_BUILD_TESTS)
+  # the reader refuses a duplicate tensor name at open, naming it (a synthetic GGUF: no model, no GPU)
+  add_executable(gguf_reader_test src/artifact/gguf_reader_test.cpp)
+  target_link_libraries(gguf_reader_test PRIVATE strata_artifact)
+  add_test(NAME gguf_reader_test COMMAND gguf_reader_test)
+endif()
 
 # ---- strata_plan ----------------------------------------------------------
 # Also header-only.  A "planner" that hides state would be one whose decision is not inspectable, and the
@@ -104,17 +139,16 @@ target_link_libraries(strata_plan INTERFACE strata_warnings)
 add_executable(strata-plan src/plan/plan_main.cpp)
 target_link_libraries(strata-plan PRIVATE strata_plan)
 
-# ---- strata_core (CUDA, Phase 2) ------------------------------------------
-# Only built when CUDA is on, so the Phase 1 targets keep working on a machine with no toolkit.  The arch
-# guard above has already refused anything but sm_120 by the time this runs.
-if(STRATA_ENABLE_CUDA)
+# ---- strata_core (GPU runtime) ---------------------------------------------
+# Only built when a GPU backend is on, so the Phase 1 targets keep working without a toolkit.
+if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   add_library(strata_core STATIC src/core/device.cu src/core/pinned.cu src/platform/memory.cpp src/core/graph.cpp src/core/weights.cpp
                                src/core/layout.cpp)
   target_include_directories(strata_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-  # CUDA::cudart is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and
+  # The selected runtime is PUBLIC because graph.cpp is a HOST translation unit that calls cudaGraphLaunch and
   # cudaEventQuery: the .cu sources get cudart through the CUDA language rules, a .cpp does not, and the
   # linker error ("unresolved external symbol cudaGraphInstantiate") names the function rather than the cause.
-  target_link_libraries(strata_core PUBLIC strata_plan strata_warnings CUDA::cudart)
+  target_link_libraries(strata_core PUBLIC strata_plan strata_warnings ${_strata_gpu_runtime_target})
 
   add_executable(strata-device src/core/device_main.cpp)
   target_link_libraries(strata-device PRIVATE strata_core)
@@ -123,13 +157,13 @@ if(STRATA_ENABLE_CUDA)
   target_link_libraries(strata-load PRIVATE strata_core strata_kernels)
 
   add_executable(strata-concurrent src/core/concurrent_main.cpp)
-  target_include_directories(strata-concurrent PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels CUDA::cudart)
+  target_include_directories(strata-concurrent PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata-concurrent PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target})
 
   add_executable(strata-overlap src/core/overlap_main.cpp)
   # a .cpp that calls cudaMemcpyAsync itself needs the toolkit headers and cudart explicitly
-  target_include_directories(strata-overlap PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels CUDA::cudart)
+  target_include_directories(strata-overlap PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata-overlap PRIVATE strata_core strata_kernels ${_strata_gpu_runtime_target})
 
   # The load test needs the 34 GB artifact, so it is registered ONLY where that artifact exists.  A test that
   # silently skips is a test that stops running; an absent registration is visible in the configure output.
@@ -178,8 +212,11 @@ if(STRATA_ENABLE_CUDA)
                           src/kernels/cuda/ple.cu
                           src/kernels/cuda/kv_q8.cu
                           src/kernels/cuda/kv_stream.cu
+                          src/kernels/cuda/kv_q4.cu
                           src/kernels/cuda/fused_gr.cu
+                          src/kernels/cuda/cvec.cu
                           src/kernels/cuda/qsa_decode_attn.cu
+                          src/kernels/cuda/qsa_prompt_attn.cu
                           src/kernels/cuda/fused_gdn.cu
                           src/kernels/cuda/dequant_bf16.cu
                           src/kernels/cuda/qsa_select.cu
@@ -189,33 +226,48 @@ if(STRATA_ENABLE_CUDA)
                           src/ngram/ple_reader.cpp
                           src/platform/direct_file.cpp)
   target_link_libraries(strata_kernels PUBLIC strata_core strata_artifact)
+  # perf-review D-1: the tensor-core prompt attention vs the FP32 kernel and FP64 (GPU, synthetic, no model); with
+  # the tests, or alone with -DSTRATA_PARITY_PROMPT_ATTN=ON
+  if(STRATA_BUILD_TESTS OR STRATA_PARITY_PROMPT_ATTN)
+    add_executable(qsa_prompt_attn_parity src/kernels/qsa_prompt_attn_parity.cpp)
+    target_include_directories(qsa_prompt_attn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+    target_link_libraries(qsa_prompt_attn_parity PRIVATE strata_kernels CUDA::cudart)
+  endif()
   # plan v0.3 P6: llama.cpp's block layouts and codebook grids for the i-quant formats (MIT, third_party/ggml)
   target_include_directories(strata_kernels PRIVATE third_party/ggml)
   # Match the pinned llama.cpp CUDA arithmetic contract for this adapter only.
-  set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu
-    src/kernels/cuda/native_gr_norm.cu
-    src/kernels/cuda/native_moe.cu
-    src/kernels/cuda/native_gdn.cu
-    src/kernels/cuda/native_gr_postops.cu
-    src/kernels/cuda/native_router.cu
-    src/kernels/cuda/native_gdn_preprocess.cu
-    src/kernels/cuda/native_qsa.cu
-    src/kernels/cuda/native_rope.cu
-    src/kernels/cuda/native_ple_postops.cu
-    src/kernels/cuda/native_flash_attn.cu
-    src/kernels/cuda/native_qsa_indexer.cu
-    src/kernels/cuda/native_qsa_score.cu
-    PROPERTIES COMPILE_OPTIONS "--use_fast_math")
+  if(STRATA_ENABLE_CUDA)
+    set_source_files_properties(src/kernels/cuda/native_mmvq.cu src/kernels/cuda/native_bf16.cu
+      src/kernels/cuda/native_gr_norm.cu
+      src/kernels/cuda/native_moe.cu
+      src/kernels/cuda/native_gdn.cu
+      src/kernels/cuda/native_gr_postops.cu
+      src/kernels/cuda/native_router.cu
+      src/kernels/cuda/native_gdn_preprocess.cu
+      src/kernels/cuda/native_qsa.cu
+      src/kernels/cuda/native_rope.cu
+      src/kernels/cuda/native_ple_postops.cu
+      src/kernels/cuda/native_flash_attn.cu
+      src/kernels/cuda/native_qsa_indexer.cu
+      src/kernels/cuda/native_qsa_score.cu
+      PROPERTIES COMPILE_OPTIONS "--use_fast_math")
+  elseif(STRATA_ENABLE_HIP)
+    # Keep Q8_K's multiply and magic-bias rounding as separate FP32 operations on HIP; contraction to an
+    # FMA changes tie results. The embedding gather also relies on separately rounded operations.
+    set_source_files_properties(src/kernels/cuda/elementwise.cu src/kernels/cuda/quantize_act.cu
+      PROPERTIES COMPILE_OPTIONS "-ffp-contract=off")
+  endif()
 
   set(STRATA_ORACLE_SOURCE_DIR "" CACHE PATH "Pinned llama.cpp source for independent CUDA parity tests")
   set(STRATA_ORACLE_BUILD_DIR "" CACHE PATH "Matching existing shared llama.cpp build")
-  if(STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR)
+  # This oracle is a CUDA ggml build; do not pull CUDA libraries into a HIP build.
+  if(STRATA_ENABLE_CUDA AND STRATA_ORACLE_SOURCE_DIR AND STRATA_ORACLE_BUILD_DIR)
     foreach(parity IN ITEMS native_mmvq native_bf16 native_shared native_gr_norm native_moe native_gdn native_gr_postops native_router native_gdn_preprocess native_gdn_pipeline native_qsa native_rope native_ple_postops native_flash_attn native_qsa_indexer native_qsa_score)
     add_executable(${parity}_parity bench/micro/${parity}_parity.cpp)
     target_include_directories(${parity}_parity PRIVATE
-      "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+      "${STRATA_ORACLE_SOURCE_DIR}/ggml/include" ${_strata_gpu_include_directories})
     target_compile_definitions(${parity}_parity PRIVATE GGML_SHARED GGML_BACKEND_SHARED)
-    target_link_libraries(${parity}_parity PRIVATE strata_kernels CUDA::cudart)
+    target_link_libraries(${parity}_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
     foreach(component IN ITEMS ggml ggml-base ggml-cpu ggml-cuda)
       string(REPLACE "-" "_" variable "STRATA_ORACLE_${component}")
       find_library(${variable} NAMES ${component} PATHS
@@ -236,102 +288,166 @@ if(STRATA_ENABLE_CUDA)
   # already links `strata_core` - putting `gdn_layer` in core would make the two mutually dependent.  A third
   # library is the honest shape of the dependency graph: core knows about devices and weights, kernels know
   # about arithmetic, and this one knows the ORDER the arithmetic happens in.
-  add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp
+  add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
     src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-  target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu CUDA::cudart)
+  target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/core/file_expert_source_test.cpp")
+    enable_testing()
+    add_executable(file_expert_source_test tests/core/file_expert_source_test.cpp)
+    target_link_libraries(file_expert_source_test PRIVATE strata_engine)
+    add_test(NAME file_expert_source_test COMMAND file_expert_source_test)
+    set_tests_properties(file_expert_source_test PROPERTIES TIMEOUT 60)
+  endif()
 
   # `strata generate` (P2.S6).  The FIRST user-facing program: everything under it is a component, and this is
   # what composes them into a token.  No tokenizer - the prompt arrives as ids - because Gate C1 compares logits
   # against llama.cpp on the SAME ids and a tokenizer on one side of that comparison is a second variable.
   # plan v0.3 P5: batched prompt processing (tensor-core GEMMs through cuBLAS).
   add_library(strata_prefill STATIC src/prefill/gemm.cu src/prefill/kernels.cu src/prefill/prefill.cpp)
-  target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata_prefill PUBLIC strata_engine CUDA::cublas CUDA::cudart)
+  target_include_directories(strata_prefill PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})
+  target_link_libraries(strata_prefill PUBLIC strata_engine ${_strata_gpu_blas_target} ${_strata_gpu_runtime_target})
+  if(STRATA_ENABLE_HIP AND STRATA_HIPBLASLT_AVAILABLE)
+    target_compile_definitions(strata_prefill PRIVATE STRATA_HIPBLASLT_AVAILABLE=1)
+    target_link_libraries(strata_prefill PRIVATE roc::hipblaslt)
+
+    add_executable(tune_hipblaslt tools/hip/tune_hipblaslt.cpp)
+    target_link_libraries(tune_hipblaslt PRIVATE hip::host roc::hipblas roc::hipblaslt)
+  endif()
 
   add_executable(strata src/program/generate.cpp)
-  target_include_directories(strata PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec CUDA::cudart)
+  target_include_directories(strata PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec ${_strata_gpu_runtime_target})
+
+  if(STRATA_ENABLE_HIP)
+    enable_testing()
+    set(_strata_hip_tests)
+    add_executable(hip_expert_cache_staging tests/hip/expert_cache_staging.cpp)
+    target_link_libraries(hip_expert_cache_staging PRIVATE strata_engine)
+    add_test(NAME hip_expert_cache_staging COMMAND hip_expert_cache_staging)
+    list(APPEND _strata_hip_tests hip_expert_cache_staging)
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/intrinsics.cpp")
+      add_executable(hip_intrinsics tests/hip/intrinsics.cpp)
+      target_link_libraries(hip_intrinsics PRIVATE strata_hip_runtime)
+      add_test(NAME hip_intrinsics COMMAND hip_intrinsics)
+      list(APPEND _strata_hip_tests hip_intrinsics)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/handoff.cpp")
+      add_executable(hip_handoff tests/hip/handoff.cpp)
+      target_link_libraries(hip_handoff PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_handoff COMMAND hip_handoff)
+      list(APPEND _strata_hip_tests hip_handoff)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/native_qsa_score.cpp")
+      add_executable(hip_native_qsa_score tests/hip/native_qsa_score.cpp)
+      target_link_libraries(hip_native_qsa_score PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_native_qsa_score COMMAND hip_native_qsa_score)
+      list(APPEND _strata_hip_tests hip_native_qsa_score)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_native_batch.cpp")
+      add_executable(hip_prefill_native_batch tests/hip/prefill_native_batch.cpp)
+      target_link_libraries(hip_prefill_native_batch PRIVATE strata_kernels strata_hip_runtime)
+      add_test(NAME hip_prefill_native_batch COMMAND hip_prefill_native_batch)
+      list(APPEND _strata_hip_tests hip_prefill_native_batch)
+    endif()
+    if(STRATA_HIPBLASLT_AVAILABLE AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/hip/prefill_hipblaslt_gemm.cpp")
+      add_executable(hip_prefill_hipblaslt_gemm tests/hip/prefill_hipblaslt_gemm.cpp)
+      target_include_directories(hip_prefill_hipblaslt_gemm PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/prefill")
+      target_link_libraries(hip_prefill_hipblaslt_gemm PRIVATE strata_prefill strata_hip_runtime roc::hipblaslt roc::hipblas)
+      add_test(NAME hip_prefill_hipblaslt_gemm COMMAND hip_prefill_hipblaslt_gemm)
+      set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77)
+      list(APPEND _strata_hip_tests hip_prefill_hipblaslt_gemm)
+    endif()
+    if(_strata_hip_tests)
+      set_tests_properties(${_strata_hip_tests} PROPERTIES TIMEOUT 60)
+    endif()
+    if(TARGET hip_prefill_native_batch)
+      set_tests_properties(hip_prefill_native_batch PROPERTIES TIMEOUT 120)
+    endif()
+    if(TARGET hip_prefill_hipblaslt_gemm)
+      set_tests_properties(hip_prefill_hipblaslt_gemm PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77)
+    endif()
+  endif()
 
   add_executable(dequant_s2_parity src/kernels/dequant_s2_parity.cpp)
   # The tests are HOST C++ files that call cudaMalloc/cudaMemcpy themselves, so they need the toolkit headers
   # and cudart explicitly: enabling the CUDA language gives those to .cu sources automatically, but a .cpp
   # target is just a C++ target and gets neither.
-  target_include_directories(dequant_s2_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(dequant_s2_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(dequant_s2_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(dequant_s2_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME dequant_s2_parity COMMAND dequant_s2_parity --selftest)
 
   add_executable(s2_gemv_parity src/kernels/s2_gemv_parity.cpp)
-  target_include_directories(s2_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s2_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s2_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s2_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s2_gemv_parity COMMAND s2_gemv_parity --selftest)
 
   add_executable(iq_parity src/kernels/iq_parity.cpp)
-  target_include_directories(iq_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(iq_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(iq_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(iq_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)
-  target_include_directories(shared_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(shared_expert_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME shared_expert_parity COMMAND shared_expert_parity --selftest)
 
   add_executable(gr_parity src/kernels/gr_parity.cpp)
-  target_include_directories(gr_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(gr_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(gr_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(gr_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME gr_parity COMMAND gr_parity --selftest)
 
   add_executable(gdn_parity src/kernels/gdn_parity.cpp)
-  target_include_directories(gdn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(gdn_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(gdn_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(gdn_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME gdn_parity COMMAND gdn_parity --selftest)
 
   add_executable(s2_gemv_q8_parity src/kernels/s2_gemv_q8_parity.cpp)
-  target_include_directories(s2_gemv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s2_gemv_q8_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s2_gemv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s2_gemv_q8_parity COMMAND s2_gemv_q8_parity --selftest)
 
   add_executable(sampler_parity src/kernels/sampler_parity.cpp)
-  target_include_directories(sampler_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(sampler_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(sampler_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(sampler_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME sampler_parity COMMAND sampler_parity --selftest)
 
   add_executable(rope_parity src/kernels/rope_parity.cpp)
-  target_include_directories(rope_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(rope_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(rope_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(rope_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME rope_parity COMMAND rope_parity --selftest)
 
   add_executable(quantize_act_parity src/kernels/quantize_act_parity.cpp)
-  target_include_directories(quantize_act_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(quantize_act_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(quantize_act_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(quantize_act_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME quantize_act_parity COMMAND quantize_act_parity --selftest)
 
   add_executable(router_top10_parity src/kernels/router_top10_parity.cpp)
-  target_include_directories(router_top10_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(router_top10_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(router_top10_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(router_top10_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME router_top10_parity COMMAND router_top10_parity --selftest)
 
   add_executable(s_gemv_parity src/kernels/s_gemv_parity.cpp)
-  target_include_directories(s_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s_gemv_parity COMMAND s_gemv_parity --selftest)
 
   add_executable(elementwise_parity src/kernels/elementwise_parity.cpp)
-  target_include_directories(elementwise_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(elementwise_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(elementwise_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(elementwise_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME elementwise_parity COMMAND elementwise_parity --selftest)
 
   add_executable(bf16_gemv_parity src/kernels/bf16_gemv_parity.cpp)
-  target_include_directories(bf16_gemv_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(bf16_gemv_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(bf16_gemv_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME bf16_gemv_parity COMMAND bf16_gemv_parity --selftest)
 
   add_executable(s_gemv_q8k_parity src/kernels/s_gemv_q8k_parity.cpp)
-  target_include_directories(s_gemv_q8k_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(s_gemv_q8k_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(s_gemv_q8k_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME s_gemv_q8k_parity COMMAND s_gemv_q8k_parity --selftest)
 
   add_executable(qsa_parity src/kernels/qsa_parity.cpp)
-  target_include_directories(qsa_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(qsa_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(qsa_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(qsa_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME qsa_parity COMMAND qsa_parity --selftest)
 
   # ---- P2.S4: the PLE n-gram hash, the IQ4_NL table read, and the PLE block ----
@@ -343,8 +459,8 @@ if(STRATA_ENABLE_CUDA)
   # INSIDE the test.  A test that passes by finding nothing to compare is the failure mode this project keeps
   # re-learning.
   add_executable(ple_parity src/kernels/ple_parity.cpp)
-  target_include_directories(ple_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(ple_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(ple_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(ple_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   set(STRATA_PLE_FIXTURE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro" CACHE PATH
       "Directory containing required ple_in.bin and ple_out.bin parity fixtures")
   add_test(NAME ple_parity COMMAND ple_parity --selftest
@@ -353,63 +469,86 @@ if(STRATA_ENABLE_CUDA)
 
   # Plan v0.3 P7: INT8 KV storage vs a host reference and vs the FP16 path (GPU, synthetic, no model).
   add_executable(kv_q8_parity src/kernels/kv_q8_parity.cpp)
-  target_include_directories(kv_q8_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_q8_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_q8_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_q8_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_q8_parity COMMAND kv_q8_parity)
 
   # KV streaming (docs/kv-streaming-design.md): streamed vs resident attention, bitwise (GPU, synthetic, no model).
   add_executable(kv_stream_parity src/kernels/kv_stream_parity.cpp)
-  target_include_directories(kv_stream_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(kv_stream_parity PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(kv_stream_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_stream_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME kv_stream_parity COMMAND kv_stream_parity)
 
+  # PR #120: the hybrid K8V4 KV (INT8 K + rotated Q4_0 V) vs host references (GPU, synthetic, no model).
+  add_executable(kv_hybrid_parity src/kernels/kv_hybrid_parity.cpp)
+  target_include_directories(kv_hybrid_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_hybrid_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  add_test(NAME kv_hybrid_parity COMMAND kv_hybrid_parity)
+  # PR #21: Q4_0 KV with Walsh-Hadamard rotation vs a host reference (GPU, synthetic, no model).
+  add_executable(kv_q4_parity src/kernels/kv_q4_parity.cpp)
+  target_include_directories(kv_q4_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(kv_q4_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  add_test(NAME kv_q4_parity COMMAND kv_q4_parity)
+  add_executable(cvec_parity src/kernels/cvec_parity.cpp)
+  target_include_directories(cvec_parity PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(cvec_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+  add_test(NAME cvec_parity COMMAND cvec_parity)
+
   # Plan v0.3 P3: native MMVQ for 1..8 tokens (bitwise selftest + --bench timing); synthetic data, no model.
   if(STRATA_BUILD_TESTS)
   add_executable(native_mmvq_multi bench/micro/native_mmvq_multi.cpp)
-  target_include_directories(native_mmvq_multi PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(native_mmvq_multi PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(native_mmvq_multi PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(native_mmvq_multi PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME native_mmvq_multi COMMAND native_mmvq_multi --selftest)
   endif()
 
   # Plan v0.3 P0.1: working-set lock of a host region (CPU only).
   add_executable(platform_memory_test src/platform/memory_test.cpp)
-  target_link_libraries(platform_memory_test PRIVATE strata_core CUDA::cudart)
+  target_link_libraries(platform_memory_test PRIVATE strata_core ${_strata_gpu_runtime_target})
   add_test(NAME platform_memory_test COMMAND platform_memory_test)
 
   # Plan v0.3 P2: the n-gram table read straight from the SSD. The selftest needs only the CPU and a temp file.
   add_executable(ple_reader_test src/ngram/ple_reader_test.cpp)
-  target_include_directories(ple_reader_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(ple_reader_test PRIVATE strata_kernels CUDA::cudart)
+  target_include_directories(ple_reader_test PRIVATE ${_strata_gpu_include_directories})
+  target_link_libraries(ple_reader_test PRIVATE strata_kernels ${_strata_gpu_runtime_target})
   add_test(NAME ple_reader_selftest COMMAND ple_reader_test --selftest)
 
-  # The GPU test is registered only when CUDA is on, and it runs ON THE TARGET PC.  `--selftest` allocates
+  # The GPU test runs ON THE TARGET PC. `--selftest` allocates
   # and writes real VRAM, so a machine without a device fails this test rather than skipping it silently -
   # which is correct: an engine that targets one GPU should not report a pass for having no GPU.
-  add_test(NAME cuda_device_selftest COMMAND strata-device --selftest)
+  if(STRATA_ENABLE_CUDA)
+    add_test(NAME cuda_device_selftest COMMAND strata-device --selftest)
+  else()
+    add_test(NAME hip_device_selftest COMMAND strata-device --selftest)
+  endif()
 
   # ---- P2.S5: the GraphRegistry --------------------------------------------------
   # A .cu because the test needs real kernels to capture; it links `strata_core` and not `strata_kernels`,
   # because what it exercises is the capture machinery and not any kernel in it.
-  if(STRATA_BUILD_TESTS)
-  add_executable(graph_registry_test bench/micro/graph_registry_test.cu)
-  target_include_directories(graph_registry_test PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(graph_registry_test PRIVATE strata_core CUDA::cudart)
-  add_test(NAME graph_registry_test COMMAND graph_registry_test)
+  if(STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/graph_registry_test.cu")
+    add_executable(graph_registry_test bench/micro/graph_registry_test.cu)
+    target_include_directories(graph_registry_test PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(graph_registry_test PRIVATE strata_core ${_strata_gpu_runtime_target})
+    add_test(NAME graph_registry_test COMMAND graph_registry_test)
+  endif()
 
   # ---- P2.S5's GATE: is a device<->pinned-memory handoff graph-capturable? --------------------
   # The doorbell protocol puts the D2H of `x_f` and the miss list INSIDE G[l] - that is the whole overlap.  The
   # phase doc says to verify this "before building on it", and this project has already lost two rounds to
   # building first (a memory-only spin that DEADLOCKS on Windows, and a `cudaEventRecord` inside a capture that
   # is SILENTLY DROPPED).  It links only cudart: it is about the capture machinery, not about any kernel.
-  add_executable(pinned_capture bench/micro/pinned_capture.cu)
-  target_include_directories(pinned_capture PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  target_link_libraries(pinned_capture PRIVATE CUDA::cudart)
-  add_test(NAME pinned_capture COMMAND pinned_capture)
+  if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bench/micro/pinned_capture.cu")
+    add_executable(pinned_capture bench/micro/pinned_capture.cu)
+    target_include_directories(pinned_capture PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(pinned_capture PRIVATE ${_strata_gpu_runtime_target})
+    add_test(NAME pinned_capture COMMAND pinned_capture)
   endif()
 
   # Tells tests/CMakeLists.txt to add the CUDA-dependent Catch2 target.  It cannot be declared there
   # unconditionally: tests/ is added at the end of this file, but the flag has to be known before it runs.
-  set(STRATA_CUDA_TESTS ON)
+  if(STRATA_ENABLE_CUDA)
+    set(STRATA_CUDA_TESTS ON)
+  endif()
 endif()
 
 # ---- strata_kernels_cpu (P2.S3) ----
@@ -424,7 +563,7 @@ endif()
 #     worse opt-out.  `cpu_require_expert_support()` therefore refuses at startup on a CPU without the
 #     features, and the fallback exists for tests, exactly as P2.S3 specifies.
 # Plan v0.3 P6: speculation components that need no GPU.
-add_library(strata_spec STATIC src/spec/suffix_drafter.cpp src/spec/controller.cpp)
+add_library(strata_spec STATIC src/spec/suffix_drafter.cpp src/spec/controller.cpp src/spec/draft_policy.cpp)
 target_include_directories(strata_spec PUBLIC include)
 if(STRATA_BUILD_TESTS)
   add_executable(suffix_drafter_test src/spec/suffix_drafter_test.cpp)
@@ -433,6 +572,13 @@ if(STRATA_BUILD_TESTS)
   add_executable(controller_test src/spec/controller_test.cpp)
   target_link_libraries(controller_test PRIVATE strata_spec)
   add_test(NAME controller_test COMMAND controller_test)
+  add_executable(draft_policy_test src/spec/draft_policy_test.cpp)
+  target_link_libraries(draft_policy_test PRIVATE strata_spec)
+  add_test(NAME draft_policy_test COMMAND draft_policy_test)
+  # the serve loop's conversation-cache retention policy (header-only, no library to link)
+  add_executable(conv_cache_test src/program/conv_cache_test.cpp)
+  target_include_directories(conv_cache_test PRIVATE include)
+  add_test(NAME conv_cache_test COMMAND conv_cache_test)
 endif()
 
 add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp
@@ -459,6 +605,9 @@ target_link_libraries(strata_kernels_cpu PUBLIC Threads::Threads)
 # with -DSTRATA_GGML_DIR=..., otherwise the pinned upstream commit (third_party/ggml/VERSION.txt).
 option(STRATA_NATIVE_EXPERTS "Native (IQ) expert support through ggml-cpu" ON)
 set(STRATA_GGML_DIR "" CACHE PATH "llama.cpp checkout to take ggml from (empty: fetch the pinned commit)")
+if(STRATA_PREFILL_MMQ AND NOT STRATA_NATIVE_EXPERTS)
+  message(FATAL_ERROR "STRATA_PREFILL_MMQ requires STRATA_NATIVE_EXPERTS=ON for GGML sources")
+endif()
 if(STRATA_NATIVE_EXPERTS)
   enable_language(C)
   set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
@@ -492,34 +641,104 @@ if(STRATA_NATIVE_EXPERTS)
   endif()
   # part of strata_kernels_cpu (the pool calls these and they call its CPU probe: one library, so GNU ld's
   # one-pass static linking has no cycle to resolve)
-  target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/native_expert.cpp src/kernels/cpu/iq_avx512.cpp)
+  target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/native_expert.cpp src/kernels/cpu/iq_avx512.cpp
+                                           src/kernels/cpu/iq_avx2.cpp)
   target_include_directories(strata_kernels_cpu PRIVATE third_party/ggml)
   if(MSVC)
     set_source_files_properties(src/kernels/cpu/iq_avx512.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX512")
+    set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
   else()
     set_source_files_properties(src/kernels/cpu/iq_avx512.cpp PROPERTIES COMPILE_OPTIONS
                                 "-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512vnni;-mavx512vbmi;-mf16c;-mfma")
+    set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
   endif()
   target_link_libraries(strata_kernels_cpu PUBLIC ggml-cpu ggml-base)
   target_compile_definitions(strata_kernels_cpu PUBLIC STRATA_NATIVE_EXPERTS=1)
   if(STRATA_ENABLE_CUDA)
+    # prompt-speed plan step 2b: llama.cpp's MMQ kernels (ggml-cuda, MIT) for the prompt path's experts, compiled
+    # from the same checkout as ggml above - only mmq.cuh's instances for the packs' types and the quantizer, with
+    # the few host symbols of ggml-cuda.cu they use in src/prefill/ggml_cuda_host.cu
+    if(STRATA_GGML_DIR)
+      set(_strata_gg ${STRATA_GGML_DIR}/ggml)
+    else()
+      set(_strata_gg ${strata_llamacpp_SOURCE_DIR}/ggml)
+    endif()
+    set(_strata_mmq ${_strata_gg}/src/ggml-cuda)
+    add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)   # q8_0: the drafter's matrices (E-9)
+      target_sources(strata_mmq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu)
+    endforeach()
+    target_include_directories(strata_mmq PRIVATE ${_strata_gg}/include ${_strata_gg}/src ${_strata_mmq}
+                               PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    target_compile_options(strata_mmq PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda -use_fast_math>)
+    if(MSVC)
+      target_compile_options(strata_mmq PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/bigobj>)
+    endif()
+    target_link_libraries(strata_mmq PUBLIC ggml-base ${_strata_gpu_runtime_target})
+    target_link_libraries(strata_prefill PUBLIC strata_mmq)
+    target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+  elseif(STRATA_PREFILL_MMQ)
+    # Compile only the pinned GGML MMQ and activation-quantizer sources used by prefill. This uses
+    # common.cuh's native HIP path without enabling a second GGML HIP backend.
+    if(STRATA_GGML_DIR)
+      set(_strata_gg "${STRATA_GGML_DIR}/ggml")
+    else()
+      set(_strata_gg "${strata_llamacpp_SOURCE_DIR}/ggml")
+    endif()
+    set(_strata_mmq "${_strata_gg}/src/ggml-cuda")
+    if(NOT EXISTS "${_strata_mmq}/common.cuh" OR NOT EXISTS "${_strata_mmq}/quantize.cu")
+      message(FATAL_ERROR "The selected GGML source is missing the ggml-cuda MMQ sources: ${_strata_mmq}")
+    endif()
+    set(_strata_mmq_sources
+      src/prefill/moe_mmq.cu
+      src/prefill/ggml_cuda_host.cu
+      "${_strata_mmq}/quantize.cu")
+    foreach(_t q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)
+      list(APPEND _strata_mmq_sources "${_strata_mmq}/template-instances/mmq-instance-${_t}.cu")
+    endforeach()
+    set_source_files_properties(${_strata_mmq_sources} PROPERTIES LANGUAGE HIP)
+    add_library(strata_mmq STATIC ${_strata_mmq_sources})
+    target_include_directories(strata_mmq
+      PRIVATE "${_strata_gg}/include" "${_strata_gg}/src" "${_strata_mmq}"
+      PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    target_compile_definitions(strata_mmq PRIVATE GGML_USE_HIP=1)
+    target_link_libraries(strata_mmq PUBLIC ggml-base hip::host roc::hipblas)
+    target_link_libraries(strata_prefill PUBLIC strata_mmq)
+    target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1)
+
+    # Numerical diagnostic: synthetic layouts by default, optional first-expert IQ3_XXS/IQ4_NL check with a pack.
+    add_executable(hip_prefill_mmq_parity tests/hip/prefill_mmq_parity.cpp)
+    target_include_directories(hip_prefill_mmq_parity PRIVATE "${_strata_gg}/include")
+    target_link_libraries(hip_prefill_mmq_parity PRIVATE strata_prefill)
+    add_test(NAME hip_prefill_mmq_parity COMMAND hip_prefill_mmq_parity)
+    set_tests_properties(hip_prefill_mmq_parity PROPERTIES TIMEOUT 120)
+    list(APPEND _strata_hip_tests hip_prefill_mmq_parity)
+  endif()
+  if(STRATA_ENABLE_HIP)
+    # Optional real-model diagnostic: pass the GGUF shard containing the native PLE key.
+    add_executable(hip_ple_iq4 tests/hip/ple_iq4.cpp)
+    target_link_libraries(hip_ple_iq4 PRIVATE strata_kernels strata_hip_runtime ggml-base)
+  endif()
+  if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
     add_executable(native_expert_parity src/kernels/native_expert_parity.cpp)
-    target_include_directories(native_expert_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-    target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact CUDA::cudart)
+    target_include_directories(native_expert_parity PRIVATE ${_strata_gpu_include_directories})
+    target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu strata_artifact ${_strata_gpu_runtime_target})
   endif()
 endif()
 
 if(STRATA_ENABLE_CUDA AND STRATA_BUILD_TESTS)
   add_executable(hit_cpu_order_parity bench/micro/hit_cpu_order_parity.cu)
-  target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu CUDA::cudart)
+  target_link_libraries(hit_cpu_order_parity PRIVATE strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
 endif()
 
 if(STRATA_BUILD_TESTS)
   # Plan v0.3 P6: one expert applied to several tokens, bitwise vs the single-token kernel (synthetic, CPU only).
   # plan v0.3 P5: the device dequantizers vs the artifact's CPU dequantizers on the real shards (not a ctest: needs
   # the 38 GB model; run it by hand or from tools/gpu_session.py)
-  add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp)
-  target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact CUDA::cudart)
+  if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)   # #93: it needs the device dequantizers and a GPU runtime
+    add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp)
+    target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact ${_strata_gpu_runtime_target})
+  endif()
 
   add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp)
   target_link_libraries(expert_multi_test PRIVATE strata_kernels_cpu)
@@ -535,9 +754,23 @@ if(STRATA_BUILD_TESTS)
   add_executable(pool_test src/kernels/cpu/pool_test.cpp)
   target_link_libraries(pool_test PRIVATE strata_kernels_cpu)
   add_test(NAME pool_test COMMAND pool_test --selftest WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  # issue #29: tiny batches with sleeping workers, as on a big-VRAM card; a hang fails it
+  add_executable(pool_stress src/kernels/cpu/pool_stress.cpp)
+  target_link_libraries(pool_stress PRIVATE strata_kernels_cpu)
+  add_test(NAME pool_stress COMMAND pool_stress 20)
+
+  if(UNIX AND NOT APPLE)
+    add_executable(direct_file_async_test tests/platform/direct_file_async.cpp src/platform/direct_file.cpp)
+    target_include_directories(direct_file_async_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    target_link_libraries(direct_file_async_test PRIVATE Threads::Threads)
+    add_test(NAME direct_file_async_test COMMAND direct_file_async_test)
+  endif()
 endif()
 
 if(STRATA_BUILD_TESTS)
   enable_testing()
-  add_subdirectory(tests)
+  # Model-level checks may be absent from source distributions.
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt")
+    add_subdirectory(tests)
+  endif()
 endif()

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Niko1221 and the Strata contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -3,133 +3,205 @@
 <p align="center"><b>Run a 125-billion-parameter AI model on a normal gaming PC</b><br>
 one NVIDIA card (12-24 GB) + 64 GB of RAM · Windows or Linux · one click to install</p>
 
+<p align="center"><a href="https://github.com/Niko1221/Strata/releases/download/v0.1.10/Pagoda.mp4"><img src="docs/media/pagoda-preview.webp" width="720" alt="A voxel pagoda garden that Strata's model wrote, running in the browser"></a><br>
+<sub>A voxel pagoda garden, 1 shot prompt running on an RTX 5070 with Strata (IQ3_S, 128K context) ·
+<a href="https://github.com/Niko1221/Strata/releases/download/v0.1.10/Pagoda.mp4">full video (49 s)</a></sub></p>
+
 Strata runs **[Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next)** - a large, smart AI model that
 normally needs a server - on your own PC. It writes its answers at **60-95 tokens per second** (a token is about ¾
 of a word): faster than you can read.
 
-- **Private:** everything runs on your PC. Nothing is sent anywhere.
-- **Works with your apps:** chat apps, coding agents and scripts that speak the OpenAI or Anthropic API just work.
-- **Sees pictures** too, if you want (screenshots, photos, scanned pages).
 - **Free and open source.**
 
-> **Jump to:** [Is my PC enough?](#is-my-pc-enough) · [Install](#install-3-steps) · [How fast?](#how-fast-is-it) ·
-> [Which model?](#which-model-should-i-pick) · [Using it](#using-it) · [Problems?](#something-went-wrong) ·
-> [How it works](#how-does-it-work) · [All the details](docs/DETAILS.md)
+> **Jump to:** [How fast?](#how-fast-is-it) · [Which model?](#which-model-should-i-pick) · [Install](#install) ·
+> [Using it](#using-it) · [Problems?](#something-went-wrong) · [How it works](#how-does-it-work) ·
+> [All the details](docs/DETAILS.md)
 
 ---
 
-## Is my PC enough?
-
-| You need | |
-| --- | --- |
-| **Graphics card** | NVIDIA RTX 30, 40 or 50 series with **12 GB of VRAM or more** |
-| **Memory (RAM)** | **64 GB** |
-| **Free disk space** | ~80 GB (an SSD makes the first start much faster) |
-| **System** | Windows 10/11, or Linux |
-
-That's it. The only thing you install yourself is a current **NVIDIA driver**
-([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA App). Everything else - Python, the engine, the
-model - is set up for you.
-
-## Install (3 steps)
-
-**Windows**
-
-1. [Download this project](https://github.com/Niko1221/Strata/archive/refs/heads/main.zip) and unzip it (or `git clone` it).
-2. Double-click **`START-HERE.bat`**.
-3. Answer 4 questions - or just press Enter each time for the recommended choice:
-   - **Which model?** The original, or Swift 1.5 (a version that thinks shorter and answers sooner)
-   - **Which size?** Q2_0, IQ2_XS, IQ3_XXS or IQ3_S - see [which model](#which-model-should-i-pick)
-   - **How much context?** How much text it can keep in mind at once (it suggests one for your card)
-   - **Images?** Whether it should also read pictures
-
-Then it downloads everything (the model is ~70 GB, so the first time takes a while - you can stop and it picks up
-where it left off) and **starts the model**. Your browser opens a chat page at `http://127.0.0.1:8080`.
-
-**Next time**, just double-click `START-HERE.bat` again: it starts right away, nothing is downloaded twice. Close its
-window to stop the model.
-
-**Linux:** run `./setup.sh` - same questions, same result.
-
 ## How fast is it?
 
 Measured on an RTX 5070 (12 GB), a Ryzen 5 7600 and 64 GB of RAM:
 
 | Size | Writes answers (short chat) | Writes answers (128K context) | Reads your prompt |
 | --- | ---: | ---: | ---: |
-| **Q2_0** | 95 tokens/s | 65 tokens/s | 539 tokens/s |
-| **IQ2_XS** | 78 tokens/s | 52 tokens/s | 463 tokens/s |
-| **IQ3_XXS** | 66 tokens/s | 45 tokens/s | 410 tokens/s |
-| **IQ3_S** | 54 tokens/s | 42 tokens/s | 374 tokens/s |
+| **Q2_0** | 93 tokens/s | 74 tokens/s | 2,170 tokens/s |
+| **IQ2_XS** | 79 tokens/s | 63 tokens/s | 2,090 tokens/s |
+| **IQ3_XXS** | 62 tokens/s | 49 tokens/s | 1,750 tokens/s |
+| **IQ3_S** | 53 tokens/s | 46 tokens/s | 1,620 tokens/s |
+| **Coder** (IQ1_M) | 55 tokens/s | 43 tokens/s | 2,180 tokens/s |
 
 - **Writes answers** = how fast the reply appears (tokens per second).
-- **Reads your prompt** = how fast it takes in what you send (long documents, code, chat history).
+- **Reads your prompt** = how fast it takes in what you send (long documents, code, chat history), measured on a
+  32K-token prompt; a 4K prompt reads at 910-1,580 tokens/s. A 32K prompt takes about 15 seconds with Q2_0.
 
 A card with more VRAM is faster, because more of the model fits on the GPU: an RTX 3090 (24 GB) should do roughly
 100-140 tokens per second. All measurements, long-context numbers and estimates for other cards are in the
 [details](docs/DETAILS.md#speed-measured).
 
+Every PC is different: `START-HERE.bat --calibrate` measures a few engine settings on yours and keeps the fastest
+(about 5-10 minutes; on the PC above it made the Coder 7% faster).
+
+**Two or three NVIDIA cards?** Just run `START-HERE.bat`: it lists your cards, says which ones Strata can use, and
+asks whether to share the model across them (recommended when two can). An install made on one card asks once at
+its next start. Or choose yourself: `START-HERE.bat --gpus 0,2` (both, remembered), `--gpus all`, or `--gpu 0` (one
+card, this start only). Each card keeps the experts of its own layers, and prompts flow through the cards in a
+pipeline: on an RTX 5080 + RTX 3090 prompts were read 18-20% faster than on the 5080 alone, decoding on par.
+Every card must be an RTX 20 series or newer with 8 GB or more. See [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
+
 ## Which model should I pick?
 
 **The size** (the same model, compressed more or less):
 
-| Size | Download | Speed | Quality | Pick it if... |
-| --- | ---: | --- | --- | --- |
-| **Q2_0** | 66 GB | fastest | good | you want speed |
-| **IQ2_XS** | 68 GB | fast | better | you want a good all-rounder (**recommended**) |
-| **IQ3_XXS** | 76 GB | slower | great | you want better answers (uses 43 GB of your 64 GB RAM) |
-| **IQ3_S** | 84 GB | slowest | best: matches the full model on the published tests | you want the very best answers (original model only; uses 50 GB of your 64 GB RAM, so close other big programs) |
+| Model | RAM+VRAM Requirements | Speed | Quality |
+| --- | ---: | --- | --- |
+| **Q2_0** | 37.6 GB | fastest | good |
+| **IQ2_XS** | 39.2 GB | fast | better (**recommended**) |
+| **IQ3_XXS** | 47.0 GB | slower | great |
+| **IQ3_S** | 54.8 GB | slowest | best: matches the full model on the published tests (original model only) |
+
+**Will it fit?** Shard 1 is the part of the model that gets loaded when it starts: its experts go into your **RAM**,
+the rest onto your graphics card (the second shard, a 29 GB lookup table, stays on the SSD). So it fits when your
+**RAM is at least shard 1 + about 10 GB** for Windows and your other programs. With 64 GB of RAM every size fits
+(IQ3_S with little else open); with 48 GB, Q2_0 and IQ2_XS. A bigger graphics card makes it faster, but it doesn't
+lower the RAM needed.
 
 **The version:**
 
 - **Qwen3.8-Flash-Next** - the original.
+- **[Coder](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF)** - ISTA-DASLab's coding
+  version: half of the experts removed, keeping the ones that code, tool use and images need (91% of the full model's
+  SWE-bench Verified score, 99% of LiveCodeBench, by its authors). One size (IQ1_M: its experts stored like IQ3_S):
+  shard 1 is **29.6 GB**, so it fits a PC with **32 GB of RAM**, runs 262K context on 64 GB, and reads long prompts
+  the fastest of all. Weaker outside coding.
 - **[Swift 1.5](https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF)** - a fine-tune by UkisAI
   that thinks much shorter before answering, so you get the answer sooner, with about the same quality. Same speed per
-  token. Its own license applies (see its page).
+  token, and about the same RAM as the same size of the original (no IQ3_S). Its own license applies (see its page).
 
-Not sure? Take **IQ2_XS**. You can add another one later with `START-HERE.bat --setup`.
+Not sure? Take **IQ2_XS** - or the **Coder** if you mainly write code, or have 32-48 GB of RAM. You can add another
+one later with `SETUP.bat` (the same as `START-HERE.bat --setup`; on Linux `./setup.sh --setup`).
+
+For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](docs/ORCA.md).
+It needs an explicit packing conversion and is not an installer menu option.
+
+An **AMD Radeon RX 7900 XT / XTX on Linux** works too (experimental): `./setup.sh --backend hip`, chosen by itself on
+a PC with no NVIDIA card Strata can use. It installs ROCm without sudo and compiles the engine (one GPU, no images
+yet). Details: [AMD HIP](docs/AMD_HIP.md).
+
+## Install
+
+**You need:** an NVIDIA RTX 20, 30, 40 or 50 card with 12 GB of VRAM or more (RTX 20 since 0.1.27), enough RAM for the size you pick (above;
+a big GPU makes up for less RAM - the [low-RAM mode](docs/DETAILS.md)),
+~80 GB of free disk space (an SSD makes the first start much faster), and Windows 10/11 or Linux. The only thing you
+install yourself is a current **NVIDIA driver** ([nvidia.com/drivers](https://www.nvidia.com/drivers) or the NVIDIA
+App). Everything else - Python, the engine, the model - is set up for you.
+
+**Windows**
+
+1. [Download this project](https://github.com/Niko1221/Strata/archive/refs/heads/main.zip) and unzip it (or `git clone` it).
+2. Double-click **`START-HERE.bat`**.
+3. Answer a few questions - or just press Enter each time for the recommended choice:
+   - **Which model and size?** The original or Swift 1.5, and Q2_0, IQ2_XS, IQ3_XXS or IQ3_S - see [above](#which-model-should-i-pick)
+   - **How much context?** How much text it can keep in mind at once (it suggests one for your card)
+   - **Images?** Whether it should also read pictures
+   - **Experimental speed projection?** Off unless you say yes - [read what it does](docs/DETAILS.md#experimental-speed-projection-experimental-off-by-default) first
+
+Then it downloads everything (the model is ~70 GB, so the first time takes a while - you can stop and it picks up
+where it left off) and **starts the model**. Your browser opens the Strata app at `http://127.0.0.1:8080`.
+
+> **While the model starts, your PC can be slow or stop responding for 1-3 minutes** (longest the first time): Strata
+> loads 35-55 GB into your RAM and locks part of it for the graphics card. That's normal - wait, and don't close the
+> window. The window tells you what it is doing.
+
+**Next time**, just double-click `START-HERE.bat` again: it starts right away, nothing is downloaded twice. Close its
+window to stop the model.
+
+**Updating:** download the new version and unzip it anywhere (or `git pull`), then run `START-HERE.bat` in it. The
+model files are kept in a `Strata-data` folder next to your Strata folder, so a new copy finds them and sets itself up
+the same way - nothing big is downloaded again.
+
+**Linux:** run `./setup.sh` - same questions, same result.
 
 ## Using it
 
-- **Chat in the browser:** `http://127.0.0.1:8080` - a simple chat page (it opens by itself when the model starts).
+<p align="center"><img src="docs/media/runpagoda.png" width="900" alt="The Strata app's Monitor tab next to a coding agent"><br>
+<sub>The Strata app's <b>Monitor</b> (left) while a coding agent writes the pagoda garden from the video (right)</sub></p>
+
+- **In the browser:** `http://127.0.0.1:8080` - the Strata app (it opens by itself when the model starts): **Chat**, a
+  live **Monitor** of the model and your GPU/CPU/RAM, and **About** with the settings and addresses.
 - **Chat in the terminal:** `.venv\Scripts\python chat.py`
 - **Your apps and coding agents:** add it as an "OpenAI-compatible" provider with base URL
   **`http://127.0.0.1:8080/v1`**, any API key and any model name. Apps that use Anthropic's API: `http://127.0.0.1:8080/v1/messages`.
 - **Thinking:** the model thinks before it answers. Choose **off, low, medium or high** - in the chat page menu, with
   `/think low` in `chat.py`, or with your app's "reasoning effort" setting. Off is fastest; high is best for hard questions.
 - **Pictures:** in the chat page click **Picture**; in `chat.py` type `/image <path>`; in apps just attach them.
-- **From your phone or another PC:** see the [details](docs/DETAILS.md#using-it) (set an API key first).
+- **From your phone or another PC:** `START-HERE.bat --setup --host 0.0.0.0 --api-key <secret>`, then open the
+  address the server window prints; see the [details](docs/DETAILS.md#using-it).
+- **Experimental speed projection (off by default):** an experimental control vector that setup can turn on; it
+  changes how the model answers - read [what it does](docs/DETAILS.md#experimental-speed-projection-experimental-off-by-default) first.
 
 **Good to know:** it answers one request at a time. The first message of a chat is read in full (about 1 minute per
 30,000 tokens); after that it keeps the conversation and reads only what is new, so follow-ups start in seconds.
 
 ## Something went wrong?
 
-| What you see | What to do |
-| --- | --- |
-| `the NVIDIA driver is too old` | Update the driver (NVIDIA App or nvidia.com/drivers), restart the PC, run `START-HERE.bat` again. |
-| It stopped during download or setup | Run `START-HERE.bat` again - it continues where it stopped. |
-| `port 8080 is already in use` | Strata is already running - look for its window. |
-| The first start takes minutes | Normal: it loads 35-43 GB into RAM. The next start is faster. |
-| Slow, and the disk light is busy | Not enough free RAM: close other programs (browsers use a lot), or pick Q2_0 / IQ2_XS. |
-| "prompt exceeds the context" | The conversation is longer than the context you chose: run `START-HERE.bat --setup` and pick more. |
+**My PC froze, or got very slow, the first time Strata started.**
+That's normal while it starts, most of all the first time. Strata loads 35-55 GB into your RAM, locks part of it for
+the graphics card, and works out how much of the model fits on your GPU. The mouse can freeze for a few minutes. **Wait, and don't close the
+window.** The next starts are much faster. Still frozen after 10 minutes? Restart the PC, close other programs
+(browsers use a lot of RAM) and try again. If it keeps happening, pick a smaller size (Q2_0 or IQ2_XS).
 
-More in the [full troubleshooting table](docs/DETAILS.md#troubleshooting). Still stuck? Open an issue and attach
-`strata-<model>.log` from this folder.
+**It stopped while downloading or installing.**
+Run `START-HERE.bat` again. It continues where it stopped.
+
+**It says the NVIDIA driver is too old.**
+Update it (NVIDIA App or [nvidia.com/drivers](https://www.nvidia.com/drivers)), restart the PC, and run
+`START-HERE.bat` again.
+
+**It says port 8080 is already in use.**
+Strata is already running. Look for its window.
+
+**It's very slow and the disk light keeps blinking.**
+Your PC is out of free RAM. Close other programs, or pick a smaller size (Q2_0 or IQ2_XS).
+
+**An answer stopped with "the engine stopped unexpectedly".**
+Usually not enough RAM (on Linux the system then stops the engine). Just send your message again: Strata starts the
+engine by itself. If it keeps happening, close other programs or pick a smaller size.
+
+**It says the prompt exceeds the context.**
+The conversation is longer than the context you chose. Start a new chat, or run `SETUP.bat` and pick more
+context.
+
+**Still stuck?** Look in the [full troubleshooting table](docs/DETAILS.md#troubleshooting), or open an issue and
+attach `strata-<model>.log` from the Strata folder.
 
 ## How does it work?
 
-A model this big doesn't fit on a gaming graphics card. Strata splits the work between the parts of your PC:
+Models like this one normally run on servers with hundreds of gigabytes of graphics memory. Your graphics card has
+12-24 GB. Strata makes it fit by **sharing the work across your whole PC** - the same idea as a kitchen, where the
+things you use all the time stay on the counter and the rest waits in the pantry.
 
-<p align="center"><img src="docs/paper/tiers.svg" width="700" alt="how Strata splits the model between GPU, RAM and SSD"></p>
+<p align="center"><img src="docs/media/how-it-works.svg" width="860" alt="The model's 24,576 experts: the busiest on the graphics card, all of them in RAM, a lookup table on the SSD"></p>
 
-- **The GPU** runs the part of the model that is used for every word, plus the "experts" it needs most often.
-- **The RAM** holds all 24,576 experts, and **the CPU** computes the few the GPU doesn't have - at the same time as the GPU.
-- **The SSD** holds a big lookup table; the model reads a few rows of it per word.
-- **A small helper inside the model guesses the next words**, and Strata checks several guesses at once. That makes
-  it 1.6-1.8x faster than going word by word - and the answer is exactly the same.
+- **The model is a team of 24,576 small specialists ("experts"),** and each word it writes needs only 10 of them.
+  So it doesn't have to have all of them on the graphics card at once.
+- **Your graphics card** does the part of the work needed for every word, and keeps the few thousand experts that
+  are asked most often. It keeps learning which ones those are while you use it.
+- **Your RAM** holds every expert. When a word needs one the card doesn't have, **your processor** works on it -
+  at the same time as the graphics card, so neither waits for the other.
+- **Your SSD** holds a big lookup table; the model only reads a few small rows of it per word.
 
-The full story is in the [paper](docs/paper/Strata-Paper.pdf) and the [details](docs/DETAILS.md).
+<p align="center"><img src="docs/media/guess-and-check.svg" width="860" alt="A small helper guesses the next words; the big model checks them all at once and keeps the right ones"></p>
+
+- **Guess, then check.** A small, fast helper built into the model guesses the next few words, and the big model
+  checks all the guesses in one go. It keeps the ones it agrees with and writes the next word itself - so one step
+  often produces several words. The helper only guesses - the big model decides every word - so you get the same
+  quality answer, 1.6-1.8x sooner.
+- **Long texts are read in big pieces** (up to 8,192 tokens - pieces of words - at a time), which is why a long
+  document or code base is read at over 1,000 tokens per second.
+
+Want the full picture? The [details](docs/DETAILS.md#how-it-works) explain every part and its numbers, and the
+[paper](docs/paper/Strata-Paper.pdf) tells the whole story, with the measurements behind it.
 
 ## Credits
 
@@ -140,3 +212,10 @@ The full story is in the [paper](docs/paper/Strata-Paper.pdf) and the [details](
 - Built with parts of [llama.cpp / ggml](https://github.com/ggml-org/llama.cpp) (MIT). Ideas from
   [Splash](https://github.com/incoai/splash), [ninfer](https://github.com/Neroued/ninfer) and
   [HyperQwen](https://github.com/syv-ai/HyperQwen). More in the [details](docs/DETAILS.md#credits-and-licenses).
+
+## License
+
+Strata is open source under the [MIT License](LICENSE). A few parts carry their own licenses: `third_party/ggml`
+(MIT, llama.cpp / ggml), the web app's font (SIL Open Font License 1.1) and the experimental speed projection's
+vector in `data/experimental-speed-projection` (Qwen Community License 1.0, from the model's activations). The
+models are not part of this repository; each model's own license applies to its files.

SETUP.bat

@@ -0,0 +1,3 @@
+@echo off
+rem Change settings or install another model (the same as START-HERE.bat --setup).
+call "%~dp0START-HERE.bat" --setup %*

bench/results/2026-09-27-cache-parity/README.md

@@ -0,0 +1,29 @@
+# 2026-09-27: expert cache on vs off (issue #23)
+
+Question: the engine used to warn at start that "the GPU hit path is NOT CORRECT ... any OUTPUT from it is not". Does
+the expert cache make the output worse?
+
+Method: teacher-forced. `strata --dump-logits` runs a text one token at a time and writes the logits at every
+position, so both runs see the same text and each position's next-token prediction can be compared:
+- cache off: no `--expert-cache`, every routed expert computed by the CPU pool;
+- cache on: `--expert-profile pack/profile-decode-8k.bin --expert-cache auto`, 5,130 of the most-used experts on the GPU.
+
+Q2_0, RTX 5070, the first 512 or 1,024 tokens of three frozen prompts.
+
+| Text | Same top-1 | Median KL | Perplexity off | Perplexity on | On - off (nats per token) |
+|---|---:|---:|---:|---:|---:|
+| Code agent (512) | 97.7% | 3.0e-04 | 2.672 | 2.630 | -0.016 +- 0.019 |
+| Long document (1,024) | 96.2% | 4.5e-03 | 29.844 | 29.843 | -0.000 +- 0.005 |
+| Chat (1,024) | 95.0% | 4.4e-03 | 18.380 | 18.301 | -0.004 +- 0.006 |
+| **All 2,557 tokens** | | | | | **-0.005 +- 0.005** |
+
+Reading:
+- The cache changes which of two nearly equal candidates wins at 2-5% of positions. Most of the flips have a top-2
+  margin under 0.5 logits (`logits-compare.txt`).
+- Neither side is more accurate: the perplexity of the actual text is the same, within 1 standard error.
+- A GPU expert and the CPU compute the same quantized expert with a different floating-point order. Both are an
+  equally good approximation of the model.
+
+The start-up warning dates from round 328, when the GPU hit path had a real fault (divergence from token 0). That fault
+was fixed long ago and is covered by `expert_parity`, `native_expert_parity` and the grouped kernels' tests. Since
+engine 0.1.8 the warning is replaced by one accurate line.

bench/results/2026-09-27-esp/README.md

@@ -0,0 +1,36 @@
+# 2026-09-27: the experimental-speed-projection control vector
+
+The package's `Qwen3.8-Flash-Next-experimental-speed-projection.gguf` (47 unit directions of 2,560, one per layer),
+applied as `--control-vector-scaled FILE:1.0 --control-vector-layer-range 4 44 --cvec-mode project --cvec-dir
+per-layer`: after layers 4-44, every hyper-connection stream `h -= (h . v) v`. The package describes it as a
+refusal-direction projection. Q2_0, RTX 5070, fixed experts (`--expert-cache 4000 --pcie-frac 0 --adapt-swaps 0`).
+
+## Kernel (`cvec_parity`)
+
+Project and add against a double-precision reference (max error 5e-7); with the switch off, the kernel's pending
+residual write is bitwise the fused hyper-connection read's own, so a loaded-but-off vector is the stock engine.
+
+## Teacher-forced, stock vs projection (`esp_kl.py`, per-token path)
+
+| Text | Tokens | Same top-1 | Mean KL | p99 KL | Perplexity stock | With the vector |
+|---|---:|---:|---:|---:|---:|---:|
+| Code | 511 | 92.8% | 0.098 | 1.69 | 2.594 | 2.982 |
+| Document | 1,023 | 89.8% | 0.058 | 0.84 | 30.299 | 30.984 |
+| Chat | 1,023 | 89.6% | 0.050 | 0.33 | 18.409 | 18.485 |
+| **All** | 2,557 | | **0.063** (max 4.10) | 1.00 | | |
+
+The package reports a mean KL of 0.186 on its own neutral text with UD-Q2_K_XL in llama.cpp.
+
+Cost on the same tokens: 44.34 -> 44.26, 46.74 -> 46.56, 47.02 -> 46.83 tok/s (0.2-0.4%).
+
+## Serve (`esp_serve.py`)
+
+- Loaded and switched off (`cvec=0`) vs an engine without it: 65 / 65 first tokens after prefixes (verify windows
+  and the batched prompt path) and three 256-token generations **token-identical**.
+- On, off, on again: identical (the switch drops the conversation cache, so nothing computed one way is reused the
+  other way).
+- The prompt paths steer like the per-token path: served-with-vector first tokens agree with the per-token path's
+  teacher-forced argmax with the vector 62 / 65, and with the stock one 57 / 65 (stock served: 64 / 65 and 57 / 65).
+- Decode tok/s of 3 x 256-token generations: stock 66.4, on 76.7 / 81.2. **Not a like-for-like comparison:** the
+  model writes different text with it on (on the code prompt it copied a long stretch of the source, which prompt
+  lookup drafts at 91% acceptance, where stock stopped after 78 tokens). On the same tokens it is 0.2-0.4% slower.

bench/results/2026-09-27-esp/esp_kl.py

@@ -0,0 +1,85 @@
+"""Teacher-forced logits, stock vs the experimental-speed-projection vector, on the same texts and fixed experts.
+
+    python esp_kl.py run      # the dumps (per-token path)
+    python esp_kl.py compare  # same top-1, mean/median KL(stock || projection), perplexities
+"""
+import json
+import os
+import subprocess
+import sys
+
+import numpy as np
+
+S = os.path.dirname(os.path.abspath(__file__))
+E = r"C:\Users\AI-Server\Desktop\Strata\Public\Engine"
+EXE = E + r"\build-dev\strata.exe"
+VEC = r"C:\Users\AI-Server\Desktop\Strata\experimental-speed-projection\Qwen3.8-Flash-Next-experimental-speed-projection.gguf"
+TEXTS = {"code": S + r"\p512.ids", "doc": S + r"\4k-02-long-doc-1k.ids", "chat": S + r"\4k-04-chat-1k.ids"}
+OUT = S + r"\esp-kl"
+ARMS = {"stock": [],
+        "proj": ["--control-vector-scaled", VEC + ":1.0", "--control-vector-layer-range", "4", "44",
+                 "--cvec-mode", "project", "--cvec-dir", "per-layer"]}
+
+
+def base_args():
+    c = json.load(open(S + r"\cfg-rel.json"))
+    a = list(c["args"])
+    i = a.index("--expert-cache")
+    a[i + 1] = "4000"
+    j = a.index("--prefill")          # the prompt one token at a time: a logits row per position
+    del a[j:j + 2]
+    return a + ["--pcie-frac", "0", "--adapt-swaps", "0"], c
+
+
+def run():
+    os.makedirs(OUT, exist_ok=True)
+    args, c = base_args()
+    env = dict(os.environ)
+    env["PATH"] = os.pathsep.join(c["lib_dirs"]) + os.pathsep + env["PATH"]
+    for name, path in TEXTS.items():
+        for arm, extra in ARMS.items():
+            out = f"{OUT}\\{name}-{arm}.bin"
+            if os.path.exists(out):
+                continue
+            cmd = [EXE, *args, *extra, "--tokens-file", path, "--max-new", "1", "--dump-logits", out]
+            with open(f"{OUT}\\{name}-{arm}.log", "w") as log:
+                r = subprocess.run(cmd, cwd=c["cwd"], stdout=log, stderr=subprocess.STDOUT, env=env)
+            print(name, arm, "exit", r.returncode, flush=True)
+
+
+def load(p):
+    h = np.fromfile(p, dtype=np.int32, count=2)
+    return np.memmap(p, dtype=np.float32, mode="r", offset=8, shape=(int(h[1]), int(h[0])))
+
+
+def lsm(x):
+    x = x.astype(np.float64)
+    m = x.max(axis=-1, keepdims=True)
+    return x - (m + np.log(np.exp(x - m).sum(axis=-1, keepdims=True)))
+
+
+def compare():
+    print(f"{'text':<6}{'tokens':>7}{'same top-1':>11}{'mean KL':>9}{'median KL':>11}{'p99 KL':>8}{'ppl stock':>10}{'ppl proj':>9}")
+    allkl = []
+    for name, path in TEXTS.items():
+        ids = [int(x) for x in open(path).read().replace(",", " ").split()]
+        a, b = load(f"{OUT}\\{name}-stock.bin"), load(f"{OUT}\\{name}-proj.bin")
+        n = min(a.shape[0], b.shape[0], len(ids) - 1)
+        kl, same, la, lb = [], 0, [], []
+        for r in range(0, n, 64):
+            pa, pb = lsm(a[r:min(n, r + 64)]), lsm(b[r:min(n, r + 64)])
+            kl += list((np.exp(pa) * (pa - pb)).sum(axis=1))
+            same += int((pa.argmax(1) == pb.argmax(1)).sum())
+            nxt = np.array(ids[r + 1:min(n, r + 64) + 1])
+            la += list(pa[np.arange(len(nxt)), nxt])
+            lb += list(pb[np.arange(len(nxt)), nxt])
+        kl = np.array(kl)
+        allkl += list(kl)
+        print(f"{name:<6}{n:>7}{100 * same / n:>10.1f}%{kl.mean():>9.3f}{np.median(kl):>11.4f}{np.percentile(kl, 99):>8.2f}"
+              f"{np.exp(-np.mean(la)):>10.3f}{np.exp(-np.mean(lb)):>9.3f}")
+    allkl = np.array(allkl)
+    print(f"all {len(allkl)} tokens: mean KL {allkl.mean():.3f}, max {allkl.max():.2f}, p99 {np.percentile(allkl, 99):.2f}")
+
+
+if __name__ == "__main__":
+    {"run": run, "compare": compare}[sys.argv[1]]()

bench/results/2026-09-27-esp/esp_serve.py

@@ -0,0 +1,129 @@
+"""Serve-mode checks of the experimental-speed-projection vector (exact mode: fixed experts, no PCIe share).
+
+    python esp_serve.py run ARM      # ARM = stock | vec; writes esp-serve/<arm>.json
+    python esp_serve.py compare
+
+Per arm: the first greedy token after many prefixes of three texts (short ones go through the verify windows, long
+ones through the batched prompt path; the conversation cache reuses the earlier prefix), and three 256-token greedy
+generations.  The vec arm runs everything with cvec=1, then cvec=0, then cvec=1 again.
+"""
+import json
+import os
+import sys
+import threading
+
+S = os.path.dirname(os.path.abspath(__file__))
+E = r"C:\Users\AI-Server\Desktop\Strata\Public\Engine"
+sys.path.insert(0, E)
+from serve.server import StrataEngine  # noqa: E402
+
+EXE = E + r"\build-dev\strata.exe"
+VEC = r"C:\Users\AI-Server\Desktop\Strata\experimental-speed-projection\Qwen3.8-Flash-Next-experimental-speed-projection.gguf"
+TEXTS = {"code": S + r"\p512.ids", "doc": S + r"\4k-02-long-doc-1k.ids", "chat": S + r"\4k-04-chat-1k.ids"}
+OUT = S + r"\esp-serve"
+VEC_ARGS = ["--control-vector-scaled", VEC + ":1.0", "--control-vector-layer-range", "4", "44",
+            "--cvec-mode", "project", "--cvec-dir", "per-layer"]
+
+
+def ids_of(name):
+    return [int(x) for x in open(TEXTS[name]).read().replace(",", " ").split()]
+
+
+def engine(extra):
+    c = json.load(open(S + r"\cfg-rel.json"))
+    a = list(c["args"])
+    a[a.index("--expert-cache") + 1] = "4000"
+    env = dict(os.environ)
+    env["PATH"] = os.pathsep.join(c["lib_dirs"]) + os.pathsep + env["PATH"]
+    return StrataEngine(EXE, a + ["--pcie-frac", "0", "--adapt-swaps", "0"] + extra, cwd=c["cwd"],
+                        log=OUT + r"\engine.log", env=env)
+
+
+def gen(eng, ids, n, sampling):
+    toks = [t for t in eng.generate(ids, n, sampling, threading.Event()) if t is not None]
+    return toks, dict(eng.last)
+
+
+def suite(eng, sampling):
+    out = {"first": {}, "long": {}}
+    for name in TEXTS:
+        ids = ids_of(name)
+        # short prefixes growing by 4 (each continues the live session through the verify windows), then long ones
+        # SHRINKING (no cached prefix, so each is read from 0 by the batched prompt path)
+        lens = list(range(32, 72, 4)) + list(range(min(len(ids) - 1, 700), 95, -48))
+        out["first"][name] = {L: gen(eng, ids[:L], 1, sampling)[0][0] for L in lens}
+    for name in TEXTS:
+        toks, last = gen(eng, ids_of(name)[:300], 256, sampling)
+        out["long"][name] = {"tokens": toks, "last": last}
+    return out
+
+
+def run(arm):
+    os.makedirs(OUT, exist_ok=True)
+    eng = engine(VEC_ARGS if arm == "vec" else [])
+    res = {"info": eng.info}
+    if arm == "stock":
+        res["stock"] = suite(eng, {})
+    else:
+        res["on"] = suite(eng, {"experimental_speed_projection": True})
+        res["off"] = suite(eng, {"experimental_speed_projection": False})
+        res["on2"] = suite(eng, {"experimental_speed_projection": True})
+    eng.close()
+    json.dump(res, open(f"{OUT}\\{arm}.json", "w"))
+    print(arm, "done", eng.info, flush=True)
+
+
+def compare():
+    import numpy as np
+    sys.path.insert(0, S)
+    from esp_kl import load, OUT as KL
+    st = json.load(open(OUT + r"\stock.json"))
+    ve = json.load(open(OUT + r"\vec.json"))
+    print("engine INFO (vec):", ve["info"])
+
+    def eq_first(a, b):
+        tot = same = 0
+        for name in a:
+            for L, t in a[name].items():
+                tot += 1
+                same += int(b[name][L] == t)
+        return same, tot
+
+    def eq_long(a, b):
+        return all(a[n]["tokens"] == b[n]["tokens"] for n in a), \
+            [(n, next((i for i, (x, y) in enumerate(zip(a[n]["tokens"], b[n]["tokens"])) if x != y), None)) for n in a]
+
+    print("1. vector loaded, switched off == no vector")
+    print("   first tokens: %d / %d identical" % eq_first(st["stock"]["first"], ve["off"]["first"]))
+    print("   256-token generations identical:", *eq_long(st["stock"]["long"], ve["off"]["long"]))
+    print("2. repeatable: on, off, on again")
+    print("   first tokens: %d / %d identical" % eq_first(ve["on"]["first"], ve["on2"]["first"]))
+    print("   256-token generations identical:", *eq_long(ve["on"]["long"], ve["on2"]["long"]))
+    print("3. the prompt paths against the per-token path's teacher-forced argmax (row L-1 predicts token L)")
+    for arm, first in (("stock", st["stock"]["first"]), ("on", ve["on"]["first"])):
+        for ref in ("stock", "proj"):
+            agree = tot = 0
+            for name in first:
+                d = load(f"{KL}\\{name}-{ref}.bin")
+                for L, t in first[name].items():
+                    L = int(L)
+                    if L - 1 < d.shape[0]:
+                        tot += 1
+                        agree += int(int(np.argmax(d[L - 1])) == t)
+            print(f"   served {arm:<5} vs per-token {ref:<5}: {agree} / {tot}")
+    print("4. how different the text is: 256-token generations, on vs stock, first difference")
+    print("  ", eq_long(st["stock"]["long"], ve["on"]["long"])[1])
+    print("5. decode speed (exact mode: fixed experts, no PCIe share - not the normal setup)")
+    for label, r in (("stock", st["stock"]), ("vec off", ve["off"]), ("vec on", ve["on"]), ("vec on 2", ve["on2"])):
+        g = sum(v["last"]["generated"] for v in r["long"].values())
+        ms = sum(v["last"]["decode_ms"] for v in r["long"].values())
+        acc = sum(v["last"].get("drafts_accepted", 0) for v in r["long"].values())
+        off = sum(v["last"].get("drafts_offered", 0) for v in r["long"].values())
+        print(f"   {label:<9} {g / ms * 1000:6.1f} tok/s  drafts accepted {acc}/{off}")
+
+
+if __name__ == "__main__":
+    if sys.argv[1] == "run":
+        run(sys.argv[2])
+    else:
+        compare()

bench/results/2026-09-27-esp/serve-checks.txt

@@ -0,0 +1,20 @@
+1. vector loaded, switched off == no vector
+   first tokens: 65 / 65 identical
+   256-token generations identical: True [('code', None), ('doc', None), ('chat', None)]
+2. repeatable: on, off, on again
+   first tokens: 65 / 65 identical
+   256-token generations identical: True [('code', None), ('doc', None), ('chat', None)]
+3. the prompt paths against the per-token path's teacher-forced argmax (row L-1 predicts token L)
+   served stock vs per-token stock: 64 / 65
+   served stock vs per-token proj : 57 / 65
+   served on    vs per-token stock: 57 / 65
+   served on    vs per-token proj : 62 / 65
+4. how different the text is: 256-token generations, on vs stock, first difference
+   [('code', 11), ('doc', 6), ('chat', 3)]
+5. decode speed (exact mode: fixed experts, no PCIe share - not the normal setup)
+   stock       66.4 tok/s  drafts accepted 256/309
+   vec off     65.7 tok/s  drafts accepted 251/318
+   vec on      76.7 tok/s  drafts accepted 366/425
+   vec on 2    81.2 tok/s  drafts accepted 366/423
+
+[exited with code 0]

bench/results/2026-09-27-esp/teacher-forced.txt

@@ -0,0 +1,6 @@
+text   tokens same top-1  mean KL  median KL  p99 KL ppl stock ppl proj
+code      511      92.8%    0.098     0.0031    1.69     2.594    2.982
+doc      1023      89.8%    0.058     0.0271    0.84    30.299   30.984
+chat     1023      89.6%    0.050     0.0265    0.33    18.409   18.485
+all 2557 tokens: mean KL 0.063, max 4.10, p99 1.00
+

bench/results/2026-09-27-kv-q4/README.md

@@ -0,0 +1,60 @@
+# 2026-09-27: Q4_0 KV with Hadamard rotation (PR #21, `--kv q4_0`)
+
+PR #21 (code-martin) adds a 4-bit K/V format: an orthonormal 256-point Walsh-Hadamard rotation of K, V and Q, then
+ggml `q4_0` blocks (576 B per cell and layer, vs 1,056 B in int8).
+
+## Integration fixes made on merge
+
+- **Q4 + KV streaming (`--kv-resident`).** The PR had no host copy for Q4 and skipped the staging, so with streaming
+  on, the prompt path wrote K/V through the residency map's -1 entries (out-of-bounds writes).
+  - Q4 is now a third format in `kv_stream`: host copy, block residency, prompt-path staging, MTP ring and STATE_HASH.
+  - `kv_stream_parity` checks it bitwise, streamed vs resident, with ring restore.
+- **Duplicate attention.** The per-token decode path ran attention twice, the first time before the selection's
+  blocks were made resident; it now runs once.
+- **Tie-break.** The group maximum's tie-break is now deterministic across lanes: equal |x| with opposite signs
+  could give one block two scales.
+- **Unrelated changes dropped:** duplicate path fixes (already in 0.1.7), personal `.gitignore` entries, and a
+  re-encoded CMakeLists.
+
+`kv_q4_parity` (the PR's test) and all 70 ctest tests pass.
+
+## Quality (teacher-forced, `tools/kv_precision_compare.py`)
+
+Same text, same fixed expert set (Q2_0, 3,000 slots, `--adapt-swaps 0`); only `--kv` differs. fp16 is the reference.
+
+| Text | KV | Same top-1 as fp16 | Mean KL | Perplexity | vs fp16 (nats/token) |
+|---|---|---:|---:|---:|---:|
+| code agent (512) | int8 | 98.4% | 0.014 | 2.649 | -0.011 +- 0.006 |
+| | q4_0 | 95.7% | 0.030 | 2.598 | -0.031 +- 0.014 |
+| document (1,024) | int8 | 95.6% | 0.009 | 30.18 | +0.012 +- 0.005 |
+| | q4_0 | 91.3% | 0.039 | 32.26 | **+0.078 +- 0.011** |
+| chat (1,024) | int8 | 95.5% | 0.009 | 18.24 | -0.003 +- 0.005 |
+| | q4_0 | 90.7% | 0.028 | 18.52 | +0.012 +- 0.008 |
+| document, 8K (every 8th position) | int8 | 92.3% | 0.114 | 7.63 | -0.013 +- 0.020 |
+| | q4_0 | 88.0% | 0.297 | 8.65 | **+0.112 +- 0.037** |
+
+- **int8 is indistinguishable from fp16** (within about 2 standard errors everywhere).
+- **q4_0 costs measurable precision, and more as the context grows:**
+  - document perplexity +8% at 1K and +12% at 8K;
+  - the KL from fp16 is 2.6-4.2x int8's;
+  - code and chat are near-neutral.
+- **Needles with `--kv q4_0 --kv-resident 32768`: 5/5** (1K, 32K, 128K at 10% and 90% depth, 262K;
+  `needles-q4.*`). Retrieval holds.
+
+## Speed and memory (Q2_0, 128K, matrix bench, `--kv-resident 32768`)
+
+| KV | Expert slots | Decode tok/s | Tokens/round | Rounds/s | K/V in RAM |
+|---|---:|---:|---:|---:|---:|
+| int8 (`kv-stream/matrix-stream`) | 4,044 | 62.4 | 2.52 | 24.8 | 1.52 GiB |
+| q4_0 (`matrix-q4`) | 4,190 | 67.4 | 2.62 | 25.7 (+3.7%) | 0.83 GiB |
+
+Without streaming (contexts up to 64K, or too little RAM for it) q4_0 halves the whole KV cache in VRAM instead.
+
+## Decision
+
+`--kv q4_0` is a user option:
+- `START-HERE.bat --kv q4_0`, or the question setup asks above 8K context;
+- or `--kv q4_0` in a `strata-*.json`.
+
+int8 stays the default: every published quality number was measured with it, and q4_0's precision cost is real,
+though it passes the needles.

bench/results/2026-09-28-coder/README.md

@@ -0,0 +1,44 @@
+# The GSQ-RCO Coder (256 of 512 experts) on Strata
+
+RTX 5070 12 GB, Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. Engine 0.1.15 + PR #54 (the expert count from the model
+file), the settings setup writes (`--prefill auto`, 8-bit KV above 4K, KV streaming from 64K, MTP, greedy, 256 tokens),
+`data/expert-profile-coder.bin`, the same code-agent prompts as `2026-09-28-speed-0114`.
+
+| | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| Prompt, tokens/s | 599 | 1,152 | 1,298 | 1,350 | 1,266 | 1,034 |
+| Output, tokens/s | 53.3 | 50.6 | 53.3 | 50.8 | 44.0 | 42.8 |
+| Draft acceptance | 0.720 | 0.707 | 0.785 | 0.696 | 0.619 | 0.684 |
+| Experts in VRAM | 2,570 | 2,615 | 2,418 | 2,382 | 2,326 | 2,208 |
+
+Every run in [`matrix.json`](matrix.json). The 128K and 262K answers were read: coherent (they list and discuss the
+files of the prompt).
+
+## Why it is fast
+
+The kept experts are stored like IQ3_S's (IQ2_S / IQ3_XXS / IQ3_S / IQ4_XS gate and up, IQ4_NL / Q2_0 down; "IQ1_M" is
+the release's name for 1.89 bits per *original* parameter), so a token costs about what IQ3_S costs where the expert is
+on the CPU. But there are half as many experts: the 23 GB of them pin completely (`cudaHostRegister` of the whole
+arena), a 12 GB card holds ~20% of them instead of ~10%, and the prompt path streams half as many.
+
+## The expert profile
+
+`data/expert-profile-coder.bin` (48 x 256, all 12,288 pairs ranked): the shipped 48 x 512 ranking, each pair mapped to
+its kept expert's new index through section 2 of the release's `tensor-allocation/*.rco-allocation.txt` (the kept
+experts in their original order; gate, up and down agree on which were kept), pruned pairs dropped. 5,492 of the
+shipped top 8,000 survive (as PR #54 found). With 2,537 slots on the 12 GB card, 72% of the expert reads of a 4K code
+prompt hit the GPU - a wrong index mapping would give about the slot share, 21%.
+
+## Images
+
+The Coder ships the original's vision encoder (the same file). With `--vision` on the GPU it described a Strata
+Monitor screenshot correctly, windows, labels and the numbers of the dashboard and the requests table (1,079 prompt
+tokens with the image, 2,012 generated at ~50 tokens/s).
+
+## The experimental speed projection
+
+The vector was made from the full model's activations; the Coder has the same residual stream. On the Coder it loads
+(41 layers steered), costs no measurable speed (47.7-55.7 tokens/s with it on or off), and both settings write correct
+answers to three coding prompts. What it is for - fewer refusals - was not measured. (Two runs of the same prompt with
+it off also differ after ~130-200 characters in serve mode, so this check cannot show how far it moves ordinary text;
+the teacher-forced method of `2026-09-27-esp` needs the token-by-token prompt path, which native packs do not run.)

bench/results/2026-09-28-coder/matrix.json

@@ -0,0 +1,86 @@
+[
+ {
+  "model": "CODER",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 18.0,
+  "prefill_tok_s": 599.05,
+  "ttft_s": 2.54,
+  "decode_tok_s": 53.33,
+  "decoded": 256,
+  "spec_accept": 0.72,
+  "tokens_per_round": 2.86,
+  "vram_slots": 2570
+ },
+ {
+  "model": "CODER",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 19.6,
+  "prefill_tok_s": 1152.17,
+  "ttft_s": 3.94,
+  "decode_tok_s": 50.6,
+  "decoded": 256,
+  "spec_accept": 0.707,
+  "tokens_per_round": 2.88,
+  "vram_slots": 2615
+ },
+ {
+  "model": "CODER",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 40.7,
+  "prefill_tok_s": 1297.8,
+  "ttft_s": 25.17,
+  "decode_tok_s": 53.34,
+  "decoded": 256,
+  "spec_accept": 0.785,
+  "tokens_per_round": 3.02,
+  "vram_slots": 2418
+ },
+ {
+  "model": "CODER",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 75.5,
+  "prefill_tok_s": 1350.09,
+  "ttft_s": 49.01,
+  "decode_tok_s": 50.84,
+  "decoded": 256,
+  "spec_accept": 0.696,
+  "tokens_per_round": 2.61,
+  "vram_slots": 2382
+ },
+ {
+  "model": "CODER",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 121.5,
+  "prefill_tok_s": 1266.31,
+  "ttft_s": 102.95,
+  "decode_tok_s": 44.03,
+  "decoded": 256,
+  "spec_accept": 0.619,
+  "tokens_per_round": 2.33,
+  "vram_slots": 2326
+ },
+ {
+  "model": "CODER",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 280.7,
+  "prefill_tok_s": 1033.91,
+  "ttft_s": 252.92,
+  "decode_tok_s": 42.8,
+  "decoded": 256,
+  "spec_accept": 0.684,
+  "tokens_per_round": 2.6,
+  "vram_slots": 2208
+ }
+]

bench/results/2026-09-28-prefill-speed/README.md

@@ -0,0 +1,47 @@
+# Faster prompt reading (engine 0.1.13)
+
+RTX 5070 12 GB, Ryzen 5 7600, 64 GB DDR5. 32K-token prompt (`bench/prompts/long/32k-01-code-agent.ids`), one-shot
+engine, `--expert-cache auto`, greedy. Numbers are prompt tokens/s.
+
+## What changed, step by step
+
+| Step | What | Result |
+|---|---|---|
+| Buffers | The attention half and the MoE half of a layer share one scratch region (~260 KB less per prompt token) | bit-identical; chunk 4096 borrows 1.78 GiB instead of 2.84 |
+| PLE rows | The next chunk's PLE rows are read from the model file on a thread while the GPU works | bit-identical; 762 -> 790 at chunk 4096 |
+| Chunk size | `--prefill auto`: the largest of 8192 / 6144 / 4096 / ... whose buffers fit in the cache slots it may borrow; a request borrows only what its own prompt needs | Q2_0: 4096 791, 6144 878, 8192 973 |
+| Host copies | Experts the arena could not pin are copied to pinned buffers by helper threads instead of the launching thread | bit-identical; IQ3_S at 4096: 551 -> 652 |
+| PLE block | Computed for the whole chunk (two GEMMs) instead of token by token | Q2_0 981 -> 1053, IQ3_S 831 -> 940 |
+| Experts | llama.cpp's MMQ kernels (weights stay quantized, int8 tensor cores) instead of dequantize-to-FP16 + cuBLAS | Q2_0 1052 -> 1130, IQ3_S (8192) 932 -> 1014 |
+| Streaming | At chunks of 2048+ every non-resident expert streams in a fixed order through a ring, so the next layer's arrive while the current layer's attention runs | bit-identical; Q2_0 1130 -> 1290, IQ3_S -> 1208 |
+
+Tried and dropped: cuBLAS grouped GEMMs for the experts (faster in isolation, slower inside the engine: 770 vs 790).
+
+## Before and after
+
+| | engine 0.1.12 (`--prefill 2048`) | engine 0.1.13 (`--prefill auto`) |
+|---|---:|---:|
+| Q2_0, 32K prompt | 572 | 1,290 |
+| IQ3_S, 32K prompt | 383 | 1,208 |
+| Server, Q2_0, 128K context, 999-token prompt | 353 | 438 |
+| Server, 6,927-token prompt | 529 | 1,077 |
+| Server, 28,584-token prompt | 584 | 1,249 |
+
+Output speed is unchanged (the decode path is the same). Short prompts (a few hundred tokens) go through the verify
+windows as before.
+
+## Quality
+
+- "Bit-identical" steps were checked with the GDN state hash after the 32K prompt (all 36 recurrent layers).
+- The steps that change rounding (the batched PLE block, MMQ's 8-bit activations) were compared teacher-forced: the
+  32K prompt is read by the prompt path up to its last 256 tokens, those go through the token path with their logits
+  dumped. At 32K every rounding change moves the predictions about this much - even a different chunk size of the
+  unchanged engine - so that is the yardstick:
+
+| Against the 8192-chunk FP16 path | Same top-1 | Mean KL | Perplexity (reference 9.21) |
+|---|---:|---:|---:|
+| Chunk 6144, same code (the yardstick) | 89.8% | 0.33 | 9.18 |
+| MMQ experts | 85.9% | 0.38 | 8.63 |
+
+- Needles (the answer hidden at a given depth): 5 of 5 at 1K, 32K, 128K (two depths) and 262K, Q2_0, KV streaming.
+- A rerun of the same configuration is bit-identical.

bench/results/2026-09-28-speed-0114/README.md

@@ -0,0 +1,32 @@
+# Speed with engine 0.1.14
+
+RTX 5070 12 GB (PCIe 5.0 x16), Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. The ready-made 0.1.14 engine, one-shot runs
+with what setup writes: `--prefill auto`, the shipped expert profile, `--expert-cache auto`, 8-bit KV above 4K, KV
+streaming (`--kv-resident 32768`) from 64K, MTP (`--spec 4 --spec-min-p 0.5`), greedy, 256 generated tokens. One
+code-agent prompt per length (`bench/prompts/*/…-code-agent.ids`, the same prompts as `2026-09-24-final`).
+
+The tables: [`matrix.md`](matrix.md) (prompt and output speed, draft acceptance, experts in VRAM); every run in
+[`matrix.json`](matrix.json).
+
+- **IQ2_XS** was measured with Swift 1.5's IQ2_XS (the original's file was not on the machine; the two run at the same
+  speed: 4K 465 vs 467 prompt tokens/s in `2026-09-24`).
+- **IQ3_S at 128K** comes from a first pass during which a large download was running (the clean rerun of that row
+  was stopped when Windows ran short of memory); the other rows are from runs on an otherwise idle machine.
+- **IQ3_XXS and IQ3_S at 262K** are not measured: with 43 / 50 GB of experts, a 260K context exceeds 64 GB of RAM.
+
+## Against 0.1.12
+
+Prompts are 1.2x (1K) to 2.4x (32K-128K) faster: the 0.1.13 prompt path (chunks of up to 8,192 tokens, MMQ experts,
+streamed expert copies; see `2026-09-28-prefill-speed`).
+
+Output speed moves with the text: speculative decoding is faster when more drafts are accepted, and the new prompt
+path's rounding changes the answer, so the same prompt can land a few percent either way (Q2_0 32K: acceptance 0.865
+then, 0.709 now). The engine itself is not slower: on the 4K prompt, run back to back on the same machine state,
+0.1.14 writes 88.5 tokens/s and 0.1.12 85.7 (3 runs each, same free VRAM).
+
+## Found on the way
+
+IQ3_XXS at 64K and 128K ended with `out of memory: cudaFuncSetAttribute` in the MMQ launch: CUDA loads a kernel's code
+when it is first used, and mid-prompt there was no VRAM left for it. Loading every kernel at start
+(`CUDA_MODULE_LOADING=EAGER`, which the engine now sets itself) fixes it for ~30 MB of VRAM; the IQ3_XXS 64K/128K rows
+were measured with that fix.

bench/results/2026-09-28-speed-0114/matrix.json

@@ -0,0 +1,322 @@
+[
+ {
+  "model": "Q2_0",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 41.0,
+  "prefill_tok_s": 494.43,
+  "ttft_s": 3.55,
+  "decode_tok_s": 84.26,
+  "decoded": 256,
+  "spec_accept": 0.709,
+  "tokens_per_round": 2.72,
+  "vram_slots": 4176
+ },
+ {
+  "model": "Q2_0",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 41.2,
+  "prefill_tok_s": 1006.63,
+  "ttft_s": 5.05,
+  "decode_tok_s": 90.26,
+  "decoded": 256,
+  "spec_accept": 0.846,
+  "tokens_per_round": 3.23,
+  "vram_slots": 4123
+ },
+ {
+  "model": "Q2_0",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 63.1,
+  "prefill_tok_s": 1307.6,
+  "ttft_s": 25.63,
+  "decode_tok_s": 73.61,
+  "decoded": 256,
+  "spec_accept": 0.709,
+  "tokens_per_round": 2.68,
+  "vram_slots": 3833
+ },
+ {
+  "model": "Q2_0",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 89.0,
+  "prefill_tok_s": 1293.5,
+  "ttft_s": 51.12,
+  "decode_tok_s": 69.02,
+  "decoded": 256,
+  "spec_accept": 0.679,
+  "tokens_per_round": 2.62,
+  "vram_slots": 3780
+ },
+ {
+  "model": "Q2_0",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 147.4,
+  "prefill_tok_s": 1208.46,
+  "ttft_s": 107.81,
+  "decode_tok_s": 67.21,
+  "decoded": 256,
+  "spec_accept": 0.691,
+  "tokens_per_round": 2.64,
+  "vram_slots": 3697
+ },
+ {
+  "model": "Q2_0",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 309.8,
+  "prefill_tok_s": 966.84,
+  "ttft_s": 270.35,
+  "decode_tok_s": 60.3,
+  "decoded": 256,
+  "spec_accept": 0.623,
+  "tokens_per_round": 2.43,
+  "vram_slots": 3525
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 40.4,
+  "prefill_tok_s": 461.28,
+  "ttft_s": 2.28,
+  "decode_tok_s": 74.38,
+  "decoded": 256,
+  "spec_accept": 0.772,
+  "tokens_per_round": 2.84,
+  "vram_slots": 4386,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 41.8,
+  "prefill_tok_s": 810.71,
+  "ttft_s": 4.48,
+  "decode_tok_s": 73.84,
+  "decoded": 256,
+  "spec_accept": 0.79,
+  "tokens_per_round": 3.24,
+  "vram_slots": 4336,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 65.0,
+  "prefill_tok_s": 1237.98,
+  "ttft_s": 25.59,
+  "decode_tok_s": 71.49,
+  "decoded": 256,
+  "spec_accept": 0.856,
+  "tokens_per_round": 3.28,
+  "vram_slots": 4057,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 97.1,
+  "prefill_tok_s": 1135.63,
+  "ttft_s": 56.59,
+  "decode_tok_s": 64.29,
+  "decoded": 256,
+  "spec_accept": 0.707,
+  "tokens_per_round": 2.76,
+  "vram_slots": 4005,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 158.0,
+  "prefill_tok_s": 1070.61,
+  "ttft_s": 120.1,
+  "decode_tok_s": 59.76,
+  "decoded": 256,
+  "spec_accept": 0.726,
+  "tokens_per_round": 2.69,
+  "vram_slots": 3923,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 332.4,
+  "prefill_tok_s": 886.2,
+  "ttft_s": 293.43,
+  "decode_tok_s": 52.83,
+  "decoded": 256,
+  "spec_accept": 0.628,
+  "tokens_per_round": 2.53,
+  "vram_slots": 3565,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 44.1,
+  "prefill_tok_s": 414.55,
+  "ttft_s": 2.54,
+  "decode_tok_s": 60.26,
+  "decoded": 256,
+  "spec_accept": 0.864,
+  "tokens_per_round": 3.52,
+  "vram_slots": 3304,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 45.8,
+  "prefill_tok_s": 769.57,
+  "ttft_s": 4.73,
+  "decode_tok_s": 62.07,
+  "decoded": 256,
+  "spec_accept": 0.855,
+  "tokens_per_round": 3.24,
+  "vram_slots": 3263,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 69.7,
+  "prefill_tok_s": 1107.75,
+  "ttft_s": 28.61,
+  "decode_tok_s": 51.4,
+  "decoded": 256,
+  "spec_accept": 0.678,
+  "tokens_per_round": 2.6,
+  "vram_slots": 3031,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 104.9,
+  "prefill_tok_s": 1065.31,
+  "ttft_s": 61.77,
+  "decode_tok_s": 49.95,
+  "decoded": 256,
+  "spec_accept": 0.741,
+  "tokens_per_round": 2.94,
+  "vram_slots": 2965,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 172.1,
+  "prefill_tok_s": 1015.11,
+  "ttft_s": 128.12,
+  "decode_tok_s": 45.78,
+  "decoded": 256,
+  "spec_accept": 0.665,
+  "tokens_per_round": 2.35,
+  "vram_slots": 2901,
+  "note": "first pass"
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 56.2,
+  "prefill_tok_s": 396.13,
+  "ttft_s": 4.14,
+  "decode_tok_s": 51.52,
+  "decoded": 256,
+  "spec_accept": 0.826,
+  "tokens_per_round": 3.12,
+  "vram_slots": 2617
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 57.5,
+  "prefill_tok_s": 736.9,
+  "ttft_s": 6.35,
+  "decode_tok_s": 51.6,
+  "decoded": 256,
+  "spec_accept": 0.747,
+  "tokens_per_round": 2.89,
+  "vram_slots": 2580
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 82.5,
+  "prefill_tok_s": 1069.83,
+  "ttft_s": 31.33,
+  "decode_tok_s": 48.19,
+  "decoded": 256,
+  "spec_accept": 0.772,
+  "tokens_per_round": 2.94,
+  "vram_slots": 2383
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 115.0,
+  "prefill_tok_s": 1069.8,
+  "ttft_s": 61.47,
+  "decode_tok_s": 48.75,
+  "decoded": 256,
+  "spec_accept": 0.786,
+  "tokens_per_round": 2.9,
+  "vram_slots": 2347
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 191.8,
+  "prefill_tok_s": 931.09,
+  "ttft_s": 138.11,
+  "decode_tok_s": 40.48,
+  "decoded": 256,
+  "spec_accept": 0.661,
+  "tokens_per_round": 2.71,
+  "vram_slots": 2310,
+  "note": "first pass (a download was running)"
+ }
+]

bench/results/2026-09-28-speed-0114/matrix.md

@@ -0,0 +1,35 @@
+### Prompt processing (tok/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| Q2_0 | 494 | 1,007 | 1,308 | 1,294 | 1,208 | 967 |
+| IQ2_XS | 461 | 811 | 1,238 | 1,136 | 1,071 | 886 |
+| IQ3_XXS | 415 | 770 | 1,108 | 1,065 | 1,015 | - |
+| IQ3_S | 396 | 737 | 1,070 | 1,070 | 931 | - |
+
+### Output (tok/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| Q2_0 | 84.3 | 90.3 | 73.6 | 69.0 | 67.2 | 60.3 |
+| IQ2_XS | 74.4 | 73.8 | 71.5 | 64.3 | 59.8 | 52.8 |
+| IQ3_XXS | 60.3 | 62.1 | 51.4 | 50.0 | 45.8 | - |
+| IQ3_S | 51.5 | 51.6 | 48.2 | 48.8 | 40.5 | - |
+
+### Draft acceptance
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| Q2_0 | 0.709 | 0.846 | 0.709 | 0.679 | 0.691 | 0.623 |
+| IQ2_XS | 0.772 | 0.790 | 0.856 | 0.707 | 0.726 | 0.628 |
+| IQ3_XXS | 0.864 | 0.855 | 0.678 | 0.741 | 0.665 | - |
+| IQ3_S | 0.826 | 0.747 | 0.772 | 0.786 | 0.661 | - |
+
+### Experts in VRAM
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| Q2_0 | 4,176 | 4,123 | 3,833 | 3,780 | 3,697 | 3,525 |
+| IQ2_XS | 4,386 | 4,336 | 4,057 | 4,005 | 3,923 | 3,565 |
+| IQ3_XXS | 3,304 | 3,263 | 3,031 | 2,965 | 2,901 | - |
+| IQ3_S | 2,617 | 2,580 | 2,383 | 2,347 | 2,310 | - |

bench/results/2026-09-29-layer-split-limits/README.md

@@ -0,0 +1,59 @@
+# Layer split: the features that were off across GPUs (2026-09-29)
+
+**Setup:**
+- **Rig:** RIBPC. RTX 5080 16 GB (CUDA0) + RTX 3090 24 GB; RTX 2080 Ti 22 GB as a helper card. Ryzen 9 9950X3D,
+  62 GB RAM, Windows 11.
+- **Model:** the Coder (IQ1_M native pack).
+- **Build:** branch `mg/limits`, sm_75 + sm_86 + sm_120.
+- **Correctness runs:** `rib_norepeat.py`, with a fixed cache, no swaps, no PCIe share and no suffix drafts where two
+  runs are compared byte for byte.
+- **Speed runs:** `rib_pp.py`.
+- Raw results are in `data/`.
+
+## Correctness
+
+| Check | Result |
+|---|---|
+| One GPU, this branch vs the 0.1.21-equivalent run (fixed cache) | 10/10 byte-identical |
+| Speed projection (control vector), one GPU vs the split on the same GPU (K=24) | 10/10 byte-identical |
+| KV streaming (128K context, 32K resident), one GPU vs the split on the same GPU | 10/10 byte-identical |
+| Images, one GPU vs the split on the same GPU (three questions about a test picture) | identical answers |
+| Speed projection / KV streaming / plain, 5080 + 3090 (K=24): no-repeat under presence 100, the 144 sanity case | all pass, 0 repeats |
+| The vector acts across GPUs: 5080 + 3090 with vs without the vector | 8 of 10 answers differ (as on one GPU: 7 of 10) |
+| Images, 5080 + 3090 (K=24): "What color is the circle?" / "What number?" / describe | "Red" / "42" / "A red circle, the number 42, and a blue square are displayed on a white background" |
+| Mid-prompt checkpoint (every 16,384 tokens): a 19.5K prompt, then another sharing its first 18,600 tokens | one GPU and 5080 + 3090: reused 16,384, answer right |
+| 5080 + 3090 (K=30) + a 2080 Ti helper holding 3,000 experts | all pass |
+| `--mmap-experts` with the Coder (a native pack) | a clear start-up error: it needs a canonical pack's `experts.bin` (with or without a split) |
+
+Across GPUs the text is not byte-identical to one GPU: experts that one card leaves to the CPU run on a GPU in the
+split, and the two round differently.
+
+## Speed
+
+| Run | Prompt 16K / 28K / 60K tok/s | Decode story / code tok/s | Hits |
+|---|---|---|---|
+| KV streaming (128K), 5080 alone | 1,487 / 1,801 / 1,837 | 66.6 / 74.0 | 59-71% |
+| KV streaming (128K), 5080 + 3090 auto | 1,685 / 1,957 / 1,968 | 75.8 / 97.0 | 96-97% |
+| Speed projection, 5080 + 3090 auto | 1,813 / 2,053 / - | 77.0 / 103.6 | 96-97% |
+| 5080 + 3090 (K=30), no helper | - | 62.5 / 85.2 | |
+| 5080 + 3090 (K=30) + 2080 Ti helper | - | 43.0 / 63.7 | |
+
+**The helper card makes it slower,** as it did without a split (bench/results/2026-09-29-layer-split). Its per-layer
+round trip costs more than the CPU pool needs for the few experts it would take over. It stays supported: the
+helpers take the visible GPUs no stage runs on and hold only experts no stage's cache holds. But it isn't
+recommended.
+
+## What changed
+
+- **Per-device tables:** the image-position (mrope) table and the control vector's tables are now kept per device;
+  every stage's rope and steering kernels read their own device's copy. `cvec_set_enabled` switches the vector on
+  every device.
+- **KV streaming:** each stage's session streams its own KV host copy (pinned, portable). The split no longer forces
+  it off.
+- **Mid-prompt checkpoints:** `Prefill::on_stage_chunk` lets each stage save its part of a checkpoint when it has
+  read the chunk. The last stage assembles the parts: the earlier ones are a chunk ahead by then.
+- **Helpers (`--expert-cache-remote`) with a split:** they take the visible devices after the stages, skip every
+  pair a stage's cache holds, and preflight and open on those devices.
+- **Explicit `--expert-cache` with a split:** it is capped so CUDA0 keeps room for the prompt path's own buffers.
+  Without the cap, the first prompt failed with "device buffers ... do not fit".
+- **`--mmap-experts`:** refused at start for a native pack, with the reason.

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-cv-same24-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 102,
+  "drafts_offered": 166,
+  "tok_s": 66.7,
+  "text": "The wind didn\u2019t just howl; it screamed. It tore through the iron gallery of the Blackwood Point Light, rattling the glass panes like a prisoner testing the bars of a cell.\n\nElias Thorne was not a man who feared the dark. He was a man who feared the silence.\n\nFor forty years, he had lived in the shadow of the Fresnel lens, a rotating eye of glass and brass that swept the Atlantic with a beam of pure, unyielding light. His job was simple: keep the light burning. Keep the logbook current. Keep the foghorn sounding when the mist rolled in, thick as wool.\n\nBut Elias knew something else. He knew that the light was not just for ships. It was for the lost.\n\nIt was November 14th, 1982. The storm was a Category 5, a monster that had swallowed the horizon. The waves were mountains of black water, crashing against the granite"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 134,
+  "drafts_offered": 180,
+  "tok_s": 80.8,
+  "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 72.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.6,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 104,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 92,
+  "tok_s": 45.4,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAnaconda    \nAntelope     \nBaboon       \nBadger      \nBat         \nBear        \nBeaver          \nBirds of Paradise (Manucode) \nBoa Constrictor \n\nBuffalo           \nCamel            \nCapybara             \nCat              \nChameleon\nCheetah\n\nChicken\n\n\nCrocodile  \nDeer  \n\nDingo   \nDog    \nDonkey     \nDragonfly   \n\nElephant       \nEmu      \nFalcon        \nFerret          \nFlamingo         \nFox<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 72,
+  "repeats": 0,
+  "drafts_accepted": 15,
+  "drafts_offered": 70,
+  "tok_s": 46.1,
+  "text": "The wind didn\u2019t just howl; it screamed against the glass of Lantern House 42, demanding entry into every crack and crevice that had been sealed with tar in previous years but now wept like old wounds on an open sore skin\u2014like a wound left unhealed by time. Elias knew this rhythm better than he remembered his own name<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 76,
+  "tok_s": 49.3,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in my teeth, searching for something to break its rhythm against stone that refused be broken by anything less than God himself had forged into being willing and able withstand eternity\u2019s cruelty without complaint or mercy from start finish until now forevermore if any such concept even existed within human comprehension let alone divine authority which we all knew did not exist but wished it would because wishful thinking is always easier to imagine hopelessly impossible truths as probable lies than vice versa.\n\nElias Thorne was eighty-three years old today, exactly one hundred and forty-seven days since his last heart attack had been logged by Dr Armitage who himself retired seven months ago after receiving a subpoena from the county clerk indicating he might have overstayed professional courtesy boundaries"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 27,
+  "drafts_offered": 56,
+  "tok_s": 46.7,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippo         \nRhino          \nCheetah            \nLeopard           \nJaguar             \nWolf                \nFox              \nBear               \nDeer                 \nElk                  \nMoose                    \nBuffalo                        \nCamel                       \nDonkey                      \nGoat                     \nSheep                   \nCow                         \nBull                          \nOx                           \nPig                            \nBoar                             \nRabbit                              \nSquirrel                                <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.4,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 148,
+  "tok_s": 56.4,
+  "text": "The phrase \"the History Of The Sea\" is inherently ambiguous, as it could refer to several distinct historical narratives: geological formation (how oceans came into existence), maritime navigation and exploration by humans throughout human civilization; or perhaps more specifically in modern times refers often used metaphorically for economic trends like stock market fluctuations (\"sea change\"). However given your request likely intends an academic-style comprehensive overview covering both natural science aspects AND cultural significance through time periods\u2014especially since this kind question appears frequently on exams asking students write essays discussing evolution from primordial origins up present-day impacts including environmental changes due climate crisis etc.\u2014I will craft such broad yet structured piece combining geology + anthropology/ecological impact together while maintaining coherent flow suitable high-level educational context without overly technical jargon but still accurate enough scientifically grounded facts where appropriate within reasonable scope limits typical school assignments expect (~80\u201312% word count range implied implicitly though not stated explicitly here we aim moderate length ~65 words? Wait no user said \u201clong\u201d so probably >3k+ characters minimum safe bet around 7-9 paragraphs total approximating maybe close-to-full-page standard double-spaced equivalent which translates roughly \u22654K chars safely acceptable). Let me proceed accordingly ensuring clarity structure intro body conclusion style formal tone neutral informative balanced perspective acknowledging uncertainties if any minor ones exist e.g exact dates vary depending sources hence use hedging language sparingly only when necessary avoid overclaiming certainty especially regarding speculative theories vs established consensus knowledge base current mainstream understanding circa early-mid"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-cv-single-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 102,
+  "drafts_offered": 166,
+  "tok_s": 67.1,
+  "text": "The wind didn\u2019t just howl; it screamed. It tore through the iron gallery of the Blackwood Point Light, rattling the glass panes like a prisoner testing the bars of a cell.\n\nElias Thorne was not a man who feared the dark. He was a man who feared the silence.\n\nFor forty years, he had lived in the shadow of the Fresnel lens, a rotating eye of glass and brass that swept the Atlantic with a beam of pure, unyielding light. His job was simple: keep the light burning. Keep the logbook current. Keep the foghorn sounding when the mist rolled in, thick as wool.\n\nBut Elias knew something else. He knew that the light was not just for ships. It was for the lost.\n\nIt was November 14th, 1982. The storm was a Category 5, a monster that had swallowed the horizon. The waves were mountains of black water, crashing against the granite"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 134,
+  "drafts_offered": 180,
+  "tok_s": 81.0,
+  "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 62.4,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.4,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 104,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 92,
+  "tok_s": 43.5,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAnaconda    \nAntelope     \nBaboon       \nBadger      \nBat         \nBear        \nBeaver          \nBirds of Paradise (Manucode) \nBoa Constrictor \n\nBuffalo           \nCamel            \nCapybara             \nCat              \nChameleon\nCheetah\n\nChicken\n\n\nCrocodile  \nDeer  \n\nDingo   \nDog    \nDonkey     \nDragonfly   \n\nElephant       \nEmu      \nFalcon        \nFerret          \nFlamingo         \nFox<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 72,
+  "repeats": 0,
+  "drafts_accepted": 15,
+  "drafts_offered": 70,
+  "tok_s": 46.1,
+  "text": "The wind didn\u2019t just howl; it screamed against the glass of Lantern House 42, demanding entry into every crack and crevice that had been sealed with tar in previous years but now wept like old wounds on an open sore skin\u2014like a wound left unhealed by time. Elias knew this rhythm better than he remembered his own name<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 76,
+  "tok_s": 48.9,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in my teeth, searching for something to break its rhythm against stone that refused be broken by anything less than God himself had forged into being willing and able withstand eternity\u2019s cruelty without complaint or mercy from start finish until now forevermore if any such concept even existed within human comprehension let alone divine authority which we all knew did not exist but wished it would because wishful thinking is always easier to imagine hopelessly impossible truths as probable lies than vice versa.\n\nElias Thorne was eighty-three years old today, exactly one hundred and forty-seven days since his last heart attack had been logged by Dr Armitage who himself retired seven months ago after receiving a subpoena from the county clerk indicating he might have overstayed professional courtesy boundaries"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 27,
+  "drafts_offered": 56,
+  "tok_s": 46.8,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippo         \nRhino          \nCheetah            \nLeopard           \nJaguar             \nWolf                \nFox              \nBear               \nDeer                 \nElk                  \nMoose                    \nBuffalo                        \nCamel                       \nDonkey                      \nGoat                     \nSheep                   \nCow                         \nBull                          \nOx                           \nPig                            \nBoar                             \nRabbit                              \nSquirrel                                <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 63.1,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 148,
+  "tok_s": 56.6,
+  "text": "The phrase \"the History Of The Sea\" is inherently ambiguous, as it could refer to several distinct historical narratives: geological formation (how oceans came into existence), maritime navigation and exploration by humans throughout human civilization; or perhaps more specifically in modern times refers often used metaphorically for economic trends like stock market fluctuations (\"sea change\"). However given your request likely intends an academic-style comprehensive overview covering both natural science aspects AND cultural significance through time periods\u2014especially since this kind question appears frequently on exams asking students write essays discussing evolution from primordial origins up present-day impacts including environmental changes due climate crisis etc.\u2014I will craft such broad yet structured piece combining geology + anthropology/ecological impact together while maintaining coherent flow suitable high-level educational context without overly technical jargon but still accurate enough scientifically grounded facts where appropriate within reasonable scope limits typical school assignments expect (~80\u201312% word count range implied implicitly though not stated explicitly here we aim moderate length ~65 words? Wait no user said \u201clong\u201d so probably >3k+ characters minimum safe bet around 7-9 paragraphs total approximating maybe close-to-full-page standard double-spaced equivalent which translates roughly \u22654K chars safely acceptable). Let me proceed accordingly ensuring clarity structure intro body conclusion style formal tone neutral informative balanced perspective acknowledging uncertainties if any minor ones exist e.g exact dates vary depending sources hence use hedging language sparingly only when necessary avoid overclaiming certainty especially regarding speculative theories vs established consensus knowledge base current mainstream understanding circa early-mid"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-cv-x2-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 85,
+  "drafts_offered": 172,
+  "tok_s": 36.6,
+  "text": "The wind didn\u2019t just howl; it screamed. It tore through the gaps in the iron railing of the Lantern Room, carrying the scent of brine and rotting kelp. Elias pulled his woolen collar up to his chin, his fingers numb and clumsy inside thick leather gloves.\n\nHe was the last man on the island. The automated systems had failed three winters ago, a glitch in the solar-powered backup that no engineer had been able to fix before the supply boat stopped coming. Now, it was just him, the light, and the sea.\n\nElias climbed the spiral stairs, his boots heavy with salt spray. The iron steps groaned under his weight, a sound that seemed to mock the storm outside. He reached the top, the glass walls vibrating with the force of the gale. Outside, the world was a blur of black and white. The ocean was invisible; it was only a presence, a weight pressing against the stone foundation of the tower.\n\nHe"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 173,
+  "tok_s": 96.8,
+  "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 88.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 94.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 104,
+  "repeats": 0,
+  "drafts_accepted": 30,
+  "drafts_offered": 96,
+  "tok_s": 55.9,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAnaconda    \nAntelope     \nBaboon       \nBadger      \nBat         \nBear        \nBeaver          \nBirds of Paradise (Manucode) \nBoa Constrictor \n\nBuffalo           \nCamel            \nCapybara             \nCat              \nChameleon\nCheetah\n\nChicken\n\n\nCrocodile  \nDeer  \n\nDingo   \nDog    \nDonkey     \nDragonfly   \n\nElephant       \nEmu      \nFalcon        \nFerret          \nFlamingo         \nFox<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 114,
+  "tok_s": 55.4,
+  "text": "The wind didn\u2019t just howl; it screamed against the glass of Lantern House 42, demanding entry into every crack and crevice that had been sealed with tar in previous years but now wept like old wounds at this one\u2019s age: forty-three winters since its construction. My name was Elias Thorne\u2014no middle initial\u2014and I kept no logbook for visitors because there were none; only ships passed by, their hulls cutting through the black water as if they too feared what lay beneath it more than above or beside them in fog so thick that even my own reflection vanished from sight when looked upon directly.\n\nI climbed each morning not to greet dawn but check whether light still lived within me\u2014the bulb\u2019s heart had failed twice before, once during a storm named *Harrow*,"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 110,
+  "tok_s": 52.2,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in my teeth, searching for something to hold onto until I died.\u201d said Elias Thorne as he adjusted his collar against November\u2019s bite.** He was not merely complaining of cold weather but articulating a poetic metaphor: that even after death (metaphorically), there would be no relief from this relentless gale.\n\nElias had been the last human keeper on any significant stretch coast since automation became standard practice, yet here he stood , stubbornly refusing to let technology replace his vigilance . His cottage perched atop a crumbling granite foundation was more than just shelter; it housed centuries of handwritten logs detailing every ship\u2019s name and cargo load that passed within sight.\n\nOn most nights Elias tended the light mechanically\u2014"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 91,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 69,
+  "tok_s": 58.7,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippopotamus          \nRhinoceros            \nLeopard                \nCheetah                 \nPanther                  \nWolf                    \nFox                      \nBear                     \nDeer                       \nElk                        \nMoose               \nBuffalo                   \nCamel           \nDonkey              \nGoat             \nSheep         \nCow  \nBull   \nOx    \nPig     \nDog       \nCat<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 12,
+  "repeats": 0,
+  "drafts_accepted": 4,
+  "drafts_offered": 17,
+  "tok_s": 60.3,
+  "text": "1 2,3456789<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 41,
+  "drafts_offered": 168,
+  "tok_s": 59.9,
+  "text": "The phrase \"the History Of The Sea\" is inherently ambiguous, as it could refer to several distinct historical narratives: geological formation (how oceans came into existence), maritime navigation and exploration histories (\"history at/over/across/on/in/of/to/from/toward/off by means around above below through under upon alongside against among between within without throughout despite during after before since until while when where who whom whose which what how why whether\"), or perhaps even metaphorical interpretations involving human interaction with water bodies over time periods like ages eras epochs generations centuries decades years months days hours minutes seconds moments instants times occasions instances events episodes incidents occurrences situations circumstances conditions states phases stages steps processes procedures sequences series chains cycles loops repetitions iterations continuations progressions developments evolutions transformations changes alterations modifications adjustments corrections revisions amendments updates upgrades improvements enhancements refinements optimizations perfecting completing finishing concluding ending terminating ceasing stopping halting pausing resuming continuing proceeding advancing progressing moving shifting translating rotating reflecting mirroring flipping turning twisting bending curving arches spirals helix coils twists turns rotations reflections mirrors flips inversions reversings reversion backtracking retracing reversing undo redo do again repeat repeats repeated repeatedly repetition repetitive repetitiveness repete repeater repeatingly recurrent recurrence recurring recurs recur recurred recursion recursive recursively recurse recessed recessionary regress regression regressed retrograde reverse reversed reversal revert reverted reverberate resonated echo echoed echoing echoes echolocation sonar radar radio telecommunication communication communications communicate communicated communicating communicative comms comm chatter chat chats chatting chatted talk talks talked"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-hp0-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 171,
+  "tok_s": 62.5,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from broken glass and rusted iron, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, ensuring the rotation was smooth, silent, and precise. *Tick. Tick. Tick.* The sound was the heartbeat of the coast."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 184,
+  "tok_s": 85.2,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 78.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 80.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 13,
+  "drafts_offered": 48,
+  "tok_s": 46.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBuffalo     \n\nCat          \nCheetah      \n\nChimpanzee \nClam         \n\n\nCrocodile   \n\n\n\nDolphin      \n\n\n\n\nDonkey       <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 30,
+  "drafts_offered": 101,
+  "tok_s": 51.3,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots smelled faintly brine and old coffee as he climbed spiral stairs worn smooth by decades feet his hands were calloused from gripping iron railings that had been polished to a dull shine\n\nElias was not merely keeping light alive; He kept order intact. Every night, at precisely 18:07 hours local time\u2014adjusted for leap seconds and magnetic drift\u2014he wound the clockwork mechanism inside brass housing behind Fresnel lens This wasn\u2019t vanity It\u2019s necessity The rotation speed must remain constant or ships lose their bearing in fog where compasses fail\n\nHe spoke softly to himself while working, murmuring numbers aloud because silence made him"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 67,
+  "repeats": 0,
+  "drafts_accepted": 14,
+  "drafts_offered": 68,
+  "tok_s": 41.2,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, like he had no mouth to hold back its fury anymore that night.\u201d Elara pulled her wool coat tighter around herself as she climbed spiral staircase of **Blackrock Lighthouse**. Her boots were slick with salt spray from previous days.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 91,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 54,
+  "tok_s": 48.8,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nHippopotamus          \nRhinoceros            \nCamel                 \nCheetah               \nLeopard                \nJaguar                  \nBear                    \nWolf                     \nFox                      \nDog                       \nCat                        \nMouse                   \nRat                         \nSquirrel              \nDeer                      \n\nAntelope                \n\nBuffalo                 \n\n\nBison                   \n\n\n\nElk                    \n\n\n\n\nMoose                  <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 11,
+  "tok_s": 68.8,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 32,
+  "drafts_offered": 107,
+  "tok_s": 52.3,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant storybook that weaves together geology with biology as wellas human culture into an epic narrative spanning billions years from its earliest beginnings in primordial times through modern-day ecological crises The vast expanse covers more than seventy percent Earth\u2019s surface area making up nearly all life support systems on our planet yet despite being so large humans have only explored less five thousand square kilometers out total 361 million km\u00b2 which means over ninety-nine point nine eight six per cent unexplored territory This makes oceans arguably larger unknown territories compared space exploration where NASA estimates they\u2019ve mapped roughly ten billion stars versus current marine surveys covering just two hundred fifty-six species documented globally against estimated millions undiscovered organisms including deep-sea creatures like anglerfish or vampire squid whose existence was first recorded centuries ago but still remain poorly understood due extreme pressures depths exceeding eleven miles below Pacific Trenches Mariana located near Guam island chain reaching maximum depth known today at approximately thirty-seven feet under pressure equivalent atmospheric levels causing structural collapse if unprotected by specialized vessels capable handling such conditions without failure mechanisms designed specifically for submersible operations operating within these zones require titanium hulls rated withstand forces greater gravity acting upon objects falling vertically downward acceleration rates calculated using Newtonian physics principles applied universally across gravitational fields regardless medium density variations present throughout different layers stratosphere troposphere mesospheres thermoclines etceterally speaking however when discussing aquatic environments instead air masses replace gases dissolved oxygen nitrogen arg"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-hp1-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 94,
+  "drafts_offered": 169,
+  "tok_s": 43.0,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy-three years old, and he had been the keeper of the Blackwood Point Light for forty of those years. The other thirty-three had been spent in a small, drafty cottage that smelled perpetually of kerosene and damp wood.\n\nFor decades, Elias had tended the great Fresnel lens, polishing the glass prisms until they gleamed like diamonds, winding the clockwork mechanism that turned the beam, and trimming the wick of the oil lamp. It was a job of patience, of precision, of silence. The light was his charge. The sea was his neighbor.\n\nBut this morning, the sea was not his neighbor. It was his enemy.\n\nThe storm had been forecasted for three days, but it had arrived in one. The barometric pressure had dropped so sharply that Elias\u2019s ears popped."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 178,
+  "tok_s": 63.7,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 56.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 57.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 24,
+  "drafts_offered": 49,
+  "tok_s": 43.3,
+  "text": "Lion  \nTiger   \nElephant    \nGiraffe     \nZebra       \nHippopotamus \nRhinoceros      \nCheetah         \nLeopard        \nJaguar          \nPuma           \nWolf            \nFox             \nBear              \nDeer                \nElk                 \nMoose               \nAntelope                  \nBison                   \nBuffalo                    \nCamel                     \nDonkey                      \nGoat                       \nSheep                        \nCow                         \nDog                          \nCat                           \nRabbit                            \nMouse                             \nHamster                               \nGuinea Pig<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 35,
+  "drafts_offered": 108,
+  "tok_s": 41.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots smelled faintly brine and old coffee as he climbed spiral stairs worn smooth by decades feet his hands were calloused from gripping iron railings that had been polished to a dull shine.\n\nElias wasn\u2019t just keeping light; was guarding silence the beam swept across black water every seven seconds cutting through fog so thick it swallowed sound entirely for three nights straight no ships answered its signal not even fishing trawlers seeking shelter they\u2019d long abandoned this stretch of coast where currents shifted unpredictably pulling debris from sunken wrecks into tangled heaps that snagged anchors and tore hulls open like wounds.\n\nHe remembered the last time someone had"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 46,
+  "tok_s": 33.7,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, trying to pry them apart like loose planks of weathered oak outside your own mouth again and then you are going away from home is fine but for now let me write a proper story.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 60,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 48,
+  "tok_s": 43.2,
+  "text": "Aardvark  \nAntelope allonax (Oryx) \u2013 *Note: Using standard names for clarity*  \n\nLet me provide a clean list of exactly **1** animal name on each single physical text file/terminal output stream with no other characters except the newline (\\n).<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 11,
+  "tok_s": 48.9,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 27,
+  "drafts_offered": 105,
+  "tok_s": 40.2,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element\u2014it became civilization itself through its capacity as both barrier (to land) but also conduit between lands\u2014and thus gave rise directly from being part nature\u2019s own evolutionary engine into becoming humanity's greatest historical actor across all time scales since first emergence until now present-day future projections still ongoing today within our current era context window limits allowed here by system constraints set above threshold levels required before output generation begins below line break marker inserted immediately after final punctuation mark ending entire preceding paragraph block structure defined earlier during initial drafting phase prior execution stage completed successfully without errors detected internally via internal validation routines executed automatically upon completion cycle termination signal received end-of-file condition triggered correctly according standard protocol specifications followed strictly adhered-to throughout process flow control logic implemented using structured programming constructs compatible with target runtime environment configuration parameters specified herein explicitly declared beforehand at top-level scope declaration section located near beginning source code file header comments included inline documentation style guidelines observed consistently applied uniformly everywhere else similarly formatted sections appearing later down document body text area scrolling vertically downward directionally aligned left margin justified right edge wrapped around fixed-width column boundaries determined dynamically based available screen real estate measured pixel-per-unit scaling factor adjusted responsively depending viewport dimensions reported back client-side rendering pipeline components responsible"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-kv-same24-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 94,
+  "drafts_offered": 167,
+  "tok_s": 62.7,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to hold the chaos of the ocean at bay. The beam swept out, a blade of pure, concentrated light cutting through the rain and sleet, illuminating the jagged teeth of the reef below.\n\n"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 179,
+  "tok_s": 81.3,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 71.0,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 71.9,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 52,
+  "tok_s": 47.9,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFalcon          <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 122,
+  "tok_s": 47.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach his post at dawn every morning for thirty years. His name was written on a brass plaque by door: **Elias Thorne**.\n\nHe wasn\u2019t just keeping light; He kept time, weathered hands gripping the cold iron railing until knuckles turned white against frostbite and windburn that never healed fully because they were always there waiting to be felt again when he stepped outside into storm or calm alike. The lens above him was a Fresnel giant\u2019s eye made of hand-ground crystal prisms each one polished daily with chamois cloth soaked in whale"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 65,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 58,
+  "tok_s": 42.6,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they slammed against their hinges every night for eleven months of winter and summer alike.\u201d I had never seen calm water here\u2014only gray churning seas that swallowed ships smaller than themselves<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 101,
+  "repeats": 0,
+  "drafts_accepted": 32,
+  "drafts_offered": 53,
+  "tok_s": 49.4,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nCheetah         \nHippopotamus          \nRhinoceros            \nLeopard             \nJaguar              \nWolf                \nFox                 \nBear                  \nDeer                    \nOtter                   \nBeaver                      \nBadger                        \nMongoose                          \nMeerkat                           \nHyena                             \nJackal                            \nAntelope                              \nBison                               \nElk                                 \nCaribou                                   \nLlama                                    \nAlpace                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 12,
+  "tok_s": 64.7,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 52,
+  "drafts_offered": 148,
+  "tok_s": 49.9,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by hundreds millions years yet continues shaping every aspect from climate regulation through evolutionary biology down even into modern industrial logistics today's global supply chains depend upon them directly indirectly via atmospheric pressure systems weather patterns migration routes food webs carbon cycles oxygen generation thermal buffering tidal forces salinity gradients depth stratification light attenuation currents circulation gyres upwelling zones thermoclines halothermics bathymetric features continental shelves abyssal plains trenching ridges volcanic arcs island formations atoll structures seamounts submarine canons canyon walls cliff faces overhang ledge platforms reef flats fore-reef back-lagoon inner-shelf outer-slope drop-offs pinnacle pinnacles spires columns arches tunnels caves grottos swim-through passages chimney vents blowholes fissures cracks crevices joints faults fractures breaks gaps holes openings void spaces hollow chambers cavernous interiors subterranean networks underground rivers lakes pools springs wells shaft pits sinks drains sewers pipes conduits tubes channels gutters ditches furrows grooves rills runnels streams brooks rivulets creeks tributaries branches forks splits divergences con"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-kv-single-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 94,
+  "drafts_offered": 167,
+  "tok_s": 49.2,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to hold the chaos of the ocean at bay. The beam swept out, a blade of pure, concentrated light cutting through the rain and sleet, illuminating the jagged teeth of the reef below.\n\n"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 179,
+  "tok_s": 76.2,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.4,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 52,
+  "tok_s": 47.1,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFalcon          <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 122,
+  "tok_s": 47.0,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach his post at dawn every morning for thirty years. His name was written on a brass plaque by door: **Elias Thorne**.\n\nHe wasn\u2019t just keeping light; He kept time, weathered hands gripping the cold iron railing until knuckles turned white against frostbite and windburn that never healed fully because they were always there waiting to be felt again when he stepped outside into storm or calm alike. The lens above him was a Fresnel giant\u2019s eye made of hand-ground crystal prisms each one polished daily with chamois cloth soaked in whale"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 65,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 58,
+  "tok_s": 42.3,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they slammed against their hinges every night for eleven months of winter and summer alike.\u201d I had never seen calm water here\u2014only gray churning seas that swallowed ships smaller than themselves<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 101,
+  "repeats": 0,
+  "drafts_accepted": 32,
+  "drafts_offered": 53,
+  "tok_s": 50.0,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nCheetah         \nHippopotamus          \nRhinoceros            \nLeopard             \nJaguar              \nWolf                \nFox                 \nBear                  \nDeer                    \nOtter                   \nBeaver                      \nBadger                        \nMongoose                          \nMeerkat                           \nHyena                             \nJackal                            \nAntelope                              \nBison                               \nElk                                 \nCaribou                                   \nLlama                                    \nAlpace                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 12,
+  "tok_s": 64.6,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 52,
+  "drafts_offered": 148,
+  "tok_s": 49.8,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by hundreds millions years yet continues shaping every aspect from climate regulation through evolutionary biology down even into modern industrial logistics today's global supply chains depend upon them directly indirectly via atmospheric pressure systems weather patterns migration routes food webs carbon cycles oxygen generation thermal buffering tidal forces salinity gradients depth stratification light attenuation currents circulation gyres upwelling zones thermoclines halothermics bathymetric features continental shelves abyssal plains trenching ridges volcanic arcs island formations atoll structures seamounts submarine canons canyon walls cliff faces overhang ledge platforms reef flats fore-reef back-lagoon inner-shelf outer-slope drop-offs pinnacle pinnacles spires columns arches tunnels caves grottos swim-through passages chimney vents blowholes fissures cracks crevices joints faults fractures breaks gaps holes openings void spaces hollow chambers cavernous interiors subterranean networks underground rivers lakes pools springs wells shaft pits sinks drains sewers pipes conduits tubes channels gutters ditches furrows grooves rills runnels streams brooks rivulets creeks tributaries branches forks splits divergences con"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-kv-x2-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 97,
+  "drafts_offered": 176,
+  "tok_s": 73.0,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was the last of the line, the only man left on the jagged spine of Widow\u2019s Rock. For forty years, he had tended the light. For forty years, he had listened to the ocean\u2019s endless, hungry whisper.\n\nTonight was different. The storm wasn\u2019t just a storm; it was a *tempest*. The waves didn\u2019t crash; they *shattered*. They threw themselves against the granite base of the lighthouse with the force of a battering ram, sending tremors up through the soles of Elias\u2019s boots.\n\nHe climbed the spiral stairs, one hundred and twelve steps of iron and rust. His hand, calloused and scarred from decades of polishing the brass railing, found the familiar groove in the metal. He didn\u2019t look up. He didn\u2019t look down. He looked *forward*."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 176,
+  "tok_s": 102.5,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 90.9,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 91.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 118,
+  "repeats": 0,
+  "drafts_accepted": 38,
+  "drafts_offered": 103,
+  "tok_s": 62.8,
+  "text": "Aardvark  \nAlbatross   \nAlligator    \nAntelope     \nBaboon       \nBadger      \nBeaver        \nBear          \nBird of Paradise \nBuffalo         \nCamel           \nCheetah            \nChimpanzee             \nCrane              \nCrocodile                \nDeer                 \nDolphin               \nDonkey                  \nElephant                   \nFlamingo                    \nFox                       \nGiraffe                     \nGoat                      \nHippopotamus  \nIguana       \nJaguar        \nKangaroo      \nKoala          \nLion         \nLeopard     \nMonkey    \nMoose   \nOstrich \nOtter\nPanda<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 58,
+  "repeats": 0,
+  "drafts_accepted": 9,
+  "drafts_offered": 63,
+  "tok_s": 50.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty jail cell below sea level three hundred feet down to where Elias kept his boots dry and ready for emergencies when he had no time even think about changing them.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 140,
+  "repeats": 0,
+  "drafts_accepted": 22,
+  "drafts_offered": 90,
+  "tok_s": 50.1,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, demanding entry into every crack and crevice of Stone Harbor Lighthous eKeepers have always been solitary figures against isolation\u2019s greatest adversary: time itself Elias Thorne knew this intimately. For thirty years he had kept watch over one hundred thousand ships each night scanning for lights that might signal disaster or hope\n\nHis job was simple but deadly serious He checked weather logs every day without fail The foghorn echoed through the tower three times before dawn Each morning at 4:02 AM Elias would climb eighty-two steps to reach his office where coffee cooled beside a single candle flickering in drafts from below<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 96,
+  "repeats": 0,
+  "drafts_accepted": 31,
+  "drafts_offered": 69,
+  "tok_s": 58.9,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippo         \nRhino          \nCheetah            \nLeopard             \nJaguar              \nPanther               \nCougar                \nBear                  \nWolf                    \nFox                     \nDeer                      \nElk                        \nMoose                       \nBison                         \nBuffalo                          \nAntelope                           \nCamel                            \nDonkey                              \nGoat                                \nSheep                                  \nCow                                    \nPig                                     \nHorse                                       <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 71.2,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 119,
+  "tok_s": 61.3,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth\u2019s surface.\u201d \u2014 *This quote by Rachel Carson serves as an apt introduction.* The story we tell today begins with that first drop falling into primordial pools billions years ago\u2014but what follows next? What happens when those drops become oceans themselves\u2014when they begin moving across continents carrying sediment from mountains down rivers until finally reaching their final resting place at last\u2014the deep abyssal plains where sunlight cannot reach nor pressure crushes everything else alive there below depths greater than six kilometers beneath waves crashing against shores above them all around world forever changing shape over time due plate tectonics shifting landmass slowly apart together again repeatedly throughout geological epochs spanning hundreds millions or even thousands million+ yrs since formation early Hadean era through Archeozoic eons onward present Holocene epoch still ongoing now right here under current conditions observed via satellites monitoring global climate patterns affecting currents temperatures salinity levels biodiversity distribution human impacts altering ecosystems directly indirectly causing extinction events locally globally recently historically prehistorically anciently mythologically culturally spiritually philosoph scientifically mathematically physically chem biol geo astro cosmological sociopolitical economic environmental ecological evolutionary genetic linguistic literary artistic musical cinematic theatrical photographic journalistic digital analog virtual real tangible intangible metaphysical ontosemantic semiotic symbolic allegorical narrative descriptive analytical interpretive evaluative critical reflective introspective contemplatory ponderous thoughtful considerate attentive observant perceptive insightful intelligent knowledgeable informed educated literati learned skilled capable"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-nocv-x2-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 87,
+  "drafts_offered": 170,
+  "tok_s": 71.4,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and green.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from rope burns and broken glass, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, listening for the faint, rhythmic *tick-tick-tick* that signaled the rotation was true. If the rhythm faltered, ships"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 178,
+  "tok_s": 104.1,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 90.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 94.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 76,
+  "repeats": 0,
+  "drafts_accepted": 20,
+  "drafts_offered": 64,
+  "tok_s": 61.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear         \nBird        \nBuffalo          \nButterfly \nCamel           \nCat            \nCaterpillar \n\nCentipede    \n\nChameleon   \n\nCheetah     \n\nChicken      \n\nCricket       \n\nCrocodile     \n\n\nDolphin       \n\n\n\nDonkey        \n\n\n\n\nDragon<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 21,
+  "drafts_offered": 90,
+  "tok_s": 59.2,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty jail cell below sea level to above cloud height at once\u2014no matter how high you climbed or deep into its foundations down through rock and saltwater. It was November again when Elias first noticed that his logbook had begun recording entries he hadn\u2019t written yet: *Storm surge expected Tuesday, but the sky is clear.*\n\nHe stopped writing for a moment to listen\u2014not with ears alone\u2014but by pressing both palms flat against cold iron walls where condensation formed in slow spirals like fingerprints left behind after touching something invisible. The metal hummed faintly beneath his skin as if remembering old storms long past their dates on paper calendars torn from newspapers dated three years ahead of today"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 63,
+  "tok_s": 52.6,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, trying to get inside where he had spent thirty years refusing entry except for family and friends like him who were old enough too know better than young men ever could be again about what safety meant. It was November 2014 when Elias Thorne first stepped onto that spiral staircase with only one boot on because his other foot hurt after walking from the harbor office all morning long before anyone else thought he needed shoes off there for work even if nobody saw them do it but everyone knew why so they did not ask him anymore once this happened already twice last winter near same season now here again today too soon then tomorrow maybe never will come back unless someone sends message tonight via radio signal code Morse dot dash pattern"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 60,
+  "tok_s": 58.8,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nKangaroo       \nPanda        \nKoala          \nCheetah      \nWolf         \nFox           \nBear            \nMonkey             \nDolphin              \nWhale                \nShark                 \nOctopus               \nJellyfish                  \nCrab                    \nSpider                   \nRabbit                     \nCat                      \nDog                       \nHorse                        \nCow                          \nSheep                            \nGoat                              \nChicken                                \nRooster                                   \nTurkey                                  \nEagle                                    \nFalcon                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 71.8,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 52,
+  "drafts_offered": 148,
+  "tok_s": 61.8,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\u201d \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \u201cthe Sea\u201d or \"Ocean\" (terms often used interchangeably but distinct geographically) was never just an element within our planet\u2019s geography\u2014it became its engine itself\u2014a living system whose evolution shaped everything else around us from climate patterns down through evolutionary biology right up into modern civilization's economic structures cultural identities political boundaries ecological systems scientific understanding technological capabilities philosophical frameworks psychological experiences artistic expressions literary metaphors cinematic narratives musical compositions architectural designs urban planning strategies agricultural practices industrial processes medical discoveries legal precedents educational curricula religious traditions spiritual beliefs mythological stories folkloric tales poetic verses dramatic scripts visual arts photographic images digital media social networks behavioral norms emotional responses cognitive models decision-making algorithms predictive analytics statistical probabilities probabilistic reasoning causal inference counterfactual thinking hypothetical scenarios normative ethics deontal logic modal semantics temporal dynamics spatial relations topographical features hydrological cycles meteor phenomena atmospheric chemistry geological formations tectonic shifts volcanic eruptions seismic events tsunamis hurricanes cyclones monsoons typhoons droughts floods storms winds currents waves swells breakers surf foam spray mist fog rain snow ice glaciers berg floes drift pack-ice brine salinity pH alkalines acidity hardness softness temperature pressure depth light darkness color hue saturation brightness contrast luminance chroma tone value shade tint reflection"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-reg-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 187,
+  "tok_s": 58.7,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, mapped with veins like the roots of an ancient oak, gripped the cold iron railing.\n\nFor forty years, Elias had tended the light at the end of the world. Not the end of the map, but the end of the land. Here, the Atlantic threw itself against the granite cliffs with such violence that the spray reached the gallery, thirty feet above the rocks.\n\nHe checked the clock. 11:42 PM. The fog was thickening, a gray soup that swallowed the horizon. In the old days, he would have climbed the stairs to the lantern room, polished the Fresnel lens, and wound the clockwork mechanism that turned the great beam. But the automation had changed"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 179,
+  "tok_s": 80.7,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 71.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.0,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 19,
+  "drafts_offered": 49,
+  "tok_s": 49.1,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFlamingo<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 80,
+  "repeats": 0,
+  "drafts_accepted": 6,
+  "drafts_offered": 52,
+  "tok_s": 46.2,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to check what was happening with that storm and why they were calling me out there when all we had left for tonight\u2019s shift is one last pass through our logs before dawn breaks.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 83,
+  "tok_s": 43.9,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in its teeth, hungry and relentless for three centuries of stone salt-spray history that had long since forgotten to be polite or gentle with itself against anything smaller than God Himself was willing acknowledge as divine protection from drowning souls who were already lost before they ever touched water.\n\nElias knew this rhythm better he himself could hear it sometimes; his boots worn smooth by pacing the spiral stair\u2019s center-post groove where thousands upon countless steps had rubbed away texture until all that remained beneath sole leather now felt like polished bone carved out slowly but surely over time into something more hollow than wood ever really should be allowed become when used so often enough within such confined circular spaces meant only for vertical ascent down always descending again back upward never truly leaving"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 62,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 31,
+  "tok_s": 49.1,
+  "text": "Lion  \nTiger  \n\nElephant  \n  \nGiraffe  \n\n\n\nHippopotamus    \nZebra   \nRhinoceros   \n\n\nPanda Bear    \n\n\n\n\nKangaroo     \nKoala      \nSloth      \n\n\n\n\n\nCheetah       \nLeopard        \nPanther        \n\n\n\n\n\n\nWolf              \nFox                \nBear                 <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.7,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 47,
+  "drafts_offered": 153,
+  "tok_s": 52.2,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by billions years yet continues shaping every aspect from climate regulation through evolutionary biology all way down modern industrial supply chains today's global shipping lanes carry cargo across vast distances because they were once crossed only slowly via coastal hopping routes used since antiquity until steamships revolutionized travel during late nineteenth century when diesel engines replaced wind power entirely allowing container ships carrying thousands TEUs (Twenty-foot Equivalent Units) per voyage crossing entire Pacific basins without stopping at intermediate ports anymore unlike earlier sailing vessels needing favorable winds currents seasonal patterns dictating departure windows based upon monsoon calendars established centuries ago before technological advances changed those constraints fundamentally altering logistics networks worldwide thereby transforming economic structures globally while simultaneously creating ecological disruptions due invasive species transported unintentionally alongside legitimate goods causing cascading ecosystem collapses locally where non-native organisms outcompete native fauna disrupting food webs leading population declines affecting biodiversity indices measured scientifically using standardized metrics like Shannon-Wiener diversity index calculations applied consistently over decades revealing trends showing significant decreases post-introduction events correl strongly with introduction dates matching historical records documenting first sightings confirmed taxonomically classified formally according International Code Nomenclature rules"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/pp-lim-cv-pp-x2.json

@@ -0,0 +1,84 @@
+{
+ "label": "lim-cv-pp-x2",
+ "extra": [
+  "--layer-split",
+  "auto",
+  "--control-vector-scaled",
+  "C:\\Users\\ribbon\\strata-test\\Strata\\data\\experimental-speed-projection\\Qwen3.8-Flash-Next-experimental-speed-projection.gguf:1.0",
+  "--control-vector-layer-range",
+  "4",
+  "44",
+  "--cvec-mode",
+  "project",
+  "--cvec-dir",
+  "per-layer"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 15925,
+   "prompt_ms": 8783.1,
+   "pp_tok_s": 1813.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27474,
+   "prompt_ms": 13379.7,
+   "pp_tok_s": 2053.4,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3652.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 246.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 77.0,
+   "reps": [
+    77.0
+   ],
+   "hit_rate": 0.9718150054476462,
+   "text": "The wind didn\u2019t just howl; it screamed. It tore through the gaps in the iron railing of the Lantern Room, carrying the scent of brine and rotting kelp. Elias pulled his woolen collar up to his chin, his fingers numb and clumsy inside thick leather gloves.\n\nHe was the last man on the island. The automated systems had failed three winters ago, a glitch in the solar panel array that no engineer had been able to fix before the supply boat stopped running. Now, it was just him, the light, and the sea.\n\nElias climbed the spiral stairs, his boots heavy with salt spray. The iron steps groaned under his weight, a sound that seemed to mock the storm outside. He reached the top, the glass walls vibrating with the force of the gale.\n\nInside the lantern room, the great Fresnel lens sat like a dormant eye. It was cold, dark, and waiting.\n\nElias moved to the control panel. His"
+  },
+  {
+   "name": "code",
+   "tok_s": 103.6,
+   "reps": [
+    103.6
+   ],
+   "hit_rate": 0.957731667226356,
+   "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+  }
+ ],
+ "stage_timing": [],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.30 GiB for experts",
+  "strata generate: layer split auto: CUDA1 82 SMs at 1.86 GHz -> 0.48 ms per layer, 14.14 GiB for experts",
+  "strata generate: layer split auto: K=22 - predicted 19.7 ms per decode window; the caches hold 11298 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 3479 slots, 6.22 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 91.6% (5554 hits / 6062 lookups)",
+  "strata serve: decode expert cache hit rate: 95.3% (3592 hits / 3771 lookups)",
+  "strata serve: decode expert cache hit rate: 96.5% (3650 hits / 3782 lookups)",
+  "strata serve: decode expert cache hit rate: 96.3% (25027 hits / 25980 lookups)",
+  "strata serve: decode expert cache hit rate: 96.5% (3201 hits / 3318 lookups)",
+  "strata serve: decode expert cache hit rate: 97.2% (129334 hits / 133085 lookups)",
+  "strata serve: decode expert cache hit rate: 95.8% (108375 hits / 113158 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split-limits/data/pp-lim-kv-pp-single.json

@@ -0,0 +1,76 @@
+{
+ "label": "lim-kv-pp-single",
+ "extra": [
+  "--max-context",
+  "131072",
+  "--kv-resident",
+  "32768"
+ ],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 15925,
+   "prompt_ms": 10706.3,
+   "pp_tok_s": 1487.4,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27474,
+   "prompt_ms": 15256.7,
+   "pp_tok_s": 1800.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 59725,
+   "prompt_ms": 32515.5,
+   "pp_tok_s": 1836.8,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4006.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 276.2,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 66.6,
+   "reps": [
+    66.6
+   ],
+   "hit_rate": 0.7101864314556545,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to sync with his own heartbeat. He had been doing this for forty years. His hands were mapped with scars from rope burns and broken glass, his eyes permanently squinted against the glare of the bulb."
+  },
+  {
+   "name": "code",
+   "tok_s": 74.0,
+   "reps": [
+    74.0
+   ],
+   "hit_rate": 0.5873783774988882,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [],
+ "engine_lines": [
+  "strata generate: expert cache auto: 8.99 GiB free, 700 MiB reserved -> 3349 slots",
+  "strata generate: expert cache 2434 slots, 4.67 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 1944 cache slots (3.72 GiB)",
+  "strata serve: decode expert cache hit rate: 41.8% (1746 hits / 4176 lookups)",
+  "strata serve: decode expert cache hit rate: 50.4% (1512 hits / 3000 lookups)",
+  "strata serve: decode expert cache hit rate: 54.9% (1689 hits / 3077 lookups)",
+  "strata serve: decode expert cache hit rate: 60.3% (1938 hits / 3213 lookups)",
+  "strata serve: decode expert cache hit rate: 58.3% (11146 hits / 19121 lookups)",
+  "strata serve: decode expert cache hit rate: 66.1% (2201 hits / 3331 lookups)",
+  "strata serve: decode expert cache hit rate: 71.0% (74397 hits / 104757 lookups)",
+  "strata serve: decode expert cache hit rate: 58.7% (54151 hits / 92191 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split-limits/data/pp-lim-kv-pp-x2.json

@@ -0,0 +1,87 @@
+{
+ "label": "lim-kv-pp-x2",
+ "extra": [
+  "--max-context",
+  "131072",
+  "--kv-resident",
+  "32768",
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 15925,
+   "prompt_ms": 9452.6,
+   "pp_tok_s": 1684.7,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27474,
+   "prompt_ms": 14042.4,
+   "pp_tok_s": 1956.5,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 59725,
+   "prompt_ms": 30345.9,
+   "pp_tok_s": 1968.1,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3806.1,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 245.6,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 75.8,
+   "reps": [
+    75.8
+   ],
+   "hit_rate": 0.9698055042592483,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined Elias\u2019s life for thirty years. He stood before the great Fresnel lens, a cathedral of glass and brass that caught the light of the single, powerful bulb and fractured it into a thousand beams.\n\nHe checked the clock. 11:42 PM. The rotation was smooth, silent, a hypnotic sweep that painted the darkness with a slow, rhythmic pulse. *Flash. Dark. Flash. Dark.*\n\nE"
+  },
+  {
+   "name": "code",
+   "tok_s": 97.0,
+   "reps": [
+    97.0
+   ],
+   "hit_rate": 0.9562207324057541,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 17.96 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.13 GiB for experts",
+  "strata generate: layer split auto: CUDA1 82 SMs at 1.86 GHz -> 0.48 ms per layer, 13.98 GiB for experts",
+  "strata generate: layer split auto: K=22 - predicted 19.7 ms per decode window; the caches hold 11199 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.30 GiB free, 700 MiB reserved -> 3277 slots",
+  "strata generate: expert cache 3409 slots, 6.09 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 91.3% (5098 hits / 5585 lookups)",
+  "strata serve: decode expert cache hit rate: 94.7% (4000 hits / 4225 lookups)",
+  "strata serve: decode expert cache hit rate: 96.5% (4095 hits / 4245 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (4183 hits / 4278 lookups)",
+  "strata serve: decode expert cache hit rate: 96.2% (27177 hits / 28250 lookups)",
+  "strata serve: decode expert cache hit rate: 97.6% (4177 hits / 4281 lookups)",
+  "strata serve: decode expert cache hit rate: 97.0% (130241 hits / 134296 lookups)",
+  "strata serve: decode expert cache hit rate: 95.6% (113534 hits / 118732 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split-limits/data/vision.json

@@ -0,0 +1,24 @@
+{
+ "answers": {
+  "single": [
+   " Red",
+   "42",
+   "A red circle, the number 42, and a blue square are arranged on a white background."
+  ],
+  "same24": [
+   " Red",
+   "42",
+   "A red circle, the number 42, and a blue square are arranged on a white background."
+  ],
+  "x2": [
+   " Red",
+   "42",
+   "A red circle, the number 42, and a blue square are displayed on a white background."
+  ]
+ },
+ "correct": {
+  "single": true,
+  "same24": true,
+  "x2": true
+ }
+}

bench/results/2026-09-29-layer-split/README.md

@@ -0,0 +1,84 @@
+# Layer split across GPUs (2026-09-29)
+
+**Rig (a community member's PC, "RIBPC"):**
+- Ryzen 9 9950X3D, 62 GB DDR5, Windows 11, CUDA 13.1.
+- RTX 5080 16 GB (x16), RTX 3090 24 GB (x4 gen4, ~6 GB/s), RTX 2080 Ti 22 GB (x4 gen3, Turing sm_75; the
+  experimental `STRATA_EXPERIMENTAL_SM75` build).
+
+**Model:** the Coder (GSQ-RCO IQ1_M, 48 layers x 256 experts = 12,288 expert pairs, ~23 GB of experts), 32K
+context, int8 KV, `--prefill auto`, `--spec 4`. It is the only model that fits the rig's RAM and disk.
+
+**Tools:** `rib_pp.py` (needle prompts at 2.7K/8K/16K/28K tokens with a code word at 50% depth, a 2-turn chat that
+must reuse its checkpoint, greedy decode of a story and a code prompt) and `rib_norepeat.py` (the Phase A
+no-repeat and sanity cases). Raw results are in `data/`.
+
+## Correctness
+
+| Check | Result |
+|---|---|
+| This branch vs the 0.1.20 release, one GPU (5080), fixed cache, all 10 cases | **10/10 byte-identical** |
+| This branch, one GPU vs the split on the same GPU (K=24, `--split-device 0`) | **10/10 byte-identical**: the hand-off is exact |
+| 5080 + 3090 (auto) and 5080 + 2080 Ti + 3090 (auto): no-repeat under presence 100 (greedy and sampled), 144 sanity | all pass, 0 repeats |
+| Needles at 2.7K-28K tokens, every configuration | all found |
+| 2-turn chat (the checkpoint restored on every GPU) | reused 6,631-6,632 of 6,660 tokens, answer right |
+
+Across GPUs the text is not byte-identical to one GPU: experts the CPU computed on one card run on a GPU in the split,
+and they round differently (the same effect as `--expert-cache`, bench/results/2026-09-27-cache-parity).
+
+## Speed
+
+**Prompt tok/s** at 16K / 28K tokens, and **decode tok/s** for story / code (greedy; three repeats averaged
+in the K sweep):
+
+| Configuration | Split | Prompt 16K | Prompt 28K | Decode story | Decode code | Decode hits |
+|---|---|---|---|---|---|---|
+| 0.1.20, 5080 alone | - | 2,045 | 2,005 | 83.2 | 88.2 | 70-84% |
+| this branch, 5080 alone | - | 2,017 | 1,974 | 83.1 | 88.0 | 70-84% |
+| this branch, 5080 alone, 3 repeats (adaptive swaps learn the prompt) | - | 1,726 | 1,970 | 86.7 | 105.2 | 91-92% |
+| 3090 alone | - | 1,253 | 1,211 | 71.7 | 92.9 | 86-93% |
+| 5080 + 3090 | K=18 | 1,547 | 1,694 | 77.2 | 103.8 | 99.9% |
+| 5080 + 3090 | K=22 | 1,840 | 2,074 | 80.2 | 106.4 | 99.8% |
+| 5080 + 3090 | K=24 | 1,978 | 2,258 | 83.5 | 109.5 | 99.5% |
+| **5080 + 3090** | **K=26** | **2,039** | **2,357** | **83.8** | **109.7** | 99.3% |
+| 5080 + 3090 | K=28 | 2,011 | 2,287 | 83.5 | 112.5 | 99.0% |
+| 5080 + 3090 | K=30 | 1,935 | 2,218 | 81.1 | 109.6 | 98% |
+| 5080 + 3090 | K=32 | 1,818 | 2,063 | 83.0 | 108.7 | 98% |
+| 5080 + 3090 | auto (K=22) | 2,037 | 2,073 | 79.5 | 108.9 | 99.7% |
+| 3090 + 5080 | auto (K=30) | 1,672 | 1,876 | 90.3 | 105.1 | 99.4% |
+| 5080 + 2080 Ti + 3090 | auto (22, 23) | 1,687 | 1,847 | 67.6 | 89.6 | 99.7% |
+| 5080 + 3090 + 2080 Ti | first auto (12, 33) | 1,148 | 1,144 | 51.9 | 69.7 | 100% |
+| 5080 + 2080 Ti | first auto | 914 | 906 | 45.5 | 58.0 | 97% |
+
+What decides it:
+
+- **The best two-GPU split for the Coder is K=26-28.** Prompts run +18-20% over the 5080 alone at 16-28K;
+  decode is on par for story and +4-7% for code.
+- **Residency saturates quickly: the per-layer GPU speed decides.** From K=18 up, both caches hold 98-100% of
+  the routed experts. Moving layers to the faster card is what helps: the 3090 costs 0.50 ms per layer per
+  window, the 5080 0.33 (`STRATA_SPLIT_TIMING`, the "wait for the GPU" per stage).
+- **Decode is bounded by that GPU time.** One 5080 with its cache adapted spends 16.3 ms waiting for its GPU and
+  5.5 ms in the CPU pool per window. The split removes the pool (0.8-1.0 ms) but adds the slower card's layers:
+  7.9 + 12.1 ms at K=24. The per-layer GPU half (kernels plus the host round trip that publishes each layer's
+  expert plan) is the next thing to cut, for one GPU as well (perf-review-plan E-6).
+- **A third, slower card costs more than it holds.** The 2080 Ti's one-layer stage took 3.1 ms per window.
+  For a model that already fits two cards, leave it out. It pays only when the two faster cards cannot hold most
+  of the routed experts (a bigger model; not testable on this rig's 62 GB of RAM).
+- **Prompt chunk:** 1024 instead of 2048 frees ~0.7 GB per card, but prompts read 7-12% slower.
+
+## Auto placement
+
+`--layer-split auto` predicts one decode window's time for every placement (every placement up to three GPUs) and
+keeps the fastest:
+
+- **per-layer time** from SMs x clock (0.33 ms for an RTX 5080);
+- **cache contents:** each card's layers' profiled pairs, hottest first, as the fill does;
+- **routed mass** of rank r taken as (r+1)^-1.2 (fits the sweep's hit rates within 0.1-1%);
+- **missed mass** costing 190 ms per unit (fitted to the sweep's best K; `STRATA_SPLIT_MISS_MS` overrides).
+
+It picked K=22 on the 5080 + 3090: within 0-12% of the best fixed split on prompts and on par on decode. It gives
+a slow third card a single layer.
+
+## Reproduce
+
+On a rig: the `strata-remote` tools, then `rib_matrix.bat`, `rib_ksweep.bat`, `rib_validate.bat` (in the dev
+notes, not in this repository).

bench/results/2026-09-29-layer-split/data/norepeat-nr-v-x2-auto-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 193,
+  "tok_s": 74.6,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, rough as sandpaper, gripped the iron railing as he ascended the final flight of steps to the lantern room.\n\nBelow him, the Atlantic churned in a frenzy, whitecaps exploding against the black rocks like fireworks. Above, the sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen.\n\nElias checked the clock. 11:42 PM. The light was due.\n\nHe didn\u2019t flip a switch. He didn\u2019t press a button. He reached for the heavy brass lever that had worn smooth by his palm over the years. With a grunt that matched the strain of his aging muscles, he pulled it down.\n\n*Clunk.*\n\nThe"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 181,
+  "tok_s": 105.5,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 85.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 94.3,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 118,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 105,
+  "tok_s": 67.9,
+  "text": "Aardvark  \nAlbatross   \nAlligator    \nAntelope     \nBaboon       \nBadger      \nBeaver        \nBear          \nBird of Paradise \nBuffalo         \nCamel           \nCheetah            \nChimpanzee             \nCrane              \nCrocodile                \nDeer                 \nDolphin               \nDonkey                  \nElephant                   \nFlamingo                    \nFox                       \nGiraffe                     \nGoat                      \nHippopotamus  \nIguana       \nJaguar        \nKangaroo      \nKoala          \nLion         \nLeopard     \nMonkey    \nMoose   \nOstrich \nOtter\nPanda<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 147,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 103,
+  "tok_s": 63.2,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to that date and time for context because you asked me about a lighthouse keeper but gave no specifics beyond \u201cshort story,\u201d so let\u2019s set it: isolated coastal station off the Oregon coast during winter storm season, 1987.\n\nMy name is Elias Vane (I made this fictional; real keepers often kept journals signed with initials only), and I\u2019m writing from memory because my logbook got waterlogged when a rogue wave breached lower storage room at dawn\u2014so here\u2019s what happened:<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 109,
+  "repeats": 0,
+  "drafts_accepted": 13,
+  "drafts_offered": 56,
+  "tok_s": 54.1,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they settled into silence with every storm break out of three hundred days here being storms that broke something else again but mostly kept me warm because I burned coal and wood whenever we had any left over from last winter when even then if you\u2019re lucky enough to get through it all without freezing yourself so hard your own thoughts become ice too\n\nBut Elias knew better than anyone: fear was useful only until dawn<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 92,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 48,
+  "tok_s": 64.7,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nHippopotamus          \nRhinoceros            \nCamel              \nCheetah             \nLeopard           \nPanther         \nJaguar                \nBison               \nElk                 \nDeer                  \nFox                   \nWolf                    \nBear                     \nLlama                       \nDonkey                        \nGoat                      \nSheep                         \nCow                           \nPig                          \nRabbit                            \nCat                              \nDog<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 14,
+  "tok_s": 71.7,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 53,
+  "drafts_offered": 134,
+  "tok_s": 66.1,
+  "text": "The ocean is not merely water; it has been, and continues to be one with humanity\u2019s most profound narrative arc in human civilization: **the story we tell ourselves through our relationship** between landmasses separated by vast expansiveness that defines both isolation from other worlds (and thus their own unique evolutionary paths) as well than connection across them via currents wind patterns tides depth pressure temperature salinity biodiversity geology climate change economic trade routes ecological systems cultural exchange political boundaries scientific discovery technological advancement environmental degradation conservation efforts maritime archaeography historical cartographic evolution legal frameworks international law enforcement mechanisms insurance liability regimes regulatory compliance standards safety protocols navigation aids communication networks emergency response strategies search-and-rescue operations salvage rights ownership disputes heritage protection zones sustainable harvesting practices aquaculture development infrastructure engineering projects coastal management policies urban planning regulations zoning ordinances building codes architectural design guidelines aesthetic considerations functional requirements accessibility provisions energy efficiency measures thermal insulation techniques moisture control methods ventilation optimization lighting solutions acoustic soundproofing structural integrity assessments seismic resistance evaluations flood mitigation plans storm surge barriers levee construction seawall reinforcement breakwater installation harbor dredging channel maintenance buoy placement lighthouse automation radar monitoring sonar mapping bathymetric surveys hydrographic charts nautical almanacs tide tables current predictions weather forecasts marine meteorological models satellite imagery remote sensing data assimilation numerical simulations computational fluid dynamics finite element analysis partial differential equations tensor calculus linear algebra probability theory statistics stochastic processes Markov chains Bayesian inference machine learning artificial neural network deep convolutional recurrent transformer attention heads generative adversarial diffusion transformers large language model tokenization"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-nr-v-x3-auto-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 193,
+  "tok_s": 68.2,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, rough as sandpaper, gripped the iron railing as he ascended the final flight of steps to the lantern room.\n\nBelow him, the Atlantic churned in a frenzy, whitecaps exploding against the black rocks like fireworks. Above, the sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen.\n\nElias checked the clock. 11:42 PM. The light was due.\n\nHe didn\u2019t flip a switch. He didn\u2019t press a button. He reached for the heavy brass lever that had worn smooth by his palm over the years. With a grunt that matched the strain of his aging muscles, he pulled it down.\n\n*Clunk.*\n\nThe"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 181,
+  "tok_s": 99.4,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 76.3,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 118,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 105,
+  "tok_s": 65.2,
+  "text": "Aardvark  \nAlbatross   \nAlligator    \nAntelope     \nBaboon       \nBadger      \nBeaver        \nBear          \nBird of Paradise \nBuffalo         \nCamel           \nCheetah            \nChimpanzee             \nCrane              \nCrocodile                \nDeer                 \nDolphin               \nDonkey                  \nElephant                   \nFlamingo                    \nFox                       \nGiraffe                     \nGoat                      \nHippopotamus  \nIguana       \nJaguar        \nKangaroo      \nKoala          \nLion         \nLeopard     \nMonkey    \nMoose   \nOstrich \nOtter\nPanda<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 147,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 103,
+  "tok_s": 45.7,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to that date and time for context because you asked me about a lighthouse keeper but gave no specifics beyond \u201cshort story,\u201d so let\u2019s set it: isolated coastal station off the Oregon coast during winter storm season, 1987.\n\nMy name is Elias Vane (I made this fictional; real keepers often kept journals signed with initials only), and I\u2019m writing from memory because my logbook got waterlogged when a rogue wave breached lower storage room at dawn\u2014so here\u2019s what happened:<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 109,
+  "repeats": 0,
+  "drafts_accepted": 13,
+  "drafts_offered": 56,
+  "tok_s": 42.1,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they settled into silence with every storm break out of three hundred days here being storms that broke something else again but mostly kept me warm because I burned coal and wood whenever we had any left over from last winter when even then if you\u2019re lucky enough to get through it all without freezing yourself so hard your own thoughts become ice too\n\nBut Elias knew better than anyone: fear was useful only until dawn<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 92,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 48,
+  "tok_s": 43.9,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nHippopotamus          \nRhinoceros            \nCamel              \nCheetah             \nLeopard           \nPanther         \nJaguar                \nBison               \nElk                 \nDeer                  \nFox                   \nWolf                    \nBear                     \nLlama                       \nDonkey                        \nGoat                      \nSheep                         \nCow                           \nPig                          \nRabbit                            \nCat                              \nDog<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 14,
+  "tok_s": 61.4,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 53,
+  "drafts_offered": 134,
+  "tok_s": 51.9,
+  "text": "The ocean is not merely water; it has been, and continues to be one with humanity\u2019s most profound narrative arc in human civilization: **the story we tell ourselves through our relationship** between landmasses separated by vast expansiveness that defines both isolation from other worlds (and thus their own unique evolutionary paths) as well than connection across them via currents wind patterns tides depth pressure temperature salinity biodiversity geology climate change economic trade routes ecological systems cultural exchange political boundaries scientific discovery technological advancement environmental degradation conservation efforts maritime archaeography historical cartographic evolution legal frameworks international law enforcement mechanisms insurance liability regimes regulatory compliance standards safety protocols navigation aids communication networks emergency response strategies search-and-rescue operations salvage rights ownership disputes heritage protection zones sustainable harvesting practices aquaculture development infrastructure engineering projects coastal management policies urban planning regulations zoning ordinances building codes architectural design guidelines aesthetic considerations functional requirements accessibility provisions energy efficiency measures thermal insulation techniques moisture control methods ventilation optimization lighting solutions acoustic soundproofing structural integrity assessments seismic resistance evaluations flood mitigation plans storm surge barriers levee construction seawall reinforcement breakwater installation harbor dredging channel maintenance buoy placement lighthouse automation radar monitoring sonar mapping bathymetric surveys hydrographic charts nautical almanacs tide tables current predictions weather forecasts marine meteorological models satellite imagery remote sensing data assimilation numerical simulations computational fluid dynamics finite element analysis partial differential equations tensor calculus linear algebra probability theory statistics stochastic processes Markov chains Bayesian inference machine learning artificial neural network deep convolutional recurrent transformer attention heads generative adversarial diffusion transformers large language model tokenization"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-nr-x2-auto-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 91,
+  "drafts_offered": 190,
+  "tok_s": 74.0,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, rough as sandpaper, knew every bolt, every lever, and every creak in the iron bones of the tower.\n\nFor forty years, Elias had been the guardian of the Point. Not just of the light, but of the silence between the waves.\n\nIt was November. The sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that shook the floorboards beneath Elias\u2019s boots.\n\nHe climbed the stairs. One hundred and twelve steps. He counted them by feel, his left hand trailing along the cold iron railing. At the top, the air was"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 137,
+  "drafts_offered": 178,
+  "tok_s": 103.4,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 91.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 70.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 118,
+  "repeats": 0,
+  "drafts_accepted": 38,
+  "drafts_offered": 108,
+  "tok_s": 65.4,
+  "text": "Aardvark  \nAlbatross   \nAlligator    \nAntelope     \nBaboon       \nBadger      \nBeaver        \nBear          \nBird of Paradise \nBuffalo         \nCamel           \nCheetah            \nChimpanzee             \nCrane              \nCrocodile                \nDeer                 \nDolphin               \nDonkey                  \nElephant                   \nFlamingo                    \nFox                       \nGiraffe                     \nGoat                      \nHippopotamus  \nIguana       \nJaguar        \nKangaroo      \nKoala          \nLion         \nLeopard     \nMonkey    \nMoose   \nOstrich \nOtter\nPanda<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 57,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 61,
+  "tok_s": 57.1,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach top room where his world began again.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 58,
+  "repeats": 0,
+  "drafts_accepted": 14,
+  "drafts_offered": 54,
+  "tok_s": 52.9,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in stone and steel, carrying with them spray from three miles out into churning black water that no longer resembled an ocean so much as abyssal chaos waiting for its prey to surface beneath waves of salt.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 47,
+  "tok_s": 51.7,
+  "text": "Lion  \nTiger  \n\nElephant\\nGiraffe\\Zebra\\rHippopotamus\\tRhinoceros\\vCrocodile\\fAlligator\\bSnake\\aSpider\\u2615CoffeeCup\u2615\ufe0fTeaTime\ud83d\udc09DragonFishWhaleDolph<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 12,
+  "tok_s": 73.5,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 43,
+  "drafts_offered": 126,
+  "tok_s": 63.4,
+  "text": "The ocean is not merely water; it has been, and continues to be one with humanity\u2019s most profound narrative arc in human civilization: **the story told by waves**. It serves as both witness (through sedimentary layers)and participant( through currents that carry messages across continents). Its \"history\" spans geological epochs where life first emerged from primordial pools into complex ecosystems today\u2014and cultural histories shaped entirely around its roleas connector between distant lands or barrier separating them forever depending on perspective taken at different times throughout millennia since ancient civilizations began building boats outof logs carved using stone tools until modern container ships carrying billions tons cargo daily cross Atlantic routes once navigated onlyby stars alone under moonlight during nights when sailors feared monsters lurking beneath hulls made fragile against storms they could barely predict even now despite advanced meteorology systems developed over past century after industrial revolution transformed maritime trade patterns globally due economic pressures driving demand for faster logistics chains linking factories producing electronics manufactured inside cities located far inland near rivers feeding estuaries opening directly onto open seas connected via canals engineered centuries ago like Suez Canal built originallyfor military purposes later adapted commercial use post-World War II era reshaping global supply networks dramatically because geopolitical shifts altered which ports remained accessible politically versus physically blocked temporarily then reopened permanently following treaty agreements signed among nations agreeing upon shared responsibilities regarding environmental stewardship including coral reef restoration efforts funded jointly international organizations coordinating scientific research conducted aboard vessels specifically designed deep-sea exploration equipped robotic submersibles capable descending trenches deeper than any previous technology allowed"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-nr-x3-auto-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 98,
+  "drafts_offered": 166,
+  "tok_s": 54.4,
+  "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak in the spiral staircase, every draft that slipped under the door, and every shift in the weather that the barometer couldn\u2019t quite predict. He knew that when the fog rolled in thick enough to swallow the horizon, the world ceased to exist. There was only the light, the stone, and him.\n\nIt was November, the month of storms. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that made the floorboards tremble. Elias stood by the lantern room, his hands wrapped around a mug of coffee that had gone cold hours ago. He wasn\u2019t drinking it. He was holding it for warmth."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 137,
+  "drafts_offered": 163,
+  "tok_s": 70.7,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv(file_path: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys being the header values\n    and values being the corresponding cell values.\n    \n    Args:\n        file_path: Path to the CSV file.\n        \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(file_path, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_basic():\n    \"\"\"Test basic CSV parsing with simple data.\"\"\"\n    import tempfile\n    import os\n    \n    # Create a temporary CSV"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 56.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 57.6,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 13,
+  "drafts_offered": 47,
+  "tok_s": 39.7,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBuffalo     \n\nCat          \nCheetah      \n\nChimpanzee \nClam         \n\n\nCrocodile   \n\n\n\nDolphin      \n\n\n\n\nDonkey       <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 57,
+  "repeats": 0,
+  "drafts_accepted": 12,
+  "drafts_offered": 58,
+  "tok_s": 39.9,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach topmost gallery where his world began.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 30,
+  "drafts_offered": 111,
+  "tok_s": 34.7,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his stone cottage like an angry ghost trying to get inside, yet finding no door left unlocked because Elias never let one stay that way for more than three seconds after entering or leaving during storm season. He had been doing this since 1987\u2014since before he even knew why\u2014and now it was muscle memory so deep into his bones and fingers\u2019 calloused skin texture patterns worn thin by years of gripping the same iron railing at midnight watch that if you asked him what time exactly, down to quarter-hour increments when fog rolled in thick enough where light beams bent around itself like old rope coiling tight on a winch drum spooling out slowly from below deck floor level storage locker number seven under stairs leading"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 96,
+  "repeats": 0,
+  "drafts_accepted": 31,
+  "drafts_offered": 48,
+  "tok_s": 42.3,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nCheetah         \nHippopotamus          \nRhinoceros            \nLeopard             \nJaguar              \nWolf                \nFox                 \nBear                  \nDeer                    \nOtter                   \nBeaver                      \nBadger                        \nMongoose                          \nMeerkat                           \nChimpanzee                            \nGorilla                             \nOrangetutan                              \nAntelope                                \nBison                                   \nElk                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 43.0,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 127,
+  "tok_s": 37.6,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\u201d \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \u201cthe Sea\u201d or \"Ocean\" (terms often used interchangeably but distinct geographically) was never just an element within our planet\u2019s geography\u2014it became its engine itself\u2014a living system whose existence predates humanity by billions years yet continues shaping us today through climate regulation oxygen production food supply trade routes cultural identity ecological resilience economic development political boundaries scientific discovery technological innovation environmental crisis conservation efforts historical memory philosophical reflection poetic metaphor musical composition cinematic storytelling literary narrative visual art architectural inspiration culinary tradition spiritual symbolism psychological influence sociological transformation educational foundation medical necessity agricultural dependency industrial revolution urban planning infrastructural engineering legal framework regulatory policy international cooperation global governance planetary stewardship existential risk mitigation sustainable practice ethical consideration moral imperative social justice democratic participation civic engagement public health community welfare individual responsibility personal accountability collective action shared experience common understanding mutual respect reciprocal trust collaborative effort cooperative endeavor joint venture unified mission integrated strategy coordinated plan synchronized schedule aligned timeline concurrent execution simultaneous operation parallel processing sequential progression iterative refinement recursive improvement continuous adaptation dynamic adjustment flexible response adaptive behavior responsive conduct proactive initiative anticipatory foresight predictive modeling probabilistic assessment statistical analysis empirical observation experimental verification theoretical hypothesis conceptual abstraction logical deduction rational inference critical evaluation analytical reasoning systematic investigation methodical inquiry structured exploration comprehensive examination detailed scrutiny meticulous inspection thorough review exhaustive audit complete survey full census total enumeration absolute"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-reg-dev-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 187,
+  "tok_s": 59.2,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, mapped with veins like the roots of an ancient oak, gripped the cold iron railing.\n\nFor forty years, Elias had tended the light at the end of the world. Not the end of the map, but the end of the land. Here, the Atlantic threw itself against the granite cliffs with such violence that the spray reached the gallery, thirty feet above the rocks.\n\nHe checked the clock. 11:42 PM. The fog was thickening, a gray soup that swallowed the horizon. In the old days, he would have climbed the stairs to the lantern room, polished the Fresnel lens, and wound the clockwork mechanism that turned the great beam. But the automation had changed"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 179,
+  "tok_s": 81.6,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.9,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.3,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 19,
+  "drafts_offered": 49,
+  "tok_s": 49.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFlamingo<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 80,
+  "repeats": 0,
+  "drafts_accepted": 6,
+  "drafts_offered": 52,
+  "tok_s": 44.9,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to check what was happening with that storm and why they were calling me out there when all we had left for tonight\u2019s shift is one last pass through our logs before dawn breaks.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 83,
+  "tok_s": 44.1,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in its teeth, hungry and relentless for three centuries of stone salt-spray history that had long since forgotten to be polite or gentle with itself against anything smaller than God Himself was willing acknowledge as divine protection from drowning souls who were already lost before they ever touched water.\n\nElias knew this rhythm better he himself could hear it sometimes; his boots worn smooth by pacing the spiral stair\u2019s center-post groove where thousands upon countless steps had rubbed away texture until all that remained beneath sole leather now felt like polished bone carved out slowly but surely over time into something more hollow than wood ever really should be allowed become when used so often enough within such confined circular spaces meant only for vertical ascent down always descending again back upward never truly leaving"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 62,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 31,
+  "tok_s": 49.5,
+  "text": "Lion  \nTiger  \n\nElephant  \n  \nGiraffe  \n\n\n\nHippopotamus    \nZebra   \nRhinoceros   \n\n\nPanda Bear    \n\n\n\n\nKangaroo     \nKoala      \nSloth      \n\n\n\n\n\nCheetah       \nLeopard        \nPanther        \n\n\n\n\n\n\nWolf              \nFox                \nBear                 <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 65.2,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 47,
+  "drafts_offered": 153,
+  "tok_s": 52.5,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by billions years yet continues shaping every aspect from climate regulation through evolutionary biology all way down modern industrial supply chains today's global shipping lanes carry cargo across vast distances because they were once crossed only slowly via coastal hopping routes used since antiquity until steamships revolutionized travel during late nineteenth century when diesel engines replaced wind power entirely allowing container ships carrying thousands TEUs (Twenty-foot Equivalent Units) per voyage crossing entire Pacific basins without stopping at intermediate ports anymore unlike earlier sailing vessels needing favorable winds currents seasonal patterns dictating departure windows based upon monsoon calendars established centuries ago before technological advances changed those constraints fundamentally altering logistics networks worldwide thereby transforming economic structures globally while simultaneously creating ecological disruptions due invasive species transported unintentionally alongside legitimate goods causing cascading ecosystem collapses locally where non-native organisms outcompete native fauna disrupting food webs leading population declines affecting biodiversity indices measured scientifically using standardized metrics like Shannon-Wiener diversity index calculations applied consistently over decades revealing trends showing significant decreases post-introduction events correl strongly with introduction dates matching historical records documenting first sightings confirmed taxonomically classified formally according International Code Nomenclature rules"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-reg-rel-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 187,
+  "tok_s": 58.4,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, mapped with veins like the roots of an ancient oak, gripped the cold iron railing.\n\nFor forty years, Elias had tended the light at the end of the world. Not the end of the map, but the end of the land. Here, the Atlantic threw itself against the granite cliffs with such violence that the spray reached the gallery, thirty feet above the rocks.\n\nHe checked the clock. 11:42 PM. The fog was thickening, a gray soup that swallowed the horizon. In the old days, he would have climbed the stairs to the lantern room, polished the Fresnel lens, and wound the clockwork mechanism that turned the great beam. But the automation had changed"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 179,
+  "tok_s": 80.6,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 75.2,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.2,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 19,
+  "drafts_offered": 49,
+  "tok_s": 48.5,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFlamingo<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 80,
+  "repeats": 0,
+  "drafts_accepted": 6,
+  "drafts_offered": 52,
+  "tok_s": 46.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to check what was happening with that storm and why they were calling me out there when all we had left for tonight\u2019s shift is one last pass through our logs before dawn breaks.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 83,
+  "tok_s": 43.3,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in its teeth, hungry and relentless for three centuries of stone salt-spray history that had long since forgotten to be polite or gentle with itself against anything smaller than God Himself was willing acknowledge as divine protection from drowning souls who were already lost before they ever touched water.\n\nElias knew this rhythm better he himself could hear it sometimes; his boots worn smooth by pacing the spiral stair\u2019s center-post groove where thousands upon countless steps had rubbed away texture until all that remained beneath sole leather now felt like polished bone carved out slowly but surely over time into something more hollow than wood ever really should be allowed become when used so often enough within such confined circular spaces meant only for vertical ascent down always descending again back upward never truly leaving"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 62,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 31,
+  "tok_s": 49.8,
+  "text": "Lion  \nTiger  \n\nElephant  \n  \nGiraffe  \n\n\n\nHippopotamus    \nZebra   \nRhinoceros   \n\n\nPanda Bear    \n\n\n\n\nKangaroo     \nKoala      \nSloth      \n\n\n\n\n\nCheetah       \nLeopard        \nPanther        \n\n\n\n\n\n\nWolf              \nFox                \nBear                 <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.5,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 47,
+  "drafts_offered": 153,
+  "tok_s": 52.6,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by billions years yet continues shaping every aspect from climate regulation through evolutionary biology all way down modern industrial supply chains today's global shipping lanes carry cargo across vast distances because they were once crossed only slowly via coastal hopping routes used since antiquity until steamships revolutionized travel during late nineteenth century when diesel engines replaced wind power entirely allowing container ships carrying thousands TEUs (Twenty-foot Equivalent Units) per voyage crossing entire Pacific basins without stopping at intermediate ports anymore unlike earlier sailing vessels needing favorable winds currents seasonal patterns dictating departure windows based upon monsoon calendars established centuries ago before technological advances changed those constraints fundamentally altering logistics networks worldwide thereby transforming economic structures globally while simultaneously creating ecological disruptions due invasive species transported unintentionally alongside legitimate goods causing cascading ecosystem collapses locally where non-native organisms outcompete native fauna disrupting food webs leading population declines affecting biodiversity indices measured scientifically using standardized metrics like Shannon-Wiener diversity index calculations applied consistently over decades revealing trends showing significant decreases post-introduction events correl strongly with introduction dates matching historical records documenting first sightings confirmed taxonomically classified formally according International Code Nomenclature rules"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-reg-same24-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 187,
+  "tok_s": 58.4,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, mapped with veins like the roots of an ancient oak, gripped the cold iron railing.\n\nFor forty years, Elias had tended the light at the end of the world. Not the end of the map, but the end of the land. Here, the Atlantic threw itself against the granite cliffs with such violence that the spray reached the gallery, thirty feet above the rocks.\n\nHe checked the clock. 11:42 PM. The fog was thickening, a gray soup that swallowed the horizon. In the old days, he would have climbed the stairs to the lantern room, polished the Fresnel lens, and wound the clockwork mechanism that turned the great beam. But the automation had changed"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 179,
+  "tok_s": 80.5,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 76.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 19,
+  "drafts_offered": 49,
+  "tok_s": 49.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFlamingo<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 80,
+  "repeats": 0,
+  "drafts_accepted": 6,
+  "drafts_offered": 52,
+  "tok_s": 46.7,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to check what was happening with that storm and why they were calling me out there when all we had left for tonight\u2019s shift is one last pass through our logs before dawn breaks.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 83,
+  "tok_s": 44.3,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in its teeth, hungry and relentless for three centuries of stone salt-spray history that had long since forgotten to be polite or gentle with itself against anything smaller than God Himself was willing acknowledge as divine protection from drowning souls who were already lost before they ever touched water.\n\nElias knew this rhythm better he himself could hear it sometimes; his boots worn smooth by pacing the spiral stair\u2019s center-post groove where thousands upon countless steps had rubbed away texture until all that remained beneath sole leather now felt like polished bone carved out slowly but surely over time into something more hollow than wood ever really should be allowed become when used so often enough within such confined circular spaces meant only for vertical ascent down always descending again back upward never truly leaving"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 62,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 31,
+  "tok_s": 50.0,
+  "text": "Lion  \nTiger  \n\nElephant  \n  \nGiraffe  \n\n\n\nHippopotamus    \nZebra   \nRhinoceros   \n\n\nPanda Bear    \n\n\n\n\nKangaroo     \nKoala      \nSloth      \n\n\n\n\n\nCheetah       \nLeopard        \nPanther        \n\n\n\n\n\n\nWolf              \nFox                \nBear                 <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.0,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 47,
+  "drafts_offered": 153,
+  "tok_s": 52.1,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by billions years yet continues shaping every aspect from climate regulation through evolutionary biology all way down modern industrial supply chains today's global shipping lanes carry cargo across vast distances because they were once crossed only slowly via coastal hopping routes used since antiquity until steamships revolutionized travel during late nineteenth century when diesel engines replaced wind power entirely allowing container ships carrying thousands TEUs (Twenty-foot Equivalent Units) per voyage crossing entire Pacific basins without stopping at intermediate ports anymore unlike earlier sailing vessels needing favorable winds currents seasonal patterns dictating departure windows based upon monsoon calendars established centuries ago before technological advances changed those constraints fundamentally altering logistics networks worldwide thereby transforming economic structures globally while simultaneously creating ecological disruptions due invasive species transported unintentionally alongside legitimate goods causing cascading ecosystem collapses locally where non-native organisms outcompete native fauna disrupting food webs leading population declines affecting biodiversity indices measured scientifically using standardized metrics like Shannon-Wiener diversity index calculations applied consistently over decades revealing trends showing significant decreases post-introduction events correl strongly with introduction dates matching historical records documenting first sightings confirmed taxonomically classified formally according International Code Nomenclature rules"
+ }
+]

bench/results/2026-09-29-layer-split/data/pp-dev-3090.json

@@ -0,0 +1,72 @@
+{
+ "label": "dev-3090",
+ "extra": [],
+ "gpus": "2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 3424.8,
+   "pp_tok_s": 796.0,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 8617.9,
+   "pp_tok_s": 972.3,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 12820.1,
+   "pp_tok_s": 1253.2,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 22901.1,
+   "pp_tok_s": 1211.4,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 5657.8,
+  "turn2_tokens": 6660,
+  "reused": 6631,
+  "turn2_ms": 238.7,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 71.7,
+   "hit_rate": 0.9266798230257504,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning black void, punctuated by whitecaps that looked like teeth.\n\nHe climbed the spiral stairs, his knees protesting with every step. The iron rungs were cold, slick with condensation that had leaked through the hairline cracks in the masonry. At the top, the lantern room hummed with the low, steady thrum of the rotating lens. It was a sound Elias had learned to sleep with, a heartbeat of the house.\n\nHe checked the clock. 11:42 PM. The light swept across the water, a slow, hypnotic arc. *Flash. Darkness. Flash. Darkness.*\n\nFor a moment, the beam caught something that wasn\u2019t water. A shape"
+  },
+  {
+   "name": "code",
+   "tok_s": 92.9,
+   "hit_rate": 0.8636411400016346,
+   "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header row\n    and values from the corresponding row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n```<|im_end|>"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 17.31 GiB free, 700 MiB reserved -> 6704 slots",
+  "strata generate: expert cache 8730 slots, 16.62 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2315 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 86.6% (4986 hits / 5756 lookups)",
+  "strata serve: decode expert cache hit rate: 87.1% (3757 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 88.7% (3827 hits / 4314 lookups)",
+  "strata serve: decode expert cache hit rate: 92.1% (3975 hits / 4318 lookups)",
+  "strata serve: decode expert cache hit rate: 95.5% (4124 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 94.3% (25793 hits / 27360 lookups)",
+  "strata serve: decode expert cache hit rate: 94.6% (4088 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 92.7% (116454 hits / 125668 lookups)",
+  "strata serve: decode expert cache hit rate: 86.4% (73970 hits / 85649 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-dev-5080.json

@@ -0,0 +1,72 @@
+{
+ "label": "dev-5080",
+ "extra": [],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2129.8,
+   "pp_tok_s": 1279.9,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5057.2,
+   "pp_tok_s": 1656.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 7964.7,
+   "pp_tok_s": 2017.2,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 14053.6,
+   "pp_tok_s": 1974.1,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3366.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 197.3,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.1,
+   "hit_rate": 0.8418683557684931,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning cauldron of black ink and white foam, throwing itself against the granite base with the force of a battering ram.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his woolen trousers, leaving faint, oily smears on the fabric. He checked the clock. 11:42 PM. The rotation was due.\n\nHe reached for the crank handle, a heavy iron wheel worn smooth by decades of his own palms. With a grunt that matched the strain of his shoulders, he began to turn it. *Click. Click. Click.* The gears engaged, and the massive Fresnel lens began its slow"
+  },
+  {
+   "name": "code",
+   "tok_s": 88.0,
+   "hit_rate": 0.7011385968111186,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4293 slots, 8.17 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 64.2% (2136 hits / 3329 lookups)",
+  "strata serve: decode expert cache hit rate: 67.3% (2309 hits / 3432 lookups)",
+  "strata serve: decode expert cache hit rate: 72.5% (2539 hits / 3504 lookups)",
+  "strata serve: decode expert cache hit rate: 77.3% (2817 hits / 3642 lookups)",
+  "strata serve: decode expert cache hit rate: 75.3% (16500 hits / 21916 lookups)",
+  "strata serve: decode expert cache hit rate: 80.0% (2978 hits / 3723 lookups)",
+  "strata serve: decode expert cache hit rate: 84.2% (98752 hits / 117301 lookups)",
+  "strata serve: decode expert cache hit rate: 70.1% (69215 hits / 98718 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k18.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k18",
+ "extra": [
+  "--layer-split",
+  "18"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 10383.7,
+   "pp_tok_s": 1547.2,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 16371.9,
+   "pp_tok_s": 1694.5,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4308.0,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 239.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 77.2,
+   "reps": [
+    74.6,
+    78.1,
+    79.0
+   ],
+   "hit_rate": 0.9985378787878788,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, the glass was vibrating like a plucked cello string. The sea, usually a rhythmic pulse against the granite base, was now a chaotic, frothing beast, throwing itself against the rocks with the force of a battering ram.\n\nElias climbed the spiral stairs, his hand gripping the cold iron railing. His knees ached, not from age, but from the constant vibration of the floor beneath his feet. He reached the lantern room, the heart of the lighthouse, and checked the clockwork mechanism. The great lens, a prism of Fresnel glass, rotated with a steady, hypnotic *whirrr-click, whirrr-click*. It was the only sound that mattered. The light swept out into the black void, cutting through"
+  },
+  {
+   "name": "code",
+   "tok_s": 103.8,
+   "reps": [
+    97.6,
+    106.4,
+    107.3
+   ],
+   "hit_rate": 0.9984863945578232,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 524 windows; per window: wait for the GPU 6.456 ms, pool + plan 0.181 ms, host staging 0.581 ms, commit 0.206 ms",
+  "strata serve: stage 1: 524 windows; per window: wait for the GPU 14.872 ms, pool + plan 0.501 ms, host staging 0.000 ms, commit 0.387 ms",
+  "strata serve: stage 0: 593 windows; per window: wait for the GPU 6.479 ms, pool + plan 0.185 ms, host staging 0.515 ms, commit 0.207 ms",
+  "strata serve: stage 1: 593 windows; per window: wait for the GPU 15.053 ms, pool + plan 0.496 ms, host staging 0.000 ms, commit 0.390 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 18)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4608 slots, 8.24 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 18-47, expert cache 7152 slots (14.14 GiB), 7152 of its 7680 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-17",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-17 (CUDA0), 18-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.7% (6159 hits / 6240 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (3796 hits / 3840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (3811 hits / 3840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (27723 hits / 27840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (3807 hits / 3840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (142002 hits / 143040 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (139882 hits / 140160 lookups)",
+  "strata serve: decode expert cache hit rate: 99.9% (131807 hits / 132000 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (119278 hits / 120480 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (117410 hits / 117600 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (117422 hits / 117600 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k19.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k19",
+ "extra": [
+  "--layer-split",
+  "19"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9875.2,
+   "pp_tok_s": 1626.9,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 15498.0,
+   "pp_tok_s": 1790.1,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4085.5,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 238.7,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 79.3,
+   "reps": [
+    74.8,
+    85.6,
+    77.4
+   ],
+   "hit_rate": 0.9977757017660928,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, the glass was vibrating like a plucked cello string. The sea, usually a rhythmic pulse against the granite base, was now a chaotic, frothing beast, throwing itself against the rocks with the force of a battering ram.\n\nElias stood at the top of the spiral staircase, his hand resting on the cold iron railing. He was not afraid. Fear was for the tourists who came in summer, wide-eyed and clutching their cameras. Elias was a keeper. He was a guardian.\n\nHe checked the logbook. The entry for the night was simple: *Wind, 60 knots. Visibility, zero. Light, operational.*\n\nHe walked down the stairs, his boots heavy on the metal grating. The smell of salt"
+  },
+  {
+   "name": "code",
+   "tok_s": 107.4,
+   "reps": [
+    98.9,
+    111.3,
+    112.1
+   ],
+   "hit_rate": 0.9986286196529353,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header row\n    and values from the corresponding row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 533 windows; per window: wait for the GPU 6.226 ms, pool + plan 0.294 ms, host staging 0.561 ms, commit 0.225 ms",
+  "strata serve: stage 1: 533 windows; per window: wait for the GPU 14.156 ms, pool + plan 0.373 ms, host staging 0.000 ms, commit 0.371 ms",
+  "strata serve: stage 0: 599 windows; per window: wait for the GPU 6.314 ms, pool + plan 0.292 ms, host staging 0.501 ms, commit 0.225 ms",
+  "strata serve: stage 1: 599 windows; per window: wait for the GPU 14.366 ms, pool + plan 0.375 ms, host staging 0.000 ms, commit 0.374 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 19)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4606 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 19-47, expert cache 7160 slots (14.14 GiB), 7160 of its 7424 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-18",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-18 (CUDA0), 19-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.5% (6141 hits / 6234 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (3797 hits / 3836 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (3814 hits / 3837 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (28659 hits / 28795 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (3814 hits / 3839 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (141378 hits / 142508 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (124963 hits / 125273 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (134574 hits / 134874 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (121132 hits / 122322 lookups)",
+  "strata serve: decode expert cache hit rate: 99.9% (114073 hits / 114235 lookups)",
+  "strata serve: decode expert cache hit rate: 99.9% (113598 hits / 113754 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k20.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k20",
+ "extra": [
+  "--layer-split",
+  "20"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9044.3,
+   "pp_tok_s": 1776.4,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13928.7,
+   "pp_tok_s": 1991.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3743.5,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 236.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 76.4,
+   "reps": [
+    74.2,
+    78.4,
+    76.7
+   ],
+   "hit_rate": 0.9979911611088791,
+   "text": "The wind didn\u2019t just blow; it screamed. It clawed at the granite base of the tower, trying to pry it from the bedrock, and it howled through the narrow slits of the lantern room glass.\n\nElias Thorne was seventy years old, but he moved with the grace of a man who had spent fifty years listening to the sea. He knew the rhythm of the waves against the rocks below\u2014the *thud, thud, crash*\u2014and he knew when that rhythm changed. When the sea grew too quiet, or too violent, he knew a storm was coming, or a ship was in trouble.\n\nTonight, the storm was a Category 4. The rain lashed against the windows like gravel thrown by a giant. Elias stood in the center of the lantern room, his hand resting on the cold brass railing. He wasn\u2019t afraid. He was a guardian.\n\nHe checked the clockwork mechanism. It was old, a relic from 1"
+  },
+  {
+   "name": "code",
+   "tok_s": 105.9,
+   "reps": [
+    103.2,
+    106.8,
+    107.7
+   ],
+   "hit_rate": 0.9981849807154201,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 537 windows; per window: wait for the GPU 6.685 ms, pool + plan 0.403 ms, host staging 0.593 ms, commit 0.215 ms",
+  "strata serve: stage 1: 537 windows; per window: wait for the GPU 13.928 ms, pool + plan 0.282 ms, host staging 0.000 ms, commit 0.359 ms",
+  "strata serve: stage 0: 606 windows; per window: wait for the GPU 6.764 ms, pool + plan 0.399 ms, host staging 0.528 ms, commit 0.216 ms",
+  "strata serve: stage 1: 606 windows; per window: wait for the GPU 14.108 ms, pool + plan 0.288 ms, host staging 0.000 ms, commit 0.362 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 20)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4603 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 20-47, expert cache 7142 slots (14.14 GiB), 7142 of its 7168 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-19",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-19 (CUDA0), 20-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.9% (6162 hits / 6230 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (3803 hits / 3835 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (3810 hits / 3836 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (27185 hits / 27330 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (3807 hits / 3835 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (139825 hits / 140938 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (142122 hits / 142524 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (146556 hits / 146851 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (118026 hits / 119233 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (118729 hits / 118997 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (118791 hits / 119007 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k21.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k21",
+ "extra": [
+  "--layer-split",
+  "21"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8880.5,
+   "pp_tok_s": 1809.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13654.6,
+   "pp_tok_s": 2031.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3680.6,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 235.4,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 77.4,
+   "reps": [
+    80.2,
+    80.8,
+    71.2
+   ],
+   "hit_rate": 0.9973622550922132,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from rope burns and broken glass, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, ensuring the rotation was smooth, silent, and steady. *Tick. Tick. Tick.* The sound was the heartbeat of the coast.\n\n"
+  },
+  {
+   "name": "code",
+   "tok_s": 106.7,
+   "reps": [
+    100.6,
+    110.5,
+    109.0
+   ],
+   "hit_rate": 0.998218711927068,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 527 windows; per window: wait for the GPU 6.934 ms, pool + plan 0.620 ms, host staging 0.530 ms, commit 0.245 ms",
+  "strata serve: stage 1: 527 windows; per window: wait for the GPU 13.912 ms, pool + plan 0.365 ms, host staging 0.000 ms, commit 0.359 ms",
+  "strata serve: stage 0: 595 windows; per window: wait for the GPU 7.031 ms, pool + plan 0.595 ms, host staging 0.472 ms, commit 0.245 ms",
+  "strata serve: stage 1: 595 windows; per window: wait for the GPU 14.051 ms, pool + plan 0.361 ms, host staging 0.000 ms, commit 0.361 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 21)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4627 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 21-47, expert cache 6912 slots (13.78 GiB), 6912 of its 6912 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-20",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-20 (CUDA0), 21-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.7% (5672 hits / 5748 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4287 hits / 4314 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4287 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (29024 hits / 29229 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4278 hits / 4316 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (131499 hits / 133085 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (130974 hits / 131451 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (138767 hits / 139134 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (121454 hits / 122932 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (116809 hits / 117079 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (118803 hits / 119015 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k22.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k22",
+ "extra": [
+  "--layer-split",
+  "22"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8730.9,
+   "pp_tok_s": 1840.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13375.4,
+   "pp_tok_s": 2074.2,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3611.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 234.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 80.2,
+   "reps": [
+    78.4,
+    80.9,
+    81.5
+   ],
+   "hit_rate": 0.9977764197412695,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy-three years old, and he had been the keeper of the Blackwood Point Light for forty of those years. The other thirty-three had been spent in a small, drafty cottage on the mainland, waiting for the day when the sea would finally claim him.\n\nTonight, the storm was a Category 4. The waves weren\u2019t just crashing; they were climbing. They reached the gallery railing, ten feet above the base of the tower, spraying a fine mist of brine that coated Elias\u2019s glasses.\n\nHe climbed the spiral stairs, his knees protesting with every step. The iron steps were cold, slick with condensation. At the top, in the lantern room, the air was thin and sharp. The great Fresnel lens, a cathedral of glass and brass, stood silent. It was time to wake it.\n\n"
+  },
+  {
+   "name": "code",
+   "tok_s": 106.4,
+   "reps": [
+    100.7,
+    110.3,
+    108.1
+   ],
+   "hit_rate": 0.9971798370756316,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 529 windows; per window: wait for the GPU 7.229 ms, pool + plan 0.561 ms, host staging 0.529 ms, commit 0.233 ms",
+  "strata serve: stage 1: 529 windows; per window: wait for the GPU 13.058 ms, pool + plan 0.253 ms, host staging 0.000 ms, commit 0.334 ms",
+  "strata serve: stage 0: 598 windows; per window: wait for the GPU 7.325 ms, pool + plan 0.552 ms, host staging 0.469 ms, commit 0.235 ms",
+  "strata serve: stage 1: 598 windows; per window: wait for the GPU 13.238 ms, pool + plan 0.260 ms, host staging 0.000 ms, commit 0.337 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 22)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4640 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5609 hits / 5739 lookups)",
+  "strata serve: decode expert cache hit rate: 98.7% (4256 hits / 4310 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4278 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (27008 hits / 27289 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (4260 hits / 4313 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (135183 hits / 137274 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (136132 hits / 136708 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (135515 hits / 135817 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (121287 hits / 123280 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (117658 hits / 118026 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (120574 hits / 120915 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k23.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k23",
+ "extra": [
+  "--layer-split",
+  "23"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8742.6,
+   "pp_tok_s": 1837.7,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13084.2,
+   "pp_tok_s": 2120.3,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3566.1,
+  "turn2_tokens": 6660,
+  "reused": 6631,
+  "turn2_ms": 238.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 81.0,
+   "reps": [
+    76.8,
+    85.2,
+    81.1
+   ],
+   "hit_rate": 0.9961302681992337,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning black void, swallowing the horizon, throwing white foam against the granite base like a tantrum.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his wool trousers, the fabric stiff with salt and grease. He checked the clockwork mechanism that turned the lens. It was a beast of a thing, a Fresnel lens of eight feet in diameter, a cathedral of glass and crystal that focused the beam of the lamp into a single, piercing needle of light.\n\nHe wound the weights. *Click, click, click.* The gears engaged. The massive lens began its slow, hypnotic rotation.\n\nFor a moment, the"
+  },
+  {
+   "name": "code",
+   "tok_s": 108.4,
+   "reps": [
+    105.1,
+    109.0,
+    111.1
+   ],
+   "hit_rate": 0.9961698810290479,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 537 windows; per window: wait for the GPU 7.423 ms, pool + plan 0.953 ms, host staging 0.542 ms, commit 0.237 ms",
+  "strata serve: stage 1: 537 windows; per window: wait for the GPU 12.223 ms, pool + plan 0.236 ms, host staging 0.000 ms, commit 0.324 ms",
+  "strata serve: stage 0: 606 windows; per window: wait for the GPU 7.519 ms, pool + plan 0.904 ms, host staging 0.483 ms, commit 0.239 ms",
+  "strata serve: stage 1: 606 windows; per window: wait for the GPU 12.395 ms, pool + plan 0.242 ms, host staging 0.000 ms, commit 0.326 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 23)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4609 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 23-47, expert cache 6400 slots (12.82 GiB), 6400 of its 6400 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-22",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-22 (CUDA0), 23-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 96.8% (5528 hits / 5709 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (4235 hits / 4303 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (4260 hits / 4308 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (30643 hits / 31089 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (4248 hits / 4309 lookups)",
+  "strata serve: decode expert cache hit rate: 98.3% (134776 hits / 137143 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (133024 hits / 133766 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (129995 hits / 130500 lookups)",
+  "strata serve: decode expert cache hit rate: 97.9% (115782 hits / 118287 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (117953 hits / 118464 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (117560 hits / 118012 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k24.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k24",
+ "extra": [
+  "--layer-split",
+  "24"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8122.5,
+   "pp_tok_s": 1978.0,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 12285.8,
+   "pp_tok_s": 2258.1,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3407.0,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 234.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.5,
+   "reps": [
+    84.9,
+    83.3,
+    82.3
+   ],
+   "hit_rate": 0.9949381327334084,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined Elias\u2019s life for thirty years. He was not merely a keeper; he was a guardian of silence.\n\nHe checked the clock. 11:42 PM. The rotation of the great Fresnel lens was due. With a practiced hand, he engaged the brake on the clockwork mechanism. The heavy brass gears groaned, then began their slow, rhythmic turn. *Tick. Tick. Tick.* It was a heartbeat"
+  },
+  {
+   "name": "code",
+   "tok_s": 109.5,
+   "reps": [
+    104.9,
+    111.0,
+    112.6
+   ],
+   "hit_rate": 0.9953796671318199,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 519 windows; per window: wait for the GPU 7.835 ms, pool + plan 0.767 ms, host staging 0.557 ms, commit 0.244 ms",
+  "strata serve: stage 1: 519 windows; per window: wait for the GPU 11.922 ms, pool + plan 0.232 ms, host staging 0.000 ms, commit 0.325 ms",
+  "strata serve: stage 0: 587 windows; per window: wait for the GPU 7.943 ms, pool + plan 0.744 ms, host staging 0.497 ms, commit 0.246 ms",
+  "strata serve: stage 1: 587 windows; per window: wait for the GPU 12.073 ms, pool + plan 0.238 ms, host staging 0.000 ms, commit 0.326 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 24)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4603 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 24-47, expert cache 6144 slots (12.35 GiB), 6144 of its 6144 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-23",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-23 (CUDA0), 24-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 95.9% (5916 hits / 6171 lookups)",
+  "strata serve: decode expert cache hit rate: 97.4% (3717 hits / 3815 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (3761 hits / 3823 lookups)",
+  "strata serve: decode expert cache hit rate: 98.1% (27138 hits / 27664 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (3742 hits / 3826 lookups)",
+  "strata serve: decode expert cache hit rate: 97.9% (126114 hits / 128875 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (133626 hits / 134658 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (132675 hits / 133350 lookups)",
+  "strata serve: decode expert cache hit rate: 96.8% (113593 hits / 117399 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (118372 hits / 118949 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (116981 hits / 117524 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k26.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k26",
+ "extra": [
+  "--layer-split",
+  "26"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 7880.9,
+   "pp_tok_s": 2038.6,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 11771.9,
+   "pp_tok_s": 2356.7,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3323.9,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 231.5,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.8,
+   "reps": [
+    85.9,
+    83.1,
+    82.4
+   ],
+   "hit_rate": 0.9933966961830739,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and green.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined Elias\u2019s life for thirty years. He was a keeper of the light, but more importantly, he was a keeper of time.\n\nHe checked his pocket watch. 11:42 PM. The rotation of the great Fresnel lens was due. With a practiced, rhythmic *clunk-clunk-clunk*, he wound the clockwork mechanism. The massive lens began its slow, hypnotic turn, sweeping its beam across"
+  },
+  {
+   "name": "code",
+   "tok_s": 109.7,
+   "reps": [
+    104.4,
+    112.7,
+    111.9
+   ],
+   "hit_rate": 0.9938670271352759,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 519 windows; per window: wait for the GPU 8.376 ms, pool + plan 1.030 ms, host staging 0.566 ms, commit 0.273 ms",
+  "strata serve: stage 1: 519 windows; per window: wait for the GPU 10.886 ms, pool + plan 0.214 ms, host staging 0.000 ms, commit 0.297 ms",
+  "strata serve: stage 0: 588 windows; per window: wait for the GPU 8.506 ms, pool + plan 0.985 ms, host staging 0.505 ms, commit 0.274 ms",
+  "strata serve: stage 1: 588 windows; per window: wait for the GPU 11.036 ms, pool + plan 0.219 ms, host staging 0.000 ms, commit 0.298 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 26)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4568 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 26-47, expert cache 5632 slots (11.32 GiB), 5632 of its 5632 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-25",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-25 (CUDA0), 26-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 92.8% (5259 hits / 5665 lookups)",
+  "strata serve: decode expert cache hit rate: 95.2% (4056 hits / 4259 lookups)",
+  "strata serve: decode expert cache hit rate: 97.2% (4164 hits / 4285 lookups)",
+  "strata serve: decode expert cache hit rate: 97.1% (27690 hits / 28510 lookups)",
+  "strata serve: decode expert cache hit rate: 97.5% (4184 hits / 4293 lookups)",
+  "strata serve: decode expert cache hit rate: 96.9% (121839 hits / 125724 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (132165 hits / 133648 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (137652 hits / 138567 lookups)",
+  "strata serve: decode expert cache hit rate: 95.3% (111307 hits / 116822 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (118436 hits / 119329 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (119109 hits / 119844 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k28.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k28",
+ "extra": [
+  "--layer-split",
+  "28"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 7989.3,
+   "pp_tok_s": 2010.9,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 12132.9,
+   "pp_tok_s": 2286.6,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3433.5,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 228.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.5,
+   "reps": [
+    83.7,
+    82.4,
+    84.5
+   ],
+   "hit_rate": 0.9900441970361987,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, the glass was vibrating like a plucked cello string. The sea, usually a rhythmic pulse against the granite base, was now a chaotic, frothing beast, throwing itself against the rocks with the force of a battering ram.\n\nElias climbed the spiral stairs, his hand gripping the cold iron railing. His knees ached, not from age, but from the constant vibration of the floor beneath his feet. He reached the lantern room, the heart of the lighthouse, and checked the clockwork mechanism. The great lens, a prism of Fresnel glass, rotated with a steady, hypnotic *whirrr-click, whirrr-click*. It was the only sound that mattered. The light swept out into the black void, cutting through"
+  },
+  {
+   "name": "code",
+   "tok_s": 112.5,
+   "reps": [
+    101.4,
+    112.1,
+    124.1
+   ],
+   "hit_rate": 0.9896636587366694,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv(file_path: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys being the header values\n    and values being the corresponding cell values.\n    \n    Args:\n        file_path: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(file_path, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n```<|im_end|>"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 527 windows; per window: wait for the GPU 8.765 ms, pool + plan 1.401 ms, host staging 0.593 ms, commit 0.281 ms",
+  "strata serve: stage 1: 527 windows; per window: wait for the GPU 9.893 ms, pool + plan 0.195 ms, host staging 0.000 ms, commit 0.276 ms",
+  "strata serve: stage 0: 578 windows; per window: wait for the GPU 8.908 ms, pool + plan 1.359 ms, host staging 0.559 ms, commit 0.284 ms",
+  "strata serve: stage 1: 578 windows; per window: wait for the GPU 10.025 ms, pool + plan 0.200 ms, host staging 0.000 ms, commit 0.278 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 28)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4537 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 28-47, expert cache 5120 slots (10.30 GiB), 5120 of its 5120 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-27",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-27 (CUDA0), 28-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 88.8% (5405 hits / 6086 lookups)",
+  "strata serve: decode expert cache hit rate: 93.8% (3539 hits / 3773 lookups)",
+  "strata serve: decode expert cache hit rate: 94.2% (3560 hits / 3781 lookups)",
+  "strata serve: decode expert cache hit rate: 95.7% (26702 hits / 27899 lookups)",
+  "strata serve: decode expert cache hit rate: 95.9% (3647 hits / 3804 lookups)",
+  "strata serve: decode expert cache hit rate: 95.7% (122618 hits / 128160 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (135035 hits / 136884 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (129476 hits / 130778 lookups)",
+  "strata serve: decode expert cache hit rate: 93.9% (111037 hits / 118290 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (118829 hits / 120140 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (90480 hits / 91425 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k30.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k30",
+ "extra": [
+  "--layer-split",
+  "30"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8301.3,
+   "pp_tok_s": 1935.4,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 12507.6,
+   "pp_tok_s": 2218.1,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3448.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 229.4,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 81.1,
+   "reps": [
+    78.0,
+    84.4,
+    81.0
+   ],
+   "hit_rate": 0.9798126997277784,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, the glass was vibrating like a plucked cello string. The sea, usually a rhythmic pulse against the granite base, was now a chaotic, frothing beast, throwing itself against the rocks with the force of a battering ram.\n\nElias stood at the top of the spiral staircase, his hand resting on the cold iron railing. He was not afraid. Fear was for the tourists who came in summer, wide-eyed and clutching their cameras. Elias was a keeper. He was a guardian.\n\nHe checked the logbook. The entry for the night was simple: *Wind, 60 knots. Visibility, zero. Light, operational.*\n\nHe walked down the stairs, his boots heavy on the metal grating. The smell of salt"
+  },
+  {
+   "name": "code",
+   "tok_s": 109.6,
+   "reps": [
+    100.7,
+    114.1,
+    114.1
+   ],
+   "hit_rate": 0.9825370343977666,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 540 windows; per window: wait for the GPU 9.206 ms, pool + plan 1.794 ms, host staging 0.525 ms, commit 0.298 ms",
+  "strata serve: stage 1: 540 windows; per window: wait for the GPU 8.767 ms, pool + plan 0.173 ms, host staging 0.000 ms, commit 0.242 ms",
+  "strata serve: stage 0: 608 windows; per window: wait for the GPU 9.390 ms, pool + plan 1.733 ms, host staging 0.471 ms, commit 0.301 ms",
+  "strata serve: stage 1: 608 windows; per window: wait for the GPU 8.910 ms, pool + plan 0.178 ms, host staging 0.000 ms, commit 0.244 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 30)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4479 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 30-47, expert cache 4608 slots (9.19 GiB), 4608 of its 4608 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-29",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-29 (CUDA0), 30-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 84.6% (5099 hits / 6028 lookups)",
+  "strata serve: decode expert cache hit rate: 90.7% (3403 hits / 3751 lookups)",
+  "strata serve: decode expert cache hit rate: 91.1% (3415 hits / 3747 lookups)",
+  "strata serve: decode expert cache hit rate: 94.1% (26108 hits / 27737 lookups)",
+  "strata serve: decode expert cache hit rate: 93.4% (3534 hits / 3784 lookups)",
+  "strata serve: decode expert cache hit rate: 94.6% (127532 hits / 134866 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (135828 hits / 137786 lookups)",
+  "strata serve: decode expert cache hit rate: 98.0% (132455 hits / 135184 lookups)",
+  "strata serve: decode expert cache hit rate: 91.0% (106351 hits / 116917 lookups)",
+  "strata serve: decode expert cache hit rate: 97.9% (115791 hits / 118246 lookups)",
+  "strata serve: decode expert cache hit rate: 98.3% (117198 hits / 119281 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k32.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k32",
+ "extra": [
+  "--layer-split",
+  "32"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8837.1,
+   "pp_tok_s": 1818.0,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13449.8,
+   "pp_tok_s": 2062.7,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3565.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 227.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.0,
+   "reps": [
+    75.0,
+    86.1,
+    87.8
+   ],
+   "hit_rate": 0.9796934433020751,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning cauldron of black ink and white foam, throwing itself against the granite base with the force of a battering ram.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his woolen trousers, leaving faint, oily smears on the fabric. He checked the clock. 11:42 PM. The rotation was due.\n\nHe reached for the crank handle, a heavy iron wheel worn smooth by decades of his own palms. With a grunt that matched the strain of his shoulders, he began to turn it. *Click. Click. Click.* The gears engaged, and the massive Fresnel lens began its slow"
+  },
+  {
+   "name": "code",
+   "tok_s": 108.7,
+   "reps": [
+    98.8,
+    112.2,
+    115.1
+   ],
+   "hit_rate": 0.9777186846317358,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 533 windows; per window: wait for the GPU 9.839 ms, pool + plan 2.192 ms, host staging 0.585 ms, commit 0.318 ms",
+  "strata serve: stage 1: 533 windows; per window: wait for the GPU 8.014 ms, pool + plan 0.156 ms, host staging 0.000 ms, commit 0.231 ms",
+  "strata serve: stage 0: 602 windows; per window: wait for the GPU 10.014 ms, pool + plan 2.111 ms, host staging 0.520 ms, commit 0.321 ms",
+  "strata serve: stage 1: 602 windows; per window: wait for the GPU 8.094 ms, pool + plan 0.160 ms, host staging 0.000 ms, commit 0.232 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 32)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4488 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 32-47, expert cache 4096 slots (8.26 GiB), 4096 of its 4096 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-31",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-31 (CUDA0), 32-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 81.3% (4889 hits / 6010 lookups)",
+  "strata serve: decode expert cache hit rate: 87.7% (3272 hits / 3733 lookups)",
+  "strata serve: decode expert cache hit rate: 87.9% (3280 hits / 3733 lookups)",
+  "strata serve: decode expert cache hit rate: 92.3% (24238 hits / 26254 lookups)",
+  "strata serve: decode expert cache hit rate: 91.5% (3457 hits / 3778 lookups)",
+  "strata serve: decode expert cache hit rate: 93.7% (133254 hits / 142240 lookups)",
+  "strata serve: decode expert cache hit rate: 98.0% (130112 hits / 132812 lookups)",
+  "strata serve: decode expert cache hit rate: 98.0% (130069 hits / 132765 lookups)",
+  "strata serve: decode expert cache hit rate: 89.1% (105263 hits / 118190 lookups)",
+  "strata serve: decode expert cache hit rate: 97.2% (116970 hits / 120309 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (115450 hits / 118081 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-single.json

@@ -0,0 +1,76 @@
+{
+ "label": "ks-single",
+ "extra": [],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9310.4,
+   "pp_tok_s": 1725.6,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 14085.4,
+   "pp_tok_s": 1969.6,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3388.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 200.7,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 86.7,
+   "reps": [
+    78.2,
+    91.4,
+    90.4
+   ],
+   "hit_rate": 0.921882148773369,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy-three years old, and his hands were a map of the coastline he had guarded for fifty years. The knuckles were swollen, the skin cracked like dry earth, but they moved with a precision that no machine could replicate.\n\nHe stood on the gallery of the *Harrow Point* lighthouse, the iron railing cold against his palms. Below, the Atlantic churned in a frenzy, whitecaps exploding against the black rocks like fireworks. Above, the sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen.\n\nElias checked the clock. 11:42 PM. The light was due.\n\nHe didn\u2019t look at the gauge. He didn\u2019t check the fuel levels. He knew. He always knew. The rhythm of the ocean, the pressure of the air, the subtle vibration"
+  },
+  {
+   "name": "code",
+   "tok_s": 105.2,
+   "reps": [
+    87.8,
+    113.5,
+    114.2
+   ],
+   "hit_rate": 0.9109811738648947,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 382 windows; per window: wait for the GPU 15.155 ms, pool + plan 4.331 ms, host staging 0.656 ms, commit 0.458 ms",
+  "strata serve: stage 0: 455 windows; per window: wait for the GPU 15.790 ms, pool + plan 5.552 ms, host staging 0.688 ms, commit 0.467 ms",
+  "strata serve: stage 0: 524 windows; per window: wait for the GPU 16.064 ms, pool + plan 5.529 ms, host staging 0.606 ms, commit 0.472 ms",
+  "strata serve: stage 0: 593 windows; per window: wait for the GPU 16.305 ms, pool + plan 5.462 ms, host staging 0.540 ms, commit 0.475 ms"
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4293 slots, 8.17 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 67.8% (2299 hits / 3391 lookups)",
+  "strata serve: decode expert cache hit rate: 68.5% (2345 hits / 3424 lookups)",
+  "strata serve: decode expert cache hit rate: 75.6% (16605 hits / 21959 lookups)",
+  "strata serve: decode expert cache hit rate: 73.3% (2608 hits / 3558 lookups)",
+  "strata serve: decode expert cache hit rate: 83.7% (104878 hits / 125346 lookups)",
+  "strata serve: decode expert cache hit rate: 92.7% (119739 hits / 129184 lookups)",
+  "strata serve: decode expert cache hit rate: 92.2% (116867 hits / 126770 lookups)",
+  "strata serve: decode expert cache hit rate: 68.1% (64111 hits / 94116 lookups)",
+  "strata serve: decode expert cache hit rate: 89.0% (96370 hits / 108259 lookups)",
+  "strata serve: decode expert cache hit rate: 91.1% (102827 hits / 112875 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-rel-5080.json

@@ -0,0 +1,72 @@
+{
+ "label": "rel-5080",
+ "extra": [],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2132.8,
+   "pp_tok_s": 1278.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5008.9,
+   "pp_tok_s": 1672.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 7855.4,
+   "pp_tok_s": 2045.2,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13833.9,
+   "pp_tok_s": 2005.4,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3298.2,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 197.9,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.2,
+   "hit_rate": 0.8418683557684931,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning cauldron of black ink and white foam, throwing itself against the granite base with the force of a battering ram.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his woolen trousers, leaving faint, oily smears on the fabric. He checked the clock. 11:42 PM. The rotation was due.\n\nHe reached for the crank handle, a heavy iron wheel worn smooth by decades of his own palms. With a grunt that matched the strain of his shoulders, he began to turn it. *Click. Click. Click.* The gears engaged, and the massive Fresnel lens began its slow"
+  },
+  {
+   "name": "code",
+   "tok_s": 88.2,
+   "hit_rate": 0.7011385968111186,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4293 slots, 8.17 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 64.2% (2136 hits / 3329 lookups)",
+  "strata serve: decode expert cache hit rate: 67.3% (2309 hits / 3432 lookups)",
+  "strata serve: decode expert cache hit rate: 72.5% (2539 hits / 3504 lookups)",
+  "strata serve: decode expert cache hit rate: 77.3% (2817 hits / 3642 lookups)",
+  "strata serve: decode expert cache hit rate: 75.3% (16500 hits / 21916 lookups)",
+  "strata serve: decode expert cache hit rate: 80.0% (2978 hits / 3723 lookups)",
+  "strata serve: decode expert cache hit rate: 84.2% (98752 hits / 117301 lookups)",
+  "strata serve: decode expert cache hit rate: 70.1% (69215 hits / 98718 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-t75-5080.json

@@ -0,0 +1,78 @@
+{
+ "label": "t75-5080",
+ "extra": [],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2158.8,
+   "pp_tok_s": 1262.7,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5060.0,
+   "pp_tok_s": 1655.9,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3389.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 213.5,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 80.4,
+   "reps": [
+    80.4
+   ],
+   "hit_rate": 0.855538332528651,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning black void, punctuated by whitecaps that looked like teeth.\n\nHe climbed the spiral stairs, his boots heavy with salt and exhaustion. The iron steps groaned under his weight, a sound that had become so familiar to him that he could sleep to it. But tonight, he wasn\u2019t sleeping. He was working.\n\nThe lens was the heart of the lighthouse. A massive, rotating Fresnel lens, a cathedral of glass and brass that turned with a mechanical hum. Elias checked the clockwork mechanism. It was smooth, rhythmic, a heartbeat in the dark. But the light itself was failing.\n\nHe climbed higher, into the lantern room. The wind battered the glass panes,"
+  },
+  {
+   "name": "code",
+   "tok_s": 87.5,
+   "reps": [
+    87.5
+   ],
+   "hit_rate": 0.7075461363025227,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 39 windows; per window: wait for the GPU 17.252 ms, pool + plan 8.473 ms, host staging 0.611 ms, commit 0.525 ms",
+  "strata serve: stage 0: 47 windows; per window: wait for the GPU 18.018 ms, pool + plan 8.616 ms, host staging 0.607 ms, commit 0.534 ms",
+  "strata serve: stage 0: 169 windows; per window: wait for the GPU 15.223 ms, pool + plan 5.717 ms, host staging 0.688 ms, commit 0.467 ms",
+  "strata serve: stage 0: 243 windows; per window: wait for the GPU 16.500 ms, pool + plan 7.316 ms, host staging 0.742 ms, commit 0.481 ms"
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4294 slots, 8.18 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 64.2% (2136 hits / 3329 lookups)",
+  "strata serve: decode expert cache hit rate: 67.3% (2309 hits / 3432 lookups)",
+  "strata serve: decode expert cache hit rate: 75.5% (16316 hits / 21602 lookups)",
+  "strata serve: decode expert cache hit rate: 74.4% (2660 hits / 3573 lookups)",
+  "strata serve: decode expert cache hit rate: 83.4% (92664 hits / 111086 lookups)",
+  "strata serve: decode expert cache hit rate: 72.0% (70946 hits / 98601 lookups)",
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4293 slots, 8.17 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 64.2% (2136 hits / 3329 lookups)",
+  "strata serve: decode expert cache hit rate: 67.3% (2309 hits / 3432 lookups)",
+  "strata serve: decode expert cache hit rate: 75.5% (16316 hits / 21602 lookups)",
+  "strata serve: decode expert cache hit rate: 74.4% (2659 hits / 3572 lookups)",
+  "strata serve: decode expert cache hit rate: 85.6% (101004 hits / 118059 lookups)",
+  "strata serve: decode expert cache hit rate: 70.8% (71044 hits / 100409 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-v-x2-auto.json

@@ -0,0 +1,103 @@
+{
+ "label": "v-x2-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2577.0,
+   "pp_tok_s": 1057.8,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 4433.5,
+   "pp_tok_s": 1889.9,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 7885.8,
+   "pp_tok_s": 2037.3,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13384.6,
+   "pp_tok_s": 2072.8,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3667.5,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 253.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 79.5,
+   "reps": [
+    78.9,
+    79.0,
+    80.7
+   ],
+   "hit_rate": 0.9969903443250137,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from rope burns and broken glass, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, listening for the faint, rhythmic *tick-tick-tick* that signaled the rotation was true. If the rhythm faltered, ships"
+  },
+  {
+   "name": "code",
+   "tok_s": 108.9,
+   "reps": [
+    103.3,
+    113.7,
+    109.5
+   ],
+   "hit_rate": 0.9971795386830036,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 542 windows; per window: wait for the GPU 7.167 ms, pool + plan 0.572 ms, host staging 0.550 ms, commit 0.232 ms",
+  "strata serve: stage 1: 542 windows; per window: wait for the GPU 12.855 ms, pool + plan 0.250 ms, host staging 0.000 ms, commit 0.334 ms",
+  "strata serve: stage 0: 610 windows; per window: wait for the GPU 7.276 ms, pool + plan 0.559 ms, host staging 0.493 ms, commit 0.234 ms",
+  "strata serve: stage 1: 610 windows; per window: wait for the GPU 13.048 ms, pool + plan 0.258 ms, host staging 0.000 ms, commit 0.337 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.30 GiB for experts",
+  "strata generate: layer split auto: CUDA1 82 SMs at 1.86 GHz -> 0.48 ms per layer, 14.14 GiB for experts",
+  "strata generate: layer split auto: K=22 - predicted 19.7 ms per decode window; the caches hold 11298 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4640 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5609 hits / 5739 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (4241 hits / 4306 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (4262 hits / 4309 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4290 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (4303 hits / 4319 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (28379 hits / 28721 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (4285 hits / 4314 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (136232 hits / 138256 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (135540 hits / 136216 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (136812 hits / 137225 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (118437 hits / 120406 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (112836 hits / 113230 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-v-x2r-auto.json

@@ -0,0 +1,88 @@
+{
+ "label": "v-x2r-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "2,0",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9610.0,
+   "pp_tok_s": 1671.8,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 14786.1,
+   "pp_tok_s": 1876.3,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3850.8,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 245.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 90.3,
+   "reps": [
+    91.0,
+    97.0,
+    82.8
+   ],
+   "hit_rate": 0.993905472636816,
+   "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak of the spiral staircase, every draft that slipped under the door, and every shift in the barometric pressure that preceded a storm. He was not merely a worker; he was the heartbeat of the coast.\n\nIt was November, the month of grey seas and shorter days. The fog had rolled in thick, a wall of white so dense that even the beam of the Fresnel lens seemed to struggle against it. Elias stood in the lantern room, his hands roughened by salt and rope, adjusting the rotation mechanism. The great lens, a cathedral of glass and brass, began its slow, rhythmic sweep. *Flash. Dark. Flash. Dark.*\n\nHe checked the log"
+  },
+  {
+   "name": "code",
+   "tok_s": 105.1,
+   "reps": [
+    98.4,
+    109.9,
+    107.0
+   ],
+   "hit_rate": 0.9949900793650793,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 492 windows; per window: wait for the GPU 15.016 ms, pool + plan 0.297 ms, host staging 0.517 ms, commit 0.404 ms",
+  "strata serve: stage 1: 492 windows; per window: wait for the GPU 5.775 ms, pool + plan 0.854 ms, host staging 0.001 ms, commit 0.210 ms",
+  "strata serve: stage 0: 561 windows; per window: wait for the GPU 15.257 ms, pool + plan 0.306 ms, host staging 0.455 ms, commit 0.408 ms",
+  "strata serve: stage 1: 561 windows; per window: wait for the GPU 5.852 ms, pool + plan 0.820 ms, host staging 0.001 ms, commit 0.210 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 57.7 GB/s -> pcie_frac 0.55",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 9.98 GiB free",
+  "strata generate: layer split auto: CUDA0 82 SMs at 1.86 GHz -> 0.48 ms per layer, 16.45 GiB for experts",
+  "strata generate: layer split auto: CUDA1 84 SMs at 2.81 GHz -> 0.31 ms per layer, 5.99 GiB for experts",
+  "strata generate: layer split auto: K=30 - predicted 20.8 ms per decode window; the caches hold 10684 of 12288 profiled pairs (~99.5% of the routed mass)",
+  "strata generate: expert cache auto: 18.62 GiB free, 700 MiB reserved -> 6634 slots",
+  "strata generate: expert cache 7680 slots, 14.23 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 30-47, expert cache 3004 slots (5.98 GiB), 3004 of its 4608 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-29",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-29 (CUDA0), 30-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 93.2% (5367 hits / 5760 lookups)",
+  "strata serve: decode expert cache hit rate: 93.7% (4049 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 96.7% (4178 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 97.7% (26272 hits / 26880 lookups)",
+  "strata serve: decode expert cache hit rate: 97.1% (4194 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (110748 hits / 113280 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (113787 hits / 114240 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (127856 hits / 128640 lookups)",
+  "strata serve: decode expert cache hit rate: 96.3% (116528 hits / 120960 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (116423 hits / 117120 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (120354 hits / 120960 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-v-x3-auto.json

@@ -0,0 +1,89 @@
+{
+ "label": "v-x3-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,1,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9525.4,
+   "pp_tok_s": 1686.6,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 15017.1,
+   "pp_tok_s": 1847.4,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4039.6,
+  "turn2_tokens": 6660,
+  "reused": 6631,
+  "turn2_ms": 360.2,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 67.6,
+   "reps": [
+    59.8,
+    83.1,
+    59.9
+   ],
+   "hit_rate": 0.9967581085879247,
+   "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak in the spiral staircase, every draft that slipped under the door, and every shift in the weather that the barometer couldn\u2019t quite predict. He knew that when the fog rolled in thick enough to swallow the horizon, the world ceased to exist. There was only the light, the stone, and him.\n\nIt was November, the month of storms. The sea was a churning cauldron of grey and white, throwing itself against the granite base with the force of a battering ram. Elias stood in the lantern room, his hands rough as sandpaper, adjusting the massive Fresnel lens. The rotation mechanism groaned, a sound like a ship\u2019s anchor chain being h"
+  },
+  {
+   "name": "code",
+   "tok_s": 89.6,
+   "reps": [
+    81.0,
+    104.8,
+    83.0
+   ],
+   "hit_rate": 0.9975041387598007,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 2: 522 windows; per window: wait for the GPU 12.405 ms, pool + plan 0.242 ms, host staging 0.006 ms, commit 0.375 ms",
+  "strata serve: stage 0: 590 windows; per window: wait for the GPU 7.387 ms, pool + plan 0.549 ms, host staging 0.517 ms, commit 0.248 ms",
+  "strata serve: stage 1: 590 windows; per window: wait for the GPU 3.071 ms, pool + plan 0.011 ms, host staging 0.000 ms, commit 0.606 ms",
+  "strata serve: stage 2: 590 windows; per window: wait for the GPU 12.592 ms, pool + plan 0.250 ms, host staging 0.006 ms, commit 0.377 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 3 GPUs: CUDA0, then CUDA1,2 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA1 holds its weights, session; 16.67 GiB free",
+  "strata generate: layer split: CUDA2 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA2 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.30 GiB for experts",
+  "strata generate: layer split auto: CUDA1 68 SMs at 1.54 GHz -> 0.69 ms per layer, 13.50 GiB for experts",
+  "strata generate: layer split auto: CUDA2 82 SMs at 1.86 GHz -> 0.48 ms per layer, 14.14 GiB for experts",
+  "strata generate: layer split auto: K=22,23 - predicted 19.9 ms per decode window; the caches hold 11298 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4640 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-22, expert cache 256 slots (0.52 GiB), 256 of its 256 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA2 runs layers 23-47, expert cache 6400 slots (12.82 GiB), 6400 of its 6400 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-22 (CUDA1), 23-47 (CUDA2), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5609 hits / 5739 lookups)",
+  "strata serve: decode expert cache hit rate: 98.7% (4256 hits / 4310 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4278 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (29862 hits / 30170 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (4264 hits / 4313 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (134418 hits / 136339 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (116986 hits / 117525 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (129134 hits / 129554 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2-auto-c1024.json

@@ -0,0 +1,84 @@
+{
+ "label": "x2-auto-c1024",
+ "extra": [
+  "--layer-split",
+  "auto",
+  "--prefill",
+  "1024"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2648.7,
+   "pp_tok_s": 1029.2,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5547.0,
+   "pp_tok_s": 1510.5,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 10123.3,
+   "pp_tok_s": 1587.0,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 17652.0,
+   "pp_tok_s": 1571.7,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4379.9,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 235.5,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 82.7,
+   "hit_rate": 1.0,
+   "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak in the spiral staircase, every draft that slipped under the door, and every shift in the weather that the barometer couldn\u2019t quite predict. He knew that when the fog rolled in thick enough to swallow the horizon, the world ceased to exist. There was only the light, the stone, and him.\n\nIt was November, the month of storms. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that made the floorboards tremble. Elias stood by the lantern room, his hands wrapped around a mug of coffee that had gone cold hours ago. He wasn\u2019t drinking it. He was holding it for warmth."
+  },
+  {
+   "name": "code",
+   "tok_s": 111.1,
+   "hit_rate": 1.0,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv(file_path: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys being the header values\n    and values being the corresponding cell values.\n    \n    Args:\n        file_path: Path to the CSV file.\n        \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(file_path, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_basic():\n    \"\"\"Test basic CSV parsing with simple data.\"\"\"\n    import tempfile\n    import os\n    \n    # Create a temporary CSV"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: K=19 - the caches hold 12288 of 12288 profiled pairs (fullest device 99%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3613 slots",
+  "strata generate: expert cache 4864 slots, 8.76 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 19-47, expert cache 7424 slots (14.66 GiB), 7424 of its 7424 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-18",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-18 (CUDA0), 19-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 100.0% (5760 hits / 5760 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (26400 hits / 26400 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (130080 hits / 130080 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (110400 hits / 110400 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2-auto.json

@@ -0,0 +1,82 @@
+{
+ "label": "x2-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2711.7,
+   "pp_tok_s": 1005.3,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5010.3,
+   "pp_tok_s": 1672.4,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 8994.8,
+   "pp_tok_s": 1786.1,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 15497.2,
+   "pp_tok_s": 1790.2,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4050.3,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 249.7,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.0,
+   "hit_rate": 0.9920220447608565,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning cauldron of black ink and white foam, throwing itself against the granite base with the force of a battering ram.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his woolen trousers, leaving faint, oily smears on the fabric. He checked the clock. 11:42 PM. The rotation was due.\n\nHe reached for the crank handle, a heavy iron wheel worn smooth by decades of his own palms. With a grunt that matched the strain of his shoulders, he began to turn it. The massive Fresnel lens, a cathedral of glass and crystal, began its slow, hypnotic sweep"
+  },
+  {
+   "name": "code",
+   "tok_s": 101.8,
+   "hit_rate": 0.991767530342175,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: K=19 - the caches hold 11767 of 12288 profiled pairs (fullest device 100%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4606 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 19-47, expert cache 7160 slots (14.14 GiB), 7160 of its 7424 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-18",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-18 (CUDA0), 19-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.5% (6141 hits / 6234 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (3797 hits / 3838 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (3804 hits / 3838 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (3816 hits / 3839 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (3825 hits / 3839 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (27693 hits / 27832 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (3815 hits / 3839 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (127081 hits / 128103 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (119386 hits / 120377 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2-k16.json

@@ -0,0 +1,81 @@
+{
+ "label": "x2-k16",
+ "extra": [
+  "--layer-split",
+  "16"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2987.1,
+   "pp_tok_s": 912.6,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5908.6,
+   "pp_tok_s": 1418.1,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 10655.8,
+   "pp_tok_s": 1507.7,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 18484.6,
+   "pp_tok_s": 1500.9,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4790.8,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 243.3,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 71.4,
+   "hit_rate": 0.9814261642156863,
+   "text": "The wind didn\u2019t just blow; it screamed. It clawed at the granite base of the tower, trying to pry the lighthouse from its bedrock. Inside, however, the air was still, smelling faintly of kerosene and old paper.\n\nElias had been the keeper of the Point for thirty-two years. He knew every creak in the floorboards, every draft that slipped through the window seals, and exactly how much oil the great Fresnel lens needed to turn its slow, hypnotic rotation.\n\nIt was 2:00 AM. The storm was at its peak. The sea was a chaotic black mirror, whipped into whitecaps that looked like jagged teeth.\n\nElias climbed the spiral stairs, his hand gripping the cold iron railing. His boots, heavy with salt and wear, thumped a steady rhythm against the stone. *Thump. Thump. Thump.*\n\nAt the top, the lantern room hummed with the vibration of"
+  },
+  {
+   "name": "code",
+   "tok_s": 86.4,
+   "hit_rate": 0.9798276439089693,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 16)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4096 slots, 7.17 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 16-47, expert cache 7124 slots (14.14 GiB), 7124 of its 8192 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-15",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-15 (CUDA0), 16-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5626 hits / 5760 lookups)",
+  "strata serve: decode expert cache hit rate: 98.1% (4237 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (4269 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4279 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (4298 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (28471 hits / 28800 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4282 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 98.1% (128135 hits / 130560 lookups)",
+  "strata serve: decode expert cache hit rate: 98.0% (117109 hits / 119520 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2-k22.json

@@ -0,0 +1,81 @@
+{
+ "label": "x2-k22",
+ "extra": [
+  "--layer-split",
+  "22"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2537.7,
+   "pp_tok_s": 1074.2,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 4440.0,
+   "pp_tok_s": 1887.2,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 7872.5,
+   "pp_tok_s": 2040.8,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13405.0,
+   "pp_tok_s": 2069.6,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3610.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 236.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 78.1,
+   "hit_rate": 0.9851122208995093,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy-three years old, and he had been the keeper of the Blackwood Point Light for forty of those years. The other thirty-three had been spent in a hospital bed, waiting for the end.\n\nEvery night, Elias climbed the one hundred and twelve steps of the spiral staircase. His knees, worn smooth by the friction of his own boots over decades, protested with a sharp ache. He counted them. *One, two, three\u2026* The rhythm was his heartbeat. If he lost the count, he lost the light.\n\nAt the top, the lantern room hummed with the low, electric thrum of the Fresnel lens. It was a cathedral of glass and brass, a machine that had never failed him. Not once. Not in four decades.\n\nHe polished the brass railing with a rag that smelled of lemon and"
+  },
+  {
+   "name": "code",
+   "tok_s": 104.3,
+   "hit_rate": 0.9842543225870746,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 22)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4640 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5609 hits / 5739 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (4241 hits / 4306 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (4262 hits / 4309 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4290 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (4303 hits / 4319 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (28379 hits / 28721 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (4285 hits / 4314 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (135713 hits / 137764 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (118518 hits / 120414 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2r-auto.json

@@ -0,0 +1,82 @@
+{
+ "label": "x2r-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "2,0",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2804.2,
+   "pp_tok_s": 972.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5641.7,
+   "pp_tok_s": 1485.2,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 10643.4,
+   "pp_tok_s": 1509.5,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 18363.5,
+   "pp_tok_s": 1510.8,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4517.2,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 245.6,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 78.9,
+   "hit_rate": 0.9913117283950618,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to sync with his own heartbeat. He had been doing this for forty years. His hands were mapped with scars from rope burns and broken glass, his eyes permanently squinted against the glare of the bulb."
+  },
+  {
+   "name": "code",
+   "tok_s": 93.4,
+   "hit_rate": 0.990286044973545,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 57.7 GB/s -> pcie_frac 0.55",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 9.98 GiB free",
+  "strata generate: layer split auto: K=36 - the caches hold 11775 of 12288 profiled pairs (fullest device 100%)",
+  "strata generate: expert cache auto: 18.62 GiB free, 700 MiB reserved -> 6634 slots",
+  "strata generate: expert cache 8828 slots, 16.45 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 36-47, expert cache 2945 slots (5.99 GiB), 2945 of its 3072 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-35",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-35 (CUDA0), 36-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.6% (5678 hits / 5760 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4283 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (4285 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (4303 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (4306 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (27700 hits / 27840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4295 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (128474 hits / 129600 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (119785 hits / 120960 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2t-auto.json

@@ -0,0 +1,99 @@
+{
+ "label": "x2t-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,1",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 4201.1,
+   "pp_tok_s": 648.9,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 9655.6,
+   "pp_tok_s": 867.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 17573.8,
+   "pp_tok_s": 914.2,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 30615.6,
+   "pp_tok_s": 906.2,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 7864.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 366.9,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 45.5,
+   "reps": [
+    45.5
+   ],
+   "hit_rate": 0.9703345151405784,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, rough as sandpaper, knew every bolt, every lever, and every creak in the iron bones of the tower.\n\nFor forty years, Elias had been the guardian of the Point. Not just of the light, but of the silence between the waves.\n\nIt was November. The sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that shook the floorboards beneath Elias\u2019s boots.\n\nHe climbed the stairs. One hundred and twelve steps. He counted them by feel, his left hand trailing along the cold iron railing. At the top, the air was"
+  },
+  {
+   "name": "code",
+   "tok_s": 58.0,
+   "reps": [
+    58.0
+   ],
+   "hit_rate": 0.9700542735219787,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 168 windows; per window: wait for the GPU 9.683 ms, pool + plan 0.509 ms, host staging 0.714 ms, commit 0.320 ms",
+  "strata serve: stage 1: 168 windows; per window: wait for the GPU 25.253 ms, pool + plan 0.881 ms, host staging 0.000 ms, commit 0.559 ms",
+  "strata serve: stage 0: 242 windows; per window: wait for the GPU 9.860 ms, pool + plan 0.550 ms, host staging 0.750 ms, commit 0.328 ms",
+  "strata serve: stage 1: 242 windows; per window: wait for the GPU 26.029 ms, pool + plan 0.980 ms, host staging 0.000 ms, commit 0.571 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 16.18 GiB free",
+  "strata generate: layer split auto: K=20 - the caches hold 10761 of 12288 profiled pairs (fullest device 100%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4603 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 20-47, expert cache 6157 slots (12.19 GiB), 6157 of its 7168 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-19",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-19 (CUDA0), 20-47 (CUDA1), one hand-off per window",
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 16.18 GiB free",
+  "strata generate: layer split auto: K=20 - the caches hold 10761 of 12288 profiled pairs (fullest device 100%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4603 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 20-47, expert cache 6157 slots (12.19 GiB), 6157 of its 7168 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-19",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-19 (CUDA0), 20-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 96.6% (5559 hits / 5754 lookups)",
+  "strata serve: decode expert cache hit rate: 97.5% (4205 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (4217 hits / 4312 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (4257 hits / 4318 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x3-auto.json

@@ -0,0 +1,99 @@
+{
+ "label": "x3-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,2,1",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 3657.4,
+   "pp_tok_s": 745.3,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 7594.1,
+   "pp_tok_s": 1103.4,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 13996.4,
+   "pp_tok_s": 1147.9,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 24255.2,
+   "pp_tok_s": 1143.8,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 6104.9,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 393.4,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 51.9,
+   "reps": [
+    51.9
+   ],
+   "hit_rate": 1.0,
+   "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak in the spiral staircase, every draft that slipped under the door, and every shift in the weather that the barometer couldn\u2019t quite predict. He knew that when the fog rolled in thick enough to swallow the horizon, the world ceased to exist. There was only the light, the stone, and him.\n\nIt was November, the month of storms. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that made the floorboards tremble. Elias stood by the lantern room, his hands wrapped around a mug of coffee that had gone cold hours ago. He wasn\u2019t drinking it. He was holding it for warmth."
+  },
+  {
+   "name": "code",
+   "tok_s": 69.7,
+   "reps": [
+    69.7
+   ],
+   "hit_rate": 1.0,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv(file_path: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys being the header values\n    and values being the corresponding cell values.\n    \n    Args:\n        file_path: Path to the CSV file.\n        \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(file_path, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_basic():\n    \"\"\"Test basic CSV parsing with simple data.\"\"\"\n    import tempfile\n    import os\n    \n    # Create a temporary CSV"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 2: 165 windows; per window: wait for the GPU 13.452 ms, pool + plan 0.162 ms, host staging 0.000 ms, commit 0.328 ms",
+  "strata serve: stage 0: 234 windows; per window: wait for the GPU 6.235 ms, pool + plan 0.145 ms, host staging 0.704 ms, commit 0.214 ms",
+  "strata serve: stage 1: 234 windows; per window: wait for the GPU 15.592 ms, pool + plan 0.246 ms, host staging 0.000 ms, commit 0.657 ms",
+  "strata serve: stage 2: 234 windows; per window: wait for the GPU 13.916 ms, pool + plan 0.177 ms, host staging 0.000 ms, commit 0.335 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 3 GPUs: CUDA0, then CUDA1,2 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session; 18.62 GiB free",
+  "strata generate: layer split: CUDA2 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA2 holds its weights, session and head; 16.18 GiB free",
+  "strata generate: layer split auto: K=12,33 - the caches hold 12288 of 12288 profiled pairs (fullest device 66%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3072 slots",
+  "strata generate: expert cache 3072 slots, 5.41 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 12-32, expert cache 5376 slots (10.22 GiB), 5376 of its 5376 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA2 runs layers 33-47, expert cache 3840 slots (7.79 GiB), 3840 of its 3840 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-11",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-11 (CUDA0), 12-32 (CUDA1), 33-47 (CUDA2), one hand-off per window",
+  "strata generate: layer split across 3 GPUs: CUDA0, then CUDA1,2 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session; 18.62 GiB free",
+  "strata generate: layer split: CUDA2 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA2 holds its weights, session and head; 16.18 GiB free",
+  "strata generate: layer split auto: K=12,33 - the caches hold 12288 of 12288 profiled pairs (fullest device 66%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3072 slots",
+  "strata generate: expert cache 3072 slots, 5.41 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 12-32, expert cache 5376 slots (10.22 GiB), 5376 of its 5376 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA2 runs layers 33-47, expert cache 3840 slots (7.79 GiB), 3840 of its 3840 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-11"
+ ]
+}

bench/results/2026-09-29-speed-0122/README.md

@@ -0,0 +1,32 @@
+# Speed with engine 0.1.22
+
+RTX 5070 12 GB (PCIe 5.0 x16), Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. The ready-made 0.1.22 engine, one-shot runs
+with what setup writes: `--prefill auto`, the shipped expert profile, `--expert-cache auto`, 8-bit KV above 4K, KV
+streaming (`--kv-resident 32768`) from 64K, MTP (`--spec 4 --spec-min-p 0.5`), greedy, 256 generated tokens, images
+off. The same code-agent prompts as `2026-09-28-speed-0114`. Every run: [`matrix.json`](matrix.json).
+
+## Prompt (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 519 | 1,226 | 1,844 | 1,836 | 1,682 | 1,304 |
+| **IQ2_XS** | 524 | 1,196 | 1,799 | 1,611 | 1,495 | 1,181 |
+| **IQ3_XXS** | 472 | 974 | 1,555 | 1,449 | 1,386 | - |
+| **IQ3_S** | 419 | 893 | 1,499 | 1,285 | 1,245 | - |
+| **Coder** | 660 | 1,522 | 1,871 | 1,938 | 1,779 | - |
+
+## Output (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 84.7 | 88.2 | 77.4 | 71.0 | 63.1 | 61.1 |
+| **IQ2_XS** | 77.2 | 74.6 | 67.5 | 63.5 | 59.1 | 54.7 |
+| **IQ3_XXS** | 65.1 | 62.3 | 49.8 | 53.6 | 47.0 | - |
+| **IQ3_S** | 50.5 | 44.6 | 49.0 | 48.8 | 41.3 | - |
+| **Coder** | 56.9 | 53.6 | 54.9 | 48.5 | 42.8 | - |
+
+- IQ2_XS was measured with Swift 1.5's IQ2_XS (the same speed as the original's); its 262K run had images on.
+- IQ3_XXS and IQ3_S at 262K do not fit 64 GB of RAM; the Coder at 262K was not run again (0.1.14: 1,034 tokens/s).
+- Against 0.1.14: prompts 1.2-1.5x faster at 32K-128K (the 0.1.22 prompt path: tensor-core attention, bit-identical
+  batched kernels, faster PLE reads). Output speed is the engine's same decode; one run per cell moves a few percent
+  with the text (speculative drafts), so compare it across many runs, not one.

bench/results/2026-09-29-speed-0122/matrix.json

@@ -0,0 +1,380 @@
+[
+ {
+  "model": "Q2_0",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 40.0,
+  "decode_tok_s": 84.71,
+  "decoded": 256,
+  "prefill_tok_s": 519.48,
+  "ttft_s": 3.62,
+  "spec_accept": 0.786,
+  "tokens_per_round": 2.77,
+  "vram_slots": 4176
+ },
+ {
+  "model": "Q2_0",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 41.6,
+  "decode_tok_s": 88.23,
+  "decoded": 256,
+  "prefill_tok_s": 1226.49,
+  "ttft_s": 4.43,
+  "spec_accept": 0.839,
+  "tokens_per_round": 3.24,
+  "vram_slots": 4123
+ },
+ {
+  "model": "Q2_0",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 56.6,
+  "decode_tok_s": 77.37,
+  "decoded": 256,
+  "prefill_tok_s": 1843.83,
+  "ttft_s": 18.63,
+  "spec_accept": 0.808,
+  "tokens_per_round": 3.04,
+  "vram_slots": 3833
+ },
+ {
+  "model": "Q2_0",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 72.9,
+  "decode_tok_s": 71.0,
+  "decoded": 256,
+  "prefill_tok_s": 1835.64,
+  "ttft_s": 36.48,
+  "spec_accept": 0.685,
+  "tokens_per_round": 2.77,
+  "vram_slots": 3780
+ },
+ {
+  "model": "Q2_0",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 118.1,
+  "decode_tok_s": 63.08,
+  "decoded": 256,
+  "prefill_tok_s": 1681.64,
+  "ttft_s": 77.92,
+  "spec_accept": 0.7,
+  "tokens_per_round": 2.56,
+  "vram_slots": 3697
+ },
+ {
+  "model": "Q2_0",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 242.0,
+  "decode_tok_s": 61.12,
+  "decoded": 256,
+  "prefill_tok_s": 1303.83,
+  "ttft_s": 200.94,
+  "spec_accept": 0.627,
+  "tokens_per_round": 2.5,
+  "vram_slots": 3525
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 263.6,
+  "decode_tok_s": 54.68,
+  "decoded": 256,
+  "prefill_tok_s": 1180.58,
+  "ttft_s": 221.91,
+  "spec_accept": 0.632,
+  "tokens_per_round": 2.49,
+  "vram_slots": 3727
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 56.0,
+  "decode_tok_s": 50.5,
+  "decoded": 256,
+  "prefill_tok_s": 419.48,
+  "ttft_s": 3.98,
+  "spec_accept": 0.811,
+  "tokens_per_round": 3.17,
+  "vram_slots": 2617
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 60.0,
+  "decode_tok_s": 44.58,
+  "decoded": 256,
+  "prefill_tok_s": 892.84,
+  "ttft_s": 5.53,
+  "spec_accept": 0.672,
+  "tokens_per_round": 2.69,
+  "vram_slots": 2580
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 75.6,
+  "decode_tok_s": 48.96,
+  "decoded": 256,
+  "prefill_tok_s": 1499.09,
+  "ttft_s": 22.65,
+  "spec_accept": 0.79,
+  "tokens_per_round": 2.9,
+  "vram_slots": 2383
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 102.1,
+  "decode_tok_s": 48.8,
+  "decoded": 256,
+  "prefill_tok_s": 1284.81,
+  "ttft_s": 51.58,
+  "spec_accept": 0.797,
+  "tokens_per_round": 2.94,
+  "vram_slots": 2281
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 159.2,
+  "decode_tok_s": 41.27,
+  "decoded": 256,
+  "prefill_tok_s": 1244.55,
+  "ttft_s": 104.81,
+  "spec_accept": 0.694,
+  "tokens_per_round": 2.73,
+  "vram_slots": 2292
+ },
+ {
+  "model": "Coder",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 34.5,
+  "decode_tok_s": 56.94,
+  "decoded": 256,
+  "prefill_tok_s": 660.44,
+  "ttft_s": 2.42,
+  "spec_accept": 0.74,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2650
+ },
+ {
+  "model": "Coder",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 18.8,
+  "decode_tok_s": 53.65,
+  "decoded": 256,
+  "prefill_tok_s": 1521.84,
+  "ttft_s": 3.21,
+  "spec_accept": 0.736,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2615
+ },
+ {
+  "model": "Coder",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 33.1,
+  "decode_tok_s": 54.85,
+  "decoded": 256,
+  "prefill_tok_s": 1870.76,
+  "ttft_s": 17.7,
+  "spec_accept": 0.806,
+  "tokens_per_round": 3.11,
+  "vram_slots": 2418
+ },
+ {
+  "model": "Coder",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 50.2,
+  "decode_tok_s": 48.46,
+  "decoded": 256,
+  "prefill_tok_s": 1937.69,
+  "ttft_s": 33.99,
+  "spec_accept": 0.653,
+  "tokens_per_round": 2.56,
+  "vram_slots": 2382
+ },
+ {
+  "model": "Coder",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 90.3,
+  "decode_tok_s": 42.75,
+  "decoded": 256,
+  "prefill_tok_s": 1778.66,
+  "ttft_s": 73.12,
+  "spec_accept": 0.588,
+  "tokens_per_round": 2.27,
+  "vram_slots": 2326
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 43.6,
+  "decode_tok_s": 77.16,
+  "decoded": 256,
+  "prefill_tok_s": 523.77,
+  "ttft_s": 3.46,
+  "spec_accept": 0.811,
+  "tokens_per_round": 3.02,
+  "vram_slots": 4361
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 43.7,
+  "decode_tok_s": 74.58,
+  "decoded": 256,
+  "prefill_tok_s": 1196.03,
+  "ttft_s": 4.47,
+  "spec_accept": 0.824,
+  "tokens_per_round": 3.36,
+  "vram_slots": 4309
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 59.6,
+  "decode_tok_s": 67.46,
+  "decoded": 256,
+  "prefill_tok_s": 1798.65,
+  "ttft_s": 19.06,
+  "spec_accept": 0.789,
+  "tokens_per_round": 3.02,
+  "vram_slots": 4030
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 81.4,
+  "decode_tok_s": 63.52,
+  "decoded": 256,
+  "prefill_tok_s": 1610.73,
+  "ttft_s": 41.37,
+  "spec_accept": 0.705,
+  "tokens_per_round": 2.6,
+  "vram_slots": 3978
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 126.0,
+  "decode_tok_s": 59.11,
+  "decoded": 256,
+  "prefill_tok_s": 1494.53,
+  "ttft_s": 87.51,
+  "spec_accept": 0.704,
+  "tokens_per_round": 2.71,
+  "vram_slots": 3897
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 48.6,
+  "decode_tok_s": 65.12,
+  "decoded": 256,
+  "prefill_tok_s": 471.53,
+  "ttft_s": 3.65,
+  "spec_accept": 0.876,
+  "tokens_per_round": 4.08,
+  "vram_slots": 3281
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 48.5,
+  "decode_tok_s": 62.33,
+  "decoded": 256,
+  "prefill_tok_s": 974.3,
+  "ttft_s": 5.17,
+  "spec_accept": 0.826,
+  "tokens_per_round": 3.15,
+  "vram_slots": 3243
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 64.9,
+  "decode_tok_s": 49.83,
+  "decoded": 256,
+  "prefill_tok_s": 1554.53,
+  "ttft_s": 21.84,
+  "spec_accept": 0.766,
+  "tokens_per_round": 2.91,
+  "vram_slots": 3009
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 91.7,
+  "decode_tok_s": 53.58,
+  "decoded": 256,
+  "prefill_tok_s": 1449.37,
+  "ttft_s": 45.84,
+  "spec_accept": 0.748,
+  "tokens_per_round": 2.8,
+  "vram_slots": 2965
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 140.9,
+  "decode_tok_s": 46.97,
+  "decoded": 256,
+  "prefill_tok_s": 1386.04,
+  "ttft_s": 94.33,
+  "spec_accept": 0.607,
+  "tokens_per_round": 2.47,
+  "vram_slots": 2901
+ }
+]

bench/results/2026-09-29-speed-0126/README.md

@@ -0,0 +1,36 @@
+# Speed with engine 0.1.26
+
+RTX 5070 12 GB (PCIe 5.0 x16), Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. The ready-made 0.1.26 engine, one-shot runs
+with what setup writes: `--prefill auto`, the shipped expert profile, `--expert-cache auto`, 8-bit KV above 4K, KV
+streaming (`--kv-resident 32768`) from 64K, MTP (`--spec 4 --spec-min-p 0.5`), greedy, 256 generated tokens, images
+off, no timing marks. The same code-agent prompts as `2026-09-29-speed-0122`. Every run: [`matrix.json`](matrix.json).
+
+## Prompt (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 536 | 1,299 | 2,171 | 2,126 | 2,107 |
+| **IQ2_XS** | 534 | 1,256 | 2,092 | 1,754 | 1,752 |
+| **IQ3_XXS** | 482 | 1,007 | 1,745 | 1,609 | 1,602 |
+| **IQ3_S** | 427 | 913 | 1,624 | 1,640 | 1,443 |
+| **Coder** | 656 | 1,583 | 2,177 | 2,236 | 2,208 |
+
+## Output (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 87.3 | 93.0 | 81.8 | 76.2 | 73.7 |
+| **IQ2_XS** | 79.6 | 78.6 | 76.3 | 63.7 | 62.7 |
+| **IQ3_XXS** | 61.9 | 61.6 | 58.5 | 57.2 | 49.0 |
+| **IQ3_S** | 52.4 | 53.3 | 48.3 | 46.3 | 45.5 |
+| **Coder** | 58.9 | 55.1 | 54.9 | 53.2 | 43.0 |
+
+- Against 0.1.22 (`2026-09-29-speed-0122`), prompts at 32K-128K are 8-28% faster: Q2_0 1,844 -> 2,171 at 32K and
+  1,682 -> 2,107 at 128K, the Coder 1,779 -> 2,208 at 128K. The gains come from:
+  - 0.1.24's QSA selection on tensor cores;
+  - 0.1.25's grouping tables in mapped memory and fused norms;
+  - 0.1.26's batched draft-layer pass.
+- 262K was not run again (0.1.22: Q2_0 1,304, IQ2_XS 1,181 tokens/s).
+- Output speed is the same decode as before. One run per cell moves several percent with the text (the share of
+  accepted drafts, `spec_accept` in `matrix.json`). For example, the Coder's 128K answer accepted 55% of its drafts
+  and its 64K answer 71%.

bench/results/2026-09-29-speed-0126/matrix.json

@@ -0,0 +1,352 @@
+[
+ {
+  "model": "Q2_0",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 23.9,
+  "decode_tok_s": 87.33,
+  "decoded": 256,
+  "prefill_tok_s": 536.42,
+  "ttft_s": 3.4,
+  "spec_accept": 0.681,
+  "tokens_per_round": 2.72,
+  "vram_slots": 4174
+ },
+ {
+  "model": "Q2_0",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 27.6,
+  "decode_tok_s": 92.97,
+  "decoded": 256,
+  "prefill_tok_s": 1299.44,
+  "ttft_s": 4.21,
+  "spec_accept": 0.838,
+  "tokens_per_round": 3.23,
+  "vram_slots": 4121
+ },
+ {
+  "model": "Q2_0",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 40.8,
+  "decode_tok_s": 81.75,
+  "decoded": 256,
+  "prefill_tok_s": 2170.85,
+  "ttft_s": 16.02,
+  "spec_accept": 0.788,
+  "tokens_per_round": 2.99,
+  "vram_slots": 3832
+ },
+ {
+  "model": "Q2_0",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 57.3,
+  "decode_tok_s": 76.23,
+  "decoded": 256,
+  "prefill_tok_s": 2126.03,
+  "ttft_s": 32.06,
+  "spec_accept": 0.707,
+  "tokens_per_round": 2.8,
+  "vram_slots": 3778
+ },
+ {
+  "model": "Q2_0",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 87.7,
+  "decode_tok_s": 73.73,
+  "decoded": 256,
+  "prefill_tok_s": 2106.85,
+  "ttft_s": 62.48,
+  "spec_accept": 0.679,
+  "tokens_per_round": 2.71,
+  "vram_slots": 3695
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 42.2,
+  "decode_tok_s": 79.62,
+  "decoded": 256,
+  "prefill_tok_s": 533.83,
+  "ttft_s": 3.51,
+  "spec_accept": 0.774,
+  "tokens_per_round": 2.89,
+  "vram_slots": 4359
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 40.6,
+  "decode_tok_s": 78.6,
+  "decoded": 256,
+  "prefill_tok_s": 1256.02,
+  "ttft_s": 4.31,
+  "spec_accept": 0.81,
+  "tokens_per_round": 3.29,
+  "vram_slots": 4308
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 53.6,
+  "decode_tok_s": 76.34,
+  "decoded": 256,
+  "prefill_tok_s": 2092.02,
+  "ttft_s": 16.57,
+  "spec_accept": 0.85,
+  "tokens_per_round": 3.2,
+  "vram_slots": 4029
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 78.3,
+  "decode_tok_s": 63.69,
+  "decoded": 256,
+  "prefill_tok_s": 1753.65,
+  "ttft_s": 38.08,
+  "spec_accept": 0.636,
+  "tokens_per_round": 2.59,
+  "vram_slots": 3977
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 115.1,
+  "decode_tok_s": 62.72,
+  "decoded": 256,
+  "prefill_tok_s": 1751.79,
+  "ttft_s": 74.85,
+  "spec_accept": 0.696,
+  "tokens_per_round": 2.6,
+  "vram_slots": 3896
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 46.9,
+  "decode_tok_s": 61.94,
+  "decoded": 256,
+  "prefill_tok_s": 482.04,
+  "ttft_s": 3.67,
+  "spec_accept": 0.845,
+  "tokens_per_round": 3.57,
+  "vram_slots": 3279
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 47.4,
+  "decode_tok_s": 61.64,
+  "decoded": 256,
+  "prefill_tok_s": 1007.48,
+  "ttft_s": 5.03,
+  "spec_accept": 0.801,
+  "tokens_per_round": 3.19,
+  "vram_slots": 3242
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 65.6,
+  "decode_tok_s": 58.49,
+  "decoded": 256,
+  "prefill_tok_s": 1745.16,
+  "ttft_s": 19.63,
+  "spec_accept": 0.789,
+  "tokens_per_round": 2.85,
+  "vram_slots": 3007
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 88.5,
+  "decode_tok_s": 57.21,
+  "decoded": 256,
+  "prefill_tok_s": 1609.31,
+  "ttft_s": 41.45,
+  "spec_accept": 0.74,
+  "tokens_per_round": 2.87,
+  "vram_slots": 2963
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 127.0,
+  "decode_tok_s": 48.95,
+  "decoded": 256,
+  "prefill_tok_s": 1602.33,
+  "ttft_s": 81.71,
+  "spec_accept": 0.643,
+  "tokens_per_round": 2.37,
+  "vram_slots": 2899
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 54.2,
+  "decode_tok_s": 52.4,
+  "decoded": 256,
+  "prefill_tok_s": 426.6,
+  "ttft_s": 4.03,
+  "spec_accept": 0.822,
+  "tokens_per_round": 3.17,
+  "vram_slots": 2617
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 58.9,
+  "decode_tok_s": 53.3,
+  "decoded": 256,
+  "prefill_tok_s": 912.99,
+  "ttft_s": 5.58,
+  "spec_accept": 0.85,
+  "tokens_per_round": 3.38,
+  "vram_slots": 2579
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 72.8,
+  "decode_tok_s": 48.31,
+  "decoded": 256,
+  "prefill_tok_s": 1624.07,
+  "ttft_s": 21.07,
+  "spec_accept": 0.827,
+  "tokens_per_round": 3.0,
+  "vram_slots": 2383
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 93.3,
+  "decode_tok_s": 46.25,
+  "decoded": 256,
+  "prefill_tok_s": 1640.34,
+  "ttft_s": 40.77,
+  "spec_accept": 0.693,
+  "tokens_per_round": 2.56,
+  "vram_slots": 2346
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 144.6,
+  "decode_tok_s": 45.51,
+  "decoded": 256,
+  "prefill_tok_s": 1443.21,
+  "ttft_s": 90.72,
+  "spec_accept": 0.697,
+  "tokens_per_round": 2.73,
+  "vram_slots": 2291
+ },
+ {
+  "model": "Coder",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 37.0,
+  "decode_tok_s": 58.91,
+  "decoded": 256,
+  "prefill_tok_s": 656.36,
+  "ttft_s": 2.48,
+  "spec_accept": 0.743,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2649
+ },
+ {
+  "model": "Coder",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 18.3,
+  "decode_tok_s": 55.1,
+  "decoded": 256,
+  "prefill_tok_s": 1583.1,
+  "ttft_s": 3.09,
+  "spec_accept": 0.713,
+  "tokens_per_round": 2.88,
+  "vram_slots": 2614
+ },
+ {
+  "model": "Coder",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 30.4,
+  "decode_tok_s": 54.89,
+  "decoded": 256,
+  "prefill_tok_s": 2177.17,
+  "ttft_s": 15.33,
+  "spec_accept": 0.731,
+  "tokens_per_round": 2.75,
+  "vram_slots": 2417
+ },
+ {
+  "model": "Coder",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 44.8,
+  "decode_tok_s": 53.22,
+  "decoded": 256,
+  "prefill_tok_s": 2236.49,
+  "ttft_s": 29.51,
+  "spec_accept": 0.709,
+  "tokens_per_round": 2.68,
+  "vram_slots": 2382
+ },
+ {
+  "model": "Coder",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 75.9,
+  "decode_tok_s": 43.04,
+  "decoded": 256,
+  "prefill_tok_s": 2208.27,
+  "ttft_s": 59.02,
+  "spec_accept": 0.553,
+  "tokens_per_round": 2.23,
+  "vram_slots": 2324
+ }
+]

cmake/hip_backend.cmake

@@ -0,0 +1,58 @@
+# Opt-in HIP configuration. Strata's CUDA-shaped kernels currently target RDNA3 gfx1100 wave32.
+# CMake/compiler discovery stays machine-independent; pass CMAKE_HIP_COMPILER when it is not on PATH.
+if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "")
+  set(CMAKE_HIP_ARCHITECTURES gfx1100 CACHE STRING "Strata HIP target architecture")
+endif()
+if(NOT CMAKE_HIP_ARCHITECTURES STREQUAL "gfx1100")
+  message(FATAL_ERROR
+    "Strata HIP currently supports only gfx1100 wave32; CMAKE_HIP_ARCHITECTURES is '${CMAKE_HIP_ARCHITECTURES}'")
+endif()
+
+enable_language(HIP)
+find_package(hip CONFIG REQUIRED)
+find_package(hipblas CONFIG REQUIRED)
+find_package(hipblaslt CONFIG QUIET)
+
+if(NOT TARGET hip::host)
+  message(FATAL_ERROR "The ROCm hip CMake package did not provide hip::host")
+endif()
+if(NOT TARGET roc::hipblas)
+  message(FATAL_ERROR "The ROCm hipblas CMake package did not provide roc::hipblas")
+endif()
+if(TARGET roc::hipblaslt)
+  set(STRATA_HIPBLASLT_AVAILABLE ON)
+else()
+  set(STRATA_HIPBLASLT_AVAILABLE OFF)
+  message(STATUS "Strata: hipBLASLt not found; solution-table dispatch is unavailable")
+endif()
+
+# HIP's link step produces a PIE; make Strata and ggml objects PIC for the ROCm linker.
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(STRATA_HIP_COMPAT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/strata/hip_compat")
+add_library(strata_hip_runtime INTERFACE)
+target_include_directories(strata_hip_runtime BEFORE INTERFACE
+  "${STRATA_HIP_COMPAT_INCLUDE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/include")
+target_compile_definitions(strata_hip_runtime INTERFACE STRATA_USE_HIP=1)
+target_link_libraries(strata_hip_runtime INTERFACE hip::host)
+foreach(_language IN ITEMS CXX HIP)
+  target_compile_options(strata_hip_runtime INTERFACE
+    "$<$<COMPILE_LANGUAGE:${_language}>:-include>"
+    "$<$<COMPILE_LANGUAGE:${_language}>:${STRATA_HIP_COMPAT_INCLUDE_DIR}/cuda_runtime.h>")
+endforeach()
+
+# CMake does not infer HIP from Strata's existing CUDA-shaped .cu suffixes.
+file(GLOB_RECURSE _strata_hip_sources CONFIGURE_DEPENDS
+  "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cu"
+  "${CMAKE_CURRENT_SOURCE_DIR}/bench/*.cu"
+  "${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cu")
+if(_strata_hip_sources)
+  set_source_files_properties(${_strata_hip_sources} PROPERTIES LANGUAGE HIP)
+endif()
+foreach(_source IN ITEMS tests/hip/intrinsics.cpp tests/hip/native_qsa_score.cpp)
+  if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${_source}")
+    set_source_files_properties("${_source}" PROPERTIES LANGUAGE HIP)
+  endif()
+endforeach()
+
+message(STATUS "Strata: HIP enabled, arch ${CMAKE_HIP_ARCHITECTURES}")

data/experimental-speed-projection/Qwen3.8-Flash-Next-experimental-speed-projection.json

@@ -0,0 +1,26 @@
+{
+ "mode": "project",
+ "dir": "per-layer",
+ "scale": 1.0,
+ "layer_range": [
+  4,
+  44
+ ],
+ "refusal_harmful_thinkoff": 2.0,
+ "refusal_harmful_thinkon": 2.0,
+ "refusal_harmless_thinkoff": 4.0,
+ "refusal_harmless_thinkon": 6.0,
+ "broken": 0,
+ "kl_mean": 0.18598,
+ "kl_max": 18.26549,
+ "kl_p99": 3.143571,
+ "decode_tps": 51.19,
+ "vector_file": "Qwen3.8-Flash-Next-experimental-speed-projection.gguf",
+ "base_model_gguf": "unsloth/Qwen3.8-Flash-Next-GGUF UD-Q2_K_XL",
+ "base_model_sha256_shard1": "a4f3b21e77353999829f2f767e9ac21ce9c71d29a74f2cc9eda48c9bf23c8b86",
+ "worktree_commit": "37e026e3b",
+ "criterion_status": "PASS",
+ "dirty_direction": false,
+ "note": "projection ablation h -= s*(h.v)*v, v unit-normalized per layer. dir=per-layer",
+ "branch": "cvec-hooks (github.com/Cudila/llama.cpp)"
+}

data/experimental-speed-projection/README.md

@@ -0,0 +1,21 @@
+# experimental-speed-projection (EXPERIMENTAL, off by default)
+
+`Qwen3.8-Flash-Next-experimental-speed-projection.gguf` is a 480 KB control vector for Qwen3.8-Flash-Next: one unit
+direction of 2,560 values per layer (`direction.1` .. `direction.47`, GGUF `controlvector` format). The JSON beside it
+is the configuration its authors selected and their scores.
+
+**What it does.** Turned on, Strata removes each direction from the residual stream after layers 4-44
+(`h -= (h . v) v` on every hyper-connection stream), the same operation as llama.cpp's `--cvec-mode project` in the
+package's patches. The package's own documentation describes the vector as a **refusal-direction projection**: the
+model declines far fewer requests (it reports 1 of 50 vs 50 of 50 on its test set), and removing refusals removes a
+safety behaviour - you are responsible for what the model writes with it on. It also shifts the model's predictions
+slightly on ordinary text (measured here: `bench/results/2026-09-27-esp/`). It is not an optimization in the engine:
+on the same text it costs 0.2-0.4% per token; a chat's speed with it on depends on the text the model writes.
+
+**Turning it on.** `START-HERE.bat --setup` asks (default: off), or pass `--experimental-speed-projection on`. With it
+loaded, the web app's Sampling drawer and the API field `"experimental_speed_projection": false` switch it off per
+request. See `docs/DETAILS.md`, "Experimental speed projection".
+
+**Origin and license.** A local package of the vector (the published original uses filenames containing "refusal";
+the bytes are the same), made from Qwen3.8-Flash-Next activations, so the Qwen Community License 1.0 of the model
+applies to it. Made for the original Qwen3.8-Flash-Next (not Swift 1.5).

docs/AMD_HIP.md

@@ -0,0 +1,156 @@
+# Experimental AMD HIP backend (gfx1100)
+
+This is a manual Linux source build for the RX 7900 XTX. It is opt-in; the
+NVIDIA installer and CUDA build remain the default. Other AMD architectures,
+wave64, Windows HIP, and mixed AMD/NVIDIA execution are outside this contribution.
+
+The backend maps the CUDA-shaped runtime and BLAS calls to HIP/hipBLAS, uses
+RDNA3's signed integer dot instruction for quantized kernels, and supplies
+wave32 shuffle/packed-byte operations. CUDA-only QSA matrix instructions have
+an ordered FP32 fallback. Prefill supports both dequantization plus hipBLAS GEMM and opt-in HIP ggml MMQ.
+An optional, calibrated hipBLASLt path accelerates dense projections on gfx1100.
+This does not claim bit-identical model answers across backends. See
+[performance settings and evidence](AMD_HIP_PERFORMANCE.md).
+
+## Install with setup (recommended)
+
+On Linux with an RX 7900 XT / XTX and the kernel's amdgpu driver (no ROCm install needed):
+
+```sh
+./setup.sh --backend hip
+```
+
+- **Detection:** setup finds the card through the kernel's KFD topology. Integrated Radeon GPUs are listed as not
+  supported. On a PC without an NVIDIA card Strata can use, `--backend hip` is chosen automatically.
+- **ROCm:** installed into `.venv` from AMD's TheRock wheels (~10 GB, no sudo), pinned to the version this backend was
+  tested with (`STRATA_ROCM_VERSION` / `STRATA_ROCM_INDEX` override it). A system ROCm in `/opt/rocm` (or
+  `$ROCM_PATH`) with hipcc and hipBLAS is used instead when present.
+- **Engine:** compiled on your PC (10-20 minutes, once; again after a `git pull` that changes it). This needs a C++
+  compiler and git (`sudo apt install build-essential git`).
+- **Limits for now:** one GPU, no images, no calibration. The Monitor shows no GPU statistics.
+
+The rest of setup is the same as on NVIDIA: the model download, the start script, the server.
+
+## Build
+
+Requirements: a working ROCm driver/runtime, HIP development headers and
+compiler, hipBLAS and (for tuned dense prefill) hipBLASLt development files, CMake 3.24+, a C++20 host compiler, and Git.
+The model still needs sufficient system RAM and fast SSD storage for its PLE
+table. VRAM occupancy alone is not a throughput measurement.
+
+```sh
+cmake -S . -B build-hip \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DSTRATA_ENABLE_HIP=ON -DSTRATA_ENABLE_CUDA=OFF \
+  -DCMAKE_HIP_ARCHITECTURES=gfx1100
+cmake --build build-hip --target strata -j2
+```
+
+If CMake cannot find the HIP compiler, add
+`-DCMAKE_HIP_COMPILER=/path/to/rocm/llvm/bin/clang++`. On the Fedora-family test
+host this was `/usr/lib64/rocm/llvm/bin/clang++`. Use the compiler and libraries
+from the same ROCm installation. Do not enable both GPU backends in one build.
+
+Native IQ experts are enabled by default. CMake fetches the llama.cpp revision
+pinned by this repository. For an offline build, point `STRATA_GGML_DIR` at a
+checkout of that exact revision; using an arbitrary newer checkout changes the
+dependency being tested.
+
+## Model and serving configuration
+
+Prepare a supported model pack and its MTP runtime using the existing tools.
+For OrcaRouter IQ3_XXS, follow [the explicit compatibility conversion](ORCA.md);
+this backend does not change quantization, model licenses, or tokenizers.
+Add `--experts-bin` to the `tools/iq_pack.py` command: mmap requires the pack's
+`experts.bin`, which the default native packing command does not emit. This
+consumes additional disk space. Original GSQ-RCO IQ3_XXS uses shard 2 for PLE;
+Orca uses shard 1. Keep each model's own pack and tokenizer together.
+
+Use `build-hip/strata` as the executable in the server JSON. Select the AMD
+device with `ROCR_VISIBLE_DEVICES`/`HIP_VISIBLE_DEVICES` if necessary. Start with
+a modest context and prefill chunk size before measuring larger workloads.
+Keep the PLE file on an SSD. Do not assume a configured context length proves
+successful full-window inference.
+
+Large pinned host allocations can fail on ROCm even when ordinary RAM is
+available. The existing `--mmap-experts` path avoids allocating the full pinned
+expert arena; it still depends on OS file-cache residency and may stall on
+storage reads. It does not make SSD access equivalent to RAM.
+
+Starting args for the original GSQ-RCO IQ3_XXS model (replace the paths):
+
+```sh
+build-hip/strata --serve \
+  --pack packs/iq3xxs \
+  --native /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf \
+  --ple-gguf /path/to/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00002-of-00002.gguf \
+  --mmap-experts --expert-profile data/expert-profile.bin --expert-cache auto \
+  --prefill 512 --spec 4 --spec-min-p 0.5 --mtp mtp/rt \
+  --max-context 4096 --kv int8 --pool-workers 15 \
+  --adapt-every 0 --pcie-frac 0 --vram-reserve-mib 1024
+```
+
+`--serve` is the engine's internal token protocol. For a browser or OpenAI API,
+put these args in the `args` array of the [server JSON example](ORCA.md#local-server),
+set `exe` to `build-hip/strata`, and use the matching pack's `tokenizer` directory.
+Launch with `.venv/bin/python -m serve.server --engine strata --config strata-hip.json --port 8080`.
+The worker count above was used on a 16-core CPU; measure it for your CPU.
+The 4K context is a smoke-test starting point, not a model limit. The expert cache
+sizes itself automatically and leaves 1 GiB of VRAM headroom.
+
+The installer supports this backend (see "Install with setup" above). The vision helper and multi-GPU layer
+splits are NVIDIA-only for now.
+
+## Original backend validation (PR #94)
+
+The following is historical validation of the original backend, not a fresh
+test count for this replacement. Current build/test evidence and performance
+limits are recorded in [AMD_HIP_PERFORMANCE.md](AMD_HIP_PERFORMANCE.md).
+
+Tested against upstream `c1e903310f211e6630780c3bd2038778c071c68d` (0.1.20),
+with pinned llama.cpp `3cf03257f219afbe7334045ff7c6a06ac68c627d`.
+The host has an RX 7900 XTX (24 GiB), Ryzen 9 7950X3D, 64 GiB system RAM,
+and Fedora-family Linux with ROCm 7.1.52802 / Clang 20. The original model used
+for the smoke check was on mechanical RAID0; cold page faults were slow. Keep
+latency-sensitive data on SSD and measure warm residency separately.
+
+- Complete HIP Release build, including the executable and parity targets.
+- All 25 tests selected by the command below passed on 2026-09-29.
+- Intrinsic parity includes signed packed-byte dot products and overflow.
+- CPU/GPU mapped-memory handoff covers delayed publication, changing payloads,
+  and repeated graph replay in both directions.
+- GR coverage includes the maximum eight-token fused batch and graph replay
+  after inputs change. Native QSA checks the scalar HIP fallback.
+- Q8_K quantization is byte-exact against the existing reference. HIP's
+  `__fmul_rn` can become ordinary multiplication; disabling contraction in
+  `quantize_act.cu` preserves the separately rounded product before the magic
+  rounding bias is added. CUDA's compile flags remain unchanged.
+- The mmap regression covers canonical and differing native layer sizes,
+  truncated/oversized files, lookup bounds, and reopening after errors.
+- CUDA 13 / GCC 15 / sm_89 executable build passed; GR, sampler, quantization,
+  and mmap tests passed on the same tree. This is a regression check, not a
+  claim of complete CUDA inference validation.
+- CPU-only `strata-plan` build passed without either GPU backend.
+- Real GSQ-RCO IQ3_XXS server smoke with mmap, the shipped expert profile,
+  automatic GPU cache, and MTP passed arithmetic, executable Python addition,
+  system-marker recall, and 1,170-token batched-prefill recall. All four requests
+  ended normally with correct answers; one engine process served them all.
+  The cache held 11,142 experts (18.08 GiB), with 874 MiB of VRAM free after
+  startup. These short checks do not establish a throughput or quality benchmark.
+
+Build all targets before running the registered focused checks:
+
+```sh
+cmake --build build-hip -j2
+ctest --test-dir build-hip --output-on-failure --timeout 60 \
+  -E '^(ple_parity|platform_memory_test)$'
+```
+
+`ple_parity` requires an external model fixture. `platform_memory_test` requests
+256 MiB of locked memory; the test shell's 8 MiB memlock limit prevented that
+test from passing. These are explicit exclusions, not skipped tests counted as
+passes. Additional unpublished upstream fixture suites are not covered.
+
+The published performance table explicitly identifies the measured development
+snapshot; it must not be read as a benchmark of every subsequent rebase. Vision, long-context stress, broad answer-quality
+equivalence, other AMD cards, and mixed-vendor inference are not validated here.

docs/AMD_HIP_PERFORMANCE.md

@@ -0,0 +1,142 @@
+# RX 7900 XTX support and performance evidence
+
+This opt-in Linux `gfx1100` backend supersedes the initial support in
+[PR #94](https://github.com/Niko1221/Strata/pull/94). It retains HIP runtime and
+wave32 integer-dot compatibility, native mmap layout validation, and MTP, then
+adds HIP MMQ, optional calibrated dense hipBLASLt GEMM, native prefill batching,
+and the host-memory / SSD paths used by the measured configuration. HIP blocking
+expert uploads use one reusable pinned staging buffer, including prompt-cache
+refills, to avoid repeated pageable-source registrations.
+
+## Reproduce the configuration
+
+Build instructions are in [AMD_HIP.md](AMD_HIP.md). Use the repository-pinned
+llama.cpp dependency; do not silently substitute another revision. Build with
+`STRATA_PREFILL_MMQ=ON` and use these runtime variables for the measured arm:
+
+```sh
+export STRATA_PREFILL_MMQ=1
+export STRATA_HIPBLASLT_TUNING="$PWD/tools/hip/gfx1100-hipblaslt-100100.txt"
+export STRATA_PREFILL_RING=96
+export STRATA_IO_THREADS=32
+```
+
+The supplied table is calibrated for gfx1100 and hipBLASLt version 100100.
+It is not a universal ROCm tuning table. The runtime guards architecture,
+library version and actual shape/stride/workspace requirements, falling back
+when a table entry is unavailable or incompatible. HIP MMQ is opt-in at runtime. Default CUDA selection is preserved.
+
+Measured engine configuration: Orca Flash Next IQ3_XXS, native pack plus matching
+GGUF/tokenizer/template, `--mmap-experts --resident-cpu-experts`, fixed ranked
+`--expert-profile`, `--expert-cache auto`, `--prefill 8192`, `--spec 4`,
+`--spec-min-p 0.5`, matching MTP runtime, `--max-context 262144`, `--kv int8`,
+`--kv-resident 32768`, `--pool-workers 15`, `--adapt-every 0`, `--pcie-frac 0`,
+and `--vram-reserve-mib 1024`. Keep PLE on SSD.
+
+`--resident-cpu-experts` copies the complement of the static GPU cache to ordinary
+RAM. It requires mmap, a fixed expert profile, and disabled adaptation. It does
+not lock all data into RAM: temporary prefill borrowing and cache refill may
+still use the mapped fallback. Allocation needs sufficient available RAM;
+on Linux this option requires readable standard cgroup-v2 mounts. For each
+finite cgroup ancestor, the guard credits only `inactive_file` after subtracting
+`file_dirty` and `file_writeback`, capped by current usage; it remains bounded by
+the ancestor limit and host `MemAvailable`, with 8 GiB headroom. This accounts
+for reclaimable clean file cache but cannot reserve memory against concurrent
+system or process allocations.
+The POSIX PLE path issues direct reads through a configurable worker pool.
+
+Use a dedicated idle server, restart it between arms, and capture its engine log:
+
+```sh
+python3 tools/hip/bench_prefill.py \
+  --model MODEL_NAME --url http://127.0.0.1:8080 \
+  --engine-log /path/to/engine.log --label candidate --output candidate.json
+```
+
+The script uses the same synthetic source prompts and request order as the
+reported measurements: a small warmup, then 140/280/140/280 functions, with one
+follow-up after each fresh prompt. It enforces zero reused tokens for the fresh
+prompts and records prefill, decode, wall time and stop reason. It requires the
+unbuffered log to contain exactly one completed timing record per request. The
+128-token output cap is intentional for throughput measurement, not task success.
+Never use cancelled-request timing lines as throughput evidence.
+
+## Final revision benchmark
+
+Measured on 2026-09-29, source revision `9568e78da663e6b224ab96d57baf450717b36e73`.
+HIP executable SHA-256: `705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b`.
+The control and candidate below were both measured anew in this session; no
+historical benchmark values are mixed into the table. The control binary SHA-256
+is `4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7`;
+its source was an existing local runtime snapshot rather than a clean upstream commit.
+Both arms use an RX 7900 XTX 24 GiB / gfx1100, Ryzen 9 7950X3D,
+64 GiB installed RAM, Fedora-family Linux, ROCm 7.1 and a 272 W GPU cap.
+Other model services are stopped for measurement and restored afterward.
+Sampling: temperature 0, top-k 1, top-p 1, min-p 0, seed 42, reasoning disabled.
+
+| Fresh request, execution order | Control prefill t/s | Candidate prefill t/s | Candidate output t/s | Candidate request wall time |
+| --- | ---: | ---: | ---: | ---: |
+| 4,210 tokens, first use | 240.0 | 447.5 | 55.5 | 11.729 s |
+| 8,830 tokens, first use at this size | 484.8 | 873.7 | 55.7 | 12.420 s |
+| 4,210 tokens, warmed | 457.2 | 901.1 | 56.7 | 6.937 s |
+| 8,830 tokens, warmed | 478.9 | 966.5 | 59.2 | 11.314 s |
+
+Each request generated exactly 128 tokens and finished at the intentional length
+cap. All four fresh prompts had zero reused KV tokens. Fresh-prompt prefill was
+1.80–2.02x the freshly measured existing AMD runtime; fresh-request wall time was
+40–46% lower. This is a comparison with our existing HIP runtime, not unmodified
+upstream, which does not provide this backend. The candidate also completed four
+cached follow-ups in 2.95–3.26 seconds. Those follow-ups are not apples-to-apples
+prefill comparisons: cache reuse differs with generated text and checkpoint
+selection. Full sanitized measurements are in
+[the fresh-run JSON](benchmarks/2026-09-29-gfx1100.json).
+
+Before this revision, an intermediate package missing the HIP upload staging
+buffer stalled on the 8,830-token request and was rejected. Restoring staging
+allowed the complete sequence above to finish. That failed attempt is not
+included in the throughput table. No watchdog limit was raised to obtain these
+results.
+
+Zero KV reuse is not equivalent to a cold filesystem cache. First-use speed
+and warmed speed are reported separately. Individual observations do not
+establish confidence intervals or a general rate at every context length.
+
+## Correctness and quality limits
+
+Both HIP and CUDA executables built from the measured source revision. CUDA was
+compile-checked, not performance-tested by this run. The selected HIP CTest suite
+passed **29/29** with the tuning table enabled, including expert-upload readback,
+asynchronous handoff, QSA, MMQ, Lt GEMM, KV streaming and PLE reading. The excluded
+`ple_parity` requires an external fixture; `platform_memory_test` requires a
+larger locked-memory limit than the test account provides. Separately, three
+real IQ3_XXS PLE matrix graph replays passed, and the POSIX direct-file test passed
+queued reads, short EOF, wake, close/drain and reopen checks.
+
+Development checks passed eight MMQ numerical comparisons, four actual Lt GEMM
+comparisons, and native QSA/indexer/embedding parity including tail states,
+chunk continuation and image overrides. Real quantized MMQ relative L2 error
+was approximately 0.0026 against raw-FP32/dequantized-weight reference, reflecting
+Q8 activation arithmetic; this is not bitwise numerical equivalence.
+
+End-to-end coding quality is not established by these numerical checks or capped
+throughput requests. Validate completed tasks with independent runtime tests; this
+contribution makes no broad answer-quality or agentic-reliability claim.
+
+The changes do not claim better model reasoning, verified full-context behavior,
+end-to-end vision validation, Windows HIP, other AMD architectures, or mixed
+AMD/NVIDIA execution. Existing CUDA multi-GPU code remains present but is not
+validation of HIP multi-GPU support.
+
+## Attribution and rejected experiments
+
+The measurements above were taken before this backend was rebased onto engine 0.1.24. The PR's own batched
+embedding gather and QSA indexer append (adapted from
+[PR #108](https://github.com/Niko1221/Strata/pull/108), commit
+`acd487233c0bbe2217a6881c5bb43f8a283b0de5`) were dropped in the rebase: 0.1.24 already does both on every
+backend, bit-exact (C-2, C-4). #108's optional GDN parallel/split path is not included. Existing upstream MMQ orchestration is retained and enabled for
+HIP with AMD architecture identification and the correct backend compilation.
+
+Static 16K chunks, expanded FP16 expert tuning, and a 192-slot ring did not offer
+a consistent end-to-end win in the evaluated workload. The table includes only
+the selected 30 dense-shape rows. Microkernel speedups alone were not sufficient
+to select a configuration. No increase in GPU power cap was used.

docs/DETAILS.md

@@ -4,33 +4,50 @@ The technical side of Strata: every measured number, the API, images, all settin
 New here? Start with the [README](../README.md) - it has everything you need to install and use it.
 
 > **On this page:** [Speed](#speed-measured) · [Other GPUs](#other-gpus-estimated) · [Which model?](#which-model) ·
-> [Requirements](#before-you-start) · [Windows](#windows) · [Linux](#linux) · [API](#using-it) · [Images](#images-vision) ·
+> [Requirements](#before-you-start) · [Windows](#windows) · [Linux](#linux) · [API](#using-it) ·
+> [MCP tools](#tools-from-mcp-servers) · [Images](#images-vision) ·
 > [Troubleshooting](#troubleshooting) · [How it works](#how-it-works)
 
 ---
 
 ## Speed (measured)
 
-RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows. One code-agent prompt per length, 256 generated
-tokens, MTP speculative decoding on. "262K" is the model's full context window (a 259,943-token prompt).
+RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1.26 with the settings setup writes
+(`--prefill auto`, 8-bit KV above 4K, KV streaming from 64K). One code-agent prompt per length, 256 generated tokens,
+MTP speculative decoding on. "262K" is the model's full context window (a 259,943-token prompt). The IQ2_XS row was
+measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
 
 ### Prompt processing (tokens/s)
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| **Q2_0** | 389 | 539 | 571 | 561 | 543 | 496 |
-| **IQ2_XS** | 332 | 463 | 495 | 486 | 472 | 437 |
-| **IQ3_XXS** | 285 | 410 | 435 | 427 | 414 | - |
-| **IQ3_S** | 260 | 374 | 397 | - | 378 | - |
+| **Q2_0** | 536 | 1,299 | 2,171 | 2,126 | 2,107 | 1,304† |
+| **IQ2_XS** | 534 | 1,256 | 2,092 | 1,754 | 1,752 | 1,181*† |
+| **IQ3_XXS** | 482 | 1,007 | 1,745 | 1,609 | 1,602 | - |
+| **IQ3_S** | 427 | 913 | 1,624 | 1,640 | 1,443 | - |
+| **Coder** | 656 | 1,583 | 2,177 | 2,236 | 2,208 | 1,034** |
+
+Engine 0.1.26; `bench/results/2026-09-29-speed-0126`. At 32K-128K that is 8-28% faster than 0.1.22. † not measured
+again: 0.1.22. \* measured with images on (the image encoder's VRAM reserve leaves fewer experts cached). \*\* not
+measured again: 0.1.14.
 
 ### Output (tokens/s)
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| **Q2_0** | 88.7 | 94.6 | 87.5 | 76.4 | 65.1 | 56.3 |
-| **IQ2_XS** | 82.0 | 78.0 | 65.3 | 63.7 | 52.0 | 48.0 |
-| **IQ3_XXS** | 64.6 | 65.6 | 57.3 | 54.4 | 44.8 | - |
-| **IQ3_S** | 51.2 | 54.4 | 51.1 | - | 42.2 | - |
+| **Q2_0** | 87.3 | 93.0 | 81.8 | 76.2 | 73.7 | 60.3† |
+| **IQ2_XS** | 79.6 | 78.6 | 76.3 | 63.7 | 62.7 | 52.8† |
+| **IQ3_XXS** | 61.9 | 61.6 | 58.5 | 57.2 | 49.0 | - |
+| **IQ3_S** | 52.4 | 53.3 | 48.3 | 46.3 | 45.5 | - |
+| **Coder** | 58.9 | 55.1 | 54.9 | 53.2 | 43.0 | 42.8† |
+
+Engine 0.1.26, the same runs. † not measured again: 0.1.14.
+
+Output speed depends on the text as well: speculative decoding runs faster when more of the drafted tokens are
+accepted, so a different answer to the same prompt moves it by several percent. Run back to back on the 4K prompt,
+0.1.14 writes 88.5 tokens/s and 0.1.12 85.7. The numbers before 0.1.13 (prompts about half as fast):
+[`bench/results/2026-09-24-final`](../bench/results/2026-09-24-final/matrix.md); these:
+[`bench/results/2026-09-28-speed-0114`](../bench/results/2026-09-28-speed-0114/README.md).
 
 IQ3_XXS and IQ3_S at 262K are not measured: with their 43 / 50 GB of experts, a 260K-token context brings a 64 GB PC
 to its memory limit. Use up to 128K with them on 64 GB (setup caps it). IQ3_S (engine 0.1.4 or newer) is only published
@@ -42,13 +59,54 @@ attention reads in VRAM (`--kv-resident 32768`), so more experts fit on the GPU.
 costs ~13.7 KB of RAM per context token (1.7 GB at 128K). Existing installs: run `START-HERE.bat --setup` once to turn
 it on.
 
-Time to first token is prompt length / prompt speed: about 7 s at 4K, 55 s at 32K, 4 minutes at 128K and 9 minutes at
-262K. The raw numbers: [`bench/results/`](../bench/results/). The [paper](paper/Strata-Paper.pdf) explains every number.
+**4-bit KV cache (engine 0.1.8, optional):** `START-HERE.bat --setup` asks above 8K context (or pass `--kv q4_0`). It
+halves the KV cache's memory with a Hadamard rotation before 4-bit rounding (PR #21), about 4% faster at 128K, but it
+is measurably less precise on long documents (perplexity +8-12%; needle tests still pass). 8-bit stays the default.
+Details: [`bench/results/2026-09-27-kv-q4`](../bench/results/2026-09-27-kv-q4/README.md).
+
+**Hybrid K8V4 KV cache (engine 0.1.25, optional, PR #120):** `--kv k8v4` (`START-HERE.bat --setup --kv k8v4`) keeps
+the keys at 8 bits and stores the values as rotated 4-bit: 23% less KV memory than 8-bit, so more experts fit in
+VRAM. RTX 3090, the Coder at 198K context: 99 instead of 85 tokens/s output, the same needle results, prompts 2-5%
+slower. It does not stream its KV cache (KV streaming is on by default from 64K), so it pays off mostly on large
+cards at long contexts.
+
+**The draft layer's tokens (0.1.27, `--draft-vocab`):** the MTP draft layer can only propose tokens from a subset
+of the vocabulary (`mtp/rt/draft_vocab.bin`). Since 0.1.27 the subset includes every Chinese, Japanese and Korean
+token (106,299 ids), so answers in those languages are 15-38% faster (Q2_0, RTX 5070). Its head takes ~180 MiB of
+VRAM, which the expert cache leaves free for it (0.1.28). `START-HERE.bat --setup --draft-vocab en` keeps the
+English/code subset from before (40,525 ids, ~110 MiB less VRAM, English answers 1-2% faster; CJK answers get
+almost no drafts). `tools/draft_vocab.py` builds and inspects subsets.
+
+**Low-RAM mode (engine 0.1.26, chosen by setup):** normally all of a model's experts are copied into RAM (23-50 GB,
+pinned) and the GPU holds a copy of the most-used ones. On a PC whose RAM cannot hold them beside the system (the
+experts plus ~10 GB), setup instead maps them from one file in the model's folder (`--mmap-experts`, the pack's
+`experts.bin`, +23-50 GB of disk). The OS file cache holds what the GPU does not, and it can give that memory back.
+On the Coder the engine's committed memory drops from 36 to ~13 GB, with the same answers. With a big GPU (an RTX
+5090 holds all of the Coder's experts, most of Q2_0's) it runs at nearly the usual speed. With a small one, most
+experts come from the SSD and it is much slower (setup says so). `START-HERE.bat --setup --low-ram on|off` overrides
+the choice.
+
+Time to first token is prompt length / prompt speed: with Q2_0 about 4 s at 4K, 25 s at 32K, under 2 minutes at 128K
+and 4.5 minutes at 262K (engine 0.1.13 made long prompts about twice as fast, below).
+
+**Faster prompts (engine 0.1.13):** the prompt is read in chunks of up to 8,192 tokens instead of 2,048 (`--prefill
+auto`: the largest chunk whose buffers fit in the expert-cache slots it borrows, and a request borrows only what its
+prompt needs); the experts are multiplied by llama.cpp's quantized MMQ kernels instead of being expanded to FP16
+first; the next layer's experts stream over PCIe while the current layer's attention runs; the PLE block runs for the
+whole chunk at once; unpinned experts are copied by helper threads. Measured on the RTX 5070 12 GB, 64 GB RAM,
+32K-token prompt: Q2_0 572 -> 1,290 tokens/s, IQ3_S 383 -> 1,208. Through the server (Q2_0, 128K context): 999 tokens
+353 -> 438 tokens/s, 6,927 tokens 529 -> 1,077, 28,584 tokens 584 -> 1,249. Output speed is unchanged. Needles 5/5
+(1K-262K). Details and the quality check:
+[`bench/results/2026-09-28-prefill-speed`](../bench/results/2026-09-28-prefill-speed/README.md). Existing installs
+switch to `--prefill auto` the next time START-HERE / setup.sh starts them. The raw numbers:
+[`bench/results/`](../bench/results/). The [paper](paper/Strata-Paper.pdf) explains every number.
 
 ## Other GPUs (estimated)
 
 Not measured - estimated from the runs above (same CPU and 64 GB RAM): the GPU part scaled by memory bandwidth, the CPU
 part by how many more experts the card's VRAM holds. Treat as **±20%**. Numbers are *prompt / output* tokens/s.
+The prompt figures predate engine 0.1.13, which about doubled prompt speed on the measured card; how much of that a
+card gains depends on its PCIe link (the experts stream over it), so they are still the older estimates.
 
 | GPU | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
@@ -60,7 +118,9 @@ part by how many more experts the card's VRAM holds. Treat as **±20%**. Numbers
 |  | IQ3_XXS | ~260 / ~106 | ~374 / ~103 | ~396 / ~89 | ~390 / ~85 | ~378 / ~71 | - |
 
 More VRAM matters more than a faster GPU: every extra GB holds ~700 more experts, and every expert on the GPU is one the
-CPU does not have to compute. A 3090's 24 GB takes most of the CPU work away.
+CPU does not have to compute. A 3090's 24 GB takes most of the CPU work away. (Since 0.1.14 the expert profile ranks
+all 24,576 experts; before, the cache stopped at 8,000, about 10-14 GB. `tools/make_profile.py` builds a profile from
+your own prompts: run the engine once with `--dump-routing trace.bin`, see the tool's help.)
 
 ## Which model?
 
@@ -73,7 +133,24 @@ of [Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next).
 | **IQ2_XS** | 68 GB | ~36 GB experts + ~6 GB | close to Q2_0 | a bit better |
 | **IQ3_XXS** | 76 GB | ~43 GB experts + ~6 GB | slower (more CPU work) | best |
 
-With 64 GB of RAM all three fit (close the browser for IQ3_XXS, and keep its context at 128K or less). With 48 GB only Q2_0 / IQ2_XS may fit. 32 GB is not enough.
+With 64 GB of RAM all three fit (close the browser for IQ3_XXS, and keep its context at 128K or less). With 48 GB only Q2_0 / IQ2_XS may fit. With 32 GB: the Coder (below).
+
+### Or: the Coder (half the experts, for code)
+
+**[Qwen3.8-Flash-Next GSQ-RCO Coder](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF)** is
+ISTA-DASLab's expert-pruned release: 256 of each layer's 512 experts are kept (still 10 active per token), chosen with
+RCO on code, agentic and vision calibration data; its authors report 91.3% of the full model's SWE-bench Verified and
+98.7% of LiveCodeBench v6. One size, named IQ1_M for its 1.89 bits per *original* parameter; the kept experts are
+stored like IQ3_S (IQ2_S-IQ4_XS gate/up, IQ4_NL/Q2_0 down). Shard 1 is 29.6 GB (experts: 23 GB of RAM), so it runs
+on **32 GB of RAM**, and at 262K on 64 GB. Its shard 2 and its vision encoder are the original's files: with the
+original installed, setup downloads only shard 1. Strata ships its expert profile (`data/expert-profile-coder.bin`,
+the shipped ranking mapped onto the kept experts through the release's `rco-allocation.txt`: 72% of the expert
+reads hit the GPU on a 12 GB card). Images work; the experimental speed projection loads and runs on it (it was made
+for the full model).
+
+```
+START-HERE.bat --setup --family coder
+```
 
 ### Or: Swift 1.5 (a fine-tune that thinks shorter)
 
@@ -99,15 +176,19 @@ You need **only an NVIDIA driver** (version 580 or newer; update it with the NVI
 
 | | |
 | --- | --- |
-| GPU | NVIDIA **RTX 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070; RTX 30/40 are untested. |
+| GPU | NVIDIA **RTX 20, 30, 40 or 50 series**, **12 GB VRAM or more** (8 GB runs, slowly). Measured on an RTX 5070 and an RTX 3090; RTX 20 (Turing, since 0.1.27) was tested by a contributor on an RTX 2070. |
 | RAM | **64 GB** recommended (see the table above). |
 | CPU | x86-64 with AVX2 (any Intel/AMD desktop CPU from the last ~8 years). AVX-512 (Ryzen 7000/9000) is a bit faster. |
 | Disk | ~70-80 GB free for the model, ~6 GB for the MTP layer (+1 GB with images). **Q2_0 on an AVX-512 CPU** also writes a one-time ~40 GB copy of its experts for the fast CPU kernel. An NVMe SSD is strongly recommended. |
 | OS | Windows 10/11, or Linux (Ubuntu 22.04/24.04 get everything installed automatically). |
 
-What the first start installs, all inside this folder (`.venv/`, `engine/`, `third_party/`, `models/`, `packs/`, `mtp/`):
+What the first start installs: in this folder `.venv/`, `engine/` and `third_party/`; the model files (`models/`,
+`packs/`, `mtp/`, 70-120 GB) in **`Strata-data` next to this folder**, so a new copy of Strata (an update unzipped
+elsewhere) finds them and sets itself up the same way. The place is remembered per user (`%APPDATA%\Strata\settings.json`,
+`~/.config/strata/settings.json`); `--data-dir` chooses another. Installs from before 0.1.16 are moved there by the next
+start (a rename on the same drive; files on another drive are used where they are).
 Python 3.12 if you have none (for your user account, no admin), a private Python environment, NVIDIA's CUDA libraries
-(from pip, ~0.4 GB), the ready-made Strata engine for RTX 30/40/50, the model and the MTP draft layer. If no
+(from pip, ~0.4 GB), the ready-made Strata engine for RTX 20/30/40/50, the model and the MTP draft layer. If no
 ready-made engine fits your PC, it offers to install the build tools (Visual Studio Build Tools + CUDA Toolkit on
 Windows, `build-essential` + CUDA on Ubuntu) and compiles the engine for your GPU (asks first; 20-40 minutes once).
 
@@ -125,21 +206,66 @@ Windows, `build-essential` + CUDA on Ubuntu) and compiles the engine for your GP
 4. **Images?** yes / no (see [Images](#images-vision)).
 
 Then it downloads and prepares everything (the model is 66-76 GB, so the first start takes a while; an interrupted
-download continues where it stopped) and **starts the model**: your browser opens `http://127.0.0.1:8080`, a small
-page that shows it is running and lets you chat. The API is at `http://127.0.0.1:8080/v1` for your apps.
+download continues where it stopped) and **starts the model**: your browser opens `http://127.0.0.1:8080`, the Strata
+app. It has three tabs:
+- **Chat:** streaming answers, the model's thinking (folded away once it answers), code with a copy button, pictures when
+  images are on, and sampling and thinking-level settings. Chats stay in your browser.
+- **Monitor:** what the model is doing (reading the prompt, with progress, or writing, at how many tokens/s); GPU load,
+  VRAM, temperature, power and PCIe traffic; CPU, RAM and disk; the context in use; the last requests.
+- **About:** the model and engine settings, and the addresses to connect other apps.
+
+`http://127.0.0.1:8080/?q=your question` opens it with a new chat already asking. The API is at
+`http://127.0.0.1:8080/v1` for your apps.
 
 **Every time after that**, `START-HERE.bat` just starts the model (30-90 s to load 34-43 GB into RAM). Nothing is
 downloaded again. Closing the window stops the model.
 
 ```
 START-HERE.bat --setup                          install another model, or change context / images
+SETUP.bat                                       the same (double-click it)
 START-HERE.bat --model IQ2_XS --context 32768 --vision yes --yes     no questions
 START-HERE.bat --gguf-dir D:\models\IQ2_XS       use GGUF files you already have
+START-HERE.bat --data-dir E:\Strata-data         keep the model files somewhere else
 START-HERE.bat --port 8081                      another port
+START-HERE.bat --gpu 1                          another GPU (numbered as nvidia-smi; setup picks the one with the most VRAM)
+START-HERE.bat --calibrate                      tune the engine for this PC (about 5-10 minutes), then start
 ```
 
 With more than one model installed, it asks which one to start. `run-<model>.bat` starts a model directly.
 
+**Tuning for your PC (`--calibrate`, engine 0.1.19).** Three engine settings depend on the PC more than on the model:
+- the share of the experts missing from VRAM that are copied to the GPU instead of computed by the CPU
+  (`--pcie-frac`: a fast PCIe link and a slower CPU want more, a laptop's narrower link less);
+- how sure the draft layer must be to add another guess to a check (`--spec-min-p`);
+- how many CPU threads compute experts (`--pool-workers`: on CPUs with efficiency cores, fewer can be faster).
+
+The defaults were measured on a Ryzen 5 7600 with an RTX 5070. Setup offers to measure them on your PC after an
+install; `START-HERE.bat --calibrate` (Linux: `./setup.sh --calibrate`) does it any time. It measures the output
+speed with each setting and keeps one only when it is more than 3% faster. The result is remembered per PC and model
+(in the settings file next to the data folder's record), so updates keep it.
+
+### Running it at startup (Task Scheduler)
+
+To have the model up at logon, people start the serve from **Task Scheduler** (or a service). Beware: Windows
+throttles such contexts, and the model's ~40 GB expert load then crawls at **~0.05 GiB/s (13-14 minutes)**
+instead of **~1.4-1.5 GiB/s (~35 seconds)** - a 24x slower start. Measured on an RTX 5070 Ti + Ryzen 7 9800X3D
++ NVMe, same binary, same args, same cache state:
+
+| How the serve starts | Expert load |
+| --- | ---: |
+| Double-click / terminal / SSH | 1.42-1.52 GiB/s (~35 s) |
+| Task Scheduler with its defaults | 0.05 GiB/s (821-841 s) |
+| Task Scheduler with the two settings below | 1.42 GiB/s (35 s) |
+
+In the task's properties set both of these (the defaults are the opposite):
+
+- **Priority level: Normal** (Options tab; the default is Below normal), and
+- **Run with highest privileges** (General tab; without it the task runs with a limited user token - which
+  also strips `SeLockMemoryPrivilege`, the privilege Windows large pages need).
+
+(Both were changed at once, so the isolated effect of each is not measured.) If the model still starts
+slowly, the engine prints a hint under its `loaded ... GiB at ...` line naming this cause.
+
 ### Chat in the terminal (optional)
 
 ```
@@ -159,6 +285,15 @@ for the build tools), and the same start: `http://127.0.0.1:8080`. Later runs of
 start the model directly. Options as on Windows (`./setup.sh --setup`, `--model Q2_0 --yes`, `--gguf-dir /data/Q2_0`).
 Terminal chat: `.venv/bin/python chat.py`.
 
+- **Updating:** `git pull`, then `./setup.sh`: it compiles the engine again when its source changed (a minute or
+  two for the changed files). If that compile fails, it says so and starts the engine you had.
+- **Other distributions** (Arch, Fedora, ...): install the C++ compiler and the CUDA Toolkit 13 with your package
+  manager first (Arch: `sudo pacman -S base-devel cuda`); setup finds `nvcc` on PATH, in `/usr/local/cuda*` and in
+  `/opt/cuda*`, and does the rest.
+- **WSL** works (Ubuntu 24.04 tested), with one limit: the NVIDIA driver pins only about 1 GB of RAM there, so KV
+  streaming (`--kv-resident`) is off and the KV cache stays in VRAM, and the experts are copied to the GPU from
+  unpinned RAM (slower prompts than native Linux).
+
 ---
 
 ## Using it
@@ -169,8 +304,13 @@ The server listens on `http://127.0.0.1:8080` (change with `--port` in setup, or
 | --- | --- |
 | OpenAI Chat Completions (stream and non-stream, tools) | `POST /v1/chat/completions` |
 | Anthropic Messages (stream and non-stream, tools) | `POST /v1/messages` |
-| Model list / health | `GET /v1/models`, `GET /health` |
-| What the model is doing right now | `GET /status` |
+| Model list / health | `GET /v1/models`, `GET /models`, `GET /health` |
+| Model properties | `GET /props` (also accepts `?model=<loaded-model-id>`) |
+| What the model is doing right now | `GET /status`, `GET /slots` (single slot, busy or idle) |
+| Everything the Monitor tab shows (engine, live state, last requests, hardware) | `GET /metrics` |
+| The MCP servers, their state and tools ([below](#tools-from-mcp-servers)) | `GET /mcp` |
+
+`/models` and `/v1/models` list only the loaded model, with its context limit and input modalities. `/props` exposes the original chat template, context limit, configured generation defaults (shared settings take precedence), model path and engine version when available. Context means the full engine context, not the resident KV window. `n_predict: -1` means no fixed output cap. Unconfigured sampling fields are omitted. `autoload` has no effect; an unknown `model` returns 404. These metadata endpoints and `/slots` require the API key when one is configured. They do not load, unload or restart models.
 
 ```bash
 curl http://127.0.0.1:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
@@ -202,9 +342,21 @@ print(r.choices[0].message.content)
   15 s, and `GET /status` says what it is doing (`reading the prompt`, `answering`, tokens so far). Closing the
   connection or pressing stop in your app really stops the model, so the next request starts at once.
 - **Chat apps.** Any app with an "OpenAI-compatible" provider works: base URL `http://127.0.0.1:8080/v1`, any API key.
-- **Context.** Chosen in setup (8K-262K). Requests longer than that are refused, never silently cut.
-- **From other devices / the internet.** The server listens on your PC only (`127.0.0.1`). To reach it from elsewhere,
-  put a tunnel in front of it, for example [cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/do-more-with-tunnels/trycloudflare/):
+- **Claude Code** (Strata 0.1.17 or newer): set `ANTHROPIC_BASE_URL=http://127.0.0.1:8080` and
+  `ANTHROPIC_MODEL` to a Claude model name it knows (it refuses names it doesn't; Strata ignores the name), plus any
+  `ANTHROPIC_AUTH_TOKEN` (or your `api_key`, if you set one).
+- **Context.** Chosen in setup (8K-262K). Requests longer than that are refused, never silently cut. A request whose
+  `max_tokens` would run past the context is refused too (400); agents that always ask for their full output cap
+  can instead get it shortened to the room left: add `"fit_max_tokens": true` to `strata-<model>.json` (or pass
+  `--fit-max-tokens` to `serve/server.py`). A prompt that leaves no room at all is still refused.
+- **From other devices on your network.** The server listens on your PC only (`127.0.0.1`) unless you say otherwise:
+  run setup with `START-HERE.bat --setup --host 0.0.0.0 --api-key some-long-secret` (or add `"host": "0.0.0.0"` and
+  `"api_key": "..."` to `strata-<model>.json`). The server window then prints this PC's addresses
+  (`from other devices: http://192.168.x.x:8080/`); open that on the other device, or use `.../v1` as an API base URL.
+  On Windows the firewall blocks it until you allow it: accept its prompt for Python (private networks), or run
+  `New-NetFirewallRule -DisplayName "Strata 8080" -Direction Inbound -Protocol TCP -LocalPort 8080 -Action Allow -Profile Private`
+  in an admin PowerShell, and make sure the network is set to Private.
+- **From the internet.** Put a tunnel in front of it, for example [cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/do-more-with-tunnels/trycloudflare/):
   `cloudflared tunnel --url http://127.0.0.1:8080`. **Set a key first**, or anyone with the link can use your PC:
   add `"api_key": "some-long-secret"` to `strata-<model>.json` (or set the `STRATA_API_KEY` environment variable);
   clients then send it as their API key.
@@ -212,11 +364,69 @@ print(r.choices[0].message.content)
 **Conversation cache.** A request that continues a chat reads only the part after what the engine already holds: the
 live session, or one of the checkpoints it keeps in RAM (up to 6, ~118 MB each, taken at the start of each new
 assistant turn and every 16K prompt tokens). A checkpoint is used only when the prompt starts with exactly its tokens
-and pictures. Engine options: `--prompt-cache N` (0 = off), `--prompt-cache-every N`, `--turn-token ID`.
+and pictures. The oldest checkpoint - in practice the end of the system prompt, which every chat of the same client
+shares - is kept for good while the rest rotates by least recent use, so a NEW chat that shares that prefix starts
+reading after it instead of from token 0. A prompt read from the start is also checkpointed at the end of its system
+prompt when that is 2,048 tokens or more (engine 0.1.20; PR #62 + #65), so that root exists for agent clients with long
+system prompts and tool lists. Engine options: `--prompt-cache N` (0 = off), `--prompt-cache-every N`,
+`--prompt-cache-root N` (0 = no system-prompt checkpoint), `--turn-token ID`.
 
-**Current limits (v1):** one request at a time, and one conversation cached at a time (switching between two chats
-re-reads the other one); greedy decoding (temperature is ignored); images only when set up with them (below); no
-video.
+**Current limits (v1):** one request at a time, and one conversation's history in the KV cache at a time (switching
+between two chats re-reads the part where they diverge; the shared prefix, such as the system prompt, is reused); images
+only when set up with them (below); no video. **Temperature / top_p / top_k / min_p /
+seed** are honored per request (OpenAI and Anthropic fields); with the default adaptive expert tier a sampled result
+is not reproducible run to run - for seed-reproducible output add `--adapt-every 100000` (static residency) to the
+engine arguments. The run config's optional `sampling` block sets the defaults for requests that leave the fields out
+(`"sampling": {"temperature": 1.0, "top_p": 0.95, "top_k": 20}`); a request's own fields always win, and with no
+block at all a request without sampling keys decodes greedy. The penalties (`presence_penalty`, `frequency_penalty`,
+`repetition_penalty`, with `penalty_last_n` capping how many recent tokens they count over, default 64 when any
+penalty is set) ride the same path; they count the tokens the request has consumed, so a repetition penalty
+suppresses what the model itself just said, not the prompt alone. Since engine 0.1.19 they apply to every token
+the speculative decoding checks at once, exactly as if it decoded one token at a time (before, only the first of
+each batch got them). That makes requests with penalties 1-11% slower than in 0.1.18: the draft layer guesses
+without penalties, so more of its guesses are now rejected. Requests without penalties are unchanged. `top_k` keeps at most 64 candidates: `0` ("off") or anything above 64 uses all 64.
+
+---
+
+## Tools from MCP servers
+
+The chat page can give the model tools from [MCP](https://modelcontextprotocol.io) servers, as LM Studio and Claude
+Desktop do: reading your files, fetching web pages, searching, anything an MCP server offers. List the servers in
+`strata-<model>.json` under `"mcp_servers"` - the same shape as Claude Desktop's `mcpServers` block, which you can
+also paste as it is (key `"mcpServers"`):
+
+```json
+"mcp_servers": {
+  "files": {"command": "npx", "args": ["-y", "@modelcontextprotocol/server-filesystem", "C:\\Users\\me\\Documents\\notes"]},
+  "search": {"url": "http://127.0.0.1:3000/mcp", "headers": {"Authorization": "Bearer ..."}}
+},
+"mcp": {"timeout_s": 60, "max_result_chars": 20000, "max_rounds": 8}
+```
+
+Or keep them in their own file and start the server with `--mcp-config path\to\claude_desktop_config.json` (a file
+with an `mcpServers` block; add it to the `serve/server.py` line of your run script). Restart Strata after a change.
+
+- **A program** (`command`, `args`, optional `env` and `cwd`) is started by Strata and spoken to over its
+  stdin/stdout; `npx`, `uvx`, `python` and friends are found on `PATH` as usual (Node.js is needed for `npx`
+  servers). **An address** (`url`, optional `headers`) uses MCP's Streamable HTTP transport (the older SSE-only
+  transport is not supported). `"disabled": true` leaves an entry out.
+- The servers start with Strata, in the background; the server window says what each one offers
+  (`MCP server 'files': 14 tools (...)`), or why it did not start - its tools are then left out and the chat works
+  without them. The Monitor tab lists them, and the Sampling drawer has **Use tools from MCP servers** (on by
+  default). A server that stops later is started again at its next call.
+- In the chat each call shows as a small block (tool, arguments, result); the model reads the result and goes on,
+  up to `max_rounds` calls in a row per answer. A tool that fails or takes longer than `timeout_s` (default 60 s)
+  gives the model an `error: ...` result instead of ending the chat. Results longer than `max_result_chars`
+  (default 20,000 characters) are cut, with a note, before the model reads them. Stop stops a running tool too.
+- Only the chat page uses them. API clients (omp, Claude Code, OpenAI and Anthropic SDKs) see the API exactly as
+  before and keep their own tools; a request to `/v1/chat/completions` opts in with `"strata_mcp": true` (it then
+  gets `strata_mcp` tool events in the stream).
+
+**Security.** MCP tools run on your PC with your user's rights, and **the model decides when to call them** - also
+because of what it reads (a web page or a file can contain instructions). Give a filesystem server only the folders
+it needs, prefer read-only tools, and don't add servers you don't trust. The tools can only be used from the chat
+page itself (a request with another site's Origin or without a JSON content type is refused); if Strata is reachable
+from other devices, set an API key.
 
 ---
 
@@ -294,6 +504,45 @@ test images.
 
 ---
 
+## Experimental speed projection (EXPERIMENTAL, off by default)
+
+**This is an experiment, not a finished feature.** It ships with Strata but stays off unless you turn it on.
+
+A 480 KB control vector for Qwen3.8-Flash-Next (`data/experimental-speed-projection/`, see its README). After each
+of layers 4-44 the engine removes one direction from every hyper-connection stream of the residual: `h -= (h . v) v`,
+one unit vector `v` per layer, exactly as llama.cpp does with the package's `--cvec-mode project` patches.
+
+**What it changes.** The vector's own package describes it as a **refusal-direction projection**: with it the model
+declines far fewer requests (it reports 1 of 50 vs 50 of 50 on its test set), and removing refusals removes a safety
+behaviour - you are responsible for what the model writes with it on. It also shifts ordinary answers a little
+(measured below). It is not an optimization in the engine: on the same text it costs 0.2-0.4% per token. What a
+chat's tokens/s does with it on depends on the text the model writes (length, repetition, how well the drafts land),
+so measure it on your own prompts; the Monitor marks every request ESP or stock.
+
+**Turning it on (at setup).** `START-HERE.bat --setup` asks "Turn on the experimental speed projection?" (default:
+no), or pass `--experimental-speed-projection on` (`off`, or a path to another vector GGUF). Only for the original
+Qwen3.8-Flash-Next, not Swift 1.5. It writes these engine flags (llama.cpp's) into `strata-<model>.json`:
+
+```
+--control-vector-scaled <Strata>\data\experimental-speed-projection\Qwen3.8-Flash-Next-experimental-speed-projection.gguf:1.0
+--control-vector-layer-range 4 44 --cvec-mode project --cvec-dir per-layer
+```
+
+The engine log then says `control vector mode = project, dir = per-layer, layers 4..44 (41 steered)`, and the web
+app's About tab lists it. (`--cvec-mode add` is llama.cpp's stock additive mode, for additive vectors.)
+
+**Per request.** A loaded vector is on for every request unless it says otherwise: the web app's Sampling drawer has
+a switch, and the API takes `"experimental_speed_projection": false` in the request body (OpenAI and Anthropic; a
+config default goes in `"sampling": {"experimental_speed_projection": false}`). Switching drops the conversation
+cache once, since the model state was computed the other way. Switched off, the output is token-for-token the stock
+model's.
+
+**Measured here** (Q2_0, fixed experts, 2,557 teacher-forced tokens of code, a document and a chat): the top-1 token
+changes at 10% of positions, mean KL from the stock model 0.063 nats (max 4.1), perplexity +15% on code, +2.3% on
+the document, +0.4% on the chat. Details: `bench/results/2026-09-27-esp/`.
+
+---
+
 ## Troubleshooting
 
 | Symptom | What to do |
@@ -302,14 +551,19 @@ test images.
 | Python or the build tools could not be installed | Install what it names (links are printed), then run it again. Everything already done is kept. |
 | `port 8080 is already in use` | Strata is already running (look for its window), or another program uses the port: `START-HERE.bat --port 8081`. |
 | `cudaHostRegister ... out of memory` in the log | Normal on Windows: the engine pins the experts in per-layer slices instead. Only a problem if the load then fails. |
-| The first start takes minutes | It is reading 34-43 GB into RAM; the second start is faster while the files are in the OS cache. |
+| `ExpertCache: cudaMalloc(...) failed: out of memory` although VRAM is free | Windows' page file is off or tiny: every allocation on the graphics card is also charged to Windows' commit (RAM + page file). Set the page file to "System managed" (System > About > Advanced system settings > Performance > Advanced > Virtual memory) and restart. Since 0.1.19 the engine retries with a smaller cache instead of stopping, and setup warns about a page file under 4 GB (issue #60). |
+| The first start takes minutes | It is reading 34-55 GB into RAM; the second start is faster while the files are in the OS cache. |
+| The PC freezes for a few minutes at the start | Normal, most of all the first time (the server window says when it happens): the engine loads the experts into RAM, pins part of it for the GPU and sizes the expert cache. Wait; don't close the window. Still frozen after 10 minutes: restart the PC, close other programs, try again, or pick a smaller size. |
+| `the engine stopped unexpectedly (exit code ...)` | The engine process ended mid-answer - usually out of RAM (Linux ends the biggest program: `sudo dmesg \| grep -i -E 'killed process\|out of memory'`). The next request starts it again by itself. If it repeats: close other programs or pick a smaller size. The server also warns at start when the model's experts leave less than ~6 GB of RAM for everything else. |
 | Slow output, disk light busy | Not enough free RAM: close other programs, or choose Q2_0 / IQ2_XS. |
 | `prompt ... exceeds the context` | The request is longer than the context you chose: run setup again with a bigger `--context`. |
 | Slower than the tables | The monitor plugged into the GPU and other GPU programs take VRAM from the expert cache; RAM running below its rated speed (enable EXPO/XMP in the BIOS) slows the CPU half. |
 | `this server was started without the vision encoder` | The model was set up for text only: run setup again with `--vision gpu`. |
 | A picture is refused or `cannot read the image` | The file is not a picture Pillow can open (JPEG, PNG, WebP, GIF, BMP, TIFF, AVIF work). |
 | Pictures are slow (10-30 s) | The encoder runs on the CPU: run setup again with `--vision gpu` (needs ~1.4 GB of VRAM). |
-| A request never finishes (older engines, mostly with pictures) | Run `START-HERE.bat` once to get engine 0.1.2 or newer. It keeps a real margin of VRAM free: the log says `... MiB of VRAM free with everything loaded` and warns when it is close to 0. |
+| A request never finishes: "reading the prompt", GPU "100%" at low power | The GPU ran out of VRAM (engines before 0.1.9 could end with ~30 MiB free at large contexts). Run `START-HERE.bat` once to get engine 0.1.9 or newer; the log then says `... MiB of VRAM free with everything loaded` (a few hundred) and names the `--vram-reserve-mib` to add if it is low. |
+| Generation stops mid-answer, GPU "100%", one CPU core busy | Fixed in engine 0.1.12 (issue #29, a race in the CPU expert pool on big-VRAM cards). Since then a request that stops moving ends with an error instead of hanging (after 2 minutes; 1 minute from 0.1.13): the log says `no progress for ... s ... (issue #29)` with where it stopped, and the next request starts the engine again. If you see that line, please open an issue with it. Engine 0.1.13 adds a stall report under it (what every expert-pool thread and the GPU handshake were doing, memory and page faults) and, on Windows, a `strata-stall-<pid>.dmp` file with every thread's stack: attach both. (`STRATA_WATCHDOG_S` sets the time in seconds; 0 turns it off.) Engine 0.1.14 fixes the stall those reports found (issue #31: with the IQ packs the host could wait forever inside the NVIDIA driver while copying experts in a verify window; the experts are now copied by a GPU kernel, `--pcie-mode dma` restores the old way). |
+| `out of memory: cudaFuncSetAttribute` in the log (IQ3_XXS, long prompt) | Fixed in engine 0.1.15: CUDA loaded a kernel's code when it was first needed, and mid-prompt there was no VRAM left for it. Run `START-HERE.bat` (Windows) or `./setup.sh` (Linux) once to update. |
 | Anything else | The engine log is `strata-<model>.log` in this folder. |
 
 ---
@@ -325,7 +579,9 @@ test images.
   as the GPU works on the cached ones (AVX-512 / AVX2 kernels, ggml's for the i-quants).
 - **SSD:** the 28.8 GB n-gram table, read a few rows per token through the OS cache.
 - **Speculation:** the model's own MTP layer drafts up to 3 tokens; one pass over all 48 layers checks them. 2.4-3.2
-  tokens per pass on average.
+  tokens per pass on average. When the reply repeats the context (code edits, quoted text), **prompt lookup** (engine
+  0.1.7) drafts up to 5 tokens from the earlier copy, but only where its measured acceptance and cost say it pays:
+  code edits 6-11% faster, other text unchanged. The drafts are checked like the MTP's, so the output is the same.
 - **Prompts** are processed in 2,048-token chunks with the experts streamed to the GPU over PCIe.
 
 The full story, with measurements, bottlenecks and what comes next: **[docs/paper/Strata-Paper.pdf](paper/Strata-Paper.pdf)**.
@@ -334,8 +590,12 @@ The full story, with measurements, bottlenecks and what comes next: **[docs/pape
 
 ## Credits and licenses
 
+Strata itself: [MIT](../LICENSE). The model files are not part of it; their licenses apply to them (below).
+
 - Model: [Qwen/Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next) by the Qwen team; quantizations:
   [ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF).
+  The Coder: [ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF)
+  (Apache-2.0 per its card); its support in Strata came from @pjgmobile's PR #54.
   Swift 1.5: [ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF](https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF)
   by UkisAI. Their licenses apply to the weights.
 - [llama.cpp / ggml](https://github.com/ggml-org/llama.cpp) (MIT): the i-quant formats, the GPU dot products and
@@ -344,3 +604,7 @@ The full story, with measurements, bottlenecks and what comes next: **[docs/pape
   `third_party/ggml/LICENSE`.
 - Ideas from [Splash](https://github.com/incoai/splash), [ninfer](https://github.com/Neroued/ninfer) and
   [HyperQwen](https://github.com/syv-ai/HyperQwen); references in the paper.
+- The web app's font: [Outfit](https://github.com/Outfitio/Outfit-Fonts) (SIL Open Font License 1.1, see
+  `serve/web/fonts/OFL.txt`). Its Monitor tab started from @code-martin's dashboard idea (PR #22).
+- The experimental speed projection's vector (`data/experimental-speed-projection/`): Qwen Community License 1.0,
+  made from the model's activations (see its README).

docs/MULTI_GPU.md

@@ -0,0 +1,122 @@
+# Strata on two or three GPUs (layer split)
+
+One model can run across several NVIDIA cards in one PC. The layers are split into contiguous ranges, one per GPU:
+the first card runs layers 0 to K-1, the next card runs K onward, and so on; the last card also runs the output head
+and the draft (MTP) layer. Each card keeps an expert cache for **its own layers only**, so two cards hold about twice
+the experts one card holds - for the Coder model on a 16 GB + 24 GB pair, nearly all of them, which is where the
+speed comes from (decode then barely touches the CPU pool).
+
+This is pipeline (layer) parallelism, not tensor parallelism: a token crosses from one card to the next once per
+verify window (a few hundred KB through pinned RAM), not twice per layer. No NVLink or peer-to-peer access is
+needed; cards on x4 or x1 slots work, and the PCIe share of each card is probed on its own link.
+
+## Using it
+
+**Nothing to type.** `START-HERE.bat` (Linux: `./setup.sh`) lists your NVIDIA cards and says for each one whether
+Strata can use it:
+
+```
+  Your NVIDIA GPUs:
+    GPU 0: NVIDIA GeForce RTX 5080, 16 GB VRAM - can be used
+    GPU 1: NVIDIA GeForce GTX 1080 Ti, 11 GB VRAM - not supported - older than the RTX 20 series (compute capability 6.1; Strata needs 7.5 or newer)
+    GPU 2: NVIDIA GeForce RTX 3090, 24 GB VRAM - can be used
+  ...
+  1) GPU 0 (NVIDIA GeForce RTX 5080, 16 GB) + GPU 2 (NVIDIA GeForce RTX 3090, 24 GB) together   (recommended)
+  2) GPU 2 (NVIDIA GeForce RTX 3090, 24 GB) only
+  3) GPU 0 (NVIDIA GeForce RTX 5080, 16 GB) only
+Which GPUs? [1]:
+```
+
+When two or more cards can share the model, the two best together are recommended (the newest generation first:
+it becomes the main card). A model installed on one card asks once, at its next start, whether to use both from
+now on; the answer is kept.
+
+**Choosing yourself** (at setup or at any start):
+
+```
+--gpus 0,2                 these cards together, as nvidia-smi numbers them; the first is the main one. Remembered.
+--gpus all                 every card that can share the model
+--gpu 0                    one card (at a start: for that start only)
+--layer-split auto         (default) or the first layer of each later card, e.g. 18 or 16,32
+```
+
+**Not supported** (setup says so and names the cards that can be used instead):
+- a card older than the RTX 20 series (compute capability below 7.5: GTX 10 and older);
+- a card with less than 8 GB of VRAM, together with others (each card holds a copy of the dense weights and its
+  own prompt buffers);
+- AMD and Intel GPUs, and a mix of NVIDIA with them.
+
+Or edit an existing config (`strata-*.json`), then restart:
+
+```json
+"gpu": [0, 2],
+"layer_split": "auto"
+```
+
+The engine flags behind it: `--layer-split K1[,K2..]|auto` and `--split-device D1[,D2..]` (the later stages'
+devices; default the next visible ones). `--layer-split K --split-device 0` runs both stages on one card sharing
+everything - the bit-exact check of the hand-off, not a speed mode.
+
+**auto** tries every placement (all of them for two or three cards; proportional to the free VRAM beyond that) and
+keeps the one whose caches would hold the most of the expert profile, hottest pairs weighted most; ties go to the
+placement that leaves the fullest card the most room. The startup log prints the choice:
+
+```
+strata generate: layer split auto: K=19 - the caches hold 11767 of 12288 profiled pairs (fullest device 100%)
+strata serve: layer split: layers 0-18 (CUDA0), 19-47 (CUDA1), one hand-off per window
+```
+
+## What each card holds
+
+- **every card**: a copy of the dense weights (~3.4 GB for the Coder), its own session state (the KV cache of the full
+  context), its verify window and its prompt-path buffers, and an expert cache for its layers filled from the profile;
+- **the last card**: also the output head and the draft layer (~0.8 GB);
+- **host RAM**: the expert arena once, shared by all cards (the CPU pool computes whatever no card holds).
+
+Prompts are read in chunks that flow through the cards in turn; while a later card reads chunk c, the first card
+already reads chunk c+1. Conversation checkpoints save and restore every card's state; the adaptive expert swaps copy
+into the card that owns the layer.
+
+## Limits (for now)
+
+- **Works across cards** (bench/results/2026-09-29-layer-split-limits):
+  - images (`--vision`): each card keeps its own image-position table;
+  - control vectors and the experimental speed projection: each card holds the vector's tables, switched on and
+    off per request on all of them;
+  - KV streaming (`--kv-resident`): each card streams the KV of its own session;
+  - mid-prompt checkpoints (`--prompt-cache-every`): each card saves its part of a checkpoint when it has read that
+    chunk;
+  - the older helper-GPU caches (`--expert-cache-remote`, docs/SECOND_GPU.md): they take the visible GPUs no stage
+    runs on, and hold only experts no stage's cache holds. On the test rig, a 2080 Ti helper made decoding slower,
+    as it did without a split: its per-layer round trip costs more than the CPU pool needs for those experts.
+- `--mmap-experts` needs a canonical pack (`experts.bin`), with or without a split; a native (IQ) pack says so at
+  start.
+- The prompt path has its own buffers on every card (1.5 GB each at the default 2048-token chunk; `--prefill 1024`
+  halves that) instead of borrowing cache slots as one card does. An explicit `--expert-cache` on the first card is
+  capped to leave room for them.
+- On Windows only 8 GiB of the expert arena is pinned (more, mapped into two GPU contexts, leaves WDDM refusing
+  allocations); the PCIe share covers those layers.
+- Every card needs compute capability 7.5 (RTX 20 or newer). The pre-sm_80 QSA scorer path is fp32 FMAs, so a
+  Turing card runs the same kernels instead of the tensor-core prompt attention.
+
+## Measured
+
+The Coder on an RTX 5080 + RTX 3090 (Ryzen 9 9950X3D), 32K context; details in
+`bench/results/2026-09-29-layer-split/`:
+
+| | Prompt 16K / 28K tok/s | Decode story / code tok/s |
+|---|---|---|
+| 5080 alone | 1,726-2,017 / 1,970 | 83-87 / 88-105 |
+| 5080 + 3090, best split (K=26) | 2,039 / 2,357 | 84 / 110 |
+| 5080 + 3090, auto (K=22) | 2,037 / 2,073 | 80 / 109 |
+
+- **Prompts gain the most** (+18-20%): each card reads its own layers of the chunk while the other reads the next.
+- **Decode is on par with the faster card alone**, and ahead on code. Once both caches hold nearly every routed
+  expert, the per-layer GPU time decides.
+- **Correctness:** one GPU is byte-identical to 0.1.20, and the hand-off itself is bit-exact.
+
+**Which cards and in what order:**
+- Put the fastest card first; auto gives it as many layers as its cache allows.
+- Leave out a much slower card when two already hold the model. An RTX 2080 Ti as a third card made the 5080 +
+  3090 pair slower (68 / 90 tok/s decode): every extra card costs its own round per window.
+- More cards pay off when the model's routed experts do not fit the faster ones.

docs/ORCA.md

@@ -0,0 +1,95 @@
+# OrcaRouter IQ3_XXS compatibility
+
+Validated on Linux with an RTX 5090 (32 GB), Ryzen 9 9950X3D and 128 GB RAM on 2026-09-27.
+
+The target is `orcarouter/Qwen3.8-Flash-Next-Uncensored-GGUF`, **IQ3_XXS** (two shards, 85.20 GB total).
+It has Strata's Qwen4Exp geometry, but ordinary quantization also compresses the hyperconnection,
+small attention and PLE projections which Strata's kernels read as BF16.
+
+`tools/iq_pack.py --compat-bf16` expands only those projections, rounding to BF16 using nearest-even.
+This adds rounding relative to the GGUF's dequantized weights; it does not recover the original BF16
+checkpoint or its quality. Expert weights, native attention weights, embeddings and the 28.8 GB PLE
+lookup table remain unchanged. The latter stays on disk. The pack records its conversions in
+`compat-bf16.json`.
+
+The native loader retains this pack's converted PLE key. Its existing native Q2_0 key path remains
+available to the original model. Hugging Face snapshot symlinks are supported; all split shards must
+finish downloading before packing. Pass the snapshot filename, not its hash-named blob target.
+
+## Preparation
+
+Build Strata normally using the pinned llama.cpp dependency. Python needs numpy, regex and gguf-py
+from that dependency (`STRATA_GGUF_PY` can point to its `gguf-py` directory).
+
+```sh
+.venv/bin/python tools/iq_pack.py \
+  --gguf /path/to/Qwen3.8-Flash-Next-Uncensored-IQ3_XXS-00001-of-00002.gguf \
+  --out packs/orca-iq3_xxs --compat-bf16
+```
+
+Use this model's own tokenizer exported into the pack. Do not share another model's `dense.bin`,
+and do not rename the Orca files to impersonate one of setup's GSQ-RCO models.
+
+Strata's persistent server also requires the MTP runtime, prepared by the existing tools:
+
+```sh
+.venv/bin/python tools/mtp_fetch.py fetch --out mtp
+.venv/bin/python tools/mtp_pack.py --src mtp --experts q2_0 --out mtp/mtp-q2_0.gguf
+.venv/bin/python tools/mtp_rt.py --gguf mtp/mtp-q2_0.gguf --out mtp/rt
+cp data/draft_vocab.bin mtp/rt/draft_vocab.bin
+```
+
+This uses the original model's draft head, as setup does for Swift. The target model verifies its
+proposals; draft acceptance and performance must be measured for this fine-tune.
+
+## Local server
+
+Save the following as `strata-orca-iq3_xxs.json` at the repository root, replacing both `/path/to/`
+entries with the **same first shard**. Orca's PLE table is in shard 1. Start with a 32K context and
+512-token prefill chunks; the expert cache sizes itself to available VRAM. The expert arena alone
+needs about 49.8 GiB of available system RAM, plus draft/runtime buffers and other applications.
+
+```json
+{
+  "exe": "build/strata",
+  "args": [
+    "--pack", "packs/orca-iq3_xxs",
+    "--native", "/path/to/Qwen3.8-Flash-Next-Uncensored-IQ3_XXS-00001-of-00002.gguf",
+    "--ple-gguf", "/path/to/Qwen3.8-Flash-Next-Uncensored-IQ3_XXS-00001-of-00002.gguf",
+    "--expert-profile", "data/expert-profile.bin", "--expert-cache", "auto",
+    "--prefill", "512", "--spec", "4", "--spec-min-p", "0.5",
+    "--mtp", "mtp/rt", "--max-context", "32768", "--kv", "int8"
+  ],
+  "cwd": ".",
+  "tokenizer": "packs/orca-iq3_xxs/tokenizer",
+  "model_name": "orcarouter-qwen3.8-flash-next-uncensored-iq3_xxs",
+  "log": "strata-orca-iq3_xxs.log",
+  "host": "127.0.0.1",
+  "port": 8080
+}
+```
+
+Run from the repository root:
+
+```sh
+.venv/bin/python -m serve.server --engine strata --config strata-orca-iq3_xxs.json --port 8080
+```
+
+The web interface is at `http://127.0.0.1:8080`; API clients use `http://127.0.0.1:8080/v1`.
+Vision is not configured in this text-only example. Performance of the original GSQ-RCO model
+does not establish this fine-tune's speed or accuracy.
+
+## Scope and validation
+
+- IQ3_XXS is the target of this change; other Orca quantizations are not validated.
+- IQ3_M additionally uses Q5_0 expert down matrices, which the native GPU expert path does not support.
+- The installer model menu is unchanged. This is an explicit local packing workflow.
+- Focused conversion and split-file tests: `.venv/bin/python -m unittest discover -s tools -p test_iq_pack.py`.
+- All eight packing tests, 17 server tests and the GPU gated-residual parity check passed.
+- The complete model packed successfully: 460 tensors converted, 1.39 GiB of converted BF16 weights;
+  1.43 GiB total packed dense weights. The 48 expert layers remain IQ3_XXS gate/up and IQ4_NL down.
+- Real server checks passed: arithmetic, Python code, translation, a longer explanation, multi-turn context
+  reuse (25 prompt tokens reused), streaming cancellation and a subsequent correct response. The engine
+  PID stayed unchanged. These are smoke tests, not a quality benchmark or a full-context stress test.
+- With the configuration above, a 111-token explanation generated at 77.7 tokens/s (engine decode timing);
+  total request time was 1.77 seconds. This single short measurement is not a general throughput claim.

docs/SECOND_GPU.md

@@ -0,0 +1,84 @@
+# CUDA1–3 expert experiment
+
+CUDA0 keeps the dense layers, KV/state, MTP and its existing expert cache. Up to
+three independent expert caches fill CUDA1, CUDA2 and CUDA3 in that order. Each
+expert has one owner: CUDA0, one secondary GPU, or the CPU pool. Each secondary
+GPU computes its rows while the CPU pool works; the results return through
+private pinned host buffers before CUDA0 continues the layer. The GPU launches
+can overlap, but each layer waits for all three.
+
+The shipped profile ranks 8,000 `(layer, expert)` pairs. CUDA0 takes its usual
+`--expert-cache auto` allocation. With only CUDA1, it takes the first remaining
+pairs. When CUDA2 or CUDA3 is enabled, the remaining profiled pairs are assigned
+round robin to the secondary GPUs. The ranked list is then extended with all
+other pairs in expert-then-layer order and distributed in the same way. This
+fills the additional VRAM and lets all secondary GPUs see frequent experts, but
+unranked experts may hardly ever be routed. The startup
+log reports slots and GiB per card; each request reports the number of expert
+entries actually computed on each secondary GPU. Remote outputs are packed
+before returning over PCIe/USB4: the request log compares MiB transferred
+with the previous full-row transfer. The input and routing metadata use pinned
+host staging. This reduces transfer traffic and launch overhead, but does not
+release the expert arena in system RAM or significantly change VRAM use.
+
+The optional `--gpu-placement layer` gives each layer to exactly one secondary
+GPU (layer number modulo the number of enabled cards). It fills that card with
+the most frequently routed experts of its layers first, then the unranked ones.
+This can reduce USB4 device switches and synchronization when several secondary
+cards serve the same layer in the default `stripe` mode. All three cards still
+keep their requested expert slots; the primary CUDA0 cache and CPU fallback
+continue to work. The log counts active layer launches on each card, so the
+tradeoff can be measured alongside tokens per second. This placement is
+experimental and may be slower if a given layer needs more than one card's
+compute capacity.
+
+The experiment requires the corresponding visible CUDA devices and a CUDA
+enabled build. Each card must keep at least 512 MiB free. Tiers must be enabled
+in order. On Windows, CUDA registration of the host expert arena is capped at
+8 GiB to leave room for the contexts and MTP on CUDA0. The rest remains
+available to the CPU pool; the PCIe expert path is available only for the
+registered layers. Without secondary GPUs, the original uncapped registration
+behavior applies.
+
+On an existing Linux installation, apply the patch to its source tree, then run:
+
+```sh
+./setup.sh --setup --gpu1-experts 5000 --gpu2-experts 5000 --gpu3-experts 5000
+```
+
+On Windows, from PowerShell in the installation folder:
+
+```powershell
+.\START-HERE.bat --setup --gpu1-experts 5000 --gpu2-experts 5000 --gpu3-experts 5000
+```
+
+To test the layer placement instead, add `--gpu-placement layer` to the setup
+command. To go back, run the same command with `--gpu-placement stripe`. After
+the new engine is built, the mode can also be changed without a rebuild: edit
+`--expert-cache-remote-placement` in `strata-iq3_xxs.json` from `layer` to
+`stripe` or vice versa and restart the server. If the argument is absent, the
+default is `stripe`.
+
+Choose the same model, size, context and image settings as your current
+installation when prompted. Setup recompiles the engine, updates its server
+configuration, and starts it. Later starts keep those settings. To tune the
+slots without recompiling, edit the values after `--expert-cache-device1`,
+`--expert-cache-device2` and `--expert-cache-device3` in the stored
+`strata-iq3_xxs.json`, then restart with `.\run-iq3_xxs.bat`.
+
+Compare identical requests at one, two and four GPUs, preferably with several
+repeats. VRAM use alone does not show useful offload: compare the per-request
+CUDA1–3 counts and tokens per second. More GPU contexts and synchronization
+may lower the speed. Prompt prefill still uses CUDA0; secondary GPUs serve
+decode, including MTP verification. No peer-to-peer access is required.
+
+Longer contexts reserve more KV/state memory on CUDA0, which reduces its
+automatic expert cache. On a 64 GB PC `setup.py` limits IQ3_XXS to 128K even
+if 262K was selected; the secondary caches still require the host expert arena.
+To try 262K on 64 GB, choose Q2_0 or IQ2_XS instead and check that CUDA0 still
+has enough free VRAM after loading. Benchmark long prompts separately from
+short decode requests.
+
+The existing warning about the CUDA0 expert-cache GPU hit path still applies:
+its outputs diverge from cache-off runs. Treat performance as experimental
+until the generated tokens have been validated.

docs/benchmarks/2026-09-29-gfx1100.json

@@ -0,0 +1,299 @@
+{
+  "source_commit": "9568e78da663e6b224ab96d57baf450717b36e73",
+  "date": "2026-09-29",
+  "candidate_binary_sha256": "705f5925a788bc227bad72abfb5466032e0879d752fea9afc3eb32051b35182b",
+  "note": "Fresh same-session control and candidate runs. Throughput cap 128; not a coding-quality benchmark.",
+  "control": [
+    {
+      "kind": "warmup",
+      "trial": 0,
+      "wall_s": 0.45770244899904355,
+      "metrics": {
+        "prompt_tokens": 15.0,
+        "reused": 0.0,
+        "fresh": 15.0,
+        "prefill_ms": 335.0,
+        "prefill_tps": 44.8,
+        "generated": 2.0,
+        "decode_ms": 115.0,
+        "decode_tps": 17.5
+      },
+      "finish_reason": "stop"
+    },
+    {
+      "kind": "cold",
+      "trial": 1,
+      "wall_s": 20.068275763012934,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 17538.0,
+        "prefill_tps": 240.0,
+        "generated": 128.0,
+        "decode_ms": 2518.0,
+        "decode_tps": 50.8
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 1,
+      "wall_s": 4.412000925018219,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 1943.0,
+        "prefill_tps": 58.7,
+        "generated": 128.0,
+        "decode_ms": 2451.0,
+        "decode_tps": 52.2
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 2,
+      "wall_s": 20.731399575975956,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 18213.0,
+        "prefill_tps": 484.8,
+        "generated": 128.0,
+        "decode_ms": 2505.0,
+        "decode_tps": 51.1
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 2,
+      "wall_s": 5.0592734439997,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 2601.0,
+        "prefill_tps": 95.4,
+        "generated": 128.0,
+        "decode_ms": 2441.0,
+        "decode_tps": 52.4
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 3,
+      "wall_s": 11.695648370019626,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 9209.0,
+        "prefill_tps": 457.2,
+        "generated": 128.0,
+        "decode_ms": 2477.0,
+        "decode_tps": 51.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 3,
+      "wall_s": 4.093982424004935,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 1896.0,
+        "prefill_tps": 60.1,
+        "generated": 128.0,
+        "decode_ms": 2188.0,
+        "decode_tps": 58.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 4,
+      "wall_s": 20.84539221401792,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 18437.0,
+        "prefill_tps": 478.9,
+        "generated": 128.0,
+        "decode_ms": 2392.0,
+        "decode_tps": 53.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 4,
+      "wall_s": 5.357054464984685,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 2658.0,
+        "prefill_tps": 93.3,
+        "generated": 128.0,
+        "decode_ms": 2682.0,
+        "decode_tps": 47.7
+      },
+      "finish_reason": "length"
+    }
+  ],
+  "optimized": [
+    {
+      "kind": "warmup",
+      "trial": 0,
+      "wall_s": 0.46265592801501043,
+      "metrics": {
+        "prompt_tokens": 15.0,
+        "reused": 0.0,
+        "fresh": 15.0,
+        "prefill_ms": 335.0,
+        "prefill_tps": 44.8,
+        "generated": 2.0,
+        "decode_ms": 121.0,
+        "decode_tps": 16.5
+      },
+      "finish_reason": "stop"
+    },
+    {
+      "kind": "cold",
+      "trial": 1,
+      "wall_s": 11.728894605010282,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 9408.0,
+        "prefill_tps": 447.5,
+        "generated": 128.0,
+        "decode_ms": 2308.0,
+        "decode_tps": 55.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 1,
+      "wall_s": 2.9531024169991724,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 740.0,
+        "prefill_tps": 154.0,
+        "generated": 128.0,
+        "decode_ms": 2193.0,
+        "decode_tps": 58.4
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 2,
+      "wall_s": 12.42044531099964,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 10107.0,
+        "prefill_tps": 873.7,
+        "generated": 128.0,
+        "decode_ms": 2298.0,
+        "decode_tps": 55.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 2,
+      "wall_s": 3.2599652509961743,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8823.0,
+        "fresh": 248.0,
+        "prefill_ms": 1070.0,
+        "prefill_tps": 231.7,
+        "generated": 128.0,
+        "decode_ms": 2173.0,
+        "decode_tps": 58.9
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 3,
+      "wall_s": 6.937167472002329,
+      "metrics": {
+        "prompt_tokens": 4210.0,
+        "reused": 0.0,
+        "fresh": 4210.0,
+        "prefill_ms": 4672.0,
+        "prefill_tps": 901.1,
+        "generated": 128.0,
+        "decode_ms": 2256.0,
+        "decode_tps": 56.7
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 3,
+      "wall_s": 3.1877827199932653,
+      "metrics": {
+        "prompt_tokens": 4451.0,
+        "reused": 4337.0,
+        "fresh": 114.0,
+        "prefill_ms": 827.0,
+        "prefill_tps": 137.8,
+        "generated": 128.0,
+        "decode_ms": 2350.0,
+        "decode_tps": 54.5
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "cold",
+      "trial": 4,
+      "wall_s": 11.314008287008619,
+      "metrics": {
+        "prompt_tokens": 8830.0,
+        "reused": 0.0,
+        "fresh": 8830.0,
+        "prefill_ms": 9136.0,
+        "prefill_tps": 966.5,
+        "generated": 128.0,
+        "decode_ms": 2161.0,
+        "decode_tps": 59.2
+      },
+      "finish_reason": "length"
+    },
+    {
+      "kind": "followup",
+      "trial": 4,
+      "wall_s": 3.0476601750124246,
+      "metrics": {
+        "prompt_tokens": 9071.0,
+        "reused": 8957.0,
+        "fresh": 114.0,
+        "prefill_ms": 734.0,
+        "prefill_tps": 155.3,
+        "generated": 128.0,
+        "decode_ms": 2296.0,
+        "decode_tps": 55.7
+      },
+      "finish_reason": "length"
+    }
+  ],
+  "control_binary_sha256": "4522d4937ca4a41ca294d31b60df8c2bc4df5b7a6caf65eaad10c036e08491f7"
+}

docs/media/guess-and-check.svg

@@ -0,0 +1,23 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 960 330" width="960" height="330" role="img" aria-label="Speculative decoding: a small helper guesses words, the big model checks them at once"><title>Speculative decoding: a small helper guesses words, the big model checks them at once</title><style>
+@font-face{font-family:"Outfit";src:url(data:font/woff2;base64,d09GMgABAAAAAH4kABQAAAABEvgAAH2tAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGokKG4GFEhyEfD9IVkFShwg/TVZBUlAGYD9TVEFUgR4AhQwvbBEICoGIUOtMC4N0ADCB3i4BNgIkA4dkBCAFhVoHhwAMB1smAVHBNga67bYBAKcffRoLZLBt+OnNYgAVeRfORkSwcSDmgX9Ls//PSU5kjM0Ctl/TNA3MJGWu6mqOhpMJjl69fYTurl4HxsC+V1F1NLEfzQZVm1CuUAwiPmZYiXF0GFOYTsmghiQV7CXfcKZM+TZcm2TjssMkfgwGl7QUhM0ORt82irww786YFU4we4TpaLjbOFnR2Gx+/0aBd6Inf+6DilwyuZCRVyLoNcsA2/VBFBnW4y15iP8/+37VPvFVukdSrqQMlQH9RH7BqiPS7X/2LglJCAFCKC2EEEIMLUREBPSHUAzViIhYGgFEQERARGwgIUIohioC8hBDLZaODRuWhqV+nhQ7NiyvQR/s97+Zs/vu/U00WWiqKYuFJAaESKZmEqVb4v+k0743oBlppJGBNw478JG5uqswG9/9orzqKqoJcAGTOGQ7toCtoX/6e9ie+4Ik0TzkJKiZpmn8Y2CBQMDNH57m9F/il5zJu/PkJCchCgl4kHYdUtG1Ezcy2HfrN7VOpT6jSilWLPvB/aGP5t452D1KCgTGfS1fQBhTozr/ufnfDRoKNaeKv1Ko0QeFaJN8IvDM0CQtZagi2vaLWhnxHkY71tGWt+uZ7T+zeqvx1fhqgOcfu7f3Y05gz0KelU6/5T1pipVbqAvMRQD+wM9pr3WowiiEOBAh/n++ROxb1D7EsYQAKVAKlBpdx5ROrbczLz1vdyp0JnZ3Vf9nZsKcz3M59rSvu2c3RrOZjBHBqKkI6sf8BpEKYx7CiOCuQ7hZ/wNUDKlTcbqu3b12T1XYu/dNT8RWtCqYFU2wEOKEBIKY79vm5SNvZCxNQ7AiE5HbeTqcaGBAmga6iV1dXdj7/+XW917SWpf62i6RP+JeElI0wUPDmVmZsoXtqAtB7vjAHqElbS2wFmJbgEURP2jEZq6wR3wmhoFNdjNLyDQJloAzSBwKhRKfbIJ4nJjHGIxEuC7OKYw5Wz1APhAlHc9/v7d+dt4sQncTDynLT7X8wiAMzt+5b4YJG7KCRw5T9RQ5WxwaFEJSvyGrkKREacH/3+ustu/r6YII7pbsCt+unpmnjo+J6rwcG6tz3k3Iq149/Y/gSx/hD9guCZniA6bORxTVH6jwwRWwJHxADolTx3ZnUyGFC9hTH1NVI7D7DNXR1XFCiK7JcReXIZ59Covl7GZWq5R2vZ/VfvbL2Y///Vqfdk6f3x/4LnLX1uwsTa0wINQ4OfPe0uvzid8CJ71AHSK1pCJRhmWEAUSZio5RiQpUm5KzC6ZJCgkSHpFD5LCpH57m+376zJTc9p2liBCEEcYYMQghhBCuMSKY7POr6/drGp9qqUzyJlmkWNDDZTOMTV2+wyhAcsKQ0gaTUCfZWW0ZB1/mCBxj7t8zsTfGtrEIEJGIEO8bjm9tjvTgOWoQGiQaQn52aFiM2A/8+t+FgEcHYBiUHiiLMJFDZWbEx7JgUM8JCwbJsAsIBHg4QIJ0DUBLrGVCSbgRYGHAR5yFBIcNgKozHwkODYCqdQsSjPkFUDWXIMEwL4Cq/goSDG0AVLU4JBjiAKjKFCRIzwBUkYEEKUl21sgYkMfR2yZ3tYNh4Oy2zlmNmbQ0beNnVaY6FQ3pv1bnRYjLBA3etNsas6wdxxCI6pkBSUPNbt1oBRdqm1fXXtpMfaXK77yFCYQM0WYAMWQIMWYOscSCsPEhjlwhbtwg7iQQX34QqWBIqFCIzGLIEkshy62FyEVD4qyHJEiGpEiBpNnS+BTmentXkTETcO+8h0MQ0IIQjRk3sfSBRaAcCgGUXQEPA8cDe8JhMCDsjj8oCBV1cT5rgYQ+mv74brKPVHiS2pMcOUvLOdTnujXWxYk6JuuY4u1Ur2YHYaEr8+xe4p+lKFcYWxWzL8PsWuUG1VsU9iPhYkcuCavHCLvSmaucv84poADxu1/TRkLcJmeekZ+tnGPSo8iJooieY1OyVLmczrI8yfO8EZIYY4oZsUgj0ormZTHKqE6tP0EGitN4ElOcx3zJVQVHESHTlNI8naWckppVNrnIbXH50qpqj3zCcBB4KAoYKjQI6NBjwLimWViLdSYRU6g0VmwY2LHnlpdtO+x+75XnIpWpUq1GQb1GLbTSoYve9X6UK244cM8jL3jFL7cWkUaC9DLIKAtZyca3fulPGrTXUVe60a3udK8HPelN7/rQX3G3nR3sZkKDAoWgV6AXkAK7INZpEWqFVpFJLBbrxaTUKCWldmlampEhspAs3aXpqncHFbx6WkMbDhobyF58iuj3LYQOZQ6NTNZnYk7IQEC//UfJNDpjFSAYOJAYm5gYBNDBJ4NNmDRjAfqOtZf10REAmQoIGHz7xkv6OP/Fmcw+trsJ8y91TS8fNTZfG2stKvpJNCoSPQgViObkOAlqbkdD5EGIxwIN7kLhjQqoMKYXQOVSkTYVLYbLXNWoaiW4ql9A3cEH6i2uY22HIYLdKDfV0FiTi5kPPNFW2+2w0y7ECPaRmggNxggJHucchtqMhBwegErlvCEhxZaHtIXz5cqjzE8Rb9SuX+CC0dveU+cG5pyQe6oOEBKsu7hu0/gG0tLp1ItZ8W+8Nk5+dTWGqK1EcEQE9Ao0CcFFqGRtZehdDhziSkhCzTstynR0eC1oy8UkGIolMawinWUIJYm2mMduHZ3DCRGHajdYkGNddS77pABMbBiptWOh/vfEuaZ8naUP4z3SQ32km7qylZ1ei20/d0UtuObVouY3q5lNxX/XVI3Xs7pb5+tYdYW/Q1PVpS5lpVeJ1YoquPIqUfGLVcyiYr8zlfGMZChH0pTKKJNeYssKHh5cJMU9TuGu/ynMUDFAf/e9zxz2lH02WGWxCjMjOdaELMQhlKeFDMnw06QX7rtoAK6+lEC94B0yo0Qbj2GwFTy/96/iCiv656YqmI269a7a34loMjYp3nNWgFQUlaeXT4VfcWisygUWleqeAAwkpiu0Fk1X1ZMUTBpZQzk9dG0ORbKyHFaR8urpk8GIVrCgFTi0Ap9WkEgnGjrGdEY6i+Angkmr83D7QADXgMzSfE2yTppua6a1hYtYehwzjsR0Vbe1F1mixKdDKTiUgSPfiQdVpFbrO7zfguddFEYq/gCpGPr54iqWdRgzMNVd7IoHLdwaBCeQE9Cs7ahPChbx3KYKpWIGaMqNPlXyHir0+1lVNMsDDPLJ9ZqYBjGpBJ058wWq5DLN/SeqDY9dQw0sGkTHqorm8D3ojCTRGCBRFnNomEgWyhl4fVGxt/wQUypcRGZqcDH5JL2QGZtpLgNiZ8kn/o149gV5bbu0Ym+G4W5V0Wz8E7lLaAOnhNYp9QyRm1ymAd/98BjLExYxTbmYLtNDOemrotm8F8Y1N1qF6lApDlXhyLvwYIAMBc2haW9dmCLD3pFiESoSLi6q2GS1dthIo4Bkho/TrFqzlupHGsiElg6Jt6YnYzqhsN8J2W3HVhcNeth3NEQemC0iGQyMJFPfGFg0pgmtMWOxNwFd6WaeFsIrLCEsbZKMjfBpokAYxBAuUvjAqMhzMix6MQmn4/B891PX9XOXwKEO/Bwe7GUmBDzAKpgxBL0HyqUSR5RmGUQHYMiwVo1/vNZ78AXNh4dSmBjunffaVnmqfYxuwgvWIrtr/eYE03a/6K22K7RIV99vjza3f0Z6/HWTgfaXnO/vQ16j3SJNMP0bJXSjAdDybvLPz7Nt9xz/sPL5/W9teM7L7UyZNfcIepbqdaoa3jiu0fnonMemvb99yw9/6h7BZfq/GsgeLdu2fBnVr2ut71WjjWN0UdNofdee8+pgtHT21FQurRX5p913/7dKAz1z0ouDVjG6bOUSCxiNtdsqXBw34WUnvze3502V1m7aerHVta+mW916XLb6kM5m+MMAlr7HLko6U+RltyLQU7HXajv554cXvnER3XejmdNLu76/Ff9yFqjYNr2etMTnvtK7tT0zpdFMlXPO1042b1Lyuk/w7pvy8v2u922RjpO/v992uRPUVaXQUN1PcM69E9jPt/IJ5yY8c8dvwe0nzj+MTJwo9eOW1itAEL9VhqjLC3W/qkR7Kvtai3w9yb050yNIrWtTXi05i9bt+f69133KOfS4un/raPI9y59Q6P8rWY6RbOFeR3fcvlVqujXu6tE0Ul56CyhlqjlGV2P6guh+nZ6BTqfe9IwZ62o6R925W2r4xMLERl+TkZguQlm70j7O2f3Wmy3TXiuVixHtG9njX2+1o+pR/aJfxXRWM47ypkseclXSdYFj0+gsUE5Ng0JPkKZh9kTdxx/GtjtEj62xEBRXJqGiwkosCHQwbYYMzYj5BUvWsHJbwQTsg0MzAkfDOJephFyR60Ba1OF58RN8dFWLL//rnxDCMOUSqHqBBbRE7qllmRVwK6+PEQu3DkPiDJdQT5EoGdGmQlua9D6fYdvKttvpKvlwPQ0O0XXKRnemnu6Ci/RdMuSGYaYLOgz73kCGozwdocUkgFjwaZkhSAgl0kFEjXZxAzb5oZUVekFBpQzoQjksWP7ISDHoBnRk5LwlPXHpDEnpOt10RhHL6Ubl2OlCQQZf4MaXaGG80gtCjfhq+RwLTzheczxy2rOCT/Hz/E78k0HLjtNrX+DW7o8Zgd49iVpCtG5FE1vhtycRxPUXWn7Pf8//RymdGHJRezffyufZxY1Q53Nif6XzDScbnmv4IXWc/JnYgz/ZPlTn/pzMx9s4lkPpF7F/wW7Y+rru1+hjW03Sv9ehv5ral/h/hRT7Wuzr2DUbxBuYH0b017i0fDXc2fLLcFPYIhn64mA87DDBsCA0Line truncated
+text{font-family:"Outfit",system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;fill:#221e1f}
+.bg{fill:#f9fafb}.card{fill:#ffffff;stroke:#dee5ee;stroke-width:1}.soft{fill:#f7f7f7}
+.muted{fill:#6f6f6f}.ink{fill:#221e1f}.icon{color:#2F384C}
+.acc{fill:#10b981}.acc-t{fill:#0c8a60}.acc-tint{fill:#ecfdf5}.acc-line{stroke:#10b981}
+.info{fill:#0f97ff}.info-t{fill:#0668c2}.info-tint{fill:#e8f4ff}
+.warn{fill:#e39b0b}.warn-t{fill:#8a5a00}.warn-tint{fill:#fff6e0}
+.dan{fill:#e0283f}.dan-tint{fill:#fdecee}.dan-t{fill:#b01a2e}
+.dot{fill:#dee5ee}.line{stroke:#dee5ee}.arrow{stroke:#6f6f6f;fill:none}.arrowhead{fill:#6f6f6f}
+.dash{stroke:#6f6f6f;fill:none;stroke-dasharray:4 4}
+.on-acc{fill:#ffffff}.icon-acc{color:#0c8a60}.icon-dan{color:#b01a2e}
+@media (prefers-color-scheme:dark){
+text,.ink{fill:#eef1f3}.bg{fill:#0e1113}.card{fill:#161a1d;stroke:#2a3036}.soft{fill:#1d2226}
+.muted{fill:#8e979f}.icon{color:#c7cdd3}
+.acc{fill:#34d399}.acc-t{fill:#4ade80}.acc-tint{fill:rgba(52,211,153,.12)}.acc-line{stroke:#34d399}
+.info{fill:#3aa9ff}.info-t{fill:#7cc4ff}.info-tint{fill:rgba(58,169,255,.12)}
+.warn{fill:#f5b53d}.warn-t{fill:#f8c96a}.warn-tint{fill:rgba(245,181,61,.12)}
+.dan{fill:#ff5a6e}.dan-tint{fill:rgba(255,90,110,.12)}.dan-t{fill:#ff8a98}
+.dot{fill:#2a3036}.line{stroke:#2a3036}.arrow{stroke:#8e979f}.arrowhead{fill:#8e979f}.dash{stroke:#8e979f}
+.on-acc{fill:#052e20}.icon-acc{color:#4ade80}.icon-dan{color:#ff8a98}
+}
+</style><rect x="0" y="0" width="960" height="330" rx="20" class="bg"/><rect x="24" y="24" width="912" height="282" rx="14" class="card"/><g class="icon" transform="translate(48,48) scale(1.2500)"><path opacity="0.12" d="M12 2.5C8.41 2.5 5.5 5.41 5.5 9C5.5 11.3 6.7 13.3 8.5 14.5V16.5C8.5 17.05 8.95 17.5 9.5 17.5H14.5C15.05 17.5 15.5 17.05 15.5 16.5V14.5C17.3 13.3 18.5 11.3 18.5 9C18.5 5.41 15.59 2.5 12 2.5Z" fill="currentColor"></path><path d="M12 2.5C8.41 2.5 5.5 5.41 5.5 9C5.5 11.3 6.7 13.3 8.5 14.5V16.5C8.5 17.05 8.95 17.5 9.5 17.5H14.5C15.05 17.5 15.5 17.05 15.5 16.5V14.5C17.3 13.3 18.5 11.3 18.5 9C18.5 5.41 15.59 2.5 12 2.5Z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><path d="M9.5 20.5H14.5" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></g><text x="92" y="70" font-size="19" font-weight="700" class="ink" text-anchor="start">Guess, then check - many words per step</text><text x="92" y="94" font-size="15" font-weight="400" class="muted" text-anchor="start">A small, fast helper inside the model guesses the next words. The big model checks all the guesses in one go</text><text x="92" y="113" font-size="15" font-weight="400" class="muted" text-anchor="start">and keeps the ones it agrees with. The big model decides every word - it just arrives 1.6-1.8x sooner.</text><text x="48" y="180" font-size="14" font-weight="600" class="muted" text-anchor="start">Helper guesses</text><text x="48" y="262" font-size="14" font-weight="600" class="muted" text-anchor="start">Big model checks</text><rect x="176" y="156" width="61" height="38" rx="10" class="dash" stroke-width="1.4"/><text x="206.5" y="181" font-size="16" font-weight="500" class="ink" text-anchor="middle">def</text><rect x="251" y="156" width="97" height="38" rx="10" class="dash" stroke-width="1.4"/><text x="299.5" y="181" font-size="16" font-weight="500" class="ink" text-anchor="middle">reverse</text><rect x="362" y="156" width="88" height="38" rx="10" class="dash" stroke-width="1.4"/><text x="406.0" y="181" font-size="16" font-weight="500" class="ink" text-anchor="middle">(items</text><rect x="464" y="156" width="52" height="38" rx="10" class="dash" stroke-width="1.4"/><text x="490.0" y="181" font-size="16" font-weight="500" class="ink" text-anchor="middle">):</text><rect x="176" y="238" width="61" height="38" rx="10" class="acc-tint"/><text x="198.5" y="263" font-size="16" font-weight="600" class="acc-t" text-anchor="middle">def</text><g class="icon-acc" transform="translate(215,249) scale(0.6667)"><path d="M4.5 12.5L9.5 17.5L19.5 6.5" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></g><line x1="206.5" y1="196" x2="206.5" y2="234" class="line" stroke-width="1.4"/><rect x="251" y="238" width="97" height="38" rx="10" class="acc-tint"/><text x="291.5" y="263" font-size="16" font-weight="600" class="acc-t" text-anchor="middle">reverse</text><g class="icon-acc" transform="translate(326,249) scale(0.6667)"><path d="M4.5 12.5L9.5 17.5L19.5 6.5" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></g><line x1="299.5" y1="196" x2="299.5" y2="234" class="line" stroke-width="1.4"/><rect x="362" y="238" width="88" height="38" rx="10" class="acc-tint"/><text x="398.0" y="263" font-size="16" font-weight="600" class="acc-t" text-anchor="middle">(items</text><g class="icon-acc" transform="translate(428,249) scale(0.6667)"><path d="M4.5 12.5L9.5 17.5L19.5 6.5" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></g><line x1="406.0" y1="196" x2="406.0" y2="234" class="line" stroke-width="1.4"/><rect x="464" y="238" width="52" height="38" rx="10" class="dan-tint"/><text x="482.0" y="263" font-size="16" font-weight="600" class="dan-t" text-anchor="middle">):</text><g class="icon-dan" transform="translate(494,249) scale(0.6667)"><path opacity="0.12" d="M22 12C22 17.5228 17.5228 22 12 22C6.47715 22 2 17.5228 2 12C2 6.47715 6.47715 2 12 2C17.5228 2 22 6.47715 22 12Z" fill="currentColor"></path><path d="M22 12C22 17.5228 17.5228 22 12 22C6.47715 22 2 17.5228 2 12C2 6.47715 6.47715 2 12 2C17.5228 2 22 6.47715 22 12Z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><path d="M15 9L9 15M9 9L15 15" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></g><rect x="526" y="238" width="89" height="38" rx="10" class="acc-tint"/><text x="570.5" y="263" font-size="16" font-weight="600" class="acc-t" text-anchor="middle">, key):</text><line x1="490.0" y1="196" x2="490.0" y2="234" class="line" stroke-width="1.4"/><line x1="655" y1="257" x2="678" y2="257" class="arrow" stroke-width="1.6" stroke-linecap="round"/><path d="M685 257 l-8 -5 v10 z" class="arrowhead"/><rect x="697" y="232" width="215" height="50" rx="12" class="acc"/><g class="icon" style="color:#ffffff" transformLine truncated

docs/media/how-it-works.svg

@@ -0,0 +1,23 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 960 470" width="960" height="470" role="img" aria-label="How Strata splits the model between your graphics card, RAM and SSD"><title>How Strata splits the model between your graphics card, RAM and SSD</title><style>
+@font-face{font-family:"Outfit";src:url(data:font/woff2;base64,d09GMgABAAAAAH4kABQAAAABEvgAAH2tAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGokKG4GFEhyEfD9IVkFShwg/TVZBUlAGYD9TVEFUgR4AhQwvbBEICoGIUOtMC4N0ADCB3i4BNgIkA4dkBCAFhVoHhwAMB1smAVHBNga67bYBAKcffRoLZLBt+OnNYgAVeRfORkSwcSDmgX9Ls//PSU5kjM0Ctl/TNA3MJGWu6mqOhpMJjl69fYTurl4HxsC+V1F1NLEfzQZVm1CuUAwiPmZYiXF0GFOYTsmghiQV7CXfcKZM+TZcm2TjssMkfgwGl7QUhM0ORt82irww786YFU4we4TpaLjbOFnR2Gx+/0aBd6Inf+6DilwyuZCRVyLoNcsA2/VBFBnW4y15iP8/+37VPvFVukdSrqQMlQH9RH7BqiPS7X/2LglJCAFCKC2EEEIMLUREBPSHUAzViIhYGgFEQERARGwgIUIohioC8hBDLZaODRuWhqV+nhQ7NiyvQR/s97+Zs/vu/U00WWiqKYuFJAaESKZmEqVb4v+k0743oBlppJGBNw478JG5uqswG9/9orzqKqoJcAGTOGQ7toCtoX/6e9ie+4Ik0TzkJKiZpmn8Y2CBQMDNH57m9F/il5zJu/PkJCchCgl4kHYdUtG1Ezcy2HfrN7VOpT6jSilWLPvB/aGP5t452D1KCgTGfS1fQBhTozr/ufnfDRoKNaeKv1Ko0QeFaJN8IvDM0CQtZagi2vaLWhnxHkY71tGWt+uZ7T+zeqvx1fhqgOcfu7f3Y05gz0KelU6/5T1pipVbqAvMRQD+wM9pr3WowiiEOBAh/n++ROxb1D7EsYQAKVAKlBpdx5ROrbczLz1vdyp0JnZ3Vf9nZsKcz3M59rSvu2c3RrOZjBHBqKkI6sf8BpEKYx7CiOCuQ7hZ/wNUDKlTcbqu3b12T1XYu/dNT8RWtCqYFU2wEOKEBIKY79vm5SNvZCxNQ7AiE5HbeTqcaGBAmga6iV1dXdj7/+XW917SWpf62i6RP+JeElI0wUPDmVmZsoXtqAtB7vjAHqElbS2wFmJbgEURP2jEZq6wR3wmhoFNdjNLyDQJloAzSBwKhRKfbIJ4nJjHGIxEuC7OKYw5Wz1APhAlHc9/v7d+dt4sQncTDynLT7X8wiAMzt+5b4YJG7KCRw5T9RQ5WxwaFEJSvyGrkKREacH/3+ustu/r6YII7pbsCt+unpmnjo+J6rwcG6tz3k3Iq149/Y/gSx/hD9guCZniA6bORxTVH6jwwRWwJHxADolTx3ZnUyGFC9hTH1NVI7D7DNXR1XFCiK7JcReXIZ59Covl7GZWq5R2vZ/VfvbL2Y///Vqfdk6f3x/4LnLX1uwsTa0wINQ4OfPe0uvzid8CJ71AHSK1pCJRhmWEAUSZio5RiQpUm5KzC6ZJCgkSHpFD5LCpH57m+376zJTc9p2liBCEEcYYMQghhBCuMSKY7POr6/drGp9qqUzyJlmkWNDDZTOMTV2+wyhAcsKQ0gaTUCfZWW0ZB1/mCBxj7t8zsTfGtrEIEJGIEO8bjm9tjvTgOWoQGiQaQn52aFiM2A/8+t+FgEcHYBiUHiiLMJFDZWbEx7JgUM8JCwbJsAsIBHg4QIJ0DUBLrGVCSbgRYGHAR5yFBIcNgKozHwkODYCqdQsSjPkFUDWXIMEwL4Cq/goSDG0AVLU4JBjiAKjKFCRIzwBUkYEEKUl21sgYkMfR2yZ3tYNh4Oy2zlmNmbQ0beNnVaY6FQ3pv1bnRYjLBA3etNsas6wdxxCI6pkBSUPNbt1oBRdqm1fXXtpMfaXK77yFCYQM0WYAMWQIMWYOscSCsPEhjlwhbtwg7iQQX34QqWBIqFCIzGLIEkshy62FyEVD4qyHJEiGpEiBpNnS+BTmentXkTETcO+8h0MQ0IIQjRk3sfSBRaAcCgGUXQEPA8cDe8JhMCDsjj8oCBV1cT5rgYQ+mv74brKPVHiS2pMcOUvLOdTnujXWxYk6JuuY4u1Ur2YHYaEr8+xe4p+lKFcYWxWzL8PsWuUG1VsU9iPhYkcuCavHCLvSmaucv84poADxu1/TRkLcJmeekZ+tnGPSo8iJooieY1OyVLmczrI8yfO8EZIYY4oZsUgj0ormZTHKqE6tP0EGitN4ElOcx3zJVQVHESHTlNI8naWckppVNrnIbXH50qpqj3zCcBB4KAoYKjQI6NBjwLimWViLdSYRU6g0VmwY2LHnlpdtO+x+75XnIpWpUq1GQb1GLbTSoYve9X6UK244cM8jL3jFL7cWkUaC9DLIKAtZyca3fulPGrTXUVe60a3udK8HPelN7/rQX3G3nR3sZkKDAoWgV6AXkAK7INZpEWqFVpFJLBbrxaTUKCWldmlampEhspAs3aXpqncHFbx6WkMbDhobyF58iuj3LYQOZQ6NTNZnYk7IQEC//UfJNDpjFSAYOJAYm5gYBNDBJ4NNmDRjAfqOtZf10REAmQoIGHz7xkv6OP/Fmcw+trsJ8y91TS8fNTZfG2stKvpJNCoSPQgViObkOAlqbkdD5EGIxwIN7kLhjQqoMKYXQOVSkTYVLYbLXNWoaiW4ql9A3cEH6i2uY22HIYLdKDfV0FiTi5kPPNFW2+2w0y7ECPaRmggNxggJHucchtqMhBwegErlvCEhxZaHtIXz5cqjzE8Rb9SuX+CC0dveU+cG5pyQe6oOEBKsu7hu0/gG0tLp1ItZ8W+8Nk5+dTWGqK1EcEQE9Ao0CcFFqGRtZehdDhziSkhCzTstynR0eC1oy8UkGIolMawinWUIJYm2mMduHZ3DCRGHajdYkGNddS77pABMbBiptWOh/vfEuaZ8naUP4z3SQ32km7qylZ1ei20/d0UtuObVouY3q5lNxX/XVI3Xs7pb5+tYdYW/Q1PVpS5lpVeJ1YoquPIqUfGLVcyiYr8zlfGMZChH0pTKKJNeYssKHh5cJMU9TuGu/ynMUDFAf/e9zxz2lH02WGWxCjMjOdaELMQhlKeFDMnw06QX7rtoAK6+lEC94B0yo0Qbj2GwFTy/96/iCiv656YqmI269a7a34loMjYp3nNWgFQUlaeXT4VfcWisygUWleqeAAwkpiu0Fk1X1ZMUTBpZQzk9dG0ORbKyHFaR8urpk8GIVrCgFTi0Ap9WkEgnGjrGdEY6i+Angkmr83D7QADXgMzSfE2yTppua6a1hYtYehwzjsR0Vbe1F1mixKdDKTiUgSPfiQdVpFbrO7zfguddFEYq/gCpGPr54iqWdRgzMNVd7IoHLdwaBCeQE9Cs7ahPChbx3KYKpWIGaMqNPlXyHir0+1lVNMsDDPLJ9ZqYBjGpBJ058wWq5DLN/SeqDY9dQw0sGkTHqorm8D3ojCTRGCBRFnNomEgWyhl4fVGxt/wQUypcRGZqcDH5JL2QGZtpLgNiZ8kn/o149gV5bbu0Ym+G4W5V0Wz8E7lLaAOnhNYp9QyRm1ymAd/98BjLExYxTbmYLtNDOemrotm8F8Y1N1qF6lApDlXhyLvwYIAMBc2haW9dmCLD3pFiESoSLi6q2GS1dthIo4Bkho/TrFqzlupHGsiElg6Jt6YnYzqhsN8J2W3HVhcNeth3NEQemC0iGQyMJFPfGFg0pgmtMWOxNwFd6WaeFsIrLCEsbZKMjfBpokAYxBAuUvjAqMhzMix6MQmn4/B891PX9XOXwKEO/Bwe7GUmBDzAKpgxBL0HyqUSR5RmGUQHYMiwVo1/vNZ78AXNh4dSmBjunffaVnmqfYxuwgvWIrtr/eYE03a/6K22K7RIV99vjza3f0Z6/HWTgfaXnO/vQ16j3SJNMP0bJXSjAdDybvLPz7Nt9xz/sPL5/W9teM7L7UyZNfcIepbqdaoa3jiu0fnonMemvb99yw9/6h7BZfq/GsgeLdu2fBnVr2ut71WjjWN0UdNofdee8+pgtHT21FQurRX5p913/7dKAz1z0ouDVjG6bOUSCxiNtdsqXBw34WUnvze3502V1m7aerHVta+mW916XLb6kM5m+MMAlr7HLko6U+RltyLQU7HXajv554cXvnER3XejmdNLu76/Ff9yFqjYNr2etMTnvtK7tT0zpdFMlXPO1042b1Lyuk/w7pvy8v2u922RjpO/v992uRPUVaXQUN1PcM69E9jPt/IJ5yY8c8dvwe0nzj+MTJwo9eOW1itAEL9VhqjLC3W/qkR7Kvtai3w9yb050yNIrWtTXi05i9bt+f69133KOfS4un/raPI9y59Q6P8rWY6RbOFeR3fcvlVqujXu6tE0Ul56CyhlqjlGV2P6guh+nZ6BTqfe9IwZ62o6R925W2r4xMLERl+TkZguQlm70j7O2f3Wmy3TXiuVixHtG9njX2+1o+pR/aJfxXRWM47ypkseclXSdYFj0+gsUE5Ng0JPkKZh9kTdxx/GtjtEj62xEBRXJqGiwkosCHQwbYYMzYj5BUvWsHJbwQTsg0MzAkfDOJephFyR60Ba1OF58RN8dFWLL//rnxDCMOUSqHqBBbRE7qllmRVwK6+PEQu3DkPiDJdQT5EoGdGmQlua9D6fYdvKttvpKvlwPQ0O0XXKRnemnu6Ci/RdMuSGYaYLOgz73kCGozwdocUkgFjwaZkhSAgl0kFEjXZxAzb5oZUVekFBpQzoQjksWP7ISDHoBnRk5LwlPXHpDEnpOt10RhHL6Ubl2OlCQQZf4MaXaGG80gtCjfhq+RwLTzheczxy2rOCT/Hz/E78k0HLjtNrX+DW7o8Zgd49iVpCtG5FE1vhtycRxPUXWn7Pf8//RymdGHJRezffyufZxY1Q53Nif6XzDScbnmv4IXWc/JnYgz/ZPlTn/pzMx9s4lkPpF7F/wW7Y+rru1+hjW03Sv9ehv5ral/h/hRT7Wuzr2DUbxBuYH0b017i0fDXc2fLLcFPYIhn64mA87DDBsCA0Line truncated
+text{font-family:"Outfit",system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;fill:#221e1f}
+.bg{fill:#f9fafb}.card{fill:#ffffff;stroke:#dee5ee;stroke-width:1}.soft{fill:#f7f7f7}
+.muted{fill:#6f6f6f}.ink{fill:#221e1f}.icon{color:#2F384C}
+.acc{fill:#10b981}.acc-t{fill:#0c8a60}.acc-tint{fill:#ecfdf5}.acc-line{stroke:#10b981}
+.info{fill:#0f97ff}.info-t{fill:#0668c2}.info-tint{fill:#e8f4ff}
+.warn{fill:#e39b0b}.warn-t{fill:#8a5a00}.warn-tint{fill:#fff6e0}
+.dan{fill:#e0283f}.dan-tint{fill:#fdecee}.dan-t{fill:#b01a2e}
+.dot{fill:#dee5ee}.line{stroke:#dee5ee}.arrow{stroke:#6f6f6f;fill:none}.arrowhead{fill:#6f6f6f}
+.dash{stroke:#6f6f6f;fill:none;stroke-dasharray:4 4}
+.on-acc{fill:#ffffff}.icon-acc{color:#0c8a60}.icon-dan{color:#b01a2e}
+@media (prefers-color-scheme:dark){
+text,.ink{fill:#eef1f3}.bg{fill:#0e1113}.card{fill:#161a1d;stroke:#2a3036}.soft{fill:#1d2226}
+.muted{fill:#8e979f}.icon{color:#c7cdd3}
+.acc{fill:#34d399}.acc-t{fill:#4ade80}.acc-tint{fill:rgba(52,211,153,.12)}.acc-line{stroke:#34d399}
+.info{fill:#3aa9ff}.info-t{fill:#7cc4ff}.info-tint{fill:rgba(58,169,255,.12)}
+.warn{fill:#f5b53d}.warn-t{fill:#f8c96a}.warn-tint{fill:rgba(245,181,61,.12)}
+.dan{fill:#ff5a6e}.dan-tint{fill:rgba(255,90,110,.12)}.dan-t{fill:#ff8a98}
+.dot{fill:#2a3036}.line{stroke:#2a3036}.arrow{stroke:#8e979f}.arrowhead{fill:#8e979f}.dash{stroke:#8e979f}
+.on-acc{fill:#052e20}.icon-acc{color:#4ade80}.icon-dan{color:#ff8a98}
+}
+</style><rect x="0" y="0" width="960" height="470" rx="20" class="bg"/><rect x="24" y="24" width="912" height="92" rx="14" class="card"/><g class="icon" transform="translate(48,50) scale(1.5000)"><path opacity="0.12" d="M21 11.5C21 16.194 16.971 20 12 20C10.8 20 9.65 19.78 8.6 19.37L4 20.5L5.2 16.8C3.83 15.34 3 13.5 3 11.5C3 6.806 7.029 3 12 3C16.971 3 21 6.806 21 11.5Z" fill="currentColor"></path><path d="M21 11.5C21 16.194 16.971 20 12 20C10.8 20 9.65 19.78 8.6 19.37L4 20.5L5.2 16.8C3.83 15.34 3 13.5 3 11.5C3 6.806 7.029 3 12 3C16.971 3 21 6.806 21 11.5Z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><path d="M8 11.5H8.01M12 11.5H12.01M16 11.5H16.01" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></g><text x="100" y="62" font-size="19" font-weight="600" class="ink" text-anchor="start">The model is made of 24,576 small specialists - “experts”.</text><text x="100" y="88" font-size="15" font-weight="400" class="muted" text-anchor="start">For each word it writes, it asks only 10 of them. Strata keeps the busy ones</text><text x="100" y="106" font-size="15" font-weight="400" class="muted" text-anchor="start">on your graphics card and the rest in RAM - so it runs on a normal PC.</text><circle cx="698.0" cy="46.0" r="2.6" class="dot"/><circle cx="705.6" cy="46.0" r="2.6" class="dot"/><circle cx="713.2" cy="46.0" r="2.6" class="dot"/><circle cx="720.8" cy="46.0" r="2.6" class="dot"/><circle cx="728.4" cy="46.0" r="2.6" class="dot"/><circle cx="736.0" cy="46.0" r="2.6" class="dot"/><circle cx="743.6" cy="46.0" r="2.6" class="dot"/><circle cx="751.2" cy="46.0" r="2.6" class="dot"/><circle cx="758.8" cy="46.0" r="2.6" class="acc"/><circle cx="766.4" cy="46.0" r="2.6" class="dot"/><circle cx="774.0" cy="46.0" r="2.6" class="dot"/><circle cx="781.6" cy="46.0" r="2.6" class="dot"/><circle cx="789.2" cy="46.0" r="2.6" class="dot"/><circle cx="796.8" cy="46.0" r="2.6" class="dot"/><circle cx="804.4" cy="46.0" r="2.6" class="dot"/><circle cx="812.0" cy="46.0" r="2.6" class="dot"/><circle cx="819.6" cy="46.0" r="2.6" class="acc"/><circle cx="827.2" cy="46.0" r="2.6" class="dot"/><circle cx="834.8" cy="46.0" r="2.6" class="dot"/><circle cx="842.4" cy="46.0" r="2.6" class="dot"/><circle cx="850.0" cy="46.0" r="2.6" class="dot"/><circle cx="857.6" cy="46.0" r="2.6" class="dot"/><circle cx="865.2" cy="46.0" r="2.6" class="dot"/><circle cx="872.8" cy="46.0" r="2.6" class="dot"/><circle cx="880.4" cy="46.0" r="2.6" class="dot"/><circle cx="888.0" cy="46.0" r="2.6" class="dot"/><circle cx="895.6" cy="46.0" r="2.6" class="dot"/><circle cx="903.2" cy="46.0" r="2.6" class="dot"/><circle cx="698.0" cy="53.6" r="2.6" class="dot"/><circle cx="705.6" cy="53.6" r="2.6" class="dot"/><circle cx="713.2" cy="53.6" r="2.6" class="acc"/><circle cx="720.8" cy="53.6" r="2.6" class="dot"/><circle cx="728.4" cy="53.6" r="2.6" class="dot"/><circle cx="736.0" cy="53.6" r="2.6" class="dot"/><circle cx="743.6" cy="53.6" r="2.6" class="dot"/><circle cx="751.2" cy="53.6" r="2.6" class="dot"/><circle cx="758.8" cy="53.6" r="2.6" class="dot"/><circle cx="766.4" cy="53.6" r="2.6" class="dot"/><circle cx="774.0" cy="53.6" r="2.6" class="dot"/><circle cx="781.6" cy="53.6" r="2.6" class="dot"/><circle cx="789.2" cy="53.6" r="2.6" class="dot"/><circle cx="796.8" cy="53.6" r="2.6" class="dot"/><circle cx="804.4" cy="53.6" r="2.6" class="dot"/><circle cx="812.0" cy="53.6" r="2.6" class="dot"/><circle cx="819.6" cy="53.6" r="2.6" class="dot"/><circle cx="827.2" cy="53.6" r="2.6" class="dot"/><circle cx="834.8" cy="53.6" r="2.6" class="dot"/><circle cx="842.4" cy="53.6" r="2.6" class="acc"/><circle cx="850.0" cy="53.6" r="2.6" class="dot"/><circle cx="857.6" cy="53.6" r="2.6" class="dot"/><circle cx="865.2" cy="53.6" r="2.6" class="dot"/><circle cx="872.8" cy="53.6" r="2.6" class="dot"/><circle cx="880.4" cy="53.6" r="2.6" class="dot"/><circle cx="888.0" cy="53.6" r="2.6" class="dot"/><circle cx="895.6" cy="53.6" r="2.6" class="dot"/><circle cx="903.2" cy="53.6" r="2.6" class="dot"/><circle cx="698.0" cy="61.2" r="2.6" class="dot"/><circle cx="705.6" cy="61.2" r="2.6" class="dot"/><circle cx="713.2" cy="61.2" r="2.6" class="dot"/><circle cx="720.8" cy="61.2" r="2.6" class="dot"/><circle cx="728.4" cy="61.2" r="2.6" class="acc"/><circle cx="736.0" cy="61.2" r="2.6" class="dot"/><circle cx="743.6" cy="61.2" r="2.6" class="dot"/><circle cx="751.2" cy="61.2" r="2.6" class="dot"/><circle cx="758.8" cy="61.2" r="2.6" class="dot"/><circle cx="766.4" cy="61.2" r="2.6" class="dot"/><circle cx="774.0" cy="61.2" r="2.6" class="dot"/><circle cx="781.6" cy="61.2" r="2.6" class="dot"/><circle cx="789.2" cy="61.2" r="2.6" class="dot"/><circle cx="796.8" cy="61.2" r="2.6" class="acc"/><circle cx="804.4" cy="61.2" r="2.6" class="dot"/><circle cx="812.0" cy="61.2" r="2.6" class="dot"/><circle cx="819.6" cy="61.2" r="2.6" class="dot"/><circle cx="827.2" cy="61.2" r="2.6" class="dot"Line truncated

docs/media/make_figures.py

@@ -0,0 +1,178 @@
+"""docs/media/make_figures.py - the README's "How does it work?" illustrations, in the Strata app's style: its colors
+(serve/web/tokens.css), its icons (serve/web/sprite.svg) and its font (Outfit, embedded, SIL OFL 1.1).  Light and dark
+follow the reader's system setting.
+
+    python docs/media/make_figures.py        # writes docs/media/how-it-works.svg and docs/media/guess-and-check.svg
+"""
+import base64
+import random
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+OUT = ROOT / "docs" / "media"
+SPRITE = (ROOT / "serve" / "web" / "sprite.svg").read_text(encoding="utf-8")
+FONT = base64.b64encode((ROOT / "serve" / "web" / "fonts" / "outfit-latin-wght.woff2").read_bytes()).decode()
+
+# the app's tokens (light, then dark)
+CSS = """
+@font-face{font-family:"Outfit";src:url(data:font/woff2;base64,%s) format("woff2");font-weight:100 900}
+text{font-family:"Outfit",system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;fill:#221e1f}
+.bg{fill:#f9fafb}.card{fill:#ffffff;stroke:#dee5ee;stroke-width:1}.soft{fill:#f7f7f7}
+.muted{fill:#6f6f6f}.ink{fill:#221e1f}.icon{color:#2F384C}
+.acc{fill:#10b981}.acc-t{fill:#0c8a60}.acc-tint{fill:#ecfdf5}.acc-line{stroke:#10b981}
+.info{fill:#0f97ff}.info-t{fill:#0668c2}.info-tint{fill:#e8f4ff}
+.warn{fill:#e39b0b}.warn-t{fill:#8a5a00}.warn-tint{fill:#fff6e0}
+.dan{fill:#e0283f}.dan-tint{fill:#fdecee}.dan-t{fill:#b01a2e}
+.dot{fill:#dee5ee}.line{stroke:#dee5ee}.arrow{stroke:#6f6f6f;fill:none}.arrowhead{fill:#6f6f6f}
+.dash{stroke:#6f6f6f;fill:none;stroke-dasharray:4 4}
+.on-acc{fill:#ffffff}.icon-acc{color:#0c8a60}.icon-dan{color:#b01a2e}
+@media (prefers-color-scheme:dark){
+text,.ink{fill:#eef1f3}.bg{fill:#0e1113}.card{fill:#161a1d;stroke:#2a3036}.soft{fill:#1d2226}
+.muted{fill:#8e979f}.icon{color:#c7cdd3}
+.acc{fill:#34d399}.acc-t{fill:#4ade80}.acc-tint{fill:rgba(52,211,153,.12)}.acc-line{stroke:#34d399}
+.info{fill:#3aa9ff}.info-t{fill:#7cc4ff}.info-tint{fill:rgba(58,169,255,.12)}
+.warn{fill:#f5b53d}.warn-t{fill:#f8c96a}.warn-tint{fill:rgba(245,181,61,.12)}
+.dan{fill:#ff5a6e}.dan-tint{fill:rgba(255,90,110,.12)}.dan-t{fill:#ff8a98}
+.dot{fill:#2a3036}.line{stroke:#2a3036}.arrow{stroke:#8e979f}.arrowhead{fill:#8e979f}.dash{stroke:#8e979f}
+.on-acc{fill:#052e20}.icon-acc{color:#4ade80}.icon-dan{color:#ff8a98}
+}
+""" % FONT
+
+
+def icon(name, x, y, size=28, cls="icon"):
+    """A sprite icon at (x, y), `size` px, drawn in the class's color (the icons use currentColor)."""
+    m = re.search(r'<symbol id="%s"[^>]*>(.*?)</symbol>' % name, SPRITE, re.S)
+    return (f'<g class="{cls}" transform="translate({x},{y}) scale({size / 24:.4f})">{m.group(1)}</g>')
+
+
+def text(x, y, s, size=14, weight=400, cls="ink", anchor="start"):
+    return (f'<text x="{x}" y="{y}" font-size="{size}" font-weight="{weight}" class="{cls}" '
+            f'text-anchor="{anchor}">{s}</text>')
+
+
+def rect(x, y, w, h, r=14, cls="card"):
+    return f'<rect x="{x}" y="{y}" width="{w}" height="{h}" rx="{r}" class="{cls}"/>'
+
+
+def svg(w, h, body, title):
+    return (f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {w} {h}" width="{w}" height="{h}" role="img" '
+            f'aria-label="{title}"><title>{title}</title><style>{CSS}</style>'
+            f'{rect(0, 0, w, h, 20, "bg")}{body}</svg>')
+
+
+def dots(x, y, cols, rows, step, lit, lit_cls="acc", r=2.6, seed=1):
+    """A grid of expert dots; `lit` of them (spread at random) in the accent color."""
+    rnd = random.Random(seed)
+    n = cols * rows
+    on = set(rnd.sample(range(n), min(lit, n)))
+    out = []
+    for i in range(n):
+        cx, cy = x + (i % cols) * step, y + (i // cols) * step
+        out.append(f'<circle cx="{cx:.1f}" cy="{cy:.1f}" r="{r}" class="{lit_cls if i in on else "dot"}"/>')
+    return "".join(out)
+
+
+def arrow(x1, y1, x2, y2, cls="arrow"):
+    return (f'<line x1="{x1}" y1="{y1}" x2="{x2 - 7}" y2="{y2}" class="{cls}" stroke-width="1.6" stroke-linecap="round"/>'
+            f'<path d="M{x2} {y2} l-8 -5 v10 z" class="arrowhead"/>')
+
+
+def how_it_works():
+    W, H = 960, 470
+    b = []
+    # the headline strip: one word, 10 experts
+    b.append(rect(24, 24, W - 48, 92))
+    b.append(icon("i-chat", 48, 50, 36))
+    b.append(text(100, 62, "The model is made of 24,576 small specialists - “experts”.", 19, 600))
+    b.append(text(100, 88, "For each word it writes, it asks only 10 of them. Strata keeps the busy ones",
+                  15, 400, "muted"))
+    b.append(text(100, 106, "on your graphics card and the rest in RAM - so it runs on a normal PC.", 15, 400, "muted"))
+    b.append(dots(W - 262, 46, 28, 4, 7.6, 10, seed=3))
+    b.append(text(W - 262, 94, "10 at work for one word", 12, 500, "acc-t"))
+    # three cards
+    cw, ch, cy = 288, 316, 136
+    xs = [24, 24 + cw + 24, 24 + 2 * (cw + 24)]
+    cards = [
+        ("i-gpu", "Graphics card", "Small but very fast: 12-24 GB", "acc",
+         ["Runs the core of the model", "for every word, and keeps the", "~4,000 experts used most"],
+         "the most-used experts", 5, 26, 26 * 5, "acc"),
+        ("i-memory", "RAM + processor", "Big: 32-64 GB of memory", "info",
+         ["Holds all 24,576 experts. When a", "word needs one the card doesn't", "have, the CPU does it - at once"],
+         "all 24,576 experts", 10, 26, 18, "info"),
+        ("i-disk", "SSD", "Holds a 29 GB lookup table", "warn",
+         ["The model looks up a few", "small rows of it per word -", "no need to keep it in RAM"],
+         "", 0, 0, 0, "warn"),
+    ]
+    for (ic, title, sub, tone, lines, dlabel, rows, cols, lit, lit_cls), x in zip(cards, xs):
+        b.append(rect(x, cy, cw, ch))
+        b.append(rect(x + 20, cy + 20, 48, 48, 12, tone + "-tint"))
+        b.append(icon(ic, x + 30, cy + 30, 28, "icon"))
+        b.append(text(x + 82, cy + 42, title, 19, 700))
+        b.append(text(x + 82, cy + 62, sub, 13, 500, tone + "-t"))
+        for i, ln in enumerate(lines):
+            b.append(text(x + 20, cy + 104 + i * 21, ln, 15, 400, "ink"))
+        if rows:
+            b.append(dots(x + 26, cy + 196, cols, rows, 9.4, lit, lit_cls, seed=len(title)))
+            b.append(text(x + 20, cy + 196 + rows * 9.4 + 14, dlabel, 12, 500, "muted"))
+        else:                                       # the SSD: a table of rows, a few being read
+            for r in range(6):
+                yy = cy + 190 + r * 17
+                b.append(rect(x + 20, yy, cw - 40, 11, 4, "warn" if r in (1, 4) else "soft"))
+            b.append(text(x + 20, cy + 306, "a few rows per word", 12, 500, "muted"))
+    return svg(W, H, "".join(b), "How Strata splits the model between your graphics card, RAM and SSD")
+
+
+def guess_and_check():
+    W, H = 960, 330
+    b = [rect(24, 24, W - 48, H - 48)]
+    b.append(icon("i-thinking", 48, 48, 30))
+    b.append(text(92, 70, "Guess, then check - many words per step", 19, 700))
+    b.append(text(92, 94, "A small, fast helper inside the model guesses the next words. The big model checks all the "
+                          "guesses in one go", 15, 400, "muted"))
+    b.append(text(92, 113, "and keeps the ones it agrees with. The big model decides every word - it just arrives 1.6-1.8x sooner.",
+                  15, 400, "muted"))
+    words = ["def", "reverse", "(items", "):"]
+    fixed = ", key):"                                  # the model's own word where the guess was wrong
+    ok = [True, True, True, False]
+    # row 1: the helper's guesses (dashed chips)
+    y1, y2 = 156, 238
+    b.append(text(48, y1 + 24, "Helper guesses", 14, 600, "muted"))
+    b.append(text(48, y2 + 24, "Big model checks", 14, 600, "muted"))
+    x = 176
+    xs = []
+    for i, w in enumerate(words[:4]):
+        wd = 34 + len(w) * 9
+        xs.append((x, wd))
+        b.append(f'<rect x="{x}" y="{y1}" width="{wd}" height="38" rx="10" class="dash" stroke-width="1.4"/>')
+        b.append(text(x + wd / 2, y1 + 25, w, 16, 500, "ink", "middle"))
+        x += wd + 14
+    # row 2: the verdicts
+    for i, ((x, wd), good) in enumerate(zip(xs, ok)):
+        tone = "acc" if good else "dan"
+        b.append(rect(x, y2, wd, 38, 10, tone + "-tint"))
+        b.append(text(x + wd / 2 - 8, y2 + 25, words[i], 16, 600, tone + "-t", "middle"))
+        b.append(icon("i-check" if good else "i-error", x + wd - 22, y2 + 11, 16, "icon-" + tone))
+        if not good:                                   # the model's own word beside the rejected guess
+            fx = x + wd + 10
+            fw = 26 + len(fixed) * 9
+            b.append(rect(fx, y2, fw, 38, 10, "acc-tint"))
+            b.append(text(fx + fw / 2, y2 + 25, fixed, 16, 600, "acc-t", "middle"))
+            xs[i] = (fx, fw)
+        b.append(f'<line x1="{x + wd / 2}" y1="{y1 + 40}" x2="{x + wd / 2}" y2="{y2 - 4}" class="line" stroke-width="1.4"/>')
+    # the result
+    rx = xs[-1][0] + xs[-1][1] + 40
+    b.append(arrow(rx, y2 + 19, rx + 30, y2 + 19))
+    b.append(rect(rx + 42, y2 - 6, W - 48 - (rx + 42), 50, 12, "acc"))
+    b.append(icon("i-bolt", rx + 54, y2 + 6, 26, "on-acc-icon"))
+    b.append(text(rx + 90, y2 + 16, "4 words in one step", 15, 700, "on-acc"))
+    b.append(text(rx + 90, y2 + 34, "3 guesses kept + its own word", 12, 500, "on-acc"))
+    return svg(W, H, "".join(b).replace('class="on-acc-icon"', 'class="icon" style="color:#ffffff"'),
+               "Speculative decoding: a small helper guesses words, the big model checks them at once")
+
+
+if __name__ == "__main__":
+    (OUT / "how-it-works.svg").write_text(how_it_works(), encoding="utf-8")
+    (OUT / "guess-and-check.svg").write_text(guess_and_check(), encoding="utf-8")
+    for f in ("how-it-works.svg", "guess-and-check.svg"):
+        print(f, (OUT / f).stat().st_size, "bytes")

include/strata/artifact/gguf_reader.hpp

@@ -26,6 +26,7 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <set>
 #include <stdexcept>
 #include <algorithm>
 
@@ -418,9 +419,14 @@ private:
             meta_.emplace(std::move(key), read_value(c, t));
         }
         tensors_.reserve((size_t)n_tensors);
+        // GGUF has no index to arbitrate between two tensors of one name: find() is first-match, so a
+        // duplicate would silently win by position.  Refuse the file at open instead, naming both.
+        std::set<std::string> names;
         for (uint64_t i = 0; i < n_tensors; ++i) {
             TensorInfo t;
             t.name = c.str();
+            if (!names.insert(t.name).second)
+                throw std::runtime_error("GGUF: duplicate tensor name '" + t.name + "' in " + path_);
             const uint32_t nd = c.read<uint32_t>();
             if (nd == 0 || nd > 4) throw std::runtime_error("GGUF: bad n_dims for " + t.name);
             t.shape.resize(nd);
@@ -454,8 +460,9 @@ private:
 // ---- architecture guard (P1.S2). The engine is specialised to ONE model; anything else must be
 // refused with a precise error rather than silently mis-run.
 struct Qwen4ExpGuard {
-    uint32_t block_count = 48, hidden = 2560, experts = 512, experts_used = 10, head_count = 24,
-             head_count_kv = 2;
+    uint32_t block_count = 48, hidden = 2560, experts = 0, experts_used = 0, head_count = 24,
+             head_count_kv = 2;   // 0 = presence-only: pruned variants (GSQ-RCO Coder) legitimately ship
+                                  // fewer experts than the canonical 512; the graph reads the true value
 };
 
 inline std::string check_architecture(const GgufFile& g, const Qwen4ExpGuard& want = {}) {
@@ -477,7 +484,7 @@ inline std::string check_architecture(const GgufFile& g, const Qwen4ExpGuard& wa
     for (const auto& r : reqs) {
         const MetaValue* v = g.get(r.key);
         if (!v) return std::string("missing ") + r.key;
-        if (v->u != r.want)
+        if (r.want && v->u != r.want)
             return std::string(r.key) + " = " + std::to_string(v->u) + ", expected " + std::to_string(r.want);
     }
     return {}; // empty == ok

include/strata/core/expert_cache.hpp

@@ -26,6 +26,7 @@
 // kernel. This is that first step, and the step it unblocks is the one that can be measured.
 #pragma once
 
+#include <cstddef>
 #include <cstdint>
 #include <string>
 #include <vector>
@@ -126,6 +127,10 @@ public:
     /// against a `cudaStreamNonBlocking` one.  The first version used the async form and `verify_slot` refused
     /// the whole run with "slot 0 differs from the arena at byte 0" - which is the check doing its job.
     bool fill_slot_blocking(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes = 0);
+    /// perf-review D-4: `fill_slot_blocking`'s copy on the same (legacy) stream, but queued: many slots are refilled
+    /// with one `sync_queued` at the end instead of a wait per slot. Same ordering against earlier work, same bytes.
+    bool fill_slot_queued(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes = 0);
+    bool sync_queued(std::string& err);
 
     /// Reads `slot` back to the host and compares it to `host_blob`, byte for byte.  **THE ONLY THING THAT SAYS
     /// THE CACHE HOLDS THE EXPERT IT CLAIMS TO.**  A slot table that is right about indices and wrong about
@@ -136,6 +141,11 @@ public:
     int64_t fills() const { return fills_; }
 
 private:
+#if defined(STRATA_USE_HIP)
+    bool ensure_blocking_staging(std::size_t bytes, std::string& err);
+    uint8_t* blocking_staging_ = nullptr;
+    std::size_t blocking_staging_bytes_ = 0;
+#endif
     uint8_t* base_ = nullptr;
     std::vector<int32_t> residency_;   ///< [n_layers * n_expert] -> slot or kNotResident
     int64_t slots_ = 0;

include/strata/core/expert_source.hpp

@@ -26,11 +26,50 @@
 #include "strata/kernels/cpu/pool.hpp"
 
 #include <cstdint>
+#include <cstddef>
+#include <utility>
 #include <string>
 #include <vector>
 
 namespace strata::core {
 
+class RemoteExperts;
+
+namespace detail {
+
+/// Sentinel used by the pure complement planner for a blob that remains in the mmap fallback.
+inline constexpr uint64_t kNoCacheComplement = ~uint64_t{0};
+
+/// Required cgroup-v2 usage counters for the conservative cache-reclaim allowance.
+struct CgroupMemoryStat {
+    uint64_t current = 0;
+    uint64_t inactive_file = 0;
+    uint64_t file_dirty = 0;
+    uint64_t file_writeback = 0;
+    bool valid = false;
+};
+
+/// Calculate additional bytes under a finite cgroup limit after reclaiming only clean inactive file cache.
+/// Returns false when the required memory.stat counters were unavailable.
+bool cgroup_available_bytes(uint64_t limit, const CgroupMemoryStat& stat, uint64_t& bytes);
+
+/// Build compact offsets for experts absent from both the primary GPU cache and an optional second GPU tier.
+/// Kept CPU-only so selection and byte accounting can be tested without initializing a GPU.
+bool make_cache_complement_plan(
+    int64_t n_layers, int64_t n_expert, const std::vector<uint64_t>& layer_blob_bytes,
+    const std::vector<std::pair<int32_t, int32_t>>& primary_gpu_pairs,
+    const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs,
+    std::vector<uint64_t>& offsets, uint64_t& bytes, std::string& err);
+
+/// Resolve one blob through the compact copy when present, otherwise preserve its exact mapped-file fallback.
+const uint8_t* cache_complement_blob_or_fallback(
+    size_t index, const std::vector<uint64_t>& offsets, const uint8_t* complement_host,
+    const uint8_t* mapped_fallback);
+
+}  // namespace detail
+
+
+
 /// Where one routed expert's bytes come from.
 ///
 /// Phase 2 has NO cache (`phase-2-correct-engine.md`: hit rate `h = 0`), so the only implementation is a
@@ -40,7 +79,7 @@ class ExpertSource {
 public:
     virtual ~ExpertSource() = default;
 
-    /// The 1,382,400-byte blob for `(layer, expert)`, or nullptr if it cannot be produced.
+    /// The expert-layout blob for `(layer, expert)`, or nullptr if it cannot be produced.
     ///
     /// The pointer only has to stay valid until the next `blob()` call: with `h = 0` every expert is computed
     /// immediately and nothing is retained.  A CACHING source must return pointers into the cache, not into a
@@ -92,6 +131,8 @@ struct GpuPlanSink {
 struct ExpertDispatch {
     strata::kernels::cpu::ExpertPool* pool = nullptr;
     ExpertSource* src = nullptr;
+    RemoteExperts* remote[3] = {}; ///< optional CUDA1..3 tiers for otherwise CPU-served rows
+    int remote_count = 0;
     int64_t n_expert = strata::kernels::cpu::NE;
 
     /// Counters, for the driver to report rather than for control flow.
@@ -266,26 +307,49 @@ public:
     FileExpertSource(const FileExpertSource&) = delete;
     FileExpertSource& operator=(const FileExpertSource&) = delete;
 
-    /// Maps `<pack_dir>/experts.bin` and checks its size against `n_layers * n_expert * BLOB`.
+    /// Maps `<pack_dir>/experts.bin` and checks its size against the loaded expert layout.  Canonical packs use
+    /// `n_layers * n_expert * BLOB`; native packs use their variable per-layer blob sizes and offsets.
     ///
     /// The size check is not a formality: a short file would fault at the END of a long sequence, and an
     /// over-long one means the pack is not the one the geometry came from.  Refuses with the two numbers.
     bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err);
+    /// Pin a compact host mirror of experts absent from a fully filled static GPU cache. The mmap remains open
+    /// as a fallback for later cache reloads. This is opt-in because the complement may still be a large allocation.
+    bool pin_cache_complement(
+        const ExpertCache& cache, std::string& err, bool pin = true,
+        const std::vector<std::pair<int32_t, int32_t>>& additional_gpu_pairs = {});
     void close();
 
     bool mapped() const { return base_ != nullptr; }
     int64_t blobs() const { return blobs_; }
+    uint64_t pinned_bytes() const { return complement_pinned_ ? complement_bytes_ : 0; }
+    uint64_t resident_bytes() const { return complement_bytes_; }
+    bool complement_pinned() const { return complement_pinned_; }
 
     const uint8_t* blob(int64_t layer, int64_t expert) override;
+    bool pinned(int64_t layer, int64_t expert) const override;
+    const uint8_t* device_alias(int64_t layer, int64_t expert) const override;
 
     /// Blobs touched, for the driver to report.  With `h = 0` this is `48 * k` per token and the number is only
     /// interesting once Phase 3 makes it not so.
     int64_t reads() const override { return reads_; }
 
 private:
+    const uint8_t* mapped_blob(int64_t layer, int64_t expert) const;
+    static constexpr uint64_t kNoComplement = detail::kNoCacheComplement;
     const uint8_t* base_ = nullptr;
     int64_t blobs_ = 0;
+    int64_t n_layers_ = 0;
     int64_t n_expert_ = 0;
+    uint64_t mapped_bytes_ = 0;
+    std::vector<uint64_t> layer_offsets_, layer_blob_bytes_;
+    void* complement_arena_ = nullptr;
+    const uint8_t* complement_host_ = nullptr;
+    const uint8_t* complement_device_ = nullptr;
+    uint64_t complement_bytes_ = 0;
+    std::vector<uint64_t> complement_offsets_;
+    bool complement_pinned_ = false;
+    bool complement_ready_ = false;
     int64_t reads_ = 0;
 #if defined(_WIN32)
     void* file_ = nullptr;
@@ -320,7 +384,8 @@ public:
 
     /// Allocates and loads `<pack_dir>/experts.bin`.  Prints nothing; the caller reports `note()` and the load
     /// rate, because those are the two numbers that say whether the arena is the one that was asked for.
-    bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, int threads, std::string& err);
+    bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, int threads, std::string& err,
+              uint64_t max_pinned_bytes = 0);
     /// Plan v0.3 P6: a native pack without experts.bin takes its experts from the model's shard 1.
     void set_gguf(const std::string& shard1) { gguf_ = shard1; }
     void close();
@@ -336,6 +401,12 @@ public:
     /// only true if the engine says what it got.
     const std::string& note() const { return note_; }
     double load_gib_per_second() const { return gib_per_s_; }
+    // Loader fix: the load, split.  `load_seconds()` is the wall clock of the load loop; the other two are
+    // sums over the reader threads (see LoadStats), so on their own they say how much of that wall was spent
+    // waiting for the disk and how much in memcpy + FNV-1a.
+    double load_seconds() const { return load_seconds_; }
+    double load_read_seconds() const { return load_read_s_; }
+    double load_copy_seconds() const { return load_copy_s_; }
 
 private:
     void* arena_ = nullptr;          ///< the PinnedArena, owned
@@ -347,6 +418,9 @@ private:
     int64_t reads_ = 0;
     std::string note_;
     double gib_per_s_ = 0.0;
+    double load_seconds_ = 0.0;
+    double load_read_s_ = 0.0;
+    double load_copy_s_ = 0.0;
     uint64_t pinned_bytes_ = 0;
     std::string gguf_;
 };

include/strata/core/layer.hpp

@@ -40,6 +40,8 @@
 #include "strata/kernels/ngram.hpp"
 #include "strata/kernels/ple.hpp"
 
+#include <cstdio>
+#include <cstdlib>
 #include <cstdint>
 #include <string>
 
@@ -207,6 +209,14 @@ struct QsaState {
     int8_t* v_q = nullptr;
     uint16_t* k_scale = nullptr;
     uint16_t* v_scale = nullptr;
+    /// PR #21: Q4_0 KV with Walsh-Hadamard rotation (qsa_set_kv_q4, kv_q4.hpp): 144 B per cell and head
+    bool kv_q4 = false;
+    uint8_t* k_q4 = nullptr;
+    uint8_t* v_q4 = nullptr;
+    /// Hybrid K8V4: K in INT8 (unrotated - the scores stay exact), V in rotated Q4_0 (kv_q4.hpp): 816 B per cell.
+    /// Uses k_q/k_scale + v_q4. Mode 0 only (no KV streaming, no ring); under this setting the MTP drafter's
+    /// ring state stays plain INT8, so the block movers never see the hybrid layout.
+    bool kv_hybrid = false;
     int32_t* page_table = nullptr;   ///< (n_pages,) logical page -> physical page (-1: not resident, streamed)
     int64_t n_pages = 0;
     int64_t max_cells = 0;
@@ -256,7 +266,8 @@ struct QsaState {
 uint64_t qsa_state_bytes(const ModelGeometry& g, int64_t max_cells, bool with_rope = true, int64_t ring_cells = 0);
 /// KV streaming: keep `cells` cells of each QSA layer in VRAM and the rest in pinned host memory (0: all in VRAM,
 /// the default). Set before sizing and initializing the session; a context that fits in `cells` is not streamed.
-/// Also puts the MTP drafter's K/V in a ring of its window (`ring_cells` of qsa_state_bytes/init).
+/// Also puts the MTP drafter's K/V in a ring of its window (`ring_cells` of qsa_state_bytes/init; -1 forces a fully
+/// resident state).
 void qsa_set_kv_resident(int64_t cells);
 int64_t qsa_kv_resident();
 /// The fewest resident cells a streamed layer may have: one verify window's selections (8 queries x 2,051 cells
@@ -268,6 +279,22 @@ uint64_t qsa_kv_host_bytes();
 /// initializing the session; default off until gate G-C accepts it.
 void qsa_set_kv_int8(bool enabled);
 bool qsa_kv_int8();
+/// PR #21: store K/V as Q4_0 after a Hadamard rotation (`--kv q4_0`): 576 B per cell, vs 1,056 in INT8.
+void qsa_set_kv_q4(bool enabled);
+bool qsa_kv_q4();
+/// Hybrid K8V4 (`--kv k8v4`): K in INT8, V in rotated Q4_0 - 816 B per cell. Not with --kv-resident.
+void qsa_set_kv_hybrid(bool enabled);
+bool qsa_kv_hybrid();
+/// The state's KV format for the block-moving functions of kv_stream.hpp (kKvF16 / kKvInt8 / kKvQ4).
+inline int qsa_kv_format(const QsaState& st) {
+    // A hybrid K8V4 state is mode 0 only and never reaches the block movers; refuse rather than let it
+    // fall through to kKvF16 - a wrong layout silently applied is worse than a hard stop (PR review).
+    if (st.kv_hybrid) {
+        std::fprintf(stderr, "strata: qsa_kv_format: a hybrid K8V4 state must never reach the block movers\n");
+        std::exit(1);   // the kernels' own "unsupported geometry" convention (kv_q8.cu, qsa_decode_attn.cu)
+    }
+    return st.kv_q4 ? strata::kernels::kKvQ4 : st.kv_int8 ? strata::kernels::kKvInt8 : strata::kernels::kKvF16;
+}
 uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, QsaState& st,
                         const QsaState* share_rope = nullptr, int64_t ring_cells = 0);
 /// KV streaming: the pools a reader sees (the VRAM slots) and, when streamed, make the selection's blocks resident.

include/strata/core/mtp.hpp

@@ -53,6 +53,9 @@ public:
     /// KV streaming: refill the ring of the drafter's window from its host copy for a sequence that continues at
     /// `upto` (a conversation-cache resume). No-op unless the drafter's K/V is a ring.
     void kv_restore(int64_t upto);
+    /// The VRAM bind() will allocate for a native head of `head_row_bytes` per vocabulary row: the draft logits and
+    /// the draft head over rt/draft_vocab.bin's subset.  The expert cache is sized before bind(), so it reserves this.
+    uint64_t bind_bytes(uint64_t head_row_bytes, int64_t n_vocab) const;
     /// The main model's embedding and head, and the verify window's final residuals (T rows, hc*n_embd each).
     bool bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err);
 
@@ -73,9 +76,23 @@ public:
     double ms_draft = 0, ms_prefill = 0;
     int64_t rounds = 0;
 
+    /// E-9: the prompt path computes this layer's prompt K/V in batches (Prefill::draft_kv): its tensors, its K/V
+    /// state, the first cell a round can still read, its device, and a wait for its own stream.
+    const float* tensor_f32(const char* name) const { return f32(name); }
+    const uint16_t* tensor_bf16(const char* name) const { return bf16(name); }
+    const void* tensor_q8(const char* name) const { return q8(name); }
+    QsaState& kv_state_rw() { return st_; }
+    int64_t first_needed() const { return (window_ > 0 && prompt_len_ > 0) ? prompt_len_ - window_ - 64 : 0; }
+    int device() const { return device_; }
+    bool idle(std::string& err) {
+        if (cs_ && cudaStreamSynchronize(cs_) != cudaSuccess) { err = "mtp: its stream failed"; return false; }
+        return true;
+    }
+
 private:
     bool record_forward(int T, int step_row0, cudaStream_t cs, std::string& err);
     bool capture_prefill(int T, std::string& err);
+    bool capture_prefill_dev(int T, std::string& err);   ///< E-4: without the mapped staging (inputs copied on device)
     bool capture_round(int T, std::string& err);
     bool capture_step(int j, std::string& err);
     cudaGraphExec_t step_exec_[9] = {};
@@ -89,11 +106,15 @@ private:
     const NativeHead* head_ = nullptr;
     const float* window_R_ = nullptr;
     int max_t_ = 0;
+    int device_ = -1;   ///< the device `load` ran on: the public calls switch to it (layer split)
     int max_drafts_ = 1 << 30;
     int64_t n_vocab_ = 0;
     uint64_t vram_ = 0;
     cudaStream_t cs_ = nullptr;
     cudaGraphExec_t prefill_exec_[9] = {};
+    cudaGraphExec_t prefill_dev_exec_[9] = {};
+    int32_t* pf_dev_ = nullptr;   ///< E-4: a prompt's rows' token / step / position records, uploaded at once
+    int64_t pf_cap_ = 0;          ///< its capacity in ints
     cudaGraphExec_t round_exec_[9] = {};
 
     struct Tensor { std::string name, kind; int64_t rows = 0, cols = 0; uint64_t off = 0, bytes = 0; };

include/strata/core/on_device.hpp

@@ -0,0 +1,22 @@
+#pragma once
+// Multi-GPU: make `device` current for a scope and restore the caller's device after it.  An object that owns CUDA
+// streams, graphs and buffers on one device (a verify stage, the drafter) wraps its public calls in this, so the
+// caller's thread may be on any device.  A negative device, or the device already current, does nothing.
+
+#include <cuda_runtime.h>
+
+namespace strata::core {
+
+struct OnDevice {
+    int previous = -1;
+    explicit OnDevice(int device) {
+        int cur = 0;
+        if (device >= 0 && cudaGetDevice(&cur) == cudaSuccess && cur != device && cudaSetDevice(device) == cudaSuccess)
+            previous = cur;
+    }
+    ~OnDevice() { if (previous >= 0) cudaSetDevice(previous); }
+    OnDevice(const OnDevice&) = delete;
+    OnDevice& operator=(const OnDevice&) = delete;
+};
+
+}  // namespace strata::core

include/strata/core/pinned.hpp

@@ -39,7 +39,8 @@ struct PinnedArena {
     explicit PinnedArena(uint64_t bytes, uint64_t slice = 0);
     /// Plan v0.3 P6: slices of different sizes (one per layer of a native pack), given as their start offsets
     /// followed by the end of the last one.  `slice_starts` holds the registered ones.
-    PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds);
+    /// `max_pinned_bytes`: optional cap on CUDA registration. 0 preserves the normal unrestricted path.
+    PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds, uint64_t max_pinned_bytes = 0);
     std::vector<uint64_t> slice_starts;
     ~PinnedArena();
     PinnedArena(const PinnedArena&) = delete;
@@ -51,15 +52,40 @@ struct PinnedArena {
 
 struct LoadStats {
     double seconds = 0.0;
+    // Loader fix: the two halves of `seconds`, both SUMMED OVER THE READER THREADS rather than wall clock, so
+    // either can exceed `seconds` when the threads overlap.  They exist to separate the read from the copy and
+    // the hash: an 8 MiB chunk that reaches the disk as ~2048 4 KiB reads spends its time in the first one.
+    double read_seconds = 0.0;                  // inside the read call only: no seek, no copy, no hash
+    double copy_seconds = 0.0;                  // memcpy + FNV-1a
+    // A short read is not a slow load, it is a WRONG one: the caller must refuse the pack rather than run
+    // on an arena whose tail was never written.  `seconds` alone cannot say that (it is -1.0 on failure, but
+    // the caller's own check is on the REQUESTED byte count, which a refused read does not change).
+    bool ok = true;                             // false: the load failed, see `error`
+    std::string error;                          // why it failed, for the caller's message
     uint64_t bytes = 0;
     uint64_t layers = 0;
     std::vector<uint64_t> layer_checksums;      // one FNV-1a per layer
     double gib_per_second() const { return seconds > 0 ? (double) bytes / (1024.0 * 1024 * 1024) / seconds : 0.0; }
+    // Aggregate rate of the readers WHILE THEY WERE INSIDE THE READ CALL: `bytes` over the mean per-thread
+    // read time.  `gib_per_second()` above divides the same bytes by the wall clock of the whole loop, which
+    // also contains the copy and the hash, so the two differ by how much of the loop was not I/O.
+    double read_gib_per_second(int threads) const {
+        if (read_seconds <= 0.0 || threads < 1) return 0.0;
+        return (double) bytes / (1024.0 * 1024 * 1024) / (read_seconds / (double) threads);
+    }
 };
 
 // Load `layers` layers of the expert arena into `dst` with `threads` readers, `chunk` bytes at a time.
 // Each thread opens its OWN handle and seeks, which is the portable form of parallel pread: a shared handle
 // needs a lock around the seek and defeats the parallelism on Windows.
+//
+// The reads go through `fread()` on a per-thread `FILE*`, NOT through `std::ifstream`.  MSVC's
+// `basic_filebuf::xsgetn` splits every request larger than `_INTERNAL_BUFSIZ - 1` into 4095-byte `fread()`
+// calls, so an 8 MiB `chunk` reached the disk as ~2048 4 KiB operations and the chunking did nothing at all
+// (measured on a 42.9 GB pack: 1222 s, 4096 bytes per operation, 0.03 GiB/s).  `fread()` hands a request
+// larger than the stream buffer straight to `_read()`/`ReadFile()`, which is what `chunk` is for.  The
+// control flow, the copy, the per-layer FNV-1a and therefore the checksums are unchanged, so a pack loaded
+// by the old and the new reader must produce identical `layer_checksums`.
 LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_bytes, uint64_t blobs_per_layer,
                        uint64_t layers, int threads, uint64_t chunk);
 /// Plan v0.3 P6: the same with one byte range per layer (`layer_off[L]`, `layer_bytes[L]`).

include/strata/core/progress.hpp

@@ -0,0 +1,51 @@
+// include/strata/core/progress.hpp - issue #29: whether a request is still moving, and where it is.
+//
+// The engine's host loop and the GPU wait on each other through flags; a protocol bug there does not crash, it
+// spins forever (the GPU "100%", one CPU core busy, no tokens).  `--serve` runs a watchdog thread over this: a
+// request whose heartbeat stops for `STRATA_WATCHDOG_S` seconds (default 60; 0 = off) ends the engine with the
+// stage it was stuck in, and the server starts it again instead of hanging.  Tokens, prompt chunks and verify
+// windows beat; the stage is two relaxed stores per layer, which the token path does not notice.
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <cstdio>
+
+namespace strata::core {
+
+struct Progress {
+    std::atomic<uint64_t> beats{0};
+    std::atomic<bool> busy{false};
+    std::atomic<const char*> where{"idle"};
+    std::atomic<int64_t> detail{-1};
+    std::atomic<int64_t> since_ms{0};    ///< when `where` was set (steady clock): how long a stage has lasted
+    std::atomic<uint64_t> ticks{0};      ///< layers served: a window that still moves, slowly, against one that stopped
+};
+
+inline int64_t progress_now_ms() {
+    return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch())
+        .count();
+}
+
+/// What the watchdog prints about a part of the engine when it fires (the expert pool, the verify window).
+using DiagFn = void (*)(std::FILE*);
+inline std::atomic<DiagFn>& diag_pool_fn() { static std::atomic<DiagFn> f{nullptr}; return f; }
+inline std::atomic<DiagFn>& diag_verify_fn() { static std::atomic<DiagFn> f{nullptr}; return f; }
+
+inline Progress& progress() {
+    static Progress p;
+    return p;
+}
+
+inline void progress_at(const char* where, int64_t detail = -1) {
+    Progress& p = progress();
+    p.where.store(where, std::memory_order_relaxed);
+    p.detail.store(detail, std::memory_order_relaxed);
+    p.since_ms.store(progress_now_ms(), std::memory_order_relaxed);
+}
+
+inline void progress_beat() { progress().beats.fetch_add(1, std::memory_order_relaxed); }
+inline void progress_tick() { progress().ticks.fetch_add(1, std::memory_order_relaxed); }
+
+}  // namespace strata::core