Terminal 0.2.4.1 / Control 0.1.49.1: version bump for the DeepSeek-setup hotfix (app stays 0.2.3, engine packages unchanged); release notes

Five conflicts, resolved keeping every f-024-rel behaviour (slots pin, long-prompt skip, HIPK, ZC fix) and maya2's:
- ds4_main.cpp, request struct: both fields kept (pin=N from f-024-slots and dspark_cum from maya2).
- ds4_main.cpp, request parser: both keys kept (pin, dspark_cum/spec_cum).
- ds4_main.cpp, prompt feed loop: slots' pin_at (the read stops at N for the shared-start checkpoint 3) and maya2's Stop handling (checkpoint 1 before the prompt, set_cancel, the stop check at the top of each feed) combined: the loop checks Stop first, then sizes the feed with the pin cut (n is no longer const).
- ds4_model.cu, forward chunk loop (two places): the skip's 'run.runs(k, il)' (an early chunk skipped past the yoco layer) kept, and maya2's halt_now() / halt_end(di, err) check added after it, so a skipped layer does no halt work.
win_submit (ds4_kernels.cu) is compiled to a no-op outside _WIN32 (#if defined(_WIN32) ... else (void) s), so it is already out of the Linux CUDA and Linux HIP builds.

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Strata Engine - top-level build (P1.S1).
+# Strata Engine - top-level build (P1.S1).
 #
 # Targets (phase-1 §P1.S1): `strata_artifact` (no CUDA dependency), `tests`.  `strata_plan` and the C++
 # `strata-pack` are added when their sources exist - a target with no sources would make the build graph
@@ -8,7 +8,7 @@
 # below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
 # and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.48 LANGUAGES CXX)
+project(strata VERSION 0.1.49.1 LANGUAGES CXX)
 # EXPERIMENTAL: the Intel Arc engine, the SYCL port in sycl/ (PR #423; icpx + oneMKL, docs/INTEL_ARC.md). It is its
 # own project that takes the unported sources from src/; with this on, nothing below is configured.
 option(STRATA_ENABLE_SYCL "EXPERIMENTAL: build the Intel Arc (SYCL) engine from sycl/ - docs/INTEL_ARC.md" OFF)
@@ -794,6 +794,108 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
   target_compile_definitions(strata PRIVATE STRATA_HAVE_QWEN35=1)
   endif()
 
+  # DeepSeek-V4.1-Flash: the third engine path (src/dsv41, from Project Maya's maya-ds4, MIT, third_party/project-maya).
+  # Its own executable for now (strata-dsv41); a family switch in `strata` comes after first light.  OFF by default: it
+  # changes no existing target (the vendored kernels live in strata::dsv41k, so no symbol is shared with strata_kernels).
+  option(STRATA_BUILD_DSV41 "Build the DeepSeek-V4.1 engine path (strata-dsv41)" OFF)
+  if(STRATA_BUILD_DSV41 AND NOT STRATA_HIP_GFX906)
+    add_library(strata_dsv41_kernels STATIC src/dsv41/vendor/exl3.cu src/dsv41/vendor/sampler.cu src/dsv41/vendor/moe_win.cu
+                src/dsv41/vendor/exl3_cpu.cpp)
+    if(NOT STRATA_ENABLE_HIP)   # (the pre-sm_80 NVIDIA decode kernels exl3.cu dispatches to)
+      target_sources(strata_dsv41_kernels PRIVATE src/dsv41/vendor/exl3_legacy.cu)
+    endif()
+    target_include_directories(strata_dsv41_kernels PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})
+    target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})
+    add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cu
+                   src/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp
+                   src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_packinfo.cpp)
+    # the commit the binary was built from, readable with --version (set -DSTRATA_BUILD_ID=... for a tree without .git)
+    if(NOT STRATA_BUILD_ID AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/BUILD_ID)   # (a git archive has no .git: the packer writes the commit here)
+      file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/BUILD_ID STRATA_BUILD_ID LIMIT_COUNT 1)
+    endif()
+    if(NOT STRATA_BUILD_ID)
+      execute_process(COMMAND git -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse --short=9 HEAD OUTPUT_VARIABLE STRATA_BUILD_ID
+                      OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
+    endif()
+    if(NOT STRATA_BUILD_ID)
+      set(STRATA_BUILD_ID unknown)
+    endif()
+    set_source_files_properties(src/dsv41/ds4_main.cpp PROPERTIES COMPILE_DEFINITIONS "DS4_BUILD_ID=\"${STRATA_BUILD_ID}\"")
+    target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
+                               ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)
+    target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})
+    find_package(Threads REQUIRED)
+    target_link_libraries(strata-dsv41 PRIVATE Threads::Threads)
+    if(WIN32 AND STRATA_ENABLE_HIP)   # cudaMemGetInfo is the DXGI-budgeted hip_compat::mem_get_info (src/core/device.cu): on a Strix Halo the
+      # carve-out is not all the GPU may use, and the engine sizes its expert tiers by what the budget says is free
+      target_link_libraries(strata-dsv41 PRIVATE strata_core)
+    endif()
+    if(STRATA_NATIVE_EXPERTS)   # Strata's CPU trellis kernels (tq_cpu.hpp) for the trellis experts on the CPU lane
+      target_link_libraries(strata-dsv41 PRIVATE strata_kernels_cpu)
+      target_compile_definitions(strata-dsv41 PRIVATE STRATA_DSV41_TQ_CPU=1)
+    endif()
+    if(STRATA_NATIVE_EXPERTS)
+      # the CPU lane's ggml-type rows: ggml-cpu's dots against Strata's AVX2 i-quant rows (bits and GB/s); no GPU, no model
+      add_executable(ds4_gg_bench tests/dsv41/gg_dot_bench.cpp src/dsv41/ds4_gg_cpu.cpp)
+      target_include_directories(ds4_gg_bench PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41
+                                 ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)
+      target_link_libraries(ds4_gg_bench PRIVATE strata_kernels_cpu Threads::Threads)
+      target_compile_definitions(ds4_gg_bench PRIVATE STRATA_DSV41_TQ_CPU=1)
+    endif()
+    if(STRATA_NATIVE_EXPERTS)
+      # the trellis experts (device kernels ds4_tq.cu, CPU lane ds4_tq_cpu.cpp) against a double reference; needs a GPU, no model
+      # both test .cpp files launch kernels, so they compile as HIP on the AMD backend and as CUDA on the NVIDIA one
+      if(STRATA_ENABLE_HIP)
+        set(_ds4_test_lang HIP)
+      elseif(STRATA_ENABLE_CUDA)
+        set(_ds4_test_lang CUDA)
+      else()
+        set(_ds4_test_lang CXX)
+      endif()
+      set_source_files_properties(tests/dsv41/tq_expert_test.cpp PROPERTIES LANGUAGE ${_ds4_test_lang})
+      add_executable(ds4_tq_test tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cpp
+                     src/dsv41/ds4_iq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp tools/trellis/tqenc.cpp)
+      target_include_directories(ds4_tq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
+                                 ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)
+      target_link_libraries(ds4_tq_test PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)
+      target_compile_definitions(ds4_tq_test PRIVATE STRATA_DSV41_TQ_CPU=1)
+      # the same with the plain-FMA tile instead of the gfx12 WMMA one (what CUDA and gfx11 builds run)
+      add_executable(ds4_tq_test_portable tests/dsv41/tq_expert_test.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_iq.cu src/dsv41/ds4_tq_cpu.cpp
+                     src/dsv41/ds4_iq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp tools/trellis/tqenc.cpp)
+      target_include_directories(ds4_tq_test_portable PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
+                                 ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)
+      target_link_libraries(ds4_tq_test_portable PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)
+      target_compile_definitions(ds4_tq_test_portable PRIVATE STRATA_DSV41_TQ_CPU=1 DS4_TQ_PORTABLE=1)
+      if(STRATA_ENABLE_HIP)
+        target_compile_options(ds4_tq_test_portable PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")
+      endif()
+      if(STRATA_ENABLE_HIP)
+        target_compile_options(ds4_tq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")
+      endif()
+      # the i-quant experts (ds4_iq.cu) against ggml's own decode of the same blocks + a double reference, device and CPU
+      # lane, for every type the device takes: the gate a new type (IQ2_XS ..) has to pass before the engine uses it
+      set_source_files_properties(tests/dsv41/iq_expert_test.cpp PROPERTIES LANGUAGE ${_ds4_test_lang})
+      add_executable(ds4_iq_test tests/dsv41/iq_expert_test.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp
+                     src/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp)
+      target_include_directories(ds4_iq_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
+                                 ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/cpu)
+      target_link_libraries(ds4_iq_test PRIVATE strata_kernels_cpu ${_strata_gpu_runtime_target} Threads::Threads)
+      target_compile_definitions(ds4_iq_test PRIVATE STRATA_DSV41_TQ_CPU=1)
+      if(STRATA_ENABLE_HIP)
+        target_compile_options(ds4_iq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")
+      endif()
+      # the kept conversations' .meta file and prefix matching (ds4_slotmeta.hpp): no GPU, no model
+      add_executable(ds4_slot_test tests/dsv41/slot_test.cpp)
+      target_include_directories(ds4_slot_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41)
+      target_compile_features(ds4_slot_test PRIVATE cxx_std_17)
+    endif()
+    if(STRATA_ENABLE_HIP)
+      foreach(_t strata_dsv41_kernels strata-dsv41)
+        target_compile_options(${_t} PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")
+      endforeach()
+    endif()
+  endif()
+
   if(STRATA_ENABLE_HIP)
     enable_testing()
     set(_strata_hip_tests)
@@ -1623,7 +1725,7 @@ if(STRATA_NATIVE_EXPERTS)
     set_source_files_properties(src/kernels/cpu/iq_avx1.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX")
   else()
     set_source_files_properties(src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/nvfp4_avx512.cpp PROPERTIES COMPILE_OPTIONS
-                                "-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512vnni;-mavx512vbmi;-mf16c;-mfma")
+                                "-mavx512f;-mavx512bw;-mavx512vl;-mavx512dq;-mavx512vnni;-mavx512vbmi;-mavx512vpopcntdq;-mbmi2;-mf16c;-mfma")
     set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
     set_source_files_properties(src/kernels/cpu/kq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
     set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
@@ -2265,3 +2367,19 @@ if(STRATA_HIP_GFX906)
   add_executable(gfx906_qsa_query_swizzle_probe bench/gfx906_qsa_query_swizzle_probe.cpp)
   target_link_libraries(gfx906_qsa_query_swizzle_probe PRIVATE strata_kernels ${_strata_gpu_runtime_target})
 endif()
+
+# ---- DeepSeek-V4.1 on a box with no GPU toolkit ------------------------------------------------
+# The whole DSV41 block above lives inside if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP), so with neither backend on
+# -DSTRATA_BUILD_DSV41=ON was silently unused and the host-side benchmark could not be built at all (found on p1004,
+# 2026-10-23: the CPU lane is exactly what a box without a usable GPU needs to measure).  This branch builds the one target
+# that has no device code, so the CPU-lane A/B runs anywhere.
+if(STRATA_BUILD_DSV41 AND NOT STRATA_ENABLE_CUDA AND NOT STRATA_ENABLE_HIP AND NOT STRATA_HIP_GFX906)
+  message(STATUS "STRATA_BUILD_DSV41: no GPU backend, building the host-side benchmark only (ds4_gg_bench)")
+  if(STRATA_NATIVE_EXPERTS)
+    add_executable(ds4_gg_bench tests/dsv41/gg_dot_bench.cpp src/dsv41/ds4_gg_cpu.cpp)
+    target_include_directories(ds4_gg_bench PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41
+                               ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)
+    target_link_libraries(ds4_gg_bench PRIVATE strata_kernels_cpu Threads::Threads)
+    target_compile_definitions(ds4_gg_bench PRIVATE STRATA_DSV41_TQ_CPU=1)
+  endif()
+endif()

README.de.md

@@ -190,6 +190,21 @@ Weitere Auswahl:
 
 Alle Größen, Downloads und was wo läuft: [docs/MODELS.md](docs/MODELS.md).
 
+### DeepSeek V4.1-Flash (im Terminal, ab 0.2.4)
+
+[DeepSeek V4.1-Flash](https://huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF) ist ein anderes, deutlich größeres Modell. ISTA-DASLab hat die drei Sets unten gebaut. Strata startet sie auf einem PC mit einer 12-GB-Grafikkarte und 46 bis 64 GB RAM und liest den Rest von der SSD. Es läuft auf NVIDIA-RTX-Karten der Serien 30 bis 50 und auf der Radeon AI PRO R9700. Es nutzt ein eigenes Engine-Paket, `strata-dsv41`, das `strata launch` lädt, wenn du DeepSeek wählst. Die Desktop-App bietet es noch nicht an.
+
+| Set | Download | Qualität im Vergleich zum Originalmodell | Antwortet mit |
+| --- | ---: | --- | --- |
+| **IQ1_M** (1,75 Bit) | 239 GB | KL 0,217, Perplexität 3,80 (+21 %); Code +8,9 %, Agent-Aufgaben +10,2 %; schwächer bei einfacher englischer Prosa. Laut Modellkarte von ISTA-DASLab behält es 98,7 % des Aufgabendurchschnitts des Basismodells (AIME25, GPQA-Diamond, LiveCodeBench v6). | noch nicht gemessen |
+| **IQ2_XXS** (2.00 Bit) | 257 GB | KL 0,170, Perplexität 3,63 (+15,5 %); Code +8,6 %, Agent-Aufgaben +5,3 % | 6 bis 9 Tokens/s auf einer RTX 3060, etwa 5 auf einer RTX 5070 |
+| **IQ2_XS** (2.25 Bit) | 274 GB | KL 0,107, Perplexität 3,37 (+7,3 %); Code +3,2 %, Agent-Aufgaben +2,2 % | etwa 11 Tokens/s auf einer Radeon AI PRO R9700 |
+| **IQ4_NL** (Engram-Tabelle, kein drittes Modell) | 111 GB, schon in allen drei Downloads oben enthalten | Die 4-Bit-Engram-Nachschlagetabelle, die das Modell aus RAM oder SSD liest. Alle drei Sets nutzen diese eine Datei (`DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf`). | n/a |
+
+4x Radeon AI PRO R9700: etwa 31 Tokens/s, 55 bis 69 mit DSpark.
+
+Starte es mit `strata launch --model ist-das-2.00bpw` (oder `ist-das-1.75bpw`, `ist-das-2.25bpw`). Details, Geschwindigkeiten je Karte und die manuelle Installation: [docs/DEEPSEEK41.md](docs/DEEPSEEK41.md).
+
 ## Wie schnell ist es?
 
 Gemessen mit einer Grafikkarte mit 12 GB (RTX 5070, Ryzen 5 7600, 64 GB RAM). Ergebnisse anderer Karten findest du hier:

README.es.md

@@ -190,6 +190,21 @@ Más opciones:
 
 Todos los tamaños, descargas y lo que cabe en cada equipo: [docs/MODELS.md](docs/MODELS.md).
 
+### DeepSeek V4.1-Flash (en el Terminal, desde 0.2.4)
+
+[DeepSeek V4.1-Flash](https://huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF) es otro modelo, mucho más grande. ISTA-DASLab hizo los tres conjuntos de abajo. Strata los ejecuta en un PC con una tarjeta gráfica de 12 GB y de 46 a 64 GB de RAM, y lee el resto de tu SSD. Funciona en tarjetas NVIDIA RTX de las series 30 a 50 y en la Radeon AI PRO R9700. Usa su propio paquete de motor, `strata-dsv41`, que `strata launch` descarga cuando eliges DeepSeek. La app de escritorio todavía no lo ofrece.
+
+| Conjunto | Descarga | Calidad frente al modelo original | Responde a |
+| --- | ---: | --- | --- |
+| **IQ1_M** (1,75 bits) | 239 GB | KL 0,217, perplejidad 3,80 (+21 %); código +8,9 %, tareas de agente +10,2 %; más débil con prosa inglesa corriente. Según la ficha del modelo de ISTA-DASLab, conserva el 98,7 % del promedio de tareas del modelo base (AIME25, GPQA-Diamond, LiveCodeBench v6). | aún sin medir |
+| **IQ2_XXS** (2.00 bits) | 257 GB | KL 0,170, perplejidad 3,63 (+15,5 %); código +8,6 %, tareas de agente +5,3 % | de 6 a 9 tokens/s en una RTX 3060, unos 5 en una RTX 5070 |
+| **IQ2_XS** (2.25 bits) | 274 GB | KL 0,107, perplejidad 3,37 (+7,3 %); código +3,2 %, tareas de agente +2,2 % | unos 11 tokens/s en una Radeon AI PRO R9700 |
+| **IQ4_NL** (tabla Engram, no es un tercer modelo) | 111 GB, ya incluidos en las tres descargas de arriba | La tabla de consulta Engram de 4 bits que el modelo lee de la RAM o del SSD. Los tres conjuntos usan este único archivo (`DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf`). | n/a |
+
+4x Radeon AI PRO R9700: unos 31 tokens/s, de 55 a 69 con DSpark.
+
+Inícialo con `strata launch --model ist-das-2.00bpw` (o `ist-das-1.75bpw`, `ist-das-2.25bpw`). Detalles, velocidades por tarjeta e instalación manual: [docs/DEEPSEEK41.md](docs/DEEPSEEK41.md).
+
 ## ¿Qué velocidad tiene?
 
 Medido con una tarjeta gráfica de 12 GB (RTX 5070, Ryzen 5 7600, 64 GB de RAM). Resultados con otras tarjetas:

README.fr.md

@@ -191,6 +191,21 @@ Autres choix :
 
 Toutes les tailles, les téléchargements et ce qui tient où : [docs/MODELS.md](docs/MODELS.md).
 
+### DeepSeek V4.1-Flash (dans le Terminal, depuis 0.2.4)
+
+[DeepSeek V4.1-Flash](https://huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF) est un autre modèle, beaucoup plus gros. ISTA-DASLab a produit les trois versions ci-dessous. Strata les fait tourner sur un PC avec une carte graphique de 12 Go et de 46 à 64 Go de RAM, et lit le reste sur votre SSD. Il fonctionne sur les cartes NVIDIA RTX des séries 30 à 50 et sur la Radeon AI PRO R9700. Il utilise son propre paquet de moteur, `strata-dsv41`, que `strata launch` télécharge quand vous choisissez DeepSeek. L’application de bureau ne le propose pas encore.
+
+| Version | Téléchargement | Qualité par rapport au modèle d’origine | Écrit à |
+| --- | ---: | --- | --- |
+| **IQ1_M** (1,75 bit) | 239 Go | KL 0,217, perplexité 3,80 (+21 %) ; code +8,9 %, tâches d’agent +10,2 % ; plus faible sur la prose anglaise courante. D’après la fiche du modèle d’ISTA-DASLab, elle garde 98,7 % de la moyenne des tâches du modèle de base (AIME25, GPQA-Diamond, LiveCodeBench v6). | pas encore mesurée |
+| **IQ2_XXS** (2.00 bits) | 257 GB | KL 0,170, perplexité 3,63 (+15,5 %) ; code +8,6 %, tâches d’agent +5,3 % | 6 à 9 tokens/s sur une RTX 3060, environ 5 sur une RTX 5070 |
+| **IQ2_XS** (2.25 bits) | 274 GB | KL 0,107, perplexité 3,37 (+7,3 %) ; code +3,2 %, tâches d’agent +2,2 % | environ 11 tokens/s sur une Radeon AI PRO R9700 |
+| **IQ4_NL** (table Engram, pas un troisième modèle) | 111 Go, déjà compris dans les trois téléchargements ci-dessus | La table de consultation Engram en 4 bits que le modèle lit dans la RAM ou sur le SSD. Les trois versions utilisent ce seul fichier (`DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf`). | n/a |
+
+4x Radeon AI PRO R9700 : environ 31 tokens/s, 55 à 69 avec DSpark.
+
+Lancez-le avec `strata launch --model ist-das-2.00bpw` (ou `ist-das-1.75bpw`, `ist-das-2.25bpw`). Détails, vitesses par carte et installation manuelle : [docs/DEEPSEEK41.md](docs/DEEPSEEK41.md).
+
 ## Quelle est sa vitesse ?
 
 Mesuré avec une carte graphique de 12 Go (RTX 5070, Ryzen 5 7600, 64 Go de RAM). Voir les résultats d'autres cartes :

README.ja.md

@@ -183,6 +183,21 @@ Gyro は NVIDIA と AMD のカード（AMD は実験的）、および Linux 上
 
 すべてのサイズ、ダウンロード、どの構成で何が動くかは [docs/MODELS.md](docs/MODELS.md) にあります。
 
+### DeepSeek V4.1-Flash（ターミナル版、0.2.4 から）
+
+[DeepSeek V4.1-Flash](https://huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF) は、これとは別のずっと大きなモデルです。下の 3 つのセットは ISTA-DASLab が作りました。Strata は、12 GB のグラフィックスカード 1 枚と 46 から 64 GB の RAM を持つ PC でこれを動かし、残りは SSD から読みます。NVIDIA RTX 30 から 50 シリーズのカードと Radeon AI PRO R9700 で動きます。専用のエンジンパッケージ `strata-dsv41` を使い、DeepSeek を選ぶと `strata launch` がダウンロードします。デスクトップアプリはまだ対応していません。
+
+| セット | ダウンロード | 元のモデルとの品質差 | 出力速度 |
+| --- | ---: | --- | --- |
+| **IQ1_M**（1.75 ビット） | 239 GB | KL 0.217、パープレキシティ 3.80（+21%）、コード +8.9%、エージェント作業 +10.2%、通常の英語の散文では弱めです。ISTA-DASLab のモデルカードによると、基本モデルのタスク平均の 98.7% を保ちます（AIME25、GPQA-Diamond、LiveCodeBench v6）。 | 未測定 |
+| **IQ2_XXS** (2.00 ビット) | 257 GB | KL 0.170、パープレキシティ 3.63（+15.5%）、コード +8.6%、エージェント作業 +5.3% | RTX 3060 で毎秒 6 から 9 トークン、RTX 5070 で約 5 |
+| **IQ2_XS** (2.25 ビット) | 274 GB | KL 0.107、パープレキシティ 3.37（+7.3%）、コード +3.2%、エージェント作業 +2.2% | Radeon AI PRO R9700 1 枚で毎秒約 11 トークン |
+| **IQ4_NL**（Engram テーブル、3 つ目のモデルではありません） | 111 GB、上の 3 つのダウンロードに含まれています | モデルが RAM または SSD から読む 4 ビットの Engram 参照テーブルです。3 つのセットはこの 1 つのファイル（`DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf`）を共有します。 | 該当なし |
+
+Radeon AI PRO R9700 4 枚：毎秒約 31 トークン、DSpark ありで 55 から 69。
+
+`strata launch --model ist-das-2.00bpw`（または `ist-das-1.75bpw`、`ist-das-2.25bpw`）で起動します。詳細、カードごとの速度、手動インストールは [docs/DEEPSEEK41.md](docs/DEEPSEEK41.md) にあります。
+
 ## どのくらい速い？
 
 12 GBのグラフィックスカード（RTX 5070、Ryzen 5 7600、RAM 64 GB）での数値です。ほかのカードの結果は

README.md

@@ -188,6 +188,21 @@ More choices:
 
 Every size, download and what fits where: [docs/MODELS.md](docs/MODELS.md).
 
+### DeepSeek V4.1-Flash (in the Terminal, from 0.2.4)
+
+[DeepSeek V4.1-Flash](https://huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF) is a different, much bigger model. ISTA-DASLab made the three sets below. Strata runs them on a PC with one 12 GB graphics card and 46 to 64 GB of RAM, and reads the rest from your SSD. It runs on NVIDIA RTX 30 to 50 cards and on the Radeon AI PRO R9700. It uses its own engine package, `strata-dsv41`, which `strata launch` downloads when you pick DeepSeek. The desktop app does not offer it yet.
+
+| Set | Download | Quality against the original model | Writes at |
+| --- | ---: | --- | --- |
+| **IQ1_M** (1.75 bits) | 239 GB | KL 0.217, perplexity 3.80 (+21%); code +8.9%, agent work +10.2%; weaker on plain English prose. ISTA-DASLab's model card: keeps 98.7% of the base model's task average (AIME25, GPQA-Diamond, LiveCodeBench v6). | not measured yet |
+| **IQ2_XXS** (2.00 bits) | 257 GB | KL 0.170, perplexity 3.63 (+15.5%); code +8.6%, agent work +5.3% | 6 to 9 tokens/s on an RTX 3060, about 5 on an RTX 5070 |
+| **IQ2_XS** (2.25 bits) | 274 GB | KL 0.107, perplexity 3.37 (+7.3%); code +3.2%, agent work +2.2% | about 11 tokens/s on one Radeon AI PRO R9700 |
+| **IQ4_NL** (Engram table, not a third model) | 111 GB, already in all three downloads above | The 4-bit Engram lookup table the model reads from RAM or SSD. All three sets use this one file (`DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf`). | n/a |
+
+4x Radeon AI PRO R9700: about 31 tokens/s, 55 to 69 with DSpark.
+
+Start it with `strata launch --model ist-das-2.00bpw` (or `ist-das-1.75bpw`, `ist-das-2.25bpw`). Details, speeds per card and the manual install: [docs/DEEPSEEK41.md](docs/DEEPSEEK41.md).
+
 ## How fast is it?
 
 Based on a 12 GB graphics card (RTX 5070, Ryzen 5 7600, 64 GB RAM). See results from other cards:

README.pt-BR.md

@@ -183,6 +183,21 @@ Mais opções:
 
 Todos os tamanhos, downloads e o que cabe em cada configuração: [docs/MODELS.md](docs/MODELS.md).
 
+### DeepSeek V4.1-Flash (no Terminal, a partir da 0.2.4)
+
+[DeepSeek V4.1-Flash](https://huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF) é outro modelo, bem maior. A ISTA-DASLab fez os três conjuntos abaixo. O Strata os executa em um PC com uma placa de vídeo de 12 GB e de 46 a 64 GB de RAM, e lê o resto do seu SSD. Ele roda em placas NVIDIA RTX das séries 30 a 50 e na Radeon AI PRO R9700. Usa um pacote de motor próprio, `strata-dsv41`, que o `strata launch` baixa quando você escolhe o DeepSeek. O app de desktop ainda não o oferece.
+
+| Conjunto | Download | Qualidade em relação ao modelo original | Responde a |
+| --- | ---: | --- | --- |
+| **IQ1_M** (1,75 bits) | 239 GB | KL 0,217, perplexidade 3,80 (+21%); código +8,9%, tarefas de agente +10,2%; mais fraco em prosa comum em inglês. Segundo o cartão do modelo da ISTA-DASLab, mantém 98,7% da média de tarefas do modelo base (AIME25, GPQA-Diamond, LiveCodeBench v6). | ainda não medida |
+| **IQ2_XXS** (2.00 bits) | 257 GB | KL 0,170, perplexidade 3,63 (+15,5%); código +8,6%, tarefas de agente +5,3% | 6 a 9 tokens/s em uma RTX 3060, cerca de 5 em uma RTX 5070 |
+| **IQ2_XS** (2.25 bits) | 274 GB | KL 0,107, perplexidade 3,37 (+7,3%); código +3,2%, tarefas de agente +2,2% | cerca de 11 tokens/s em uma Radeon AI PRO R9700 |
+| **IQ4_NL** (tabela Engram, não é um terceiro modelo) | 111 GB, já incluídos nos três downloads acima | A tabela de consulta Engram de 4 bits que o modelo lê da RAM ou do SSD. Os três conjuntos usam este único arquivo (`DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf`). | n/a |
+
+4x Radeon AI PRO R9700: cerca de 31 tokens/s, 55 a 69 com DSpark.
+
+Inicie com `strata launch --model ist-das-2.00bpw` (ou `ist-das-1.75bpw`, `ist-das-2.25bpw`). Detalhes, velocidades por placa e a instalação manual: [docs/DEEPSEEK41.md](docs/DEEPSEEK41.md).
+
 ## Qual é a velocidade?
 
 Com base numa placa de vídeo de 12 GB (RTX 5070, Ryzen 5 7600, 64 GB de RAM). Veja resultados de outras placas:

README.zh-CN.md

@@ -182,6 +182,21 @@ Gyro 可在 NVIDIA 和 AMD 显卡（AMD 为实验性支持）以及 Linux 上的
 
 所有尺寸、下载大小，以及各种配置能跑什么：[docs/MODELS.md](docs/MODELS.md)。
 
+### DeepSeek V4.1-Flash（终端版，0.2.4 起）
+
+[DeepSeek V4.1-Flash](https://huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF) 是另一个大得多的模型。下面三个版本由 ISTA-DASLab 制作。Strata 在一台有一块 12 GB 显卡和 46 到 64 GB 内存的电脑上运行它们，其余部分从固态硬盘读取。它可在 NVIDIA RTX 30 到 50 系列显卡和 Radeon AI PRO R9700 上运行。它使用自己的引擎包 `strata-dsv41`，你选择 DeepSeek 时由 `strata launch` 下载。桌面应用暂时不提供它。
+
+| 版本 | 下载大小 | 与原模型相比的质量 | 生成速度 |
+| --- | ---: | --- | --- |
+| **IQ1_M**（1.75 位） | 239 GB | KL 0.217，困惑度 3.80（+21%）；代码 +8.9%，智能体任务 +10.2%；普通英文散文表现较弱。据 ISTA-DASLab 的模型卡，它保留了基础模型任务平均分的 98.7%（AIME25、GPQA-Diamond、LiveCodeBench v6）。 | 尚未测量 |
+| **IQ2_XXS** (2.00 位) | 257 GB | KL 0.170，困惑度 3.63（+15.5%）；代码 +8.6%，智能体任务 +5.3% | RTX 3060 上每秒 6 到 9 个 token，RTX 5070 上约 5 个 |
+| **IQ2_XS** (2.25 位) | 274 GB | KL 0.107，困惑度 3.37（+7.3%）；代码 +3.2%，智能体任务 +2.2% | 一块 Radeon AI PRO R9700 上每秒约 11 个 token |
+| **IQ4_NL**（Engram 表，不是第三个模型） | 111 GB，已包含在上面三个下载中 | 模型从内存或固态硬盘读取的 4 位 Engram 查找表。三个版本共用这一个文件（`DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf`）。 | 不适用 |
+
+4 块 Radeon AI PRO R9700：每秒约 31 个 token，使用 DSpark 时为 55 到 69。
+
+用 `strata launch --model ist-das-2.00bpw`（或 `ist-das-1.75bpw`、`ist-das-2.25bpw`）启动。详情、各显卡的速度和手动安装见 [docs/DEEPSEEK41.md](docs/DEEPSEEK41.md)。
+
 ## 速度如何？
 
 以下数据基于 12 GB 显卡（RTX 5070、Ryzen 5 7600、64 GB 内存）。其他显卡的结果见：

data/dsv41/profile-2.00.bin.sha256

@@ -0,0 +1 @@
+8db93e483b34b15dbf63a0826601f1db2107a8cfea73774e7d21d8559e0a9568 *profile-2.00.bin

data/dsv41/profile-2.25.bin.sha256

@@ -0,0 +1 @@
+c8de062c5f3c71725ecf5328bd4ee02c71dca88478bf7f6b0f2ec258567ac6b6 *profile-2.25.bin

docs/DEEPSEEK41.md

@@ -0,0 +1,142 @@
+# DeepSeek-V4.1-Flash
+
+DeepSeek-V4.1-Flash is a large mixture-of-experts model: 40 layers, 384 experts in each (6 are used for every word), and a 203 GB
+n-gram table next to them. Strata runs it with its own engine, `strata-dsv41` (`src/dsv41`, ported from Project Maya's
+`maya-ds4`, MIT; the notice is in `third_party/project-maya`). The file is Unsloth's UD-Q8_K_XL, pinned in `setup.py`: nine
+shards, 508.4 GB, the experts as MXFP4 and the rest in BF16. The two official 2-bit sets (below) are what a normal PC runs.
+
+The 508 GB set is for a computer with several graphics cards and a lot of RAM. It was run on four Radeon AI PRO R9700 (32 GB each) with
+251 GB of RAM. Strata does not refuse a smaller computer; it tells you what to expect first.
+
+## Where it goes
+
+| | size | where |
+|---|---|---|
+| routed experts (15,360 of 18.8 MB) | 288.8 GB | as many as fit in graphics memory; the rest are computed on the CPU, read in place from the model files |
+| attention and other dense weights | 16.8 GB | graphics memory, one block of layers on each card (about 3.9 GB a card on four) |
+| n-gram table (Engram) | 202.8 GB | the SSD. A few rows are read for each word it writes. It is not counted in RAM, but a slow disk slows every word, so keep the model files on an NVMe drive |
+
+Measured on the four R9700 cards: 6,197 experts (116.5 GB) in graphics memory, 9,163 (172.3 GB) on the CPU, which needs about that
+much free RAM to stay cached. `strata launch --plan` and `setup.py --check` work this out for your cards and RAM
+(`strata_control/dsv41_fit.py`; it predicted 6,240 experts on the R9700 box).
+
+With less memory than that, part of the CPU's experts are read from the SSD while the model answers. It runs, slowly. A card
+that cannot hold its share of the dense weights (about 4 GB each on four cards, 16 GB on one) can make the engine stop at the start
+with an out-of-memory message. Both are said before anything is downloaded.
+
+## The official 2-bit versions (the ones for a normal PC)
+
+Setup offers three official sets (ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF, pinned): **IQ1_M, 1.75 bpw** (`ist-das-1.75bpw`, 121 GB, slightly lower quality; setup recommends it where 2.00 is tight in memory or disk), **IQ2_XXS, 2.00 bpw** (138 GB) and **IQ2_XS, 2.25 bpw** (155 GB). All use the
+same 111 GB n-gram table (downloaded once into `models/deepseek-v4.1-flash-shared`) and the 8 GB DSpark drafter: 257 GB of disk for
+2.00, 274 GB for 2.25, 239 GB for 1.75, 533 GB for all three. Quality against the original model (4,995 positions, the final files): 2.00 has KL 0.170 and perplexity +15.5% (code +8.6%, agent work +5.3%); 2.25 has KL 0.107 and
+perplexity +7.3% (code +3.2%, agent work +2.2%). IQ1_M: perplexity +21% (KL 0.217, top-1 82.5%); code +8.9%, agent work +10.2%, multilingual +14%, docs +24%, plain English prose +53%, so it is weaker on prose. ISTA-DASLab's model card says it keeps 98.7% of the base model's task average (AIME25, GPQA-Diamond, LiveCodeBench v6).
+
+| File | Size | What it is |
+| --- | ---: | --- |
+| IQ1_M shards (1.75 bpw), 4 files | 121 GB | the model, smallest set |
+| IQ2_XXS shards (2.00 bpw), 4 files | 138 GB | the model, middle set |
+| IQ2_XS shards (2.25 bpw), 4 files | 155 GB | the model, largest set |
+| IQ4_NL: `DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf` | 111 GB | the 4-bit Engram lookup table the model reads from RAM or SSD. Not a model to pick: all three sets use this one file, and it is already in their download sizes |
+| DSpark drafter (Q8_0) | 8 GB | the small draft model (Unsloth) |
+
+121 + 111 + 8 = 239 GB for 1.75; 138 + 111 + 8 = 257 GB for 2.00; 155 + 111 + 8 = 274 GB for 2.25.
+
+Measured: an RTX 5070 (12 GB) with 64 GB of RAM and a 3.6 GB/s NVMe drive answers at about 4.7 tokens/s and reads prompts at 121
+tokens/s (4K) to 231 tokens/s (8K, layer-major path); an RTX 3060 (12 GB) with 46 GB of RAM about 7.2 tokens/s; four R9700 about 31
+tokens/s. Under about 40 GB of RAM it is untested and slow; setup says so and still installs.
+
+Setup downloads with resume and SHA-256, packs the model with the plain recipe (`pack_gguf.py --model SET --shards ... --engram
+engram-IQ4_NL.gguf --engram-meta <UD shard 1> --dspark-gguf dspark-...-Q8_0.gguf`; do not add `--skip-tensors` or `--dense-override`:
+that packs without an error and gives garbage text), copies the shipped hot-expert profile (`data/dsv41/profile-2.00.bin` or `-2.25.bin`)
+into the pack as `expert-profile.bin`, checks `strata-dsv41 --pack-info` (every file present) and answers a 16-token test question
+(`tools/dsv41/smoke.py`, the text must name Paris). The start script passes `--dspark auto` (never `off`: `off` still loads the drafter
+and its 7 GB of experts take the graphics memory); the RAM store sizes itself. Long prompts use the engine's fast chunked path on its own (no flag is written: the engine picks the chunk size for your cards).
+
+## Install
+
+```
+./setup.sh --family deepseek41 --model IST-DAS-2.00bpw   # 257 GB: the official 2.00 set + Engram + drafter (resumable, SHA-256 checked)
+./setup.sh --family deepseek41 --model IST-DAS-2.25bpw   # 274 GB (the Engram is shared with 2.00)
+./setup.sh --family deepseek41 --model IST-DAS-1.75bpw   # 239 GB, the smallest
+./setup.sh --family deepseek41 --model UD-Q8_K_XL        # Unsloth's 508 GB set, for hundreds of GB of RAM or several cards
+./setup.sh --family deepseek41 --gguf-dir DIR            # files you already have: the shards (or the folder with 2.00bpw / 2.25bpw / UD-Q8_K_XL inside); the Engram and drafter are looked for beside them
+strata launch --model ist-das-2.00bpw --plan             # where it would go on this computer, nothing changed
+strata launch --model-path DIR --model ist-das-2.00bpw   # start from files you already have
+```
+
+Setup needs `strata-dsv41` next to the engine (or `STRATA_DSV41_EXE`) and says so before it downloads anything. Build it with
+`cmake -DSTRATA_BUILD_DSV41=ON`. The default context is 8192 tokens, the size it was checked at; `--context` asks for more.
+Loading takes a few minutes (the config sets `"engine_ready_s": 3600`).
+
+## Through the API
+
+The server speaks to the engine as it does for the other models, so `/v1/chat/completions`, `/v1/messages`, streaming, stop and
+`/metrics` work as usual (the answer speed is the tokens the model produced over the engine's decode time).
+
+- Thinking is on by default, as the model's template has it. `reasoning_effort: "none"` or `chat_template_kwargs.enable_thinking: false` turns it off.
+  Effort words: `low` (50), `medium` (62), `high` (75, the default), `max` (100), or a whole number 1 to 100.
+- Prompts are the GGUF's own chat template, the same bytes as DeepSeek's `encoding.py` on the cases checked. Project Maya's own template lost the
+  tool associations and later system instructions (their issue #299); Strata does not use it.
+- Tool calls: the model writes DSML (`<｜DSML｜ calls>` ... `<｜DSML｜ invoke name="...">` ... `<｜DSML｜ parameter name="..." string="true">`). The
+  server turns them into OpenAI `tool_calls` and Anthropic `tool_use`, including `tool_choice`, streamed arguments and calls written inside the thinking.
+  A value marked `string="true"` stays a string.
+- Defaults when a request names none: temperature 1.0, top_p 1.0.
+- Agents and sub-agents that take turns can skip reading their prompts again (off in this version, a switch for testing). With
+  `STRATA_DS4_SLOT_KEEP=1` the engine sets a conversation aside when another request takes the model (about 17 MB per 6.5K tokens) and takes it
+  back when the next prompt continues it, also after a restart. Slots live in `ds4-slots` next to the run log, are capped at 2 GB
+  (`STRATA_DS4_SLOT_GB`) and are removed after a week. `STRATA_DS4_AUTO_PIN=1` also keeps the system prompt and tools shared by an agent
+  and its sub-agents as a checkpoint.
+
+Not in this version: several requests at once (`--parallel` is ignored with a warning) and pictures (`--vision` is ignored with a warning).
+
+## Tuning
+
+`strata launch --tune quick` (or `thorough`, or the Tuning screen) works for DeepSeek with its own short tuner, `tools/dsv41/tune.py`. It starts
+the engine a few times on a fixed short question and measures the answer speed: the standard settings first, then the DSpark drafter forced on,
+then a shorter draft block (3), then one CPU thread for each physical core (thorough also tries every logical thread). A setting is kept only for a
+gain of more than 5%, and the result is written into the launch's settings. Expect about 5 to 10 minutes (each run loads the model, which takes
+1 to 3 minutes); close other heavy programs first. If a run fails or the computer is too busy to tell, the standard settings stay and one plain line
+says so. The RAM store (`--ssd-cache`) sizes itself when the model starts, so the tuner does not touch it, and the context stays what you chose.
+
+## The engine package
+
+DeepSeek's engine, `strata-dsv41`, is a separate download from the Strata engine: when you choose DeepSeek, setup (or `strata launch`) fetches the
+package for your computer from the signed update feed (resumable, signature and every file checked) and installs it in `engine-dsv41/` beside
+`engine/`. The Qwen engine's files are never changed. If the feed has no package for your graphics card, setup says so in plain words and stops before
+anything big is downloaded.
+
+### Getting the engine by hand
+
+If the automatic download does not work for you (no access to the update feed, or an offline computer), download the engine package yourself from
+the release page: the file is named `strata-dsv41-<version>-<system>-<backend>-<variant>.tar.gz` (Windows and Linux; pick the one for your system and
+graphics card). Check it against `SHA256SUMS.txt` on the release page if you like. Then give it to Strata in either of two ways:
+
+```
+strata launch --model ist-das-2.00bpw --dsv41-package PATH-TO-THE-FILE
+python setup.py --family deepseek41 --dsv41-package PATH-TO-THE-FILE
+```
+
+or put the file in the `engine-dsv41/` folder (setup tells you its exact place when it cannot find the engine) and run the same command again.
+Strata checks the package before it uses it: it must be the DeepSeek engine (not the Strata engine) for your system, its manifest must be signed by a key
+Strata trusts, and every file must match the hash in the manifest. A damaged or foreign file is refused with a plain line and nothing is installed.
+The unpacked folder works as well as the archive. The Strata (Qwen) engine is not touched either way.
+
+## Compressed experts (trellis), for testing
+
+`strata-dsv41` reads the routed experts in the trellis types as well as the source MXFP4: TQK6 and TQK7 (agentionai's files), and the
+experimental TQK5X / TQK4X (computed codebook) and TQK5T / TQK4T (table codebook), see `tools/trellis/README.md`. A layer may be converted
+while the others stay MXFP4. On the GPU they run on the tensor cores of gfx12 (and in plain FMAs elsewhere), on the CPU lane on Strata's
+AVX-512 / AVX2 kernels; the build needs `-DSTRATA_NATIVE_EXPERTS=ON` for the CPU lane.
+
+```
+# 1. the pack of the source model (also takes the DSpark and mmproj files that sit beside the model folder)
+python tools/dsv41/pack_gguf.py --model DIR/UD-Q8_K_XL --out PACK
+# 2. re-encode some layers (HIP encoder; one layer at K5 is 2.3 GB, the whole model 93 GB)
+python tools/dsv41/tq_convert.py --pack PACK --out CONV --layers 1,7,14 --type K5      # K4 K5 K4T K5T K6 K7
+# 3. the pack that reads them
+python tools/dsv41/pack_gguf.py --model DIR/UD-Q8_K_XL --out PACK2 --tq-experts CONV
+strata-dsv41 --pack PACK2 --score data/quant_quality/v1/set.ids.json --score-out RUN.tsv      # teacher-forced top-64 rows
+```
+
+The encode is plain error (no rotation, no activation weights). The tests are `ds4_tq_test` (device and CPU lane against a double
+reference, needs one GPU) and `tqw_hip_test`.

docs/release-notes-0.2.3.1.md

@@ -0,0 +1,23 @@
+**0.2.3.1: a small fix for setup stopping on the Windows NVIDIA package.**
+
+## What is fixed
+
+- **Setup no longer stops on the Windows NVIDIA engine package (#3810).** Setup read the list of graphics-card generations the picture reader was built for one character at a time, so it crashed on the Windows CUDA package from 0.2.3. It now reads that list the way the package writes it. The Linux NVIDIA package writes the same list, so the same fix covers it. An installed AMD engine also had a check that never recognised it; that is fixed too.
+
+Only the Terminal's Control changed (version 0.1.48.1). The app stays 0.2.3 and the engine packages stay 0.1.48, so there is nothing new to download for them.
+
+## Updating
+
+- **From 0.2.3:** run `strata update` in the Terminal. It fetches Control 0.1.48.1, and the next `strata` command already runs it (`strata --version` says 0.2.3.1). The app is not touched.
+- **From an older version:** follow the steps in the 0.2.3 notes. The install command (`install.ps1` on Windows, `install.sh` on Linux) now installs this fix.
+- **Mac:** the Mac Terminal stays on Control 0.1.48. This fix does not touch the Mac.
+
+## Downloads
+
+Everything from 0.2.3 is attached again, so the release page still holds every file. The files that changed are `strata-control-0.1.48.1-windows-x86_64.zip`, `strata-control-0.1.48.1-linux-x86_64.tar.gz`, the feed (`strata-feed.json` with its `.sig`), `install-manifest.txt`, `latest.json` and `SHA256SUMS.txt`. The installers, AppImages, dmg and engine packages are the 0.2.3 files.
+
+---
+
+Strata is free and open source. If it runs well on your PC, a coffee keeps the work on it going:
+
+<a href="https://buymeacoffee.com/strataengine"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" height="50"></a>

docs/release-notes-0.2.4.1.md

@@ -0,0 +1,17 @@
+# Strata 0.2.4.1: DeepSeek setup works from the installed Terminal
+
+**Fixed:** setting up DeepSeek V4.1-Flash from the installed Terminal stopped at "Preparing your model" with "can't open file .../tools/dsv41/pack_gguf.py". The Terminal and Control packages of 0.2.4 did not contain the DeepSeek preparation scripts (`pack_gguf.py`, `smoke.py` and the short tune, `tune.py`); they ran only from a source checkout. They are in the packages now, and so are three scripts that other commands run by path (`strata_inspect.py`, `strata_bench.py`, `nvfp4_local_pack.py`).
+
+Only the Terminal's Control changed (version 0.1.49.1). The app stays 0.2.3 and every engine package stays as it was, so there is nothing new to download for them.
+
+## Updating
+
+- **From 0.2.4:** run `strata update` in the Terminal. It fetches Control 0.1.49.1; the next `strata` command already runs it (`strata --version` says 0.2.4.1). Then run your DeepSeek setup again.
+- **New installs:** the install command (`install.ps1` on Windows, `install.sh` on Linux) installs this version.
+- **Mac:** the Mac Terminal gets Control 0.1.49.1 too. DeepSeek is not on the Mac yet.
+
+---
+
+Strata is free and open source. If it runs well on your PC, a coffee keeps the work on it going:
+
+<a href="https://buymeacoffee.com/strataengine"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" height="50"></a>

docs/release-notes-0.2.4.md

@@ -0,0 +1,93 @@
+# Strata 0.2.4: DeepSeek V4.1-Flash in the Terminal
+
+Strata now runs DeepSeek V4.1-Flash on a PC with one 12 GB graphics card and 46 to 64 GB of RAM. The engine keeps the experts it uses most in RAM and streams the rest from your SSD.
+
+**DeepSeek is in the Terminal (`strata launch`, `strata setup`) in this release. The desktop app is unchanged (it stays 0.2.3) and gets DeepSeek in a later update.**
+
+## What you get
+
+- **DeepSeek V4.1-Flash** in the model list of Setup, in three sizes from [ISTA-DASLab](https://huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF):
+
+  | set | download | quality against the original model |
+  |---|---|---|
+  | IQ1_M (1.75 bits) | 239 GB | perplexity +21% (KL 0.217); code and agent text +8.9% and +10.2%; weaker on plain English prose. ISTA-DASLab's model card: keeps 98.7% of the base model's task average (AIME25, GPQA-Diamond, LiveCodeBench v6) |
+  | IQ2_XXS (2.00 bits) | 257 GB | perplexity +15.5% (KL 0.170); code and agent text +8.6% and +5.3% |
+  | IQ2_XS (2.25 bits) | 274 GB | perplexity +7.3% (KL 0.107); code and agent text +3.2% and +2.2% |
+  | IQ4_NL (Engram table, not a model to pick) | 111 GB, already in all three downloads above | The 4-bit Engram lookup table the model reads from RAM or SSD; all three sets use this one file |
+
+  "Perplexity" is how surprised the model is by a fixed set of 4,995 test texts; lower is closer to the original. Pick IQ2_XS for coding and agent work when you have the disk. Setup shows all three sizes and warns if your PC is short of disk or RAM. It does not stop you.
+- Setup downloads the files, builds the model folder and starts the engine. You set no flags.
+
+## Speed on real PCs
+
+Output is the speed of the answer. Prompt is how fast Strata reads what you send it.
+
+| PC | output | prompt |
+|---|---|---|
+| RTX 5070 12 GB, 64 GB RAM | about 5 tokens/s (code) | 231 tokens/s at 8K |
+| RTX 3060 12 GB, 46 GB RAM | about 6 to 9 tokens/s (6.1-6.7 without DSpark, 8.2-8.7 with) | about 103 tokens/s at 8K |
+| 1x Radeon AI PRO R9700 (AMD), IQ2_XS | about 11 tokens/s | not measured |
+| 4x Radeon AI PRO R9700 (AMD) | about 31 tokens/s (55 to 69 with DSpark) | about 505 tokens/s at 8K, about 818 at 32K |
+
+These are the 2.00-bit set. The 2.25-bit set reads 13% more data per token, so expect it to be a little slower. Your drive matters: the 12 GB cards spend most of each token waiting on the SSD and on the CPU, and a slow NVMe drive lowers every number above.
+
+## How it works
+
+- **A RAM expert store that sizes itself.** The model is a mixture of experts: each token uses a few of thousands of small blocks. Strata keeps the busiest ones in RAM, takes whatever RAM is free, and reads the rest from the SSD with direct reads. On the RTX 5070 PC with 64 GB this gave 4.3 to 4.9 tokens/s where memory-mapping the files gave 1.8 to 2.2.
+- **Hot-expert profiles ship with the engine,** so the first prompts do not start cold.
+- **Faster prompts.** The engine now reads long prompts layer by layer in big chunks instead of 16 tokens at a time. A 200-token prompt went from 3.6 to 6.4 tokens/s on the RTX 5070 PC, and 8K prompts reach 231 tokens/s.
+- **Faster prompts on Radeon AI PRO R9700.** New matrix-core kernels (RDNA4) read prompts 36% faster at 8K and 49% faster at 32K on 4 cards (370 to 505 and 550 to 818 tokens/s). Answer speed does not change. They are on by default on these cards.
+- **Faster long prompts.** A prompt over about 3,700 tokens skips work that the last 20 layers cannot use (a 16K prompt is about 34% faster). In 99.5% of the 799 positions we compared, the model's top choice for the next word was the same with and without the skip.
+- **Thinking budget.** When neither your request nor your config sets one, DeepSeek thinks for at most 32,768 tokens. A request can set its own budget; 0 turns the limit off.
+- **DSpark** (a small draft model from Unsloth) is on where it measured faster and off on CPU-bound PCs. On the RTX 3060 PC (20 threads) it raised decode from 6.1-6.7 to 8.2-8.7 tokens/s. On the 4x R9700 box it raised 31 tokens/s to 55 to 69. On the RTX 5070 PC with a 6-core Ryzen 5 7600 it gave no gain at any setting (code 4.64 tokens/s with it off against 4.33 at best with it on; agent 3.41 against 2.86 to 3.21), because checking the drafts costs the CPU about what the saved tokens are worth. There `--dspark auto` does not load the drafter.
+- **Faster CPU rows.** The engine has new exact rows for the experts the CPU computes: AVX-512 on Zen 4 and Zen 5, AVX2 on other CPUs, chosen automatically. On the Ryzen 5 7600 PC with an RTX 5070 (2.00 set, DSpark on, three alternating pairs, identical text in all six runs), decode went from 4.08 to 5.11 tokens/s (+25%) and prompt reading rose 26%. This was measured on one CPU, a Zen 4. Zen 5 gained more in the kernel tests.
+- **Cards and engines:** tested on an RTX 3060 (Linux, full model), an RTX 5070 (Windows) and the Radeon AI PRO R9700 (gfx1201, Linux). The RTX 30 kernels (sm_86) were also checked on an RTX A4000. Windows on RTX 30 is built but was not run here. RX 9070 (gfx1200) is built but untested. DeepSeek runs on its own engine package (see below).
+
+## Needs
+
+- Disk: 239 GB (IQ1_M), 257 GB (IQ2_XXS) or 274 GB (IQ2_XS). The download includes the Engram table and the DSpark drafter.
+- RAM: 46 GB worked on the RTX 3060; 64 GB is the tested comfortable size. More RAM means a bigger store and faster answers.
+- Graphics card: 12 GB or more.
+
+## Known limits
+
+- Prompt speed on 12 GB cards depends on the drive. A slow NVMe drive holds the prompt below the numbers above.
+- DSpark is off on PCs where the CPU is the limit.
+- Not in this release: Mac and Strix Halo. DeepSeek V4.1-Flash does not run there yet.
+- Output on 12 GB cards is 4 to 8 tokens/s. That is usable for chat and slow for long agent runs.
+- The same prompt can give a slightly different answer from one run to the next. At some words the model's top choices are nearly tied, and small differences in how the numbers are added up decide between them.
+- If you set up DeepSeek on 4 or more AMD cards before 0.2.4, run setup again to get the faster prompt settings.
+
+## Updating
+
+- **From 0.2.3 or 0.2.3.1:** run `strata update` in the Terminal. It fetches Control 0.1.49. Your Qwen engine does not change: every engine package is still the 0.1.48 file, byte for byte.
+- **Mac:** the Mac Terminal gets Control 0.1.49 too. The Mac engine stays 0.1.48, and DeepSeek is not on the Mac yet.
+- **New installs:** the install command (`install.ps1` on Windows, `install.sh` on Linux) installs this version.
+
+## The DeepSeek engine is a separate download
+
+DeepSeek runs on its own engine package, `strata-dsv41`, which sits beside your Qwen engine and never replaces it. There are two ways to get it.
+
+- **Automatic:** choose DeepSeek in `strata launch` or `strata setup`. Strata downloads the package that fits your PC, checks its signature and installs it.
+- **By hand:** download the package for your system from this page, check it against `SHA256SUMS.txt`, and run `strata launch --model ist-das-2.00bpw --dsv41-package PATH-TO-FILE` (or `python setup.py --family deepseek41 --dsv41-package PATH-TO-FILE`). Strata checks that it is the DeepSeek engine, that it fits your system and that its signature and file hashes are good, and refuses anything else. You can also put the file in the `engine-dsv41` folder next to `engine`.
+
+| System | File |
+|---|---|
+| Windows, NVIDIA RTX 30 / 40 / 50 | `strata-dsv41-0.1.49-windows-x86_64-cuda-cuda13.tar.gz` |
+| Linux, NVIDIA RTX 30 / 40 / 50 | `strata-dsv41-0.1.49-linux-x86_64-cuda-cuda13.tar.gz` |
+| Linux, Radeon AI PRO R9700 / RX 9070 | `strata-dsv41-0.1.49-linux-x86_64-hip-rx9070.tar.gz` |
+
+## Downloads
+
+Everything from 0.2.3 is attached again, so the release page still holds every file. New in this release: the three Control packages (`strata-control-0.1.49-windows-x86_64.zip`, `-linux-x86_64.tar.gz`, `-macos-arm64.tar.gz`) and the three DeepSeek packages above. The installers, AppImages, dmg and the Qwen engine packages are the 0.2.3 files.
+
+## Thanks
+
+- **ISTA-DASLab** for the IQ1_M, IQ2_XXS and IQ2_XS sets of DeepSeek V4.1-Flash ([ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF](https://huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF)).
+- **Unsloth** for the DSpark drafter GGUF.
+
+---
+
+Strata is free and open source. If it runs well on your PC, a coffee keeps the work on it going:
+
+<a href="https://buymeacoffee.com/strataengine"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" height="50"></a>

include/strata/dsv41/exl3.hpp

@@ -0,0 +1,187 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// include/strata/kernels/exl3.hpp - EXL3 (exllamav3) quantized weights on the device: the decode GEMV, the routed
+// experts, and the full reconstruction for the prompt path.
+//
+// The format (exllamav3, MIT, turboderp).  A weight with k inputs and n outputs (y = x W) is stored as
+//   trellis  int16 [k/16][n/16][16 K]: each 16x16 tile is a tail-biting trellis of 256 K-bit steps read MSB-first
+//            from its 8 K uint32 words (little-endian pairs of the int16s).  Step p's state is the 16-bit window that
+//            ends at bit (p + 1) K, modulo the tile's 256 K bits; the codebook turns the state into a value:
+//              mul1 (cb 2):  fp16(1024 + bytesum(state * 0x83DCD12D)) * fp16(0x1eee) + fp16(0xc931), one FP16 FMA
+//              mcg (cb 1):   x = state * 0xCBAC1FED, 3inst (cb 0): x = state * 89226354 + 64248484; then
+//                            x = (x & 0x8fff8fff) ^ 0x3b603b60 and the value is the FP16 sum of x's two halves.
+//            Step p sits at tile row (p/8 % 4) * 2 + (p & 1) + 8 (p/2 & 1), column p/32 + 8 (p/4 & 1) (the tensor-
+//            core fragment order).
+//   suh, svh FP16 [k], [n]: the input and output channel scales (signs and magnitudes).
+// With H the 128-point Sylvester Hadamard over sqrt(128), applied to each 128 consecutive values:
+//   y = H(H(x * suh) W') * svh,   W' the decoded k x n matrix.
+//
+// Activations enter as F32 or as the engine's q8_1 blocks; the kernels rotate them on the fly (each block rotates
+// the slice it reads), so callers keep their existing buffers.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cuda_runtime.h>
+
+namespace strata::dsv41k::exl3 {
+
+/// The weight type the engine's MvJob / WSlot carry for EXL3: their weight pointer is a (host) const Mat*.
+constexpr int kTypeEXL3 = 200;
+
+/// One EXL3 matrix, or a column range of one (a view: trellis/svh advanced to its first column, ld the row stride).
+struct Mat {
+    const uint32_t* trellis = nullptr;   // device; 4-byte aligned
+    const uint16_t* suh = nullptr;       // device FP16 bits [k]
+    const uint16_t* svh = nullptr;       // device FP16 bits [n]
+    int k = 0, n = 0;
+    int K = 0;                           // bits per weight, 1..8
+    int cb = 2;                          // codebook: 0 3inst, 1 mcg, 2 mul1
+    int ld = 0;                          // tiles per 16-row slice in memory (the full matrix's n / 16); 0: n / 16
+    // a GGUF's plain matrix instead (DeepSeek-V4.1's maya-ds4: fmt 1 BF16): raw = [n][ld] (row r = output r, ld
+    // values a row; 0: k), the EXL3 fields unused - the engine sends it to its own kernels, never to these
+    int fmt = 0;
+    const void* raw = nullptr;
+};
+
+/// Trellis bytes of a k x n matrix at K bits.
+inline size_t trellis_bytes(int64_t k, int64_t n, int K) { return (size_t) (k / 16) * (size_t) (n / 16) * 32u * K; }
+/// The columns [c0, c0 + nc) of m (c0 and nc multiples of 128: the output rotation's blocks stay whole).
+Mat view(const Mat& m, int c0, int nc);
+/// Whether the kernels take this matrix (k, n multiples of 128, K 1..8, cb 0..2).
+bool supported(const Mat& m);
+
+/// y[r] = alpha * (x W)[r] + (bias ? bias[r] : 0).  x: xf (F32) when given, else xq (q8_1 blocks of k values).
+/// ncol > 1: as many activations (a verify window's tokens), activation c at xf + c xf_cs / xq + c xq_cs bytes, its
+/// y at y + c y_cs - the weights read and decoded once for up to 8 of them, and each y bit for bit its own job's.
+struct MvJob {
+    const Mat* m = nullptr;
+    const float* xf = nullptr;
+    const void* xq = nullptr;
+    float* y = nullptr;
+    const float* bias = nullptr;
+    float alpha = 1.0f;
+    int ncol = 1;
+    size_t xf_cs = 0, xq_cs = 0, y_cs = 0;
+};
+constexpr int kMaxMvJobs = 8;
+constexpr int kMaxMvCols = 16;   // a job's activations at most
+bool mv(const MvJob* jobs, int n, cudaStream_t s);
+/// The calling thread's launches of mv(), moe_gate_up() and moe_down() read `flag` (a device int) first and do no work
+/// while it is non-zero (glmf::set_skip's, which sets it: a pipelined window's tokens past a wrong draft); nullptr (the
+/// default): no check.  The flag changes only between launches (skip_gate, on the same stream).  (v1.1.8's kernels
+/// before sm_80 do not read it.)
+void set_skip(const int* flag);
+/// How many pieces mv cuts m's inputs into on the current device (0: not covered) - one piece, and every column's
+/// sum is the same whatever columns the job holds (a view's, or the whole matrix's).
+int mv_splits(const Mat& m);
+/// How many activations a lone job of m can take with its weights read once on the current device (the input split's
+/// partials of them in the scratch; 0: not covered) - a caller with more cuts them in runs of this many.
+int mv_cols(const Mat& m);
+
+/// Allocates the per-stream scratch of mv / moe / win_experts on the current device (call once per stream before a
+/// graph capture; the first use does it otherwise).
+bool prepare(cudaStream_t s);
+
+/// One routed expert's blob: three parts (gate, up, down) at byte offsets from the blob base, each holding the FP16
+/// suh, the FP16 svh and the trellis of that matrix.
+struct ExpertLayout {
+    size_t suh[3] = {0, 0, 0}, svh[3] = {0, 0, 0}, trellis[3] = {0, 0, 0};
+    int K[3] = {0, 0, 0};
+    int cb = 2;
+    int fmt = 0;   // 2: a GGUF's MXFP4 expert (maya-ds4's own kernels): trellis[r] = matrix r's offset, [out][in / 32]
+};
+/// The routed experts of one token (the engine's device plan: plan_ptr[i] = the blob in VRAM, 0 = not on the device;
+/// plan_w[i] its weight).  gate/up over x (n_embd F32) -> clamped swiglu -> h (F32 [k][n_ff], scratch the caller
+/// owns), then down over h, the weighted sum in plan order, + cpu_part when *cpu_flag, + sh_out when given -> out.
+struct MoeArgs {
+    const unsigned long long* plan_ptr = nullptr;
+    const float* plan_w = nullptr;
+    const float* cpu_part = nullptr;
+    const int* cpu_flag = nullptr;
+    ExpertLayout lay;
+    int k = 8, n_embd = 4096, n_ff = 2048;
+    float limit = 0.0f;   // 0: no clamp
+};
+void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s);
+void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s);
+
+/// A verify window's routed experts, each distinct expert read once for all the tokens it serves (after exllamav3's
+/// fused MoE, which runs the slots that picked one expert as the rows of one tile): T tokens' plans [T][8] grouped by
+/// expert (glmf::moe_win_group: group g's blob at grp_ptr[g], its entries e = t * 8 + i at grp_ent[grp_start[g] ..
+/// grp_start[g + 1])), token t's F32 input at x + t x_stride, its output at out + t out_stride (plus the CPU lane's
+/// part when cpu_flag[t], plus sh_out's row t) - every token's output is moe_gate_up + moe_down's for it alone, bit for
+/// bit.  h [T * 8][n_ff] and part [T * 8][n_embd] are the caller's scratch.
+struct WinArgs {
+    const unsigned long long* grp_ptr = nullptr;
+    const int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;
+    const unsigned long long* plan_ptr = nullptr;   // [T][8]
+    const float* plan_w = nullptr;                  // [T][8]
+    const float* cpu_part = nullptr;                // [T][n_embd] (rows cpu_stride apart when given)
+    size_t cpu_stride = 0;
+    const int* cpu_flag = nullptr;                  // [T]
+    const float* x = nullptr;
+    size_t x_stride = 0;
+    float* h = nullptr;
+    float* part = nullptr;
+    const float* sh_out = nullptr;                  // [T][n_embd] or null
+    float* out = nullptr;
+    size_t out_stride = 0;
+    ExpertLayout lay;
+    int T = 0, k = 8, n_embd = 4096, n_ff = 2048;
+    float limit = 0.0f;
+    // (given) the combine first waits on the device until *wait_ptr == wait_val (the host's answer to the CPU lane's
+    // part arrives while the device computes the experts); cpu_flag may be null (no CPU part)
+    const unsigned* wait_ptr = nullptr;
+    unsigned wait_val = 0;
+};
+bool win_experts(const WinArgs& a, cudaStream_t s);
+/// Whether the current device runs the several-activation kernels (win_experts; mv's ncol > 1): every device does
+/// (they take ~10 KB of shared memory a block).
+bool several_ok();
+
+/// The prompt path's tensor-core GEMMs (sm_80 and later; mma_supported() says whether this device runs them).
+bool mma_supported();
+/// A set of n_exp routed experts in VRAM, expert e's blob at wbase + e * stride (the layout in lay), owning the sorted
+/// rows [bounds[e], bounds[e + 1]) (device, n_exp + 1 entries); a sorted row's LOCAL row is bounds[e] - row0 + r.
+struct SetArgs {
+    const uint8_t* wbase = nullptr;
+    size_t stride = 0;
+    ExpertLayout lay;
+    int n_exp = 0;
+    const int* bounds = nullptr;
+    int row0 = 0;
+    int max_rows = 0;   // the most rows one expert of the set has
+    int n_embd = 4096, n_ff = 2048;
+    float limit = 0.0f;
+};
+/// h[local row] = FP16 clamped swiglu of gate and up over x[row_idx[local row]] (F32 rows, ldx floats apart).
+void set_gate_up(const SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh, cudaStream_t s);
+/// y[local row] = down(h[local row]) (F32 rows, ldy floats apart).
+void set_down(const SetArgs& a, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s);
+/// Y[r] = beta Y[r] + (X[r] W) for r < rows: one matrix over FP16 rows (ldx), F32 output rows (ldy).
+void gemm_rows(const Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta, cudaStream_t s);
+
+/// The full weight in the engine's row layout: dst[r][c] = W[c][r] (n rows of k values), BF16 or F16 bits.
+void reconstruct_rows(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s);
+/// The decoded W' alone in that layout: dst[c][r] = W'[r][c], F16 bits (exactly the decoded values) - for a product
+/// that rotates its activations and outputs instead of the weight (x W = H(H(x * suh) W') * svh): rows_rot_f16 before
+/// the GEMM, rows_rot_out after it.  No Hadamard over the k x n weight: a plain decode, at the memory's speed.
+void reconstruct_inner_rows(const Mat& m, uint16_t* dst, cudaStream_t s);
+/// out[r][c] = FP16(H(x_r * suh)[c]) for r < nrows, c < ncols (ncols % 128 == 0), x_r = x[idx ? idx[r] : r] (F32, ldx
+/// floats a row): an EXL3 matrix's rotated input rows, as the decode kernels rotate theirs.
+void rows_rot_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, const uint16_t* suh, uint16_t* out,
+                  cudaStream_t s);
+/// y[r][c] = H(y_r)[c] * svh[c] in place for r < nrows, c < ncols (each 128 values one block; ldy floats a row).
+void rows_rot_out(float* y, int ldy, int nrows, int ncols, const uint16_t* svh, cudaStream_t s);
+/// The decoded W' alone, row-major [k][n] F16 bits (tests).
+void reconstruct_inner(const Mat& m, uint16_t* dst, cudaStream_t s);
+
+/// out[r][c] = FP16(x[idx ? idx[r] : r][c]) for r < nrows, c < ncols (x: ldx floats a row): the prompt path's rows of
+/// one expert, gathered for its GEMMs.
+void rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out, cudaStream_t s);
+
+/// Launch failures of this family since the process started (each is also printed).
+int launch_errors();
+
+}  // namespace strata::dsv41k::exl3

include/strata/dsv41/exl3_cpu.hpp

@@ -0,0 +1,69 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// include/strata/kernels/cpu/exl3_cpu.hpp - EXL3 routed experts on the CPU (the fast path's CPU lane).
+//
+// The mul1 codebook is affine in the byte sum of state * 0x83DCD12D: w = fp16(1024 + s) * k_inv + k_bias (one FP16
+// FMA on the device).  The CPU keeps the affine form without the FP16 rounding, so a 128-output band is
+//   y' = k_inv * sum_k xh_k s_kc + (1024 k_inv + k_bias) * sum_k xh_k,   xh = H(x * suh)
+// and y = H(y') * svh.  The byte sums come from AVX-512 (permutes and funnel shifts pull 16 columns' states at once,
+// vpmaddubsw / vpmaddwd add the bytes up, an F32 FMA takes the activation), or from AVX2 + FMA the same way 8 columns
+// at a time - the same FMAs in the same order, so the two give the same bits; a CPU with neither runs a scalar loop.
+// The format: include/strata/kernels/exl3.hpp.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace strata::dsv41k::cpu {
+
+/// One EXL3 matrix (mul1 codebook): k inputs, n outputs, K bits (1..8), tiles in exllamav3's natural layout.
+struct Exl3Part {
+    const uint32_t* trellis = nullptr;
+    const uint16_t* suh = nullptr;   // FP16 bits [k]
+    const uint16_t* svh = nullptr;   // FP16 bits [n]
+    int k = 0, n = 0, K = 0;
+};
+/// Whether a vector kernel runs here (AVX-512F/BW or AVX2 + FMA); the scalar one otherwise.
+bool exl3_cpu_vector();
+/// The kernel the functions below run here: 2 AVX-512 (F, BW), 1 AVX2 + FMA (+ F16C), 0 scalar.  STRATA_FORCE_AVX2=1
+/// keeps an AVX-512 CPU on AVX2 (to compare the two; they give the same bits).
+int exl3_cpu_tier();
+/// xh = H(x * suh) (k values, k % 128 == 0); returns the sum of xh.
+float exl3_cpu_prepare(const Exl3Part& m, const float* x, float* xh);
+/// One 128-value block of that: xh[i] = H(x * suh)[i] for i < 128 (suh: the block's 128 scales); returns its sum.
+float exl3_cpu_prepare_block(const uint16_t* suh, const float* x, float* xh);
+/// The 128 outputs from column c0 (a multiple of 128) of y = H(x W') * svh, from exl3_cpu_prepare's xh and sum.
+void exl3_cpu_band(const Exl3Part& m, const float* xh, float xsum, int c0, float* y);
+/// The same in parts: the byte-sum products of the 16-row slices [ks0, ks1) (xh from slice 0) ...
+void exl3_cpu_band_acc(const Exl3Part& m, const float* xh, int c0, int ks0, int ks1, float* acc);
+/// exl3_cpu_band_acc for n activations of the same matrix at once (xh[i] -> acc[i], each 128 floats): each tile row is
+/// decoded once for up to kExl3CpuMaxN of them, and acc[i] is exl3_cpu_band_acc's for xh[i] alone, bit for bit.
+constexpr int kExl3CpuMaxN = 8;
+void exl3_cpu_band_acc_n(const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,
+                         float* const* acc);
+/// The byte-sum products of the slices [ks0, ks1) for EVERY column (acc: m.n floats, overwritten) - a split-K piece,
+/// whose tile rows lie in memory order where a band's are 1 KB runs a whole tile row apart (from DRAM, ~1.5x the
+/// bands' speed).  Each column's sums are exl3_cpu_band_acc's over the same slices, bit for bit.
+void exl3_cpu_rows_acc(const Exl3Part& m, const float* xh, int ks0, int ks1, float* acc);
+/// ... for n activations at once (xh[i] -> acc[i]), each tile row decoded once; acc[i] is the call for xh[i] alone.
+void exl3_cpu_rows_acc_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, float* const* acc);
+/// ... over the columns [c0, c1) alone (multiples of 128; acc[i] indexed by column, only those overwritten): a piece cut
+/// by columns too, for more jobs than pieces - each column's sums are the whole piece's, bit for bit.
+void exl3_cpu_rows_acc_cols_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, int c0, int c1,
+                              float* const* acc);
+/// The byte-sum products of nb bands from c0 (128 nb columns; acc[i]: 128 nb floats) over every input for any n
+/// activations, as a GEMM: each 256-input chunk of a band decoded once into F32 in `wc` (exl3_cpu_gemm_scratch()
+/// floats, 64-byte aligned, the caller's), then multiplied for all n, the chunk of the activations kept in L2 across
+/// the nb bands - for many tokens an expert (the prompt path), where the band kernels decode a tile row again every
+/// kExl3CpuMaxN.  The same sums as exl3_cpu_band_acc, in its order.
+size_t exl3_cpu_gemm_scratch();
+void exl3_cpu_band_gemm(const Exl3Part& m, const float* const* xh, int n, int c0, int nb, float* const* acc,
+                        float* wc);
+/// exl3_cpu_band_acc_n on a given tier (0, 1, 2 as exl3_cpu_tier) - tests and benchmarks; false (nothing computed)
+/// where this CPU cannot run it.
+bool exl3_cpu_band_acc_on(int tier, const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,
+                          float* const* acc);
+/// ... and, with the parts' acc summed over every slice, the outputs (the codebook's affine form, H, svh).
+void exl3_cpu_band_out(const Exl3Part& m, const float* acc, float xsum, int c0, float* y);
+
+}  // namespace strata::dsv41k::cpu

include/strata/dsv41/hip_extra.h

@@ -0,0 +1,50 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// include/strata/dsv41/hip_extra.h - CUDA runtime names the DeepSeek-V4.1 engine path uses that Strata's HIP shim
+// (include/strata/hip_compat) does not map.  Force-included into the dsv41 targets only, so no other target changes.
+// The mappings are Project Maya's include/strata/hip_compat (MIT, third_party/project-maya/NOTICE.md).
+#pragma once
+#include <hip/hip_runtime.h>
+#include <hipblas/hipblas.h>
+#ifndef __grid_constant__
+#define __grid_constant__
+#endif
+#ifndef cudaDevAttrMemoryClockRate
+#define cudaDevAttrMemoryClockRate hipDeviceAttributeMemoryClockRate
+#endif
+#ifndef cudaDevAttrGlobalMemoryBusWidth
+#define cudaDevAttrGlobalMemoryBusWidth hipDeviceAttributeMemoryBusWidth
+#endif
+#ifndef cudaDevAttrCanUseHostPointerForRegisteredMem
+#define cudaDevAttrCanUseHostPointerForRegisteredMem hipDeviceAttributeCanUseHostPointerForRegisteredMem
+#endif
+#ifndef cudaDeviceGetPCIBusId
+#define cudaDeviceGetPCIBusId hipDeviceGetPCIBusId
+#endif
+#ifndef cudaDeviceGetStreamPriorityRange
+#define cudaDeviceGetStreamPriorityRange hipDeviceGetStreamPriorityRange
+#endif
+#ifndef cudaOccupancyMaxActiveBlocksPerMultiprocessor
+#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor
+#endif
+#ifndef cudaErrorUnknown
+#define cudaErrorUnknown hipErrorUnknown
+#endif
+#ifndef cudaErrorInvalidConfiguration
+#define cudaErrorInvalidConfiguration hipErrorInvalidConfiguration
+#endif
+#ifndef cudaMemcpy2D
+#define cudaMemcpy2D hipMemcpy2D
+#endif
+#ifndef cudaStreamCreateWithPriority
+#define cudaStreamCreateWithPriority hipStreamCreateWithPriority
+#endif
+#ifndef cudaStreamLegacy
+#define cudaStreamLegacy hipStreamLegacy
+#endif
+#ifndef CUBLAS_GEMM_DEFAULT_TENSOR_OP
+#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT
+#endif
+#ifndef cudaDataType
+#define cudaDataType hipDataType
+#endif

include/strata/dsv41/hip_or_cuda.h

@@ -0,0 +1,45 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// include/strata/dsv41/hip_or_cuda.h - the runtime the dsv41 *tests* need, whichever backend the build has.
+//
+// The engine's own sources take their runtime from the backend's headers (ds4_*.cu compile as HIP or as CUDA).  The tests
+// are plain .cpp files that launch kernels and copy buffers, and they used to include <hip/hip_runtime.h> unconditionally,
+// which no CUDA toolkit ships: every CUDA box had to pass a shim on -I to build them (found on l620 and z440, 2026-10-23).
+// This header is that shim, in the tree: HIP names on a HIP build, mapped onto the CUDA runtime otherwise.
+//
+// Two details that a first version got wrong (z440, 2026-10-23): HIP's hipMalloc is a *template* taking T**, so a plain
+// void** overload does not bind `hipMalloc(&uint8_ptr)`; and hipMemset has no CUDA counterpart under that name.
+#pragma once
+
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+#include <hip/hip_runtime.h>
+#else
+#include <cuda_runtime.h>
+
+using hipError_t = cudaError_t;
+using hipEvent_t = cudaEvent_t;
+using hipStream_t = cudaStream_t;
+constexpr hipError_t hipSuccess = cudaSuccess;
+constexpr int hipMemcpyHostToDevice = cudaMemcpyHostToDevice;
+constexpr int hipMemcpyDeviceToHost = cudaMemcpyDeviceToHost;
+
+inline hipError_t hipMalloc(void** p, size_t n) { return cudaMalloc(p, n); }
+template <typename T>
+inline hipError_t hipMalloc(T** p, size_t n) { return cudaMalloc(reinterpret_cast<void**>(p), n); }
+inline hipError_t hipFree(void* p) { return cudaFree(p); }
+inline hipError_t hipMemcpy(void* dst, const void* src, size_t n, int kind) {
+    return cudaMemcpy(dst, src, n, static_cast<cudaMemcpyKind>(kind));
+}
+template <typename T>
+inline hipError_t hipMemcpy(T* dst, const T* src, size_t n, int kind) {
+    return cudaMemcpy(static_cast<void*>(dst), static_cast<const void*>(src), n, static_cast<cudaMemcpyKind>(kind));
+}
+inline hipError_t hipMemset(void* p, int v, size_t n) { return cudaMemset(p, v, n); }
+inline hipError_t hipDeviceSynchronize() { return cudaDeviceSynchronize(); }
+inline hipError_t hipSetDevice(int d) { return cudaSetDevice(d); }
+inline const char* hipGetErrorString(hipError_t e) { return cudaGetErrorString(e); }
+inline hipError_t hipEventCreate(hipEvent_t* e) { return cudaEventCreate(e); }
+inline hipError_t hipEventRecord(hipEvent_t e, hipStream_t s = nullptr) { return cudaEventRecord(e, s); }
+inline hipError_t hipEventSynchronize(hipEvent_t e) { return cudaEventSynchronize(e); }
+inline hipError_t hipEventElapsedTime(float* ms, hipEvent_t a, hipEvent_t b) { return cudaEventElapsedTime(ms, a, b); }
+#endif

include/strata/dsv41/moe_win.hpp

@@ -0,0 +1,33 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// include/strata/dsv41/moe_win.hpp - a verify window's MoE entries grouped by expert (the grouping alone).
+// Extracted from Project Maya's include/strata/kernels/glm_fast.hpp (MIT, see third_party/project-maya/NOTICE.md).
+#pragma once
+#include <cstddef>
+#include <cuda_runtime.h>
+
+namespace strata::dsv41k::glmf {
+constexpr int kMaxMvCols = 16;
+struct MoeWin {
+    const unsigned long long* plan_ptr = nullptr;   // [T][8] the tokens' plans (0: an entry the GPU does not compute)
+    const float* plan_w = nullptr;                  // [T][8]
+    const float* cpu_part = nullptr;                // [T][n_embd] the host's answers to disk misses, when flagged
+    const int* cpu_flag = nullptr;                  // [T]
+    const void* xq = nullptr;                       // the tokens' FFN inputs (q8_1), xq_stride bytes apart
+    size_t xq_stride = 0;
+    void* hq = nullptr;                             // scratch: [T * 8][n_ff / 32] q8_1
+    float* part = nullptr;                          // scratch: [T * 8][n_embd]
+    const float* sh_out = nullptr;                  // [T][n_embd] the shared expert's down rows, or null
+    float* out = nullptr;                           // [T][n_embd], out_stride floats apart
+    size_t out_stride = 0;
+    unsigned long long* grp_ptr = nullptr;          // scratch: [T * 8]
+    int* grp_start = nullptr;                       // scratch: [T * 8 + 1]
+    int* grp_ent = nullptr;                         // scratch: [T * 8]
+    int* n_groups = nullptr;                        // scratch: [1]
+    int T = 0, k = 0, n_embd = 0, n_ff = 0;
+    float limit = 0.0f;
+    size_t down_off = 0;
+};
+constexpr int kMaxWinTokens = kMaxMvCols;   // the window's tokens moe_win_experts groups (16 x 8 entries)
+void moe_win_group(const MoeWin& w, cudaStream_t s);
+}  // namespace strata::dsv41k::glmf

include/strata/dsv41/sampler.hpp

@@ -0,0 +1,94 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// include/strata/kernels/sampler.hpp - the sampler chain, host-callable (P2.S2).
+//
+//     penalties -> top_k -> top_p -> min_p -> temperature -> pick
+//
+// llama.cpp's default chain (issue #53): ONE penalties stage, first, and temperature AFTER the truncation filters;
+// top_p cuts before min_p.  `src/kernels/cuda/sampler.cu` has the details and `sampler_parity` pins them.
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace strata::dsv41k {
+
+struct SamplerParams {
+    int top_k = 20;              // sampled path: 1..64 as given; 0 (off) or more than 64 keep the widest list, 64
+    float top_p = 0.95f;         // 1.0 disables the filter
+    float min_p = 0.0f;          // 0 disables; keeps tokens with p >= min_p * p_max (a prefix of the top_k list)
+    float temperature = 1.0f;    // <= 0 means greedy
+    int min_keep = 1;            // top_p keeps at least this many
+    int penalty_last_n = 0;      // 0 disables; the window over `history` to count occurrences in
+    float penalty_repeat = 1.0f;
+    float penalty_freq = 0.0f;
+    float penalty_present = 0.0f;
+    uint64_t seed = 0;           // drives Philox, which is counter-based on (seed, token index)
+    uint64_t counter = 0;        // absolute draw index of row 0; advance across decode calls
+    bool greedy = false;
+    // a request's logit_bias: a DEVICE array of n_vocab floats on the sampling device, added to every row's logits
+    // before the penalties and the filters (-inf bans a token); nullptr = none
+    const float* logit_bias = nullptr;
+    // the GLM engine's penalty history, on the HOST (never read by a kernel): every token the request's state has been
+    // fed so far, oldest first - its prompt, then each token it wrote.  Glm5Model::pick_tokens hands each pick the last
+    // penalty_last_n of them (a verify window's row t: these, then the window's drafts 1..t).  nullptr = none
+    const std::vector<int32_t>* host_history = nullptr;
+};
+
+// logits (n_tokens, n_vocab) -> one sampled token id per row in `out`.
+//
+// **`logits` AND `out` ARE DEVICE POINTERS.**  This is a CUDA kernel launch, not a host function, and nothing
+// in the parameter names or the types says so - `const float* logits` reads exactly like a host buffer.
+// Measured: passing host memory for `logits` faults inside the kernel with an ILLEGAL MEMORY ACCESS that is
+// reported by whatever synchronising call happens NEXT, which will be somewhere else entirely and will name a
+// buffer that has nothing to do with it.  A sticky async fault does not know where it came from.
+//
+// `history` is (n_tokens, history_len) int32 - ONE ROW PER TOKEN ROW, at a stride of `history_len` - the most
+// recent tokens that row's pick follows, with any unused slots set to -1; only the last `p.penalty_last_n` of
+// each row are counted, and ids outside [0, n_vocab) are ignored.  Pass nullptr and 0 when no penalties apply.
+// A verify window's rows need DIFFERENT histories: row t follows the window's drafts 1..t (`penalty_rows`).
+void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
+                   const SamplerParams& p, int* out, void* stream);
+
+// SPECULATIVE SAMPLING (Leviathan et al. 2023, Chen et al. 2023) for the pipelined NextN decode - opt-in
+// (STRATA_GLM_SPEC_SAMPLE=1).  The draft is DRAWN from the draft block's own distribution q (this chain on its logits,
+// a Philox stream of its own) and its kept list is stored; the model's token is then the draft with probability
+// min(1, p(d) / q(d)), else a draw from the residual max(0, p - q) renormalised - so it is distributed exactly as the
+// plain draw from p, and a draft stands whenever the token is the draft (sum over x of min(p(x), q(x)) of the time
+// instead of p(argmax q)).  Same (seed, counter): the same tokens - but not the plain draw's tokens.
+struct SpecList {
+    int n = 0;          // the kept list's length (1..64)
+    int ids[64];        // descending, as the selection produced them
+    double p[64];       // q over the kept list (sums to 1)
+};
+
+// draft rows: logits (n_tokens, n_vocab) -> out[t] drawn through this chain, q[t] its kept list.  DEVICE pointers,
+// like sample_tokens; a sampled request without penalties only (greedy keeps the argmax draft).
+void sample_draft(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, SpecList* q, int* out,
+                  void* stream);
+// verify rows: the model's logits (n_tokens, n_vocab), the draft draft[t] drawn from q[t] (same p.seed / p.counter as
+// its sample_draft) -> out[t], distributed as sample_tokens' draw.  A draft q[t] does not hold gets the plain draw.
+void sample_verify(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, const SpecList* q,
+                   const int* draft, int* out, void* stream);
+
+// The penalty-history rows of a verify window, on the host: row t of `out` (T rows of `h` slots) is the last `h`
+// tokens of `tail[0..n_tail)` followed by `window[0..t]`, most recent LAST, -1 in the unused front slots.
+// `tail` is what the state consumed before the window, `window[0]` the fed-back token and `window[1..]` the
+// drafts: row t is exactly the history plain decode counts when it picks the token after `window[t]`, so drafting
+// cannot change which tokens are penalised.  Row 0 alone is the single row the engine staged before 0.1.19.
+inline void penalty_rows(const int32_t* tail, int64_t n_tail, const int32_t* window, int T, int h, int32_t* out) {
+    for (int t = 0; t < T; ++t) {
+        int32_t* row = out + (size_t) t * (size_t) h;
+        const int64_t avail = n_tail + t + 1;                      // tail + window[0..t]
+        const int take = (int) std::min<int64_t>(h, avail);
+        std::fill(row, row + (h - take), -1);
+        for (int j = 0; j < take; ++j) {
+            const int64_t i = avail - take + j;                     // index into (tail, window)
+            row[h - take + j] = i < n_tail ? tail[i] : window[i - n_tail];
+        }
+    }
+}
+
+}  // namespace strata::dsv41k

include/strata/kernels/cpu/iq_avx2.hpp

@@ -23,6 +23,13 @@ void iq256_gu_rows(int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_
 void iq256_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
                 float* const* out, int r0, int r1);
 
+/// The DeepSeek-V4.1 CPU lane's rows (src/dsv41/ds4_gg_cpu.cpp).  The same loops as iq256_rows / iq256_gu_rows with ggml's float
+/// order, so a token's dot is ggml-cpu's vec_dot bit for bit at any number of tokens (the main engine's iq256_* are unchanged),
+/// and IQ1_M (29), whose ggml dot is single-token.  The caller checks AVX2 support.
+bool iq256x_supported(int ggml_type) noexcept;
+void iq256x_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
+                 float* const* out, int r0, int r1);
+
 /// The kernels' variants, a mask.  Every variant computes the same bits; they differ in speed from core to core.
 /// kIq256Gather: IQ3_XXS, IQ3_S and IQ2_S read their grid entries with one gather (STRATA_IQ256_GATHER).
 /// kIq256Vnni: AVX-VNNI's vpdpwssd / vpdpbusd, every format and iq4nl256_down_rows (cpu_avxvnni_ok).

include/strata/kernels/cpu/iq_avx512.hpp

@@ -15,4 +15,11 @@ void iq512_gu_rows(int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_
 void iq512_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
                 float* const* out, int r0, int r1);
 
+/// The DeepSeek-V4.1 CPU lane's rows (src/dsv41/ds4_gg_cpu.cpp): the same 64-value decode, summed in ggml's order, so a token's
+/// dot is ggml-cpu's AVX2 vec_dot bit for bit at any number of tokens (1-6 in one pass, 7 and more as a prompt chunk's tiles).
+/// The caller checks cpu_avx512_ok() (AVX-512 F/BW/VL/DQ/VNNI/VBMI).
+bool iq512x_supported(int ggml_type) noexcept;
+void iq512x_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
+                 float* const* out, int r0, int r1);
+
 }  // namespace strata::kernels::cpu

include/strata/kernels/cpu/native_expert.hpp

@@ -36,6 +36,7 @@ struct NativeFmt {
     /// native_quant_act / native_quant_h apply y = H_block (signs * x) before quantizing.  had_block 0 = none.
     int had_block = 0;
     bool had_gu = false, had_d = false;
+    int layer = -1;                     ///< the layer this format belongs to (set by expert_layout_load; STRATA_ACT_DUMP only)
     const float* gu_signs = nullptr;    ///< n_embd signs for x (nullptr: identity)
     const float* d_signs = nullptr;     ///< n_ff signs for h (nullptr: identity)
 };

include/strata/kernels/cpu/tq_cpu.hpp

@@ -31,11 +31,18 @@ namespace strata::kernels::cpu {
 
 enum class TqIsa : int { scalar = 0, avx2 = 1, avx512 = 2 };
 
-/// TQ2_T (144), TQK6 (145), TQK7 (146).
+/// EXPERIMENTAL lower-rate versions of the TQK rule (tools/trellis/tqenc.hpp, version 1; Strata-internal ids, not ggml
+/// types, not in any GGUF): TQK5X = 22 bytes / 128 weights (1.375 bpw), TQK4X = 18 bytes (1.125 bpw).  Same hash,
+/// codebook, 15-bit mask, LSB-first circular stream and fp16 scale as TQK6/7 with K = 5 / 4.
+inline constexpr int kTqTypeK5X = 224;   ///< version 2: K = 5 with the computed codebook (tq_computed.hpp)
+inline constexpr int kTqTypeK4X = 225;   ///< version 2: K = 4, computed codebook
+inline constexpr int kTqTypeK5T = 226;   ///< version 1: K = 5 with the fork's point table (the first pilots)
+inline constexpr int kTqTypeK4T = 227;   ///< version 1: K = 4, point table
+/// TQ2_T (144), TQK6 (145), TQK7 (146) and the experimental TQK5X / TQK4X / TQK5T / TQK4T.
 bool tq_supported(int ggml_type) noexcept;
-/// Bits per step: 8, 6, 7 (0 for any other type).
+/// Bits per step: 8, 6, 7, 5, 4 (0 for any other type).
 int tq_kbits(int ggml_type) noexcept;
-/// Values per block of the trellis types, and bytes per block (2 + 4 * bits: 34, 26, 30).
+/// Values per block of the trellis types, and bytes per block (2 + 4 * bits: 34, 26, 30, 22, 18).
 constexpr int kTqBlock = 128;
 inline size_t tq_block_bytes(int ggml_type) noexcept { return (size_t) (2 + 4 * tq_kbits(ggml_type)); }
 /// The best ISA this CPU runs (capped by STRATA_TQ_ISA).
@@ -49,8 +56,12 @@ bool tq_isa_ok(TqIsa isa);
 bool tq_kernels_enabled();
 const char* tq_isa_name(TqIsa isa);
 
-/// The 32768 x 4 int16 state table (filled on first use; thread-safe).
+/// The 32768 x 4 int16 state table of the fork's types (filled on first use; thread-safe).
 const int16_t* tq_state_table();
+/// The same for the computed codebook of TQK5X / TQK4X (tq_computed.hpp).
+const int16_t* tq_state_table_computed();
+/// The table a type decodes with.
+const int16_t* tq_state_table_for(int ggml_type);
 
 /// Rows [r0, r1) of a trellis matrix (`row_bytes` per row, `n` weights per row) against `nt` (<= 8) Q8_0
 /// activations of n values: out[t][r] = w_r . a[t].
@@ -63,7 +74,7 @@ void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, si
 
 /// Largest token group per call and widest row the kernels take.
 inline constexpr int kTqMaxTokens = 8;
-inline constexpr int kTqMaxN = 4096;
+inline constexpr int kTqMaxN = 5120;   // (DeepSeek-V4.1 hidden width; was 4096)
 
 namespace tq_detail {
 /// One token's Q8_0 activation unpacked for the kernels: the int8 values widened to int16 and the scales as floats.
@@ -73,11 +84,11 @@ struct TqAct {
 };
 void unpack_act(const void* q8_0, int n, TqAct& a);
 // the per-ISA row kernels (dot products only; `out[t][r]` for rows [r0, r1))
-void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_scalar(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
                     float* const* out, int r0, int r1);
-void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_avx2(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
                   float* const* out, int r0, int r1);
-void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_avx512(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
                     float* const* out, int r0, int r1);
 }  // namespace tq_detail
 

packaging/ENGINE-COMPONENT.md

@@ -96,7 +96,7 @@ the manifest file (which holds every file's hash), used as the package's digest
 
 | Package | Built on | Contents | Notes |
 |---|---|---|---|
-| `windows-x86_64` cuda, variant `cuda13` | the owner PC, the same cmake line as the release zips (CUDA 13.0, `75;86;89;120` real + `120` PTX, `STRATA_PORTABLE=ON`) | `strata.exe`, `strata_kq.dll`, `strata-vision.exe` (built for this variant's archs by `packaging/build_vision_windows.bat cuda13 <stage>`, which writes `vision_archs` into BUILD.json), `BUILD.json` | `min_driver` nvidia 580.65 |
+| `windows-x86_64` cuda, variant `cuda13` | the owner PC, the same cmake line as the release zips (CUDA 13.0, `75;86;89;120` real + `120` PTX, `STRATA_PORTABLE=ON`) | `strata.exe`, `strata_kq.dll`, `strata-vision.exe` (built for this variant's archs by `packaging/build_vision_windows.bat cuda13 <stage>`, which writes `vision_archs` into BUILD.json), `strata-dsv41.exe` (DeepSeek-V4.1, `packaging/build_dsv41_windows.bat`, sm_86 + sm_120; `--require-dsv41`), `BUILD.json` | `min_driver` nvidia 580.65 |
 | `windows-x86_64` cuda, variant `cuda12` | the owner PC (CUDA 12.9, `60;61;70;75;80;86;89` real + `89` PTX, `STRATA_EXPERIMENTAL_SM60=ON`) | the same four files, with its own `strata-vision.exe` (`packaging/build_vision_windows.bat cuda12 <stage>`: sm_60 .. sm_89; the CUDA 13 reader has no Pascal code, #3669) | experimental, `min_driver` nvidia 575.51; the CUDA 13 build wins when both fit |
 | `windows-x86_64` hip | the owner PC, `tools/hip/build_windows.bat` (ROCm 10.2 nightly wheels) then `tools/hip/package_windows.py` | `strata.exe`, `strata-device.exe`, the ROCm DLLs and the rocBLAS/hipBLASLt kernels of the zip (`--tree`) | no Windows AMD test machine yet; with `STRATA_NO_VISION` unset it also builds the two picture readers (`strata-vision.exe` on the CPU, `strata-vision-hip.exe` on the GPU) |
 | `linux-x86_64` cuda, variant `cuda13` | a Linux box with the CUDA 13 toolkit, `VARIANT=cuda13 packaging/build_cuda_linux.sh` | `strata`, `libstrata_kq.so`, `strata-vision` (built by the same script for the same archs, `BUILD_VISION=0` for a text-only package), `libcudart/libcublas/libcublasLt` (13.x) | the GPU driver's `libcuda` stays the PC's |

packaging/build_dsv41_cuda.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# strata-dsv41 (the DeepSeek-V4.1 engine) for NVIDIA on Linux, built next to the release engine so package_engine.py can ship it
+# beside `strata` (setup.py dsv41_exe() looks for strata-dsv41 in the engine folder).  Same toolchain and RUNPATH rule as the
+# engine build: the result finds cudart/cublas beside itself ($ORIGIN), never the toolkit it was built with.
+#
+#   SRC=<checkout> B=<build dir> CUDA=/usr/local/cuda-13.0 GG=<llama.cpp checkout> [ARCH="86-real;120"] [JOBS=6]  packaging/build_dsv41_cuda.sh
+#
+# ARCH defaults to the 0.2.4 release shape: SASS for sm_86 (RTX 3090 / 3060) and sm_120 (RTX Blackwell) + PTX sm_120 (the Windows twin
+# built and ran on an RTX 5070; the Linux sm_86 build is the one tested on the RTX 3060 box).  Needs CUDA 13 (the sm_120 code); for a
+# CUDA 12.9 package pass ARCH="86-real;89-real;89-virtual" or leave strata-dsv41 out (the Pascal cuda12 variant has no DeepSeek path).
+# Prints the binary path on the last line; collect its libraries with packaging/collect_libs.py --binary <it> --search $CUDA/lib64.
+set -eu
+SRC=${SRC:-$(cd "$(dirname "$0")/.." && pwd)}
+B=${B:?B=<build dir>}
+CUDA=${CUDA:?CUDA=<toolkit root>}
+GG=${GG:?GG=<llama.cpp checkout at the pinned commit>}
+ARCH=${ARCH:-86-real;120}
+JOBS=${JOBS:-6}
+export PATH=$CUDA/bin:$PATH
+cmake -G Ninja -S "$SRC" -B "$B" -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_CUDA=ON -DSTRATA_BUILD_TESTS=OFF -DSTRATA_BUILD_DSV41=ON \
+  -DSTRATA_NATIVE_EXPERTS=ON -DSTRATA_PORTABLE=ON "-DCMAKE_CUDA_ARCHITECTURES=$ARCH" -DCMAKE_CUDA_COMPILER="$CUDA/bin/nvcc" \
+  -DCUDAToolkit_ROOT="$CUDA" -DSTRATA_GGML_DIR="$GG" -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_SKIP_BUILD_RPATH=OFF \
+  '-DCMAKE_INSTALL_RPATH=$ORIGIN'
+ninja -j "$JOBS" -C "$B" strata-dsv41
+[ -x "$B/strata-dsv41" ] || { echo "no $B/strata-dsv41" >&2; exit 1; }
+echo "$B/strata-dsv41"

packaging/build_dsv41_windows.bat

@@ -0,0 +1,26 @@
+@echo off
+rem strata-dsv41.exe (the DeepSeek-V4.1 engine) for NVIDIA on Windows, built next to the release engine so package_engine.py ships it
+rem beside strata.exe (setup.py dsv41_exe() looks for it in the engine folder).  cudart is static; the only extra DLL is cublas64_13.dll,
+rem which package_engine.py copies from CUDA_PATH like it does for strata.exe.
+rem   build_dsv41_windows.bat <build dir> [source dir]        env: STRATA_GGML_DIR (llama.cpp), CUDA_ROOT (default CUDA 13.0), BUILD_JOBS (default 4),
+rem                                                            DSV41_ARCH (default "86-real;120" = sm_86 + sm_120 SASS, PTX sm_120)
+rem Result: <build dir>\strata-dsv41.exe (about 66 MB).  Windows: only the cuda13 variant gets it (sm_120 needs CUDA 13).
+setlocal
+set "BD=%~1"
+set "SRCD=%~2"
+if "%BD%"=="" ( echo usage: build_dsv41_windows.bat ^<build dir^> [source dir] & exit /b 2 )
+if "%SRCD%"=="" set "SRCD=%~dp0.."
+if "%CUDA_ROOT%"=="" set "CUDA_ROOT=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v13.0"
+if "%DSV41_ARCH%"=="" set "DSV41_ARCH=86-real;120"
+if "%BUILD_JOBS%"=="" set "BUILD_JOBS=4"
+if "%STRATA_GGML_DIR%"=="" ( echo set STRATA_GGML_DIR to the llama.cpp checkout & exit /b 2 )
+call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" >nul 2>&1
+if not exist "%BD%\build.ninja" (
+  cmake -G Ninja -S "%SRCD%" -B "%BD%" -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_CUDA=ON -DSTRATA_BUILD_TESTS=OFF -DSTRATA_BUILD_DSV41=ON -DSTRATA_PORTABLE=ON "-DCMAKE_CUDA_ARCHITECTURES=%DSV41_ARCH%" "-DCMAKE_CUDA_COMPILER=%CUDA_ROOT%/bin/nvcc.exe" "-DCUDAToolkit_ROOT=%CUDA_ROOT%" "-DSTRATA_GGML_DIR=%STRATA_GGML_DIR%"
+  if errorlevel 1 exit /b 1
+)
+start "" /belownormal /wait /b cmake --build "%BD%" -j %BUILD_JOBS% --target strata-dsv41
+if errorlevel 1 exit /b 1
+if not exist "%BD%\strata-dsv41.exe" ( echo no strata-dsv41.exe & exit /b 1 )
+echo built %BD%\strata-dsv41.exe
+exit /b 0

packaging/build_hip_linux.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Linux AMD (HIP) engine package: strata + the CPU picture reader (strata-vision) + the GPU picture reader (strata-vision-hip),
+# Linux AMD (HIP) engine package: strata (+ strata-dsv41 for rx9070) + the CPU picture reader (strata-vision) + the GPU picture reader (strata-vision-hip),
 # the ROCm libraries both load, and the rocBLAS / hipBLASLt kernel data for the listed archs, packaged as an engine component.
 # Run on a Linux box that has ROCm (TheRock Python wheels are enough); the tester packages were built in an Ubuntu 22.04 chroot
 # (glibc 2.35) so they run on every distribution since.  All settings are environment variables:
@@ -11,6 +11,8 @@
 #   HP=<ROCm devel root>  CORE=<ROCm core root>  LIBW=<ROCm libraries root>   (TheRock: _rocm_sdk_devel, _rocm_sdk_core, _rocm_sdk_libraries_*)
 #   OUT=<work folder>  PKGVER=0.1.48  COMMIT=<source commit for the build id>  JOBS=16
 #   VISION_ONLY=1                    build only the two readers (no engine, no package)
+#   DSV41=1|0                        also build strata-dsv41 (the DeepSeek-V4.1 engine, tools/dsv41/build_hip_linux.sh's flags) and ship it beside
+#                                    strata: default 1 for the rx9070 variant (gfx1200 gfx1201; built and run on an R9700), 0 for the others
 #   NO_PACKAGE=1                     stop after the stage folder
 #   VISION_HIP_DEFS="-DGGML_CUDA_FA=OFF"   extra cmake definitions for the GPU reader only
 #
@@ -36,6 +38,7 @@ case "$V" in
   *)      ARCHS=${ARCHS:?ARCHS=gfx...} ;;
 esac
 NOTE_EXTRA=""; [ "$V" = rx6000 ] && NOTE_EXTRA=" (built, not tested on a real RX 6000 card)"
+if [ -z "${DSV41:-}" ]; then DSV41=0; [ "$V" = rx9070 ] && DSV41=1; fi
 ARCHLIST=${ARCHS//;/ }
 DATAARCH=${ARCHS//;/,}
 mkdir -p "$OUT"; S=$OUT/build.status
@@ -56,6 +59,13 @@ if [ -z "${VISION_ONLY:-}" ]; then
   nice -n 10 cmake --build "$OUT/b" --target strata -j "$JOBS" > "$OUT/build.log" 2>&1
   st "engine built rc=$?"
   [ -x "$OUT/b/strata" ] || { st "ABORT no strata"; exit 1; }
+  if [ "$DSV41" = 1 ]; then      # own build tree: the engine's flags stay as they were; STRATA_NATIVE_EXPERTS=ON is what the DeepSeek path was built with
+    cmake -G Ninja -S . -B "$OUT/bd" -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_HIP=ON -DSTRATA_ENABLE_CUDA=OFF -DSTRATA_BUILD_TESTS=OFF       -DSTRATA_BUILD_DSV41=ON -DSTRATA_NATIVE_EXPERTS=ON "-DCMAKE_HIP_ARCHITECTURES=$ARCHS" "${HIPDEFS[@]}" -DSTRATA_GGML_DIR="$LLAMA" -DSTRATA_PORTABLE=ON       "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath,'\$ORIGIN' -Wl,--enable-new-dtags" > "$OUT/cfg-dsv41.log" 2>&1
+    st "dsv41 configured rc=$?"
+    nice -n 10 cmake --build "$OUT/bd" --target strata-dsv41 -j "$JOBS" > "$OUT/build-dsv41.log" 2>&1
+    st "dsv41 built rc=$?"
+    [ -x "$OUT/bd/strata-dsv41" ] || { st "ABORT no strata-dsv41 (DSV41=0 leaves it out)"; tail -20 "$OUT/build-dsv41.log" >> "$S"; exit 1; }
+  fi
 fi
 # the CPU reader: portable (AVX2), the fallback that runs on any PC
 cmake -G Ninja -S tools/vision -B "$OUT/bv" -DCMAKE_BUILD_TYPE=Release -DLLAMA_DIR="$LLAMA" -DSTRATA_VISION_CUDA=OFF \
@@ -74,11 +84,12 @@ st "gpu reader built rc=$?"
 STAGE=$OUT/stage; rm -rf "$STAGE"; mkdir -p "$STAGE"
 ARCHJSON=$(for a in $ARCHLIST; do printf '"%s", ' "$a"; done | sed 's/, $//')
 cat > "$STAGE/BUILD.json" <<EOJ
-{"version": "$PKGVER", "source": "f-amd-vision $COMMIT", "variant": "$V", "archs": [$ARCHJSON], "portable": true, "vision": "gpu", "vision_cpu": "strata-vision", "vision_gpu": "strata-vision-hip", "vision_archs": [$ARCHJSON]}
+{"version": "$PKGVER", "source": "f-amd-vision $COMMIT", "variant": "$V", "archs": [$ARCHJSON], "portable": true, "vision": "gpu", "vision_cpu": "strata-vision", "vision_gpu": "strata-vision-hip", "vision_archs": [$ARCHJSON], "dsv41": $([ "$DSV41" = 1 ] && echo true || echo false)}
 EOJ
 # the libraries the engine and the GPU reader load (the CPU reader needs only the system's), collected from the build outputs
 DATA=(--data "$LIBW/lib/rocblas/library=rocblas/library" --data "$LIBW/lib/hipblaslt/library=hipblaslt/library" --data-arch "$DATAARCH")
-for b in "$OUT/b/strata" "$OUT/bh/bin/strata-vision-hip"; do
+BINS=("$OUT/b/strata" "$OUT/bh/bin/strata-vision-hip"); [ "$DSV41" = 1 ] && BINS+=("$OUT/bd/strata-dsv41")
+for b in "${BINS[@]}"; do
   python3 packaging/collect_libs.py --binary "$b" --search "$LIBW/lib" --search "$HP/lib" --search "$CORE/lib" --search "$CORE/lib/llvm/lib" ${SYSDEPS:+--search "$HP/lib/rocm_sysdeps/lib"} \
     --also 'libamd_comgr.so*' --also 'libhiprtc*.so*' --out "$STAGE" \
     "${DATA[@]}" >> "$OUT/collect.log" 2>&1
@@ -86,8 +97,9 @@ for b in "$OUT/b/strata" "$OUT/bh/bin/strata-vision-hip"; do
   DATA=()                                              # the kernel data once
 done
 cp "$OUT/b/strata" "$OUT/bv/bin/strata-vision" "$OUT/bh/bin/strata-vision-hip" "$STAGE/"
+[ "$DSV41" = 1 ] && cp "$OUT/bd/strata-dsv41" "$STAGE/"
 for f in "$STAGE"/*.so "$STAGE"/*.so.*; do [ -f "$f" ] && [ ! -L "$f" ] && patchelf --set-rpath '$ORIGIN' "$f"; done
-for f in "$STAGE/strata" "$STAGE/strata-vision" "$STAGE/strata-vision-hip"; do
+for f in "$STAGE/strata" "$STAGE/strata-vision" "$STAGE/strata-vision-hip" $([ "$DSV41" = 1 ] && echo "$STAGE/strata-dsv41"); do
   st "RUNPATH $(basename "$f"): $(readelf -d "$f" | grep -E 'RPATH|RUNPATH' | cut -d'[' -f2 | tr -d ']' | paste -sd' ')"
 done
 # the hipBLASLt version of this ROCm into BUILD.json, and the tuning tables of tools/hip that match it (architecture and version) into tuning/
@@ -96,8 +108,9 @@ st "tuning tables rc=$?: $(tail -n 1 "$OUT/stage_tuning.log")"
 [ -n "${NO_PACKAGE:-}" ] && { st "STAGE-DONE"; exit 0; }
 rm -rf "$OUT/components"; mkdir -p "$OUT/components/engine"
 ARCHARGS=""; for a in $ARCHLIST; do ARCHARGS="$ARCHARGS --arch $a"; done
+REQ=""; [ "$DSV41" = 1 ] && REQ="--require-dsv41"
 python3 packaging/package_engine.py --src "$STAGE" --backend hip --version "$PKGVER" --out "$OUT/components/engine" $ARCHARGS \
-  --variant "$V" --build-id "$COMMIT+rocm" --tree --notes "AMD $V build: engine, CPU picture reader, GPU picture reader${NOTE_EXTRA:-}" > "$OUT/package.log" 2>&1
+  --variant "$V" --build-id "$COMMIT+rocm" --tree $REQ --notes "AMD $V build: engine, CPU picture reader, GPU picture reader$([ "$DSV41" = 1 ] && echo ", DeepSeek-V4.1 engine")${NOTE_EXTRA:-}" > "$OUT/package.log" 2>&1
 st "packaged rc=$?"
 cat "$OUT/package.log" >> "$S"
 st "BUILD-DONE"

packaging/feed_tool.py

@@ -156,13 +156,13 @@ def _tree(folder: Path, *, links: bool = False) -> list[tuple[str, Path]]:
     return found
 
 
-def pack_engine(folder: Path, out: Path, fmt: str = "tar.gz", *, url_prefix: str = "engine/") -> dict:
+def pack_engine(folder: Path, out: Path, fmt: str = "tar.gz", *, url_prefix: str = "engine/", prefix: str = "strata-engine") -> dict:
     """Archive a (signed) engine package folder and return its feed entry. The archive holds the folder's files at its root."""
     folder = Path(folder)
     component = ec.read_manifest(folder)
     files = _tree(folder)
     installed = sum(p.stat().st_size for _, p in files)
-    name = f"strata-engine-{folder.name}.{fmt}"
+    name = f"{prefix}-{folder.name}.{fmt}"        # prefix strata-dsv41 for the separate DeepSeek engine packages
     archive = Path(out) / name
     _write_archive(files, archive, fmt, component.entrypoint)
     entry = {"version": component.version, "platform": component.platform, "backend": component.backend,
@@ -190,7 +190,7 @@ def pack_control(folder: Path, out: Path, *, version: str, platform: str, fmt: s
 
 def build_feed(out: Path, *, key: dict, sequence: int, channel: str = "stable", engine_dir: Path | None = None,
                control_dir: Path | None = None, app_json: Path | None = None, expires_days: int | None = None,
-               now: dt.datetime | None = None) -> dict:
+               now: dt.datetime | None = None, dsv41_dir: Path | None = None) -> dict:
     now = now or dt.datetime.now(dt.timezone.utc)
     install_manifest = ""
     document: dict = {"schema_version": feedmod.SCHEMA_VERSION, "feed": "strata-updates", "channel": channel,
@@ -199,6 +199,9 @@ def build_feed(out: Path, *, key: dict, sequence: int, channel: str = "stable",
     if engine_dir:
         document["engine"] = {"packages": [json.loads(p.read_text(encoding="utf-8"))
                                            for p in sorted(Path(engine_dir).glob("*.entry.json"))]}
+    if dsv41_dir:
+        document["dsv41"] = {"packages": [json.loads(p.read_text(encoding="utf-8"))
+                                          for p in sorted(Path(dsv41_dir).glob("*.entry.json"))]}
     if control_dir:
         entries = [json.loads(p.read_text(encoding="utf-8")) for p in sorted(Path(control_dir).glob("*.entry.json"))]
         versions = {e["version"] for e in entries}
@@ -307,6 +310,7 @@ def main(argv: list[str] | None = None) -> int:
     pe.add_argument("--out", type=Path, required=True)
     pe.add_argument("--format", choices=feedmod.FORMATS, default="tar.gz")
     pe.add_argument("--url-prefix", default="engine/", help='where the archive sits relative to the feed ("" for flat GitHub release assets)')
+    pe.add_argument("--prefix", default="strata-engine", help="archive name prefix (strata-dsv41 for the DeepSeek engine packages)")
     pc = sub.add_parser("pack-control")
     pc.add_argument("folder", type=Path)
     pc.add_argument("--out", type=Path, required=True)
@@ -320,6 +324,7 @@ def main(argv: list[str] | None = None) -> int:
     bf.add_argument("--sequence", type=int, required=True)
     bf.add_argument("--channel", default="stable")
     bf.add_argument("--engine", type=Path)
+    bf.add_argument("--dsv41", type=Path, help="folder of the DeepSeek engine packages' .entry.json files (feed member dsv41)")
     bf.add_argument("--control", type=Path)
     bf.add_argument("--app-json", type=Path)
     bf.add_argument("--expires-days", type=int)
@@ -344,16 +349,16 @@ def main(argv: list[str] | None = None) -> int:
     elif a.cmd == "sign-package":
         print("manifest sha256", sign_package(a.folder, _load_key(a.key)))
     elif a.cmd == "pack-engine":
-        entry = pack_engine(a.folder, a.out, a.format, url_prefix=a.url_prefix)
+        entry = pack_engine(a.folder, a.out, a.format, url_prefix=a.url_prefix, prefix=a.prefix)
         print(json.dumps({k: entry[k] for k in ("version", "backend", "variant", "size", "sha256", "manifest_sha256")}, indent=1))
     elif a.cmd == "pack-control":
         entry = pack_control(a.folder, a.out, version=a.version, platform=a.platform, fmt=a.format, url_prefix=a.url_prefix)
         print(json.dumps({k: entry[k] for k in ("version", "platform", "size", "sha256")}, indent=1))
     elif a.cmd == "build-feed":
         doc = build_feed(a.out, key=_load_key(a.key), sequence=a.sequence, channel=a.channel, engine_dir=a.engine,
-                         control_dir=a.control, app_json=a.app_json, expires_days=a.expires_days)
+                         control_dir=a.control, app_json=a.app_json, expires_days=a.expires_days, dsv41_dir=a.dsv41)
         print(f"wrote {a.out / feedmod.FEED_NAME} (+ .sig); engines {len((doc.get('engine') or {}).get('packages', []))}, "
-              f"control {'yes' if 'control' in doc else 'no'}, app {'yes' if 'app' in doc else 'no'}")
+              f"deepseek engines {len((doc.get('dsv41') or {}).get('packages', []))}, control {'yes' if 'control' in doc else 'no'}, app {'yes' if 'app' in doc else 'no'}")
     elif a.cmd == "set-app-platform":
         url = a.url or f"https://github.com/Niko1221/Strata/releases/latest/download/{a.archive.name}"
         entry = set_app_platform(a.app_json, target=a.target, archive=a.archive, signature_file=a.sig, url=url, version=a.version,

packaging/package_engine.py

@@ -14,8 +14,9 @@ checks and starts; it is not signed, not published, and nothing here uploads it.
 (`engine_component.register_signature_verifier`); `--sign-with` is deliberately absent until then.
 
 Files taken from --src, by default: the entrypoint, every `*.dll` / `*.so` / `*.so.*` / `*.dylib` / `*.metallib`
-beside it, `BUILD.json`, `LICENSE*`, and the picture reader `strata-vision[.exe]` when the build has one (without it the
-engine answers text only: setup.py Control mode keeps the person's picture choice only when it is there). `--include GLOB` (repeatable, relative to --src, `**` allowed) adds more, for
+beside it, `BUILD.json`, `LICENSE*`, the picture reader `strata-vision[.exe]` when the build has one (without it the
+engine answers text only: setup.py Control mode keeps the person's picture choice only when it is there) and the DeepSeek-V4.1
+engine `strata-dsv41[.exe]` (setup.py dsv41_exe() looks for it beside the engine; `--require-dsv41` refuses a package without it). `--include GLOB` (repeatable, relative to --src, `**` allowed) adds more, for
 example `strata-vision.exe`, `strata-device.exe` or `rocm/**`. `--tree` takes every regular file under --src (the HIP
 package ships a `rocm/` tree of kernels). Symlinks are never followed or copied.
 """
@@ -43,6 +44,9 @@ from strata_control import engine_component as ec  # noqa: E402
 from strata_control.components import BACKEND_MATRIX, host_platform  # noqa: E402
 
 
+DSV41_EXE = "strata-dsv41"
+
+
 def sha256(path: Path) -> str:
     digest = hashlib.sha256()
     with path.open("rb") as stream:
@@ -54,7 +58,8 @@ def sha256(path: Path) -> str:
 def wanted(path: Path) -> bool:
     name = path.name.lower()
     return (name.endswith((".dll", ".so", ".dylib", ".metallib")) or ".so." in name or name == "build.json"
-            or name.startswith("license") or name in ("strata-vision", "strata-vision.exe", "strata-vision-hip", "strata-vision-hip.exe"))   # the picture readers (CPU, AMD GPU) beside the engine
+            or name.startswith("license") or name in ("strata-vision", "strata-vision.exe", "strata-vision-hip", "strata-vision-hip.exe",
+                                                       DSV41_EXE, DSV41_EXE + ".exe"))   # the picture readers (CPU, AMD GPU) and the DeepSeek-V4.1 engine beside the engine
 
 
 READER_NAMES = ("strata-vision", "strata-vision.exe")
@@ -164,7 +169,8 @@ def build(src: Path, backend: str, version: str, target: str, out: Path, *, entr
           archs: tuple[str, ...] = (), min_driver: dict | None = None, build_id: str | None = None,
           min_runtime: str | None = None, notes: str = "", tree: bool = False, include: list[str] | None = None,
           variant: str = "", search: list[Path] | None = None, allow_libs: list[str] | None = None,
-          check_reader: bool = False, allow_stale_vision: bool = False, allow_no_reader: bool = False) -> Path:
+          check_reader: bool = False, allow_stale_vision: bool = False, allow_no_reader: bool = False,
+          require_dsv41: bool = False) -> Path:
     if backend not in BACKEND_MATRIX:
         raise SystemExit(f"unknown backend {backend!r}")
     if backend != "cpu" and not archs:
@@ -186,6 +192,9 @@ def build(src: Path, backend: str, version: str, target: str, out: Path, *, entr
         for note in check_picture_reader(src, backend, target, archs, variant=variant, allow_stale=allow_stale_vision,
                                          allow_none=allow_no_reader):
             print("note: " + note)
+    if require_dsv41 and not any((src / n).is_file() for n in (DSV41_EXE, DSV41_EXE + ".exe")):
+        raise SystemExit(f"refusing to package: {src} has no {DSV41_EXE}[.exe] (--require-dsv41); build it with -DSTRATA_BUILD_DSV41=ON "
+                         "(--target strata-dsv41) and copy it beside the engine")
     folder = out / (f"{version}-{target}-{backend}" + (f"-{variant}" if variant else ""))
     if folder.exists():
         raise SystemExit(f"{folder} already exists; remove it first (packages are immutable)")
@@ -213,7 +222,9 @@ def build(src: Path, backend: str, version: str, target: str, out: Path, *, entr
             manifest["notes"] = notes
         (staging / ec.MANIFEST).write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8")
         if os.name != "nt":
-            (staging / entrypoint).chmod(0o755)
+            for exe_name in (entrypoint, DSV41_EXE, "strata-vision", "strata-vision-hip"):      # the programs setup starts: executable in the tar
+                if (staging / exe_name).is_file():
+                    (staging / exe_name).chmod(0o755)
         ec.verify(ec.read_manifest(staging), full=True)
         staging.rename(folder)
     except BaseException:
@@ -295,6 +306,8 @@ def main(argv: list[str] | None = None) -> int:
                     help="folder to copy a missing imported DLL from (default on Windows: CUDA_PATH/bin/x64, HIP_PATH/bin, PATH)")
     ap.add_argument("--allow-lib", action="append", default=[], metavar="NAME",
                     help="an imported library that really is provided by the target system (rare; named so the check passes)")
+    ap.add_argument("--require-dsv41", action="store_true",
+                    help="refuse the package when --src has no strata-dsv41[.exe] (the DeepSeek-V4.1 engine that setup.py looks for beside the engine)")
     ap.add_argument("--include", action="append", default=[], metavar="GLOB", help="more files, relative to --src")
     ap.add_argument("--stamp-vision", type=Path, metavar="STAGE", help="write --vision-archs into STAGE/BUILD.json and exit")
     ap.add_argument("--vision-archs", default="", help="with --stamp-vision: the architectures the reader was built for, e.g. '60;61;70;75;80;86;89'")
@@ -309,6 +322,9 @@ def main(argv: list[str] | None = None) -> int:
         except ec.ComponentProblem as exc:
             print(f"{a.verify}: REFUSED: {exc}")
             return 1
+        if a.require_dsv41 and not any(f[0] in (DSV41_EXE, DSV41_EXE + ".exe") for f in component.files):
+            print(f"{a.verify}: REFUSED: no {DSV41_EXE}[.exe] in the manifest (--require-dsv41)")
+            return 1
         bad = binary_deps.uncovered(a.verify)
         if bad:
             for rel, name in bad:
@@ -336,7 +352,7 @@ def main(argv: list[str] | None = None) -> int:
                    archs=tuple(a.arch), min_driver=drivers, build_id=a.build_id, min_runtime=a.min_runtime,
                    notes=a.notes, tree=a.tree, include=a.include, variant=a.variant,
                    search=a.search, allow_libs=a.allow_lib, check_reader=True, allow_stale_vision=a.allow_stale_vision,
-                   allow_no_reader=a.no_picture_reader)
+                   allow_no_reader=a.no_picture_reader, require_dsv41=a.require_dsv41)
     entries = [p for p in folder.rglob("*") if p.is_file()]
     total = sum(p.stat().st_size for p in entries)
     print(f"{folder}  ({total / (1 << 20):.1f} MiB, {len(entries)} files, manifest sha256 {package_sha256(folder)})")

packaging/package_terminal.py

@@ -121,6 +121,11 @@ INSTALLER_FILES = ("setup.py", "ref/load.py")   # ref/load.py: strata_pack.py (t
 #: module it starts to import is never forgotten again; tests/terminal/test_bundle_imports.py checks the staged bundle.
 INSTALLER_TOOLS = ("strata_mcp", "iq_pack", "strata_pack", "pack_index", "mtp_fetch", "mtp_pack", "mtp_rt", "draft_vocab",
                    "strata_tokenizer", "gguf_reader", "calibrate")
+#: tools run BY PATH (`python tools/.../x.py`) by setup.py, strata_cli or strata_control: no import names them, so setup_tool_imports() cannot find
+#: them.  DeepSeek's setup and launch run dsv41/pack_gguf.py (the model folder), dsv41/smoke.py (the engine's first answer) and dsv41/tune.py
+#: (the short tune); setup.py's other by-path tools (nvfp4_local_pack, strata_bench, strata_inspect) were missing from the bundle the same way.
+#: tests/terminal/test_bundle_tools_paths.py reads the by-path references out of the source and fails when one is missing from the bundle.
+INSTALLER_SCRIPTS = ("dsv41/pack_gguf", "dsv41/smoke", "dsv41/tune", "nvfp4_local_pack", "strata_bench", "strata_inspect")
 INSTALLER_DATA = "data"           # draft vocabularies, expert profiles: 2 MB
 
 #: packages the bundle must import after the install (the API server's and the CLI's).
@@ -245,9 +250,15 @@ def local_imports(folder: Path, entry: tuple[str, ...]) -> list[str]:
         source = folder / "tools" / f"{name}.py"
         if not source.exists():
             raise SystemExit(f"installer tool missing: tools/{name}.py")
+        sub = name.rsplit("/", 1)[0] + "/" if "/" in name else ""     # (tools/dsv41/tune.py imports its neighbour smoke.py)
         for node in ast.walk(ast.parse(source.read_text(encoding="utf-8-sig"))):
             names = [a.name.split(".")[0] for a in node.names] if isinstance(node, ast.Import) else                 [node.module.split(".")[0]] if isinstance(node, ast.ImportFrom) and node.module and node.level == 0 else []
-            todo += [m for m in names if (folder / "tools" / f"{m}.py").exists() and m not in seen]
+            for m in names:
+                for rel in ((sub + m,) if sub else ()) + (m,):
+                    if (folder / "tools" / f"{rel}.py").exists():
+                        if rel not in seen:
+                            todo.append(rel)
+                        break
     return sorted(seen)
 
 
@@ -266,7 +277,7 @@ def setup_tool_imports(source: Path = ROOT) -> list[str]:
 
 def add_installer(folder: Path, source: Path = ROOT) -> dict:
     """Copy the installer's files from the source tree into a staged terminal folder (after the terminal payload is in place)."""
-    tools = local_imports(source, tuple(sorted(set(INSTALLER_TOOLS) | set(setup_tool_imports(source)))))
+    tools = local_imports(source, tuple(sorted(set(INSTALLER_TOOLS) | set(INSTALLER_SCRIPTS) | set(setup_tool_imports(source)))))
     copied = []
     for name in INSTALLER_FILES:
         (folder / name).parent.mkdir(parents=True, exist_ok=True)
@@ -274,6 +285,7 @@ def add_installer(folder: Path, source: Path = ROOT) -> dict:
         copied.append(name)
     (folder / "tools").mkdir(exist_ok=True)
     for name in tools:
+        (folder / "tools" / f"{name}.py").parent.mkdir(parents=True, exist_ok=True)
         shutil.copy2(source / "tools" / f"{name}.py", folder / "tools" / f"{name}.py")
         copied.append(f"tools/{name}.py")
     if (folder / INSTALLER_DATA).exists():

serve/dsml.py

@@ -0,0 +1,256 @@
+"""serve/dsml.py - DeepSeek-V4.1's tool-call text (DSML) for the server.
+
+DeepSeek-V4.1 writes a tool call as (the model's own chat template, the same bytes as DeepSeek's encoding.py)
+
+    <｜DSML｜ calls>
+    <｜DSML｜ invoke name="get_weather">
+    <｜DSML｜ parameter name="city" string="true">Zagreb</｜DSML｜ parameter>
+    <｜DSML｜ parameter name="days" string="false">3</｜DSML｜ parameter>
+    </｜DSML｜ invoke>
+    </｜DSML｜ calls>
+
+`｜DSML｜` is one token (id 128825, a user-defined one, so it is also text when a request is tokenized).  String
+values are written as they are; every other type is JSON (string="false").
+
+The server's OutputParser (serve/frontend.py) reads the Qwen form `<tool_call><function=NAME><parameter=P>...`, with
+its streaming of tool arguments, its calls inside the thinking and its recovery rules.  DsmlTranslator rewrites the
+DSML text into that form as it streams (holding back only a tag that is still arriving), and DsmlOutputParser is the
+OutputParser behind it.  So the OpenAI and Anthropic paths, tool_choice, streaming arguments and the thinking/answer
+split behave for DeepSeek as they do for the other families, and there is one place that knows the DSML spelling.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from serve.frontend import Event, OutputParser
+
+DSML = "｜DSML｜"
+_OPEN = "<" + DSML
+_CLOSE = "</" + DSML
+_TAG_MAX = 600                  # a tag (a tool name, a parameter name) longer than this is text, not held any further
+
+_CALLS = re.compile(r"<｜DSML｜\s*(?:tool_|function_)?calls\s*>")
+_CALLS_END = re.compile(r"</｜DSML｜\s*(?:tool_|function_)?calls\s*>")
+_INVOKE = re.compile(r'<｜DSML｜\s*invoke\s+name\s*=\s*"([^"]*)"\s*>')
+_INVOKE_END = re.compile(r"</｜DSML｜\s*invoke\s*>")
+_PARAM = re.compile(r'<｜DSML｜\s*parameter\s+name\s*=\s*"([^"]*)"(?:\s+string\s*=\s*"(true|false)")?\s*>')
+_PARAM_END = re.compile(r"</｜DSML｜\s*parameter\s*>")
+# a tag cut short is held while it can still become one of those: "<", "<｜DS", '<｜DSML｜ invoke name="get_we'
+_PARTIAL = re.compile(r"^</?(?:｜(?:D(?:S(?:M(?:L(?:｜[^<>]*)?)?)?)?)?)?$")
+
+
+def _could_still_be_a_tag(s: str) -> bool:
+    """`s` starts with '<' and has no '>' yet: it may be the start of a DSML tag."""
+    return len(s) <= _TAG_MAX and _PARTIAL.match(s) is not None
+
+
+class DsmlTranslator:
+    """DSML text -> the Qwen tool-call text, a delta at a time.  Text outside a call passes through unchanged.
+
+    on_param(tool, name, is_string) is called when a parameter tag is read, before its text is returned, so the parser
+    behind it can treat a string="true" value as the string it is."""
+
+    def __init__(self, on_param=None):
+        self.buf = ""
+        self.state = "text"        # text | calls (inside the block) | invoke | param (inside a value)
+        self.tool = None
+        self.on_param = on_param
+
+    @property
+    def held(self) -> str:
+        """Text held back because it may be the start of a tag."""
+        return self.buf
+
+    def feed(self, delta: str) -> str:
+        self.buf += delta
+        return self._run(final=False)
+
+    def flush(self) -> str:
+        """The end of the output: whatever is held is text."""
+        return self._run(final=True)
+
+    # ------------------------------------------------------------------------------------------------
+    def _run(self, final: bool) -> str:
+        out: list[str] = []
+        while self.buf:
+            lt = self.buf.find("<")
+            if lt < 0:
+                self._text(self.buf, out)
+                self.buf = ""
+                break
+            if lt:
+                self._text(self.buf[:lt], out)
+                self.buf = self.buf[lt:]
+            tag = self._tag(out)
+            if tag == "wait":
+                if final:
+                    self._text(self.buf, out)
+                    self.buf = ""
+                break
+            if tag is None:                     # an ordinary '<'
+                self._text("<", out)
+                self.buf = self.buf[1:]
+        return "".join(out)
+
+    def _text(self, text: str, out: list[str]) -> None:
+        if self.state in ("calls", "invoke"):
+            if text.strip() == "":
+                return                          # the layout between tags
+            # stray text inside the block: it is shown as text, and the block goes on
+        out.append(text)
+
+    def _tag(self, outl: list[str]):
+        """Handle the tag at the start of self.buf.  Returns True (a tag was consumed), "wait" (it may still become
+        one) or None (it is not one)."""
+        s = self.buf
+        gt = s.find(">")
+        if gt < 0:
+            return "wait" if _could_still_be_a_tag(s) else None
+        head = s[:gt + 1]
+        if not (head.startswith(_OPEN) or head.startswith(_CLOSE)):
+            return None
+        out = outl.append
+        m = _PARAM_END.fullmatch(head)
+        if m:
+            if self.state == "param":
+                self.buf = s[gt + 1:]
+                self.state = "invoke"
+                out("\n</parameter>\n")
+                return True
+            return None
+        m = _INVOKE_END.fullmatch(head) or _CALLS_END.fullmatch(head)
+        if m:
+            closing_invoke = bool(_INVOKE_END.fullmatch(head))
+            if self.state == "param":           # the value's closing tag was left out
+                out("\n</parameter>\n")
+                self.state = "invoke"
+            if closing_invoke and self.state == "invoke":
+                self.buf = s[gt + 1:]
+                self.state = "calls"
+                out("</function>\n</tool_call>\n")
+                return True
+            if not closing_invoke and self.state in ("calls", "invoke"):
+                if self.state == "invoke":
+                    out("</function>\n</tool_call>\n")
+                self.buf = s[gt + 1:]
+                self.state = "text"
+                return True
+            return None
+        if self.state == "param":
+            return None                         # inside a value only a closing tag means anything
+        m = _CALLS.fullmatch(head)
+        if m:
+            self.buf = s[gt + 1:]
+            self.state = "calls"
+            return True
+        m = _INVOKE.fullmatch(head)
+        if m:
+            if self.state == "invoke":          # the previous call's closing tag was left out
+                out("</function>\n</tool_call>\n")
+            self.buf = s[gt + 1:]
+            self.state, self.tool = "invoke", m.group(1)
+            out(f"<tool_call>\n<function={m.group(1)}>\n")
+            return True
+        m = _PARAM.fullmatch(head)
+        if m and self.state in ("calls", "invoke"):
+            self.buf = s[gt + 1:]
+            self.state = "param"
+            if self.on_param is not None:
+                self.on_param(self.tool, m.group(1), m.group(2) != "false")
+            out(f"<parameter={m.group(1)}>\n")
+            return True
+        return None
+
+
+class DsmlOutputParser(OutputParser):
+    """OutputParser for DeepSeek-V4.1: the model's DSML calls are read as the template-form calls they are."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.dsml = DsmlTranslator(self._on_param)
+        # The 2-bit sets often write the whole answer after <think> and stop (EOS) without </think>: everything would be reasoning and
+        # the reply empty (P2003 install test, 4 of 4 prompts).  The reasoning text is kept here; finish() hands it over as the answer
+        # when the turn ended on a stop with no answer and no call.
+        self._reasoning: list[str] = []
+        self._answered = False
+
+    def _watch(self, evs):
+        for ev in evs:
+            if ev.kind == "reasoning":
+                self._reasoning.append(ev.text)
+            elif ev.kind in ("tool_start", "tool_call") or (ev.kind == "content" and ev.text.strip()):
+                self._answered = True
+        return evs
+
+    def _on_param(self, tool, name, is_string) -> None:
+        """string="true": the value is a string whatever it looks like ("123", "true"), as the template wrote it."""
+        if not is_string:
+            return
+        spec = self.schemas.get(tool)
+        if spec is None:
+            return
+        params = dict(spec.get("parameters") or {})
+        props = dict(params.get("properties") or {})
+        if (props.get(name) or {}).get("type") != "string":
+            props[name] = {**(props.get(name) or {}), "type": "string"}
+            params["properties"] = props
+            self.schemas[tool] = {**spec, "parameters": params}
+        if self.scall is not None and self.scall.name == tool:
+            self.sdeclared[name] = "string"
+
+    def feed(self, delta: str):
+        return self._watch(super().feed(self.dsml.feed(delta)))
+
+    def finish(self, reason: str | None = None):
+        rest = self.dsml.flush()
+        out = self._watch(super().feed(rest) if rest else [])
+        out += self._watch(super().finish(reason))
+        text = "".join(self._reasoning).strip()
+        if reason in (None, "stop") and text and not self._answered:
+            # no </think> and no answer: the text IS the answer.  A streaming client has already seen it as reasoning and now gets
+            # it as content; the non-streaming collectors (server.py) drop the identical reasoning.
+            self._answered = True
+            out.append(Event("content", text))
+        return out
+
+    @property
+    def clean(self) -> bool:
+        """No tag is held back anywhere: a clean point to stop the thinking at."""
+        return not self.buf and not self.dsml.held
+
+
+FIXTURE_TEMPLATE = Path(__file__).resolve().parent / "fixtures" / "deepseek41" / "chat_template.jinja"
+
+
+def v41_template_ok(text: str) -> bool:
+    """The V4.1 model writes its call block as `<｜DSML｜ calls>` / `<｜DSML｜ invoke ...` (a space after the DSML token).
+    Packs made from another GGUF ship the V4-Flash template (`<｜DSML｜tool_calls>`, no space): the model then sees one
+    spelling in the prompt and writes another, and only 2 of 11 tool-call tries came back as real calls (P2003 install
+    test; 0 of 4 vs 6 of 6 with this template on the same pack)."""
+    return " calls>" in text
+
+
+def pick_template(pack_template: Path | None) -> tuple[Path, bool]:
+    """(the template file to render with, True when it replaced a pack template that spells the call block otherwise)."""
+    if pack_template is not None and pack_template.exists():
+        try:
+            if v41_template_ok(pack_template.read_text(encoding="utf-8")):
+                return pack_template, False
+        except OSError:
+            return pack_template, False
+        return FIXTURE_TEMPLATE, True
+    return FIXTURE_TEMPLATE, False
+
+
+def to_dsml_force(force: str | None) -> str | None:
+    """forced_call's opening of a call in the Qwen form -> the same opening in DSML (the model's own spelling)."""
+    if not force:
+        return force
+    head = _OPEN + " calls>\n"
+    marker = "<function="
+    at = force.find(marker)
+    name = force[at + len(marker):] if at >= 0 else ""
+    if name.endswith(">\n"):
+        return head + f'{_OPEN} invoke name="{name[:-2]}">\n'
+    return head + f'{_OPEN} invoke name="{name}'

serve/fixtures/deepseek41/chat_template.jinja

@@ -0,0 +1,291 @@
+{#- Unsloth port of DS41 #}
+{%- set IMG = "<｜deepseek_image｜>" -%}
+{%- set DSML = "｜DSML｜" -%}
+
+{%- if thinking_mode is not defined or thinking_mode is none -%}
+    {%- if enable_thinking is defined and enable_thinking is not none -%}
+        {%- set thinking_mode = "thinking" if enable_thinking else "chat" -%}
+    {%- elif thinking is defined and thinking is not none and thinking is not string -%}
+        {%- set thinking_mode = "thinking" if thinking else "chat" -%}
+    {%- else -%}
+        {%- set thinking_mode = "thinking" -%}
+    {%- endif -%}
+{%- endif -%}
+{%- if thinking_mode not in ["chat", "thinking"] -%}
+    {{- raise_exception("Invalid thinking_mode `" ~ thinking_mode ~ "`, should be \"chat\" or \"thinking\"") -}}
+{%- endif -%}
+{%- if drop_thinking is not defined or drop_thinking is none -%}{%- set drop_thinking = true -%}{%- endif -%}
+{%- if add_generation_prompt is not defined -%}{%- set add_generation_prompt = false -%}{%- endif -%}
+{%- if tools is not defined -%}{%- set tools = none -%}{%- endif -%}
+{%- if response_format is not defined -%}{%- set response_format = none -%}{%- endif -%}
+
+{%- if reasoning_effort is not defined or reasoning_effort is none -%}{%- set reasoning_effort = "high" -%}{%- endif -%}
+{%- set effort_map = {"low": 50, "medium": 62, "high": 75, "max": 100} -%}
+{%- if reasoning_effort is string and reasoning_effort in effort_map -%}
+    {%- set budget = effort_map[reasoning_effort] -%}
+{%- elif reasoning_effort is integer and reasoning_effort >= 1 and reasoning_effort <= 100 -%}
+    {%- set budget = reasoning_effort -%}
+{%- else -%}
+    {{- raise_exception("Invalid reasoning effort: " ~ reasoning_effort ~ ", should be int within [1,100] or one of \"low\", \"medium\", \"high\", \"max\"") -}}
+{%- endif -%}
+
+{#- Splits/validates "ns::name" against an explicit namespace and returns the qualified name. -#}
+{%- macro qualified_name(name, ns) -%}
+    {%- if ns is mapping -%}{%- set ns = ns.name -%}{%- endif -%}
+    {%- set parts = name.split("::", 1) -%}
+    {%- if parts | length == 2 -%}
+        {%- if ns is not none and ns != parts[0] -%}{{- raise_exception("Conflicting tool namespaces: " ~ ns ~ " != " ~ parts[0]) -}}{%- endif -%}
+        {%- set ns = parts[0] -%}
+        {%- set name = parts[1] -%}
+    {%- endif -%}
+    {%- if "::" in name -%}{{- raise_exception("Tool name must not contain '::': " ~ name) -}}{%- endif -%}
+    {%- if ns is not none and "::" in ns -%}{{- raise_exception("Tool namespace must not contain '::': " ~ ns) -}}{%- endif -%}
+    {{- name if ns is none else ns ~ "::" ~ name -}}
+{%- endmacro -%}
+
+{%- macro render_tools(tool_list) -%}
+    {%- set ts = namespace(schemas=[]) -%}
+    {%- for tool in tool_list -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {%- set ns = tool.namespace if tool.namespace is defined and tool.namespace is not none else fn.get("namespace") -%}
+        {#- JSON assembled by hand (key order kept, name/description replaced in place): llama.cpp has no ** unpacking. -#}
+        {%- set qname = qualified_name(fn.name, ns) -%}
+        {%- set sch = namespace(parts=[]) -%}
+        {%- for k, v in fn.items() if k != "namespace" -%}
+            {%- if k == "name" -%}{%- set v = qname -%}
+            {%- elif k == "description" and ns is mapping and ns.description -%}{%- set v = ns.description ~ "\n" ~ (fn.description or "") -%}
+            {%- endif -%}
+            {%- set sch.parts = sch.parts + [(k | tojson) ~ ": " ~ (v | tojson)] -%}
+        {%- endfor -%}
+        {%- if "description" not in fn and ns is mapping and ns.description -%}
+            {%- set sch.parts = sch.parts + ['"description": ' ~ ((ns.description ~ "\n") | tojson)] -%}
+        {%- endif -%}
+        {%- set ts.schemas = ts.schemas + ["{" ~ sch.parts | join(", ") ~ "}"] -%}
+    {%- endfor -%}
+    {{- "## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜ calls>\" block like the following:\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"$TOOL_NAME\">\n<｜DSML｜ parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜ parameter>\n...\n</｜DSML｜ invoke>\n<｜DSML｜ invoke name=\"$TOOL_NAME2\">\n...\n</｜DSML｜ invoke>\n</｜DSML｜ calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n" -}}
+    {{- ts.schemas | join("\n") -}}
+    {{- "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n" -}}
+{%- endmacro -%}
+
+{#- 1. Merge tool results (and consecutive user turns) into user messages made of content blocks. -#}
+{%- set mns = namespace(msgs=[]) -%}
+{%- for msg in messages -%}
+    {%- set content = msg.get("content") -%}
+    {%- set is_blocks = content is iterable and content is not string and content is not mapping -%}
+    {%- if content is string and IMG in content -%}
+        {{- raise_exception("Message content contains image special token " ~ IMG ~ ". Images should be provided as image content blocks.") -}}
+    {%- endif -%}
+    {%- set rc = msg.reasoning_content if msg.get("reasoning_content") is not none else msg.get("thinking") -%}
+    {%- if rc is string and IMG in rc -%}
+        {{- raise_exception("reasoning_content contains image special token " ~ IMG) -}}
+    {%- endif -%}
+    {%- set prev = mns.msgs[-1] if mns.msgs else none -%}
+    {%- if msg.role == "tool" -%}
+        {%- set tblock = {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content if content is not none else ""} -%}
+        {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined -%}
+            {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + [tblock])] -%}
+        {%- else -%}
+            {%- set mns.msgs = mns.msgs + [{"role": "user", "content_blocks": [tblock]}] -%}
+        {%- endif -%}
+    {%- elif msg.role == "user" -%}
+        {%- set blocks = msg.content_blocks if msg.get("content_blocks") is not none else (content if is_blocks else [{"type": "text", "text": content or ""}]) -%}
+        {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined and prev.get("task") is none -%}
+            {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + blocks)] -%}
+        {%- else -%}
+            {%- set mns.msgs = mns.msgs + [dict(msg, content_blocks=blocks)] -%}
+        {%- endif -%}
+    {%- else -%}
+        {%- set mns.msgs = mns.msgs + [msg] -%}
+    {%- endif -%}
+{%- endfor -%}
+
+{#- Top-level tools / response_format attach to a leading system message, or to a new empty one. -#}
+{%- if tools or response_format -%}
+    {%- if not (mns.msgs and mns.msgs[0].role == "system") -%}
+        {%- set mns.msgs = [{"role": "system", "content": ""}] + mns.msgs -%}
+    {%- endif -%}
+    {%- set first = mns.msgs[0] -%}
+    {%- if tools and not first.get("tools") -%}{%- set first = dict(first, tools=tools) -%}{%- endif -%}
+    {%- if response_format and not first.get("response_format") -%}{%- set first = dict(first, response_format=response_format) -%}{%- endif -%}
+    {%- set mns.msgs = [first] + mns.msgs[1:] -%}
+{%- endif -%}
+
+{#- 2. Reorder tool results to match the tool_calls order of the preceding assistant message. -#}
+{%- set sns = namespace(msgs=[], ids=[]) -%}
+{%- for m in mns.msgs -%}
+    {%- if m.role == "assistant" and m.get("tool_calls") -%}
+        {%- set sns.ids = [] -%}
+        {%- for tc in m.tool_calls -%}
+            {%- set sns.ids = sns.ids + [tc.get("id") or (tc.function.get("id", "") if tc.function is defined else "")] -%}
+        {%- endfor -%}
+        {%- set sns.msgs = sns.msgs + [m] -%}
+    {%- elif m.role == "user" and m.content_blocks -%}
+        {%- set tb = namespace(blocks=[]) -%}
+        {%- for b in m.content_blocks if b.type == "tool_result" -%}
+            {%- set order = namespace(idx=0) -%}
+            {%- for tc_id in sns.ids -%}
+                {%- if tc_id and tc_id == b.get("tool_use_id", "") -%}{%- set order.idx = loop.index0 -%}{%- endif -%}
+            {%- endfor -%}
+            {%- set tb.blocks = tb.blocks + [dict(b, _order=order.idx)] -%}
+        {%- endfor -%}
+        {%- if tb.blocks | length > 1 and sns.ids -%}
+            {%- set sorted_blocks = tb.blocks | sort(attribute="_order") -%}
+            {%- set rb = namespace(blocks=[], i=0) -%}
+            {%- for b in m.content_blocks -%}
+                {%- if b.type == "tool_result" -%}
+                    {%- set rb.blocks = rb.blocks + [sorted_blocks[rb.i]] -%}
+                    {%- set rb.i = rb.i + 1 -%}
+                {%- else -%}
+                    {%- set rb.blocks = rb.blocks + [b] -%}
+                {%- endif -%}
+            {%- endfor -%}
+            {%- set sns.msgs = sns.msgs + [dict(m, content_blocks=rb.blocks)] -%}
+        {%- else -%}
+            {%- set sns.msgs = sns.msgs + [m] -%}
+        {%- endif -%}
+    {%- else -%}
+        {%- set sns.msgs = sns.msgs + [m] -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- set msgs = sns.msgs -%}
+
+{#- Last user turn: users and mid-conversation systems count, except user turns made only of tool results.
+    Tools anywhere disable drop_thinking. -#}
+{%- set state = namespace(last_user=-1, has_tools=false) -%}
+{%- for m in msgs -%}
+    {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%}
+        {%- set real = namespace(v=true) -%}
+        {%- if m.get("content_blocks") -%}
+            {%- set real.v = false -%}
+            {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%}
+        {%- endif -%}
+        {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%}
+    {%- endif -%}
+    {%- if m.get("tools") -%}{%- set state.has_tools = true -%}{%- endif -%}
+{%- endfor -%}
+{%- set effective_drop = drop_thinking and not state.has_tools -%}
+{%- set is_thinking = thinking_mode == "thinking" -%}
+{#- Like encoding.py, thinking mode with drop_thinking silently drops messages of other roles before the last user turn. -#}
+{%- if is_thinking and effective_drop -%}
+    {%- set kept = namespace(msgs=[]) -%}
+    {%- for m in msgs -%}
+        {%- if m.role in ["user", "system", "tool", "latest_reminder", "direct_search_results", "assistant"] or loop.index0 >= state.last_user -%}
+            {%- set kept.msgs = kept.msgs + [m] -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- if kept.msgs | length != msgs | length -%}
+        {%- set msgs = kept.msgs -%}
+        {%- set state.last_user = -1 -%}
+        {%- for m in msgs -%}
+            {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%}
+                {%- set real = namespace(v=true) -%}
+                {%- if m.get("content_blocks") -%}
+                    {%- set real.v = false -%}
+                    {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%}
+                {%- endif -%}
+                {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endif -%}
+
+{{- "<｜begin▁of▁sentence｜>" -}}
+{#- 3. Render each message. -#}
+{%- for m in msgs -%}
+    {%- set idx = loop.index0 -%}
+    {%- set content = m.get("content") -%}
+    {%- if content is iterable and content is not string and content is not mapping -%}
+        {%- set sp = namespace(parts=[]) -%}
+        {%- for b in content -%}
+            {%- if b.type == "text" -%}{%- set sp.parts = sp.parts + [b.text or ""] -%}
+            {%- elif b.type in ["image", "image_url"] -%}{%- set sp.parts = sp.parts + [IMG] -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {%- set content = sp.parts | join("\n\n") -%}
+    {%- endif -%}
+
+    {%- if idx == 0 -%}
+        {%- if is_thinking or m.role == "system" -%}{{- "<｜System｜>" -}}{%- endif -%}
+        {%- if is_thinking -%}{{- "Reasoning Effort: " ~ budget ~ " (range 1-100, the higher the value, the more thorough the reasoning)\n\n" -}}{%- endif -%}
+    {%- endif -%}
+
+    {%- if m.role == "system" -%}
+        {%- if idx > 0 -%}{{- "<｜System｜>" -}}{%- endif -%}
+        {{- content or "" -}}
+        {%- if m.get("tools") -%}{{- "\n\n" ~ render_tools(m.tools) -}}{%- endif -%}
+        {%- if m.get("response_format") -%}
+            {{- "\n\n## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n" ~ (m.response_format | tojson) -}}
+        {%- endif -%}
+
+    {%- elif m.role == "user" -%}
+        {{- "<｜User｜>" -}}
+        {%- set up = namespace(parts=[]) -%}
+        {%- for b in m.content_blocks -%}
+            {%- if b.type == "text" -%}
+                {%- if IMG in (b.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%}
+                {%- set up.parts = up.parts + [b.text or ""] -%}
+            {%- elif b.type in ["image", "image_url"] -%}
+                {%- set up.parts = up.parts + [IMG] -%}
+            {%- elif b.type == "tool_result" -%}
+                {%- set tc = b.get("content", "") -%}
+                {%- if tc is iterable and tc is not string and tc is not mapping -%}
+                    {%- set sub = namespace(parts=[]) -%}
+                    {%- for s in tc -%}
+                        {%- if s.type == "text" -%}
+                            {%- if IMG in (s.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%}
+                            {%- set sub.parts = sub.parts + [s.get("text", "")] -%}
+                        {%- elif s.type in ["image", "image_url"] -%}{%- set sub.parts = sub.parts + [IMG] -%}
+                        {%- else -%}{%- set sub.parts = sub.parts + ["[Unsupported " ~ s.type ~ "]"] -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                    {%- set tc = sub.parts | join("\n\n") -%}
+                {%- endif -%}
+                {%- set up.parts = up.parts + ["<tool_result>" ~ tc ~ "</tool_result>"] -%}
+            {%- else -%}
+                {%- set up.parts = up.parts + ["[Unsupported " ~ b.type ~ "]"] -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- up.parts | join("\n\n") -}}
+
+    {%- elif m.role == "latest_reminder" -%}
+        {{- "<｜latest_reminder｜>" ~ (content or "") -}}
+
+    {%- elif m.role == "assistant" -%}
+        {%- set prev_has_task = idx > 0 and msgs[idx - 1].get("task") is not none -%}
+        {%- if is_thinking and not prev_has_task and (not effective_drop or idx > state.last_user) -%}
+            {{- ((m.reasoning_content if m.get("reasoning_content") is not none else m.get("thinking")) or "") ~ "</think>" -}}
+        {%- endif -%}
+        {{- content or "" -}}
+        {%- if m.get("tool_calls") -%}
+            {%- set tcs = namespace(lines=[]) -%}
+            {%- for tc in m.tool_calls -%}
+                {%- set fn = tc.function if tc.function is defined else tc -%}
+                {%- set params = namespace(lines=[]) -%}
+                {%- set args = fn.arguments if fn.arguments is mapping else {"arguments": fn.arguments} -%}
+                {%- for k, v in args.items() -%}
+                    {%- set params.lines = params.lines + ["<" ~ DSML ~ " parameter name=\"" ~ k ~ "\" string=\"" ~ ("true" if v is string else "false") ~ "\">" ~ (v if v is string else v | tojson) ~ "</" ~ DSML ~ " parameter>"] -%}
+                {%- endfor -%}
+                {%- set name = qualified_name(fn.name, tc.get("namespace") or fn.get("namespace")) -%}
+                {%- set tcs.lines = tcs.lines + ["<" ~ DSML ~ " invoke name=\"" ~ name ~ "\">\n" ~ params.lines | join("\n") ~ "\n</" ~ DSML ~ " invoke>"] -%}
+            {%- endfor -%}
+            {{- "\n\n<" ~ DSML ~ " calls>\n" ~ tcs.lines | join("\n") ~ "\n</" ~ DSML ~ " calls>" -}}
+        {%- endif -%}
+        {%- if not m.get("wo_eos", false) -%}{{- "<｜end▁of▁sentence｜>" -}}{%- endif -%}
+
+    {%- else -%}
+        {{- raise_exception("Unknown role: " ~ m.role) -}}
+    {%- endif -%}
+
+    {#- Transition tokens, only when the next message is an assistant/latest_reminder turn or this is the last one. -#}
+    {%- if loop.last or msgs[idx + 1].role in ["assistant", "latest_reminder"] -%}
+        {%- set task = m.get("task") -%}
+        {%- if task is not none -%}
+            {%- set task_tokens = {"action": "<｜action｜>", "query": "<｜query｜>", "authority": "<｜authority｜>", "domain": "<｜domain｜>", "title": "<｜title｜>", "read_url": "<｜read_url｜>"} -%}
+            {%- if task not in task_tokens -%}{{- raise_exception("Invalid task: '" ~ task ~ "'") -}}{%- endif -%}
+            {%- if task == "action" -%}{{- "<｜Assistant｜>" ~ ("<think>" if is_thinking else "</think>") -}}{%- endif -%}
+            {{- task_tokens[task] -}}
+        {%- elif (m.role == "user" or (m.role == "system" and idx > 0)) and (add_generation_prompt or not loop.last) -%}
+            {{- "<｜Assistant｜>" ~ ("<think>" if is_thinking and (not effective_drop or idx >= state.last_user) else "</think>") -}}
+        {%- endif -%}
+    {%- endif -%}
+{%- endfor -%}

serve/fixtures/deepseek41/golden.json

@@ -0,0 +1,965 @@
+[
+ {
+  "id": "sanity-I1",
+  "mode": "chat",
+  "request": {
+   "messages": [
+    {
+     "role": "user",
+     "content": "[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys."
+    }
+   ],
+   "chat_template_kwargs": {
+    "thinking": false,
+    "enable_thinking": false
+   }
+  },
+  "thinking": false,
+  "text": "<｜begin▁of▁sentence｜><｜User｜>[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys.<｜Assistant｜></think>",
+  "ids": [
+   0,
+   128803,
+   61,
+   11675,
+   102630,
+   20448,
+   19,
+   2296,
+   25529,
+   1353,
+   260,
+   26639,
+   2873,
+   418,
+   9109,
+   1341,
+   304,
+   3292,
+   305,
+   1895,
+   1341,
+   304,
+   270,
+   9132,
+   223,
+   21,
+   16,
+   3011,
+   915,
+   18222,
+   16,
+   128804,
+   128822
+  ],
+  "expect_text": "{\"ok\": true, \"count\": 3}<｜end▁of▁sentence｜>",
+  "finish": "stop"
+ },
+ {
+  "id": "sanity-I1",
+  "mode": "thinking",
+  "request": {
+   "messages": [
+    {
+     "role": "user",
+     "content": "[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys."
+    }
+   ]
+  },
+  "thinking": true,
+  "text": "<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<｜User｜>[case sanity-I1]\nReturn only a JSON object with ok set to true and count set to the integer 3. No other keys.<｜Assistant｜><think>",
+  "ids": [
+   0,
+   128799,
+   77666,
+   288,
+   9504,
+   482,
+   28,
+   223,
+   2402,
+   343,
+   16801,
+   223,
+   19,
+   15,
+   1457,
+   14,
+   270,
+   4077,
+   270,
+   1990,
+   14,
+   270,
+   850,
+   15146,
+   270,
+   22805,
+   868,
+   128803,
+   61,
+   11675,
+   102630,
+   20448,
+   19,
+   2296,
+   25529,
+   1353,
+   260,
+   26639,
+   2873,
+   418,
+   9109,
+   1341,
+   304,
+   3292,
+   305,
+   1895,
+   1341,
+   304,
+   270,
+   9132,
+   223,
+   21,
+   16,
+   3011,
+   915,
+   18222,
+   16,
+   128804,
+   128821
+  ],
+  "expect_text": "We need answer only JSON object with ok true and count integer 3. No other keys. Need comply.</think>{\"ok\":true,\"count\":3}<｜end▁of▁sentence｜>",
+  "finish": "stop"
+ },
+ {
+  "id": "sanity-T1",
+  "mode": "chat",
+  "request": {
+   "messages": [
+    {
+     "role": "user",
+     "content": "[case sanity-T1]\nUse lookup_record to look up record A-17."
+    }
+   ],
+   "tools": [
+    {
+     "type": "function",
+     "function": {
+      "name": "lookup_record",
+      "description": "Synthetic benchmark lookup; no external action is performed.",
+      "parameters": {
+       "type": "object",
+       "properties": {
+        "id": {
+         "type": "string"
+        }
+       },
+       "required": [
+        "id"
+       ],
+       "additionalProperties": false
+      }
+     }
+    }
+   ],
+   "chat_template_kwargs": {
+    "thinking": false,
+    "enable_thinking": false
+   }
+  },
+  "thinking": false,
+  "text": "<｜begin▁of▁sentence｜><｜System｜>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜ calls>\" block like the following:\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"$TOOL_NAME\">\n<｜DSML｜ parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜ parameter>\n...\n</｜DSML｜ invoke>\n<｜DSML｜ invoke name=\"$TOOL_NAME2\">\n...\n</｜DSML｜ invoke>\n</｜DSML｜ calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"lookup_record\", \"description\": \"Synthetic benchmark lookup; no external action is performed.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"id\": {\"type\": \"string\"}}, \"required\": [\"id\"], \"additionalProperties\": false}}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<｜User｜>[case sanity-T1]\nUse lookup_record to look up record A-17.<｜Assistant｜></think>",
+  "ids": [
+   0,
+   128799,
+   271,
+   372,
+   27193,
+   271,
+   3476,
+   611,
+   3278,
+   304,
+   260,
+   1341,
+   294,
+   6704,
+   304,
+   1694,
+   3287,
+   270,
+   3967,
+   734,
+   3417,
+   16,
+   2042,
+   588,
+   34756,
+   6704,
+   513,
+   4985,
+   260,
+   38322,
+   128825,
+   10699,
+   48902,
+   5603,
+   1277,
+   270,
+   2502,
+   979,
+   30,
+   128825,
+   10699,
+   1018,
+   30,
+   128825,
+   34756,
+   2329,
+   84476,
+   6792,
+   5896,
+   35044,
+   3816,
+   30,
+   128825,
+   10767,
+   2329,
+   84476,
+   46630,
+   4130,
+   61729,
+   35044,
+   4,
+   3418,
+   1281,
+   11476,
+   94,
+   19836,
+   3320,
+   6,
+   46630,
+   4130,
+   61729,
+   56909,
+   1718,
+   128825,
+   10767,
+   1018,
+   7835,
+   1718,
+   128825,
+   34756,
+   1018,
+   30,
+   128825,
+   34756,
+   2329,
+   84476,
+   6792,
+   5896,
+   35044,
+   20,
+   3816,
+   7835,
+   1718,
+   128825,
+   34756,
+   1018,
+   1718,
+   128825,
+   10699,
+   4697,
+   3524,
+   8252,
+   1531,
+   366,
+   12038,
+   412,
+   344,
+   305,
+   1341,
+   3608,
+   4463,
+   1281,
+   11476,
+   4,
+   37419,
+   1884,
+   710,
+   915,
+   4815,
+   343,
+   62896,
+   14,
+   2631,
+   2065,
+   634,
+   14,
+   31939,
+   14,
+   8435,
+   754,
+   2281,
+   270,
+   1990,
+   295,
+   26639,
+   8786,
+   305,
+   1341,
+   3608,
+   4463,
+   1281,
+   19836,
+   4,
+   108526,
+   3575,
+   6892,
+   71144,
+   344,
+   22104,
+   343,
+   89944,
+   284,
+   513,
+   223,
+   128821,
+   754,
+   440,
+   74366,
+   5238,
+   782,
+   5553,
+   22805,
+   6352,
+   223,
+   128821,
+   1613,
+   128822,
+   119907,
+   1117,
+   4105,
+   10699,
+   469,
+   4087,
+   4256,
+   339,
+   13079,
+   6922,
+   14,
+   5238,
+   6578,
+   1561,
+   223,
+   128822,
+   418,
+   4105,
+   10699,
+   469,
+   4087,
+   4256,
+   339,
+   795,
+   17829,
+   28249,
+   13178,
+   8380,
+   271,
+   24313,
+   2852,
+   3362,
+   582,
+   12533,
+   1425,
+   119326,
+   1760,
+   582,
+   20855,
+   3362,
+   582,
+   53,
+   48819,
+   35312,
+   64018,
+   29,
+   1119,
+   9390,
+   4271,
+   344,
+   7451,
+   66910,
+   582,
+   46172,
+   3362,
+   28612,
+   4611,
+   3362,
+   582,
+   10325,
+   1760,
+   582,
+   68838,
+   3362,
+   28612,
+   329,
+   3362,
+   28612,
+   4611,
+   3362,
+   582,
+   4463,
+   4,
+   55695,
+   582,
+   24486,
+   3362,
+   20584,
+   329,
+   45390,
+   582,
+   102970,
+   34520,
+   3362,
+   5062,
+   33236,
+   3476,
+   74366,
+   29851,
+   1605,
+   270,
+   3554,
+   6428,
+   4105,
+   2329,
+   305,
+   10767,
+   5815,
+   8380,
+   304,
+   34756,
+   4105,
+   10699,
+   603,
+   128803,
+   61,
+   11675,
+   102630,
+   8139,
+   19,
+   2296,
+   14885,
+   64018,
+   119326,
+   304,
+   1747,
+   890,
+   3676,
+   334,
+   15,
+   1002,
+   16,
+   128804,
+   128822
+  ],
+  "expect_text": "\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"lookup_record\">\n<｜DSML｜ parameter name=\"id\" string=\"true\">A-17</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n</｜DSML｜ calls><｜end▁of▁sentence｜>",
+  "finish": "tool_calls"
+ },
+ {
+  "id": "sanity-T2",
+  "mode": "thinking",
+  "request": {
+   "messages": [
+    {
+     "role": "user",
+     "content": "[case sanity-T2]\nUse get_weather for Zagreb in celsius."
+    }
+   ],
+   "tools": [
+    {
+     "type": "function",
+     "function": {
+      "name": "get_weather",
+      "description": "Synthetic benchmark lookup; no external action is performed.",
+      "parameters": {
+       "type": "object",
+       "properties": {
+        "city": {
+         "type": "string"
+        },
+        "unit": {
+         "type": "string"
+        }
+       },
+       "required": [
+        "city",
+        "unit"
+       ],
+       "additionalProperties": false
+      }
+     }
+    }
+   ]
+  },
+  "thinking": true,
+  "text": "<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜ calls>\" block like the following:\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"$TOOL_NAME\">\n<｜DSML｜ parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜ parameter>\n...\n</｜DSML｜ invoke>\n<｜DSML｜ invoke name=\"$TOOL_NAME2\">\n...\n</｜DSML｜ invoke>\n</｜DSML｜ calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"get_weather\", \"description\": \"Synthetic benchmark lookup; no external action is performed.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}, \"unit\": {\"type\": \"string\"}}, \"required\": [\"city\", \"unit\"], \"additionalProperties\": false}}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<｜User｜>[case sanity-T2]\nUse get_weather for Zagreb in celsius.<｜Assistant｜><think>",
+  "ids": [
+   0,
+   128799,
+   77666,
+   288,
+   9504,
+   482,
+   28,
+   223,
+   2402,
+   343,
+   16801,
+   223,
+   19,
+   15,
+   1457,
+   14,
+   270,
+   4077,
+   270,
+   1990,
+   14,
+   270,
+   850,
+   15146,
+   270,
+   22805,
+   86262,
+   372,
+   27193,
+   271,
+   3476,
+   611,
+   3278,
+   304,
+   260,
+   1341,
+   294,
+   6704,
+   304,
+   1694,
+   3287,
+   270,
+   3967,
+   734,
+   3417,
+   16,
+   2042,
+   588,
+   34756,
+   6704,
+   513,
+   4985,
+   260,
+   38322,
+   128825,
+   10699,
+   48902,
+   5603,
+   1277,
+   270,
+   2502,
+   979,
+   30,
+   128825,
+   10699,
+   1018,
+   30,
+   128825,
+   34756,
+   2329,
+   84476,
+   6792,
+   5896,
+   35044,
+   3816,
+   30,
+   128825,
+   10767,
+   2329,
+   84476,
+   46630,
+   4130,
+   61729,
+   35044,
+   4,
+   3418,
+   1281,
+   11476,
+   94,
+   19836,
+   3320,
+   6,
+   46630,
+   4130,
+   61729,
+   56909,
+   1718,
+   128825,
+   10767,
+   1018,
+   7835,
+   1718,
+   128825,
+   34756,
+   1018,
+   30,
+   128825,
+   34756,
+   2329,
+   84476,
+   6792,
+   5896,
+   35044,
+   20,
+   3816,
+   7835,
+   1718,
+   128825,
+   34756,
+   1018,
+   1718,
+   128825,
+   10699,
+   4697,
+   3524,
+   8252,
+   1531,
+   366,
+   12038,
+   412,
+   344,
+   305,
+   1341,
+   3608,
+   4463,
+   1281,
+   11476,
+   4,
+   37419,
+   1884,
+   710,
+   915,
+   4815,
+   343,
+   62896,
+   14,
+   2631,
+   2065,
+   634,
+   14,
+   31939,
+   14,
+   8435,
+   754,
+   2281,
+   270,
+   1990,
+   295,
+   26639,
+   8786,
+   305,
+   1341,
+   3608,
+   4463,
+   1281,
+   19836,
+   4,
+   108526,
+   3575,
+   6892,
+   71144,
+   344,
+   22104,
+   343,
+   89944,
+   284,
+   513,
+   223,
+   128821,
+   754,
+   440,
+   74366,
+   5238,
+   782,
+   5553,
+   22805,
+   6352,
+   223,
+   128821,
+   1613,
+   128822,
+   119907,
+   1117,
+   4105,
+   10699,
+   469,
+   4087,
+   4256,
+   339,
+   13079,
+   6922,
+   14,
+   5238,
+   6578,
+   1561,
+   223,
+   128822,
+   418,
+   4105,
+   10699,
+   469,
+   4087,
+   4256,
+   339,
+   795,
+   17829,
+   28249,
+   13178,
+   8380,
+   271,
+   24313,
+   2852,
+   3362,
+   582,
+   1133,
+   65,
+   50219,
+   1760,
+   582,
+   20855,
+   3362,
+   582,
+   53,
+   48819,
+   35312,
+   64018,
+   29,
+   1119,
+   9390,
+   4271,
+   344,
+   7451,
+   66910,
+   582,
+   46172,
+   3362,
+   28612,
+   4611,
+   3362,
+   582,
+   10325,
+   1760,
+   582,
+   68838,
+   3362,
+   28612,
+   37399,
+   3362,
+   28612,
+   4611,
+   3362,
+   582,
+   4463,
+   87928,
+   582,
+   15165,
+   3362,
+   28612,
+   4611,
+   3362,
+   582,
+   4463,
+   4,
+   55695,
+   582,
+   24486,
+   3362,
+   20584,
+   37399,
+   1760,
+   582,
+   15165,
+   45390,
+   582,
+   102970,
+   34520,
+   3362,
+   5062,
+   33236,
+   3476,
+   74366,
+   29851,
+   1605,
+   270,
+   3554,
+   6428,
+   4105,
+   2329,
+   305,
+   10767,
+   5815,
+   8380,
+   304,
+   34756,
+   4105,
+   10699,
+   603,
+   128803,
+   61,
+   11675,
+   102630,
+   8139,
+   20,
+   2296,
+   14885,
+   1178,
+   65,
+   50219,
+   362,
+   121648,
+   295,
+   124281,
+   16,
+   128804,
+   128821
+  ],
+  "expect_text": "The user wants weather for Zagreb in celsius. Call the tool.</think>\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"city\" string=\"true\">Zagreb</｜DSML｜ parameter>\n<｜DSML｜ parameter name=\"unit\" string=\"true\">celsius</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n</｜DSML｜ calls><｜end▁of▁sentence｜>",
+  "finish": "tool_calls"
+ },
+ {
+  "id": "sanity-I3",
+  "mode": "thinking",
+  "request": {
+   "messages": [
+    {
+     "role": "user",
+     "content": "[case sanity-I3]\nProject: Orion. Build: 17. Return only a JSON object with keys project (string) and build (integer)."
+    }
+   ]
+  },
+  "thinking": true,
+  "text": "<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<｜User｜>[case sanity-I3]\nProject: Orion. Build: 17. Return only a JSON object with keys project (string) and build (integer).<｜Assistant｜><think>",
+  "ids": [
+   0,
+   128799,
+   77666,
+   288,
+   9504,
+   482,
+   28,
+   223,
+   2402,
+   343,
+   16801,
+   223,
+   19,
+   15,
+   1457,
+   14,
+   270,
+   4077,
+   270,
+   1990,
+   14,
+   270,
+   850,
+   15146,
+   270,
+   22805,
+   868,
+   128803,
+   61,
+   11675,
+   102630,
+   20448,
+   21,
+   2296,
+   20811,
+   28,
+   84706,
+   16,
+   23718,
+   28,
+   223,
+   1002,
+   16,
+   20870,
+   1353,
+   260,
+   26639,
+   2873,
+   418,
+   18222,
+   2775,
+   343,
+   4463,
+   11,
+   305,
+   2906,
+   343,
+   39648,
+   797,
+   128804,
+   128821
+  ],
+  "expect_text": "We need answer only JSON object keys project string build integer. Need infer from prompt: Project: Orion. Build: 17. So {\"project\":\"Orion\",\"build\":17}. Ensure only JSON.</think>{\"project\":\"Orion\",\"build\":17}<｜end▁of▁sentence｜>",
+  "finish": "stop"
+ },
+ {
+  "id": "performance-short-r1",
+  "mode": "chat",
+  "request": {
+   "messages": [
+    {
+     "role": "user",
+     "content": "[case performance-short-r1]\nExplain how a hash table finds a value, including collisions and one small example."
+    }
+   ],
+   "chat_template_kwargs": {
+    "thinking": false,
+    "enable_thinking": false
+   }
+  },
+  "thinking": false,
+  "text": "<｜begin▁of▁sentence｜><｜User｜>[case performance-short-r1]\nExplain how a hash table finds a value, including collisions and one small example.<｜Assistant｜></think>",
+  "ids": [
+   0,
+   128803,
+   61,
+   11675,
+   4197,
+   10565,
+   482,
+   9954,
+   19,
+   2296,
+   65106,
+   1192,
+   260,
+   19657,
+   4184,
+   15249,
+   260,
+   1990,
+   14,
+   2622,
+   54415,
+   305,
+   834,
+   2395,
+   2511,
+   16,
+   128804,
+   128822
+  ],
+  "expect_text": "# How a Hash Table Finds a Value\n\nA hash table stores key–value pairs and uses a **hash function** to map each key to an index in an underlying array. Lookup is fast because it jumps almost directly to the right slot instead of scanning.\n\n## The lookup process\n\n1. **Hash the key.** Compute `h = hash(key)`, then reduce it to a valid array index, e.g. `index = h % capacity`.\n2. **Go to that bucket.** The array slot at `index` holds the entry (or a chain/list of entries).\n3. **Compare keys.** If the bucket holds the exact key you asked for, return its value.\n4. **Handle collisions.** If the bucket holds a *different* key that happened to hash to the same index, keep searching within that bucket (see below).\n\n## Collisions\n\nA **collision** happens when two different keys map to the same index. Two common resolutions:\n\n- **Separate chaining:** each bucket is a small list (or tree). On lookup, walk the list comparing keys until you find a match or reach the end (key absent).\n- **Open addressing:** if the slot is occupied by a different key, probe other slots (linear probing:",
+  "finish": "length"
+ }
+]

serve/frontend.py

@@ -135,15 +135,36 @@ EFFORT = {"none": None, "off": None, "minimal": None, "disabled": None, "false":
           "low": "low", "medium": "medium", "high": "xhigh", "xhigh": "xhigh", "max": "xhigh", "maximum": "xhigh"}
 
 
+# DeepSeek-V4.1's template takes low, medium, high (its default) and max, or a whole number 1..100 (the "Reasoning
+# Effort: N" line of its prompt).  use_family("deepseek41") switches the table; the other families keep the one above.
+EFFORT_DEEPSEEK = {"none": None, "off": None, "minimal": None, "disabled": None, "false": None,
+                   "low": "low", "medium": "medium", "high": "high", "xhigh": "max", "max": "max", "maximum": "max"}
+_EFFORT_QWEN = dict(EFFORT)
+_NUMERIC_EFFORT = False
+
+
+def use_family(family: str | None) -> None:
+    """The reasoning-effort words of the model family this server runs (module state, like ENV_TO_USER)."""
+    global _NUMERIC_EFFORT
+    EFFORT.clear()
+    EFFORT.update(EFFORT_DEEPSEEK if family == "deepseek41" else _EFFORT_QWEN)
+    _NUMERIC_EFFORT = family == "deepseek41"
+
+
 def effort_kwargs(value) -> dict:
     """A reasoning effort as given by a client -> the template's kwargs.  Unknown values are a 400, not a crash."""
     if value is None or value == "":
         return {}
     if value is False:
         return {"enable_thinking": False}
+    if _NUMERIC_EFFORT and isinstance(value, int) and not isinstance(value, bool):
+        if not 1 <= value <= 100:
+            raise ValueError(f"unknown reasoning effort {value!r}: use none, low, medium, high, max or 1..100")
+        return {"reasoning_effort": value}
     key = str(value).strip().lower()
     if key not in EFFORT:
-        raise ValueError(f"unknown reasoning effort {value!r}: use none, low, medium or high")
+        raise ValueError(f"unknown reasoning effort {value!r}: use none, low, medium or high"
+                         + (", max or 1..100" if _NUMERIC_EFFORT else ""))
     level = EFFORT[key]
     return {"enable_thinking": False} if level is None else {"reasoning_effort": level}
 
@@ -154,7 +175,7 @@ def budget_effort(tokens) -> dict:
         n = int(tokens)
     except (TypeError, ValueError):
         return {}
-    return {"reasoning_effort": "low" if n < 2048 else "medium" if n < 8192 else "xhigh"}
+    return {"reasoning_effort": "low" if n < 2048 else "medium" if n < 8192 else EFFORT["high"]}
 
 
 def _has_image(content) -> bool:
@@ -927,6 +948,12 @@ class OutputParser:
         self.batch = False           # the call being finished is followed by another in the same wrapper
         self._reset_scan()
 
+    @property
+    def clean(self) -> bool:
+        """Nothing is held back: a clean point to stop the thinking at (a parser that rewrites the model's text, as
+        serve/dsml.py's does, also counts what it holds)."""
+        return not self.buf
+
     def _ok_at(self, p: int) -> bool:
         """self.buf[p] would open a call: at the start of a line, outside a code fence and inline code."""
         snap = (self.fence, self.line, self.ticks)

serve/server.py

@@ -201,7 +201,9 @@ VISION_START = "<|vision_start|>"
 # sets it; 0 turns it off.
 REPEAT_STOP_TOKENS = 256
 ANSWER_ROOM_MIN_CAP = 1024    # 0.1.45: from this max_tokens on, thinking without a budget is closed early enough to leave room to answer
-ANSWER_RESERVE_MIN = 512      # #984: the tokens kept for the answer when a thinking budget is cut down to fit max_tokens
+DS41_DEFAULT_BUDGET = 32768   # DeepSeek-V4.1 only: the thinking budget when neither the request nor the config names one
+DS41_SMALL_CAP_MIN = 48       # DeepSeek-V4.1 only: from this max_tokens up to ANSWER_ROOM_MIN_CAP, thinking is capped at half of it
+ANSWER_RESERVE_MIN = 512     # #984: the tokens kept for the answer when a thinking budget is cut down to fit max_tokens
 # #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)
 # #1053 (opt-in "reasoning_close_retry": true): a reply that ends on its stop token still inside <think>, with no answer
 # and no call, is continued once with the thinking closed (the same way the budget's wrap-up closes it)
@@ -3038,6 +3040,18 @@ def layer_split_value(cfg: dict) -> str:
     return ",".join(str(x) for x in vals)
 
 
+def apply_ready_timeout(cfg: dict) -> None:
+    """The config's "engine_ready_s": how long a start may take before READY (default ENGINE_READY_S, 900).  A model of 300+ GB read
+    from a slow disk (DeepSeek-V4.1-Flash: setup writes 3600) needs more.  ValueError for anything but a number > 0."""
+    global ENGINE_READY_S
+    v = cfg.get("engine_ready_s")
+    if v is None:
+        return
+    if isinstance(v, bool) or not isinstance(v, (int, float)) or not v > 0:
+        raise ValueError(f'"engine_ready_s" must be a number of seconds above 0, not {v!r}')
+    ENGINE_READY_S = float(v)
+
+
 def layer_split_of(cfg: dict) -> bool:
     """Several GPUs in the config are a layer split, unless the args put the later card(s) to another use: with
     --peer-device the second card is an expert-cache tier, and the engine refuses that beside --layer-split (#665)."""
@@ -3465,6 +3479,23 @@ def hip_auto_devices(cfg: dict, env: dict) -> list[int] | None:
     return visible
 
 
+DS41_SLOT_KEEP_DEFAULT = "0"      # the engine keeps a displaced conversation in RAM and in the slot folder (see below)
+DS41_SLOT_GB_DEFAULT = "2"        # the folder's cap in GB (a 6.5K-token conversation is about 17 MB)
+
+
+def deepseek_slot_env(env: dict, cfg: dict) -> None:
+    """DeepSeek-V4.1 only.  The engine sets a conversation aside when another request takes the model and takes it back
+    at its checkpoints (agents and sub-agents that take turns do not read their prompts again).  STRATA_DS4_SLOT_KEEP
+    switches it (the default above; a config's env or the shell wins); the folder is `ds4-slots` next to the run log, in
+    Strata's data folder (STRATA_DS4_SLOT_DIR names another), and the engine caps it (STRATA_DS4_SLOT_GB, 2 here, 16 in the engine) and
+    removes what is older than a week or not this model's.  Off by default (0.2.4); STRATA_DS4_SLOT_KEEP=1 turns it on."""
+    env.setdefault("STRATA_DS4_SLOT_KEEP", DS41_SLOT_KEEP_DEFAULT)
+    env.setdefault("STRATA_DS4_SLOT_GB", DS41_SLOT_GB_DEFAULT)
+    log = cfg.get("log")
+    if "STRATA_DS4_SLOT_DIR" not in env and log:
+        env["STRATA_DS4_SLOT_DIR"] = str(Path(log).parent / "ds4-slots")
+
+
 def child_env(cfg: dict, *, offline: bool = False) -> dict:
     """The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
     compiled it) first on the library search path."""
@@ -3479,6 +3510,8 @@ def child_env(cfg: dict, *, offline: bool = False) -> dict:
         env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in ordered_gpus(cfg))
     for k, v in (cfg.get("env") or {}).items():      # engine settings the config carries (AMD: the GEMM tuning table)
         env[str(k)] = str(v)
+    if (cfg.get("arch") or "") == "deepseek41":      # (only this engine reads these; Qwen's environment is untouched)
+        deepseek_slot_env(env, cfg)
     # Strict offline reaches children through the environment, because a tool subprocess or a model
     # loader is code this server does not own. Applied last so a profile's own `env` cannot remove
     # the policy by naming the same variable.
@@ -3881,6 +3914,7 @@ class Service:
         self.vision_held = False                         # an elastic encoder holds VRAM the expert cache gave it
         self.last_image_at = 0.0                         # when an image turn last ended (the elastic encoder's idle)
         self.reasoning_loop_recovery = False
+        self.reasoning_budget_configured = False         # the config named reasoning_budget_tokens (0 included): no family default
         self.reasoning_budget_tokens = 0                 # #123: the config's default thinking budget (0: none)
         self.repeat_stop_tokens = REPEAT_STOP_TOKENS     # #606: one token this many times in a row ends a reply (0: off)
         self.anthropic_think_unasked = True               # #278: "anthropic_thinking": "on_request" -> False
@@ -3889,6 +3923,8 @@ class Service:
         self.restore_after_load = None                    # a session file an image job saved before it unloaded
         self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
                             tokenizer.encode("<|endoftext|>", parse_special=True))
+        self.family = None                              # the model family ("deepseek41"): set_family() below
+        self.parser_cls = OutputParser                  # reads the model's output (reasoning, content, tool calls)
         # #567: a prompt re-encodes only what follows the last special token it shares with a recent prompt (the
         # same ids as a full encode: tools/strata_tokenizer.py PromptEncoder).  Tokenizers without resume points
         # encode in full.
@@ -3901,6 +3937,25 @@ class Service:
             self.prompts = PromptEncoder(tokenizer, keep=slots) if slots is not None and slots > 0 \
                 else PromptEncoder(tokenizer)
 
+    def set_family(self, family: str | None) -> None:
+        """A family whose text differs from the Qwen form this server was written for (DeepSeek-V4.1: its own end
+        token, DSML tool calls, reasoning-effort words).  The others keep the defaults."""
+        self.family = family
+        if family == "deepseek41":
+            from serve import dsml
+            eos = (getattr(self.tok, "special_ids", None) or {}).get("eos", 1)
+            self.stop_ids = {int(eos)}
+            self.parser_cls = dsml.DsmlOutputParser
+            self._force_text = dsml.to_dsml_force
+        import serve.frontend as _frontend
+        _frontend.use_family(family)
+
+    _force_text = None
+
+    def force_text(self, force):
+        """forced_call's opening of the call the reply must make, in this model's spelling."""
+        return self._force_text(force) if self._force_text is not None else force
+
     def loaded(self) -> bool:
         return not hasattr(self.engine, "alive") or self.engine.alive()
 
@@ -4107,6 +4162,8 @@ class Service:
             value = req.get("thinking_budget_tokens")
         if value is None:
             value = self.reasoning_budget_tokens
+            if not value and self.family == "deepseek41" and not self.reasoning_budget_configured:
+                value = DS41_DEFAULT_BUDGET          # DeepSeek-V4.1 only, as Maya ships it: no budget anywhere = this one
         if isinstance(value, float) and value.is_integer():
             value = int(value)
         if isinstance(value, bool) or not isinstance(value, int):
@@ -4861,10 +4918,30 @@ class Service:
             # (an encoder restart, the cache's eviction) with other bits still matches its checkpoints
             self.embeddings.keys = [Path(p).stem[:16] for p, _ in encoded]
             write_temporary(combined, [p for p, _ in encoded])
+        if self.family == "deepseek41" and req is not None and req.get("strata_prefix") is None and not images:
+            lead = self.deepseek_shared_start(messages)
+            if lead:
+                req["strata_prefix"] = {"messages": lead}
         if req is not None and req.get("strata_prefix") is not None:
             req["strata_prefix"] = self.resolve_prefix(req["strata_prefix"], messages, tools, kwargs, ids, bool(images))
         return ids, kwargs.get("enable_thinking", True) is not False, max_new
 
+    @staticmethod
+    def deepseek_shared_start(messages) -> int:
+        """DeepSeek-V4.1 only: how many leading messages are the system prompt (its tools are rendered with them).  An
+        agent and its sub-agents that work behind the same system prompt and tools share exactly this start, so the
+        engine keeps a checkpoint of its state there (pin=N): the next agent behind the same start goes back to it
+        instead of reading it again.  0: none (off unless STRATA_DS4_AUTO_PIN=1, no system message, or nothing after it)."""
+        if os.environ.get("STRATA_DS4_AUTO_PIN", "0") != "1":
+            return 0
+        n = 0
+        for m in messages:
+            if isinstance(m, dict) and m.get("role") in ("system", "developer"):
+                n += 1
+            else:
+                break
+        return n if 0 < n < len(messages) else 0
+
     PREFIX_END = "\u0001strata-prefix-end\u0001"          # marks where a message's shared part ends in the rendered prompt
 
     def resolve_prefix(self, spec, messages, tools, kwargs, ids, has_images=False) -> dict:
@@ -5019,6 +5096,14 @@ class Service:
             budget = max_new - min(max(ANSWER_RESERVE_MIN, max_new // 4), max_new // 2)
             if max_new <= 32768: print(f"[strata] thinking limited to {budget} of max_tokens {max_new} so the answer has room "
                   "(\"answer_room\": false in the config turns this off)", flush=True)
+        if (thinking and not budget and self.answer_room and self.family == "deepseek41"
+                and max_new and DS41_SMALL_CAP_MIN <= max_new < ANSWER_ROOM_MIN_CAP):
+            # DeepSeek-V4.1 only: a short max_tokens (/v1/messages and /v1/chat/completions with 200-600) was spent
+            # entirely on the thinking, so no text block ever arrived.  With no budget of the client's own, the thinking
+            # is capped at half of max_tokens; then the wrap-up, </think> and the answer follow in the other half.
+            budget = max_new // 2
+            print(f"[strata] thinking limited to {budget} of max_tokens {max_new} so the answer has room "
+                  "(\"answer_room\": false in the config turns this off)", flush=True)
         if budget and max_new and max_new > 0:
             # #984: a budget at or near max_tokens never fires before the reply is cut off while still thinking, so
             # it has no answer.  Close the thinking early enough to leave room for one.
@@ -5027,8 +5112,8 @@ class Service:
                 print(f"[strata] thinking capped at {max_new - reserve} of max_tokens {max_new} to leave room for "
                       f"the answer (budget {budget})", flush=True)
                 budget = max(1, max_new - reserve)
-        parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery,
-                              declared_only=declared_only)
+        parser = self.parser_cls(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery,
+                                 declared_only=declared_only)
         detok, n, finish = Detokenizer(self.tok), 0, "length"
         run_tok, run_len, repeated = None, 0, False     # #606: the current run of one repeated token
         thinking_n = 0                                  # tokens written while thinking (Responses' reasoning_tokens)
@@ -5253,13 +5338,13 @@ class Service:
                                 if budget and parser.state == "reasoning":
                                     thought += 1
                                     # at a clean point: no tag held back, no character split across tokens
-                                    if thought >= budget and not parser.buf and not detok.pending():
+                                    if thought >= budget and parser.clean and not detok.pending():
                                         wrap = True
                                         break
                                 if force:
                                     if any(ev.kind in ("tool_start", "tool_call") for ev in evs):
                                         force = None    # the model is writing a call of its own
-                                    elif parser.state == "content" and not parser.buf and not detok.pending():
+                                    elif parser.state == "content" and parser.clean and not detok.pending():
                                         opens = True
                                         break
                         except EngineNonFinite:
@@ -5470,6 +5555,7 @@ class Service:
                                       "strata-<model>.json for every request) leaves room to answer", flush=True)
                             if os.environ.get("STRATA_DEBUG") and raw_ids:
                                 print(f"[strata] raw: {self.tok.decode(raw_ids)!r}", flush=True)
+                                print(f"[strata] raw ids: {list(raw_ids)}", flush=True)
                         st["busy"] = False
                         st.pop("tail", None)                     # #212: the answer's end is not kept once it is done
                         st.pop("tool", None)
@@ -5839,6 +5925,8 @@ def openai_collect(chunks) -> dict:
     if last["choices"][0]["finish_reason"] != "tool_calls":
         calls = [c for c in calls if _is_json(c["function"]["arguments"])]   # a call the output ended inside (#211)
     msg = {"role": "assistant", "content": "".join(content) or None}
+    if "".join(reasoning).strip() and "".join(reasoning).strip() == "".join(content).strip():
+        reasoning = []              # a turn with no </think> (serve/dsml.py): the text is the answer; it is not also reasoning
     if "".join(reasoning):
         msg["reasoning_content"] = "".join(reasoning)
     if calls:
@@ -5977,6 +6065,11 @@ def anthropic_collect(events) -> dict:
             msg["stop_reason"] = e["delta"]["stop_reason"]
             msg["stop_sequence"] = e["delta"].get("stop_sequence")
             msg["usage"].update(e["usage"])
+    for i in range(len(blocks) - 1):    # a turn with no </think> (serve/dsml.py): the text is the answer; it is not also a thinking block
+        a, b = blocks[i], blocks[i + 1]
+        if a.get("type") == "thinking" and b.get("type") == "text" and a.get("thinking", "").strip() == b.get("text", "").strip():
+            del blocks[i]
+            break
     msg["content"] = blocks
     return msg
 
@@ -7046,7 +7139,7 @@ def make_handler(svc: Service):
             self._no_local_images(messages)
             if tool_choice_of(req.get("tool_choice"))[0] == "none":   # as the Responses route: no tools are offered
                 tools = None
-            force = forced_call(req.get("tool_choice"), tools)      # a bad value is a 400 before anything is sent
+            force = svc.force_text(forced_call(req.get("tool_choice"), tools))      # a bad value is a 400 before anything is sent
             messages, validator = prepare_format(req.get("response_format"), messages)
             if validator is not None and (tools or req.get("strata_mcp")):
                 raise ValueError("structured response_format with tools/MCP is not supported")
@@ -7298,7 +7391,7 @@ def make_handler(svc: Service):
             self._no_local_images(messages)
             if tool_choice_of(req.get("tool_choice"))[0] == "none":   # Anthropic's {"type": "none"}: no tools offered
                 tools = None
-            force = forced_call(req.get("tool_choice"), tools)        # "any" / {"type": "tool", "name": N}
+            force = svc.force_text(forced_call(req.get("tool_choice"), tools))        # "any" / {"type": "tool", "name": N}
             max_new = int(req.get("max_tokens") or 0)                  # 0/-1: the rest of the context
             svc.reasoning_budget(req)                         # a bad value is a 400 before anything is sent
             stop_strings(req)                                 # ... and so is a bad stop / stop_sequences
@@ -7951,7 +8044,8 @@ def check_request_sampling(req) -> None:
 
 
 # the API's model name per model family (the GGUF architecture, "arch" in the config) when the config names none
-FAMILY_MODEL_NAMES = {"qwen4exp": "qwen3.8-flash-next", "qwen35moe": "qwen3.6-35b-a3b"}
+FAMILY_MODEL_NAMES = {"qwen4exp": "qwen3.8-flash-next", "qwen35moe": "qwen3.6-35b-a3b",
+                      "deepseek41": "deepseek-v4.1-flash"}
 
 
 # The sampling each model card recommends ("Best Practices"), used when neither the request, the Chat settings nor the
@@ -7964,6 +8058,8 @@ MODEL_CARD_SAMPLING = {
     # Qwen3.6-35B-A3B: the general-task sets (the card's precise-coding set is temperature 0.6, presence_penalty 0)
     "qwen35moe": {"thinking": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "presence_penalty": 1.5},
                   "instruct": {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "presence_penalty": 1.5}},
+    # DeepSeek-V4.1-Flash: temperature 1.0, top_p 1.0 for local use (thinking or not), no penalties
+    "deepseek41": {"thinking": {"temperature": 1.0, "top_p": 1.0}, "instruct": {"temperature": 1.0, "top_p": 1.0}},
 }
 
 
@@ -8224,7 +8320,12 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
             tokens[i] = t
         merges = (tpath / "merges.txt").read_text(encoding="utf-8").split("\n")
         types = json.loads((tpath / "token_type.json").read_text())
-        tok = ST.Tokenizer(tokens, merges, types)
+        tcfg = {}
+        try:                                            # the pack's pre-tokenizer name and special ids (DeepSeek's differ)
+            tcfg = json.loads((tpath / "tokenizer.json").read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            pass
+        tok = ST.Tokenizer(tokens, merges, types, tcfg.get("pre") or "qwen35", tcfg.get("special_ids") or None)
     hub = hub_from_config(cfg, a.mcp_config)            # before the minutes of loading: a bad entry stops here
     if a.offline and (hub is not None or cfg.get("before_load") or a.before_load):
         ap.error("--offline requires MCP servers and before_load commands to be disabled in this profile")
@@ -8298,6 +8399,7 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
             exe = cfg["exe"] if os.path.isabs(cfg["exe"]) else os.path.abspath(os.path.join(cfg.get("cwd") or ".", cfg["exe"]))
         try:
             silence = engine_silence_s(cfg)             # #481: checked before the (minutes-long) start
+            apply_ready_timeout(cfg)
         except ValueError as e:
             raise SystemExit(f"[strata] config {e}")
         try:
@@ -8328,6 +8430,12 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
             delay_s=getattr(a, "script_delay", 0.0) or 0.0), None, {}
     # the model's own chat template (exported with its tokenizer), else the original model's
     tpl = tpath / "chat_template.jinja"
+    if (cfg.get("arch") or "") == "deepseek41":
+        from serve import dsml
+        tpl, swapped = dsml.pick_template(tpl)
+        if swapped:
+            print("[strata] this pack's chat template spells the tool-call block differently from the V4.1 model's; "
+                  "using the V4.1 template so tool calls come back as real calls", flush=True)
     svc = Service(engine, tok, ChatTemplate(tpl if tpl.exists() else ROOT / "serve/chat_template.jinja"),
                   model_name=default_model_name(cfg), vision=vision,
                   sampling_defaults=sampling_defaults,
@@ -8338,6 +8446,7 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
     svc.queue_timeout_s = float(queue_timeout) if queue_timeout is not None else None
     if a.engine == "strata":                            # the model card's sampling when nothing else is said (see MODEL_CARD_SAMPLING)
         svc.model_arch = cfg.get("arch") or "qwen4exp"
+        svc.set_family(svc.model_arch if svc.model_arch == "deepseek41" else None)
         requested = cfg.get("parallel")
         if requested is None:
             args = cfg.get("args", [])
@@ -8456,6 +8565,7 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
     svc.repeat_stop_tokens = rs
     svc.effort_end = bool(effort_end)                   # #458: "effort_position": "end" with an engine that has it
     if cfg.get("reasoning_budget_tokens") is not None:  # #123: a default thinking budget for every request
+        svc.reasoning_budget_configured = True
         try:
             svc.reasoning_budget_tokens = cfg["reasoning_budget_tokens"]
             budget = svc.reasoning_budget({})

serve/test_dsv41.py

@@ -0,0 +1,763 @@
+"""DeepSeek-V4.1 through the server: its tokenizer, its chat template, the DSML tool calls, the effort words, the end
+token and the model-card defaults, over both APIs against a fake engine (no GPU, no pack, no model).
+
+    python -m unittest serve.test_dsv41 -v
+
+STRATA_DSV41_PACK=<dir> (the pack's tokenizer folder: vocab.json, merges.txt, token_type.json, tokenizer.json) adds
+the check of the real tokenizer against the ids llama.cpp (PR 30817 b72050d0) made of the same rendered prompts.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import unittest
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "tools"))
+import strata_tokenizer as ST  # noqa: E402
+from serve import frontend, server  # noqa: E402
+from serve.dsml import DsmlOutputParser, DsmlTranslator, to_dsml_force  # noqa: E402
+from serve.frontend import ChatTemplate, OutputParser, openai_to_messages  # noqa: E402
+from serve.server import (FAMILY_MODEL_NAMES, MockEngine, REASONING_WRAP_UP, Service, default_model_name, model_card_sampling,  # noqa: E402
+                          serve)
+
+FIX = ROOT / "serve" / "fixtures" / "deepseek41"
+TEMPLATE = FIX / "chat_template.jinja"
+GOLDEN = json.loads((FIX / "golden.json").read_text(encoding="utf-8"))
+
+BOS, EOS_TEXT = "<｜begin▁of▁sentence｜>", "<｜end▁of▁sentence｜>"
+SPECIALS = [(BOS, 3), (EOS_TEXT, 3), ("<｜User｜>", 3), ("<｜Assistant｜>", 3), ("<｜System｜>", 3),
+            ("<｜latest_reminder｜>", 3), ("<think>", 4), ("</think>", 4), ("｜DSML｜", 4), ("<｜deepseek_image｜>", 3)]
+EOS = 1
+
+
+def mini_tokenizer() -> ST.Tokenizer:
+    """The real Tokenizer class over a tiny vocabulary: DeepSeek's special tokens, then the 256 byte tokens, no merges."""
+    b2u = ST.bytes_to_unicode()
+    tokens = [s for s, _ in SPECIALS] + [b2u[b] for b in range(256)]
+    types = [t for _, t in SPECIALS] + [1] * 256
+    return ST.Tokenizer(tokens, [], types, "deepseek3", {"bos": 0, "eos": EOS})
+
+
+class DsEngine(MockEngine):
+    """Replays scripted completions as ids, then DeepSeek's end token; keeps what it was asked."""
+
+    def __init__(self, tok, script, max_context=8192):
+        super().__init__(tok, script, max_context=max_context)
+        self.scripts = [tok.encode(x, parse_special=True) + [EOS] for x in ([script] if isinstance(script, str) else script)]
+        self.script = self.scripts[0]
+        self.last_sampling = None
+        self.last = {}
+
+    def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+        self.last_sampling = dict(sampling or {})
+        got = 0
+        try:
+            for t in super().generate(ids, max_new, sampling, cancel, embeddings):
+                got += 1
+                yield t
+        finally:      # (the server stops reading at the end token: the engine's DONE comes while it is closed)
+            self.last = {"generated": got, "prompt_tokens": len(ids), "prompt_ms": 50.0, "decode_ms": got * 10.0,
+                         "finish": "stop"}
+
+
+def call(*invokes, string="true") -> str:
+    """The text DeepSeek writes for calls: [(name, [(param, value, is_string)...])...]."""
+    out = ["\n\n<｜DSML｜ calls>"]
+    for name, params in invokes:
+        out.append(f'<｜DSML｜ invoke name="{name}">')
+        for p, v, is_str in params:
+            out.append(f'<｜DSML｜ parameter name="{p}" string="{"true" if is_str else "false"}">{v}</｜DSML｜ parameter>')
+        out.append("</｜DSML｜ invoke>")
+    out.append("</｜DSML｜ calls>")
+    return "\n".join(out)
+
+
+WEATHER = {"type": "function", "function": {"name": "get_weather", "description": "weather",
+                                              "parameters": {"type": "object", "properties": {
+                                                  "city": {"type": "string"}, "days": {"type": "integer"}},
+                                                  "required": ["city"]}}}
+
+
+class Pre(unittest.TestCase):
+    """The pre-tokenizer is llama.cpp's DEEPSEEK3_LLM list: a sequence of splits, not one pattern."""
+
+    def split(self, text):
+        return ST._sequence_findall([ST.regex.compile(p) for p in ST.DEEPSEEK3_PATTERNS], text)
+
+    def test_digits_in_runs_of_three(self):
+        self.assertEqual(self.split("a12345678b"), ["a", "123", "456", "78", "b"])
+
+    def test_cjk_runs(self):
+        self.assertEqual(self.split("你好, world"), ["你好", ",", " world"])
+
+    def test_punctuation_then_letters_and_whitespace(self):
+        self.assertEqual(self.split("x_y.z  \n\nq"), ["x", "_y", ".z", "  \n\n", "q"])
+
+    def test_every_character_is_kept(self):
+        text = "Hello, 世界! 123\t\r\n  é \U0001f600 ｜DSML｜"
+        self.assertEqual("".join(self.split(text)), text)
+
+
+class Tokenizer(unittest.TestCase):
+    def test_a_special_token_decodes_to_its_text(self):
+        tok = mini_tokenizer()
+        ids = tok.encode("<｜User｜>hi｜DSML｜", parse_special=True)
+        self.assertEqual(ids[0], 2)
+        self.assertEqual(tok.decode(ids), "<｜User｜>hi｜DSML｜")
+        self.assertEqual(tok.token_bytes(8), "｜DSML｜".encode())
+
+    def test_the_dsml_token_is_matched_without_parse_special_and_a_control_token_is_not(self):
+        tok = mini_tokenizer()
+        self.assertEqual(tok.encode("｜DSML｜").__len__(), 1)
+        self.assertGreater(len(tok.encode("<｜User｜>")), 1)
+
+    def test_qwen_tokens_are_unchanged(self):
+        b2u = ST.bytes_to_unicode()
+        tok = ST.Tokenizer([b2u[b] for b in range(256)] + ["<|im_end|>"], [], [1] * 256 + [3])
+        self.assertEqual(tok.token_bytes(256), b"<|im_end|>")
+        self.assertEqual(tok.encode("a<|im_end|>", parse_special=True), [97, 256])
+
+    @unittest.skipUnless(os.environ.get("STRATA_DSV41_PACK"), "STRATA_DSV41_PACK (the pack's tokenizer folder) not set")
+    def test_the_real_tokenizer_makes_the_ids_llama_cpp_made(self):
+        d = Path(os.environ["STRATA_DSV41_PACK"])
+        vocab = json.loads((d / "vocab.json").read_text(encoding="utf-8"))
+        tokens = [None] * len(vocab)
+        for t, i in vocab.items():
+            tokens[i] = t
+        cfg = json.loads((d / "tokenizer.json").read_text())
+        tok = ST.Tokenizer(tokens, (d / "merges.txt").read_text(encoding="utf-8").split("\n"),
+                           json.loads((d / "token_type.json").read_text()), cfg["pre"], cfg["special_ids"])
+        for case in GOLDEN:
+            self.assertEqual(tok.encode(case["text"], parse_special=True), case["ids"], case["id"] + " " + case["mode"])
+
+
+class Template(unittest.TestCase):
+    """The GGUF's own chat template (the same bytes as DeepSeek's encoding.py on the cases checked), through the
+    server's request reader and renderer: the text llama.cpp tokenized for the reference run."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tpl = ChatTemplate(TEMPLATE)
+
+    def test_the_golden_prompts(self):
+        for case in GOLDEN:
+            messages, tools, kw = openai_to_messages(case["request"])
+            self.assertEqual(self.tpl.render(messages, tools=tools, **kw), case["text"], case["id"] + " " + case["mode"])
+
+    def test_thinking_is_the_default(self):
+        text = self.tpl.render([{"role": "user", "content": "hi"}])
+        self.assertTrue(text.endswith("<｜Assistant｜><think>"))
+        text = self.tpl.render([{"role": "user", "content": "hi"}], enable_thinking=False)
+        self.assertTrue(text.endswith("<｜Assistant｜></think>"))
+
+    def test_a_tool_call_and_its_result_render_in_dsml(self):
+        msgs = [{"role": "user", "content": "weather?"},
+                {"role": "assistant", "content": "", "reasoning_content": "call it",
+                 "tool_calls": [{"function": {"name": "get_weather", "arguments": {"city": "Zagreb", "days": 3}}}]},
+                {"role": "tool", "content": "sunny"}]
+        text = self.tpl.render(msgs, tools=[WEATHER["function"]], enable_thinking=False)
+        self.assertIn('<｜DSML｜ invoke name="get_weather">', text)
+        self.assertIn('<｜DSML｜ parameter name="city" string="true">Zagreb</｜DSML｜ parameter>', text)
+        self.assertIn('<｜DSML｜ parameter name="days" string="false">3</｜DSML｜ parameter>', text)
+        self.assertIn("<tool_result>sunny</tool_result>", text)
+
+
+class Translator(unittest.TestCase):
+    RAW = call(("get_weather", [("city", "Zagreb", True), ("days", "3", False)]), ("ping", []))
+
+    def all_at_once(self, text):
+        t = DsmlTranslator()
+        return t.feed(text) + t.flush()
+
+    def test_a_call_becomes_the_qwen_form(self):
+        got = self.all_at_once("Sure." + self.RAW)
+        self.assertEqual(got, "Sure.\n\n<tool_call>\n<function=get_weather>\n<parameter=city>\nZagreb\n</parameter>\n"
+                              "<parameter=days>\n3\n</parameter>\n</function>\n</tool_call>\n"
+                              "<tool_call>\n<function=ping>\n</function>\n</tool_call>\n")
+
+    def test_the_split_of_the_stream_changes_nothing(self):
+        want = self.all_at_once("a < b and 1<2" + self.RAW + "tail")
+        for step in (1, 2, 3, 5, 8, 13):
+            t = DsmlTranslator()
+            text = "a < b and 1<2" + self.RAW + "tail"
+            got = "".join(t.feed(text[i:i + step]) for i in range(0, len(text), step)) + t.flush()
+            self.assertEqual(got, want, step)
+
+    def test_text_with_angle_brackets_passes(self):
+        text = "x <div> y </｜DSML｜ nope> <｜User｜> <｜DSML｜ unknown>"
+        self.assertEqual(self.all_at_once(text), text)
+
+    def test_the_end_of_the_output_inside_a_tag_is_text(self):
+        t = DsmlTranslator()
+        got = t.feed("ok <｜DSML｜ inv") + t.flush()
+        self.assertEqual(got, "ok <｜DSML｜ inv")
+
+    def test_a_value_that_looks_like_a_tag(self):
+        raw = call(("write", [("text", "<b>bold</b> and < 3", True)]))
+        self.assertIn("<parameter=text>\n<b>bold</b> and < 3\n</parameter>", self.all_at_once(raw))
+
+    def test_a_missing_closing_tag_is_closed(self):
+        raw = '<｜DSML｜ calls>\n<｜DSML｜ invoke name="f">\n<｜DSML｜ parameter name="a" string="true">x</｜DSML｜ invoke>'
+        self.assertEqual(self.all_at_once(raw),
+                         "<tool_call>\n<function=f>\n<parameter=a>\nx\n</parameter>\n</function>\n</tool_call>\n")
+
+
+class Parser(unittest.TestCase):
+    TOOLS = [WEATHER["function"], {"name": "lookup", "description": "", "parameters": {
+        "type": "object", "properties": {"id": {}}}}]
+
+    def run_parser(self, text, thinking=True, step=7, cls=DsmlOutputParser):
+        p = cls(thinking=thinking, tools=self.TOOLS, stream_tools=True)
+        evs = []
+        for i in range(0, len(text), step):
+            evs += p.feed(text[i:i + step])
+        evs += p.finish("stop")
+        return evs
+
+    def test_reasoning_then_two_calls(self):
+        raw = call(("get_weather", [("city", "Zagreb", True), ("days", "3", False)]), ("lookup", [("id", "123", True)]))
+        for step in (1, 4, 1000):
+            evs = self.run_parser("Think < ok.</think>" + raw, step=step)
+            self.assertEqual("".join(e.text for e in evs if e.kind == "reasoning"), "Think < ok.")
+            calls = [e.call for e in evs if e.kind == "tool_call"]
+            self.assertEqual([(c.name, c.arguments) for c in calls],
+                             [("get_weather", {"city": "Zagreb", "days": 3}), ("lookup", {"id": "123"})])
+            self.assertEqual("".join(e.text for e in evs if e.kind == "tool_args"),
+                             '{"city":"Zagreb","days":3}{"id":"123"}')
+
+    def test_a_string_value_stays_a_string_even_when_the_schema_says_nothing(self):
+        evs = self.run_parser(call(("lookup", [("id", "007", True)])), thinking=False)
+        self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": "007"}])
+        evs = self.run_parser(call(("lookup", [("id", "7", False)])), thinking=False)
+        self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": 7}])
+
+    def test_a_non_string_value_is_json(self):
+        evs = self.run_parser(call(("lookup", [("id", '{"a": [1, 2]}', False)])), thinking=False)
+        self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": {"a": [1, 2]}}])
+
+    def test_content_before_the_call_and_none_after(self):
+        evs = self.run_parser("Checking.\n\n" + call(("lookup", [("id", "x", True)]))[2:], thinking=False)
+        self.assertEqual("".join(e.text for e in evs if e.kind == "content").strip(), "Checking.")
+        self.assertEqual(sum(e.kind == "tool_call" for e in evs), 1)
+
+    def test_a_call_cut_off_by_the_length_limit_is_not_a_call(self):
+        raw = call(("get_weather", [("city", "Zagr", True)]))
+        raw = raw[:raw.index("Zagr") + 4]
+        p = DsmlOutputParser(thinking=False, tools=self.TOOLS, stream_tools=True)
+        evs = p.feed(raw) + p.finish("length")
+        self.assertEqual(sum(e.kind == "tool_call" for e in evs), 0)
+
+    def test_the_plain_parser_is_unchanged(self):
+        p = OutputParser(thinking=False, tools=self.TOOLS)
+        evs = p.feed("<tool_call>\n<function=lookup>\n<parameter=id>\n5\n</parameter>\n</function>\n</tool_call>")
+        self.assertEqual([e.call.arguments for e in evs if e.kind == "tool_call"], [{"id": 5}])
+        self.assertTrue(p.clean)
+
+    def test_clean_counts_a_held_tag(self):
+        p = DsmlOutputParser(thinking=False, tools=self.TOOLS)
+        p.feed("hello <｜DS")
+        self.assertFalse(p.clean)
+        p.feed("ML｜ calls>")
+        self.assertTrue(p.clean)
+
+    def test_forced_opening(self):
+        self.assertEqual(to_dsml_force("<tool_call>\n<function=get_weather>\n"),
+                         '<｜DSML｜ calls>\n<｜DSML｜ invoke name="get_weather">\n')
+        self.assertEqual(to_dsml_force("<tool_call>\n<function="), '<｜DSML｜ calls>\n<｜DSML｜ invoke name="')
+        self.assertIsNone(to_dsml_force(None))
+        # the opening and the model's continuation are one call
+        p = DsmlOutputParser(thinking=False, tools=self.TOOLS, stream_tools=True)
+        evs = p.feed(to_dsml_force("<tool_call>\n<function=get_weather>\n"))
+        evs += p.feed('<｜DSML｜ parameter name="city" string="true">Rome</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n</｜DSML｜ calls>')
+        evs += p.finish("stop")
+        self.assertEqual([(e.call.name, e.call.arguments) for e in evs if e.kind == "tool_call"],
+                         [("get_weather", {"city": "Rome"})])
+
+
+class ReadyTimeout(unittest.TestCase):
+    """A 300+ GB model read from a slow disk needs more than the default 900 s before READY: the config's "engine_ready_s"."""
+
+    def test_the_config_sets_it_and_a_bad_value_is_said(self):
+        from serve import server
+        before = server.ENGINE_READY_S
+        self.addCleanup(setattr, server, "ENGINE_READY_S", before)
+        server.apply_ready_timeout({})
+        self.assertEqual(server.ENGINE_READY_S, before)
+        server.apply_ready_timeout({"engine_ready_s": 3600})
+        self.assertEqual(server.ENGINE_READY_S, 3600.0)
+        for bad in (0, -5, "soon", True):
+            with self.assertRaises(ValueError):
+                server.apply_ready_timeout({"engine_ready_s": bad})
+
+
+class ApiBase(unittest.TestCase):
+    """Both APIs against a fake engine that replays scripted DeepSeek text (the helpers; Api has the tests)."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tok = mini_tokenizer()
+
+    def start(self, script):
+        tok = self.tok
+        engine = DsEngine(tok, script)
+        svc = Service(engine, tok, ChatTemplate(TEMPLATE), model_name=default_model_name({"arch": "deepseek41"}))
+        svc.model_arch = "deepseek41"
+        svc.set_family("deepseek41")
+        httpd = serve(svc, port=0)
+        self.addCleanup(httpd.server_close)
+        self.addCleanup(httpd.shutdown)
+        self.addCleanup(frontend.use_family, None)
+        self.engine, self.svc = engine, svc
+        self.base = f"http://127.0.0.1:{httpd.server_address[1]}"
+        return engine
+
+    def post(self, path, body, raw=False):
+        req = urllib.request.Request(self.base + path, data=json.dumps(body).encode(),
+                                     headers={"Content-Type": "application/json"})
+        try:
+            with urllib.request.urlopen(req, timeout=30) as r:
+                data = r.read().decode()
+                return r.status, (data if raw else json.loads(data))
+        except urllib.error.HTTPError as e:
+            return e.code, json.loads(e.read().decode())
+
+    def prompt(self):
+        return self.tok.decode(self.engine.last_prompt)
+
+    def chat(self, **extra):
+        body = {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 400, **extra}
+        return self.post("/v1/chat/completions", body)
+
+
+class Api(ApiBase):
+    # -- thinking --------------------------------------------------------------------------------
+    def test_thinking_by_default(self):
+        self.start("The user greets me.</think>\n\nHello!")
+        status, r = self.chat()
+        self.assertEqual(status, 200, r)
+        self.assertTrue(self.prompt().endswith("<｜User｜>hi<｜Assistant｜><think>"))
+        msg = r["choices"][0]["message"]
+        self.assertEqual(msg["reasoning_content"], "The user greets me.")
+        self.assertEqual(msg["content"].strip(), "Hello!")
+        self.assertEqual(r["choices"][0]["finish_reason"], "stop")
+
+    ANSWER = "17 x 23 = 391. The answer is 391."
+
+    def test_no_closing_think_tag_makes_the_text_the_answer(self):
+        """The 2-bit sets write the answer after <think> and stop without </think>: it must not arrive as reasoning only (P2003)."""
+        for effort in (None, "low", "medium", "high"):
+            with self.subTest(effort=effort, api="openai"):
+                self.start(self.ANSWER)
+                status, r = self.chat(**({"reasoning_effort": effort} if effort else {}))
+                self.assertEqual(status, 200, r)
+                msg = r["choices"][0]["message"]
+                self.assertEqual(msg["content"].strip(), self.ANSWER)
+                self.assertFalse(msg.get("reasoning_content"))
+                self.assertEqual(r["choices"][0]["finish_reason"], "stop")
+            with self.subTest(effort=effort, api="anthropic"):
+                self.start(self.ANSWER)
+                status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 400,
+                                                       "messages": [{"role": "user", "content": "hi"}],
+                                                       **({"output_config": {"effort": effort}} if effort else {})})
+                self.assertEqual(status, 200, r)
+                self.assertEqual([b["type"] for b in r["content"]], ["text"])
+                self.assertEqual(r["content"][0]["text"].strip(), self.ANSWER)
+
+    def test_no_closing_think_tag_streaming_ends_with_the_answer_as_content(self):
+        self.start(self.ANSWER)
+        status, raw = self.post("/v1/chat/completions", {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 400,
+                                                         "stream": True}, raw=True)
+        self.assertEqual(status, 200)
+        chunks = [json.loads(l[6:]) for l in raw.splitlines() if l.startswith("data: ") and l != "data: [DONE]"]
+        content = "".join(c["choices"][0]["delta"].get("content") or "" for c in chunks if c["choices"])
+        self.assertEqual(content.strip(), self.ANSWER)
+        status, raw = self.post("/v1/messages", {"model": "x", "max_tokens": 400, "stream": True,
+                                                 "messages": [{"role": "user", "content": "hi"}]}, raw=True)
+        self.assertEqual(status, 200)
+        self.assertIn('"text_delta"', raw)
+        self.assertIn("391", raw)
+
+    def test_a_turn_cut_by_max_tokens_is_not_turned_into_an_answer(self):
+        self.start(self.ANSWER * 30)
+        status, r = self.chat(max_tokens=8)
+        msg = r["choices"][0]["message"]
+        self.assertEqual(r["choices"][0]["finish_reason"], "length")
+        self.assertFalse(msg.get("content"))
+        self.assertTrue(msg.get("reasoning_content"))
+
+    def test_a_normal_closed_think_is_unchanged(self):
+        self.start("Reasoning here.</think>The answer.")
+        status, r = self.chat()
+        msg = r["choices"][0]["message"]
+        self.assertEqual((msg["reasoning_content"], msg["content"].strip()), ("Reasoning here.", "The answer."))
+
+    def test_thinking_off(self):
+        self.start("Hello!")
+        for extra in ({"chat_template_kwargs": {"enable_thinking": False}}, {"reasoning_effort": "none"}):
+            status, r = self.chat(**extra)
+            self.assertEqual(status, 200, r)
+            self.assertTrue(self.prompt().endswith("<｜User｜>hi<｜Assistant｜></think>"), extra)
+            msg = r["choices"][0]["message"]
+            self.assertEqual(msg["content"].strip(), "Hello!")
+            self.assertFalse(msg.get("reasoning_content"))
+
+    def test_the_end_token_is_deepseeks_not_qwens(self):
+        self.start("one<|im_end|>two")                # (not a token here: just text, like any other)
+        status, r = self.chat(chat_template_kwargs={"enable_thinking": False})
+        self.assertEqual(r["choices"][0]["message"]["content"], "one<|im_end|>two")
+
+    def test_model_name_and_card_sampling(self):
+        self.start("ok")
+        self.assertEqual(FAMILY_MODEL_NAMES["deepseek41"], "deepseek-v4.1-flash")
+        self.assertEqual(default_model_name({"arch": "deepseek41"}), "deepseek-v4.1-flash")
+        self.assertEqual(model_card_sampling("deepseek41", True), {"temperature": 1.0, "top_p": 1.0})
+        self.chat()
+        self.assertEqual(self.engine.last_sampling.get("temperature"), 1.0)
+
+    # -- effort ------------------------------------------------------------------------------------
+    def test_effort_words(self):
+        self.start("ok")
+        for given, line in (("low", "Reasoning Effort: 50 "), ("medium", "Reasoning Effort: 62 "),
+                            ("high", "Reasoning Effort: 75 "), ("max", "Reasoning Effort: 100 "),
+                            ("xhigh", "Reasoning Effort: 100 "), (40, "Reasoning Effort: 40 ")):
+            status, r = self.chat(reasoning_effort=given)
+            self.assertEqual(status, 200, (given, r))
+            self.assertIn(line, self.prompt(), given)
+        status, r = self.chat()
+        self.assertIn("Reasoning Effort: 75 ", self.prompt())
+        status, r = self.chat(reasoning_effort="extreme")
+        self.assertEqual(status, 400)
+
+    def test_anthropic_budget_picks_the_effort(self):
+        self.start("ok")
+        status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 300, "thinking": {"type": "enabled", "budget_tokens": 1000},
+                                               "messages": [{"role": "user", "content": "hi"}]})
+        self.assertEqual(status, 200, r)
+        self.assertIn("Reasoning Effort: 50 ", self.prompt())
+
+    # -- tool calls -------------------------------------------------------------------------------
+    def test_openai_tool_call(self):
+        self.start("Need the weather.</think>" + call(("get_weather", [("city", "Zagreb", True), ("days", "2", False)])))
+        status, r = self.chat(tools=[WEATHER])
+        self.assertEqual(status, 200, r)
+        self.assertIn('"name": "get_weather"', self.prompt())
+        ch = r["choices"][0]
+        self.assertEqual(ch["finish_reason"], "tool_calls")
+        tc = ch["message"]["tool_calls"][0]
+        self.assertEqual(tc["function"]["name"], "get_weather")
+        self.assertEqual(json.loads(tc["function"]["arguments"]), {"city": "Zagreb", "days": 2})
+        self.assertEqual(ch["message"]["reasoning_content"], "Need the weather.")
+
+    def test_two_calls_in_one_turn(self):
+        self.start("</think>" + call(("get_weather", [("city", "A", True)]), ("get_weather", [("city", "B", True)])))
+        status, r = self.chat(tools=[WEATHER])
+        names = [json.loads(c["function"]["arguments"])["city"] for c in r["choices"][0]["message"]["tool_calls"]]
+        self.assertEqual(names, ["A", "B"])
+
+    def test_streamed_tool_call(self):
+        self.start("</think>" + call(("get_weather", [("city", "Zagreb", True)])))
+        status, text = self.post("/v1/chat/completions", {"messages": [{"role": "user", "content": "w?"}], "tools": [WEATHER],
+                                                          "stream": True, "max_tokens": 300}, raw=True)
+        self.assertEqual(status, 200)
+        chunks = [json.loads(x[6:]) for x in text.split("\n\n") if x.startswith("data: ") and x != "data: [DONE]"]
+        calls = [tc for c in chunks for tc in c["choices"][0]["delta"].get("tool_calls", [])]
+        self.assertEqual(calls[0]["function"]["name"], "get_weather")
+        self.assertEqual(json.loads("".join(tc["function"].get("arguments", "") for tc in calls)), {"city": "Zagreb"})
+        self.assertEqual([c["choices"][0]["finish_reason"] for c in chunks if c["choices"][0].get("finish_reason")],
+                         ["tool_calls"])
+
+    def test_a_tool_result_turn_renders_for_the_model(self):
+        self.start("The weather is sunny.")
+        msgs = [{"role": "user", "content": "weather in Zagreb?"},
+                {"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {
+                    "name": "get_weather", "arguments": "{\"city\": \"Zagreb\"}"}}]},
+                {"role": "tool", "tool_call_id": "c1", "content": "sunny, 21C"}]
+        status, r = self.post("/v1/chat/completions", {"messages": msgs, "tools": [WEATHER], "max_tokens": 200,
+                                                       "chat_template_kwargs": {"enable_thinking": False}})
+        self.assertEqual(status, 200, r)
+        p = self.prompt()
+        self.assertIn('<｜DSML｜ parameter name="city" string="true">Zagreb</｜DSML｜ parameter>', p)
+        self.assertIn("<tool_result>sunny, 21C</tool_result>", p)
+        self.assertEqual(r["choices"][0]["message"]["content"].strip(), "The weather is sunny.")
+
+    def test_forced_tool_choice(self):
+        self.start('<｜DSML｜ parameter name="city" string="true">Rome</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n</｜DSML｜ calls>')
+        status, r = self.chat(tools=[WEATHER], tool_choice={"type": "function", "function": {"name": "get_weather"}},
+                              chat_template_kwargs={"enable_thinking": False})
+        self.assertEqual(status, 200, r)
+        self.assertTrue(self.prompt().endswith('<｜Assistant｜></think><｜DSML｜ calls>\n<｜DSML｜ invoke name="get_weather">\n'), self.prompt()[-120:])
+        tc = r["choices"][0]["message"]["tool_calls"][0]
+        self.assertEqual((tc["function"]["name"], json.loads(tc["function"]["arguments"])), ("get_weather", {"city": "Rome"}))
+
+    # -- anthropic ---------------------------------------------------------------------------------
+    def test_anthropic_tool_use(self):
+        self.start("Weather needed.</think>" + call(("get_weather", [("city", "Oslo", True), ("days", "1", False)])))
+        status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 400,
+                                               "tools": [{"name": "get_weather", "description": "w", "input_schema":
+                                                          WEATHER["function"]["parameters"]}],
+                                               "messages": [{"role": "user", "content": "Oslo?"}]})
+        self.assertEqual(status, 200, r)
+        kinds = [b["type"] for b in r["content"]]
+        self.assertEqual(kinds, ["thinking", "tool_use"])
+        self.assertEqual(r["content"][1]["input"], {"city": "Oslo", "days": 1})
+        self.assertEqual(r["stop_reason"], "tool_use")
+
+    def test_anthropic_text_without_thinking(self):
+        self.start("Hello there.")
+        status, r = self.post("/v1/messages", {"model": "x", "max_tokens": 100, "thinking": {"type": "disabled"},
+                                               "messages": [{"role": "user", "content": "hi"}]})
+        self.assertEqual(status, 200, r)
+        self.assertTrue(self.prompt().endswith("<｜Assistant｜></think>"))
+        self.assertEqual([b["type"] for b in r["content"]], ["text"])
+        self.assertEqual(r["content"][0]["text"], "Hello there.")
+        self.assertEqual(r["stop_reason"], "end_turn")
+
+    # -- metrics -----------------------------------------------------------------------------------
+    def test_metrics_use_the_one_answer_speed(self):
+        self.start("</think>Hello, how are you today?")
+        self.chat()
+        status = urllib.request.urlopen(self.base + "/metrics", timeout=10)
+        m = json.loads(status.read().decode())
+        last = m["requests"][-1]
+        # produced tokens over the engine's decode time (10 ms a token in the fake), never over the prompt's time
+        self.assertEqual(last["decode_tok_s"], 100.0)
+
+
+    # -- the shared start of an agent's and its sub-agents' prompts (the engine keeps its state there) ------------
+    def test_the_system_prompt_and_tools_are_the_shared_start(self):
+        self.start("ok")
+        self.auto_pin("1")
+        sysmsg = {"role": "system", "content": "You are an agent. " * 20}
+        body = {"messages": [sysmsg, {"role": "user", "content": "task one"}], "tools": [WEATHER], "max_tokens": 40}
+        self.assertEqual(self.post("/v1/chat/completions", body)[0], 200)
+        n = self.engine.last_sampling["strata_prefix"]["tokens"]
+        ids = self.engine.last_prompt
+        self.assertTrue(0 < n < len(ids) - 1)
+        self.assertIn("get_weather", self.tok.decode(ids[:n]))            # the tools are in the shared start
+        self.assertNotIn("task one", self.tok.decode(ids[:n]))            # the first user message is not
+        # another agent behind the same system prompt and tools, another task: the same boundary, the same start
+        body["messages"] = [sysmsg, {"role": "user", "content": "a different task for the sub-agent"}]
+        self.assertEqual(self.post("/v1/chat/completions", body)[0], 200)
+        self.assertEqual(self.engine.last_sampling["strata_prefix"]["tokens"], n)
+        self.assertEqual(self.engine.last_prompt[:n], ids[:n])
+
+    def auto_pin(self, value):
+        old = os.environ.get("STRATA_DS4_AUTO_PIN")
+        if value is None:
+            os.environ.pop("STRATA_DS4_AUTO_PIN", None)
+        else:
+            os.environ["STRATA_DS4_AUTO_PIN"] = value
+        self.addCleanup(lambda: os.environ.pop("STRATA_DS4_AUTO_PIN", None) if old is None
+                        else os.environ.__setitem__("STRATA_DS4_AUTO_PIN", old))
+
+    def test_no_shared_start_without_a_system_prompt_or_when_switched_off(self):
+        self.start("ok")
+        self.auto_pin("1")
+        self.chat()
+        self.assertNotIn("strata_prefix", self.engine.last_sampling)     # no system message
+        body = {"messages": [{"role": "system", "content": "s"}, {"role": "user", "content": "u"}], "max_tokens": 40}
+        for off in (None, "0"):                                           # off by default, and when switched off
+            self.auto_pin(off)
+            self.post("/v1/chat/completions", body)
+            self.assertNotIn("strata_prefix", self.engine.last_sampling)
+
+    def test_a_clients_own_prefix_wins(self):
+        self.start("ok")
+        self.auto_pin("1")
+        body = {"messages": [{"role": "system", "content": "s " * 30}, {"role": "user", "content": "u"}],
+                "max_tokens": 40, "strata_prefix": {"tokens": 7}}
+        self.post("/v1/chat/completions", body)
+        self.assertEqual(self.engine.last_sampling["strata_prefix"], {"tokens": 7})
+
+    def test_the_qwen_path_asks_for_no_prefix(self):
+        # the same request on a server that is not DeepSeek's: nothing is added (Qwen keeps its own cache logic)
+        svc = Service(MockEngine(self.tok, "ok"), self.tok, ChatTemplate(ROOT / "serve" / "chat_template.jinja"))
+        req = {}
+        svc.prepare([{"role": "system", "content": "s"}, {"role": "user", "content": "u"}], None, {}, 16, req=req)
+        self.assertEqual(req, {})
+
+
+class SlotEnv(unittest.TestCase):
+    """The engine's kept conversations: the folder and the switch reach the DeepSeek engine only."""
+
+    KEYS = ("STRATA_DS4_SLOT_KEEP", "STRATA_DS4_SLOT_DIR")
+
+    def env(self, cfg):
+        saved = {k: os.environ.pop(k, None) for k in self.KEYS}
+        try:
+            return server.child_env(cfg)
+        finally:
+            for k, v in saved.items():
+                if v is not None:
+                    os.environ[k] = v
+
+    def test_deepseek_gets_a_folder_in_the_data_folder_and_the_default_switch(self):
+        env = self.env({"arch": "deepseek41", "log": str(Path("data") / "strata-deepseek.log"), "args": []})
+        self.assertEqual(Path(env["STRATA_DS4_SLOT_DIR"]), Path("data") / "ds4-slots")
+        self.assertEqual(env["STRATA_DS4_SLOT_KEEP"], "0")
+        self.assertEqual(env["STRATA_DS4_SLOT_GB"], "2")
+
+    def test_the_configs_own_settings_win(self):
+        env = self.env({"arch": "deepseek41", "log": "x/y.log",
+                        "env": {"STRATA_DS4_SLOT_KEEP": "1", "STRATA_DS4_SLOT_DIR": "elsewhere"}})
+        self.assertEqual((env["STRATA_DS4_SLOT_KEEP"], env["STRATA_DS4_SLOT_DIR"]), ("1", "elsewhere"))
+
+    def test_other_families_get_nothing(self):
+        for arch in ("qwen4exp", None, "gyro"):
+            env = self.env({"arch": arch, "log": "x/y.log"} if arch else {"log": "x/y.log"})
+            self.assertFalse([k for k in env if k.startswith("STRATA_DS4_")], arch)
+
+class ThinkForever(DsEngine):
+    """Thinks without end (random letters) until the thinking is closed in the prompt, then answers."""
+
+    ANSWER = "Four."
+
+    def __init__(self, tok):
+        super().__init__(tok, "")
+        self.prompts = []
+
+    def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+        self.prompts.append(list(ids))
+        self.last_prompt = list(ids)
+        got = 0
+        try:
+            if self.tok.decode(list(ids[-14:])).endswith("</think>\n\n"):
+                out = self.tok.encode(self.ANSWER) + [EOS]
+            else:
+                import random
+                rng = random.Random(len(ids))          # (no repeated token: the server's loop guard would end it)
+                out = self.tok.encode("".join(chr(97 + rng.randrange(26)) for _ in range(max_new)))
+            for t in out[:max_new]:
+                if cancel.is_set():
+                    return
+                got += 1
+                yield t
+        finally:
+            self.last = {"generated": got, "prompt_tokens": len(ids), "prompt_ms": 50.0, "decode_ms": got * 10.0,
+                         "finish": "stop"}
+
+
+class SmallCap(ApiBase):
+    """A short max_tokens (200-600) with the thinking on: it was all spent on the thinking and no text block came.
+    With no budget of the client's own the thinking is capped at half of max_tokens (this family only)."""
+
+    def start_forever(self):
+        self.start("")
+        engine = ThinkForever(self.tok)
+        self.svc.engine = engine
+        self.engine = engine
+        return engine
+
+    def text_of(self, r):
+        return "".join(c.get("text", "") for c in r["content"] if c["type"] == "text")
+
+    def test_every_short_cap_gets_an_answer_on_both_apis(self):
+        for cap in (200, 400, 600):
+            for api in ("anthropic", "openai"):
+                with self.subTest(cap=cap, api=api):
+                    self.start_forever()
+                    if api == "anthropic":
+                        status, r = self.post("/v1/messages", {"model": "x", "max_tokens": cap, "thinking": {"type": "enabled"},
+                                                               "messages": [{"role": "user", "content": "2+2?"}]})
+                        self.assertEqual(status, 200, r)
+                        self.assertEqual(self.text_of(r), ThinkForever.ANSWER)
+                        self.assertLessEqual(r["usage"]["output_tokens"], cap)
+                    else:
+                        status, r = self.chat(max_tokens=cap)
+                        self.assertEqual(status, 200, r)
+                        self.assertEqual(r["choices"][0]["message"]["content"], ThinkForever.ANSWER)
+                        self.assertLessEqual(r["usage"]["completion_tokens"], cap)
+                    first = self.engine.prompts[0]
+                    self.assertEqual(len(self.engine.prompts), 2)          # the thinking was wrapped up, then answered
+                    self.assertEqual(len(self.engine.prompts[1]) - len(first), cap // 2 - 0 + len(self.tok.encode(
+                        REASONING_WRAP_UP, parse_special=True)))
+
+    def test_thinking_off_is_untouched(self):
+        for cap in (200, 400, 600):
+            with self.subTest(cap=cap):
+                self.start("Plain answer.")
+                status, r = self.chat(max_tokens=cap, reasoning_effort="none")
+                self.assertEqual(status, 200, r)
+                self.assertEqual(r["choices"][0]["message"]["content"], "Plain answer.")
+                status, r = self.post("/v1/messages", {"model": "x", "max_tokens": cap, "thinking": {"type": "disabled"},
+                                                       "messages": [{"role": "user", "content": "hi"}]})
+                self.assertEqual(self.text_of(r), "Plain answer.")
+
+    def test_a_client_budget_wins(self):
+        self.start_forever()
+        status, r = self.chat(max_tokens=600, reasoning_budget_tokens=20)
+        self.assertEqual(status, 200, r)
+        self.assertEqual(len(self.engine.prompts[1]) - len(self.engine.prompts[0]), 20 + len(self.tok.encode(
+            REASONING_WRAP_UP, parse_special=True)))
+
+    def test_a_tool_call_under_a_short_cap_still_parses(self):
+        for cap in (200, 400, 600):
+            with self.subTest(cap=cap):
+                self.start("Need it.</think>" + call(("f", [("a", "1", True)])))
+                status, r = self.chat(max_tokens=cap, tools=[{"type": "function", "function": {
+                    "name": "f", "description": "f", "parameters": {"type": "object", "properties": {"a": {"type": "string"}}}}}])
+                self.assertEqual(status, 200, r)
+                calls = r["choices"][0]["message"]["tool_calls"]
+                self.assertEqual(calls[0]["function"]["name"], "f")
+                self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"a": "1"})
+                self.assertEqual(len(self.engine.last_prompt) > 0, True)
+
+    def test_the_family_default_thinking_budget(self):
+        """DeepSeek-V4.1 only, as Maya ships it: 32768 thinking tokens when neither the request nor the config names a budget."""
+        self.start("ok")
+        self.assertEqual(self.svc.reasoning_budget({}), 32768)
+        self.assertIsNone(self.svc.reasoning_budget({"reasoning_budget_tokens": 0}))        # a request turns it off
+        self.assertEqual(self.svc.reasoning_budget({"reasoning_budget_tokens": 50}), 50)    # and its own wins
+        self.assertEqual(self.svc.reasoning_budget({"thinking_budget_tokens": 70}), 70)
+        self.svc.reasoning_budget_tokens, self.svc.reasoning_budget_configured = 90, True
+        self.assertEqual(self.svc.reasoning_budget({}), 90)                                 # the config's wins
+        self.svc.reasoning_budget_tokens = 0                                                # the config says none
+        self.assertIsNone(self.svc.reasoning_budget({}))
+        self.svc.reasoning_budget_configured = False
+        self.svc.set_family(None)
+        self.assertIsNone(self.svc.reasoning_budget({}))                                    # other families: none
+
+    def test_the_default_budget_closes_a_long_thinking(self):
+        self.start_forever()
+        status, r = self.chat(max_tokens=5000)
+        self.assertEqual(status, 200, r)
+        self.assertEqual(r["choices"][0]["message"]["content"], ThinkForever.ANSWER)
+        status, r = self.chat(max_tokens=300, reasoning_effort="none")                      # thinking off: no budget, no wrap-up
+        self.assertEqual(len(self.engine.prompts), 3)
+
+    def test_other_families_keep_the_old_behaviour(self):
+        self.start_forever()
+        self.svc.set_family(None)
+        status, r = self.chat(max_tokens=300)
+        self.assertEqual(len(self.engine.prompts), 1)                       # no wrap-up: Qwen's serving is as it was
+
+
+class PickTemplateTests(unittest.TestCase):
+    """A pack made from another GGUF can ship the V4-Flash template (`tool_calls`, no space): the V4.1 model then
+    writes malformed calls (P2003 install test).  The server swaps in the V4.1 template for such a pack."""
+
+    def test_v4flash_pack_template_is_replaced(self):
+        import tempfile
+        from serve import dsml
+        with tempfile.TemporaryDirectory() as d:
+            old = Path(d) / "old.jinja"
+            old.write_text("{%- set dsml_token = '｜DSML｜' -%}{{ '<' + dsml_token + 'tool_calls>' }}", encoding="utf-8")
+            self.assertEqual(dsml.pick_template(old), (dsml.FIXTURE_TEMPLATE, True))
+            same = Path(d) / "same.jinja"
+            same.write_text(dsml.FIXTURE_TEMPLATE.read_text(encoding="utf-8"), encoding="utf-8")
+            self.assertEqual(dsml.pick_template(same), (same, False))
+            self.assertEqual(dsml.pick_template(Path(d) / "none.jinja"), (dsml.FIXTURE_TEMPLATE, False))
+
+    def test_fixture_is_accepted(self):
+        from serve import dsml
+        self.assertTrue(dsml.v41_template_ok(dsml.FIXTURE_TEMPLATE.read_text(encoding="utf-8")))
+
+
+if __name__ == "__main__":
+    unittest.main()

serve/test_family_name.py

@@ -19,11 +19,12 @@ class FamilyName(unittest.TestCase):
     def test_per_family(self):
         self.assertEqual(default_model_name({"arch": "qwen35moe"}), "qwen3.6-35b-a3b")
         self.assertEqual(default_model_name({"arch": "qwen4exp"}), "qwen3.8-flash-next")
+        self.assertEqual(default_model_name({"arch": "deepseek41"}), "deepseek-v4.1-flash")
 
     def test_old_configs(self):
         self.assertEqual(default_model_name({}), "qwen3.8-flash-next")
         self.assertEqual(default_model_name({"arch": "something-else"}), "qwen3.8-flash-next")
-        self.assertEqual(set(FAMILY_MODEL_NAMES), {"qwen4exp", "qwen35moe"})
+        self.assertEqual(set(FAMILY_MODEL_NAMES), {"qwen4exp", "qwen35moe", "deepseek41"})
 
 
 if __name__ == "__main__":

setup.py

@@ -98,6 +98,8 @@ HF_REVISIONS = {
     "unsloth/Qwen3.8-Flash-Next-GGUF": "38bb39ee97821de2c9009abb7e93950eec396e66",                   # 2026-09-30
     "unsloth/Qwen3.6-35B-A3B-MTP-GGUF": "5bc3e238d916f48a861bac2f8a1990a0e9b7e98d",                  # 2026-10-12
     "agentionai/Qwen3.8-Flash-Next-Gyro-GGUF": "1ada08422e404aca51ab9df665f089f34c419c6b",           # 2026-10-13
+    "unsloth/DeepSeek-V4.1-Flash-1017-GGUF": "f924b94f19c1fea02c06a65c8475b0a45e2b88b1",             # 2026-10-20 (the port's reference)
+    "ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF": "cb02422c900df489cea5833e8a92171224059b3c",    # 2026-10-24 (ISTA-DASLab's final 1.75 / 2.00 / 2.25 bpw sets and Engram)
 }
 
 
@@ -653,6 +655,127 @@ OWN_PATH_FAMILIES = {
 }
 PREVIEW_FAMILIES = OWN_PATH_FAMILIES             # the earlier name (tools and tests that import it)
 
+# DeepSeek-V4.1-Flash (GGUF architecture deepseek41), Unsloth's UD-Q8_K_XL at the revision the port was checked against: a
+# family with an engine of its own (strata-dsv41, src/dsv41, ported from Project Maya's maya-ds4, MIT) that is not a
+# Flash-Next pack and not a single GGUF: nine shards (508.4 GB, the routed experts as MXFP4, the dense weights in BF16, and
+# in shard 8 the 202.8 GB n-gram table that stays on the SSD), the engine's pack folder (tools/dsv41/pack_gguf.py: hash
+# tables, tensor map, the tokenizer and the GGUF's own chat template, a few MB) beside them.  MULTI_FILE_FAMILIES, offered
+# in the same menu and installed by install_dsv41.  Where it fits: strata_control/dsv41_fit.py (real numbers).  The DSpark
+# drafter (dspark-...-Q8_0.gguf, 8.0 GB) and the picture reader (mmproj-BF16.gguf, 1.0 GB) are pinned here but not fetched: the
+# engine does not use them yet.
+DSV41_REPO = "unsloth/DeepSeek-V4.1-Flash-1017-GGUF"
+DSV41_SHARD = "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-{i:05d}-of-00009.gguf"
+DSV41_FILES = {
+    "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00001-of-00009.gguf": (5780589, "cc15d470afcb030558bbc4d8e2ccd840182e2e95de1a95198138efe63a5fd99f"),
+    "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00002-of-00009.gguf": (50536121984, "9cb2047676bd20e448be2377b480080c97fb38fae117f4846e07de2c65bb8522"),
+    "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00003-of-00009.gguf": (50536121984, "17d398dd327e7ff1991b382677814db75d970b074661757564b6f2ac3d0c8d95"),
+    "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00004-of-00009.gguf": (50536122016, "7ea8da1143b9a6bf1424a361eb1b0d6f66c7ce79eac9711162f212cc4f39c319"),
+    "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00005-of-00009.gguf": (50536122016, "1952da19a841176b46f9681c99ea57c674f8a8861e16d76fb35cec4befe039b2"),
+    "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00006-of-00009.gguf": (50536122016, "d6711781d8effab79b410af9c40920bffcc0c28976e509bc1227ff0c6669c33a"),
+    "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00007-of-00009.gguf": (36097230048, "2ebc8d198d03a14865e8a85df637285ceb5dd79b1ffc78424bd3c3e06df57823"),
+    "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00008-of-00009.gguf": (202758032592, "12df293355efbd104b088a77e13ee22410fb1ec3f7ab3863947f09896e64bec7"),
+    "DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00009-of-00009.gguf": (16835109952, "df33722d1dcf5ffe0347ed8406f7acb3b18061f4ded2c13c2b0a184b8d41c9ea"),
+    "dspark-DeepSeek-V4.1-Flash-1017-Q8_0.gguf": (7976914240, "6ba6ac1b331c540691a209e7fc97dec2a24e295acb1e755c84c7d24a4373927a"),
+    "mmproj-BF16.gguf": (972208288, "9a1ef7c40d0d702c7ed8c5f863f60abae64091387ba3fce0ad19ce9beeda3bf9"),
+}
+# The two OFFICIAL IST-DAS sets (ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF at the revision above, a flat repository; Hub LFS sha256 = what the
+# files hash to): IQ2_XXS is the 2.00 bpw set, IQ2_XS the 2.25 bpw set and IQ1_M the 1.75 bpw set, 4 shards each, plus ONE Engram table (110.6 GB, the same file for both sets:
+# downloaded once into the shared folder) and the DSpark drafter and the 5.8 MB metadata shard of Unsloth's set that pack_gguf --engram-meta reads
+# (both pinned in DSV41_FILES above).  This is the set a normal PC runs (64 GB RAM + a 12 GB card; docs/DEEPSEEK41.md); the UD-Q8_K_XL entry stays
+# for the big machines.
+DSV41_ISTA_REPO = "ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF"
+DSV41_ISTA_ENGRAM = "DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf"
+DSV41_ISTA_SHARD = "DeepSeek-V4.1-Flash-1017-GSQ-RCO-{quant}-{i:05d}-of-00004.gguf"
+DSV41_DSPARK = "dspark-DeepSeek-V4.1-Flash-1017-Q8_0.gguf"
+DSV41_ISTA_FILES = {
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ1_M-00001-of-00004.gguf": (39358830464, "6e3f0d5ff53d00428ec32c079db9e8255cb410ccaa435e5342aaa8a47fe1339d"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ1_M-00002-of-00004.gguf": (39176113248, "34d68c99fdae9948f9f0af0dafe308e805ce0a78d2c48641b24cc57242fdc0af"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ1_M-00003-of-00004.gguf": (39997334720, "a08468c83955e0c4d3a9eb3f7b86c45b7bbac55d99c4640039a296a8a4b327ea"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ1_M-00004-of-00004.gguf": (2188841824, "b05f26a92325aa6845cebcb3ec1f93d2e65ebf89bdc3f805695960d2f0681832"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ2_XS-00001-of-00004.gguf": (39535776992, "4c65307992ab586c527ca1f6f64062aa96a33b23c02b7fed103f45f75e854798"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ2_XS-00002-of-00004.gguf": (39494617536, "933461a974fcccee741bf2f9798201e72ab4aa3f8a952be56f849896ca6ea30e"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ2_XS-00003-of-00004.gguf": (39423838496, "bbf91fa346a85ebaf32dd4fd5c456b534d780d381e54f46ac90783a481373240"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ2_XS-00004-of-00004.gguf": (36771591232, "e5b2f8781fab688cd4bc3bc7a69651b5dbb90ed94965657c6bead8bb4774f3cf"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ2_XXS-00001-of-00004.gguf": (39075714624, "d46efb67ccffd8fb5d7f18b423896381b9e06d8491e161c370ddf3b05b07e050"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ2_XXS-00002-of-00004.gguf": (39388449504, "cec1b2c3da429dc768b8bb9bc4dc4403d81f74d34db660a6a34ee3e555bf2c70"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ2_XXS-00003-of-00004.gguf": (39246891616, "9d9c2140fcb134a0b9a74dedd0e5167e68077af6d08ce7f856bce112f0a35dd3"),
+    "DeepSeek-V4.1-Flash-1017-GSQ-RCO-IQ2_XXS-00004-of-00004.gguf": (20527837312, "9b790a2297ed3cdede379fc984f4a52bb95f3b23724c797af11d1fa6a69db059"),
+    "DeepSeek-V4.1-Flash-Engram-IQ4_NL.gguf": (110595290816, "eb19f30d368d393784e1fc54fdfaffc427ebb2110bf3762fdf5a8fbb9a2d740a"),
+}
+# The files the first release of these sets was downloaded as (helcig/tmp-oct26-2, 2026-10-23: model-2.00bpw-0000N-of-00004.gguf, model-2.25bpw-...,
+# engram-IQ4_NL.gguf, usually in a 2.00bpw/ or 2.25bpw/ folder): testers and our boxes have them.  They are the same weights: shards 2-4 are the
+# same bytes as today's, and shard 1 and the Engram differ only in their GGUF header (metadata).  setup accepts them from "files already here"
+# (size and SHA-256 checked against these pins) and never downloads them.  Not in REPO_PINS: nothing is fetched under these names.
+DSV41_LEGACY_FILES = {
+    "model-2.00bpw-00001-of-00004.gguf": (39075713728, "78ab9efc46a26647c3e94b322c99cd7e2af261dd28b38f2712e128562db1ef47"),
+    "model-2.00bpw-00002-of-00004.gguf": (39388449504, "cec1b2c3da429dc768b8bb9bc4dc4403d81f74d34db660a6a34ee3e555bf2c70"),
+    "model-2.00bpw-00003-of-00004.gguf": (39246891616, "9d9c2140fcb134a0b9a74dedd0e5167e68077af6d08ce7f856bce112f0a35dd3"),
+    "model-2.00bpw-00004-of-00004.gguf": (20527837312, "9b790a2297ed3cdede379fc984f4a52bb95f3b23724c797af11d1fa6a69db059"),
+    "model-2.25bpw-00001-of-00004.gguf": (39535776064, "ddd13aca16faa23b08f5196b5e0daddf90f04688a271f5889eb22fa7d7d65526"),
+    "model-2.25bpw-00002-of-00004.gguf": (39494617536, "933461a974fcccee741bf2f9798201e72ab4aa3f8a952be56f849896ca6ea30e"),
+    "model-2.25bpw-00003-of-00004.gguf": (39423838496, "bbf91fa346a85ebaf32dd4fd5c456b534d780d381e54f46ac90783a481373240"),
+    "model-2.25bpw-00004-of-00004.gguf": (36771591232, "e5b2f8781fab688cd4bc3bc7a69651b5dbb90ed94965657c6bead8bb4774f3cf"),
+    "engram-IQ4_NL.gguf": (110595290720, "1f17d8ae37e9df3bb45f7ef1b4fcd1706d7b759af0413c117bf8e92aed459281"),
+}
+# today's name -> the name it had in the first release
+DSV41_LEGACY_NAMES = {
+    **{DSV41_ISTA_SHARD.format(quant="IQ2_XXS", i=i): "model-2.00bpw-%05d-of-00004.gguf" % i for i in range(1, 5)},
+    **{DSV41_ISTA_SHARD.format(quant="IQ2_XS", i=i): "model-2.25bpw-%05d-of-00004.gguf" % i for i in range(1, 5)},
+    DSV41_ISTA_ENGRAM: "engram-IQ4_NL.gguf",
+}
+# the files every official set shares (name -> where it comes from); they live in models/deepseek-v4.1-flash-shared, once
+DSV41_SHARED = {
+    DSV41_ISTA_ENGRAM: hf(DSV41_ISTA_REPO),
+    DSV41_SHARD.format(i=1): hf(DSV41_REPO) + "UD-Q8_K_XL/",      # only its metadata is read (pack_gguf --engram-meta)
+    DSV41_DSPARK: hf(DSV41_REPO),
+}
+DSV41_ALL_FILES = {**DSV41_FILES, **DSV41_ISTA_FILES, **DSV41_LEGACY_FILES}
+DSV41_PROFILES = ROOT / "data" / "dsv41"           # the shipped hot-expert profiles (profile-2.00.bin, profile-2.25.bin, + .sha256)
+DSV41_CONTEXT = 8192          # the context the engine was checked at (4096 and 8192; 64K is not tried yet)
+MULTI_FILE_FAMILIES = {
+    "deepseek41": {"title": "DeepSeek-V4.1-Flash", "by": "DeepSeek; the 2-bit GGUFs by IST-DAS, UD-Q8_K_XL by Unsloth", "arch": "deepseek41",
+                   "about": "a big model that runs on a PC with a 12 GB card and 64 GB of RAM (the two official 2-bit versions: 257 or "
+                            "274 GB of disk, about 5-7 tokens a second) or, in Unsloth's 508 GB version, on several graphics cards",
+                   "hf": hf(DSV41_REPO) + "UD-Q8_K_XL/", "tag": "deepseek-v4.1-flash-", "name": "deepseek-v4.1-flash",
+                   "license": "DeepSeek's model license: https://huggingface.co/unsloth/DeepSeek-V4.1-Flash-1017-GGUF",
+                   "models": {"UD-Q8_K_XL": {"about": "Unsloth Dynamic 8-bit: the experts in MXFP4 as DeepSeek made them, the rest "
+                                                       "in BF16 (the checkpoint's own precision)",
+                              "ram_gb": 306, "shards": [DSV41_SHARD.format(i=i) for i in range(1, 10)]}},
+                   "sha256": DSV41_ALL_FILES, "vision": False}
+}
+# The official sets first (a normal PC), Unsloth's 508 GB set last.  Each model: its shards, the sub-folder they are in at the repository
+# (and in a --gguf-dir folder), the files shared with the other set ("shared"), the pack recipe and the shipped expert profile.
+MULTI_FILE_FAMILIES["deepseek41"]["models"] = {
+    "IST-DAS-2.00bpw": {
+        "label": "IST-DAS 2.00 bpw",
+        "about": "the smaller official version: 138 GB of model + 111 GB n-gram table + 8 GB drafter (257 GB of disk). Runs on a PC with "
+                 "a 12 GB card and 40-64 GB of RAM at about 5-7 tokens a second; quality about 15.5% above the original model's "
+                 "perplexity (code and agent work 5-9%)",
+        "ram_gb": 40, "recipe": "ista", "subdir": None, "legacy_subdir": "2.00bpw", "profile": "profile-2.00.bin", "hf": hf(DSV41_ISTA_REPO),
+        "shards": [DSV41_ISTA_SHARD.format(quant="IQ2_XXS", i=i) for i in range(1, 5)],
+        "shared": [DSV41_ISTA_ENGRAM, DSV41_SHARD.format(i=1), DSV41_DSPARK]},
+    "IST-DAS-2.25bpw": {
+        "label": "IST-DAS 2.25 bpw",
+        "about": "the better official version: 155 GB of model + 111 GB n-gram table + 8 GB drafter (274 GB of disk, 17 GB more than "
+                 "2.00). About the same speed; quality about 7.3% above the original model's perplexity (code and agent work 2-3%)",
+        "ram_gb": 44, "recipe": "ista", "subdir": None, "legacy_subdir": "2.25bpw", "profile": "profile-2.25.bin", "hf": hf(DSV41_ISTA_REPO),
+        "shards": [DSV41_ISTA_SHARD.format(quant="IQ2_XS", i=i) for i in range(1, 5)],
+        "shared": [DSV41_ISTA_ENGRAM, DSV41_SHARD.format(i=1), DSV41_DSPARK]},
+    "IST-DAS-1.75bpw": {
+        "label": "IST-DAS 1.75 bpw",
+        "about": "the smallest official version: 121 GB of model + 111 GB n-gram table + 8 GB drafter (239 GB of disk, 18 GB less than 2.00). Runs "
+                 "where 2.00 is tight (a 12 GB card and 32-40 GB of RAM). Smallest: fine for code and agent work, noticeably weaker on plain prose; pick 2.00 if it fits",
+        "ram_gb": 34, "recipe": "ista", "subdir": None, "legacy_subdir": None, "profile": "profile-2.00.bin", "hf": hf(DSV41_ISTA_REPO),
+        "shards": [DSV41_ISTA_SHARD.format(quant="IQ1_M", i=i) for i in range(1, 5)],
+        "shared": [DSV41_ISTA_ENGRAM, DSV41_SHARD.format(i=1), DSV41_DSPARK]},
+    "UD-Q8_K_XL": {
+        "label": "Unsloth UD-Q8_K_XL",
+        "about": "Unsloth Dynamic 8-bit: the experts in MXFP4 as DeepSeek made them, the rest in BF16 (the checkpoint's own precision); "
+                 "508 GB, for machines with hundreds of GB of RAM or several cards",
+        "ram_gb": 306, "recipe": "ud", "subdir": "UD-Q8_K_XL", "shards": [DSV41_SHARD.format(i=i) for i in range(1, 10)],
+        "shared": []},
+}
+
 # Every model file Strata downloads has a fixed size and SHA-256: (repository, file name) -> (bytes, sha256).  download()
 # checks the file against it as soon as it is whole (a wrong file is deleted and setup stops), and stops before the
 # download when the server's size is not the pinned one.  The files are fetched from the commits in HF_REVISIONS.
@@ -663,6 +786,8 @@ REPO_PINS = {
     "unsloth/Qwen3.8-Flash-Next-GGUF": {**UNSLOTH_SHARDS, **UNSLOTH_IQ4_XS_SHARDS, **UNSLOTH_BIG_SHARDS},
     "unsloth/Qwen3.6-35B-A3B-MTP-GGUF": QWEN36_FILES,
     "agentionai/Qwen3.8-Flash-Next-Gyro-GGUF": GYRO_FILES,
+    "unsloth/DeepSeek-V4.1-Flash-1017-GGUF": DSV41_FILES,
+    "ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF": DSV41_ISTA_FILES,
 }
 # EXPERIMENTAL, off by default (setup asks): a control vector shipped with the repository, see its README
 ESP_VECTOR = ROOT / "data" / "experimental-speed-projection" / "Qwen3.8-Flash-Next-experimental-speed-projection.gguf"
@@ -758,7 +883,7 @@ def run(cmd, cwd=None, env=None, check=True, quiet=False):
     if CONTROL:                                        # the app: only this Python running a bundled tool
         argv = [str(c) for c in cmd]
         if len(argv) < 2 or os.path.normcase(argv[0]) != os.path.normcase(sys.executable) or argv[1] == "-m" or \
-                Path(argv[1]).resolve().parent != (ROOT / "tools").resolve():
+                Path(argv[1]).resolve().parent not in ((ROOT / "tools").resolve(), (ROOT / "tools" / "dsv41").resolve()):
             raise acquisition.AcquisitionRefused("the desktop app's installer only runs its bundled preparation tools; "
                                                  + " ".join(argv[:2]))
     say("  > " + " ".join(str(c) for c in cmd))
@@ -1578,6 +1703,30 @@ def check_gpus(sel, found, what="", yes=False, named=False) -> None:
         fail(f"GPU {i}{'' if g is None else ' (' + g['name'] + ')'} {what}cannot be used: {p}", hint)
 
 
+def arch_numbers(value) -> list:
+    """The CUDA architectures of a BUILD.json `archs` / `vision_archs` value, as numbers.  The packaging recipes write the picture
+    reader's as ONE string ("75;86;89;120", packaging/package_engine.py stamp_vision_archs, build_cuda_linux.sh), the engine's own as a
+    list of numbers ([75, 86, 89, 120]); [60, "sm_61", "70-real", "compute_89"] and "60,61 70" are read too (#3810: a string was
+    iterated character by character and setup stopped with int(';'))."""
+    if value is None or value == "":
+        return []
+    parts = list(value) if isinstance(value, (list, tuple, set)) else re.split(r"[;,\s]+", str(value))
+    out: list = []
+    for part in parts:
+        m = re.search(r"\d+", str(part))
+        if m and int(m.group()) not in out:
+            out.append(int(m.group()))
+    return out
+
+
+def arch_names(value) -> list:
+    """The AMD architectures ("gfx1100") of a BUILD.json `archs` / `vision_archs` value, as strings: a list or one ";" / "," separated string."""
+    if value is None or value == "":
+        return []
+    parts = list(value) if isinstance(value, (list, tuple, set)) else re.split(r"[;,\s]+", str(value))
+    return [str(p) for p in parts if str(p)]
+
+
 def engine_archs(toolkit=13):
     """The GPU generations the installed engine has code for: (archs, ptx), or None when there is none."""
     info = engine_dir(toolkit) / "BUILD.json"
@@ -1585,7 +1734,7 @@ def engine_archs(toolkit=13):
         meta = json.loads(info.read_text(encoding="utf-8"))
     except (OSError, ValueError):
         return None
-    return [int(x) for x in meta.get("archs", [])], bool(meta.get("ptx"))
+    return arch_numbers(meta.get("archs")), bool(meta.get("ptx"))
 
 
 def engine_archs_hip():
@@ -1594,7 +1743,7 @@ def engine_archs_hip():
         meta = json.loads((ROOT / "engine" / "BUILD.json").read_text(encoding="utf-8"))
     except (OSError, ValueError):
         return None
-    return [str(x) for x in meta.get("archs", [])] if meta.get("backend") == "hip" else None
+    return arch_names(meta.get("archs")) if meta.get("backend") == "hip" else None
 
 
 def engine_runs_on(g, toolkit=13) -> bool:
@@ -3115,6 +3264,18 @@ def requirement_lines(path: Path | None = None) -> list[str]:
     return lines
 
 
+MONITOR_REQUIREMENTS = ROOT / "requirements-monitor.txt"
+
+
+def runtime_packages() -> list[str]:
+    """The packages setup installs for a run: the pinned runtime ones, plus the Terminal's screens (textual, which brings rich: `strata launch`
+    and `strata monitor` need them; the packaged Terminal bundles them, a source install gets them here)."""
+    base = requirement_lines() if REQUIREMENTS.exists() else list(PY_PACKAGES)
+    if MONITOR_REQUIREMENTS.exists():
+        base += [r for r in requirement_lines(MONITOR_REQUIREMENTS) if r not in base]
+    return base
+
+
 def _installed(name: str) -> bool:
     try:
         import importlib.metadata as md
@@ -4169,7 +4330,7 @@ def get_prebuilt_hip(url_base, gpu, updating=False) -> Path | None:
             meta = {}
         ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:4] if x.isdigit())
         if meta.get("backend") == "hip" and meta.get("source") == "prebuilt" and ver >= WIN_HIP_MIN_ENGINE and \
-                gpu["arch"] in meta.get("archs", []) and not updating:
+                gpu["arch"] in arch_names(meta.get("archs")) and not updating:
             ok("ready-made AMD engine already installed")
             return eng
     if not url_base:
@@ -4211,11 +4372,11 @@ def get_prebuilt_hip(url_base, gpu, updating=False) -> Path | None:
         why = "it is not a HIP engine"
     elif ver < WIN_HIP_MIN_ENGINE:
         why = f"it is version {meta.get('version')}; this setup needs {'.'.join(map(str, WIN_HIP_MIN_ENGINE))}"
-    elif gpu["arch"] not in meta.get("archs", []):
-        why = (f"it is built for {', '.join(meta.get('archs', []))}; your GPU is {gpu['arch']}.  A card it has no code "
+    elif gpu["arch"] not in arch_names(meta.get("archs")):
+        why = (f"it is built for {', '.join(arch_names(meta.get('archs')))}; your GPU is {gpu['arch']}.  A card it has no code "
                "for runs from a self-build of the engine: see docs/AMD_HIP.md (--build)")   # #881
         if is_strix_halo(gpu):                         # never a zip without gfx1151 code: say so, and what to do
-            why = (f"it is built for {', '.join(meta.get('archs', [])) or 'other GPUs'} and has no gfx1151 code: your "
+            why = (f"it is built for {', '.join(arch_names(meta.get('archs'))) or 'other GPUs'} and has no gfx1151 code: your "
                    f"{gpu['name']} (Strix Halo) needs a ready-made engine from Strata 0.1.40 or newer, whose zip is "
                    "built for gfx1151 too.  Until then: the Linux guide docs/STRIX_HALO.md, or compile the engine "
                    "here with tools\\hip\\build_windows.bat (its default list has gfx1151) and start setup with "
@@ -4229,7 +4390,7 @@ def get_prebuilt_hip(url_base, gpu, updating=False) -> Path | None:
     shutil.rmtree(tmp, ignore_errors=True)
     drop_archive(z)
     hip_runtime_beside_exe(eng)                        # #468 #461
-    ok(f"ready-made AMD engine {meta.get('version', '')} for {', '.join(meta.get('archs', []))} "
+    ok(f"ready-made AMD engine {meta.get('version', '')} for {', '.join(arch_names(meta.get('archs')))} "
        f"(ROCm {meta.get('rocm', '?')})")
     return eng
 
@@ -4357,12 +4518,12 @@ def build_engine_hip(gpu, llama, vision="none") -> Path:
     meta = json.loads(stamp.read_text(encoding="utf-8")) if stamp.exists() else {}
     src, vsrc = source_hash(ENGINE_SOURCES), source_hash(VISION_SOURCES)
     archs = sorted(set(gpu.get("archs") or [gpu["arch"]]))
-    has_archs = set(archs) <= set(meta.get("archs", []))
+    has_archs = set(archs) <= set(arch_names(meta.get("archs")))
     floor = cpu_floor(cpu_info()[1])                     # "" on an AVX2 CPU: the normal engine
     engine_ok = meta.get("backend") == "hip" and (eng / EXE).exists() and meta.get("src") == src and has_archs and \
         (meta.get("isa_floor") or "") == floor and not stale_binary(eng / EXE)
     vision_ok = vision == "none" or ((eng / VEXE).exists() and meta.get("vision_src") == vsrc and
-                                     (vision != "gpu" or ((eng / HVEXE).exists() and set(archs) <= set(meta.get("vision_archs", [])))))
+                                     (vision != "gpu" or ((eng / HVEXE).exists() and set(archs) <= set(arch_names(meta.get("vision_archs"))))))
     if engine_ok and vision_ok:
         ok("engine already built for this PC")
         return eng
@@ -4422,8 +4583,8 @@ def reader_archs(eng: Path) -> list:
         except (OSError, ValueError):
             continue
         for key in keys:
-            if isinstance(data, dict) and isinstance(data.get(key), list) and data[key]:
-                return [str(x) for x in data[key]]
+            if isinstance(data, dict) and arch_names(data.get(key)):
+                return arch_names(data.get(key))
     return []
 
 
@@ -4936,7 +5097,7 @@ def get_prebuilt(url_base, gpu, vision, updating=False, toolkit=13) -> Path | No
         ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:4] if x.isdigit())
         if meta.get("source") == "local":              # compiled here: build_engine checks its source and cards
             return None
-        have = [int(a) for a in meta.get("archs", [])]
+        have = arch_numbers(meta.get("archs"))
         miss = [int(x) for x in gpu.get("archs", [gpu["arch"]])
                 if have and int(x) not in have and not (meta.get("ptx") and int(x) > max(have))]
         if miss:                                       # a card it has no code for (#128): compiled here instead
@@ -4997,7 +5158,7 @@ def get_prebuilt(url_base, gpu, vision, updating=False, toolkit=13) -> Path | No
         shutil.rmtree(tmp, ignore_errors=True)
         drop_archive(z)
         return None
-    archs = [int(a) for a in meta.get("archs", [])]
+    archs = arch_numbers(meta.get("archs"))
     miss = [int(x) for x in gpu.get("archs", [gpu["arch"]])
             if int(x) not in archs and not (meta.get("ptx") and int(x) > max(archs))]
     if miss:
@@ -5012,7 +5173,7 @@ def get_prebuilt(url_base, gpu, vision, updating=False, toolkit=13) -> Path | No
     if not (eng / EXE).exists():
         fail("the ready-made engine archive has no " + EXE)
     if not WIN:
-        for x in (EXE, VEXE):
+        for x in (EXE, VEXE, "strata-dsv41"):
             if (eng / x).exists():
                 (eng / x).chmod(0o755)
     ok(f"ready-made engine {meta.get('version', '')} for {', '.join('sm_' + str(a) for a in archs)} (CUDA "
@@ -5106,11 +5267,11 @@ def update_installed_engine(url_base, toolkit=None) -> None:
         if meta.get("src") != source_hash(ENGINE_SOURCES) or stale_binary(eng / EXE):
             try:
                 usable = [x for x in amd_gpus() if amd_problem(x) is None]
-                g = next((x for x in usable if x["arch"] in meta.get("archs", [])), usable[0] if usable else None)
+                g = next((x for x in usable if x["arch"] in arch_names(meta.get("archs"))), usable[0] if usable else None)
                 if g is None:
                     raise RuntimeError("no supported AMD GPU found")
                 # every architecture it was built for (a layer split across two families keeps both)
-                build_engine_hip({**g, "archs": [x for x in meta.get("archs", []) if x in AMD_ARCHS] or [g["arch"]]},
+                build_engine_hip({**g, "archs": [x for x in arch_names(meta.get("archs")) if x in AMD_ARCHS] or [g["arch"]]},
                                  get_llama_cpp(), meta.get("vision") or "none")
             except (Exception, SystemExit) as e:
                 warn(f"could not compile the updated engine{'' if isinstance(e, SystemExit) else f' ({e})'}: "
@@ -5138,9 +5299,9 @@ def update_installed_engine(url_base, toolkit=None) -> None:
         try:                                           # a failed compile must not stop the model from starting
             if gpu is None:
                 raise RuntimeError("no NVIDIA GPU found")
-            gpu = {**gpu, "archs": sorted({int(gpu["arch"]), *(int(x) for x in meta.get("archs", []))})}
+            gpu = {**gpu, "archs": sorted({int(gpu["arch"]), *arch_numbers(meta.get("archs"))})}
             if int(toolkit) == 12:                     # the cards it was compiled for (the main GPU may be newer)
-                gpu["archs"] = sorted({int(x) for x in meta.get("archs", [])}) or gpu["archs"]
+                gpu["archs"] = sorted(arch_numbers(meta.get("archs"))) or gpu["archs"]
             build_engine(gpu, vision, False, get_llama_cpp(), toolkit=toolkit)
         except (Exception, SystemExit) as e:
             warn(f"could not compile the updated engine{'' if isinstance(e, SystemExit) else f' ({e})'}: starting the installed one")
@@ -5149,7 +5310,7 @@ def update_installed_engine(url_base, toolkit=None) -> None:
     if gpu is not None:
         try:
             if int(toolkit) == 12:                     # the cards the CUDA 12 engine serves, not the newest one
-                gpu = {**gpu, "archs": [x for x in (int(a) for a in meta.get("archs", [])) if x < CUDA13_MIN_ARCH]
+                gpu = {**gpu, "archs": [x for x in arch_numbers(meta.get("archs")) if x < CUDA13_MIN_ARCH]
                        or [int(gpu["arch"])]}
             new = get_prebuilt(url_base, gpu, "gpu", updating=True, toolkit=toolkit)
         except Exception as e:                         # a failed download must not stop the model from starting
@@ -5398,7 +5559,7 @@ def prebuilt_vision(meta: dict, gpu: dict, vision: str) -> str:
     of compiling one, which fails on most Windows PCs (no Visual Studio / CUDA toolkit); --build compiles it."""
     if vision != "gpu":
         return vision
-    va = [int(x) for x in meta.get("vision_archs", meta.get("archs", []))]
+    va = arch_numbers(meta.get("vision_archs", meta.get("archs")))
     if va and int(gpu["arch"]) not in va and not (meta.get("ptx") and int(gpu["arch"]) > max(va)):
         warn(f"the ready-made image encoder has no code for your GPU (sm_{gpu['arch']}): it runs on the CPU instead "
              "(images take longer; setup --build compiles one for your GPU)")
@@ -5423,7 +5584,7 @@ def build_engine(gpu, vision, yes, llama, toolkit=None) -> Path:
     local = meta.get("source") == "local"
     src, vsrc = source_hash(ENGINE_SOURCES), source_hash(VISION_SOURCES)
     archs = sorted({int(x) for x in gpu.get("archs", [gpu["arch"]])})    # every card the model runs on
-    built = {int(x) for x in meta.get("archs", [])}
+    built = set(arch_numbers(meta.get("archs")))
     # a card the engine has no code for (a GPU added with --gpus, #128) needs a compile even when the source is the
     # same; the compile keeps the generations it was built for
     new_arch = local and not set(archs) <= built
@@ -5929,7 +6090,7 @@ def data_folder(requested: str | None, move: bool = False, yes: bool = True, set
                        ["keep", "move"], "keep", False) == "move"
         if held and not move:
             for folder in held:
-                say(f"  your model files stay in {folder}; {dest} is used for new downloads")
+                say(f"  Models you already have in {folder} stay there; new downloads go to {dest}.")
             if set_default:
                 save_settings({**settings, "data_dir": str(dest)})
             return dest, held
@@ -6228,6 +6389,15 @@ def choices_from_config(cfg_path: Path) -> dict:
     """The setup answers a config was written with (family, size, context, KV, images, projection, network)."""
     cfg = json.loads(cfg_path.read_text(encoding="utf-8-sig"))
     tag = cfg_path.stem[len("strata-"):]
+    big = next((f for f, d in MULTI_FILE_FAMILIES.items() if tag.startswith(d["tag"])), None)
+    if big is not None:                                # DeepSeek-V4.1-Flash: its own size; no KV / images / projection settings
+        d = MULTI_FILE_FAMILIES[big]
+        size = next((m for m in d["models"] if m.lower() == tag[len(d["tag"]):].lower()), None)
+        a = cfg.get("args", [])
+        ctx = a[a.index("--max-context") + 1] if "--max-context" in a and a.index("--max-context") + 1 < len(a) else None
+        return {"family": big, "model": size, "context": int(ctx) if ctx and ctx.isdigit() else None, "kv": None,
+                "vision": "none", "esp": "off", "host": cfg.get("host"), "api_key": cfg.get("api_key"),
+                "port": cfg.get("port"), "gpu": cfg.get("gpu"), "layer_split": None, "cuda": None, "vram_reserve_mib": None}
     own = next((f for f, d in OWN_PATH_FAMILIES.items() if tag.startswith(d["tag"])), None)
     if own is not None:                                # Qwen3.6: its own sizes; no KV / images / projection settings
         d = OWN_PATH_FAMILIES[own]
@@ -6263,11 +6433,11 @@ def choices_from_config(cfg_path: Path) -> dict:
 def calibration_targets(have: list, model: str | None, family: str | None) -> list:
     """#2036: the installed configs `--calibrate --model X` (and/or --family) means; every config with neither."""
     if not (model or family):
-        return [c for c in have if choices_from_config(c)["family"] not in OWN_PATH_FAMILIES]
+        return [c for c in have if choices_from_config(c)["family"] not in (*OWN_PATH_FAMILIES, *MULTI_FILE_FAMILIES)]
     out = []
     for c in have:
         ch = choices_from_config(c)
-        if ch["family"] in OWN_PATH_FAMILIES:          # the tuning measures Flash-Next's engine settings
+        if ch["family"] in OWN_PATH_FAMILIES or ch["family"] in MULTI_FILE_FAMILIES:   # the tuning measures Flash-Next's engine settings
             continue
         if (not model or ch["model"] == model) and (not family or ch["family"] == family):
             out.append(c)
@@ -6520,8 +6690,8 @@ def update_install(have: list, a) -> int:
         say("  No model is installed in this Strata folder yet: run START-HERE.bat (Linux: ./setup.sh) to set it up -")
         say("  it finds an earlier install's model files next to it and reuses them.")
         return 0
-    pip_install(requirement_lines() if REQUIREMENTS.exists() else PY_PACKAGES,
-                "numpy, jinja2, regex, pyyaml, tqdm, requests, pillow, psutil")
+    pip_install(runtime_packages(),
+                "numpy, jinja2, regex, pyyaml, tqdm, requests, pillow, psutil, textual (the screens)")
     if not a.build:
         update_installed_engine(a.prebuilt)
     mtp_done: set = set()
@@ -7037,7 +7207,7 @@ def ensure_engine_for(cards, cfg_path: Path, cfg: dict, yes: bool) -> dict:
     say("  The installed engine has no code for " + ", ".join(f"{g['name']} (sm_{g['arch']})" for g in missing) +
         ": it is compiled for " + ("these cards" if len(cards) > 1 else "it") + " now.")
     main = gpu_info(cards[0]["index"])
-    archs = sorted({int(x) for x in meta.get("archs", [])} | {int(g["arch"]) for g in cards})
+    archs = sorted(set(arch_numbers(meta.get("archs"))) | {int(g["arch"]) for g in cards})
     vision = meta.get("vision") or ("gpu" if (eng / VEXE).exists() else "none")
     build_engine({**main, "archs": archs}, vision, yes, get_llama_cpp(), toolkit=tk)
     dirs = json.loads(info.read_text(encoding="utf-8")).get("cuda_dirs") or []
@@ -7451,8 +7621,8 @@ def install_own_path(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adopte
         warn(f"little free disk space in {folder}: {free_gb(folder):.1f} GB free for about {left:.1f} GB. Setup goes on and should fit")
 
     step(3, "Python packages")
-    pip_install(requirement_lines() if REQUIREMENTS.exists() else PY_PACKAGES,
-                "numpy, jinja2, regex, pyyaml, tqdm, requests, cmake, ninja, pillow, psutil")
+    pip_install(runtime_packages(),
+                "numpy, jinja2, regex, pyyaml, tqdm, requests, cmake, ninja, pillow, psutil, textual (the screens)")
 
     step(4, "the Strata engine")
     llama, eng, meta, lib_dirs, gpu, vision = engine_step(a, gpu, hip, amd, cuda_tk, vision)
@@ -7562,6 +7732,390 @@ def install_own_path(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adopte
     return start(cfg_path, port)
 
 
+# DeepSeek's engine is its own package (feed member "dsv41", strata_control/dsv41_engine.py), downloaded when DeepSeek is chosen and unpacked in
+# <data>/engine-dsv41/<version>-<platform>-<backend>/ beside the engine: the Qwen engine's files are never touched.  Fetched, verified and
+# resumed like an engine update (the signed feed; --feed / STRATA_UPDATE_FEED name another one, as for `strata update`).
+DSV41_ENGINE_DIR = "engine-dsv41"
+
+
+def dsv41_engine_dir(eng: Path) -> Path:
+    """Beside the engine folder; for the app / Terminal (the engine is a component folder Strata does not write into): in Strata's own folder."""
+    return (home() if CONTROL else Path(eng).parent) / DSV41_ENGINE_DIR
+
+
+def dsv41_manual_steps(eng: Path) -> str:
+    return (f"If you downloaded the DeepSeek engine package from the release page (the file strata-dsv41-...-{'windows' if WIN else 'linux'}-....{'zip' if WIN else 'tar.gz'}), "
+            f"run the same command again with --dsv41-package PATH-OF-THE-FILE (strata launch accepts it too), or put the file in {dsv41_engine_dir(eng)} "
+            "and run it again. Strata checks it before it is used.")
+
+
+def get_dsv41_engine(eng: Path, hip: bool, package: str | None = None) -> Path | None:
+    """The DeepSeek engine package next to the engine `eng`: one the person gave (--dsv41-package) or put in engine-dsv41/ by hand, else downloaded from
+    the signed feed.  Checked first either way; its strata-dsv41, or None when it cannot be had."""
+    from strata_control import dsv41_engine
+    root = dsv41_engine_dir(eng)
+    if package:
+        exe = dsv41_engine.install_package(root, package, emit=lambda m: say("  " + m))
+        if exe is None:
+            fail("the DeepSeek engine package you gave could not be used", "see the line above; nothing else was changed")
+        return exe
+    for f in dsv41_engine.pending(root):
+        say(f"  Found {f.name} in {root}: checking it ...")
+        exe = dsv41_engine.install_package(root, f, emit=lambda m: say("  " + m))
+        if exe is not None:
+            return exe
+    acquisition.require("the DeepSeek engine")
+    from strata_cli import update_cmd
+    source = update_cmd._feed_source(argparse.Namespace(feed=None), root.parent)
+    say("  Downloading the DeepSeek engine (its own package, separate from the Strata engine) ...")
+    hw = None
+    if hip:
+        from strata_control import updater
+        hw = {**updater.detect_hardware(), "backend": "hip", "backend_source": "requested"}
+    exe = dsv41_engine.install(root, source, hardware=hw, emit=lambda m: say("  " + m))
+    if exe is None:
+        warn("no DeepSeek engine package could be fetched (see above)")
+    return exe
+
+
+def dsv41_exe(eng: Path) -> Path | None:
+    """The DeepSeek-V4.1 engine (strata-dsv41): STRATA_DSV41_EXE (a build of your own), else its own package folder beside the engine (engine-dsv41/),
+    else beside the engine in `eng` (an older all-in-one package); None when there is none."""
+    own = (os.environ.get("STRATA_DSV41_EXE") or "").strip()
+    if own:
+        return Path(own) if Path(own).is_file() else None
+    name = "strata-dsv41.exe" if WIN else "strata-dsv41"
+    from strata_control import dsv41_engine
+    exe = dsv41_engine.find(dsv41_engine_dir(eng)) or (Path(eng) / name)
+    if exe.is_file() and not WIN and not os.access(exe, os.X_OK):      # a package copied without its modes (scp / tar from Windows): fix it, do not stop
+        try:
+            exe.chmod(0o755)
+        except OSError:
+            pass
+    return exe if exe.is_file() else None
+
+
+def dsv41_folder(a, fam: dict, tag: str, model: str = "UD-Q8_K_XL") -> Path:
+    """Where the model's shards are: --gguf-dir (a folder with the shards, or the repository's folder with the set's own folder inside:
+    UD-Q8_K_XL, 2.00bpw, 2.25bpw), else the models folder's own."""
+    if a.gguf_dir:
+        folder = Path(a.gguf_dir)
+        d = fam["models"][model]
+        names = [n for s in d["shards"] for n in (s, DSV41_LEGACY_NAMES.get(s)) if n]
+        if any((folder / n).exists() for n in names):
+            return folder
+        for sub in (d.get("subdir"), d.get("legacy_subdir"), model):          # the set's own folder inside (the first release used 2.00bpw/, 2.25bpw/)
+            if sub and (folder / sub).is_dir():
+                return folder / sub
+        return folder
+    return Path(a.models_dir) / tag
+
+
+def dsv41_pick(folder: Path, name: str) -> Path:
+    """`name` in `folder`, or the file it was called in the first release (DSV41_LEGACY_NAMES; the same weights) when only that one is there;
+    else `name` (where it will be downloaded to)."""
+    if (folder / name).exists():
+        return folder / name
+    old = DSV41_LEGACY_NAMES.get(name)
+    if old and (folder / old).exists():
+        return folder / old
+    return folder / name
+
+
+def dsv41_shared_dir(a, fam: dict) -> Path:
+    """Where the files the official sets share (the Engram, the DSpark drafter, the metadata shard) are downloaded to: one folder, so
+    installing the second set does not download the 111 GB Engram again."""
+    return Path(a.models_dir) / (fam["tag"] + "shared")
+
+
+def dsv41_find_shared(a, name: str, folder: Path, shared: Path) -> Path:
+    """The file `name` (an Engram, the drafter, ...) the person already has: in the shared folder, beside the shards, or (--gguf-dir) in the
+    folder above them or one folder level down from either; else where it will be downloaded to (the shared folder).  The size must be the
+    pinned one, so a half-copied file is not taken for the file."""
+    names = [n for n in (name, DSV41_LEGACY_NAMES.get(name)) if n]            # today's name, and the first release's (the same weights)
+    roots = [shared, folder]
+    if a.gguf_dir:
+        base = Path(a.gguf_dir)
+        roots += [base, base.parent, base.parent.parent]          # (strata launch hands over the set's own folder: 2.00bpw/)
+    seen = []
+    for r in roots:
+        if r in seen:
+            continue
+        seen.append(r)
+        try:
+            kids = [r] + ([k for k in r.iterdir() if k.is_dir()] if (a.gguf_dir and r != shared) else [])
+        except OSError:
+            continue
+        for k in kids:
+            for n in names:
+                f = k / n
+                try:
+                    if f.is_file() and f.stat().st_size == DSV41_ALL_FILES[n][0]:
+                        return f
+                except OSError:
+                    pass
+    return shared / name
+
+
+def dsv41_pack_cmd(folder: Path, shards: list, engram: Path, meta: Path, dspark: Path, pack: Path) -> list:
+    """The pack recipe for an official IST-DAS set - this and nothing else: another one (--skip-tensors / --dense-override + engram_fold)
+    packs without an error and gives garbage text (P2003, 2026-10-23).  The drafter's noise token (128799) and markov rank (256) are stated:
+    the IST-DAS GGUFs do not carry them, and a pack with noise id 0 drafts deep less often (decode ~10% slower with DSpark).  --shards names the four files, so an Engram or another
+    set in the same folder is not read as part of the model."""
+    return [sys.executable, str(ROOT / "tools" / "dsv41" / "pack_gguf.py"), "--model", str(folder), "--shards", *[Path(n).name for n in shards],
+            "--engram", str(engram), "--engram-meta", str(meta), "--dspark-gguf", str(dspark),
+            "--dspark-noise-token", "128799", "--dspark-markov-rank", "256", "--out", str(pack)]
+
+
+def dsv41_engine_env(lib_dirs) -> dict:
+    env = dict(os.environ)
+    dirs = [str(d) for d in lib_dirs or [] if d]
+    if dirs:
+        env["PATH" if WIN else "LD_LIBRARY_PATH"] = os.pathsep.join(dirs + [env.get("PATH" if WIN else "LD_LIBRARY_PATH", "")])
+    return env
+
+
+def dsv41_pack_info(exe: Path, pack: Path, lib_dirs) -> tuple[bool, str]:
+    """`strata-dsv41 --pack-info --pack P`: (True, text) when every file of the pack is there (no MISSING line), else (False, text)."""
+    try:
+        r = subprocess.run([str(exe), "--pack-info", "--pack", str(pack)], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=300,
+                           env=dsv41_engine_env(lib_dirs))
+    except (OSError, subprocess.TimeoutExpired) as e:
+        return False, f"the engine could not read the pack: {e}"
+    text = (r.stdout or "") + (r.stderr or "")
+    return r.returncode == 0 and "MISSING" not in text and "all present" in text, text
+
+
+def dsv41_smoke(exe: Path, pack: Path, lib_dirs, gpus=None, timeout: int = 5400) -> tuple[bool, str]:
+    """16 tokens of a fixed prompt through the engine (tools/dsv41/smoke.py), the text checked: (True, text) or (False, why).  Takes a
+    few minutes: the engine reads what it needs from the SSD the first time."""
+    cmd = [sys.executable, str(ROOT / "tools" / "dsv41" / "smoke.py"), "--exe", str(exe), "--pack", str(pack)]
+    if gpus:
+        cmd += ["--gpus", ",".join(str(g) for g in gpus)]
+    try:
+        r = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout,
+                           env=dsv41_engine_env(lib_dirs))
+    except (OSError, subprocess.TimeoutExpired) as e:
+        return False, f"the test run did not finish: {e}"
+    text = ((r.stdout or "") + (r.stderr or "")).strip()
+    enc = getattr(sys.stdout, "encoding", None) or "utf-8"            # the reply is printed by say()/fail(): a Windows console code page cannot show DeepSeek's special tokens
+    return r.returncode == 0, text[-1500:].encode(enc, "replace").decode(enc, "replace")
+
+
+def dsv41_cards(gpu: dict, all_cards: list | None, asked: bool) -> list:
+    """The cards the model runs on: the ones asked for (--gpu / --gpus), else every card setup can use (recommend: the model is
+    larger than any one card, so the more graphics memory the more of its experts stay off the CPU)."""
+    if asked or not all_cards:
+        return list(gpu.get("cards") or [gpu])
+    return list(all_cards)
+
+
+def install_dsv41(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adopted, all_cards=None):
+    """Steps 2-7 for DeepSeek-V4.1-Flash (MULTI_FILE_FAMILIES): where it fits (strata_control/dsv41_fit.py, said plainly: a 508 GB
+    download and a model larger than most computers' memory are warnings, never a refusal), the engine (strata-dsv41), the nine
+    pinned shards (or your own, --gguf-dir), the engine's pack folder, the tokenizer and chat template from the GGUF, the config.  The
+    engine places the experts itself (as many as the cards hold in graphics memory, the rest computed on the CPU from the files)."""
+    from strata_control import dsv41_fit
+    fam = MULTI_FILE_FAMILIES[family]
+    say()
+    say(f"  {fam['title']} - {fam['by']}: {fam['about']}")
+    ok(f"model: {fam['title']}")
+    if fam.get("license"):
+        say(f"  Its license: {fam['license']}")
+    names = list(fam["models"])
+    if a.model and a.model not in names:
+        fail(f"{fam['title']} has no {a.model} model file", "choose: " + ", ".join(names))
+    if a.model:
+        model = a.model
+    else:
+        try:
+            room = free_gb(Path(a.models_dir) if Path(a.models_dir).exists() else Path(a.models_dir).parent)
+        except Exception:                                    # noqa: BLE001 - the disk is only a hint here
+            room = None
+        rec = dsv41_fit.recommend(ram, room)                 # the best version this computer's memory and disk suit; warn-never-refuse
+        for i, m in enumerate(names, 1):
+            say(f"  {i}) {fam['models'][m].get('label', m)}: {fam['models'][m]['about']}" + ("   (recommended)" if m == rec else ""))
+        model = names[int(ask("Which version?", [str(i) for i in range(1, len(names) + 1)], str(names.index(rec) + 1), a.yes)) - 1]
+    ok(f"size: {model}")
+    md = fam["models"][model]
+    say("  Its GGUF files: " + ("IST-DAS / ISTA-DASLab (huggingface.co/ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF), with the DSpark drafter and a small metadata file from Unsloth"
+                              if md.get("recipe") == "ista" else "Unsloth (huggingface.co/unsloth/DeepSeek-V4.1-Flash-1017-GGUF)"))
+    ista = md.get("recipe") == "ista"
+    tag = fam["tag"] + model
+    cards = dsv41_cards(gpu, all_cards, bool(a.gpu is not None or a.gpus))
+    ctx = a.context or DSV41_CONTEXT
+    folder = dsv41_folder(a, fam, tag, model)
+    shards = [dsv41_pick(folder, n) for n in md["shards"]]
+    shared_dir = dsv41_shared_dir(a, fam)
+    shared = [dsv41_find_shared(a, n, folder, shared_dir) for n in md.get("shared", [])]
+    own_files = {f: bool(a.gguf_dir) or f.parent != shared_dir for f in shared}         # a file found outside setup's own folder is the person's
+
+    def partial(f: Path) -> int:
+        p = f.with_name(f.name + ".part")
+        return p.stat().st_size if p.is_file() else 0
+
+    left = sum(0 if f.exists() and (done(f) or a.gguf_dir or own_files.get(f) or f.name in DSV41_LEGACY_FILES) else fam["sha256"][f.name][0] - partial(f)
+               for f in [*shards, *shared]) / 1e9
+    probe = folder if folder.exists() else (folder.parent if folder.parent.exists() else Path(a.models_dir))
+    free = free_gb(probe)
+    plan = dsv41_fit.plan([{**c, "vram_gb": float(c.get("vram_gb") or 0)} for c in cards], ram, free if left else None, ctx, model=model)
+    say()
+    for line in plan["lines"]:
+        say("  " + line)
+    if left:
+        say(f"  Download: {left:.0f} GB to fetch (checked against pinned sizes and SHA-256; it resumes if it stops). At 50 MB/s that "
+            f"is {left * 1e9 / 50e6 / 3600:.1f} hours. Files you already have: --gguf-dir DIR.")
+    else:
+        say("  Nothing to download: the files are already here.")
+    unchecked = [f for f in [*shards, *shared] if f.exists() and f"sha256 {fam['sha256'][f.name][1]}" not in
+                 (f.with_name(f.name + ".done").read_text(encoding="utf-8", errors="replace") if done(f) else "")]
+    if unchecked:
+        gb = sum(fam["sha256"][f.name][0] for f in unchecked) / 1e9
+        say(f"  The files already here ({gb:.0f} GB) are checked against their SHA-256 once, now: about {max(1, round(gb / 55))} minutes on a fast "
+            "drive (4-5 minutes for the 257 GB of 2.00), longer on a slow one. The next runs skip it.")
+    for w in plan["warnings"]:
+        if "free disk space" in w:
+            continue                                                      # the disk is a hard limit, said below
+        warn(w)
+    if plan["verdict"] == "spills":
+        confirm_risk(f"{model} runs best with about {plan['ram_needed_gb']:.0f} GB of RAM beside this PC's graphics memory and this PC has "
+                     f"{ram:.0f} GB: most of its experts are read from the SSD while it answers (slow)", bool(a.model), a.yes,
+                     f"{model} needs about {plan['ram_needed_gb']:.0f} GB of RAM", f"more RAM or cards fix it; --model {model} --yes "
+                     "installs it anyway", "  Install it anyway?")
+        warn(f"installing {model} with {ram:.0f} GB of RAM, as you chose")
+    if left and free < left:
+        fail(f"not enough free disk space in {folder}: need ~{left:.0f} GB, {free:.0f} GB free",
+             "use --models-dir on a bigger drive (an NVMe drive is best: the n-gram table is read from it while the model "
+             "answers), or --gguf-dir for files you already have")
+    if left and free < left + dsv41_fit.DISK_HEADROOM_GB:
+        warn(f"little free disk space in {folder}: {free:.1f} GB free for about {left:.1f} GB. Setup goes on and should fit")
+    if ctx > 65536:
+        warn(f"{ctx // 1024}K tokens of context: the engine was checked at 4K and 8K; longer is kept as you chose, but untested")
+    if a.vision in ("yes", "gpu", "cpu"):
+        warn(f"{fam['title']} cannot read pictures in this Strata version (its picture reader is in development): images off")
+    if a.rope_scaling or a.rope_scale or a.kv or a.resident_budget_gib is not None or a.low_ram not in (None, "auto"):
+        warn(f"--rope-scaling / --kv / --resident-budget-gib / --low-ram are Flash-Next settings: {fam['title']} does not use them")
+
+    step(3, "Python packages")
+    pip_install(runtime_packages(),
+                "numpy, jinja2, regex, pyyaml, tqdm, requests, cmake, ninja, pillow, psutil, textual (the screens)")
+
+    step(4, "the Strata engine")
+    llama, eng, meta, lib_dirs, gpu, _vision = engine_step(a, gpu, hip, amd, cuda_tk, "none")
+    exe = dsv41_exe(eng)
+    if exe is None and not getattr(a, "build", False):
+        get_dsv41_engine(eng, hip, getattr(a, "dsv41_package", None))
+        exe = dsv41_exe(eng)
+    if exe is not None and exe.parent != Path(eng):
+        lib_dirs = [*lib_dirs, str(exe.parent)]            # its own package carries the libraries it needs
+    if exe is None:
+        if os.environ.get("STRATA_VERBOSE") == "1" or "--verbose" in sys.argv:
+            say("  Details for developers: build it with cmake -DSTRATA_BUILD_DSV41=ON and point STRATA_DSV41_EXE at it.")
+        fail("DeepSeek's engine is not available for this computer yet",
+             "Supported graphics cards: NVIDIA RTX 30 and RTX 50 series, and the Radeon AI PRO R9700. " + dsv41_manual_steps(eng)
+             + " Or update Strata later (a newer version may add your card), or pick another model. Nothing has been downloaded.")
+    ok(f"engine: {exe}")
+
+    step(5, f"downloading {fam['title']} {model}")
+    for f in shards:
+        if f.exists() and done(f):
+            ok(f"{f.name} already downloaded")
+        else:
+            if a.gguf_dir and not f.exists():
+                fail(f"{f} is missing", f"put {f.name} in {folder}, or drop --gguf-dir to download it")
+            if not a.gguf_dir and f.name not in DSV41_LEGACY_FILES:      # (a first-release file is checked, never fetched under its old name)
+                download((md.get("hf") or fam["hf"]) + f.name, f)
+        if f.name in DSV41_LEGACY_FILES:
+            ok(f"{f.name}: a file of the first release of this set (the same weights); it is used as it is")
+        verify_sha256(f, *fam["sha256"][f.name], own=bool(a.gguf_dir) or f.name in DSV41_LEGACY_FILES)
+    for f in shared:                                  # the Engram, the drafter, the metadata shard: once for every official set
+        if f.exists() and (done(f) or own_files[f]):
+            ok(f"{f.name} already here ({f.parent})")
+        else:
+            download(DSV41_SHARED[f.name] + f.name, f)
+        verify_sha256(f, *fam["sha256"][f.name], own=own_files[f])
+    ok(f"model files: {folder} ({len(shards)} shards" + (f"; shared files: {shared_dir if any(not own_files[f] for f in shared) else shared[0].parent}" if shared else "") + ")")
+
+    step(6, "preparing the model for Strata")
+    data, _ = data_folder_for(a)
+    pack = data / "packs" / tag.lower()            # the engine's map of the shards, the hash tables, the tokenizer: a few MB
+    have_pack = (pack / "tokenizer" / "vocab.json").exists() and (pack / "tensors.txt").exists()
+    if have_pack and ista:                         # a pack from an earlier run is kept only when the engine finds all of its files
+        good, why = dsv41_pack_info(exe, pack, lib_dirs)
+        if not good:
+            warn("the pack from an earlier run is incomplete; building it again")
+            shutil.rmtree(pack, ignore_errors=True)
+            have_pack = False
+    if not have_pack:
+        if ista:
+            run(dsv41_pack_cmd(folder, [f.name for f in shards], shared[0], shared[1], shared[2], pack))
+        else:
+            run([sys.executable, str(ROOT / "tools" / "dsv41" / "pack_gguf.py"), "--model", str(folder), "--out", str(pack)])
+    if ista:
+        prof = DSV41_PROFILES / md["profile"]          # which experts are used most: the RAM store starts with them (the engine reads <pack>/expert-profile.bin)
+        if prof.is_file() and not (pack / "expert-profile.bin").exists():
+            pack.mkdir(parents=True, exist_ok=True)
+            shutil.copyfile(prof, pack / "expert-profile.bin")
+        good, why = dsv41_pack_info(exe, pack, lib_dirs)
+        if not good:
+            tail = "\n".join(l for l in why.splitlines() if "MISSING" in l)[:600]
+            fail(f"the model's pack is not complete ({tail or why[-300:]})",
+                 f"delete {pack} and run setup again; if it is still incomplete, the files in {folder} may be damaged (setup checks them)")
+        ok(f"pack: {pack} (checked: every file the engine needs is there)")
+        if os.environ.get("STRATA_DSV41_NO_SMOKE") == "1":
+            warn("the 16-word test run was skipped (STRATA_DSV41_NO_SMOKE=1)")
+        else:
+            say("  Testing the model with a short question (a few minutes: the engine reads the first experts from the disk) ...")
+            good, why = dsv41_smoke(exe, pack, lib_dirs, [c["index"] for c in cards] if (a.gpu is not None or a.gpus) else None)
+            if not good:
+                fail("the model does not answer sensibly after the preparation: " + (why.splitlines()[-1] if why else "no answer"),
+                     f"delete {pack} and run setup again so the pack is built fresh; if the test fails again, "
+                     f"tell us (the log is in {home()})")
+            ok("test question answered sensibly")
+    else:
+        ok(f"pack: {pack}")
+
+    step(7, "writing the start script")
+    # --dspark auto, never off: "off" still LOADS the drafter, and its 7 GB of experts take the graphics memory (found 2026-10-23);
+    # auto keeps it unloaded when the experts are far larger than the card.  The RAM store sizes itself (--ssd-cache auto is the
+    # engine's default), and <pack>/expert-profile.bin is read from the pack folder without a flag.  The layer-major chunked prompt is the engine's own default now (f-dsv41-rel d6f18babe: 1024 on a single card,
+    # its own choice for several), so no --prefill-chunk is written.  --dspark-conf 0.65: the
+    # drafts' confidence floor (H100 replay, DSPARK-DEFAULTS.md: within 0.01-0.02 of the per-class best, never below 0.3).  The engine has
+    # no flag for the draft block (3 in the same table): it reads the block size from the pack, so only the floor is written.
+    args = ["--pack", str(pack), "--max-context", str(ctx), "--eos", "1", "--dspark", "auto",
+            *(["--dspark-conf", "0.65"] if md.get("recipe") == "ista" else []), *plan["args"]]
+    cfg = {"exe": str(exe), "args": args, "cwd": str(ROOT), "tokenizer": str(pack / "tokenizer"),
+           "model_name": fam["name"], "arch": fam["arch"], "log": str(home() / f"strata-{tag.lower()}.log"),
+           "lib_dirs": lib_dirs, "port": port, "engine_ready_s": 3600}      # 300+ GB read from the files: a slow disk needs its time
+    if cuda_tk == 12:
+        cfg["cuda"] = 12
+    if hip:
+        cfg["backend"] = "hip"
+    idx = [c["index"] for c in cards]
+    if len(idx) > 1 or a.gpu is not None or a.gpus:
+        cfg["gpu"] = idx if len(idx) > 1 else idx[0]
+        cfg["gpus_asked"] = True
+    if a.host:
+        cfg["host"] = a.host
+    if a.api_key:
+        cfg["api_key"] = a.api_key
+    if a.browser is not None:
+        cfg["open_browser"] = a.browser
+    cfg_path = home() / f"strata-{tag.lower()}.json"
+    write_setup_config(cfg_path, cfg, adopted if adopted is not None and adopted.name == cfg_path.name else None, explicit_args(a))
+    script = write_run_script(tag, cfg_path, port, cfg.get("open_browser") is not False)
+    ok(f"start script: {script.name}")
+    say()
+    say("All set.")
+    say(f"  API (OpenAI):     http://127.0.0.1:{port}/v1   (any API key; model name: anything)")
+    say(f"  API (Anthropic):  http://127.0.0.1:{port}/v1/messages")
+    say(f"  Next time:        just run {'START-HERE.bat' if WIN else './setup.sh'} (or {script.name}) - it starts right away")
+    say("  Loading takes a few minutes: the CPU's share of the experts is read from the files first.")
+    if a.no_start:
+        return 0
+    return start(cfg_path, port)
+
+
 def preview_family(a) -> int:
     """--family qwen36: a family setup knows but the engine does not run yet (PREVIEW_FAMILIES).  Downloads the chosen
     size (pinned revision, size and SHA-256 checked) and, with --vision, the image encoder - and says plainly that
@@ -7822,11 +8376,13 @@ def main() -> int:
     if sys.platform == "darwin" and not CONTROL:       # a Mac: metal/setup_mac.py (the Metal engine); the app (Control mode) goes on below
         return mac_setup(sys.argv[1:])
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("--family", choices=list(FAMILIES) + list(PREVIEW_FAMILIES),
+    ap.add_argument("--family", choices=list(FAMILIES) + list(PREVIEW_FAMILIES) + list(MULTI_FILE_FAMILIES),
                     help="qwen = Qwen3.8-Flash-Next, swift = Swift 1.5, qwen36 = Qwen3.6-35B-A3B (pictures with --vision, "
-                         "several requests at once with --parallel; docs/QWEN36.md)")
+                         "several requests at once with --parallel; docs/QWEN36.md), deepseek41 = DeepSeek-V4.1-Flash "
+                         "(the official 1.75 / 2.00 / 2.25 bpw versions, 239 / 257 / 274 GB; docs/DEEPSEEK41.md)")
     ap.add_argument("--model", choices=list(dict.fromkeys([*MODELS, *(m for f in PREVIEW_FAMILIES.values()
-                                                                       for m in f["models"])])))
+                                                                       for m in f["models"]),
+                                                          *(m for f in MULTI_FILE_FAMILIES.values() for m in f["models"])])))
     ap.add_argument("--context", type=int)
     ap.add_argument("--rope-scaling", choices=["none", "linear", "yarn"],
                     help="the RoPE extension for a context past the model's trained 262144: linear (position "
@@ -7888,6 +8444,8 @@ def main() -> int:
     ap.add_argument("--set-default-data-dir", action="store_true",
                     help="with --data-dir: remember the new folder as the data folder without moving anything")
     ap.add_argument("--models-dir", help="where the GGUF files go (default: <data folder>/models)")
+    ap.add_argument("--dsv41-package", metavar="FILE", help="DeepSeek: the engine package you downloaded from the release page (a .tar.gz or .zip, "
+                                                           "or its unpacked folder); it is checked and installed instead of being fetched")
     ap.add_argument("--gguf-dir", help="use GGUF files you already have (a folder with every shard: "
                                        "<name>-00001-of-0000N.gguf ... -0000N-of-0000N.gguf)")
     ap.add_argument("--yes", action="store_true", help="accept the recommended answers")
@@ -7951,8 +8509,8 @@ def main() -> int:
                          "sycl = Intel Arc, EXPERIMENTAL: Linux, built from source (docs/INTEL_ARC.md)")
     ap.add_argument("--skip-build", action="store_true", help=argparse.SUPPRESS)
     a = ap.parse_args()
-    if a.model and a.model not in MODELS and a.family not in PREVIEW_FAMILIES:
-        ap.error(f"--model {a.model} is a size of " + ", ".join(f"--family {f}" for f, d in PREVIEW_FAMILIES.items()
+    if a.model and a.model not in MODELS and a.family not in PREVIEW_FAMILIES and a.family not in MULTI_FILE_FAMILIES:
+        ap.error(f"--model {a.model} is a size of " + ", ".join(f"--family {f}" for f, d in {**PREVIEW_FAMILIES, **MULTI_FILE_FAMILIES}.items()
                                                               if a.model in d["models"]))
     if a.source:
         os.environ["STRATA_SOURCE"] = a.source
@@ -7996,7 +8554,9 @@ def main() -> int:
             a.gpus = ",".join(str(i) for i in order)
         else:
             a.gpu = order[0]
-    say("Strata - Qwen3.8-Flash-Next on a normal PC (a GPU + system RAM + CPU)")
+    say("Strata - " + (MULTI_FILE_FAMILIES[a.family]["title"] if a.family in MULTI_FILE_FAMILIES else
+                       OWN_PATH_FAMILIES[a.family]["title"] if a.family in OWN_PATH_FAMILIES else "Qwen3.8-Flash-Next")
+        + " on a normal PC (a GPU + system RAM + CPU)")
     have = installed_configs()
     # A plain start of an installed model finds everything where it already is: it creates, moves and copies nothing
     # (the model paths are in the run config; the data folder is the one settings.json remembers).  Setting up, checking,
@@ -8283,6 +8843,20 @@ def main() -> int:
             else:
                 any_fits = any_fits or (not verdict.startswith("does not fit") and not d.get("mem_total"))
             say(f"  {m:8s} needs ~{d['ram_gb']} GB {'of VRAM + RAM' if d.get('mem_total') else 'RAM'}: {verdict}")
+        try:                                           # DeepSeek-V4.1-Flash: a line, from the real numbers (strata_control/dsv41_fit.py)
+            from strata_control import dsv41_fit
+            cards_ok = amd_ok if hip else [g for g in found if gpu_problem(g) is None]
+            for mname in ("IST-DAS-2.00bpw", "IST-DAS-2.25bpw", "IST-DAS-1.75bpw", "UD-Q8_K_XL"):
+                ds = dsv41_fit.plan(cards_ok, ram, None, model=mname)
+                tail = {"fits": f"fits ({ds['experts_in_vram']:,} of {ds['experts_total']:,} experts in graphics memory, the rest on the CPU)",
+                        "tight": ("runs, with the experts RAM cannot hold read from the SSD (about 5-7 tokens a second measured)"
+                                  if ds.get("variant") else "fits, but RAM is tight"),
+                        "spills": (f"works but slow: less RAM than it was measured with (46-64 GB)" if ds.get("variant")
+                                   else f"about {ds['spill_gb']:.0f} GB of its experts would be read from the SSD (slow)"),
+                        "no_card": "needs one or more graphics cards"}.get(ds["verdict"], "not checked")
+                say(f"  DeepSeek-V4.1-Flash {mname}: a {ds['download_gb']:.0f} GB download: {tail}   (--family deepseek41 --model {mname})")
+        except Exception:                              # noqa: BLE001 - a line for information: never break the check
+            pass
         if not any_fits:
             # #977: every size says "does not fit", so the verdict says so too (and the exit code, for scripts). It only
             # reports: --model NAME --yes still installs one anyway.
@@ -8305,16 +8879,20 @@ def main() -> int:
             say(f"  {i}) {d['title']:20s} {d['by']} - {d['about']}" + ("   [experimental]" if d.get("experimental") else "")
                 + family_menu_note(f, hip, gpu, avx2, avx512)
                 + (f"   (recommended for Strix Halo: {STRIX_HALO_MODEL})" if f == rec_fam and rec_fam != fams[0] else ""))
-        for j, f in enumerate(OWN_PATH_FAMILIES, len(fams) + 1):   # families with an engine path of their own
-            d = OWN_PATH_FAMILIES[f]
+        for j, f in enumerate([*OWN_PATH_FAMILIES, *MULTI_FILE_FAMILIES], len(fams) + 1):   # families with an engine path of their own
+            d = {**OWN_PATH_FAMILIES, **MULTI_FILE_FAMILIES}[f]
             say(f"  {j}) {d['title']:20s} {d['by']} - {d['about']}   [new]")
-        allf = fams + list(OWN_PATH_FAMILIES)
+        allf = fams + list(OWN_PATH_FAMILIES) + list(MULTI_FILE_FAMILIES)
         if tier_note(ram, low_ram_vram(gpu)) and rec_fam == "swift":
             say("  " + tier_note(ram, low_ram_vram(gpu)))
         family = allf[int(ask("Which model?", [str(i) for i in range(1, len(allf) + 1)],
                               str(allf.index(rec_fam) + 1), a.yes)) - 1]
     if family in OWN_PATH_FAMILIES:                    # Qwen3.6: its own engine path, its own steps 2-7
         return install_own_path(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adopted)
+    if family in MULTI_FILE_FAMILIES:                  # DeepSeek-V4.1-Flash: nine shards, its own engine, several cards
+        all_cards = amd_ok if hip else [g for g in found if gpu_problem(g) is None]
+        gpu = {**gpu, "cards": chosen}
+        return install_dsv41(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adopted, all_cards)
     fam = FAMILIES[family]
     if fam.get("ready") and hip:
         # not a risk the user could take: the HIP engine has no NVFP4 expert kernels at all, so it would not start
@@ -8637,8 +9215,8 @@ def main() -> int:
 
     # ---- 3. python packages
     step(3, "Python packages")
-    pip_install(requirement_lines() if REQUIREMENTS.exists() else PY_PACKAGES,
-                "numpy, jinja2, regex, pyyaml, tqdm, requests, pillow, psutil")
+    pip_install(runtime_packages(),
+                "numpy, jinja2, regex, pyyaml, tqdm, requests, pillow, psutil, textual (the screens)")
 
     # ---- 4. the engine
     step(4, "the Strata engine")

src/dsv41/ds4_attn_split.cuh

@@ -0,0 +1,206 @@
+// Split-KV sparse attention: Strata (dsv41 CUDA lab), used with the DeepSeek attention of Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// (the surrounding types, AttnRows and the packed-row formats, are theirs; see third_party/project-maya/NOTICE.md).
+// src/dsv41/ds4_attn_split.cuh - split-KV ("flash-decoding") version of DeepSeek's sparse attention for a few queries
+// (decode T = 1, DSpark verify T = 2..5): 64 heads x d 512, MQA (K = V), window rows (BF16) + selected compressed rows (kKvFp4:
+// E2M1 codes + E4M3 scales per 16) + a learned sink logit per head.  Same inputs and outputs as strata::ds4::sparse_attn
+// (AttnRows gen 0, ckv kKvFp4); the shipped kernel runs one 256-thread block per (head, query) = 64 blocks on 48 SMs and takes
+// ~125 us per query; this one splits the keys over R-row chunks and groups G heads per block, which share the decoded rows.
+//   phase 1: block (head group, chunk, query): decode R rows to BF16 in shared memory (exact: window rows are BF16, packed
+//            rows have <= 7 significant bits), scores for G heads (warp per row, lanes 16 values, shuffle reduction), local
+//            softmax, partial output (thread = 2 output values) -> scratch {m, l, o[512]} per (query, head, chunk)
+//   phase 2: block (head, query): merge the chunks and the sink.
+// Plain C++ (no PTX), shuffles and shared memory only: the HIP port is a rename.
+#pragma once
+#include "ds4_kernels.cuh"
+#include "ds4_numerics.cuh"
+#include <cuda_fp16.h>
+
+namespace strata::ds4 {
+namespace attnsplit {
+
+
+
+__device__ __forceinline__ float bf2f_(uint16_t b) { return __uint_as_float((uint32_t) b << 16); }
+__device__ __forceinline__ uint16_t f2bf_exact(float f) { return (uint16_t) (__float_as_uint(f) >> 16); }   // exact values only
+__device__ __forceinline__ float e2m1_x2p14(uint32_t c) {
+    return __half2float(__ushort_as_half((unsigned short) (((c & 8u) << 12) | ((c & 7u) << 9))));
+}
+__device__ __forceinline__ float wsum(float v) {
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
+    return v;
+}
+__device__ __forceinline__ float wmax(float v) {
+#pragma unroll
+    for (int o = 16; o > 0; o >>= 1) v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, o));
+    return v;
+}
+
+// d = 512 only (a thread owns 2 output values, a lane 16 values of a row)
+template <int G, int R>
+__global__ void __launch_bounds__(256) attn_part(const float* __restrict__ q, const uint16_t* __restrict__ win,
+                                                 const uint8_t* __restrict__ ckv, size_t ckv_row_bytes, const int* __restrict__ sel,
+                                                 int sel_stride, AttnRows ar, int heads, float scale, float* __restrict__ part,
+                                                 float* __restrict__ ml, int nsplit) {
+    constexpr int D = 512;
+    __shared__ __align__(16) uint16_t rows[R][D];
+    __shared__ float sc[G][R];
+    __shared__ float4 pr[R][(G + 3) / 4];
+    __shared__ float mloc[G], lloc[G];
+    const int hg = blockIdx.x, chunk = blockIdx.y, t = blockIdx.z;
+    const int n_win = ar.nwin[t], n_sel = ar.nsel[t], total = n_win + n_sel;
+    const int j0 = chunk * R;
+    if (j0 >= total) return;
+    const int nr = min(R, total - j0);
+    const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
+    // q of this block's heads: lane's 16 values per head in registers (issued first: its latency overlaps the row loads)
+    float qv[G][16];
+#pragma unroll
+    for (int g = 0; g < G; ++g) {
+        const float4* qp = (const float4*) (q + ((size_t) t * heads + hg * G + g) * D + lane * 16);
+#pragma unroll
+        for (int i = 0; i < 4; ++i) { const float4 v = qp[i]; qv[g][4 * i] = v.x; qv[g][4 * i + 1] = v.y; qv[g][4 * i + 2] = v.z; qv[g][4 * i + 3] = v.w; }
+    }
+    // ---- stage rows: warp per row, lane = 16 values; all of a warp's row loads are issued before any is converted
+    constexpr int RPW = (R + 7) / 8;
+    uint4 wa[RPW][2];
+    uint2 cwv[RPW];
+    uint32_t sbv[RPW];
+    int rowoff[RPW];
+#pragma unroll
+    for (int i = 0; i < RPW; ++i) {
+        const int jj = wp + 8 * i, j = j0 + jj;
+        rowoff[i] = -1;
+        if (jj < nr) {
+            if (j < n_win) {
+                const uint4* src = (const uint4*) (win + (size_t) (ar.win0[t] + j) * D + lane * 16);
+                wa[i][0] = src[0];
+                wa[i][1] = src[1];
+            } else {
+                rowoff[i] = sel[(size_t) t * sel_stride + (j - n_win)];
+            }
+        }
+    }
+#pragma unroll
+    for (int i = 0; i < RPW; ++i)
+        if (rowoff[i] >= 0) {
+            const uint8_t* r = ckv + (size_t) rowoff[i] * ckv_row_bytes;
+            cwv[i] = *(const uint2*) (r + lane * 8);
+            sbv[i] = r[D / 2 + lane];
+        }
+#pragma unroll
+    for (int i = 0; i < RPW; ++i) {
+        const int jj = wp + 8 * i;
+        if (jj >= nr) continue;
+        uint4* dst = (uint4*) &rows[jj][lane * 16];
+        if (rowoff[i] < 0) {
+            dst[0] = wa[i][0];
+            dst[1] = wa[i][1];
+        } else {
+            const float s = e4m3_value((uint8_t) sbv[i]) * 16384.0f;   // (the nibbles decode to value * 2^-14)
+            uint32_t o[8];
+#pragma unroll
+            for (int w = 0; w < 2; ++w) {
+                const uint32_t c = w ? cwv[i].y : cwv[i].x;
+#pragma unroll
+                for (int x = 0; x < 4; ++x) {
+                    const uint32_t b = c >> (8 * x);
+                    const uint16_t lo = f2bf_exact(e2m1_x2p14(b) * s), hi = f2bf_exact(e2m1_x2p14(b >> 4) * s);
+                    o[4 * w + x] = (uint32_t) lo | ((uint32_t) hi << 16);
+                }
+            }
+            dst[0] = make_uint4(o[0], o[1], o[2], o[3]);
+            dst[1] = make_uint4(o[4], o[5], o[6], o[7]);
+        }
+    }
+    __syncthreads();
+    // ---- scores
+    for (int jj = wp; jj < nr; jj += 8) {
+        const uint4 a = *(const uint4*) &rows[jj][lane * 16], b = *(const uint4*) &rows[jj][lane * 16 + 8];
+        float kv[16];
+        const uint32_t w[8] = {a.x, a.y, a.z, a.w, b.x, b.y, b.z, b.w};
+#pragma unroll
+        for (int i = 0; i < 8; ++i) { kv[2 * i] = __uint_as_float(w[i] << 16); kv[2 * i + 1] = __uint_as_float(w[i] & 0xffff0000u); }
+#pragma unroll
+        for (int g = 0; g < G; ++g) {
+            float s = 0.f;
+#pragma unroll
+            for (int i = 0; i < 16; ++i) s += qv[g][i] * kv[i];
+            s = wsum(s);
+            if (lane == 0) sc[g][jj] = s * scale;
+        }
+    }
+    __syncthreads();
+    // ---- local softmax: warp g handles head g
+    if (wp < G) {
+        float m = -INFINITY;
+        for (int j = lane; j < nr; j += 32) m = fmaxf(m, sc[wp][j]);
+        m = wmax(m);
+        float l = 0.f;
+        for (int j = lane; j < nr; j += 32) { const float p = __expf(sc[wp][j] - m); sc[wp][j] = p; l += p; }
+        l = wsum(l);
+        if (lane == 0) { mloc[wp] = m; lloc[wp] = l; }
+    }
+    __syncthreads();
+    for (int i = threadIdx.x; i < nr * G; i += 256) { const int j = i / G, g = i % G; ((float*) &pr[j][0])[g] = sc[g][j]; }
+    __syncthreads();
+    // ---- partial output: thread = values 2 tid, 2 tid + 1 of every head
+    float o0[G], o1[G];
+#pragma unroll
+    for (int g = 0; g < G; ++g) o0[g] = o1[g] = 0.f;
+    const int c = threadIdx.x * 2;
+    for (int j = 0; j < nr; ++j) {
+        const uint32_t kw = *(const uint32_t*) &rows[j][c];
+        const float k0 = __uint_as_float(kw << 16), k1 = __uint_as_float(kw & 0xffff0000u);
+        const float* p = (const float*) &pr[j][0];
+#pragma unroll
+        for (int g = 0; g < G; ++g) { o0[g] += p[g] * k0; o1[g] += p[g] * k1; }
+    }
+#pragma unroll
+    for (int g = 0; g < G; ++g) {
+        const size_t bi = ((size_t) t * heads + hg * G + g) * nsplit + chunk;
+        *(float2*) (part + bi * D + c) = make_float2(o0[g], o1[g]);
+        if (threadIdx.x == 0) { ml[bi * 2] = mloc[g]; ml[bi * 2 + 1] = lloc[g]; }
+    }
+}
+
+__global__ void __launch_bounds__(256) attn_merge(const float* __restrict__ part, const float* __restrict__ ml, const float* __restrict__ sink,
+                                                  int heads, int nsplit, int R, AttnRows ar, float* __restrict__ out) {
+    constexpr int D = 512;
+    const int h = blockIdx.x, t = blockIdx.y;
+    const int total = ar.nwin[t] + ar.nsel[t];
+    const int ns = (total + R - 1) / R;
+    const size_t base = ((size_t) t * heads + h) * nsplit;
+    float M = sink[h];
+    for (int s = 0; s < ns; ++s) M = fmaxf(M, ml[(base + s) * 2]);
+    float den = __expf(sink[h] - M);
+    for (int s = 0; s < ns; ++s) den += ml[(base + s) * 2 + 1] * __expf(ml[(base + s) * 2] - M);
+    const int c = threadIdx.x * 2;
+    float a0 = 0.f, a1 = 0.f;
+    for (int s = 0; s < ns; ++s) {
+        const float w = __expf(ml[(base + s) * 2] - M);
+        const float2 v = *(const float2*) (part + (base + s) * D + c);
+        a0 += w * v.x;
+        a1 += w * v.y;
+    }
+    *(float2*) (out + ((size_t) t * heads + h) * D + c) = make_float2(a0 / den, a1 / den);
+}
+
+struct AttnSplitScratch {
+    float* part = nullptr;   // [T][heads][nsplit][512]
+    float* ml = nullptr;     // [T][heads][nsplit][2]
+    int nsplit = 0, max_t = 0;
+};
+
+// total_max = the largest nwin + nsel of the T queries
+template <int G, int R>
+void sparse_attn_split(const float* q, const uint16_t* win, const uint8_t* ckv, size_t ckv_row_bytes, const int* sel, int sel_stride,
+                       const AttnRows& ar, const float* sink, int heads, int T, float scale, float* out, const AttnSplitScratch& sc,
+                       int total_max, cudaStream_t s) {
+    const int nsplit = (total_max + R - 1) / R;
+    attn_part<G, R><<<dim3(heads / G, nsplit, T), 256, 0, s>>>(q, win, ckv, ckv_row_bytes, sel, sel_stride, ar, heads, scale, sc.part, sc.ml, sc.nsplit);
+    attn_merge<<<dim3(heads, T), 256, 0, s>>>(sc.part, sc.ml, sink, heads, sc.nsplit, R, ar, out);
+}
+
+}  // namespace attnsplit
+}  // namespace strata::ds4

src/dsv41/ds4_engram_io.hpp

@@ -0,0 +1,275 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_engram_io.hpp - DeepSeek's Engram (n-gram memory) rows read off the decode's critical path.
+//
+// A token's Engram rows depend only on the token ids, so they are asked for as soon as the window's tokens are known
+// and collected only right before the layer that adds them (layer 1's hide behind the embedding and layer 0, layer
+// 14's behind layers 0-13).  The reads run on the reader's own threads, each with its own handles to the model's files
+// (on Windows the ReadFile calls on one synchronous handle run one at a time), FIFO: the first Engram layer's rows
+// first.  A bounded row cache (clock eviction) keeps rows already fetched: a row that recurs (the same n-gram again,
+// in the prompt or later) is a copy, not a read.  The rows are the file's bytes either way, so the output is the same.
+#pragma once
+
+#include "ds4_os.hpp"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <deque>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace strata::ds4 {
+
+inline int64_t now_ns() {
+    return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch())
+        .count();
+}
+
+/// Rows already fetched: key -> `bytes` of payload, at most `cap` rows (grown on demand in blocks), clock eviction.
+/// Not thread-safe (the caller locks).
+class EgRowCache {
+public:
+    void init(size_t cap, size_t bytes) {
+        cap_ = cap;
+        bytes_ = bytes;
+        map_.reserve(std::min<size_t>(cap, 1u << 20));
+    }
+    size_t capacity() const { return cap_; }
+    size_t size() const { return keys_.size(); }
+    /// The row's payload, or nullptr (a hit marks it recently used).
+    const uint8_t* find(uint64_t key) {
+        const auto it = map_.find(key);
+        if (it == map_.end()) return nullptr;
+        ref_[it->second] = 1;
+        return at(it->second);
+    }
+    /// Room for key's payload (the caller fills it): a new slot while below capacity, else the clock's victim.
+    uint8_t* put(uint64_t key) {
+        if (cap_ == 0 || bytes_ == 0) return nullptr;
+        const auto it = map_.find(key);
+        if (it != map_.end()) return at(it->second);
+        uint32_t s;
+        if (keys_.size() < cap_) {
+            s = (uint32_t) keys_.size();
+            if (s % kBlock == 0) blocks_.emplace_back(new uint8_t[(size_t) kBlock * bytes_]);
+            keys_.push_back(key);
+            ref_.push_back(0);
+        } else {
+            while (ref_[hand_]) {   // (a second chance for rows used since the hand last passed)
+                ref_[hand_] = 0;
+                hand_ = (hand_ + 1) % (uint32_t) cap_;
+            }
+            s = hand_;
+            hand_ = (hand_ + 1) % (uint32_t) cap_;
+            map_.erase(keys_[s]);
+            keys_[s] = key;
+            ref_[s] = 0;
+        }
+        map_.emplace(key, s);
+        return at(s);
+    }
+
+private:
+    static constexpr uint32_t kBlock = 4096;
+    uint8_t* at(uint32_t s) { return blocks_[s / kBlock].get() + (size_t) (s % kBlock) * bytes_; }
+    size_t cap_ = 0, bytes_ = 0;
+    uint32_t hand_ = 0;
+    std::unordered_map<uint64_t, uint32_t> map_;
+    std::vector<uint64_t> keys_;
+    std::vector<uint8_t> ref_;
+    std::vector<std::unique_ptr<uint8_t[]>> blocks_;
+};
+
+/// One row to fetch: its bytes to dst, its scales to scales.  mx8: n bytes at off of a file, [scale, 32 codes] blocks
+/// (a GGUF's MXFP8 row); else n row bytes at off and n2 scale bytes at off2 of file2 (two reads; n2 0: none - a GGUF's
+/// Q6_K or IQ4_NL row, their scales inside their n bytes: 210 a 256 values, 18 a 32).
+struct EgRead {
+    int file = -1;
+    uint64_t off = 0;
+    uint32_t n = 0;
+    uint8_t* dst = nullptr;
+    uint8_t* scales = nullptr;
+    bool mx8 = false;
+    int file2 = -1;
+    uint64_t off2 = 0;
+    uint32_t n2 = 0;
+    uint64_t key = 0;
+};
+
+/// A batch of rows (one Engram layer of a window): reads, the other places a row read once goes, and its state.
+struct EgBatch {
+    std::vector<EgRead> reads;
+    std::vector<std::pair<int, std::pair<uint8_t*, uint8_t*>>> dups;   // read i -> another (codes, scales)
+    std::vector<uint8_t> ok;                                            // per read: 1 read whole
+    std::atomic<int> left{0};                                           // reads not yet done
+    std::atomic<int> post{0};                                           // 0 pending, 1 finishing, 2 finished
+    int64_t t0 = 0;                                                     // submitted (now_ns())
+    std::atomic<int64_t> t1{0};                                         // the last read done
+    bool live = false;                                                  // submitted and not yet finished
+    void clear() {
+        reads.clear();
+        dups.clear();
+        ok.clear();
+        left.store(0);
+        post.store(0);
+        live = false;
+    }
+};
+
+/// The reader's threads: each read on whichever is free, in submission order, with that thread's own file handles.
+class EgReader {
+public:
+    ~EgReader() { stop(); }
+    /// direct[i]: file i read unbuffered (Windows, a file nothing maps: os::open_unbuffered), else buffered
+    void start(const std::vector<std::string>& paths, int threads, std::vector<char> direct = {}) {
+        paths_ = paths;
+        direct_ = std::move(direct);
+        for (int i = 0; i < threads; ++i) th_.emplace_back([this] { loop(); });
+    }
+    void stop() {
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            quit_ = true;
+        }
+        cv_.notify_all();
+        for (auto& t : th_) t.join();
+        th_.clear();
+        if (check_n_.load() > 0)
+            std::fprintf(stderr, "ds4 Engram check (DS4_EG_CHECK): %lld rows read unbuffered, %lld differed from a buffered read\n",
+                         (long long) check_n_.load(), (long long) check_bad_.load());
+    }
+    int threads() const { return (int) th_.size(); }
+    void submit(EgBatch& b) {
+        b.ok.assign(b.reads.size(), 0);
+        b.post.store(0);
+        b.t0 = now_ns();
+        b.t1.store(b.t0);
+        b.live = true;
+        b.left.store((int) b.reads.size(), std::memory_order_release);
+        const int n = (int) b.reads.size();
+        if (n == 0) return;
+        // a token's rows one job each; a prompt chunk's in runs (fewer trips through the queue's lock)
+        const int step = std::max(1, std::min(32, n / (int) (th_.size() * 4)));
+        int jobs = 0;
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            for (int i = 0; i < n; i += step, ++jobs) q_.push_back({&b, i, std::min(n, i + step)});
+        }
+        if (jobs >= (int) th_.size()) cv_.notify_all();
+        else
+            for (int i = 0; i < jobs; ++i) cv_.notify_one();
+    }
+    /// Until every read of b is done.
+    void wait(EgBatch& b) {
+        if (b.left.load(std::memory_order_acquire) == 0) return;
+        std::unique_lock<std::mutex> lk(done_mu_);
+        done_cv_.wait(lk, [&] { return b.left.load(std::memory_order_acquire) == 0; });
+    }
+
+private:
+    void loop() {
+        std::vector<os::fd_t> fds(paths_.size(), os::kBadFd);   // this thread's own handles, opened on first use
+        std::vector<os::fd_t> ufds(paths_.size(), os::kBadFd);  // (and its unbuffered ones)
+        std::vector<uint8_t> buf, abuf(3 * os::kDirectAlign);
+        uint8_t* al = abuf.data() + (os::kDirectAlign - (uintptr_t) abuf.data() % os::kDirectAlign) % os::kDirectAlign;
+        for (;;) {
+            Job job;
+            {
+                std::unique_lock<std::mutex> lk(mu_);
+                cv_.wait(lk, [&] { return quit_ || !q_.empty(); });
+                if (quit_ && q_.empty()) break;
+                job = q_.front();
+                q_.pop_front();
+            }
+            EgBatch& b = *job.b;
+            for (int i = job.i0; i < job.i1; ++i) b.ok[(size_t) i] = read(fds, ufds, al, b.reads[(size_t) i], buf) ? 1 : 0;
+            const int64_t t = now_ns();
+            for (int64_t prev = b.t1.load(); t > prev && !b.t1.compare_exchange_weak(prev, t);) {
+            }
+            if (b.left.fetch_sub(job.i1 - job.i0, std::memory_order_acq_rel) == job.i1 - job.i0) {   // (b is the waiter's again)
+                std::lock_guard<std::mutex> lk(done_mu_);
+                done_cv_.notify_all();
+            }
+        }
+        for (os::fd_t f : fds) os::close_fd(f);
+        for (os::fd_t f : ufds) os::close_fd(f);
+    }
+    bool read(std::vector<os::fd_t>& fds, std::vector<os::fd_t>& ufds, uint8_t* al, const EgRead& r,
+              std::vector<uint8_t>& buf) {
+        if ((r.mx8 || r.n2 == 0) && r.file >= 0 && (size_t) r.file < direct_.size() && direct_[(size_t) r.file] &&
+            os::direct_span(r.off, r.n) <= 2 * os::kDirectAlign) {
+            os::fd_t& u = ufds[(size_t) r.file];
+            if (u == os::kBadFd) u = os::open_unbuffered(paths_[(size_t) r.file]);
+            if (u != os::kBadFd && os::pread_unbuffered(u, al, r.n, r.off)) {
+                const uint8_t* src = al + r.off % os::kDirectAlign;
+                static const bool check = std::getenv("DS4_EG_CHECK") != nullptr;   // (each row read buffered too)
+                if (check) {
+                    std::vector<uint8_t> ref(r.n);
+                    os::fd_t& f2 = fds[(size_t) r.file];
+                    if (f2 == os::kBadFd) f2 = os::open_ro(paths_[(size_t) r.file]);
+                    const bool ok = f2 != os::kBadFd && os::pread_full(f2, ref.data(), r.n, r.off);
+                    if (!ok || std::memcmp(ref.data(), src, r.n) != 0) ++check_bad_;
+                    ++check_n_;
+                }
+                if (!r.mx8) {   // (a GGUF's Q6_K / IQ4_NL row: its bytes as they are, the scales inside)
+                    std::memcpy(r.dst, src, r.n);
+                    return true;
+                }
+                for (uint32_t k = 0; k < r.n / 33; ++k) {   // [scale, 32 codes] -> codes | scales
+                    r.scales[k] = src[(size_t) k * 33];
+                    std::memcpy(r.dst + (size_t) k * 32, src + (size_t) k * 33 + 1, 32);
+                }
+                return true;
+            }   // (else buffered, below)
+        }
+        auto fd = [&](int file) {
+            if (file < 0 || file >= (int) fds.size()) return os::kBadFd;
+            if (fds[(size_t) file] == os::kBadFd) fds[(size_t) file] = os::open_ro(paths_[(size_t) file]);
+            return fds[(size_t) file];
+        };
+        const os::fd_t f = fd(r.file);
+        if (f == os::kBadFd) return false;
+        if (!r.mx8) {
+            if (!os::pread_full(f, r.dst, r.n, r.off)) return false;
+            if (r.n2 == 0) return true;
+            const os::fd_t f2 = fd(r.file2);
+            return f2 != os::kBadFd && os::pread_full(f2, r.scales, r.n2, r.off2);
+        }
+        buf.resize(r.n);
+        if (!os::pread_full(f, buf.data(), r.n, r.off)) return false;
+        for (uint32_t k = 0; k < r.n / 33; ++k) {   // [scale, 32 codes] -> codes | scales
+            r.scales[k] = buf[(size_t) k * 33];
+            std::memcpy(r.dst + (size_t) k * 32, buf.data() + (size_t) k * 33 + 1, 32);
+        }
+        return true;
+    }
+
+    std::vector<std::string> paths_;
+    std::vector<char> direct_;
+
+public:
+    std::atomic<int64_t> check_n_{0}, check_bad_{0};   // (DS4_EG_CHECK: rows compared, rows that differed)
+    std::vector<std::thread> th_;
+    std::mutex mu_;
+    std::condition_variable cv_;
+    struct Job {
+        EgBatch* b = nullptr;
+        int i0 = 0, i1 = 0;   // reads [i0, i1)
+    };
+    std::deque<Job> q_;
+    bool quit_ = false;
+    std::mutex done_mu_;
+    std::condition_variable done_cv_;
+};
+
+}  // namespace strata::ds4

src/dsv41/ds4_gg_cpu.cpp

@@ -0,0 +1,293 @@
+// Strata. The CPU lane of the DeepSeek-V4.1 engine path for ggml's block types (IQ2_XXS .. Q4_K, the experts of a requantized model):
+// ggml-cpu's own row dots (type traits: vec_dot over the Q8_K / Q8_0 activation of vec_dot_type), one row at a time.  Maya's lane for
+// these types (native_expert split-base rows) is not in Strata's tree; this one is plain and complete, not tuned.
+#include "ds4_gguf_cpu.hpp"
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+
+#if defined(STRATA_DSV41_TQ_CPU)
+#include "ggml-cpu.h"
+#include "ggml.h"
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/iq_avx2.hpp"
+#include "strata/kernels/cpu/iq_avx512.hpp"
+
+#include <mutex>
+#include <vector>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#elif defined(__x86_64__)
+#include <cpuid.h>
+#endif
+
+namespace strata::ds4::gguf {
+namespace {
+void init_once() {
+    static std::once_flag f;
+    std::call_once(f, [] { ggml_cpu_init(); });
+}
+size_t row_size(int type, int64_t k) { return ggml_row_size((ggml_type) type, k); }
+}  // namespace
+
+bool gg_cpu_type(int type) {
+    // every type ggml-cpu has a row dot for (ds4_gguf.cu's known types); the i-quants with a device kernel are usually
+    // placed in VRAM, but a pack may hand any of them to this lane
+    switch (type) {
+        case 10: case 11: case 12: case 13: case 14: case 16: case 17: case 18: case 19: case 20: case 21: case 22:
+        case 23: case 29: return true;
+        default: return false;
+    }
+}
+
+static bool dbg() { static const bool on = std::getenv("DS4_DBG_GG") != nullptr; return on; }
+bool gg_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
+    if (dbg()) std::fprintf(stderr, "gg_cpu_fmt %d %d\n", tg, td);
+    init_once();
+    if (!gg_cpu_type(tg) || !gg_cpu_type(td)) {
+        err = "not ggml block types";
+        return false;
+    }
+    const ggml_type_traits_cpu* a = ggml_get_type_traits_cpu((ggml_type) tg);
+    const ggml_type_traits_cpu* b = ggml_get_type_traits_cpu((ggml_type) td);
+    if (a == nullptr || b == nullptr || a->vec_dot == nullptr || b->vec_dot == nullptr) {
+        err = "ggml-cpu has no row dot for these types";
+        return false;
+    }
+    f.tg = tg;
+    f.td = td;
+    f.n_embd = n_embd;
+    f.n_ff = n_ff;
+    f.gu_act = (int) a->vec_dot_type;
+    f.d_act = (int) b->vec_dot_type;
+    f.gu_row = row_size(tg, n_embd);
+    f.d_row = row_size(td, n_ff);
+    f.act_bytes = row_size(f.gu_act, n_embd);
+    f.h_bytes = row_size(f.d_act, n_ff);
+    return true;
+}
+
+// x (k values) -> the activation type (Q8_K, Q8_0 ..) ggml-cpu's dot takes
+void gg_cpu_quant(int act, const float* x, int k, void* dst) {
+    init_once();
+    if (dbg()) std::fprintf(stderr, "gg_cpu_quant act %d k %d\n", act, k);
+    const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) act);
+    if (tc != nullptr && tc->from_float != nullptr) {
+        tc->from_float(x, dst, k);
+        return;
+    }
+    const ggml_type_traits* t = ggml_get_type_traits((ggml_type) act);
+    if (dbg()) std::fprintf(stderr, "gg_cpu_quant: no cpu from_float, ref %p\n", (void*) (t ? t->from_float_ref : nullptr));
+    t->from_float_ref(x, dst, k);
+}
+
+// Strata's AVX2 multi-token rows for the i-quants (iq256x_rows: each 32-value chunk decoded once, every token applies it).  They
+// are ggml-cpu's vec_dot bit for bit (the integer sums are ggml's and the float additions come in ggml's order: ds4_gg_bench --exact,
+// and own_ok() below checks it once per type on this CPU against the ggml-cpu build in use), so taking them or not never changes an
+// answer, at any number of tokens - the choice is speed only.  Measured on the rows alone: no faster than ggml's dot at one token
+// (both wait for the codebook lookups), x1.7-2.9 at 2-4 tokens, x1.4 on IQ3_S already at one; so by default ggml's dot below
+// min_tokens(type) and these rows from there.  DS4_GG_IQ256=0 never takes them, =1 always (the old opt-in).
+static int g_iq256 = -1;   // -1: from the environment and the CPU, 0 off, 1 always, 2 from min_tokens(type)
+void gg_cpu_set_iq256(int mode) { g_iq256 = mode; }
+static int min_tokens(int type) { return type == 21 ? 1 : type == 29 ? 3 : 2; }
+static bool own_ok(int type, int impl) {
+    static std::once_flag once[2][32];
+    static bool ok[2][32];
+    const int k = type & 31;
+    std::call_once(once[impl][k], [&] {
+        ok[impl][k] = false;
+        const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) type);
+        const ggml_type_traits_cpu* q8 = tc ? ggml_get_type_traits_cpu(tc->vec_dot_type) : nullptr;
+        if (tc == nullptr || tc->vec_dot == nullptr || q8 == nullptr || q8->from_float == nullptr) return;
+        const int n = 1024, rows = 7;
+        const size_t rb = ggml_row_size((ggml_type) type, n), bb = ggml_type_size((ggml_type) type), ab = ggml_row_size(tc->vec_dot_type, n);
+        uint64_t st = 0x9E3779B97F4A7C15ull + (uint64_t) type;
+        auto next = [&st] { st ^= st << 13; st ^= st >> 7; st ^= st << 17; return st; };
+        std::vector<uint8_t> w(rb * rows);
+        for (auto& b : w) b = (uint8_t) next();
+        for (size_t o = 0; o < w.size(); o += bb) {
+            const ggml_fp16_t d = ggml_fp32_to_fp16(0.002f + 0.001f * (float) (next() % 16));
+            if (type == 29) {   // IQ1_M keeps its f16 in the top nibbles of the four scale words
+                uint16_t sc[4];
+                std::memcpy(sc, &w[o + 48], 8);
+                for (int j = 0; j < 4; ++j) sc[j] = (uint16_t) ((sc[j] & 0x0fff) | ((d >> (4 * j)) & 0xf) << 12);
+                std::memcpy(&w[o + 48], sc, 8);
+            } else {
+                std::memcpy(&w[o], &d, sizeof d);
+            }
+        }
+        for (int nt : {1, 2, 3, 4, 5, 6, 8, 16, 33}) {
+            std::vector<std::vector<uint8_t>> a((size_t) nt, std::vector<uint8_t>(ab));
+            std::vector<const void*> ap((size_t) nt);
+            std::vector<std::vector<float>> o0((size_t) nt, std::vector<float>(rows)), o1 = o0;
+            std::vector<float*> p1((size_t) nt);
+            for (int t = 0; t < nt; ++t) {
+                std::vector<float> x((size_t) n);
+                for (auto& v : x) v = ((float) (next() % 2001) - 1000.0f) * 1e-3f;
+                q8->from_float(x.data(), a[(size_t) t].data(), n);
+                ap[(size_t) t] = a[(size_t) t].data();
+                p1[(size_t) t] = o1[(size_t) t].data();
+                for (int r = 0; r < rows; ++r) tc->vec_dot(n, &o0[(size_t) t][(size_t) r], 0, w.data() + (size_t) r * rb, 0, ap[(size_t) t], 0, 1);
+            }
+            if (impl == 1) strata::kernels::cpu::iq512x_rows(type, w.data(), rb, n, ap.data(), nt, p1.data(), 0, rows);
+            else strata::kernels::cpu::iq256x_rows(type, w.data(), rb, n, ap.data(), nt, p1.data(), 0, rows);
+            for (int t = 0; t < nt; ++t)
+                if (std::memcmp(o0[(size_t) t].data(), o1[(size_t) t].data(), rows * sizeof(float)) != 0) {
+                    if (dbg() || std::getenv("DS4_GG_VERBOSE")) std::fprintf(stderr, "ds4: CPU lane type %d: AVX2 rows differ from ggml-cpu's dot at %d tokens, not used\n", type, nt);
+                    return;
+                }
+        }
+        ok[impl][k] = true;
+    });
+    return ok[impl][k];
+}
+// the AMD family (0x19 Zen 3/4, 0x1A Zen 5), 0 on any other vendor or if it cannot be read
+static int amd_family() {
+    static const int fam = [] {
+#if defined(_MSC_VER)
+        int r[4];
+        __cpuid(r, 0);
+        char v[13];
+        std::memcpy(v, &r[1], 4); std::memcpy(v + 4, &r[3], 4); std::memcpy(v + 8, &r[2], 4);
+        v[12] = 0;
+        if (std::strcmp(v, "AuthenticAMD") != 0) return 0;
+        __cpuid(r, 1);
+        const unsigned eax = (unsigned) r[0];
+#elif defined(__x86_64__)
+        unsigned a, b, c, d;
+        if (!__get_cpuid(0, &a, &b, &c, &d)) return 0;
+        char v[13];
+        std::memcpy(v, &b, 4); std::memcpy(v + 4, &d, 4); std::memcpy(v + 8, &c, 4);
+        v[12] = 0;
+        if (std::strcmp(v, "AuthenticAMD") != 0) return 0;
+        if (!__get_cpuid(1, &a, &b, &c, &d)) return 0;
+        const unsigned eax = a;
+#else
+        return 0;
+#endif
+        const unsigned base = (eax >> 8) & 0xF;
+        return (int) (base == 0xF ? base + ((eax >> 20) & 0xFF) : base);
+    }();
+    return fam;
+}
+// DS4_GG_IQ512: 1 takes the AVX-512 rows (bit-identical too) where the CPU has AVX-512 F/BW/VL/DQ/VNNI/VBMI, 0 never; unset: the rule
+// below, from the measurements on Zen 4.
+static int g_iq512 = -1;
+static bool want_512(int type, int nt) {
+    if (g_iq512 < 0) {
+        const char* v = std::getenv("DS4_GG_IQ512");
+        g_iq512 = !strata::kernels::cpu::cpu_avx512_ok() ? 0 : v == nullptr ? 2 : std::atoi(v) != 0 ? 1 : 0;
+    }
+    if (g_iq512 == 0 || !strata::kernels::cpu::iq512x_supported(type)) return false;
+    if (g_iq512 == 1) return true;
+    // by CPU (measured, ds4_gg_bench --expert; the rows are bit-identical to ggml's dot everywhere):
+    //  Zen 5 (AMD family 0x1A, Strix Halo, 16 threads, GB/s of weights ggml -> AVX-512 rows): one token IQ2_XXS 35 -> 44, IQ2_XS 37 -> 48,
+    //  IQ1_M 32 -> 36 (the sign masks by popcount, IQ1_M's indices by pdep: no table loads), IQ3_XXS 43 -> 45; 4 tokens x2.5-3.5; 16 tokens x4.4-4.8;
+    //  Zen 4 (family 0x19, Ryzen 5 7600, 6 threads): one token IQ2_XXS 15.9 -> 19.7, IQ2_XS 20.6 -> 24.2, IQ1_M 16.7 -> 17.6-18.2, IQ3_XXS 22.4 -> 23.8
+    //  (noise-level); 2 tokens x1.5-2.2; 4 tokens x2.4-3.3; 16 tokens x2.8-4.6;
+    //  anything else with AVX-512 (Intel): not measured, so the AVX2 rows
+    const int fam = amd_family();
+    if (fam < 0x19) return false;
+    if (type == 16 || type == 17) return true;
+    if (type == 18) return nt >= 2;
+    if (type == 29) return nt >= 2 || fam >= 0x1A;
+    return false;
+}
+// 0: ggml-cpu's dot, 1: the AVX2 rows, 2: the AVX-512 rows
+static int use_iq256(int type, int nt) {
+    if (g_iq256 < 0) {
+        const char* v = std::getenv("DS4_GG_IQ256");
+        // (ggml-cpu's own answer: built with AVX2 + FMA + F16C, which is also what makes its dots the AVX2 ones these rows match)
+        const bool avx2 = ggml_cpu_has_avx2() && ggml_cpu_has_fma() && ggml_cpu_has_f16c();
+        g_iq256 = !avx2 ? 0 : v == nullptr ? 2 : std::atoi(v) != 0 ? 1 : 0;
+    }
+    if (g_iq256 == 0) return 0;
+    if (want_512(type, nt) && own_ok(type, 1)) return 2;
+    if (!strata::kernels::cpu::iq256x_supported(type)) return 0;
+    if (g_iq256 == 2 && nt < min_tokens(type)) return 0;
+    return own_ok(type, 0) ? 1 : 0;
+}
+
+static void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {
+    if (nt > 0) {
+        const int impl = use_iq256(type, nt);
+        if (impl == 2) {
+            strata::kernels::cpu::iq512x_rows(type, w, rb, n, act, nt, out, r0, r1);
+            return;
+        }
+        if (impl == 1) {
+            strata::kernels::cpu::iq256x_rows(type, w, rb, n, act, nt, out, r0, r1);
+            return;
+        }
+    }
+    const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) type);
+    if (dbg()) std::fprintf(stderr, "gg dots type %d n %d rows %d..%d nt %d\n", type, n, r0, r1, nt);
+    for (int r = r0; r < r1; ++r)
+        for (int t = 0; t < nt; ++t) tc->vec_dot(n, &out[t][r], 0, w + (size_t) r * rb, 0, act[t], 0, 1);
+}
+
+void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff, int r0,
+                    int r1, float limit) {
+    constexpr int kMaxTok = 64;   // (the prompt path: a chunk's tokens an expert)
+    static thread_local std::vector<float> tmp[kMaxTok];
+    float* uo[kMaxTok];
+    if (nt > kMaxTok) {   // (more than a pass takes: in pieces)
+        for (int t0 = 0; t0 < nt; t0 += kMaxTok)
+            gg_cpu_gate_up(f, gate, up, act + t0, nt - t0 < kMaxTok ? nt - t0 : kMaxTok, ff + t0, r0, r1, limit);
+        return;
+    }
+    for (int t = 0; t < nt; ++t) {
+        tmp[t].resize((size_t) r1);
+        uo[t] = tmp[t].data();
+    }
+    dots(f.tg, gate, f.gu_row, f.n_embd, act, nt, ff, r0, r1);
+    dots(f.tg, up, f.gu_row, f.n_embd, act, nt, uo, r0, r1);
+    for (int t = 0; t < nt; ++t)
+        for (int r = r0; r < r1; ++r) {
+            float g = ff[t][r], u = uo[t][r];
+            if (limit > 0.0f) {
+                g = g < limit ? g : limit;
+                u = u < -limit ? -limit : (u > limit ? limit : u);
+            }
+            ff[t][r] = g / (1.0f + std::exp(-g)) * u;
+        }
+}
+
+void gg_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
+    dots(f.td, down, f.d_row, f.n_ff, hq, nt, out, r0, r1);
+}
+
+// a row of any ggml block type -> BF16 on the host (the embedding table's reader; Q5_K / Q4_K / Q6_K / Q8_0 rows of a
+// GGUF whose dense tensors are not BF16): ggml's own to_float, then the same round-to-nearest-even to BF16 the Q8_0 path uses
+bool gg_row_to_bf16(int type, const uint8_t* row, int k, uint16_t* out) {
+    init_once();
+    const ggml_type_traits* t = ggml_get_type_traits((ggml_type) type);
+    if (t == nullptr || t->to_float == nullptr) return false;
+    static thread_local std::vector<float> f;
+    f.resize((size_t) k);
+    t->to_float(row, f.data(), k);
+    for (int i = 0; i < k; ++i) {
+        uint32_t u;
+        std::memcpy(&u, &f[(size_t) i], 4);
+        u += 0x7fffu + ((u >> 16) & 1u);
+        out[i] = (uint16_t) (u >> 16);
+    }
+    return true;
+}
+}  // namespace strata::ds4::gguf
+#else
+namespace strata::ds4::gguf {
+bool gg_cpu_type(int) { return false; }
+bool gg_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {
+    err = "this build has no ggml-cpu lane (STRATA_NATIVE_EXPERTS=OFF)";
+    return false;
+}
+void gg_cpu_quant(int, const float*, int, void*) {}
+void gg_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int, float) {}
+void gg_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}
+bool gg_row_to_bf16(int, const uint8_t*, int, uint16_t*) { return false; }
+}  // namespace strata::ds4::gguf
+#endif

src/dsv41/ds4_gguf.cuh

@@ -0,0 +1,81 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_gguf.cuh - a GGUF's weights on the device for maya-ds4 (unsloth's DeepSeek-V4.1-Flash-1017 GGUFs,
+// tools/ds4/pack_gguf.py): the dense matrices in BF16 and the routed experts in MXFP4.  They stand in for
+// strata::dsv41k::exl3's matrices and experts (an x3::Mat with fmt kFmtBF16, an x3::ExpertLayout with fmt kFmtMXFP4)
+// with the same arguments and the same results' layout, so the engine's paths (decode, verify windows, chunked prompts,
+// tensor parallel) take either format.
+//
+// MXFP4 is ggml's block_mxfp4: 32 values in 17 bytes, an E8M0 scale e then 16 bytes of E2M1 codes - value j of the
+// block in byte j's low nibble (j < 16) or byte j - 16's high nibble, worth kvalues[code] * 2^(e - 128) with kvalues
+// {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12} (the E2M1 values doubled).  A matrix is [out][in / 32] blocks
+// (row r = output r), as the GGUF stores it, so an expert's blob is its gate, up and down matrices one after another.
+// Every product is an F32 sum of exactly dequantised weights times F32 activations, and a window's entry gets the bits
+// the token path gives it (the same kernel, the same order).
+#pragma once
+
+#include "strata/dsv41/exl3.hpp"
+
+#include <cstddef>
+#include <cstdint>
+#include <cuda_runtime.h>
+
+namespace strata::ds4::gguf {
+
+namespace x3 = strata::dsv41k::exl3;
+
+constexpr int kFmtBF16 = 1;    // x3::Mat::fmt
+constexpr int kFmtMXFP4 = 2;   // x3::ExpertLayout::fmt (and an expert's matrix as an x3::Mat)
+constexpr int kFmtQ8_0 = 3;    // x3::Mat::fmt: ggml's Q8_0, an FP16 scale and 32 int8 in 34 bytes (UD-Q4_K_XL's dense)
+constexpr int kFmtGG = 4;      // x3::Mat::fmt: ggml's low-bit block type m.K (ds4_iq.cuh's: IQ2_XXS .. Q4_K), [n][k / 256]
+constexpr int kMxBlock = 32, kMxBytes = 17, kQ8Bytes = 34;
+constexpr int kTypeMXFP4 = 39;   // ggml's type id
+
+/// Bytes of an MXFP4 row of k values (k % 32 == 0), and of a matrix of n such rows.
+__host__ __device__ inline size_t mx_row_bytes(int64_t k) { return (size_t) (k / kMxBlock) * kMxBytes; }
+__host__ __device__ inline size_t mx_bytes(int64_t n, int64_t k) { return (size_t) n * mx_row_bytes(k); }
+__host__ __device__ inline size_t q8_row_bytes(int64_t k) { return (size_t) (k / 32) * kQ8Bytes; }
+
+/// A GGUF expert layer (x3::ExpertLayout::fmt kFmtMXFP4: block-quantized experts read in place) keeps each role's ggml
+/// type in lay.K[r] (r: gate, up, down; 0 = MXFP4): MXFP4 takes these kernels, the low-bit types ds4_iq.cuh's (gate
+/// and up of one type).  Roles of one layer may differ (down of another type); MXFP4 and a low-bit type do not mix.
+inline int expert_type(const x3::ExpertLayout& l, int r) { return l.K[r] == 0 ? kTypeMXFP4 : l.K[r]; }
+inline bool mx_experts(const x3::ExpertLayout& l) {
+    return expert_type(l, 0) == kTypeMXFP4 && expert_type(l, 1) == kTypeMXFP4 && expert_type(l, 2) == kTypeMXFP4;
+}
+/// Bytes of a row of k values of an expert type (0: not one the device kernels take, or k not whole blocks).
+size_t type_row_bytes(int type, int64_t k);
+/// Values a block of an expert type (32: MXFP4, 256: the low-bit ones; 0: neither).
+int type_block(int type);
+/// The type's name (ggml's), "?" when unknown.
+const char* type_name(int type);
+/// Whether the device kernels take this type (MXFP4 is its own path; the i-quants and K-quants ds4_iq.cu instantiates).
+/// A false here with a layout known to type_row_bytes() means the CPU lane reads it and it must not enter the VRAM tier.
+bool type_on_device(int type);
+
+/// The per-stream scratch of the MXFP4 experts on the current device (once per stream, as x3::prepare).
+bool prepare(cudaStream_t s);
+
+/// x3::mv's contract for GGUF matrices (F32 activations): each job's m->fmt kFmtBF16 (raw = BF16 [n][ld], ld values a
+/// row), kFmtMXFP4 (raw = MXFP4 [n][k / 32] blocks, ld bytes a row; 0: mx_row_bytes(k)) - an expert's matrix in VRAM -
+/// or kFmtQ8_0 (raw = Q8_0 [n][k / 32] blocks, ld bytes a row; 0: q8_row_bytes(k)).
+bool mv(const x3::MvJob* jobs, int n, cudaStream_t s);
+/// The columns [c0, c0 + nc) of such a matrix: its rows c0 .. c0 + nc - 1, a contiguous range.
+x3::Mat view(const x3::Mat& m, int c0, int nc);
+/// x3::reconstruct_rows' layout from a BF16 or Q8_0 matrix: dst[r][c] = W[c][r] (n rows of k values), F16 or BF16
+/// bits (a Q8_0 weight rounded to them).
+void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s);
+
+/// x3::moe_gate_up / moe_down / win_experts / set_gate_up / set_down for a GGUF's experts (lay.fmt == kFmtMXFP4; the
+/// roles' types in lay.K, see expert_type).
+void moe_gate_up(const x3::MoeArgs& a, const float* x, float* h, cudaStream_t s);
+void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s);
+bool win_experts(const x3::WinArgs& a, cudaStream_t s);
+void set_gate_up(const x3::SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh,
+                 cudaStream_t s);
+void set_down(const x3::SetArgs& a, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s);
+
+/// Launch failures of these kernels since the process started (each is also printed).
+int launch_errors();
+
+}  // namespace strata::ds4::gguf

src/dsv41/ds4_gguf_cpu.cpp

@@ -0,0 +1,418 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_gguf_cpu.cpp - a GGUF's MXFP4 experts on the CPU lane (see ds4_gguf_cpu.hpp).
+#include "ds4_gguf_cpu.hpp"
+
+#include "strata/dsv41/exl3_cpu.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#if defined(__x86_64__) || defined(_M_X64)
+#define MX_X86 1
+#include <immintrin.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <cpuid.h>
+#endif
+#endif
+
+// a function's instruction set (GCC / Clang, clang-cl too; MSVC compiles any intrinsic as it is): the file is built for
+// the baseline x86-64, and each tier's code runs only where detect() says the CPU and the OS take it
+#if defined(MX_X86) && (defined(__GNUC__) || defined(__clang__))
+#define MX_TARGET(s) __attribute__((target(s)))
+#else
+#define MX_TARGET(s)
+#endif
+#define MX_AVX2 MX_TARGET("avx2,fma")
+// AVX-VNNI's intrinsics: GCC 11, Clang 12, MSVC 2022
+#if defined(MX_X86) && (defined(_MSC_VER) || (defined(__clang__) && __clang_major__ >= 12) || \
+                        (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 11))
+#define MX_AVXVNNI 1
+#endif
+
+namespace strata::ds4::gguf {
+namespace {
+
+constexpr int kBlock = 32, kBytes = 17;
+constexpr float kValues[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f,
+                               0.0f, -1.0f, -2.0f, -3.0f, -4.0f, -6.0f, -8.0f, -12.0f};
+constexpr int kCodes[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+
+float e8m0_half(uint32_t e) {   // 2^(e - 128), as the device's
+    const uint32_t bits = e < 2 ? 0x00200000u << e : (e - 1) << 23;
+    float f;
+    std::memcpy(&f, &bits, 4);
+    return f;
+}
+struct E8m0Table {
+    float v[256];
+    E8m0Table() {
+        for (int e = 0; e < 256; ++e) v[e] = e8m0_half((uint32_t) e);
+    }
+};
+const E8m0Table& e8() {
+    static const E8m0Table t;
+    return t;
+}
+
+void rows_scalar(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n, float* const* y) {
+    for (int r = r0; r < r1; ++r) {
+        const uint8_t* p = w + (size_t) r * rb;
+        float acc[kMxCpuMaxN] = {};
+        for (int b = 0; b < k / kBlock; ++b, p += kBytes) {
+            const float d = e8m0_half(p[0]);
+            float wv[kBlock];
+            for (int j = 0; j < 16; ++j) {
+                wv[j] = kValues[p[1 + j] & 15] * d;
+                wv[j + 16] = kValues[p[1 + j] >> 4] * d;
+            }
+            for (int i = 0; i < n; ++i) {
+                const float* xb = x[i] + b * kBlock;
+                float s = 0.0f;
+                for (int j = 0; j < kBlock; ++j) s += wv[j] * xb[j];
+                acc[i] += s;
+            }
+        }
+        for (int i = 0; i < n; ++i) y[i][r - r0] = acc[i];
+    }
+}
+
+#if defined(MX_X86)
+MX_AVX2 float hsum(__m256 v) {
+    const __m128 a = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1));
+    const __m128 b = _mm_add_ps(a, _mm_movehl_ps(a, a));
+    return _mm_cvtss_f32(_mm_add_ss(b, _mm_shuffle_ps(b, b, 1)));
+}
+
+// a block: the 16 code bytes -> 32 int8 values through a shuffle table (low nibbles: values 0..15, high: 16..31),
+// widened to F32 in 4 x 8, times the block's scale (exact), FMAs against each activation
+MX_AVX2 void rows_avx2(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n,
+                       float* const* y) {
+    const __m128i lut = _mm_setr_epi8(0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12);
+    const __m128i m4 = _mm_set1_epi8(0x0f);
+    for (int r = r0; r < r1; ++r) {
+        const uint8_t* p = w + (size_t) r * rb;
+        __m256 acc[kMxCpuMaxN];
+        for (int i = 0; i < n; ++i) acc[i] = _mm256_setzero_ps();
+        for (int b = 0; b < k / kBlock; ++b, p += kBytes) {
+            const __m256 d = _mm256_set1_ps(e8m0_half(p[0]));
+            const __m128i q = _mm_loadu_si128((const __m128i*) (p + 1));
+            const __m128i lo = _mm_shuffle_epi8(lut, _mm_and_si128(q, m4));
+            const __m128i hi = _mm_shuffle_epi8(lut, _mm_and_si128(_mm_srli_epi16(q, 4), m4));
+            const __m256 w0 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(lo)));
+            const __m256 w1 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(lo, 8))));
+            const __m256 w2 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(hi)));
+            const __m256 w3 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(hi, 8))));
+            for (int i = 0; i < n; ++i) {
+                const float* xb = x[i] + b * kBlock;
+                __m256 a = acc[i];
+                a = _mm256_fmadd_ps(w0, _mm256_loadu_ps(xb), a);
+                a = _mm256_fmadd_ps(w1, _mm256_loadu_ps(xb + 8), a);
+                a = _mm256_fmadd_ps(w2, _mm256_loadu_ps(xb + 16), a);
+                a = _mm256_fmadd_ps(w3, _mm256_loadu_ps(xb + 24), a);
+                acc[i] = a;
+            }
+        }
+        for (int i = 0; i < n; ++i) y[i][r - r0] = hsum(acc[i]);
+    }
+}
+
+// ---- the int8 path's vector tiers (ds4_mx_kern.inc)
+#define MXK_NS mx_avx2
+#define MXK_ATTR MX_TARGET("avx2,fma")
+#define MXK_256 1
+#include "ds4_mx_kern.inc"
+#if defined(MX_AVXVNNI)
+#define MXK_NS mx_avxvnni
+#define MXK_ATTR MX_TARGET("avx2,fma,avxvnni")
+#define MXK_256 1
+#define MXK_VNNI 1
+#include "ds4_mx_kern.inc"
+#endif
+#define MXK_NS mx_avx512
+#define MXK_ATTR MX_TARGET("avx512f,avx512bw,fma")
+#define MXK_512 1
+#include "ds4_mx_kern.inc"
+#define MXK_NS mx_avx512vnni
+#define MXK_ATTR MX_TARGET("avx512f,avx512bw,avx512vnni,fma")
+#define MXK_512 1
+#define MXK_VNNI 1
+#include "ds4_mx_kern.inc"
+#endif
+
+// ---- the int8 path's scalar tier: ds4_mx_kern.inc's order (8 accumulators, block j of a group of 8 into L(j)), so it
+// gives the vector tiers' bits - also the tests' check on them.  The activation's layout (ds4_mx_kern.inc), for any k
+// % 32 == 0: the int8 values in quads (a partial last quad padded), then per group of 8 blocks the scales and the
+// offset corrections in lane order (a partial last group padded)
+inline int act_lane(int b) { return 8 * (b >> 3) + ((b & 7) & 1) * 4 + ((b & 7) >> 1); }
+inline size_t act_qbytes(int k) { return (size_t) 128 * (size_t) ((k / kBlock + 3) / 4); }
+inline size_t act_groups(int k) { return (size_t) ((k / kBlock + 7) / 8); }
+inline int act_q(const uint8_t* act, int b, int j) {   // block b's value j (0..31)
+    return ((const int8_t*) act)[(b >> 2) * 128 + (j >= 16 ? 64 : 0) + 16 * (b & 3) + (j & 15)];
+}
+inline float act_d(const uint8_t* act, int k, int b) { return ((const float*) (act + act_qbytes(k)))[act_lane(b)]; }
+
+void quant_scalar(const float* x, int b0, int b1, uint8_t* act, int k) {
+    int8_t* q = (int8_t*) act;
+    float* dx = (float*) (act + act_qbytes(k));
+    int32_t* corr = (int32_t*) (act + act_qbytes(k) + 32 * act_groups(k));
+    for (int b = b0; b < b1; ++b, x += kBlock) {
+        float am = 0.0f;
+        for (int j = 0; j < kBlock; ++j) am = std::max(am, std::fabs(x[j]));
+        const float sc = am / 127.0f;
+        int8_t v[kBlock];
+        int s = 0;
+        for (int j = 0; j < kBlock; ++j) {
+            v[j] = am > 0.0f ? (int8_t) (int) std::nearbyint(x[j] / sc) : (int8_t) 0;
+            s += v[j];
+        }
+        int8_t* lo = q + (b >> 2) * 128 + 16 * (b & 3);
+        for (int j = 0; j < 16; ++j) {
+            lo[j] = v[j];
+            lo[64 + j] = v[16 + j];
+        }
+        dx[act_lane(b)] = sc;
+        corr[act_lane(b)] = -12 * s;
+    }
+}
+
+void rows_q_scalar(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n, float* const* y) {
+    const float* t = e8().v;
+    for (int r = r0; r < r1; ++r) {
+        const uint8_t* p = w + (size_t) r * rb;
+        float acc[kMxCpuMaxN][8] = {};
+        for (int b = 0; b < k / kBlock; ++b, p += kBytes) {
+            int wv[kBlock];
+            for (int j = 0; j < 16; ++j) {
+                wv[j] = kCodes[p[1 + j] & 15];
+                wv[j + 16] = kCodes[p[1 + j] >> 4];
+            }
+            const int l = act_lane(b) & 7;
+            for (int i = 0; i < n; ++i) {
+                const uint8_t* a = (const uint8_t*) act[i];
+                int s = 0;
+                for (int j = 0; j < kBlock; ++j) s += wv[j] * act_q(a, b, j);
+                acc[i][l] = std::fma((float) s, t[p[0]] * act_d(a, k, b), acc[i][l]);
+            }
+        }
+        for (int i = 0; i < n; ++i) {
+            const float* A = acc[i];
+            y[i][r - r0] = ((A[0] + A[4]) + (A[2] + A[6])) + ((A[1] + A[5]) + (A[3] + A[7]));
+        }
+    }
+}
+
+// ---- which tiers this CPU runs
+struct Cpu {
+    bool avx2 = false, vnni256 = false, avx512 = false, vnni512 = false, vbmi = false, bmi2 = false;
+};
+Cpu detect() {
+    Cpu c;
+#if defined(MX_X86)
+    unsigned r[4] = {0, 0, 0, 0};
+    auto cpuid = [&](unsigned leaf, unsigned sub) {
+#if defined(_MSC_VER)
+        int x[4];
+        __cpuidex(x, (int) leaf, (int) sub);
+        for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];
+#else
+        __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);
+#endif
+    };
+    cpuid(0, 0);
+    const unsigned max_leaf = r[0];
+    if (max_leaf < 7) return c;
+    cpuid(1, 0);
+    const bool fma = (r[2] >> 12) & 1u, osxsave = (r[2] >> 27) & 1u, avx = (r[2] >> 28) & 1u;
+    if (!fma || !osxsave || !avx) return c;
+#if defined(_MSC_VER)
+    const unsigned long long xcr0 = _xgetbv(0);
+#else
+    unsigned lo = 0, hi = 0;
+    __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
+    const unsigned long long xcr0 = ((unsigned long long) hi << 32) | lo;
+#endif
+    if ((xcr0 & 0x6) != 0x6) return c;   // the OS saves the YMM state
+    cpuid(7, 0);
+    const unsigned b7 = r[1], c7 = r[2];
+    c.avx2 = (b7 >> 5) & 1u;
+    const bool zmm_os = (xcr0 & 0xE6) == 0xE6;
+    c.avx512 = c.avx2 && zmm_os && ((b7 >> 16) & 1u) && ((b7 >> 30) & 1u);   // F, BW
+    c.vnni512 = c.avx512 && ((c7 >> 11) & 1u);
+    c.vbmi = c.avx512 && ((c7 >> 1) & 1u);   // AVX-512 VBMI
+    c.bmi2 = (b7 >> 8) & 1u;
+    cpuid(7, 1);
+    c.vnni256 = c.avx2 && ((r[0] >> 4) & 1u);   // AVX-VNNI
+#endif
+    return c;
+}
+const Cpu& cpu() {
+    static const Cpu c = detect();
+    return c;
+}
+
+// The tier for this CPU, measured: every vector tier it runs times the same small warm job (64 rows of a 5120-wide
+// matrix, in L2, against 1, 2 and 4 activations - the compute a row takes, which is where the tiers differ; streaming
+// from RAM they all reach the memory's speed) in interleaved rounds, the best round of each counting.  The choice never
+// changes an answer (the tiers give the same bits), only the speed.  DS4_MX_TIER=<name> picks one by hand.
+std::string g_tier_report;
+int measure_tier(const std::vector<int>& cand);
+int pick_tier() {
+    if (const char* f = std::getenv("DS4_MX_TIER"))
+        for (int t = 0; t < kMxTiers; ++t)
+            if (std::string(f) == mx_tier_name(t) && mx_tier_ok(t)) {
+                g_tier_report = std::string("DS4_MX_TIER=") + f;
+                return t;
+            }
+    std::vector<int> cand;
+    for (int t = 1; t < kMxTiers; ++t)
+        if (mx_tier_ok(t)) cand.push_back(t);
+    if (cand.empty()) return kMxScalar;
+    if (cand.size() == 1) return cand[0];
+    return measure_tier(cand);
+}
+int measure_tier(const std::vector<int>& cand) {
+    constexpr int K = 5120, R = 64, NT = 4;
+    const size_t rb = (size_t) K / kBlock * kBytes;
+    std::vector<uint8_t> w(rb * R), act(mx_act_bytes(K) * NT);
+    uint32_t s = 0x9E3779B9u;
+    for (size_t i = 0; i < w.size(); ++i) {
+        s = s * 1664525u + 1013904223u;
+        w[i] = (uint8_t) (s >> 24);
+    }
+    for (size_t b = 0; b < w.size() / kBytes; ++b) w[b * kBytes] = (uint8_t) (118 + b % 6);
+    std::vector<float> x(K);
+    for (int i = 0; i < NT; ++i) {
+        for (int j = 0; j < K; ++j) {
+            s = s * 1664525u + 1013904223u;
+            x[(size_t) j] = (float) ((int) (s >> 16) - 32768) / 32768.0f;
+        }
+        quant_scalar(x.data(), 0, K / kBlock, act.data() + (size_t) i * mx_act_bytes(K), K);
+    }
+    const void* ap[NT];
+    for (int i = 0; i < NT; ++i) ap[i] = act.data() + (size_t) i * mx_act_bytes(K);
+    float y[NT][R];
+    float* yp[NT] = {y[0], y[1], y[2], y[3]};
+    std::vector<double> best(cand.size(), 1e30);
+    for (int round = 0; round < 6; ++round)
+        for (size_t c = 0; c < cand.size(); ++c) {
+            const auto t0 = std::chrono::steady_clock::now();
+            for (int rep = 0; rep < 8; ++rep)
+                for (int n : {1, 2, 4}) mx_rows_tier(cand[c], w.data(), rb, K, 0, R, ap, n, yp, 0);
+            const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+            if (round > 0) best[c] = std::min(best[c], us);   // (round 0: the clock and the caches warm)
+        }
+    size_t bi = 0;
+    for (size_t c = 1; c < cand.size(); ++c)
+        if (best[c] < best[bi]) bi = c;
+    char buf[64];
+    for (size_t c = 0; c < cand.size(); ++c) {
+        std::snprintf(buf, sizeof buf, "%s%s %.0f us", c ? ", " : "", mx_tier_name(cand[c]), best[c]);
+        g_tier_report += buf;
+    }
+    return cand[bi];
+}
+int prefetch_bytes() {
+    const char* f = std::getenv("DS4_MX_PF");
+    return f != nullptr ? std::max(0, std::atoi(f)) : 0;
+}
+
+}  // namespace
+
+void q8_round(const float* x, int k, float* out) {
+    for (int b = 0; b < k / kBlock; ++b) {
+        const float* xb = x + b * kBlock;
+        float am = 0.0f;
+        for (int j = 0; j < kBlock; ++j) am = std::max(am, std::fabs(xb[j]));
+        const float sc = am / 127.0f;
+        for (int j = 0; j < kBlock; ++j)
+            out[b * kBlock + j] = am > 0.0f ? std::nearbyint(xb[j] / sc) * sc : 0.0f;
+    }
+}
+
+void mx_cpu_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n, float* const* y) {
+#if defined(MX_X86)
+    static const bool avx2 = strata::dsv41k::cpu::exl3_cpu_tier() >= 1;
+    if (avx2) return rows_avx2(w, rb, k, r0, r1, x, n, y);
+#endif
+    rows_scalar(w, rb, k, r0, r1, x, n, y);
+}
+
+// ---- the int8 path
+bool mx_tier_ok(int t) {
+    switch (t) {
+        case kMxScalar: return true;
+        case kMxAvx2: return cpu().avx2;
+#if defined(MX_AVXVNNI)
+        case kMxAvxVnni: return cpu().vnni256;
+#endif
+        case kMxAvx512: return cpu().avx512;
+        case kMxAvx512Vnni: return cpu().vnni512;
+        default: return false;
+    }
+}
+const char* mx_tier_name(int t) {
+    switch (t) {
+        case kMxScalar: return "scalar";
+        case kMxAvx2: return "avx2";
+        case kMxAvxVnni: return "avxvnni";
+        case kMxAvx512: return "avx512";
+        case kMxAvx512Vnni: return "avx512vnni";
+        default: return "?";
+    }
+}
+int mx_tier() {
+    static const int t = pick_tier();
+    return t;
+}
+const char* mx_tier_report() {
+    (void) mx_tier();
+    return g_tier_report.c_str();
+}
+
+void mx_quant_blocks(const float* x, int b0, int b1, void* act, int k) {
+#if defined(MX_X86)
+    if (cpu().avx2 && k % 256 == 0) return mx_avx2::quant(x, b0, b1, (uint8_t*) act, k);
+#endif
+    quant_scalar(x, b0, b1, (uint8_t*) act, k);
+}
+void mx_quant_blocks_scalar(const float* x, int b0, int b1, void* act, int k) {
+    quant_scalar(x, b0, b1, (uint8_t*) act, k);
+}
+int mx_act_value(const void* act, int b, int j) { return act_q((const uint8_t*) act, b, j); }
+float mx_act_scale(const void* act, int k, int b) { return act_d((const uint8_t*) act, k, b); }
+
+void mx_rows_tier(int t, const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n,
+                  float* const* y, int pf) {
+    if (k % 256 != 0 || !mx_tier_ok(t)) t = kMxScalar;   // (the vector tiers take a group of 8 blocks a step)
+    const float* tb = e8().v;
+#if defined(MX_X86)
+    switch (t) {
+        case kMxAvx2: return mx_avx2::rows(w, rb, k, r0, r1, act, n, y, tb, pf);
+#if defined(MX_AVXVNNI)
+        case kMxAvxVnni: return mx_avxvnni::rows(w, rb, k, r0, r1, act, n, y, tb, pf);
+#endif
+        case kMxAvx512: return mx_avx512::rows(w, rb, k, r0, r1, act, n, y, tb, pf);
+        case kMxAvx512Vnni: return mx_avx512vnni::rows(w, rb, k, r0, r1, act, n, y, tb, pf);
+        default: break;
+    }
+#endif
+    (void) tb, (void) pf;
+    for (int i0 = 0; i0 < n; i0 += kMxCpuMaxN)
+        rows_q_scalar(w, rb, k, r0, r1, act + i0, std::min(kMxCpuMaxN, n - i0), y + i0);
+}
+
+void mx_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n, float* const* y) {
+    static const int pf = prefetch_bytes();
+    mx_rows_tier(mx_tier(), w, rb, k, r0, r1, act, n, y, pf);
+}
+
+}  // namespace strata::ds4::gguf

src/dsv41/ds4_gguf_cpu.hpp

@@ -0,0 +1,101 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_gguf_cpu.hpp - a GGUF's MXFP4 experts on the CPU lane of maya-ds4 (the format: ds4_gguf.cuh).  The
+// weights are read where they lie (the GGUF file's mapping), each block decoded once for all the activations of a
+// call (mx_rows: int8 dots, the CPU's best vector tier at run time; a scalar loop otherwise).
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+
+namespace strata::ds4::gguf {
+
+constexpr int kMxCpuMaxN = 8;   // activations a call
+
+/// y[i][r - r0] = (row r of W) . x[i] for r0 <= r < r1 and i < n (n <= kMxCpuMaxN): W's rows hold k values
+/// (k % 32 == 0) as MXFP4 blocks, rb bytes apart.  (F32 activations: the tests' and the benchmark's old path.)
+void mx_cpu_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n, float* const* y);
+
+/// A GGUF's low-bit experts (ds4_iq.cuh's types) on the CPU lane - ggml-cpu's arithmetic through the GLM CPU lane's
+/// native experts (strata::dsv41k::cpu::native_*: its AVX-512 / AVX2 multi-token rows where they are ggml's bit for bit,
+/// ggml's dots otherwise; ds4_iq_cpu.cpp): an activation quantized once to the weight type's vec_dot_type (Q8_K: an F32
+/// scale per 256 values - the device takes one per 32, so the two sides agree to that rounding), each row one dot.
+struct IqCpu {
+    int tg = 0, td = 0, n_embd = 0, n_ff = 0;
+    int gu_act = 0, d_act = 0;                 // the activations' ggml types
+    size_t gu_row = 0, d_row = 0;              // bytes a weight row
+    size_t act_bytes = 0, h_bytes = 0;         // bytes of x (n_embd) and of h (n_ff) as those
+};
+/// The layer's geometry; false (err) when this build or ggml-cpu cannot take it.
+bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);
+/// x (k values) as the activation type `act` (f.gu_act / f.d_act) into dst.
+void iq_cpu_quant(int act, const float* x, int k, void* dst);
+/// ff[t][r] = clamped swiglu(gate_r . a[t], up_r . a[t]) for rows [r0, r1) (limit 0: no clamp), nt tokens.
+void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt,
+                    float* const* ff, int r0, int r1, float limit);
+/// out[t][r] = down_r . hq[t] for rows [r0, r1).
+void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);
+/// The trellis types (ds4_iq.cuh's is_tq; ds4_tq_cpu.cpp, Strata's tq_cpu kernels): the same four calls, reached from the
+/// iq_cpu_* ones above for those types (their activations: F32 scale a 32 values, then the int8 values).
+bool tq_cpu_type(int type);
+int tq_cpu_act();
+bool tq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);
+void tq_cpu_quant(const float* x, int k, void* dst);
+void tq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
+                    int r0, int r1, float limit);
+void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);
+
+/// ggml block types on the CPU lane through ggml-cpu's row dots (ds4_gg_cpu.cpp), for the IQ / K-quant experts.
+bool gg_cpu_type(int type);
+/// One row of a GGUF block type (Q4_K / Q5_K / Q6_K / Q8_0 / any type ggml has a to_float for) -> k values as BF16
+/// (round to nearest even), the host embedding's format.  False: ggml has no decode for this type.
+bool gg_row_to_bf16(int type, const uint8_t* row, int k, uint16_t* out);
+bool gg_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);
+void gg_cpu_quant(int act, const float* x, int k, void* dst);
+void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
+                    int r0, int r1, float limit);
+/// -1: from the environment (default: Strata's AVX2 i-quant rows, bit-identical to ggml-cpu's dot, from 2 tokens - 1 on IQ3_S, 3 on IQ1_M;
+/// DS4_GG_IQ256=0 never, =1 always), 0 ggml-cpu, 1 always (benches), 2 the default rule.
+void gg_cpu_set_iq256(int mode);
+void gg_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);
+
+/// The CPU lane's MXFP4 rows in the device's arithmetic (ds4_gguf.cu's mx_dot): the activations rounded to int8 per 32
+/// values as the device rounds them (q8_round's q and scale), each block's products summed exactly in integers, then
+/// times (weight scale x activation scale) into F32 sums - the same numbers as the device up to the order of the
+/// blocks' sums.  The vector tiers (ds4_mx_kern.inc): AVX2 vpmaddubsw, AVX-VNNI and AVX-512 (VNNI) vpdpbusd; the CPU's
+/// fastest is taken at run time (DS4_MX_TIER=<name> picks another one it runs).  Every tier, the scalar one included,
+/// sums in the same order: they give the same bits, so the choice never changes an answer.
+enum MxTier { kMxScalar = 0, kMxAvx2, kMxAvxVnni, kMxAvx512, kMxAvx512Vnni, kMxTiers };
+/// Bytes of an activation of k values (k % 32 == 0) for mx_rows: the int8 values in quads of blocks, then a scale and
+/// an offset correction a block in groups of 8 (ds4_mx_kern.inc) - 1.25 k for k % 256 == 0.
+inline size_t mx_act_bytes(int k) {
+    const size_t nb = (size_t) k / 32;
+    return 128 * ((nb + 3) / 4) + 64 * ((nb + 7) / 8);
+}
+/// (tests) value j (0..31) of block b of an activation, and the block's scale
+int mx_act_value(const void* act, int b, int j);
+float mx_act_scale(const void* act, int k, int b);
+/// Blocks [b0, b1) of a k-value activation (x at block b0's first value: x[32 (b - b0) + j]) into act.
+void mx_quant_blocks(const float* x, int b0, int b1, void* act, int k);
+void mx_quant_blocks_scalar(const float* x, int b0, int b1, void* act, int k);   // (the same bytes, no vectors)
+inline void mx_quant(const float* x, int k, void* act) { mx_quant_blocks(x, 0, k / 32, act, k); }
+/// y[i][r - r0] = (row r of W) . act[i] for r0 <= r < r1, i < n (any n; act[i] mx_quant's of k values).
+void mx_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n, float* const* y);
+/// The same with tier t (the scalar one when the CPU does not run t, or k % 256 != 0); pf: bytes prefetched ahead.
+void mx_rows_tier(int t, const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n,
+                  float* const* y, int pf = 0);
+int mx_tier();                   // the tier mx_rows takes (measured on first use: the fastest a core, ds4_gguf_cpu.cpp)
+const char* mx_tier_report();    // the measurement behind it (each candidate's time), or how it was picked
+bool mx_tier_ok(int t);          // this CPU runs tier t
+const char* mx_tier_name(int t);
+/// Whether the rows of `type` are ds4's own here (IQ3_XXS / IQ3_S on AVX2 / AVX-512 VBMI: ggml-cpu's numbers bit for
+/// bit, checked against its dot on first use; DS4_IQ_OWN=0 turns them off) rather than ggml's / the GLM lane's.
+bool iq_cpu_own_rows(int type);
+
+/// x rounded as the device's MXFP4 kernels round their activations (q8_quant: per 32 values sc = max|x| / 127, q =
+/// round-half-even(x / sc), the same F32 operations, so the same q), into out as q sc - an expert gives the same sums
+/// on the CPU lane as in VRAM (up to their order), whichever side the expert cache puts it on.
+void q8_round(const float* x, int k, float* out);
+
+}  // namespace strata::ds4::gguf

src/dsv41/ds4_iq.cuh

@@ -0,0 +1,133 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_iq.cuh - a GGUF's routed experts in ggml's low-bit block types on the device for maya-ds4: the i-quants
+// IQ2_XXS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS and the K-quants Q2_K, Q3_K, Q4_K (256 values a block, ggml's layouts:
+// ggml-common.h, llama.cpp MIT).  ds4_gguf.cu sends an expert layer here when its experts are not MXFP4
+// (x3::ExpertLayout::fmt kFmtMXFP4 with K[r] = role r's ggml type; K[r] == kTypeMXFP4: ds4_gguf.cu's own kernels).
+//
+// Every weight is decoded exactly (ggml's dequantize_row_*: a block scale times an integer scale times a grid or code
+// value) and every product takes the activations rounded to int8 per 32 values with an F32 scale (ds4_gguf.cu's
+// q8_quant): a 32-value sub-block's sum is an exact integer dot (dp4a) times its scales in F32 - no integer division as
+// llama.cpp's vecdotq takes (its sums truncate), so these are the double reference's sums to F32 rounding.  A window's
+// entry gets the token path's bits (the same kernel, the same order).
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cuda_runtime.h>
+
+namespace strata::ds4::gguf::iq {
+
+// ggml type ids
+constexpr int kQ5_K = 13, kQ6_K = 14;   // (the dense matrices' K-quants, GEMV only)
+constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ2_XS = 17, kIQ3_XXS = 18, kIQ1_S = 19,
+              kIQ4_NL = 20, kIQ3_S = 21, kIQ2_S = 22, kIQ4_XS = 23, kIQ1_M = 29;
+
+/// The trellis types (Strata's, ds4_tq.cu): ggml's TQK6 / TQK7 (145 / 146, the table codebook) and the experimental
+/// TQK5X / TQK4X (224 / 225, computed codebook) and TQK5T / TQK4T (226 / 227, table): a block of 128 values is
+/// {fp16 d; 32 steps of K bits}, 2 + 4 K bytes (tools/trellis/tqenc.hpp).  They go through the same entry points below.
+constexpr int kTQK6 = 145, kTQK7 = 146, kTQK5X = 224, kTQK4X = 225, kTQK5T = 226, kTQK4T = 227;
+constexpr int tq_kbits(int type) {
+    return type == kTQK6 ? 6 : type == kTQK7 ? 7 : (type == kTQK5X || type == kTQK5T) ? 5 : (type == kTQK4X || type == kTQK4T) ? 4 : 0;
+}
+constexpr bool is_tq(int type) { return tq_kbits(type) > 0; }
+
+/// Whether the device kernels take this ggml type.
+bool supported(int type);
+/// Whether the type's block layout is known (its row bytes can be computed).  It covered more than supported() while
+/// IQ1_S, IQ1_M and IQ4_NL had no device kernel; they have one now, so both take the same types.
+bool known(int type);
+/// Values a block of the type holds (256; the trellis types 128; 0: not one of these).
+int block_values(int type);
+/// Bytes of a block of the type (0: not one of these).
+int block_bytes(int type);
+/// Bytes of a row of k values (k % block_values == 0).
+inline size_t row_bytes(int type, int64_t k) { return (size_t) (k / block_values(type)) * (size_t) block_bytes(type); }
+/// The type's name ("?" when unknown).
+const char* name(int type);
+
+/// The token / window experts (ds4_gguf.cu's MxArgs, with the roles' types): group g = an expert (grp_ptr null: expert g
+/// of the plan, plan_ptr[g] its blob or 0, its entry g, input row 0; else blob grp_ptr[g], entries grp_ent[grp_start[g]
+/// ..], entry e's input row e >> 3).  x / h as int8 rows with an F32 scale per 32 (xq, xd: rows of n_embd; hq, hd: row e
+/// = entry e, rows of n_ff).
+struct Args {
+    const unsigned long long* plan_ptr = nullptr;
+    const unsigned long long* grp_ptr = nullptr;
+    const int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;
+    size_t tr[3] = {0, 0, 0};   // the roles' offsets in a blob
+    int n_embd = 0, n_ff = 0;
+    float limit = 0.0f;
+    const int8_t* xq = nullptr;
+    const float* xd = nullptr;
+    const int8_t* hq = nullptr;
+    const float* hd = nullptr;
+    float* h = nullptr;      // [entries][n_ff]: the clamped swiglu
+    float* part = nullptr;   // [entries][n_embd]: each entry's down output
+    int tg = 0, td = 0;      // gate / up's type, down's
+};
+/// grid (n_ff / 8, groups): h of every entry (ne_max: a group's most entries).  false: a type not covered.
+bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s);
+/// grid (n_embd / 8, groups): part of every entry.
+bool down(const Args& a, int groups, int ne_max, cudaStream_t s);
+
+/// The prompt path's expert sets (ds4_gguf.cu's SetK): expert i's blob at wbase + i stride owns the sorted rows
+/// [bounds[i], bounds[i + 1]) (local row = sorted row - row0); gate / up: h16[local] = F16 swiglu over x[idx[local]]
+/// (F32 rows, ldx); down: y[local] = down(h16[local]) (F16 rows in, ldx; F32 out, ldo).  A set's experts of at most 16
+/// rows take the token kernels' form (each weight read once, the rows' inputs rounded to int8 per 32 as the token path
+/// rounds them; STRATA_DS4_NO_LIGHT=1: never), the others GEMM tiles (tensor cores on CUDA, F16 operands).
+struct SetArgs {
+    const uint8_t* wbase = nullptr;
+    size_t stride = 0;
+    size_t tr[3] = {0, 0, 0};
+    const int* bounds = nullptr;
+    int row0 = 0, max_rows = 0, n_exp = 0;
+    int n_embd = 0, n_ff = 0;
+    float limit = 0.0f;
+    const float* xf = nullptr;
+    const int* idx = nullptr;
+    const uint16_t* xh = nullptr;
+    int ldx = 0;
+    uint16_t* h = nullptr;
+    float* y = nullptr;
+    int ldo = 0;
+    int tg = 0, td = 0;
+    int skip_le = 0;   // (the tiles) the experts of at most this many rows are left out: the light kernels took them
+};
+bool set_gate_up(const SetArgs& a, cudaStream_t s);
+bool set_down(const SetArgs& a, cudaStream_t s);
+/// The light form alone (the set's experts of at most 16 rows; the others untouched), for any of these types and MXFP4
+/// (tg / td 39: ds4_gguf.cu's experts): false (nothing launched) when it is switched off - the caller's tiles take every
+/// expert then; true: they take those of more rows (skip_le 16).
+bool light_gate_up(const SetArgs& a, cudaStream_t s);
+bool light_down(const SetArgs& a, cudaStream_t s);
+
+/// GEMV jobs on such matrices (an expert's views on the host-routed path): y[c][r] = alpha (row r . x_c) + bias[r] for
+/// ncol activations, x as int8 rows (xq + c k, scales xd + c k / 32).
+struct MvJob {
+    const uint8_t* w = nullptr;
+    size_t ld = 0;   // bytes a row
+    int type = 0, k = 0, n = 0, ncol = 1;
+    const int8_t* xq = nullptr;
+    const float* xd = nullptr;
+    float* y = nullptr;
+    size_t ycs = 0;
+    const float* bias = nullptr;
+    float alpha = 1.0f;
+};
+bool mv(const MvJob* jobs, int n, cudaStream_t s);
+
+/// n rows of k values of a matrix of the type (ld bytes a row) as BF16 (bf16) or F16 bits, dst[n][k]: the chunked prompt path's
+/// dense GEMM operand.  false: a type not covered.
+bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s);
+
+/// The trellis types' kernels (ds4_tq.cu), reached from the entry points above for those types: arithmetic as the CPU lane's
+/// (ds4_tq_cpu.cpp: int8 activations with an F32 scale a 32, exact weights), on the tensor cores of gfx12 (WMMA) or in FMAs.
+namespace tq {
+bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s);
+bool down(const Args& a, int groups, int ne_max, cudaStream_t s);
+bool set_gate_up(const SetArgs& a, cudaStream_t s);
+bool set_down(const SetArgs& a, cudaStream_t s);
+bool mv(const MvJob* jobs, int n, cudaStream_t s);
+}  // namespace tq
+
+}  // namespace strata::ds4::gguf::iq

src/dsv41/ds4_iq1m_codes.inc

@@ -0,0 +1,515 @@
+// Generated from ggml-common.h iq1s_grid_gpu (third_party/ggml): entry (neg << 11 | grid index) -> the 8 int8 codes of an
+// IQ1_M / IQ1_S group (iq1_codes of its low and its high nibbles), the delta sign folded in.  Checked against the device
+// computation by ds4_iq_test (bit-compare) and against ggml by its parity checks.
+{0xf9f9f9f9,0xf9f9f9f9},{0xf9f9f909,0xf9f9f9f9},{0xf9f90101,0xf9f9f9f9},{0xf9f909f9,0xf9f9f9f9},{0xf9f90909,0xf9f9f9f9},{0xf901f901,0xf9f9f9f9},{0xf9010101,0xf9f9f9f9},{0xf909f9f9,0xf9f9f9f9},
+{0xf909f909,0xf9f9f9f9},{0xf90909f9,0xf9f9f9f9},{0xf9090909,0xf9f9f9f9},{0x01f90101,0xf9f9f9f9},{0x0101f901,0xf9f9f9f9},{0x010101f9,0xf9f9f9f9},{0x01010109,0xf9f9f9f9},{0x01090101,0xf9f9f9f9},
+{0x09f9f9f9,0xf9f9f9f9},{0x09f9f909,0xf9f9f9f9},{0x09f909f9,0xf9f9f9f9},{0x09f90909,0xf9f9f9f9},{0x09010101,0xf9f9f9f9},{0x0909f9f9,0xf9f9f9f9},{0x0909f909,0xf9f9f9f9},{0x090909f9,0xf9f9f9f9},
+{0x09090909,0xf9f9f9f9},{0xf9f901f9,0xf9f9f901},{0xf9f90101,0xf9f9f901},{0xf901f901,0xf9f9f901},{0xf90101f9,0xf9f9f901},{0xf9010109,0xf9f9f901},{0xf9010901,0xf9f9f901},{0xf9010909,0xf9f9f901},
+{0xf9090101,0xf9f9f901},{0x01f9f901,0xf9f9f901},{0x01f90109,0xf9f9f901},{0x01f90901,0xf9f9f901},{0x0101f909,0xf9f9f901},{0x01010101,0xf9f9f901},{0x01010909,0xf9f9f901},{0x0109f901,0xf9f9f901},
+{0x010901f9,0xf9f9f901},{0x01090109,0xf9f9f901},{0x010909f9,0xf9f9f901},{0x09f90101,0xf9f9f901},{0x0901f901,0xf9f9f901},{0x090101f9,0xf9f9f901},{0x09010109,0xf9f9f901},{0x09090101,0xf9f9f901},
+{0xf9f9f9f9,0xf9f9f909},{0xf9f9f909,0xf9f9f909},{0xf9f909f9,0xf9f9f909},{0xf9f90909,0xf9f9f909},{0xf9010101,0xf9f9f909},{0xf909f9f9,0xf9f9f909},{0xf909f909,0xf9f9f909},{0xf90909f9,0xf9f9f909},
+{0xf9090909,0xf9f9f909},{0x01f90101,0xf9f9f909},{0x0101f901,0xf9f9f909},{0x01010901,0xf9f9f909},{0x010901f9,0xf9f9f909},{0x01090901,0xf9f9f909},{0x09f9f9f9,0xf9f9f909},{0x09f9f909,0xf9f9f909},
+{0x09f909f9,0xf9f9f909},{0x09f90909,0xf9f9f909},{0x0901f901,0xf9f9f909},{0x09010101,0xf9f9f909},{0x09010901,0xf9f9f909},{0x0909f9f9,0xf9f9f909},{0x0909f909,0xf9f9f909},{0x090909f9,0xf9f9f909},
+{0x09090909,0xf9f9f909},{0xf901f901,0xf9f901f9},{0xf90101f9,0xf9f901f9},{0xf9010109,0xf9f901f9},{0xf9090101,0xf9f901f9},{0x01f9f901,0xf9f901f9},{0x01f90901,0xf9f901f9},{0x01010101,0xf9f901f9},
+{0x01010909,0xf9f901f9},{0x010901f9,0xf9f901f9},{0x01090101,0xf9f901f9},{0x0901f901,0xf9f901f9},{0x09010901,0xf9f901f9},{0x09090101,0xf9f901f9},{0xf9f9f901,0xf9f90101},{0xf9f901f9,0xf9f90101},
+{0xf9f90101,0xf9f90101},{0xf9f90109,0xf9f90101},{0xf9010101,0xf9f90101},{0xf90109f9,0xf9f90101},{0xf9010909,0xf9f90101},{0xf9090901,0xf9f90101},{0x01f9f9f9,0xf9f90101},{0x01f90101,0xf9f90101},
+{0x01f90909,0xf9f90101},{0x0101f9f9,0xf9f90101},{0x0101f901,0xf9f90101},{0x010101f9,0xf9f90101},{0x01010101,0xf9f90101},{0x01010109,0xf9f90101},{0x01010901,0xf9f90101},{0x0109f9f9,0xf9f90101},
+{0x0109f909,0xf9f90101},{0x01090101,0xf9f90101},{0x010909f9,0xf9f90101},{0x01090909,0xf9f90101},{0x09f9f901,0xf9f90101},{0x0901f901,0xf9f90101},{0x09010101,0xf9f90101},{0x090109f9,0xf9f90101},
+{0x09010909,0xf9f90101},{0x0909f901,0xf9f90101},{0x090901f9,0xf9f90101},{0x09090101,0xf9f90101},{0x09090109,0xf9f90101},{0x09090901,0xf9f90101},{0xf90101f9,0xf9f90109},{0xf9010901,0xf9f90109},
+{0x01f9f901,0xf9f90109},{0x01f901f9,0xf9f90109},{0x0101f9f9,0xf9f90109},{0x0101f909,0xf9f90109},{0x01010101,0xf9f90109},{0x010109f9,0xf9f90109},{0x0109f9f9,0xf9f90109},{0x0109f901,0xf9f90109},
+{0x01090109,0xf9f90109},{0x01090901,0xf9f90109},{0x09f90101,0xf9f90109},{0x0901f901,0xf9f90109},{0x090101f9,0xf9f90109},{0x09010901,0xf9f90109},{0xf9f9f9f9,0xf9f909f9},{0xf9f9f909,0xf9f909f9},
+{0xf9f909f9,0xf9f909f9},{0xf9f90909,0xf9f909f9},{0xf9010101,0xf9f909f9},{0xf909f9f9,0xf9f909f9},{0xf909f909,0xf9f909f9},{0xf90909f9,0xf9f909f9},{0xf9090909,0xf9f909f9},{0x01f90101,0xf9f909f9},
+{0x0101f901,0xf9f909f9},{0x01010109,0xf9f909f9},{0x01090101,0xf9f909f9},{0x09f9f9f9,0xf9f909f9},{0x09f9f909,0xf9f909f9},{0x09f909f9,0xf9f909f9},{0x09f90909,0xf9f909f9},{0x09010101,0xf9f909f9},
+{0x0909f9f9,0xf9f909f9},{0x0909f909,0xf9f909f9},{0x090909f9,0xf9f909f9},{0x09090909,0xf9f909f9},{0xf9f90101,0xf9f90901},{0xf901f901,0xf9f90901},{0xf90101f9,0xf9f90901},{0xf9010901,0xf9f90901},
+{0xf90901f9,0xf9f90901},{0xf9090101,0xf9f90901},{0x01f9f901,0xf9f90901},{0x0101f9f9,0xf9f90901},{0x0101f901,0xf9f90901},{0x01010101,0xf9f90901},{0x0109f901,0xf9f90901},{0x010901f9,0xf9f90901},
+{0x01090901,0xf9f90901},{0x0901f901,0xf9f90901},{0x090101f9,0xf9f90901},{0x09010109,0xf9f90901},{0x09010901,0xf9f90901},{0x09090101,0xf9f90901},{0xf9f9f9f9,0xf9f90909},{0xf9f9f909,0xf9f90909},
+{0xf9f909f9,0xf9f90909},{0xf9f90909,0xf9f90909},{0xf9010101,0xf9f90909},{0xf909f9f9,0xf9f90909},{0xf909f909,0xf9f90909},{0xf90909f9,0xf9f90909},{0xf9090909,0xf9f90909},{0x01f90101,0xf9f90909},
+{0x0101f901,0xf9f90909},{0x01010901,0xf9f90909},{0x0109f901,0xf9f90909},{0x01090101,0xf9f90909},{0x09f9f9f9,0xf9f90909},{0x09f9f909,0xf9f90909},{0x09f90101,0xf9f90909},{0x09f909f9,0xf9f90909},
+{0x09f90909,0xf9f90909},{0x09010101,0xf9f90909},{0x0909f9f9,0xf9f90909},{0x0909f909,0xf9f90909},{0x090909f9,0xf9f90909},{0x09090909,0xf9f90909},{0xf901f9f9,0xf901f9f9},{0xf901f901,0xf901f9f9},
+{0xf90101f9,0xf901f9f9},{0xf9010901,0xf901f9f9},{0xf90901f9,0xf901f9f9},{0xf9090101,0xf901f9f9},{0x01f9f901,0xf901f9f9},{0x01f901f9,0xf901f9f9},{0x0101f9f9,0xf901f9f9},{0x01010101,0xf901f9f9},
+{0x010109f9,0xf901f9f9},{0x0109f901,0xf901f9f9},{0x010901f9,0xf901f9f9},{0x01090101,0xf901f9f9},{0x01090901,0xf901f9f9},{0x0901f901,0xf901f9f9},{0x090101f9,0xf901f9f9},{0x09010109,0xf901f9f9},
+{0x0909f901,0xf901f9f9},{0x09090101,0xf901f9f9},{0xf9f9f901,0xf901f901},{0xf9f901f9,0xf901f901},{0xf9f90109,0xf901f901},{0xf9f90901,0xf901f901},{0xf901f9f9,0xf901f901},{0xf901f909,0xf901f901},
+{0xf9010101,0xf901f901},{0xf90109f9,0xf901f901},{0xf909f901,0xf901f901},{0xf90901f9,0xf901f901},{0xf9090901,0xf901f901},{0x01f90101,0xf901f901},{0x01f90909,0xf901f901},{0x0101f9f9,0xf901f901},
+{0x0101f901,0xf901f901},{0x0101f909,0xf901f901},{0x010101f9,0xf901f901},{0x01010101,0xf901f901},{0x01010109,0xf901f901},{0x01010901,0xf901f901},{0x0109f9f9,0xf901f901},{0x01090101,0xf901f901},
+{0x09f901f9,0xf901f901},{0x0901f909,0xf901f901},{0x09010101,0xf901f901},{0x0909f901,0xf901f901},{0x090901f9,0xf901f901},{0xf901f901,0xf901f909},{0xf90101f9,0xf901f909},{0xf9010109,0xf901f909},
+{0xf9090101,0xf901f909},{0x01f9f9f9,0xf901f909},{0x01f90109,0xf901f909},{0x01f90901,0xf901f909},{0x0101f909,0xf901f909},{0x01010101,0xf901f909},{0x010109f9,0xf901f909},{0x01010909,0xf901f909},
+{0x010901f9,0xf901f909},{0x01090109,0xf901f909},{0x09f90101,0xf901f909},{0x0901f901,0xf901f909},{0x090101f9,0xf901f909},{0x09010109,0xf901f909},{0x09090101,0xf901f909},{0xf9f9f901,0xf90101f9},
+{0xf9f90109,0xf90101f9},{0xf9f90901,0xf90101f9},{0xf90101f9,0xf90101f9},{0xf9010101,0xf90101f9},{0xf90109f9,0xf90101f9},{0xf9010901,0xf90101f9},{0xf909f901,0xf90101f9},{0xf9090109,0xf90101f9},
+{0x01f9f901,0xf90101f9},{0x01f90101,0xf90101f9},{0x01f90109,0xf90101f9},{0x01f909f9,0xf90101f9},{0x01f90909,0xf90101f9},{0x0101f901,0xf90101f9},{0x010101f9,0xf90101f9},{0x01010101,0xf90101f9},
+{0x01010109,0xf90101f9},{0x01010901,0xf90101f9},{0x0109f909,0xf90101f9},{0x01090101,0xf90101f9},{0x010909f9,0xf90101f9},{0x09f901f9,0xf90101f9},{0x09f90901,0xf90101f9},{0x0901f9f9,0xf90101f9},
+{0x090101f9,0xf90101f9},{0x09010101,0xf90101f9},{0x090109f9,0xf90101f9},{0x09010901,0xf90101f9},{0x09010909,0xf90101f9},{0x0909f901,0xf90101f9},{0x090901f9,0xf90101f9},{0x09090101,0xf90101f9},
+{0x09090901,0xf90101f9},{0xf9f9f909,0xf9010101},{0xf9f90101,0xf9010101},{0xf9f90909,0xf9010101},{0xf901f901,0xf9010101},{0xf90101f9,0xf9010101},{0xf9010101,0xf9010101},{0xf9010109,0xf9010101},
+{0xf9010901,0xf9010101},{0xf909f9f9,0xf9010101},{0xf909f909,0xf9010101},{0xf9090101,0xf9010101},{0xf90909f9,0xf9010101},{0xf9090909,0xf9010101},{0x01f9f901,0xf9010101},{0x01f901f9,0xf9010101},
+{0x01f90101,0xf9010101},{0x01f90109,0xf9010101},{0x0101f901,0xf9010101},{0x0101f909,0xf9010101},{0x010101f9,0xf9010101},{0x01010101,0xf9010101},{0x01010109,0xf9010101},{0x01010901,0xf9010101},
+{0x01010909,0xf9010101},{0x0109f901,0xf9010101},{0x010901f9,0xf9010101},{0x01090101,0xf9010101},{0x01090109,0xf9010101},{0x01090901,0xf9010101},{0x09f9f9f9,0xf9010101},{0x09f9f909,0xf9010101},
+{0x09f901f9,0xf9010101},{0x09f90101,0xf9010101},{0x09f909f9,0xf9010101},{0x09f90909,0xf9010101},{0x0901f9f9,0xf9010101},{0x0901f901,0xf9010101},{0x090101f9,0xf9010101},{0x09010101,0xf9010101},
+{0x09010109,0xf9010101},{0x09010901,0xf9010101},{0x09010909,0xf9010101},{0x0909f9f9,0xf9010101},{0x0909f909,0xf9010101},{0x09090101,0xf9010101},{0xf9f9f901,0xf9010109},{0xf9f901f9,0xf9010109},
+{0xf9f90101,0xf9010109},{0xf9f90109,0xf9010109},{0xf9010101,0xf9010109},{0xf9010109,0xf9010109},{0xf90109f9,0xf9010109},{0xf9010909,0xf9010109},{0xf909f901,0xf9010109},{0xf9090109,0xf9010109},
+{0x01f9f9f9,0xf9010109},{0x01f9f909,0xf9010109},{0x01f901f9,0xf9010109},{0x01f90101,0xf9010109},{0x01f909f9,0xf9010109},{0x01f90909,0xf9010109},{0x0101f901,0xf9010109},{0x01010101,0xf9010109},
+{0x01010109,0xf9010109},{0x010109f9,0xf9010109},{0x01010901,0xf9010109},{0x0109f901,0xf9010109},{0x010901f9,0xf9010109},{0x01090101,0xf9010109},{0x010909f9,0xf9010109},{0x01090901,0xf9010109},
+{0x01090909,0xf9010109},{0x09f90109,0xf9010109},{0x09f90909,0xf9010109},{0x0901f909,0xf9010109},{0x09010101,0xf9010109},{0x090901f9,0xf9010109},{0x09090901,0xf9010109},{0xf901f901,0xf90109f9},
+{0xf9010109,0xf90109f9},{0xf9090101,0xf90109f9},{0x01f9f901,0xf90109f9},{0x01f901f9,0xf90109f9},{0x01f90109,0xf90109f9},{0x01f90901,0xf90109f9},{0x0101f9f9,0xf90109f9},{0x01010101,0xf90109f9},
+{0x010109f9,0xf90109f9},{0x01010909,0xf90109f9},{0x0109f9f9,0xf90109f9},{0x0109f901,0xf90109f9},{0x010901f9,0xf90109f9},{0x01090109,0xf90109f9},{0x01090901,0xf90109f9},{0x09f90101,0xf90109f9},
+{0x0901f901,0xf90109f9},{0x090101f9,0xf90109f9},{0x09090101,0xf90109f9},{0xf901f9f9,0xf9010901},{0xf901f909,0xf9010901},{0xf9010101,0xf9010901},{0xf9010909,0xf9010901},{0xf909f901,0xf9010901},
+{0xf9090101,0xf9010901},{0x01f9f909,0xf9010901},{0x01f901f9,0xf9010901},{0x01f90101,0xf9010901},{0x01f909f9,0xf9010901},{0x0101f901,0xf9010901},{0x010101f9,0xf9010901},{0x01010101,0xf9010901},
+{0x01010109,0xf9010901},{0x01010901,0xf9010901},{0x01010909,0xf9010901},{0x0109f9f9,0xf9010901},{0x01090101,0xf9010901},{0x01090909,0xf9010901},{0x09f90901,0xf9010901},{0x0901f901,0xf9010901},
+{0x0901f909,0xf9010901},{0x09010101,0xf9010901},{0x090109f9,0xf9010901},{0x0909f901,0xf9010901},{0x09090109,0xf9010901},{0x09090901,0xf9010901},{0xf9f90901,0xf9010909},{0xf9010109,0xf9010909},
+{0xf90901f9,0xf9010909},{0xf9090109,0xf9010909},{0x01f901f9,0xf9010909},{0x01f90109,0xf9010909},{0x01f90901,0xf9010909},{0x0101f9f9,0xf9010909},{0x0101f909,0xf9010909},{0x01010101,0xf9010909},
+{0x010109f9,0xf9010909},{0x0109f901,0xf9010909},{0x01090109,0xf9010909},{0x01090901,0xf9010909},{0x09f90101,0xf9010909},{0x0901f901,0xf9010909},{0x09010109,0xf9010909},{0x09010909,0xf9010909},
+{0xf9f9f9f9,0xf909f9f9},{0xf9f9f909,0xf909f9f9},{0xf9f909f9,0xf909f9f9},{0xf9f90909,0xf909f9f9},{0xf9010101,0xf909f9f9},{0xf909f9f9,0xf909f9f9},{0xf909f909,0xf909f9f9},{0xf9090101,0xf909f9f9},
+{0xf90909f9,0xf909f9f9},{0xf9090909,0xf909f9f9},{0x01f90101,0xf909f9f9},{0x0101f901,0xf909f9f9},{0x01010901,0xf909f9f9},{0x0109f901,0xf909f9f9},{0x01090101,0xf909f9f9},{0x09f9f9f9,0xf909f9f9},
+{0x09f9f909,0xf909f9f9},{0x09f909f9,0xf909f9f9},{0x09f90909,0xf909f9f9},{0x09010101,0xf909f9f9},{0x0909f9f9,0xf909f9f9},{0x0909f909,0xf909f9f9},{0x09090101,0xf909f9f9},{0x090909f9,0xf909f9f9},
+{0x09090909,0xf909f9f9},{0xf9f90101,0xf909f901},{0xf901f901,0xf909f901},{0xf90101f9,0xf909f901},{0xf9010901,0xf909f901},{0xf9090101,0xf909f901},{0x01f9f909,0xf909f901},{0x01f901f9,0xf909f901},
+{0x01f90901,0xf909f901},{0x01010101,0xf909f901},{0x010109f9,0xf909f901},{0x01010909,0xf909f901},{0x0109f901,0xf909f901},{0x010901f9,0xf909f901},{0x01090101,0xf909f901},{0x01090109,0xf909f901},
+{0x09f90101,0xf909f901},{0x0901f9f9,0xf909f901},{0x09010109,0xf909f901},{0x09010901,0xf909f901},{0x09090101,0xf909f901},{0xf9f9f901,0xf909f909},{0xf9f909f9,0xf909f909},{0xf9f90909,0xf909f909},
+{0xf901f901,0xf909f909},{0xf9010101,0xf909f909},{0xf909f9f9,0xf909f909},{0xf909f909,0xf909f909},{0xf90909f9,0xf909f909},{0xf9090909,0xf909f909},{0x01f90101,0xf909f909},{0x0101f901,0xf909f909},
+{0x01010109,0xf909f909},{0x01010901,0xf909f909},{0x01090101,0xf909f909},{0x09f9f901,0xf909f909},{0x09f909f9,0xf909f909},{0x09f90909,0xf909f909},{0x0901f901,0xf909f909},{0x09010101,0xf909f909},
+{0x0909f9f9,0xf909f909},{0x0909f909,0xf909f909},{0x090909f9,0xf909f909},{0x09090909,0xf909f909},{0xf9f90101,0xf90901f9},{0xf90101f9,0xf90901f9},{0xf9010109,0xf90901f9},{0xf9010901,0xf90901f9},
+{0xf9090101,0xf90901f9},{0x01f901f9,0xf90901f9},{0x01f90101,0xf90901f9},{0x01f90109,0xf90901f9},{0x01f90901,0xf90901f9},{0x0101f909,0xf90901f9},{0x01010101,0xf90901f9},{0x010109f9,0xf90901f9},
+{0x01010909,0xf90901f9},{0x01090109,0xf90901f9},{0x09f90101,0xf90901f9},{0x0901f901,0xf90901f9},{0x090101f9,0xf90901f9},{0x09010901,0xf90901f9},{0x0909f901,0xf90901f9},{0x09090101,0xf90901f9},
+{0xf9f90901,0xf9090101},{0xf9010101,0xf9090101},{0xf909f901,0xf9090101},{0xf9090901,0xf9090101},{0x01f9f9f9,0xf9090101},{0x01f90101,0xf9090101},{0x01f909f9,0xf9090101},{0x0101f901,0xf9090101},
+{0x010101f9,0xf9090101},{0x01010101,0xf9090101},{0x01010901,0xf9090101},{0x0109f909,0xf9090101},{0x01090101,0xf9090101},{0x010909f9,0xf9090101},{0x09f90901,0xf9090101},{0x0901f9f9,0xf9090101},
+{0x090101f9,0xf9090101},{0x09010101,0xf9090101},{0x090109f9,0xf9090101},{0x09010909,0xf9090101},{0x0909f901,0xf9090101},{0x090901f9,0xf9090101},{0x09090109,0xf9090101},{0x09090901,0xf9090101},
+{0xf9f90101,0xf9090109},{0xf901f9f9,0xf9090109},{0xf901f909,0xf9090109},{0xf9010901,0xf9090109},{0xf9090101,0xf9090109},{0x01f9f901,0xf9090109},{0x01f90901,0xf9090109},{0x01010101,0xf9090109},
+{0x0109f9f9,0xf9090109},{0x0109f901,0xf9090109},{0x01090901,0xf9090109},{0x09f901f9,0xf9090109},{0x09f90109,0xf9090109},{0x0901f9f9,0xf9090109},{0x09010909,0xf9090109},{0xf9f9f9f9,0xf90909f9},
+{0xf9f9f909,0xf90909f9},{0xf9f909f9,0xf90909f9},{0xf9f90909,0xf90909f9},{0xf9010101,0xf90909f9},{0xf909f9f9,0xf90909f9},{0xf909f909,0xf90909f9},{0xf90909f9,0xf90909f9},{0xf9090909,0xf90909f9},
+{0x01f90101,0xf90909f9},{0x0101f901,0xf90909f9},{0x010101f9,0xf90909f9},{0x01090101,0xf90909f9},{0x09f9f9f9,0xf90909f9},{0x09f9f909,0xf90909f9},{0x09f909f9,0xf90909f9},{0x09f90909,0xf90909f9},
+{0x0909f9f9,0xf90909f9},{0x0909f909,0xf90909f9},{0x090909f9,0xf90909f9},{0x09090909,0xf90909f9},{0xf9f90901,0xf9090901},{0xf901f901,0xf9090901},{0xf90101f9,0xf9090901},{0xf9010901,0xf9090901},
+{0xf9090101,0xf9090901},{0x01f90109,0xf9090901},{0x01f90901,0xf9090901},{0x0101f909,0xf9090901},{0x01010101,0xf9090901},{0x0109f901,0xf9090901},{0x010901f9,0xf9090901},{0x01090109,0xf9090901},
+{0x01090901,0xf9090901},{0x09f90101,0xf9090901},{0x0901f9f9,0xf9090901},{0x09010109,0xf9090901},{0x09010901,0xf9090901},{0x090901f9,0xf9090901},{0x09090101,0xf9090901},{0xf9f9f9f9,0xf9090909},
+{0xf9f9f909,0xf9090909},{0xf9f909f9,0xf9090909},{0xf9f90909,0xf9090909},{0xf909f9f9,0xf9090909},{0xf909f909,0xf9090909},{0xf90909f9,0xf9090909},{0xf9090909,0xf9090909},{0x01f90101,0xf9090909},
+{0x0101f901,0xf9090909},{0x01010109,0xf9090909},{0x01010901,0xf9090909},{0x01090101,0xf9090909},{0x09f9f9f9,0xf9090909},{0x09f9f909,0xf9090909},{0x09f909f9,0xf9090909},{0x09f90909,0xf9090909},
+{0x09010101,0xf9090909},{0x0909f9f9,0xf9090909},{0x0909f909,0xf9090909},{0x090909f9,0xf9090909},{0x09090909,0xf9090909},{0xf9f90101,0x01f9f9f9},{0xf901f901,0x01f9f9f9},{0xf9010109,0x01f9f9f9},
+{0xf9090101,0x01f9f9f9},{0x01f90901,0x01f9f9f9},{0x0101f909,0x01f9f9f9},{0x01010101,0x01f9f9f9},{0x010109f9,0x01f9f9f9},{0x01010909,0x01f9f9f9},{0x0109f901,0x01f9f9f9},{0x010901f9,0x01f9f9f9},
+{0x01090109,0x01f9f9f9},{0x090101f9,0x01f9f9f9},{0x09010901,0x01f9f9f9},{0x0909f901,0x01f9f9f9},{0x09090109,0x01f9f9f9},{0xf9f9f9f9,0x01f9f901},{0xf9f9f901,0x01f9f901},{0xf9f901f9,0x01f9f901},
+{0xf9f90109,0x01f9f901},{0xf9f90901,0x01f9f901},{0xf901f909,0x01f9f901},{0xf9010101,0x01f9f901},{0xf9010109,0x01f9f901},{0xf90109f9,0x01f9f901},{0xf9010909,0x01f9f901},{0xf909f901,0x01f9f901},
+{0xf9090109,0x01f9f901},{0xf9090901,0x01f9f901},{0x01f90101,0x01f9f901},{0x01f909f9,0x01f9f901},{0x01f90909,0x01f9f901},{0x0101f901,0x01f9f901},{0x010101f9,0x01f9f901},{0x01010101,0x01f9f901},
+{0x01010109,0x01f9f901},{0x01010901,0x01f9f901},{0x01010909,0x01f9f901},{0x01090101,0x01f9f901},{0x010909f9,0x01f9f901},{0x01090909,0x01f9f901},{0x09f9f901,0x01f9f901},{0x09f901f9,0x01f9f901},
+{0x09f90109,0x01f9f901},{0x0901f9f9,0x01f9f901},{0x0901f909,0x01f9f901},{0x09010101,0x01f9f901},{0x0909f9f9,0x01f9f901},{0x0909f901,0x01f9f901},{0x0909f909,0x01f9f901},{0xf9f90101,0x01f9f909},
+{0xf901f901,0x01f9f909},{0xf90101f9,0x01f9f909},{0xf9010109,0x01f9f909},{0xf9090101,0x01f9f909},{0x01f9f901,0x01f9f909},{0x0101f909,0x01f9f909},{0x01010101,0x01f9f909},{0x01010909,0x01f9f909},
+{0x010901f9,0x01f9f909},{0x01090901,0x01f9f909},{0x09f90901,0x01f9f909},{0x090101f9,0x01f9f909},{0x09090101,0x01f9f909},{0xf9f9f901,0x01f901f9},{0xf9010101,0x01f901f9},{0xf9010901,0x01f901f9},
+{0xf9090901,0x01f901f9},{0x01f90101,0x01f901f9},{0x01f909f9,0x01f901f9},{0x01f90909,0x01f901f9},{0x0101f901,0x01f901f9},{0x010101f9,0x01f901f9},{0x01010101,0x01f901f9},{0x01010109,0x01f901f9},
+{0x0109f901,0x01f901f9},{0x0109f909,0x01f901f9},{0x01090101,0x01f901f9},{0x010909f9,0x01f901f9},{0x01090909,0x01f901f9},{0x09f9f901,0x01f901f9},{0x09f90109,0x01f901f9},{0x09f90901,0x01f901f9},
+{0x0901f9f9,0x01f901f9},{0x0901f909,0x01f901f9},{0x09010101,0x01f901f9},{0x0909f9f9,0x01f901f9},{0x0909f901,0x01f901f9},{0x09090901,0x01f901f9},{0xf9f9f901,0x01f90101},{0xf9f9f909,0x01f90101},
+{0xf9f90101,0x01f90101},{0xf9f90909,0x01f90101},{0xf901f901,0x01f90101},{0xf90101f9,0x01f90101},{0xf9010101,0x01f90101},{0xf9010109,0x01f90101},{0xf9010901,0x01f90101},{0xf909f9f9,0x01f90101},
+{0xf9090101,0x01f90101},{0xf9090909,0x01f90101},{0x01f9f901,0x01f90101},{0x01f901f9,0x01f90101},{0x01f90101,0x01f90101},{0x01f90109,0x01f90101},{0x01f90901,0x01f90101},{0x0101f9f9,0x01f90101},
+{0x0101f901,0x01f90101},{0x010101f9,0x01f90101},{0x01010101,0x01f90101},{0x01010109,0x01f90101},{0x010109f9,0x01f90101},{0x01010901,0x01f90101},{0x0109f901,0x01f90101},{0x010901f9,0x01f90101},
+{0x01090101,0x01f90101},{0x01090109,0x01f90101},{0x01090901,0x01f90101},{0x09f9f909,0x01f90101},{0x09f901f9,0x01f90101},{0x09f90101,0x01f90101},{0x09f909f9,0x01f90101},{0x0901f901,0x01f90101},
+{0x090101f9,0x01f90101},{0x09010101,0x01f90101},{0x09010109,0x01f90101},{0x09010901,0x01f90101},{0x09010909,0x01f90101},{0x09090101,0x01f90101},{0x090909f9,0x01f90101},{0x09090909,0x01f90101},
+{0xf9f9f901,0x01f90109},{0xf9f90101,0x01f90109},{0xf9f90901,0x01f90109},{0xf90101f9,0x01f90109},{0xf9010101,0x01f90109},{0xf90109f9,0x01f90109},{0xf9010909,0x01f90109},{0xf909f901,0x01f90109},
+{0xf90901f9,0x01f90109},{0xf9090901,0x01f90109},{0x01f9f9f9,0x01f90109},{0x01f9f909,0x01f90109},{0x01f90101,0x01f90109},{0x01f909f9,0x01f90109},{0x0101f9f9,0x01f90109},{0x0101f901,0x01f90109},
+{0x0101f909,0x01f90109},{0x01010101,0x01f90109},{0x01010109,0x01f90109},{0x01010901,0x01f90109},{0x0109f909,0x01f90109},{0x01090101,0x01f90109},{0x010909f9,0x01f90109},{0x09f9f901,0x01f90109},
+{0x09f90101,0x01f90109},{0x09f90909,0x01f90109},{0x090101f9,0x01f90109},{0x09010101,0x01f90109},{0x0909f901,0x01f90109},{0x090901f9,0x01f90109},{0x09090109,0x01f90109},{0xf9f90101,0x01f909f9},
+{0xf901f901,0x01f909f9},{0xf9010101,0x01f909f9},{0xf9010909,0x01f909f9},{0xf9090101,0x01f909f9},{0x01f9f909,0x01f909f9},{0x01f90901,0x01f909f9},{0x0101f9f9,0x01f909f9},{0x01010101,0x01f909f9},
+{0x010109f9,0x01f909f9},{0x0109f901,0x01f909f9},{0x010901f9,0x01f909f9},{0x01090109,0x01f909f9},{0x01090901,0x01f909f9},{0x09f90101,0x01f909f9},{0x0901f901,0x01f909f9},{0x090101f9,0x01f909f9},
+{0x09010109,0x01f909f9},{0x09010901,0x01f909f9},{0x09090101,0x01f909f9},{0xf9f9f901,0x01f90901},{0xf9f90101,0x01f90901},{0xf9f90109,0x01f90901},{0xf9f90909,0x01f90901},{0xf901f9f9,0x01f90901},
+{0xf90101f9,0x01f90901},{0xf9010101,0x01f90901},{0xf90109f9,0x01f90901},{0xf909f901,0x01f90901},{0xf90901f9,0x01f90901},{0xf9090109,0x01f90901},{0x01f9f9f9,0x01f90901},{0x01f90101,0x01f90901},
+{0x01f90909,0x01f90901},{0x0101f901,0x01f90901},{0x0101f909,0x01f90901},{0x010101f9,0x01f90901},{0x01010101,0x01f90901},{0x01010109,0x01f90901},{0x01010901,0x01f90901},{0x0109f9f9,0x01f90901},
+{0x0109f909,0x01f90901},{0x01090101,0x01f90901},{0x01090109,0x01f90901},{0x01090909,0x01f90901},{0x09f90109,0x01f90901},{0x09f90901,0x01f90901},{0x0901f909,0x01f90901},{0x09010101,0x01f90901},
+{0x09010109,0x01f90901},{0x090109f9,0x01f90901},{0x0909f901,0x01f90901},{0x090901f9,0x01f90901},{0x09090109,0x01f90901},{0x09090901,0x01f90901},{0xf9010109,0x01f90909},{0x01f901f9,0x01f90909},
+{0x01f90109,0x01f90909},{0x01f90901,0x01f90909},{0x01010101,0x01f90909},{0x010109f9,0x01f90909},{0x01010909,0x01f90909},{0x010901f9,0x01f90909},{0x01090901,0x01f90909},{0x090101f9,0x01f90909},
+{0x09090101,0x01f90909},{0xf9f9f901,0x0101f9f9},{0xf9f901f9,0x0101f9f9},{0xf9f90101,0x0101f9f9},{0xf9f90109,0x0101f9f9},{0xf9f90901,0x0101f9f9},{0xf901f909,0x0101f9f9},{0xf9010101,0x0101f9f9},
+{0xf9010909,0x0101f9f9},{0xf909f901,0x0101f9f9},{0xf90901f9,0x0101f9f9},{0xf9090901,0x0101f9f9},{0x01f9f9f9,0x0101f9f9},{0x01f90101,0x0101f9f9},{0x01f909f9,0x0101f9f9},{0x0101f901,0x0101f9f9},
+{0x010101f9,0x0101f9f9},{0x01010101,0x0101f9f9},{0x01010109,0x0101f9f9},{0x01010901,0x0101f9f9},{0x01090101,0x0101f9f9},{0x010909f9,0x0101f9f9},{0x09f90109,0x0101f9f9},{0x09f90901,0x0101f9f9},
+{0x09010101,0x0101f9f9},{0x090109f9,0x0101f9f9},{0x0909f9f9,0x0101f9f9},{0x0909f901,0x0101f9f9},{0x09090109,0x0101f9f9},{0x09090901,0x0101f9f9},{0xf9f90101,0x0101f901},{0xf9f909f9,0x0101f901},
+{0xf9f90901,0x0101f901},{0xf9f90909,0x0101f901},{0xf901f901,0x0101f901},{0xf90101f9,0x0101f901},{0xf9010101,0x0101f901},{0xf9010109,0x0101f901},{0xf90109f9,0x0101f901},{0xf9010901,0x0101f901},
+{0xf909f9f9,0x0101f901},{0xf9090101,0x0101f901},{0xf9090109,0x0101f901},{0xf90909f9,0x0101f901},{0xf9090909,0x0101f901},{0x01f9f901,0x0101f901},{0x01f901f9,0x0101f901},{0x01f90101,0x0101f901},
+{0x01f90109,0x0101f901},{0x01f90901,0x0101f901},{0x0101f9f9,0x0101f901},{0x0101f901,0x0101f901},{0x0101f909,0x0101f901},{0x010101f9,0x0101f901},{0x01010101,0x0101f901},{0x01010109,0x0101f901},
+{0x010109f9,0x0101f901},{0x01010901,0x0101f901},{0x01010909,0x0101f901},{0x0109f901,0x0101f901},{0x010901f9,0x0101f901},{0x01090101,0x0101f901},{0x01090109,0x0101f901},{0x01090901,0x0101f901},
+{0x09f9f909,0x0101f901},{0x09f90101,0x0101f901},{0x0901f901,0x0101f901},{0x090101f9,0x0101f901},{0x09010101,0x0101f901},{0x09010109,0x0101f901},{0x09010901,0x0101f901},{0x0909f9f9,0x0101f901},
+{0x09090101,0x0101f901},{0x09090909,0x0101f901},{0xf9f9f901,0x0101f909},{0xf9f90109,0x0101f909},{0xf901f909,0x0101f909},{0xf9010101,0x0101f909},{0xf9010909,0x0101f909},{0xf909f901,0x0101f909},
+{0xf90901f9,0x0101f909},{0x01f9f909,0x0101f909},{0x01f90101,0x0101f909},{0x01f90909,0x0101f909},{0x0101f901,0x0101f909},{0x010101f9,0x0101f909},{0x01010101,0x0101f909},{0x01010109,0x0101f909},
+{0x01010901,0x0101f909},{0x0109f909,0x0101f909},{0x01090101,0x0101f909},{0x09f90101,0x0101f909},{0x0901f9f9,0x0101f909},{0x0901f909,0x0101f909},{0x09010101,0x0101f909},{0x09010901,0x0101f909},
+{0x09010909,0x0101f909},{0x090901f9,0x0101f909},{0xf9f901f9,0x010101f9},{0xf9f90101,0x010101f9},{0xf901f901,0x010101f9},{0xf90101f9,0x010101f9},{0xf9010101,0x010101f9},{0xf9010109,0x010101f9},
+{0xf90109f9,0x010101f9},{0xf9010901,0x010101f9},{0xf909f901,0x010101f9},{0xf9090101,0x010101f9},{0xf90909f9,0x010101f9},{0xf9090909,0x010101f9},{0x01f9f901,0x010101f9},{0x01f901f9,0x010101f9},
+{0x01f90101,0x010101f9},{0x01f90109,0x010101f9},{0x01f90901,0x010101f9},{0x01f90909,0x010101f9},{0x0101f9f9,0x010101f9},{0x0101f901,0x010101f9},{0x010101f9,0x010101f9},{0x01010101,0x010101f9},
+{0x01010109,0x010101f9},{0x010109f9,0x010101f9},{0x01010901,0x010101f9},{0x01010909,0x010101f9},{0x0109f901,0x010101f9},{0x0109f909,0x010101f9},{0x010901f9,0x010101f9},{0x01090101,0x010101f9},
+{0x01090109,0x010101f9},{0x01090901,0x010101f9},{0x09f9f9f9,0x010101f9},{0x09f909f9,0x010101f9},{0x09f90909,0x010101f9},{0x0901f901,0x010101f9},{0x090101f9,0x010101f9},{0x09010101,0x010101f9},
+{0x09010109,0x010101f9},{0x09010901,0x010101f9},{0x0909f901,0x010101f9},{0x090901f9,0x010101f9},{0x09090101,0x010101f9},{0x09090909,0x010101f9},{0xf9f9f901,0x01010101},{0xf9f9f909,0x01010101},
+{0xf9f901f9,0x01010101},{0xf9f90101,0x01010101},{0xf9f90109,0x01010101},{0xf9f90901,0x01010101},{0xf901f9f9,0x01010101},{0xf901f901,0x01010101},{0xf901f909,0x01010101},{0xf90101f9,0x01010101},
+{0xf9010101,0x01010101},{0xf9010109,0x01010101},{0xf9010901,0x01010101},{0xf9010909,0x01010101},{0xf909f901,0x01010101},{0xf90901f9,0x01010101},{0xf9090101,0x01010101},{0xf9090109,0x01010101},
+{0xf9090901,0x01010101},{0x01f9f9f9,0x01010101},{0x01f9f901,0x01010101},{0x01f9f909,0x01010101},{0x01f901f9,0x01010101},{0x01f90101,0x01010101},{0x01f90109,0x01010101},{0x01f909f9,0x01010101},
+{0x01f90901,0x01010101},{0x0101f9f9,0x01010101},{0x0101f901,0x01010101},{0x0101f909,0x01010101},{0x010101f9,0x01010101},{0x01010101,0x01010101},{0x01010109,0x01010101},{0x010109f9,0x01010101},
+{0x01010901,0x01010101},{0x01010909,0x01010101},{0x0109f9f9,0x01010101},{0x0109f901,0x01010101},{0x010901f9,0x01010101},{0x01090101,0x01010101},{0x01090109,0x01010101},{0x010909f9,0x01010101},
+{0x01090901,0x01010101},{0x01090909,0x01010101},{0x09f9f901,0x01010101},{0x09f901f9,0x01010101},{0x09f90101,0x01010101},{0x09f90901,0x01010101},{0x09f90909,0x01010101},{0x0901f9f9,0x01010101},
+{0x0901f901,0x01010101},{0x090101f9,0x01010101},{0x09010101,0x01010101},{0x09010109,0x01010101},{0x090109f9,0x01010101},{0x09010901,0x01010101},{0x0909f901,0x01010101},{0x090901f9,0x01010101},
+{0x09090101,0x01010101},{0x09090109,0x01010101},{0x09090901,0x01010101},{0xf9f9f9f9,0x01010109},{0xf9f9f901,0x01010109},{0xf9f9f909,0x01010109},{0xf9f901f9,0x01010109},{0xf9f90109,0x01010109},
+{0xf9f909f9,0x01010109},{0xf9f90901,0x01010109},{0xf901f901,0x01010109},{0xf90101f9,0x01010109},{0xf9010101,0x01010109},{0xf90109f9,0x01010109},{0xf9010901,0x01010109},{0xf909f9f9,0x01010109},
+{0xf909f901,0x01010109},{0xf909f909,0x01010109},{0xf90901f9,0x01010109},{0xf9090101,0x01010109},{0xf9090109,0x01010109},{0xf90909f9,0x01010109},{0xf9090901,0x01010109},{0x01f9f901,0x01010109},
+{0x01f90101,0x01010109},{0x01f90109,0x01010109},{0x01f909f9,0x01010109},{0x01f90901,0x01010109},{0x01f90909,0x01010109},{0x0101f9f9,0x01010109},{0x0101f901,0x01010109},{0x0101f909,0x01010109},
+{0x010101f9,0x01010109},{0x01010101,0x01010109},{0x01010109,0x01010109},{0x010109f9,0x01010109},{0x01010901,0x01010109},{0x01010909,0x01010109},{0x0109f901,0x01010109},{0x010901f9,0x01010109},
+{0x01090101,0x01010109},{0x01090901,0x01010109},{0x09f9f909,0x01010109},{0x09f90101,0x01010109},{0x09f90109,0x01010109},{0x09f909f9,0x01010109},{0x09f90901,0x01010109},{0x09f90909,0x01010109},
+{0x0901f901,0x01010109},{0x09010101,0x01010109},{0x09010909,0x01010109},{0x0909f909,0x01010109},{0x09090101,0x01010109},{0x09090109,0x01010109},{0x090909f9,0x01010109},{0x09090901,0x01010109},
+{0xf9f901f9,0x010109f9},{0xf9f90101,0x010109f9},{0xf9f90109,0x010109f9},{0xf9f90901,0x010109f9},{0xf901f9f9,0x010109f9},{0xf9010101,0x010109f9},{0xf90109f9,0x010109f9},{0xf909f901,0x010109f9},
+{0xf9090909,0x010109f9},{0x01f90101,0x010109f9},{0x01f909f9,0x010109f9},{0x01f90909,0x010109f9},{0x0101f901,0x010109f9},{0x010101f9,0x010109f9},{0x01010101,0x010109f9},{0x01010109,0x010109f9},
+{0x010109f9,0x010109f9},{0x01010901,0x010109f9},{0x0109f9f9,0x010109f9},{0x0109f909,0x010109f9},{0x010901f9,0x010109f9},{0x01090101,0x010109f9},{0x09f9f909,0x010109f9},{0x09f90901,0x010109f9},
+{0x0901f9f9,0x010109f9},{0x0901f909,0x010109f9},{0x09010101,0x010109f9},{0x090109f9,0x010109f9},{0x0909f901,0x010109f9},{0x09090901,0x010109f9},{0xf9f9f901,0x01010901},{0xf9f9f909,0x01010901},
+{0xf9f90101,0x01010901},{0xf9f90909,0x01010901},{0xf901f901,0x01010901},{0xf90101f9,0x01010901},{0xf9010101,0x01010901},{0xf9010109,0x01010901},{0xf9010901,0x01010901},{0xf9090101,0x01010901},
+{0x01f9f901,0x01010901},{0x01f901f9,0x01010901},{0x01f90101,0x01010901},{0x01f90109,0x01010901},{0x01f90901,0x01010901},{0x0101f9f9,0x01010901},{0x0101f901,0x01010901},{0x0101f909,0x01010901},
+{0x010101f9,0x01010901},{0x01010101,0x01010901},{0x01010109,0x01010901},{0x010109f9,0x01010901},{0x01010901,0x01010901},{0x01010909,0x01010901},{0x0109f901,0x01010901},{0x010901f9,0x01010901},
+{0x01090101,0x01010901},{0x01090109,0x01010901},{0x01090901,0x01010901},{0x09f9f901,0x01010901},{0x09f90101,0x01010901},{0x09f90901,0x01010901},{0x0901f901,0x01010901},{0x090101f9,0x01010901},
+{0x09010101,0x01010901},{0x09010109,0x01010901},{0x090109f9,0x01010901},{0x09010901,0x01010901},{0x09090101,0x01010901},{0xf9f901f9,0x01010909},{0xf9f909f9,0x01010909},{0xf9010101,0x01010909},
+{0xf9010909,0x01010909},{0xf909f9f9,0x01010909},{0xf9090101,0x01010909},{0xf9090109,0x01010909},{0xf9090901,0x01010909},{0x01f90101,0x01010909},{0x01f909f9,0x01010909},{0x01f90901,0x01010909},
+{0x0101f901,0x01010909},{0x01010101,0x01010909},{0x01010109,0x01010909},{0x010109f9,0x01010909},{0x01010901,0x01010909},{0x0109f909,0x01010909},{0x01090101,0x01010909},{0x010909f9,0x01010909},
+{0x01090909,0x01010909},{0x09f9f901,0x01010909},{0x09f90909,0x01010909},{0x0901f909,0x01010909},{0x09010101,0x01010909},{0x09010109,0x01010909},{0x090109f9,0x01010909},{0x09010909,0x01010909},
+{0x0909f901,0x01010909},{0xf9f90101,0x0109f9f9},{0xf90101f9,0x0109f9f9},{0xf9010109,0x0109f9f9},{0xf9010901,0x0109f9f9},{0xf9090101,0x0109f9f9},{0x01f901f9,0x0109f9f9},{0x0101f9f9,0x0109f9f9},
+{0x01010101,0x0109f9f9},{0x01010109,0x0109f9f9},{0x010109f9,0x0109f9f9},{0x01010909,0x0109f9f9},{0x0109f901,0x0109f9f9},{0x010901f9,0x0109f9f9},{0x01090109,0x0109f9f9},{0x01090901,0x0109f9f9},
+{0x09f9f901,0x0109f9f9},{0x09010109,0x0109f9f9},{0x09090101,0x0109f9f9},{0xf9f9f901,0x0109f901},{0xf9f901f9,0x0109f901},{0xf9f90109,0x0109f901},{0xf9f90901,0x0109f901},{0xf901f909,0x0109f901},
+{0xf9010101,0x0109f901},{0xf909f901,0x0109f901},{0xf909f909,0x0109f901},{0xf9090109,0x0109f901},{0xf9090901,0x0109f901},{0x01f90101,0x0109f901},{0x01f90901,0x0109f901},{0x0101f901,0x0109f901},
+{0x01010101,0x0109f901},{0x01010109,0x0109f901},{0x01010901,0x0109f901},{0x01090101,0x0109f901},{0x01090109,0x0109f901},{0x01090909,0x0109f901},{0x09f901f9,0x0109f901},{0x09f90909,0x0109f901},
+{0x0901f909,0x0109f901},{0x09010101,0x0109f901},{0x0909f901,0x0109f901},{0x09090109,0x0109f901},{0x09090901,0x0109f901},{0xf901f901,0x0109f909},{0xf9010109,0x0109f909},{0xf9010901,0x0109f909},
+{0x01f9f9f9,0x0109f909},{0x01f9f901,0x0109f909},{0x01f90109,0x0109f909},{0x01010101,0x0109f909},{0x01010109,0x0109f909},{0x010109f9,0x0109f909},{0x0109f9f9,0x0109f909},{0x09f90101,0x0109f909},
+{0x0901f901,0x0109f909},{0x09010109,0x0109f909},{0x09090101,0x0109f909},{0xf901f901,0x010901f9},{0xf901f909,0x010901f9},{0xf9010101,0x010901f9},{0xf9010109,0x010901f9},{0xf9010909,0x010901f9},
+{0xf909f901,0x010901f9},{0xf9090109,0x010901f9},{0xf9090901,0x010901f9},{0x01f9f9f9,0x010901f9},{0x01f9f909,0x010901f9},{0x01f90101,0x010901f9},{0x01f909f9,0x010901f9},{0x01f90909,0x010901f9},
+{0x0101f901,0x010901f9},{0x010101f9,0x010901f9},{0x01010101,0x010901f9},{0x01010109,0x010901f9},{0x01010901,0x010901f9},{0x01010909,0x010901f9},{0x0109f9f9,0x010901f9},{0x0109f909,0x010901f9},
+{0x01090101,0x010901f9},{0x09f901f9,0x010901f9},{0x09f90101,0x010901f9},{0x09f90901,0x010901f9},{0x0901f9f9,0x010901f9},{0x0901f909,0x010901f9},{0x090101f9,0x010901f9},{0x09010101,0x010901f9},
+{0x09010109,0x010901f9},{0x090109f9,0x010901f9},{0x09010909,0x010901f9},{0x0909f901,0x010901f9},{0x090901f9,0x010901f9},{0x09090901,0x010901f9},{0xf9f90101,0x01090101},{0xf9f909f9,0x01090101},
+{0xf9f90909,0x01090101},{0xf901f901,0x01090101},{0xf9010101,0x01090101},{0xf9010109,0x01090101},{0xf9010901,0x01090101},{0x01f901f9,0x01090101},{0x01f90101,0x01090101},{0x01f90109,0x01090101},
+{0x01f90901,0x01090101},{0x0101f9f9,0x01090101},{0x0101f901,0x01090101},{0x010101f9,0x01090101},{0x01010101,0x01090101},{0x01010109,0x01090101},{0x01010901,0x01090101},{0x0109f901,0x01090101},
+{0x010901f9,0x01090101},{0x01090101,0x01090101},{0x01090109,0x01090101},{0x01090901,0x01090101},{0x09f90109,0x01090101},{0x09f90901,0x01090101},{0x09f90909,0x01090101},{0x0901f901,0x01090101},
+{0x09010101,0x01090101},{0x09010109,0x01090101},{0x09010901,0x01090101},{0x09010909,0x01090101},{0x0909f909,0x01090101},{0x09090101,0x01090101},{0x09090109,0x01090101},{0x090909f9,0x01090101},
+{0xf9f9f909,0x01090109},{0xf9f90901,0x01090109},{0xf9010101,0x01090109},{0xf909f9f9,0x01090109},{0xf9090109,0x01090109},{0xf90909f9,0x01090109},{0xf9090901,0x01090109},{0x01f9f9f9,0x01090109},
+{0x01f90101,0x01090109},{0x01f909f9,0x01090109},{0x01f90909,0x01090109},{0x0101f901,0x01090109},{0x010101f9,0x01090109},{0x01010101,0x01090109},{0x01010109,0x01090109},{0x010109f9,0x01090109},
+{0x01010909,0x01090109},{0x0109f9f9,0x01090109},{0x01090101,0x01090109},{0x010909f9,0x01090109},{0x09f9f9f9,0x01090109},{0x09f9f909,0x01090109},{0x09f90101,0x01090109},{0x09f90909,0x01090109},
+{0x090101f9,0x01090109},{0x09010109,0x01090109},{0x090109f9,0x01090109},{0x09010901,0x01090109},{0x0909f9f9,0x01090109},{0x090901f9,0x01090109},{0x09090109,0x01090109},{0x09090909,0x01090109},
+{0xf9010109,0x010909f9},{0xf9010901,0x010909f9},{0xf9090101,0x010909f9},{0x01f9f901,0x010909f9},{0x0101f909,0x010909f9},{0x01010101,0x010909f9},{0x01010909,0x010909f9},{0x0109f901,0x010909f9},
+{0x01090901,0x010909f9},{0x09f90101,0x010909f9},{0x0901f901,0x010909f9},{0x090109f9,0x010909f9},{0x09090109,0x010909f9},{0xf9f9f901,0x01090901},{0xf9f901f9,0x01090901},{0xf901f9f9,0x01090901},
+{0xf9010101,0x01090901},{0xf909f901,0x01090901},{0xf90901f9,0x01090901},{0xf9090109,0x01090901},{0xf9090901,0x01090901},{0x01f9f9f9,0x01090901},{0x01f9f901,0x01090901},{0x01f90101,0x01090901},
+{0x01f90109,0x01090901},{0x01f909f9,0x01090901},{0x0101f901,0x01090901},{0x010101f9,0x01090901},{0x01010101,0x01090901},{0x01010109,0x01090901},{0x01010901,0x01090901},{0x0109f9f9,0x01090901},
+{0x01090101,0x01090901},{0x01090909,0x01090901},{0x09f9f909,0x01090901},{0x09f901f9,0x01090901},{0x09f90909,0x01090901},{0x09010101,0x01090901},{0x0909f901,0x01090901},{0x090901f9,0x01090901},
+{0x09090101,0x01090901},{0x09090901,0x01090901},{0xf901f901,0x01090909},{0xf9010109,0x01090909},{0xf90109f9,0x01090909},{0x01f9f901,0x01090909},{0x01f901f9,0x01090909},{0x01f90901,0x01090909},
+{0x0101f9f9,0x01090909},{0x01010101,0x01090909},{0x010109f9,0x01090909},{0x01010909,0x01090909},{0x010901f9,0x01090909},{0x01090101,0x01090909},{0x01090901,0x01090909},{0x09f90109,0x01090909},
+{0x090101f9,0x01090909},{0x090109f9,0x01090909},{0x09010909,0x01090909},{0x09090109,0x01090909},{0xf9f9f9f9,0x09f9f9f9},{0xf9f9f909,0x09f9f9f9},{0xf9f909f9,0x09f9f9f9},{0xf9f90909,0x09f9f9f9},
+{0xf909f9f9,0x09f9f9f9},{0xf909f909,0x09f9f9f9},{0xf90909f9,0x09f9f9f9},{0xf9090909,0x09f9f9f9},{0x01f90101,0x09f9f9f9},{0x0101f9f9,0x09f9f9f9},{0x0101f901,0x09f9f9f9},{0x010101f9,0x09f9f9f9},
+{0x01010109,0x09f9f9f9},{0x01010901,0x09f9f9f9},{0x01090101,0x09f9f9f9},{0x09f9f9f9,0x09f9f9f9},{0x09f9f909,0x09f9f9f9},{0x09f909f9,0x09f9f9f9},{0x09f90909,0x09f9f9f9},{0x09010101,0x09f9f9f9},
+{0x0909f9f9,0x09f9f9f9},{0x0909f909,0x09f9f9f9},{0x090909f9,0x09f9f9f9},{0x09090909,0x09f9f9f9},{0xf9f90101,0x09f9f901},{0xf901f901,0x09f9f901},{0xf90101f9,0x09f9f901},{0xf9010109,0x09f9f901},
+{0xf9010901,0x09f9f901},{0xf9090101,0x09f9f901},{0x01f9f901,0x09f9f901},{0x01f901f9,0x09f9f901},{0x01f90901,0x09f9f901},{0x0101f9f9,0x09f9f901},{0x0101f909,0x09f9f901},{0x01010101,0x09f9f901},
+{0x01010109,0x09f9f901},{0x010109f9,0x09f9f901},{0x01010901,0x09f9f901},{0x010901f9,0x09f9f901},{0x01090109,0x09f9f901},{0x01090901,0x09f9f901},{0x09f90101,0x09f9f901},{0x09f90901,0x09f9f901},
+{0x090101f9,0x09f9f901},{0x09010109,0x09f9f901},{0x09010901,0x09f9f901},{0x09090101,0x09f9f901},{0xf9f9f9f9,0x09f9f909},{0xf9f9f909,0x09f9f909},{0xf9f909f9,0x09f9f909},{0xf9f90909,0x09f9f909},
+{0xf9010101,0x09f9f909},{0xf909f9f9,0x09f9f909},{0xf909f909,0x09f9f909},{0xf90909f9,0x09f9f909},{0xf9090909,0x09f9f909},{0x0101f901,0x09f9f909},{0x010101f9,0x09f9f909},{0x01010901,0x09f9f909},
+{0x01090101,0x09f9f909},{0x09f9f9f9,0x09f9f909},{0x09f9f909,0x09f9f909},{0x09f909f9,0x09f9f909},{0x09f90909,0x09f9f909},{0x09010101,0x09f9f909},{0x0909f9f9,0x09f9f909},{0x0909f909,0x09f9f909},
+{0x090909f9,0x09f9f909},{0x09090909,0x09f9f909},{0xf90101f9,0x09f901f9},{0xf9010901,0x09f901f9},{0x01f9f901,0x09f901f9},{0x01f901f9,0x09f901f9},{0x0101f901,0x09f901f9},{0x01010101,0x09f901f9},
+{0x01010909,0x09f901f9},{0x0109f901,0x09f901f9},{0x010901f9,0x09f901f9},{0x01090901,0x09f901f9},{0x090101f9,0x09f901f9},{0x09010901,0x09f901f9},{0xf9f9f901,0x09f90101},{0xf9f90901,0x09f90101},
+{0xf901f909,0x09f90101},{0xf9010101,0x09f90101},{0xf9010909,0x09f90101},{0xf9090109,0x09f90101},{0xf9090901,0x09f90101},{0x01f9f9f9,0x09f90101},{0x01f9f901,0x09f90101},{0x01f90101,0x09f90101},
+{0x01f909f9,0x09f90101},{0x0101f901,0x09f90101},{0x010101f9,0x09f90101},{0x01010101,0x09f90101},{0x01010109,0x09f90101},{0x01010901,0x09f90101},{0x01010909,0x09f90101},{0x01090101,0x09f90101},
+{0x01090109,0x09f90101},{0x010909f9,0x09f90101},{0x01090909,0x09f90101},{0x09f9f901,0x09f90101},{0x09f901f9,0x09f90101},{0x09f90109,0x09f90101},{0x09f90901,0x09f90101},{0x0901f9f9,0x09f90101},
+{0x0901f909,0x09f90101},{0x09010101,0x09f90101},{0x090109f9,0x09f90101},{0x09090109,0x09f90101},{0xf901f901,0x09f90109},{0xf9010109,0x09f90109},{0xf9010901,0x09f90109},{0xf9090101,0x09f90109},
+{0x01f9f901,0x09f90109},{0x01f901f9,0x09f90109},{0x01f90901,0x09f90109},{0x01f90909,0x09f90109},{0x0101f9f9,0x09f90109},{0x01010101,0x09f90109},{0x01010901,0x09f90109},{0x01010909,0x09f90109},
+{0x0109f901,0x09f90109},{0x01090109,0x09f90109},{0x01090909,0x09f90109},{0x09f90101,0x09f90109},{0x0901f901,0x09f90109},{0x09010909,0x09f90109},{0x090901f9,0x09f90109},{0xf9f9f9f9,0x09f909f9},
+{0xf9f9f909,0x09f909f9},{0xf9f909f9,0x09f909f9},{0xf9f90909,0x09f909f9},{0xf9010101,0x09f909f9},{0xf909f9f9,0x09f909f9},{0xf909f909,0x09f909f9},{0xf90909f9,0x09f909f9},{0xf9090909,0x09f909f9},
+{0x01f9f901,0x09f909f9},{0x01f90101,0x09f909f9},{0x0101f901,0x09f909f9},{0x010101f9,0x09f909f9},{0x01010901,0x09f909f9},{0x01090101,0x09f909f9},{0x01090901,0x09f909f9},{0x09f9f9f9,0x09f909f9},
+{0x09f9f909,0x09f909f9},{0x09f909f9,0x09f909f9},{0x09f90909,0x09f909f9},{0x09010101,0x09f909f9},{0x0909f9f9,0x09f909f9},{0x0909f909,0x09f909f9},{0x090909f9,0x09f909f9},{0x09090909,0x09f909f9},
+{0xf9f90101,0x09f90901},{0xf9f90109,0x09f90901},{0xf901f901,0x09f90901},{0xf90101f9,0x09f90901},{0xf9010109,0x09f90901},{0xf9090101,0x09f90901},{0x01f9f901,0x09f90901},{0x01f901f9,0x09f90901},
+{0x01f90109,0x09f90901},{0x01f90901,0x09f90901},{0x0101f9f9,0x09f90901},{0x0101f909,0x09f90901},{0x01010101,0x09f90901},{0x01010909,0x09f90901},{0x0109f901,0x09f90901},{0x010901f9,0x09f90901},
+{0x09f90101,0x09f90901},{0x09010109,0x09f90901},{0x09010901,0x09f90901},{0x09090101,0x09f90901},{0xf9f9f9f9,0x09f90909},{0xf9f9f909,0x09f90909},{0xf9f909f9,0x09f90909},{0xf9f90909,0x09f90909},
+{0xf9010101,0x09f90909},{0xf909f9f9,0x09f90909},{0xf909f909,0x09f90909},{0xf90909f9,0x09f90909},{0xf9090909,0x09f90909},{0x01f90101,0x09f90909},{0x0101f901,0x09f90909},{0x010101f9,0x09f90909},
+{0x01010109,0x09f90909},{0x09f9f9f9,0x09f90909},{0x09f9f909,0x09f90909},{0x09f909f9,0x09f90909},{0x09f90909,0x09f90909},{0x09010101,0x09f90909},{0x0909f9f9,0x09f90909},{0x0909f909,0x09f90909},
+{0x090909f9,0x09f90909},{0x09090909,0x09f90909},{0xf9f90101,0x0901f9f9},{0xf901f901,0x0901f9f9},{0xf9010109,0x0901f9f9},{0xf90109f9,0x0901f9f9},{0xf9010901,0x0901f9f9},{0xf9090101,0x0901f9f9},
+{0x01f9f901,0x0901f9f9},{0x01f90109,0x0901f9f9},{0x01f90901,0x0901f9f9},{0x01010101,0x0901f9f9},{0x010109f9,0x0901f9f9},{0x01010909,0x0901f9f9},{0x01090901,0x0901f9f9},{0x01090909,0x0901f9f9},
+{0x09f90101,0x0901f9f9},{0x0901f901,0x0901f9f9},{0x090101f9,0x0901f9f9},{0x09010109,0x0901f9f9},{0x09010901,0x0901f9f9},{0x09090101,0x0901f9f9},{0xf9f9f901,0x0901f901},{0xf9f901f9,0x0901f901},
+{0xf9f90109,0x0901f901},{0xf9f90901,0x0901f901},{0xf901f9f9,0x0901f901},{0xf9010101,0x0901f901},{0xf90109f9,0x0901f901},{0xf9010909,0x0901f901},{0xf909f901,0x0901f901},{0xf90901f9,0x0901f901},
+{0xf9090109,0x0901f901},{0xf9090901,0x0901f901},{0x01f9f9f9,0x0901f901},{0x01f90101,0x0901f901},{0x0101f9f9,0x0901f901},{0x0101f901,0x0901f901},{0x010101f9,0x0901f901},{0x01010101,0x0901f901},
+{0x01010109,0x0901f901},{0x01010901,0x0901f901},{0x0109f909,0x0901f901},{0x01090101,0x0901f901},{0x09f901f9,0x0901f901},{0x09f90109,0x0901f901},{0x0901f909,0x0901f901},{0x09010101,0x0901f901},
+{0x090109f9,0x0901f901},{0x0909f901,0x0901f901},{0x090901f9,0x0901f901},{0x09090109,0x0901f901},{0x09090901,0x0901f901},{0xf9f90101,0x0901f909},{0xf901f901,0x0901f909},{0xf90101f9,0x0901f909},
+{0xf9010901,0x0901f909},{0xf9090101,0x0901f909},{0x01f901f9,0x0901f909},{0x01f90109,0x0901f909},{0x01f90901,0x0901f909},{0x0101f9f9,0x0901f909},{0x0101f909,0x0901f909},{0x01010101,0x0901f909},
+{0x010109f9,0x0901f909},{0x01090109,0x0901f909},{0x01090901,0x0901f909},{0x09f90101,0x0901f909},{0x090101f9,0x0901f909},{0x09010109,0x0901f909},{0x09090901,0x0901f909},{0xf9f9f901,0x090101f9},
+{0xf9f901f9,0x090101f9},{0xf9f90109,0x090101f9},{0xf901f9f9,0x090101f9},{0xf9010101,0x090101f9},{0xf90109f9,0x090101f9},{0xf9090109,0x090101f9},{0x01f9f9f9,0x090101f9},{0x01f90909,0x090101f9},
+{0x0101f901,0x090101f9},{0x010101f9,0x090101f9},{0x01010101,0x090101f9},{0x01010109,0x090101f9},{0x010109f9,0x090101f9},{0x01010901,0x090101f9},{0x0109f9f9,0x090101f9},{0x0109f901,0x090101f9},
+{0x0109f909,0x090101f9},{0x01090101,0x090101f9},{0x09f901f9,0x090101f9},{0x09f90109,0x090101f9},{0x0901f909,0x090101f9},{0x090101f9,0x090101f9},{0x09010101,0x090101f9},{0x090109f9,0x090101f9},
+{0x0909f901,0x090101f9},{0x09090901,0x090101f9},{0xf9f9f9f9,0x09010101},{0xf9f90101,0x09010101},{0xf9f909f9,0x09010101},{0xf9f90909,0x09010101},{0xf901f9f9,0x09010101},{0xf901f901,0x09010101},
+{0xf90101f9,0x09010101},{0xf9010101,0x09010101},{0xf9010109,0x09010101},{0xf9010901,0x09010101},{0xf909f901,0x09010101},{0xf9090101,0x09010101},{0xf9090901,0x09010101},{0xf9090909,0x09010101},
+{0x01f9f901,0x09010101},{0x01f901f9,0x09010101},{0x01f90101,0x09010101},{0x01f90109,0x09010101},{0x01f90901,0x09010101},{0x0101f9f9,0x09010101},{0x0101f901,0x09010101},{0x0101f909,0x09010101},
+{0x010101f9,0x09010101},{0x01010101,0x09010101},{0x01010109,0x09010101},{0x010109f9,0x09010101},{0x01010901,0x09010101},{0x01010909,0x09010101},{0x0109f901,0x09010101},{0x010901f9,0x09010101},
+{0x01090101,0x09010101},{0x01090109,0x09010101},{0x01090901,0x09010101},{0x09f9f901,0x09010101},{0x09f90101,0x09010101},{0x09f909f9,0x09010101},{0x0901f901,0x09010101},{0x0901f909,0x09010101},
+{0x090101f9,0x09010101},{0x09010101,0x09010101},{0x09010109,0x09010101},{0x09010901,0x09010101},{0x09010909,0x09010101},{0x0909f9f9,0x09010101},{0x0909f909,0x09010101},{0x09090101,0x09010101},
+{0x090909f9,0x09010101},{0x09090909,0x09010101},{0xf9f9f901,0x09010109},{0xf9f901f9,0x09010109},{0xf901f9f9,0x09010109},{0xf9010101,0x09010109},{0xf9010901,0x09010109},{0xf909f9f9,0x09010109},
+{0xf9090109,0x09010109},{0xf9090901,0x09010109},{0x01f90101,0x09010109},{0x01f909f9,0x09010109},{0x01f90901,0x09010109},{0x0101f901,0x09010109},{0x0101f909,0x09010109},{0x01010101,0x09010109},
+{0x01010109,0x09010109},{0x01010901,0x09010109},{0x01090101,0x09010109},{0x010909f9,0x09010109},{0x09f9f909,0x09010109},{0x09f901f9,0x09010109},{0x09f90901,0x09010109},{0x09f90909,0x09010109},
+{0x0901f909,0x09010109},{0x090101f9,0x09010109},{0x09010101,0x09010109},{0x090901f9,0x09010109},{0x09090109,0x09010109},{0x09090901,0x09010109},{0xf9f90101,0x090109f9},{0xf9010109,0x090109f9},
+{0xf9010901,0x090109f9},{0xf9090101,0x090109f9},{0x01f9f901,0x090109f9},{0x01f90109,0x090109f9},{0x0101f9f9,0x090109f9},{0x0101f909,0x090109f9},{0x01010101,0x090109f9},{0x01010109,0x090109f9},
+{0x01010909,0x090109f9},{0x010901f9,0x090109f9},{0x01090101,0x090109f9},{0x09f90101,0x090109f9},{0x0901f901,0x090109f9},{0x09010109,0x090109f9},{0x09010901,0x090109f9},{0x09090101,0x090109f9},
+{0xf9f901f9,0x09010901},{0xf9f90109,0x09010901},{0xf9f90901,0x09010901},{0xf901f9f9,0x09010901},{0xf901f909,0x09010901},{0xf9010101,0x09010901},{0xf90109f9,0x09010901},{0xf9010909,0x09010901},
+{0xf909f9f9,0x09010901},{0xf909f901,0x09010901},{0xf90901f9,0x09010901},{0xf9090109,0x09010901},{0x01f9f9f9,0x09010901},{0x01f9f909,0x09010901},{0x01f90101,0x09010901},{0x01f909f9,0x09010901},
+{0x01f90909,0x09010901},{0x0101f901,0x09010901},{0x010101f9,0x09010901},{0x01010101,0x09010901},{0x01010109,0x09010901},{0x01010901,0x09010901},{0x0109f909,0x09010901},{0x01090101,0x09010901},
+{0x01090109,0x09010901},{0x01090909,0x09010901},{0x09f9f901,0x09010901},{0x09f901f9,0x09010901},{0x0901f9f9,0x09010901},{0x0901f909,0x09010901},{0x09010101,0x09010901},{0x09010909,0x09010901},
+{0x0909f901,0x09010901},{0x09090109,0x09010901},{0xf9f90101,0x09010909},{0xf9010101,0x09010909},{0xf9090101,0x09010909},{0x01f901f9,0x09010909},{0x01f90109,0x09010909},{0x01f90901,0x09010909},
+{0x0101f9f9,0x09010909},{0x01010101,0x09010909},{0x010109f9,0x09010909},{0x0109f901,0x09010909},{0x09f90101,0x09010909},{0x0901f901,0x09010909},{0x090101f9,0x09010909},{0x09010101,0x09010909},
+{0x09010109,0x09010909},{0xf9f9f9f9,0x0909f9f9},{0xf9f9f909,0x0909f9f9},{0xf9f909f9,0x0909f9f9},{0xf9f90909,0x0909f9f9},{0xf9010101,0x0909f9f9},{0xf909f9f9,0x0909f9f9},{0xf909f909,0x0909f9f9},
+{0xf90909f9,0x0909f9f9},{0xf9090909,0x0909f9f9},{0x01f90101,0x0909f9f9},{0x0101f901,0x0909f9f9},{0x010101f9,0x0909f9f9},{0x01010109,0x0909f9f9},{0x01010901,0x0909f9f9},{0x09f9f9f9,0x0909f9f9},
+{0x09f9f909,0x0909f9f9},{0x09f909f9,0x0909f9f9},{0x09f90909,0x0909f9f9},{0x09010101,0x0909f9f9},{0x0909f9f9,0x0909f9f9},{0x0909f909,0x0909f9f9},{0x090909f9,0x0909f9f9},{0x09090909,0x0909f9f9},
+{0xf9f90101,0x0909f901},{0xf9f90901,0x0909f901},{0xf901f901,0x0909f901},{0xf90101f9,0x0909f901},{0xf9010109,0x0909f901},{0xf9010901,0x0909f901},{0xf9010909,0x0909f901},{0x01f90109,0x0909f901},
+{0x01f90901,0x0909f901},{0x0101f901,0x0909f901},{0x01010101,0x0909f901},{0x010109f9,0x0909f901},{0x01010909,0x0909f901},{0x0109f901,0x0909f901},{0x010901f9,0x0909f901},{0x09f90101,0x0909f901},
+{0x0901f9f9,0x0909f901},{0x0901f909,0x0909f901},{0x09010109,0x0909f901},{0x09010901,0x0909f901},{0xf9f9f909,0x0909f909},{0xf9f909f9,0x0909f909},{0xf9f90909,0x0909f909},{0xf901f9f9,0x0909f909},
+{0xf9010901,0x0909f909},{0xf909f909,0x0909f909},{0xf90909f9,0x0909f909},{0xf9090909,0x0909f909},{0x01f90101,0x0909f909},{0x0101f901,0x0909f909},{0x01010109,0x0909f909},{0x01010901,0x0909f909},
+{0x01090101,0x0909f909},{0x09f9f9f9,0x0909f909},{0x09f9f909,0x0909f909},{0x09f909f9,0x0909f909},{0x09f90909,0x0909f909},{0x09010101,0x0909f909},{0x0909f9f9,0x0909f909},{0x0909f909,0x0909f909},
+{0x090909f9,0x0909f909},{0x09090909,0x0909f909},{0xf9010901,0x090901f9},{0xf9090101,0x090901f9},{0x01f9f901,0x090901f9},{0x01f901f9,0x090901f9},{0x0101f9f9,0x090901f9},{0x010101f9,0x090901f9},
+{0x01010101,0x090901f9},{0x010109f9,0x090901f9},{0x01010909,0x090901f9},{0x0109f901,0x090901f9},{0x01090101,0x090901f9},{0x01090109,0x090901f9},{0x010909f9,0x090901f9},{0x01090901,0x090901f9},
+{0x09f90101,0x090901f9},{0xf9f90109,0x09090101},{0xf9f90901,0x09090101},{0xf901f9f9,0x09090101},{0xf901f909,0x09090101},{0xf9010101,0x09090101},{0xf90109f9,0x09090101},{0xf9090109,0x09090101},
+{0xf9090901,0x09090101},{0x01f9f909,0x09090101},{0x01f90101,0x09090101},{0x0101f901,0x09090101},{0x010101f9,0x09090101},{0x01010101,0x09090101},{0x01010109,0x09090101},{0x01010901,0x09090101},
+{0x01090101,0x09090101},{0x01090909,0x09090101},{0x09f9f901,0x09090101},{0x09f901f9,0x09090101},{0x09f90101,0x09090101},{0x09f90109,0x09090101},{0x09f90901,0x09090101},{0x0901f909,0x09090101},
+{0x09010101,0x09090101},{0x090109f9,0x09090101},{0xf9f90101,0x09090109},{0xf901f901,0x09090109},{0xf9010109,0x09090109},{0xf9010909,0x09090109},{0xf909f901,0x09090109},{0xf9090101,0x09090109},
+{0x01f901f9,0x09090109},{0x01f90109,0x09090109},{0x01f90909,0x09090109},{0x0101f909,0x09090109},{0x01010101,0x09090109},{0x01010109,0x09090109},{0x010109f9,0x09090109},{0x0109f9f9,0x09090109},
+{0x0109f909,0x09090109},{0x09f90109,0x09090109},{0x0901f9f9,0x09090109},{0x09010101,0x09090109},{0x09010109,0x09090109},{0x09010901,0x09090109},{0x0909f901,0x09090109},{0x090901f9,0x09090109},
+{0x09090109,0x09090109},{0xf9f9f9f9,0x090909f9},{0xf9f9f909,0x090909f9},{0xf9f909f9,0x090909f9},{0xf9f90909,0x090909f9},{0xf909f9f9,0x090909f9},{0xf909f909,0x090909f9},{0xf90909f9,0x090909f9},
+{0xf9090909,0x090909f9},{0x0101f901,0x090909f9},{0x010101f9,0x090909f9},{0x01010109,0x090909f9},{0x01010901,0x090909f9},{0x09f9f9f9,0x090909f9},{0x09f9f909,0x090909f9},{0x09f909f9,0x090909f9},
+{0x09f90909,0x090909f9},{0x09010101,0x090909f9},{0x0909f9f9,0x090909f9},{0x0909f909,0x090909f9},{0x090909f9,0x090909f9},{0x09090909,0x090909f9},{0xf9f90101,0x09090901},{0xf90101f9,0x09090901},
+{0xf9010901,0x09090901},{0xf909f901,0x09090901},{0xf9090101,0x09090901},{0x01f9f901,0x09090901},{0x0101f9f9,0x09090901},{0x01010101,0x09090901},{0x01010909,0x09090901},{0x0109f901,0x09090901},
+{0x01090109,0x09090901},{0x01090901,0x09090901},{0x0901f9f9,0x09090901},{0x09010109,0x09090901},{0xf9f9f9f9,0x09090909},{0xf9f9f909,0x09090909},{0xf9f909f9,0x09090909},{0xf9f90909,0x09090909},
+{0xf909f9f9,0x09090909},{0xf909f909,0x09090909},{0xf90909f9,0x09090909},{0xf9090909,0x09090909},{0x0101f901,0x09090909},{0x010101f9,0x09090909},{0x01010109,0x09090909},{0x09f9f9f9,0x09090909},
+{0x09f9f909,0x09090909},{0x09f909f9,0x09090909},{0x09f90909,0x09090909},{0x09010101,0x09090909},{0x0909f9f9,0x09090909},{0x0909f909,0x09090909},{0x090909f9,0x09090909},{0x09090909,0x09090909},
+{0xf7f7f7f7,0xf7f7f7f7},{0xf7f7f707,0xf7f7f7f7},{0xf7f7ffff,0xf7f7f7f7},{0xf7f707f7,0xf7f7f7f7},{0xf7f70707,0xf7f7f7f7},{0xf7fff7ff,0xf7f7f7f7},{0xf7ffffff,0xf7f7f7f7},{0xf707f7f7,0xf7f7f7f7},
+{0xf707f707,0xf7f7f7f7},{0xf70707f7,0xf7f7f7f7},{0xf7070707,0xf7f7f7f7},{0xfff7ffff,0xf7f7f7f7},{0xfffff7ff,0xf7f7f7f7},{0xfffffff7,0xf7f7f7f7},{0xffffff07,0xf7f7f7f7},{0xff07ffff,0xf7f7f7f7},
+{0x07f7f7f7,0xf7f7f7f7},{0x07f7f707,0xf7f7f7f7},{0x07f707f7,0xf7f7f7f7},{0x07f70707,0xf7f7f7f7},{0x07ffffff,0xf7f7f7f7},{0x0707f7f7,0xf7f7f7f7},{0x0707f707,0xf7f7f7f7},{0x070707f7,0xf7f7f7f7},
+{0x07070707,0xf7f7f7f7},{0xf7f7fff7,0xf7f7f7ff},{0xf7f7ffff,0xf7f7f7ff},{0xf7fff7ff,0xf7f7f7ff},{0xf7fffff7,0xf7f7f7ff},{0xf7ffff07,0xf7f7f7ff},{0xf7ff07ff,0xf7f7f7ff},{0xf7ff0707,0xf7f7f7ff},
+{0xf707ffff,0xf7f7f7ff},{0xfff7f7ff,0xf7f7f7ff},{0xfff7ff07,0xf7f7f7ff},{0xfff707ff,0xf7f7f7ff},{0xfffff707,0xf7f7f7ff},{0xffffffff,0xf7f7f7ff},{0xffff0707,0xf7f7f7ff},{0xff07f7ff,0xf7f7f7ff},
+{0xff07fff7,0xf7f7f7ff},{0xff07ff07,0xf7f7f7ff},{0xff0707f7,0xf7f7f7ff},{0x07f7ffff,0xf7f7f7ff},{0x07fff7ff,0xf7f7f7ff},{0x07fffff7,0xf7f7f7ff},{0x07ffff07,0xf7f7f7ff},{0x0707ffff,0xf7f7f7ff},
+{0xf7f7f7f7,0xf7f7f707},{0xf7f7f707,0xf7f7f707},{0xf7f707f7,0xf7f7f707},{0xf7f70707,0xf7f7f707},{0xf7ffffff,0xf7f7f707},{0xf707f7f7,0xf7f7f707},{0xf707f707,0xf7f7f707},{0xf70707f7,0xf7f7f707},
+{0xf7070707,0xf7f7f707},{0xfff7ffff,0xf7f7f707},{0xfffff7ff,0xf7f7f707},{0xffff07ff,0xf7f7f707},{0xff07fff7,0xf7f7f707},{0xff0707ff,0xf7f7f707},{0x07f7f7f7,0xf7f7f707},{0x07f7f707,0xf7f7f707},
+{0x07f707f7,0xf7f7f707},{0x07f70707,0xf7f7f707},{0x07fff7ff,0xf7f7f707},{0x07ffffff,0xf7f7f707},{0x07ff07ff,0xf7f7f707},{0x0707f7f7,0xf7f7f707},{0x0707f707,0xf7f7f707},{0x070707f7,0xf7f7f707},
+{0x07070707,0xf7f7f707},{0xf7fff7ff,0xf7f7fff7},{0xf7fffff7,0xf7f7fff7},{0xf7ffff07,0xf7f7fff7},{0xf707ffff,0xf7f7fff7},{0xfff7f7ff,0xf7f7fff7},{0xfff707ff,0xf7f7fff7},{0xffffffff,0xf7f7fff7},
+{0xffff0707,0xf7f7fff7},{0xff07fff7,0xf7f7fff7},{0xff07ffff,0xf7f7fff7},{0x07fff7ff,0xf7f7fff7},{0x07ff07ff,0xf7f7fff7},{0x0707ffff,0xf7f7fff7},{0xf7f7f7ff,0xf7f7ffff},{0xf7f7fff7,0xf7f7ffff},
+{0xf7f7ffff,0xf7f7ffff},{0xf7f7ff07,0xf7f7ffff},{0xf7ffffff,0xf7f7ffff},{0xf7ff07f7,0xf7f7ffff},{0xf7ff0707,0xf7f7ffff},{0xf70707ff,0xf7f7ffff},{0xfff7f7f7,0xf7f7ffff},{0xfff7ffff,0xf7f7ffff},
+{0xfff70707,0xf7f7ffff},{0xfffff7f7,0xf7f7ffff},{0xfffff7ff,0xf7f7ffff},{0xfffffff7,0xf7f7ffff},{0xffffffff,0xf7f7ffff},{0xffffff07,0xf7f7ffff},{0xffff07ff,0xf7f7ffff},{0xff07f7f7,0xf7f7ffff},
+{0xff07f707,0xf7f7ffff},{0xff07ffff,0xf7f7ffff},{0xff0707f7,0xf7f7ffff},{0xff070707,0xf7f7ffff},{0x07f7f7ff,0xf7f7ffff},{0x07fff7ff,0xf7f7ffff},{0x07ffffff,0xf7f7ffff},{0x07ff07f7,0xf7f7ffff},
+{0x07ff0707,0xf7f7ffff},{0x0707f7ff,0xf7f7ffff},{0x0707fff7,0xf7f7ffff},{0x0707ffff,0xf7f7ffff},{0x0707ff07,0xf7f7ffff},{0x070707ff,0xf7f7ffff},{0xf7fffff7,0xf7f7ff07},{0xf7ff07ff,0xf7f7ff07},
+{0xfff7f7ff,0xf7f7ff07},{0xfff7fff7,0xf7f7ff07},{0xfffff7f7,0xf7f7ff07},{0xfffff707,0xf7f7ff07},{0xffffffff,0xf7f7ff07},{0xffff07f7,0xf7f7ff07},{0xff07f7f7,0xf7f7ff07},{0xff07f7ff,0xf7f7ff07},
+{0xff07ff07,0xf7f7ff07},{0xff0707ff,0xf7f7ff07},{0x07f7ffff,0xf7f7ff07},{0x07fff7ff,0xf7f7ff07},{0x07fffff7,0xf7f7ff07},{0x07ff07ff,0xf7f7ff07},{0xf7f7f7f7,0xf7f707f7},{0xf7f7f707,0xf7f707f7},
+{0xf7f707f7,0xf7f707f7},{0xf7f70707,0xf7f707f7},{0xf7ffffff,0xf7f707f7},{0xf707f7f7,0xf7f707f7},{0xf707f707,0xf7f707f7},{0xf70707f7,0xf7f707f7},{0xf7070707,0xf7f707f7},{0xfff7ffff,0xf7f707f7},
+{0xfffff7ff,0xf7f707f7},{0xffffff07,0xf7f707f7},{0xff07ffff,0xf7f707f7},{0x07f7f7f7,0xf7f707f7},{0x07f7f707,0xf7f707f7},{0x07f707f7,0xf7f707f7},{0x07f70707,0xf7f707f7},{0x07ffffff,0xf7f707f7},
+{0x0707f7f7,0xf7f707f7},{0x0707f707,0xf7f707f7},{0x070707f7,0xf7f707f7},{0x07070707,0xf7f707f7},{0xf7f7ffff,0xf7f707ff},{0xf7fff7ff,0xf7f707ff},{0xf7fffff7,0xf7f707ff},{0xf7ff07ff,0xf7f707ff},
+{0xf707fff7,0xf7f707ff},{0xf707ffff,0xf7f707ff},{0xfff7f7ff,0xf7f707ff},{0xfffff7f7,0xf7f707ff},{0xfffff7ff,0xf7f707ff},{0xffffffff,0xf7f707ff},{0xff07f7ff,0xf7f707ff},{0xff07fff7,0xf7f707ff},
+{0xff0707ff,0xf7f707ff},{0x07fff7ff,0xf7f707ff},{0x07fffff7,0xf7f707ff},{0x07ffff07,0xf7f707ff},{0x07ff07ff,0xf7f707ff},{0x0707ffff,0xf7f707ff},{0xf7f7f7f7,0xf7f70707},{0xf7f7f707,0xf7f70707},
+{0xf7f707f7,0xf7f70707},{0xf7f70707,0xf7f70707},{0xf7ffffff,0xf7f70707},{0xf707f7f7,0xf7f70707},{0xf707f707,0xf7f70707},{0xf70707f7,0xf7f70707},{0xf7070707,0xf7f70707},{0xfff7ffff,0xf7f70707},
+{0xfffff7ff,0xf7f70707},{0xffff07ff,0xf7f70707},{0xff07f7ff,0xf7f70707},{0xff07ffff,0xf7f70707},{0x07f7f7f7,0xf7f70707},{0x07f7f707,0xf7f70707},{0x07f7ffff,0xf7f70707},{0x07f707f7,0xf7f70707},
+{0x07f70707,0xf7f70707},{0x07ffffff,0xf7f70707},{0x0707f7f7,0xf7f70707},{0x0707f707,0xf7f70707},{0x070707f7,0xf7f70707},{0x07070707,0xf7f70707},{0xf7fff7f7,0xf7fff7f7},{0xf7fff7ff,0xf7fff7f7},
+{0xf7fffff7,0xf7fff7f7},{0xf7ff07ff,0xf7fff7f7},{0xf707fff7,0xf7fff7f7},{0xf707ffff,0xf7fff7f7},{0xfff7f7ff,0xf7fff7f7},{0xfff7fff7,0xf7fff7f7},{0xfffff7f7,0xf7fff7f7},{0xffffffff,0xf7fff7f7},
+{0xffff07f7,0xf7fff7f7},{0xff07f7ff,0xf7fff7f7},{0xff07fff7,0xf7fff7f7},{0xff07ffff,0xf7fff7f7},{0xff0707ff,0xf7fff7f7},{0x07fff7ff,0xf7fff7f7},{0x07fffff7,0xf7fff7f7},{0x07ffff07,0xf7fff7f7},
+{0x0707f7ff,0xf7fff7f7},{0x0707ffff,0xf7fff7f7},{0xf7f7f7ff,0xf7fff7ff},{0xf7f7fff7,0xf7fff7ff},{0xf7f7ff07,0xf7fff7ff},{0xf7f707ff,0xf7fff7ff},{0xf7fff7f7,0xf7fff7ff},{0xf7fff707,0xf7fff7ff},
+{0xf7ffffff,0xf7fff7ff},{0xf7ff07f7,0xf7fff7ff},{0xf707f7ff,0xf7fff7ff},{0xf707fff7,0xf7fff7ff},{0xf70707ff,0xf7fff7ff},{0xfff7ffff,0xf7fff7ff},{0xfff70707,0xf7fff7ff},{0xfffff7f7,0xf7fff7ff},
+{0xfffff7ff,0xf7fff7ff},{0xfffff707,0xf7fff7ff},{0xfffffff7,0xf7fff7ff},{0xffffffff,0xf7fff7ff},{0xffffff07,0xf7fff7ff},{0xffff07ff,0xf7fff7ff},{0xff07f7f7,0xf7fff7ff},{0xff07ffff,0xf7fff7ff},
+{0x07f7fff7,0xf7fff7ff},{0x07fff707,0xf7fff7ff},{0x07ffffff,0xf7fff7ff},{0x0707f7ff,0xf7fff7ff},{0x0707fff7,0xf7fff7ff},{0xf7fff7ff,0xf7fff707},{0xf7fffff7,0xf7fff707},{0xf7ffff07,0xf7fff707},
+{0xf707ffff,0xf7fff707},{0xfff7f7f7,0xf7fff707},{0xfff7ff07,0xf7fff707},{0xfff707ff,0xf7fff707},{0xfffff707,0xf7fff707},{0xffffffff,0xf7fff707},{0xffff07f7,0xf7fff707},{0xffff0707,0xf7fff707},
+{0xff07fff7,0xf7fff707},{0xff07ff07,0xf7fff707},{0x07f7ffff,0xf7fff707},{0x07fff7ff,0xf7fff707},{0x07fffff7,0xf7fff707},{0x07ffff07,0xf7fff707},{0x0707ffff,0xf7fff707},{0xf7f7f7ff,0xf7fffff7},
+{0xf7f7ff07,0xf7fffff7},{0xf7f707ff,0xf7fffff7},{0xf7fffff7,0xf7fffff7},{0xf7ffffff,0xf7fffff7},{0xf7ff07f7,0xf7fffff7},{0xf7ff07ff,0xf7fffff7},{0xf707f7ff,0xf7fffff7},{0xf707ff07,0xf7fffff7},
+{0xfff7f7ff,0xf7fffff7},{0xfff7ffff,0xf7fffff7},{0xfff7ff07,0xf7fffff7},{0xfff707f7,0xf7fffff7},{0xfff70707,0xf7fffff7},{0xfffff7ff,0xf7fffff7},{0xfffffff7,0xf7fffff7},{0xffffffff,0xf7fffff7},
+{0xffffff07,0xf7fffff7},{0xffff07ff,0xf7fffff7},{0xff07f707,0xf7fffff7},{0xff07ffff,0xf7fffff7},{0xff0707f7,0xf7fffff7},{0x07f7fff7,0xf7fffff7},{0x07f707ff,0xf7fffff7},{0x07fff7f7,0xf7fffff7},
+{0x07fffff7,0xf7fffff7},{0x07ffffff,0xf7fffff7},{0x07ff07f7,0xf7fffff7},{0x07ff07ff,0xf7fffff7},{0x07ff0707,0xf7fffff7},{0x0707f7ff,0xf7fffff7},{0x0707fff7,0xf7fffff7},{0x0707ffff,0xf7fffff7},
+{0x070707ff,0xf7fffff7},{0xf7f7f707,0xf7ffffff},{0xf7f7ffff,0xf7ffffff},{0xf7f70707,0xf7ffffff},{0xf7fff7ff,0xf7ffffff},{0xf7fffff7,0xf7ffffff},{0xf7ffffff,0xf7ffffff},{0xf7ffff07,0xf7ffffff},
+{0xf7ff07ff,0xf7ffffff},{0xf707f7f7,0xf7ffffff},{0xf707f707,0xf7ffffff},{0xf707ffff,0xf7ffffff},{0xf70707f7,0xf7ffffff},{0xf7070707,0xf7ffffff},{0xfff7f7ff,0xf7ffffff},{0xfff7fff7,0xf7ffffff},
+{0xfff7ffff,0xf7ffffff},{0xfff7ff07,0xf7ffffff},{0xfffff7ff,0xf7ffffff},{0xfffff707,0xf7ffffff},{0xfffffff7,0xf7ffffff},{0xffffffff,0xf7ffffff},{0xffffff07,0xf7ffffff},{0xffff07ff,0xf7ffffff},
+{0xffff0707,0xf7ffffff},{0xff07f7ff,0xf7ffffff},{0xff07fff7,0xf7ffffff},{0xff07ffff,0xf7ffffff},{0xff07ff07,0xf7ffffff},{0xff0707ff,0xf7ffffff},{0x07f7f7f7,0xf7ffffff},{0x07f7f707,0xf7ffffff},
+{0x07f7fff7,0xf7ffffff},{0x07f7ffff,0xf7ffffff},{0x07f707f7,0xf7ffffff},{0x07f70707,0xf7ffffff},{0x07fff7f7,0xf7ffffff},{0x07fff7ff,0xf7ffffff},{0x07fffff7,0xf7ffffff},{0x07ffffff,0xf7ffffff},
+{0x07ffff07,0xf7ffffff},{0x07ff07ff,0xf7ffffff},{0x07ff0707,0xf7ffffff},{0x0707f7f7,0xf7ffffff},{0x0707f707,0xf7ffffff},{0x0707ffff,0xf7ffffff},{0xf7f7f7ff,0xf7ffff07},{0xf7f7fff7,0xf7ffff07},
+{0xf7f7ffff,0xf7ffff07},{0xf7f7ff07,0xf7ffff07},{0xf7ffffff,0xf7ffff07},{0xf7ffff07,0xf7ffff07},{0xf7ff07f7,0xf7ffff07},{0xf7ff0707,0xf7ffff07},{0xf707f7ff,0xf7ffff07},{0xf707ff07,0xf7ffff07},
+{0xfff7f7f7,0xf7ffff07},{0xfff7f707,0xf7ffff07},{0xfff7fff7,0xf7ffff07},{0xfff7ffff,0xf7ffff07},{0xfff707f7,0xf7ffff07},{0xfff70707,0xf7ffff07},{0xfffff7ff,0xf7ffff07},{0xffffffff,0xf7ffff07},
+{0xffffff07,0xf7ffff07},{0xffff07f7,0xf7ffff07},{0xffff07ff,0xf7ffff07},{0xff07f7ff,0xf7ffff07},{0xff07fff7,0xf7ffff07},{0xff07ffff,0xf7ffff07},{0xff0707f7,0xf7ffff07},{0xff0707ff,0xf7ffff07},
+{0xff070707,0xf7ffff07},{0x07f7ff07,0xf7ffff07},{0x07f70707,0xf7ffff07},{0x07fff707,0xf7ffff07},{0x07ffffff,0xf7ffff07},{0x0707fff7,0xf7ffff07},{0x070707ff,0xf7ffff07},{0xf7fff7ff,0xf7ff07f7},
+{0xf7ffff07,0xf7ff07f7},{0xf707ffff,0xf7ff07f7},{0xfff7f7ff,0xf7ff07f7},{0xfff7fff7,0xf7ff07f7},{0xfff7ff07,0xf7ff07f7},{0xfff707ff,0xf7ff07f7},{0xfffff7f7,0xf7ff07f7},{0xffffffff,0xf7ff07f7},
+{0xffff07f7,0xf7ff07f7},{0xffff0707,0xf7ff07f7},{0xff07f7f7,0xf7ff07f7},{0xff07f7ff,0xf7ff07f7},{0xff07fff7,0xf7ff07f7},{0xff07ff07,0xf7ff07f7},{0xff0707ff,0xf7ff07f7},{0x07f7ffff,0xf7ff07f7},
+{0x07fff7ff,0xf7ff07f7},{0x07fffff7,0xf7ff07f7},{0x0707ffff,0xf7ff07f7},{0xf7fff7f7,0xf7ff07ff},{0xf7fff707,0xf7ff07ff},{0xf7ffffff,0xf7ff07ff},{0xf7ff0707,0xf7ff07ff},{0xf707f7ff,0xf7ff07ff},
+{0xf707ffff,0xf7ff07ff},{0xfff7f707,0xf7ff07ff},{0xfff7fff7,0xf7ff07ff},{0xfff7ffff,0xf7ff07ff},{0xfff707f7,0xf7ff07ff},{0xfffff7ff,0xf7ff07ff},{0xfffffff7,0xf7ff07ff},{0xffffffff,0xf7ff07ff},
+{0xffffff07,0xf7ff07ff},{0xffff07ff,0xf7ff07ff},{0xffff0707,0xf7ff07ff},{0xff07f7f7,0xf7ff07ff},{0xff07ffff,0xf7ff07ff},{0xff070707,0xf7ff07ff},{0x07f707ff,0xf7ff07ff},{0x07fff7ff,0xf7ff07ff},
+{0x07fff707,0xf7ff07ff},{0x07ffffff,0xf7ff07ff},{0x07ff07f7,0xf7ff07ff},{0x0707f7ff,0xf7ff07ff},{0x0707ff07,0xf7ff07ff},{0x070707ff,0xf7ff07ff},{0xf7f707ff,0xf7ff0707},{0xf7ffff07,0xf7ff0707},
+{0xf707fff7,0xf7ff0707},{0xf707ff07,0xf7ff0707},{0xfff7fff7,0xf7ff0707},{0xfff7ff07,0xf7ff0707},{0xfff707ff,0xf7ff0707},{0xfffff7f7,0xf7ff0707},{0xfffff707,0xf7ff0707},{0xffffffff,0xf7ff0707},
+{0xffff07f7,0xf7ff0707},{0xff07f7ff,0xf7ff0707},{0xff07ff07,0xf7ff0707},{0xff0707ff,0xf7ff0707},{0x07f7ffff,0xf7ff0707},{0x07fff7ff,0xf7ff0707},{0x07ffff07,0xf7ff0707},{0x07ff0707,0xf7ff0707},
+{0xf7f7f7f7,0xf707f7f7},{0xf7f7f707,0xf707f7f7},{0xf7f707f7,0xf707f7f7},{0xf7f70707,0xf707f7f7},{0xf7ffffff,0xf707f7f7},{0xf707f7f7,0xf707f7f7},{0xf707f707,0xf707f7f7},{0xf707ffff,0xf707f7f7},
+{0xf70707f7,0xf707f7f7},{0xf7070707,0xf707f7f7},{0xfff7ffff,0xf707f7f7},{0xfffff7ff,0xf707f7f7},{0xffff07ff,0xf707f7f7},{0xff07f7ff,0xf707f7f7},{0xff07ffff,0xf707f7f7},{0x07f7f7f7,0xf707f7f7},
+{0x07f7f707,0xf707f7f7},{0x07f707f7,0xf707f7f7},{0x07f70707,0xf707f7f7},{0x07ffffff,0xf707f7f7},{0x0707f7f7,0xf707f7f7},{0x0707f707,0xf707f7f7},{0x0707ffff,0xf707f7f7},{0x070707f7,0xf707f7f7},
+{0x07070707,0xf707f7f7},{0xf7f7ffff,0xf707f7ff},{0xf7fff7ff,0xf707f7ff},{0xf7fffff7,0xf707f7ff},{0xf7ff07ff,0xf707f7ff},{0xf707ffff,0xf707f7ff},{0xfff7f707,0xf707f7ff},{0xfff7fff7,0xf707f7ff},
+{0xfff707ff,0xf707f7ff},{0xffffffff,0xf707f7ff},{0xffff07f7,0xf707f7ff},{0xffff0707,0xf707f7ff},{0xff07f7ff,0xf707f7ff},{0xff07fff7,0xf707f7ff},{0xff07ffff,0xf707f7ff},{0xff07ff07,0xf707f7ff},
+{0x07f7ffff,0xf707f7ff},{0x07fff7f7,0xf707f7ff},{0x07ffff07,0xf707f7ff},{0x07ff07ff,0xf707f7ff},{0x0707ffff,0xf707f7ff},{0xf7f7f7ff,0xf707f707},{0xf7f707f7,0xf707f707},{0xf7f70707,0xf707f707},
+{0xf7fff7ff,0xf707f707},{0xf7ffffff,0xf707f707},{0xf707f7f7,0xf707f707},{0xf707f707,0xf707f707},{0xf70707f7,0xf707f707},{0xf7070707,0xf707f707},{0xfff7ffff,0xf707f707},{0xfffff7ff,0xf707f707},
+{0xffffff07,0xf707f707},{0xffff07ff,0xf707f707},{0xff07ffff,0xf707f707},{0x07f7f7ff,0xf707f707},{0x07f707f7,0xf707f707},{0x07f70707,0xf707f707},{0x07fff7ff,0xf707f707},{0x07ffffff,0xf707f707},
+{0x0707f7f7,0xf707f707},{0x0707f707,0xf707f707},{0x070707f7,0xf707f707},{0x07070707,0xf707f707},{0xf7f7ffff,0xf707fff7},{0xf7fffff7,0xf707fff7},{0xf7ffff07,0xf707fff7},{0xf7ff07ff,0xf707fff7},
+{0xf707ffff,0xf707fff7},{0xfff7fff7,0xf707fff7},{0xfff7ffff,0xf707fff7},{0xfff7ff07,0xf707fff7},{0xfff707ff,0xf707fff7},{0xfffff707,0xf707fff7},{0xffffffff,0xf707fff7},{0xffff07f7,0xf707fff7},
+{0xffff0707,0xf707fff7},{0xff07ff07,0xf707fff7},{0x07f7ffff,0xf707fff7},{0x07fff7ff,0xf707fff7},{0x07fffff7,0xf707fff7},{0x07ff07ff,0xf707fff7},{0x0707f7ff,0xf707fff7},{0x0707ffff,0xf707fff7},
+{0xf7f707ff,0xf707ffff},{0xf7ffffff,0xf707ffff},{0xf707f7ff,0xf707ffff},{0xf70707ff,0xf707ffff},{0xfff7f7f7,0xf707ffff},{0xfff7ffff,0xf707ffff},{0xfff707f7,0xf707ffff},{0xfffff7ff,0xf707ffff},
+{0xfffffff7,0xf707ffff},{0xffffffff,0xf707ffff},{0xffff07ff,0xf707ffff},{0xff07f707,0xf707ffff},{0xff07ffff,0xf707ffff},{0xff0707f7,0xf707ffff},{0x07f707ff,0xf707ffff},{0x07fff7f7,0xf707ffff},
+{0x07fffff7,0xf707ffff},{0x07ffffff,0xf707ffff},{0x07ff07f7,0xf707ffff},{0x07ff0707,0xf707ffff},{0x0707f7ff,0xf707ffff},{0x0707fff7,0xf707ffff},{0x0707ff07,0xf707ffff},{0x070707ff,0xf707ffff},
+{0xf7f7ffff,0xf707ff07},{0xf7fff7f7,0xf707ff07},{0xf7fff707,0xf707ff07},{0xf7ff07ff,0xf707ff07},{0xf707ffff,0xf707ff07},{0xfff7f7ff,0xf707ff07},{0xfff707ff,0xf707ff07},{0xffffffff,0xf707ff07},
+{0xff07f7f7,0xf707ff07},{0xff07f7ff,0xf707ff07},{0xff0707ff,0xf707ff07},{0x07f7fff7,0xf707ff07},{0x07f7ff07,0xf707ff07},{0x07fff7f7,0xf707ff07},{0x07ff0707,0xf707ff07},{0xf7f7f7f7,0xf70707f7},
+{0xf7f7f707,0xf70707f7},{0xf7f707f7,0xf70707f7},{0xf7f70707,0xf70707f7},{0xf7ffffff,0xf70707f7},{0xf707f7f7,0xf70707f7},{0xf707f707,0xf70707f7},{0xf70707f7,0xf70707f7},{0xf7070707,0xf70707f7},
+{0xfff7ffff,0xf70707f7},{0xfffff7ff,0xf70707f7},{0xfffffff7,0xf70707f7},{0xff07ffff,0xf70707f7},{0x07f7f7f7,0xf70707f7},{0x07f7f707,0xf70707f7},{0x07f707f7,0xf70707f7},{0x07f70707,0xf70707f7},
+{0x0707f7f7,0xf70707f7},{0x0707f707,0xf70707f7},{0x070707f7,0xf70707f7},{0x07070707,0xf70707f7},{0xf7f707ff,0xf70707ff},{0xf7fff7ff,0xf70707ff},{0xf7fffff7,0xf70707ff},{0xf7ff07ff,0xf70707ff},
+{0xf707ffff,0xf70707ff},{0xfff7ff07,0xf70707ff},{0xfff707ff,0xf70707ff},{0xfffff707,0xf70707ff},{0xffffffff,0xf70707ff},{0xff07f7ff,0xf70707ff},{0xff07fff7,0xf70707ff},{0xff07ff07,0xf70707ff},
+{0xff0707ff,0xf70707ff},{0x07f7ffff,0xf70707ff},{0x07fff7f7,0xf70707ff},{0x07ffff07,0xf70707ff},{0x07ff07ff,0xf70707ff},{0x0707fff7,0xf70707ff},{0x0707ffff,0xf70707ff},{0xf7f7f7f7,0xf7070707},
+{0xf7f7f707,0xf7070707},{0xf7f707f7,0xf7070707},{0xf7f70707,0xf7070707},{0xf707f7f7,0xf7070707},{0xf707f707,0xf7070707},{0xf70707f7,0xf7070707},{0xf7070707,0xf7070707},{0xfff7ffff,0xf7070707},
+{0xfffff7ff,0xf7070707},{0xffffff07,0xf7070707},{0xffff07ff,0xf7070707},{0xff07ffff,0xf7070707},{0x07f7f7f7,0xf7070707},{0x07f7f707,0xf7070707},{0x07f707f7,0xf7070707},{0x07f70707,0xf7070707},
+{0x07ffffff,0xf7070707},{0x0707f7f7,0xf7070707},{0x0707f707,0xf7070707},{0x070707f7,0xf7070707},{0x07070707,0xf7070707},{0xf7f7ffff,0xfff7f7f7},{0xf7fff7ff,0xfff7f7f7},{0xf7ffff07,0xfff7f7f7},
+{0xf707ffff,0xfff7f7f7},{0xfff707ff,0xfff7f7f7},{0xfffff707,0xfff7f7f7},{0xffffffff,0xfff7f7f7},{0xffff07f7,0xfff7f7f7},{0xffff0707,0xfff7f7f7},{0xff07f7ff,0xfff7f7f7},{0xff07fff7,0xfff7f7f7},
+{0xff07ff07,0xfff7f7f7},{0x07fffff7,0xfff7f7f7},{0x07ff07ff,0xfff7f7f7},{0x0707f7ff,0xfff7f7f7},{0x0707ff07,0xfff7f7f7},{0xf7f7f7f7,0xfff7f7ff},{0xf7f7f7ff,0xfff7f7ff},{0xf7f7fff7,0xfff7f7ff},
+{0xf7f7ff07,0xfff7f7ff},{0xf7f707ff,0xfff7f7ff},{0xf7fff707,0xfff7f7ff},{0xf7ffffff,0xfff7f7ff},{0xf7ffff07,0xfff7f7ff},{0xf7ff07f7,0xfff7f7ff},{0xf7ff0707,0xfff7f7ff},{0xf707f7ff,0xfff7f7ff},
+{0xf707ff07,0xfff7f7ff},{0xf70707ff,0xfff7f7ff},{0xfff7ffff,0xfff7f7ff},{0xfff707f7,0xfff7f7ff},{0xfff70707,0xfff7f7ff},{0xfffff7ff,0xfff7f7ff},{0xfffffff7,0xfff7f7ff},{0xffffffff,0xfff7f7ff},
+{0xffffff07,0xfff7f7ff},{0xffff07ff,0xfff7f7ff},{0xffff0707,0xfff7f7ff},{0xff07ffff,0xfff7f7ff},{0xff0707f7,0xfff7f7ff},{0xff070707,0xfff7f7ff},{0x07f7f7ff,0xfff7f7ff},{0x07f7fff7,0xfff7f7ff},
+{0x07f7ff07,0xfff7f7ff},{0x07fff7f7,0xfff7f7ff},{0x07fff707,0xfff7f7ff},{0x07ffffff,0xfff7f7ff},{0x0707f7f7,0xfff7f7ff},{0x0707f7ff,0xfff7f7ff},{0x0707f707,0xfff7f7ff},{0xf7f7ffff,0xfff7f707},
+{0xf7fff7ff,0xfff7f707},{0xf7fffff7,0xfff7f707},{0xf7ffff07,0xfff7f707},{0xf707ffff,0xfff7f707},{0xfff7f7ff,0xfff7f707},{0xfffff707,0xfff7f707},{0xffffffff,0xfff7f707},{0xffff0707,0xfff7f707},
+{0xff07fff7,0xfff7f707},{0xff0707ff,0xfff7f707},{0x07f707ff,0xfff7f707},{0x07fffff7,0xfff7f707},{0x0707ffff,0xfff7f707},{0xf7f7f7ff,0xfff7fff7},{0xf7ffffff,0xfff7fff7},{0xf7ff07ff,0xfff7fff7},
+{0xf70707ff,0xfff7fff7},{0xfff7ffff,0xfff7fff7},{0xfff707f7,0xfff7fff7},{0xfff70707,0xfff7fff7},{0xfffff7ff,0xfff7fff7},{0xfffffff7,0xfff7fff7},{0xffffffff,0xfff7fff7},{0xffffff07,0xfff7fff7},
+{0xff07f7ff,0xfff7fff7},{0xff07f707,0xfff7fff7},{0xff07ffff,0xfff7fff7},{0xff0707f7,0xfff7fff7},{0xff070707,0xfff7fff7},{0x07f7f7ff,0xfff7fff7},{0x07f7ff07,0xfff7fff7},{0x07f707ff,0xfff7fff7},
+{0x07fff7f7,0xfff7fff7},{0x07fff707,0xfff7fff7},{0x07ffffff,0xfff7fff7},{0x0707f7f7,0xfff7fff7},{0x0707f7ff,0xfff7fff7},{0x070707ff,0xfff7fff7},{0xf7f7f7ff,0xfff7ffff},{0xf7f7f707,0xfff7ffff},
+{0xf7f7ffff,0xfff7ffff},{0xf7f70707,0xfff7ffff},{0xf7fff7ff,0xfff7ffff},{0xf7fffff7,0xfff7ffff},{0xf7ffffff,0xfff7ffff},{0xf7ffff07,0xfff7ffff},{0xf7ff07ff,0xfff7ffff},{0xf707f7f7,0xfff7ffff},
+{0xf707ffff,0xfff7ffff},{0xf7070707,0xfff7ffff},{0xfff7f7ff,0xfff7ffff},{0xfff7fff7,0xfff7ffff},{0xfff7ffff,0xfff7ffff},{0xfff7ff07,0xfff7ffff},{0xfff707ff,0xfff7ffff},{0xfffff7f7,0xfff7ffff},
+{0xfffff7ff,0xfff7ffff},{0xfffffff7,0xfff7ffff},{0xffffffff,0xfff7ffff},{0xffffff07,0xfff7ffff},{0xffff07f7,0xfff7ffff},{0xffff07ff,0xfff7ffff},{0xff07f7ff,0xfff7ffff},{0xff07fff7,0xfff7ffff},
+{0xff07ffff,0xfff7ffff},{0xff07ff07,0xfff7ffff},{0xff0707ff,0xfff7ffff},{0x07f7f707,0xfff7ffff},{0x07f7fff7,0xfff7ffff},{0x07f7ffff,0xfff7ffff},{0x07f707f7,0xfff7ffff},{0x07fff7ff,0xfff7ffff},
+{0x07fffff7,0xfff7ffff},{0x07ffffff,0xfff7ffff},{0x07ffff07,0xfff7ffff},{0x07ff07ff,0xfff7ffff},{0x07ff0707,0xfff7ffff},{0x0707ffff,0xfff7ffff},{0x070707f7,0xfff7ffff},{0x07070707,0xfff7ffff},
+{0xf7f7f7ff,0xfff7ff07},{0xf7f7ffff,0xfff7ff07},{0xf7f707ff,0xfff7ff07},{0xf7fffff7,0xfff7ff07},{0xf7ffffff,0xfff7ff07},{0xf7ff07f7,0xfff7ff07},{0xf7ff0707,0xfff7ff07},{0xf707f7ff,0xfff7ff07},
+{0xf707fff7,0xfff7ff07},{0xf70707ff,0xfff7ff07},{0xfff7f7f7,0xfff7ff07},{0xfff7f707,0xfff7ff07},{0xfff7ffff,0xfff7ff07},{0xfff707f7,0xfff7ff07},{0xfffff7f7,0xfff7ff07},{0xfffff7ff,0xfff7ff07},
+{0xfffff707,0xfff7ff07},{0xffffffff,0xfff7ff07},{0xffffff07,0xfff7ff07},{0xffff07ff,0xfff7ff07},{0xff07f707,0xfff7ff07},{0xff07ffff,0xfff7ff07},{0xff0707f7,0xfff7ff07},{0x07f7f7ff,0xfff7ff07},
+{0x07f7ffff,0xfff7ff07},{0x07f70707,0xfff7ff07},{0x07fffff7,0xfff7ff07},{0x07ffffff,0xfff7ff07},{0x0707f7ff,0xfff7ff07},{0x0707fff7,0xfff7ff07},{0x0707ff07,0xfff7ff07},{0xf7f7ffff,0xfff707f7},
+{0xf7fff7ff,0xfff707f7},{0xf7ffffff,0xfff707f7},{0xf7ff0707,0xfff707f7},{0xf707ffff,0xfff707f7},{0xfff7f707,0xfff707f7},{0xfff707ff,0xfff707f7},{0xfffff7f7,0xfff707f7},{0xffffffff,0xfff707f7},
+{0xffff07f7,0xfff707f7},{0xff07f7ff,0xfff707f7},{0xff07fff7,0xfff707f7},{0xff07ff07,0xfff707f7},{0xff0707ff,0xfff707f7},{0x07f7ffff,0xfff707f7},{0x07fff7ff,0xfff707f7},{0x07fffff7,0xfff707f7},
+{0x07ffff07,0xfff707f7},{0x07ff07ff,0xfff707f7},{0x0707ffff,0xfff707f7},{0xf7f7f7ff,0xfff707ff},{0xf7f7ffff,0xfff707ff},{0xf7f7ff07,0xfff707ff},{0xf7f70707,0xfff707ff},{0xf7fff7f7,0xfff707ff},
+{0xf7fffff7,0xfff707ff},{0xf7ffffff,0xfff707ff},{0xf7ff07f7,0xfff707ff},{0xf707f7ff,0xfff707ff},{0xf707fff7,0xfff707ff},{0xf707ff07,0xfff707ff},{0xfff7f7f7,0xfff707ff},{0xfff7ffff,0xfff707ff},
+{0xfff70707,0xfff707ff},{0xfffff7ff,0xfff707ff},{0xfffff707,0xfff707ff},{0xfffffff7,0xfff707ff},{0xffffffff,0xfff707ff},{0xffffff07,0xfff707ff},{0xffff07ff,0xfff707ff},{0xff07f7f7,0xfff707ff},
+{0xff07f707,0xfff707ff},{0xff07ffff,0xfff707ff},{0xff07ff07,0xfff707ff},{0xff070707,0xfff707ff},{0x07f7ff07,0xfff707ff},{0x07f707ff,0xfff707ff},{0x07fff707,0xfff707ff},{0x07ffffff,0xfff707ff},
+{0x07ffff07,0xfff707ff},{0x07ff07f7,0xfff707ff},{0x0707f7ff,0xfff707ff},{0x0707fff7,0xfff707ff},{0x0707ff07,0xfff707ff},{0x070707ff,0xfff707ff},{0xf7ffff07,0xfff70707},{0xfff7fff7,0xfff70707},
+{0xfff7ff07,0xfff70707},{0xfff707ff,0xfff70707},{0xffffffff,0xfff70707},{0xffff07f7,0xfff70707},{0xffff0707,0xfff70707},{0xff07fff7,0xfff70707},{0xff0707ff,0xfff70707},{0x07fffff7,0xfff70707},
+{0x0707ffff,0xfff70707},{0xf7f7f7ff,0xfffff7f7},{0xf7f7fff7,0xfffff7f7},{0xf7f7ffff,0xfffff7f7},{0xf7f7ff07,0xfffff7f7},{0xf7f707ff,0xfffff7f7},{0xf7fff707,0xfffff7f7},{0xf7ffffff,0xfffff7f7},
+{0xf7ff0707,0xfffff7f7},{0xf707f7ff,0xfffff7f7},{0xf707fff7,0xfffff7f7},{0xf70707ff,0xfffff7f7},{0xfff7f7f7,0xfffff7f7},{0xfff7ffff,0xfffff7f7},{0xfff707f7,0xfffff7f7},{0xfffff7ff,0xfffff7f7},
+{0xfffffff7,0xfffff7f7},{0xffffffff,0xfffff7f7},{0xffffff07,0xfffff7f7},{0xffff07ff,0xfffff7f7},{0xff07ffff,0xfffff7f7},{0xff0707f7,0xfffff7f7},{0x07f7ff07,0xfffff7f7},{0x07f707ff,0xfffff7f7},
+{0x07ffffff,0xfffff7f7},{0x07ff07f7,0xfffff7f7},{0x0707f7f7,0xfffff7f7},{0x0707f7ff,0xfffff7f7},{0x0707ff07,0xfffff7f7},{0x070707ff,0xfffff7f7},{0xf7f7ffff,0xfffff7ff},{0xf7f707f7,0xfffff7ff},
+{0xf7f707ff,0xfffff7ff},{0xf7f70707,0xfffff7ff},{0xf7fff7ff,0xfffff7ff},{0xf7fffff7,0xfffff7ff},{0xf7ffffff,0xfffff7ff},{0xf7ffff07,0xfffff7ff},{0xf7ff07f7,0xfffff7ff},{0xf7ff07ff,0xfffff7ff},
+{0xf707f7f7,0xfffff7ff},{0xf707ffff,0xfffff7ff},{0xf707ff07,0xfffff7ff},{0xf70707f7,0xfffff7ff},{0xf7070707,0xfffff7ff},{0xfff7f7ff,0xfffff7ff},{0xfff7fff7,0xfffff7ff},{0xfff7ffff,0xfffff7ff},
+{0xfff7ff07,0xfffff7ff},{0xfff707ff,0xfffff7ff},{0xfffff7f7,0xfffff7ff},{0xfffff7ff,0xfffff7ff},{0xfffff707,0xfffff7ff},{0xfffffff7,0xfffff7ff},{0xffffffff,0xfffff7ff},{0xffffff07,0xfffff7ff},
+{0xffff07f7,0xfffff7ff},{0xffff07ff,0xfffff7ff},{0xffff0707,0xfffff7ff},{0xff07f7ff,0xfffff7ff},{0xff07fff7,0xfffff7ff},{0xff07ffff,0xfffff7ff},{0xff07ff07,0xfffff7ff},{0xff0707ff,0xfffff7ff},
+{0x07f7f707,0xfffff7ff},{0x07f7ffff,0xfffff7ff},{0x07fff7ff,0xfffff7ff},{0x07fffff7,0xfffff7ff},{0x07ffffff,0xfffff7ff},{0x07ffff07,0xfffff7ff},{0x07ff07ff,0xfffff7ff},{0x0707f7f7,0xfffff7ff},
+{0x0707ffff,0xfffff7ff},{0x07070707,0xfffff7ff},{0xf7f7f7ff,0xfffff707},{0xf7f7ff07,0xfffff707},{0xf7fff707,0xfffff707},{0xf7ffffff,0xfffff707},{0xf7ff0707,0xfffff707},{0xf707f7ff,0xfffff707},
+{0xf707fff7,0xfffff707},{0xfff7f707,0xfffff707},{0xfff7ffff,0xfffff707},{0xfff70707,0xfffff707},{0xfffff7ff,0xfffff707},{0xfffffff7,0xfffff707},{0xffffffff,0xfffff707},{0xffffff07,0xfffff707},
+{0xffff07ff,0xfffff707},{0xff07f707,0xfffff707},{0xff07ffff,0xfffff707},{0x07f7ffff,0xfffff707},{0x07fff7f7,0xfffff707},{0x07fff707,0xfffff707},{0x07ffffff,0xfffff707},{0x07ff07ff,0xfffff707},
+{0x07ff0707,0xfffff707},{0x0707fff7,0xfffff707},{0xf7f7fff7,0xfffffff7},{0xf7f7ffff,0xfffffff7},{0xf7fff7ff,0xfffffff7},{0xf7fffff7,0xfffffff7},{0xf7ffffff,0xfffffff7},{0xf7ffff07,0xfffffff7},
+{0xf7ff07f7,0xfffffff7},{0xf7ff07ff,0xfffffff7},{0xf707f7ff,0xfffffff7},{0xf707ffff,0xfffffff7},{0xf70707f7,0xfffffff7},{0xf7070707,0xfffffff7},{0xfff7f7ff,0xfffffff7},{0xfff7fff7,0xfffffff7},
+{0xfff7ffff,0xfffffff7},{0xfff7ff07,0xfffffff7},{0xfff707ff,0xfffffff7},{0xfff70707,0xfffffff7},{0xfffff7f7,0xfffffff7},{0xfffff7ff,0xfffffff7},{0xfffffff7,0xfffffff7},{0xffffffff,0xfffffff7},
+{0xffffff07,0xfffffff7},{0xffff07f7,0xfffffff7},{0xffff07ff,0xfffffff7},{0xffff0707,0xfffffff7},{0xff07f7ff,0xfffffff7},{0xff07f707,0xfffffff7},{0xff07fff7,0xfffffff7},{0xff07ffff,0xfffffff7},
+{0xff07ff07,0xfffffff7},{0xff0707ff,0xfffffff7},{0x07f7f7f7,0xfffffff7},{0x07f707f7,0xfffffff7},{0x07f70707,0xfffffff7},{0x07fff7ff,0xfffffff7},{0x07fffff7,0xfffffff7},{0x07ffffff,0xfffffff7},
+{0x07ffff07,0xfffffff7},{0x07ff07ff,0xfffffff7},{0x0707f7ff,0xfffffff7},{0x0707fff7,0xfffffff7},{0x0707ffff,0xfffffff7},{0x07070707,0xfffffff7},{0xf7f7f7ff,0xffffffff},{0xf7f7f707,0xffffffff},
+{0xf7f7fff7,0xffffffff},{0xf7f7ffff,0xffffffff},{0xf7f7ff07,0xffffffff},{0xf7f707ff,0xffffffff},{0xf7fff7f7,0xffffffff},{0xf7fff7ff,0xffffffff},{0xf7fff707,0xffffffff},{0xf7fffff7,0xffffffff},
+{0xf7ffffff,0xffffffff},{0xf7ffff07,0xffffffff},{0xf7ff07ff,0xffffffff},{0xf7ff0707,0xffffffff},{0xf707f7ff,0xffffffff},{0xf707fff7,0xffffffff},{0xf707ffff,0xffffffff},{0xf707ff07,0xffffffff},
+{0xf70707ff,0xffffffff},{0xfff7f7f7,0xffffffff},{0xfff7f7ff,0xffffffff},{0xfff7f707,0xffffffff},{0xfff7fff7,0xffffffff},{0xfff7ffff,0xffffffff},{0xfff7ff07,0xffffffff},{0xfff707f7,0xffffffff},
+{0xfff707ff,0xffffffff},{0xfffff7f7,0xffffffff},{0xfffff7ff,0xffffffff},{0xfffff707,0xffffffff},{0xfffffff7,0xffffffff},{0xffffffff,0xffffffff},{0xffffff07,0xffffffff},{0xffff07f7,0xffffffff},
+{0xffff07ff,0xffffffff},{0xffff0707,0xffffffff},{0xff07f7f7,0xffffffff},{0xff07f7ff,0xffffffff},{0xff07fff7,0xffffffff},{0xff07ffff,0xffffffff},{0xff07ff07,0xffffffff},{0xff0707f7,0xffffffff},
+{0xff0707ff,0xffffffff},{0xff070707,0xffffffff},{0x07f7f7ff,0xffffffff},{0x07f7fff7,0xffffffff},{0x07f7ffff,0xffffffff},{0x07f707ff,0xffffffff},{0x07f70707,0xffffffff},{0x07fff7f7,0xffffffff},
+{0x07fff7ff,0xffffffff},{0x07fffff7,0xffffffff},{0x07ffffff,0xffffffff},{0x07ffff07,0xffffffff},{0x07ff07f7,0xffffffff},{0x07ff07ff,0xffffffff},{0x0707f7ff,0xffffffff},{0x0707fff7,0xffffffff},
+{0x0707ffff,0xffffffff},{0x0707ff07,0xffffffff},{0x070707ff,0xffffffff},{0xf7f7f7f7,0xffffff07},{0xf7f7f7ff,0xffffff07},{0xf7f7f707,0xffffff07},{0xf7f7fff7,0xffffff07},{0xf7f7ff07,0xffffff07},
+{0xf7f707f7,0xffffff07},{0xf7f707ff,0xffffff07},{0xf7fff7ff,0xffffff07},{0xf7fffff7,0xffffff07},{0xf7ffffff,0xffffff07},{0xf7ff07f7,0xffffff07},{0xf7ff07ff,0xffffff07},{0xf707f7f7,0xffffff07},
+{0xf707f7ff,0xffffff07},{0xf707f707,0xffffff07},{0xf707fff7,0xffffff07},{0xf707ffff,0xffffff07},{0xf707ff07,0xffffff07},{0xf70707f7,0xffffff07},{0xf70707ff,0xffffff07},{0xfff7f7ff,0xffffff07},
+{0xfff7ffff,0xffffff07},{0xfff7ff07,0xffffff07},{0xfff707f7,0xffffff07},{0xfff707ff,0xffffff07},{0xfff70707,0xffffff07},{0xfffff7f7,0xffffff07},{0xfffff7ff,0xffffff07},{0xfffff707,0xffffff07},
+{0xfffffff7,0xffffff07},{0xffffffff,0xffffff07},{0xffffff07,0xffffff07},{0xffff07f7,0xffffff07},{0xffff07ff,0xffffff07},{0xffff0707,0xffffff07},{0xff07f7ff,0xffffff07},{0xff07fff7,0xffffff07},
+{0xff07ffff,0xffffff07},{0xff0707ff,0xffffff07},{0x07f7f707,0xffffff07},{0x07f7ffff,0xffffff07},{0x07f7ff07,0xffffff07},{0x07f707f7,0xffffff07},{0x07f707ff,0xffffff07},{0x07f70707,0xffffff07},
+{0x07fff7ff,0xffffff07},{0x07ffffff,0xffffff07},{0x07ff0707,0xffffff07},{0x0707f707,0xffffff07},{0x0707ffff,0xffffff07},{0x0707ff07,0xffffff07},{0x070707f7,0xffffff07},{0x070707ff,0xffffff07},
+{0xf7f7fff7,0xffff07f7},{0xf7f7ffff,0xffff07f7},{0xf7f7ff07,0xffff07f7},{0xf7f707ff,0xffff07f7},{0xf7fff7f7,0xffff07f7},{0xf7ffffff,0xffff07f7},{0xf7ff07f7,0xffff07f7},{0xf707f7ff,0xffff07f7},
+{0xf7070707,0xffff07f7},{0xfff7ffff,0xffff07f7},{0xfff707f7,0xffff07f7},{0xfff70707,0xffff07f7},{0xfffff7ff,0xffff07f7},{0xfffffff7,0xffff07f7},{0xffffffff,0xffff07f7},{0xffffff07,0xffff07f7},
+{0xffff07f7,0xffff07f7},{0xffff07ff,0xffff07f7},{0xff07f7f7,0xffff07f7},{0xff07f707,0xffff07f7},{0xff07fff7,0xffff07f7},{0xff07ffff,0xffff07f7},{0x07f7f707,0xffff07f7},{0x07f707ff,0xffff07f7},
+{0x07fff7f7,0xffff07f7},{0x07fff707,0xffff07f7},{0x07ffffff,0xffff07f7},{0x07ff07f7,0xffff07f7},{0x0707f7ff,0xffff07f7},{0x070707ff,0xffff07f7},{0xf7f7f7ff,0xffff07ff},{0xf7f7f707,0xffff07ff},
+{0xf7f7ffff,0xffff07ff},{0xf7f70707,0xffff07ff},{0xf7fff7ff,0xffff07ff},{0xf7fffff7,0xffff07ff},{0xf7ffffff,0xffff07ff},{0xf7ffff07,0xffff07ff},{0xf7ff07ff,0xffff07ff},{0xf707ffff,0xffff07ff},
+{0xfff7f7ff,0xffff07ff},{0xfff7fff7,0xffff07ff},{0xfff7ffff,0xffff07ff},{0xfff7ff07,0xffff07ff},{0xfff707ff,0xffff07ff},{0xfffff7f7,0xffff07ff},{0xfffff7ff,0xffff07ff},{0xfffff707,0xffff07ff},
+{0xfffffff7,0xffff07ff},{0xffffffff,0xffff07ff},{0xffffff07,0xffff07ff},{0xffff07f7,0xffff07ff},{0xffff07ff,0xffff07ff},{0xffff0707,0xffff07ff},{0xff07f7ff,0xffff07ff},{0xff07fff7,0xffff07ff},
+{0xff07ffff,0xffff07ff},{0xff07ff07,0xffff07ff},{0xff0707ff,0xffff07ff},{0x07f7f7ff,0xffff07ff},{0x07f7ffff,0xffff07ff},{0x07f707ff,0xffff07ff},{0x07fff7ff,0xffff07ff},{0x07fffff7,0xffff07ff},
+{0x07ffffff,0xffff07ff},{0x07ffff07,0xffff07ff},{0x07ff07f7,0xffff07ff},{0x07ff07ff,0xffff07ff},{0x0707ffff,0xffff07ff},{0xf7f7fff7,0xffff0707},{0xf7f707f7,0xffff0707},{0xf7ffffff,0xffff0707},
+{0xf7ff0707,0xffff0707},{0xf707f7f7,0xffff0707},{0xf707ffff,0xffff0707},{0xf707ff07,0xffff0707},{0xf70707ff,0xffff0707},{0xfff7ffff,0xffff0707},{0xfff707f7,0xffff0707},{0xfff707ff,0xffff0707},
+{0xfffff7ff,0xffff0707},{0xffffffff,0xffff0707},{0xffffff07,0xffff0707},{0xffff07f7,0xffff0707},{0xffff07ff,0xffff0707},{0xff07f707,0xffff0707},{0xff07ffff,0xffff0707},{0xff0707f7,0xffff0707},
+{0xff070707,0xffff0707},{0x07f7f7ff,0xffff0707},{0x07f70707,0xffff0707},{0x07fff707,0xffff0707},{0x07ffffff,0xffff0707},{0x07ffff07,0xffff0707},{0x07ff07f7,0xffff0707},{0x07ff0707,0xffff0707},
+{0x0707f7ff,0xffff0707},{0xf7f7ffff,0xff07f7f7},{0xf7fffff7,0xff07f7f7},{0xf7ffff07,0xff07f7f7},{0xf7ff07ff,0xff07f7f7},{0xf707ffff,0xff07f7f7},{0xfff7fff7,0xff07f7f7},{0xfffff7f7,0xff07f7f7},
+{0xffffffff,0xff07f7f7},{0xffffff07,0xff07f7f7},{0xffff07f7,0xff07f7f7},{0xffff0707,0xff07f7f7},{0xff07f7ff,0xff07f7f7},{0xff07fff7,0xff07f7f7},{0xff07ff07,0xff07f7f7},{0xff0707ff,0xff07f7f7},
+{0x07f7f7ff,0xff07f7f7},{0x07ffff07,0xff07f7f7},{0x0707ffff,0xff07f7f7},{0xf7f7f7ff,0xff07f7ff},{0xf7f7fff7,0xff07f7ff},{0xf7f7ff07,0xff07f7ff},{0xf7f707ff,0xff07f7ff},{0xf7fff707,0xff07f7ff},
+{0xf7ffffff,0xff07f7ff},{0xf707f7ff,0xff07f7ff},{0xf707f707,0xff07f7ff},{0xf707ff07,0xff07f7ff},{0xf70707ff,0xff07f7ff},{0xfff7ffff,0xff07f7ff},{0xfff707ff,0xff07f7ff},{0xfffff7ff,0xff07f7ff},
+{0xffffffff,0xff07f7ff},{0xffffff07,0xff07f7ff},{0xffff07ff,0xff07f7ff},{0xff07ffff,0xff07f7ff},{0xff07ff07,0xff07f7ff},{0xff070707,0xff07f7ff},{0x07f7fff7,0xff07f7ff},{0x07f70707,0xff07f7ff},
+{0x07fff707,0xff07f7ff},{0x07ffffff,0xff07f7ff},{0x0707f7ff,0xff07f7ff},{0x0707ff07,0xff07f7ff},{0x070707ff,0xff07f7ff},{0xf7fff7ff,0xff07f707},{0xf7ffff07,0xff07f707},{0xf7ff07ff,0xff07f707},
+{0xfff7f7f7,0xff07f707},{0xfff7f7ff,0xff07f707},{0xfff7ff07,0xff07f707},{0xffffffff,0xff07f707},{0xffffff07,0xff07f707},{0xffff07f7,0xff07f707},{0xff07f7f7,0xff07f707},{0x07f7ffff,0xff07f707},
+{0x07fff7ff,0xff07f707},{0x07ffff07,0xff07f707},{0x0707ffff,0xff07f707},{0xf7fff7ff,0xff07fff7},{0xf7fff707,0xff07fff7},{0xf7ffffff,0xff07fff7},{0xf7ffff07,0xff07fff7},{0xf7ff0707,0xff07fff7},
+{0xf707f7ff,0xff07fff7},{0xf707ff07,0xff07fff7},{0xf70707ff,0xff07fff7},{0xfff7f7f7,0xff07fff7},{0xfff7f707,0xff07fff7},{0xfff7ffff,0xff07fff7},{0xfff707f7,0xff07fff7},{0xfff70707,0xff07fff7},
+{0xfffff7ff,0xff07fff7},{0xfffffff7,0xff07fff7},{0xffffffff,0xff07fff7},{0xffffff07,0xff07fff7},{0xffff07ff,0xff07fff7},{0xffff0707,0xff07fff7},{0xff07f7f7,0xff07fff7},{0xff07f707,0xff07fff7},
+{0xff07ffff,0xff07fff7},{0x07f7fff7,0xff07fff7},{0x07f7ffff,0xff07fff7},{0x07f707ff,0xff07fff7},{0x07fff7f7,0xff07fff7},{0x07fff707,0xff07fff7},{0x07fffff7,0xff07fff7},{0x07ffffff,0xff07fff7},
+{0x07ffff07,0xff07fff7},{0x07ff07f7,0xff07fff7},{0x07ff0707,0xff07fff7},{0x0707f7ff,0xff07fff7},{0x0707fff7,0xff07fff7},{0x070707ff,0xff07fff7},{0xf7f7ffff,0xff07ffff},{0xf7f707f7,0xff07ffff},
+{0xf7f70707,0xff07ffff},{0xf7fff7ff,0xff07ffff},{0xf7ffffff,0xff07ffff},{0xf7ffff07,0xff07ffff},{0xf7ff07ff,0xff07ffff},{0xfff7fff7,0xff07ffff},{0xfff7ffff,0xff07ffff},{0xfff7ff07,0xff07ffff},
+{0xfff707ff,0xff07ffff},{0xfffff7f7,0xff07ffff},{0xfffff7ff,0xff07ffff},{0xfffffff7,0xff07ffff},{0xffffffff,0xff07ffff},{0xffffff07,0xff07ffff},{0xffff07ff,0xff07ffff},{0xff07f7ff,0xff07ffff},
+{0xff07fff7,0xff07ffff},{0xff07ffff,0xff07ffff},{0xff07ff07,0xff07ffff},{0xff0707ff,0xff07ffff},{0x07f7ff07,0xff07ffff},{0x07f707ff,0xff07ffff},{0x07f70707,0xff07ffff},{0x07fff7ff,0xff07ffff},
+{0x07ffffff,0xff07ffff},{0x07ffff07,0xff07ffff},{0x07ff07ff,0xff07ffff},{0x07ff0707,0xff07ffff},{0x0707f707,0xff07ffff},{0x0707ffff,0xff07ffff},{0x0707ff07,0xff07ffff},{0x070707f7,0xff07ffff},
+{0xf7f7f707,0xff07ff07},{0xf7f707ff,0xff07ff07},{0xf7ffffff,0xff07ff07},{0xf707f7f7,0xff07ff07},{0xf707ff07,0xff07ff07},{0xf70707f7,0xff07ff07},{0xf70707ff,0xff07ff07},{0xfff7f7f7,0xff07ff07},
+{0xfff7ffff,0xff07ff07},{0xfff707f7,0xff07ff07},{0xfff70707,0xff07ff07},{0xfffff7ff,0xff07ff07},{0xfffffff7,0xff07ff07},{0xffffffff,0xff07ff07},{0xffffff07,0xff07ff07},{0xffff07f7,0xff07ff07},
+{0xffff0707,0xff07ff07},{0xff07f7f7,0xff07ff07},{0xff07ffff,0xff07ff07},{0xff0707f7,0xff07ff07},{0x07f7f7f7,0xff07ff07},{0x07f7f707,0xff07ff07},{0x07f7ffff,0xff07ff07},{0x07f70707,0xff07ff07},
+{0x07fffff7,0xff07ff07},{0x07ffff07,0xff07ff07},{0x07ff07f7,0xff07ff07},{0x07ff07ff,0xff07ff07},{0x0707f7f7,0xff07ff07},{0x0707fff7,0xff07ff07},{0x0707ff07,0xff07ff07},{0x07070707,0xff07ff07},
+{0xf7ffff07,0xff0707f7},{0xf7ff07ff,0xff0707f7},{0xf707ffff,0xff0707f7},{0xfff7f7ff,0xff0707f7},{0xfffff707,0xff0707f7},{0xffffffff,0xff0707f7},{0xffff0707,0xff0707f7},{0xff07f7ff,0xff0707f7},
+{0xff0707ff,0xff0707f7},{0x07f7ffff,0xff0707f7},{0x07fff7ff,0xff0707f7},{0x07ff07f7,0xff0707f7},{0x0707ff07,0xff0707f7},{0xf7f7f7ff,0xff0707ff},{0xf7f7fff7,0xff0707ff},{0xf7fff7f7,0xff0707ff},
+{0xf7ffffff,0xff0707ff},{0xf707f7ff,0xff0707ff},{0xf707fff7,0xff0707ff},{0xf707ff07,0xff0707ff},{0xf70707ff,0xff0707ff},{0xfff7f7f7,0xff0707ff},{0xfff7f7ff,0xff0707ff},{0xfff7ffff,0xff0707ff},
+{0xfff7ff07,0xff0707ff},{0xfff707f7,0xff0707ff},{0xfffff7ff,0xff0707ff},{0xfffffff7,0xff0707ff},{0xffffffff,0xff0707ff},{0xffffff07,0xff0707ff},{0xffff07ff,0xff0707ff},{0xff07f7f7,0xff0707ff},
+{0xff07ffff,0xff0707ff},{0xff070707,0xff0707ff},{0x07f7f707,0xff0707ff},{0x07f7fff7,0xff0707ff},{0x07f70707,0xff0707ff},{0x07ffffff,0xff0707ff},{0x0707f7ff,0xff0707ff},{0x0707fff7,0xff0707ff},
+{0x0707ffff,0xff0707ff},{0x070707ff,0xff0707ff},{0xf7fff7ff,0xff070707},{0xf7ffff07,0xff070707},{0xf7ff07f7,0xff070707},{0xfff7f7ff,0xff070707},{0xfff7fff7,0xff070707},{0xfff707ff,0xff070707},
+{0xfffff7f7,0xff070707},{0xffffffff,0xff070707},{0xffff07f7,0xff070707},{0xffff0707,0xff070707},{0xff07fff7,0xff070707},{0xff07ffff,0xff070707},{0xff0707ff,0xff070707},{0x07f7ff07,0xff070707},
+{0x07fffff7,0xff070707},{0x07ff07f7,0xff070707},{0x07ff0707,0xff070707},{0x0707ff07,0xff070707},{0xf7f7f7f7,0x07f7f7f7},{0xf7f7f707,0x07f7f7f7},{0xf7f707f7,0x07f7f7f7},{0xf7f70707,0x07f7f7f7},
+{0xf707f7f7,0x07f7f7f7},{0xf707f707,0x07f7f7f7},{0xf70707f7,0x07f7f7f7},{0xf7070707,0x07f7f7f7},{0xfff7ffff,0x07f7f7f7},{0xfffff7f7,0x07f7f7f7},{0xfffff7ff,0x07f7f7f7},{0xfffffff7,0x07f7f7f7},
+{0xffffff07,0x07f7f7f7},{0xffff07ff,0x07f7f7f7},{0xff07ffff,0x07f7f7f7},{0x07f7f7f7,0x07f7f7f7},{0x07f7f707,0x07f7f7f7},{0x07f707f7,0x07f7f7f7},{0x07f70707,0x07f7f7f7},{0x07ffffff,0x07f7f7f7},
+{0x0707f7f7,0x07f7f7f7},{0x0707f707,0x07f7f7f7},{0x070707f7,0x07f7f7f7},{0x07070707,0x07f7f7f7},{0xf7f7ffff,0x07f7f7ff},{0xf7fff7ff,0x07f7f7ff},{0xf7fffff7,0x07f7f7ff},{0xf7ffff07,0x07f7f7ff},
+{0xf7ff07ff,0x07f7f7ff},{0xf707ffff,0x07f7f7ff},{0xfff7f7ff,0x07f7f7ff},{0xfff7fff7,0x07f7f7ff},{0xfff707ff,0x07f7f7ff},{0xfffff7f7,0x07f7f7ff},{0xfffff707,0x07f7f7ff},{0xffffffff,0x07f7f7ff},
+{0xffffff07,0x07f7f7ff},{0xffff07f7,0x07f7f7ff},{0xffff07ff,0x07f7f7ff},{0xff07fff7,0x07f7f7ff},{0xff07ff07,0x07f7f7ff},{0xff0707ff,0x07f7f7ff},{0x07f7ffff,0x07f7f7ff},{0x07f707ff,0x07f7f7ff},
+{0x07fffff7,0x07f7f7ff},{0x07ffff07,0x07f7f7ff},{0x07ff07ff,0x07f7f7ff},{0x0707ffff,0x07f7f7ff},{0xf7f7f7f7,0x07f7f707},{0xf7f7f707,0x07f7f707},{0xf7f707f7,0x07f7f707},{0xf7f70707,0x07f7f707},
+{0xf7ffffff,0x07f7f707},{0xf707f7f7,0x07f7f707},{0xf707f707,0x07f7f707},{0xf70707f7,0x07f7f707},{0xf7070707,0x07f7f707},{0xfffff7ff,0x07f7f707},{0xfffffff7,0x07f7f707},{0xffff07ff,0x07f7f707},
+{0xff07ffff,0x07f7f707},{0x07f7f7f7,0x07f7f707},{0x07f7f707,0x07f7f707},{0x07f707f7,0x07f7f707},{0x07f70707,0x07f7f707},{0x07ffffff,0x07f7f707},{0x0707f7f7,0x07f7f707},{0x0707f707,0x07f7f707},
+{0x070707f7,0x07f7f707},{0x07070707,0x07f7f707},{0xf7fffff7,0x07f7fff7},{0xf7ff07ff,0x07f7fff7},{0xfff7f7ff,0x07f7fff7},{0xfff7fff7,0x07f7fff7},{0xfffff7ff,0x07f7fff7},{0xffffffff,0x07f7fff7},
+{0xffff0707,0x07f7fff7},{0xff07f7ff,0x07f7fff7},{0xff07fff7,0x07f7fff7},{0xff0707ff,0x07f7fff7},{0x07fffff7,0x07f7fff7},{0x07ff07ff,0x07f7fff7},{0xf7f7f7ff,0x07f7ffff},{0xf7f707ff,0x07f7ffff},
+{0xf7fff707,0x07f7ffff},{0xf7ffffff,0x07f7ffff},{0xf7ff0707,0x07f7ffff},{0xf707ff07,0x07f7ffff},{0xf70707ff,0x07f7ffff},{0xfff7f7f7,0x07f7ffff},{0xfff7f7ff,0x07f7ffff},{0xfff7ffff,0x07f7ffff},
+{0xfff707f7,0x07f7ffff},{0xfffff7ff,0x07f7ffff},{0xfffffff7,0x07f7ffff},{0xffffffff,0x07f7ffff},{0xffffff07,0x07f7ffff},{0xffff07ff,0x07f7ffff},{0xffff0707,0x07f7ffff},{0xff07ffff,0x07f7ffff},
+{0xff07ff07,0x07f7ffff},{0xff0707f7,0x07f7ffff},{0xff070707,0x07f7ffff},{0x07f7f7ff,0x07f7ffff},{0x07f7fff7,0x07f7ffff},{0x07f7ff07,0x07f7ffff},{0x07f707ff,0x07f7ffff},{0x07fff7f7,0x07f7ffff},
+{0x07fff707,0x07f7ffff},{0x07ffffff,0x07f7ffff},{0x07ff07f7,0x07f7ffff},{0x0707ff07,0x07f7ffff},{0xf7fff7ff,0x07f7ff07},{0xf7ffff07,0x07f7ff07},{0xf7ff07ff,0x07f7ff07},{0xf707ffff,0x07f7ff07},
+{0xfff7f7ff,0x07f7ff07},{0xfff7fff7,0x07f7ff07},{0xfff707ff,0x07f7ff07},{0xfff70707,0x07f7ff07},{0xfffff7f7,0x07f7ff07},{0xffffffff,0x07f7ff07},{0xffff07ff,0x07f7ff07},{0xffff0707,0x07f7ff07},
+{0xff07f7ff,0x07f7ff07},{0xff07ff07,0x07f7ff07},{0xff070707,0x07f7ff07},{0x07f7ffff,0x07f7ff07},{0x07fff7ff,0x07f7ff07},{0x07ff0707,0x07f7ff07},{0x0707fff7,0x07f7ff07},{0xf7f7f7f7,0x07f707f7},
+{0xf7f7f707,0x07f707f7},{0xf7f707f7,0x07f707f7},{0xf7f70707,0x07f707f7},{0xf7ffffff,0x07f707f7},{0xf707f7f7,0x07f707f7},{0xf707f707,0x07f707f7},{0xf70707f7,0x07f707f7},{0xf7070707,0x07f707f7},
+{0xfff7f7ff,0x07f707f7},{0xfff7ffff,0x07f707f7},{0xfffff7ff,0x07f707f7},{0xfffffff7,0x07f707f7},{0xffff07ff,0x07f707f7},{0xff07ffff,0x07f707f7},{0xff0707ff,0x07f707f7},{0x07f7f7f7,0x07f707f7},
+{0x07f7f707,0x07f707f7},{0x07f707f7,0x07f707f7},{0x07f70707,0x07f707f7},{0x07ffffff,0x07f707f7},{0x0707f7f7,0x07f707f7},{0x0707f707,0x07f707f7},{0x070707f7,0x07f707f7},{0x07070707,0x07f707f7},
+{0xf7f7ffff,0x07f707ff},{0xf7f7ff07,0x07f707ff},{0xf7fff7ff,0x07f707ff},{0xf7fffff7,0x07f707ff},{0xf7ffff07,0x07f707ff},{0xf707ffff,0x07f707ff},{0xfff7f7ff,0x07f707ff},{0xfff7fff7,0x07f707ff},
+{0xfff7ff07,0x07f707ff},{0xfff707ff,0x07f707ff},{0xfffff7f7,0x07f707ff},{0xfffff707,0x07f707ff},{0xffffffff,0x07f707ff},{0xffff0707,0x07f707ff},{0xff07f7ff,0x07f707ff},{0xff07fff7,0x07f707ff},
+{0x07f7ffff,0x07f707ff},{0x07ffff07,0x07f707ff},{0x07ff07ff,0x07f707ff},{0x0707ffff,0x07f707ff},{0xf7f7f7f7,0x07f70707},{0xf7f7f707,0x07f70707},{0xf7f707f7,0x07f70707},{0xf7f70707,0x07f70707},
+{0xf7ffffff,0x07f70707},{0xf707f7f7,0x07f70707},{0xf707f707,0x07f70707},{0xf70707f7,0x07f70707},{0xf7070707,0x07f70707},{0xfff7ffff,0x07f70707},{0xfffff7ff,0x07f70707},{0xfffffff7,0x07f70707},
+{0xffffff07,0x07f70707},{0x07f7f7f7,0x07f70707},{0x07f7f707,0x07f70707},{0x07f707f7,0x07f70707},{0x07f70707,0x07f70707},{0x07ffffff,0x07f70707},{0x0707f7f7,0x07f70707},{0x0707f707,0x07f70707},
+{0x070707f7,0x07f70707},{0x07070707,0x07f70707},{0xf7f7ffff,0x07fff7f7},{0xf7fff7ff,0x07fff7f7},{0xf7ffff07,0x07fff7f7},{0xf7ff07f7,0x07fff7f7},{0xf7ff07ff,0x07fff7f7},{0xf707ffff,0x07fff7f7},
+{0xfff7f7ff,0x07fff7f7},{0xfff7ff07,0x07fff7f7},{0xfff707ff,0x07fff7f7},{0xffffffff,0x07fff7f7},{0xffff07f7,0x07fff7f7},{0xffff0707,0x07fff7f7},{0xff0707ff,0x07fff7f7},{0xff070707,0x07fff7f7},
+{0x07f7ffff,0x07fff7f7},{0x07fff7ff,0x07fff7f7},{0x07fffff7,0x07fff7f7},{0x07ffff07,0x07fff7f7},{0x07ff07ff,0x07fff7f7},{0x0707ffff,0x07fff7f7},{0xf7f7f7ff,0x07fff7ff},{0xf7f7fff7,0x07fff7ff},
+{0xf7f7ff07,0x07fff7ff},{0xf7f707ff,0x07fff7ff},{0xf7fff7f7,0x07fff7ff},{0xf7ffffff,0x07fff7ff},{0xf7ff07f7,0x07fff7ff},{0xf7ff0707,0x07fff7ff},{0xf707f7ff,0x07fff7ff},{0xf707fff7,0x07fff7ff},
+{0xf707ff07,0x07fff7ff},{0xf70707ff,0x07fff7ff},{0xfff7f7f7,0x07fff7ff},{0xfff7ffff,0x07fff7ff},{0xfffff7f7,0x07fff7ff},{0xfffff7ff,0x07fff7ff},{0xfffffff7,0x07fff7ff},{0xffffffff,0x07fff7ff},
+{0xffffff07,0x07fff7ff},{0xffff07ff,0x07fff7ff},{0xff07f707,0x07fff7ff},{0xff07ffff,0x07fff7ff},{0x07f7fff7,0x07fff7ff},{0x07f7ff07,0x07fff7ff},{0x07fff707,0x07fff7ff},{0x07ffffff,0x07fff7ff},
+{0x07ff07f7,0x07fff7ff},{0x0707f7ff,0x07fff7ff},{0x0707fff7,0x07fff7ff},{0x0707ff07,0x07fff7ff},{0x070707ff,0x07fff7ff},{0xf7f7ffff,0x07fff707},{0xf7fff7ff,0x07fff707},{0xf7fffff7,0x07fff707},
+{0xf7ff07ff,0x07fff707},{0xf707ffff,0x07fff707},{0xfff7fff7,0x07fff707},{0xfff7ff07,0x07fff707},{0xfff707ff,0x07fff707},{0xfffff7f7,0x07fff707},{0xfffff707,0x07fff707},{0xffffffff,0x07fff707},
+{0xffff07f7,0x07fff707},{0xff07ff07,0x07fff707},{0xff0707ff,0x07fff707},{0x07f7ffff,0x07fff707},{0x07fffff7,0x07fff707},{0x07ffff07,0x07fff707},{0x070707ff,0x07fff707},{0xf7f7f7ff,0x07fffff7},
+{0xf7f7fff7,0x07fffff7},{0xf7f7ff07,0x07fffff7},{0xf7fff7f7,0x07fffff7},{0xf7ffffff,0x07fffff7},{0xf7ff07f7,0x07fffff7},{0xf707ff07,0x07fffff7},{0xfff7f7f7,0x07fffff7},{0xfff70707,0x07fffff7},
+{0xfffff7ff,0x07fffff7},{0xfffffff7,0x07fffff7},{0xffffffff,0x07fffff7},{0xffffff07,0x07fffff7},{0xffff07f7,0x07fffff7},{0xffff07ff,0x07fffff7},{0xff07f7f7,0x07fffff7},{0xff07f7ff,0x07fffff7},
+{0xff07f707,0x07fffff7},{0xff07ffff,0x07fffff7},{0x07f7fff7,0x07fffff7},{0x07f7ff07,0x07fffff7},{0x07fff707,0x07fffff7},{0x07fffff7,0x07fffff7},{0x07ffffff,0x07fffff7},{0x07ff07f7,0x07fffff7},
+{0x0707f7ff,0x07fffff7},{0x070707ff,0x07fffff7},{0xf7f7f7f7,0x07ffffff},{0xf7f7ffff,0x07ffffff},{0xf7f707f7,0x07ffffff},{0xf7f70707,0x07ffffff},{0xf7fff7f7,0x07ffffff},{0xf7fff7ff,0x07ffffff},
+{0xf7fffff7,0x07ffffff},{0xf7ffffff,0x07ffffff},{0xf7ffff07,0x07ffffff},{0xf7ff07ff,0x07ffffff},{0xf707f7ff,0x07ffffff},{0xf707ffff,0x07ffffff},{0xf70707ff,0x07ffffff},{0xf7070707,0x07ffffff},
+{0xfff7f7ff,0x07ffffff},{0xfff7fff7,0x07ffffff},{0xfff7ffff,0x07ffffff},{0xfff7ff07,0x07ffffff},{0xfff707ff,0x07ffffff},{0xfffff7f7,0x07ffffff},{0xfffff7ff,0x07ffffff},{0xfffff707,0x07ffffff},
+{0xfffffff7,0x07ffffff},{0xffffffff,0x07ffffff},{0xffffff07,0x07ffffff},{0xffff07f7,0x07ffffff},{0xffff07ff,0x07ffffff},{0xffff0707,0x07ffffff},{0xff07f7ff,0x07ffffff},{0xff07fff7,0x07ffffff},
+{0xff07ffff,0x07ffffff},{0xff07ff07,0x07ffffff},{0xff0707ff,0x07ffffff},{0x07f7f7ff,0x07ffffff},{0x07f7ffff,0x07ffffff},{0x07f707f7,0x07ffffff},{0x07fff7ff,0x07ffffff},{0x07fff707,0x07ffffff},
+{0x07fffff7,0x07ffffff},{0x07ffffff,0x07ffffff},{0x07ffff07,0x07ffffff},{0x07ff07ff,0x07ffffff},{0x07ff0707,0x07ffffff},{0x0707f7f7,0x07ffffff},{0x0707f707,0x07ffffff},{0x0707ffff,0x07ffffff},
+{0x070707f7,0x07ffffff},{0x07070707,0x07ffffff},{0xf7f7f7ff,0x07ffff07},{0xf7f7fff7,0x07ffff07},{0xf7fff7f7,0x07ffff07},{0xf7ffffff,0x07ffff07},{0xf7ff07ff,0x07ffff07},{0xf707f7f7,0x07ffff07},
+{0xf707ff07,0x07ffff07},{0xf70707ff,0x07ffff07},{0xfff7ffff,0x07ffff07},{0xfff707f7,0x07ffff07},{0xfff707ff,0x07ffff07},{0xfffff7ff,0x07ffff07},{0xfffff707,0x07ffff07},{0xffffffff,0x07ffff07},
+{0xffffff07,0x07ffff07},{0xffff07ff,0x07ffff07},{0xff07ffff,0x07ffff07},{0xff0707f7,0x07ffff07},{0x07f7f707,0x07ffff07},{0x07f7fff7,0x07ffff07},{0x07f707ff,0x07ffff07},{0x07f70707,0x07ffff07},
+{0x07fff707,0x07ffff07},{0x07fffff7,0x07ffff07},{0x07ffffff,0x07ffff07},{0x0707fff7,0x07ffff07},{0x0707ff07,0x07ffff07},{0x070707ff,0x07ffff07},{0xf7f7ffff,0x07ff07f7},{0xf7ffff07,0x07ff07f7},
+{0xf7ff07ff,0x07ff07f7},{0xf707ffff,0x07ff07f7},{0xfff7f7ff,0x07ff07f7},{0xfff7ff07,0x07ff07f7},{0xfffff7f7,0x07ff07f7},{0xfffff707,0x07ff07f7},{0xffffffff,0x07ff07f7},{0xffffff07,0x07ff07f7},
+{0xffff0707,0x07ff07f7},{0xff07fff7,0x07ff07f7},{0xff07ffff,0x07ff07f7},{0x07f7ffff,0x07ff07f7},{0x07fff7ff,0x07ff07f7},{0x07ffff07,0x07ff07f7},{0x07ff07ff,0x07ff07f7},{0x0707ffff,0x07ff07f7},
+{0xf7f7fff7,0x07ff07ff},{0xf7f7ff07,0x07ff07ff},{0xf7f707ff,0x07ff07ff},{0xf7fff7f7,0x07ff07ff},{0xf7fff707,0x07ff07ff},{0xf7ffffff,0x07ff07ff},{0xf7ff07f7,0x07ff07ff},{0xf7ff0707,0x07ff07ff},
+{0xf707f7f7,0x07ff07ff},{0xf707f7ff,0x07ff07ff},{0xf707fff7,0x07ff07ff},{0xf707ff07,0x07ff07ff},{0xfff7f7f7,0x07ff07ff},{0xfff7f707,0x07ff07ff},{0xfff7ffff,0x07ff07ff},{0xfff707f7,0x07ff07ff},
+{0xfff70707,0x07ff07ff},{0xfffff7ff,0x07ff07ff},{0xfffffff7,0x07ff07ff},{0xffffffff,0x07ff07ff},{0xffffff07,0x07ff07ff},{0xffff07ff,0x07ff07ff},{0xff07f707,0x07ff07ff},{0xff07ffff,0x07ff07ff},
+{0xff07ff07,0x07ff07ff},{0xff070707,0x07ff07ff},{0x07f7f7ff,0x07ff07ff},{0x07f7fff7,0x07ff07ff},{0x07fff7f7,0x07ff07ff},{0x07fff707,0x07ff07ff},{0x07ffffff,0x07ff07ff},{0x07ff0707,0x07ff07ff},
+{0x0707f7ff,0x07ff07ff},{0x0707ff07,0x07ff07ff},{0xf7f7ffff,0x07ff0707},{0xf7ffffff,0x07ff0707},{0xf707ffff,0x07ff0707},{0xfff7fff7,0x07ff0707},{0xfff7ff07,0x07ff0707},{0xfff707ff,0x07ff0707},
+{0xfffff7f7,0x07ff0707},{0xffffffff,0x07ff0707},{0xffff07f7,0x07ff0707},{0xff07f7ff,0x07ff0707},{0x07f7ffff,0x07ff0707},{0x07fff7ff,0x07ff0707},{0x07fffff7,0x07ff0707},{0x07ffffff,0x07ff0707},
+{0x07ffff07,0x07ff0707},{0xf7f7f7f7,0x0707f7f7},{0xf7f7f707,0x0707f7f7},{0xf7f707f7,0x0707f7f7},{0xf7f70707,0x0707f7f7},{0xf7ffffff,0x0707f7f7},{0xf707f7f7,0x0707f7f7},{0xf707f707,0x0707f7f7},
+{0xf70707f7,0x0707f7f7},{0xf7070707,0x0707f7f7},{0xfff7ffff,0x0707f7f7},{0xfffff7ff,0x0707f7f7},{0xfffffff7,0x0707f7f7},{0xffffff07,0x0707f7f7},{0xffff07ff,0x0707f7f7},{0x07f7f7f7,0x0707f7f7},
+{0x07f7f707,0x0707f7f7},{0x07f707f7,0x0707f7f7},{0x07f70707,0x0707f7f7},{0x07ffffff,0x0707f7f7},{0x0707f7f7,0x0707f7f7},{0x0707f707,0x0707f7f7},{0x070707f7,0x0707f7f7},{0x07070707,0x0707f7f7},
+{0xf7f7ffff,0x0707f7ff},{0xf7f707ff,0x0707f7ff},{0xf7fff7ff,0x0707f7ff},{0xf7fffff7,0x0707f7ff},{0xf7ffff07,0x0707f7ff},{0xf7ff07ff,0x0707f7ff},{0xf7ff0707,0x0707f7ff},{0xfff7ff07,0x0707f7ff},
+{0xfff707ff,0x0707f7ff},{0xfffff7ff,0x0707f7ff},{0xffffffff,0x0707f7ff},{0xffff07f7,0x0707f7ff},{0xffff0707,0x0707f7ff},{0xff07f7ff,0x0707f7ff},{0xff07fff7,0x0707f7ff},{0x07f7ffff,0x0707f7ff},
+{0x07fff7f7,0x0707f7ff},{0x07fff707,0x0707f7ff},{0x07ffff07,0x0707f7ff},{0x07ff07ff,0x0707f7ff},{0xf7f7f707,0x0707f707},{0xf7f707f7,0x0707f707},{0xf7f70707,0x0707f707},{0xf7fff7f7,0x0707f707},
+{0xf7ff07ff,0x0707f707},{0xf707f707,0x0707f707},{0xf70707f7,0x0707f707},{0xf7070707,0x0707f707},{0xfff7ffff,0x0707f707},{0xfffff7ff,0x0707f707},{0xffffff07,0x0707f707},{0xffff07ff,0x0707f707},
+{0xff07ffff,0x0707f707},{0x07f7f7f7,0x0707f707},{0x07f7f707,0x0707f707},{0x07f707f7,0x0707f707},{0x07f70707,0x0707f707},{0x07ffffff,0x0707f707},{0x0707f7f7,0x0707f707},{0x0707f707,0x0707f707},
+{0x070707f7,0x0707f707},{0x07070707,0x0707f707},{0xf7ff07ff,0x0707fff7},{0xf707ffff,0x0707fff7},{0xfff7f7ff,0x0707fff7},{0xfff7fff7,0x0707fff7},{0xfffff7f7,0x0707fff7},{0xfffffff7,0x0707fff7},
+{0xffffffff,0x0707fff7},{0xffff07f7,0x0707fff7},{0xffff0707,0x0707fff7},{0xff07f7ff,0x0707fff7},{0xff07ffff,0x0707fff7},{0xff07ff07,0x0707fff7},{0xff0707f7,0x0707fff7},{0xff0707ff,0x0707fff7},
+{0x07f7ffff,0x0707fff7},{0xf7f7ff07,0x0707ffff},{0xf7f707ff,0x0707ffff},{0xf7fff7f7,0x0707ffff},{0xf7fff707,0x0707ffff},{0xf7ffffff,0x0707ffff},{0xf7ff07f7,0x0707ffff},{0xf707ff07,0x0707ffff},
+{0xf70707ff,0x0707ffff},{0xfff7f707,0x0707ffff},{0xfff7ffff,0x0707ffff},{0xfffff7ff,0x0707ffff},{0xfffffff7,0x0707ffff},{0xffffffff,0x0707ffff},{0xffffff07,0x0707ffff},{0xffff07ff,0x0707ffff},
+{0xff07ffff,0x0707ffff},{0xff070707,0x0707ffff},{0x07f7f7ff,0x0707ffff},{0x07f7fff7,0x0707ffff},{0x07f7ffff,0x0707ffff},{0x07f7ff07,0x0707ffff},{0x07f707ff,0x0707ffff},{0x07fff707,0x0707ffff},
+{0x07ffffff,0x0707ffff},{0x07ff07f7,0x0707ffff},{0xf7f7ffff,0x0707ff07},{0xf7fff7ff,0x0707ff07},{0xf7ffff07,0x0707ff07},{0xf7ff0707,0x0707ff07},{0xf707f7ff,0x0707ff07},{0xf707ffff,0x0707ff07},
+{0xfff7fff7,0x0707ff07},{0xfff7ff07,0x0707ff07},{0xfff70707,0x0707ff07},{0xfffff707,0x0707ff07},{0xffffffff,0x0707ff07},{0xffffff07,0x0707ff07},{0xffff07f7,0x0707ff07},{0xff07f7f7,0x0707ff07},
+{0xff07f707,0x0707ff07},{0x07f7ff07,0x0707ff07},{0x07fff7f7,0x0707ff07},{0x07ffffff,0x0707ff07},{0x07ffff07,0x0707ff07},{0x07ff07ff,0x0707ff07},{0x0707f7ff,0x0707ff07},{0x0707fff7,0x0707ff07},
+{0x0707ff07,0x0707ff07},{0xf7f7f7f7,0x070707f7},{0xf7f7f707,0x070707f7},{0xf7f707f7,0x070707f7},{0xf7f70707,0x070707f7},{0xf707f7f7,0x070707f7},{0xf707f707,0x070707f7},{0xf70707f7,0x070707f7},
+{0xf7070707,0x070707f7},{0xfffff7ff,0x070707f7},{0xfffffff7,0x070707f7},{0xffffff07,0x070707f7},{0xffff07ff,0x070707f7},{0x07f7f7f7,0x070707f7},{0x07f7f707,0x070707f7},{0x07f707f7,0x070707f7},
+{0x07f70707,0x070707f7},{0x07ffffff,0x070707f7},{0x0707f7f7,0x070707f7},{0x0707f707,0x070707f7},{0x070707f7,0x070707f7},{0x07070707,0x070707f7},{0xf7f7ffff,0x070707ff},{0xf7fffff7,0x070707ff},
+{0xf7ff07ff,0x070707ff},{0xf707f7ff,0x070707ff},{0xf707ffff,0x070707ff},{0xfff7f7ff,0x070707ff},{0xfffff7f7,0x070707ff},{0xffffffff,0x070707ff},{0xffff0707,0x070707ff},{0xff07f7ff,0x070707ff},
+{0xff07ff07,0x070707ff},{0xff0707ff,0x070707ff},{0x07fff7f7,0x070707ff},{0x07ffff07,0x070707ff},{0xf7f7f7f7,0x07070707},{0xf7f7f707,0x07070707},{0xf7f707f7,0x07070707},{0xf7f70707,0x07070707},
+{0xf707f7f7,0x07070707},{0xf707f707,0x07070707},{0xf70707f7,0x07070707},{0xf7070707,0x07070707},{0xfffff7ff,0x07070707},{0xfffffff7,0x07070707},{0xffffff07,0x07070707},{0x07f7f7f7,0x07070707},
+{0x07f7f707,0x07070707},{0x07f707f7,0x07070707},{0x07f70707,0x07070707},{0x07ffffff,0x07070707},{0x0707f7f7,0x07070707},{0x0707f707,0x07070707},{0x070707f7,0x07070707},{0x07070707,0x07070707},

src/dsv41/ds4_iq_cpu.cpp

@@ -0,0 +1,782 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_iq_cpu.cpp - a GGUF's low-bit experts on the CPU lane (see ds4_gguf_cpu.hpp): the GLM CPU lane's native
+// experts (src/kernels/cpu/native_expert.cpp), whose arithmetic is ggml-cpu's - and for IQ3_XXS / IQ3_S (Maya-S's
+// down, Maya-M's gate / up / down) rows of our own, the same numbers bit for bit, faster.
+#include "ds4_gguf_cpu.hpp"
+// Strata port: the low-bit (IQ) experts have no CPU lane here (they need Maya's native_expert split-base rows); the stub
+// branch below answers for them.  MXFP4, the source format, has its own CPU rows (ds4_gguf_cpu.cpp).
+#undef STRATA_NATIVE_EXPERTS
+#if defined(STRATA_NATIVE_EXPERTS)
+#include "strata/kernels/cpu/native_expert.hpp"  // (only with STRATA_NATIVE_EXPERTS)
+
+#define GGML_COMMON_DECL_CPP
+#define GGML_COMMON_IMPL_CPP
+#include "ggml-common.h"
+#include "ggml-cpu.h"
+#include "ggml.h"
+#endif
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#if defined(__x86_64__) || defined(_M_X64)
+#define IQ_X86 1
+#include <immintrin.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <cpuid.h>
+#endif
+#endif
+#if defined(IQ_X86) && (defined(__GNUC__) || defined(__clang__))
+#define IQ_AVX2 __attribute__((target("avx2,fma")))
+#else
+#define IQ_AVX2
+#endif
+
+namespace strata::ds4::gguf {
+
+#if defined(STRATA_NATIVE_EXPERTS)
+namespace kc = strata::dsv41k::cpu;
+namespace {
+kc::NativeFmt native(const IqCpu& f) {   // (the fields the rows read; no lookups)
+    kc::NativeFmt n;
+    n.gu_type = f.tg;
+    n.d_type = f.td;
+    n.gu_act = f.gu_act;
+    n.d_act = f.d_act;
+    n.n_embd = f.n_embd;
+    n.n_ff = f.n_ff;
+    n.gu_row = f.gu_row;
+    n.d_row = f.d_row;
+    n.up_off = f.gu_row * (size_t) f.n_ff;
+    n.down_off = 2 * n.up_off;
+    n.bytes = n.down_off + f.d_row * (size_t) f.n_embd;
+    n.act_bytes = f.act_bytes;
+    n.h_bytes = f.h_bytes;
+    return n;
+}
+
+// ---- what this CPU runs: AVX2 + FMA (the OS saving the YMM state), and AVX-512 F / BW + VBMI with BMI2 (the ZMM state)
+struct IqCpuIsa {
+    bool avx2 = false, w512 = false;
+};
+IqCpuIsa iq_isa() {
+    IqCpuIsa c;
+#if defined(IQ_X86)
+    unsigned r[4] = {0, 0, 0, 0};
+    auto cpuid = [&](unsigned leaf, unsigned sub) {
+#if defined(_MSC_VER)
+        int x[4];
+        __cpuidex(x, (int) leaf, (int) sub);
+        for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];
+#else
+        __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);
+#endif
+    };
+    cpuid(0, 0);
+    if (r[0] < 7) return c;
+    cpuid(1, 0);
+    if (!((r[2] >> 12) & 1u) || !((r[2] >> 27) & 1u) || !((r[2] >> 28) & 1u)) return c;   // FMA, OSXSAVE, AVX
+#if defined(_MSC_VER)
+    const unsigned long long xcr0 = _xgetbv(0);
+#else
+    unsigned lo = 0, hi = 0;
+    __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
+    const unsigned long long xcr0 = ((unsigned long long) hi << 32) | lo;
+#endif
+    if ((xcr0 & 0x6) != 0x6) return c;
+    cpuid(7, 0);
+    c.avx2 = (r[1] >> 5) & 1u;
+    c.w512 = c.avx2 && (xcr0 & 0xE6) == 0xE6 && ((r[1] >> 16) & 1u) && ((r[1] >> 30) & 1u) && ((r[2] >> 1) & 1u) &&
+             ((r[1] >> 8) & 1u);   // AVX-512 F, BW, VBMI; BMI2
+#endif
+    return c;
+}
+const IqCpuIsa& isa() {
+    static const IqCpuIsa c = iq_isa();
+    return c;
+}
+
+// ---- IQ3_XXS / IQ3_S rows of our own.  ggml-cpu's AVX2 dots build each 32-value chunk's grid magnitudes from eight
+// scalar table loads and inserts, and redo it for every token; here the grid comes in with one vpgatherdd from the
+// same table, the signs are built in registers (the GLM rows' sgn7_vec / sgn_vec), and every token of a call applies
+// the decoded chunk.  The arithmetic is ggml's to the bit: the same exact int32 lanes per block (lane l: the chunks'
+// grid entries l, each times its 2 s + 1), then for each block fma(d_x d_y, lanes, acc) and hsum_float_8 (IQ3_XXS:
+// times 0.25) - checked against ggml-cpu's own dot once per type (iq3_check), else ggml's rows are used.
+inline float h2f(uint16_t h) {   // F16 -> F32, exactly
+    const uint32_t s = (uint32_t) (h & 0x8000u) << 16, e = (h >> 10) & 0x1fu, m = h & 0x3ffu;
+    uint32_t b;
+    if (e == 0) {
+        const float f = (float) m * 5.9604644775390625e-8f;   // m 2^-24 (exact)
+        std::memcpy(&b, &f, 4);
+        b |= s;
+    } else if (e == 31) {
+        b = s | 0x7f800000u | (m << 13);
+    } else {
+        b = s | ((e + 112u) << 23) | (m << 13);
+    }
+    float f;
+    std::memcpy(&f, &b, 4);
+    return f;
+}
+inline uint32_t u32(const uint8_t* p) {
+    uint32_t v;
+    std::memcpy(&v, p, 4);
+    return v;
+}
+
+#if defined(IQ_X86)
+IQ_AVX2 inline float hsum_ggml(__m256 x) {   // ggml's hsum_float_8, the same additions in the same order
+    __m128 r = _mm256_extractf128_ps(x, 1);
+    r = _mm_add_ps(r, _mm256_castps256_ps128(x));
+    r = _mm_add_ps(r, _mm_movehl_ps(r, r));
+    r = _mm_add_ss(r, _mm_movehdup_ps(r));
+    return _mm_cvtss_f32(r);
+}
+// IQ2_S / IQ3_S: 32 explicit sign bits -> 32 bytes of -1 / +1 (ggml's bit selector)
+IQ_AVX2 inline __m256i sgn32(uint32_t m) {
+    const __m256i sh = _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,   //
+                                        2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);
+    const __m256i sel = _mm256_set1_epi64x((long long) 0x8040201008040201ull);
+    const __m256i b = _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) m), sh), sel);
+    return _mm256_or_si256(_mm256_cmpeq_epi8(b, sel), _mm256_set1_epi8(1));
+}
+// table entries into a vector without the shuffle port (ggml's _mm256_set_epi64x / _epi32 of table loads ends up in
+// inserts, one shuffle each; here each entry is a broadcast load and the lanes are put together by dword blends)
+IQ_AVX2 inline __m256i bq(const void* p) { return _mm256_castpd_si256(_mm256_broadcast_sd((const double*) p)); }
+IQ_AVX2 inline __m256i bd(const void* p) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*) p)); }
+IQ_AVX2 inline __m256i tab4(const uint64_t* t, unsigned a, unsigned b, unsigned c, unsigned d) {   // qwords t[a] t[b] t[c] t[d]
+    return _mm256_blend_epi32(_mm256_blend_epi32(bq(t + a), bq(t + b), 0x0C), _mm256_blend_epi32(bq(t + c), bq(t + d), 0xC0),
+                              0xF0);
+}
+IQ_AVX2 inline __m256i tab8(const uint32_t* t, const unsigned* i) {   // dwords t[i[0]] .. t[i[7]]
+    const __m256i a = _mm256_blend_epi32(bd(t + i[0]), bd(t + i[1]), 0x02), b = _mm256_blend_epi32(bd(t + i[2]), bd(t + i[3]), 0x08);
+    const __m256i c = _mm256_blend_epi32(bd(t + i[4]), bd(t + i[5]), 0x20), e = _mm256_blend_epi32(bd(t + i[6]), bd(t + i[7]), 0x80);
+    return _mm256_blend_epi32(_mm256_blend_epi32(a, b, 0x0C), _mm256_blend_epi32(c, e, 0xC0), 0xF0);
+}
+// the 8 sign bytes (-1 / +1) of each 7-bit sign index with its parity bit: ggml's keven_signs_q2xs, as +-1 bytes
+alignas(64) uint64_t g_ks64[128];
+void build_ks64() {
+    for (int i = 0; i < 128; ++i) {
+        int par = 0;
+        for (int b = 0; b < 7; ++b) par ^= (i >> b) & 1;
+        const int s = i | (par << 7);
+        uint64_t v = 0;
+        for (int j = 0; j < 8; ++j) v |= (uint64_t) (((s >> j) & 1) ? 0xFFu : 0x01u) << (8 * j);
+        g_ks64[i] = v;
+    }
+}
+IQ_AVX2 inline __m256i sgn7t(uint32_t w) {   // a chunk's four 7-bit sign indices (bits 0..27 of w)
+    return tab4(g_ks64, w & 127u, (w >> 7) & 127u, (w >> 14) & 127u, (w >> 21) & 127u);
+}
+IQ_AVX2 inline __m256i sc16(int a, int b) {   // int16 lanes 0-7 a (values 0-15), 8-15 b (values 16-31)
+    return _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_set1_epi16((short) a)), _mm_set1_epi16((short) b), 1);
+}
+bool g_gather18 = false, g_gather21 = false;   // the IQ3_XXS / IQ3_S grids by vpgatherdd (own(): if faster here)
+// IQ2_XXS's grid entry with its signs applied, for every grid index g and sign index i, at g << 7 | i (256 KB, built on
+// first use): one lookup instead of two; the products are then (q8 + 128) x signed grid less 128 x the grid's pair sums
+// (vpmaddubsw takes u8 x s8), the same integers as ggml's |grid| x signed q8
+alignas(64) uint64_t g_ct2x[256 * 128];
+bool g_ct16 = false;   // (own(): if faster here)
+void build_ct2x() {
+    for (int g = 0; g < 256; ++g)
+        for (int i = 0; i < 128; ++i) {
+            const uint64_t sg = g_ks64[i];   // (+-1 bytes)
+            uint64_t v = 0;
+            for (int j = 0; j < 8; ++j) {
+                const int gb = (int) ((iq2xxs_grid[g] >> (8 * j)) & 255u);
+                const int sb = ((sg >> (8 * j)) & 255u) == 0xFFu ? -gb : gb;
+                v |= (uint64_t) (uint8_t) (int8_t) sb << (8 * j);
+            }
+            g_ct2x[(size_t) g << 7 | (size_t) i] = v;
+        }
+}
+
+// rows [r0, r1) (out[t][r - r0]) of NT tokens; TY 16 = IQ2_XXS, 22 = IQ2_S, 18 = IQ3_XXS, 21 = IQ3_S.  A pair of
+// 32-value chunks a step: grid magnitudes g1 / g2, sign vectors s1 / s2 (-1 / +1), scales c1 / c2 - decoded once - then
+// every token: q8 times the signs, vpmaddubsw against the grid, vpmaddwd by the scales, into ggml's 8 int32 lanes
+template <int TY, int NT>
+IQ_AVX2 void iq_rows_avx2(const uint8_t* w, size_t rb, int nb, const block_q8_K* const* y, float* const* out, int r0,
+                          int r1) {
+    const __m256i sub = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
+    const bool gat = TY == 18 ? g_gather18 : TY == 21 ? g_gather21 : false;
+    const bool ct = TY == 16 && g_ct16;
+    const __m256i k80 = _mm256_set1_epi8((char) 0x80);
+    constexpr float K = TY == 18 ? 0.25f : TY == 21 ? 1.0f : 0.125f;
+    constexpr size_t BB = TY == 16 ? sizeof(block_iq2_xxs) : TY == 22 ? sizeof(block_iq2_s) :
+                          TY == 18 ? sizeof(block_iq3_xxs) : sizeof(block_iq3_s);
+    for (int r = r0; r < r1; ++r) {
+        const uint8_t* row = w + (size_t) r * rb;
+        __m256 accf[NT];
+        for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
+        for (int i = 0; i < nb; ++i) {
+            const uint8_t* blk = row + (size_t) i * BB;
+            __m256i sumi[NT];
+            for (int t = 0; t < NT; ++t) sumi[t] = _mm256_setzero_si256();
+            uint16_t dh;
+            std::memcpy(&dh, blk, 2);
+            for (int ib = 0; ib < QK_K / 32; ib += 2) {
+                __m256i g1, g2, s1, s2, c1, c2;
+                if (TY == 16) {   // IQ2_XXS: per chunk a word of 4 grid indices, a word of 4 sign indices + the scale
+                    const uint8_t* q = blk + 2 + 8 * ib;
+                    const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);
+                    if (ct) {   // (g1 / g2 the signed grids; s1 / s2 their 128 x pair sums)
+                        g1 = tab4(g_ct2x, (a0 & 255u) << 7 | (a1 & 127u), ((a0 >> 8) & 255u) << 7 | ((a1 >> 7) & 127u),
+                                  ((a0 >> 16) & 255u) << 7 | ((a1 >> 14) & 127u), (a0 >> 24) << 7 | ((a1 >> 21) & 127u));
+                        g2 = tab4(g_ct2x, (b0 & 255u) << 7 | (b1 & 127u), ((b0 >> 8) & 255u) << 7 | ((b1 >> 7) & 127u),
+                                  ((b0 >> 16) & 255u) << 7 | ((b1 >> 14) & 127u), (b0 >> 24) << 7 | ((b1 >> 21) & 127u));
+                        s1 = _mm256_maddubs_epi16(k80, g1);
+                        s2 = _mm256_maddubs_epi16(k80, g2);
+                    } else {
+                        g1 = tab4(iq2xxs_grid, a0 & 255u, (a0 >> 8) & 255u, (a0 >> 16) & 255u, a0 >> 24);
+                        g2 = tab4(iq2xxs_grid, b0 & 255u, (b0 >> 8) & 255u, (b0 >> 16) & 255u, b0 >> 24);
+                        s1 = sgn7t(a1);
+                        s2 = sgn7t(b1);
+                    }
+                    c1 = _mm256_set1_epi16((short) (2 * (a1 >> 28) + 1));
+                    c2 = _mm256_set1_epi16((short) (2 * (b1 >> 28) + 1));
+                } else if (TY == 22) {   // IQ2_S: 10-bit grid indices (qs | 2 bits of qh), explicit signs, 2 scales a chunk
+                    const uint8_t* qs = blk + 2 + 4 * ib;
+                    const uint8_t* qh = blk + 2 + QK_K / 4;
+                    const uint8_t* sg = blk + 2 + QK_K / 8 + 4 * ib;
+                    const uint8_t* scl = blk + 2 + QK_K / 4 + QK_K / 32;
+                    const unsigned h0 = qh[ib], h1 = qh[ib + 1];
+                    g1 = tab4(iq2s_grid, qs[0] | ((h0 << 8) & 0x300u), qs[1] | ((h0 << 6) & 0x300u), qs[2] | ((h0 << 4) & 0x300u),
+                              qs[3] | ((h0 << 2) & 0x300u));
+                    g2 = tab4(iq2s_grid, qs[4] | ((h1 << 8) & 0x300u), qs[5] | ((h1 << 6) & 0x300u), qs[6] | ((h1 << 4) & 0x300u),
+                              qs[7] | ((h1 << 2) & 0x300u));
+                    s1 = sgn32(u32(sg));
+                    s2 = sgn32(u32(sg + 4));
+                    c1 = sc16(2 * (scl[ib] & 0xf) + 1, 2 * (scl[ib] >> 4) + 1);
+                    c2 = sc16(2 * (scl[ib + 1] & 0xf) + 1, 2 * (scl[ib + 1] >> 4) + 1);
+                } else if (TY == 18) {   // IQ3_XXS: 8 grid indices a chunk, then the chunks' sign / scale words
+                    const uint8_t* q3 = blk + 2 + 8 * ib;
+                    const uint8_t* gas = blk + 2 + QK_K / 4 + 4 * ib;
+                    if (gat) {
+                        g1 = _mm256_i32gather_epi32((const int*) iq3xxs_grid, _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) q3)), 4);
+                        g2 = _mm256_i32gather_epi32((const int*) iq3xxs_grid,
+                                                    _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) (q3 + 8))), 4);
+                    } else {
+                        unsigned i1[8], i2[8];
+                        for (int k = 0; k < 8; ++k) {
+                            i1[k] = q3[k];
+                            i2[k] = q3[8 + k];
+                        }
+                        g1 = tab8(iq3xxs_grid, i1);
+                        g2 = tab8(iq3xxs_grid, i2);
+                    }
+                    const uint32_t a0 = u32(gas), a1 = u32(gas + 4);
+                    s1 = sgn7t(a0);
+                    s2 = sgn7t(a1);
+                    c1 = _mm256_set1_epi16((short) (2 * (a0 >> 28) + 1));
+                    c2 = _mm256_set1_epi16((short) (2 * (a1 >> 28) + 1));
+                } else {   // IQ3_S: 9-bit grid indices (qs | a bit of qh), explicit signs
+                    const uint8_t* qs = blk + 2 + 8 * ib;
+                    const uint8_t* qh = blk + 2 + QK_K / 4;
+                    const uint8_t* sg = blk + 2 + QK_K / 4 + QK_K / 32 + 4 * ib;
+                    const int sc = blk[2 + QK_K / 4 + QK_K / 32 + QK_K / 8 + ib / 2];
+                    const unsigned h0 = qh[ib], h1 = qh[ib + 1];
+                    if (gat) {
+                        const __m256i hb1 = _mm256_slli_epi32(_mm256_and_si256(_mm256_srlv_epi32(_mm256_set1_epi32((int) h0), sub), _mm256_set1_epi32(1)), 8);
+                        const __m256i hb2 = _mm256_slli_epi32(_mm256_and_si256(_mm256_srlv_epi32(_mm256_set1_epi32((int) h1), sub), _mm256_set1_epi32(1)), 8);
+                        g1 = _mm256_i32gather_epi32((const int*) iq3s_grid,
+                                                    _mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) qs)), hb1), 4);
+                        g2 = _mm256_i32gather_epi32((const int*) iq3s_grid,
+                                                    _mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) (qs + 8))), hb2), 4);
+                    } else {
+                        unsigned i1[8], i2[8];
+                        for (int k = 0; k < 8; ++k) {
+                            i1[k] = qs[k] | (((h0 >> k) & 1u) << 8);
+                            i2[k] = qs[8 + k] | (((h1 >> k) & 1u) << 8);
+                        }
+                        g1 = tab8(iq3s_grid, i1);
+                        g2 = tab8(iq3s_grid, i2);
+                    }
+                    s1 = sgn32(u32(sg));
+                    s2 = sgn32(u32(sg + 4));
+                    c1 = _mm256_set1_epi16((short) (2 * (sc & 0xf) + 1));
+                    c2 = _mm256_set1_epi16((short) (2 * (sc >> 4) + 1));
+                }
+                if (ct) {
+                    for (int t = 0; t < NT; ++t) {
+                        const int8_t* q8 = y[t][i].qs + 32 * ib;
+                        const __m256i u1 = _mm256_xor_si256(_mm256_loadu_si256((const __m256i*) q8), k80);   // (q8 + 128)
+                        const __m256i u2 = _mm256_xor_si256(_mm256_loadu_si256((const __m256i*) (q8 + 32)), k80);
+                        const __m256i d1 = _mm256_sub_epi16(_mm256_maddubs_epi16(u1, g1), s1);
+                        const __m256i d2 = _mm256_sub_epi16(_mm256_maddubs_epi16(u2, g2), s2);
+                        sumi[t] = _mm256_add_epi32(sumi[t], _mm256_add_epi32(_mm256_madd_epi16(d1, c1), _mm256_madd_epi16(d2, c2)));
+                    }
+                    continue;
+                }
+                for (int t = 0; t < NT; ++t) {
+                    const int8_t* q8 = y[t][i].qs + 32 * ib;
+                    const __m256i d1 = _mm256_maddubs_epi16(g1, _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*) q8), s1));
+                    const __m256i d2 = _mm256_maddubs_epi16(g2, _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*) (q8 + 32)), s2));
+                    sumi[t] = _mm256_add_epi32(sumi[t], _mm256_add_epi32(_mm256_madd_epi16(d1, c1), _mm256_madd_epi16(d2, c2)));
+                }
+            }
+            const float dx = h2f(dh);
+            for (int t = 0; t < NT; ++t)
+                accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(sumi[t]), accf[t]);
+        }
+        for (int t = 0; t < NT; ++t) out[t][r - r0] = K * hsum_ggml(accf[t]);
+    }
+}
+
+// ---- the same rows in 512-bit lanes (AVX-512 BW + VBMI, BMI2: Ice Lake and later Intel, Zen 4 / Zen 5), the GLM
+// lane's one-token IQ2_XXS scheme (src/kernels/cpu/iq_avx512_one.cpp) for 3-bit grids: every grid byte is one of 8
+// levels, so an entry is four 3-bit codes (a u16) and the grid's codes fit in registers - IQ3_XXS's 256 in 8 zmm,
+// IQ3_S's 512 in 16; a block's 64 indices are looked up 32 at a time with two-register word permutes and blends, each
+// 64 values' 16 codes spread to their bytes by a word permute, a multishift, an and and a byte shuffle; IQ3_XXS's signs
+// come from pdep and a parity fold, IQ3_S's are a mask register's bits.  Two chunks a register, folded to ggml's 8
+// lanes, then ggml's float steps: the same numbers to the bit (own_check).
+#if defined(__GNUC__) || defined(__clang__)
+#define IQ_512 __attribute__((target("avx512f,avx512bw,avx512vbmi,bmi2,avx2,fma")))
+#else
+#define IQ_512
+#endif
+alignas(64) uint16_t g_code3x[256], g_code3s[512];
+bool g_codes_ok = false;
+
+IQ_512 inline uint32_t signs4(uint32_t w) {   // ksigns_iq2xs for the four 7-bit fields of w: i | parity(i) << 7
+    const uint64_t s = _pdep_u64(w & 0x0FFFFFFFu, 0x7F7F7F7Full);
+    uint64_t p = s ^ ((s >> 4) & 0x0F0F0F0Full);
+    p ^= (p >> 2) & 0x03030303ull;
+    p ^= (p >> 1) & 0x01010101ull;
+    return (uint32_t) (s | ((p & 0x01010101ull) << 7));
+}
+// each grid entry as four 3-bit level codes; false if a byte is not one of the 8 levels (a grid this file does not know)
+bool build_codes3() {
+    static const int lx[8] = {4, 12, 20, 28, 36, 44, 52, 62}, ls[8] = {1, 3, 5, 7, 9, 11, 13, 15};
+    auto code = [](uint32_t e, const int* lv) -> int {
+        int c = 0;
+        for (int k = 0; k < 4; ++k) {
+            const int b = (int) ((e >> (8 * k)) & 255u);
+            int l = -1;
+            for (int j = 0; j < 8; ++j)
+                if (lv[j] == b) l = j;
+            if (l < 0) return -1;
+            c |= l << (3 * k);
+        }
+        return c;
+    };
+    for (int i = 0; i < 256; ++i) {
+        const int c = code(iq3xxs_grid[i], lx);
+        if (c < 0) return false;
+        g_code3x[i] = (uint16_t) c;
+    }
+    for (int i = 0; i < 512; ++i) {
+        const int c = code(iq3s_grid[i], ls);
+        if (c < 0) return false;
+        g_code3s[i] = (uint16_t) c;
+    }
+    return true;
+}
+IQ_512 inline __m256i fold(__m512i a) {   // a block's 16 int32 lanes -> ggml's 8 (lane k + lane k + 8), exact
+    return _mm256_add_epi32(_mm512_castsi512_si256(a), _mm512_extracti64x4_epi64(a, 1));
+}
+IQ_512 inline __m512i scales4(int s0, int s1, int s2, int s3) {   // int16 lanes 0-7 s0, 8-15 s1, 16-23 s2, 24-31 s3
+    const __m512i idx = _mm512_set_epi16(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1,   //
+                                         0, 0, 0, 0, 0, 0, 0, 0);
+    const uint64_t packed = (uint64_t) (uint16_t) s0 | ((uint64_t) (uint16_t) s1 << 16) |
+                            ((uint64_t) (uint16_t) s2 << 32) | ((uint64_t) (uint16_t) s3 << 48);
+    return _mm512_permutexvar_epi16(idx, _mm512_set1_epi64((long long) packed));
+}
+// the codes of 32 word indices: 64 entries a register pair, NP pairs, blended by the index's bits 6, 7 (and 8)
+template <int NP>
+IQ_512 inline __m512i lookup(const __m512i* T, __m512i idx) {
+    __m512i r[NP];
+    for (int j = 0; j < NP; ++j) r[j] = _mm512_permutex2var_epi16(T[2 * j], idx, T[2 * j + 1]);
+    const __mmask32 m6 = _mm512_test_epi16_mask(idx, _mm512_set1_epi16(64));
+    const __mmask32 m7 = _mm512_test_epi16_mask(idx, _mm512_set1_epi16(128));
+    __m512i a = _mm512_mask_blend_epi16(m7, _mm512_mask_blend_epi16(m6, r[0], r[1]), _mm512_mask_blend_epi16(m6, r[2], r[3]));
+    if (NP == 8) {
+        const __mmask32 m8 = _mm512_test_epi16_mask(idx, _mm512_set1_epi16(256));
+        const __m512i b = _mm512_mask_blend_epi16(m7, _mm512_mask_blend_epi16(m6, r[NP > 4 ? 4 : 0], r[NP > 5 ? 5 : 0]),
+                                                  _mm512_mask_blend_epi16(m6, r[NP > 6 ? 6 : 0], r[NP > 7 ? 7 : 0]));
+        a = _mm512_mask_blend_epi16(m8, a, b);
+    }
+    return a;
+}
+
+template <int TY, int NT>
+IQ_512 void iq3_rows_512(const uint8_t* w, size_t rb, int nb, const block_q8_K* const* y, float* const* out, int r0,
+                         int r1) {
+    constexpr int NTAB = TY == 18 ? 8 : 16;
+    constexpr size_t BB = TY == 18 ? sizeof(block_iq3_xxs) : sizeof(block_iq3_s);
+    __m512i T[NTAB];
+    for (int k = 0; k < NTAB; ++k)
+        T[k] = _mm512_load_si512((const void*) ((TY == 18 ? g_code3x : g_code3s) + 32 * k));
+    // code -> grid byte (IQ3_XXS 4 .. 62, IQ3_S 1 .. 15) in every 128-bit lane; byte b of an output qword <- 3 bits at
+    // [0, 3, 6, 9] (its first entry's code, word 0) or [32, 35, 38, 41] (the second's, word 2)
+    const __m512i lut = TY == 18 ? _mm512_set4_epi32(0, 0, 0x3E342C24, 0x1C140C04) : _mm512_set4_epi32(0, 0, 0x0F0D0B09, 0x07050301);
+    const __m512i ctl = _mm512_set1_epi64(0x2926232009060300LL);
+    const __m512i seven = _mm512_set1_epi8(7), zero = _mm512_setzero_si512();
+    __m512i pidx[2];   // a 64-value group h of 32 codes: output qword m <- entries 16 h + 2 m (word 0), + 1 (word 2)
+    for (int h = 0; h < 2; ++h) {
+        alignas(64) uint16_t p[32];
+        for (int m = 0; m < 8; ++m) {
+            p[4 * m] = p[4 * m + 1] = (uint16_t) (16 * h + 2 * m);
+            p[4 * m + 2] = p[4 * m + 3] = (uint16_t) (16 * h + 2 * m + 1);
+        }
+        pidx[h] = _mm512_load_si512((const void*) p);
+    }
+    for (int r = r0; r < r1; ++r) {
+        const uint8_t* row = w + (size_t) r * rb;
+        __m256 accf[NT];
+        for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
+        for (int i = 0; i < nb; ++i) {
+            const uint8_t* blk = row + (size_t) i * BB;
+            __m512i codes[2];
+            if (TY == 18) {
+                const __m512i q = _mm512_loadu_si512((const void*) (blk + 2));
+                codes[0] = lookup<4>(T, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(q)));
+                codes[1] = lookup<4>(T, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(q, 1)));
+            } else {
+                const uint8_t* qh = blk + 2 + QK_K / 4;
+                const __m512i hi = _mm512_set1_epi16(256);
+                const __m512i ia = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*) (blk + 2)));
+                const __m512i ib = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*) (blk + 34)));
+                // (index k of chunk c: qs[8 c + k] | bit k of qh[c] << 8 - the 32 bits of qh[0..3] are chunks 0..3's)
+                codes[0] = lookup<8>(T, _mm512_or_si512(ia, _mm512_maskz_mov_epi16((__mmask32) u32(qh), hi)));
+                codes[1] = lookup<8>(T, _mm512_or_si512(ib, _mm512_maskz_mov_epi16((__mmask32) u32(qh + 4), hi)));
+            }
+            __m512i acc[NT];
+            for (int t = 0; t < NT; ++t) acc[t] = _mm512_setzero_si512();
+            for (int p = 0; p < 4; ++p) {
+                const __m512i rep = _mm512_permutexvar_epi16(pidx[p & 1], codes[p >> 1]);
+                const __m512i g = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_multishift_epi64_epi8(ctl, rep), seven));
+                uint64_t sm;
+                int sa, sb;
+                if (TY == 18) {
+                    const uint8_t* sw = blk + 2 + QK_K / 4 + 8 * p;
+                    const uint32_t wa = u32(sw), wb = u32(sw + 4);
+                    sm = (uint64_t) signs4(wa) | ((uint64_t) signs4(wb) << 32);
+                    sa = 2 * (int) (wa >> 28) + 1;
+                    sb = 2 * (int) (wb >> 28) + 1;
+                } else {
+                    std::memcpy(&sm, blk + 2 + QK_K / 4 + QK_K / 32 + 8 * p, 8);
+                    const int sc = blk[2 + QK_K / 4 + QK_K / 32 + QK_K / 8 + p];
+                    sa = 2 * (sc & 0xf) + 1;
+                    sb = 2 * (sc >> 4) + 1;
+                }
+                const __mmask64 m = _cvtu64_mask64(sm);
+                const __m512i S = scales4(sa, sa, sb, sb);
+                for (int t = 0; t < NT; ++t) {
+                    const __m512i yv = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * p));
+                    const __m512i ys = _mm512_mask_sub_epi8(yv, m, zero, yv);
+                    acc[t] = _mm512_add_epi32(acc[t], _mm512_madd_epi16(_mm512_maddubs_epi16(g, ys), S));
+                }
+            }
+            uint16_t dh;
+            std::memcpy(&dh, blk, 2);
+            const float dx = h2f(dh);
+            for (int t = 0; t < NT; ++t)
+                accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(fold(acc[t])), accf[t]);
+        }
+        for (int t = 0; t < NT; ++t) out[t][r - r0] = (TY == 18 ? 0.25f : 1.0f) * hsum_ggml(accf[t]);
+    }
+}
+
+template <int TY>
+void iq_rows_n(const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {
+    static const bool w512 = [] {
+        const char* v = std::getenv("DS4_IQ_AVX2");   // (=1: the AVX2 rows on an AVX-512 CPU, to compare)
+        return (TY == 18 || TY == 21) && g_codes_ok && isa().w512 && !(v != nullptr && std::atoi(v) != 0);
+    }();
+    const int nb = n / QK_K;
+    for (int t0 = 0; t0 < nt; t0 += 4) {
+        const block_q8_K* y[4];
+        float* o[4];
+        const int c = nt - t0 < 4 ? nt - t0 : 4;
+        for (int t = 0; t < c; ++t) {
+            y[t] = (const block_q8_K*) act[t0 + t];
+            o[t] = out[t0 + t];
+        }
+        if constexpr (TY == 18 || TY == 21) {
+            if (w512) {
+                switch (c) {
+                    case 1: iq3_rows_512<TY, 1>(w, rb, nb, y, o, r0, r1); break;
+                    case 2: iq3_rows_512<TY, 2>(w, rb, nb, y, o, r0, r1); break;
+                    case 3: iq3_rows_512<TY, 3>(w, rb, nb, y, o, r0, r1); break;
+                    default: iq3_rows_512<TY, 4>(w, rb, nb, y, o, r0, r1); break;
+                }
+                continue;
+            }
+        }
+        switch (c) {
+            case 1: iq_rows_avx2<TY, 1>(w, rb, nb, y, o, r0, r1); break;
+            case 2: iq_rows_avx2<TY, 2>(w, rb, nb, y, o, r0, r1); break;
+            case 3: iq_rows_avx2<TY, 3>(w, rb, nb, y, o, r0, r1); break;
+            default: iq_rows_avx2<TY, 4>(w, rb, nb, y, o, r0, r1); break;
+        }
+    }
+}
+#endif  // IQ_X86
+
+// out[t][r - r0] for the rows of `type` (an own type only)
+void own_rows(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0,
+              int r1) {
+#if defined(IQ_X86)
+    if (type == GGML_TYPE_IQ2_XXS) return iq_rows_n<16>(w, rb, n, act, nt, out, r0, r1);
+    if (type == GGML_TYPE_IQ2_S) return iq_rows_n<22>(w, rb, n, act, nt, out, r0, r1);
+    if (type == GGML_TYPE_IQ3_XXS) return iq_rows_n<18>(w, rb, n, act, nt, out, r0, r1);
+    if (type == GGML_TYPE_IQ3_S) return iq_rows_n<21>(w, rb, n, act, nt, out, r0, r1);
+#endif
+    (void) type, (void) w, (void) rb, (void) n, (void) act, (void) nt, (void) out, (void) r0, (void) r1;
+}
+
+// our rows against ggml-cpu's dot, bit for bit: random blocks of the type (its F16 scale a small normal value), Q8_K
+// activations from ggml's quantizer, 1, 2, 3, 5 and 9 tokens, 13 rows of 8 blocks
+bool own_check(int type) {
+    ggml_cpu_init();
+    const ggml_type t = (ggml_type) type;
+    const int n = 2048, rows = 13;
+    const size_t rb = ggml_row_size(t, n), bb = ggml_type_size(t), ab = ggml_row_size(GGML_TYPE_Q8_K, n);
+    uint64_t st = 0x9E3779B97F4A7C15ull;
+    auto next = [&st] {
+        st ^= st << 13;
+        st ^= st >> 7;
+        st ^= st << 17;
+        return st;
+    };
+    std::vector<uint8_t> w(rb * rows);
+    for (auto& b : w) b = (uint8_t) next();
+    for (size_t o = 0; o < w.size(); o += bb) {
+        const ggml_fp16_t d = ggml_fp32_to_fp16(0.002f + 0.001f * (float) (next() % 16));
+        std::memcpy(&w[o], &d, sizeof d);
+    }
+    const auto* q8 = ggml_get_type_traits_cpu(GGML_TYPE_Q8_K);
+    const auto dot = ggml_get_type_traits_cpu(t)->vec_dot;
+    for (int nt : {1, 2, 3, 5, 9}) {
+        std::vector<uint8_t> a(ab * (size_t) nt);
+        std::vector<float> x((size_t) n), got((size_t) nt * rows), ref((size_t) nt * rows);
+        std::vector<const void*> ap((size_t) nt);
+        std::vector<float*> op((size_t) nt);
+        for (int k = 0; k < nt; ++k) {
+            for (auto& v : x) v = ((float) (next() % 2001) - 1000.0f) * 1e-3f;
+            q8->from_float(x.data(), a.data() + ab * (size_t) k, n);
+            ap[(size_t) k] = a.data() + ab * (size_t) k;
+            op[(size_t) k] = got.data() + (size_t) k * rows;
+        }
+        own_rows(type, w.data(), rb, n, ap.data(), nt, op.data(), 0, rows);
+        for (int k = 0; k < nt; ++k)
+            for (int r = 0; r < rows; ++r) dot(n, &ref[(size_t) k * rows + r], 0, w.data() + (size_t) r * rb, 0, ap[(size_t) k], 0, 1);
+        if (std::memcmp(got.data(), ref.data(), got.size() * sizeof(float)) != 0) return false;
+    }
+    return true;
+}
+
+// the time (us, best of 5 rounds) of 32 rows of 16 blocks of the type against one token: ours with the IQ3 grids as
+// broadcasts + blends (variant 1) or by vpgatherdd (2), or the native lane's rows (0: the GLM lane's / ggml-cpu's)
+double time_variant(int type, int variant) {
+    const ggml_type t = (ggml_type) type;
+    const int n = 4096, rows = 32;
+    const size_t rb = ggml_row_size(t, n), bb = ggml_type_size(t), ab = ggml_row_size(GGML_TYPE_Q8_K, n);
+    std::vector<uint8_t> w(rb * rows), a(ab);
+    uint32_t s = 12345u;
+    for (auto& b : w) b = (uint8_t) ((s = s * 1664525u + 1013904223u) >> 24);
+    for (size_t o = 0; o < w.size(); o += bb) {
+        const ggml_fp16_t d = ggml_fp32_to_fp16(0.003f);
+        std::memcpy(&w[o], &d, sizeof d);
+    }
+    std::vector<float> x((size_t) n), y((size_t) rows);
+    for (auto& v : x) v = (float) ((int) ((s = s * 1664525u + 1013904223u) >> 16) - 32768) / 32768.0f;
+    ggml_get_type_traits_cpu(GGML_TYPE_Q8_K)->from_float(x.data(), a.data(), n);
+    const void* ap[1] = {a.data()};
+    float* yp[1] = {y.data()};
+    kc::NativeFmt nf;   // (the native rows timed as a gate / up pair of the same rows: 2 x rows dots, as ours below)
+    std::string err;
+    if (variant == 0 && !kc::native_fmt(type, type, n, QK_K, nf, err)) return 1e30;
+    bool* gflag = type == GGML_TYPE_IQ3_XXS ? &g_gather18 : type == GGML_TYPE_IQ3_S ? &g_gather21 :
+                  type == GGML_TYPE_IQ2_XXS ? &g_ct16 : nullptr;
+    const bool keep = gflag != nullptr && *gflag;
+    if (gflag != nullptr) *gflag = variant >= 2;
+    double best = 1e30;
+    for (int round = 0; round < 6; ++round) {
+        const auto t0 = std::chrono::steady_clock::now();
+        for (int rep = 0; rep < 4; ++rep) {
+            if (variant == 0) {
+                kc::native_gu_rows_split(nf, w.data(), w.data(), ap, 1, yp, 0, rows, std::numeric_limits<float>::infinity());
+            } else {
+                own_rows(type, w.data(), rb, n, ap, 1, yp, 0, rows);
+                own_rows(type, w.data(), rb, n, ap, 1, yp, 0, rows);
+            }
+        }
+        const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+        if (round > 0) best = std::min(best, us);
+    }
+    if (gflag != nullptr) *gflag = keep;
+    return best;
+}
+
+// our rows for this type, here: IQ3_XXS / IQ3_S on AVX2 or AVX-512 VBMI, IQ2_XXS / IQ2_S on AVX2 - checked once
+// against ggml-cpu's dot (each variant), then the fastest of ours and the native lane's measured on this CPU (they
+// give the same bits, so the choice changes no answer).  DS4_IQ_OWN=0: the native lane's rows; =1: ours without
+// measuring (DS4_IQ_GATHER=1: the IQ3 grids by vpgatherdd); DS4_IQ_AVX2=1: our AVX2 rows on an AVX-512 CPU too.
+bool own(int type) {
+#if defined(IQ_X86)
+    int k;
+    switch (type) {
+        case GGML_TYPE_IQ3_XXS: k = 0; break;
+        case GGML_TYPE_IQ3_S: k = 1; break;
+        case GGML_TYPE_IQ2_XXS: k = 2; break;
+        case GGML_TYPE_IQ2_S: k = 3; break;
+        default: return false;
+    }
+    static int mode = -1;   // DS4_IQ_OWN: -1 measured, 0 never, 1 always
+    static bool force_avx2 = false, force_gather = false;
+    static const bool allowed = [] {
+        const char* v = std::getenv("DS4_IQ_OWN");
+        mode = v == nullptr ? -1 : std::atoi(v) != 0 ? 1 : 0;
+        const char* a2 = std::getenv("DS4_IQ_AVX2");
+        force_avx2 = a2 != nullptr && std::atoi(a2) != 0;
+        const char* gt = std::getenv("DS4_IQ_GATHER");
+        force_gather = gt != nullptr && std::atoi(gt) != 0;
+        g_codes_ok = build_codes3();   // (the 512-bit rows' tables; without them those rows are not used)
+        build_ks64();
+        build_ct2x();
+        return mode != 0 && isa().avx2;
+    }();
+    // (IQ2 on an AVX-512 CPU: the GLM lane's AVX-512 rows serve them, unless DS4_IQ_AVX2)
+    if (!allowed || (k >= 2 && isa().w512 && !force_avx2)) return false;
+    static std::once_flag once[4];
+    static bool ok[4];
+    std::call_once(once[k], [&] {
+        bool* gflag = k == 0 ? &g_gather18 : k == 1 ? &g_gather21 : k == 2 ? &g_ct16 : nullptr;   // (the second form)
+        bool good = own_check(type);   // (the blend form; the 512-bit rows where they serve)
+        bool good_g = false;
+        if (gflag != nullptr) {
+            *gflag = true;
+            good_g = own_check(type);
+            *gflag = false;
+        }
+        if (mode == 1) {
+            if (gflag != nullptr) *gflag = force_gather && good_g;
+            ok[k] = good || (force_gather && good_g);
+            return;
+        }
+        // the variants timed in turns, the best of each kept (other work on the machine - the load's read-ahead, the
+        // prompt - then hits them alike), and ours taken only when clearly faster (10 %): the choice changes no answer,
+        // a wrong one costs time (Uranus' Golden Cove: one-shot timings 30-80 % apart run to run, the choice flipping)
+        double tn = 1e30, tb = 1e30, tg = 1e30;
+        for (int it = 0; it < 5; ++it) {
+            tn = std::min(tn, time_variant(type, 0));
+            if (good) tb = std::min(tb, time_variant(type, 1));
+            if (good_g) tg = std::min(tg, time_variant(type, 2));
+        }
+        if (gflag != nullptr) *gflag = tg < tb;
+        ok[k] = std::min(tb, tg) < 0.9 * tn;
+        if (std::getenv("DS4_IQ_VERBOSE") != nullptr)
+            std::fprintf(stderr, "ds4: CPU lane %s rows: native %.0f us, ours %.0f us%s -> %s\n", ggml_type_name((ggml_type) type),
+                         tn, tb, gflag != nullptr ? (std::string(" (second form: gather / one table ") + std::to_string((int) tg) + " us)").c_str() : "",
+                         ok[k] ? (gflag != nullptr && *gflag ? "ours (second form)" : "ours") : "native");
+    });
+    return ok[k];
+#else
+    (void) type;
+    return false;
+#endif
+}
+}  // namespace
+
+bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
+    kc::NativeFmt n;
+    if (!kc::native_fmt(tg, td, n_embd, n_ff, n, err)) return false;
+    f.tg = tg;
+    f.td = td;
+    f.n_embd = n_embd;
+    f.n_ff = n_ff;
+    f.gu_act = n.gu_act;
+    f.d_act = n.d_act;
+    f.gu_row = n.gu_row;
+    f.d_row = n.d_row;
+    f.act_bytes = n.act_bytes;
+    f.h_bytes = n.h_bytes;
+    return true;
+}
+
+void iq_cpu_quant(int act, const float* x, int k, void* dst) {
+    kc::NativeFmt n;
+    n.gu_act = act;
+    n.n_embd = k;
+    kc::native_quant_act(n, x, dst);
+}
+
+void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt,
+                    float* const* ff, int r0, int r1, float limit) {
+    if (r1 > r0 && own(f.tg)) {   // (native_gu_rows_split's epilogue, on the same values)
+        const int nr = r1 - r0;
+        thread_local std::vector<float> buf;
+        thread_local std::vector<float*> go, uo;
+        buf.resize((size_t) 2 * nt * nr);
+        go.resize((size_t) nt);
+        uo.resize((size_t) nt);
+        for (int t = 0; t < nt; ++t) {
+            go[(size_t) t] = buf.data() + (size_t) t * nr;
+            uo[(size_t) t] = buf.data() + (size_t) (nt + t) * nr;
+        }
+        own_rows(f.tg, gate, f.gu_row, f.n_embd, act, nt, go.data(), r0, r1);
+        own_rows(f.tg, up, f.gu_row, f.n_embd, act, nt, uo.data(), r0, r1);
+        for (int t = 0; t < nt; ++t)
+            for (int r = 0; r < nr; ++r) {
+                float g = go[(size_t) t][r], u = uo[(size_t) t][r];
+                if (limit > 0.0f) {
+                    g = std::fmin(g, limit);
+                    u = std::fmin(std::fmax(u, -limit), limit);
+                }
+                ff[t][r0 + r] = (g / (1.f + std::exp(-g))) * u;
+            }
+        return;
+    }
+    kc::native_gu_rows_split(native(f), gate, up, act, nt, ff, r0, r1,
+                             limit > 0.0f ? limit : std::numeric_limits<float>::infinity());
+}
+
+void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
+    if (r1 > r0 && own(f.td)) {
+        thread_local std::vector<float*> o;
+        o.resize((size_t) nt);
+        for (int t = 0; t < nt; ++t) o[(size_t) t] = out[t] + r0;
+        own_rows(f.td, down, f.d_row, f.n_ff, hq, nt, o.data(), r0, r1);
+        return;
+    }
+    kc::native_down_rows_split(native(f), down, hq, nt, out, r0, r1);
+}
+
+bool iq_cpu_own_rows(int type) { return own(type); }
+#else
+// (no ggml-cpu lane for the low-bit types here; the trellis types have their own: ds4_tq_cpu.cpp)
+bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
+    if (tq_cpu_type(tg) && tq_cpu_type(td)) return tq_cpu_fmt(tg, td, n_embd, n_ff, f, err);
+    if (gg_cpu_type(tg) && gg_cpu_type(td)) return gg_cpu_fmt(tg, td, n_embd, n_ff, f, err);
+    err = "this build has no ggml-cpu (STRATA_NATIVE_EXPERTS=OFF): the low-bit experts CPU lane is not in it";
+    return false;
+}
+void iq_cpu_quant(int act, const float* x, int k, void* dst) {
+    if (act == tq_cpu_act()) tq_cpu_quant(x, k, dst);
+    else gg_cpu_quant(act, x, k, dst);
+}
+void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
+                    int r0, int r1, float limit) {
+    if (tq_cpu_type(f.tg)) tq_cpu_gate_up(f, gate, up, act, nt, ff, r0, r1, limit);
+    else gg_cpu_gate_up(f, gate, up, act, nt, ff, r0, r1, limit);
+}
+void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
+    if (tq_cpu_type(f.td)) tq_cpu_down(f, down, hq, nt, out, r0, r1);
+    else gg_cpu_down(f, down, hq, nt, out, r0, r1);
+}
+bool iq_cpu_own_rows(int) { return false; }
+#endif
+
+}  // namespace strata::ds4::gguf

src/dsv41/ds4_iq_hip.inc

@@ -0,0 +1,274 @@
+// Strata. The prompt sets of ds4_iq.cu on the gfx12 (RDNA4, R9700 = gfx1201) matrix cores: v_wmma_f32_16x16x16_f16, F16 operands,
+// F32 accumulation.  Included by ds4_iq.cu inside its anonymous namespace (it uses its Ty, decode<>, load_tab<>, swiglu, SetArgs).
+//
+// The idea is the CUDA prompt tiles' (set_h16ws_kernel on f-dsv41-cuk-int2): a block-tile of the expert's rows x weight rows, K in
+// steps of 64; each i-quant block is decoded ONCE into shared memory as F16 (code * the sub-block's scale: the product is exact up
+// to the F16 rounding of the scale and the product) and then shared by every row of the tile; x as F16 rows next to it, both
+// double-buffered, one barrier a step.  The F32 FMA tile of today (set_kernel) decodes the weights once per 64 rows and does
+// scalar FMAs.  Default ON for gfx12, STRATA_DS4_HIPK=0 switches it off (gfx12 only, the types below, experts of more than 16 rows).
+#pragma once
+#if defined(STRATA_USE_HIP)
+
+namespace hipk {
+constexpr int kStH = 72;   // halves a shared row: 64 + 8 (the pad spreads the banks; rows stay 16-byte aligned)
+using hv2 = _Float16 __attribute__((ext_vector_type(2)));
+using hv8 = _Float16 __attribute__((ext_vector_type(8)));
+using fv8 = float __attribute__((ext_vector_type(8)));
+
+template <int T>
+constexpr bool kTypes = T == kIQ2_XXS || T == kIQ2_XS || T == kIQ2_S || T == kIQ3_XXS || T == kIQ3_S || T == kIQ4_XS ||
+                        T == kIQ1_S || T == kIQ1_M || T == kIQ4_NL;
+
+__device__ __forceinline__ fv8 wmma16(hv8 a, hv8 b, fv8 c) {
+#if defined(__gfx1200__) || defined(__gfx1201__)
+    return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, b, c);
+#else
+    return c;
+#endif
+}
+__device__ __forceinline__ uint32_t as_u32(hv2 v) { return __builtin_bit_cast(uint32_t, v); }
+__device__ __forceinline__ hv2 as_hv2(uint32_t v) { return __builtin_bit_cast(hv2, v); }
+// 4 int8 codes (a word) -> 4 halves times the scale: 1024 + (code + 128) is exact in a half's mantissa, less 1152
+__device__ __forceinline__ uint2 codes4(int w, hv2 sc) {
+    const uint32_t u = (uint32_t) w ^ 0x80808080u;
+    const uint32_t lo = __byte_perm(u, 0x64646464u, 0x5140), hi = __byte_perm(u, 0x64646464u, 0x5342);
+    const hv2 m = {(_Float16) 1152.0f, (_Float16) 1152.0f};
+    const hv2 a = (as_hv2(lo) - m) * sc, b = (as_hv2(hi) - m) * sc;
+    return make_uint2(as_u32(a), as_u32(b));
+}
+
+// BM rows x BN weight rows (a matrix: gate and up: two) per block, 8 waves WM x WN, each wave (BM/WM) x (BN/WN)
+template <int T, bool GU, int BM, int BN, int WM, int WN>
+__global__ void __launch_bounds__(256) set_kernel(const SetArgs a, int dbg) {
+    static_assert(WM * WN == 8, "8 waves");
+    constexpr int NM = GU ? 2 : 1, MT = BM / WM / 16, NT = BN / WN / 16;
+    static_assert(MT >= 1 && NT >= 1 && MT * WM * 16 == BM && NT * WN * 16 == BN, "wave tiling");
+    __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+    __shared__ int srow[BM];
+    __shared__ __align__(16) _Float16 xs[2][BM][kStH];
+    __shared__ __align__(16) _Float16 ws[2][NM][BN][kStH];
+    const int e = blockIdx.z;
+    const int lo = a.bounds[e] - a.row0, hi_r = a.bounds[e + 1] - a.row0;
+    const int r0 = lo + blockIdx.x * BM;
+    if (r0 >= hi_r || hi_r - lo <= a.skip_le) return;   // (uniform)
+    load_tab<T>(tab);
+    const int nr = min(BM, hi_r - r0);
+    const int c0 = blockIdx.y * BN;
+    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+    const int K = GU ? a.n_embd : a.n_ff;
+    const size_t rb = row_bytes_t<T>(K);
+    const int tid = (int) threadIdx.x, warp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;
+    if (tid < BM) srow[tid] = GU ? a.idx[r0 + min(tid, nr - 1)] : r0 + min(tid, nr - 1);
+    __syncthreads();
+    const int nst = K / 64;
+
+    // a step's operands are fetched and decoded into registers (the global loads and the codebook's lookups) while the matrix
+    // cores work on the step before, and written to shared memory after them
+    constexpr int NI = (NM * BN * 2 + 255) / 256;                // weight items a thread
+    constexpr int NX = GU ? (BM * 16 + 255) / 256 : (BM * 8 + 255) / 256;   // row chunks a thread
+    struct Regs {
+        int q[NI][8];
+        _Float16 sc0[NI], sc1[NI];
+        float4 xv[GU ? NX : 1];
+        uint4 xu[GU ? 1 : NX];
+    };
+    auto fetch = [&](int s, Regs& R) {
+#pragma unroll
+        for (int ii = 0; ii < NI; ++ii) {
+            const int it = tid + ii * 256;
+            if (it < NM * BN * 2) {
+                const int kbl = it & 1, col = (it >> 1) % BN, mat = it / (2 * BN);
+                Sub sb;
+                if (dbg & 2) { for (int w = 0; w < 8; ++w) sb.q[w] = tid + w; sb.d = 1.0f; sb.s0 = sb.s1 = 1; } else
+                decode<T>(blob + a.tr[GU ? mat : 2] + (size_t) (c0 + col) * rb, s * 2 + kbl, tab, sb);
+#pragma unroll
+                for (int w = 0; w < 8; ++w) R.q[ii][w] = sb.q[w];
+                R.sc0[ii] = (_Float16) (sb.d * (float) sb.s0);
+                R.sc1[ii] = (_Float16) (sb.d * (float) sb.s1);
+            }
+        }
+#pragma unroll
+        for (int xi = 0; xi < NX; ++xi) {
+            const int c = tid + xi * 256;
+            if constexpr (GU) {
+                if (c < BM * 16) {
+                    const int row = c >> 4, ch = c & 15;
+                    R.xv[xi] = *(const float4*) (a.xf + (size_t) srow[row] * a.ldx + (size_t) s * 64 + ch * 4);
+                }
+            } else {
+                if (c < BM * 8) {
+                    const int row = c >> 3, ch = c & 7;
+                    R.xu[xi] = *(const uint4*) (a.xh + (size_t) srow[row] * a.ldx + (size_t) s * 64 + ch * 8);
+                }
+            }
+        }
+    };
+    auto store = [&](const Regs& R, int b) {
+#pragma unroll
+        for (int ii = 0; ii < NI; ++ii) {
+            const int it = tid + ii * 256;
+            if (it < NM * BN * 2) {
+                const int kbl = it & 1, col = (it >> 1) % BN, mat = it / (2 * BN);
+                const hv2 h0 = {R.sc0[ii], R.sc0[ii]}, h1 = {R.sc1[ii], R.sc1[ii]};
+                uint2* dst = (uint2*) &ws[b][mat][col][kbl * 32];   // (values 4 w .. 4 w + 3 of the half at dst[w], 16 + 4 w .. at dst[4 + w])
+#pragma unroll
+                for (int w = 0; w < 4; ++w) {
+                    dst[w] = codes4(R.q[ii][w], h0);
+                    dst[4 + w] = codes4(R.q[ii][4 + w], h1);
+                }
+            }
+        }
+#pragma unroll
+        for (int xi = 0; xi < NX; ++xi) {
+            const int c = tid + xi * 256;
+            if constexpr (GU) {
+                if (c < BM * 16) {
+                    const int row = c >> 4, ch = c & 15;
+                    const float4 v = R.xv[xi];
+                    const hv2 p = {(_Float16) v.x, (_Float16) v.y}, q = {(_Float16) v.z, (_Float16) v.w};
+                    *(uint2*) &xs[b][row][ch * 4] = make_uint2(as_u32(p), as_u32(q));
+                }
+            } else {
+                if (c < BM * 8) {
+                    const int row = c >> 3, ch = c & 7;
+                    *(uint4*) &xs[b][row][ch * 8] = R.xu[xi];
+                }
+            }
+        }
+    };
+
+    const int wm = warp / WN, wn = warp % WN;
+    fv8 acc[NM][MT][NT];
+#pragma unroll
+    for (int m = 0; m < NM; ++m)
+#pragma unroll
+        for (int i = 0; i < MT; ++i)
+#pragma unroll
+            for (int j = 0; j < NT; ++j) acc[m][i][j] = fv8{0, 0, 0, 0, 0, 0, 0, 0};
+
+    // read-ahead: the first and last byte of the weights' block and a word of the rows' line of a step PF steps on, summed into a
+    // value that is never used (the loads only warm the caches; their latency is not waited for until the end)
+    int touched = 0;
+    auto touch = [&](int s) {
+        if (dbg & 4) return;
+#pragma unroll
+        for (int ii = 0; ii < NI; ++ii) {
+            const int it = tid + ii * 256;
+            if (it < NM * BN * 2) {
+                const int col = (it >> 1) % BN, mat = it / (2 * BN);
+                const uint8_t* p = blob + a.tr[GU ? mat : 2] + (size_t) (c0 + col) * rb + (size_t) ((s * 2) >> 3) * Ty<T>::B;
+                touched += p[0] + p[Ty<T>::B - 1];
+            }
+        }
+        if constexpr (GU) {
+            for (int c = tid; c < BM * 2; c += 256) touched += (int) a.xf[(size_t) srow[c >> 1] * a.ldx + (size_t) s * 64 + (c & 1) * 32];
+        } else {
+            for (int c = tid; c < BM * 2; c += 256) touched += (int) a.xh[(size_t) srow[c >> 1] * a.ldx + (size_t) s * 64 + (c & 1) * 32];
+        }
+    };
+    constexpr int PF = 4;
+    for (int s = 1; s <= PF && s < nst; ++s) touch(s + 1 < nst ? s + 1 : s);
+    {
+        Regs R0;
+        fetch(0, R0);
+        store(R0, 0);
+    }
+    __syncthreads();
+    for (int s = 0; s < nst; ++s) {
+        const int b = s & 1;
+        Regs R;
+        if (s + 1 < nst) fetch(s + 1, R);
+        if (s + 1 + PF < nst) touch(s + 1 + PF);
+#pragma unroll
+        for (int kk = 0; kk < 4; ++kk) {
+            hv8 af[MT];
+#pragma unroll
+            for (int i = 0; i < MT; ++i) af[i] = *(const hv8*) &xs[b][wm * (16 * MT) + i * 16 + l16][kk * 16 + hi * 8];
+#pragma unroll
+            for (int m = 0; m < NM; ++m)
+#pragma unroll
+                for (int j = 0; j < NT; ++j) {
+                    const hv8 bf = *(const hv8*) &ws[b][m][wn * (16 * NT) + j * 16 + l16][kk * 16 + hi * 8];
+#pragma unroll
+                    for (int i = 0; i < MT; ++i)
+                        if (!(dbg & 1) && wm * (16 * MT) + i * 16 < nr) acc[m][i][j] = wmma16(af[i], bf, acc[m][i][j]);   // (a tile past the rows: skipped, uniform)
+                }
+        }
+        if (s + 1 < nst) store(R, b ^ 1);
+        __syncthreads();
+    }
+    if (touched == 0x7fffffff && tid == 12345) a.y[0] = 0.0f;
+    // D[row = 8 hi + i][col = l16] of every tile
+#pragma unroll
+    for (int i = 0; i < MT; ++i)
+#pragma unroll
+        for (int q = 0; q < 8; ++q) {
+            const int row = wm * (16 * MT) + i * 16 + 8 * hi + q;
+            if (row >= nr) continue;
+            const size_t lr = (size_t) (r0 + row);
+#pragma unroll
+            for (int j = 0; j < NT; ++j) {
+                const int col = c0 + wn * (16 * NT) + j * 16 + l16;
+                if (GU) a.h[lr * a.ldo + col] = __half_as_ushort(__float2half_rn(swiglu(acc[0][i][j][q], acc[1][i][j][q], a.limit)));
+                else a.y[lr * a.ldo + col] = acc[0][i][j][q];
+            }
+        }
+}
+
+#include "ds4_iq_hip_r.inc"
+
+inline bool ok_device() {   // gfx12 unless STRATA_DS4_HIPK=0
+    static std::atomic<int> state[16];
+    int dev = 0;
+    cudaGetDevice(&dev);
+    if (dev < 0 || dev >= 16) return false;
+    int v = state[dev].load(std::memory_order_relaxed);
+    if (v == 0) {
+        cudaDeviceProp prop{};
+        const bool gfx12 = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx12", 5) == 0;
+        const char* e = std::getenv("STRATA_DS4_HIPK");
+        v = gfx12 && !(e != nullptr && e[0] == '0') ? 1 : 2;   // (on by default on gfx12; STRATA_DS4_HIPK=0 off)
+        state[dev].store(v, std::memory_order_relaxed);
+    }
+    return v == 1;
+}
+
+template <int T, bool GU, int BM, int BN, int WM, int WN>
+inline bool run(const SetArgs& a, cudaStream_t s) {
+    const int N = GU ? a.n_ff : a.n_embd;
+    const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);
+    static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;
+    set_kernel<T, GU, BM, BN, WM, WN><<<grid, 256, 0, s>>>(a, dbg);
+    return cudaPeekAtLastError() == cudaSuccess;
+}
+// false: not taken (the caller's tiles do it)
+template <int T, bool GU>
+inline bool launch(const SetArgs& a, cudaStream_t s) {
+    if constexpr (!kTypes<T>) {
+        return false;
+    } else {
+        if (!ok_device() || a.max_rows <= 16) return false;
+        const int K = GU ? a.n_embd : a.n_ff, N = GU ? a.n_ff : a.n_embd;
+        if (K % 256 != 0 || N % 64 != 0 || a.ldx % 8 != 0 || a.ldo % 2 != 0) return false;
+        if (((uintptr_t) (GU ? (const void*) a.xf : (const void*) a.xh) & 15) != 0) return false;
+        static const int var = std::getenv("STRATA_DS4_HIPK") ? std::atoi(std::getenv("STRATA_DS4_HIPK")) : 0;
+        if (var != 2) {   // 2: the shared-memory weights kernel
+            static const int cfg = std::getenv("HIPK_CFG") ? std::atoi(std::getenv("HIPK_CFG")) : 0;
+            if (cfg == 1 && a.max_rows <= 32) return runr<T, GU, 32, 4, 1>(a, s);
+            if (cfg == 2 && a.max_rows <= 32) return runr<T, GU, 32, 8, 1>(a, s);
+            if (cfg == 3 && a.max_rows <= 32) return runr<T, GU, 32, 2, 2>(a, s);
+            if (cfg == 4 && a.max_rows <= 32) return runr<T, GU, 32, 8, 2>(a, s);
+            if (cfg == 5 && a.max_rows <= 32) return runr<T, GU, 32, 2, 1>(a, s);
+            if (a.max_rows <= 32) return runr<T, GU, 32, 4, 2>(a, s);
+            if (a.max_rows <= 64) return runr<T, GU, 64, 4, 2>(a, s);
+            return runr<T, GU, 128, 8, 1>(a, s);
+        }
+        if (a.max_rows <= 32) return run<T, GU, 32, 64, 2, 4>(a, s);
+        if (a.max_rows <= 64) return run<T, GU, 64, 64, 2, 4>(a, s);
+        if constexpr (GU) return run<T, GU, 128, 32, 4, 2>(a, s);
+        else return run<T, GU, 128, 64, 4, 2>(a, s);
+    }
+}
+}  // namespace hipk
+
+#endif

src/dsv41/ds4_iq_hip_r.inc

@@ -0,0 +1,177 @@
+// Strata. Part of ds4_iq_hip.inc (included from it): the weights straight into the matrix cores' registers.
+// One lane of the B operand of a 16x16x16 step holds 8 consecutive k of one weight row: exactly one 8-value group of a 32-value
+// sub-block of an i-quant block (the grid entry, its signs).  grp_dec decodes that group alone: 8 int8 codes (two words) and the
+// scale (weight = scale * code), so a wave needs no shared memory for the weights and no barrier: every wave decodes its own
+// columns once and uses them for every row of the tile.  kb: the 32-value sub-block of the row, p: its group (0..3).
+// The decode is in two halves so that the loads of a k-step can be issued one k-step ahead of its use: grp_load fetches the
+// group raw bytes into registers (Raw), grp_dec turns them into codes and scale (the codebook lookups, the signs).
+__device__ __forceinline__ float hf(uint32_t h) { return __half2float(__ushort_as_half((uint16_t) h)); }
+#include "ds4_iq_hip_s.inc"
+
+constexpr int kKC = 64;   // K a step: two sub-blocks
+// BM rows (MT tiles of 16) x BN = NW 16 NTW weight rows per block; NW waves, each NTW column tiles of 16 and every row tile
+template <int T, bool GU, int BM, int NW, int NTW>
+__global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg) {
+    constexpr int NM = GU ? 2 : 1, MT = BM / 16, BN = NW * 16 * NTW, NTH = NW * 32;
+    static_assert(BM % 16 == 0, "row tiles");
+    __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+    __shared__ int srow[BM];
+    __shared__ __align__(16) _Float16 xs[2][BM][kStH];
+    const int e = blockIdx.z;
+    const int lo = a.bounds[e] - a.row0, hi_r = a.bounds[e + 1] - a.row0;
+    const int r0 = lo + blockIdx.x * BM;
+    if (r0 >= hi_r || hi_r - lo <= a.skip_le) return;   // (uniform)
+    load_tab<T>(tab);
+    const int nr = min(BM, hi_r - r0);
+    const int c0 = blockIdx.y * BN;
+    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+    const int K = GU ? a.n_embd : a.n_ff;
+    const size_t rb = row_bytes_t<T>(K);
+    const int tid = (int) threadIdx.x, warp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;
+    for (int i = tid; i < BM; i += NTH) srow[i] = GU ? a.idx[r0 + min(i, nr - 1)] : r0 + min(i, nr - 1);
+    __syncthreads();
+    const int nst = K / kKC;
+    // the weight rows of this lane column tiles
+    const uint8_t* wrow[NM][NTW];
+#pragma unroll
+    for (int m = 0; m < NM; ++m)
+#pragma unroll
+        for (int j = 0; j < NTW; ++j)
+            wrow[m][j] = blob + a.tr[GU ? m : 2] + (size_t) ((c0 + warp * (16 * NTW) + j * 16 + l16)) * rb;
+
+    constexpr int NX = GU ? (BM * 16 + NTH - 1) / NTH : (BM * 8 + NTH - 1) / NTH;
+    struct XR {
+        float4 f[GU ? NX : 1];
+        uint4 u[GU ? 1 : NX];
+    };
+    auto fetchx = [&](int s, XR& R) {
+#pragma unroll
+        for (int xi = 0; xi < NX; ++xi) {
+            const int c = tid + xi * NTH;
+            if constexpr (GU) {
+                if (c < BM * 16) {
+                    const int row = c >> 4, ch = c & 15;
+                    R.f[xi] = *(const float4*) (a.xf + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 4);
+                }
+            } else {
+                if (c < BM * 8) {
+                    const int row = c >> 3, ch = c & 7;
+                    R.u[xi] = *(const uint4*) (a.xh + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 8);
+                }
+            }
+        }
+    };
+    auto storex = [&](const XR& R, int b) {
+#pragma unroll
+        for (int xi = 0; xi < NX; ++xi) {
+            const int c = tid + xi * NTH;
+            if constexpr (GU) {
+                if (c < BM * 16) {
+                    const int row = c >> 4, ch = c & 15;
+                    const float4 v = R.f[xi];
+                    const hv2 p = {(_Float16) v.x, (_Float16) v.y}, q = {(_Float16) v.z, (_Float16) v.w};
+                    *(uint2*) &xs[b][row][ch * 4] = make_uint2(as_u32(p), as_u32(q));
+                }
+            } else {
+                if (c < BM * 8) {
+                    const int row = c >> 3, ch = c & 7;
+                    *(uint4*) &xs[b][row][ch * 8] = R.u[xi];
+                }
+            }
+        }
+    };
+
+    fv8 acc[NM][MT][NTW];
+#pragma unroll
+    for (int m = 0; m < NM; ++m)
+#pragma unroll
+        for (int i = 0; i < MT; ++i)
+#pragma unroll
+            for (int j = 0; j < NTW; ++j) acc[m][i][j] = fv8{0, 0, 0, 0, 0, 0, 0, 0};
+
+    {
+        XR R0;
+        fetchx(0, R0);
+        storex(R0, 0);
+    }
+    // the weights of a step (64 values) are loaded one step ahead: two sets of registers, the step loop unrolled by two
+    SRaw sr0[NM][NTW], sr1[NM][NTW];
+    auto load_stage = [&](SRaw (&dst)[NM][NTW], int s) {
+#pragma unroll
+        for (int m = 0; m < NM; ++m)
+#pragma unroll
+            for (int j = 0; j < NTW; ++j) stage_load<T>(wrow[m][j], s, hi, dst[m][j]);
+    };
+    auto do_stage = [&](int s, SRaw (&cur)[NM][NTW], SRaw (&nxt)[NM][NTW]) {
+        const int b = s & 1;
+        XR R;
+        if (s + 1 < nst) {
+            fetchx(s + 1, R);
+            load_stage(nxt, s + 1);
+        }
+        hv8 bf[NM][NTW];
+        auto step = [&](auto tt) {
+            constexpr int t = decltype(tt)::value;
+            constexpr int sub = t >> 1, ks = t & 1;
+#pragma unroll
+            for (int m = 0; m < NM; ++m)
+#pragma unroll
+                for (int j = 0; j < NTW; ++j) {
+                    int c0_, c1_;
+                    float sc;
+                    step_dec<T, t>(cur[m][j], s, hi, tab, c0_, c1_, sc);
+                    const hv2 h = {(_Float16) sc, (_Float16) sc};
+                    const uint2 u0 = codes4(c0_, h), u1 = codes4(c1_, h);
+                    bf[m][j] = __builtin_bit_cast(hv8, make_uint4(u0.x, u0.y, u1.x, u1.y));
+                }
+            const int kcol = sub * 32 + ks * 16 + hi * 8;
+#pragma unroll
+            for (int i = 0; i < MT; ++i) {
+                if (i * 16 < nr) {   // (a tile past the rows: skipped, uniform)
+                    const hv8 af = *(const hv8*) &xs[b][i * 16 + l16][kcol];
+#pragma unroll
+                    for (int m = 0; m < NM; ++m)
+#pragma unroll
+                        for (int j = 0; j < NTW; ++j) acc[m][i][j] = wmma16(af, bf[m][j], acc[m][i][j]);
+                }
+            }
+        };
+        step(std::integral_constant<int, 0>{});
+        step(std::integral_constant<int, 1>{});
+        step(std::integral_constant<int, 2>{});
+        step(std::integral_constant<int, 3>{});
+        if (s + 1 < nst) storex(R, b ^ 1);
+        __syncthreads();
+    };
+    load_stage(sr0, 0);
+    __syncthreads();
+    for (int s = 0; s < nst; s += 2) {   // (nst is a multiple of 4)
+        do_stage(s, sr0, sr1);
+        do_stage(s + 1, sr1, sr0);
+    }
+#pragma unroll
+    for (int i = 0; i < MT; ++i)
+#pragma unroll
+        for (int q = 0; q < 8; ++q) {
+            const int row = i * 16 + 8 * hi + q;
+            if (row >= nr) continue;
+            const size_t lr = (size_t) (r0 + row);
+#pragma unroll
+            for (int j = 0; j < NTW; ++j) {
+                const int col = c0 + warp * (16 * NTW) + j * 16 + l16;
+                if (GU) a.h[lr * a.ldo + col] = __half_as_ushort(__float2half_rn(swiglu(acc[0][i][j][q], acc[1][i][j][q], a.limit)));
+                else a.y[lr * a.ldo + col] = acc[0][i][j][q];
+            }
+        }
+}
+
+template <int T, bool GU, int BM, int NW, int NTW>
+inline bool runr(const SetArgs& a, cudaStream_t s) {
+    const int N = GU ? a.n_ff : a.n_embd;
+    constexpr int BN = NW * 16 * NTW;
+    if (N % BN != 0) return false;
+    const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);
+    static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;
+    set_kernel_r<T, GU, BM, NW, NTW><<<grid, NW * 32, 0, s>>>(a, dbg);
+    return cudaPeekAtLastError() == cudaSuccess;
+}

src/dsv41/ds4_iq_hip_s.inc

@@ -0,0 +1,162 @@
+// Strata. Part of ds4_iq_hip_r.inc: the weights of a whole 64-value step (two 32-value sub-blocks) of one weight row in a few wide
+// loads, and the decode of its four 8-value groups (k-steps) from registers.  The loads are unaligned (the blocks of the i-quants are
+// 2-byte aligned; the AMD hardware takes unaligned vector loads).  A lane holds the group p = 2 ks + hi of every sub-block, so
+// the two lanes of a column (hi 0 / 1) share most of their bytes.  s: the step (64 values), b: the block (256 values: 4 steps),
+// jb = 2 (s & 3): the first sub-block of the step in its block.
+struct SRaw {
+    uint32_t w[10];
+};
+__device__ __forceinline__ uint4 ldu128(const void* p) {
+    uint4 v;
+    __builtin_memcpy(&v, p, 16);
+    return v;
+}
+__device__ __forceinline__ uint2 ldu64(const void* p) {
+    uint2 v;
+    __builtin_memcpy(&v, p, 8);
+    return v;
+}
+__device__ __forceinline__ uint32_t ldu32(const void* p) {
+    uint32_t v;
+    __builtin_memcpy(&v, p, 4);
+    return v;
+}
+__device__ __forceinline__ uint32_t ldu16(const void* p) {
+    uint16_t v;
+    __builtin_memcpy(&v, p, 2);
+    return v;
+}
+template <int T>
+__device__ __forceinline__ void stage_load(const uint8_t* __restrict__ row, int s, int hi, SRaw& r) {
+    if constexpr (T == kIQ4_NL) {   // 32-value blocks of 18 bytes: the step is blocks 2 s and 2 s + 1
+        const uint8_t* b0 = row + (size_t) (2 * s) * 18;
+        const uint2 qa = ldu64(b0 + 2 + 8 * hi), qb = ldu64(b0 + 18 + 2 + 8 * hi);
+        r.w[0] = qa.x, r.w[1] = qa.y, r.w[2] = qb.x, r.w[3] = qb.y;
+        r.w[4] = ldu16(b0);
+        r.w[5] = ldu16(b0 + 18);
+    } else {
+        const uint8_t* b = row + (size_t) (s >> 2) * Ty<T>::B;
+        const int jb = 2 * (s & 3);
+        if constexpr (T == kIQ2_XXS) {
+            const uint4 q = ldu128(b + 2 + 8 * jb);
+            r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;
+            r.w[4] = ldu16(b);
+        } else if constexpr (T == kIQ2_XS) {
+            const uint4 q = ldu128(b + 2 + 8 * jb);
+            r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;
+            r.w[4] = ldu16(b + 66 + jb);
+            r.w[5] = ldu16(b);
+        } else if constexpr (T == kIQ2_S) {
+            const uint2 q = ldu64(b + 2 + 4 * jb), sg = ldu64(b + 34 + 4 * jb);
+            r.w[0] = q.x, r.w[1] = q.y, r.w[2] = sg.x, r.w[3] = sg.y;
+            r.w[4] = ldu16(b + 66 + jb);
+            r.w[5] = ldu16(b + 74 + jb);
+            r.w[6] = ldu16(b);
+        } else if constexpr (T == kIQ3_XXS) {
+            const uint4 q = ldu128(b + 2 + 8 * jb);
+            const uint2 ax = ldu64(b + 66 + 4 * jb);
+            r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;
+            r.w[4] = ax.x, r.w[5] = ax.y;
+            r.w[6] = ldu16(b);
+        } else if constexpr (T == kIQ3_S) {
+            const uint4 q = ldu128(b + 2 + 8 * jb);
+            const uint2 sg = ldu64(b + 74 + 4 * jb);
+            r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;
+            r.w[4] = ldu16(b + 66 + jb);
+            r.w[5] = sg.x, r.w[6] = sg.y;
+            r.w[7] = b[106 + (jb >> 1)];
+            r.w[8] = ldu16(b);
+        } else if constexpr (T == kIQ4_XS) {
+            const uint2 qa = ldu64(b + 8 + 16 * jb + 8 * hi), qb = ldu64(b + 8 + 16 * (jb + 1) + 8 * hi);
+            r.w[0] = qa.x, r.w[1] = qa.y, r.w[2] = qb.x, r.w[3] = qb.y;
+            r.w[4] = ldu16(b + 2);
+            r.w[5] = b[4 + (jb >> 1)];
+            r.w[6] = ldu16(b);
+        } else if constexpr (T == kIQ1_S) {
+            const uint2 q = ldu64(b + 2 + 4 * jb);
+            r.w[0] = q.x, r.w[1] = q.y;
+            r.w[2] = ldu32(b + 34 + 2 * jb);
+            r.w[3] = ldu16(b);
+        } else {   // IQ1_M
+            const uint2 q = ldu64(b + 4 * jb);
+            const uint2 d4 = ldu64(b + 48);
+            r.w[0] = q.x, r.w[1] = q.y;
+            r.w[2] = ldu32(b + 32 + 2 * jb);
+            r.w[3] = ldu16(b + 48 + (jb >> 1) * 2);
+            r.w[4] = d4.x, r.w[5] = d4.y;
+        }
+    }
+}
+// the k-step t (0..3, compile time) of a loaded step: sub-block sub = t >> 1, group p = 2 (t & 1) + hi
+template <int T, int TT>
+__device__ __forceinline__ void step_dec(const SRaw& r, int s, int hi, const uint32_t* tab, int& c0, int& c1, float& sc) {
+    constexpr int sub = TT >> 1, ks = TT & 1;
+    const int p = 2 * ks + hi;
+    if constexpr (T == kIQ2_XXS) {
+        const uint2 g = ((const uint2*) tab)[(r.w[2 * sub] >> (8 * p)) & 255u];
+        const uint32_t a1 = r.w[2 * sub + 1];
+        const uint32_t sg = ksigns8(a1 >> (7 * p));
+        c0 = sgn4(g.x, sg);
+        c1 = sgn4(g.y, sg >> 4);
+        sc = hf(r.w[4]) * (float) (2 * (int) (a1 >> 28) + 1) * 0.125f;
+    } else if constexpr (T == kIQ2_XS) {
+        const uint32_t v = (r.w[2 * sub + (ks)] >> (16 * hi)) & 0xffffu;
+        const uint2 g = ((const uint2*) tab)[v & 511u];
+        const uint32_t sg = ksigns8(v >> 9);
+        const uint32_t scb = (r.w[4] >> (8 * sub)) & 255u;
+        c0 = sgn4(g.x, sg);
+        c1 = sgn4(g.y, sg >> 4);
+        sc = hf(r.w[5]) * (float) (2 * (int) (ks == 0 ? scb & 15 : scb >> 4) + 1) * 0.125f;
+    } else if constexpr (T == kIQ2_S) {
+        const uint32_t qh = (r.w[4] >> (8 * sub)) & 255u;
+        const uint32_t idx = ((r.w[sub] >> (8 * p)) & 255u) | ((qh << (8 - 2 * p)) & 0x300u);
+        const uint32_t sg = (r.w[2 + sub] >> (8 * p)) & 255u;
+        const uint32_t scb = (r.w[5] >> (8 * sub)) & 255u;
+        const uint2 g = ((const uint2*) tab)[idx];
+        c0 = sgn4(g.x, sg);
+        c1 = sgn4(g.y, sg >> 4);
+        sc = hf(r.w[6]) * (float) (2 * (int) (ks == 0 ? scb & 15 : scb >> 4) + 1) * 0.125f;
+    } else if constexpr (T == kIQ3_XXS) {
+        const uint32_t word = r.w[2 * sub + ks];
+        const uint32_t aux = r.w[4 + sub];
+        const uint32_t sg = ksigns8(aux >> (7 * p));
+        c0 = sgn4(tab[(word >> (16 * hi)) & 255u], sg);
+        c1 = sgn4(tab[(word >> (16 * hi + 8)) & 255u], sg >> 4);
+        sc = hf(r.w[6]) * (float) (2 * (int) (aux >> 28) + 1) * 0.25f;
+    } else if constexpr (T == kIQ3_S) {
+        const uint32_t word = r.w[2 * sub + ks];
+        const uint32_t h = (r.w[4] >> (8 * sub)) & 255u;
+        const uint32_t sg = (r.w[5 + sub] >> (8 * p)) & 255u;
+        c0 = sgn4(tab[((word >> (16 * hi)) & 255u) | ((h << (8 - 2 * p)) & 256u)], sg);
+        c1 = sgn4(tab[((word >> (16 * hi + 8)) & 255u) | ((h << (7 - 2 * p)) & 256u)], sg >> 4);
+        sc = hf(r.w[8]) * (float) (1 + 2 * (int) ((r.w[7] >> (4 * sub)) & 15));
+    } else if constexpr (T == kIQ4_XS) {
+        const int jb = 2 * (s & 3);
+        const int ls = ((r.w[5] >> (4 * sub)) & 15) | (((r.w[4] >> (2 * (jb + sub))) & 3) << 4);
+        const int2 v0 = tab16(r.w[2 * sub], tab), v1 = tab16(r.w[2 * sub + 1], tab);
+        c0 = ks == 0 ? v0.x : v0.y;
+        c1 = ks == 0 ? v1.x : v1.y;
+        sc = hf(r.w[6]) * (float) (ls - 32);
+    } else if constexpr (T == kIQ1_S) {
+        const uint32_t qh = (r.w[2] >> (16 * sub)) & 0xffffu;
+        const uint32_t g = tab[((r.w[sub] >> (8 * p)) & 255u) | (((qh >> (3 * p)) & 7u) << 8)];
+        const bool neg = (qh & 0x8000u) != 0;
+        c0 = (int) iq1_codes(g & 0x0F0F0F0Fu, neg);
+        c1 = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);
+        sc = hf(r.w[3]) * (float) (2 * (int) ((qh >> 12) & 7u) + 1) * 0.125f;
+    } else if constexpr (T == kIQ1_M) {
+        const uint32_t qh = (r.w[2] >> (8 * (2 * sub + ks))) & 255u;   // (p < 2: the first byte of the sub-block, else the second)
+        const uint32_t g = tab[((r.w[sub] >> (8 * p)) & 255u) | ((qh << (hi ? 4 : 8)) & 0x700u)];
+        const bool neg = (hi ? qh & 0x80u : qh & 0x08u) != 0;
+        c0 = (int) iq1_codes(g & 0x0F0F0F0Fu, neg);
+        c1 = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);
+        const int sh = (int) ((r.w[3] >> (6 * sub + (ks == 0 ? 0 : 3))) & 7u);
+        const uint32_t d16 = (r.w[4] >> 12 & 0xfu) | ((r.w[4] >> 24) & 0xf0u) | ((r.w[5] >> 4) & 0xf00u) | ((r.w[5] >> 16) & 0xf000u);
+        sc = hf(d16) * (float) (2 * sh + 1) * 0.125f;
+    } else {   // IQ4_NL
+        const int2 v0 = tab16(r.w[2 * sub], tab), v1 = tab16(r.w[2 * sub + 1], tab);
+        c0 = ks == 0 ? v0.x : v0.y;
+        c1 = ks == 0 ? v1.x : v1.y;
+        sc = hf(r.w[4 + sub]);
+    }
+}

src/dsv41/ds4_kernels.cuh

@@ -0,0 +1,306 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_kernels.cuh - DeepSeek-V4.1's own device kernels for a window of T tokens (1 <= T <= kWin) at positions
+// p0 .. p0 + T - 1 (the EXL3 matrices go through strata::dsv41k::exl3).  The math is DeepSeek's inference/model.py
+// (MIT); tools/ds4/ref.py is the oracle the engine is checked against.  Every pointer is device memory, every activation
+// F32, rows of a window back to back; the KV caches keep the reference's FP8 / FP4 roundings packed as DeepSeek stores
+// them (codes and scales, kv_row_bytes), the window attention's working rows BF16 (those values are exact in BF16).
+// What differs from query to query (its window, how much of the compressed KV it sees, how many rows it selects) is
+// known on the host and passed by value.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cuda_runtime.h>
+
+namespace strata::ds4 {
+
+constexpr int kWin = 16;      // tokens a window takes at most (the EXL3 GEMV's activations per job)
+constexpr int kPicks = 8;     // routed experts a token can take at most
+
+/// The caches' rows: DeepSeek's own roundings kept as codes and scales - lossless, the values are on those grids already
+/// (the numbers are Finish::quant's; a finish with that quant and a pdst writes the format), a row of d values in
+/// kv_row_bytes(fmt, d) bytes, padded to 16 (the hand-offs copy 16-byte units):
+///   kKvFp8   the window ring: d E4M3 codes, then d / 32 power-of-two scales (UE8M0: the F32 exponent byte)
+///   kKvFp4   the compressed KV: d / 2 bytes of E2M1 codes (value 2i in the low nibble), then d / 16 E4M3 scales
+///   kKvFp4p  the index keys: d / 2 bytes of E2M1 codes, then d / 32 UE8M0 scales
+/// kKvBf16: plain BF16 rows (a DSpark block's own rows, and the reference the tests check the packed rows against).
+constexpr int kKvBf16 = 0, kKvFp8 = 1, kKvFp4 = 2, kKvFp4p = 3;
+__host__ __device__ constexpr size_t kv_row_bytes(int fmt, int d) {
+    return fmt == kKvBf16  ? (size_t) d * 2
+           : fmt == kKvFp8 ? ((size_t) d + (size_t) d / 32 + 15) / 16 * 16
+           : fmt == kKvFp4 ? ((size_t) d / 2 + (size_t) d / 16 + 15) / 16 * 16
+                           : ((size_t) d / 2 + (size_t) d / 32 + 15) / 16 * 16;
+}
+
+/// h[t][s][e] = BF16 rows[t][e] for each of the hc streams.
+void embed(const uint16_t* rows_bf16, int T, float* h, int dim, int hc, cudaStream_t s);
+
+/// The per-vector finish: n_vec vectors of len values at x + v * stride, each in order
+///   norm  (w != null): x = x * rsqrt(mean(x^2) + eps) * w
+///   rope  (rd > 0): the last rd values as adjacent complex pairs rotated by pos_v * freqs[i] (inverse: the conjugate),
+///         pos_v = pos + (v / vpp) * pos_step
+///   quant 0 none, 1 FP8 E4M3 with a power-of-two scale per 32, 2 FP4 E2M1 with an E4M3 scale per 16, 3 FP4 with a
+///         power-of-two scale per 32 (the reference kernels' act_quant / fp4_act_quant, inplace)
+/// and written back to x, or as BF16 to dst + v * dst_stride when dst is given, or packed (quant > 0: the cache format of
+/// that number, kv_row_bytes) to pdst + v * pdst_stride bytes when pdst is given (x then left as it was).
+struct Finish {
+    const float* w = nullptr;
+    float eps = 1e-20f;
+    int rd = 0, pos = 0, inverse = 0, vpp = 1, pos_step = 1;
+    const float* freqs = nullptr;   // rd / 2 values
+    int quant = 0;
+    uint16_t* dst = nullptr;
+    long long dst_stride = 0;
+    uint8_t* pdst = nullptr;
+    long long pdst_stride = 0;
+};
+void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudaStream_t s);
+
+/// Hyper-Connections, row t of each [T][..] array: mixes[t][m] = (h_flat . fn[m]) * rsqrt(mean(h_flat^2) + eps), then
+/// pre / post / comb as hc_split_sinkhorn (comb[t][j * hc + k], j the source stream, k the destination).  Strides: mixes
+/// 32, pre / post 8, comb 64 floats a row.
+void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);
+void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
+              float* pre, float* post, float* comb, cudaStream_t s);
+/// x[t][e] = sum_s mix[t][s] h[t][s][e], then x = rmsnorm(x) * w (w null: no norm).
+void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,
+                 cudaStream_t s);
+/// out[t][k][e] = post[t][k] y[t][e] + sum_j comb[t][j][k] res[t][j][e]  (out must not alias res).
+void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,
+             float* out, cudaStream_t s);
+/// hc_post with out = res (h), the same arithmetic: a chunk keeps one copy of its streams.
+void hc_post_inplace(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T,
+                     cudaStream_t s);
+/// A chunk's hc_mix: the dots of each row with the n_mix weights already in mixes (a GEMM's, rows 32 apart), times
+/// rsqrt(mean(h_row^2) + eps) - hc_mix's value up to the dots' summation order.
+void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);
+
+/// y[t][r] = alpha * sum_k W[r][k] x[t][k], W F16 bits [n][k] (the indexer's weights_proj).
+void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alpha, float* y, cudaStream_t s);
+/// The same for T activations x[t] = x + t ldx into y + t ldy, each weight read once for all of them; each output is
+/// gemv_f16's bit for bit.
+void gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy,
+                   cudaStream_t s);
+/// y[t ldy + r] = sum_k W[r][k] x[t][k] in F32 (W [n][k], k % 128 == 0): the router's logits, as DeepSeek's
+/// inference computes them (linear(x.float(), weight.float())) - a warp a row, the same summing order for any T, so
+/// the token, window and chunk paths get the same bits.
+void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy, cudaStream_t s);
+
+/// The router, row t: scores = sqrt(softplus(logits[t])) (n experts, rows 1024 apart), the top `k` of scores + bias;
+/// weights from the unbiased scores, normalized to sum 1 (+1e-20) and times route_scale -> idx[t][i], w[t][i] (rows
+/// kPicks apart).  Rows of image tokens (bit t of vl) take bias_vl instead (training's noaux_tc_for_vl).
+constexpr int kVlRows = 8192;
+struct VlBits {
+    uint32_t w[kVlRows / 32];
+};
+void route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,
+           cudaStream_t s, const float* bias_vl = nullptr, const VlBits* vl = nullptr);
+
+/// h = silu(min(g, lim)) * clamp(u, -lim, lim)  (lim <= 0: no clamp); n values.
+void swiglu(const float* g, const float* u, int n, float lim, float* h, cudaStream_t s);
+
+/// The ratio-r compressor over a window: token t (position p0 + t) takes slot (p0 + t) % r of the state; each token that
+/// completes a group writes the group's pooled latent (softmax over the slots, per value, then rmsnorm(w)) to the next
+/// row of latent.
+void compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,
+                  int d, const float* w, float eps, float* latent, cudaStream_t s);
+
+/// Per-query sizes, by value - or (gen > 0, a chunk of any length) made on the device from the query's position: query
+/// t at position p0 + t has nn = (p0 + t + 1) / R compressed positions and cb = min(ccap, ceil(nn / blk)) candidate
+/// blocks;  gen 1: n = nn, k = min(kcap, nn)  (the indexer's scores and their top-k)
+///          gen 2: n = nn, k = cb             (index_score over the candidates)
+///          gen 3: n = cb * blk, k = min(kcap, nn)   (the top-k of the candidates' scores)
+///          gen 4: n = ceil(nn / blk), k = kcap      (the candidate blocks' top-k)
+struct Rows {
+    int n[kWin];
+    int k[kWin];
+    int gen = 0, R = 1, kcap = 0, ccap = 0, blk = 1;
+    long long p0 = 0;
+};
+
+/// The indexer's scores, row t: out[t][i] = sum_h relu(q[t][h] . k[p]) w[t][h] for i < n (the widest query's), p = i, or
+/// with candidates p = cand[t][i / blk] * blk + i % blk for i < lim.k[t] * blk; positions >= lim.n[t] (or past the
+/// query's candidates) -> -inf.  q rows heads * hd apart, w rows 64, cand rows cand_stride, out rows out_stride; the keys
+/// k rows of format kfmt (the cache's kKvFp4p, or kKvBf16).
+void index_score(const float* q, const float* w, const void* k, int heads, int hd, int T, int n, const int* cand,
+                 int cand_stride, int blk, const Rows& lim, float* out, long long out_stride, cudaStream_t s,
+                 int kfmt = kKvFp4p);
+/// The top r.k[t] of vals[t][0, r.n[t]) (row t at vals + t * stride): their positions (unordered; with candidates
+/// cand[t][i / blk] * blk + i % blk) into out[t][0, k).  r.k[t] >= r.n[t] takes them all.
+void topk(const float* vals, long long stride, int T, const Rows& r, const int* cand, int cand_stride, int blk,
+          int* out, int out_stride, cudaStream_t s);
+/// Level one of the candidate pre-filter, row t: block b's score = the max over its blk positions of the r.n[t] the
+/// query sees; the block of its newest position is pinned; the top r.k[t] blocks -> out[t] (bm: scratch rows bm_stride).
+void cand_blocks(const float* score, long long stride, int T, const Rows& r, int blk, float* bm, long long bm_stride,
+                 int* out, int out_stride, cudaStream_t s);
+
+/// Sparse attention, query t: heads x d values of q[t] against win rows win0[t] .. win0[t] + nwin[t] - 1 of the window
+/// buffer (d BF16 each) and the nsel[t] compressed rows ckv[sel[t][i]] (format ckv_fmt: the cache's kKvFp4, or kKvBf16);
+/// K = V, a learned sink logit per head.  The packed rows, the same bits either way: up to kAttnScratchT queries (with
+/// a scratch, heads_per_block 0; or -1, any T) decoded once a query into scr's BF16 rows for the BF16 rows' kernel;
+/// else a kernel that decodes each staged row once for heads_per_block heads of a query (1, 2, 4, 8; 0: as many as
+/// keep two blocks a multiprocessor).
+struct AttnScratch {
+    uint16_t* rows = nullptr;   // [max_t * cap][512] BF16
+    int max_t = 0, cap = 0;     // queries, rows a query (at least the sel_stride)
+};
+constexpr int kAttnScratchT = 7;   // (the staged kernel wins from 8 queries: V100, Sun)
+struct AttnRows {
+    int win0[kWin], nwin[kWin], nsel[kWin];
+    // gen: query t is a chunk's row t0 + t at position p0 + t0 + t (the window buffer: the ring's win rows before p0,
+    // then the chunk's), selecting min(kcap, (its position + 1) / R) compressed rows when R > 0
+    int gen = 0, t0 = 0, win = 0, R = 0, kcap = 0;
+    long long p0 = 0;
+};
+void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride,
+                 const AttnRows& a, const float* sink, int heads, int d, int T, float scale, float* out, cudaStream_t s,
+                 int ckv_fmt = kKvFp4, int heads_per_block = 0, const AttnScratch* scr = nullptr);
+/// The window buffer: ext[r] = ring[(p0 - n + r) % n] for the n rows before p0 (zeros before position 0); afterwards
+/// ring[(p0 + t) % n] = ext[n + t] for t < T.  The ring's rows kKvFp8 (each group of 32 at the smallest power-of-two
+/// scale its BF16 values fit under: exact for values on the FP8 grid, the window's), the window buffer's BF16 (p0 = n:
+/// the ring's rows as they lie).
+void ring_to_ext(const uint8_t* ring, uint16_t* ext, int64_t p0, int n, int d, cudaStream_t s);
+void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, int d, cudaStream_t s,
+                 uint8_t* backup = nullptr);
+/// A speculative window's rollback: ring[(p0 + t) % n] = backup[t] for keep <= t < T (the rows ext_to_ring replaced).
+void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s);
+/// The same for up to kMaxRingBatch layers' rings at once (one launch).
+constexpr int kMaxRingBatch = 64;
+struct RingBatch {
+    uint8_t* ring[kMaxRingBatch] = {};
+    const uint8_t* bak[kMaxRingBatch] = {};
+    int n = 0;
+};
+void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s);
+
+/// Engram.  rows: n rows of hd values -> out F32 [n * hd].  fmt, the pack's row layout: 0 FP8 E4M3 values with E8M0
+/// scales (scales: n x (hd / 32)); 1 ggml Q6_K superblocks (hd / 256 of 210 bytes at the start of each row's hd bytes;
+/// no scales), decoded as ggml's dequantize_row_q6_K; 2 ggml IQ4_NL blocks (hd / 32 of 18 bytes, an fp16 d and 16
+/// nibble bytes at the start of each row's hd bytes; no scales), decoded as ggml's dequantize_row_iq4_nl.
+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int fmt = 0);
+/// The same over the host's per-token layout (a chunk): token t's cols rows, then their scales, `per` bytes a token;
+/// T tokens -> out F32 [T * cols][hd].
+void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s,
+                        int fmt = 0);
+/// h[t][k] += gate[t][k] * value[t] with the gate from the normalized dot of h[t][k] * w[k] * key[t][k]
+/// (kv[t] = [key hc x dim | value dim]).
+void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s);
+
+/// out[r] = x[src[r]] for r < n (rows of cols floats).
+struct Gather {
+    int src[kWin * kPicks];
+};
+void gather_rows(const float* x, const Gather& g, int n, int cols, float* out, cudaStream_t s);
+/// The MoE's sum, token t: out[t] = sum_j w[t][j] ed[row[t][j]] (its VRAM experts, in pick order) + cpu[t] (when its
+/// cpu flag is set) + sh[t].
+struct Combine {
+    int n[kWin];
+    int row[kWin][kPicks];
+    float w[kWin][kPicks];
+    int cpu[kWin];
+};
+void combine(const float* ed, const float* cpu, const float* sh, const Combine& c, int T, int dim, float* out,
+             cudaStream_t s);
+
+/// The mailboxes of a small window (decode, a draft's verification, the drafter's block): host-mapped memory, a ring of
+/// kRing.  The device publishes a layer's routing and its FFN inputs for the window's T tokens (seq written last, after
+/// a system fence); the host answers with the CPU lane's weighted part of each token that took a CPU-lane expert.
+constexpr int kRing = 64;
+constexpr int kMaxDim = 8192;
+constexpr int kAhead1 = 16, kAhead = kAhead1 + kPicks;   // a token's guesses for the next layer, then the one after
+struct Req {
+    unsigned seq;
+    int T;
+    int na[2];   // lookahead: ahead[t][0 .. na[0]) the next layer's likely picks, ahead[t][kAhead1 ..) +na[1] the one after's
+    int ids[kWin][kPicks];
+    float w[kWin][kPicks];
+    int ahead[kWin][kAhead];
+    float x[kWin][kMaxDim];
+};
+struct Resp {
+    unsigned seq;
+    int pad[3];
+    int cpu[kWin];   // 1: part[t] is the CPU lane's weighted sum for token t
+    float part[kWin][kMaxDim];
+};
+/// route() for the T tokens of a window, plus: plan_ptr[t][i] = vram[idx[t][i]] (0: not on the device), each token's x
+/// (dim floats) and picks into the request slot, then T and seq (req null: no request, the plan alone).
+void route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale,
+                   const unsigned long long* vram, const float* x, int dim, int* idx, float* w,
+                   unsigned long long* plan_ptr, Req* req, unsigned seq, cudaStream_t s, const float* bias_vl = nullptr,
+                   unsigned vl = 0, int na0 = 0, int na1 = 0);
+/// Lookahead: the m picks a later layer will likely make (its gate's logits on this layer's FFN input, the routing's
+/// score + bias, the m best of n) into req->ahead[t][off ..) for the T tokens - before route_publish, which publishes them.
+void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s);
+/// The RAM store's picks for the device (a decode token's k plan pointers): those the VRAM table left at 0 whose expert zc_tab has (the
+/// slot's device address) are copied into stage slot i (stride apart, `bytes` each) and the plan then points there.
+void zc_stage(unsigned long long* plan, const int* idx, const unsigned long long* zc_tab, int k, uint8_t* stage, size_t stride, size_t bytes,
+              cudaStream_t s);
+/// Waits (one thread, on the device) until *seq_ptr == seq.
+void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s);
+/// A chunk's routed experts as sorted rows: the picks (t, i < k) of T tokens (idx rows kPicks apart) by the virtual
+/// slot vs[expert] (n_vs of them: the VRAM slots, then the experts streamed in), each slot's rows in token order:
+/// bounds[n_vs + 1], row_tok[r], pos[t][i] (the pick's sorted row, rows kPicks apart).
+void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,
+              cudaStream_t s);
+/// acc[t] += w[t][i] * y[pos[t][i] - r0] for each pick of t (in pick order) whose sorted row is in [r0, r1).
+void moe_combine(float* acc, const float* y, const int* pos, const float* w, int T, int k, int r0, int r1, int dim,
+                 cudaStream_t s);
+/// rows[t][0] = 1, the rest 0 (the first layer's pre-mix: stream 0).
+void init_pre(float* pre, int T, cudaStream_t s);
+
+/// out[t][e] = mean over the hc streams of h[t][s][e] (the DSpark drafter's taps), rows out_stride floats apart.
+void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s);
+/// out[t][i] = i for i < n, rows `stride` apart (the drafter's block rows as every query's selection).
+void iota_rows(int* out, int T, int n, int stride, cudaStream_t s);
+/// The DSpark markov head: logits[v] += E[*tok] . H[v] for v < V (E BF16 [V][R], H F16 [V][R]).
+void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s);
+/// The markov bias over a head group: a member's rows [v0, v0 + vn) of it added to its share of a logits row (erow:
+/// the previous token's E row, BF16, on the device) - markov_add's sums; the share's argmax (the greedy sampler's rule)
+/// into a host-mapped slot {value, index, seq}; every member's slot of the step -> the global argmax into *tok and its
+/// E row (E host-mapped) into erow.
+void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s);
+void argmax_share(const float* row, int vn, int v0, unsigned* slot, unsigned seq, cudaStream_t s);
+void markov_pick(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E, uint16_t* erow, int R,
+                 cudaStream_t s);
+/// The DSpark confidence: out[t] = sigmoid(w . [x[t] | E[tok[t]]]) (x rows dim floats, w F32 [dim + R]).
+void confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out,
+                cudaStream_t s);
+
+/// The pipeline's hand-offs and tensor parallelism, through host-mapped memory (these cards copy peer to peer only
+/// through the driver's staging).  xfer_put copies up to 4 segments (16-byte multiples, 16-byte aligned) device ->
+/// host-mapped, then (flag given) sets *flag = seq after a system fence (cnt: a device counter of the stream, 0
+/// between uses); xfer_get waits on the device until *flag >= seq, then copies host-mapped -> device.
+struct Xfer {
+    const void* src[4] = {nullptr, nullptr, nullptr, nullptr};
+    void* dst[4] = {nullptr, nullptr, nullptr, nullptr};
+    size_t bytes[4] = {0, 0, 0, 0};
+    int n = 0;
+};
+void xfer_put(const Xfer& x, unsigned* flag, unsigned seq, unsigned* cnt, cudaStream_t s);
+// Windows (WDDM, CUDA and HIP alike) holds launches back until the host waits or queries the stream: the work queued on
+// `s` submitted now (cudaStreamQuery, which does not wait).  Elsewhere nothing.  DS4_WIN_SUBMIT=0: off (A/B tests)
+void win_submit(cudaStream_t s);
+void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s);
+/// Waits (one thread) until flags[i * stride] >= seq for i < n (a pipelined chunk's readers have taken a buffer);
+/// set_flag: *flag = v after a system fence (a reader's acknowledgement).
+void wait_flags(const unsigned* flags, int n, int stride, unsigned seq, cudaStream_t s);
+void set_flag(unsigned* flag, unsigned v, cudaStream_t s);
+/// A tensor-parallel group's all-reduce: host-mapped rows data[member][slot] (stride floats), flags[member][slot]
+/// (32 apart).  x (n floats, n % 4 == 0: this member's partial) -> the sum of the nm members' partials in member order,
+/// the same bits on every member; seq counts the group's all-reduces (the same on each member), slot = seq % slots.
+struct ArGroup {
+    float* data = nullptr;
+    unsigned* flags = nullptr;
+    size_t stride = 0;
+    int nm = 1, slots = 4;
+    // three members or more: a reduce-scatter + all-gather (each member sums its chunk of every block's range, in
+    // member order, then reads the others' sums) - [slots][stride] sums, [2][nm][slots][kArBlocks][8] flags
+    float* sums = nullptr;
+    unsigned* bflags = nullptr;
+};
+constexpr int kArBlocks = 64;
+void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsigned* cnt, cudaStream_t s);
+
+}  // namespace strata::ds4

src/dsv41/ds4_model.hpp

@@ -0,0 +1,196 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_model.hpp - DeepSeek-V4.1-Flash on Maya: an EXL3 model (tools/ds4/pack.py's pack beside its files) over
+// the machine's GPUs, layer-split, with the routed experts that do not fit in VRAM on the CPU lane.
+//
+// Windows of up to 16 tokens through every layer (a prompt in windows, a decode step a window of one): each query of a
+// window sees exactly what the reference's decode path gives it (DeepSeek's inference/model.py at start_pos > 0) -
+// its own 128-token window, the compressed groups completed by its position.  The state a token leaves is the
+// 128-token window ring of every layer, the compressors' partial groups, the compressed KV and index keys of the KV
+// source layers (append-only), and the Engram n-gram history; checkpoint() / restore() keep the first two (the rest is
+// truncated by position) so a conversation can go back to the end of its last prompt.  A conversation that leaves the
+// model altogether (another chat answered in between) is a slot: slot_save / slot_load take the same state - the
+// compressed KV and index keys included, which a checkpoint leaves in place - through a file or a host buffer.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace strata::ds4 {
+
+struct Options {
+    std::string pack;                 // <model>/maya-ds4
+    int64_t max_context = 32768;
+    std::vector<int> gpus;            // empty: every visible GPU
+    int64_t reserve_mib = 600;        // VRAM left free on each GPU
+    double vram_experts = 1.0;        // the share of the expert budget to use (tests: less, so the CPU lane works)
+    int cpu_threads = 0;              // 0: the physical cores
+    std::string profile;              // expert hit counts (read to place the hottest experts in VRAM, written back)
+    bool expert_cache = true;         // experts move between VRAM and the CPU lane as they are used
+    // the CPU lane's experts read in place from the model's files, the page cache keeping what fits, instead of
+    // copied into RAM: 1 on, 0 off, -1 (auto) where the RAM this process can get does not hold them
+    int mmap_experts = -1;
+    // the experts the VRAM leaves over are cached in RAM by a store of their own (ds4_store.hpp): whole experts read unbuffered into
+    // slots, a replacement policy (0 lru, 1 lfu, 2 lfu then lru), filled on demand and by the lookahead's guesses.  GB of RAM at
+    // most (never past the RAM the process can still get less 8 GB); 0: off, the OS page cache as before
+    double ssd_cache_gb = -1.0;   // < 0: auto (the RAM the process can get less a floor, at most 48 GB; off below 4 GB), 0: off
+    int ssd_policy = 0;
+    // dspark_auto: the drafter is loaded only where the routed experts fit the GPUs (more than 1.5x what they have, the verify rounds
+    // read 3.3x the bytes of a token and measured no faster than plain decoding, and the drafter's 7 GB of experts take the VRAM the
+    // trunk's hottest would have had); --dspark on / off / none says it outright
+    bool dspark_auto = true;
+    bool dspark = true;              // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decoding
+    int dspark_block = 0;             // at most this many drafts a round (--dspark-block N; 0: the pack's block size)
+    float dspark_conf = 0.5f;         // a draft is verified while its confidence (sigmoid) is at least this
+    // tensor parallelism: runs of tp GPUs of one kind (the same memory) take the same layers, each a share of the
+    // heads, of the output projection and of every expert's FFN, summed through host-mapped memory (a GPU unlike the
+    // next stays alone); tp > 1 runs with threads (--tp 1 --threads off: one host thread, layer split only)
+    int tp = 2;
+    bool threads = false;             // a host thread per GPU enqueues its layers (the hand-offs host-mapped)
+    // prompts in chunks of this many tokens through each layer at once (threads): the GEMMs on the tensor cores, the
+    // CPU lane's experts streamed to the GPUs a chunk at a time instead of computed on the host; 0: windows of 16;
+    // -1: 2048, or 8192 with the experts read in place (every chunk streams them all from the files)
+    int prefill_chunk = -1;
+    bool vision = true;               // the vision tower (the checkpoint's vision.* / aligner.*), when it has one
+    bool vision_only = false;         // (tests) load the vision tower alone: encode_image() and nothing else
+    bool verbose = false;
+};
+
+/// A picture in the prompt: its span's rows (every position of the span carries the image token), BF16 [span][dim]
+struct Image {
+    int64_t start = 0;
+    std::shared_ptr<const std::vector<uint16_t>> rows;
+};
+
+struct Stats {
+    int64_t tokens = 0;
+    int64_t experts = 0, cpu_experts = 0;   // routed experts run, and how many of them on the CPU lane
+    double cpu_ms = 0.0, gpu_wait_ms = 0.0;   // the CPU lane's time; the host's waits for the routing
+    double layer_ms = 0.0, head_ms = 0.0, engram_ms = 0.0;   // the layers' host time in all; the head's wait; Engram reads
+    int64_t eg_rows = 0, eg_hits = 0, eg_reads = 0;   // Engram rows asked for, served without a read, read from the files
+    double eg_io_ms = 0.0;                            // the windows' Engram reads, issue to last done (mostly hidden)
+    int64_t eg_cached = 0, eg_cache_cap = 0;          // the row cache's rows held, its capacity
+    int64_t swaps = 0;                // experts the cache moved into VRAM
+    int64_t skipped = 0;              // routed experts STRATA_DS4_DISK_SKIP_W left out (light, not in RAM)
+    int64_t ram_experts = 0, ram_moves = 0;   // CPU-lane experts read from the RAM tier; experts moved into it
+    double prof_ms[6] = {};          // DS4_PROF: the GPU's time in decode by stage (between layers, attention, routing +
+    int64_t prof_tokens = 0;         // shared expert, experts' gate / up, the wait for the CPU lane, down + post)
+    // lookahead (decode windows of one): the picks of layers guessed a layer before (the next layer's gate on this
+    // layer's FFN input), of them in the guess's top k / in all its guesses / in the guess two layers before; the picks
+    // read from the files, of them prefetched early; experts prefetched early
+    int64_t ahead_picks = 0, ahead_hit_k = 0, ahead_hit = 0, ahead_hit2 = 0;
+    int64_t ahead_cold = 0, ahead_cold_hit = 0, ahead_pf = 0;
+    int64_t ahead_rank_n[16] = {}, ahead_rank_hit[16] = {};   // the next-layer guesses by rank, and how many were picked
+    // the RAM store (--ssd-cache), totals since the load: reads asked for and found there (hits), missed (misses), started ahead
+    // of use (prefetched) of which used / given up unused, slots given up, bytes read, ms the CPU lane waited for the reads
+    int64_t store_hits = 0, store_misses = 0, store_prefetched = 0, store_pf_used = 0, store_pf_wasted = 0, store_evictions = 0;
+    int64_t store_bytes = 0, store_fails = 0;
+    int64_t store_zc = 0, store_zc_visible = 0;   // experts the GPU computed from the store (zero-copy over PCIe); slots it may read
+    double store_wait_ms = 0.0, store_busy_ms = 0.0;
+    int64_t store_slots = 0, store_slots_max = 0;
+    int64_t drafts_offered = 0, drafts_accepted = 0, drafts_rounds = 0;
+    // per draft position (the 1st .. 8th of a round): verified with all the drafts before it accepted, and accepted
+    int64_t draft_reached[8] = {}, draft_kept[8] = {};
+    double draft_ms = 0.0, verify_ms = 0.0;   // the drafter's time, the verification windows' time
+};
+
+class Model {
+public:
+    Model();
+    ~Model();
+    bool load(const Options& o, std::string& err);
+
+    /// Feeds `token` at position pos(); with logits, the next token's logits are left on the head's device.
+    bool step(int token, bool logits, std::string& err);
+    /// Feeds n tokens (windows of up to 16, or a long run in chunks of prefill_chunk); the last `logits_rows` of them get
+    /// logits (logits_rows > 16: only a run of one chunk or window).
+    bool feed(const int* tokens, int n, int logits_rows, std::string& err);
+    /// feed() of a prompt's end that also keeps the state before its last token in checkpoint `slot` (a prompt's
+    /// last token then rides along in the run of chunks instead of a window of its own); the same as
+    /// feed(n - 1), checkpoint(slot), feed(1)
+    bool feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err);
+    /// The next token from the last feed's last logit row (greedy when temperature <= 0).
+    int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter);
+    /// The last feed's logit rows on the host (rows x vocab floats).
+    bool logits(std::vector<float>& out);
+
+    /// The DSpark drafter (speculative decoding): its block size (0: not loaded), drafts for the positions after `y`
+    /// (y: the token at pos(), not fed yet) with their confidences, and the verification of [y, drafts]: how many of the
+    /// drafts stand and the token after them (the state then holds y and the accepted drafts).  The drafts are the
+    /// drafter's argmax; a sampled request passes its chain (the same temperature / top_p / top_k / min_p / seed /
+    /// counter as its verify()) so that with DS4_DSPARK_SAMPLE=1 they are drawn from the drafter's distribution, as
+    /// DeepSeek's reference does.  verify() keeps a draft d with probability min(1, p(d) / q(d)) over the request's
+    /// chain - the speculative-sampling rule, so every token is still distributed as the model's own draw.
+    int draft_block() const;
+    bool draft(int y, std::vector<int>& drafts, std::vector<float>& conf, std::string& err, float temperature = 0.0f,
+               float top_p = 1.0f, int top_k = 0, float min_p = 0.0f, uint64_t seed = 0, uint64_t counter = 0);
+    bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,
+                uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err);
+
+    /// Tokens a feed call takes at a time to use the chunked prompt path (all of a prompt: its chunks run as a
+    /// pipeline; 16 without it).
+    int feed_unit() const;
+    /// Called from feed() while a run of chunks goes through the layers, with the tokens done so far.
+    void set_progress(std::function<void(int64_t)> f);
+    /// Asked while a run of chunks goes through the layers: true (a Stop) ends it early - one GPU after the layer it is
+    /// on, several at a chunk all of them reach - and feed() returns false with cancelled() true.  The state past the
+    /// prompt's start is then not usable: restore() a checkpoint kept before the feed, or reset().
+    void set_cancel(std::function<bool()> f);
+    bool cancelled() const;   // the last feed() / feed_mark() ended early on a Stop
+    int64_t pos() const;
+    int64_t max_context() const;
+    int vocab() const;
+    int dim() const;
+    const std::vector<int32_t>& tokens() const;   // what the state has consumed, in order
+
+    void reset();
+    /// Keep the state at the current position in checkpoint slot 0 or 1 (the end of a prompt, the position before it);
+    /// restore(limit) goes back to the latest one at or before limit (-1: none).
+    void checkpoint(int slot = 0);
+    bool restore(int64_t limit);
+    int64_t restorable(int64_t limit) const;
+    /// Checkpoint 1 also becomes the anchor (kept until a restore goes before it): the position before the end of the
+    /// first prompt of a run that continued in place, where a re-rendered history (earlier reasoning dropped) splits.
+    void pin();
+    /// The positions restore() can go back to, latest first (a slot keeps them: it can be taken back at any of them).
+    std::vector<int64_t> check_positions() const;
+
+    /// The conversation in the model into `out` (slot_save): its position, its tokens, every window ring (the DSpark
+    /// blocks' included), the compressors' partial groups and each KV source's compressed KV and index keys up to it,
+    /// the pieces the state a prompt continues is made of.  0 and err: nothing was set aside (it holds pictures, there
+    /// is no conversation, or the conversation is already set aside - unchanged since).
+    uint64_t slot_save(std::vector<uint8_t>& out, std::string& err);
+    /// `data` (a slot's bytes) back into the model: the state at the slot's position n_pos, as slot_save took it -
+    /// what a prompt that continues the slot's tokens reads on from (the caller checked the tokens are a prefix).
+    bool slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err);
+    /// What a saved conversation must match: this engine's version, this model's geometry and files and the drafter's
+    /// blocks - a slot naming anything else is not this model's state (a file's .meta says so; slot_load refuses).
+    std::string slot_identity() const;
+
+    /// The vision tower: the token every position of an image's span carries (-1: no tower).
+    int image_token() const;
+    /// An image's pixels (RGB rows, width x height, both multiples of the patch size: the server's resize) -> the
+    /// rows of its span, BF16 [n_h * (n_w + 1) + 2][dim]: image_start, n_h rows of (n_w aligner rows, image_newline),
+    /// image_end.
+    bool encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err);
+    /// The pictures of the prompt about to be fed ({}: none): positions inside a span read its rows (and route with
+    /// the image bias, and take no part in the Engram n-grams); the rest the embedding table's.
+    void set_images(std::vector<Image> images);
+
+    void save_profile() const;
+    const Stats& stats() const;
+    std::string describe() const;
+    /// INFO / VRAM lines for the server's Monitor.
+    std::vector<std::string> info_lines() const;
+
+    struct Impl;
+
+private:
+    std::unique_ptr<Impl> p_;
+};
+
+}  // namespace strata::ds4

src/dsv41/ds4_mx_kern.inc

@@ -0,0 +1,251 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_mx_kern.inc - the CPU lane's int8 MXFP4 rows (ds4_gguf_cpu.hpp: mx_rows), one body for the x86 tiers.
+// ds4_gguf_cpu.cpp includes it once a tier, inside its unnamed namespace, after <immintrin.h>, with
+//   MXK_NS    the tier's namespace
+//   MXK_ATTR  the tier's instruction set as a function attribute (GCC / Clang: target(...); MSVC: nothing - it takes
+//             any intrinsic) on every function here, so the file is built for the baseline and each tier's code runs
+//             only where ds4_gguf_cpu.cpp's CPU check says it can
+//   MXK_256 or MXK_512, and MXK_VNNI (vpdpbusd; else vpmaddubsw + vpmaddwd)
+// and gets MXK_NS::rows(...) (and with MXK_256 and no MXK_VNNI, MXK_NS::quant(...)); the macros are undefined at the end.
+//
+// The arithmetic is the device's (ds4_gguf.cu, mx_dot) - a block's 32 codes times the int8 activations summed exactly
+// in integers, times (weight scale x activation scale) into F32 sums - in ONE order for every tier, the scalar one
+// included (ds4_gguf_cpu.cpp's rows_q_scalar), so every tier gives the same bits and the choice of tier never changes
+// an answer:
+//   * the blocks go in groups of 8; block j of a group adds into accumulator L(j), L = {0, 4, 1, 5, 2, 6, 3, 7} (the
+//     order a 256-bit hadd tree leaves the 8 blocks' totals in: lanes = blocks 0, 2, 4, 6, 1, 3, 5, 7):
+//     acc[L(j)] = fma(float(total_j), dw_j * dx_j, acc[L(j)]), the groups in turn;
+//   * the row is ((acc0 + acc4) + (acc2 + acc6)) + ((acc1 + acc5) + (acc3 + acc7)) (hsum256's tree).
+// The codes come out of a shuffle table as w + 12 (0..24, unsigned bytes), so the products go through vpmaddubsw /
+// vpdpbusd as u8 x s8; each block's total then gets the activation's -12 x (sum of its 32 values) back, exactly.
+//
+// The activation of a call (mx_act_bytes(k): k % 256 == 0 here; quant() / ds4_gguf_cpu.cpp's quant_scalar):
+//   [0, k)            int8 values in quads of 4 blocks, quad g at 128 g: the 4 blocks' values 0..15 (block j at
+//                     +16 j), then their 16..31 - one 256-bit step takes a pair of blocks, one 512-bit step a quad
+//   [k, k + k/8)      F32 scale of each block, group G (8 blocks) at 8 G in lane order (blocks 0, 2, 4, 6, 1, 3, 5, 7)
+//   [k + k/8, + k/8)  int32 -12 x (sum of the block's 32 values), the same order
+namespace MXK_NS {
+
+constexpr int kBlockBytes = 17;   // E8M0 scale + 16 code bytes
+
+MXK_ATTR inline float hsum256(__m256 v) {
+    const __m128 a = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1));
+    const __m128 b = _mm_add_ps(a, _mm_movehl_ps(a, a));
+    return _mm_cvtss_f32(_mm_add_ss(b, _mm_shuffle_ps(b, b, 1)));
+}
+// the 8 weight scales of a group (its first block at p), in lane order
+MXK_ATTR inline __m256 group_dw(const uint8_t* p, const float* e8) {
+    return _mm256_setr_ps(e8[p[0]], e8[p[2 * kBlockBytes]], e8[p[4 * kBlockBytes]], e8[p[6 * kBlockBytes]],
+                          e8[p[kBlockBytes]], e8[p[3 * kBlockBytes]], e8[p[5 * kBlockBytes]], e8[p[7 * kBlockBytes]]);
+}
+
+#if defined(MXK_256)
+// ---------------------------------------------------------------- 256-bit: a pair of blocks a step
+// a pair's codes (a1: values 0..15 of both blocks, a2: 16..31) times x1 / x2: 8 int32 lanes, block A's 4 then B's 4
+MXK_ATTR inline __m256i pair_dot(__m256i a1, __m256i a2, __m256i x1, __m256i x2) {
+#if defined(MXK_VNNI)
+    return _mm256_dpbusd_avx_epi32(_mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), a1, x1), a2, x2);
+#else
+    const __m256i p = _mm256_add_epi16(_mm256_maddubs_epi16(a1, x1), _mm256_maddubs_epi16(a2, x2));
+    return _mm256_madd_epi16(p, _mm256_set1_epi16(1));
+#endif
+}
+
+template <int N>
+MXK_ATTR inline void rows_n(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, float* const* y,
+                            const float* e8, int pf) {
+    const __m256i lut = _mm256_setr_epi8(12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0,   //
+                                         12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0);
+    const __m256i m4 = _mm256_set1_epi8(0x0f);
+    const int ng = k / 256;
+    const int8_t* xq[N];
+    const float* xd[N];
+    const int32_t* xc[N];
+    for (int i = 0; i < N; ++i) {
+        const uint8_t* a = (const uint8_t*) act[i];
+        xq[i] = (const int8_t*) a;
+        xd[i] = (const float*) (a + k);
+        xc[i] = (const int32_t*) (a + k + k / 8);
+    }
+    for (int r = r0; r < r1; ++r) {
+        const uint8_t* p = w + (size_t) r * rb;
+        __m256 acc[N];
+        for (int i = 0; i < N; ++i) acc[i] = _mm256_setzero_ps();
+        for (int G = 0; G < ng; ++G, p += 8 * kBlockBytes) {
+            if (pf > 0) _mm_prefetch((const char*) p + pf, _MM_HINT_T0);
+            const __m256 dw = group_dw(p, e8);
+            // the group's 4 pairs of blocks (2 q, 2 q + 1) decoded once; then each activation in turn: its 4 pair dots,
+            // the hadd tree to the 8 blocks' totals (lanes: blocks 0 2 4 6 | 1 3 5 7), the FMA - only the accumulators
+            // stay live from one activation to the next (16 YMM registers on AVX2)
+            __m256i a1[4], a2[4];
+            for (int q = 0; q < 4; ++q) {
+                const uint8_t* pb = p + 2 * q * kBlockBytes;
+                const __m256i c = _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*) (pb + 1))),
+                                                          _mm_loadu_si128((const __m128i*) (pb + 18)), 1);
+                a1[q] = _mm256_shuffle_epi8(lut, _mm256_and_si256(c, m4));
+                a2[q] = _mm256_shuffle_epi8(lut, _mm256_and_si256(_mm256_srli_epi16(c, 4), m4));
+            }
+            const int o = G * 256;   // (quads 2 G and 2 G + 1: pair q at (q / 2) * 128 + (q % 2) * 32)
+            for (int i = 0; i < N; ++i) {
+                const int8_t* x = xq[i] + o;
+                const __m256i s0 = pair_dot(a1[0], a2[0], _mm256_loadu_si256((const __m256i*) x),
+                                            _mm256_loadu_si256((const __m256i*) (x + 64)));
+                const __m256i s1 = pair_dot(a1[1], a2[1], _mm256_loadu_si256((const __m256i*) (x + 32)),
+                                            _mm256_loadu_si256((const __m256i*) (x + 96)));
+                const __m256i s2 = pair_dot(a1[2], a2[2], _mm256_loadu_si256((const __m256i*) (x + 128)),
+                                            _mm256_loadu_si256((const __m256i*) (x + 192)));
+                const __m256i s3 = pair_dot(a1[3], a2[3], _mm256_loadu_si256((const __m256i*) (x + 160)),
+                                            _mm256_loadu_si256((const __m256i*) (x + 224)));
+                const __m256i u = _mm256_hadd_epi32(_mm256_hadd_epi32(s0, s1), _mm256_hadd_epi32(s2, s3));
+                const __m256i tot = _mm256_add_epi32(u, _mm256_loadu_si256((const __m256i*) (xc[i] + 8 * G)));
+                acc[i] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(tot), _mm256_mul_ps(dw, _mm256_loadu_ps(xd[i] + 8 * G)), acc[i]);
+            }
+        }
+        for (int i = 0; i < N; ++i) y[i][r - r0] = hsum256(acc[i]);
+    }
+}
+
+#if !defined(MXK_VNNI)
+// blocks [b0, b1) of x (x at block b0's first value) into act, the bytes of quant_scalar (ds4_gguf_cpu.cpp): the same
+// IEEE division and round-half-even conversion, the same max
+MXK_ATTR inline void quant(const float* x, int b0, int b1, uint8_t* act, int k) {
+    int8_t* q = (int8_t*) act;
+    float* dx = (float*) (act + k);
+    int32_t* corr = (int32_t*) (act + k + k / 8);
+    const __m256 sign = _mm256_set1_ps(-0.0f);
+    const __m256i order = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
+    for (int b = b0; b < b1; ++b, x += 32) {
+        const __m256 v0 = _mm256_loadu_ps(x), v1 = _mm256_loadu_ps(x + 8);
+        const __m256 v2 = _mm256_loadu_ps(x + 16), v3 = _mm256_loadu_ps(x + 24);
+        const __m256 m = _mm256_max_ps(_mm256_max_ps(_mm256_andnot_ps(sign, v0), _mm256_andnot_ps(sign, v1)),
+                                       _mm256_max_ps(_mm256_andnot_ps(sign, v2), _mm256_andnot_ps(sign, v3)));
+        __m128 m2 = _mm_max_ps(_mm256_castps256_ps128(m), _mm256_extractf128_ps(m, 1));
+        m2 = _mm_max_ps(m2, _mm_movehl_ps(m2, m2));
+        m2 = _mm_max_ss(m2, _mm_shuffle_ps(m2, m2, 1));
+        const float am = _mm_cvtss_f32(m2);
+        const float sc = am / 127.0f;
+        int8_t* lo = q + (b >> 2) * 128 + 16 * (b & 3);
+        int8_t* hi = lo + 64;
+        int s = 0;
+        if (am > 0.0f) {
+            const __m256 vs = _mm256_set1_ps(sc);
+            const __m256i q0 = _mm256_cvtps_epi32(_mm256_div_ps(v0, vs)), q1 = _mm256_cvtps_epi32(_mm256_div_ps(v1, vs));
+            const __m256i q2 = _mm256_cvtps_epi32(_mm256_div_ps(v2, vs)), q3 = _mm256_cvtps_epi32(_mm256_div_ps(v3, vs));
+            const __m256i pk = _mm256_permutevar8x32_epi32(
+                _mm256_packs_epi16(_mm256_packs_epi32(q0, q1), _mm256_packs_epi32(q2, q3)), order);
+            _mm_storeu_si128((__m128i*) lo, _mm256_castsi256_si128(pk));
+            _mm_storeu_si128((__m128i*) hi, _mm256_extracti128_si256(pk, 1));
+            __m256i t = _mm256_add_epi32(_mm256_add_epi32(q0, q1), _mm256_add_epi32(q2, q3));
+            __m128i t2 = _mm_add_epi32(_mm256_castsi256_si128(t), _mm256_extracti128_si256(t, 1));
+            t2 = _mm_add_epi32(t2, _mm_shuffle_epi32(t2, 0x4E));
+            t2 = _mm_add_epi32(t2, _mm_shuffle_epi32(t2, 0xB1));
+            s = _mm_cvtsi128_si32(t2);
+        } else {
+            _mm_storeu_si128((__m128i*) lo, _mm_setzero_si128());
+            _mm_storeu_si128((__m128i*) hi, _mm_setzero_si128());
+        }
+        const int j = b & 7, lane = 8 * (b >> 3) + (j & 1) * 4 + (j >> 1);
+        dx[lane] = sc;
+        corr[lane] = -12 * s;
+    }
+}
+#endif
+#endif  // MXK_256
+
+#if defined(MXK_512)
+// ---------------------------------------------------------------- 512-bit: a quad of blocks a step
+MXK_ATTR inline __m512i quad_dot(__m512i a1, __m512i a2, __m512i x1, __m512i x2) {
+#if defined(MXK_VNNI)
+    return _mm512_dpbusd_epi32(_mm512_dpbusd_epi32(_mm512_setzero_si512(), a1, x1), a2, x2);
+#else
+    const __m512i p = _mm512_add_epi16(_mm512_maddubs_epi16(a1, x1), _mm512_maddubs_epi16(a2, x2));
+    return _mm512_madd_epi16(p, _mm512_set1_epi16(1));
+#endif
+}
+
+template <int N>
+MXK_ATTR inline void rows_n(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, float* const* y,
+                            const float* e8, int pf) {
+    const __m512i lut = _mm512_broadcast_i32x4(_mm_setr_epi8(12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0));
+    const __m512i m4 = _mm512_set1_epi8(0x0f);
+    // a quad's 16 lanes after adjacent-pair sums: block j's halves in lanes 4 j and 4 j + 2; the group's two quads ->
+    // the 8 blocks in lane order (0 2 4 6 1 3 5 7), first halves (pa) and second halves (pb)
+    const __m512i pa = _mm512_setr_epi32(0, 8, 16, 24, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0);
+    const __m512i pb = _mm512_setr_epi32(2, 10, 18, 26, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0);
+    const int ng = k / 256;
+    const int8_t* xq[N];
+    const float* xd[N];
+    const int32_t* xc[N];
+    for (int i = 0; i < N; ++i) {
+        const uint8_t* a = (const uint8_t*) act[i];
+        xq[i] = (const int8_t*) a;
+        xd[i] = (const float*) (a + k);
+        xc[i] = (const int32_t*) (a + k + k / 8);
+    }
+    for (int r = r0; r < r1; ++r) {
+        const uint8_t* p = w + (size_t) r * rb;
+        __m256 acc[N];
+        for (int i = 0; i < N; ++i) acc[i] = _mm256_setzero_ps();
+        for (int G = 0; G < ng; ++G, p += 8 * kBlockBytes) {
+            if (pf > 0) _mm_prefetch((const char*) p + pf, _MM_HINT_T0);
+            const __m256 dw = group_dw(p, e8);
+            __m512i h0[N];
+            for (int qd = 0; qd < 2; ++qd) {   // the group's quads: blocks 4 qd .. 4 qd + 3
+                const uint8_t* pq = p + 4 * qd * kBlockBytes;
+                __m512i c = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*) (pq + 1)));
+                c = _mm512_inserti32x4(c, _mm_loadu_si128((const __m128i*) (pq + 18)), 1);
+                c = _mm512_inserti32x4(c, _mm_loadu_si128((const __m128i*) (pq + 35)), 2);
+                c = _mm512_inserti32x4(c, _mm_loadu_si128((const __m128i*) (pq + 52)), 3);
+                const __m512i a1 = _mm512_shuffle_epi8(lut, _mm512_and_si512(c, m4));
+                const __m512i a2 = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_srli_epi16(c, 4), m4));
+                const int o = (2 * G + qd) * 128;
+                for (int i = 0; i < N; ++i) {
+                    const __m512i s = quad_dot(a1, a2, _mm512_loadu_si512((const void*) (xq[i] + o)),
+                                               _mm512_loadu_si512((const void*) (xq[i] + o + 64)));
+                    const __m512i h = _mm512_add_epi32(s, _mm512_shuffle_epi32(s, (_MM_PERM_ENUM) 0xB1));
+                    if (qd == 0) {
+                        h0[i] = h;
+                        continue;
+                    }
+                    const __m512i ta = _mm512_permutex2var_epi32(h0[i], pa, h), tb = _mm512_permutex2var_epi32(h0[i], pb, h);
+                    const __m256i tot = _mm256_add_epi32(
+                        _mm256_add_epi32(_mm512_castsi512_si256(ta), _mm512_castsi512_si256(tb)),
+                        _mm256_loadu_si256((const __m256i*) (xc[i] + 8 * G)));
+                    acc[i] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(tot), _mm256_mul_ps(dw, _mm256_loadu_ps(xd[i] + 8 * G)), acc[i]);
+                }
+            }
+        }
+        for (int i = 0; i < N; ++i) y[i][r - r0] = hsum256(acc[i]);
+    }
+}
+#endif  // MXK_512
+
+// y[i][r - r0] for every i < n, in calls of up to 8 activations; e8 the 256 E8M0 scales as F32 (2^(e - 128)), pf the
+// bytes to prefetch ahead in a row (0: none)
+MXK_ATTR inline void rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n,
+                          float* const* y, const float* e8, int pf) {
+    for (int i0 = 0; i0 < n; i0 += 8) {
+        float* yy[8];
+        const int c = n - i0 < 8 ? n - i0 : 8;
+        for (int i = 0; i < c; ++i) yy[i] = y[i0 + i];
+        switch (c) {
+            case 1: rows_n<1>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+            case 2: rows_n<2>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+            case 3: rows_n<3>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+            case 4: rows_n<4>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+            case 5: rows_n<5>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+            case 6: rows_n<6>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+            case 7: rows_n<7>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+            default: rows_n<8>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+        }
+    }
+}
+
+}  // namespace MXK_NS
+
+#undef MXK_NS
+#undef MXK_ATTR
+#undef MXK_256
+#undef MXK_512
+#undef MXK_VNNI

src/dsv41/ds4_numerics.cuh

@@ -0,0 +1,65 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_numerics.cuh - the BF16 and FP8 (E4M3) conversions of maya-ds4's kernels, on CUDA through its
+// intrinsics (cuda_bf16.h / cuda_fp8.h) and on HIP in plain arithmetic that gives the same bits: BF16 round to
+// nearest even; E4M3 (OCP "fn": no infinities, max 448) decoded exactly, rounded to nearest even and saturated.
+#pragma once
+
+#include <cstdint>
+#include <cuda_runtime.h>
+#if !defined(STRATA_USE_HIP)
+#include <cuda_bf16.h>
+#include <cuda_fp16.h>
+#include <cuda_fp8.h>
+#endif
+
+namespace strata::ds4 {
+
+/// F32 -> BF16 bits, round to nearest even (a NaN stays a NaN).
+__device__ __forceinline__ uint16_t bf16_rn(float f) {
+#if !defined(STRATA_USE_HIP)
+    return __bfloat16_as_ushort(__float2bfloat16_rn(f));
+#else
+    const uint32_t u = __float_as_uint(f);
+    if ((u & 0x7fffffffu) > 0x7f800000u) return (uint16_t) ((u >> 16) | 0x40u);
+    return (uint16_t) ((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);
+#endif
+}
+
+/// An E4M3 code's value (exact in F32).
+__device__ __forceinline__ float e4m3_value(uint8_t q) {
+#if !defined(STRATA_USE_HIP)
+    const __half_raw h = __nv_cvt_fp8_to_halfraw((__nv_fp8_storage_t) q, __NV_E4M3);
+    return __half2float(__half(h));
+#else
+    const uint32_t e = (q >> 3) & 15u, m = q & 7u;
+    float v;
+    if (e == 15u && m == 7u) v = __uint_as_float(0x7fc00000u);                         // NaN
+    else if (e == 0u) v = (float) m * 0.001953125f;                                      // m 2^-9
+    else v = __uint_as_float(((e + 120u) << 23) | (m << 20));                           // (1 + m / 8) 2^(e - 7)
+    return (q & 0x80u) ? -v : v;
+#endif
+}
+
+/// v rounded to the nearest E4M3 value (ties to even), saturated at +-448.
+__device__ __forceinline__ float e4m3_round(float v) {
+#if !defined(STRATA_USE_HIP)
+    const __nv_fp8_storage_t q = __nv_cvt_float_to_fp8(v, __NV_SATFINITE, __NV_E4M3);
+    const __half_raw h = __nv_cvt_fp8_to_halfraw(q, __NV_E4M3);
+    return __half2float(__half(h));
+#else
+    const float a = fabsf(v);
+    if (!(a == a)) return v;
+    float r;
+    if (a >= 448.0f) r = 448.0f;
+    else {
+        const int e = (int) ((__float_as_uint(a) >> 23) & 0xffu) - 127;   // a's binade (a > 0)
+        const float step = e < -6 ? 0.001953125f : __uint_as_float((uint32_t) (e - 3 + 127) << 23);
+        r = a == 0.0f ? 0.0f : rintf(a / step) * step;                      // (a power of two: exact)
+        r = fminf(r, 448.0f);
+    }
+    return v < 0.0f ? -r : r;
+#endif
+}
+
+}  // namespace strata::ds4

src/dsv41/ds4_os.hpp

@@ -0,0 +1,561 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_os.hpp - the few OS services maya-ds4 needs, on Linux and on Windows: files read at an offset and
+// mapped read-only (the experts read in place), page-cache hints, anonymous host memory on a NUMA node, thread pinning,
+// each NUMA node's physical cores, the RAM this process can still get.  (On Windows as src/core/expert_source.cpp
+// does it for GLM: no FILE_FLAG_RANDOM_ACCESS - the page cache must keep what it read.)
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <sstream>
+#include <string>
+#include <thread>
+#include <vector>
+
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <psapi.h>   // (K32QueryWorkingSetEx: kernel32's, no psapi.lib)
+#include <timeapi.h>
+#ifdef _MSC_VER
+#pragma comment(lib, "winmm.lib")
+#endif
+#else
+#include <fcntl.h>
+#include <pthread.h>
+#include <sched.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#endif
+
+namespace strata::ds4::os {
+
+/// Windows sleeps in 15.6 ms ticks unless a process asks for finer ones: the CPU lane's service thread naps (sleep_for 20 us)
+/// between tokens, and the device's first wait of the next token then waits out the tick.  1 ms for the process's life
+/// (Project Maya #358).  Off by default (owner PC, shipped profile, 3 pairs: not faster in every pair); DS4_TIMER_MS=1 turns it on.
+/// Elsewhere nothing.
+inline void fine_timer() {
+#if defined(_WIN32)
+    const char* v = std::getenv("DS4_TIMER_MS");
+    const int ms = v != nullptr ? std::atoi(v) : 0;
+    if (ms > 0) timeBeginPeriod((UINT) ms);
+#endif
+}
+
+using fd_t = intptr_t;
+constexpr fd_t kBadFd = -1;
+
+#if defined(_WIN32)
+inline std::wstring wide(const std::string& s) {   // UTF-8 -> UTF-16 (a path with non-ASCII characters)
+    const int n = MultiByteToWideChar(CP_UTF8, 0, s.c_str(), -1, nullptr, 0);
+    std::wstring w((size_t) (n > 0 ? n : 1), L'\0');
+    if (n > 0) MultiByteToWideChar(CP_UTF8, 0, s.c_str(), -1, &w[0], n);
+    return w;
+}
+#endif
+
+/// A file opened for reading (kBadFd: it could not be).
+inline fd_t open_ro(const std::string& path) {
+#if defined(_WIN32)
+    HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                           FILE_ATTRIBUTE_NORMAL, nullptr);
+    return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;
+#else
+    const int fd = ::open(path.c_str(), O_RDONLY);
+    return fd < 0 ? kBadFd : (fd_t) fd;
+#endif
+}
+/// The same file opened for unbuffered reads (Linux: O_DIRECT - no page cache, the disk straight into the buffer;
+/// kBadFd where the filesystem refuses it, or on Windows, whose unbuffered reads of a mapped file take a slow path)
+inline fd_t open_direct(const std::string& path) {
+#if defined(_WIN32)
+    // FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED: the disk straight into an aligned buffer, any number of reads in flight on
+    // one handle (DS4_WIN_DIRECT=0: none, the page cache's buffered reads as before)
+    static const bool off = [] {
+        const char* v = std::getenv("DS4_WIN_DIRECT");
+        return v != nullptr && std::atoi(v) == 0;
+    }();
+    if (off) return kBadFd;
+    HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
+                           FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);
+    return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;
+#else
+    const int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT);
+    return fd < 0 ? kBadFd : (fd_t) fd;
+#endif
+}
+/// [off, off + n) through an unbuffered descriptor into dst, a kDirectAlign-aligned buffer of at least
+/// direct_span(off, n) bytes: the file read from off rounded down, so the bytes land at dst + (off % kDirectAlign).
+/// False if a read fails (the caller reads buffered instead).
+constexpr uint64_t kDirectAlign = 4096;
+inline uint64_t direct_span(uint64_t off, uint64_t n) {
+    const uint64_t a0 = off & ~(kDirectAlign - 1), a1 = (off + n + kDirectAlign - 1) & ~(kDirectAlign - 1);
+    return a1 - a0;
+}
+inline bool pread_direct(fd_t f, void* dst, uint64_t n, uint64_t off) {
+#if defined(_WIN32)
+    struct Ev {   // (each thread its own, closed when it ends)
+        HANDLE h = CreateEventW(nullptr, TRUE, FALSE, nullptr);
+        ~Ev() {
+            if (h != nullptr) CloseHandle(h);
+        }
+    };
+    static thread_local Ev ev;
+    if (ev.h == nullptr) return false;
+    const uint64_t a0 = off & ~(kDirectAlign - 1), need = direct_span(off, n), want = (off - a0) + n;
+    uint8_t* p = (uint8_t*) dst;
+    uint64_t got = 0;
+    while (got < need) {
+        const DWORD ask = (DWORD) std::min<uint64_t>(need - got, 1ull << 30);   // (a multiple of 4096)
+        OVERLAPPED ov{};
+        const uint64_t at = a0 + got;
+        ov.Offset = (DWORD) (at & 0xffffffffu);
+        ov.OffsetHigh = (DWORD) (at >> 32);
+        ov.hEvent = ev.h;
+        ResetEvent(ev.h);
+        DWORD r = 0;
+        if (!ReadFile((HANDLE) f, p + got, ask, nullptr, &ov)) {
+            const DWORD e = GetLastError();
+            if (e == ERROR_HANDLE_EOF) break;
+            if (e != ERROR_IO_PENDING) return false;
+        }
+        if (!GetOverlappedResult((HANDLE) f, &ov, &r, TRUE)) {
+            if (GetLastError() == ERROR_HANDLE_EOF) break;
+            return false;
+        }
+        if (r == 0) break;   // (the file's end: the last aligned read is short)
+        got += r;
+        if (r % kDirectAlign != 0) break;
+    }
+    return got >= want;
+#else
+    const uint64_t a0 = off & ~(kDirectAlign - 1), need = direct_span(off, n), want = (off - a0) + n;
+    uint8_t* p = (uint8_t*) dst;
+    uint64_t got = 0;
+    while (got < need) {
+        const ssize_t r = ::pread((int) f, p + got, (size_t) std::min<uint64_t>(need - got, 1ull << 30), (off_t) (a0 + got));
+        if (r < 0) return false;
+        if (r == 0) break;   // (the file's end: the last aligned read is short)
+        got += (uint64_t) r;
+        if ((uint64_t) r % kDirectAlign != 0) break;
+    }
+    return got >= want;
+#endif
+}
+/// Windows: the file opened for unbuffered reads (FILE_FLAG_NO_BUFFERING), for a file no view of this process maps -
+/// unbuffered reads of a mapped file take a slow path there (open_direct); kBadFd elsewhere
+inline fd_t open_unbuffered(const std::string& path) {
+#if defined(_WIN32)
+    HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                           FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING, nullptr);
+    return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;
+#else
+    (void) path;
+    return kBadFd;
+#endif
+}
+/// [off, off + n) through open_unbuffered's handle (one thread's) into buf, kDirectAlign-aligned and at least
+/// direct_span(off, n) bytes: the bytes land at buf + off % kDirectAlign
+inline bool pread_unbuffered(fd_t f, void* buf, uint64_t n, uint64_t off) {
+#if defined(_WIN32)
+    const uint64_t a0 = off & ~(kDirectAlign - 1), span = direct_span(off, n);
+    OVERLAPPED ov{};
+    ov.Offset = (DWORD) a0;
+    ov.OffsetHigh = (DWORD) (a0 >> 32);
+    DWORD got = 0;
+    return ReadFile((HANDLE) f, buf, (DWORD) span, &got, &ov) && got >= (off - a0) + n;
+#else
+    (void) f, (void) buf, (void) n, (void) off;
+    return false;
+#endif
+}
+/// The same file opened for reads from many threads at once (Windows runs the reads of a synchronous handle one at a
+/// time, so an overlapped one; Linux: a plain descriptor, pread is concurrent anyway)
+inline fd_t open_par(const std::string& path) {
+#if defined(_WIN32)
+    HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                           FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, nullptr);
+    return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;
+#else
+    const int fd = ::open(path.c_str(), O_RDONLY);
+    return fd < 0 ? kBadFd : (fd_t) fd;
+#endif
+}
+/// n bytes at off through open_par's handle (any thread, concurrently)
+inline bool pread_par(fd_t f, void* dst, uint64_t n, uint64_t off) {
+#if defined(_WIN32)
+    struct Ev {   // (each thread its own, closed when it ends)
+        HANDLE h = CreateEventW(nullptr, TRUE, FALSE, nullptr);
+        ~Ev() {
+            if (h != nullptr) CloseHandle(h);
+        }
+    };
+    static thread_local Ev ev;
+    if (ev.h == nullptr) return false;
+    uint8_t* p = (uint8_t*) dst;
+    while (n > 0) {
+        const DWORD want = (DWORD) std::min<uint64_t>(n, 1ull << 30);
+        OVERLAPPED ov{};
+        ov.Offset = (DWORD) (off & 0xffffffffu);
+        ov.OffsetHigh = (DWORD) (off >> 32);
+        ov.hEvent = ev.h;
+        ResetEvent(ev.h);
+        DWORD got = 0;
+        if (!ReadFile((HANDLE) f, p, want, nullptr, &ov) && GetLastError() != ERROR_IO_PENDING) return false;
+        if (!GetOverlappedResult((HANDLE) f, &ov, &got, TRUE) || got == 0) return false;
+        p += got, off += got, n -= got;
+    }
+    return true;
+#else
+    uint8_t* p = (uint8_t*) dst;
+    while (n > 0) {
+        const ssize_t r = ::pread((int) f, p, (size_t) std::min<uint64_t>(n, 1ull << 30), (off_t) off);
+        if (r <= 0) return false;
+        p += r, off += (uint64_t) r, n -= (uint64_t) r;
+    }
+    return true;
+#endif
+}
+/// `from` renamed to `to`, replacing `to` if it exists (rename(2) does; Windows' rename fails on an existing file, so
+/// MoveFileExW with MOVEFILE_REPLACE_EXISTING)
+inline bool replace_file(const std::string& from, const std::string& to) {
+#if defined(_WIN32)
+    return MoveFileExW(wide(from).c_str(), wide(to).c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0;
+#else
+    return std::rename(from.c_str(), to.c_str()) == 0;
+#endif
+}
+inline void close_fd(fd_t f) {
+    if (f == kBadFd) return;
+#if defined(_WIN32)
+    CloseHandle((HANDLE) f);
+#else
+    ::close((int) f);
+#endif
+}
+inline uint64_t file_size(fd_t f) {
+#if defined(_WIN32)
+    LARGE_INTEGER sz{};
+    return GetFileSizeEx((HANDLE) f, &sz) ? (uint64_t) sz.QuadPart : 0;
+#else
+    struct stat sb;
+    return ::fstat((int) f, &sb) == 0 && sb.st_size > 0 ? (uint64_t) sb.st_size : 0;
+#endif
+}
+/// n bytes at offset off into dst (any thread, concurrently); false if the file ends first or a read fails.
+inline bool pread_full(fd_t f, void* dst, uint64_t n, uint64_t off) {
+    uint8_t* p = (uint8_t*) dst;
+    while (n > 0) {
+        const uint64_t want = std::min<uint64_t>(n, 1ull << 30);
+#if defined(_WIN32)
+        OVERLAPPED ov{};
+        ov.Offset = (DWORD) (off & 0xffffffffu);
+        ov.OffsetHigh = (DWORD) (off >> 32);
+        DWORD got = 0;
+        if (!ReadFile((HANDLE) f, p, (DWORD) want, &got, &ov) || got == 0) return false;
+        const uint64_t r = got;
+#else
+        const ssize_t rr = ::pread((int) f, p, (size_t) want, (off_t) off);
+        if (rr <= 0) return false;
+        const uint64_t r = (uint64_t) rr;
+#endif
+        p += r, off += r, n -= r;
+    }
+    return true;
+}
+/// The whole file mapped read-only (nullptr: it could not be).
+inline void* map_ro(fd_t f, uint64_t bytes) {
+    if (bytes == 0) return nullptr;
+#if defined(_WIN32)
+    HANDLE m = CreateFileMappingW((HANDLE) f, nullptr, PAGE_READONLY, 0, 0, nullptr);
+    if (m == nullptr) return nullptr;
+    void* v = MapViewOfFile(m, FILE_MAP_READ, 0, 0, 0);
+    CloseHandle(m);   // (the view keeps the mapping)
+    return v;
+#else
+    void* p = ::mmap(nullptr, (size_t) bytes, PROT_READ, MAP_SHARED, (int) f, 0);
+    return p == MAP_FAILED ? nullptr : p;
+#endif
+}
+inline void unmap(void* p, uint64_t bytes) {
+    if (p == nullptr) return;
+#if defined(_WIN32)
+    (void) bytes;
+    UnmapViewOfFile(p);
+#else
+    ::munmap(p, (size_t) bytes);
+#endif
+}
+/// A hint that [off, off + n) of the file is needed soon (need) or not again soon (!need); nothing where the OS has
+/// no such hint for a file (Windows' page cache reads ahead on its own).
+inline void advise(fd_t f, uint64_t off, uint64_t n, bool need) {
+#if defined(_WIN32)
+    (void) f, (void) off, (void) n, (void) need;
+#else
+    posix_fadvise((int) f, (off_t) off, (off_t) n, need ? POSIX_FADV_WILLNEED : POSIX_FADV_DONTNEED);
+#endif
+}
+
+/// Ranges of a file's mapping about to be read: the OS reads them in now, with large requests side by side, instead of
+/// a page fault at a time later (Windows: PrefetchVirtualMemory - cold expert-sized pieces came in 5x faster on an NVMe;
+/// Linux: madvise WILLNEED, the kernel's read-ahead of the whole range).  Best effort: a failure changes nothing.
+inline void prefetch(const std::vector<std::pair<const void*, size_t>>& ranges) {
+    if (ranges.empty()) return;
+#if defined(_WIN32)
+    std::vector<WIN32_MEMORY_RANGE_ENTRY> r(ranges.size());
+    for (size_t i = 0; i < ranges.size(); ++i) {
+        r[i].VirtualAddress = const_cast<void*>(ranges[i].first);
+        r[i].NumberOfBytes = ranges[i].second;
+    }
+    PrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) r.size(), r.data(), 0);
+#else
+    const uintptr_t pg = 4096;
+    for (const auto& x : ranges) {
+        const uintptr_t a = (uintptr_t) x.first, b = a & ~(pg - 1);
+        madvise((void*) b, x.second + (a - b), MADV_WILLNEED);
+    }
+#endif
+}
+
+/// Whether every page of [p, p + n) of a file's mapping is in the page cache (Linux mincore; elsewhere false: the
+/// caller reads buffered anyway, which the cache serves)
+inline bool resident(const void* p, size_t n) {
+#if defined(_WIN32)
+    (void) p, (void) n;
+    return false;
+#else
+    const uintptr_t pg = 4096, a = (uintptr_t) p, b = a & ~(pg - 1);
+    const size_t len = n + (a - b), np = (len + pg - 1) / pg;
+    thread_local std::vector<unsigned char> v;
+    if (v.size() < np) v.resize(np);
+    if (mincore((void*) b, len, v.data()) != 0) return false;
+    for (size_t i = 0; i < np; ++i)
+        if ((v[i] & 1) == 0) return false;
+    return true;
+#endif
+}
+
+/// Whether a range of a file's mapping looks resident, so that prefetch() would only cost CPU time (it walks every page:
+/// ~50 ns a 4 KiB page on Windows - 0.34 ms for a 6.3 MB expert matrix on a Strix Halo): `samples` pages spread over
+/// it all resident (Linux: mincore, the page cache; Windows: QueryWorkingSetEx, the process's working set - a page on
+/// the standby list counts as not resident, and prefetch() is what brings those back cheaply).  An expert's pages are
+/// read together, so they come and go together: a wrong "yes" costs page faults, a wrong "no" a prefetch.
+inline bool resident_sampled(const void* p, size_t bytes, int samples = 8) {
+    if (bytes == 0) return true;
+    const uintptr_t pg = 4096, a = (uintptr_t) p, span = bytes - 1;
+    samples = std::max(1, std::min(samples, 16));
+#if defined(_WIN32)
+    PSAPI_WORKING_SET_EX_INFORMATION w[16] = {};
+    for (int i = 0; i < samples; ++i)
+        w[i].VirtualAddress = (PVOID) ((a + (samples > 1 ? span / (samples - 1) * i : 0)) & ~(pg - 1));
+    if (!K32QueryWorkingSetEx(GetCurrentProcess(), w, (DWORD) (samples * sizeof w[0]))) return false;
+    for (int i = 0; i < samples; ++i)
+        if (!w[i].VirtualAttributes.Valid) return false;
+    return true;
+#else
+    for (int i = 0; i < samples; ++i) {
+        unsigned char v = 0;
+        const uintptr_t at = (a + (samples > 1 ? span / (samples - 1) * i : 0)) & ~(pg - 1);
+        if (mincore((void*) at, pg, &v) != 0 || !(v & 1)) return false;
+    }
+    return true;
+#endif
+}
+/// prefetch() of the ranges that do not look resident (resident_sampled) - on Windows, where PrefetchVirtualMemory walks
+/// every page of a resident range (0.34 ms a 6.3 MB expert matrix on a Strix Halo); Linux's madvise(WILLNEED) on cached
+/// pages costs ~1 us a matrix (a Core i5-12600T), less than the residency check (2.5 us): every range there
+inline void prefetch_cold(const std::vector<std::pair<const void*, size_t>>& ranges) {
+#if defined(_WIN32)
+    static const bool all = [] {   // (DS4_PF_ALL=1: every range, as before)
+        const char* v = std::getenv("DS4_PF_ALL");
+        return v != nullptr && std::atoi(v) != 0;
+    }();
+    if (all) return prefetch(ranges);
+    std::vector<std::pair<const void*, size_t>> cold;
+    for (const auto& x : ranges)
+        if (!resident_sampled(x.first, x.second)) cold.push_back(x);
+    prefetch(cold);
+#else
+    prefetch(ranges);
+#endif
+}
+
+/// Whether [p, p + n) of a file's mapping is in RAM now: every page (Linux, one mincore), sampled pages of the working
+/// set (Windows; a page on the standby list counts as not in RAM)
+inline bool in_ram(const void* p, size_t n) {
+#if defined(_WIN32)
+    return resident_sampled(p, n, 8);
+#else
+    return resident(p, n);
+#endif
+}
+
+/// Anonymous memory on NUMA node `node` of `nodes` (node < 0 or nodes < 2: wherever the first touch puts it), on huge
+/// pages where the OS gives them; nullptr if it cannot be had.
+inline void* anon_alloc(size_t bytes, int node, int nodes) {
+#if defined(_WIN32)
+    if (node >= 0 && nodes > 1)
+        return VirtualAllocExNuma(GetCurrentProcess(), nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE,
+                                  (DWORD) node);
+    return VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+#else
+    void* m = ::mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (m == MAP_FAILED) return nullptr;
+    madvise(m, bytes, MADV_HUGEPAGE);
+    if (node >= 0 && nodes > 1) {
+        unsigned long mask[4] = {0, 0, 0, 0};
+        mask[node / 64] = 1ul << (node % 64);
+        syscall(SYS_mbind, m, bytes, 2 /* MPOL_BIND */, mask, 256, 0);
+    }
+    return m;
+#endif
+}
+/// An address range of `bytes` reserved (no memory behind it yet), and pieces of it made real: the store's arena is one range so a
+/// kernel can tell its pointers by [lo, hi).  nullptr: it could not be had.
+inline void* range_reserve(size_t bytes) {
+#if defined(_WIN32)
+    return VirtualAlloc(nullptr, bytes, MEM_RESERVE, PAGE_READWRITE);
+#else
+    void* m = ::mmap(nullptr, bytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+    return m == MAP_FAILED ? nullptr : m;
+#endif
+}
+inline bool range_commit(void* p, size_t bytes) {
+#if defined(_WIN32)
+    return VirtualAlloc(p, bytes, MEM_COMMIT, PAGE_READWRITE) != nullptr;
+#else
+    return ::mprotect(p, bytes, PROT_READ | PROT_WRITE) == 0;
+#endif
+}
+/// A committed piece of a reserved range given back to the system (its address range stays reserved)
+inline void range_decommit(void* p, size_t bytes) {
+#if defined(_WIN32)
+    VirtualFree(p, bytes, MEM_DECOMMIT);
+#else
+    ::madvise(p, bytes, MADV_DONTNEED);
+    ::mprotect(p, bytes, PROT_NONE);
+#endif
+}
+inline void range_release(void* p, size_t bytes) {
+    if (p == nullptr) return;
+#if defined(_WIN32)
+    (void) bytes;
+    VirtualFree(p, 0, MEM_RELEASE);
+#else
+    ::munmap(p, bytes);
+#endif
+}
+inline void anon_free(void* p, size_t bytes) {
+    if (p == nullptr) return;
+#if defined(_WIN32)
+    (void) bytes;
+    VirtualFree(p, 0, MEM_RELEASE);
+#else
+    ::munmap(p, bytes);
+#endif
+}
+
+/// The calling thread, or `t`, on logical CPU `cpu` (Windows: the first 64 CPUs, processor group 0).
+inline void pin_self(int cpu) {
+#if defined(_WIN32)
+    if (cpu >= 0 && cpu < 64) SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR) 1 << cpu);
+#else
+    cpu_set_t set;
+    CPU_ZERO(&set);
+    CPU_SET(cpu, &set);
+    sched_setaffinity(0, sizeof set, &set);
+#endif
+}
+inline void pin_thread(std::thread& t, int cpu) {
+#if defined(_WIN32)
+    if (cpu >= 0 && cpu < 64) SetThreadAffinityMask((HANDLE) t.native_handle(), (DWORD_PTR) 1 << cpu);
+#else
+    cpu_set_t set;
+    CPU_ZERO(&set);
+    CPU_SET(cpu, &set);
+    pthread_setaffinity_np(t.native_handle(), sizeof(set), &set);
+#endif
+}
+
+#if defined(_WIN32)
+/// Each NUMA node's physical cores (the first logical CPU of each), processor group 0 - Windows' answer to
+/// /sys/devices/system/node; empty if the OS does not say.
+inline std::vector<std::vector<int>> node_cores(bool smt = false) {   // (smt: every logical CPU of a core, not its first)
+    std::vector<std::vector<int>> nodes;
+    auto query = [](LOGICAL_PROCESSOR_RELATIONSHIP rel, std::vector<uint8_t>& buf) {
+        DWORD len = 0;
+        GetLogicalProcessorInformationEx(rel, nullptr, &len);
+        buf.resize(len);
+        return len > 0 && GetLogicalProcessorInformationEx(rel, (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) buf.data(), &len);
+    };
+    std::vector<uint8_t> nb, cb;
+    if (!query(RelationNumaNode, nb) || !query(RelationProcessorCore, cb)) return nodes;
+    std::vector<KAFFINITY> masks;
+    for (size_t o = 0; o < nb.size();) {
+        auto* e = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (nb.data() + o);
+        if (e->NumaNode.GroupMask.Group == 0) masks.push_back(e->NumaNode.GroupMask.Mask);
+        o += e->Size;
+    }
+    nodes.resize(masks.size());
+    for (size_t o = 0; o < cb.size();) {
+        auto* e = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (cb.data() + o);
+        if (e->Processor.GroupCount > 0 && e->Processor.GroupMask[0].Group == 0) {
+            const KAFFINITY m = e->Processor.GroupMask[0].Mask;
+            int first = -1;
+            for (int b = 0; b < 64; ++b)
+                if ((m >> b) & 1) {
+                    first = b;
+                    break;
+                }
+            for (size_t n = 0; n < masks.size() && first >= 0; ++n)
+                if ((masks[n] >> first) & 1) {
+                    nodes[n].push_back(first);
+                    if (smt)
+                        for (int b = first + 1; b < 64; ++b)
+                            if ((m >> b) & 1) nodes[n].push_back(b);
+                }
+        }
+        o += e->Size;
+    }
+    nodes.erase(std::remove_if(nodes.begin(), nodes.end(), [](const std::vector<int>& v) { return v.empty(); }),
+                nodes.end());
+    for (auto& v : nodes) std::sort(v.begin(), v.end());
+    return nodes;
+}
+/// The RAM this process can still get (~0: not known).
+inline uint64_t ram_avail() {
+    MEMORYSTATUSEX s{};
+    s.dwLength = sizeof(s);
+    return GlobalMemoryStatusEx(&s) ? (uint64_t) s.ullAvailPhys : ~uint64_t{0};
+}
+#endif
+
+/// Windows' commit (RAM + page file, what every private allocation is charged to - on an integrated GPU the GPU's
+/// own allocations too): the limit, what this process can still commit, the RAM; all ~0 where there is no such limit
+/// (Linux overcommits; its RAM is ram_avail's business).
+struct Commit {
+    uint64_t limit = ~uint64_t{0}, avail = ~uint64_t{0}, ram = ~uint64_t{0};
+};
+inline Commit commit() {
+    Commit c;
+#if defined(_WIN32)
+    MEMORYSTATUSEX s{};
+    s.dwLength = sizeof(s);
+    if (GlobalMemoryStatusEx(&s)) {
+        c.limit = (uint64_t) s.ullTotalPageFile;
+        c.avail = (uint64_t) s.ullAvailPageFile;
+        c.ram = (uint64_t) s.ullTotalPhys;
+    }
+#endif
+    return c;
+}
+
+}  // namespace strata::ds4::os

src/dsv41/ds4_packinfo.cpp

@@ -0,0 +1,273 @@
+// Strata. `strata-dsv41 --pack-info`: the pack as a description, before anything is loaded onto a GPU - the type map
+// (layer x role), the bytes an expert and a token, the tier sizes, and the tensors whose type has no device kernel (they
+// would go to the CPU lane, or stop the load).  Host only, no device, no weights read: tensors.txt and ds4.txt are the
+// pack's whole description (tools/dsv41/pack_gguf.py).  The same report is the first line of every benchmark log.
+#include "ds4_packinfo.hpp"
+#include "ds4_iq.cuh"      // iq::supported / iq::name: the device-type list below is derived from the kernels, not copied
+
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#if defined(_MSC_VER)
+#include <cstring>
+#define strtok_r strtok_s
+#endif
+namespace strata::ds4 {
+namespace {
+
+// The ggml block types the device kernels take (ds4_iq.cu's Ty<> / DS4_IQ_SWITCH + ds4_gguf.cu's MXFP4).  A name here
+// and not in the pack's tensor list is fine; a tensor whose type is not here is the CPU lane's (ggml-cpu's row dots).
+// It was a copy of the supported() switch and fell behind when IQ1_S, IQ1_M and IQ4_NL got kernels (2026-10-23), so it is
+// now the switch itself, by type name.
+
+bool device_type(const std::string& t) {
+    static const std::set<std::string> ok = [] {
+        std::set<std::string> s = {"MXFP4", "TQK6", "TQK7", "TQK5X", "TQK4X", "TQK5T", "TQK4T"};   // ds4_gguf.cu / trellis
+        for (int ty = 0; ty < 64; ++ty) {
+            const char* n = gguf::iq::name(ty);
+            if (n && std::strcmp(n, "?") != 0 && gguf::iq::supported(ty)) s.insert(n);
+        }
+        return s;
+    }();
+    return ok.count(t) != 0;
+}
+// The types the dense path reads as they are (ds4_gguf.cu: BF16 / F32 / MXFP8 rows, Q8_0 rows; ds4_iq's GEMV for the
+// K-quants is the same device support as the experts').
+bool dense_type(const std::string& t) {
+    return t == "BF16" || t == "F32" || t == "F16" || t == "MXFP8" || t == "Q8_0" || device_type(t);
+}
+
+struct Tensor {
+    std::string dtype;
+    double bytes = 0;
+    std::vector<long long> shape;
+};
+
+double gb(double b) { return b / 1e9; }
+
+}  // namespace
+
+int pack_info(const std::string& dir, std::FILE* out) {
+    std::map<std::string, std::string> kv;
+    {
+        std::ifstream f(dir + "/ds4.txt");
+        if (!f) {
+            std::fprintf(out, "pack-info: %s/ds4.txt: not found\n", dir.c_str());
+            return 1;
+        }
+        std::string line;
+        while (std::getline(f, line)) {
+            const size_t sp = line.find(' ');
+            if (sp != std::string::npos) kv[line.substr(0, sp)] = line.substr(sp + 1);
+        }
+    }
+    auto num = [&](const char* k, long long d) {
+        auto it = kv.find(k);
+        return it == kv.end() ? d : std::atoll(it->second.c_str());
+    };
+    const long long nl = num("num_hidden_layers", 0), ne = num("n_routed_experts", 0), topk = num("num_experts_per_tok", 0);
+    const long long dim = num("hidden_size", 0), ff = num("moe_intermediate_size", 0), vocab = num("vocab_size", 0);
+    std::map<std::string, Tensor> t;
+    std::vector<std::string> files;
+    std::map<int, double> need;   // bytes the pack needs from each file (its largest tensor end)
+    {
+        std::FILE* f = std::fopen((dir + "/tensors.txt").c_str(), "r");
+        if (f == nullptr) {
+            std::fprintf(out, "pack-info: %s/tensors.txt: not found\n", dir.c_str());
+            return 1;
+        }
+        std::vector<char> buf(1 << 16);
+        while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {
+            char* s = buf.data();
+            if (s[0] == 'f') {   // file <i> <name>
+                char name[4096];
+                int i = 0;
+                if (std::sscanf(s, "file %d %4095s", &i, name) == 2) {
+                    if ((int) files.size() <= i) files.resize((size_t) i + 1);
+                    files[(size_t) i] = name;
+                }
+                continue;
+            }
+            if (s[0] != 't') continue;
+            char* save = nullptr;
+            ::strtok_r(s, " \n", &save);
+            const char* name = ::strtok_r(nullptr, " \n", &save);
+            const char* dt = ::strtok_r(nullptr, " \n", &save);
+            const char* fi = ::strtok_r(nullptr, " \n", &save);
+            const char* off = ::strtok_r(nullptr, " \n", &save);
+            const char* nb = ::strtok_r(nullptr, " \n", &save);
+            if (name == nullptr || dt == nullptr || fi == nullptr || off == nullptr || nb == nullptr) continue;
+            Tensor x;
+            x.dtype = dt;
+            x.bytes = (double) std::strtoull(nb, nullptr, 10);
+            const int idx = std::atoi(fi);
+            need[idx] = std::max(need[idx], (double) std::strtoull(off, nullptr, 10) + x.bytes);
+            while (const char* d = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(d));
+            t[name] = std::move(x);
+        }
+        std::fclose(f);
+    }
+    if (t.empty()) {
+        std::fprintf(out, "pack-info: %s/tensors.txt: no tensors\n", dir.c_str());
+        return 1;
+    }
+    // the files the pack points at: they must be there, and at least as long as the largest tensor end says (the weights
+    // live elsewhere; a pack whose model_dir was cleaned up still describes itself perfectly - this is the check)
+    const std::string md = kv.count("model_dir") ? kv["model_dir"] : "";
+    int missing = 0;
+    for (const auto& [i, n] : need) {
+        const std::string name = (size_t) i < files.size() ? files[(size_t) i] : "";
+        if (name.empty()) {
+            std::fprintf(out, "  file %d: no name in tensors.txt\n", i);
+            ++missing;
+            continue;
+        }
+        const std::string path = md + "/" + name;
+        std::FILE* f = std::fopen(path.c_str(), "rb");
+        if (f == nullptr) {
+            std::fprintf(out, "  MISSING %s (%.2f GB wanted)\n", path.c_str(), n / 1e9);
+            ++missing;
+            continue;
+        }
+#if defined(_WIN32)
+        _fseeki64(f, 0, SEEK_END);   // (long is 32 bits there: the weights are far past 2 GB)
+        const long long sz = _ftelli64(f);
+#else
+        std::fseek(f, 0, SEEK_END);
+        const long long sz = std::ftell(f);
+#endif
+        std::fclose(f);
+        if ((double) sz + 0.5 < n) {
+            std::fprintf(out, "  SHORT %s: %.2f GB there, %.2f GB wanted\n", path.c_str(), (double) sz / 1e9, n / 1e9);
+            ++missing;
+        }
+    }
+
+    // ---- the expert map: layer -> role types and bytes an expert
+    struct Role {
+        std::string type[3];
+        double bytes[3] = {0, 0, 0};
+        bool mixed = false;
+        int n = 0;
+    };
+    std::vector<Role> lay((size_t) nl);
+    std::map<std::string, long long> type_count;   // tensor count a type (the experts' 3 roles)
+    double experts_bytes = 0;
+    std::set<std::string> no_kernel;
+    for (long long il = 0; il < nl; ++il) {
+        Role& r = lay[(size_t) il];
+        const std::string pre = "layers." + std::to_string(il) + ".ffn.experts.";
+        for (long long e = 0; e < ne; ++e) {
+            const char* role[3] = {"w1", "w3", "w2"};
+            for (int k = 0; k < 3; ++k) {
+                auto it = t.find(pre + std::to_string(e) + "." + role[k] + ".weight");
+                if (it == t.end()) {
+                    std::fprintf(out, "pack-info: %s%s%lld.%s.weight: missing\n", pre.c_str(), "", e, role[k]);
+                    return 1;
+                }
+                if (e == 0) {
+                    r.type[k] = it->second.dtype;
+                    r.bytes[k] = it->second.bytes;
+                } else if (it->second.dtype != r.type[k] || it->second.bytes != r.bytes[k]) {
+                    r.mixed = true;
+                }
+                if (e == 0) {
+                    ++type_count[it->second.dtype];
+                    if (!device_type(it->second.dtype)) no_kernel.insert("layer " + std::to_string(il) + " " + role[k] + " " + it->second.dtype);
+                }
+            }
+        }
+        r.n = (int) ne;
+        experts_bytes += (double) ne * (r.bytes[0] + r.bytes[1] + r.bytes[2]);
+    }
+    const double per_expert = nl > 0 ? experts_bytes / ((double) nl * (double) ne) : 0;
+    // (a token picks `topk` experts a layer: sum over the layers of topk x one expert of that layer)
+    double token_bytes = 0;
+    for (long long il = 0; il < nl; ++il) token_bytes += (double) topk * (lay[(size_t) il].bytes[0] + lay[(size_t) il].bytes[1] + lay[(size_t) il].bytes[2]);
+    const double expert_params = (double) nl * (double) ne * 3.0 * (double) ff * (double) dim;
+
+    // ---- the dense side: everything that is not an expert tensor
+    double dense = 0, dense_embed = 0, dense_head = 0, dense_mtp = 0, dense_vision = 0, dense_engram = 0;
+    std::set<std::string> dense_types;
+    for (const auto& [name, x] : t) {
+        if (name.find(".ffn.experts.") != std::string::npos && name.rfind("layers.", 0) == 0) continue;
+        dense += x.bytes;
+        dense_types.insert(x.dtype);
+        if (name == "embed.weight") dense_embed += x.bytes;
+        else if (name == "head.weight") dense_head += x.bytes;
+        else if (name.rfind("mtp.", 0) == 0) dense_mtp += x.bytes;
+        else if (name.find(".engram.") != std::string::npos) dense_engram += x.bytes;
+        else if (name.rfind("vision.", 0) == 0 || name.rfind("aligner.", 0) == 0) dense_vision += x.bytes;
+        if (!dense_type(x.dtype)) no_kernel.insert("dense " + name + " " + x.dtype);
+    }
+
+    // ---- report
+    std::fprintf(out, "pack-info %s\n", dir.c_str());
+    std::fprintf(out, "  model_dir %s   format %s   %lld layers x %lld experts, %lld used; dim %lld, ff %lld, vocab %lld\n",
+                 kv.count("model_dir") ? kv["model_dir"].c_str() : "?", kv.count("format") ? kv["format"].c_str() : "?", nl, ne, topk,
+                 dim, ff, vocab);
+    std::fprintf(out, "  layer  gate+up   down      bytes/expert   MB/token\n");
+    for (long long il = 0; il < nl; ++il) {
+        const Role& r = lay[(size_t) il];
+        const double eb = r.bytes[0] + r.bytes[1] + r.bytes[2];
+        std::fprintf(out, "  %5lld  %-9s %-9s %13.0f %10.2f%s\n", il, r.type[0].c_str(), r.type[2].c_str(), eb,
+                     (double) topk * eb / 1e6, r.mixed ? "  (mixed)" : "");
+    }
+    std::fprintf(out, "  expert tensors by type:");
+    for (const auto& [ty, n] : type_count) std::fprintf(out, " %s %lld", ty.c_str(), n);
+    std::fprintf(out, "\n");
+    std::fprintf(out, "  an expert %.2f MB, %lld picks a token: %.3f GB a token\n", per_expert / 1e6, topk, token_bytes / 1e9);
+    std::fprintf(out, "  all experts %.1f GB at %.2f bpw (%.0f G parameters), dense %.2f GB (embed %.2f, head %.2f, mtp %.2f, vision %.2f, engram %.2f)\n",
+                 gb(experts_bytes), expert_params > 0 ? experts_bytes * 8.0 / expert_params : 0.0, expert_params / 1e9, gb(dense),
+                 gb(dense_embed), gb(dense_head), gb(dense_mtp), gb(dense_vision), gb(dense_engram));
+    std::fprintf(out, "  dense types:");
+    for (const auto& d : dense_types) std::fprintf(out, " %s", d.c_str());
+    std::fprintf(out, "\n");
+    // the geometry the engine needs at run time but that is easy to lose in a pack built by another tool: DSpark's block
+    // size / noise token / markov rank (0 there means it drafts one token a run, i.e. it is off) and the Engram's layers
+    {
+        auto kvn = [&](const char* k) {
+            auto it = kv.find(k);
+            return it == kv.end() ? 0 : std::atoll(it->second.c_str());
+        };
+        const int n_mtp = (int) std::count_if(t.begin(), t.end(), [](const auto& p) { return p.first.rfind("mtp.", 0) == 0; });
+        const long long nl_eg = num("engram_layer_ids", -1);
+        std::fprintf(out, "  dspark: block_size %lld, noise_token_id %lld, markov_rank %lld, routed experts %lld of %lld%s\n",
+                     num("dspark_block_size", 0), num("dspark_noise_token_id", 0), num("dspark_markov_rank", 0),
+                     num("dspark_n_routed_experts", 0), num("dspark_num_experts_per_tok", 0),
+                     num("dspark_block_size", 0) == 0 ? "  <- DSpark cannot draft (block size 0)" : "");
+        std::fprintf(out, "  mtp tensors: %d%s\n", n_mtp, n_mtp == 0 ? "  <- no DSpark drafter in the pack" : "");
+        auto it = kv.find("engram_layer_ids");
+        std::fprintf(out, "  engram layers: %s%s\n", it == kv.end() ? "(none)" : it->second.c_str(),
+                     it == kv.end() ? "  <- the Engram tables are not used" : "");
+    }
+    if (no_kernel.empty()) {
+        std::fprintf(out, "  no GPU kernel: none (every tensor is a device type)\n");
+    } else {
+        std::fprintf(out, "  no GPU kernel (%zu): the CPU lane takes them, they cannot go in the VRAM tier\n", no_kernel.size());
+        int shown = 0;
+        for (const std::string& s : no_kernel) {
+            std::fprintf(out, "    %s\n", s.c_str());
+            if (++shown >= 40) {
+                std::fprintf(out, "    ... %zu more\n", no_kernel.size() - (size_t) shown);
+                break;
+            }
+        }
+    }
+    if (missing > 0) {
+        std::fprintf(out, "  %d of %zu files missing or short: this pack cannot load (build it again, or point model_dir at "
+                          "the weights)\n", missing, need.size());
+        return 2;
+    }
+    std::fprintf(out, "  files: %zu, all present and long enough\n", need.size());
+    return 0;
+}
+
+}  // namespace strata::ds4

src/dsv41/ds4_packinfo.hpp

@@ -0,0 +1,13 @@
+// Strata. The pack report (ds4_packinfo.cpp): `strata-dsv41 --pack-info --pack <dir>`.  Host only: no device, no weights.
+#pragma once
+
+#include <cstdio>
+#include <string>
+
+namespace strata::ds4 {
+
+/// Prints the pack's type map, its bytes an expert / a token, its tier sizes and the tensors whose type has no device
+/// kernel.  0 when the pack's description was read, non-zero (and a line saying what is missing) otherwise.
+int pack_info(const std::string& dir, std::FILE* out);
+
+}  // namespace strata::ds4

src/dsv41/ds4_pool.hpp

@@ -0,0 +1,237 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_pool.hpp - the DeepSeek engine's CPU workers: run(n, fn) runs fn(0..n-1) on the workers and the caller and
+// returns when every job is done.  The design is glmfast::Workers' (src/core/glm_fast_state.hpp): one 64-bit ticket
+// (generation << 32 | next index) claimed by CAS, so a worker that wakes late for an older batch can never take a job
+// of the current one; idle workers spin for a while, then sleep on a condition variable.
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <cstring>
+#include <deque>
+#include <functional>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+#include "ds4_os.hpp"
+#if defined(_MSC_VER)
+#include <intrin.h>
+#endif
+
+namespace strata::ds4 {
+
+inline void cpu_relax() {
+#if defined(_MSC_VER)
+    _mm_pause();
+#elif defined(__x86_64__) || defined(_M_X64)
+    __builtin_ia32_pause();
+#endif
+}
+
+class Pool {
+public:
+    explicit Pool(int n, const std::vector<int>& cpus = {}, int spin_us = 200) : spin_us_(spin_us) {
+        for (int i = 0; i < n; ++i) {
+            th_.emplace_back([this] { loop(); });
+            if (!cpus.empty()) os::pin_thread(th_.back(), cpus[(size_t) i % cpus.size()]);
+        }
+    }
+    ~Pool() {
+        quit_.store(true);
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+        }
+        cv_.notify_all();
+        for (auto& t : th_) t.join();
+    }
+    int size() const { return (int) th_.size() + 1; }
+
+    void run(int n, const std::function<void(int)>& fn) {
+        start(n, fn);
+        help();
+        wait();
+    }
+    /// The batch goes to the workers; the caller may help() and must wait() before fn goes out of scope.  (Two pools
+    /// - one a NUMA node - run at once this way: start both, help one, wait for both.)
+    void start(int n, const std::function<void(int)>& fn) {
+        n_ = n;
+        if (n <= 0) return;
+        const uint64_t g = (gen_.load(std::memory_order_relaxed) + 1) & 0xffffffffu;
+        ticket_.store((g << 32) | 0x7fffffffu, std::memory_order_seq_cst);
+        fn_ = &fn;
+        total_.store(n, std::memory_order_release);
+        done_.store(0, std::memory_order_relaxed);
+        ticket_.store(g << 32, std::memory_order_release);
+        gen_.store(g, std::memory_order_release);
+        if (sleeping_.load(std::memory_order_acquire) > 0) {
+            std::lock_guard<std::mutex> lk(mu_);
+            cv_.notify_all();
+        }
+    }
+    void help() {
+        if (n_ > 0) work(gen_.load(std::memory_order_acquire));
+    }
+    void wait() {
+        if (n_ <= 0) return;
+        while (done_.load(std::memory_order_acquire) < n_) cpu_relax();
+        n_ = 0;
+    }
+
+private:
+    void work(uint64_t g) {
+        for (;;) {
+            uint64_t t = ticket_.load(std::memory_order_acquire);
+            if ((t >> 32) != g) return;
+            const int n = total_.load(std::memory_order_acquire);
+            if ((int) (t & 0xffffffffu) >= n) return;
+            if (!ticket_.compare_exchange_weak(t, t + 1, std::memory_order_acq_rel)) continue;
+            (*fn_)((int) (t & 0xffffffffu));
+            done_.fetch_add(1, std::memory_order_acq_rel);
+        }
+    }
+    void loop() {
+        uint64_t seen = gen_.load();
+        for (;;) {
+            const auto t0 = std::chrono::steady_clock::now();
+            uint64_t g;
+            for (;;) {
+                g = gen_.load(std::memory_order_acquire);
+                if (g != seen || quit_.load(std::memory_order_relaxed)) break;
+                cpu_relax();
+                if (std::chrono::steady_clock::now() - t0 > std::chrono::microseconds(spin_us_)) {
+                    std::unique_lock<std::mutex> lk(mu_);
+                    sleeping_.fetch_add(1);
+                    cv_.wait(lk, [&] { return gen_.load() != seen || quit_.load(); });
+                    sleeping_.fetch_sub(1);
+                }
+            }
+            if (quit_.load()) return;
+            seen = g;
+            work(g);
+        }
+    }
+
+    std::vector<std::thread> th_;
+    int spin_us_;
+    int n_ = 0;   // the current batch's size (the caller's side)
+    const std::function<void(int)>* fn_ = nullptr;
+    alignas(64) std::atomic<uint64_t> ticket_{0};
+    alignas(64) std::atomic<uint64_t> gen_{0};
+    alignas(64) std::atomic<int> total_{0};
+    alignas(64) std::atomic<int> done_{0};
+    std::atomic<int> sleeping_{0};
+    std::atomic<bool> quit_{false};
+    std::mutex mu_;
+    std::condition_variable cv_;
+};
+
+/// File reads on threads of their own, in the order they were pushed, the pusher never waiting: the prompt path's
+/// streamed experts (each part of an expert a job), unbuffered where the job has a direct descriptor and its range is
+/// not all in the page cache already (the disk straight into dst, the cache neither used nor disturbed), buffered
+/// otherwise or when the unbuffered read fails - the bytes land at dst + off % os::kDirectAlign either way.  A job's `left` is
+/// counted down when it is done (`bad` set if it failed); wait_any() sleeps until some job finished or the timeout.
+class IoQueue {
+public:
+    struct Job {
+        os::fd_t fd = os::kBadFd, dfd = os::kBadFd;
+        uint64_t off = 0, len = 0;
+        uint8_t* dst = nullptr;
+        std::atomic<int>* left = nullptr;
+        std::atomic<bool>* bad = nullptr;
+        const uint8_t* mem = nullptr;   // non-null: a copy of len bytes from memory to dst instead of a read
+        const void* map = nullptr;      // the range in the file's mapping: all in the page cache, it is read buffered
+        uint8_t* keep = nullptr;        // non-null (a read): the len bytes read copied here as well
+        std::atomic<bool>* cancel = nullptr;   // set before the job starts: it is dropped unread (bad is set, left counted down)
+    };
+    explicit IoQueue(int n) {
+        for (int i = 0; i < n; ++i) th_.emplace_back([this] { loop(); });
+    }
+    ~IoQueue() {
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            quit_ = true;
+        }
+        cv_.notify_all();
+        for (auto& t : th_) t.join();
+    }
+    /// low: a read that can wait (a guess) - the jobs of the first queue are all taken before any of this one's
+    void push(const Job* jobs, int n, bool low = false) {
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            for (int i = 0; i < n; ++i) (low ? ql_ : q_).push_back(jobs[i]);
+        }
+        if (n == 1) cv_.notify_one();
+        else cv_.notify_all();
+    }
+    void wait_any(int us) {
+        std::unique_lock<std::mutex> lk(done_mu_);
+        const uint64_t seen = done_;
+        done_cv_.wait_for(lk, std::chrono::microseconds(us), [&] { return done_ != seen; });
+    }
+    double busy_ms() const { return busy_us_.load(std::memory_order_relaxed) / 1000.0; }
+    uint64_t direct_bytes() const { return direct_.load(std::memory_order_relaxed); }
+    uint64_t buffered_bytes() const { return buffered_.load(std::memory_order_relaxed); }
+
+private:
+    void loop() {
+        for (;;) {
+            Job j;
+            {
+                std::unique_lock<std::mutex> lk(mu_);
+                cv_.wait(lk, [&] { return quit_ || !q_.empty() || !ql_.empty(); });
+                if (quit_ && q_.empty() && ql_.empty()) return;
+                std::deque<Job>& qq = q_.empty() ? ql_ : q_;
+                j = qq.front();
+                qq.pop_front();
+            }
+            if (j.cancel != nullptr && j.cancel->load(std::memory_order_acquire)) {   // (dropped unread)
+                if (j.bad != nullptr) j.bad->store(true, std::memory_order_release);
+                if (j.left != nullptr) j.left->fetch_sub(1, std::memory_order_acq_rel);
+                {
+                    std::lock_guard<std::mutex> lk(done_mu_);
+                    ++done_;
+                }
+                done_cv_.notify_all();
+                continue;
+            }
+            const auto t0 = std::chrono::steady_clock::now();
+            const uint64_t lead = j.off % os::kDirectAlign;
+            bool ok = false;
+            if (j.mem != nullptr) {
+                std::memcpy(j.dst, j.mem, (size_t) j.len);
+                ok = true;
+            } else if (j.dfd != os::kBadFd && !(j.map != nullptr && os::resident(j.map, (size_t) j.len)) &&
+                       os::pread_direct(j.dfd, j.dst, j.len, j.off)) {   // (the page cache's ones: buffered, below)
+                ok = true;
+                direct_.fetch_add(j.len, std::memory_order_relaxed);
+            } else if (os::pread_par(j.fd, j.dst + lead, j.len, j.off)) {
+                ok = true;
+                buffered_.fetch_add(j.len, std::memory_order_relaxed);
+            }
+            if (ok && j.keep != nullptr && j.mem == nullptr) std::memmove(j.keep, j.dst + lead, (size_t) j.len);   // (keep may overlap dst: the store reads in place and moves up)
+            busy_us_.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::microseconds>(
+                                   std::chrono::steady_clock::now() - t0).count(),
+                               std::memory_order_relaxed);
+            if (!ok && j.bad != nullptr) j.bad->store(true, std::memory_order_release);
+            if (j.left != nullptr) j.left->fetch_sub(1, std::memory_order_acq_rel);
+            {
+                std::lock_guard<std::mutex> lk(done_mu_);
+                ++done_;
+            }
+            done_cv_.notify_all();
+        }
+    }
+    std::vector<std::thread> th_;
+    std::mutex mu_, done_mu_;
+    std::condition_variable cv_, done_cv_;
+    std::deque<Job> q_, ql_;
+    uint64_t done_ = 0;
+    bool quit_ = false;
+    std::atomic<uint64_t> busy_us_{0}, direct_{0}, buffered_{0};
+};
+
+}  // namespace strata::ds4

src/dsv41/ds4_q6k.cuh

@@ -0,0 +1,30 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_q6k.cuh - one value of a ggml Q6_K superblock (256 values, 210 bytes: ql[128] the low 4 bits, qh[64] the
+// high 2, scales[16] int8 a 16 values, d F16), as ggml's dequantize_row_q6_K computes it: d * scale * (q - 32), the two
+// products in that order in F32 - the same bits (the Engram tables a GGUF may hold in Q6_K).
+#pragma once
+
+#include <cuda_fp16.h>
+#include <cstdint>
+
+namespace strata::ds4 {
+
+constexpr int kQ6kBytes = 210;
+
+__device__ __forceinline__ float q6k_value(const uint8_t* __restrict__ b, int i) {
+    const uint8_t* ql = b;
+    const uint8_t* qh = b + 128;
+    const int8_t* sc = (const int8_t*) (b + 192);
+    // (the F16 scale's bits as a half: HIP's headers take no brace-initialized __half_raw from them)
+    const float d = __half2float(__ushort_as_half((unsigned short) (b[208] | (b[209] << 8))));
+    const int h = i >> 7, w = i & 127, quarter = w >> 5, l = w & 31;   // (ggml's loop: half h, 32-value quarters)
+    ql += h * 64;
+    qh += h * 32;
+    sc += h * 8;
+    const int lo = quarter & 1 ? ql[l + 32] : ql[l];
+    const int q = ((quarter & 2 ? lo >> 4 : lo & 0xF) | (((qh[l] >> (2 * quarter)) & 3) << 4)) - 32;
+    return __fmul_rn(__fmul_rn(d, (float) sc[(l >> 4) + 2 * quarter]), (float) q);
+}
+
+}  // namespace strata::ds4

src/dsv41/ds4_slotmeta.hpp

@@ -0,0 +1,98 @@
+// The kept-conversation (slot) bookkeeping that needs no GPU: a slot's .meta file (identity, size, tokens, checkpoint
+// positions) and the choice of which kept conversation a prompt continues, and from where.  The engine's serve loop
+// (ds4_main.cpp) uses these; tests/dsv41/slot_test.cpp checks them alone.
+//
+// Slot .meta layout (little endian), "DS4META2":
+//   u64 magic, u64 identity length, identity bytes, u64 .bin bytes, u64 token count, i32 tokens[],
+//   u64 checkpoint count (0..4), i64 checkpoints[], u64 magic
+// The magic at both ends and the .bin size make a half-written or foreign file refused, never read.
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+namespace strata::ds4::slotmeta {
+
+constexpr uint64_t kMagic = 0x324154454d345344ull;   // "DS4META2"
+
+// Written under a temporary name and renamed over `path` (an interrupted write leaves no half file).
+inline bool write(const std::string& path, const std::string& identity, const std::vector<int32_t>& toks,
+                  uint64_t bytes, const std::vector<int64_t>& cks) {
+    const std::string tmp = path + ".tmp";
+    {
+        std::ofstream f(tmp, std::ios::binary | std::ios::trunc);
+        const uint64_t il = identity.size(), nt = toks.size(), nc = cks.size();
+        f.write((const char*) &kMagic, 8);
+        f.write((const char*) &il, 8);
+        f.write(identity.data(), (std::streamsize) il);
+        f.write((const char*) &bytes, 8);
+        f.write((const char*) &nt, 8);
+        f.write((const char*) toks.data(), (std::streamsize) (nt * sizeof(int32_t)));
+        f.write((const char*) &nc, 8);
+        f.write((const char*) cks.data(), (std::streamsize) (nc * sizeof(int64_t)));
+        f.write((const char*) &kMagic, 8);
+        f.flush();
+        if (!f) return false;
+    }
+    std::error_code ec;
+    std::filesystem::rename(tmp, path, ec);
+    return !ec;
+}
+
+// false: foreign (another identity), damaged, cut short, or not plausible (no tokens, more than max_ctx, a checkpoint
+// outside the tokens).
+inline bool read(const std::string& path, const std::string& identity, int64_t max_ctx, std::vector<int32_t>& toks,
+                 uint64_t& bytes, std::vector<int64_t>& cks) {
+    std::ifstream f(path, std::ios::binary);
+    uint64_t m0 = 0, il = 0, nt = 0, nc = 0, m1 = 0;
+    if (!f.read((char*) &m0, 8) || m0 != kMagic || !f.read((char*) &il, 8) || il != identity.size()) return false;
+    std::string id(il, '\0');
+    if (!f.read(id.data(), (std::streamsize) il) || id != identity) return false;
+    if (!f.read((char*) &bytes, 8) || !f.read((char*) &nt, 8) || nt == 0 || nt > (uint64_t) max_ctx) return false;
+    toks.resize(nt);
+    if (!f.read((char*) toks.data(), (std::streamsize) (nt * sizeof(int32_t))) || !f.read((char*) &nc, 8) || nc > 4)
+        return false;
+    cks.resize(nc);
+    if (!f.read((char*) cks.data(), (std::streamsize) (nc * sizeof(int64_t))) || !f.read((char*) &m1, 8) || m1 != kMagic)
+        return false;
+    for (int64_t c : cks)
+        if (c <= 0 || (uint64_t) c > nt) return false;
+    return true;
+}
+
+// The kept conversation `prompt` continues the furthest, and where: *len is the number of the slot's tokens that are
+// the prompt's (0: none).  A conversation the prompt continues whole comes back whole (len == its size); one the prompt
+// leaves part way - a retried prompt, a history re-rendered without the earlier reasoning - comes back at its latest
+// checkpoint the prompt still agrees with.  Only a match longer than `floor` (what the model itself can go back to)
+// counts.  The last prompt token is never covered (a prompt always reads its last token).  Ties keep the earlier slot.
+struct Cand {
+    const std::vector<int32_t>* tokens;
+    const std::vector<int64_t>* cks;
+};
+inline size_t pick(const std::vector<Cand>& slots, const std::vector<int>& prompt, size_t floor, size_t* len) {
+    *len = 0;
+    if (prompt.size() < 2) return SIZE_MAX;
+    size_t best = SIZE_MAX;
+    for (size_t i = 0; i < slots.size(); ++i) {
+        const std::vector<int32_t>& t = *slots[i].tokens;
+        const size_t lim = std::min(t.size(), prompt.size() - 1);
+        size_t c = 0;
+        while (c < lim && t[c] == prompt[c]) ++c;
+        size_t u = c == t.size() ? c : 0;
+        if (u == 0)
+            for (int64_t k : *slots[i].cks)
+                if ((size_t) k <= c && (size_t) k > u) u = (size_t) k;
+        if (u > *len && u > floor) {
+            best = i;
+            *len = u;
+        }
+    }
+    return best;
+}
+
+}  // namespace strata::ds4::slotmeta

src/dsv41/ds4_store.hpp

@@ -0,0 +1,530 @@
+// Strata. src/dsv41/ds4_store.hpp - the RAM tier of the DeepSeek-V4.1 routed experts as a cache of its own: whole experts read
+// from the model's files by unbuffered requests (the OS page cache neither used nor disturbed), kept in a fixed arena of slots
+// that a replacement policy owns, filled on demand (the CPU lane's picks) and ahead of time (the lookahead's guesses).
+//
+// A slot holds an expert in the engine's blob layout (gate | up | down at the layer's offsets tr[r], the same as a VRAM slot), so
+// the CPU lane's rows read it in place and - when the arena is registered with the GPU - the device kernels read it over PCIe
+// (zero-copy, 24 GB/s on a 5070) through the same plan pointers as a VRAM expert.  An expert is three matrices at three file
+// offsets (the GGUF's one tensor a role and layer): each is read aligned (4096) into the slot and moved into place (the read's
+// lead of up to 4095 bytes), the blob layout leaving 8192 bytes between matrices (and around the blob) for the aligned spans.
+//
+// Memory: slots are allocated in chunks as they are first needed (private memory), never past the budget nor past the RAM this
+// process can still get less `floor_bytes` (the owner's session and the system keep theirs).
+//
+// Policy (the slot to give up, among the unpinned, not GPU-visible ones): lru (oldest touch), lfu (lowest decayed hit count).
+// A slot whose expert a window is computing is pinned from ensure() to release(); a slot the GPU may read is `visible` until
+// sync() takes it back at a window boundary (the GPU idle), which also publishes the newly ready ones.
+#pragma once
+
+#include "ds4_os.hpp"
+#include "ds4_pool.hpp"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+namespace strata::ds4 {
+
+class ExpertStore {
+public:
+    struct LayerGeo {
+        int n_exp = 0;
+        int file[3] = {-1, -1, -1};
+        uint64_t off[3] = {0, 0, 0}, stride[3] = {0, 0, 0};   // matrix r of expert e: file[r], off[r] + e stride[r]
+        uint64_t bytes[3] = {0, 0, 0};
+        uint64_t tr[3] = {0, 0, 0};                            // matrix r's place in the blob
+        uint64_t blob = 0;                                     // the blob's bytes (gaps of 8192 between matrices)
+    };
+    enum Policy { kLru = 0, kLfu = 1 };
+    struct Stats {
+        std::atomic<int64_t> hits{0}, misses{0}, prefetched{0}, prefetch_used{0}, prefetch_wasted{0}, evictions{0}, fails{0};
+        std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0}, cancelled{0};
+    };
+    struct Vis {                 // a slot the GPU may read from now on (dev != 0), or no longer (dev == 0)
+        int layer = 0, e = 0;
+        unsigned long long dev = 0;
+    };
+
+    struct Cfg {
+        int threads = 8;
+        size_t budget_bytes = 0;           // the arena at most (grown in chunks as slots are first needed)
+        size_t floor_bytes = 8ull << 30;   // RAM left alone
+        Policy policy = kLru;
+        float decay = 0.97f;               // lfu: heat x decay every `decay_every` accesses of its layer
+        int decay_every = 240;
+        std::function<uint64_t()> ram_avail;   // the RAM this process can still get (~0: unknown)
+        // a new chunk of the arena for the GPU to read in place: returns the device address of `host` (null: not visible)
+        std::function<void*(void* host, size_t bytes)> map_chunk;
+        std::function<void(void* host)> unmap_chunk;
+        size_t reserve = 192;              // slots kept evictable (not visible) for the window's demand and guesses
+        int vis_percent = 100;             // the share of experts the GPU may take (by a fixed hash): the rest stay the CPU lane's, so both lanes work
+    };
+
+    bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& dfds, const std::vector<os::fd_t>& pfds) {
+        cfg_ = c;
+        geo_ = layers;
+        dfds_ = dfds;
+        pfds_ = pfds;
+        size_t blob = 0;
+        for (const LayerGeo& g : geo_) blob = std::max<size_t>(blob, g.blob);
+        slot_bytes_ = (blob + 2 * kSlack + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1);
+        if (blob == 0 || c.budget_bytes < slot_bytes_) return false;
+        nmax_ = c.budget_bytes / slot_bytes_;
+        // one reserved range for the whole arena (chunks committed as they are needed), 64 KB aligned chunks
+        arena_bytes_ = ((nmax_ + kChunk - 1) / kChunk) * kChunk * slot_bytes_;
+        arena_ = (uint8_t*) os::range_reserve(arena_bytes_);
+        if (arena_ == nullptr) return false;
+        slots_.reset(new Slot[nmax_]);
+        where_.resize(geo_.size());
+        heat_.resize(geo_.size());
+        hits_in_layer_.assign(geo_.size(), 0);
+        for (size_t l = 0; l < geo_.size(); ++l) {
+            where_[l].assign((size_t) geo_[l].n_exp, -1);
+            heat_[l].assign((size_t) geo_[l].n_exp, 0.0f);
+        }
+        io_.reset(new IoQueue(std::max(1, c.threads)));
+        grow();   // (the first chunk now: whether the GPU can map it is known from the start)
+        // DS4_STORE_PREGROW=1 (default on Linux with the GPU mapping, =0 off): the whole arena now.  cuMemHostRegister (the GPU mapping of a chunk) taken while a kernel of
+        // the decode is running waited on it for good on Linux (H100, driver 580): registering all chunks before the first
+        // kernel avoids it
+#if defined(__linux__)
+        const bool pregrow = cfg_.map_chunk && !(std::getenv("DS4_STORE_PREGROW") != nullptr && std::atoi(std::getenv("DS4_STORE_PREGROW")) == 0);   // (Linux: on with the GPU mapping)
+#else
+        const bool pregrow = std::getenv("DS4_STORE_PREGROW") != nullptr && std::atoi(std::getenv("DS4_STORE_PREGROW")) != 0;
+#endif
+        if (pregrow)
+            while (grow()) {
+            }
+        return true;
+    }
+    ~ExpertStore() {
+        io_.reset();
+        for (size_t i = 0; i < chunks_.size(); ++i)
+            if (cfg_.unmap_chunk && chunk_dev_[i] != nullptr) cfg_.unmap_chunk(chunks_[i].first);
+        os::range_release(arena_, arena_bytes_);
+    }
+
+    size_t slot_bytes() const { return slot_bytes_; }
+    size_t slots_allocated() const { return nalloc_; }
+    size_t slots_max() const { return nmax_; }
+    const Stats& stats() const { return st_; }
+    double busy_ms() const { return io_ ? io_->busy_ms() : 0.0; }
+    static constexpr size_t kSlack = 8192;
+
+    /// The slot holding (layer, e) ready to read, or -1.  (Any thread; a slot found stays valid while it is pinned or visible.)
+    int find(int layer, int e) const {
+        if (layer < 0 || layer >= (int) where_.size()) return -1;   // (a layer the store does not cover: the DSpark drafter's)
+        const int s = where_[(size_t) layer][(size_t) e];
+        if (s < 0) return -1;
+        Slot& sp = slots_[(size_t) s];
+        try_ready(sp);
+        return sp.state.load(std::memory_order_acquire) == kReady && sp.layer == layer && sp.expert == e ? s : -1;
+    }
+    const uint8_t* mat(int slot, int r) const { return slots_[(size_t) slot].base + geo_[(size_t) slots_[(size_t) slot].layer].tr[r]; }
+    const uint8_t* blob(int slot) const { return slots_[(size_t) slot].base; }
+    /// The device address of the slot's blob (0: the arena is not mapped for the GPU)
+    unsigned long long dev_blob(int slot) const {
+        const size_t c = (size_t) slot / kChunk;
+        if (c >= chunk_dev_.size() || chunk_dev_[c] == nullptr) return 0;
+        return (unsigned long long) ((uint8_t*) chunk_dev_[c] + (slots_[(size_t) slot].base - (uint8_t*) chunks_[c].first));
+    }
+
+    /// Makes the experts es[0..n) of `layer` ready and pinned (reading what is not here), waiting for the reads; the
+    /// caller releases them.  False if a read failed or no slot could be had for one (ok[i] says which are fine; the caller
+    /// reads the others another way).
+    bool ensure(int layer, const int* es, int n, std::vector<char>& ok) {
+        ok.assign((size_t) n, 0);
+        if (layer < 0 || layer >= (int) geo_.size()) return false;
+        std::vector<int> sl((size_t) n, -1);
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            for (int i = 0; i < n; ++i) {
+                const int e = es[i];
+                touch_heat(layer, e);
+                int s = where_[(size_t) layer][(size_t) e];
+                if (s >= 0) {
+                    Slot& sp = slots_[(size_t) s];
+                    try_ready(sp);
+                    if (sp.state.load() == kReady && !sp.prefetch) st_.hits.fetch_add(1, std::memory_order_relaxed);
+                    if (sp.prefetch) st_.prefetch_used.fetch_add(1, std::memory_order_relaxed);
+                    sp.prefetch = false;
+                    sp.pin++;
+                    sp.tick = ++clock_;
+                    sl[(size_t) i] = s;
+                    continue;
+                }
+                st_.misses.fetch_add(1, std::memory_order_relaxed);
+                s = start_fill(layer, e, true);
+                sl[(size_t) i] = s;
+            }
+        }
+        const auto t0 = std::chrono::steady_clock::now();
+        bool all = true;
+        for (int i = 0; i < n; ++i) {
+            const int s = sl[(size_t) i];
+            if (s < 0) {
+                all = false;
+                continue;
+            }
+            Slot& sp = slots_[(size_t) s];
+            while (sp.left.load(std::memory_order_acquire) > 0) io_->wait_any(2000);
+            std::lock_guard<std::mutex> lk(mu_);
+            finish(sp);
+            if (sp.state.load() == kReady) ok[(size_t) i] = 1;
+            else {
+                sp.pin--;
+                sl[(size_t) i] = -1;
+                all = false;
+            }
+        }
+        st_.wait_ns.fetch_add(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count(),
+                              std::memory_order_relaxed);
+        return all;
+    }
+    void release(int layer, const int* es, int n, const std::vector<char>& ok) {
+        if (layer < 0 || layer >= (int) geo_.size()) return;
+        std::lock_guard<std::mutex> lk(mu_);
+        for (int i = 0; i < n; ++i)
+            if (ok[(size_t) i]) {
+                const int s = where_[(size_t) layer][(size_t) es[i]];
+                if (s >= 0 && slots_[(size_t) s].pin > 0) slots_[(size_t) s].pin--;
+            }
+    }
+    /// Starts reading (layer, e) if it is neither here nor on its way (a guess: unpinned, may be given up before it is used).
+    void prefetch(int layer, int e) {
+        if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) return;
+        if (where_[(size_t) layer][(size_t) e] >= 0) return;
+        std::lock_guard<std::mutex> lk(mu_);
+        if (where_[(size_t) layer][(size_t) e] >= 0) return;
+        if (start_fill(layer, e, false) >= 0) st_.prefetched.fetch_add(1, std::memory_order_relaxed);
+    }
+    /// Reads the experts of `list` (layer, expert; hottest first) into free slots and returns when they are in (a start that
+    /// is already warm: the profile's hottest, so the first tokens find them).  The first `list.size()` fit the arena less
+    /// `leave` slots; returns how many were read.
+    size_t preload(const std::vector<std::pair<int, int>>& list, size_t leave) {
+        size_t n = 0;
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            const size_t room = nmax_ > leave ? nmax_ - leave : 0;
+            const size_t take = std::min(list.size(), room);
+            for (size_t k = take; k-- > 0;) {   // (coldest first: the hottest get the newest ticks)
+                const int layer = list[k].first, e = list[k].second;
+                if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) continue;
+                if (where_[(size_t) layer][(size_t) e] >= 0) continue;
+                const int s = start_fill(layer, e, false);
+                if (s < 0) break;
+                slots_[(size_t) s].prefetch = false;   // (warm, not a guess)
+                ++n;
+            }
+        }
+        for (size_t i = 0;; ++i) {   // until every read is done
+            bool busy = false;
+            {
+                std::lock_guard<std::mutex> lk(mu_);
+                for (size_t j = 0; j < nalloc_; ++j) {
+                    Slot& sp = slots_[j];
+                    if (sp.state.load(std::memory_order_acquire) != kLoading) continue;
+                    if (sp.left.load(std::memory_order_acquire) > 0) busy = true;
+                    else {
+                        try_ready(sp);
+                        if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
+                    }
+                }
+            }
+            if (!busy) break;
+            io_->wait_any(5000);
+        }
+        return n;
+    }
+    /// Guesses still waiting in the queue for a layer the token is past: dropped unread (their slots free once the queue has gone by)
+    void cancel_stale(int layer_now) {
+        std::lock_guard<std::mutex> lk(mu_);
+        for (size_t i = 0; i < nalloc_; ++i) {
+            Slot& sp = slots_[i];
+            if (sp.prefetch && sp.pin == 0 && sp.layer >= 0 && sp.layer <= layer_now && sp.state.load(std::memory_order_acquire) == kLoading)
+                sp.cancel.store(true, std::memory_order_release);
+        }
+    }
+    /// An access the host saw for an expert the GPU computed (keeps it warm in the policy's eyes).
+    void touch(int layer, int e) {
+        if (layer < 0 || layer >= (int) where_.size()) return;
+        std::lock_guard<std::mutex> lk(mu_);
+        touch_heat(layer, e);
+        const int s = where_[(size_t) layer][(size_t) e];
+        if (s >= 0) slots_[(size_t) s].tick = ++clock_;
+    }
+    bool gpu_mapped() const { return !chunk_dev_.empty() && chunk_dev_[0] != nullptr; }
+    /// The arena's address range (the device sees its host pointers as they are when the mapping is the identity)
+    unsigned long long arena_lo() const { return (unsigned long long) (uintptr_t) arena_; }
+    unsigned long long arena_hi() const { return (unsigned long long) (uintptr_t) (arena_ + arena_bytes_); }
+    bool identity_mapped() const {
+        for (size_t i = 0; i < chunks_.size(); ++i)
+            if (chunk_dev_[i] != nullptr && chunk_dev_[i] != chunks_[i].first) return false;
+        return true;
+    }
+    size_t max_blob() const { return slot_bytes_ - 2 * kSlack; }
+
+    /// At a window boundary (nothing reading the slots; the caller applies `ev` to the device's table before the next window):
+    /// every ready slot not visible yet becomes visible (`gpu_ok(layer, e)`: false for an expert VRAM holds), and visible
+    /// slots of the lowest score are taken back until `cfg.reserve` slots can be given up.  Returns the number of events.
+    size_t sync(std::vector<Vis>& ev, const std::function<bool(int, int)>& gpu_ok) {
+        std::lock_guard<std::mutex> lk(mu_);
+        size_t nvis = 0, nfree = 0;
+        for (size_t i = 0; i < nalloc_; ++i) {
+            Slot& sp = slots_[i];
+            if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {
+                try_ready(sp);
+                if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
+            }
+            const int st = sp.state.load(std::memory_order_acquire);
+            if (st == kFree) {
+                ++nfree;
+                continue;
+            }
+            if (st != kReady) continue;
+            if (sp.visible) {
+                ++nvis;
+                continue;
+            }
+            if (sp.pin == 0) ++nfree;
+            if (sp.pin == 0 && dev_blob((int) i) != 0 && gpu_ok(sp.layer, sp.expert) && !sp.prefetch && vis_hash(sp.layer, sp.expert) < cfg_.vis_percent) {
+                sp.visible = true;
+                ++nvis;
+                if (sp.pin == 0) --nfree;
+                ev.push_back({sp.layer, sp.expert, dev_blob((int) i)});
+            }
+        }
+        // give slots back until the reserve is there: the visible ones with the lowest score
+        if (nfree < cfg_.reserve && nvis > 0) {
+            std::vector<std::pair<double, size_t>> cand;
+            cand.reserve(nvis);
+            for (size_t i = 0; i < nalloc_; ++i)
+                if (slots_[i].visible && slots_[i].state.load(std::memory_order_acquire) == kReady) cand.push_back({score(slots_[i]), i});
+            const size_t want = std::min(cand.size(), cfg_.reserve - nfree);
+            std::partial_sort(cand.begin(), cand.begin() + (long) want, cand.end());
+            for (size_t k = 0; k < want; ++k) {
+                Slot& sp = slots_[cand[k].second];
+                sp.visible = false;
+                ev.push_back({sp.layer, sp.expert, 0ull});
+            }
+        }
+        return ev.size();
+    }
+    /// RAM is short: gives back whole chunks from the end of the arena (their experts forgotten) - up to `want`, stopping at one
+    /// with a slot in use (pinned, loading, or GPU-visible).  Returns the chunks given back.
+    size_t shrink_chunks(size_t want) {
+        std::lock_guard<std::mutex> lk(mu_);
+        size_t done = 0;
+        while (done < want && !chunks_.empty()) {
+            const size_t c = chunks_.size() - 1, first = c * kChunk;
+            bool busy = false;
+            for (size_t i = first; i < nalloc_ && !busy; ++i) {
+                Slot& sp = slots_[i];
+                if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {   // (read by now)
+                    try_ready(sp);
+                    if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
+                }
+                busy = sp.pin > 0 || sp.visible || sp.state.load(std::memory_order_acquire) == kLoading;
+            }
+            if (busy) break;
+            for (size_t i = first; i < nalloc_; ++i) {
+                Slot& sp = slots_[i];
+                if (sp.layer >= 0 && sp.expert >= 0) {
+                    where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
+                    st_.evictions.fetch_add(1, std::memory_order_relaxed);
+                }
+                sp.layer = sp.expert = -1;
+                sp.prefetch = false;
+                sp.state.store(kFree, std::memory_order_release);
+                sp.base = nullptr;
+            }
+            if (cfg_.unmap_chunk && chunk_dev_[c] != nullptr) cfg_.unmap_chunk(chunks_[c].first);
+            os::range_decommit(chunks_[c].first, chunks_[c].second);
+            chunks_.pop_back();
+            chunk_dev_.pop_back();
+            nalloc_ = first;
+            free_hint_ = std::min(free_hint_, first);
+            ++done;
+            ++shrunk_;
+        }
+        return done;
+    }
+    size_t shrunk() const { return shrunk_; }
+    /// An expert VRAM took over: no longer a visible slot
+    void unmark(int layer, int e) {
+        if (layer < 0 || layer >= (int) where_.size()) return;
+        std::lock_guard<std::mutex> lk(mu_);
+        const int s = where_[(size_t) layer][(size_t) e];
+        if (s >= 0) slots_[(size_t) s].visible = false;
+    }
+    size_t visible_count() const {
+        size_t n = 0;
+        for (size_t i = 0; i < nalloc_; ++i) n += slots_[i].visible ? 1 : 0;
+        return n;
+    }
+
+private:
+    static constexpr int kFree = 0, kLoading = 1, kReady = 2, kFailed = 3;
+    static constexpr size_t kChunk = 64;   // slots a chunk of the arena
+    struct Slot {
+        uint8_t* base = nullptr;   // the blob's first byte
+        int layer = -1, expert = -1, pin = 0;
+        bool prefetch = false, visible = false;
+        std::atomic<int> state{kFree}, left{0};
+        std::atomic<bool> bad{false}, cancel{false};
+        uint64_t tick = 0;
+    };
+
+    static int vis_hash(int layer, int e) { return (int) (((unsigned) layer * 2654435761u + (unsigned) e * 40503u) >> 7) % 100; }
+    double score(const Slot& sp) const {
+        if (cfg_.policy == kLru) return (double) sp.tick;
+        return (double) heat_[(size_t) sp.layer][(size_t) sp.expert] + 1e-9 * (double) sp.tick;
+    }
+    void touch_heat(int layer, int e) {
+        heat_[(size_t) layer][(size_t) e] += 1.0f;
+        if (cfg_.policy != kLru && ++hits_in_layer_[(size_t) layer] % (uint64_t) cfg_.decay_every == 0)
+            for (float& h : heat_[(size_t) layer]) h *= cfg_.decay;   // (a linear pass over 384 floats every 240 accesses of that layer)
+    }
+    // a slot for (layer, e) and its reads started; -1 if there is none to be had.  mu_ held.
+    int start_fill(int layer, int e, bool demand) {
+        int s = take_slot(demand);
+        if (s < 0) {
+            st_.no_slot.fetch_add(1, std::memory_order_relaxed);
+            return -1;
+        }
+        Slot& sp = slots_[(size_t) s];
+        if (sp.layer >= 0 && sp.expert >= 0) {
+            where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
+            if (sp.prefetch) st_.prefetch_wasted.fetch_add(1, std::memory_order_relaxed);
+            st_.evictions.fetch_add(1, std::memory_order_relaxed);
+        }
+        sp.layer = layer;
+        sp.expert = e;
+        sp.pin = demand ? 1 : 0;
+        sp.prefetch = !demand;
+        sp.visible = false;
+        sp.tick = ++clock_;
+        sp.state.store(kLoading, std::memory_order_release);
+        sp.bad.store(false);
+        sp.cancel.store(false);
+        sp.left.store(3, std::memory_order_release);
+        where_[(size_t) layer][(size_t) e] = s;
+        const LayerGeo& g = geo_[(size_t) layer];
+        IoQueue::Job jobs[3];
+        for (int r = 0; r < 3; ++r) {
+            const uint64_t off = g.off[r] + (uint64_t) e * g.stride[r];
+            const uint64_t lead = off % os::kDirectAlign;
+            uint8_t* fin = sp.base + g.tr[r];   // where the matrix belongs
+            // the aligned read lands the file's byte `off` at dst + lead: dst is fin - lead rounded down to 4096 (its span spills
+            // under 4096 bytes either way into the blob's gaps), the matrix is then moved up to fin
+            uint8_t* dst = (uint8_t*) ((uintptr_t) (fin - lead) & ~(uintptr_t) (os::kDirectAlign - 1));
+            IoQueue::Job& j = jobs[r];
+            j.fd = pfds_[(size_t) g.file[r]];
+            j.dfd = dfds_[(size_t) g.file[r]];
+            j.off = off;
+            j.len = g.bytes[r];
+            j.dst = dst;
+            j.keep = fin;
+            j.left = &sp.left;
+            j.bad = &sp.bad;
+            j.cancel = demand ? nullptr : &sp.cancel;
+        }
+        st_.bytes_read.fetch_add((int64_t) (g.bytes[0] + g.bytes[1] + g.bytes[2]), std::memory_order_relaxed);
+        io_->push(jobs, 3, !demand);   // (a guess waits behind every demand read)
+        return s;
+    }
+    // loading -> ready once its reads are done and whole (lock-free; a failed one is cleaned up by finish / take_slot)
+    void try_ready(Slot& sp) const {
+        int want = kLoading;
+        if (sp.left.load(std::memory_order_acquire) == 0 && !sp.bad.load(std::memory_order_acquire))
+            sp.state.compare_exchange_strong(want, kReady, std::memory_order_acq_rel);
+    }
+    void finish(Slot& sp) {   // mu_ held; reads done
+        if (sp.state.load() != kLoading) return;
+        if (sp.bad.load()) {
+            where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
+            sp.layer = sp.expert = -1;
+            sp.state.store(kFree, std::memory_order_release);
+            sp.prefetch = false;
+            if (sp.cancel.load()) st_.cancelled.fetch_add(1, std::memory_order_relaxed);
+            else st_.fails.fetch_add(1, std::memory_order_relaxed);
+            return;
+        }
+        sp.state.store(kReady, std::memory_order_release);
+    }
+    bool grow() {
+        if (nalloc_ >= nmax_) return false;
+        const size_t n = std::min(kChunk, nmax_ - nalloc_);
+        const size_t bytes = n * slot_bytes_;
+        const uint64_t avail = cfg_.ram_avail ? cfg_.ram_avail() : ~uint64_t{0};
+        if (avail != ~uint64_t{0} && avail < bytes + cfg_.floor_bytes) return false;
+        void* p = arena_ + nalloc_ * slot_bytes_;
+        if (!os::range_commit(p, bytes)) return false;
+        chunks_.push_back({p, bytes});
+        void* dev = cfg_.map_chunk ? cfg_.map_chunk(p, bytes) : nullptr;
+        chunk_dev_.push_back(dev);
+        for (size_t i = 0; i < n; ++i) slots_[nalloc_ + i].base = (uint8_t*) p + i * slot_bytes_ + kSlack;
+        nalloc_ += n;
+        return true;
+    }
+    // the slot to give up (or a free one): the policy's lowest among the unpinned, not visible, ready ones.  mu_ held.
+    int take_slot(bool demand) {
+        for (size_t i = free_hint_; i < nalloc_; ++i)
+            if (slots_[i].state.load(std::memory_order_acquire) == kFree) {
+                free_hint_ = i + 1;
+                return (int) i;
+            }
+        if (grow()) return take_slot(demand);
+        int best = -1;
+        double bv = 0;
+        for (size_t i = 0; i < nalloc_; ++i) {
+            Slot& sp = slots_[i];
+            if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {
+                try_ready(sp);
+                if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
+                if (sp.state.load(std::memory_order_acquire) == kFree) {
+                    free_hint_ = std::min(free_hint_, i);
+                    return (int) i;
+                }
+            }
+            if (sp.pin > 0 || sp.visible || sp.state.load(std::memory_order_acquire) != kReady) continue;
+            const double v = score(sp);
+            if (best < 0 || v < bv) {
+                best = (int) i;
+                bv = v;
+            }
+        }
+        if (best < 0) return -1;
+        free_hint_ = std::min<size_t>(free_hint_, (size_t) best);
+        return best;
+    }
+
+    Cfg cfg_;
+    std::vector<LayerGeo> geo_;
+    std::vector<os::fd_t> dfds_, pfds_;
+    size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0, arena_bytes_ = 0;
+    uint8_t* arena_ = nullptr;
+    size_t shrunk_ = 0;
+    std::unique_ptr<Slot[]> slots_;
+    std::vector<std::pair<void*, size_t>> chunks_;
+    std::vector<void*> chunk_dev_;
+    std::vector<std::vector<int>> where_;
+    std::vector<std::vector<float>> heat_;
+    std::vector<uint64_t> hits_in_layer_;
+    uint64_t clock_ = 0;
+    std::mutex mu_;
+    std::unique_ptr<IoQueue> io_;
+    Stats st_;
+};
+
+}  // namespace strata::ds4

src/dsv41/ds4_tq.cu

@@ -0,0 +1,539 @@
+// Strata. The trellis routed experts (TQK6 / TQK7 / TQK5X / TQK4X / TQK5T / TQK4T, ds4_iq.cuh's is_tq) on the device for the
+// DeepSeek-V4.1 engine path.  The weights stay as the file has them (a row of k values = k / 128 blocks of {fp16 d; 4 K stream
+// bytes}); the blob in VRAM is the file's bytes, so the expert cache, the tensor-parallel cuts and the CPU lane read the same
+// layout.  Arithmetic as the CPU lane's (ds4_tq_cpu.cpp): the activations are int8 with an F32 scale per 32 values (the
+// engine's q8_quant), the weights exact (fp16 codebook values times the block's fp16 scale); the sums are F32.
+//
+// gfx12 (RDNA4, R9700): v_wmma_f32_16x16x16_f16.  The weights are the B operand (16 rows of a tile = the 16 columns, a lane
+// holds one row's 8 consecutive k), the activations the A operand (16 entries = the 16 rows, split x = hi + lo in two fp16
+// so the product keeps ~2^-22), D[entry][weight row] in F32: a lane holds the entries 8 kg .. 8 kg + 7 of its row, so the
+// block scale d is the lane's own.  Lane l = r + 16 kg owns row r and the 16 steps 16 kg .. 16 kg + 15 of every block, whose
+// 15-bit states are one 128-bit window of the stream (the tile code of tools/dsv41_lab/wmma_emu.hpp, checked there against a
+// double reference and on the card by tests/dsv41/tqw_hip_test.cpp).  Elsewhere the same lane mapping in plain FMAs.
+#include "ds4_iq.cuh"
+
+#include <cuda_fp16.h>
+
+#include "ggml-common-tq.h"   // tq_lut_f16 (the 2048-point codebook as fp16 bit patterns, __device__)
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+
+#if defined(STRATA_USE_HIP) && !defined(DS4_TQ_PORTABLE) && (defined(__GFX12__) || defined(__gfx1200__) || defined(__gfx1201__))
+#define DS4_TQ_WMMA 1
+#endif
+
+#if !defined(STRATA_USE_HIP) && defined(__CUDACC__)   // (HIP's non-sync shuffle name, for the portable lane on CUDA)
+#define __shfl_xor(v, m) __shfl_xor_sync(0xffffffffu, (v), (m))
+#endif
+namespace strata::ds4::gguf::iq::tq {
+namespace {
+
+constexpr int kWaves = 4, kThreads = kWaves * 32;   // a block: 4 waves, a wave = one 16-row tile of each matrix
+constexpr int kTileRows = 16;
+constexpr int kMaxEnt = 16;   // entries (columns) a pass
+
+#if defined(DS4_TQ_WMMA)
+typedef _Float16 half8_t __attribute__((ext_vector_type(8)));
+typedef float float8_t __attribute__((ext_vector_type(8)));
+#endif
+
+constexpr bool computed_code(int type) { return type == kTQK5X || type == kTQK4X; }
+
+// ---------------------------------------------------------------- the codebook and the stream
+// a step's 15-bit state -> the four weights as two words of two fp16 (lo, hi): w0 w1 | w2 w3
+template <bool COMPUTED>
+__device__ __forceinline__ void step_words(const uint32_t* lut, uint32_t s, uint32_t& w0, uint32_t& w1) {
+    if (COMPUTED) {
+        w0 = ((s * 89226354u + 64248484u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+        w1 = ((s * 0x9E3779B1u + 0x7F4A7C15u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+    } else {
+        const uint32_t x = s * 0x9e3779b1u;
+        w0 = lut[x >> 21];
+        w1 = lut[(x >> 10) & 2047u];
+    }
+}
+__device__ __forceinline__ float half_bits_f(uint32_t b) { return __half2float(__ushort_as_half((unsigned short) b)); }
+
+// the 128-bit window of a lane's 16 steps in one block record: record = {u16 d; stream}, lane half kg takes stream bits
+// [(16 - 16 kg) K, (32 - 16 kg) K) plus the 15 - K bits after them (circular), so state(u), u = 0..15, is the 15 bits at
+// (15 - u) K of the window.
+template <int K>
+__device__ __forceinline__ void stream_window(const uint8_t* rec, int kg, uint64_t& lo, uint64_t& hi) {
+    const uint16_t* st = (const uint16_t*) (rec + 2);
+    constexpr int NW = (15 * K + 15 + 15) / 16;   // u16 words that hold 15 K + 15 bits
+    uint64_t w[2] = {0ull, 0ull};
+    if (kg == 1) {
+#pragma unroll
+        for (int i = 0; i < NW; ++i) w[i >> 2] |= (uint64_t) st[i] << (16 * (i & 3));
+    } else {   // the stream's upper half [16 K, 32 K) = K words; the bits past its end wrap to the stream's start
+        const uint16_t* up = st + K;
+#pragma unroll
+        for (int i = 0; i < K; ++i) w[i >> 2] |= (uint64_t) up[i] << (16 * (i & 3));
+        const uint64_t wrap = (uint64_t) st[0] & ((1ull << (15 - K)) - 1ull);   // 15 - K <= 11 bits: inside word 0
+        const int p = 16 * K;
+        w[p >> 6] |= wrap << (p & 63);   // (p & 63) + 15 - K <= 56: never past the word
+    }
+    lo = w[0];
+    hi = w[1];
+}
+__device__ __forceinline__ uint32_t state_at(uint64_t lo, uint64_t hi, int pos) {   // 15 bits at bit pos of the 128-bit window
+    uint64_t v;
+    if (pos + 15 <= 64) v = lo >> pos;
+    else if (pos >= 64) v = hi >> (pos - 64);
+    else v = (lo >> pos) | (hi << (64 - pos));
+    return (uint32_t) v & 0x7fffu;
+}
+
+// ---------------------------------------------------------------- activations: where a lane's 8 values of an entry come from
+// Each loader: valid(m) says entry m (this pass's column) exists; load8(m, k0, v) the 8 values k0..k0+7 as F32.
+struct ActQ8 {   // int8 rows with an F32 scale a 32: entry m = row rows[m] of q (n values a row), scales d (n / 32 a row)
+    const int8_t* q;
+    const float* d;
+    const int* rows;
+    int n;
+    __device__ __forceinline__ void load8(int m, int k0, float* v) const {
+        const size_t r = (size_t) rows[m];
+        const int2 w = *(const int2*) (q + r * n + k0);
+        const float sc = d[r * (n / 32) + (k0 >> 5)];
+        const int8_t* b = (const int8_t*) &w;
+#pragma unroll
+        for (int j = 0; j < 8; ++j) v[j] = (float) b[j] * sc;
+    }
+};
+struct ActF32 {   // F32 rows: entry m = row rows[m] (ld floats a row)
+    const float* x;
+    const int* rows;
+    size_t ld;
+    __device__ __forceinline__ void load8(int m, int k0, float* v) const {
+        const float4* p = (const float4*) (x + (size_t) rows[m] * ld + k0);
+        const float4 a = p[0], b = p[1];
+        v[0] = a.x; v[1] = a.y; v[2] = a.z; v[3] = a.w; v[4] = b.x; v[5] = b.y; v[6] = b.z; v[7] = b.w;
+    }
+};
+struct ActF16 {   // F16 rows: entry m = row rows[m] (ld halves a row)
+    const uint16_t* x;
+    const int* rows;
+    size_t ld;
+    __device__ __forceinline__ void load8(int m, int k0, float* v) const {
+        const uint4 w = *(const uint4*) (x + (size_t) rows[m] * ld + k0);
+        const uint32_t u[4] = {w.x, w.y, w.z, w.w};
+#pragma unroll
+        for (int j = 0; j < 4; ++j) {
+            v[2 * j] = half_bits_f(u[j] & 0xffffu);
+            v[2 * j + 1] = half_bits_f(u[j] >> 16);
+        }
+    }
+};
+
+// ---------------------------------------------------------------- the stream window by 32-bit loads
+// The same 128-bit window as stream_window, from aligned 32-bit loads (a record is 2 + 4 K bytes: its scale, then the stream, at
+// any 2-byte alignment): the array starts at the 4-byte floor of the record, the stream at bit 16 or 32 of it.  Loads past the
+// end of the record are skipped (the window bits above what a lane needs are junk anyway).
+template <int K>
+__device__ __forceinline__ void stream_window_u32(const uint8_t* rec, int kg, uint64_t& lo, uint64_t& hi) {
+    constexpr int bsz = 2 + 4 * K;
+    const uintptr_t ad = (uintptr_t) rec;
+    const uint32_t* pa = (const uint32_t*) (ad & ~(uintptr_t) 3);
+    const int off = 16 + 8 * (int) (ad & 3);                    // the stream's first bit in the array: 16 or 32
+    const int start = off + (kg ? 0 : 16 * K);
+    const int w0 = start >> 5, s = start & 31;
+    const int nwords = (int) (((ad & 3) + bsz + 3) >> 2);       // words that hold the record
+    uint32_t v[5];
+#pragma unroll
+    for (int i = 0; i < 5; ++i) v[i] = (w0 + i < nwords) ? pa[w0 + i] : 0u;
+    uint32_t r[4];
+#pragma unroll
+    for (int i = 0; i < 4; ++i) r[i] = (uint32_t) ((((uint64_t) v[i + 1] << 32) | v[i]) >> s);
+    lo = ((uint64_t) r[1] << 32) | r[0];
+    hi = ((uint64_t) r[3] << 32) | r[2];
+    if (!kg) {   // the upper half [16 K, 32 K) of the stream (16 K >= 64 bits); the bits past its end wrap to the stream's start
+        hi &= (1ull << (16 * K - 64)) - 1ull;
+        const uint32_t first = off == 16 ? (pa[0] >> 16) : pa[1];
+        const uint64_t wrap = (uint64_t) first & ((1ull << (15 - K)) - 1ull);
+        const int p = 16 * K - 64;
+        hi |= wrap << p;
+    }
+}
+
+// ---------------------------------------------------------------- NM 16-row tiles (matrices sharing the activations) over up to 16 entries
+// acc_out[mi][i] (i < 8) = D[entry 8 kg + i][row r] of matrix mi's tile: sum over the row's k values of x_entry * w, k % 128 == 0.
+// wrows[mi]: the tile's row r (this lane's) of matrix mi in the blob; nblk blocks.  The activation fragments are made once for all NM.
+template <int K, bool COMPUTED, int NM, bool WORDS, class Act>
+__device__ __forceinline__ void tile_dot(const uint8_t* const* wrows, int nblk, const uint32_t* lut, const Act& act, int ne,
+                                         float (*acc_out)[8]) {
+    constexpr int bsz = 2 + 4 * K;
+    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4;
+    float tot[NM][8];
+#pragma unroll
+    for (int mi = 0; mi < NM; ++mi)
+#pragma unroll
+        for (int i = 0; i < 8; ++i) tot[mi][i] = 0.0f;
+    const bool have = r < ne;   // this lane's A rows (the entry m = r)
+    for (int b = 0; b < nblk; ++b) {
+        float d[NM];
+        uint64_t lo[NM], hi[NM];
+#pragma unroll
+        for (int mi = 0; mi < NM; ++mi) {
+            const uint8_t* rec = wrows[mi] + (size_t) b * bsz;
+            d[mi] = half_bits_f(*(const uint16_t*) rec);
+            if (WORDS) stream_window_u32<K>(rec, kg, lo[mi], hi[mi]);
+            else stream_window<K>(rec, kg, lo[mi], hi[mi]);
+        }
+#if defined(DS4_TQ_WMMA)
+        float8_t acc[NM];
+#pragma unroll
+        for (int mi = 0; mi < NM; ++mi) acc[mi] = float8_t{0, 0, 0, 0, 0, 0, 0, 0};
+#pragma unroll
+        for (int m = 0; m < 8; ++m) {
+            half8_t ah, al;
+            {
+                float v[8];
+                if (have) act.load8(r, b * 128 + 64 * kg + 8 * m, v);
+                else {
+#pragma unroll
+                    for (int j = 0; j < 8; ++j) v[j] = 0.0f;
+                }
+#pragma unroll
+                for (int j = 0; j < 8; ++j) {
+                    const _Float16 h = (_Float16) v[j];
+                    ah[j] = h;
+                    al[j] = (_Float16) (v[j] - (float) h);
+                }
+            }
+#pragma unroll
+            for (int mi = 0; mi < NM; ++mi) {
+                half8_t wf;
+#pragma unroll
+                for (int s2 = 0; s2 < 2; ++s2) {
+                    const int u = 2 * m + s2;
+                    uint32_t w0, w1;
+                    step_words<COMPUTED>(lut, state_at(lo[mi], hi[mi], (15 - u) * K), w0, w1);
+                    wf[4 * s2 + 0] = __builtin_bit_cast(_Float16, (uint16_t) (w0 & 0xffffu));
+                    wf[4 * s2 + 1] = __builtin_bit_cast(_Float16, (uint16_t) (w0 >> 16));
+                    wf[4 * s2 + 2] = __builtin_bit_cast(_Float16, (uint16_t) (w1 & 0xffffu));
+                    wf[4 * s2 + 3] = __builtin_bit_cast(_Float16, (uint16_t) (w1 >> 16));
+                }
+                acc[mi] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(ah, wf, acc[mi]);   // D[m = entry][n = weight row]
+                acc[mi] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(al, wf, acc[mi]);
+            }
+        }
+#pragma unroll
+        for (int mi = 0; mi < NM; ++mi)
+#pragma unroll
+            for (int i = 0; i < 8; ++i) tot[mi][i] += d[mi] * acc[mi][i];
+#else
+        // plain FMAs: this lane's row r, its 64 weights of the block against every entry (lane r + 16 kg' sums the halves)
+        float part[NM][kMaxEnt];
+#pragma unroll
+        for (int mi = 0; mi < NM; ++mi)
+#pragma unroll
+            for (int e = 0; e < kMaxEnt; ++e) part[mi][e] = 0.0f;
+#pragma unroll
+        for (int m = 0; m < 8; ++m) {
+            float w[NM][8];
+#pragma unroll
+            for (int mi = 0; mi < NM; ++mi)
+#pragma unroll
+                for (int s2 = 0; s2 < 2; ++s2) {
+                    uint32_t w0, w1;
+                    step_words<COMPUTED>(lut, state_at(lo[mi], hi[mi], (15 - (2 * m + s2)) * K), w0, w1);
+                    w[mi][4 * s2 + 0] = half_bits_f(w0 & 0xffffu);
+                    w[mi][4 * s2 + 1] = half_bits_f(w0 >> 16);
+                    w[mi][4 * s2 + 2] = half_bits_f(w1 & 0xffffu);
+                    w[mi][4 * s2 + 3] = half_bits_f(w1 >> 16);
+                }
+#pragma unroll
+            for (int e = 0; e < kMaxEnt; ++e)
+                if (e < ne) {
+                    float v[8];
+                    act.load8(e, b * 128 + 64 * kg + 8 * m, v);
+#pragma unroll
+                    for (int mi = 0; mi < NM; ++mi)
+#pragma unroll
+                        for (int j = 0; j < 8; ++j) part[mi][e] = fmaf(w[mi][j], v[j], part[mi][e]);
+                }
+        }
+        // entries 0..7 into tot[i] of the kg = 0 lane, 8..15 of the kg = 1 lane, so the output indexing matches the WMMA's
+#pragma unroll
+        for (int mi = 0; mi < NM; ++mi)
+#pragma unroll
+            for (int i = 0; i < 8; ++i) {
+                const float p0 = part[mi][i], p1 = part[mi][8 + i];
+                const float q0 = __shfl_xor_sync(0xffffffffu, p0, 16), q1 = __shfl_xor_sync(0xffffffffu, p1, 16);
+                tot[mi][i] += d[mi] * (kg == 0 ? p0 + q0 : p1 + q1);
+            }
+#endif
+    }
+#pragma unroll
+    for (int mi = 0; mi < NM; ++mi)
+#pragma unroll
+        for (int i = 0; i < 8; ++i) acc_out[mi][i] = tot[mi][i];
+}
+
+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {   // ds4_iq.cu's
+    if (limit > 0.0f) {
+        gv = fminf(gv, limit);
+        uv = fminf(fmaxf(uv, -limit), limit);
+    }
+    return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
+}
+
+template <int K, bool COMPUTED>
+__device__ __forceinline__ void load_lut(uint32_t* lut) {
+    if (!COMPUTED) {
+        // the table as the CPU decoder has it (tq_cpu.cpp's int16 state table): fp16 * 2^13 rounded half away from zero, over
+        // 2^13 - exact for values >= 2^-3 and within 2^-14 below; the result is still an exact fp16 (fewer significant bits)
+        for (int i = threadIdx.x; i < 2048; i += blockDim.x) {
+            uint32_t pair = 0;
+#pragma unroll
+            for (int j = 0; j < 2; ++j) {
+                const float v = half_bits_f(tq_lut_f16[2 * i + j]) * 8192.0f;
+                const int n = (int) (v < 0.0f ? v - 0.5f : v + 0.5f);
+                pair |= (uint32_t) __half_as_ushort(__float2half_rn((float) n / 8192.0f)) << (16 * j);
+            }
+            lut[i] = pair;
+        }
+        __syncthreads();
+    }
+}
+
+// ---------------------------------------------------------------- token / window experts
+__device__ __forceinline__ bool group_of(const Args& a, int grp, const uint8_t*& blob, int& m0, int& m1) {
+    if (a.grp_ptr == nullptr) {
+        const unsigned long long p = a.plan_ptr[grp];
+        if (p == 0ull) return false;
+        blob = (const uint8_t*) p;
+        m0 = grp;
+        m1 = grp + 1;
+        return true;
+    }
+    if (grp >= *a.n_groups) return false;
+    blob = (const uint8_t*) a.grp_ptr[grp];
+    m0 = a.grp_start[grp];
+    m1 = a.grp_start[grp + 1];
+    return true;
+}
+
+// grid (n_ff / 64, groups): a wave = a 16-row tile of the gate and of the up of the group's expert, 16 entries a pass
+template <int K, bool COMPUTED, bool WORDS>
+__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {
+    __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
+    __shared__ int ent[kMaxEnt], xr[kMaxEnt];
+    const uint8_t* blob;
+    int m0, m1;
+    if (!group_of(a, blockIdx.y, blob, m0, m1)) return;   // (uniform)
+    load_lut<K, COMPUTED>(lut);
+    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
+    const int tile = blockIdx.x * kWaves + wave;
+    const int row = tile * kTileRows + r;
+    const size_t rb = (size_t) (a.n_embd / 128) * (2 + 4 * K);
+    const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;
+    const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;
+    for (int mm = m0; mm < m1; mm += kMaxEnt) {
+        const int ne = min(kMaxEnt, m1 - mm);
+        __syncthreads();
+        if ((int) threadIdx.x < ne) {
+            const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
+            ent[threadIdx.x] = e;
+            xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;
+        }
+        __syncthreads();
+        if (tile * kTileRows >= a.n_ff) continue;
+        const ActQ8 act{a.xq, a.xd, xr, a.n_embd};
+        const uint8_t* wr[2] = {g, u};
+        float gu[2][8];
+        tile_dot<K, COMPUTED, 2, WORDS>(wr, a.n_embd / 128, lut, act, ne, gu);
+#pragma unroll
+        for (int i = 0; i < 8; ++i) {
+            const int m = 8 * kg + i;
+            if (m < ne) a.h[(size_t) ent[m] * a.n_ff + row] = swiglu(gu[0][i], gu[1][i], a.limit);
+        }
+    }
+}
+
+// grid (n_embd / 64, groups): the down's tile of a wave against the entries' int8 h
+template <int K, bool COMPUTED, bool WORDS>
+__global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {
+    __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
+    __shared__ int ent[kMaxEnt];
+    const uint8_t* blob;
+    int m0, m1;
+    if (!group_of(a, blockIdx.y, blob, m0, m1)) return;
+    load_lut<K, COMPUTED>(lut);
+    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
+    const int tile = blockIdx.x * kWaves + wave;
+    const int row = tile * kTileRows + r;
+    const size_t rb = (size_t) (a.n_ff / 128) * (2 + 4 * K);
+    const uint8_t* d = blob + a.tr[2] + (size_t) row * rb;
+    for (int mm = m0; mm < m1; mm += kMaxEnt) {
+        const int ne = min(kMaxEnt, m1 - mm);
+        __syncthreads();
+        if ((int) threadIdx.x < ne) ent[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
+        __syncthreads();
+        if (tile * kTileRows >= a.n_embd) continue;
+        const ActQ8 act{a.hq, a.hd, ent, a.n_ff};
+        const uint8_t* wr[1] = {d};
+        float dd[1][8];
+        tile_dot<K, COMPUTED, 1, WORDS>(wr, a.n_ff / 128, lut, act, ne, dd);
+#pragma unroll
+        for (int i = 0; i < 8; ++i) {
+            const int m = 8 * kg + i;
+            if (m < ne) a.part[(size_t) ent[m] * a.n_embd + row] = dd[0][i];
+        }
+    }
+}
+
+// ---------------------------------------------------------------- the prompt path's sets
+// grid (n_ff / 64, chunks of 16 rows, n_exp): expert e = blockIdx.z owns the sorted rows [bounds[e], bounds[e + 1]); local
+// row = sorted row - row0.  gate / up: F32 x[idx[local]] (ldx) -> F16 swiglu h[local * ldo + c]; down: F16 xh[local] (ldx) ->
+// F32 y[local * ldo + c].
+template <int K, bool COMPUTED, bool GATE_UP, bool WORDS>
+__global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {
+    __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
+    __shared__ int rowsel[kMaxEnt];
+    const int e = blockIdx.z;
+    const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
+    const int r0 = lo + blockIdx.y * kMaxEnt;
+    if (r0 >= hi) return;   // (uniform)
+    load_lut<K, COMPUTED>(lut);
+    const int ne = min(kMaxEnt, hi - r0);
+    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
+    const int N = GATE_UP ? a.n_ff : a.n_embd, Kin = GATE_UP ? a.n_embd : a.n_ff;
+    const int tile = blockIdx.x * kWaves + wave;
+    const int row = tile * kTileRows + r;
+    const size_t rb = (size_t) (Kin / 128) * (2 + 4 * K);
+    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+    if ((int) threadIdx.x < ne) rowsel[threadIdx.x] = GATE_UP ? a.idx[r0 + threadIdx.x] : r0 + (int) threadIdx.x;
+    __syncthreads();
+    if (tile * kTileRows >= N) return;
+    float res[2][8];
+    if (GATE_UP) {
+        const ActF32 act{a.xf, rowsel, (size_t) a.ldx};
+        const uint8_t* wr[2] = {blob + a.tr[0] + (size_t) row * rb, blob + a.tr[1] + (size_t) row * rb};
+        tile_dot<K, COMPUTED, 2, WORDS>(wr, Kin / 128, lut, act, ne, res);
+    } else {
+        const ActF16 act{a.xh, rowsel, (size_t) a.ldx};
+        const uint8_t* wr[1] = {blob + a.tr[2] + (size_t) row * rb};
+        tile_dot<K, COMPUTED, 1, WORDS>(wr, Kin / 128, lut, act, ne, res);
+    }
+    const int kgl = threadIdx.x >> 4 & 1;
+#pragma unroll
+    for (int i = 0; i < 8; ++i) {
+        const int m = 8 * kgl + i;
+        if (m >= ne) continue;
+        const size_t lr = (size_t) (r0 + m);
+        if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(res[0][i], res[1][i], a.limit)));
+        else a.y[lr * a.ldo + row] = res[0][i];
+    }
+}
+
+// ---------------------------------------------------------------- GEMV jobs
+// grid (n / 64): y[c * ycs + r] = alpha (row r . x_c) + bias[r] for ncol (<= 16) int8 activation columns
+template <int K, bool COMPUTED, bool WORDS>
+__global__ void __launch_bounds__(kThreads) mv_kernel(const MvJob j) {
+    __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
+    __shared__ int cols[kMaxEnt];
+    load_lut<K, COMPUTED>(lut);
+    if ((int) threadIdx.x < kMaxEnt) cols[threadIdx.x] = (int) threadIdx.x;
+    __syncthreads();
+    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
+    const int tile = blockIdx.x * kWaves + wave;
+    if (tile * kTileRows >= j.n) return;
+    const int row = tile * kTileRows + r;
+    const ActQ8 act{j.xq, j.xd, cols, j.k};
+    const uint8_t* wr[1] = {j.w + (size_t) row * j.ld};
+    float accm[1][8];
+    tile_dot<K, COMPUTED, 1, WORDS>(wr, j.k / 128, lut, act, j.ncol, accm);
+#pragma unroll
+    for (int i = 0; i < 8; ++i) {
+        const int m = 8 * kg + i;
+        if (m < j.ncol && row < j.n) j.y[(size_t) m * j.ycs + row] = j.alpha * accm[0][i] + (j.bias != nullptr ? j.bias[row] : 0.0f);
+    }
+}
+
+// ---------------------------------------------------------------- dispatch
+#define DS4_TQ_SWITCH(t, F)                         \
+    switch (t) {                                    \
+        case kTQK6: F(6, false); break;             \
+        case kTQK7: F(7, false); break;             \
+        case kTQK5X: F(5, true); break;             \
+        case kTQK4X: F(4, true); break;             \
+        case kTQK5T: F(5, false); break;            \
+        case kTQK4T: F(4, false); break;            \
+        default: return false;                      \
+    }
+
+// 32-bit stream loads (STRATA_DS4_TQ_WORDS=1) or the 16-bit ones (default)
+bool use_words() {
+    static const bool on = [] {
+        const char* e = std::getenv("STRATA_DS4_TQ_WORDS");
+        return e != nullptr && std::atoi(e) != 0;   // (no gain measured on the R9700: 13.8 vs 14.2 ms a token)
+    }();
+    return on;
+}
+
+bool bad(const char* what) {
+    std::fprintf(stderr, "ds4 tq: %s\n", what);
+    return false;
+}
+}  // namespace
+
+bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
+    const bool w = use_words();
+    (void) ne_max;
+    if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("gate_up: geometry not covered");
+    const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) groups);
+#define GU_(K, C) { if (w) gate_up_kernel<K, C, true><<<grid, kThreads, 0, s>>>(a); else gate_up_kernel<K, C, false><<<grid, kThreads, 0, s>>>(a); }
+    DS4_TQ_SWITCH(a.tg, GU_)
+#undef GU_
+    return true;
+}
+
+bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {
+    const bool w = use_words();
+    (void) ne_max;
+    if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("down: geometry not covered");
+    const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) groups);
+#define DN_(K, C) { if (w) down_kernel<K, C, true><<<grid, kThreads, 0, s>>>(a); else down_kernel<K, C, false><<<grid, kThreads, 0, s>>>(a); }
+    DS4_TQ_SWITCH(a.td, DN_)
+#undef DN_
+    return true;
+}
+
+bool set_gate_up(const SetArgs& a, cudaStream_t s) {
+    const bool w = use_words();
+    if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("set_gate_up: geometry not covered");
+    const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);
+#define SG_(K, C) { if (w) set_kernel<K, C, true, true><<<grid, kThreads, 0, s>>>(a); else set_kernel<K, C, true, false><<<grid, kThreads, 0, s>>>(a); }
+    DS4_TQ_SWITCH(a.tg, SG_)
+#undef SG_
+    return true;
+}
+
+bool set_down(const SetArgs& a, cudaStream_t s) {
+    const bool w = use_words();
+    if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("set_down: geometry not covered");
+    const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);
+#define SD_(K, C) { if (w) set_kernel<K, C, false, true><<<grid, kThreads, 0, s>>>(a); else set_kernel<K, C, false, false><<<grid, kThreads, 0, s>>>(a); }
+    DS4_TQ_SWITCH(a.td, SD_)
+#undef SD_
+    return true;
+}
+
+bool mv(const MvJob* jobs, int n, cudaStream_t s) {
+    for (int i = 0; i < n; ++i) {
+        const MvJob& j = jobs[i];
+        const bool w = use_words();
+        if (j.k % 128 != 0 || j.n % kTileRows != 0 || j.ncol < 1 || j.ncol > kMaxEnt) return bad("mv: geometry not covered");
+        const dim3 grid((unsigned) ((j.n + kWaves * kTileRows - 1) / (kWaves * kTileRows)));
+#define MV_(K, C) { if (w) mv_kernel<K, C, true><<<grid, kThreads, 0, s>>>(j); else mv_kernel<K, C, false><<<grid, kThreads, 0, s>>>(j); }
+        DS4_TQ_SWITCH(j.type, MV_)
+#undef MV_
+    }
+    return true;
+}
+
+}  // namespace strata::ds4::gguf::iq::tq

src/dsv41/ds4_tq_cpu.cpp

@@ -0,0 +1,136 @@
+// Strata. The trellis routed experts (ds4_iq.cuh's is_tq types) on the CPU lane of the DeepSeek-V4.1 engine path: Strata's
+// own decode-and-dot kernels (include/strata/kernels/cpu/tq_cpu.hpp: AVX-512 / AVX2 / scalar, one result whatever the ISA)
+// over the file's rows read in place.  The activations are the device's (ds4_gguf.cu's q8_quant: int8 with an F32 scale a 32
+// values), so an expert gives the same sums on either lane up to the order of the blocks' float sums.
+#include "ds4_gguf_cpu.hpp"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#if defined(STRATA_DSV41_TQ_CPU)
+#include "strata/kernels/cpu/tq_cpu.hpp"
+#endif
+
+namespace strata::ds4::gguf {
+
+#if defined(STRATA_DSV41_TQ_CPU)
+namespace kc = strata::kernels::cpu;
+namespace {
+
+int kbits_of(int type) {
+    switch (type) {
+        case 145: return 6;
+        case 146: return 7;
+        case 224: case 226: return 5;
+        case 225: case 227: return 4;
+        default: return 0;
+    }
+}
+constexpr int kActTq = 100;   // IqCpu::gu_act / d_act of these types: the layout below
+
+// an activation of k values: k / 32 F32 scales, then k int8
+size_t act_bytes_of(int k) { return (size_t) k / 32 * 4 + (size_t) k; }
+
+void unpack(const void* act, int k, kc::tq_detail::TqAct& a) {
+    const uint8_t* p = (const uint8_t*) act;
+    std::memcpy(a.d, p, (size_t) k / 32 * 4);
+    const int8_t* q = (const int8_t*) (p + (size_t) k / 32 * 4);
+    for (int i = 0; i < k; ++i) a.q[i] = q[i];
+}
+
+// dots of rows [r0, r1) of one matrix against nt activations: out[t][r]
+void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {
+    static thread_local std::vector<kc::tq_detail::TqAct> unp(kc::kTqMaxTokens);
+    const kc::tq_detail::TqAct* ap[kc::kTqMaxTokens];
+    for (int t = 0; t < nt; ++t) {
+        unpack(act[t], n, unp[(size_t) t]);
+        ap[t] = &unp[(size_t) t];
+    }
+    const int kb = kc::tq_kbits(type);
+    const int16_t* tab = kc::tq_state_table_for(type);
+    switch (kc::tq_isa()) {
+        case kc::TqIsa::avx512: kc::tq_detail::tq_rows_avx512(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
+        case kc::TqIsa::avx2: kc::tq_detail::tq_rows_avx2(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
+        default: kc::tq_detail::tq_rows_scalar(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
+    }
+}
+}  // namespace
+
+bool tq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
+    if (kbits_of(tg) == 0 || kbits_of(td) == 0) {
+        err = "not trellis types";
+        return false;
+    }
+    if (n_embd % 128 != 0 || n_ff % 128 != 0 || n_embd > kc::kTqMaxN || n_ff > kc::kTqMaxN) {
+        err = "the trellis CPU kernels take widths of 128 blocks up to " + std::to_string(kc::kTqMaxN);
+        return false;
+    }
+    f.tg = tg;
+    f.td = td;
+    f.n_embd = n_embd;
+    f.n_ff = n_ff;
+    f.gu_act = f.d_act = kActTq;
+    f.gu_row = (size_t) (n_embd / 128) * (2 + 4 * kbits_of(tg));
+    f.d_row = (size_t) (n_ff / 128) * (2 + 4 * kbits_of(td));
+    f.act_bytes = act_bytes_of(n_embd);
+    f.h_bytes = act_bytes_of(n_ff);
+    return true;
+}
+
+// x (k values) -> the layout above (the device's q8_quant: sc = max|x| / 127, q = round-half-even(x / sc))
+void tq_cpu_quant(const float* x, int k, void* dst) {
+    float* d = (float*) dst;
+    int8_t* q = (int8_t*) ((uint8_t*) dst + (size_t) k / 32 * 4);
+    for (int b = 0; b < k / 32; ++b) {
+        const float* xb = x + b * 32;
+        float am = 0.0f;
+        for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(xb[j]));
+        const float sc = am / 127.0f;
+        d[b] = sc;
+        for (int j = 0; j < 32; ++j) q[b * 32 + j] = (int8_t) (am > 0.0f ? (int) std::nearbyint(xb[j] / sc) : 0);
+    }
+}
+
+void tq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
+                    int r0, int r1, float limit) {
+    static thread_local std::vector<float> tmp[kc::kTqMaxTokens];
+    float* up_out[kc::kTqMaxTokens];
+    for (int t = 0; t < nt; ++t) {
+        tmp[t].resize((size_t) r1);
+        up_out[t] = tmp[t].data();
+    }
+    dots(f.tg, gate, f.gu_row, f.n_embd, act, nt, ff, r0, r1);
+    dots(f.tg, up, f.gu_row, f.n_embd, act, nt, up_out, r0, r1);
+    for (int t = 0; t < nt; ++t)
+        for (int r = r0; r < r1; ++r) {
+            float g = ff[t][r], u = up_out[t][r];
+            if (limit > 0.0f) {
+                g = std::min(g, limit);
+                u = std::min(std::max(u, -limit), limit);
+            }
+            ff[t][r] = g / (1.0f + std::exp(-g)) * u;
+        }
+}
+
+void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
+    dots(f.td, down, f.d_row, f.n_ff, hq, nt, out, r0, r1);
+}
+bool tq_cpu_type(int type) { return kbits_of(type) > 0; }
+int tq_cpu_act() { return kActTq; }
+#else
+bool tq_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {
+    err = "this build has no CPU trellis kernels (STRATA_NATIVE_EXPERTS=OFF)";
+    return false;
+}
+void tq_cpu_quant(const float*, int, void*) {}
+void tq_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int, float) {}
+void tq_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}
+bool tq_cpu_type(int) { return false; }
+int tq_cpu_act() { return 100; }
+#endif
+
+}  // namespace strata::ds4::gguf

src/dsv41/ds4_tqw.cuh

@@ -0,0 +1,138 @@
+// Strata. gfx12 (RDNA4, R9700 = gfx1201) WMMA form of the trellis expert GEMV of the dsv41 CUDA lab (tools/dsv41_lab:
+// tqk_gemv3.cuh, wmma_emu.hpp = the specification, hip/tqk_gemv_wmma.hip.cpp = its first HIP form).  Layout "TQW" (see
+// wmma_emu.hpp::repack_tqw): a wave = T tiles of 16 rows, lane l = r + 16 kg owns row r and 16 consecutive steps of every
+// 128-weight block; activations are the B operand (NB <= 16 columns), split x = hi + lo in two fp16 for ~2^-22 accuracy.
+// CODE 0: the table codebook (K5/K6/K7 of the shipped files: two lookups into an 8 KB LDS table of half2 per step);
+// CODE 1: the computed "3single" code (no table; the new K4/K5 types if they take it).
+#pragma once
+#include <hip/hip_runtime.h>
+#include <cstdint>
+
+namespace strata::ds4::tqw {
+
+typedef _Float16 half8_t __attribute__((ext_vector_type(8)));
+typedef float float8_t __attribute__((ext_vector_type(8)));
+
+constexpr uint32_t kMask = 0x7fffu;
+
+struct Job {
+    const uint32_t* words;     // TQW words  [tile][P][K][32 lanes]
+    const uint16_t* scales;    // [tile][P][16 rows][2 blocks] fp16
+    const half8_t* xb;         // prepared B fragments (prep_b)
+    float* y;                  // [NB][ldy]
+    int rows, in, ldy;
+};
+
+__device__ __forceinline__ void step_words(const uint32_t* lut, uint32_t s, uint32_t& w0, uint32_t& w1, int code) {
+    if (code == 1) {
+        w0 = ((s * 89226354u + 64248484u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+        w1 = ((s * 0x9E3779B1u + 0x7F4A7C15u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+    } else {
+        const uint32_t x = s * 0x9e3779b1u;
+        w0 = lut[x >> 21];
+        w1 = lut[(x >> 10) & 2047u];
+    }
+}
+
+// activations -> B fragments.  out[(((blk * 8 + m) * 2 + kg) * nb + n) * 2 + {0 = hi, 1 = lo}] = 8 halves of x[n][blk * 128 + 4 * (16 kg + 2 m) + j]
+__global__ void prep_b(const float* x, int ldx, int nb, int in, half8_t* out) {
+    const int nblk = in / 128;
+    const size_t gid = (size_t) blockIdx.x * blockDim.x + threadIdx.x, n_all = (size_t) nblk * 8 * 2 * nb;
+    if (gid >= n_all) return;
+    const int n = (int) (gid % nb), kg = (int) ((gid / nb) % 2), m = (int) ((gid / (2 * nb)) % 8), blk = (int) (gid / (16 * nb));
+    const float* p = x + (size_t) n * ldx + blk * 128 + 4 * (16 * kg + 2 * m);
+    half8_t hi, lo;
+#pragma unroll
+    for (int j = 0; j < 8; ++j) { hi[j] = (_Float16) p[j]; lo[j] = (_Float16) (p[j] - (float) hi[j]); }
+    out[gid * 2] = hi;
+    out[gid * 2 + 1] = lo;
+}
+
+template <int K, int NB, int T, int EXACT, int CODE>
+__global__ void __launch_bounds__(128) gemv(const Job* jobs, const uint32_t* __restrict__ lut_g) {
+    __shared__ uint32_t lut[CODE == 0 ? 2048 : 1];
+    if (CODE == 0) {
+        for (int i = threadIdx.x; i < 2048; i += blockDim.x) lut[i] = lut_g[i];
+        __syncthreads();
+    }
+    const Job job = jobs[blockIdx.y];
+    const int wave = threadIdx.x >> 5, lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4;
+    const int tile0 = (blockIdx.x * (blockDim.x >> 5) + wave) * T;
+    if (tile0 * 16 >= job.rows) return;
+    const int nblk = job.in / 128, NP = nblk / 2;
+    float tot[T][8];
+#pragma unroll
+    for (int t = 0; t < T; ++t)
+#pragma unroll
+        for (int i = 0; i < 8; ++i) tot[t][i] = 0.f;
+    for (int P = 0; P < NP; ++P) {
+        uint32_t rec[T][K];
+#pragma unroll
+        for (int t = 0; t < T; ++t)
+#pragma unroll
+            for (int j = 0; j < K; ++j) rec[t][j] = __builtin_nontemporal_load(job.words + (((size_t) (tile0 + t) * NP + P) * K + j) * 32 + lane);
+#pragma unroll
+        for (int bk = 0; bk < 2; ++bk) {
+            half8_t bh[8], bl[8];
+            if (r < NB) {
+#pragma unroll
+                for (int m = 0; m < 8; ++m) {
+                    const half8_t* p = job.xb + (((((size_t) (2 * P + bk) * 8 + m) * 2 + kg) * NB + r) * 2);
+                    bh[m] = p[0];
+                    if (EXACT) bl[m] = p[1];
+                }
+            } else {
+#pragma unroll
+                for (int m = 0; m < 8; ++m) { bh[m] = half8_t{0, 0, 0, 0, 0, 0, 0, 0}; bl[m] = bh[m]; }
+            }
+#pragma unroll
+            for (int t = 0; t < T; ++t) {
+                // chunk bk of the record: bits [16 K bk, 16 K bk + 16 K); window 0 = steps u 0..7 (+ 15-K bits of the partner chunk),
+                // window 1 = steps u 8..15
+                constexpr int NW = K;   // (words of a record)
+                auto bits = [&](int pos, int n) -> uint64_t {   // n <= 56 bits at record bit pos (pos is a compile-time constant here)
+                    const int a = pos >> 5, sh = pos & 31;
+                    const uint32_t w0 = rec[t][a], w1 = a + 1 < NW ? rec[t][a + 1 < NW ? a + 1 : 0] : 0u, w2 = a + 2 < NW ? rec[t][a + 2 < NW ? a + 2 : 0] : 0u;
+                    // (w2:w1:w0) >> sh, low 64 bits
+                    const uint64_t out = (sh == 0) ? (((uint64_t) w1 << 32) | w0) : ((((uint64_t) w1 << 32) | w0) >> sh) | ((uint64_t) w2 << (64 - sh));
+                    return out & ((1ull << n) - 1ull);
+                };
+                const int b0 = 16 * K * bk;
+                const uint64_t lo8 = bits(b0, 8 * K), hi8 = bits(b0 + 8 * K, 8 * K);
+                const uint32_t mine = (uint32_t) (lo8 & 0xffffffffu);   // the bits the PARTNER needs: this lane's chunk bits [0, 15 - K)
+                const uint32_t pc = __shfl_xor((int) mine, 16);
+                const uint64_t win0 = hi8 | ((uint64_t) (pc & ((1u << (15 - K)) - 1u)) << (8 * K));
+                const uint64_t win1 = lo8 | ((hi8 & ((1ull << (15 - K)) - 1ull)) << (8 * K));
+                float8_t acc = {0, 0, 0, 0, 0, 0, 0, 0};
+#pragma unroll
+                for (int m = 0; m < 8; ++m) {
+                    half8_t a;
+#pragma unroll
+                    for (int s2 = 0; s2 < 2; ++s2) {
+                        const int u = 2 * m + s2;
+                        const uint32_t st = (uint32_t) (u < 8 ? win0 >> ((7 - u) * K) : win1 >> ((15 - u) * K)) & kMask;
+                        uint32_t w0, w1;
+                        step_words(lut, st, w0, w1, CODE);
+                        a[4 * s2 + 0] = __builtin_bit_cast(_Float16, (uint16_t) (w0 & 0xffffu));
+                        a[4 * s2 + 1] = __builtin_bit_cast(_Float16, (uint16_t) (w0 >> 16));
+                        a[4 * s2 + 2] = __builtin_bit_cast(_Float16, (uint16_t) (w1 & 0xffffu));
+                        a[4 * s2 + 3] = __builtin_bit_cast(_Float16, (uint16_t) (w1 >> 16));
+                    }
+                    acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, bh[m], acc);
+                    if (EXACT) acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, bl[m], acc);
+                }
+                // scales of rows 8 kg .. 8 kg + 7 of this block: 8 consecutive (row, bk) pairs of 2 -> stride 2
+                const uint16_t* sp = job.scales + (((size_t) (tile0 + t) * NP + P) * 16 + 8 * kg) * 2 + bk;
+#pragma unroll
+                for (int i = 0; i < 8; ++i) tot[t][i] += (float) __builtin_bit_cast(_Float16, sp[2 * i]) * acc[i];
+            }
+        }
+    }
+#pragma unroll
+    for (int t = 0; t < T; ++t)
+        if (r < NB)
+#pragma unroll
+            for (int i = 0; i < 8; ++i) job.y[(size_t) r * job.ldy + (tile0 + t) * 16 + 8 * kg + i] = tot[t][i];
+}
+
+}  // namespace strata::ds4::tqw

src/dsv41/ds4_vision.cu

@@ -0,0 +1,519 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_vision.cu - DeepSeek-V4.1's vision tower (see ds4_vision.cuh).
+//
+// One encode, N patches on a gh x gw grid:
+//   patchify (the reference's (x / 255 - 0.5) / 0.5, BF16) -> x = patch_embed(patches)               FP32 [N][1024]
+//   per block, its weights in slot (b % 2) while block b + 1's copy runs:
+//     xn = norm1(x); qkv = rope(wqkv(xn) + bias)                       (rows in pieces of rb_: the FP32 GEMM output)
+//     per query piece of qb_ rows, per head: S = Q K^T / 8 (FP32), P = softmax(S) (BF16), O[:, head] = P V;
+//       x[piece] += wo(O) + bias
+//     xn = norm2(x); per row piece: x += w2(silu(gate) * up), gate | up = w1(xn)
+//   xn = norm(x); the 3 x 3 unfold; GELU(w1 + b1); w2 + b2 -> BF16 rows (the aligner's weights stream in pieces too)
+#include "ds4_vision.cuh"
+
+#include <cublas_v2.h>
+#include "ds4_numerics.cuh"
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cfloat>
+#include <cmath>
+#include <cstring>
+
+namespace strata::ds4 {
+namespace {
+
+#define VK(call)                                                                      \
+    do {                                                                              \
+        cudaError_t e_ = (call);                                                      \
+        if (e_ != cudaSuccess) {                                                      \
+            err = std::string("vision: ") + #call + ": " + cudaGetErrorString(e_);    \
+            return false;                                                             \
+        }                                                                             \
+    } while (0)
+#define VB(call)                                                                                      \
+    do {                                                                                              \
+        cublasStatus_t e_ = (call);                                                                   \
+        if (e_ != CUBLAS_STATUS_SUCCESS) {                                                            \
+            err = std::string("vision: ") + #call + ": cuBLAS status " + std::to_string((int) e_);   \
+            return false;                                                                             \
+        }                                                                                             \
+    } while (0)
+
+constexpr int kT = 256;                  // threads of the row kernels
+constexpr int kSmT = 512, kSmPer = 20;   // softmax: a row of up to 10240 keys in registers
+constexpr size_t kWs = 16u << 20;        // cuBLAS workspace
+
+__device__ __forceinline__ float bf(uint16_t b) { return __uint_as_float((uint32_t) b << 16); }
+__device__ __forceinline__ uint16_t bfr(float f) { return bf16_rn(f); }
+
+__device__ float block_reduce(float v, bool is_max) {
+    __shared__ float red[32];
+    __shared__ float tot;
+    for (int o = 16; o > 0; o >>= 1) {
+        const float u = __shfl_xor_sync(0xffffffffu, v, o);
+        v = is_max ? fmaxf(v, u) : v + u;
+    }
+    const int lane = threadIdx.x & 31, w = threadIdx.x >> 5, nw = (blockDim.x + 31) >> 5;
+    __syncthreads();   // (an earlier call's readers of red / tot are done)
+    if (lane == 0) red[w] = v;
+    __syncthreads();
+    if (w == 0) {
+        v = lane < nw ? red[lane] : (is_max ? -FLT_MAX : 0.0f);
+        for (int o = 16; o > 0; o >>= 1) {
+            const float u = __shfl_xor_sync(0xffffffffu, v, o);
+            v = is_max ? fmaxf(v, u) : v + u;
+        }
+        if (lane == 0) tot = v;
+    }
+    __syncthreads();
+    return tot;
+}
+
+// patch i of the gh x gw grid: its 3 x p x p values, channel-major (the reference's reshape / permute)
+__global__ void patchify_kernel(const uint8_t* __restrict__ rgb, int width, int gw, int p, uint16_t* __restrict__ out) {
+    const int i = blockIdx.x, e = threadIdx.x, pp = p * p;
+    if (e >= 3 * pp) return;
+    const int c = e / pp, r = e % pp, y = (i / gw) * p + r / p, x = (i % gw) * p + r % p;
+    const float u = (float) rgb[((size_t) y * width + x) * 3 + c];
+    out[(size_t) i * 3 * pp + e] = bfr(__fdiv_rn(__fsub_rn(__fdiv_rn(u, 255.0f), 0.5f), 0.5f));
+}
+
+// x[r] += y[r] (y null: nothing) + bias
+__global__ void add_rows_kernel(float* __restrict__ x, const float* __restrict__ y, int ldy,
+                                const uint16_t* __restrict__ bias, int dim) {
+    const int r = blockIdx.x;
+    for (int e = threadIdx.x; e < dim; e += blockDim.x) {
+        float v = x[(size_t) r * dim + e];
+        if (y != nullptr) v += y[(size_t) r * ldy + e];
+        if (bias != nullptr) v += bf(bias[e]);
+        x[(size_t) r * dim + e] = v;
+    }
+}
+
+__global__ void rmsnorm_kernel(const float* __restrict__ x, const uint16_t* __restrict__ w, int dim, float eps,
+                               uint16_t* __restrict__ out) {
+    const int r = blockIdx.x;
+    const float* xr = x + (size_t) r * dim;
+    float ss = 0.0f;
+    for (int e = threadIdx.x; e < dim; e += blockDim.x) ss += xr[e] * xr[e];
+    ss = block_reduce(ss, false);
+    const float rs = rsqrtf(ss / (float) dim + eps);
+    for (int e = threadIdx.x; e < dim; e += blockDim.x) out[(size_t) r * dim + e] = bfr(bf(w[e]) * (xr[e] * rs));
+}
+
+// a piece's rows of wqkv's output (+ bias, rounded to BF16 as the reference's linear) into qkv, q and k rotated: the
+// head's first half pairs with its second; frequency j < hd / 4 turns with the patch's row, the rest with its column
+__global__ void qkv_rope_kernel(const float* __restrict__ y, const uint16_t* __restrict__ bias, int r0, int gw,
+                                const float* __restrict__ inv_freq, int heads, int hd, uint16_t* __restrict__ qkv) {
+    const int r = blockIdx.x, gr = r0 + r, dim = heads * hd, half = hd / 2, nf = hd / 4;
+    const int row = gr / gw, col = gr % gw;
+    const float* yr = y + (size_t) r * 3 * dim;
+    uint16_t* o = qkv + (size_t) gr * 3 * dim;
+    for (int i = threadIdx.x; i < 2 * heads * half; i += blockDim.x) {
+        const int base = (i / half) * hd, d = i % half;
+        const float x1 = bf(bfr(yr[base + d] + bf(bias[base + d])));
+        const float x2 = bf(bfr(yr[base + d + half] + bf(bias[base + d + half])));
+        float sn, cs;
+        sincosf((float) (d < nf ? row : col) * inv_freq[d % nf], &sn, &cs);
+        o[base + d] = bfr(x1 * cs - x2 * sn);
+        o[base + d + half] = bfr(x2 * cs + x1 * sn);
+    }
+    for (int e = 2 * dim + threadIdx.x; e < 3 * dim; e += blockDim.x) o[e] = bfr(yr[e] + bf(bias[e]));
+}
+
+__global__ void __launch_bounds__(kSmT) softmax_kernel(const float* __restrict__ s, int n, uint16_t* __restrict__ p) {
+    const float* sr = s + (size_t) blockIdx.x * n;
+    uint16_t* pr = p + (size_t) blockIdx.x * n;
+    float v[kSmPer];
+    float m = -FLT_MAX;
+#pragma unroll
+    for (int k = 0; k < kSmPer; ++k) {
+        const int e = threadIdx.x + k * kSmT;
+        v[k] = e < n ? sr[e] : -FLT_MAX;
+        m = fmaxf(m, v[k]);
+    }
+    m = block_reduce(m, true);
+    float sum = 0.0f;
+#pragma unroll
+    for (int k = 0; k < kSmPer; ++k) {
+        const int e = threadIdx.x + k * kSmT;
+        v[k] = e < n ? expf(v[k] - m) : 0.0f;
+        sum += v[k];
+    }
+    const float inv = 1.0f / block_reduce(sum, false);
+#pragma unroll
+    for (int k = 0; k < kSmPer; ++k) {
+        const int e = threadIdx.x + k * kSmT;
+        if (e < n) pr[e] = bfr(v[k] * inv);
+    }
+}
+
+// mid = silu(gate) * up, gate | up the halves of w1's row (BF16 steps, as the reference's)
+__global__ void silu_mul_kernel(const float* __restrict__ y, int inter, uint16_t* __restrict__ mid) {
+    const int r = blockIdx.y, e = blockIdx.x * blockDim.x + threadIdx.x;
+    if (e >= inter) return;
+    const float gt = bf(bfr(y[(size_t) r * 2 * inter + e])), up = bf(bfr(y[(size_t) r * 2 * inter + inter + e]));
+    mid[(size_t) r * inter + e] = bfr(bf(bfr(gt / (1.0f + expf(-gt)))) * up);
+}
+
+// the aligner's input: block (bi, bj) of the m x m patches as one row, value c * m * m + kh * m + kw (F.unfold's
+// order), zero past the grid
+__global__ void unfold_kernel(const uint16_t* __restrict__ xn, int gh, int gw, int m, int dim, int lw,
+                              uint16_t* __restrict__ un) {
+    const int l = blockIdx.x, bi = l / lw, bj = l % lw, mm = m * m;
+    for (int f = threadIdx.x; f < dim * mm; f += blockDim.x) {
+        const int c = f / mm, k = f % mm, y = bi * m + k / m, x = bj * m + k % m;
+        un[(size_t) l * dim * mm + f] = y < gh && x < gw ? xn[((size_t) y * gw + x) * dim + c] : (uint16_t) 0;
+    }
+}
+
+// out = act(bf16(y + bias)): GELU (erf) or none
+__global__ void bias_act_kernel(const float* __restrict__ y, const uint16_t* __restrict__ bias, int n, int gelu,
+                                uint16_t* __restrict__ out) {
+    const int r = blockIdx.y, e = blockIdx.x * blockDim.x + threadIdx.x;
+    if (e >= n) return;
+    float v = bf(bfr(y[(size_t) r * n + e] + bf(bias[e])));
+    if (gelu) v = 0.5f * v * (1.0f + erff(v * 0.70710678118654752f));
+    out[(size_t) r * n + e] = bfr(v);
+}
+
+int cdiv(int a, int b) { return (a + b - 1) / b; }
+
+// y[r][o] = alpha * sum_k x[r][k] W[o][k] (row-major W [out][in] with row stride ldw, x [rows][ldx], y [rows][ldy])
+bool gemm(cublasHandle_t h, const uint16_t* W, int ldw, const uint16_t* x, int ldx, void* y, cudaDataType yt, int ldy,
+          int rows, int in, int out, float alpha, std::string& err) {
+    const float beta = 0.0f;
+    VB(cublasGemmEx(h, CUBLAS_OP_T, CUBLAS_OP_N, out, rows, in, &alpha, W, CUDA_R_16BF, ldw, x, CUDA_R_16BF, ldx, &beta,
+                    y, yt, ldy, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+    return true;
+}
+
+}  // namespace
+
+template <class T>
+bool VisionTower::dalloc(T*& p, size_t n, std::string& err) {
+    void* v = nullptr;
+    const size_t b = (n * sizeof(T) + 255) / 256 * 256;
+    if (cudaMalloc(&v, b) != cudaSuccess) {
+        err = "vision: cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(at_) + " failed";
+        return false;
+    }
+    dallocs_.push_back(v);
+    dev_bytes_ += b;
+    p = (T*) v;
+    return true;
+}
+
+VisionTower::~VisionTower() { release(); }
+
+void VisionTower::release() {
+    if (at_ < 0) return;
+    int cur = 0;
+    cudaGetDevice(&cur);
+    cudaSetDevice(at_);
+    for (int i = 0; i < 2; ++i) {
+        if (copied_[i]) cudaEventDestroy((cudaEvent_t) copied_[i]);
+        if (used_[i]) cudaEventDestroy((cudaEvent_t) used_[i]);
+        copied_[i] = used_[i] = nullptr;
+    }
+    if (blas_) cublasDestroy((cublasHandle_t) blas_);
+    if (st_) cudaStreamDestroy((cudaStream_t) st_);
+    if (cs_) cudaStreamDestroy((cudaStream_t) cs_);
+    for (void* p : dallocs_) cudaFree(p);
+    if (host_) cudaFreeHost(host_);
+    cudaSetDevice(cur);
+    blas_ = st_ = cs_ = nullptr;
+    dallocs_.clear();
+    host_ = nullptr;
+    dev_ = at_ = -1;
+    dev_bytes_ = 0;
+}
+
+bool VisionTower::load(const VisionGeo& g, int device, const Reader& read, std::string& err) {
+    g_ = g;
+    const int D = g.dim, I = g.inter, P = g.patch, M = g.merge, hd = g.heads > 0 ? g.dim / g.heads : 0;
+    if (g.layers <= 0 || hd * g.heads != D || hd % 4 != 0 || P <= 0 || M <= 0 || g.max_tokens < 5) {
+        err = "vision: geometry not covered";
+        return false;
+    }
+    // the largest aligner grid within max_tokens (n_h * (n_w + 1) + 2): the work buffers' size
+    int cells = 0;
+    for (int h = 1; h <= g.max_tokens - 2; ++h) {
+        const int w = (g.max_tokens - 2) / h - 1;
+        if (w >= 1) cells = std::max(cells, h * w);
+    }
+    max_n_ = cells * M * M;
+    if (max_n_ > kSmT * kSmPer) {
+        err = "vision: " + std::to_string(max_n_) + " patches are more than the attention covers";
+        return false;
+    }
+    cudaSetDevice(device);
+    at_ = device;
+    dev_bytes_ = 0;
+    // the checkpoint's tensors, checked before anything is read
+    struct Want {
+        std::string name;
+        std::vector<int64_t> shape;
+    };
+    std::vector<Want> want = {{"vision.patch_embed.proj.weight", {D, 3 * P * P}},
+                              {"vision.patch_embed.proj.bias", {D}},
+                              {"vision.norm.weight", {D}},
+                              {"aligner.w1.weight", {g.out, (int64_t) D * M * M}},
+                              {"aligner.w1.bias", {g.out}},
+                              {"aligner.w2.weight", {g.out, g.out}},
+                              {"aligner.w2.bias", {g.out}}};
+    for (int l = 0; l < g.layers; ++l) {
+        const std::string B = "vision.blocks." + std::to_string(l) + ".";
+        want.push_back({B + "norm1.weight", {D}});
+        want.push_back({B + "norm2.weight", {D}});
+        want.push_back({B + "attn.wqkv.weight", {3 * D, D}});
+        want.push_back({B + "attn.wqkv.bias", {3 * D}});
+        want.push_back({B + "attn.wo.weight", {D, D}});
+        want.push_back({B + "attn.wo.bias", {D}});
+        want.push_back({B + "mlp.w1.weight", {2 * I, D}});
+        want.push_back({B + "mlp.w2.weight", {D, I}});
+    }
+    for (const Want& w : want) {
+        std::vector<int64_t> s;
+        if (!read(w.name, s, nullptr)) {
+            err = "vision: " + w.name + " is not in the checkpoint";
+            return false;
+        }
+        if (s != w.shape) {
+            err = "vision: " + w.name + " has another shape than the config's";
+            return false;
+        }
+    }
+    // the streamed weights, pinned
+    block_n_ = (size_t) 3 * D * D + (size_t) D * D + (size_t) 2 * I * D + (size_t) D * I;
+    al1_off_ = block_n_ * g.layers;
+    al2_off_ = al1_off_ + (size_t) g.out * D * M * M;
+    const size_t host_n = al2_off_ + (size_t) g.out * g.out;
+    if (cudaHostAlloc((void**) &host_, host_n * 2, cudaHostAllocDefault) != cudaSuccess) {
+        host_ = nullptr;
+        err = "vision: pinning " + std::to_string((host_n * 2) >> 20) + " MiB of host memory failed";
+        return false;
+    }
+    std::vector<int64_t> s;
+    for (int l = 0; l < g.layers; ++l) {
+        const std::string B = "vision.blocks." + std::to_string(l) + ".";
+        uint16_t* b = host_ + (size_t) l * block_n_;
+        if (!read(B + "attn.wqkv.weight", s, b) || !read(B + "attn.wo.weight", s, b + (size_t) 3 * D * D) ||
+            !read(B + "mlp.w1.weight", s, b + (size_t) 4 * D * D) ||
+            !read(B + "mlp.w2.weight", s, b + (size_t) 4 * D * D + (size_t) 2 * I * D)) {
+            err = "vision: block " + std::to_string(l) + ": read failed";
+            return false;
+        }
+    }
+    if (!read("aligner.w1.weight", s, host_ + al1_off_) || !read("aligner.w2.weight", s, host_ + al2_off_)) {
+        err = "vision: the aligner: read failed";
+        return false;
+    }
+    // the small ones on the device
+    auto up = [&](const std::string& name, size_t n, uint16_t*& d) -> bool {
+        std::vector<uint16_t> h(n);
+        std::vector<int64_t> sh;
+        if (!read(name, sh, h.data())) {
+            err = "vision: " + name + ": read failed";
+            return false;
+        }
+        if (!dalloc(d, n, err)) return false;
+        VK(cudaMemcpy(d, h.data(), n * 2, cudaMemcpyHostToDevice));
+        return true;
+    };
+    bool ok = up("vision.patch_embed.proj.weight", (size_t) D * 3 * P * P, pe_w_) &&
+              up("vision.patch_embed.proj.bias", D, pe_b_) && up("vision.norm.weight", D, norm_) &&
+              up("aligner.w1.bias", g.out, al1_b_) && up("aligner.w2.bias", g.out, al2_b_);
+    n1_.assign(g.layers, nullptr);
+    n2_.assign(g.layers, nullptr);
+    bqkv_.assign(g.layers, nullptr);
+    bo_.assign(g.layers, nullptr);
+    for (int l = 0; l < g.layers && ok; ++l) {
+        const std::string B = "vision.blocks." + std::to_string(l) + ".";
+        ok = up(B + "norm1.weight", D, n1_[l]) && up(B + "norm2.weight", D, n2_[l]) &&
+             up(B + "attn.wqkv.bias", 3 * D, bqkv_[l]) && up(B + "attn.wo.bias", D, bo_[l]);
+    }
+    if (!ok) return false;
+    {   // the rotary's frequencies: theta ^ -(2i / (hd / 2)), hd / 4 of them
+        const int nf = hd / 4;
+        std::vector<float> f(nf);
+        for (int i = 0; i < nf; ++i) f[i] = 1.0f / std::pow(g.theta, (float) (2 * i) / (float) (hd / 2));
+        if (!dalloc(inv_freq_, nf, err)) return false;
+        VK(cudaMemcpy(inv_freq_, f.data(), nf * 4, cudaMemcpyHostToDevice));
+    }
+    // the work buffers: the patch grid's rows whole (the residual, its norm, q / k / v), the GEMM outputs a piece of
+    // rows at a time, the attention's scores a piece of queries at a time (~48 MB)
+    const int L = cells;
+    rb_ = 2048;
+    qb_ = std::max(16, (int) ((48u << 20) / ((size_t) max_n_ * 4)) / 16 * 16);
+    slot_n_ = std::max(block_n_, (size_t) 1 << 22);
+    const size_t y_n = (size_t) rb_ * std::max({3 * D, 2 * I, D});
+    const size_t qkv_n = std::max({(size_t) max_n_ * 3 * D,
+                                   (size_t) max_n_ * 3 * P * P + ((size_t) max_n_ * 3 * P * P + 1) / 2,   // patches, pixels
+                                   (size_t) L * D * M * M + (size_t) 2 * L * g.out});                      // the aligner's
+    if ((size_t) L * g.out > y_n || (size_t) max_n_ * D > y_n) {
+        err = "vision: work buffer sizes";
+        return false;
+    }
+    ok = dalloc(x_, (size_t) max_n_ * D, err) && dalloc(xn_, (size_t) max_n_ * D, err) && dalloc(qkv_, qkv_n, err) &&
+         dalloc(y_, y_n, err) && dalloc(o_, (size_t) max_n_ * D, err) && dalloc(mid_, (size_t) rb_ * I, err) &&
+         dalloc(s_, (size_t) qb_ * max_n_, err) && dalloc(p_, (size_t) qb_ * max_n_, err) &&
+         dalloc(slot_[0], slot_n_, err) && dalloc(slot_[1], slot_n_, err) && dalloc(ws_, kWs, err);
+    if (!ok) return false;
+    patch_ = qkv_;   // (before block 0's qkv)
+    rgb_ = (uint8_t*) (qkv_ + (size_t) max_n_ * 3 * P * P);
+    un_ = qkv_;      // (after the last block's attention)
+    a16_ = un_ + (size_t) L * D * M * M;
+    outb_ = a16_ + (size_t) L * g.out;
+    cublasHandle_t h = nullptr;
+    VB(cublasCreate(&h));
+    blas_ = h;
+    cudaStream_t st = nullptr, cs = nullptr;
+    VK(cudaStreamCreateWithFlags(&st, cudaStreamNonBlocking));
+    VK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));
+    st_ = st;
+    cs_ = cs;
+    VB(cublasSetStream(h, st));
+    VB(cublasSetWorkspace(h, ws_, kWs));
+    for (int i = 0; i < 2; ++i) {
+        cudaEvent_t a = nullptr, b = nullptr;
+        VK(cudaEventCreateWithFlags(&a, cudaEventDisableTiming));
+        VK(cudaEventCreateWithFlags(&b, cudaEventDisableTiming));
+        copied_[i] = a;
+        used_[i] = b;
+    }
+    dev_ = device;
+    return true;
+}
+
+bool VisionTower::encode(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& rows, int& n_h, int& n_w,
+                         std::string& err) {
+    if (dev_ < 0) {
+        err = "vision: not loaded";
+        return false;
+    }
+    const VisionGeo& g = g_;
+    const int D = g.dim, I = g.inter, P = g.patch, M = g.merge, hd = D / g.heads, H3 = 3 * D;
+    if (width <= 0 || height <= 0 || width % P != 0 || height % P != 0) {
+        err = "vision: an image of " + std::to_string(width) + " x " + std::to_string(height) +
+              " pixels (both sides must be multiples of " + std::to_string(P) + ")";
+        return false;
+    }
+    const int gh = height / P, gw = width / P, N = gh * gw;
+    n_h = cdiv(gh, M);
+    n_w = cdiv(gw, M);
+    const int L = n_h * n_w;
+    if (N > max_n_ || n_h * (n_w + 1) + 2 > g.max_tokens) {
+        err = "vision: an image of " + std::to_string(width) + " x " + std::to_string(height) + " pixels is more than " +
+              std::to_string(g.max_tokens) + " image tokens";
+        return false;
+    }
+    cudaSetDevice(dev_);
+    cublasHandle_t h = (cublasHandle_t) blas_;
+    cudaStream_t s = (cudaStream_t) st_, cs = (cudaStream_t) cs_;
+    // the weights in the order the encode takes them: the blocks, then the aligner's in pieces of a slot
+    std::vector<Piece> pcs;
+    for (int l = 0; l < g.layers; ++l) pcs.push_back({host_ + (size_t) l * block_n_, block_n_});
+    const int in1 = D * M * M, r1 = (int) std::min<size_t>(g.out, slot_n_ / in1), r2 = (int) std::min<size_t>(g.out, slot_n_ / g.out);
+    for (int o0 = 0; o0 < g.out; o0 += r1)
+        pcs.push_back({host_ + al1_off_ + (size_t) o0 * in1, (size_t) std::min(r1, g.out - o0) * in1});
+    for (int o0 = 0; o0 < g.out; o0 += r2)
+        pcs.push_back({host_ + al2_off_ + (size_t) o0 * g.out, (size_t) std::min(r2, g.out - o0) * g.out});
+    auto fetch = [&](size_t i) -> bool {
+        const int k = (int) (i % 2);
+        VK(cudaStreamWaitEvent(cs, (cudaEvent_t) used_[k], 0));
+        VK(cudaMemcpyAsync(slot_[k], pcs[i].host, pcs[i].n * 2, cudaMemcpyHostToDevice, cs));
+        VK(cudaEventRecord((cudaEvent_t) copied_[k], cs));
+        return true;
+    };
+    size_t pi = 0;   // the piece being used
+    auto take = [&]() -> const uint16_t* {   // the next piece's slot, once its copy is in (the one after it started)
+        const int k = (int) (pi % 2);
+        if (pi + 1 < pcs.size() && !fetch(pi + 1)) return nullptr;
+        if (cudaStreamWaitEvent(s, (cudaEvent_t) copied_[k], 0) != cudaSuccess) {
+            err = "vision: waiting for a weight copy failed";
+            return nullptr;
+        }
+        return slot_[k];
+    };
+    auto done = [&]() {   // the piece's work is queued: its slot is free once that ran
+        cudaEventRecord((cudaEvent_t) used_[pi % 2], s);
+        ++pi;
+    };
+    if (!fetch(0)) return false;
+
+    // the patches
+    VK(cudaMemcpyAsync(rgb_, rgb, (size_t) N * 3 * P * P, cudaMemcpyHostToDevice, s));
+    patchify_kernel<<<N, 3 * P * P, 0, s>>>(rgb_, width, gw, P, patch_);
+    if (!gemm(h, pe_w_, 3 * P * P, patch_, 3 * P * P, x_, CUDA_R_32F, D, N, 3 * P * P, D, 1.0f, err)) return false;
+    add_rows_kernel<<<N, kT, 0, s>>>(x_, nullptr, 0, pe_b_, D);
+
+    const float scale = 1.0f / std::sqrt((float) hd);
+    const int qb = (int) std::max<size_t>(1, std::min<size_t>((size_t) N, (size_t) qb_ * max_n_ / (size_t) N));
+    for (int l = 0; l < g.layers; ++l) {
+        const uint16_t* wqkv = take();
+        if (wqkv == nullptr) return false;
+        const uint16_t *wo = wqkv + (size_t) H3 * D, *w1 = wo + (size_t) D * D, *w2 = w1 + (size_t) 2 * I * D;
+        rmsnorm_kernel<<<N, kT, 0, s>>>(x_, n1_[l], D, g.eps, xn_);
+        for (int r0 = 0; r0 < N; r0 += rb_) {
+            const int nr = std::min(rb_, N - r0);
+            if (!gemm(h, wqkv, D, xn_ + (size_t) r0 * D, D, y_, CUDA_R_32F, H3, nr, D, H3, 1.0f, err)) return false;
+            qkv_rope_kernel<<<nr, kT, 0, s>>>(y_, bqkv_[l], r0, gw, inv_freq_, g.heads, hd, qkv_);
+        }
+        for (int q0 = 0; q0 < N; q0 += qb) {
+            const int nq = std::min(qb, N - q0);
+            const float one = 1.0f, zero = 0.0f;
+            for (int hh = 0; hh < g.heads; ++hh) {
+                // S^T (keys x queries, column-major) = K Q^T: row-major S [query][key]
+                VB(cublasGemmEx(h, CUBLAS_OP_T, CUBLAS_OP_N, N, nq, hd, &scale, qkv_ + D + hh * hd, CUDA_R_16BF, H3,
+                                qkv_ + (size_t) q0 * H3 + hh * hd, CUDA_R_16BF, H3, &zero, s_, CUDA_R_32F, N,
+                                CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+                softmax_kernel<<<nq, kSmT, 0, s>>>(s_, N, p_);
+                // O^T (hd x queries) = V^T P^T: row-major O [query][head's hd] at column hh * hd
+                VB(cublasGemmEx(h, CUBLAS_OP_N, CUBLAS_OP_N, hd, nq, N, &one, qkv_ + 2 * D + hh * hd, CUDA_R_16BF, H3,
+                                p_, CUDA_R_16BF, N, &zero, o_ + hh * hd, CUDA_R_16BF, D, CUBLAS_COMPUTE_32F,
+                                CUBLAS_GEMM_DEFAULT));
+            }
+            if (!gemm(h, wo, D, o_, D, y_, CUDA_R_32F, D, nq, D, D, 1.0f, err)) return false;
+            add_rows_kernel<<<nq, kT, 0, s>>>(x_ + (size_t) q0 * D, y_, D, bo_[l], D);
+        }
+        rmsnorm_kernel<<<N, kT, 0, s>>>(x_, n2_[l], D, g.eps, xn_);
+        for (int r0 = 0; r0 < N; r0 += rb_) {
+            const int nr = std::min(rb_, N - r0);
+            if (!gemm(h, w1, D, xn_ + (size_t) r0 * D, D, y_, CUDA_R_32F, 2 * I, nr, D, 2 * I, 1.0f, err)) return false;
+            silu_mul_kernel<<<dim3(cdiv(I, kT), nr), kT, 0, s>>>(y_, I, mid_);
+            if (!gemm(h, w2, I, mid_, I, y_, CUDA_R_32F, D, nr, I, D, 1.0f, err)) return false;
+            add_rows_kernel<<<nr, kT, 0, s>>>(x_ + (size_t) r0 * D, y_, D, nullptr, D);
+        }
+        done();
+    }
+    rmsnorm_kernel<<<N, kT, 0, s>>>(x_, norm_, D, g.eps, xn_);
+
+    // the aligner
+    unfold_kernel<<<L, kT, 0, s>>>(xn_, gh, gw, M, D, n_w, un_);
+    for (int o0 = 0; o0 < g.out; o0 += r1) {
+        const uint16_t* w = take();
+        if (w == nullptr) return false;
+        if (!gemm(h, w, in1, un_, in1, y_ + o0, CUDA_R_32F, g.out, L, in1, std::min(r1, g.out - o0), 1.0f, err))
+            return false;
+        done();
+    }
+    bias_act_kernel<<<dim3(cdiv(g.out, kT), L), kT, 0, s>>>(y_, al1_b_, g.out, 1, a16_);
+    for (int o0 = 0; o0 < g.out; o0 += r2) {
+        const uint16_t* w = take();
+        if (w == nullptr) return false;
+        if (!gemm(h, w, g.out, a16_, g.out, y_ + o0, CUDA_R_32F, g.out, L, g.out, std::min(r2, g.out - o0), 1.0f, err))
+            return false;
+        done();
+    }
+    bias_act_kernel<<<dim3(cdiv(g.out, kT), L), kT, 0, s>>>(y_, al2_b_, g.out, 0, outb_);
+    rows.resize((size_t) L * g.out);
+    VK(cudaMemcpyAsync(rows.data(), outb_, rows.size() * 2, cudaMemcpyDeviceToHost, s));
+    VK(cudaStreamSynchronize(s));
+    VK(cudaGetLastError());
+    return true;
+}
+
+}  // namespace strata::ds4

src/dsv41/ds4_vision.cuh

@@ -0,0 +1,79 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_vision.cuh - DeepSeek-V4.1's vision tower on one GPU (DeepSeek's inference/vision.py):
+//   patches (14 x 14 RGB, scaled to [-1, 1]) -> Linear(588 -> 1024)
+//   32 blocks: x += attn(rmsnorm(x)) (16 heads of 64, full bidirectional attention over the image, a 2D rotary on q
+//              and k: the first half of each head's frequencies the patch row, the second its column);
+//              x += w2(silu(gate) * up) with gate, up = w1(rmsnorm(x))
+//   rmsnorm, then the aligner: each 3 x 3 block of patches (zero-padded at the right and bottom) as one 9216 row,
+//   Linear(9216 -> 5120), GELU, Linear(5120 -> 5120): one LLM embedding row per block, in reading order.
+// The BF16 weights live in pinned host memory and stream to the GPU a block at a time (two slots, the next block's
+// copy under this one's work), so the tower costs the model ~350 MB of the GPU's memory, not ~1 GB; the GEMMs are
+// cuBLAS (BF16 in, FP32 accumulation), the residual stream FP32.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace strata::ds4 {
+
+struct VisionGeo {
+    int layers = 0, dim = 1024, heads = 16, inter = 2816, patch = 14, merge = 3, out = 5120;
+    float theta = 10000.0f, eps = 1e-6f;
+    int max_tokens = 1024;   // an image's span in the prompt at most: n_h * (n_w + 1) + 2 tokens
+};
+
+class VisionTower {
+public:
+    /// read(name, shape, dst): the tensor's shape, and (dst not null) its BF16 data into dst; false: not there
+    using Reader = std::function<bool(const std::string& name, std::vector<int64_t>& shape, void* dst)>;
+    VisionTower() = default;
+    VisionTower(const VisionTower&) = delete;
+    VisionTower& operator=(const VisionTower&) = delete;
+    ~VisionTower();
+    bool load(const VisionGeo& g, int device, const Reader& read, std::string& err);
+    /// Frees everything load took (a load that failed half way too); loaded() is false after it.
+    void release();
+    bool loaded() const { return dev_ >= 0; }
+    int device() const { return dev_; }
+    size_t device_bytes() const { return dev_bytes_; }
+    /// The patches an image may have (its aligner grid within max_tokens).
+    int max_patches() const { return max_n_; }
+    /// rgb: height x width x 3 bytes, both sides multiples of the patch size -> the aligner's rows, BF16
+    /// [n_h * n_w][out] in reading order (n_h x n_w: the patch grid over 3, rounded up).
+    bool encode(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& rows, int& n_h, int& n_w,
+                std::string& err);
+
+private:
+    struct Piece {             // a weight the encode streams in: host bits, its size
+        const uint16_t* host = nullptr;
+        size_t n = 0;
+    };
+    VisionGeo g_;
+    int dev_ = -1, at_ = -1, max_n_ = 0, qb_ = 0, rb_ = 0;   // (at_: the device its memory is on)
+    size_t dev_bytes_ = 0, slot_n_ = 0;
+    std::vector<void*> dallocs_;
+    uint16_t* host_ = nullptr;          // pinned: every block's [wqkv, wo, w1, w2], then aligner w1, w2
+    size_t block_n_ = 0;                // a block's weights (BF16 values)
+    size_t al1_off_ = 0, al2_off_ = 0;  // the aligner's weights in host_
+    // resident (small): the patch embedding, the norms and biases
+    uint16_t *pe_w_ = nullptr, *pe_b_ = nullptr, *norm_ = nullptr, *al1_b_ = nullptr, *al2_b_ = nullptr;
+    std::vector<uint16_t*> n1_, n2_, bqkv_, bo_;
+    // work buffers (max_n_ patches)
+    uint8_t* rgb_ = nullptr;
+    uint16_t *patch_ = nullptr, *xn_ = nullptr, *qkv_ = nullptr, *o_ = nullptr, *mid_ = nullptr, *p_ = nullptr,
+             *un_ = nullptr, *a16_ = nullptr, *outb_ = nullptr;
+    float *x_ = nullptr, *y_ = nullptr, *s_ = nullptr, *inv_freq_ = nullptr;
+    uint16_t* slot_[2] = {nullptr, nullptr};
+    void* blas_ = nullptr;              // cublasHandle_t
+    uint8_t* ws_ = nullptr;
+    void *st_ = nullptr, *cs_ = nullptr;   // cudaStream_t: the work, the weight copies
+    void *copied_[2] = {nullptr, nullptr}, *used_[2] = {nullptr, nullptr};   // cudaEvent_t per slot
+    template <class T>
+    bool dalloc(T*& p, size_t n, std::string& err);
+};
+
+}  // namespace strata::ds4

src/dsv41/vendor/exl3_legacy.cu

@@ -0,0 +1,967 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/kernels/cuda/exl3_legacy.cu - v1.1.8's EXL3 decode kernels, kept for NVIDIA GPUs before sm_80 (V100, T4,
+// RTX 20): #180's decode kernels run the tiles on the tensor cores from sm_80 on, and their CUDA-core fallback was
+// slower here than these (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s).  exl3.cu's mv / moe_gate_up / moe_down
+// / win_experts / prepare / several_ok call these on such a GPU (STRATA_GLM_EXL3_DECODE=legacy|new picks one on any
+// GPU); the prompt path is exl3.cu's on every GPU.  The code below is v1.1.8's as it was, the prompt path's kernels
+// left out.
+//
+// The codebooks and the trellis window arithmetic follow exllamav3 (MIT, turboderp - exllamav3_ext/quant/
+// codebook.cuh, exl3_dq.cuh, hadamard_inner.cuh); the kernels around them are shaped for this engine's decode:
+//
+//   * a block owns 128 outputs (8 tiles - exactly one output rotation block) over a slice of at most 4096 inputs;
+//     its 8 warps take the slice's 16-row tile rows in turn, each staging the row's 8 tiles in shared memory with
+//     coalesced loads and decoding 8 weights per lane (one fragment: 2 columns x 4 rows), F32 accumulation;
+//   * the input rotation runs per block on the slice it reads (F32 or q8_1 in, FP16 in shared memory), so the
+//     engine's activation buffers are used as they are;
+//   * a matrix with more than 4096 inputs, or too few output blocks to fill the device, splits its inputs over
+//     blocks: the partial sums go to a per-stream scratch and the LAST block of an output block adds them up in
+//     split order (deterministic) and runs the output rotation.
+#if !defined(STRATA_USE_HIP)   // (AMD runs exl3.cu's kernels; EXL3 does not run there anyway)
+#include "strata/dsv41/exl3.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cstdio>
+#include <map>
+#include <mutex>
+#include <utility>
+
+namespace strata::dsv41k::exl3::legacy {
+namespace {
+
+constexpr int kThreads = 256, kWarps = 8;
+constexpr int kCols = 128;                // outputs per block: 8 tiles, one rotation block
+constexpr int kMaxKR = 4096;              // inputs per block (the rotated slice lives in shared memory)
+constexpr float kHad = 0.088388347648f;   // 1 / sqrt(128)
+
+struct Q81 {   // the engine's q8_1 block (ggml block_q8_1): d, sum, 32 int8
+    __half2 ds;
+    int8_t qs[32];
+};
+
+std::atomic<int> g_launch_errors{0};
+void launch_check(const char* what) {
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "exl3 %s: %s\n", what, cudaGetErrorString(e));
+        g_launch_errors.fetch_add(1, std::memory_order_relaxed);
+    }
+}
+
+// ---------------------------------------------------------------- the codebooks
+__device__ __forceinline__ uint32_t bytesum_plus(uint32_t x, uint32_t acc) {
+#if defined(STRATA_USE_HIP)
+    return acc + (x & 0xffu) + ((x >> 8) & 0xffu) + ((x >> 16) & 0xffu) + (x >> 24);
+#else
+    return __dp4a(x, 0x01010101u, acc);
+#endif
+}
+
+// two states -> two values, FP16 arithmetic as exllamav3's decode_3inst_2 (the tensor-core kernels take the half2)
+template <int cb>
+__device__ __forceinline__ __half2 decode2h(uint32_t s0, uint32_t s1) {
+    if constexpr (cb == 2) {
+        const uint32_t a = bytesum_plus(s0 * 0x83DCD12Du, 0x6400u);
+        const uint32_t b = bytesum_plus(s1 * 0x83DCD12Du, 0x6400u);
+        const __half2 h = __halves2half2(__ushort_as_half((unsigned short) a), __ushort_as_half((unsigned short) b));
+        return __hfma2(h, __half2half2(__ushort_as_half(0x1eee)), __half2half2(__ushort_as_half(0xc931)));
+    } else {
+        uint32_t x0 = cb == 1 ? s0 * 0xCBAC1FEDu : s0 * 89226354u + 64248484u;
+        uint32_t x1 = cb == 1 ? s1 * 0xCBAC1FEDu : s1 * 89226354u + 64248484u;
+        x0 = (x0 & 0x8fff8fffu) ^ 0x3b603b60u;
+        x1 = (x1 & 0x8fff8fffu) ^ 0x3b603b60u;
+        const __half2 lo = __halves2half2(__ushort_as_half((unsigned short) (x0 & 0xffffu)),
+                                          __ushort_as_half((unsigned short) (x1 & 0xffffu)));
+        const __half2 hi = __halves2half2(__ushort_as_half((unsigned short) (x0 >> 16)),
+                                          __ushort_as_half((unsigned short) (x1 >> 16)));
+        return __hadd2(lo, hi);
+    }
+}
+template <int cb>
+__device__ __forceinline__ float2 decode2(uint32_t s0, uint32_t s1) {
+    return __half22float2(decode2h<cb>(s0, s1));
+}
+
+// step p's state: the 16-bit window ending at bit (p + 1) K of the tile's MSB-first bit stream (w: its 8 K words),
+// wrapping around the tile (the + 256 K keeps the start non-negative)
+template <int K>
+__device__ __forceinline__ uint32_t state_at(const uint32_t* w, int p) {
+    constexpr int W = 8 * K;
+    const int b1 = (p + 1 + 256) * K;
+    const int i1 = (b1 - 1) >> 5, i0 = (b1 - 16) >> 5;
+    const int sh = ((i1 + 1) << 5) - b1;
+    const uint64_t m = ((uint64_t) w[i0 % W] << 32) | (uint64_t) w[i1 % W];
+    return (uint32_t) (m >> sh) & 0xffffu;
+}
+
+// lane's 8 steps (lane * 8 + j): v[j] sits at tile row (lane & 3) * 2 + (j & 1) + 8 (j >> 1 & 1), column
+// (lane >> 2) + 8 (j >> 2)
+template <int K, int cb>
+__device__ __forceinline__ void decode8(const uint32_t* w, int lane, float* v) {
+    const int p0 = lane * 8;
+#pragma unroll
+    for (int j = 0; j < 8; j += 2) {
+        const float2 d = decode2<cb>(state_at<K>(w, p0 + j), state_at<K>(w, p0 + j + 1));
+        v[j] = d.x;
+        v[j + 1] = d.y;
+    }
+}
+
+// ---------------------------------------------------------------- the rotation
+// the 128-point transform of the 128 values a warp holds 4 per lane (value lane * 4 + i), over sqrt(128)
+__device__ __forceinline__ void had128(float* v, int lane) {
+    const float s0 = v[0] + v[1], d0 = v[0] - v[1], s1 = v[2] + v[3], d1 = v[2] - v[3];
+    v[0] = s0 + s1;
+    v[1] = d0 + d1;
+    v[2] = s0 - s1;
+    v[3] = d0 - d1;
+#pragma unroll
+    for (int m = 1; m < 32; m <<= 1) {
+#pragma unroll
+        for (int i = 0; i < 4; ++i) {
+            const float o = __shfl_xor_sync(0xffffffffu, v[i], m);
+            v[i] = (lane & m) ? o - v[i] : v[i] + o;
+        }
+    }
+#pragma unroll
+    for (int i = 0; i < 4; ++i) v[i] *= kHad;
+}
+
+__device__ __forceinline__ float h2f(uint16_t b) { return __half2float(__ushort_as_half(b)); }
+
+// xs[i] = H(x * suh)[k0 + i] for i < kr (kr % 128 == 0); x from xf (F32) or xq (q8_1); every warp of the block
+__device__ void rotate_in(const float* __restrict__ xf, const Q81* __restrict__ xq, const uint16_t* __restrict__ suh,
+                          int k0, int kr, __half* xs) {
+    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+    for (int c = warp; c < kr / 128; c += kWarps) {
+        const int base = k0 + c * 128 + lane * 4;
+        float v[4];
+        if (xf != nullptr) {
+#pragma unroll
+            for (int i = 0; i < 4; ++i) v[i] = xf[base + i];
+        } else {
+            const Q81* b = xq + base / 32;
+            const float d = __low2float(b->ds);
+#pragma unroll
+            for (int i = 0; i < 4; ++i) v[i] = d * (float) b->qs[base % 32 + i];
+        }
+#pragma unroll
+        for (int i = 0; i < 4; ++i) v[i] *= h2f(suh[base + i]);
+        had128(v, lane);
+#pragma unroll
+        for (int i = 0; i < 4; ++i) xs[c * 128 + lane * 4 + i] = __float2half_rn(v[i]);
+    }
+}
+
+// ---------------------------------------------------------------- the block GEMV
+// a lane's 4 weights of one column times the activation's 4 inputs, added to a: the rounding spelled out, so the
+// one-activation and the several-activation GEMVs (block_gemv, block_gemv_n) give each activation the same bits
+__device__ __forceinline__ float dot4_acc(float a, const float* v, float x0, float x1, float x2, float x3) {
+    return __fadd_rn(a, __fmaf_rn(v[3], x3, __fmaf_rn(v[2], x2, __fmaf_rn(v[1], x1, __fmul_rn(v[0], x0)))));
+}
+
+// red[c] (c < 128) = sum over the 16-row slices [ks0, ks1) of x' . W'[:, tc0 * 16 + c], x' = xs from slice ks0 on.
+// tr: the matrix's first tile, ld tiles per slice row.  stage: kWarps * 64 K words, red: kWarps * 128 floats.
+// Ends with a __syncthreads (red[0..127] complete).
+template <int K, int cb>
+__device__ void block_gemv(const uint32_t* __restrict__ tr, int ld, int tc0, int ks0, int ks1, const __half* xs,
+                           uint32_t* stage, float* red) {
+    constexpr int W = 8 * K, TW = 8 * W;
+    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+    uint32_t* st = stage + warp * TW;
+    float acc[8][2];
+#pragma unroll
+    for (int t = 0; t < 8; ++t) acc[t][0] = acc[t][1] = 0.0f;
+    const int kk = (lane & 3) * 2;
+    for (int ks = ks0 + warp; ks < ks1; ks += kWarps) {
+        const uint32_t* src = tr + ((size_t) ks * ld + tc0) * W;
+        __syncwarp();
+#pragma unroll
+        for (int i = lane; i < TW; i += 32) st[i] = src[i];
+        __syncwarp();
+        const __half* xk = xs + (ks - ks0) * 16;
+        const float x0 = __half2float(xk[kk]), x1 = __half2float(xk[kk + 1]);
+        const float x2 = __half2float(xk[kk + 8]), x3 = __half2float(xk[kk + 9]);
+#pragma unroll
+        for (int t = 0; t < 8; ++t) {
+            float v[8];
+            decode8<K, cb>(st + t * W, lane, v);
+            acc[t][0] = dot4_acc(acc[t][0], v, x0, x1, x2, x3);
+            acc[t][1] = dot4_acc(acc[t][1], v + 4, x0, x1, x2, x3);
+        }
+    }
+    // the 4 lanes of one lane / 4 hold the same two columns of every tile
+#pragma unroll
+    for (int t = 0; t < 8; ++t)
+#pragma unroll
+        for (int c = 0; c < 2; ++c) {
+            acc[t][c] += __shfl_xor_sync(0xffffffffu, acc[t][c], 1);
+            acc[t][c] += __shfl_xor_sync(0xffffffffu, acc[t][c], 2);
+        }
+    if ((lane & 3) == 0)
+#pragma unroll
+        for (int t = 0; t < 8; ++t) {
+            red[warp * kCols + t * 16 + (lane >> 2)] = acc[t][0];
+            red[warp * kCols + t * 16 + 8 + (lane >> 2)] = acc[t][1];
+        }
+    __syncthreads();
+    if (threadIdx.x < kCols) {
+        float s = 0.0f;
+#pragma unroll
+        for (int w = 0; w < kWarps; ++w) s += red[w * kCols + threadIdx.x];
+        red[threadIdx.x] = s;
+    }
+    __syncthreads();
+}
+
+// block_gemv for nc activations at once (nc <= kNC; activation c's rotated slice at xs + c * kMaxKR): each lane decodes
+// its 8 weights of a tile once and adds them into every activation's sums - as exllamav3's small-m GEMV and fused MoE
+// kernels share a tile's decode among the rows of an MMA, but with block_gemv's own F32 arithmetic, so outc[c * 128 +
+// i] (i < 128) is bit for bit block_gemv's red[i] for activation c alone.  Ends with a __syncthreads.
+constexpr int kNC = 4;
+template <int K, int cb>
+__device__ void block_gemv_n(const uint32_t* __restrict__ tr, int ld, int tc0, int ks0, int ks1, const __half* xs,
+                             int nc, uint32_t* stage, float* red, float* outc) {
+    constexpr int W = 8 * K, TW = 8 * W;
+    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+    uint32_t* st = stage + warp * TW;
+    float acc[kNC][8][2];
+#pragma unroll
+    for (int c = 0; c < kNC; ++c)
+#pragma unroll
+        for (int t = 0; t < 8; ++t) acc[c][t][0] = acc[c][t][1] = 0.0f;
+    const int kk = (lane & 3) * 2;
+    for (int ks = ks0 + warp; ks < ks1; ks += kWarps) {
+        const uint32_t* src = tr + ((size_t) ks * ld + tc0) * W;
+        __syncwarp();
+#pragma unroll
+        for (int i = lane; i < TW; i += 32) st[i] = src[i];
+        __syncwarp();
+        float x[kNC][4];
+#pragma unroll
+        for (int c = 0; c < kNC; ++c) {
+            const __half* xk = xs + c * kMaxKR + (ks - ks0) * 16;
+            if (c < nc) {
+                x[c][0] = __half2float(xk[kk]);
+                x[c][1] = __half2float(xk[kk + 1]);
+                x[c][2] = __half2float(xk[kk + 8]);
+                x[c][3] = __half2float(xk[kk + 9]);
+            }
+        }
+#pragma unroll
+        for (int t = 0; t < 8; ++t) {
+            float v[8];
+            decode8<K, cb>(st + t * W, lane, v);
+#pragma unroll
+            for (int c = 0; c < kNC; ++c)
+                if (c < nc) {
+                    acc[c][t][0] = dot4_acc(acc[c][t][0], v, x[c][0], x[c][1], x[c][2], x[c][3]);
+                    acc[c][t][1] = dot4_acc(acc[c][t][1], v + 4, x[c][0], x[c][1], x[c][2], x[c][3]);
+                }
+        }
+    }
+    // per activation, block_gemv's reduction
+#pragma unroll
+    for (int c = 0; c < kNC; ++c) {
+        if (c >= nc) break;
+#pragma unroll
+        for (int t = 0; t < 8; ++t)
+#pragma unroll
+            for (int h = 0; h < 2; ++h) {
+                acc[c][t][h] += __shfl_xor_sync(0xffffffffu, acc[c][t][h], 1);
+                acc[c][t][h] += __shfl_xor_sync(0xffffffffu, acc[c][t][h], 2);
+            }
+        if ((lane & 3) == 0)
+#pragma unroll
+            for (int t = 0; t < 8; ++t) {
+                red[warp * kCols + t * 16 + (lane >> 2)] = acc[c][t][0];
+                red[warp * kCols + t * 16 + 8 + (lane >> 2)] = acc[c][t][1];
+            }
+        __syncthreads();
+        if (threadIdx.x < kCols) {
+            float s = 0.0f;
+#pragma unroll
+            for (int w = 0; w < kWarps; ++w) s += red[w * kCols + threadIdx.x];
+            outc[c * kCols + threadIdx.x] = s;
+        }
+        __syncthreads();
+    }
+}
+
+// one warp: y[c] = alpha * (H(red) * svh)[c] + bias[c] for the block's 128 outputs
+__device__ __forceinline__ void epilogue(const float* red, const uint16_t* svh, float alpha, const float* bias,
+                                         float* y) {
+    const int lane = threadIdx.x & 31;
+    float v[4];
+#pragma unroll
+    for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];
+    had128(v, lane);
+#pragma unroll
+    for (int i = 0; i < 4; ++i) {
+        const int c = lane * 4 + i;
+        y[c] = __fmaf_rn(__fmul_rn(alpha, v[i]), h2f(svh[c]), bias != nullptr ? bias[c] : 0.0f);
+    }
+}
+
+// ---------------------------------------------------------------- mv
+struct KJob {
+    const uint32_t* tr;
+    const uint16_t* suh;
+    const uint16_t* svh;
+    const float* xf;
+    const Q81* xq;
+    float* y;
+    const float* bias;
+    float* part;       // S x n partial sums (S > 1); mv_n_kernel: nc x S x n
+    unsigned* cnt;     // n / 128 counters, zero between launches
+    float alpha;
+    int k, n, ld, S, kr;
+    int nc;            // mv_n_kernel: the activations (1..kNC), xf / xq / y advanced by these per activation
+    size_t xf_cs, xq_cs, y_cs;   // (floats, bytes, floats)
+};
+struct KBatch {
+    KJob j[kMaxMvJobs];
+    int blk_end[kMaxMvJobs];
+    int n;
+};
+
+// the dynamic shared memory of the several-activation kernels (mv_n_kernel, win_gu_kernel, win_down_kernel):
+// red[kWarps][128] outc[kNC][128] gu[2][kNC][128] (F32), stage[kWarps * 64 K] (words), xs[kNC][kMaxKR] (FP16) - up to
+// 58 KB, past the 48 KB a launch gets without asking (set_smem)
+constexpr size_t smem_n_bytes(int K) {
+    return (size_t) (kWarps * kCols + kNC * kCols + 2 * kNC * kCols) * 4 + (size_t) kWarps * 64 * K * 4 +
+           (size_t) kNC * kMaxKR * 2;
+}
+struct SmemN {
+    float *red, *outc, *gu;
+    uint32_t* stage;
+    __half* xs;
+};
+template <int K>
+__device__ __forceinline__ SmemN smem_n(unsigned char* p) {
+    SmemN s;
+    s.red = (float*) p;
+    s.outc = s.red + kWarps * kCols;
+    s.gu = s.outc + kNC * kCols;
+    s.stage = (uint32_t*) (s.gu + 2 * kNC * kCols);
+    s.xs = (__half*) (s.stage + kWarps * 64 * K);
+    return s;
+}
+
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) mv_kernel(const KBatch b) {
+    __shared__ __half xs[kMaxKR];
+    __shared__ uint32_t stage[kWarps * 64 * K];
+    __shared__ float red[kWarps * kCols];
+    __shared__ int last;
+    int ji = 0;
+    while ((int) blockIdx.x >= b.blk_end[ji]) ++ji;
+    const KJob& J = b.j[ji];
+    const int local = (int) blockIdx.x - (ji > 0 ? b.blk_end[ji - 1] : 0);
+    const int g = local / J.S, sp = local % J.S;
+    const int k0 = sp * J.kr, k1 = min(J.k, k0 + J.kr);
+    rotate_in(J.xf, J.xq, J.suh, k0, k1 - k0, xs);
+    __syncthreads();
+    block_gemv<K, cb>(J.tr, J.ld, g * 8, k0 / 16, k1 / 16, xs, stage, red);
+    const int warp = threadIdx.x >> 5;
+    if (J.S == 1) {
+        if (warp == 0)
+            epilogue(red, J.svh + g * kCols, J.alpha, J.bias != nullptr ? J.bias + g * kCols : nullptr,
+                     J.y + g * kCols);
+        return;
+    }
+    if (threadIdx.x < kCols) J.part[(size_t) sp * J.n + g * kCols + threadIdx.x] = red[threadIdx.x];
+    __threadfence();
+    __syncthreads();
+    if (threadIdx.x == 0) last = atomicAdd(&J.cnt[g], 1u) == (unsigned) (J.S - 1);
+    __syncthreads();
+    if (!last) return;
+    __threadfence();
+    if (threadIdx.x < kCols) {
+        float s = 0.0f;
+        for (int i = 0; i < J.S; ++i) s += *(volatile const float*) &J.part[(size_t) i * J.n + g * kCols + threadIdx.x];
+        red[threadIdx.x] = s;
+    }
+    __syncthreads();
+    if (warp == 0)
+        epilogue(red, J.svh + g * kCols, J.alpha, J.bias != nullptr ? J.bias + g * kCols : nullptr, J.y + g * kCols);
+    if (threadIdx.x == 0) J.cnt[g] = 0u;
+}
+
+// mv_kernel for a job's nc activations at once (a verify window's tokens): the same input split, the weights read and
+// decoded once for all of them (block_gemv_n), each activation's partials and sums in mv_kernel's order - every
+// activation's y is mv_kernel's for it alone, bit for bit
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) mv_n_kernel(const KBatch b) {
+    extern __shared__ __align__(16) unsigned char smem_raw[];
+    const SmemN sm = smem_n<K>(smem_raw);
+    __shared__ int last;
+    int ji = 0;
+    while ((int) blockIdx.x >= b.blk_end[ji]) ++ji;
+    const KJob& J = b.j[ji];
+    const int local = (int) blockIdx.x - (ji > 0 ? b.blk_end[ji - 1] : 0);
+    const int g = local / J.S, sp = local % J.S;
+    const int k0 = sp * J.kr, k1 = min(J.k, k0 + J.kr);
+    for (int c = 0; c < J.nc; ++c)
+        rotate_in(J.xf != nullptr ? J.xf + c * J.xf_cs : nullptr,
+                  J.xq != nullptr ? (const Q81*) ((const uint8_t*) J.xq + c * J.xq_cs) : nullptr, J.suh, k0, k1 - k0,
+                  sm.xs + c * kMaxKR);
+    __syncthreads();
+    block_gemv_n<K, cb>(J.tr, J.ld, g * 8, k0 / 16, k1 / 16, sm.xs, J.nc, sm.stage, sm.red, sm.outc);
+    const int warp = threadIdx.x >> 5;
+    const float* bias = J.bias != nullptr ? J.bias + g * kCols : nullptr;
+    if (J.S == 1) {
+        if (warp < J.nc)
+            epilogue(sm.outc + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);
+        return;
+    }
+    if (threadIdx.x < kCols)
+        for (int c = 0; c < J.nc; ++c)
+            J.part[((size_t) c * J.S + sp) * J.n + g * kCols + threadIdx.x] = sm.outc[c * kCols + threadIdx.x];
+    __threadfence();
+    __syncthreads();
+    if (threadIdx.x == 0) last = atomicAdd(&J.cnt[g], 1u) == (unsigned) (J.S - 1);
+    __syncthreads();
+    if (!last) return;
+    __threadfence();
+    if (threadIdx.x < kCols)
+        for (int c = 0; c < J.nc; ++c) {
+            float s = 0.0f;
+            for (int i = 0; i < J.S; ++i)
+                s += *(volatile const float*) &J.part[((size_t) c * J.S + i) * J.n + g * kCols + threadIdx.x];
+            sm.outc[c * kCols + threadIdx.x] = s;
+        }
+    __syncthreads();
+    if (warp < J.nc) epilogue(sm.outc + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);
+    if (threadIdx.x == 0) J.cnt[g] = 0u;
+}
+
+// ---------------------------------------------------------------- the routed experts
+struct KMoe {
+    const unsigned long long* plan_ptr;
+    const float* plan_w;
+    const float* cpu_part;
+    const int* cpu_flag;
+    const float* x;
+    float* h;            // [k][n_ff]
+    const float* sh_out;
+    float* out;
+    float* part;         // [k][n_embd]
+    unsigned* cnt;       // n_embd / 128
+    size_t suh[3], svh[3], tr[3];
+    int k, n_embd, n_ff;
+    float limit;
+};
+
+// the pieces the token's experts (moe_*_kernel) and a window's grouped experts (win_*_kernel) share, so each window
+// token gets the token path's bits: one warp's output block of 128 (H, then svh) ...
+__device__ __forceinline__ void block_out(const float* red, const uint16_t* svh, float* dst) {
+    const int lane = threadIdx.x & 31;
+    float v[4];
+#pragma unroll
+    for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];
+    had128(v, lane);
+#pragma unroll
+    for (int i = 0; i < 4; ++i) dst[lane * 4 + i] = __fmul_rn(v[i], h2f(svh[lane * 4 + i]));
+}
+// ... the clamped swiglu ...
+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {
+    if (limit > 0.0f) {
+        gv = fminf(gv, limit);
+        uv = fminf(fmaxf(uv, -limit), limit);
+    }
+    return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
+}
+// ... and one output's sum: the experts' rows in plan order (p[i]: expert i's, 0 for one not on the device), the CPU
+// lane's part when flagged (cpu_part: null when not), the shared expert's row (sh: 0 without one)
+__device__ __forceinline__ float combine(const float* plan_w, const float* p, int k, const float* cpu_part, float sh) {
+    float m = 0.0f;
+    for (int i = 0; i < k; ++i) m = __fmaf_rn(plan_w[i], p[i], m);
+    if (cpu_part != nullptr) m = __fadd_rn(m, *cpu_part);
+    return __fadd_rn(m, sh);
+}
+
+// grid (n_ff / 128, k): expert blockIdx.y's gate and up for 128 h values, then the clamped swiglu
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) moe_gate_up_kernel(const KMoe a) {
+    __shared__ __half xs[2][kMaxKR];
+    __shared__ uint32_t stage[kWarps * 64 * K];
+    __shared__ float red[kWarps * kCols];
+    __shared__ float gu[2][kCols];
+    const int ei = blockIdx.y, g = blockIdx.x;
+    const unsigned long long ptr = a.plan_ptr[ei];
+    if (ptr == 0ull) return;
+    const uint8_t* blob = (const uint8_t*) ptr;
+    const int warp = threadIdx.x >> 5;
+    for (int r = 0; r < 2; ++r) rotate_in(a.x, nullptr, (const uint16_t*) (blob + a.suh[r]), 0, a.n_embd, xs[r]);
+    __syncthreads();
+    for (int r = 0; r < 2; ++r) {
+        block_gemv<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd / 16, xs[r], stage, red);
+        if (warp == 0) block_out(red, (const uint16_t*) (blob + a.svh[r]) + g * kCols, gu[r]);
+        __syncthreads();
+    }
+    if (threadIdx.x < kCols)
+        a.h[(size_t) ei * a.n_ff + g * kCols + threadIdx.x] = swiglu(gu[0][threadIdx.x], gu[1][threadIdx.x], a.limit);
+}
+
+// grid (n_embd / 128, k): expert blockIdx.y's down rows for 128 outputs; the last block of each output block adds
+// the experts up in plan order (as the engine's moe_down), then the CPU lane's part and the shared expert
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) moe_down_kernel(const KMoe a) {
+    __shared__ __half xs[kMaxKR];
+    __shared__ uint32_t stage[kWarps * 64 * K];
+    __shared__ float red[kWarps * kCols];
+    __shared__ int last;
+    const int ei = blockIdx.y, g = blockIdx.x;
+    const unsigned long long ptr = a.plan_ptr[ei];
+    const int warp = threadIdx.x >> 5;
+    float* part = a.part + (size_t) ei * a.n_embd + g * kCols;
+    if (ptr != 0ull) {
+        const uint8_t* blob = (const uint8_t*) ptr;
+        rotate_in(a.h + (size_t) ei * a.n_ff, nullptr, (const uint16_t*) (blob + a.suh[2]), 0, a.n_ff, xs);
+        __syncthreads();
+        block_gemv<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff / 16, xs, stage, red);
+        if (warp == 0) block_out(red, (const uint16_t*) (blob + a.svh[2]) + g * kCols, part);
+    } else if (threadIdx.x < kCols) {
+        part[threadIdx.x] = 0.0f;
+    }
+    __threadfence();
+    __syncthreads();
+    if (threadIdx.x == 0) last = atomicAdd(&a.cnt[g], 1u) == (unsigned) (a.k - 1);
+    __syncthreads();
+    if (!last) return;
+    __threadfence();
+    if (threadIdx.x < kCols) {
+        const int r = g * kCols + threadIdx.x;
+        float p[8];
+        for (int i = 0; i < a.k; ++i) p[i] = *(volatile const float*) &a.part[(size_t) i * a.n_embd + r];
+        a.out[r] = combine(a.plan_w, p, a.k, a.cpu_flag != nullptr && *a.cpu_flag ? a.cpu_part + r : nullptr,
+                           a.sh_out != nullptr ? a.sh_out[r] : 0.0f);
+    }
+    if (threadIdx.x == 0) a.cnt[g] = 0u;
+}
+
+// ---------------------------------------------------------------- a verify window's routed experts
+// gf::moe_win_experts' grouping (group g: one expert, its blob at grp_ptr[g]; its entries e = t * 8 + i at
+// grp_ent[grp_start[g] ..)), each group's weights read and decoded once for up to kNC of its tokens at a time (as
+// exllamav3's fused MoE runs a run of slots that picked one expert as the rows of one MMA tile), with the token
+// path's arithmetic per entry: every token's output is moe_gate_up + moe_down's for it alone, bit for bit
+struct KWin {
+    const unsigned long long* grp_ptr;
+    const int *grp_start, *grp_ent, *n_groups;
+    const unsigned long long* plan_ptr;
+    const float* plan_w;
+    const float* cpu_part;
+    const int* cpu_flag;
+    const float* x;
+    size_t x_stride;
+    float* h;            // [T * 8][n_ff]
+    float* part;         // [T * 8][n_embd]
+    const float* sh_out; // [T][n_embd]
+    float* out;
+    size_t out_stride;
+    size_t suh[3], svh[3], tr[3];
+    int k, n_embd, n_ff;
+    float limit;
+};
+
+// grid (n_ff / 128, groups at most): group blockIdx.y's gate and up for 128 h values of each entry, the clamped swiglu
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) win_gu_kernel(const KWin a) {
+    extern __shared__ __align__(16) unsigned char smem_raw[];
+    const SmemN sm = smem_n<K>(smem_raw);
+    const int grp = blockIdx.y, g = blockIdx.x;
+    if (grp >= *a.n_groups) return;
+    const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];
+    const int warp = threadIdx.x >> 5;
+    const int m1 = a.grp_start[grp + 1];
+    for (int m0 = a.grp_start[grp]; m0 < m1; m0 += kNC) {
+        const int nc = min(kNC, m1 - m0);
+        for (int r = 0; r < 2; ++r) {
+            for (int c = 0; c < nc; ++c)
+                rotate_in(a.x + (size_t) (a.grp_ent[m0 + c] >> 3) * a.x_stride, nullptr,
+                          (const uint16_t*) (blob + a.suh[r]), 0, a.n_embd, sm.xs + c * kMaxKR);
+            __syncthreads();
+            block_gemv_n<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd / 16, sm.xs, nc,
+                                sm.stage, sm.red, sm.outc);
+            if (warp < nc)
+                block_out(sm.outc + warp * kCols, (const uint16_t*) (blob + a.svh[r]) + g * kCols,
+                          sm.gu + (r * kNC + warp) * kCols);
+            __syncthreads();
+        }
+        if (threadIdx.x < kCols)
+            for (int c = 0; c < nc; ++c)
+                a.h[(size_t) a.grp_ent[m0 + c] * a.n_ff + g * kCols + threadIdx.x] =
+                    swiglu(sm.gu[c * kCols + threadIdx.x], sm.gu[(kNC + c) * kCols + threadIdx.x], a.limit);
+        __syncthreads();
+    }
+}
+
+// grid (n_embd / 128, groups at most): group blockIdx.y's down rows for 128 outputs of each entry into its part rows
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) win_down_kernel(const KWin a) {
+    extern __shared__ __align__(16) unsigned char smem_raw[];
+    const SmemN sm = smem_n<K>(smem_raw);
+    const int grp = blockIdx.y, g = blockIdx.x;
+    if (grp >= *a.n_groups) return;
+    const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];
+    const int warp = threadIdx.x >> 5;
+    const int m1 = a.grp_start[grp + 1];
+    for (int m0 = a.grp_start[grp]; m0 < m1; m0 += kNC) {
+        const int nc = min(kNC, m1 - m0);
+        for (int c = 0; c < nc; ++c)
+            rotate_in(a.h + (size_t) a.grp_ent[m0 + c] * a.n_ff, nullptr, (const uint16_t*) (blob + a.suh[2]), 0,
+                      a.n_ff, sm.xs + c * kMaxKR);
+        __syncthreads();
+        block_gemv_n<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff / 16, sm.xs, nc,
+                            sm.stage, sm.red, sm.outc);
+        if (warp < nc)
+            block_out(sm.outc + warp * kCols, (const uint16_t*) (blob + a.svh[2]) + g * kCols,
+                      a.part + (size_t) a.grp_ent[m0 + warp] * a.n_embd + g * kCols);
+        __syncthreads();
+    }
+}
+
+// grid (n_embd / 128, T): each token's outputs, moe_down_kernel's combine (an entry off the device adds 0)
+__global__ void __launch_bounds__(kCols) win_combine_kernel(const KWin a) {
+    const int t = blockIdx.y, r = blockIdx.x * kCols + threadIdx.x;
+    float p[8];
+    for (int i = 0; i < a.k; ++i) {
+        const int e = t * 8 + i;
+        p[i] = a.plan_ptr[e] != 0ull ? a.part[(size_t) e * a.n_embd + r] : 0.0f;
+    }
+    a.out[(size_t) t * a.out_stride + r] =
+        combine(a.plan_w + t * 8, p, a.k, a.cpu_flag[t] ? a.cpu_part + (size_t) t * a.n_embd + r : nullptr,
+                a.sh_out != nullptr ? a.sh_out[(size_t) t * a.n_embd + r] : 0.0f);
+}
+
+// ---------------------------------------------------------------- dispatch
+// F(K, cb) for K 1..8 and the three codebooks
+template <template <int, int> class F, typename... A>
+bool dispatch(int K, int cb, A&&... args) {
+#define EXL3_CASE(KK)                                                    \
+    case KK:                                                             \
+        if (cb == 2) F<KK, 2>::run(std::forward<A>(args)...);            \
+        else if (cb == 1) F<KK, 1>::run(std::forward<A>(args)...);       \
+        else F<KK, 0>::run(std::forward<A>(args)...);                    \
+        return true;
+    switch (K) {
+        EXL3_CASE(1) EXL3_CASE(2) EXL3_CASE(3) EXL3_CASE(4) EXL3_CASE(5) EXL3_CASE(6) EXL3_CASE(7) EXL3_CASE(8)
+        default: return false;
+    }
+#undef EXL3_CASE
+}
+
+template <int K, int cb> struct RunMv {
+    static void run(const KBatch& b, int blocks, cudaStream_t s) { mv_kernel<K, cb><<<blocks, kThreads, 0, s>>>(b); }
+};
+template <int K, int cb> struct RunMvN {
+    static void run(const KBatch& b, int blocks, cudaStream_t s) {
+        mv_n_kernel<K, cb><<<blocks, kThreads, smem_n_bytes(K), s>>>(b);
+    }
+};
+template <int K, int cb> struct RunWin {
+    static void run(const KWin& a, int groups, cudaStream_t s) {
+        win_gu_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) groups), kThreads, smem_n_bytes(K), s>>>(a);
+    }
+};
+template <int K, int cb> struct RunWinDown {
+    static void run(const KWin& a, int groups, cudaStream_t s) {
+        win_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, smem_n_bytes(K),
+                                 s>>>(a);
+    }
+};
+template <int K, int cb> struct AllowSmemN {
+    static void run() {
+        const int b = (int) smem_n_bytes(K);
+        cudaFuncSetAttribute(mv_n_kernel<K, cb>, cudaFuncAttributeMaxDynamicSharedMemorySize, b);
+        cudaFuncSetAttribute(win_gu_kernel<K, cb>, cudaFuncAttributeMaxDynamicSharedMemorySize, b);
+        cudaFuncSetAttribute(win_down_kernel<K, cb>, cudaFuncAttributeMaxDynamicSharedMemorySize, b);
+    }
+};
+template <int K, int cb> struct RunGU {
+    static void run(const KMoe& a, cudaStream_t s) {
+        moe_gate_up_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);
+    }
+};
+template <int K, int cb> struct RunDown {
+    static void run(const KMoe& a, cudaStream_t s) {
+        moe_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);
+    }
+};
+// ---------------------------------------------------------------- per-stream scratch
+constexpr size_t kPartFloats = (size_t) 2 << 20;   // split partials: 8 MB
+constexpr size_t kCounters = 16384;
+constexpr size_t kHFloats = (size_t) 8 * 8192;     // the experts' h
+struct Scratch {
+    float* part = nullptr;
+    unsigned* cnt = nullptr;
+    float* h = nullptr;
+};
+std::mutex g_mu;
+std::map<std::pair<int, cudaStream_t>, Scratch> g_scratch;
+
+Scratch* scratch(cudaStream_t s) {
+    int dev = 0;
+    cudaGetDevice(&dev);
+    std::lock_guard<std::mutex> lk(g_mu);
+    auto it = g_scratch.find({dev, s});
+    if (it != g_scratch.end()) return &it->second;
+    cudaStreamCaptureStatus cs = cudaStreamCaptureStatusNone;
+    if (cudaStreamIsCapturing(s, &cs) == cudaSuccess && cs != cudaStreamCaptureStatusNone) {
+        std::fprintf(stderr, "exl3: scratch for a stream first seen inside a graph capture (call exl3::prepare)\n");
+        return nullptr;
+    }
+    Scratch sc;
+    if (cudaMalloc(&sc.part, kPartFloats * sizeof(float)) != cudaSuccess ||
+        cudaMalloc(&sc.cnt, kCounters * sizeof(unsigned)) != cudaSuccess ||
+        cudaMalloc(&sc.h, kHFloats * sizeof(float)) != cudaSuccess ||
+        cudaMemsetAsync(sc.cnt, 0, kCounters * sizeof(unsigned), s) != cudaSuccess) {
+        std::fprintf(stderr, "exl3: scratch did not allocate\n");
+        return nullptr;
+    }
+    cudaStreamSynchronize(s);
+    return &(g_scratch[{dev, s}] = sc);
+}
+
+int sm_count() {
+    int dev = 0, sms = 0;
+    cudaGetDevice(&dev);
+    cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
+    return std::max(1, sms);
+}
+
+// the several-activation kernels may use smem_n_bytes of shared memory: asked once per device (prepare, before a graph
+// capture; else at the first such launch).  A device that cannot give a block that much (48 KB GPUs) runs one
+// activation at a time instead
+constexpr int kMaxDev = 64;
+std::atomic<int> g_smem_n[kMaxDev];   // 0 not asked, 1 allowed, 2 refused
+bool allow_smem_n() {
+    int dev = 0;
+    cudaGetDevice(&dev);
+    if (dev < 0 || dev >= kMaxDev) return false;
+    const int st = g_smem_n[dev].load(std::memory_order_acquire);
+    if (st != 0) return st == 1;
+    std::lock_guard<std::mutex> lk(g_mu);
+    if (g_smem_n[dev].load(std::memory_order_relaxed) != 0) return g_smem_n[dev].load() == 1;
+    int optin = 0;
+    cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
+    bool ok = (size_t) optin >= smem_n_bytes(8);
+    if (ok) {
+        for (int K = 1; K <= 8; ++K)
+            for (int cb = 0; cb <= 2; ++cb) dispatch<AllowSmemN>(K, cb);
+        ok = cudaGetLastError() == cudaSuccess;
+    }
+    if (!ok)
+        std::fprintf(stderr, "exl3: CUDA%d gives a block %d bytes of shared memory, the several-activation kernels "
+                             "need %zu - a verify window's tokens run one at a time there\n", dev, optin,
+                     smem_n_bytes(8));
+    g_smem_n[dev].store(ok ? 1 : 2, std::memory_order_release);
+    return ok;
+}
+
+// mv's input split for m: S blocks of kr inputs per output block, at most kMaxKR inputs a block and about two blocks
+// per SM (the same for every activation count: mv_kernel and mv_n_kernel sum each activation alike)
+void mv_split(const Mat& m, int sms, int& S, int& kr) {
+    const int groups = m.n / kCols;
+    S = (m.k + kMaxKR - 1) / kMaxKR;
+    while (groups * S < 2 * sms && m.k / (S * 2) >= 256) S *= 2;
+    kr = ((m.k + S - 1) / S + 127) / 128 * 128;
+    S = (m.k + kr - 1) / kr;
+}
+
+}  // namespace
+
+int launch_errors() { return g_launch_errors.load(std::memory_order_relaxed); }
+
+bool prepare(cudaStream_t s) {
+    if (scratch(s) == nullptr) return false;
+    (void) allow_smem_n();
+    return true;
+}
+
+bool several_ok() { return allow_smem_n(); }
+
+bool mv(const MvJob* jobs, int n, cudaStream_t s) {
+    if (n <= 0 || n > kMaxMvJobs) return false;
+    Scratch* sc = scratch(s);
+    if (sc == nullptr) return false;
+    const int sms = sm_count();
+    // the launch units: a job of one activation is mv_kernel's, a job of several mv_n_kernel's in runs of up to kNC
+    // (one mv_kernel unit an activation where the device lacks the shared memory)
+    struct Unit {
+        int j, c0, nc;
+    };
+    Unit units[kMaxMvJobs * kMaxMvCols];
+    int nu = 0;
+    bool several = false;
+    for (int j = 0; j < n; ++j) several = several || jobs[j].ncol > 1;
+    const int run = several && allow_smem_n() ? kNC : 1;
+    for (int j = 0; j < n; ++j) {
+        const Mat& m = *jobs[j].m;
+        if (!supported(m) || jobs[j].y == nullptr || (jobs[j].xf == nullptr && jobs[j].xq == nullptr) ||
+            jobs[j].ncol < 1 || jobs[j].ncol > kMaxMvCols) {
+            std::fprintf(stderr, "exl3 mv: job %d unsupported (k %d n %d K %d cb %d, %d activations)\n", j, m.k, m.n,
+                         m.K, m.cb, jobs[j].ncol);
+            return false;
+        }
+        for (int c0 = 0; c0 < jobs[j].ncol; c0 += run) units[nu++] = {j, c0, std::min(run, jobs[j].ncol - c0)};
+    }
+    bool done[sizeof(units) / sizeof(units[0])] = {};
+    for (int i = 0; i < nu; ++i) {
+        if (done[i]) continue;
+        const Mat& m0 = *jobs[units[i].j].m;
+        const bool multi = units[i].nc > 1;
+        // a launch: the units of one (K, codebook, kernel), as many as KBatch and the split scratch take (the next
+        // launch on the stream may use the scratch again: this one's last blocks are done with it by then)
+        KBatch b{};
+        int blocks = 0, nb = 0;
+        size_t part_at = 0, cnt_at = 0;
+        for (int u = i; u < nu && nb < kMaxMvJobs; ++u) {
+            const MvJob& job = jobs[units[u].j];
+            const Mat& m = *job.m;
+            if (done[u] || m.K != m0.K || m.cb != m0.cb || (units[u].nc > 1) != multi) continue;
+            const int c0 = units[u].c0, nc = units[u].nc, groups = m.n / kCols;
+            int S = 1, kr = 0;
+            mv_split(m, sms, S, kr);
+            const size_t need = S > 1 ? (size_t) S * m.n * nc : 0;
+            if (S > 1 && (part_at + need > kPartFloats || cnt_at + (size_t) groups > kCounters)) {
+                if (nb > 0) continue;   // the next launch
+                std::fprintf(stderr, "exl3 mv: scratch too small for k %d n %d (%d activations)\n", m.k, m.n, nc);
+                return false;
+            }
+            done[u] = true;
+            KJob& J = b.j[nb];
+            J.tr = m.trellis;
+            J.suh = m.suh;
+            J.svh = m.svh;
+            J.xf = job.xf != nullptr ? job.xf + c0 * job.xf_cs : nullptr;
+            J.xq = job.xq != nullptr ? (const Q81*) ((const uint8_t*) job.xq + c0 * job.xq_cs) : nullptr;
+            J.y = job.y + c0 * job.y_cs;
+            J.bias = job.bias;
+            J.alpha = job.alpha;
+            J.k = m.k;
+            J.n = m.n;
+            J.ld = m.ld > 0 ? m.ld : m.n / 16;
+            J.S = S;
+            J.kr = kr;
+            J.part = sc->part + part_at;
+            J.cnt = sc->cnt + cnt_at;
+            J.nc = nc;
+            J.xf_cs = job.xf_cs;
+            J.xq_cs = job.xq_cs;
+            J.y_cs = job.y_cs;
+            part_at += need;
+            if (S > 1) cnt_at += (size_t) groups;
+            blocks += groups * S;
+            b.blk_end[nb++] = blocks;
+        }
+        b.n = nb;
+        if (!(multi ? dispatch<RunMvN>(m0.K, m0.cb, b, blocks, s) : dispatch<RunMv>(m0.K, m0.cb, b, blocks, s)))
+            return false;
+    }
+    launch_check("mv");
+    return true;
+}
+
+static bool moe_args(const MoeArgs& a, KMoe& m, cudaStream_t s) {
+    Scratch* sc = scratch(s);
+    if (sc == nullptr) return false;
+    if (a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxKR ||
+        a.n_ff > kMaxKR || (size_t) a.k * a.n_ff > kHFloats || (size_t) a.k * a.n_embd > kPartFloats ||
+        (size_t) (a.n_embd / kCols) > kCounters) {
+        std::fprintf(stderr, "exl3 moe: geometry not covered (k %d n_embd %d n_ff %d)\n", a.k, a.n_embd, a.n_ff);
+        return false;
+    }
+    m = KMoe{};
+    m.plan_ptr = a.plan_ptr;
+    m.plan_w = a.plan_w;
+    m.cpu_part = a.cpu_part;
+    m.cpu_flag = a.cpu_flag;
+    m.h = sc->h;
+    m.part = sc->part;
+    m.cnt = sc->cnt;
+    for (int r = 0; r < 3; ++r) {
+        m.suh[r] = a.lay.suh[r];
+        m.svh[r] = a.lay.svh[r];
+        m.tr[r] = a.lay.trellis[r];
+    }
+    m.k = a.k;
+    m.n_embd = a.n_embd;
+    m.n_ff = a.n_ff;
+    m.limit = a.limit;
+    return true;
+}
+
+void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s) {
+    KMoe m;
+    if (!moe_args(a, m, s)) return;
+    if (a.lay.K[0] != a.lay.K[1]) {
+        std::fprintf(stderr, "exl3 moe_gate_up: gate K %d != up K %d\n", a.lay.K[0], a.lay.K[1]);
+        return;
+    }
+    m.x = x;
+    if (h != nullptr) m.h = h;
+    if (!dispatch<RunGU>(a.lay.K[0], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_gate_up: K %d\n", a.lay.K[0]);
+    launch_check("moe_gate_up");
+}
+
+void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s) {
+    KMoe m;
+    if (!moe_args(a, m, s)) return;
+    if (h != nullptr) m.h = const_cast<float*>(h);
+    m.sh_out = sh_out;
+    m.out = out;
+    if (!dispatch<RunDown>(a.lay.K[2], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_down: K %d\n", a.lay.K[2]);
+    launch_check("moe_down");
+}
+
+bool win_experts(const WinArgs& a, cudaStream_t s) {
+    if (a.T < 1 || a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxKR ||
+        a.n_ff > kMaxKR || a.lay.K[0] != a.lay.K[1] || a.h == nullptr || a.part == nullptr || a.out == nullptr) {
+        std::fprintf(stderr, "exl3 win_experts: not covered (T %d k %d n_embd %d n_ff %d, gate K %d up K %d)\n", a.T,
+                     a.k, a.n_embd, a.n_ff, a.lay.K[0], a.lay.K[1]);
+        return false;
+    }
+    if (!allow_smem_n()) return false;
+    KWin w{};
+    w.grp_ptr = a.grp_ptr;
+    w.grp_start = a.grp_start;
+    w.grp_ent = a.grp_ent;
+    w.n_groups = a.n_groups;
+    w.plan_ptr = a.plan_ptr;
+    w.plan_w = a.plan_w;
+    w.cpu_part = a.cpu_part;
+    w.cpu_flag = a.cpu_flag;
+    w.x = a.x;
+    w.x_stride = a.x_stride;
+    w.h = a.h;
+    w.part = a.part;
+    w.sh_out = a.sh_out;
+    w.out = a.out;
+    w.out_stride = a.out_stride;
+    for (int r = 0; r < 3; ++r) {
+        w.suh[r] = a.lay.suh[r];
+        w.svh[r] = a.lay.svh[r];
+        w.tr[r] = a.lay.trellis[r];
+    }
+    w.k = a.k;
+    w.n_embd = a.n_embd;
+    w.n_ff = a.n_ff;
+    w.limit = a.limit;
+    const int groups = a.T * a.k;   // at most (the blocks past the real count return at once)
+    if (!dispatch<RunWin>(a.lay.K[0], a.lay.cb, w, groups, s) ||
+        !dispatch<RunWinDown>(a.lay.K[2], a.lay.cb, w, groups, s))
+        return false;
+    win_combine_kernel<<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s>>>(w);
+    launch_check("win_experts");
+    return true;
+}
+
+}  // namespace strata::dsv41k::exl3::legacy
+#endif

src/dsv41/vendor/moe_win.cu

@@ -0,0 +1,57 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/dsv41/vendor/moe_win.cu - the window's expert grouping kernel.
+// Extracted from Project Maya's src/kernels/cuda/glm_fast.cu (MIT, see third_party/project-maya/NOTICE.md).
+#include "strata/dsv41/moe_win.hpp"
+#include <cstdio>
+#include <cstdlib>
+
+namespace strata::dsv41k::glmf {
+namespace {
+constexpr int kWinEnt = kMaxWinTokens * 8;
+
+// the entries grouped by expert slot (in order of first appearance), each group's members in entry order (t * 8 + i)
+// (the tables in shared memory: as locals they were a 2.6 KB stack a thread, which the driver reserves for every thread
+// the device can hold - ~420 MB on a V100 - and a launch failed "out of memory" once the expert pool had the rest)
+__global__ void moe_win_group_kernel(const __grid_constant__ MoeWin w) {
+    __shared__ unsigned long long gp[kWinEnt];
+    __shared__ int cnt[kWinEnt], gof[kWinEnt], fill[kWinEnt + 1];
+    if (threadIdx.x != 0) return;
+    int ng = 0;
+    for (int t = 0; t < w.T; ++t)
+        for (int i = 0; i < 8; ++i) {
+            const int e = t * 8 + i;
+            gof[e] = -1;
+            const unsigned long long p = i < w.k ? w.plan_ptr[e] : 0ull;
+            if (p == 0ull) continue;
+            int g = 0;
+            while (g < ng && gp[g] != p) ++g;
+            if (g == ng) {
+                gp[ng] = p;
+                cnt[ng++] = 0;
+            }
+            ++cnt[g];
+            gof[e] = g;
+        }
+    fill[0] = 0;
+    for (int g = 0; g < ng; ++g) {
+        w.grp_ptr[g] = gp[g];
+        w.grp_start[g] = fill[g];
+        fill[g + 1] = fill[g] + cnt[g];
+    }
+    w.grp_start[ng] = fill[ng];
+    for (int e = 0; e < w.T * 8; ++e)
+        if (gof[e] >= 0) w.grp_ent[fill[gof[e]]++] = e;
+    *w.n_groups = ng;
+}
+}  // namespace
+
+void moe_win_group(const MoeWin& w, cudaStream_t s) {
+    moe_win_group_kernel<<<1, 32, 0, s>>>(w);
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "dsv41: moe_win_group: %s\n", cudaGetErrorString(e));
+        std::abort();
+    }
+}
+}  // namespace strata::dsv41k::glmf

src/dsv41/vendor/sampler.cu

@@ -0,0 +1,581 @@
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/kernels/cuda/sampler.cu - P2.S2: the sampler chain, in llama.cpp's order.
+//
+//     penalties -> top_k -> top_p -> min_p -> temperature -> pick
+//
+// THE ORDER IS THE WHOLE CONTENT OF THIS FILE.  llama.cpp builds its chain by walking `params.samplers`, whose
+// default is { PENALTIES, DRY, TOP_N_SIGMA, TOP_K, TYPICAL_P, TOP_P, MIN_P, XTC, TEMPERATURE } (`common/common.h`
+// at 3cf03257) - ONE penalties stage, first, and TEMPERATURE AFTER THE TRUNCATION FILTERS.  (Issue #53: this file
+// used to apply the penalties a second time after the temperature, and min_p before top_p - both taken from the
+// order of the `case` labels in `common/sampling.cpp`, which is not the order the chain runs.)  Every order
+// produces a valid token, so only a comparison at the distribution level can tell them apart; the parity test
+// does that against an independently computed distribution.
+//
+// Both kernels put ONE BLOCK per token over the vocabulary: `sampler_greedy_kernel` is the plain argmax,
+// `sampler_kernel` runs the sampled chain as `top_k` block-argmax rounds followed by the top_p / temperature /
+// draw chain (its header says why the selection must be parallel and why the tie rule keeps the semantics).
+#include "strata/dsv41/sampler.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+
+namespace strata::dsv41k {
+namespace {
+
+// Philox 4x32-10, the counter-based generator the phase asks for.  Counter-based matters because it makes the
+// stream a function of (seed, position) rather than of how many draws came before - so a batch can be sampled
+// in any order and a run is reproducible.
+__device__ __forceinline__ uint32_t philox4x32_round(uint32_t& c0, uint32_t& c1, uint32_t& c2, uint32_t& c3,
+                                                     uint32_t k0, uint32_t k1) {
+    const uint32_t hi0 = __umulhi(0x9E3779B9u, c0);
+    const uint32_t hi1 = __umulhi(0xBB67AE85u, c2);
+    const uint32_t lo0 = 0x9E3779B9u * c0;
+    const uint32_t lo1 = 0xBB67AE85u * c2;
+    const uint32_t n0 = hi1 ^ c1 ^ k0;
+    const uint32_t n1 = lo1;
+    const uint32_t n2 = hi0 ^ c3 ^ k1;
+    const uint32_t n3 = lo0;
+    c0 = n0; c1 = n1; c2 = n2; c3 = n3;
+    return 0;
+}
+
+__device__ __forceinline__ float philox_uniform(uint64_t seed, uint64_t counter) {
+    uint32_t c0 = (uint32_t) counter, c1 = (uint32_t) (counter >> 32);
+    uint32_t c2 = (uint32_t) seed, c3 = (uint32_t) (seed >> 32);
+    for (int i = 0; i < 10; ++i) {
+        philox4x32_round(c0, c1, c2, c3, (uint32_t) i, 0u);
+    }
+    // 24 bits of mantissa, so the value is uniform in [0,1) with no rounding to 1.0
+    return (float) (c0 >> 8) * (1.0f / 16777216.0f);
+}
+
+// `count_in_history` and the penalty application, transcribed from `llama_sampler_penalties_apply`.
+// The repeat penalty MULTIPLIES for non-positive logits and DIVIDES for positive ones - dividing
+// unconditionally is the natural reading of the source paper and it INVERTS the penalty on half the
+// vocabulary.  The presence penalty is `float(count > 0)`, a boolean, not the count.
+__device__ __forceinline__ int history_count(const int* __restrict__ h, int n, int v) {
+    int c = 0;
+    for (int i = 0; i < n; ++i) if (h[i] == v) ++c;
+    return c;
+}
+
+__device__ __forceinline__ float apply_penalties(float logit, int count, const SamplerParams& p) {
+    if (count <= 0) return logit;
+    if (logit <= 0.0f) logit *= p.penalty_repeat;
+    else               logit /= p.penalty_repeat;
+    logit -= (float) count * p.penalty_freq + (count > 0 ? 1.0f : 0.0f) * p.penalty_present;
+    return logit;
+}
+
+/// **THE GREEDY ARGMAX, ONE BLOCK PER TOKEN, COVERING THE VOCABULARY.**
+///
+/// **WHY THIS IS A SEPARATE KERNEL AND NOT A BRANCH.**  `sampler_kernel` is launched as a grid over TOKENS
+/// with 64 threads and a `if (t >= n_tokens) return;` at the top.  The decode path has `n_tokens == 1`, so
+/// that launch was `<<<1, 64>>>`, 63 threads exited on the first line, and ONE THREAD walked all 248,320
+/// logits in a dependent loop on one SM of 48.  Measured in isolation (`bench/micro/sampler_cost.cu`):
+/// **3.11 ms per token**, 5.7% of an ~54 ms token, and the whole of round 309's `sample` phase - the two
+/// synchronisations around it are 0.03 ms each.
+///
+/// The obvious repair is to parallelise the scan inside `sampler_kernel`, and it is WRONG: with one thread
+/// per token, a block reduction over the vocabulary has nothing to reduce, and the threads that returned
+/// early are not there for `__syncthreads` or `__shfl_down_sync`.  The first attempt did exactly that and
+/// produced the token `5120` thirty-two times.  The grid has to be over tokens with the BLOCK over the
+/// vocabulary, which is a different launch configuration and therefore a different kernel.
+///
+/// **THE TIE RULE IS UNCHANGED AND THAT IS THE WHOLE CORRECTNESS ARGUMENT.**  The serial scan walked `v`
+/// ascending with `if (s > bv)`, so the LOWEST index wins a tie.  Each thread keeps that rule over its own
+/// strided subset and the reduction resolves two candidates by taking the larger value and, on equality, the
+/// SMALLER index - the same total order, so `sampler_parity` and C1 see no change.
+__global__ void sampler_greedy_kernel(const float* __restrict__ logits, int n_vocab,
+                                      const int* __restrict__ history, int history_len, const SamplerParams p,
+                                      int pmin, int plen, int* __restrict__ out) {
+    const int t = blockIdx.x;
+    const float* l = logits + (size_t) t * n_vocab;
+    (void) pmin;
+    const int* hrow = history ? history + (size_t) t * history_len : nullptr;
+    int hlen = 0;
+    if (hrow) {
+        hlen = plen < history_len ? plen : history_len;
+        if (hlen < 0) hlen = 0;
+        hrow += history_len - hlen;          // the window is the TAIL
+    }
+
+    // PENALTY MEMBERSHIP AS A BITMAP.  The history touches at most `hlen` tokens of a quarter-million
+    // vocabulary, but the naive `history_count` per candidate per argmax round costs O(k x n_vocab x hlen)
+    // integer compares (~318 M per token at k=20, hlen=64 - measured 45 -> 31 tok/s on a real workload).
+    // A shared bitmap gives an O(1) membership test, and only the (at most hlen) hits pay the count scan;
+    // the counts - and therefore every sampled value - are exactly what the per-candidate scan produced.
+    extern __shared__ unsigned int penal_bits[];
+    const int bits_words = (int) ((n_vocab + 31) / 32);
+    // The gate needs a NON-EMPTY WINDOW (`hlen > 0`): the launch sizes the shared bitmap only when penalties
+    // are on, so a caller handing over a history buffer with `penalty_last_n == 0` must not touch it.
+    const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;
+    if (use_bits) {
+        for (int w = threadIdx.x; w < bits_words; w += blockDim.x) penal_bits[w] = 0u;
+        __syncthreads();
+        for (int i = threadIdx.x; i < hlen; i += blockDim.x)
+            if (hrow[i] >= 0 && hrow[i] < n_vocab)   // an id outside the vocabulary is never a candidate
+                atomicOr(&penal_bits[hrow[i] >> 5], 1u << (hrow[i] & 31));
+        __syncthreads();
+    }
+    auto hit_count = [&](int v) -> int {
+        if (!use_bits || !(penal_bits[v >> 5] & (1u << (v & 31)))) return 0;
+        return history_count(hrow, hlen, v);
+    };
+
+    // `n_vocab` is the "no candidate" index: it loses every comparison to a real one, so a thread with no
+    // elements contributes nothing rather than contributing a bogus zero.
+    float bv = __int_as_float(0xff800000);   // -inf
+    int best = n_vocab;
+    for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {
+        const float s = apply_penalties(p.logit_bias ? l[v] + p.logit_bias[v] : l[v], hit_count(v), p);
+        if (s > bv) { bv = s; best = v; }
+    }
+    for (int off = 16; off > 0; off >>= 1) {
+        const float ov = __shfl_down_sync(0xFFFFFFFFu, bv, off);
+        const int oi = __shfl_down_sync(0xFFFFFFFFu, best, off);
+        if (ov > bv || (ov == bv && oi < best)) { bv = ov; best = oi; }
+    }
+    __shared__ float sv[32];
+    __shared__ int si[32];
+    const int warp = (int) (threadIdx.x >> 5), lane = (int) (threadIdx.x & 31);
+    if (lane == 0) { sv[warp] = bv; si[warp] = best; }
+    __syncthreads();
+    if (warp == 0) {
+        const int nw = (int) ((blockDim.x + 31) >> 5);
+        float wv = lane < nw ? sv[lane] : __int_as_float(0xff800000);
+        int wi = lane < nw ? si[lane] : n_vocab;
+        for (int off = 16; off > 0; off >>= 1) {
+            const float ov = __shfl_down_sync(0xFFFFFFFFu, wv, off);
+            const int oi = __shfl_down_sync(0xFFFFFFFFu, wi, off);
+            if (ov > wv || (ov == wv && oi < wi)) { wv = ov; wi = oi; }
+        }
+        // A tie between two `-inf` candidates leaves `wi == n_vocab`, and the serial version answered 0.
+        if (lane == 0) out[t] = (wi < n_vocab) ? wi : 0;
+    }
+}
+
+/// **THE SAMPLED PATH, ONE BLOCK PER TOKEN.**  The kernel below replaced a version that ran the whole chain
+/// in ONE THREAD per token (`<<<ceil(T/64), 64>>>`, so a 4-token window fielded four threads): `top_k` alone
+/// was `k` sequential scans of the vocabulary with an inner sweep over the already-taken list - 20 x 248,320
+/// iterations of dependent work on one SM - and a verify window measured **1.6 s in the sampler**, which made
+/// every temperature-bearing request ~30x slower than a greedy one.  The selection is `k` argmax rounds, and
+/// an argmax over the vocabulary parallelises exactly like `sampler_greedy_kernel` (block over the vocab), so
+/// the rounds run back to back inside a block-per-token launch: the per-token cost falls to
+/// `k x n_vocab / 1024` plus `k` block reductions.
+///
+/// THE SEMANTICS ARE THE SERIAL ONES, EXACTLY.  Each round's argmax resolves ties to the LOWEST index (the
+/// serial scan's strict `>` keeps the first maximum it meets), so the kept sequence - both its set and its
+/// order - is unchanged; `top_p`'s cut reads that order in double arithmetic as before; temperature and the
+/// Philox draw apply after the cut.  `sampler_parity` pins all of it against the host reference.
+// MODE 0 = the draw (sample_tokens, unchanged); 1 = speculative sampling's draft (sample_draft: the draw from the
+// draft block's logits, on a Philox stream of its own, and its kept list in ql[t]); 2 = its verify (sample_verify:
+// the draft draft[t] with probability min(1, p/q), else a draw from max(0, p - q)).  The selection is shared.
+constexpr uint64_t kSpecDraftKey = 0x5bd1e9955bd1e995ull, kSpecAcceptKey = 0x9e3779b97f4a7c15ull,
+                   kSpecResidKey = 0xc2b2ae3d27d4eb4full;
+// one block of 1024 threads per row (the launches below): the bound keeps every mode at 64 registers a thread, so a
+// block fits the 64K-register file - unbounded, sm_75 (Turing) compiled the draft mode to 68 and its launch failed
+// ("too many resources requested for launch") wherever the NextN block sat on a Turing card
+template <int MODE>
+__global__ void __launch_bounds__(1024) sampler_kernel(const float* __restrict__ logits, int n_vocab, int n_tokens,
+                               const int* __restrict__ history, int history_len, const SamplerParams p,
+                               int* __restrict__ out, SpecList* __restrict__ ql, const int* __restrict__ draft) {
+    const int t = blockIdx.x;
+    if (t >= n_tokens) return;
+    const float* l = logits + (size_t) t * n_vocab;
+
+    // Temperature is needed by BOTH stages below, so it is computed here; the chain still APPLIES it after
+    // the truncation filters - the survivors are chosen on the raw logits and only then scaled.
+    const float inv_t = p.temperature > 0.0f ? 1.0f / p.temperature : 0.0f;
+
+    // The penalty window is the last `penalty_last_n` entries of this row's history (disabled at this
+    // launch: `sample_tokens` refuses a non-zero `penalty_last_n` without a history buffer).
+    const int* hrow = history ? history + (size_t) t * history_len : nullptr;
+    int hlen = 0;
+    if (hrow) {
+        hlen = p.penalty_last_n < history_len ? p.penalty_last_n : history_len;
+        if (hlen < 0) hlen = 0;
+        hrow += history_len - hlen;          // the window is the TAIL
+    }
+
+    // the membership bitmap, as in `sampler_greedy_kernel` - see the cost note there.  The gate needs an
+    // NON-EMPTY WINDOW too: the launch sizes the bitmap only when penalties are on, so a caller that hands over
+    // a stale history buffer with `penalty_last_n == 0` must not touch it.
+    extern __shared__ unsigned int penal_bits[];
+    const int bits_words = (int) ((n_vocab + 31) / 32);
+    const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;
+    if (use_bits) {
+        for (int w = threadIdx.x; w < bits_words; w += blockDim.x) penal_bits[w] = 0u;
+        __syncthreads();
+        for (int i = threadIdx.x; i < hlen; i += blockDim.x)
+            if (hrow[i] >= 0 && hrow[i] < n_vocab)   // an id outside the vocabulary is never a candidate
+                atomicOr(&penal_bits[hrow[i] >> 5], 1u << (hrow[i] & 31));
+        __syncthreads();
+    }
+    auto hit_count = [&](int v) -> int {
+        if (!use_bits || !(penal_bits[v >> 5] & (1u << (v & 31)))) return 0;
+        return history_count(hrow, hlen, v);
+    };
+
+    // top_k in 1..64 is taken as given; 0 ("off") and anything wider mean the widest shortlist the kernel
+    // keeps, 64.  Every row writes out[t]: a verify window reads all of them.
+    const int KMAX = 64;
+    int k = (p.top_k > 0 && p.top_k < KMAX) ? p.top_k : KMAX;
+    if (k > n_vocab) k = n_vocab;
+
+    // ---- top_k: an exact RADIX SELECT, then a rank sort of the k survivors.  The order is the serial one:
+    // descending by (penalised) logit, ties to the lower index - i.e. descending by the composite key
+    // (ordered(score) << 32) | ~index, which is distinct for every token.  So: find the k-th largest ordered score
+    // T byte by byte (4 histogram passes over the vocabulary), keep every token above T and the lowest-indexed
+    // ones equal to T, then rank the k of them by the composite key.  The k block-argmax rounds this replaces
+    // re-scanned the taken list for every token in every round: 1.5 ms a sample at top_k 20 and 11 ms at 64 on an
+    // RTX 4090 (the tail card's critical path in every sampled decode step).
+    // THE COMMON CASE takes two passes: a coarse histogram of the top 11 bits finds the bin that holds the k-th
+    // largest; every token in that bin or above (at most kCap) is copied to shared memory and ranked there by the
+    // composite key.  A flat distribution with more than kCap tokens there takes the full radix select below.
+    __shared__ int sel_ids[KMAX];
+    __shared__ float sel_logit[KMAX];
+    __shared__ unsigned int sh_prefix, sh_remaining, sh_ncand, sh_tie_base;
+    // ONE shared pool, its parts reused phase by phase: the penalty bitmap (dynamic shared memory, 31 KB at a Qwen
+    // vocabulary) must still fit beside it under the 48 KB a launch gets without opting in.
+    //   [0, 2048)     the coarse histogram, then (once its bin is known) the candidates' scores and ids;
+    //                 in the fallback, the byte histogram and the k candidates
+    //   [2048, 3072)  the block scan
+    constexpr int kBins = 2048, kCap = 1024;   // the coarse histogram: the ordered score's top 11 bits
+    __shared__ unsigned int pool[kBins + 1024];
+    unsigned int* coarse = pool;
+    unsigned int* ccand_u = pool;
+    int* ccand_i = (int*) pool + kCap;
+    unsigned int* hist = pool;
+    unsigned int* cand_u = pool + 256;
+    int* cand_i = (int*) pool + 256 + KMAX;
+    unsigned int* scan = pool + kBins;
+    __shared__ int sh_bin;
+    __shared__ unsigned int sh_ncoarse;
+    // a score's ordered bits: larger float -> larger unsigned; NaN below everything (never kept before a number)
+    auto ordered = [](float x) -> unsigned int {
+        unsigned int b = __float_as_uint(x);
+        if ((b & 0x7fffffffu) > 0x7f800000u) return 0u;
+        if (b == 0x80000000u) b = 0u;   // -0 ties with +0, as the float comparison it replaces
+        return (b & 0x80000000u) ? ~b : (b | 0x80000000u);
+    };
+    auto score = [&](int v) -> float {
+        return apply_penalties(p.logit_bias ? l[v] + p.logit_bias[v] : l[v], hit_count(v), p);
+    };
+    for (int b = threadIdx.x; b < kBins; b += blockDim.x) coarse[b] = 0u;
+    if (threadIdx.x == 0) sh_ncoarse = 0u;
+    __syncthreads();
+    for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) atomicAdd(&coarse[ordered(score(v)) >> 21], 1u);
+    __syncthreads();
+    {   // the bin holding the k-th largest: suffix counts over the bins (2 bins a thread, a block scan)
+        const int nt = (int) blockDim.x, per = (kBins + nt - 1) / nt;
+        const int b_hi = kBins - 1 - (int) threadIdx.x * per;   // this thread's bins, from the top down
+        unsigned int mine = 0u;
+        for (int j = 0; j < per && b_hi - j >= 0; ++j) mine += coarse[b_hi - j];
+        scan[threadIdx.x] = mine;
+        __syncthreads();
+        for (int off = 1; off < nt; off <<= 1) {
+            const unsigned int add = (int) threadIdx.x >= off ? scan[threadIdx.x - off] : 0u;
+            __syncthreads();
+            scan[threadIdx.x] += add;
+            __syncthreads();
+        }
+        unsigned int above = scan[threadIdx.x] - mine;   // tokens in the bins above this thread's
+        if (above < (unsigned int) k && above + mine >= (unsigned int) k) {
+            for (int j = 0; j < per && b_hi - j >= 0; ++j) {
+                above += coarse[b_hi - j];
+                if (above >= (unsigned int) k) { sh_bin = b_hi - j; sh_tie_base = above; break; }
+            }
+        }
+        __syncthreads();
+    }
+    const int bin = sh_bin;
+    const bool coarse_ok = sh_tie_base <= (unsigned int) kCap;   // the tokens in that bin and above fit
+    if (coarse_ok) {
+        for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {
+            const unsigned int u = ordered(score(v));
+            if ((int) (u >> 21) >= bin) {
+                const unsigned int at = atomicAdd(&sh_ncoarse, 1u);
+                if (at < (unsigned int) kCap) { ccand_u[at] = u; ccand_i[at] = v; }
+            }
+        }
+        __syncthreads();
+        const int nc = (int) min(sh_ncoarse, (unsigned int) kCap);
+        for (int i = threadIdx.x; i < nc; i += blockDim.x) {
+            const unsigned int ui = ccand_u[i];
+            const int ii = ccand_i[i];
+            int r = 0;
+            for (int j = 0; j < nc && r < k; ++j) {
+                const unsigned int uj = ccand_u[j];
+                r += uj > ui || (uj == ui && ccand_i[j] < ii);
+            }
+            if (r < k) {
+                sel_ids[r] = ii;
+                sel_logit[r] = score(ii);
+            }
+        }
+        __syncthreads();
+    } else {
+        if (threadIdx.x == 0) {
+            sh_prefix = 0u;
+            sh_remaining = (unsigned int) k;
+            sh_ncand = 0u;
+        }
+        unsigned int mask = 0u;
+        for (int pass = 0; pass < 4; ++pass) {
+            const int shift = 24 - 8 * pass;
+            for (int b = threadIdx.x; b < 256; b += blockDim.x) hist[b] = 0u;
+            __syncthreads();
+            const unsigned int prefix = sh_prefix;
+            for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {
+                const unsigned int u = ordered(score(v));
+                if ((u & mask) == prefix) atomicAdd(&hist[(u >> shift) & 255u], 1u);
+            }
+            __syncthreads();
+            if (threadIdx.x == 0) {   // the digit that holds the remaining-th largest, from the top
+                unsigned int cum = 0u, rem = sh_remaining;
+                int d = 255;
+                for (; d > 0; --d) {
+                    if (cum + hist[d] >= rem) break;
+                    cum += hist[d];
+                }
+                sh_remaining = rem - cum;
+                sh_prefix = prefix | ((unsigned int) d << shift);
+            }
+            mask |= 255u << shift;
+            __syncthreads();
+        }
+        const unsigned int T = sh_prefix;      // the k-th largest ordered score
+        const unsigned int need = sh_remaining; // how many tokens equal to T are kept (the lowest-indexed ones)
+        // every token above T (fewer than k of them)
+        for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {
+            const unsigned int u = ordered(score(v));
+            if (u > T) {
+                const unsigned int at = atomicAdd(&sh_ncand, 1u);
+                if (at < (unsigned int) KMAX) { cand_u[at] = u; cand_i[at] = v; }
+            }
+        }
+        __syncthreads();
+        if (threadIdx.x == 0) sh_tie_base = sh_ncand;
+        // the ties in index order: each thread counts its own contiguous range, an exclusive scan orders the ranges
+        {
+            const int nt = (int) blockDim.x;
+            const int chunk = (n_vocab + nt - 1) / nt;
+            const int lo = (int) threadIdx.x * chunk, hi = min(n_vocab, lo + chunk);
+            unsigned int mine = 0u;
+            for (int v = lo; v < hi; ++v) mine += ordered(score(v)) == T;
+            scan[threadIdx.x] = mine;
+            __syncthreads();
+            for (int off = 1; off < nt; off <<= 1) {   // Hillis-Steele inclusive scan
+                const unsigned int add = (int) threadIdx.x >= off ? scan[threadIdx.x - off] : 0u;
+                __syncthreads();
+                scan[threadIdx.x] += add;
+                __syncthreads();
+            }
+            unsigned int rank = scan[threadIdx.x] - mine;   // ties before this range
+            for (int v = lo; v < hi && rank < need; ++v)
+                if (ordered(score(v)) == T) {
+                    const unsigned int at = sh_tie_base + rank;
+                    if (at < (unsigned int) KMAX) { cand_u[at] = T; cand_i[at] = v; }
+                    ++rank;
+                }
+        }
+        __syncthreads();
+        // rank sort: candidate i's place is the number of candidates with a larger composite key
+        if ((int) threadIdx.x < k) {
+            const unsigned int ui = cand_u[threadIdx.x];
+            const int ii = cand_i[threadIdx.x];
+            int r = 0;
+            for (int j = 0; j < k; ++j) {
+                const unsigned int uj = cand_u[j];
+                r += uj > ui || (uj == ui && cand_i[j] < ii);
+            }
+            sel_ids[r] = ii;
+            sel_logit[r] = score(ii);
+        }
+        __syncthreads();
+    }
+
+    // ---- top_p over the top_k list (penalised logits, descending as the selection produced them), then min_p,
+    // then temperature and one Philox draw - llama.cpp's order (issue #53).  Every thread computes the same chain
+    // redundantly over `sel_*` - the arithmetic is the serial kernel's, instruction for instruction - so they
+    // agree on `pick` and thread 0 writes it.
+    // (one thread: the chain is serial and in double precision, which consumer cards run at 1/64 rate - 1024
+    // threads computing it redundantly cost more than the whole selection above)
+    if (threadIdx.x != 0) return;
+    int n_keep = k;
+    float mx = sel_logit[0];
+    for (int i = 1; i < k; ++i) mx = fmaxf(mx, sel_logit[i]);
+    if (p.top_p < 1.0f) {
+        double sum = 0.0;
+        for (int i = 0; i < k; ++i) sum += exp((double) sel_logit[i] - (double) mx);
+        double cum = 0.0;
+        int cut = k;
+        for (int i = 0; i < k; ++i) {
+            cum += exp((double) sel_logit[i] - (double) mx) / sum;
+            if (cum >= (double) p.top_p) { cut = i + 1; break; }
+        }
+        if (cut < p.min_keep) cut = p.min_keep < k ? p.min_keep : k;
+        n_keep = cut;
+    }
+    // ---- min_p on top_p's survivors: the descending prefix whose probability is at least `min_p` of the top
+    // token's.  In logit space the threshold is `sel_logit[0] + logf(min_p)` - equivalent to `p >= min_p * p_max`
+    // without the overflow an exp of raw logits risks.  0 disables, and the head itself always survives
+    // (`expf(0) == 1 >= min_p` for min_p in 0..1), so the count never reaches zero.
+    if (p.min_p > 0.0f) {
+        const float thresh = sel_logit[0] + logf(p.min_p);
+        for (int i = 0; i < n_keep; ++i)
+            if (sel_logit[i] < thresh) { n_keep = i; break; }
+    }
+    // temperature only: the penalties were applied once, before the selection (issue #53: they were applied a
+    // second time here, after the temperature scaling - llama.cpp's chain has one penalties stage)
+    auto scaled = [&](int i) { return sel_logit[i] * inv_t; };
+    float smx = scaled(0);
+    for (int i = 1; i < n_keep; ++i) smx = fmaxf(smx, scaled(i));
+    double sum = 0.0;
+    for (int i = 0; i < n_keep; ++i) sum += exp((double) scaled(i) - (double) smx);
+    if constexpr (MODE == 0) {
+        const float u = philox_uniform(p.seed, p.counter + (uint64_t) t);
+        double cum = 0.0;
+        int pick = sel_ids[n_keep - 1];
+        for (int i = 0; i < n_keep; ++i) {
+            cum += exp((double) scaled(i) - (double) smx) / sum;
+            if ((double) u < cum) { pick = sel_ids[i]; break; }
+        }
+        if (threadIdx.x == 0) out[t] = pick;
+    } else {
+        double w[KMAX];   // the kept list's probabilities, as the plain draw computes them
+        for (int i = 0; i < n_keep; ++i) w[i] = exp((double) scaled(i) - (double) smx) / sum;
+        // the plain draw over w with a given uniform (MODE 0's loop)
+        auto draw = [&](float u) -> int {
+            double cum = 0.0;
+            int pick = sel_ids[n_keep - 1];
+            for (int i = 0; i < n_keep; ++i) {
+                cum += w[i];
+                if ((double) u < cum) { pick = sel_ids[i]; break; }
+            }
+            return pick;
+        };
+        if constexpr (MODE == 1) {
+            SpecList* q = ql + t;
+            q->n = n_keep;
+            for (int i = 0; i < n_keep; ++i) { q->ids[i] = sel_ids[i]; q->p[i] = w[i]; }
+            out[t] = draw(philox_uniform(p.seed ^ kSpecDraftKey, p.counter + (uint64_t) t));
+        } else {
+            const SpecList* q = ql + t;
+            const int d = draft[t];
+            const int qn = q->n < KMAX ? q->n : KMAX;
+            auto qof = [&](int id) -> double {
+                for (int j = 0; j < qn; ++j)
+                    if (q->ids[j] == id) return q->p[j];
+                return 0.0;
+            };
+            double pd = 0.0;
+            for (int i = 0; i < n_keep; ++i)
+                if (sel_ids[i] == d) { pd = w[i]; break; }
+            const double qd = qof(d);
+            int pick = -1;
+            if (qd <= 0.0) {
+                pick = draw(philox_uniform(p.seed, p.counter + (uint64_t) t));   // not a draft from q: the plain draw
+            } else if ((double) philox_uniform(p.seed ^ kSpecAcceptKey, p.counter + (uint64_t) t) * qd < pd) {
+                pick = d;                                                        // kept: probability min(1, p(d)/q(d))
+            } else {
+                // rejected: a draw from max(0, p - q), renormalised - over p's kept list (p is 0 outside it)
+                double r[KMAX], z = 0.0;
+                for (int i = 0; i < n_keep; ++i) {
+                    const double x = w[i] - qof(sel_ids[i]);
+                    r[i] = x > 0.0 ? x : 0.0;
+                    z += r[i];
+                }
+                const float u = philox_uniform(p.seed ^ kSpecResidKey, p.counter + (uint64_t) t);
+                if (z > 0.0) {
+                    double cum = 0.0;
+                    for (int i = 0; i < n_keep; ++i) {
+                        if (r[i] <= 0.0) continue;
+                        pick = sel_ids[i];   // the last one with residual mass, if rounding leaves cum below u
+                        cum += r[i] / z;
+                        if ((double) u < cum) break;
+                    }
+                } else {
+                    pick = draw(u);   // p == q to the last bit: a rejection had probability 0 - never in practice
+                }
+            }
+            out[t] = pick;
+        }
+    }
+}
+
+}  // namespace
+
+void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
+                   const SamplerParams& p, int* out, void* stream) {
+    if (n_tokens <= 0 || n_vocab <= 0) return;
+    if (p.penalty_last_n > 0 && (history == nullptr || history_len <= 0)) {
+        std::fprintf(stderr, "sample_tokens: penalty_last_n %d needs a history (got %p, len %d)\n",
+                     p.penalty_last_n, (const void*) history, history_len);
+        std::exit(1);
+    }
+    const unsigned shmem = (history != nullptr && history_len > 0 && p.penalty_last_n > 0)
+                               ? (unsigned) ((n_vocab + 31) / 32) * sizeof(unsigned)   // the penalty bitmap
+                               : 0;
+    if (p.greedy || p.temperature <= 0.0f) {
+        // One block per token, 1,024 threads over the vocabulary.  See `sampler_greedy_kernel`.
+        const int gthreads = 1024;
+        sampler_greedy_kernel<<<(unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream>>>(
+            logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);
+    } else {
+        // The same block-per-token shape: the selection's k argmax rounds reduce inside the block.  See
+        // `sampler_kernel`'s header for what the old one-thread-per-token launch cost.
+        sampler_kernel<0><<<(unsigned) n_tokens, 1024, shmem, (cudaStream_t) stream>>>(
+            logits, n_vocab, n_tokens, history, history_len, p, out, nullptr, nullptr);
+    }
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "sample_tokens launch: %s\n", cudaGetErrorString(e));
+        std::exit(1);
+    }
+    if (stream == nullptr) cudaDeviceSynchronize();
+}
+
+namespace {
+void spec_check(const char* who, int n_tokens, int n_vocab, const SamplerParams& p) {
+    if (p.greedy || p.temperature <= 0.0f || p.penalty_last_n > 0 || n_vocab <= 0) {
+        std::fprintf(stderr, "%s: a sampled request without penalties only (greedy %d, temperature %g, penalty_last_n "
+                             "%d, n_vocab %d)\n", who, (int) p.greedy, p.temperature, p.penalty_last_n, n_vocab);
+        std::exit(1);
+    }
+    (void) n_tokens;
+}
+void spec_launched(const char* who, void* stream) {
+    const cudaError_t e = cudaGetLastError();
+    if (e != cudaSuccess) {
+        std::fprintf(stderr, "%s launch: %s\n", who, cudaGetErrorString(e));
+        std::exit(1);
+    }
+    if (stream == nullptr) cudaDeviceSynchronize();
+}
+}  // namespace
+
+void sample_draft(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, SpecList* q, int* out,
+                  void* stream) {
+    if (n_tokens <= 0) return;
+    spec_check("sample_draft", n_tokens, n_vocab, p);
+    sampler_kernel<1><<<(unsigned) n_tokens, 1024, 0, (cudaStream_t) stream>>>(logits, n_vocab, n_tokens, nullptr, 0,
+                                                                                p, out, q, nullptr);
+    spec_launched("sample_draft", stream);
+}
+
+void sample_verify(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, const SpecList* q,
+                   const int* draft, int* out, void* stream) {
+    if (n_tokens <= 0) return;
+    spec_check("sample_verify", n_tokens, n_vocab, p);
+    sampler_kernel<2><<<(unsigned) n_tokens, 1024, 0, (cudaStream_t) stream>>>(logits, n_vocab, n_tokens, nullptr, 0,
+                                                                                p, out, const_cast<SpecList*>(q), draft);
+    spec_launched("sample_verify", stream);
+}
+
+}  // namespace strata::dsv41k

src/kernels/cpu/expert_layout.cpp

@@ -503,6 +503,7 @@ bool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t n
                   ": the pack is inconsistent (repack with tools/iq_pack.py)";
             return false;
         }
+        for (int64_t l = 0; l < n_layers; ++l) L.fmt[(size_t) l].layer = (int) l;
         HadamardSpec h;
         if (have && !hadamard_parse(pack_dir + "/hadamard.txt", n_layers, h, err)) return false;
         hadamard_set(std::move(h));

src/kernels/cpu/iq_avx2.cpp

@@ -32,6 +32,7 @@
 #include <cmath>
 #include <cstdlib>
 #include <cstring>
+#include <vector>
 
 // STRATA_AVXVNNI (CMake: the compiler has the AVX-VNNI intrinsics): GCC and Clang compile them only inside functions
 // with target("avxvnni") - the VNNI copy of iq_avx2_rows.inl.
@@ -534,6 +535,23 @@ void iq256_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void*
     iq256_rows_v(iq256_variant(), type, w, row_bytes, n, act, nt, out, r0, r1);
 }
 
+bool iq256x_supported(int type) noexcept { return type == 16 || type == 17 || type == 18 || type == 21 || type == 22 || type == 29; }
+
+void iq256x_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out,
+                 int r0, int r1) {
+    const int variant = iq256_variant();
+    const bool g = (variant & kIq256Gather) != 0;
+#if STRATA_AVXVNNI
+    if (variant & kIq256Vnni) {
+        if (g) vnni::dot_type<true, true>(type, nt, w, row_bytes, n, act, out, r0, r1);
+        else vnni::dot_type<false, true>(type, nt, w, row_bytes, n, act, out, r0, r1);
+        return;
+    }
+#endif
+    if (g) plain::dot_type<true, true>(type, nt, w, row_bytes, n, act, out, r0, r1);
+    else plain::dot_type<false, true>(type, nt, w, row_bytes, n, act, out, r0, r1);
+}
+
 void iq4nl256_down_rows_v(int variant, const uint8_t* w, size_t row_bytes, int n, const void* const* hq, int nt,
                           float* const* out, int r0, int r1) {
     const block_q8_0* y[8];

src/kernels/cpu/iq_avx2_rows.inl

@@ -78,7 +78,7 @@ void gu_ggml_one(const uint8_t* blob, size_t gu_row, size_t up_off, int n, const
     }
 }
 
-template <int TY, int NT> STRATA_ROWS_FN
+template <int TY, int NT, bool EX = false> STRATA_ROWS_FN
 inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res) {
     const int pf = prefetch_distance();
     __m256 accf[NT];
@@ -86,8 +86,12 @@ inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y,
     for (int i = 0; i < nblocks; ++i) {
         const uint8_t* blk = row + (size_t) i * Fmt32<TY>::bytes;
         rows_ahead(blk, pf);
-        __m256i acci[NT];
-        for (int t = 0; t < NT; ++t) acci[t] = _mm256_setzero_si256();
+        // EX: independent integer accumulators a token (A of them) cut the dependent madd+add chain from eight steps to 8/A -
+        // few tokens have too little else in flight to hide it (integer sums, the same bits)
+        constexpr int A = EX ? (NT == 1 ? 4 : NT == 2 ? 2 : 1) : 1;
+        __m256i acci[NT][A];
+        for (int t = 0; t < NT; ++t)
+            for (int a = 0; a < A; ++a) acci[t][a] = _mm256_setzero_si256();
         for (int j = 0; j < 4; ++j) {
             for (int half = 0; half < 2; ++half) {
                 __m256i g, sgn, sc;
@@ -96,15 +100,20 @@ inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y,
                 for (int t = 0; t < NT; ++t) {
                     const __m256i yv = _mm256_loadu_si256((const __m256i*) (y[t][i].qs + off));
                     const __m256i ys = _mm256_sign_epi8(yv, sgn);
-                    acci[t] = madd_add(acci[t], _mm256_maddubs_epi16(g, ys), sc);
+                    acci[t][(2 * j + half) % A] = madd_add(acci[t][(2 * j + half) % A], _mm256_maddubs_epi16(g, ys), sc);
                 }
             }
         }
-        const float dx = h2f(u16(blk)) * Fmt32<TY>::K;
         for (int t = 0; t < NT; ++t)
-            accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(acci[t]), accf[t]);
+            for (int a = 1; a < A; ++a) acci[t][0] = _mm256_add_epi32(acci[t][0], acci[t][a]);
+        // EX (the DeepSeek CPU lane's rows): ggml's float order, so a token's dot is the same bits as ggml-cpu's whatever
+        // number of tokens share the pass - block scale f16(d) * y.d, the format's constant K after the sum, ggml's hsum.
+        // Without EX the order of the main engine's rows (unchanged).
+        const float dx = EX ? h2f(u16(blk)) : h2f(u16(blk)) * Fmt32<TY>::K;
+        for (int t = 0; t < NT; ++t)
+            accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(acci[t][0]), accf[t]);
     }
-    for (int t = 0; t < NT; ++t) res[t] = hsum8(accf[t]);
+    for (int t = 0; t < NT; ++t) res[t] = EX ? Fmt32<TY>::K * ggml_hsum8(accf[t]) : hsum8(accf[t]);
 }
 
 // ---- IQ2_XS (17) with ggml's vectorized sign decode (arch/x86/quants.c, ggml_vec_dot_iq2_xs_q8_K):
@@ -156,8 +165,10 @@ inline void row_dot_iq2xs(const uint8_t* row, int nblocks, const block_q8_K* con
         __m128i st = _mm_set1_epi64x((long long) u64(blk + 66));
         st = _mm_unpacklo_epi8(_mm_and_si128(st, m4), _mm_and_si128(_mm_srli_epi16(st, 4), m4));
         const __m128i scales = _mm_add_epi8(_mm_slli_epi16(st, 1), m1);
-        __m256i acc[NT][2];
-        for (int t = 0; t < NT; ++t) acc[t][0] = acc[t][1] = _mm256_setzero_si256();
+        constexpr int CH = GGML ? 4 : 2;   // (EX: four independent integer chains a token)
+        __m256i acc[NT][CH];
+        for (int t = 0; t < NT; ++t)
+            for (int c = 0; c < CH; ++c) acc[t][c] = _mm256_setzero_si256();
         for (int jj = 0; jj < 2; ++jj) {   // 128 values (16 u16) per step
             const __m256i q2 = _mm256_loadu_si256((const __m256i*) (blk + 2 + 32 * jj));
             const __m256i p7 = _mm256_srli_epi16(q2, 9);              // 7 sign bits per u16, even bytes
@@ -183,7 +194,7 @@ inline void row_dot_iq2xs(const uint8_t* row, int nblocks, const block_q8_K* con
                 for (int t = 0; t < NT; ++t) {
                     const __m256i yv = _mm256_loadu_si256((const __m256i*) (y[t][i].qs + off));
                     const __m256i ys = _mm256_sign_epi8(yv, sgn);
-                    acc[t][h & 1] = madd_add(acc[t][h & 1], _mm256_maddubs_epi16(g, ys), sc);
+                    acc[t][h % CH] = madd_add(acc[t][h % CH], _mm256_maddubs_epi16(g, ys), sc);
                 }
             }
         }
@@ -191,98 +202,404 @@ inline void row_dot_iq2xs(const uint8_t* row, int nblocks, const block_q8_K* con
             const float d = h2f(u16(blk)) * (GGML ? y[t][i].d : 0.125f);
             const float scale = GGML ? d : d * y[t][i].d;
             accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(scale),
-                _mm256_cvtepi32_ps(_mm256_add_epi32(acc[t][0], acc[t][1])), accf[t]);
+                _mm256_cvtepi32_ps(CH == 4 ? _mm256_add_epi32(_mm256_add_epi32(acc[t][0], acc[t][1]), _mm256_add_epi32(acc[t][2], acc[t][3]))
+                                                       : _mm256_add_epi32(acc[t][0], acc[t][1])), accf[t]);
         }
     }
     for (int t = 0; t < NT; ++t) res[t] = GGML ? 0.125f * ggml_hsum8(accf[t]) : hsum8(accf[t]);
 }
 
-template <int TY, int NT> STRATA_ROWS_FN
-inline void row_dot_any(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res) {
-    if constexpr (TY == 17) row_dot_iq2xs<NT>(row, nblocks, y, res);
-    else                    row_dot<TY, NT>(row, nblocks, y, res);
+// ---- IQ1_M (29): 56-byte blocks (qs[32], qh[16], scales[8] holding the 3-bit sub-scales and, in their top nibbles, the
+// block's f16 d), ggml's ggml_vec_dot_iq1_m_q8_K (AVX2): s = hsum(sum d*sumi1) + 0.125 * hsum(sum d*sumi2), sumi1 the
+// grid dot, sumi2 the +-1 delta dot.  Both are kept lane for lane as ggml has them, so the bits are ggml's.
+//  - the grid values are -1/0/1 bytes: |g| and g's sign (the two halves of ggml's mul_add_epi8) are taken once per weight chunk;
+//  - the delta dot needs no weight bytes beyond one sign a group of 8: sign(q8, delta) summed in pairs and times the scale is
+//    (pair sums of q8) times (+-scale), so the pair sums of every activation chunk are made once per call (iq1m_pre_build) and
+//    a token's delta dot is one madd per 32 values;
+//  - the indices' high bits and the delta signs of eight groups come out of one 32-bit load by variable shifts (ggml's
+//    non-BMI2 path branches on every bit).
+
+// the pair sums maddubs(1, q8) of every 32-value chunk of an activation: 16 int16 a chunk, 8 chunks a block
+STRATA_ROWS_FN inline void iq1m_pre_build(const block_q8_K* y, int nblocks, int16_t* p) {
+    const __m256i one8 = _mm256_set1_epi8(1);
+    for (int i = 0; i < nblocks; ++i)
+        for (int c = 0; c < 8; ++c)
+            _mm256_storeu_si256((__m256i*) (p + ((size_t) i * 8 + c) * 16),
+                                _mm256_maddubs_epi16(one8, _mm256_loadu_si256((const __m256i*) (y[i].qs + 32 * c))));
 }
 
-template <int TY, int NT> STRATA_ROWS_FN
+template <int NT> STRATA_ROWS_FN
+inline void row_dot_iq1m(const uint8_t* row, int nblocks, const block_q8_K* const* y, const int16_t* const* P, float* res) {
+    const __m256i mask = _mm256_set1_epi16(0x7);
+    const __m256i mone = _mm256_set1_epi16(1);
+    const __m256i mtwo8 = _mm256_set1_epi8(2);
+    const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);
+    const __m256i nib_shift = _mm256_setr_epi32(0, 4, 8, 12, 16, 20, 24, 28);
+    const __m256i m7 = _mm256_set1_epi32(7), m8 = _mm256_set1_epi32(8);
+    const int pf = prefetch_distance();
+    __m256 acc1[NT], acc2[NT];
+    for (int t = 0; t < NT; ++t) acc1[t] = acc2[t] = _mm256_setzero_ps();
+    for (int i = 0; i < nblocks; ++i) {
+        const uint8_t* blk = row + (size_t) i * 56;
+        rows_ahead(blk, pf);
+        const uint8_t* qs = blk;
+        const uint8_t* qh = blk + 32;
+        const uint8_t* scb = blk + 48;
+        const uint16_t sc0 = u16(scb), sc1 = u16(scb + 2), sc2 = u16(scb + 4), sc3 = u16(scb + 6);
+        const uint16_t d16 = (uint16_t) ((sc0 >> 12) | ((sc1 >> 8) & 0x00f0) | ((sc2 >> 4) & 0x0f00) | (sc3 & 0xf000));
+        __m256i scales = _mm256_set1_epi64x((long long) u64(scb));
+        scales = _mm256_srlv_epi64(scales, scales_shift);
+        scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);
+        __m256i sidx1 = _mm256_set1_epi16(0x0100);
+        __m256i sidx2 = _mm256_add_epi8(sidx1, _mm256_set1_epi8(8));
+        // the block's four 64-value steps decoded once into a small stack frame, then each token runs over them with two
+        // accumulators in registers (NT tokens' worth of accumulators and decode together do not fit 16 registers)
+        struct Dec { __m256i a1, g1, a2, g2, sa, sb, ssa, ssb; };
+        Dec dec[4];
+        for (int it = 0; it < 4; ++it) {
+            const __m256i v8 = _mm256_srlv_epi32(_mm256_set1_epi32((int) u32(qh + 4 * it)), nib_shift);   // group k's qh nibble
+            const __m256i idx = _mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) (qs + 8 * it))),
+                                                _mm256_slli_epi32(_mm256_and_si256(v8, m7), 8));
+            alignas(32) int32_t ix[8];
+            _mm256_store_si256((__m256i*) ix, idx);
+            const __m256i g1 = _mm256_set_epi64x((long long) iq1s_grid[ix[3]], (long long) iq1s_grid[ix[2]],
+                                                 (long long) iq1s_grid[ix[1]], (long long) iq1s_grid[ix[0]]);
+            const __m256i g2 = _mm256_set_epi64x((long long) iq1s_grid[ix[7]], (long long) iq1s_grid[ix[6]],
+                                                 (long long) iq1s_grid[ix[5]], (long long) iq1s_grid[ix[4]]);
+            const __m256i neg = _mm256_cmpeq_epi32(_mm256_and_si256(v8, m8), m8);   // delta -1: all ones a group
+            const __m256i n1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(neg));   // 4 int16 lanes a group of 8 values
+            const __m256i n2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(neg, 1));
+            const __m256i sc_a = _mm256_shuffle_epi8(scales, sidx1);
+            const __m256i sc_b = _mm256_shuffle_epi8(scales, sidx2);
+            dec[it] = Dec{_mm256_sign_epi8(g1, g1), g1, _mm256_sign_epi8(g2, g2), g2, sc_a, sc_b,
+                          _mm256_sign_epi16(sc_a, _mm256_or_si256(n1, mone)), _mm256_sign_epi16(sc_b, _mm256_or_si256(n2, mone))};
+            sidx1 = _mm256_add_epi8(sidx1, mtwo8);
+            sidx2 = _mm256_add_epi8(sidx2, mtwo8);
+        }
+        const float dx = h2f(d16);
+        for (int t = 0; t < NT; ++t) {
+            __m256i s1 = _mm256_setzero_si256(), s2 = _mm256_setzero_si256();
+            for (int it = 0; it < 4; ++it) {
+                const Dec& dc = dec[it];
+                const __m256i q8a = _mm256_loadu_si256((const __m256i*) (y[t][i].qs + 64 * it));
+                const __m256i q8b = _mm256_loadu_si256((const __m256i*) (y[t][i].qs + 64 * it + 32));
+                const __m256i dot1 = _mm256_maddubs_epi16(dc.a1, _mm256_sign_epi8(q8a, dc.g1));
+                const __m256i dot2 = _mm256_maddubs_epi16(dc.a2, _mm256_sign_epi8(q8b, dc.g2));
+                s1 = _mm256_add_epi32(s1, _mm256_add_epi32(_mm256_madd_epi16(dot1, dc.sa), _mm256_madd_epi16(dot2, dc.sb)));
+                const int16_t* pp = P[t] + ((size_t) i * 8 + 2 * it) * 16;
+                s2 = _mm256_add_epi32(s2, _mm256_add_epi32(_mm256_madd_epi16(_mm256_loadu_si256((const __m256i*) pp), dc.ssa),
+                                                           _mm256_madd_epi16(_mm256_loadu_si256((const __m256i*) (pp + 16)), dc.ssb)));
+            }
+            const __m256 d = _mm256_set1_ps(y[t][i].d * dx);
+            acc1[t] = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(s1), acc1[t]);
+            acc2[t] = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(s2), acc2[t]);
+        }
+    }
+    for (int t = 0; t < NT; ++t) res[t] = ggml_hsum8(acc1[t]) + 0.125f * ggml_hsum8(acc2[t]);
+}
+
+template <int TY, int NT, bool EX = false> STRATA_ROWS_FN
+inline void row_dot_any(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res,
+                        const int16_t* const* aux = nullptr) {
+    if constexpr (TY == 29) row_dot_iq1m<NT>(row, nblocks, y, aux, res);
+    else if constexpr (TY == 17) row_dot_iq2xs<NT, EX>(row, nblocks, y, res);
+    else                    row_dot<TY, NT, EX>(row, nblocks, y, res);
+}
+
+template <int TY, int NT, bool EX> STRATA_ROWS_FN
 void gu_rows(const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, float* const* ff,
              int r0, int r1) {
     const block_q8_K* y[NT];
     for (int t = 0; t < NT; ++t) y[t] = (const block_q8_K*) act[t];
     const int nb = n / QK_K;
+    const int16_t* aux[NT] = {};
+    if constexpr (TY == 29) {   // (IQ1_M's activation pair sums, once for the rows of this call)
+        static thread_local std::vector<int16_t> pre;
+        pre.resize((size_t) NT * nb * 128);
+        for (int t = 0; t < NT; ++t) {
+            iq1m_pre_build(y[t], nb, pre.data() + (size_t) t * nb * 128);
+            aux[t] = pre.data() + (size_t) t * nb * 128;
+        }
+    }
     float g[NT], u[NT];
     for (int r = r0; r < r1; ++r) {
-        row_dot_any<TY, NT>(blob + (size_t) r * gu_row, nb, y, g);
-        row_dot_any<TY, NT>(blob + up_off + (size_t) r * gu_row, nb, y, u);
+        row_dot_any<TY, NT, EX>(blob + (size_t) r * gu_row, nb, y, g, aux);
+        row_dot_any<TY, NT, EX>(blob + up_off + (size_t) r * gu_row, nb, y, u, aux);
         for (int t = 0; t < NT; ++t) ff[t][r] = (g[t] / (1.f + std::exp(-g[t]))) * u[t];
     }
 }
 
-template <int TY, int NT> STRATA_ROWS_FN
+template <int TY, int NT, bool EX> STRATA_ROWS_FN
 void dot_rows(const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out, int r0, int r1) {
     const block_q8_K* y[NT];
     for (int t = 0; t < NT; ++t) y[t] = (const block_q8_K*) act[t];
+    const int nb = n / QK_K;
+    const int16_t* aux[NT] = {};
+    if constexpr (TY == 29) {   // (IQ1_M's activation pair sums, once for the rows of this call)
+        static thread_local std::vector<int16_t> pre;
+        pre.resize((size_t) NT * nb * 128);
+        for (int t = 0; t < NT; ++t) {
+            iq1m_pre_build(y[t], nb, pre.data() + (size_t) t * nb * 128);
+            aux[t] = pre.data() + (size_t) t * nb * 128;
+        }
+    }
     float res[NT];
     for (int r = r0; r < r1; ++r) {
-        row_dot_any<TY, NT>(w + (size_t) r * row_bytes, n / QK_K, y, res);
+        row_dot_any<TY, NT, EX>(w + (size_t) r * row_bytes, nb, y, res, aux);
         for (int t = 0; t < NT; ++t) out[t][r] = res[t];
     }
 }
 
-template <int TY> STRATA_ROWS_FN
-void gu_rows_nt(int nt, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act,
-                float* const* ff, int r0, int r1) {
-    switch (nt) {
-        case 1: gu_rows<TY, 1>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 2: gu_rows<TY, 2>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 3: gu_rows<TY, 3>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 4: gu_rows<TY, 4>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 5: gu_rows<TY, 5>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 6: gu_rows<TY, 6>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 7: gu_rows<TY, 7>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        default: gu_rows<TY, 8>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
-    }
+// ---- many tokens an expert (the prompt: a chunk's tokens routed to one expert), EX rows only.
+// The weights of a 256-value block are decoded once into signed int8 (the grid magnitude times the sign) and every token of the
+// call applies them.  The activations come as u8 (q8 + 128, made once per call for all the rows): maddubs(act_u8, w_s8) needs no
+// per-token sign, and sum(w * q) = sum(w * (q + 128)) - 128 * sum(w) with the second sum made once per weight block (corr), so the
+// integer sums are exactly ggml's, lane for lane (no int16 overflow: |w| <= 62 for these grids, 255 * 62 * 2 < 32767).
+// A token costs, per 32 values, one maddubs, one madd and one add (the single-token form has a sign on top).
+constexpr int kWideTokens = 64;
+
+STRATA_ROWS_FN inline void wide_prep(const block_q8_K* y, int nb, uint8_t* u) {
+    const __m256i flip = _mm256_set1_epi8((char) 0x80);
+    for (int i = 0; i < nb; ++i)
+        for (int c = 0; c < 8; ++c)
+            _mm256_storeu_si256((__m256i*) (u + (size_t) i * 256 + 32 * c),
+                                _mm256_xor_si256(_mm256_loadu_si256((const __m256i*) (y[i].qs + 32 * c)), flip));
 }
 
 template <int TY> STRATA_ROWS_FN
+void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const* act, int T, float* const* out, int r0, int r1) {
+    // tokens in tiles of kTile and rows in groups of kRows: a tile's activation slice of one block (kTile * 256 B) and the group's
+    // accumulators (kRows * kTile * 32 B) stay in L1 while the group's rows go past, the tile's whole activation in L2
+#ifndef STRATA_WIDE_TILE
+#define STRATA_WIDE_TILE 16
+#endif
+#ifndef STRATA_WIDE_ROWS
+#define STRATA_WIDE_ROWS 8
+#endif
+    constexpr int kTile = STRATA_WIDE_TILE, kRows = STRATA_WIDE_ROWS;
+    const int nb = n / QK_K;
+    static thread_local std::vector<uint8_t> ubuf;
+    static thread_local std::vector<float> accb;
+    ubuf.resize((size_t) (T < kTile ? T : kTile) * n);
+    accb.resize((size_t) kRows * kTile * 8);
+    const __m256i one8 = _mm256_set1_epi8(1);
+    const int pf = prefetch_distance();
+    for (int t0 = 0; t0 < T; t0 += kTile) {
+        const int tt = T - t0 < kTile ? T - t0 : kTile;
+        const block_q8_K* y[kTile];
+        for (int t = 0; t < tt; ++t) {
+            y[t] = (const block_q8_K*) act[t0 + t];
+            wide_prep(y[t], nb, ubuf.data() + (size_t) t * n);
+        }
+        for (int rg = r0; rg < r1; rg += kRows) {
+            const int rr = r1 - rg < kRows ? r1 - rg : kRows;
+            for (int k = 0; k < rr * tt; ++k) _mm256_storeu_ps(&accb[(size_t) k * 8], _mm256_setzero_ps());
+            for (int i = 0; i < nb; ++i) {
+                for (int q = 0; q < rr; ++q) {
+                    const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * Fmt32<TY>::bytes;
+                    rows_ahead(blk, pf);
+                    __m256i w8[8], sc[8];
+                    __m256i corr = _mm256_setzero_si256();
+                    for (int j = 0; j < 4; ++j)
+                        for (int half = 0; half < 2; ++half) {
+                            __m256i g, sgn, s;
+                            Fmt32<TY>::decode(blk, j, half, g, sgn, s);
+                            const int k = 2 * j + half;
+                            w8[k] = _mm256_sign_epi8(g, sgn);
+                            sc[k] = s;
+                            corr = madd_add(corr, _mm256_maddubs_epi16(one8, w8[k]), s);
+                        }
+                    const __m256i corr128 = _mm256_slli_epi32(corr, 7);
+                    const float dx = h2f(u16(blk));
+                    float* accq = &accb[(size_t) q * tt * 8];
+                    for (int t = 0; t < tt; ++t) {
+                        const uint8_t* u = ubuf.data() + (size_t) t * n + (size_t) i * 256;
+                        __m256i a0 = _mm256_setzero_si256(), a1 = _mm256_setzero_si256();
+                        for (int k = 0; k < 8; k += 2) {
+                            a0 = madd_add(a0, _mm256_maddubs_epi16(_mm256_loadu_si256((const __m256i*) (u + 32 * k)), w8[k]), sc[k]);
+                            a1 = madd_add(a1, _mm256_maddubs_epi16(_mm256_loadu_si256((const __m256i*) (u + 32 * k + 32)), w8[k + 1]), sc[k + 1]);
+                        }
+                        const __m256i s = _mm256_sub_epi32(_mm256_add_epi32(a0, a1), corr128);
+                        float* ap = accq + (size_t) t * 8;
+                        _mm256_storeu_ps(ap, _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(s), _mm256_loadu_ps(ap)));
+                    }
+                }
+            }
+            for (int q = 0; q < rr; ++q)
+                for (int t = 0; t < tt; ++t)
+                    out[t0 + t][rg + q] = Fmt32<TY>::K * ggml_hsum8(_mm256_loadu_ps(&accb[((size_t) q * tt + t) * 8]));
+        }
+    }
+}
+
+// IQ1_M the same way: its grid bytes are already signed (-1, 0, 1), and the delta dot keeps the pair sums of the activation
+// (maddubs(1, q8), made once per tile).  Two float accumulators a (row, token), ggml's sum d*sumi1 and sum d*sumi2.
+STRATA_ROWS_FN
+void dot_rows_wide_iq1m(const uint8_t* w, size_t row_bytes, int n, const void* const* act, int T, float* const* out, int r0, int r1) {
+    constexpr int kTile = STRATA_WIDE_TILE, kRows = STRATA_WIDE_ROWS;
+    const int nb = n / QK_K;
+    static thread_local std::vector<uint8_t> ubuf;
+    static thread_local std::vector<int16_t> pbuf;
+    static thread_local std::vector<float> accb;
+    ubuf.resize((size_t) (T < kTile ? T : kTile) * n);
+    pbuf.resize((size_t) (T < kTile ? T : kTile) * n);
+    accb.resize((size_t) kRows * kTile * 16);
+    const __m256i one8 = _mm256_set1_epi8(1);
+    const __m256i mask = _mm256_set1_epi16(0x7);
+    const __m256i mone = _mm256_set1_epi16(1);
+    const __m256i mtwo8 = _mm256_set1_epi8(2);
+    const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);
+    const __m256i nib_shift = _mm256_setr_epi32(0, 4, 8, 12, 16, 20, 24, 28);
+    const __m256i m7 = _mm256_set1_epi32(7), m8 = _mm256_set1_epi32(8);
+    const int pf = prefetch_distance();
+    for (int t0 = 0; t0 < T; t0 += kTile) {
+        const int tt = T - t0 < kTile ? T - t0 : kTile;
+        const block_q8_K* y[kTile];
+        for (int t = 0; t < tt; ++t) {
+            y[t] = (const block_q8_K*) act[t0 + t];
+            wide_prep(y[t], nb, ubuf.data() + (size_t) t * n);
+            iq1m_pre_build(y[t], nb, pbuf.data() + (size_t) t * n);
+        }
+        for (int rg = r0; rg < r1; rg += kRows) {
+            const int rr = r1 - rg < kRows ? r1 - rg : kRows;
+            for (int k = 0; k < rr * tt * 2; ++k) _mm256_storeu_ps(&accb[(size_t) k * 8], _mm256_setzero_ps());
+            for (int i = 0; i < nb; ++i) {
+                for (int q = 0; q < rr; ++q) {
+                    const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * 56;
+                    rows_ahead(blk, pf);
+                    const uint8_t* qs = blk;
+                    const uint8_t* qh = blk + 32;
+                    const uint8_t* scb = blk + 48;
+                    const uint16_t sc0 = u16(scb), sc1 = u16(scb + 2), sc2 = u16(scb + 4), sc3 = u16(scb + 6);
+                    const uint16_t d16 = (uint16_t) ((sc0 >> 12) | ((sc1 >> 8) & 0x00f0) | ((sc2 >> 4) & 0x0f00) | (sc3 & 0xf000));
+                    __m256i scales = _mm256_set1_epi64x((long long) u64(scb));
+                    scales = _mm256_srlv_epi64(scales, scales_shift);
+                    scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);
+                    __m256i sidx1 = _mm256_set1_epi16(0x0100);
+                    __m256i sidx2 = _mm256_add_epi8(sidx1, _mm256_set1_epi8(8));
+                    __m256i g1[4], g2[4], sa[4], sb[4], ssa[4], ssb[4];
+                    __m256i corr = _mm256_setzero_si256();
+                    for (int it = 0; it < 4; ++it) {
+                        const __m256i v8 = _mm256_srlv_epi32(_mm256_set1_epi32((int) u32(qh + 4 * it)), nib_shift);
+                        const __m256i idx = _mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) (qs + 8 * it))),
+                                                            _mm256_slli_epi32(_mm256_and_si256(v8, m7), 8));
+                        alignas(32) int32_t ix[8];
+                        _mm256_store_si256((__m256i*) ix, idx);
+                        g1[it] = _mm256_set_epi64x((long long) iq1s_grid[ix[3]], (long long) iq1s_grid[ix[2]],
+                                                   (long long) iq1s_grid[ix[1]], (long long) iq1s_grid[ix[0]]);
+                        g2[it] = _mm256_set_epi64x((long long) iq1s_grid[ix[7]], (long long) iq1s_grid[ix[6]],
+                                                   (long long) iq1s_grid[ix[5]], (long long) iq1s_grid[ix[4]]);
+                        const __m256i neg = _mm256_cmpeq_epi32(_mm256_and_si256(v8, m8), m8);
+                        const __m256i n1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(neg));
+                        const __m256i n2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(neg, 1));
+                        sa[it] = _mm256_shuffle_epi8(scales, sidx1);
+                        sb[it] = _mm256_shuffle_epi8(scales, sidx2);
+                        ssa[it] = _mm256_sign_epi16(sa[it], _mm256_or_si256(n1, mone));
+                        ssb[it] = _mm256_sign_epi16(sb[it], _mm256_or_si256(n2, mone));
+                        sidx1 = _mm256_add_epi8(sidx1, mtwo8);
+                        sidx2 = _mm256_add_epi8(sidx2, mtwo8);
+                        corr = madd_add(madd_add(corr, _mm256_maddubs_epi16(one8, g1[it]), sa[it]), _mm256_maddubs_epi16(one8, g2[it]), sb[it]);
+                    }
+                    const __m256i corr128 = _mm256_slli_epi32(corr, 7);
+                    const float dx = h2f(d16);
+                    float* accq = &accb[(size_t) q * tt * 16];
+                    for (int t = 0; t < tt; ++t) {
+                        const uint8_t* u = ubuf.data() + (size_t) t * n + (size_t) i * 256;
+                        const int16_t* pp = pbuf.data() + (size_t) t * n + (size_t) i * 128;
+                        __m256i a0 = _mm256_setzero_si256(), a1 = _mm256_setzero_si256();
+                        __m256i b0 = _mm256_setzero_si256(), b1 = _mm256_setzero_si256();
+                        for (int it = 0; it < 4; ++it) {
+                            a0 = madd_add(a0, _mm256_maddubs_epi16(_mm256_loadu_si256((const __m256i*) (u + 64 * it)), g1[it]), sa[it]);
+                            a1 = madd_add(a1, _mm256_maddubs_epi16(_mm256_loadu_si256((const __m256i*) (u + 64 * it + 32)), g2[it]), sb[it]);
+                            b0 = madd_add(b0, _mm256_loadu_si256((const __m256i*) (pp + 32 * it)), ssa[it]);
+                            b1 = madd_add(b1, _mm256_loadu_si256((const __m256i*) (pp + 32 * it + 16)), ssb[it]);
+                        }
+                        const __m256i s1 = _mm256_sub_epi32(_mm256_add_epi32(a0, a1), corr128);
+                        const __m256i s2 = _mm256_add_epi32(b0, b1);
+                        const __m256 d = _mm256_set1_ps(y[t][i].d * dx);
+                        float* ap = accq + (size_t) t * 16;
+                        _mm256_storeu_ps(ap, _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(s1), _mm256_loadu_ps(ap)));
+                        _mm256_storeu_ps(ap + 8, _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(s2), _mm256_loadu_ps(ap + 8)));
+                    }
+                }
+            }
+            for (int q = 0; q < rr; ++q)
+                for (int t = 0; t < tt; ++t) {
+                    const float* ap = &accb[((size_t) q * tt + t) * 16];
+                    out[t0 + t][rg + q] = ggml_hsum8(_mm256_loadu_ps(ap)) + 0.125f * ggml_hsum8(_mm256_loadu_ps(ap + 8));
+                }
+        }
+    }
+}
+
+template <int TY, bool EX> STRATA_ROWS_FN
+void gu_rows_nt(int nt, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act,
+                float* const* ff, int r0, int r1) {
+    switch (nt) {
+        case 1: gu_rows<TY, 1, EX>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 2: gu_rows<TY, 2, EX>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 3: gu_rows<TY, 3, EX>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 4: gu_rows<TY, 4, EX>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 5: gu_rows<TY, 5, EX>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 6: gu_rows<TY, 6, EX>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 7: gu_rows<TY, 7, EX>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        default: gu_rows<TY, 8, EX>(blob, gu_row, up_off, n, act, ff, r0, r1); break;
+    }
+}
+
+template <int TY, bool EX> STRATA_ROWS_FN
 void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out, int r0,
                  int r1) {
+    if constexpr (EX) {
+        if (nt >= 7) {   // (a prompt chunk's tokens: one decode of a block for all of them)
+            if constexpr (TY == 29) dot_rows_wide_iq1m(w, row_bytes, n, act, nt, out, r0, r1);
+            else dot_rows_wide<TY>(w, row_bytes, n, act, nt, out, r0, r1);
+            return;
+        }
+    }
     switch (nt) {
-        case 1: dot_rows<TY, 1>(w, row_bytes, n, act, out, r0, r1); break;
-        case 2: dot_rows<TY, 2>(w, row_bytes, n, act, out, r0, r1); break;
-        case 3: dot_rows<TY, 3>(w, row_bytes, n, act, out, r0, r1); break;
-        case 4: dot_rows<TY, 4>(w, row_bytes, n, act, out, r0, r1); break;
+        case 1: dot_rows<TY, 1, EX>(w, row_bytes, n, act, out, r0, r1); break;
+        case 2: dot_rows<TY, 2, EX>(w, row_bytes, n, act, out, r0, r1); break;
+        case 3: dot_rows<TY, 3, EX>(w, row_bytes, n, act, out, r0, r1); break;
+        case 4: dot_rows<TY, 4, EX>(w, row_bytes, n, act, out, r0, r1); break;
+        case 5: if constexpr (EX) { dot_rows<TY, 5, EX>(w, row_bytes, n, act, out, r0, r1); break; }   // (the DeepSeek lane's verify
+        [[fallthrough]];                                                                              //  rounds: one decode of 5 or 6)
+        case 6: if constexpr (EX) { dot_rows<TY, 6, EX>(w, row_bytes, n, act, out, r0, r1); break; }
+        [[fallthrough]];
         default: for (int t0 = 0; t0 < nt; t0 += 4) {
             const int k = nt - t0 < 4 ? nt - t0 : 4;
-            dot_rows_nt<TY>(k, w, row_bytes, n, act + t0, out + t0, r0, r1);
+            dot_rows_nt<TY, EX>(k, w, row_bytes, n, act + t0, out + t0, r0, r1);
         }
     }
 }
 
 // The decode of each format for a variant: the gathered one (kIq256Gather) for IQ3_XXS, IQ3_S and IQ2_S.
-template <bool G> STRATA_ROWS_FN
+template <bool G, bool EX = false> STRATA_ROWS_FN
 void gu_type(int type, int nt, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act,
              float* const* ff, int r0, int r1) {
     switch (type) {
-        case 16: gu_rows_nt<16>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 17: gu_rows_nt<17>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 18: gu_rows_nt<G ? 118 : 18>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 21: gu_rows_nt<G ? 121 : 21>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 22: gu_rows_nt<G ? 122 : 22>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
-        case 23: gu_rows_nt<23>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 16: gu_rows_nt<16, EX>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 17: gu_rows_nt<17, EX>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 18: gu_rows_nt<G ? 118 : 18, EX>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 21: gu_rows_nt<G ? 121 : 21, EX>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 22: gu_rows_nt<G ? 122 : 22, EX>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 23: gu_rows_nt<23, EX>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
+        case 29: gu_rows_nt<29, EX>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;
         default: break;
     }
 }
 
-template <bool G> STRATA_ROWS_FN
+template <bool G, bool EX = false> STRATA_ROWS_FN
 void dot_type(int type, int nt, const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out,
               int r0, int r1) {
     switch (type) {
-        case 16: dot_rows_nt<16>(nt, w, row_bytes, n, act, out, r0, r1); break;
-        case 17: dot_rows_nt<17>(nt, w, row_bytes, n, act, out, r0, r1); break;
-        case 18: dot_rows_nt<G ? 118 : 18>(nt, w, row_bytes, n, act, out, r0, r1); break;
-        case 21: dot_rows_nt<G ? 121 : 21>(nt, w, row_bytes, n, act, out, r0, r1); break;
-        case 22: dot_rows_nt<G ? 122 : 22>(nt, w, row_bytes, n, act, out, r0, r1); break;
-        case 23: dot_rows_nt<23>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 16: dot_rows_nt<16, EX>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 17: dot_rows_nt<17, EX>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 18: dot_rows_nt<G ? 118 : 18, EX>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 21: dot_rows_nt<G ? 121 : 21, EX>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 22: dot_rows_nt<G ? 122 : 22, EX>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 23: dot_rows_nt<23, EX>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 29: dot_rows_nt<29, EX>(nt, w, row_bytes, n, act, out, r0, r1); break;
         default: break;
     }
 }

src/kernels/cpu/iq_avx512.cpp

@@ -15,10 +15,16 @@
 #include "ggml-common.h"
 
 #include <immintrin.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <cpuid.h>
+#endif
 
 #include <cmath>
 #include <cstdlib>
 #include <cstring>
+#include <vector>
 
 namespace strata::kernels::cpu {
 namespace {
@@ -240,6 +246,297 @@ void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void*
     }
 }
 
+
+// The sign masks of IQ2_XXS / IQ2_XS / IQ3_XXS come from a 7-bit index whose 8th bit is the parity of the other seven (ksigns_iq2xs[i] =
+// i | parity(i) << 7).  The table needs a load and a few shifts and ors a group of eight values; popcount does it for all eight groups
+// of 64 values in one vector: measured on Zen 5, one-token IQ2_XXS rows 33 -> 45-50 GB/s (16 threads, DRAM-sized), cache-resident 16.8 -> 13.4 ns a block.
+// (AVX512_VPOPCNTDQ + VL: Zen 4, Zen 5, Ice Lake and later; iq512x_supported() asks for it.)
+inline __mmask64 sign_mask_from_idx(__m256i si) {
+    const __m256i par = _mm256_slli_epi32(_mm256_and_si256(_mm256_popcnt_epi32(si), _mm256_set1_epi32(1)), 7);
+    return _cvtu64_mask64((uint64_t) _mm_cvtsi128_si64(_mm256_cvtepi32_epi8(_mm256_or_si256(si, par))));
+}
+inline __m256i sign_idx_xxs(uint32_t lo4, uint32_t hi4) {   // four 7-bit fields of each of two words -> eight dwords
+    const __m256i w = _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_set1_epi32((int) lo4)), _mm_set1_epi32((int) hi4), 1);
+    return _mm256_and_si256(_mm256_srlv_epi32(w, _mm256_setr_epi32(0, 7, 14, 21, 0, 7, 14, 21)), _mm256_set1_epi32(127));
+}
+
+template <int TY> struct FmtX : Fmt<TY> {};
+template <> struct FmtX<16> : Fmt<16> {
+    static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
+        const uint8_t* q = b + 2 + 16 * j;
+        const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);
+        g = _mm512_set_epi64((long long) iq2xxs_grid[b0 >> 24], (long long) iq2xxs_grid[(b0 >> 16) & 255],
+                             (long long) iq2xxs_grid[(b0 >> 8) & 255], (long long) iq2xxs_grid[b0 & 255],
+                             (long long) iq2xxs_grid[a0 >> 24], (long long) iq2xxs_grid[(a0 >> 16) & 255],
+                             (long long) iq2xxs_grid[(a0 >> 8) & 255], (long long) iq2xxs_grid[a0 & 255]);
+        m = sign_mask_from_idx(sign_idx_xxs(a1, b1));
+        const int sa = 2 * (int) (a1 >> 28) + 1, sb = 2 * (int) (b1 >> 28) + 1;
+        sc = scales4(sa, sa, sb, sb);
+    }
+};
+template <> struct FmtX<17> : Fmt<17> {
+    static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
+        const uint8_t* q = b + 2 + 16 * j;
+        uint16_t v[8];
+        std::memcpy(v, q, 16);
+        g = _mm512_set_epi64((long long) iq2xs_grid[v[7] & 511], (long long) iq2xs_grid[v[6] & 511],
+                             (long long) iq2xs_grid[v[5] & 511], (long long) iq2xs_grid[v[4] & 511],
+                             (long long) iq2xs_grid[v[3] & 511], (long long) iq2xs_grid[v[2] & 511],
+                             (long long) iq2xs_grid[v[1] & 511], (long long) iq2xs_grid[v[0] & 511]);
+        m = sign_mask_from_idx(_mm256_srli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*) q)), 9));
+        const uint8_t s0 = b[66 + 2 * j], s1 = b[66 + 2 * j + 1];
+        sc = scales4(2 * (s0 & 15) + 1, 2 * (s0 >> 4) + 1, 2 * (s1 & 15) + 1, 2 * (s1 >> 4) + 1);
+    }
+};
+template <> struct FmtX<18> : Fmt<18> {
+    static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
+        const uint8_t* q = b + 2 + 16 * j;
+        g = _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],
+                             (int) iq3xxs_grid[q[11]], (int) iq3xxs_grid[q[10]], (int) iq3xxs_grid[q[9]], (int) iq3xxs_grid[q[8]],
+                             (int) iq3xxs_grid[q[7]], (int) iq3xxs_grid[q[6]], (int) iq3xxs_grid[q[5]], (int) iq3xxs_grid[q[4]],
+                             (int) iq3xxs_grid[q[3]], (int) iq3xxs_grid[q[2]], (int) iq3xxs_grid[q[1]], (int) iq3xxs_grid[q[0]]);
+        const uint32_t a = u32(b + 2 + 64 + 8 * j), c = u32(b + 2 + 64 + 8 * j + 4);
+        m = sign_mask_from_idx(sign_idx_xxs(a, c));
+        const int sa = 2 * (int) (a >> 28) + 1, sb = 2 * (int) (c >> 28) + 1;
+        sc = scales4(sa, sa, sb, sb);
+    }
+};
+
+// ---- the DeepSeek CPU lane's exact rows (iq512x_rows): the same 64-value decode, but summed as ggml's AVX2 dots sum, so a
+// token's row is ggml-cpu's vec_dot bit for bit at any number of tokens: the integer sums of a block fold to ggml's eight lanes
+// (low half = the even 32-value chunk, high half = the odd one), the block scale is f16(d) * y.d, eight float lanes accumulate
+// across blocks, and K * ggml's hsum comes last.  vpdpwssd is the madd and the add in one.
+inline __m256i fold256(__m512i a) { return _mm256_add_epi32(_mm512_castsi512_si256(a), _mm512_extracti64x4_epi64(a, 1)); }
+
+inline float hsum8_ggml(__m256 x) {
+    __m128 s = _mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x));
+    s = _mm_add_ps(s, _mm_movehl_ps(s, s));
+    s = _mm_add_ss(s, _mm_movehdup_ps(s));
+    return _mm_cvtss_f32(s);
+}
+
+template <int TY, int NT>
+inline void row_dot_x(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res) {
+    __m256 accf[NT];
+    for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
+    const __m512i zero = _mm512_setzero_si512();
+    for (int i = 0; i < nblocks; ++i) {
+        const uint8_t* blk = row + (size_t) i * FmtX<TY>::bytes;
+        if (prefetch_ahead > 0) {
+            const uint8_t* ahead = blk + (size_t) prefetch_ahead;
+            _mm_prefetch((const char*) ahead, _MM_HINT_T0);
+            _mm_prefetch((const char*) ahead + 64, _MM_HINT_T0);
+        }
+        __m512i acci[NT];
+        for (int t = 0; t < NT; ++t) acci[t] = _mm512_setzero_si512();
+        for (int j = 0; j < 4; ++j) {
+            __m512i g, sc;
+            __mmask64 m;
+            FmtX<TY>::decode(blk, j, g, m, sc);
+            for (int t = 0; t < NT; ++t) {
+                const __m512i yv = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * j));
+                const __m512i ys = _mm512_mask_sub_epi8(yv, m, zero, yv);
+                acci[t] = _mm512_dpwssd_epi32(acci[t], _mm512_maddubs_epi16(g, ys), sc);
+            }
+        }
+        const float dx = h2f(u16(blk));
+        for (int t = 0; t < NT; ++t)
+            accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(fold256(acci[t])), accf[t]);
+    }
+    for (int t = 0; t < NT; ++t) res[t] = FmtX<TY>::K * hsum8_ggml(accf[t]);
+}
+
+template <int TY, int NT>
+void dot_rows_x(const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out, int r0, int r1) {
+    const block_q8_K* y[NT];
+    for (int t = 0; t < NT; ++t) y[t] = (const block_q8_K*) act[t];
+    float res[NT];
+    for (int r = r0; r < r1; ++r) {
+        row_dot_x<TY, NT>(w + (size_t) r * row_bytes, n / QK_K, y, res);
+        for (int t = 0; t < NT; ++t) out[t][r] = res[t];
+    }
+}
+
+// many tokens an expert (the prompt): signed int8 weights decoded once per block, activations as u8 (q8 + 128, once a call),
+// sum(w * q) = sum(w * (q + 128)) - 128 * sum(w) with the second sum once a weight block; tokens in tiles of 16 and rows in
+// groups of 8 so the tile's slice of a block and the group's accumulators stay in L1 (the AVX2 form's layout)
+template <int TY>
+void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const* act, int T, float* const* out, int r0, int r1) {
+    constexpr int kTile = 16, kRows = 8;
+    const int nb = n / QK_K;
+    static thread_local std::vector<uint8_t> ubuf;
+    static thread_local std::vector<float> accb;
+    ubuf.resize((size_t) (T < kTile ? T : kTile) * n);
+    accb.resize((size_t) kRows * kTile * 8);
+    const __m512i zero = _mm512_setzero_si512(), one8 = _mm512_set1_epi8(1), flip = _mm512_set1_epi8((char) 0x80);
+    for (int t0 = 0; t0 < T; t0 += kTile) {
+        const int tt = T - t0 < kTile ? T - t0 : kTile;
+        const block_q8_K* y[kTile];
+        for (int t = 0; t < tt; ++t) {
+            y[t] = (const block_q8_K*) act[t0 + t];
+            uint8_t* u = ubuf.data() + (size_t) t * n;
+            for (int i = 0; i < nb; ++i)
+                for (int j = 0; j < 4; ++j)
+                    _mm512_storeu_si512((void*) (u + (size_t) i * 256 + 64 * j),
+                                        _mm512_xor_si512(_mm512_loadu_si512((const void*) (y[t][i].qs + 64 * j)), flip));
+        }
+        for (int rg = r0; rg < r1; rg += kRows) {
+            const int rr = r1 - rg < kRows ? r1 - rg : kRows;
+            for (int k = 0; k < rr * tt; ++k) _mm256_storeu_ps(&accb[(size_t) k * 8], _mm256_setzero_ps());
+            for (int i = 0; i < nb; ++i)
+                for (int q = 0; q < rr; ++q) {
+                    const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * FmtX<TY>::bytes;
+                    if (prefetch_ahead > 0) {
+                        _mm_prefetch((const char*) blk + prefetch_ahead, _MM_HINT_T0);
+                        _mm_prefetch((const char*) blk + prefetch_ahead + 64, _MM_HINT_T0);
+                    }
+                    __m512i w8[4], sc[4];
+                    __m512i corr = zero;
+                    for (int j = 0; j < 4; ++j) {
+                        __m512i g, s;
+                        __mmask64 m;
+                        FmtX<TY>::decode(blk, j, g, m, s);
+                        w8[j] = _mm512_mask_sub_epi8(g, m, zero, g);
+                        sc[j] = s;
+                        corr = _mm512_dpwssd_epi32(corr, _mm512_maddubs_epi16(one8, w8[j]), s);
+                    }
+                    const __m256i corr128 = _mm256_slli_epi32(fold256(corr), 7);
+                    const float dx = h2f(u16(blk));
+                    float* accq = &accb[(size_t) q * tt * 8];
+                    for (int t = 0; t < tt; ++t) {
+                        const uint8_t* u = ubuf.data() + (size_t) t * n + (size_t) i * 256;
+                        __m512i a = zero;
+                        for (int j = 0; j < 4; ++j)
+                            a = _mm512_dpwssd_epi32(a, _mm512_maddubs_epi16(_mm512_loadu_si512((const void*) (u + 64 * j)), w8[j]), sc[j]);
+                        const __m256i s = _mm256_sub_epi32(fold256(a), corr128);
+                        float* ap = accq + (size_t) t * 8;
+                        _mm256_storeu_ps(ap, _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(s), _mm256_loadu_ps(ap)));
+                    }
+                }
+            for (int q = 0; q < rr; ++q)
+                for (int t = 0; t < tt; ++t)
+                    out[t0 + t][rg + q] = FmtX<TY>::K * hsum8_ggml(_mm256_loadu_ps(&accb[((size_t) q * tt + t) * 8]));
+        }
+    }
+}
+
+template <int TY>
+void dot_rows_x_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out, int r0, int r1) {
+    switch (nt) {
+        case 1: dot_rows_x<TY, 1>(w, row_bytes, n, act, out, r0, r1); break;
+        case 2: dot_rows_x<TY, 2>(w, row_bytes, n, act, out, r0, r1); break;
+        case 3: dot_rows_x<TY, 3>(w, row_bytes, n, act, out, r0, r1); break;
+        case 4: dot_rows_x<TY, 4>(w, row_bytes, n, act, out, r0, r1); break;
+        case 5: dot_rows_x<TY, 5>(w, row_bytes, n, act, out, r0, r1); break;
+        case 6: dot_rows_x<TY, 6>(w, row_bytes, n, act, out, r0, r1); break;
+        default: dot_rows_wide<TY>(w, row_bytes, n, act, nt, out, r0, r1); break;
+    }
+}
+
+
+// ---- IQ1_M (29), AVX-512, 1..6 tokens: ggml's dot lane for lane.  The grid indices come out of two pdep (as ggml's BMI2 path), the
+// +-1 delta signs out of one pext into an 8-bit mask a 64-value step; the signed grid bytes are split into |g| and a negative mask;
+// the delta dot is the pair sums of the activation (made once a call) times +-scale.  d is f16 in the top nibbles of the scales.
+inline void iq1m_pre(const block_q8_K* y, int nb, int16_t* p) {
+    const __m256i one8 = _mm256_set1_epi8(1);
+    for (int i = 0; i < nb; ++i)
+        for (int c = 0; c < 8; ++c)
+            _mm256_storeu_si256((__m256i*) (p + ((size_t) i * 8 + c) * 16),
+                                _mm256_maddubs_epi16(one8, _mm256_loadu_si256((const __m256i*) (y[i].qs + 32 * c))));
+}
+
+template <int NT>
+void rows_m(const uint8_t* w, size_t rb, int n, const block_q8_K* const* y, const int16_t* const* P, float* const* out, int r0, int r1) {
+    const int nb = n / QK_K;
+    const __m256i mask = _mm256_set1_epi16(0x7), mone = _mm256_set1_epi16(1), mtwo8 = _mm256_set1_epi8(2);
+    const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);
+    const __m512i zero = _mm512_setzero_si512();
+    for (int r = r0; r < r1; ++r) {
+        __m256 a1[NT], a2[NT];
+        for (int t = 0; t < NT; ++t) a1[t] = a2[t] = _mm256_setzero_ps();
+        const uint8_t* row = w + (size_t) r * rb;
+        for (int i = 0; i < nb; ++i) {
+            const uint8_t* blk = row + (size_t) i * 56;
+            const uint8_t* qs = blk;
+            const uint8_t* qh = blk + 32;
+            const uint8_t* scb = blk + 48;
+            if (prefetch_ahead > 0) {
+                _mm_prefetch((const char*) blk + prefetch_ahead, _MM_HINT_T0);
+                _mm_prefetch((const char*) blk + prefetch_ahead + 64, _MM_HINT_T0);
+            }
+            const uint16_t sc0 = u16(scb), sc1 = u16(scb + 2), sc2 = u16(scb + 4), sc3 = u16(scb + 6);
+            const uint16_t d16 = (uint16_t) ((sc0 >> 12) | ((sc1 >> 8) & 0x00f0) | ((sc2 >> 4) & 0x0f00) | (sc3 & 0xf000));
+            __m256i scales = _mm256_set1_epi64x((long long) u64(scb));
+            scales = _mm256_srlv_epi64(scales, scales_shift);
+            scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);
+            __m256i sidx1 = _mm256_set1_epi16(0x0100);
+            __m256i sidx2 = _mm256_add_epi8(sidx1, _mm256_set1_epi8(8));
+            __m512i s1[NT], s2[NT];
+            for (int t = 0; t < NT; ++t) s1[t] = s2[t] = zero;
+            for (int it = 0; it < 4; ++it) {
+                const uint32_t qh32 = u32(qh + 4 * it);
+                const uint64_t p1 = _pdep_u64(u32(qs + 8 * it), 0x00ff00ff00ff00ffULL) | _pdep_u64(qh32 & 0x7777, 0x0f000f000f000f00ULL);
+                const uint64_t p2 = _pdep_u64(u32(qs + 8 * it + 4), 0x00ff00ff00ff00ffULL) | _pdep_u64((qh32 >> 16) & 0x7777, 0x0f000f000f000f00ULL);
+                uint16_t ix[8];
+                std::memcpy(ix, &p1, 8);
+                std::memcpy(ix + 4, &p2, 8);
+                const __m512i g = _mm512_set_epi64((long long) iq1s_grid[ix[7]], (long long) iq1s_grid[ix[6]], (long long) iq1s_grid[ix[5]],
+                                                   (long long) iq1s_grid[ix[4]], (long long) iq1s_grid[ix[3]], (long long) iq1s_grid[ix[2]],
+                                                   (long long) iq1s_grid[ix[1]], (long long) iq1s_grid[ix[0]]);
+                const __m512i negv = _mm512_maskz_set1_epi64((__mmask8) _pext_u32(qh32, 0x88888888u), -1);
+                const __m256i sca = _mm256_shuffle_epi8(scales, sidx1);
+                const __m256i scb2 = _mm256_shuffle_epi8(scales, sidx2);
+                sidx1 = _mm256_add_epi8(sidx1, mtwo8);
+                sidx2 = _mm256_add_epi8(sidx2, mtwo8);
+                const __m512i sc = _mm512_inserti64x4(_mm512_castsi256_si512(sca), scb2, 1);
+                const __m512i ssc = _mm512_sub_epi16(_mm512_xor_si512(sc, negv), negv);
+                const __m512i ag = _mm512_abs_epi8(g);
+                const __mmask64 gneg = _mm512_movepi8_mask(g);
+                for (int t = 0; t < NT; ++t) {
+                    const __m512i q8 = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * it));
+                    const __m512i pp = _mm512_loadu_si512((const void*) (P[t] + ((size_t) i * 8 + 2 * it) * 16));
+                    const __m512i ys = _mm512_mask_sub_epi8(q8, gneg, zero, q8);
+                    s1[t] = _mm512_dpwssd_epi32(s1[t], _mm512_maddubs_epi16(ag, ys), sc);
+                    s2[t] = _mm512_dpwssd_epi32(s2[t], pp, ssc);
+                }
+            }
+            const float dx = h2f(d16);
+            for (int t = 0; t < NT; ++t) {
+                const __m256 d = _mm256_set1_ps(y[t][i].d * dx);
+                a1[t] = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(fold256(s1[t])), a1[t]);
+                a2[t] = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(fold256(s2[t])), a2[t]);
+            }
+        }
+        for (int t = 0; t < NT; ++t) out[t][r] = hsum8_ggml(a1[t]) + 0.125f * hsum8_ggml(a2[t]);
+    }
+}
+
+void rows_m_nt(int nt, const uint8_t* w, size_t rb, int n, const void* const* act, float* const* out, int r0, int r1) {
+    const int nb = n / QK_K;
+    static thread_local std::vector<int16_t> pre;
+    for (int t0 = 0; t0 < nt; t0 += 6) {
+        const int k = nt - t0 < 6 ? nt - t0 : 6;
+        const block_q8_K* y[6];
+        const int16_t* P[6];
+        float* o[6];
+        pre.resize((size_t) k * n);
+        for (int t = 0; t < k; ++t) {
+            y[t] = (const block_q8_K*) act[t0 + t];
+            iq1m_pre(y[t], nb, pre.data() + (size_t) t * n);
+            P[t] = pre.data() + (size_t) t * n;
+            o[t] = out[t0 + t];
+        }
+        switch (k) {
+            case 1: rows_m<1>(w, rb, n, y, P, o, r0, r1); break;
+            case 2: rows_m<2>(w, rb, n, y, P, o, r0, r1); break;
+            case 3: rows_m<3>(w, rb, n, y, P, o, r0, r1); break;
+            case 4: rows_m<4>(w, rb, n, y, P, o, r0, r1); break;
+            case 5: rows_m<5>(w, rb, n, y, P, o, r0, r1); break;
+            default: rows_m<6>(w, rb, n, y, P, o, r0, r1); break;
+        }
+    }
+}
+
 }  // namespace
 
 void iq512_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, int nt,
@@ -266,4 +563,36 @@ void iq512_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void*
     }
 }
 
+static bool has_vpopcnt() {
+    static const bool ok = [] {
+#if defined(_MSC_VER)
+        int r[4];
+        __cpuidex(r, 7, 0);
+        return ((r[2] >> 14) & 1) != 0;   // AVX512_VPOPCNTDQ
+#else
+        unsigned a, b, c, d;
+        __cpuid_count(7, 0, a, b, c, d);
+        return ((c >> 14) & 1u) != 0;
+#endif
+    }();
+    return ok;
+}
+
+bool iq512x_supported(int type) noexcept {
+    return has_vpopcnt() && (type == 16 || type == 17 || type == 18 || type == 21 || type == 22 || type == 29);
+}
+
+void iq512x_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out,
+                 int r0, int r1) {
+    switch (type) {
+        case 16: dot_rows_x_nt<16>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 17: dot_rows_x_nt<17>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 18: dot_rows_x_nt<18>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 21: dot_rows_x_nt<21>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 22: dot_rows_x_nt<22>(nt, w, row_bytes, n, act, out, r0, r1); break;
+        case 29: rows_m_nt(nt, w, row_bytes, n, act, out, r0, r1); break;
+        default: break;
+    }
+}
+
 }  // namespace strata::kernels::cpu

src/kernels/cpu/native_expert.cpp

@@ -16,11 +16,14 @@
 #include "ggml.h"
 #include "ggml-cpu.h"
 
+#include <algorithm>
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <map>
 #include <mutex>
+#include <string>
 #include <vector>
 
 namespace strata::kernels::cpu {
@@ -129,6 +132,88 @@ bool q8k_avx1(int type) {
 }
 }  // namespace
 
+// STRATA_ACT_DUMP=<prefix> (research, off by default): the second moments of the vectors that enter the CPU experts, in the
+// basis the experts use (after the Hadamard rotation of a folded layer): per layer the gate/up input (the token's hidden
+// state, once per token) and the down input (the SwiGLU output, once per routed expert).  STRATA_ACT_DUMP_LAYERS=10,20 picks
+// the layers (default none), STRATA_ACT_DUMP_COV=1 adds the full covariance.  Files <prefix>_L<l>_<gu|dn>.bin: int64 n,
+// int64 count, double sum x^2 [n], (double sum x x^T [n*n]); rewritten every 2000 records and at exit.
+namespace {
+struct ActAcc {
+    int64_t n = 0, count = 0;
+    std::vector<double> sq, cov;
+};
+struct ActDump {
+    bool on = false, cov = false;
+    std::string prefix;
+    std::vector<int> layers;
+    std::map<std::pair<int, int>, ActAcc> acc;
+    std::mutex mu;
+    int64_t records = 0;
+    ActDump() {
+        const char* p = std::getenv("STRATA_ACT_DUMP");
+        if (p == nullptr || *p == 0) return;
+        prefix = p;
+        cov = std::getenv("STRATA_ACT_DUMP_COV") != nullptr && std::atoi(std::getenv("STRATA_ACT_DUMP_COV")) != 0;
+        if (const char* l = std::getenv("STRATA_ACT_DUMP_LAYERS")) {
+            std::string s(l);
+            size_t at = 0;
+            while (at < s.size()) {
+                layers.push_back(std::atoi(s.c_str() + at));
+                const size_t c = s.find(',', at);
+                if (c == std::string::npos) break;
+                at = c + 1;
+            }
+        }
+        on = !layers.empty();
+        if (on) std::atexit([] { act_dump_flush(); });
+    }
+    static void act_dump_flush();
+};
+ActDump& act_dump() {
+    static ActDump* d = new ActDump;   // never destroyed: the atexit flush must still find it
+    return *d;
+}
+void ActDump::act_dump_flush() {
+    ActDump& d = act_dump();
+    std::lock_guard<std::mutex> lk(d.mu);
+    for (auto& kv : d.acc) {
+        const std::string path = d.prefix + "_L" + std::to_string(kv.first.first) + (kv.first.second ? "_dn.bin" : "_gu.bin");
+        FILE* f = std::fopen((path + ".tmp").c_str(), "wb");
+        if (!f) continue;
+        std::fwrite(&kv.second.n, 8, 1, f);
+        std::fwrite(&kv.second.count, 8, 1, f);
+        std::fwrite(kv.second.sq.data(), 8, kv.second.sq.size(), f);
+        if (!kv.second.cov.empty()) std::fwrite(kv.second.cov.data(), 8, kv.second.cov.size(), f);
+        std::fclose(f);
+        std::rename((path + ".tmp").c_str(), path.c_str());
+    }
+}
+void act_dump_record(int layer, int kind, const float* v, int64_t n) {
+    ActDump& d = act_dump();
+    if (std::find(d.layers.begin(), d.layers.end(), layer) == d.layers.end()) return;
+    std::lock_guard<std::mutex> lk(d.mu);
+    ActAcc& a = d.acc[{layer, kind}];
+    if (a.n == 0) {
+        a.n = n;
+        a.sq.assign((size_t) n, 0.0);
+        if (d.cov) a.cov.assign((size_t) n * (size_t) n, 0.0);
+    }
+    for (int64_t i = 0; i < n; ++i) a.sq[(size_t) i] += (double) v[i] * v[i];
+    if (d.cov)
+        for (int64_t i = 0; i < n; ++i) {
+            double* row = a.cov.data() + (size_t) i * (size_t) n;
+            const double vi = v[i];
+            for (int64_t j = 0; j < n; ++j) row[j] += vi * v[j];
+        }
+    ++a.count;
+    if (++d.records % (d.cov ? 400000 : 2000) == 0) {
+        d.mu.unlock();
+        ActDump::act_dump_flush();
+        d.mu.lock();
+    }
+}
+}  // namespace
+
 void native_quant_act(const NativeFmt& f, const float* x, void* dst) {
     if (f.had_gu) {   // Hadamard-folded gate/up: quantize H (s * x), never x itself (hadamard.hpp)
         thread_local std::vector<float> tmp;
@@ -136,6 +221,7 @@ void native_quant_act(const NativeFmt& f, const float* x, void* dst) {
         hadamard_rotate(tmp.data(), (int) f.n_embd, f.had_block, f.gu_signs);
         x = tmp.data();
     }
+    if (act_dump().on) act_dump_record(f.layer, 0, x, f.n_embd);
     if (q8k_avx2(f.gu_act)) { q8k_quant_avx2(x, dst, f.n_embd); return; }
     if (q8k_avx1(f.gu_act)) { q8k_quant_avx1(x, dst, f.n_embd); return; }
     traits(f.gu_act)->from_float(x, dst, f.n_embd);
@@ -148,6 +234,7 @@ void native_quant_h(const NativeFmt& f, const float* h, void* dst) {
         hadamard_rotate(tmp.data(), (int) f.n_ff, f.had_block, f.d_signs);
         h = tmp.data();
     }
+    if (act_dump().on) act_dump_record(f.layer, 1, h, f.n_ff);
     if (q8k_avx2(f.d_act)) { q8k_quant_avx2(h, dst, f.n_ff); return; }
     if (q8k_avx1(f.d_act)) { q8k_quant_avx1(h, dst, f.n_ff); return; }
     traits(f.d_act)->from_float(h, dst, f.n_ff);

src/kernels/cpu/tq_avx2.cpp

@@ -29,11 +29,11 @@ inline __m256i rows4(const long long* tab, uint32_t s0, uint32_t s1, uint32_t s2
 }
 
 template <int KB>
-void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
+void rows_t(const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
             int r1) {
     const int nb = n / 128;
     constexpr size_t bsz = 2 + 4 * KB;
-    const long long* tab = (const long long*) tq_state_table();
+    const long long* tab = (const long long*) table;
     for (int r = r0; r < r1; ++r) {
         const uint8_t* x = w + (size_t) r * row_bytes;
         __m128 acc[kTqMaxTokens];
@@ -77,11 +77,13 @@ void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, in
 
 }  // namespace
 
-void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_avx2(int kb, const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
                   float* const* out, int r0, int r1) {
-    if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
-    else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
-    else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
+    if (kb == 8) rows_t<8>(table, w, row_bytes, n, a, nt, out, r0, r1);
+    else if (kb == 6) rows_t<6>(table, w, row_bytes, n, a, nt, out, r0, r1);
+    else if (kb == 5) rows_t<5>(table, w, row_bytes, n, a, nt, out, r0, r1);   // experimental TQK5X
+    else if (kb == 4) rows_t<4>(table, w, row_bytes, n, a, nt, out, r0, r1);   // experimental TQK4X
+    else rows_t<7>(table, w, row_bytes, n, a, nt, out, r0, r1);
 }
 
 }  // namespace strata::kernels::cpu::tq_detail

src/kernels/cpu/tq_avx512.cpp

@@ -47,11 +47,11 @@ inline __m512i rows8(const long long* tab, const uint8_t* qs, uint64_t wr, int t
 }
 
 template <int KB>
-void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
+void rows_t(const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
             int r1) {
     const int nb = n / 128;
     constexpr size_t bsz = 2 + 4 * KB;
-    const long long* tab = (const long long*) tq_state_table();
+    const long long* tab = (const long long*) table;
     for (int r = r0; r < r1; ++r) {
         const uint8_t* x = w + (size_t) r * row_bytes;
         __m128 acc[kTqMaxTokens];
@@ -85,11 +85,13 @@ void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, in
 
 }  // namespace
 
-void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_avx512(int kb, const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
                     float* const* out, int r0, int r1) {
-    if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
-    else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
-    else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
+    if (kb == 8) rows_t<8>(table, w, row_bytes, n, a, nt, out, r0, r1);
+    else if (kb == 6) rows_t<6>(table, w, row_bytes, n, a, nt, out, r0, r1);
+    else if (kb == 5) rows_t<5>(table, w, row_bytes, n, a, nt, out, r0, r1);   // experimental TQK5X
+    else if (kb == 4) rows_t<4>(table, w, row_bytes, n, a, nt, out, r0, r1);   // experimental TQK4X
+    else rows_t<7>(table, w, row_bytes, n, a, nt, out, r0, r1);
 }
 
 }  // namespace strata::kernels::cpu::tq_detail

src/kernels/cpu/tq_computed.hpp

@@ -0,0 +1,33 @@
+// src/kernels/cpu/tq_computed.hpp - the LUT-free ("computed") codebook of the experimental trellis types TQK5X / TQK4X
+// (version 2): a state's four weights come from two 32-bit LCG words instead of the hashed 2048-point table.
+//
+//   r_i = ((s * A_i + B_i) & 0x8FFF8FFF) ^ 0x3B603B60        i = 0, 1   (s = the 15-bit state)
+//   weights 0, 1 = the low and high fp16 of r_0, weights 2, 3 = those of r_1
+//
+// (QTIP's 3INST word, Tseng et al., Cornell-RelaxML/qtip lib/codebook/bitshift.py; the variant "3single" of the
+// f-dsv41-cudalab study, which measured it at the SQNR of the table on real DeepSeek experts and a GEMV without
+// shared-memory lookups.)  The mask fixes the top exponent bits, so |w| lies in [0.125, 2) and the int16 table value
+// round(w * 2^13) used by the CPU kernels is an exact integer (1024..16383): decoder and encoder agree to the bit.
+// The existing types (TQ2_T, TQK6, TQK7) and the table-based K5 / K4 (ids 226 / 227, "version 1") do not use it.
+#pragma once
+
+#include <cstdint>
+
+#include "tq_states.hpp"
+
+namespace strata::kernels::cpu::tq_detail {
+
+inline constexpr uint32_t kCompA[2] = {89226354u, 0x9E3779B1u};
+inline constexpr uint32_t kCompB[2] = {64248484u, 0x7F4A7C15u};
+inline constexpr uint32_t kCompMask = 0x8FFF8FFFu, kCompXor = 0x3B603B60u;
+
+inline uint32_t comp_word(uint32_t s, int i) { return ((s * kCompA[i] + kCompB[i]) & kCompMask) ^ kCompXor; }
+
+/// The four weights of state s at scale 2^13 (int16, exact).
+inline void comp_state4(uint32_t s, int16_t out[4]) {
+    const uint32_t r0 = comp_word(s & 0x7fffu, 0), r1 = comp_word(s & 0x7fffu, 1);
+    const uint16_t h[4] = {(uint16_t) (r0 & 0xffffu), (uint16_t) (r0 >> 16), (uint16_t) (r1 & 0xffffu), (uint16_t) (r1 >> 16)};
+    for (int j = 0; j < 4; ++j) out[j] = (int16_t) (tq_half_to_float(h[j]) * 8192.0f);
+}
+
+}  // namespace strata::kernels::cpu::tq_detail

src/kernels/cpu/tq_cpu.cpp

@@ -5,6 +5,7 @@
 #include "strata/kernels/cpu/expert_layout.hpp"
 
 #include "tq_states.hpp"
+#include "tq_computed.hpp"
 #include "ggml-common-tq.h"
 
 #include <cmath>
@@ -25,6 +26,7 @@ namespace strata::kernels::cpu {
 namespace {
 
 alignas(64) int16_t g_table[(tq_detail::kTqStateMask + 1) * 4];
+alignas(64) int16_t g_table_c[(tq_detail::kTqStateMask + 1) * 4];
 
 void fill_table() {
     for (uint32_t s = 0; s <= tq_detail::kTqStateMask; ++s) {
@@ -39,6 +41,10 @@ void fill_table() {
     }
 }
 
+void fill_table_computed() {
+    for (uint32_t s = 0; s <= tq_detail::kTqStateMask; ++s) tq_detail::comp_state4(s, g_table_c + 4 * s);
+}
+
 int env_isa_cap() {
     const char* e = std::getenv("STRATA_TQ_ISA");
     if (e == nullptr || *e == 0) return 2;
@@ -49,11 +55,10 @@ int env_isa_cap() {
 }
 
 template <int KB>
-void rows_scalar_t(const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,
+void rows_scalar_t(const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,
                    float* const* out, int r0, int r1) {
     const int nb = n / 128;
     constexpr size_t bsz = 2 + 4 * KB;
-    const int16_t* tab = tq_state_table();
     for (int r = r0; r < r1; ++r) {
         const uint8_t* x = w + (size_t) r * row_bytes;
         float acc[kTqMaxTokens][4] = {};
@@ -89,9 +94,21 @@ const int16_t* tq_state_table() {
     return g_table;
 }
 
-bool tq_supported(int t) noexcept { return t == 144 || t == 145 || t == 146; }
+const int16_t* tq_state_table_computed() {
+    static std::once_flag once;
+    std::call_once(once, fill_table_computed);
+    return g_table_c;
+}
 
-int tq_kbits(int t) noexcept { return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : 0; }
+const int16_t* tq_state_table_for(int t) { return (t == kTqTypeK5X || t == kTqTypeK4X) ? tq_state_table_computed() : tq_state_table(); }
+
+bool tq_supported(int t) noexcept {
+    return t == 144 || t == 145 || t == 146 || t == kTqTypeK5X || t == kTqTypeK4X || t == kTqTypeK5T || t == kTqTypeK4T;
+}
+
+int tq_kbits(int t) noexcept {
+    return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : (t == kTqTypeK5X || t == kTqTypeK5T) ? 5 : (t == kTqTypeK4X || t == kTqTypeK4T) ? 4 : 0;
+}
 
 bool tq_isa_ok(TqIsa isa) {
 #if defined(STRATA_TQ_X86)
@@ -140,11 +157,13 @@ void unpack_act(const void* q8_0, int n, TqAct& a) {
     }
 }
 
-void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_scalar(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
                     float* const* out, int r0, int r1) {
-    if (kb == 8) rows_scalar_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
-    else if (kb == 6) rows_scalar_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
-    else rows_scalar_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
+    if (kb == 8) rows_scalar_t<8>(tab, w, row_bytes, n, a, nt, out, r0, r1);
+    else if (kb == 6) rows_scalar_t<6>(tab, w, row_bytes, n, a, nt, out, r0, r1);
+    else if (kb == 5) rows_scalar_t<5>(tab, w, row_bytes, n, a, nt, out, r0, r1);   // experimental TQK5X / TQK5T
+    else if (kb == 4) rows_scalar_t<4>(tab, w, row_bytes, n, a, nt, out, r0, r1);   // experimental TQK4X / TQK4T
+    else rows_scalar_t<7>(tab, w, row_bytes, n, a, nt, out, r0, r1);
 }
 
 }  // namespace tq_detail
@@ -167,14 +186,14 @@ ActSet& unpack_all(const void* const* act, int n, int nt) {
     return *set;
 }
 
-void rows_on(TqIsa isa, int kb, const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,
-             float* const* out, int r0, int r1) {
+void rows_on(TqIsa isa, int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n,
+             const tq_detail::TqAct* const* a, int nt, float* const* out, int r0, int r1) {
     switch (isa) {
 #if defined(STRATA_TQ_X86)
-        case TqIsa::avx512: tq_detail::tq_rows_avx512(kb, w, row_bytes, n, a, nt, out, r0, r1); break;
-        case TqIsa::avx2: tq_detail::tq_rows_avx2(kb, w, row_bytes, n, a, nt, out, r0, r1); break;
+        case TqIsa::avx512: tq_detail::tq_rows_avx512(kb, tab, w, row_bytes, n, a, nt, out, r0, r1); break;
+        case TqIsa::avx2: tq_detail::tq_rows_avx2(kb, tab, w, row_bytes, n, a, nt, out, r0, r1); break;
 #endif
-        default: tq_detail::tq_rows_scalar(kb, w, row_bytes, n, a, nt, out, r0, r1); break;
+        default: tq_detail::tq_rows_scalar(kb, tab, w, row_bytes, n, a, nt, out, r0, r1); break;
     }
 }
 
@@ -183,15 +202,15 @@ void rows_on(TqIsa isa, int kb, const uint8_t* w, size_t row_bytes, int n, const
 void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
              float* const* out, int r0, int r1) {
     if (!tq_isa_ok(isa)) std::abort();   // a caller bug: asked for an ISA this CPU does not have
-    tq_state_table();
+    const int16_t* tab = tq_state_table_for(ggml_type);
     const ActSet& s = unpack_all(act, n, nt);
-    rows_on(isa, tq_kbits(ggml_type), w, row_bytes, n, s.ptr, nt, out, r0, r1);
+    rows_on(isa, tq_kbits(ggml_type), tab, w, row_bytes, n, s.ptr, nt, out, r0, r1);
 }
 
 void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n,
                 const void* const* act, int nt, float* const* ff, int r0, int r1) {
     if (!tq_isa_ok(isa)) std::abort();
-    tq_state_table();
+    const int16_t* tab = tq_state_table_for(ggml_type);
     const ActSet& s = unpack_all(act, n, nt);
     thread_local std::vector<float> gbuf, ubuf;
     const size_t nr = (size_t) r1;   // row r of token t at [t * r1 + r]
@@ -203,8 +222,8 @@ void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, si
         up[t] = ubuf.data() + (size_t) t * nr;
     }
     const int kb = tq_kbits(ggml_type);
-    rows_on(isa, kb, blob, gu_row, n, s.ptr, nt, gp, r0, r1);
-    rows_on(isa, kb, blob + up_off, gu_row, n, s.ptr, nt, up, r0, r1);
+    rows_on(isa, kb, tab, blob, gu_row, n, s.ptr, nt, gp, r0, r1);
+    rows_on(isa, kb, tab, blob + up_off, gu_row, n, s.ptr, nt, up, r0, r1);
     for (int t = 0; t < nt; ++t)
         for (int r = r0; r < r1; ++r) {
             const float g = gp[t][r], u = up[t][r];

strata_cli/__init__.py

@@ -6,8 +6,8 @@ from pathlib import Path
 SCHEMA_VERSION = 1
 
 #: The version people are told: the Strata app's own (apps/desktop/src-tauri/Cargo.toml `version`; tests/terminal/test_tuiqa_version.py keeps the
-#: two equal). `version()` below is the files' build number, which the engine-compatibility checks compare, so it stays numeric.
-PRODUCT_VERSION = "0.2.3"
+#: two equal, except for a Terminal-only release such as 0.2.3.1 or 0.2.4, whose app stays 0.2.3). `version()` below is the files' build number, which the engine-compatibility checks compare, so it stays numeric.
+PRODUCT_VERSION = "0.2.4.1"
 
 
 def version() -> str:

strata_cli/launch/app.py

@@ -148,8 +148,14 @@ class Ctx:
                     tuned_args, tuned_env = list(merged.get("args") or []), dict(merged.get("env") or {})
             except Exception:                                # noqa: BLE001
                 pass
-        exe = self.engine_exe()
-        text = P.engine_help_text(exe) if exe else None
+        if (cfg or {}).get("arch") == "deepseek41" or self.settings.family == "deepseek41":   # DeepSeek runs strata-dsv41: its settings, not the Qwen engine's
+            own = (cfg or {}).get("exe")
+            exe = own if own and Path(str(own)).is_file() else None
+            text = P.dsv41_help_text(exe)
+            exe = exe or "static"
+        else:
+            exe = self.engine_exe()
+            text = P.engine_help_text(exe) if exe else None
         if exe and text:
             self.engine_flags = sorted({p.name for p in P.parse_help(text)})
             self.engine_note = ""

strata_cli/launch/catalog.py

@@ -72,6 +72,8 @@ def verdict(on_this_pc: str | None) -> tuple[str, str]:
         return TIGHT, "not checked (this computer's memory could not be read)"
     if low.startswith("does not fit"):
         return NO, "too big (can still try)"
+    if "free disk space" in low:                           # DeepSeek: fits in memory, not on the disk (still allowed)
+        return TIGHT, "tight: needs more disk space"
     if "low-ram mode" in low:
         return FITS, "fits, a bit slower"
     if "part of its experts" in low or "read from the ssd" in low:
@@ -324,11 +326,16 @@ class Catalog:
         rec = self.recommendation(hw)
         out: list[ModelRow] = []
         own = self.strata.own_families()                # Qwen3.6 is listed for the app; the Terminal's launch list is Flash-Next's
+        dsv41 = None
         for fam in self.strata.model_table(hw):
-            if fam["family"] in own:
+            if fam["family"] in own and not own[fam["family"]].get("multi_file"):
                 continue
             for size in fam["sizes"]:
                 f = fit_on(size, hw.get("ram_gb"), vram)
+                if size.get("multi_file"):            # DeepSeek-V4.1-Flash: shared over every card (strata_control/dsv41_fit.py)
+                    dsv41 = self.strata.dsv41_plan(hw, model=size["model"])
+                    f = {"vramUseGb": _round1(dsv41["vram_experts_gb"] + sum(c["dense_gb"] for c in dsv41["cards"])),
+                         "ramUseGb": _round1(dsv41.get("ram_needed_gb") or dsv41["cpu_experts_gb"]), "spillGb": _round1(dsv41.get("spill_gb") or 0.0)}
                 cat, words = verdict(size.get("on_this_pc"))
                 out.append(ModelRow(
                     family=fam["family"], family_title=fam["title"], model=size["model"], id=size["id"],
@@ -344,6 +351,9 @@ class Catalog:
         """`iq3_xxs` or `IQ3_XXS` (and an optional family) -> (family, model); raises ValueError with the closest match."""
         models, families, _ctx, _src = self.tables()
         names = list(models)
+        for fam_name, fam_d in families.items():       # DeepSeek-V4.1-Flash's size is not one of setup's MODELS
+            if fam_d.get("multi_file"):
+                names += [m for m in self.strata.sizes_of(models, fam_name) if m not in names]
         match = next((n for n in names if n.lower() == model.lower()), None)
         if match is None:
             near = S.closest(model, names)
@@ -376,6 +386,8 @@ class Catalog:
     def max_context(self, family: str, model: str, hw: dict | None = None) -> int | None:
         """The recommended context for this computer and model (setup.py: VRAM rule, then the RAM rule)."""
         hw = hw or self.hardware()
+        if self.strata.multi_family(family):                   # DeepSeek-V4.1-Flash: the context the engine was checked at
+            return int(getattr(self.strata.setup_module(), "DSV41_CONTEXT", 8192))
         if hw.get("mac") and hw.get("ram_gb"):                 # a Mac: the context by unified memory (strata_control/mac_fit.py)
             mac = self.strata.mac()
             if mac is not None:

strata_cli/launch/cmd.py

@@ -84,6 +84,8 @@ def add_parsers(sp: argparse._SubParsersAction) -> list[argparse.ArgumentParser]
     g.add_argument("--preset", metavar="NAME", help="start a saved preset (other flags change it for this start)")
     g.add_argument("--model", metavar="NAME", help="the model size, e.g. iq3_xxs, q2_0, ud-q4_k_xl (any case)")
     g.add_argument("--family", metavar="NAME", help="the model family when a size is offered by several (qwen, swift, coder, ...)")
+    g.add_argument("--dsv41-package", metavar="FILE", help="DeepSeek: the engine package you downloaded from the release page (.tar.gz or .zip); "
+                                                          "it is checked and installed instead of being fetched")
     g.add_argument("--model-path", metavar="PATH", help="use model files you already have: a folder of shards or one .gguf file")
     g.add_argument("--data-dir", metavar="DIR", help="where downloaded model files are stored (default: Strata's data folder)")
     g.add_argument("--move-data", action="store_true",
@@ -199,6 +201,8 @@ def settings_from_args(a: argparse.Namespace, catalog=None) -> tuple[S.LaunchSet
     s = store.load(a.preset) if a.preset else S.LaunchSettings()
     if a.preset:
         s.name = s.name or a.preset
+    if getattr(a, "dsv41_package", None):
+        s.dsv41_package = str(Path(a.dsv41_package).expanduser())
     if a.model_path:
         s.model_path = str(Path(a.model_path).expanduser())
         if not a.model:
@@ -277,6 +281,24 @@ def settings_from_args(a: argparse.Namespace, catalog=None) -> tuple[S.LaunchSet
     return s, notices
 
 
+def _placement_lines(placement: dict | None) -> None:
+    """DeepSeek-V4.1-Flash's plan: the cards, the experts in graphics memory and on the CPU, the n-gram table on the SSD, the warnings."""
+    if not placement:
+        return
+    import textwrap
+    _say("  placement")
+    for line in placement.get("lines") or []:
+        wrapped = textwrap.wrap(line, 94) or [""]
+        _say("    - " + wrapped[0])
+        for w in wrapped[1:]:
+            _say("      " + w)
+    for line in placement.get("warnings") or []:
+        wrapped = textwrap.wrap(line, 90) or [""]
+        _say("  [!] " + wrapped[0])
+        for w in wrapped[1:]:
+            _say("      " + w)
+
+
 def apply_defaults(s: S.LaunchSettings, catalog) -> tuple[S.LaunchSettings, list[S.Notice]]:
     """Fill what was left open with the recommendation for this computer, and collect warnings (never errors for a bad fit)."""
     notices: list[S.Notice] = []
@@ -308,6 +330,13 @@ def apply_defaults(s: S.LaunchSettings, catalog) -> tuple[S.LaunchSettings, list
                 if s.vision == "off" and r.images and catalog.reader_state() == "missing":
                     notices.append(S.Notice("warn", "The engine installed here has no picture reader, so pictures are off.",
                                             "Update the engine to read pictures, or add --vision off to silence this."))
+        if catalog.strata.multi_family(s.family):           # DeepSeek-V4.1-Flash: one request at a time, no tuning, no pictures yet
+            if s.parallel and s.parallel > 1:
+                notices.append(S.Notice("warn", f"{s.model} answers one request at a time in this Strata version, so --parallel {s.parallel} is not used.", ""))
+                s.parallel = None
+            if s.vision not in (None, "off"):
+                notices.append(S.Notice("warn", "DeepSeek-V4.1-Flash cannot read pictures in this Strata version, so pictures are off.", ""))
+            s.vision = "off"                                 # (tuning is DeepSeek's own short tune: tools/dsv41/tune.py, run by the launch)
         notices += S.check_context(s, max_fit_tokens=None)
     if s.access != "local" and not s.api_key and not s.no_key:
         from strata_cli.launch import access
@@ -506,6 +535,22 @@ def open_page(url: str) -> bool:
         return False
 
 
+def _screens_missing() -> str | None:
+    """The Python package the screens need that this install lacks ("rich" / "textual"), None when both are there.  A source install made before
+    setup installed them has neither: `strata launch` then goes on in plain lines instead of stopping with a traceback."""
+    import importlib.util
+    return next((m for m in ("rich", "textual") if importlib.util.find_spec(m) is None), None)
+
+
+def final_hint_lines(s, preset_saved: bool) -> list[str]:
+    """The last lines of a flag launch: how to connect an app and how to stop.  `--preset "<name>"` is suggested only when a preset by that name exists
+    (--preset NAME, or --save-preset NAME); otherwise the plain command, and how to keep the settings."""
+    if preset_saved:
+        return [f"  Connect an app:  strata connect opencode      Stop:  strata launch --stop --preset \"{s.name}\""]
+    return ["  Connect an app:  strata connect opencode      Stop:  strata launch --stop",
+            "  To keep these settings for next time, add --save-preset NAME."]
+
+
 def _tui(run, *args, **kw) -> int:
     """Open the screens. An error of the screens themselves ends with a plain sentence and a non-zero exit code, never a traceback and exit 0
     (tester report T02)."""
@@ -643,14 +688,21 @@ def run_launch(a: argparse.Namespace) -> int:
         have_choices = bool(a.preset or a.model or a.model_path or a.context or a.vision or a.parallel or a.access or a.tune or a.drafting)
         interactive = sys.stdin.isatty() and sys.stdout.isatty() and not a.no_tui and not machine and not a.plan
         plain_questions = False
+        no_screens_package = None
+        if interactive and _screens_missing():
+            no_screens_package = _screens_missing()
+            interactive = False
+            _say(f"The screens need a Python package this install does not have yet ({no_screens_package}), so Strata goes on in plain lines. "
+                 "Run setup again (START-HERE / ./setup.sh) to get the screens.")
         if not have_choices and not interactive and not a.plan:
             if machine or a.no_tui or catalog is None or not _can_ask():     # --no-tui never asks (round 8, M)
                 return fail("`strata launch` with no choices opens screens, and this is not a terminal (or --no-tui was given). Name what to start: --preset NAME, or --model NAME "
                             "(strata launch --help has examples; --plan shows what would happen without changing anything).", 2)
             # Git Bash, mintty and some editor panels cannot show the screens although a person is typing: go on in plain lines, and ask first
             plain_questions = True
-            _say("This window cannot show Strata's screens (Git Bash and similar windows do not pass the keyboard through). "
-                 "For the screens, open Windows Terminal, PowerShell or cmd, or run `winpty strata launch`. Going on with plain questions.")
+            if no_screens_package is None:
+                _say("This window cannot show Strata's screens (Git Bash and similar windows do not pass the keyboard through). "
+                     "For the screens, open Windows Terminal, PowerShell or cmd, or run `winpty strata launch`. Going on with plain questions.")
         if interactive and not have_choices:
             if catalog is None:
                 return fail("This Terminal does not carry the model list, so the screens cannot open. Install it again with the one-line installer.", 5)
@@ -691,10 +743,11 @@ def run_launch(a: argparse.Namespace) -> int:
                     _say(f"  model     {m.get('title') or 'your model files'}{size}   (your files in {s.model_path})")
                 else:
                     _say(f"  model     {m.get('title') or f'{s.family} {s.model}'}{size}")
-                _say(f"  context   {S.context_label(s.context)}    pictures {PICTURE_WORDS.get(s.vision or 'off', s.vision)}    parallel {s.parallel or 'fit to your card'}")
+                _say(f"  context   {S.context_label(s.context)}    pictures {PICTURE_WORDS.get(s.vision or 'off', s.vision)}    parallel {('one at a time' if plan.get('placement') else s.parallel or 'fit to your card')}")
                 _say(f"  download  {(plan.get('download') or {}).get('text', '')}")
                 if plan.get("drafting"):
                     _say(f"  drafting  {plan['drafting'].replace('Drafting: ', '', 1)}")
+                _placement_lines(plan.get("placement"))
                 _say(f"  tuning    {s.tune}    access {S.ACCESS_WORDS[s.access]}, port {s.port}"
                      + ("    API key on" if s.api_key else ""))
                 for n in notes:
@@ -760,7 +813,8 @@ def run_launch(a: argparse.Namespace) -> int:
             _say(f"  {'Anthropic apps':<14} {result.urls[0]}   (Claude Code: ANTHROPIC_BASE_URL, no /v1 at the end)")
         if result.key:
             from strata_cli.launch import access
-            _say(f"  {'API key':<14} {access.mask_key(result.key)}   (show it: strata launch --show-key --preset \"{s.name}\")")
+            _say(f"  {'API key':<14} {access.mask_key(result.key)}   (show it: strata launch --show-key"
+                 + (f" --preset \"{s.name}\")" if (a.preset or a.save_preset) else ")"))
         for n in result.notices:
             _say(f"[!] {n}")
         local = next((u for u in result.urls if "127.0.0.1" in u), result.urls[0] if result.urls else "")
@@ -772,7 +826,8 @@ def run_launch(a: argparse.Namespace) -> int:
             _say(f"  Watch it:    STRATA_API_KEY=$(strata launch --show-key) strata monitor --url {local} --credential-env STRATA_API_KEY")
         else:
             _say(f"  Watch it:    strata monitor --url {local}")
-        _say(f"  Connect an app:  strata connect opencode      Stop:  strata launch --stop --preset \"{s.name}\"")
+        for line in final_hint_lines(s, bool(a.preset or a.save_preset)):
+            _say(line)
         return 0
     except store.PresetError as e:
         return fail(str(e), 2)

strata_cli/launch/params.py

@@ -212,6 +212,45 @@ def flag_list(engine: str | Path | None) -> list[str] | None:
         return None
 
 
+#: strata-dsv41's settings in plain words (its own --help is one bracketed usage line, which the Qwen engine's list parser cannot read).
+DSV41_FLAGS = {
+    "--pack": ("DIR", "The folder with the model's prepared data."),
+    "--max-context": ("N", "How much text the model remembers at once."),
+    "--eos": ("ID", "The token that ends an answer (1 for DeepSeek)."),
+    "--dspark": ("on|off|none|auto", "The DSpark drafter that guesses ahead to speed up answers (auto loads it only where it helps)."),
+    "--dspark-conf": ("F", "How sure a DSpark guess must be to be used (0 to 1)."),
+    "--dspark-block": ("N", "The most words DSpark guesses in one round."),
+    "--expert-profile": ("FILE|none", "A table of which experts are used most, to decide what is kept in memory first."),
+    "--ssd-cache": ("GB", "How much RAM keeps the most used experts so they are not read from the SSD again (sizes itself when absent)."),
+    "--ssd-policy": ("lru|lfu|lfulru", "Which experts the RAM store drops first."),
+    "--prefill-chunk": ("N", "How many prompt words are read at once (the fast prompt path)."),
+    "--threads": ("on|off", "Read long prompts layer by layer on all CPU threads."),
+    "--cpu-threads": ("N", "How many CPU threads compute the experts the graphics card does not hold."),
+    "--tp": ("N", "Share each layer between this many graphics cards."),
+    "--gpus": ("LIST", "Which graphics cards to use together."),
+    "--reserve-mib": ("N", "Graphics memory (in MiB) kept free for other programs."),
+    "--vram-experts": ("F", "How many experts to keep in graphics memory."),
+    "--mmap-experts": ("on|off|auto", "Read the experts in place from the model's files."),
+    "--no-expert-cache": ("", "Do not keep the most used experts in graphics memory."),
+    "--vision": ("on|off", "Let the model read pictures."),
+    "--verbose": ("", "Print more about what the engine does."),
+}
+
+
+def dsv41_help_text(engine: str | Path | None) -> str:
+    """A list of strata-dsv41's settings the Advanced screen can read: ours in plain words, plus any flag its own --help names that we do not
+    describe.  Works before the engine is installed too (the plain words are static)."""
+    lines = [f"  {flag} {meta}  {desc}" if meta else f"  {flag}  {desc}" for flag, (meta, desc) in DSV41_FLAGS.items()]
+    if engine:
+        import subprocess
+        try:
+            proc = subprocess.run([str(engine), "--help"], capture_output=True, text=True, timeout=20)
+            seen = set(re.findall(r"--[a-z][a-z0-9-]*", (proc.stdout or "") + " " + (proc.stderr or "")))
+            lines += [f"  {f}  An engine setting (the engine's help has no sentence for it)." for f in sorted(seen) if f not in DSV41_FLAGS]
+        except Exception:                                    # noqa: BLE001 - the static list stands
+            pass
+    return "\n".join(lines) + "\n"
+
 def engine_help_text(engine: str | Path | None) -> str | None:
     if not engine:
         return None

strata_cli/launch/runner.py

@@ -81,6 +81,11 @@ def missing_file(failure: str) -> bool:
     return failure.startswith(("ModuleNotFoundError", "ImportError", "FileNotFoundError"))
 
 
+def capital(text: str) -> str:
+    """A reason that was written to stand alone ("the update feed could not be read: ...") starts the sentence it is joined to with a capital."""
+    return text[:1].upper() + text[1:]
+
+
 class LaunchError(RuntimeError):
     """A step could not be done. `text` says what it means; `next` says what to do."""
 
@@ -615,6 +620,7 @@ class Runner:
                             why = str(res["error"])
                             break
             why = why or (tail[-1] if tail else "")
+            why = capital(why)                         # (joined after a sentence: "the update feed could not be read ..." starts a new one)
             raise LaunchError("The engine could not be installed." + (f" {why}" if why else ""),
                               "Check your internet connection, or point Strata at an update source with --feed, then try again.", 6)
         self.emit("engine", "The engine is installed.", 100)
@@ -739,6 +745,8 @@ class Runner:
         if s.model_path:
             p = Path(s.model_path)
             args += ["--gguf-dir", str(p if p.is_dir() else p.parent)]
+        if s.dsv41_package and s.family == "deepseek41":
+            args += ["--dsv41-package", s.dsv41_package]
         if s.gpu_roles:
             args += ["--gpu-roles", ",".join(f"{k}={v}" for k, v in sorted(s.gpu_roles.items(), key=lambda kv: int(kv[0])))]
         elif s.gpus:
@@ -917,7 +925,8 @@ class Runner:
         if not (s.family and s.model):
             raise LaunchError("No model was chosen.", "Choose a model, or use --model NAME (strata launch --help lists them).", 2)
         path, desc = self.installed_state()
-        if path is not None and desc is not None and desc_matches(desc, s) and not files_present(read_config(path)):
+        if path is not None and desc is not None and desc_matches(desc, s) and not files_present(read_config(path)) \
+                and not self.dsv41_engine_missing(read_config(path)):
             if self.installed_serves_request(path):
                 self.emit("model", f"{s.family} {s.model} is already installed with these choices.", 100)
                 return path
@@ -1079,6 +1088,47 @@ class Runner:
         self.emit("model", f"Used your prepared model files and wrote the run settings ({path.name}); this card is not one the built-in downloader sets up.", 100, level="warn")
         return path
 
+    def deepseek_files(self) -> bool:
+        """--model-path names DeepSeek-V4.1-Flash's shards (a folder, the repository's folder with the set's own folder inside - UD-Q8_K_XL,
+        2.00bpw, 2.25bpw - or one shard): they are that model, whatever the fingerprint table of the Flash-Next files says.  True when it did
+        (s.family / s.model / s.model_path set); a shard missing from the folder is said now, in plain words, before an engine or a download
+        is touched."""
+        s = self.s
+        try:
+            mod = self.catalog.strata.setup_module()
+            fam = getattr(mod, "MULTI_FILE_FAMILIES", {}).get("deepseek41")
+            if fam is None or not s.model_path:
+                return False
+            p = Path(s.model_path)
+            found = None
+            order = ([s.model] if s.model in fam["models"] else []) + [m for m in fam["models"] if m != s.model]
+            for m in order:
+                d = fam["models"][m]
+                folders = [p, p / (d.get("subdir") or m)] if p.is_dir() else [p.parent]
+                folder = next((f for f in folders if any((f / n).exists() for n in d["shards"])), None)
+                if folder is not None:
+                    found = (m, d, folder)
+                    break
+        except Exception:                                    # noqa: BLE001 - not DeepSeek's: the usual check goes on
+            return False
+        if found is None:
+            return False
+        model, d, folder = found
+        if s.family and s.family != "deepseek41":
+            raise LaunchError(f"These are DeepSeek-V4.1-Flash's files, not {s.family} {s.model}'s.",
+                              "Leave --model out, or use --model " + model + ".", 2)
+        others = list(d["shards"])
+        missing = [n for n in others if not (folder / n).exists()]
+        if missing:
+            parts = {4: "four", 9: "nine"}.get(len(others), str(len(others)))
+            raise LaunchError(f"DeepSeek-V4.1-Flash {d.get('label', model)} is in {parts} parts and {len(missing)} of them are not in {folder} "
+                              f"(first missing: {missing[0]}).",
+                              "Put every part in the folder (or drop --model-path and let Strata download them). "
+                              "Nothing was downloaded or changed.", 5)
+        s.family, s.model, s.model_path = "deepseek41", model, str(folder)
+        self.emit("model", f"These files are DeepSeek-V4.1-Flash {model}; they are used where they are.", None)
+        return True
+
     def check_own_files(self) -> None:
         """The person's own model files, looked at once: a path that is not there or holds no model is said in plain words first, and the
         files are matched to the catalog model (resolve_own_files). Nothing else is done with a path that was already checked."""
@@ -1095,6 +1145,8 @@ class Runner:
         problem = models.own_path_problem(s.model_path)
         if problem:                                          # before the engine or setup are touched, with the path as typed
             raise LaunchError(problem[0], problem[1], 2)
+        if self.deepseek_files():
+            return
         given = bool(s.family and s.model)
         try:
             plan = models.plan_import(models.absolute(s.model_path))
@@ -1168,6 +1220,7 @@ class Runner:
         base = read_config(base_path)
         cfg = derive_config(base, self.s, bool_flags=tune_knobs.BOOL_FLAGS,
                             log=str(launch_dir() / f"{self.name}.engine.log"), gpus=cards_for_launch(self.s, self.catalog, base=base))
+        self.use_own_dsv41_engine(cfg)
         out = launch_dir() / f"{self.name}.json"
         write_private(out, cfg)
         self.config_path = out
@@ -1192,7 +1245,16 @@ class Runner:
         if cur.get("overrides") != want:
             profiles.save(name, {**cur, "overrides": want}, expected_revision=rev)
 
+    def multi_file(self) -> bool:
+        """The chosen model is DeepSeek-V4.1-Flash (setup.py MULTI_FILE_FAMILIES): nine shards, its own engine; no tuning and no parallel slots."""
+        try:
+            return bool(self.s.family and self.catalog.strata.multi_family(self.s.family))
+        except Exception:                                    # noqa: BLE001
+            return False
+
     def catalog_parallel(self) -> int | None:
+        if self.multi_file():                                # the DeepSeek engine answers one request at a time: no --batch to pass
+            return None
         try:
             comp = self.catalog.computer()
             return S.fit_parallel(comp.main_vram_gb) if comp.usable_gpus else None
@@ -1200,10 +1262,56 @@ class Runner:
             return None
 
     # ---- 4. tuning
+    def tune_deepseek(self) -> None:
+        """DeepSeek-V4.1-Flash has its own short tuner (tools/dsv41/tune.py: the drafter on or off, the CPU threads; the RAM store sizes itself):
+        a few test runs of the engine, a result written into this launch's config, and the standard settings on any failure."""
+        s = self.s
+        cfg = getattr(self, "config_path", None)
+        if cfg is None:
+            self.emit("tune", "There are no DeepSeek settings to tune yet, so it starts with the standard ones.", 100, level="warn")
+            return
+        mode = "thorough" if s.tune == "thorough" else "quick"
+        argv = [sys.executable, str(Path(__file__).resolve().parents[2] / "tools" / "dsv41" / "tune.py"), "--config", str(cfg), "--mode", mode,
+                "--output", "json"]
+        self.emit("tune", "Tuning DeepSeek: it times a short question with your standard settings, then with the DSpark drafter on (and, if that helps, a shorter "
+                          "draft block) and the CPU threads set to your physical cores; each run loads the model once. It keeps only a clear gain. Close other heavy programs.", 0)
+        try:
+            p = self._spawn(argv)
+        except Exception as e:                           # noqa: BLE001 - a tune never stops a launch
+            self.emit("tune", f"The DeepSeek tuner could not start ({type(e).__name__}), so the standard settings run.", 100, level="warn")
+            return
+        lines: list[str] = []
+        reader = threading.Thread(target=lambda: lines.extend(p.stdout.readlines()), daemon=True)   # type: ignore[union-attr]
+        reader.start()
+        while p.poll() is None:
+            if self.cancel.is_set():
+                self._stop_children()
+                raise Cancelled()
+            time.sleep(1.0)
+        reader.join(timeout=3)
+        result = None
+        for line in reversed(lines):
+            try:
+                result = json.loads(line).get("result")
+                break
+            except (ValueError, AttributeError):
+                continue
+        if result is None:
+            self.tune_word = "failed"
+            self.emit("tune", "The DeepSeek tuner did not finish, so the standard settings run. Nothing was changed.", 100, level="warn")
+            return
+        self.tune_result = result
+        self.tune_word = "faster" if result.get("changed") else "already fastest"
+        s.tuned_at = time.strftime("%Y-%m-%d")
+        self.emit("tune", " ".join(result.get("lines") or ["Tuning finished."]), 100, level="info", word=self.tune_word)
+
     def tune(self) -> None:
         s = self.s
         if s.tune == "skip":
             return
+        if self.multi_file():
+            self.tune_deepseek()
+            return
         from strata_control import presets
         if not self.retune:
             if s.tuned_preset:
@@ -1712,12 +1820,17 @@ class Runner:
         title = row.title if row is not None else (f"{s.family} {s.model}" if s.model else None)
         model = {"title": title, "family": s.family, "model": s.model, "model_gb": row.download_gb if row is not None else None,
                  "own_files": bool(s.model_path), "folder": s.model_path}
-        return {"profile": self.profile, "settings": s.to_dict(with_key=False), "warnings": warns,
-                "flags": S.flags_text(s), "steps": list(STAGES), "model": model, "download": download, "drafting": self.drafting_plan_line()}
+        out = {"profile": self.profile, "settings": s.to_dict(with_key=False), "warnings": warns,
+               "flags": S.flags_text(s), "steps": list(STAGES), "model": model, "download": download, "drafting": self.drafting_plan_line()}
+        if self.multi_file():
+            out["placement"] = self.placement()
+        return out
 
     def drafting_plan_line(self) -> str:
         """The drafting line of --plan: what drafts, or why not what was asked for. Nothing is changed or downloaded."""
         from strata_control import drafting
+        if self.multi_file():                                # DeepSeek drafts with its own DSpark drafter (the --dspark auto the config writes), not MTP
+            return "Drafting: DSpark (used when it speeds up answers on this computer)"
         try:
             base, cfg = self.installed_state()
             want = self.drafting_inputs(cfg or {})
@@ -1726,6 +1839,109 @@ class Runner:
         except Exception:                                    # noqa: BLE001
             return ""
 
+    def dsv41_engine_missing(self, cfg: dict) -> bool:
+        """An installed DeepSeek model whose config names an engine that is gone (an older package folder an update removed, or another install's) while
+        this install has no DeepSeek engine package either: not 'already installed' - setup runs, which fetches the package through the feed or says in
+        one plain line what to do (--dsv41-package, or the engine-dsv41 folder), instead of the start failing on a missing file."""
+        if cfg.get("arch") != "deepseek41":
+            return False
+        exe = str(cfg.get("exe") or "")
+        if exe and Path(exe).is_file():
+            return False
+        return self.dsv41_engine_found()[0] is None
+
+    def use_own_dsv41_engine(self, cfg: dict) -> None:
+        """DeepSeek: the run config's engine is the DeepSeek engine package of THIS install (engine-dsv41/ here), not the path the installed model's
+        config was written with: that config can come from another Strata install on this computer (the model is shared between installs) or name
+        an older package folder that an update has replaced.  Only when this install has no package does the config's own path stay."""
+        if cfg.get("arch") != "deepseek41":
+            return
+        found, _known = self.dsv41_engine_found()
+        if found is None:
+            return
+        old = str(cfg.get("exe") or "")
+        if os.path.normcase(os.path.normpath(old)) != os.path.normcase(os.path.normpath(str(found))):
+            cfg["exe"] = str(found)
+            if old:
+                self.notices.append("DeepSeek runs on this install's own engine package, not the one the model's earlier settings named.")
+        libs = [d for d in (cfg.get("lib_dirs") or []) if "engine-dsv41" not in str(d)]
+        cfg["lib_dirs"] = [*libs, str(Path(found).parent)]          # its own folder carries the libraries it needs
+
+    def dsv41_engine_found(self) -> tuple[Path | None, bool | None]:
+        """(strata-dsv41 already installed, or None; whether the engine can be told at all).  Looks where setup puts DeepSeek's own engine package
+        (engine-dsv41/ beside the engine, in Strata's own folder, and in the data folder), and at STRATA_DSV41_EXE (a build of your own)."""
+        from strata_control import dsv41_engine
+        own = (os.environ.get("STRATA_DSV41_EXE") or "").strip()
+        if own:
+            return (Path(own) if Path(own).is_file() else None), True
+        folders: list[Path] = [Path(self.catalog.root) / dsv41_engine.DIR]       # this install's own package first
+        try:
+            from strata_control import engine_component as ec
+            rep = ec.status()
+            exe = (rep.get("engine") or {}).get("exe") if rep.get("usable") else None
+            if exe:
+                folder = Path(str(exe)).parent
+                folders += [folder.parent / dsv41_engine.DIR]
+                try:
+                    folders.append(self.catalog.strata.setup_module().dsv41_engine_dir(folder))
+                except Exception:                            # noqa: BLE001
+                    pass
+                known = True
+            else:
+                known = None
+        except Exception:                                    # noqa: BLE001
+            known = None
+        try:                                                 # where the setup this Terminal runs puts it (its own engine folder, or Strata's folder)
+            mod = self.catalog.strata.setup_module()
+            folders += [Path(self.catalog.root) / dsv41_engine.DIR,      # setup runs here in the app's installer mode: its home() is this folder
+                        mod.dsv41_engine_dir(Path(mod.ROOT) / "engine"), Path(mod.home()) / dsv41_engine.DIR]
+        except Exception:                                    # noqa: BLE001
+            pass
+        try:
+            folders.append((Path(self.s.data_dir) if self.s.data_dir else self.catalog.data_dir()) / dsv41_engine.DIR)
+        except Exception:                                    # noqa: BLE001
+            pass
+        for f in folders:
+            found = dsv41_engine.find(f)
+            if found is not None:
+                return found, True
+        for f in folders[:1]:                                # the engine's own folder: an all-in-one package carries it
+            for n in ("strata-dsv41", "strata-dsv41.exe"):
+                if (f.parent / n).is_file():
+                    return f.parent / n, True
+        return None, known
+
+    def dsv41_engine(self) -> bool | None:
+        """Whether the installed engine carries strata-dsv41 (True / False); None when no engine is installed yet or it cannot be told."""
+        found, known = self.dsv41_engine_found()
+        return True if found is not None else (False if known else None)
+
+    def placement(self) -> dict:
+        """Where DeepSeek-V4.1-Flash would go on this computer's cards (strata_control/dsv41_fit.py): the cards this launch names, else every
+        card that counts; the real numbers in plain words, with the n-gram table's SSD note and the RAM / disk warnings (never a refusal)."""
+        s = self.s
+        hw = self.catalog.hardware()
+        try:
+            data = Path(s.data_dir) if s.data_dir else self.catalog.data_dir()
+            free = None if (s.model_path or self.model_state().remaining_gb < 1) else self.catalog.disk_free(data)
+        except Exception:                                    # noqa: BLE001 - a plan that cannot read the disk still shows the rest
+            free = None
+        p = self.catalog.strata.dsv41_plan(hw, free, s.gpus, model=s.model)
+        warnings = list(p["warnings"])
+        found, known = self.dsv41_engine_found()
+        engine = True if found is not None else (False if known else None)
+        if found is not None:
+            ver = next((part for part in (found.parent.name.split("-")[0], ) if part[:1].isdigit()), "")
+            p["lines"].append("DeepSeek engine: installed" + (f", {ver}" if ver else "") + ".")
+        if engine is False:
+            warnings.append("DeepSeek runs on its own engine package (about 70 MB, separate from the Strata engine): setup downloads it first. "
+                            "If it cannot be fetched, setup stops with a message before the model is downloaded.")
+        return {"verdict": p["verdict"], "lines": p["lines"], "warnings": warnings, "cards": p["cards"], "args": p["args"],
+                "engine_has_dsv41": engine,
+                "experts_in_vram": p["experts_in_vram"], "experts_total": p["experts_total"], "vram_experts_gb": p["vram_experts_gb"],
+                "cpu_experts_gb": p["cpu_experts_gb"], "ram_needed_gb": p.get("ram_needed_gb"), "engram_gb": p["engram_gb"],
+                "download_gb": p["download_gb"], "measured": p["measured"]}
+
     def run(self, *, start: bool = True) -> Result:
         self.restart_after = start
         try:
@@ -1787,6 +2003,11 @@ def _read_json(path: Path) -> dict | None:
 #: lines of an engine log that say why it stopped (the last one wins), and a plain next step for the causes we know. Recommend, never force.
 CAUSE_LINE = re.compile(r"error|exception|failed|cannot|can't|could not|no device|not found|out of memory|unsupported|abort|invalid|missing|"
                         r"no kernel|illegal|denied|refus", re.I)
+# The GPU runtime (ROCm) failed while the card was being set up, often right after another engine exited: the runtime's own text
+# ("terminate called after throwing an instance of 'rocr::AMD::hsa_exception'") stays in the log, not in the sentence shown.
+HSA_START = re.compile(r"hsa_exception|HSA_STATUS_ERROR", re.I)
+HSA_START_NEXT = ("The graphics card was not ready yet; wait a few seconds and run it again. If it keeps happening, restart the PC "
+                  "and report the log.")
 KNOWN_CAUSES = (
     (re.compile(r"no device of requested type|device not found|no (gpu|sycl|level.zero) device|permission denied.*(dri|render)", re.I),
      "The engine could not open the graphics card. On Linux the account must be allowed to use it: add it to the render group "
@@ -1865,10 +2086,19 @@ def explain_engine_exit(result: dict, config: Path | None, marks: dict | None =
         stage = "The engine stopped while it was loading the model" + (f" (exit code {code})" if code not in (None, 0) else "")
     else:
         stage = f"The server did not become ready ({state}: {why or 'see the engine log'})"
+    hsa = HSA_START.search(" ".join(lines + [str(why or "")]))   # the runtime's own text is kept out of the sentence (it is in the log)
+    if hsa:
+        # a real cause in the same log (out of memory, no permission on the card ...) wins: its line is the cause shown
+        real = next((x for x in reversed(lines) if any(p.search(x) for p, _ in KNOWN_CAUSES)), None)
+        if real:
+            hsa = None
+            cause = re.sub(r"^(\[strata\]\s*)?(\d{4}-\d\d-\d\d[ T][\d:.]+\s*)?", "", real.strip())[:240]
+        else:
+            cause = ""
     text = stage + (f": {cause}" if cause else ".")
     if not text.endswith((".", "!")):
         text += "."
-    step = next((words for pattern, words in KNOWN_CAUSES if pattern.search(" ".join([cause, str(why or "")]))), "Run it again; if it repeats, report the log.")
+    step = HSA_START_NEXT if hsa else next((words for pattern, words in KNOWN_CAUSES if pattern.search(" ".join([cause, str(why or "")]))), "Run it again; if it repeats, report the log.")
     return text, step + (f" The engine's log: {log}" if log else " The engine's log has the reason (strata logs, or the log named in the run config).")
 
 

strata_cli/launch/screens.py

@@ -1041,9 +1041,15 @@ class TuningScreen(Base):
     def on_mount(self) -> None:
         s = self.ctx.settings
         first = self.ctx.fresh_setup and not s.is_tuned
-        items = [Item(["Quick", "about 20 minutes", "checks the most likely gains" + ("; suggested the first time" if first else "")], key="quick"),
-                 Item(["Thorough", "up to 2 hours", "tries more settings, measures each one several times"], key="thorough"),
-                 Item(["Skip", "", "use the default settings (already good on most computers)"], key="skip")]
+        if s.family == "deepseek41":                         # DeepSeek's own short tune (tools/dsv41/tune.py): its real length and what it tries
+            items = [Item(["Quick", "about 5-10 minutes", "tries the DSpark drafter on or off, a shorter draft block and the CPU threads; keeps only clear gains"
+                           + ("; suggested the first time" if first else "")], key="quick"),
+                     Item(["Thorough", "about 10-15 minutes", "also tries every CPU thread"], key="thorough"),
+                     Item(["Skip", "", "use the default settings (already good on most computers)"], key="skip")]
+        else:
+            items = [Item(["Quick", "about 20 minutes", "checks the most likely gains" + ("; suggested the first time" if first else "")], key="quick"),
+                     Item(["Thorough", "up to 2 hours", "tries more settings, measures each one several times"], key="thorough"),
+                     Item(["Skip", "", "use the default settings (already good on most computers)"], key="skip")]
         if not self.ctx.tune_chosen and s.tune == "skip" and not s.is_tuned and self.ctx.fresh_setup:
             s.tune = "quick"
         self.query_one(Menu).set_items(items, {"quick": 0, "thorough": 1, "skip": 2}.get(s.tune, 0))
@@ -1449,7 +1455,8 @@ class SaveScreen(Base):
                  ("Context", S.context_label(s.context) + f", memory {s.kv or 'int8'}"),
                  ("Pictures", "on" if s.vision != "off" else "off"),
                  ("Parallel", f"{s.parallel} at a time" if s.parallel else f"fit to your card ({S.fit_parallel(self.ctx.computer.main_vram_gb)})"),
-                 ("Tuning", {"quick": "quick (about 20 min)", "thorough": "thorough (up to 2 h)", "skip": "skipped"}[s.tune]),
+                 ("Tuning", ({"quick": "quick (about 5-10 min)", "thorough": "thorough (about 10-15 min)", "skip": "skipped"} if s.family == "deepseek41"
+                              else {"quick": "quick (about 20 min)", "thorough": "thorough (up to 2 h)", "skip": "skipped"})[s.tune]),
                  ("Access", f"{S.ACCESS_WORDS[s.access]}, port {s.port}" + (", API key on" if s.api_key else ", no API key")),
                  ("Storage", s.data_dir or str(self.ctx.default_folder()))]
         if s.extra:

strata_cli/launch/settings.py

@@ -82,6 +82,7 @@ class LaunchSettings:
     family: str | None = None                  # catalog family id (qwen, swift, coder, unsloth, gyro, nvfp4...)
     model: str | None = None                   # catalog size (IQ3_XXS ...)
     model_path: str | None = None              # "Use model files I already have": a .gguf file or a folder
+    dsv41_package: str | None = None           # DeepSeek: the engine package the person downloaded by hand (checked, then installed)
     data_dir: str | None = None                # where model files are stored (None: the Strata default)
     gpus: list[int] | None = None              # None = the engine's choice (all usable cards)
     gpu_roles: dict[str, str] | None = None    # {"1": "main", "0": "helper", "2": "off"}: a role for every card; None = the recommendation

strata_control/dsv41_engine.py

@@ -0,0 +1,153 @@
+"""DeepSeek-V4.1's engine as its own package (feed member ``dsv41``), downloaded when DeepSeek is chosen.
+
+It goes in ``<data>/engine-dsv41/<version>-<platform>-<backend>[-<variant>]/``, beside the Strata engine and never inside it, so the Qwen
+engine's files are not touched.  The package is picked, downloaded (resumable), unpacked and checked with the same code as an engine update
+(strata_control/updater.py: the signed feed, the archive's SHA-256, the manifest hash the feed names, the package signature, every file's hash);
+there is no test start here (setup's own 16-token question follows the pack).
+"""
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+from typing import Callable
+
+NAME = "strata-dsv41"
+DIR = "engine-dsv41"
+
+
+def exe_name() -> str:
+    return NAME + (".exe" if os.name == "nt" else "")
+
+
+def find(dest: Path) -> Path | None:
+    """The newest installed package's strata-dsv41 in `dest` (engine-dsv41/), or None."""
+    try:
+        folders = sorted((d for d in Path(dest).iterdir() if d.is_dir() and (d / exe_name()).is_file()), key=lambda d: d.name)
+    except OSError:
+        return None
+    return folders[-1] / exe_name() if folders else None
+
+
+ARCHIVE_SUFFIXES = (".tar.gz", ".tar.xz", ".zip")
+MAX_UNPACKED = 8 << 30                  # a DeepSeek engine package is under a GB; this only stops an archive that unpacks into something huge
+
+
+def pending(dest: Path) -> list[Path]:
+    """Packages the person put in `dest` (engine-dsv41/) by hand: strata-dsv41-*.tar.gz / .zip, newest first."""
+    try:
+        files = [f for f in Path(dest).iterdir() if f.is_file() and f.name.startswith("strata-dsv41") and f.name.endswith(ARCHIVE_SUFFIXES)]
+    except OSError:
+        return []
+    return sorted(files, key=lambda f: f.stat().st_mtime, reverse=True)
+
+
+def install_package(dest: Path, package: str | os.PathLike, *, emit: Callable[[str], None] = print) -> Path | None:
+    """Install a DeepSeek engine package the person downloaded from the release page (a .tar.gz / .zip, or the unpacked folder) into `dest`.
+    Checked before anything is installed: it is a DeepSeek engine package for this system, its manifest is signed by a key Strata trusts, and every
+    file matches the hash in the manifest.  Returns the new strata-dsv41, or None with a plain line through `emit`."""
+    from strata_control import components, engine_component as ec, signing, updater
+    from strata_control.targets import host_platform
+
+    src, dest = Path(package).expanduser(), Path(dest)
+    if not src.exists():
+        emit(f"{src} is not there. Download the DeepSeek engine package for your system from the release page and give its path again.")
+        return None
+    work = dest.parent / (DIR + ".manual")
+    shutil.rmtree(work, ignore_errors=True)
+    work.mkdir(parents=True)
+    try:
+        if src.is_dir():
+            root = work / "pkg"
+            shutil.copytree(src, root)
+        else:
+            fmt = "zip" if src.name.endswith(".zip") else "tar.xz" if src.name.endswith(".tar.xz") else "tar.gz"
+            if not src.name.endswith(ARCHIVE_SUFFIXES):
+                emit(f"{src.name} is not a package file (.tar.gz or .zip). Download the DeepSeek engine package for your system from the release page.")
+                return None
+            root = work / "pkg"
+            updater.extract(src, root, fmt, MAX_UNPACKED)
+        if not (root / ec.MANIFEST).is_file():
+            inner = next((d for d in root.iterdir() if d.is_dir() and (d / ec.MANIFEST).is_file()), None) if root.is_dir() else None
+            if inner is None:
+                emit(f"{src.name} is not an engine package (it has no {ec.MANIFEST}).")
+                return None
+            root = inner
+        component = ec.read_manifest(root)
+        if component.entrypoint != exe_name():
+            emit(f"{src.name} is not the DeepSeek engine (its program is {component.entrypoint}, not {exe_name()}). "
+                 "The Strata engine package is a different download.")
+            return None
+        if component.platform != host_platform():
+            emit(f"{src.name} is for {component.platform}, and this computer is {host_platform()}. Download the one for your system.")
+            return None
+        outer = ec._verifier
+        signing.install_component_verifier(signing.load_trusted_keys())
+        try:
+            ec.verify_signature(component, {**os.environ, ec.ENV_REQUIRE_SIGNED: "1"})
+            ec.verify(component, full=True)
+        except ec.ComponentProblem as exc:
+            emit(f"{src.name} did not pass its check ({exc}). It may be damaged or not from the Strata release: download it again from the release page.")
+            return None
+        finally:
+            ec.register_signature_verifier(outer)
+        name = f"{component.version}-{component.platform}-{component.backend}" + (f"-{component.variant}" if component.variant else "")
+        dest.mkdir(parents=True, exist_ok=True)
+        target = dest / name
+        shutil.rmtree(target, ignore_errors=True)
+        shutil.move(str(root), str(target))
+        if os.name != "nt":
+            (target / exe_name()).chmod(0o755)
+        for old in dest.iterdir():
+            if old.is_dir() and old.name != target.name:
+                shutil.rmtree(old, ignore_errors=True)
+        emit(f"DeepSeek engine {component.version} ({component.backend}) checked (signed, every file matches) and installed: {target}")
+        return target / exe_name()
+    except (updater.UpdateError, ec.ComponentProblem, OSError) as exc:
+        emit(f"{src.name} could not be installed: {exc}")
+        return None
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+
+
+def install(dest: Path, source: str, *, hardware: dict | None = None, emit: Callable[[str], None] = print, cancel=None) -> Path | None:
+    """Fetch the DeepSeek engine package for this PC from the signed feed at `source` (a URL or a folder) into `dest`.
+    Returns the new strata-dsv41, or None with a plain line through `emit` when there is none (not in the feed, nothing for this PC)."""
+    from strata_control import feed as feedmod, signing, updater
+
+    dest = Path(dest)
+    work = dest.parent / (DIR + ".work")
+    hw = hardware or updater.detect_hardware()
+    state_dir = work / "state"
+    state_dir.mkdir(parents=True, exist_ok=True)
+    ctx = updater.Context(components=work, source=source, trusted=signing.load_trusted_keys(), hardware=hw, cancel=cancel,
+                          emit=lambda event: emit(event["message"]) if event.get("event") == "phase" and event.get("message") else None)
+    state = updater.State(work)
+    try:
+        feed = updater.open_feed(ctx, state)
+        pkg, reasons = feedmod.select_engine(list(feed.dsv41), hw)
+        if pkg is None:
+            emit("There is no DeepSeek engine package for this computer in the update feed"
+                 + (": " + "; ".join(reasons[:2]) if reasons else "") + ".")
+            return None
+        archive = updater._download(ctx, "dsv41", pkg.url, pkg.size, pkg.sha256, "the DeepSeek engine", feed)
+        stage_root, component = updater.stage_engine(ctx, feed, pkg, archive)
+        try:
+            folder = stage_root / "engine" / updater.folder_name(pkg)
+            dest.mkdir(parents=True, exist_ok=True)
+            target = dest / folder.name
+            shutil.rmtree(target, ignore_errors=True)
+            shutil.move(str(folder), str(target))
+        finally:
+            shutil.rmtree(stage_root, ignore_errors=True)
+        if os.name != "nt":
+            (target / exe_name()).chmod(0o755)
+        for old in dest.iterdir():                              # one generation: the older folders of this engine go
+            if old.is_dir() and old.name != target.name:
+                shutil.rmtree(old, ignore_errors=True)
+        emit(f"DeepSeek engine {pkg.version} installed: {target}")
+        shutil.rmtree(work, ignore_errors=True)               # (on a failure the partial download stays in `work`: the next try resumes it)
+        return target / exe_name()
+    except (updater.UpdateError, feedmod.FeedError, signing.SignatureError) as e:
+        emit(f"The DeepSeek engine could not be fetched: {e}")
+        return None

strata_control/dsv41_fit.py

@@ -0,0 +1,389 @@
+"""Where DeepSeek-V4.1-Flash goes on a computer: the real numbers of the model and its engine, in one place.
+
+The model is 40 layers of 384 routed experts (top 6) beside dense attention weights.  Unsloth's UD-Q8_K_XL (the
+file Strata runs, pinned in setup.py) is 508.4 GB in nine shards:
+
+  routed experts, MXFP4      288.8 GB   15,360 experts of 18.8 MB (40 layers x 384)
+  dense weights, BF16 / F32   16.8 GB   attention, router, shared experts, the head (the embedding table stays on the CPU)
+  the n-gram table (Engram)  202.8 GB   MXFP8, shard 8: read from the SSD, a few rows a token, never held in memory
+
+so the memory the model needs is the experts and the dense weights, 305.6 GB, with the n-gram table left out.  strata-dsv41
+(src/dsv41) puts the dense weights and as many experts as the cards hold in graphics memory, one block of layers on each
+card, and computes the rest of the experts on the CPU, reading them in place from the model's files (the page cache).
+
+Measured (r9700, 4 x Radeon AI PRO R9700 32 GB, 251 GB RAM, 2026-10-20, f-dsv41-port 01389205e): per card 3.4-4.7 GB of dense
+weights, 28.5-29.8 GB of experts (1,505-1,575 of its 3,840), ~0.6 GB free; 6,197 experts (116.5 GB) in graphics memory and
+9,163 (172.3 GB) on the CPU lane; --tp 2: prompts 143-181 tokens a second (3K prompts), answers 25-31 tokens a second.  Pure functions, no I/O:
+setup.py, tools/strata_mcp.py and strata_cli/launch import it.  Recommend, never force: nothing here refuses a computer.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from strata_control import gpu_fit
+
+GB = 1e9
+GIB = 2 ** 30
+MAIN_BYTES = 508_376_763_197          # the nine shards of UD-Q8_K_XL (setup.py DSV41_FILES)
+ENGRAM_BYTES = 202_758_032_592        # shard 8: the n-gram table
+EXPERT_BYTES = 288_777_830_400        # MXFP4: 543,581,798,400 values x 17/32 bytes
+LAYERS = 40
+EXPERTS_PER_LAYER = 384
+EXPERTS = LAYERS * EXPERTS_PER_LAYER  # 15,360
+EXPERT_GB = EXPERT_BYTES / GB / EXPERTS     # 18.8 MB each
+WEIGHTS_GB = (MAIN_BYTES - ENGRAM_BYTES) / GB            # 305.6
+EXPERTS_GB = EXPERT_BYTES / GB                           # 288.8
+DENSE_GB = WEIGHTS_GB - EXPERTS_GB                       # 16.8
+DENSE_ON_GPU_GB = 15.5                # measured: 3.72 + 3.69 + 3.40 + 4.70 (the embedding table, ~1.3 GB, stays on the CPU)
+ENGRAM_GB = ENGRAM_BYTES / GB                            # 202.8
+DOWNLOAD_GB = MAIN_BYTES / GB                            # 508.4
+RESERVE_GB = 1.15                     # per card: the 600 MiB the engine leaves free (--reserve-mib) and ~0.5 GB of its buffers
+                                      # (measured: dense weights + experts + free = 33.8 of an R9700's 34.4 GB)
+REST_RAM_GB = 9.0                     # RAM the engine and the system keep besides the experts ("beside 9 GB for the rest")
+SPARE_FITS_GB = 4.0                   # RAM left beyond the CPU lane's experts below which "fits" becomes "tight"
+DISK_HEADROOM_GB = 6.0                # the pack folder (hash tables, tokenizer) and room to work
+SMALLEST_CARD_GB = 8.0                # a card under this adds little: the dense weights alone are ~4 GB of it per card of four
+MEASURED = ("4 x Radeon AI PRO R9700 (32 GB) with 251 GB of RAM: prompts of 3,000 tokens read at 143-181 tokens a second, answers "
+            "at 25-31 tokens a second (--tp 2); one answer at a time")
+
+
+# The two official IST-DAS sets (ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF, setup.py DSV41_ISTA_FILES): 4 shards each (the experts as IQ1_M / IQ2 / IQ3 types, the
+# dense weights Q8_0 / Q5_K), plus ONE Engram (110.6 GB, IQ4_NL, shared by both sets, stays on the SSD) and the DSpark drafter (8.0 GB).
+# Measured 2026-10-23 (f-dsv41-rel): quality against the original model (final files, 4,995 positions: ppl +15.5% for 2.00 [code +8.6%, agent +5.3%], +7.3% for 2.25 [code +3.2%, agent +2.2%]
+# on 2.25); speed on the owner PC (RTX 5070 12 GB, 64 GB RAM, 3.6 GB/s NVMe): answers ~4.7 tokens a second, prompts 121 tokens a second at 4K and
+# 231 tokens a second at 8K; RTX 3060 12 GB with 46 GB of RAM (the engine held to 36 GB): answers ~7.2 tokens a second on 2.00; 4 x Radeon AI PRO
+# R9700: answers ~31 tokens a second, prompts 330-373 tokens a second at 8K.
+ISTA_ENGRAM_BYTES = 110_595_290_816
+ISTA_DSPARK_BYTES = 7_976_914_240
+ISTA_META_BYTES = 5_780_589          # the first shard of Unsloth's UD-Q8_K_XL: only the metadata --engram-meta reads
+ISTA_SETS = {
+    "IST-DAS-2.00bpw": {"title": "IST-DAS 2.00 bpw", "set_bytes": 138_238_893_056, "experts_gb": 132.07, "profile": "profile-2.00.bin",
+                        "ppl": "15.5%", "ram_ok_gb": 40.0, "ram_great_gb": 160.0, "disk_total": 257.0,
+                        "quality": "about 15.5% above the original model's perplexity (code and agent work about 5-9% above)"},
+    # IQ1_M (1.75 bpw): 120,721,120,256 bytes of shards; experts = the shards minus the 6.16 GB of dense weights.  RAM class from the experts' size against
+    # the 2.00 set's measured 40 GB (not measured on a computer yet); quality: the H100's 4,995-position score (KL 0.217, ppl 3.80, +21%; code +8.9%, agent +10.2%, English prose +53%).
+    "IST-DAS-1.75bpw": {"title": "IST-DAS 1.75 bpw", "set_bytes": 120_721_120_256, "experts_gb": 114.56, "profile": "profile-2.00.bin",
+                        "ppl": "21%", "ram_ok_gb": 34.0, "disk_total": 239.0,
+                        "quality": "about 21% above the original model's perplexity: the smallest, fine for code and agent work, noticeably weaker "
+                                   "on plain prose; pick 2.00 if it fits"},
+    "IST-DAS-2.25bpw": {"title": "IST-DAS 2.25 bpw", "set_bytes": 155_225_824_256, "experts_gb": 149.06, "profile": "profile-2.25.bin",
+                        "ppl": "7.3%", "ram_ok_gb": 44.0, "ram_great_gb": 176.0, "disk_total": 274.0,
+                        "quality": "about 7.3% above the original model's perplexity (code and agent work about 2-3% above), "
+                                   "the better of the two, 17 GB more disk"},
+}
+ISTA_DENSE_GB = 6.16                  # the dense weights in the shards (attention, router, shared experts, the head; the embedding stays on the CPU)
+ISTA_DENSE_ON_GPU_GB = 4.5            # what a card holds of them with a 12 GB card (measured on the 5070 / 3060: the engine places the rest itself)
+ISTA_MEASURED = ("an RTX 5070 (12 GB) with 64 GB of RAM and a 3.6 GB/s NVMe drive: answers about 4.7 tokens a second, a 4,000-token prompt "
+                 "read at 121 tokens a second; an RTX 3060 (12 GB) with 46 GB of RAM: about 7.2 tokens a second; four Radeon AI PRO R9700 cards: "
+                 "about 31 tokens a second")
+
+
+def variant(model: str | None) -> dict | None:
+    """The official-set entry for a model name (case-insensitive), None for UD-Q8_K_XL and anything else."""
+    if not model:
+        return None
+    return next((v for k, v in ISTA_SETS.items() if k.lower() == str(model).lower()), None)
+
+
+def variant_key(model: str | None) -> str | None:
+    return next((k for k in ISTA_SETS if k.lower() == str(model or "").lower()), None)
+
+
+def ista_download_bytes(model: str, have_engram: bool = False) -> int:
+    v = variant(model)
+    return v["set_bytes"] + ISTA_DSPARK_BYTES + ISTA_META_BYTES + (0 if have_engram else ISTA_ENGRAM_BYTES)
+
+
+def recommend(ram_gb: float | None, disk_free_gb: float | None = None) -> str:
+    """The official set that suits this computer: 2.00 bpw, unless its memory (under the measured 40 GB of RAM) or its disk (under 2.00's need but
+    enough for the smaller one) make the 1.75 bpw set the better start.  Only a recommendation: every set can still be chosen."""
+    big, small = "IST-DAS-2.00bpw", "IST-DAS-1.75bpw"
+    if ram_gb is not None and float(ram_gb) < ISTA_SETS[big]["ram_ok_gb"]:
+        return small
+    if disk_free_gb is not None:
+        need_big = ista_download_bytes(big) / GB + DISK_HEADROOM_GB
+        need_small = ista_download_bytes(small) / GB + DISK_HEADROOM_GB
+        if float(disk_free_gb) < need_big and float(disk_free_gb) >= need_small:
+            return small
+    return big
+
+
+def _round1(x: float) -> float:
+    return round(float(x) * 10) / 10
+
+
+def _cap_gb(vram_gb: float) -> float:
+    """Decimal GB a card of `vram_gb` (as the catalogs state it, GiB: 32 for an R9700) has."""
+    return float(vram_gb) * GIB / GB
+
+
+def split_layers(weights: list[float], layers: int = LAYERS) -> list[tuple[int, int]]:
+    """The layer ranges (first, last) of consecutive cards, in proportion to `weights` (each card at least one layer).
+    The engine balances its own split; on cards of one size this is what it picked (0-9, 10-19, 20-29, 30-39)."""
+    n = len(weights)
+    if n <= 1:
+        return [(0, layers - 1)]
+    total = float(sum(weights)) or 1.0
+    raw = [w / total * layers for w in weights]
+    counts = [max(1, int(r)) for r in raw]
+    order = sorted(range(n), key=lambda i: raw[i] - int(raw[i]), reverse=True)
+    k = 0
+    while sum(counts) < layers:
+        counts[order[k % n]] += 1
+        k += 1
+    while sum(counts) > layers:
+        i = max(range(n), key=lambda j: counts[j])
+        counts[i] -= 1
+    out, first = [], 0
+    for c in counts:
+        out.append((first, first + c - 1))
+        first += c
+    return out
+
+
+def tp_for(n_cards: int, cards: list[dict] | None = None) -> int:
+    """--tp for this many cards.  Pairs of cards sharing each layer's matrices (--tp 2, the engine's default for two or more cards) were
+    measured 2x faster for prompts than a plain layer split on the first rig; on four AMD Radeon AI PRO R9700 cards the plain layer split
+    (--tp 1) is faster for both: 8K prompt 370 against 196 tokens a second and 32 against 29 for answers (2026-10-23, r9700), so the IST-DAS plans give four
+    or more AMD cards 1 (cards passed); the Unsloth plan is unmeasured and keeps 2.  Two and three cards keep 2 (two R9700: prompts 220 against 140, answers slower; not changed).
+    (The engine's own check refuses 4 - 576 is not a multiple of 128 - and drops to 2.)"""
+    if n_cards >= 4 and cards and all(str(c.get("vendor") or "").lower() == "amd" for c in cards):
+        return 1
+    return 2 if n_cards >= 2 else 1
+
+
+def _tp_args(n_cards: int, tp: int) -> list[str]:
+    """The engine's --tp when it is not the engine's own pick: it takes 2 with two or more cards when told nothing, so 1 is written out."""
+    return ["--tp", str(tp)] if tp > 1 or n_cards >= 2 else []
+
+
+def _plan_ista(model: str, cards: list[dict] | None, ram_gb: float | None, disk_free_gb: float | None, context: int,
+               vendor: str | None) -> dict[str, Any]:
+    """plan() for an official IST-DAS set (ISTA_SETS): the same keys, the measured speeds in plain words.  RAM classes: with RAM for every
+    expert the CPU lane keeps ("fits"), with 40 GB or more the engine's RAM store keeps the hot experts and reads the rest from the SSD
+    ("tight": measured 4.7 tokens a second on 64 GB, 7.2 on 46 GB), under 40 GB it is untested and slower ("spills").  Never a refusal."""
+    v = variant(model)
+    keep, ignored = gpu_fit.usable_cards(gpu_fit.for_vendor(list(cards or []), vendor)[0]) if vendor else gpu_fit.usable_cards(list(cards or []))
+    keep = [c for c in keep if float(c.get("vram_gb") or 0) >= SMALLEST_CARD_GB]
+    keep.sort(key=lambda c: c.get("index", 0))
+    n = len(keep)
+    expert_gb = v["experts_gb"] / EXPERTS
+    set_gb = v["set_bytes"] / GB
+    download = ista_download_bytes(model) / GB
+    out: dict[str, Any] = {"variant": model, "cards": [], "experts_total": EXPERTS, "experts_in_vram": 0, "vram_experts_gb": 0.0,
+                           "cpu_experts_gb": v["experts_gb"], "weights_gb": _round1(set_gb), "experts_gb": _round1(v["experts_gb"]),
+                           "dense_gb": _round1(ISTA_DENSE_GB), "engram_gb": _round1(ISTA_ENGRAM_BYTES / GB), "download_gb": _round1(download),
+                           "tp": tp_for(n, keep), "args": [], "measured": ISTA_MEASURED, "lines": [], "warnings": [], "ram_gb": ram_gb,
+                           "context": context, "disk_total_gb": v["disk_total"]}
+    need_disk = download + DISK_HEADROOM_GB
+    out["disk_need_gb"] = _round1(need_disk)
+    out["disk_ok"] = None if disk_free_gb is None else disk_free_gb >= need_disk
+    lines, warns = out["lines"], out["warnings"]
+    lines.append(f"DeepSeek-V4.1-Flash, {v['title']}: a {download:.0f} GB download ({set_gb:.0f} GB of model, {ISTA_ENGRAM_BYTES / GB:.0f} GB "
+                 f"n-gram table, {ISTA_DSPARK_BYTES / GB:.0f} GB drafter). Quality: {v['quality']}.")
+    lines.append("The n-gram table stays on the SSD and is read a few rows for each word: use an NVMe drive for the model files (a slow "
+                 "disk makes every word slower). The same table serves every IST-DAS set, so installing another one downloads it once.")
+    if n == 0:
+        out["verdict"] = "no_card"
+        warns.append("No graphics card Strata can use was found: this model needs one or more cards (8 GB or more; 12 GB is what it was "
+                     "measured with).")
+    else:
+        per_dense = ISTA_DENSE_ON_GPU_GB / n
+        cap = [max(0.0, _cap_gb(float(c.get("vram_gb") or 0)) - per_dense - RESERVE_GB) for c in keep]
+        vram_exp = min(v["experts_gb"], sum(cap))
+        ranges = split_layers([float(c.get("vram_gb") or 0) for c in keep])
+        in_vram = 0
+        for c, room, (a, b) in zip(keep, cap, ranges):
+            gb = min(room, vram_exp * (room / sum(cap))) if sum(cap) else 0.0
+            ne = min(int(gb / expert_gb), (b - a + 1) * EXPERTS_PER_LAYER)
+            in_vram += ne
+            out["cards"].append({"index": c.get("index"), "name": c.get("name"), "vram_gb": float(c.get("vram_gb") or 0), "layers": [a, b],
+                                 "dense_gb": _round1(per_dense), "experts_gb": _round1(ne * expert_gb), "experts": ne})
+        out["experts_in_vram"] = in_vram
+        out["vram_experts_gb"] = _round1(in_vram * expert_gb)
+        out["cpu_experts_gb"] = _round1(v["experts_gb"] - in_vram * expert_gb)
+        out["tp"] = tp_for(n, keep)
+        out["args"] = _tp_args(n, out["tp"])
+        names = " + ".join(f"{c['name'] or 'card ' + str(c['index'])} ({c['vram_gb']:.0f} GB)" for c in out["cards"])
+        lines.append(f"Graphics memory: {names}. About {in_vram:,} of the {EXPERTS:,} experts ({out['vram_experts_gb']:.0f} GB) stay on "
+                     f"the card{'s' if n > 1 else ''}; the other {EXPERTS - in_vram:,} ({out['cpu_experts_gb']:.0f} GB) are computed on the "
+                     "CPU from RAM, and the ones RAM cannot hold are read from the SSD when a word needs them.")
+        if n >= 2 and out["tp"] == 2:
+            lines.append("Two cards share each layer's matrices (--tp 2): measured about twice as fast at reading prompts.")
+        if ram_gb:
+            all_cached = out["cpu_experts_gb"] + REST_RAM_GB
+            out["ram_needed_gb"] = _round1(all_cached)
+            out["spill_gb"] = _round1(max(0.0, all_cached - float(ram_gb)))
+            out["spare_ram_gb"] = _round1(max(0.0, float(ram_gb) - all_cached))
+            if float(ram_gb) >= all_cached:
+                out["verdict"] = "fits"
+                lines.append(f"That fits this computer: {ram_gb:.0f} GB of RAM keeps all {out['cpu_experts_gb']:.0f} GB of the CPU's experts cached.")
+            elif float(ram_gb) >= v["ram_ok_gb"]:
+                out["verdict"] = "tight"
+                lines.append(f"This computer has {ram_gb:.0f} GB of RAM: Strata keeps the experts you use most in RAM (it sizes this itself) and "
+                             "reads the rest from the SSD. Measured: about 4.7 tokens a second on 64 GB of RAM and a 12 GB card, about 7 on "
+                             "46 GB with a faster drive. More RAM or a faster NVMe drive makes it quicker.")
+            else:
+                out["verdict"] = "spills"
+                warns.append(f"This computer has {ram_gb:.0f} GB of RAM and this model was measured with 46 GB and 64 GB. With less, most "
+                             "experts come from the SSD while it answers: it works but is slow (a word can take seconds). More RAM fixes "
+                             "it. It is still allowed.")
+        else:
+            out["verdict"] = "unknown"
+        short = [c for c in keep if _cap_gb(float(c.get("vram_gb") or 0)) < per_dense + RESERVE_GB + 1.0]
+        if short:
+            warns.append(f"The dense weights need ~{per_dense:.0f} GB of graphics memory and {short[0].get('name') or 'a card'} "
+                         f"({float(short[0].get('vram_gb') or 0):.0f} GB) has hardly that: the engine may stop at the start. It is still allowed.")
+        if ignored:
+            lines.append("Not used: " + "; ".join(f"{c.get('name') or 'a card'} ({c.get('why_ignored')})" for c in ignored) + ".")
+        lines.append("Measured on " + ISTA_MEASURED + ". Other computers are slower or faster by what their cards, RAM and drive can do.")
+    if disk_free_gb is not None and disk_free_gb < need_disk:
+        warns.append(f"The download needs about {need_disk:.0f} GB of free disk space and {disk_free_gb:.0f} GB are free where the files go. "
+                     "Use --models-dir on a bigger drive, or --gguf-dir for files you already have.")
+    out["words"] = " ".join(lines)
+    return out
+
+
+def plan(cards: list[dict] | None, ram_gb: float | None, disk_free_gb: float | None = None, context: int = 8192,
+         vendor: str | None = None, model: str | None = None) -> dict[str, Any]:
+    """Where the model goes on this computer.  `cards`: the graphics cards ({"index", "name", "vram_gb", ...}, as gpu_fit takes them),
+    `ram_gb` the RAM, `disk_free_gb` the free space where the files go (None: not checked).  `model`: an official IST-DAS set
+    (ISTA_SETS: "IST-DAS-2.00bpw" / "IST-DAS-2.25bpw" / "IST-DAS-1.75bpw") gets its own numbers; None or "UD-Q8_K_XL" is Unsloth's 508 GB set.
+
+    Keys: cards (per card: index, name, vram_gb, layers, dense_gb, experts_gb, experts), experts_in_vram, experts_total, vram_experts_gb,
+    cpu_experts_gb, ram_needed_gb, spill_gb, spare_ram_gb, verdict (fits / tight / spills / unknown / no_card), tp, args,
+    download_gb, engram_gb, disk_need_gb, disk_ok, lines (plain words, one idea each), warnings (the risks, plainly), measured."""
+    if variant(model):
+        return _plan_ista(variant_key(model), cards, ram_gb, disk_free_gb, context, vendor)
+    keep, ignored = gpu_fit.usable_cards(gpu_fit.for_vendor(list(cards or []), vendor)[0]) if vendor else gpu_fit.usable_cards(list(cards or []))
+    keep = [c for c in keep if float(c.get("vram_gb") or 0) >= SMALLEST_CARD_GB]
+    keep.sort(key=lambda c: c.get("index", 0))                 # the engine takes the layers in card order
+    n = len(keep)
+    out: dict[str, Any] = {"cards": [], "experts_total": EXPERTS, "experts_in_vram": 0, "vram_experts_gb": 0.0,
+                           "cpu_experts_gb": EXPERTS_GB, "weights_gb": _round1(WEIGHTS_GB), "experts_gb": _round1(EXPERTS_GB),
+                           "dense_gb": _round1(DENSE_GB), "engram_gb": _round1(ENGRAM_GB), "download_gb": _round1(DOWNLOAD_GB),
+                           "tp": tp_for(n), "args": [], "measured": MEASURED, "lines": [], "warnings": [],
+                           "ram_gb": ram_gb, "context": context}
+    need_disk = DOWNLOAD_GB + DISK_HEADROOM_GB
+    out["disk_need_gb"] = _round1(need_disk)
+    out["disk_ok"] = None if disk_free_gb is None else disk_free_gb >= need_disk
+    lines, warns = out["lines"], out["warnings"]
+    lines.append(f"DeepSeek-V4.1-Flash (UD-Q8_K_XL): a {DOWNLOAD_GB:.0f} GB download. In memory it needs {WEIGHTS_GB:.0f} GB: "
+                 f"{EXPERTS_GB:.0f} GB of experts ({EXPERTS:,} of {EXPERT_GB * 1000:.1f} MB) and {DENSE_GB:.0f} GB of attention "
+                 "and other dense weights.")
+    lines.append(f"Its n-gram table ({ENGRAM_GB:.0f} GB of the download) stays on the SSD and is read a few rows for each word "
+                 "it writes: it is not counted in RAM, but a slow disk (SATA, a network drive, a USB disk) makes every word "
+                 "slower. Use an NVMe drive for the model files.")
+    if n == 0:
+        out["verdict"] = "no_card"
+        warns.append("No graphics card Strata can use was found: this model needs one or more cards (it was run on AMD Radeon "
+                     "AI PRO R9700 cards).")
+    else:
+        per_dense = DENSE_ON_GPU_GB / n
+        cap = []
+        for c in keep:
+            room = _cap_gb(float(c.get("vram_gb") or 0)) - per_dense - RESERVE_GB
+            cap.append(max(0.0, room))
+        vram_exp = min(EXPERTS_GB, sum(cap))
+        share = [x / sum(cap) if sum(cap) else 1.0 / n for x in cap]
+        ranges = split_layers([float(c.get("vram_gb") or 0) for c in keep])
+        in_vram = 0
+        for c, room, (a, b), s in zip(keep, cap, ranges, share):
+            gb = min(room, vram_exp * s) if sum(cap) else 0.0
+            ne = int(gb / EXPERT_GB)
+            ne = min(ne, (b - a + 1) * EXPERTS_PER_LAYER)
+            in_vram += ne
+            out["cards"].append({"index": c.get("index"), "name": c.get("name"), "vram_gb": float(c.get("vram_gb") or 0),
+                                 "layers": [a, b], "dense_gb": _round1(per_dense), "experts_gb": _round1(ne * EXPERT_GB),
+                                 "experts": ne})
+        out["experts_in_vram"] = in_vram
+        out["vram_experts_gb"] = _round1(in_vram * EXPERT_GB)
+        out["cpu_experts_gb"] = _round1(EXPERTS_GB - in_vram * EXPERT_GB)
+        out["tp"] = tp_for(n)
+        out["args"] = ["--tp", str(out["tp"])] if out["tp"] > 1 else []
+        if ram_gb:
+            needed = out["cpu_experts_gb"] + REST_RAM_GB
+            spare = float(ram_gb) - needed
+            out["ram_needed_gb"] = _round1(needed)
+            out["spare_ram_gb"] = _round1(max(0.0, spare))
+            out["spill_gb"] = _round1(max(0.0, -spare))
+            out["verdict"] = "spills" if spare < 0 else "tight" if spare < SPARE_FITS_GB else "fits"
+        else:
+            out["verdict"] = "unknown"
+        # the placement, in words
+        short = [c for c in keep if _cap_gb(float(c.get("vram_gb") or 0)) < per_dense + RESERVE_GB + 1.0]
+        if short:
+            warns.append(f"The dense weights need ~{per_dense:.0f} GB of graphics memory on {'each card' if n > 1 else 'the card'} "
+                         f"and {short[0].get('name') or 'a card'} ({float(short[0].get('vram_gb') or 0):.0f} GB) has "
+                         "hardly that: the engine may stop at the start (out of graphics memory). More or bigger cards fix it; "
+                         "it is still allowed.")
+        if n == 1:
+            c = out["cards"][0]
+            lines.append(f"One card ({c['name'] or 'card ' + str(c['index'])}, {c['vram_gb']:.0f} GB): the dense weights ({c['dense_gb']:.0f} GB) "
+                         f"and {c['experts']:,} of the {EXPERTS:,} experts ({c['experts_gb']:.0f} GB) go in its memory, the other "
+                         f"{EXPERTS - c['experts']:,} experts ({out['cpu_experts_gb']:.0f} GB) are computed on the CPU from RAM.")
+        else:
+            blocks = "  |  ".join(f"card {c['index']}: layers {c['layers'][0]}-{c['layers'][1]}" for c in out["cards"])
+            lines.append(f"{n} cards, one block of layers on each ({blocks}); the engine balances the split. On each card "
+                         f"~{out['cards'][0]['dense_gb']:.1f} GB of dense weights and ~{out['cards'][0]['experts_gb']:.0f} GB of experts "
+                         f"(about {out['cards'][0]['experts']:,}); in all {in_vram:,} of {EXPERTS:,} experts ({out['vram_experts_gb']:.0f} GB) "
+                         f"are in graphics memory.")
+            lines.append("Two cards share each layer's matrices (--tp 2): measured about twice as fast at reading prompts as a plain "
+                         "split of the layers.")
+        lines.append(f"The other {EXPERTS - in_vram:,} experts ({out['cpu_experts_gb']:.0f} GB) are computed on the CPU, read in place "
+                     "from the model's files: they need that much free RAM to stay cached "
+                     f"({out.get('ram_needed_gb', out['cpu_experts_gb'] + REST_RAM_GB):.0f} GB with {REST_RAM_GB:.0f} GB for the system and the engine).")
+        if out["verdict"] == "spills":
+            warns.append(f"This computer has {ram_gb:.0f} GB of RAM and {out['ram_needed_gb']:.0f} GB would keep the CPU's share of the experts "
+                         f"cached: about {out['spill_gb']:.0f} GB of them are read from the SSD while the model answers, which is slow "
+                         "(a word can take seconds). More RAM or more graphics memory fixes it. It is still allowed.")
+        elif out["verdict"] == "tight":
+            warns.append(f"RAM is tight: {out['spare_ram_gb']:.0f} GB are left beside the experts; close other programs.")
+        elif out["verdict"] == "fits":
+            lines.append(f"That fits this computer ({ram_gb:.0f} GB of RAM, {out['spare_ram_gb']:.0f} GB to spare).")
+        if n >= 2 and any(abs(float(c.get("vram_gb") or 0) - float(keep[0].get("vram_gb") or 0)) > 1 for c in keep):
+            lines.append("The cards are not the same size: the layers are shared in proportion to their memory.")
+        if ignored:
+            lines.append("Not used: " + "; ".join(f"{c.get('name') or 'a card'} ({c.get('why_ignored')})" for c in ignored) + ".")
+        lines.append("Measured on " + MEASURED + ". Other computers are slower or faster by what their cards and RAM can hold; "
+                     "this engine has been run on AMD cards.")
+    if disk_free_gb is not None and disk_free_gb < need_disk:
+        warns.append(f"The download needs about {need_disk:.0f} GB of free disk space and {disk_free_gb:.0f} GB are free where the "
+                     "files go. Use --models-dir on a bigger drive, or --gguf-dir for files you already have.")
+    out["words"] = " ".join(lines)
+    return out
+
+
+def _option(key: str, cards: list[dict], ram_gb: float | None, ctx: int, vendor: str | None, model: str | None = None) -> dict[str, Any]:
+    p = plan(cards, ram_gb, None, ctx, vendor, model)
+    names = " + ".join(gpu_fit._name(c) for c in cards)
+    label = f"{names} + RAM" if len(cards) == 1 else f"{names} + RAM (layers shared, --tp {p['tp']})"
+    words = {"fits": f"fits: {p['experts_in_vram']:,} of {p['experts_total']:,} experts in graphics memory, the rest computed on the CPU "
+                     f"({p.get('ram_needed_gb', 0):.0f} GB of RAM)",
+             "tight": f"fits, but tight: RAM left over is under {SPARE_FITS_GB:.0f} GB",
+             "spills": f"does not fit in memory: about {p.get('spill_gb', 0):.0f} GB of the experts are read from the SSD while answering (slow)",
+             "unknown": "memory of this computer unknown", "no_card": "no usable card"}[p["verdict"]]
+    return {"key": key, "cards": [c.get("index") for c in cards], "label": label, "verdict": p["verdict"], "summary": f"{label}: {words}",
+            "words": words, "vram_gb": _round1(sum(float(c.get("vram_gb") or 0) for c in cards)), "spill_gb": p.get("spill_gb", 0.0),
+            "ram_use_gb": p.get("cpu_experts_gb", 0.0), "spare_ram_gb": p.get("spare_ram_gb", 0.0),
+            "notes": [l for l in p["lines"] if l.startswith(("One card", f"{len(cards)} cards", "Two cards share"))],
+            "recommended": False, "plan": p}
+
+
+def options(cards: list[dict], ram_gb: float | None, vendor: str | None = None, context: int = 8192, model: str | None = None) -> dict[str, Any]:
+    """gpu_fit.options for this model: every card that counts (recommended: the model is bigger than any one card, so each card adds
+    experts that stay off the CPU), and the largest card alone.  Same keys as gpu_fit.options."""
+    mine, others = gpu_fit.for_vendor(list(cards or []), vendor) if vendor else (list(cards or []), [])
+    keep, ignored = gpu_fit.usable_cards(mine)
+    keep = [c for c in keep if float(c.get("vram_gb") or 0) >= SMALLEST_CARD_GB]
+    out: dict[str, Any] = {"options": [], "recommended": None, "ignored": ignored, "cards": len(keep), "vendor": vendor,
+                           "other_vendor_note": gpu_fit.other_vendor_note(list(cards or []), vendor)}
+    if not keep:
+        return out
+    opts = [_option("all", sorted(keep, key=lambda c: c.get("index", 0)), ram_gb, context, vendor, model)]
+    if len(keep) > 1:
+        opts.append(_option("single", [keep[0]], ram_gb, context, vendor, model))
+    opts[0]["recommended"] = True
+    out["options"], out["recommended"] = opts, "all"
+    return out

strata_control/feed.py

@@ -153,6 +153,9 @@ class Feed:
     source: str                    # where it was read from (the base for relative URLs)
     key_id: str                    # who signed it
     raw: bytes = field(default=b"", repr=False)
+    #: the DeepSeek-V4.1 engine packages (feed member ``dsv41``, same entry shape as ``engine``): a separate download that setup fetches when
+    #: DeepSeek is chosen. Kept out of ``engine`` so that `strata update` and select_engine never pick one as THE engine. Older Controls ignore the member.
+    dsv41: tuple[EnginePackage, ...] = ()
 
     def url(self, ref: str) -> str:
         return transfer.resolve_ref(transfer.base_of(self.source), ref)
@@ -278,8 +281,20 @@ def parse(payload: bytes, *, source: str, key_id: str) -> Feed:
             if ident in seen:
                 raise FeedError(f"the feed lists engine {pkg.label} twice")
             seen.add(ident)
+    dsv41 = []
+    raw_dsv41 = data.get("dsv41")
+    if raw_dsv41 is not None:
+        if not isinstance(raw_dsv41, dict) or not isinstance(raw_dsv41.get("packages"), list):
+            raise FeedError("the feed's dsv41 member lists no packages")
+        dsv41 = [_engine_package(item) for item in raw_dsv41["packages"]]
+        seen = set()
+        for pkg in dsv41:
+            ident = (pkg.version, pkg.platform, pkg.backend, pkg.variant)
+            if ident in seen:
+                raise FeedError(f"the feed lists DeepSeek engine {pkg.label} twice")
+            seen.add(ident)
     return Feed(sequence=sequence, channel=channel, generated_at=generated, expires_at=expires, app=app,
-                control=tuple(control), engine=tuple(engine), source=source, key_id=key_id, raw=payload)
+                control=tuple(control), engine=tuple(engine), source=source, key_id=key_id, raw=payload, dsv41=tuple(dsv41))
 
 
 def _expired_words(expires_at: str, now: dt.datetime, server_now: dt.datetime | None) -> str:

strata_control/folders.py

@@ -76,7 +76,7 @@ def problem(raw: str | os.PathLike | None) -> tuple[str, str] | None:
                 if os.name == "nt" else "Type the whole path, starting with /, like /data/strata-models. Nothing was downloaded or changed.")
     full = os.path.abspath(path)
     for root in _system_roots():
-        if _inside(full, root):
+        if _inside(full, root) and not _inside(full, os.path.expanduser("~")):         # the running user's own home is theirs (root's /root included)
             return (f"{full} is a system folder, and Strata should not keep model files there.", _next())
     try:
         if path.exists():